def test_create_project(pydss_project):
project_name = "test-project"
project_dir = os.path.join(PATH, project_name)
# Intentionally not in alphabetic order so that we verify our designated
# ordering.
scenarios = [
PyDssScenario("b_scenario1"),
PyDssScenario("a_scenario2"),
]
project = PyDssProject.create_project(PATH, project_name, scenarios)
assert os.path.exists(project_dir)
for dir_name in PyDssScenario._SCENARIO_DIRECTORIES:
for scenario in scenarios:
path = os.path.join(
project_dir,
SCENARIOS,
scenario.name,
dir_name,
)
assert os.path.exists(path)
project2 = PyDssProject.load_project(project_dir)
assert project.name == project2.name
scenarios1 = project.scenarios
scenarios1.sort(key=lambda x: x.name)
scenarios2 = project2.scenarios
scenarios2.sort(key=lambda x: x.name)
assert len(scenarios1) == len(scenarios2)
for i in range(len(project.scenarios)):
assert scenarios1[i].name == scenarios2[i].name
assert scenarios1[i].controllers == scenarios2[i].controllers
def add_post_process(project_path, scenario_name, script, config_file):
"""Add post-process script to PyDSS scenario."""
setup_logging("PyDSS", console_level=logging.INFO)
project = PyDssProject.load_project(project_path)
scenario = project.get_scenario(scenario_name)
pp_info = {"script": script, "config_file": config_file}
scenario.add_post_process(pp_info)
project.serialize()
def test_auto_snapshot_time_point(cleanup_project):
PyDssProject.run_project(
AUTO_SNAPSHOT_TIME_POINT_PROJECT_PATH,
simulation_file=SIMULATION_SETTINGS_FILENAME,
)
project = PyDssProject.load_project(AUTO_SNAPSHOT_TIME_POINT_PROJECT_PATH)
settings = project.read_scenario_time_settings("max_pv_load_ratio")
assert str(settings["start_time"]) == "2020-01-01 11:15:00"
def test_space_estimation_with_dry_run(cleanup_project):
"""Should generate esimated space with dry run simulation"""
# dry run pydss project
project = PyDssProject.load_project(RUN_PROJECT_PATH)
project.run(dry_run=True)
assert "scenario1" in project.estimated_space
assert project.estimated_space["scenario1"] is not None
assert project.estimated_space["scenario1"] > 0
def __init__(self,
project_path=None,
project=None,
in_memory=False,
frequency=False,
mode=False):
"""Constructs PyDssResults object.
Parameters
----------
project_path : str | None
Load project from files in path
project : PyDssProject | None
Existing project object
in_memory : bool
If true, load all exported data into memory.
frequency : bool
If true, add frequency column to all dataframes.
mode : bool
If true, add mode column to all dataframes.
"""
options = ElementOptions()
if project_path is not None:
# TODO: handle old version?
self._project = PyDssProject.load_project(
project_path,
simulation_file=RUN_SIMULATION_FILENAME,
)
elif project is None:
raise InvalidParameter("project_path or project must be set")
else:
self._project = project
self._fs_intf = self._project.fs_interface
self._scenarios = []
filename = self._project.get_hdf_store_filename()
driver = "core" if in_memory else None
self._hdf_store = h5py.File(filename, "r", driver=driver)
if self._project.simulation_config.exports.export_results:
for name in self._project.list_scenario_names():
metadata = self._project.read_scenario_export_metadata(name)
scenario_result = PyDssScenarioResults(
name,
self.project_path,
self._hdf_store,
self._fs_intf,
metadata,
options,
frequency=frequency,
mode=mode,
)
self._scenarios.append(scenario_result)
def test_create_project(pydss_project):
project_name = "test-project"
project_dir = os.path.join(PATH, project_name)
thermal_upgrade = {
"script": "AutomatedThermalUpgrade",
"config_file": THERMAL_CONFIG,
}
voltage_upgrade = {
"script": "AutomatedVoltageUpgrade",
"config_file": VOLTAGE_CONFIG,
}
# Intentionally not in alphabetic order so that we verify our designated
# ordering.
scenarios = [
PyDssScenario("b_scenario1", post_process_infos=[thermal_upgrade]),
PyDssScenario("a_scenario2", post_process_infos=[voltage_upgrade]),
]
project = PyDssProject.create_project(PATH, project_name, scenarios)
assert os.path.exists(project_dir)
for dir_name in PyDssScenario._SCENARIO_DIRECTORIES:
for scenario in scenarios:
path = os.path.join(
project_dir,
SCENARIOS,
scenario.name,
dir_name,
)
assert os.path.exists(path)
project2 = PyDssProject.load_project(project_dir)
assert project.name == project2.name
scenarios1 = project.scenarios
scenarios1.sort(key=lambda x: x.name)
scenarios2 = project2.scenarios
scenarios2.sort(key=lambda x: x.name)
assert len(scenarios1) == len(scenarios2)
for i in range(len(project.scenarios)):
assert scenarios1[i].name == scenarios2[i].name
assert scenarios1[i].controllers == scenarios2[i].controllers
assert scenarios1[i].post_process_infos == scenarios2[
i].post_process_infos
def extract_element_files(project_path, output_dir=None, verbose=False):
"""Extract the element info files from an archived PyDSS project."""
if not os.path.exists(project_path):
print(f"project-path={project_path} does not exist")
sys.exit(1)
filename = "pydss_extract.log"
console_level = logging.INFO
file_level = logging.INFO
if verbose:
console_level = logging.DEBUG
file_level = logging.DEBUG
setup_logging(
"PyDSS",
filename=filename,
console_level=console_level,
file_level=file_level,
)
logger.info("CLI: [%s]", get_cli_string())
project = PyDssProject.load_project(project_path)
fs_intf = project.fs_interface
results = PyDssResults(project_path)
for scenario in results.scenarios:
for filename in scenario.list_element_info_files():
text = fs_intf.read_file(filename)
if output_dir is None:
path = os.path.join(project_path, filename)
else:
path = os.path.join(output_dir, filename)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f_out:
f_out.write(text)
print(f"Extracted {filename} to {path}")
def extract(project_path, file_path, output_dir=None, verbose=False):
"""Extract a file from an archived PyDSS project."""
if not os.path.exists(project_path):
print(f"project-path={project_path} does not exist")
sys.exit(1)
filename = "pydss_extract.log"
console_level = logging.INFO
file_level = logging.INFO
if verbose:
console_level = logging.DEBUG
file_level = logging.DEBUG
setup_logging(
"PyDSS",
filename=filename,
console_level=console_level,
file_level=file_level,
)
logger.info("CLI: [%s]", get_cli_string())
project = PyDssProject.load_project(project_path)
data = project.fs_interface.read_file(file_path)
if output_dir is None:
path = os.path.join(project_path, file_path)
else:
path = os.path.join(output_dir, file_path)
os.makedirs(os.path.dirname(path), exist_ok=True)
ext = os.path.splitext(file_path)[1]
mode = "wb" if ext in (".h5", ".feather") else "w"
with open(path, mode) as f_out:
f_out.write(data)
print(f"Extracted {file_path} to {path}")
def run(project_path,
options=None,
tar_project=False,
zip_project=False,
verbose=False,
simulations_file=None,
dry_run=False):
"""Run a PyDSS simulation."""
project_path = Path(project_path)
settings = PyDssProject.load_simulation_settings(project_path,
simulations_file)
if verbose:
# Override the config file.
settings.logging.logging_level = logging.DEBUG
filename = None
console_level = logging.INFO
file_level = logging.INFO
if not settings.logging.enable_console:
console_level = logging.ERROR
if verbose:
console_level = logging.DEBUG
file_level = logging.DEBUG
if settings.logging.enable_file:
logs_path = project_path / "Logs"
if not logs_path.exists():
logger.error("Logs path %s does not exist", logs_path)
sys.exit(1)
filename = logs_path / "pydss.log"
setup_logging(
"PyDSS",
filename=filename,
console_level=console_level,
file_level=file_level,
)
logger.info("CLI: [%s]", get_cli_string())
if options is not None:
options = ast.literal_eval(options)
if not isinstance(options, dict):
logger.error("options are invalid: %s", options)
sys.exit(1)
project = PyDssProject.load_project(project_path,
options=options,
simulation_file=simulations_file)
project.run(tar_project=tar_project,
zip_project=zip_project,
dry_run=dry_run)
if dry_run:
maxlen = max([len(k) for k in project.estimated_space.keys()])
if len("ScenarioName") > maxlen:
maxlen = len("ScenarioName")
template = "{:<{width}} {}\n".format("ScenarioName",
"EstimatedSpace",
width=maxlen)
total_size = 0
for k, v in project.estimated_space.items():
total_size += v
vstr = make_human_readable_size(v)
template += "{:<{width}} : {}\n".format(k, vstr, width=maxlen)
template = template.strip()
logger.info(template)
logger.info("-" * 30)
logger.info(f"TotalSpace: {make_human_readable_size(total_size)}")
logger.info("=" * 30)
logger.info(
"Note: compression may reduce the size by ~90% depending on the data."
)
def run_test_project_by_property(tar_project, zip_project):
project = PyDssProject.load_project(RUN_PROJECT_PATH)
PyDssProject.run_project(
RUN_PROJECT_PATH,
tar_project=tar_project,
zip_project=zip_project,
simulation_file=SIMULATION_SETTINGS_FILENAME,
)
results = PyDssResults(RUN_PROJECT_PATH)
assert len(results.scenarios) == 1
assert results._hdf_store.attrs["version"] == DATA_FORMAT_VERSION
scenario = results.scenarios[0]
assert isinstance(scenario, PyDssScenarioResults)
elem_classes = scenario.list_element_classes()
expected_elem_classes = list(EXPECTED_ELEM_CLASSES_PROPERTIES.keys())
expected_elem_classes.sort()
assert elem_classes == expected_elem_classes
for elem_class in elem_classes:
expected_properties = EXPECTED_ELEM_CLASSES_PROPERTIES[elem_class]
expected_properties.sort()
properties = scenario.list_element_properties(elem_class)
assert properties == expected_properties
for prop in properties:
element_names = scenario.list_element_names(elem_class, prop)
for name in element_names:
df = scenario.get_dataframe(elem_class, prop, name)
assert isinstance(df, pd.DataFrame)
assert len(df) == 96
for name, df in scenario.iterate_dataframes(elem_class, prop):
assert name in element_names
assert isinstance(df, pd.DataFrame)
# Test with an option.
assert scenario.list_element_property_options(
"Lines", "Currents") == ["phase_terminal"]
df = scenario.get_dataframe("Lines",
"Currents",
"Line.sw0",
phase_terminal="A1")
assert isinstance(df, pd.DataFrame)
assert len(df) == 96
assert len(df.columns) == 1
step = datetime.timedelta(
seconds=project.simulation_config["Project"]["Step resolution (sec)"])
assert df.index[1] - df.index[0] == step
df = scenario.get_dataframe("Lines",
"CurrentsMagAng",
"Line.sw0",
phase_terminal="A1",
mag_ang="mag")
assert isinstance(df, pd.DataFrame)
assert len(df) == 96
assert len(df.columns) == 1
df = scenario.get_dataframe("Lines",
"CurrentsMagAng",
"Line.sw0",
phase_terminal=None,
mag_ang="ang")
assert isinstance(df, pd.DataFrame)
assert len(df.columns) == 2
assert len(df) == 96
regex = re.compile(r"[ABCN]1")
df = scenario.get_dataframe("Lines",
"Currents",
"Line.sw0",
phase_terminal=regex)
assert isinstance(df, pd.DataFrame)
assert len(df.columns) == 1
assert len(df) == 96
option_values = scenario.get_option_values("Lines", "Currents", "Line.sw0")
assert option_values == ["A1", "A2"]
prop = "Currents"
full_df = scenario.get_full_dataframe("Lines", prop)
assert len(full_df.columns) >= len(
scenario.list_element_names("Lines", prop))
for column in full_df.columns:
assert "Unnamed" not in column
assert len(full_df) == 96
element_info_files = scenario.list_element_info_files()
assert element_info_files
for filename in element_info_files:
df = scenario.read_element_info_file(filename)
assert isinstance(df, pd.DataFrame)
# Test the shortcut.
df = scenario.read_element_info_file("PVSystems")
assert isinstance(df, pd.DataFrame)
cap_changes = scenario.read_capacitor_changes()
def run(project_path, options=None, tar_project=False, zip_project=False, verbose=False, simulations_file=None, dry_run=False):
"""Run a PyDSS simulation."""
if not os.path.exists(project_path):
print(f"project-path={project_path} does not exist")
sys.exit(1)
config = PyDssProject.load_simulation_config(project_path, simulations_file)
if verbose:
# Override the config file.
config["Logging"]["Logging Level"] = logging.DEBUG
filename = None
console_level = logging.INFO
file_level = logging.INFO
if not config["Logging"]["Display on screen"]:
console_level = logging.ERROR
if verbose:
console_level = logging.DEBUG
file_level = logging.DEBUG
if config["Logging"]["Log to external file"]:
logs_path = os.path.join(project_path, "Logs")
filename = os.path.join(
logs_path,
os.path.basename(project_path) + ".log",
)
if not os.path.exists(logs_path):
print("Logs path does not exist. 'run' is not supported on a tarred project.")
sys.exit(1)
setup_logging(
"PyDSS",
filename=filename,
console_level=console_level,
file_level=file_level,
)
logger.info("CLI: [%s]", get_cli_string())
if options is not None:
options = ast.literal_eval(options)
if not isinstance(options, dict):
print(f"options must be of type dict; received {type(options)}")
sys.exit(1)
project = PyDssProject.load_project(project_path, options=options, simulation_file=simulations_file)
project.run(tar_project=tar_project, zip_project=zip_project, dry_run=dry_run)
if dry_run:
print("="*30)
maxlen = max([len(k) for k in project.estimated_space.keys()])
if len("ScenarioName") > maxlen:
maxlen = len("ScenarioName")
template = "{:<{width}} {}\n".format("ScenarioName", "EstimatedSpace", width=maxlen)
total_size = 0
for k, v in project.estimated_space.items():
total_size += v
vstr = make_human_readable_size(v)
template += "{:<{width}} : {}\n".format(k, vstr, width=maxlen)
template = template.strip()
print(template)
print("-"*30)
print(f"TotalSpace: {make_human_readable_size(total_size)}")
print("="*30)
print("Note: compression may reduce the size by ~90% depending on the data.")
async def get_pydss_project_info(self, request, path):
"""
---
summary: Returns a dictionary of valid project and scenarios in the provided path
tags:
- PyDSS project
parameters:
- name: path
in: query
required: true
schema:
type: string
example: C:/Users/alatif/Desktop/PyDSS_2.0/PyDSS/examples
responses:
'200':
description: Successfully retrieved project information
content:
application/json:
schema:
type: object
examples:
get_instance_status:
value:
Status: 200
Message: PyDSS instance with the provided UUID is currently running
UUID: 96c21e00-cd3c-4943-a914-14451f5f7ab6
"Data": {'Project1' : {'Scenario1', 'Scenario2'}, 'Project2' : {'Scenario1'}}
'406':
description: Provided path does not exist
content:
application/json:
schema:
type: object
examples:
get_instance_status:
value:
Status: 406
Message: Provided path does not exist
UUID: None
"""
logger.info(f"Exploring {path} for valid projects")
if not os.path.exists(path):
return web.json_response({
"Status": 404,
"Message": f"Provided path does not exist",
"UUID": None
})
subfolders = [f.path for f in os.scandir(path) if f.is_dir()]
projects = {}
for folder in subfolders:
try:
pydss_project = PyDssProject.load_project(folder)
projects[pydss_project._name] = [x.name for x in pydss_project.scenarios]
except:
pass
n = len(projects)
if n > 0:
return web.json_response({"Status": 200,
"Message": f"{n} valid projects found",
"UUID": None,
"Data": projects})
else:
web.json_response({"Status": 404,
"Message": f"No valid PyDSS project in provided base path",
"UUID": None})