def _check_sum_groups(self, sum_groups_file): if sum_groups_file is not None: if self._sum_groups: raise InvalidConfiguration( f"Cannot set both sum_groups and sum_groups_file") # This path needs to be relative to the current directory, not the Exports.toml. # This might need to be changed. if not Path(sum_groups_file).exists(): raise InvalidConfiguration( f"{sum_groups_file} does not exist. The path must be relative to the current directory." ) self._sum_groups = load_data(sum_groups_file)["sum_groups"] self._sum_elements = True # Ignore the user setting. This must be true. # Ensure that there are no duplicate names. orig_length = 0 all_names = set() for group in self._sum_groups: orig_length += len(group["elements"]) group["elements"] = set(group["elements"]) all_names = all_names.union(group["elements"]) if orig_length != len(all_names): tag = f"{self.elem_class}/{self.name}" raise InvalidConfiguration( f"{tag} has duplicate element names in sum_groups")
def append_required_exports(exports, settings: SimulationSettingsModel): """Append export properties required by the configured reports. Parameters ---------- exports : ExportListReader settings : SimulationSettingsModel """ all_reports = Reports.get_all_reports() report_settings = settings.reports if not report_settings: return existing_scenarios = {x.name for x in settings.project.scenarios} for report in report_settings.types: if not report.enabled: continue name = report.name if name not in all_reports: raise InvalidConfiguration(f"{name} is not a valid report") required_scenarios = all_reports[name].get_required_scenario_names( ) missing = required_scenarios.difference(existing_scenarios) if missing: text = " ".join(missing) raise InvalidConfiguration( f"{name} requires these scenarios: {text}") scenarios = report.scenarios active_scenario = settings.project.active_scenario if scenarios and active_scenario not in scenarios: logger.debug("report %s is not enabled for scenario %s", name, active_scenario) continue required = all_reports[name].get_required_exports(settings) for elem_class, required_properties in required.items(): for req_prop in required_properties: found = False store_type = req_prop.get("store_values_type", "all") for prop in exports.list_element_properties(elem_class): if prop.name == req_prop["property"] and \ prop.store_values_type.value == store_type: if prop.opendss_classes or req_prop.get( "opendss_classes"): assert prop.sum_elements == req_prop.get( "sum_elements", False) assert prop.data_conversion == \ req_prop.get("data_conversion", DataConversion.NONE) prop.append_opendss_classes( req_prop["opendss_classes"]) found = True if not found: exports.append_property(elem_class, req_prop) logger.debug("Add required property: %s %s", elem_class, req_prop) all_reports[name].set_required_project_settings(settings)
def _get_window_size_by_name_index(self, prop): """Returns a list of window sizes per element name corresponding to self._names.""" if not prop.opendss_classes: raise InvalidConfiguration( f"window_sizes requires opendss_classes: {prop.name}" ) window_sizes = [None] * len(self._names) for opendss_class, window_size in prop.window_sizes.items(): if opendss_class not in prop.opendss_classes: raise InvalidConfiguration( f"{opendss_class} is not defined in opendss_classes: {prop.name}" ) # Note: names have singluar class names, such as Line.line1. # opendss_classes are plural, such as Lines mapping = {} for i, name in enumerate(self._names): opendss_class_singular = name.split(".")[0] size = mapping.get(opendss_class_singular) if size is None: for opendss_class in prop.opendss_classes: if opendss_class.startswith(opendss_class_singular): size = prop.window_sizes[opendss_class] mapping[opendss_class_singular] = size if size is None: raise InvalidConfiguration(f"Failed to find window_size for {name}") window_sizes[i] = size return window_sizes
def _get_window_sizes(inputs, resolution): line_window_size = timedelta(hours=inputs["line_window_size_hours"]) if line_window_size % resolution != timedelta(0): raise InvalidConfiguration( f"line_window_size={line_window_size} must be a multiple of {resolution}" ) transformer_window_size = timedelta(hours=inputs["transformer_window_size_hours"]) if transformer_window_size % resolution != timedelta(0): raise InvalidConfiguration( f"transformer_window_size={transformer_window_size} must be a multiple of {resolution}" ) return line_window_size // resolution, transformer_window_size // resolution
def run(self, logging_configured=True, tar_project=False, zip_project=False, dry_run=False): """Run all scenarios in the project.""" if isinstance(self._fs_intf, PyDssArchiveFileInterfaceBase): raise InvalidConfiguration("cannot run from an archived project") if tar_project and zip_project: raise InvalidParameter("tar_project and zip_project cannot both be True") if self._simulation_config['Project']['DSS File'] == "": raise InvalidConfiguration("a valid opendss file needs to be passed") inst = instance() self._simulation_config["Logging"]["Pre-configured logging"] = logging_configured if dry_run: store_filename = os.path.join(tempfile.gettempdir(), STORE_FILENAME) else: store_filename = os.path.join(self._project_dir, STORE_FILENAME) driver = None if self._simulation_config["Exports"].get("Export Data In Memory", True): driver = "core" with h5py.File(store_filename, mode="w", driver=driver) as hdf_store: self._hdf_store = hdf_store self._hdf_store.attrs["version"] = DATA_FORMAT_VERSION for scenario in self._scenarios: self._simulation_config["Project"]["Active Scenario"] = scenario.name inst.run(self._simulation_config, self, scenario, dry_run=dry_run) self._estimated_space[scenario.name] = inst.get_estimated_space() if not dry_run: results = None export_tables = self._simulation_config["Exports"].get( "Export Data Tables", False ) generate_reports = self._simulation_config.get("Reports", False) if export_tables or generate_reports: # Hack. Have to import here. Need to re-organize to fix. from PyDSS.pydss_results import PyDssResults results = PyDssResults(self._project_dir) if export_tables: for scenario in results.scenarios: scenario.export_data() if generate_reports: results.generate_reports() if tar_project: self._tar_project_files() elif zip_project: self._zip_project_files() if dry_run and os.path.exists(store_filename): os.remove(store_filename)
def __init__(self, elem_class, data): self.elem_class = elem_class self._opendss_classes = data.get("opendss_classes", []) self.name = data["property"] self.publish = data.get("publish", False) self._data_conversion = DataConversion( data.get("data_conversion", "none")) self._sum_elements = data.get("sum_elements", False) self._sum_groups = data.get("sum_groups", []) sum_groups_file = data.get("sum_groups_file") self._limits = self._parse_limits(data, "limits") self._limits_filter = LimitsFilter(data.get("limits_filter", "outside")) self._limits_b = self._parse_limits(data, "limits_b") self._limits_filter_b = LimitsFilter( data.get("limits_filter_b", "outside")) self._store_values_type = StoreValuesType( data.get("store_values_type", "all")) self._names, self._are_names_regex, self._are_names_filtered = self._parse_names( data) self._sample_interval = data.get("sample_interval", 1) self._window_size = data.get("window_size", 100) self._window_sizes = data.get("window_sizes", {}) custom_prop = f"{elem_class}.{self.name}" self._custom_metric = CUSTOM_METRICS.get(custom_prop) if self._sum_groups or sum_groups_file: self._check_sum_groups(sum_groups_file) # Note to devs: any field added here needs to be handled in serialize() if self._sum_elements and self._store_values_type not in \ (StoreValuesType.ALL, StoreValuesType.SUM): raise InvalidConfiguration( "sum_elements requires store_values_types = all or sum") if self._is_max() and self._limits is not None: raise InvalidConfiguration("limits are not allowed with max types") requires_opendss_classes = ( "ExportLoadingsMetric", "OverloadsMetricInMemory", "ExportPowersMetric", ) if elem_class == "CktElement" and self.name in requires_opendss_classes and not self._opendss_classes: raise InvalidConfiguration( f"Exporting {elem_class}.{self.name} requires that opendss_classes be specifed" )
def make_storage_container( self, path, prop, num_steps, max_chunk_bytes, values, **kwargs ): """Make a storage container. Returns ------- StorageFilterBase """ if prop.store_values_type not in STORAGE_TYPE_MAP: raise InvalidConfiguration(f"unsupported {prop.store_values_type}") elem_names = self._make_elem_names() cls = STORAGE_TYPE_MAP[prop.store_values_type] container = cls( self._hdf_store, path, prop, num_steps, max_chunk_bytes, values, elem_names, **kwargs, ) return container
def create(cls, controllers: list, settings: ProjectModel): """Create controllers. The circuit must be loaded in OpenDSS.""" solver = get_solver_from_simulation_type(settings) buses = OpenDSS.CreateBusObjects() elements, elements_by_class = OpenDSS.CreateDssObjects(buses) controllers_by_class = defaultdict(dict) for circuit_element_controller in controllers: controller_class = circuit_element_controller.get_controller_class( ) element_class = circuit_element_controller.get_element_class() for name in circuit_element_controller.element_names: element = elements.get(name) if element is None: raise InvalidConfiguration(f"{name} is not in the circuit") controller = controller_class( element, circuit_element_controller.controller_model.dict( by_alias=True), dss, elements, solver, ) controllers_by_class[element_class]["Controller." + name] = controller return cls( controllers_by_class, solver, settings.max_control_iterations, settings.error_tolerance, )
def serialize(self): """Serialize object to a dictionary.""" if self._are_names_regex: #raise InvalidConfiguration("cannot serialize when names are regex") logger.warning("cannot serialize when names are regex") names = None else: names = self._names data = { "property": self.name, "data_conversion": self._data_conversion.value, "opendss_classes": self._opendss_classes, "sample_interval": self._sample_interval, "names": names, "publish": self.publish, "store_values_type": self.store_values_type.value, "sum_elements": self.sum_elements, "sum_groups": self.sum_groups, } if self._limits is not None: data["limits"] = [self._limits.min, self._limits.max] data["limits_filter"] = self._limits_filter.value if self._limits_b is not None: data["limits_b"] = [self._limits_b.min, self._limits_b.max] data["limits_filter_b"] = self._limits_filter_b.value if self.is_moving_average(): if self.window_sizes: data["window_sizes"] = self._window_sizes if not self._opendss_classes: raise InvalidConfiguration( f"window_sizes requires opendss_classes: {self.name}") else: data["window_size"] = self._window_size return data
def run_scenario(self, project, scenario, settings: SimulationSettingsModel, dry_run=False): if dry_run: dss = OpenDSS(settings) self._dump_scenario_simulation_settings(settings) #dss.init(dss_args) logger.info('Dry run scenario: %s', settings.project.active_scenario) if settings.monte_carlo.num_scenarios > 0: raise InvalidConfiguration( "Dry run does not support MonteCarlo simulation.") else: self._estimated_space = dss.DryRunSimulation(project, scenario) return None, None opendss = OpenDSS(settings) self._dump_scenario_simulation_settings(settings) logger.info('Running scenario: %s', settings.project.active_scenario) if settings.monte_carlo.num_scenarios > 0: opendss.RunMCsimulation(project, scenario, samples=settings.monte_carlo.num_scenarios) else: for is_complete, _, _, _ in opendss.RunSimulation( project, scenario): if is_complete: break
def run_scenario(self, project, scenario, simulation_config, dry_run=False): dss_args = self.update_scenario_settings(simulation_config) self._dump_scenario_simulation_settings(dss_args) if dry_run: dss = dssInstance.OpenDSS(dss_args) logger.info('Dry run scenario: %s', dss_args["Project"]["Active Scenario"]) if dss_args["MonteCarlo"]["Number of Monte Carlo scenarios"] > 0: raise InvalidConfiguration( "Dry run does not support MonteCarlo simulation.") else: self._estimated_space = dss.DryRunSimulation(project, scenario) return None, None dss = dssInstance.OpenDSS(dss_args) logger.info('Running scenario: %s', dss_args["Project"]["Active Scenario"]) if dss_args["MonteCarlo"]["Number of Monte Carlo scenarios"] > 0: dss.RunMCsimulation(project, scenario, samples=dss_args["MonteCarlo"] ['Number of Monte Carlo scenarios']) else: dss.RunSimulation(project, scenario) return dss_args
def append_required_exports(exports, options): """Append export properties required by the configured reports. Parameters ---------- exports : ExportListReader options : dict Simulation options """ report_options = options.get("Reports") if report_options is None: return for report in report_options["Types"]: if not report["enabled"]: continue name = report["name"] if name not in REPORTS: raise InvalidConfiguration(f"{name} is not a valid report") required = REPORTS[name].get_required_reports() for elem_class, required_properties in required.items(): for req_prop in required_properties: found = False store_type = req_prop["store_values_type"] for prop in exports.list_element_properties(elem_class): if prop.name == req_prop["property"] and \ prop.store_values_type.value == store_type: found = True break if not found: exports.append_property(elem_class, req_prop) logger.debug("Add required property: %s %s", elem_class, req_prop)
def _GetActiveScenario(self): active_scenario = self._settings.project.active_scenario for scenario in self._settings.project.scenarios: if scenario.name == active_scenario: return scenario raise InvalidConfiguration( f"Active Scenario {active_scenario} is not present")
def _parse_limits(data): limits = data.get("limits") if limits is None: return None if not isinstance(limits, list) or len(limits) != 2: raise InvalidConfiguration(f"invalid limits format: {limits}") return MinMax(limits[0], limits[1])
def add_property(self, prop): """Add an instance of ExportListProperty for tracking.""" if prop.are_names_filtered != self._are_names_filtered: raise InvalidConfiguration(f"All properties for shared elements must have the same filters: " f"{self._elem_class.__name__} / {prop.name}.") existing = self._properties.get(prop.store_values_type) if existing is None: self._properties[prop.store_values_type] = prop elif prop != existing: raise InvalidParameter(f"{prop.store_values_type} is already stored")
def _parse_names(data): names = data.get("names") name_regexes = data.get("name_regexes") if names and name_regexes: raise InvalidConfiguration( f"names and name_regexes cannot both be set") for obj in (names, name_regexes): if obj is None: continue if not isinstance(obj, list) or not isinstance(obj[0], str): raise InvalidConfiguration(f"invalid name format: {obj}") if names: return set(names), False if name_regexes: return [re.compile(r"{}".format(x)) for x in name_regexes], True return None, False
def hdf_store(self): """Return the HDFStore Returns ------- pd.HDFStore """ if self._hdf_store is None: raise InvalidConfiguration("hdf_store is not defined") return self._hdf_store
def make_empty_storage_container(self, path, prop): """Make an empty storage container.""" if prop.store_values_type not in STORAGE_TYPE_MAP: raise InvalidConfiguration(f"unsupported {prop.store_values_type}") elem_names = self._make_elem_names() cls = STORAGE_TYPE_MAP[prop.store_values_type] values = [ValueByNumber(x.FullName, self.label(), 0.0) for x in self._dss_objs] container = cls( self._hdf_store, path, prop, 1, self._max_chunk_bytes, values, elem_names ) return container
def __init__(self, name, prop, value): super().__init__() assert not isinstance(value, list), str(value) self._name = name self._prop = prop self._value_type = type(value) if self._value_type == str: raise InvalidConfiguration( f"Data export feature does not support strings: name={name} prop={prop} value={value}" ) self._value = value
def __init__(self, registry_filename=None): if registry_filename is None: self._registry_filename = Path.home() / self._REGISTRY_FILENAME else: self._registry_filename = Path(registry_filename) self._controllers = {x: {} for x in CONTROLLER_TYPES} data = copy.deepcopy(DEFAULT_REGISTRY) for controller_type, controllers in DEFAULT_REGISTRY[ "Controllers"].items(): for controller in controllers: path = Path(controller["filename"]) if not path.exists(): raise InvalidConfiguration( f"Default controller file={path} does not exist") # This is written to work with legacy versions where default controllers were # written to the registry. if self._registry_filename.exists(): registered = load_data(self._registry_filename) to_delete = [] for controller_type, controllers in registered[ "Controllers"].items(): for i, controller in enumerate(controllers): path = Path(controller["filename"]) if not path.exists(): name = controller["name"] msg = f"The registry contains a controller with an invalid file. " \ f"Type={controller_type} name={name} file={path}.\nWould you like to " \ "delete it? (y/n) -> " response = input(msg).lower() if response == "y": to_delete.append((controller_type, i)) continue else: logger.error( "Exiting because the registry %s is invalid", self._registry_filename) sys.exit(1) if not self._is_default_controller(controller_type, controller["name"]): data["Controllers"][controller_type].append(controller) if to_delete: for ref in reversed(to_delete): registered["Controllers"][ref[0]].pop(ref[1]) backup = str(self._registry_filename) + ".bk" self._registry_filename.rename(backup) dump_data(registered, self._registry_filename, indent=2) logger.info("Fixed the registry and moved the original to %s", backup) for controller_type, controllers in data["Controllers"].items(): for controller in controllers: self._add_controller(controller_type, controller)
def fs_interface(self): """Return the interface object used to read files. Returns ------- PyDssFileSystemInterface """ if self._fs_intf is None: raise InvalidConfiguration("fs interface is not defined") return self._fs_intf
def get_load_shape_resolution_secs(): def func(): if dss.LoadShape.Name() == "default": return None return dss.LoadShape.SInterval() res = [x for x in iter_elements(dss.LoadShape, func) if x is not None] if len(set(res)) != 1: raise InvalidConfiguration( f"SInterval for all LoadShapes must be the same: {res}") return res[0]
def _check_scenarios(self): scenarios = self._list_scenario_names() if scenarios is None: return exp_scenarios = self.scenario_names exp_scenarios.sort() for scenario in exp_scenarios: if scenario not in scenarios: raise InvalidConfiguration( f"{scenario} is not a valid scenario. Valid scenarios: {scenarios}" )
def _SetSnapshotTimePoint(self, scenario): """Adjusts the time parameters based on the mode.""" p_settings = self._settings.project config = scenario.snapshot_time_point_selection_config mode = config.mode assert mode != SnapshotTimePointSelectionMode.NONE, mode if mode != SnapshotTimePointSelectionMode.NONE: if p_settings.simulation_type != SimulationType.QSTS: raise InvalidConfiguration( f"{mode} is only supported with QSTS simulations") # These settings have to be temporarily overridden because of the underlying # implementation to create a load shape dataframes.. orig_start = p_settings.start_time orig_duration = p_settings.simulation_duration_min if orig_duration != p_settings.step_resolution_sec / 60: raise InvalidConfiguration( "Simulation duration must be the same as resolution") try: p_settings.start_time = config.start_time p_settings.simulation_duration_min = config.search_duration_min new_start = get_snapshot_timepoint(self._settings, mode).strftime(DATE_FORMAT) p_settings.start_time = new_start self._Logger.info( "Changed simulation start time from %s to %s", orig_start, new_start, ) except Exception: p_settings.start_time = orig_start raise finally: p_settings.simulation_duration_min = orig_duration else: assert False, f"unsupported mode {mode}"
def DryRunSimulation(self, project, scenario): """Run one time point for getting estimated space.""" if not self._Options['Exports']['Log Results']: raise InvalidConfiguration("Log Reults must set to be True.") Steps, _, _ = self._dssSolver.SimulationSteps() self._Logger.info('Dry run simulation...') self.ResultContainer.InitializeDataStore(project.hdf_store, Steps) try: self.RunStep(0) finally: self.ResultContainer.FlushData() return self.ResultContainer.max_num_bytes()
def compute_chunk_count(num_columns, max_size, dtype, max_chunk_bytes=DEFAULT_MAX_CHUNK_BYTES): assert max_size > 0, f"max_size={max_size}" tmp = np.empty((1, num_columns), dtype=dtype) size_row = tmp.size * tmp.itemsize chunk_count = min(int(max_chunk_bytes / size_row), max_size) if chunk_count == 0: raise InvalidConfiguration( f"HDF Max Chunk Bytes is smaller than the size of a row. Please increase it. " \ f"max_chunk_bytes={max_chunk_bytes} num_columns={num_columns} " \ f"size_row={size_row}" ) return chunk_count
def serialize(self): """Serialize object to a dictionary.""" if self._are_names_regex: raise InvalidConfiguration("cannot serialize when names are regex") data = { "property": self.name, "sample_interval": self._sample_interval, "names": self._names, "publish": self.publish, "store_values_type": self.store_values_type.value, } if self._limits is not None: data["limits"] = [self._limits.min, self._limits.max] data["limits_filter"] = self._limits_filter.value if self._store_values_type == StoreValuesType.MOVING_AVERAGE: data["moving_average_store_interval"] = self._ma_store_interval data["window_size"] = self._window_size return data
def _export_pv_profiles(self): dss = self._dss_instance pv_systems = self._objects_by_class.get("PVSystems") if pv_systems is None: raise InvalidConfiguration("PVSystems are not exported") pv_infos = [] profiles = set() for full_name, obj in pv_systems.items(): profile_name = obj.GetParameter("yearly").lower() if profile_name != "": profiles.add(profile_name) pv_infos.append({ "irradiance": obj.GetParameter("irradiance"), "name": full_name, "pmpp": obj.GetParameter("pmpp"), "load_shape_profile": profile_name, }) pmult_sums = {} dss.LoadShape.First() sim_resolution = self._options["Project"]["Step resolution (sec)"] while True: name = dss.LoadShape.Name().lower() if name in profiles: sinterval = dss.LoadShape.SInterval() assert sim_resolution >= sinterval offset = int(sim_resolution / dss.LoadShape.SInterval()) pmult_sums[name] = sum(dss.LoadShape.PMult()[::offset]) if dss.LoadShape.Next() == 0: break for pv_info in pv_infos: profile = pv_info["load_shape_profile"] if profile == "": pv_info["load_shape_pmult_sum"] = 0 else: pv_info["load_shape_pmult_sum"] = pmult_sums[profile] data = {"pv_systems": pv_infos} filename = os.path.join(self._export_dir, "pv_profiles.json") dump_data(data, filename, indent=2) self._logger.info("Exported PV profile information to %s", filename)
def get_hdf_store_filename(self): """Return the HDFStore filename. Returns ------- str Path to the HDFStore. Raises ------ InvalidConfiguration Raised if no store exists. """ filename = os.path.join(self._project_dir, STORE_FILENAME) if not os.path.exists(filename): raise InvalidConfiguration(f"HDFStore does not exist") return filename
def read_scenario_export_metadata(self, scenario_name): """Return the metadata for a scenario's exported data. Parameters ---------- scenario_name : str Returns ------- dict """ if self._fs_intf is None: raise InvalidConfiguration("pydss fs interface is not defined") if scenario_name not in self.list_scenario_names(): raise InvalidParameter(f"invalid scenario: {scenario_name}") return self._fs_intf.read_scenario_export_metadata(scenario_name)