def GetVariable(self, VarName, convert=False): if VarName not in self._Variables: raise InvalidParameter( f'{VarName} is an invalid variable name for element {self._FullName}' ) self.SetActiveObject() func = self._Variables[VarName] if func is None: print(func, VarName) raise InvalidParameter( f"get function for {self._FullName} / {VarName} is None") value = func() if not convert: return value if VarName in self.VARIABLE_OUTPUTS_BY_LABEL: info = self.VARIABLE_OUTPUTS_BY_LABEL[VarName] is_complex = info["is_complex"] units = info["units"] return ValueByLabel(self._FullName, VarName, value, self._Nodes, is_complex, units) elif VarName in self.VARIABLE_OUTPUTS_COMPLEX: assert isinstance(value, list) and len(value) == 2, str(value) value = complex(value[0], value[1]) return ValueByNumber(self._FullName, VarName, value)
def get_summed_element_total(self, element_class, prop, group=None): """Return the total value for a summed element property. Parameters ---------- element_class : str prop : str group : str | None Specify a group name if sum_groups was assigned. Returns ------- dict Raises ------ InvalidParameter Raised if the element class is not stored. """ if group is not None: prop = ValueStorageBase.DELIMITER.join((prop, group)) if element_class not in self._summed_elem_props: raise InvalidParameter(f"{element_class} is not stored") if prop not in self._summed_elem_props[element_class]: raise InvalidParameter(f"{prop} is not stored") return self._summed_elem_props[element_class][prop]
def GetVariable(self, VarName, convert=False): if VarName not in self._Variables: raise InvalidParameter( f'{VarName} is an invalid variable name for element {self._FullName}' ) if self._dssInstance.Element.Name() != self._FullName: self.SetActiveObject() func = self._Variables[VarName] if func is None: raise InvalidParameter( f"get function for {self._FullName} / {VarName} is None") value = func() if not convert: return value if VarName in self.VARIABLE_OUTPUTS_BY_LABEL: info = self.VARIABLE_OUTPUTS_BY_LABEL[VarName] is_complex = info["is_complex"] units = info["units"] return ValueByLabel(self._FullName, VarName, value, self._Nodes, is_complex, units) elif VarName in self.VARIABLE_OUTPUTS_COMPLEX: assert isinstance(value, list) and len(value) == 2, str(value) value = complex(value[0], value[1]) elif VarName in self.VARIABLE_OUTPUTS_BY_LIST: assert isinstance(value, list), str(value) labels = [f"_bus_index_{i}" for i in range(len(value))] return ValueByList(self._FullName, VarName, value, labels) return ValueByNumber(self._FullName, VarName, value)
def read_report(self, report_name): """Return the report data. Parameters ---------- report_name : str Returns ------- str """ all_reports = Reports.get_all_reports() if report_name not in all_reports: raise InvalidParameter(f"invalid report name {report_name}") report_cls = all_reports[report_name] # This bypasses self._fs_intf because reports are always extracted. reports_dir = os.path.join(self._project.project_path, REPORTS_DIR) for filename in os.listdir(reports_dir): name, ext = os.path.splitext(filename) if name == os.path.splitext(report_cls.FILENAME)[0]: path = os.path.join(reports_dir, filename) if ext in (".json", ".toml"): return load_data(path) if ext in (".csv", ".h5"): return read_dataframe(path) raise InvalidParameter( f"did not find report {report_name} in {reports_dir}")
def get_columns(df, names, options, **kwargs): """Return the column names in the dataframe that match names and kwargs. Parameters ---------- df : pd.DataFrame names : str | list single name or list of names kwargs : dict Filter on options; values can be strings or regular expressions. Returns ------- list """ if isinstance(names, str): names = set([names]) elif isinstance(names, set): pass else: names = set(names) field_indices = {option: i + 1 for i, option in enumerate(options)} columns = [] for column in df.columns: col = column index = column.find(" [") if index != -1: col = column[:index] # [name, option1, option2, ...] fields = ValueStorageBase.get_fields(col, next(iter(names))) if options and kwargs: assert len(fields) == 1 + len( options), f"fields={fields} options={options}" _name = fields[0] if _name not in names: continue match = True for key, val in kwargs.items(): if isinstance(val, str): if fields[field_indices[key]] != val: match = False elif isinstance(val, re.Pattern): if val.search(fields[field_indices[key]]) is None: match = False elif val is None: continue else: raise InvalidParameter(f"unhandled option value '{val}'") if not match: break if match: columns.append(column) if not columns: raise InvalidParameter(f"{names} does not exist in DataFrame") return columns
def get_element_property_number(self, element_class, prop, element_name): """Return the number stored for the element property.""" if element_class not in self._elem_prop_nums: raise InvalidParameter(f"{element_class} is not stored") if prop not in self._elem_prop_nums[element_class]: raise InvalidParameter(f"{prop} is not stored") if element_name not in self._elem_prop_nums[element_class][prop]: raise InvalidParameter(f"{element_name} is not stored") return self._elem_prop_nums[element_class][prop][element_name]
def add_profiles(self, data, name, pType, startTime, resolution_sec=900, units="", info=""): if type(startTime) is not datetime.datetime: raise InvalidParameter("startTime should be a python datetime object") if pType not in PROFILE_TYPES.names(): raise InvalidParameter("Valid values for pType are {}".format(PROFILE_TYPES.names())) if data: self.add_from_arrays(data, name, pType, startTime, resolution_sec, units=units, info=info) self.store.flush() return
def _add_controller(self, controller_type, controller): name = controller["name"] filename = controller["filename"] if self.is_controller_registered(controller_type, name): raise InvalidParameter( f"{controller_type} / {name} is already registered") if not os.path.exists(filename): raise InvalidParameter(f"{filename} does not exist.") # Make sure the file can be parsed. load_data(filename) self._controllers[controller_type][name] = controller
def get_summed_element_dataframe(self, element_class, prop, real_only=False, abs_val=False, group=None): """Return the dataframe for a summed element property. Parameters ---------- element_class : str prop : str group : str | None Specify a group name if sum_groups was assigned. real_only : bool If dtype of any column is complex, drop the imaginary component. abs_val : bool If dtype of any column is complex, compute its absolute value. Returns ------- pd.DataFrame Raises ------ InvalidParameter Raised if the element class is not stored. """ if group is not None: prop = ValueStorageBase.DELIMITER.join((prop, group)) if element_class not in self._summed_elem_timeseries_props: raise InvalidParameter(f"{element_class} is not stored") if prop not in self._summed_elem_timeseries_props[element_class]: raise InvalidParameter(f"{prop} is not stored") elem_group = self._group[element_class]["SummedElementProperties"] dataset = elem_group[prop] df = DatasetBuffer.to_dataframe(dataset) self._add_indices_to_dataframe(df) if real_only: for column in df.columns: if df[column].dtype == complex: df[column] = np.real(df[column]) elif abs_val: for column in df.columns: if df[column].dtype == complex: df[column] = df[column].apply(np.absolute) return df
def get_units(prop, index=None): units = unit_info.get(prop) if units is None: raise InvalidParameter(f"no units are stored for {prop}") if isinstance(units, dict): if index is None: raise InvalidParameter(f"index must be provided for {prop}") if index == 0: return units["E"] if index == 1: return units["O"] raise InvalidParameter("index must be 0 or 1") return units
def __init__(self, Path): self.pyPlots = {} filenames = os.listdir(Path) found_config_file = False found_excel_file = False for filename in filenames: pyPlotType, ext = os.path.splitext(filename) if filename.startswith('~$'): continue elif ext == '.xlsx': filename = convert_config_data_to_toml(filename) elif ext != ".toml": continue if pyPlotType not in self.pyPlots: self.pyPlots[pyPlotType] = {} filepath = os.path.join(Path, filename) assert (os.path.exists(filepath) ), 'path: "{}" does not exist!'.format(filepath) assert (os.path.exists(filepath) ), 'path: "{}" does not exist!'.format(filepath) for name, plot in load_data(filepath).items(): if name in self.pyPlots[pyPlotType]: raise InvalidParameter( f"Multiple PyDSS dynamic plot definitions of the same type with the same name not allowed: " f"{name} already exists for plot type {pyPlotType}") self.pyPlots[pyPlotType][name] = plot
def SetVariable(self, VarName, Value): if self._dssInstance.Element.Name() != self._FullName: self.SetActiveObject() if VarName not in self._Variables: raise InvalidParameter(f"invalid variable name {VarName}") return self._Variables[VarName](Value)
def get_option_values(df, name): """Return the option values parsed from the column names. Parameters ---------- df : pd.DataFrame name : str Returns ------- list """ values = [] for column in df.columns: col = column index = column.find(" [") if index != -1: col = column[:index] # [name, option1, option2, ...] fields = col.split(ValueStorageBase.DELIMITER) _name = fields[0] if _name != name: continue values += fields[1:] if not values: raise InvalidParameter(f"{name} does not exist in DataFrame") return values
def read_element_info_file(self, filename): """Return the contents of file describing an OpenDSS element object. Parameters ---------- filename : str full path to a file (returned by list_element_info_files) or an element class, like "Transformers" Returns ------- pd.DataFrame """ if "." not in filename: actual = None for _file in self.list_element_info_files(): basename = os.path.splitext(os.path.basename(_file))[0] if basename.replace("Info", "") == filename: actual = _file if actual is None: raise InvalidParameter( f"element info file for {filename} is not stored") filename = actual return self._fs_intf.read_csv(filename)
def get_element_property_value(self, element_class, prop, element_name): """Return the number stored for the element property.""" if element_class not in self._elem_values_by_prop: raise InvalidParameter(f"{element_class} is not stored") if prop not in self._elem_values_by_prop[element_class]: raise InvalidParameter(f"{prop} is not stored") if element_name not in self._elem_values_by_prop[element_class][prop]: raise InvalidParameter(f"{element_name} is not stored") dataset = self._group[f"{element_class}/ElementProperties/{prop}"] col_range = self._get_element_column_range(element_class, prop, element_name) start = col_range[0] length = col_range[1] if length == 1: return dataset[:][0][start] return dataset[:][0][start:start + length]
def _create_exports(self): elements = {} # element name to ElementData for elem_class in self._export_list.list_element_classes(): if elem_class == "Buses": objs = self._buses elif elem_class in self._objects_by_class: objs = self._objects_by_class[elem_class] else: continue for name, obj in objs.items(): if not obj.Enabled: continue for prop in self._export_list.iter_export_properties( elem_class=elem_class): if prop.custom_function is None and not obj.IsValidAttribute( prop.name): raise InvalidParameter( f"{name} / {prop.name} cannot be exported") if prop.should_store_name(name): if name not in elements: elements[name] = ElementData( name, obj, max_chunk_bytes=self._max_chunk_bytes, options=self._options) elements[name].append_property(prop) self._logger.debug("Store %s %s name=%s", elem_class, prop.name, name) self._elements = elements.values()
def _check_input_fields(self): required_fields = self._get_required_input_fields() fields = set(self.config.keys()) for field in required_fields: if field not in fields: raise InvalidParameter( f"{self.__class__.__name__} requires input field {field}")
def __init__(self, name, controller_types=None, controllers=None, export_modes=None, exports=None, visualizations=None, post_process_infos=None, visualization_types=None): self.name = name self.post_process_infos = [] if visualization_types is None and visualizations is None: self.visualizations = { x: self.load_visualization_config_from_type(x) for x in PyDssScenario.DEFAULT_VISUALIZATION_TYPES } elif visualization_types is not None: self.visualizations = { x: self.load_visualization_config_from_type(x) for x in visualization_types } elif isinstance(visualizations, str): basename = os.path.splitext(os.path.basename(visualizations))[0] visualization_type = VisualizationType(basename) self.visualizations = {visualization_type: load_data(controllers)} else: assert isinstance(visualizations, dict) self.visualizations = visualizations if (controller_types is None and controllers is None): self.controllers = {} elif controller_types is not None: self.controllers = { x: self.load_controller_config_from_type(x) for x in controller_types } elif isinstance(controllers, str): basename = os.path.splitext(os.path.basename(controllers))[0] controller_type = ControllerType(basename) self.controllers = {controller_type: load_data(controllers)} else: assert isinstance(controllers, dict) self.controllers = controllers if export_modes is not None and exports is not None: raise InvalidParameter( "export_modes and exports cannot both be set" ) if (export_modes is None and exports is None): mode = PyDssScenario.DEFAULT_EXPORT_MODE self.exports = {mode: self.load_export_config_from_mode(mode)} elif export_modes is not None: self.exports = { x: self.load_export_config_from_mode(x) for x in export_modes } elif isinstance(exports, str): mode = ExportMode(os.path.splitext(os.path.basename(exports))[0]) self.exports = {mode: load_data(exports)} else: assert isinstance(exports, dict) self.exports = exports if post_process_infos is not None: for pp_info in post_process_infos: self.add_post_process(pp_info)
def get_dataframe(self, element_class, prop, element_name, real_only=False, **kwargs): """Return the dataframe for an element. Parameters ---------- element_class : str prop : str element_name : str real_only : bool If dtype of any column is complex, drop the imaginary component. kwargs : **kwargs Filter on options. Option values can be strings or regular expressions. Returns ------- pd.DataFrame Raises ------ InvalidParameter Raised if the element is not stored. """ if element_name not in self._elem_props: raise InvalidParameter(f"element {element_name} is not stored") elem_group = self._group[element_class][element_name] dataset = elem_group[prop] df = DatasetBuffer.to_dataframe(dataset) if kwargs: options = self._check_options(element_class, prop, **kwargs) columns = ValueStorageBase.get_columns(df, element_name, options, **kwargs) df = df[columns] if self._data_format_version == "1.0.0": dataset_property_type = DatasetPropertyType.ELEMENT_PROPERTY else: dataset_property_type = get_dataset_property_type(dataset) if dataset_property_type == DatasetPropertyType.FILTERED: timestamp_path = get_timestamp_path(dataset) timestamp_dataset = self._hdf_store[timestamp_path] df["Timestamp"] = DatasetBuffer.to_datetime(timestamp_dataset) df.set_index("Timestamp", inplace=True) else: self._add_indices_to_dataframe(df) if real_only: for column in df.columns: if df[column].dtype == np.complex: df[column] = [x.real for x in df[column]] return df
def unregister_controller(self, controller_type, name): """Unregisters a controller. Parameters ---------- controller_type : str name : str """ if not self.is_controller_registered(controller_type, name): raise InvalidParameter( f"{controller_type} / {name} isn't registered") if self._is_default_controller(controller_type, name): raise InvalidParameter(f"Cannot unregister a default controller") self._controllers[controller_type].pop(name) self._serialize_registry()
def _check_options(self, element_class, prop, **kwargs): """Checks that kwargs are valid and returns available option names.""" for option in kwargs: if not self._options.is_option_valid(element_class, prop, option): raise InvalidParameter( f"class={element_class} property={prop} option={option} is invalid" ) return self._options.list_options(element_class, prop)
def get_solver_from_simulation_type(settings: ProjectModel): """Return a solver from the simulation type.""" if settings.simulation_type == SimulationType.SNAPSHOT: return Snapshot(dssInstance=dss, settings=settings) elif settings.simulation_type == SimulationType.QSTS: return QSTS(dssInstance=dss, settings=settings) elif settings.simulation_type == SimulationType.DYNAMIC: return Dynamic(dssInstance=dss, settings=settings) raise InvalidParameter( f"{settings.simulation_type} does not have a supported solver")
def add_property(self, prop): """Add an instance of ExportListProperty for tracking.""" if prop.are_names_filtered != self._are_names_filtered: raise InvalidConfiguration(f"All properties for shared elements must have the same filters: " f"{self._elem_class.__name__} / {prop.name}.") existing = self._properties.get(prop.store_values_type) if existing is None: self._properties[prop.store_values_type] = prop elif prop != existing: raise InvalidParameter(f"{prop.store_values_type} is already stored")
def add_post_process(self, post_process_info): """Add a post-process script to a scenario. Parameters ---------- post_process_info : dict Must define all fields in PyDssScenario.REQUIRED_POST_PROCESS_FIELDS """ for field in self.REQUIRED_POST_PROCESS_FIELDS: if field not in post_process_info: raise InvalidParameter(f"missing post-process field={field}") config_file = post_process_info["config_file"] if not os.path.exists(config_file): raise InvalidParameter(f"{config_file} does not exist") self.post_process_infos.append(post_process_info) logger.info("Appended post-process script %s to %s", post_process_info["script"], self.name)
def __init__(self, project_path=None, project=None, in_memory=False, frequency=False, mode=False): """Constructs PyDssResults object. Parameters ---------- project_path : str | None Load project from files in path project : PyDssProject | None Existing project object in_memory : bool If true, load all exported data into memory. frequency : bool If true, add frequency column to all dataframes. mode : bool If true, add mode column to all dataframes. """ options = ElementOptions() if project_path is not None: # TODO: handle old version? self._project = PyDssProject.load_project( project_path, simulation_file=RUN_SIMULATION_FILENAME, ) elif project is None: raise InvalidParameter("project_path or project must be set") else: self._project = project self._fs_intf = self._project.fs_interface self._scenarios = [] filename = self._project.get_hdf_store_filename() driver = "core" if in_memory else None self._hdf_store = h5py.File(filename, "r", driver=driver) if self._project.simulation_config.exports.export_results: for name in self._project.list_scenario_names(): metadata = self._project.read_scenario_export_metadata(name) scenario_result = PyDssScenarioResults( name, self.project_path, self._hdf_store, self._fs_intf, metadata, options, frequency=frequency, mode=mode, ) self._scenarios.append(scenario_result)
def get_dataframe(self, element_class, prop, element_name, real_only=False, abs_val=False, **kwargs): """Return the dataframe for an element. Parameters ---------- element_class : str prop : str element_name : str real_only : bool If dtype of any column is complex, drop the imaginary component. abs_val : bool If dtype of any column is complex, compute its absolute value. kwargs Filter on options; values can be strings or regular expressions. Returns ------- pd.DataFrame Raises ------ InvalidParameter Raised if the element is not stored. """ if element_name not in self._elem_props: raise InvalidParameter(f"element {element_name} is not stored") dataset = self._group[f"{element_class}/ElementProperties/{prop}"] prop_type = get_dataset_property_type(dataset) if prop_type == DatasetPropertyType.PER_TIME_POINT: return self._get_elem_prop_dataframe(element_class, prop, element_name, dataset, real_only=real_only, abs_val=abs_val, **kwargs) elif prop_type == DatasetPropertyType.FILTERED: return self._get_filtered_dataframe(element_class, prop, element_name, dataset, real_only=real_only, abs_val=abs_val, **kwargs) assert False, str(prop_type)
def __init__(self, value, hdf_store, path, max_size, dataset_property_type, max_chunk_bytes=None, store_timestamp=False): group_name = os.path.dirname(path) basename = os.path.basename(path) try: if basename in hdf_store[group_name].keys(): raise InvalidParameter(f"duplicate dataset name {basename}") except KeyError: # Don't bother checking each sub path. pass dtype = self._TYPE_MAPPING.get(value.value_type) assert dtype is not None scaleoffset = None if dtype == np.float: scaleoffset = 4 elif dtype == np.int: scaleoffset = 0 attributes = {"type": dataset_property_type.value} timestamp_path = None if store_timestamp: timestamp_path = self.timestamp_path(path) self._timestamps = DatasetBuffer( hdf_store, timestamp_path, max_size, np.float, ["Timestamp"], scaleoffset=scaleoffset, max_chunk_bytes=max_chunk_bytes, attributes={"type": DatasetPropertyType.TIMESTAMP.value}, ) attributes["timestamp_path"] = timestamp_path else: self._timestamps = None self._dataset = DatasetBuffer( hdf_store, path, max_size, dtype, value.make_columns(), scaleoffset=scaleoffset, max_chunk_bytes=max_chunk_bytes, attributes=attributes, )
def run(self, logging_configured=True, tar_project=False, zip_project=False, dry_run=False): """Run all scenarios in the project.""" if isinstance(self._fs_intf, PyDssArchiveFileInterfaceBase): raise InvalidConfiguration("cannot run from an archived project") if tar_project and zip_project: raise InvalidParameter("tar_project and zip_project cannot both be True") if self._simulation_config['Project']['DSS File'] == "": raise InvalidConfiguration("a valid opendss file needs to be passed") inst = instance() self._simulation_config["Logging"]["Pre-configured logging"] = logging_configured if dry_run: store_filename = os.path.join(tempfile.gettempdir(), STORE_FILENAME) else: store_filename = os.path.join(self._project_dir, STORE_FILENAME) driver = None if self._simulation_config["Exports"].get("Export Data In Memory", True): driver = "core" with h5py.File(store_filename, mode="w", driver=driver) as hdf_store: self._hdf_store = hdf_store self._hdf_store.attrs["version"] = DATA_FORMAT_VERSION for scenario in self._scenarios: self._simulation_config["Project"]["Active Scenario"] = scenario.name inst.run(self._simulation_config, self, scenario, dry_run=dry_run) self._estimated_space[scenario.name] = inst.get_estimated_space() if not dry_run: results = None export_tables = self._simulation_config["Exports"].get( "Export Data Tables", False ) generate_reports = self._simulation_config.get("Reports", False) if export_tables or generate_reports: # Hack. Have to import here. Need to re-organize to fix. from PyDSS.pydss_results import PyDssResults results = PyDssResults(self._project_dir) if export_tables: for scenario in results.scenarios: scenario.export_data() if generate_reports: results.generate_reports() if tar_project: self._tar_project_files() elif zip_project: self._zip_project_files() if dry_run and os.path.exists(store_filename): os.remove(store_filename)
def get_full_dataframe(self, element_class, prop, real_only=False): """Return a dataframe containing all data. The dataframe is copied. Parameters ---------- element_class : str prop : str real_only : bool If dtype of any column is complex, drop the imaginary component. Returns ------- pd.DataFrame """ if prop not in self.list_element_properties(element_class): raise InvalidParameter(f"property {prop} is not stored") master_df = None length = None for _, df in self.iterate_dataframes(element_class, prop, real_only=real_only): cur_len = len(df) if master_df is None: master_df = df length = cur_len else: if cur_len != length: raise InvalidParameter( "cannot create full dataframe when elements have different indices" ) for column in ("Frequency", "Simulation Mode"): if column in df.columns: df.drop(column, axis=1, inplace=True) master_df = master_df.join(df) return master_df
def track_reg_control_tap_number_changes( reg_control, timestamp, step_number, options, last_value, count ): reg_control.dss.RegControls.Name(reg_control.Name) if reg_control.dss.CktElement.Name() != reg_control.dss.Element.Name(): raise InvalidParameter( f"Object is not a circuit element {reg_control.Name()}" ) tap_number = reg_control.dss.RegControls.TapNumber() if last_value is not None: count += abs(tap_number - last_value) logger.debug("%s changed count from %s to %s count=%s", reg_control.Name, last_value, tap_number, count) return tap_number, count