def run_with_base_class(): client = IncoreClient(pyglobals.INCORE_API_DEV_URL) client.clear_cache() pipeline_restoration = PipelineRestoration(client) # shelby county pipelines pipeline_restoration.load_remote_input_dataset("pipeline", "5a284f28c7d30d13bc081d14") pipeline_restoration.load_remote_input_dataset("pipeline_damage", "61f36023c53b3620b6b614c6") # Load fragility mapping restoration_service = RestorationService(client) mapping_set = MappingSet( restoration_service.get_mapping("61f35f09903e515036cee106")) pipeline_restoration.set_input_dataset('dfr3_mapping_set', mapping_set) pipeline_restoration.set_parameter("result_name", "pipeline_restoration_times") pipeline_restoration.set_parameter("restoration_key", "Restoration ID Code") pipeline_restoration.set_parameter("num_available_workers", 4) pipeline_restoration.set_parameter("num_cpu", 4) # Run pipeline restoration analysis pipeline_restoration.run_analysis()
def run_with_base_class(): client = IncoreClient(pyglobals.INCORE_API_DEV_URL) epf_rest = EpfRestoration(client) restorationsvc = RestorationService(client) mapping_set = MappingSet( restorationsvc.get_mapping( "61f302e6e3a03e465500b3eb")) # new format of mapping epf_rest.load_remote_input_dataset('epfs', '6189c103d5b02930aa3efc35') epf_rest.set_input_dataset('dfr3_mapping_set', mapping_set) epf_rest.set_parameter("result_name", "epf_restoration.csv") epf_rest.set_parameter("restoration_key", "Restoration ID Code") epf_rest.set_parameter("end_time", 100.0) epf_rest.set_parameter("time_interval", 1.0) epf_rest.set_parameter("pf_interval", 0.01) epf_rest.run_analysis() # test utility function epf_rest_util = EpfRestorationUtil(epf_rest) functionality = epf_rest_util.get_percentage_func( guid="60748fbd-67c3-4f8d-beb9-26685a53d3c5", damage_state="DS_0", time=2.0) time = epf_rest_util.get_restoration_time( guid="60748fbd-67c3-4f8d-beb9-26685a53d3c5", damage_state="DS_1", pf=0.81) print(functionality, time)
def run_with_base_class(): client = IncoreClient(pyglobals.INCORE_API_DEV_URL) wf_rest = WaterFacilityRestoration(client) # Load restoration mapping restorationsvc = RestorationService(client) mapping_set = MappingSet( restorationsvc.get_mapping( "61f075ee903e515036cee0a5")) # new format of mapping wf_rest.load_remote_input_dataset( "water_facilities", "5a284f2ac7d30d13bc081e52") # water facility wf_rest.set_input_dataset('dfr3_mapping_set', mapping_set) wf_rest.set_parameter("result_name", "wf_restoration") wf_rest.set_parameter("restoration_key", "Restoration ID Code") wf_rest.set_parameter("end_time", 100.0) wf_rest.set_parameter("time_interval", 1.0) wf_rest.set_parameter("pf_interval", 0.05) wf_rest.run_analysis() # test utility function wf_util = WaterFacilityRestorationUtil(wf_rest) functionality = wf_util.get_percentage_func( guid="e1bce78d-00a1-4605-95f3-3776ff907f73", damage_state="DS_0", time=2.0) time = wf_util.get_restoration_time( guid="e1bce78d-00a1-4605-95f3-3776ff907f73", damage_state="DS_1", pf=0.81) print(functionality, time)
def waterfacility_restoration(self, mapping_set, restoration_key, end_time, time_interval, pf_interval): """Gets applicable restoration curve set and calculates restoration time and functionality Args: mapping_set (class): Restoration Mapping Set restoration_key (str): Restoration Key to determine which curve to use. E.g. Restoration ID Code end_time (float): User specified end repair time time_interval (float): Increment interval of repair time. Default to 1 (1 day) pf_interval (float): Increment interval of percentage of functionality. Default 0.1 (10%) Returns: time_results (list): Given Percentage of functionality, the change of repair time pf_results (list): Given Repair time, change of the percentage of functionality """ time_results = [] pf_results = [] for mapping in mapping_set.mappings: # parse rules to get inventory class. e.g. treatment plan, tank, pump etc if isinstance(mapping.rules, list): inventory_class = RestorationService.extract_inventory_class_legacy(mapping.rules) elif isinstance(mapping.rules, dict): inventory_class = RestorationService.extract_inventory_class(mapping.rules) else: raise ValueError("Unsupported mapping rules!") # get restoration curves # if it's string:id; then need to fetch it from remote and cast to restorationcurveset object restoration_curve_set = mapping.entry[restoration_key] if isinstance(restoration_curve_set, str): restoration_curve_set = RestorationCurveSet(self.restorationsvc.get_dfr3_set(restoration_curve_set)) # given time calculate pf time = np.arange(0, end_time + time_interval, time_interval) for t in time: pf_results.append({ "inventory_class": inventory_class, "time": t, **restoration_curve_set.calculate_restoration_rates(time=t) }) # given pf calculate time pf = np.arange(0, 1 + pf_interval, pf_interval) for p in pf: new_dict = {} t_res = restoration_curve_set.calculate_inverse_restoration_rates(time=p) for key, value in t_res.items(): new_dict.update({"time_" + key: value}) time_results.append({ "inventory_class": inventory_class, "percentage_of_functionality": p, **new_dict }) return pf_results, time_results
def run_with_base_class(): client = IncoreClient(pyglobals.INCORE_API_DEV_URL) epf_rest = ElectricPowerFacilityRestoration(client) restorationsvc = RestorationService(client) mapping_set = MappingSet( restorationsvc.get_mapping( "61f302e6e3a03e465500b3eb")) # new format of mapping epf_rest.set_input_dataset('dfr3_mapping_set', mapping_set) epf_rest.set_parameter("result_name", "epf_restoration.csv") epf_rest.set_parameter("restoration_key", "Restoration ID Code") epf_rest.set_parameter("end_time", 100.0) epf_rest.set_parameter("time_interval", 1.0) epf_rest.set_parameter("pf_interval", 0.01) epf_rest.run_analysis()
def pytest_sessionstart(session): """ Called after the Session object has been created and before performing collection and entering the run test loop. """ try: with open( os.path.join(os.path.dirname(__file__), "pyincore/.incorepw"), 'r') as f: cred = f.read().splitlines() except EnvironmentError: assert False credentials = jwt.decode(cred[0], cred[1]) monkeypatch = MonkeyPatch() monkeypatch.setattr("builtins.input", lambda x: credentials["username"]) monkeypatch.setattr("getpass.getpass", lambda y: credentials["password"]) client = IncoreClient(service_url=pyglobals.INCORE_API_DEV_URL, token_file_name=".incrtesttoken") pytest.client = client pytest.datasvc = DataService(client) pytest.fragilitysvc = FragilityService(client) pytest.repairsvc = RepairService(client) pytest.restorationsvc = RestorationService(client) pytest.hazardsvc = HazardService(client) pytest.spacesvc = SpaceService(client) print( f"Successfully initialized Incore client and services. Using {pyglobals.INCORE_API_DEV_URL}" )
def run_with_base_class(): client = IncoreClient(pyglobals.INCORE_API_DEV_URL) wf_rest = WaterFacilityRestoration(client) # Load restoration mapping restorationsvc = RestorationService(client) mapping_set = MappingSet( restorationsvc.get_mapping( "61f075ee903e515036cee0a5")) # new format of mapping wf_rest.set_input_dataset('dfr3_mapping_set', mapping_set) wf_rest.set_parameter("result_name", "wf_restoration") wf_rest.set_parameter("restoration_key", "Restoration ID Code") wf_rest.set_parameter("end_time", 100.0) wf_rest.set_parameter("time_interval", 1.0) wf_rest.set_parameter("pf_interval", 0.05) wf_rest.run_analysis()
def __init__(self, incore_client): self.restorationsvc = RestorationService(incore_client) super(WaterFacilityRestoration, self).__init__(incore_client)
class WaterFacilityRestoration(BaseAnalysis): """Computes water facility restoration for an earthquake, tsunami, tornado, or hurricane exposure. """ def __init__(self, incore_client): self.restorationsvc = RestorationService(incore_client) super(WaterFacilityRestoration, self).__init__(incore_client) def run(self): """Performs Water facility restoration analysis by using the parameters from the spec and creates an output dataset in csv format Returns: bool: True if successful, False otherwise """ mapping_set = self.get_input_dataset("dfr3_mapping_set") restoration_key = self.get_parameter("restoration_key") if restoration_key is None: restoration_key = "Restoration ID Code" end_time = self.get_parameter("end_time") if end_time is None: end_time = 365.0 time_interval = self.get_parameter("time_interval") if time_interval is None: time_interval = 1 pf_interval = self.get_parameter("pf_interval") if pf_interval is None: pf_interval = 0.05 (pf_results, time_results) = self.waterfacility_restoration(mapping_set, restoration_key, end_time, time_interval, pf_interval) self.set_result_csv_data("pf_results", time_results, name="percentage_of_functionality_" + self.get_parameter("result_name")) self.set_result_csv_data("time_results", pf_results, name="reptime_" + self.get_parameter("result_name")) return True def waterfacility_restoration(self, mapping_set, restoration_key, end_time, time_interval, pf_interval): """Gets applicable restoration curve set and calculates restoration time and functionality Args: mapping_set (class): Restoration Mapping Set restoration_key (str): Restoration Key to determine which curve to use. E.g. Restoration ID Code end_time (float): User specified end repair time time_interval (float): Increment interval of repair time. Default to 1 (1 day) pf_interval (float): Increment interval of percentage of functionality. Default 0.1 (10%) Returns: time_results (list): Given Percentage of functionality, the change of repair time pf_results (list): Given Repair time, change of the percentage of functionality """ time_results = [] pf_results = [] for mapping in mapping_set.mappings: # parse rules to get inventory class. e.g. treatment plan, tank, pump etc if isinstance(mapping.rules, list): inventory_class = RestorationService.extract_inventory_class_legacy(mapping.rules) elif isinstance(mapping.rules, dict): inventory_class = RestorationService.extract_inventory_class(mapping.rules) else: raise ValueError("Unsupported mapping rules!") # get restoration curves # if it's string:id; then need to fetch it from remote and cast to restorationcurveset object restoration_curve_set = mapping.entry[restoration_key] if isinstance(restoration_curve_set, str): restoration_curve_set = RestorationCurveSet(self.restorationsvc.get_dfr3_set(restoration_curve_set)) # given time calculate pf time = np.arange(0, end_time + time_interval, time_interval) for t in time: pf_results.append({ "inventory_class": inventory_class, "time": t, **restoration_curve_set.calculate_restoration_rates(time=t) }) # given pf calculate time pf = np.arange(0, 1 + pf_interval, pf_interval) for p in pf: new_dict = {} t_res = restoration_curve_set.calculate_inverse_restoration_rates(time=p) for key, value in t_res.items(): new_dict.update({"time_" + key: value}) time_results.append({ "inventory_class": inventory_class, "percentage_of_functionality": p, **new_dict }) return pf_results, time_results def get_spec(self): return { 'name': 'water-facility-restoration', 'description': 'water facility restoration analysis', 'input_parameters': [ { 'id': 'restoration_key', 'required': False, 'description': 'restoration key to use in mapping dataset', 'type': str }, { 'id': 'result_name', 'required': True, 'description': 'result dataset name', 'type': str }, { 'id': 'end_time', 'required': False, 'description': 'end time in days. Default to 365.', 'type': float }, { 'id': 'time_interval', 'required': False, 'description': 'incremental interval for time in days. Default to 1', 'type': float }, { 'id': 'pf_interval', 'required': False, 'description': 'incremental interval for percentage of functionality. Default to 0.05', 'type': float } ], 'input_datasets': [ { 'id': 'dfr3_mapping_set', 'required': True, 'description': 'DFR3 Mapping Set Object', 'type': ['incore:dfr3MappingSet'], } ], 'output_datasets': [ { 'id': 'pf_results', 'parent_type': '', 'description': 'A csv file recording functionality change with time for each class and limit ' 'state.', 'type': 'incore:waterFacilityRestorationFunc' }, { 'id': 'time_results', 'parent_type': '', 'description': 'A csv file recording repair time at certain functionality recovery for each class ' 'and limit state.', 'type': 'incore:waterFacilityRestorationTime' }, ] }
class WaterFacilityRestoration(BaseAnalysis): """Computes water facility restoration for an earthquake, tsunami, tornado, or hurricane exposure. """ def __init__(self, incore_client): self.restorationsvc = RestorationService(incore_client) super(WaterFacilityRestoration, self).__init__(incore_client) def run(self): """Performs Water facility restoration analysis by using the parameters from the spec and creates an output dataset in csv format Returns: bool: True if successful, False otherwise """ inventory_list = list( self.get_input_dataset("water_facilities").get_inventory_reader()) mapping_set = self.get_input_dataset("dfr3_mapping_set") restoration_key = self.get_parameter("restoration_key") if restoration_key is None: restoration_key = "Restoration ID Code" end_time = self.get_parameter("end_time") if end_time is None: end_time = 365.0 time_interval = self.get_parameter("time_interval") if time_interval is None: time_interval = 1 pf_interval = self.get_parameter("pf_interval") if pf_interval is None: pf_interval = 0.05 (inventory_restoration_map, pf_results, time_results) = self.waterfacility_restoration( inventory_list, mapping_set, restoration_key, end_time, time_interval, pf_interval) self.set_result_csv_data("inventory_restoration_map", inventory_restoration_map, name="inventory_restoration_map_" + self.get_parameter("result_name")) self.set_result_csv_data("pf_results", time_results, name="percentage_of_functionality_" + self.get_parameter("result_name")) self.set_result_csv_data("time_results", pf_results, name="reptime_" + self.get_parameter("result_name")) return True def waterfacility_restoration(self, inventory_list, mapping_set, restoration_key, end_time, time_interval, pf_interval): """Gets applicable restoration curve set and calculates restoration time and functionality Args: inventory_list (list): Multiple water facilities from input inventory set. mapping_set (class): Restoration Mapping Set restoration_key (str): Restoration Key to determine which curve to use. E.g. Restoration ID Code end_time (float): User specified end repair time time_interval (float): Increment interval of repair time. Default to 1 (1 day) pf_interval (float): Increment interval of percentage of functionality. Default 0.1 (10%) Returns: inventory_restoration_map (list): A map between inventory and restoration being applied time_results (list): Given Percentage of functionality, the change of repair time pf_results (list): Given Repair time, change of the percentage of functionality """ # Obtain the restoration id for each water facility inventory_restoration_map = [] restoration_sets = self.restorationsvc.match_inventory( self.get_input_dataset("dfr3_mapping_set"), inventory_list, restoration_key) for inventory in inventory_list: if inventory["id"] in restoration_sets.keys(): restoration_set_id = restoration_sets[inventory["id"]].id else: restoration_set_id = None inventory_restoration_map.append({ "guid": inventory['properties']['guid'], "restoration_id": restoration_set_id }) time_results = [] pf_results = [] for mapping in mapping_set.mappings: # get restoration curves # if it's string:id; then need to fetch it from remote and cast to restorationcurveset object restoration_curve_set = mapping.entry[restoration_key] if isinstance(restoration_curve_set, str): restoration_curve_set = RestorationCurveSet( self.restorationsvc.get_dfr3_set(restoration_curve_set)) # given time calculate pf time = np.arange(0, end_time + time_interval, time_interval) for t in time: pf_results.append({ "restoration_id": restoration_curve_set.id, "time": t, **restoration_curve_set.calculate_restoration_rates(time=t) }) # given pf calculate time pf = np.arange(0, 1 + pf_interval, pf_interval) for p in pf: new_dict = {} t_res = restoration_curve_set.calculate_inverse_restoration_rates( time=p) for key, value in t_res.items(): new_dict.update({"time_" + key: value}) time_results.append({ "restoration_id": restoration_curve_set.id, "percentage_of_functionality": p, **new_dict }) return inventory_restoration_map, pf_results, time_results def get_spec(self): return { 'name': 'water-facility-restoration', 'description': 'water facility restoration analysis', 'input_parameters': [{ 'id': 'restoration_key', 'required': False, 'description': 'restoration key to use in mapping dataset', 'type': str }, { 'id': 'result_name', 'required': True, 'description': 'result dataset name', 'type': str }, { 'id': 'end_time', 'required': False, 'description': 'end time in days. Default to 365.', 'type': float }, { 'id': 'time_interval', 'required': False, 'description': 'incremental interval for time in days. Default to 1', 'type': float }, { 'id': 'pf_interval', 'required': False, 'description': 'incremental interval for percentage of functionality. Default to 0.05', 'type': float }], 'input_datasets': [{ 'id': 'water_facilities', 'required': True, 'description': 'Water Facility Inventory', 'type': ['ergo:waterFacilityTopo'], }, { 'id': 'dfr3_mapping_set', 'required': True, 'description': 'DFR3 Mapping Set Object', 'type': ['incore:dfr3MappingSet'], }], 'output_datasets': [ { 'id': "inventory_restoration_map", 'parent_type': '', 'description': 'A csv file recording the mapping relationship between GUID and restoration id ' 'applicable.', 'type': 'incore:inventoryRestorationMap' }, { 'id': 'pf_results', 'parent_type': '', 'description': 'A csv file recording functionality change with time for each class and limit ' 'state.', 'type': 'incore:waterFacilityRestorationFunc' }, { 'id': 'time_results', 'parent_type': '', 'description': 'A csv file recording repair time at certain functionality recovery for each class ' 'and limit state.', 'type': 'incore:waterFacilityRestorationTime' }, ] }
def __init__(self, incore_client): self.restorationsvc = RestorationService(incore_client) super(EpfRestoration, self).__init__(incore_client)
class PipelineRestoration(BaseAnalysis): """ Args: incore_client (IncoreClient): Service authentication. """ def __init__(self, incore_client): self.restorationsvc = RestorationService(incore_client) super(PipelineRestoration, self).__init__(incore_client) def get_spec(self): """Get specifications of the Pipeline Restoration analysis. Returns: obj: A JSON object of specifications of the pipeline restoration analysis. """ return { 'name': 'pipeline-restoration', 'description': 'calculate the restoration times for damaged pipelines', 'input_parameters': [ { 'id': 'result_name', 'required': True, 'description': 'name of the result csv dataset', 'type': str }, { 'id': 'num_cpu', 'required': False, 'description': 'If using parallel execution, the number of cpus to request', 'type': int }, { 'id': 'num_available_workers', 'required': True, 'description': 'Number of available workers to work on the repairs', 'type': int }, { 'id': 'restoration_key', 'required': False, 'description': 'restoration key to use in mapping dataset', 'type': str }, ], 'input_datasets': [{ 'id': 'pipeline', 'required': True, 'description': 'Pipeline Inventory', 'type': ['ergo:buriedPipelineTopology', 'ergo:pipeline'], }, { 'id': 'pipeline_damage', 'required': True, 'description': 'pipeline damage results with repairs', 'type': ['ergo:pipelineDamageVer2', 'ergo:pipelineDamageVer3'] }, { 'id': 'dfr3_mapping_set', 'required': True, 'description': 'DFR3 Mapping Set Object', 'type': ['incore:dfr3MappingSet'], }], 'output_datasets': [{ 'id': 'pipeline_restoration', 'description': 'CSV file of pipeline restoration times', 'type': 'incore:pipelineRestorationVer1' }] } def run(self): """Executes pipeline restoration analysis.""" pipelines_df = self.get_input_dataset( "pipeline").get_dataframe_from_shapefile() pipeline_dmg = self.get_input_dataset( "pipeline_damage").get_csv_reader() pipelines_dmg_df = pd.DataFrame(list(pipeline_dmg)) damage_result = pipelines_dmg_df.merge(pipelines_df, on='guid') damage_result = damage_result.to_dict(orient='records') user_defined_cpu = 1 if not self.get_parameter("num_cpu") is None and self.get_parameter( "num_cpu") > 0: user_defined_cpu = self.get_parameter("num_cpu") num_workers = AnalysisUtil.determine_parallelism_locally( self, len(damage_result), user_defined_cpu) avg_bulk_input_size = int(len(damage_result) / num_workers) inventory_args = [] count = 0 inventory_list = damage_result while count < len(inventory_list): inventory_args.append(inventory_list[count:count + avg_bulk_input_size]) count += avg_bulk_input_size restoration_results = self.pipeline_restoration_concurrent_future( self.pipeline_restoration_bulk_input, num_workers, inventory_args) self.set_result_csv_data("pipeline_restoration", restoration_results, name=self.get_parameter("result_name")) return True def pipeline_restoration_concurrent_future(self, function_name, parallelism, *args): """Utilizes concurrent.future module. Args: function_name (function): The function to be parallelized. parallelism (int): Number of workers in parallelization. *args: All the arguments in order to pass into parameter function_name. Returns: list: A list of dictionary with restoration details """ res_output = [] with concurrent.futures.ProcessPoolExecutor( max_workers=parallelism) as executor: for res_ret in executor.map(function_name, *args): res_output.extend(res_ret) return res_output def pipeline_restoration_bulk_input(self, damage): """Run analysis for pipeline restoration calculation Args: damage (obj): An output of pipeline damage with repair rate Returns: restoration_results (list): A list of dictionary restoration times and inventory details """ restoration_results = [] num_available_workers = self.get_parameter("num_available_workers") restoration_key = self.get_parameter("restoration_key") if restoration_key is None: restoration_key = "Restoration ID Code" restoration_sets = self.restorationsvc.match_list_of_dicts( self.get_input_dataset("dfr3_mapping_set"), damage, restoration_key) for dmg in damage: res = self.restoration_time(dmg, num_available_workers, restoration_sets[dmg['guid']]) restoration_results.append(res) return restoration_results @staticmethod def restoration_time(dmg, num_available_workers, restoration_set): """Calculates restoration time for a single pipeline. Args: dmg (obj): Pipeline damage analysis output for a single entry. num_available_workers (int): Number of available workers working on the repairs. restoration_set(obj): Restoration curve(s) to be be used Returns: dict: A dictionary with id/guid and restoration time, along with some inventory metadata """ res_result = collections.OrderedDict() if 'guid' in dmg.keys(): res_result['guid'] = dmg['guid'] else: res_result['guid'] = 'NA' res_result[ 'repair_time'] = restoration_set.calculate_restoration_rates( **{ "break_rate": float(dmg['breakrate']), "leak_rate": float(dmg['leakrate']), "pipe_length": dmg['length'], "num_workers": num_available_workers })['RT'] return res_result