def run_with_base_class():
client = IncoreClient(pyglobals.INCORE_API_DEV_URL)
client.clear_cache()
pipeline_restoration = PipelineRestoration(client)
# shelby county pipelines
pipeline_restoration.load_remote_input_dataset("pipeline",
"5a284f28c7d30d13bc081d14")
pipeline_restoration.load_remote_input_dataset("pipeline_damage",
"61f36023c53b3620b6b614c6")
# Load fragility mapping
restoration_service = RestorationService(client)
mapping_set = MappingSet(
restoration_service.get_mapping("61f35f09903e515036cee106"))
pipeline_restoration.set_input_dataset('dfr3_mapping_set', mapping_set)
pipeline_restoration.set_parameter("result_name",
"pipeline_restoration_times")
pipeline_restoration.set_parameter("restoration_key",
"Restoration ID Code")
pipeline_restoration.set_parameter("num_available_workers", 4)
pipeline_restoration.set_parameter("num_cpu", 4)
# Run pipeline restoration analysis
pipeline_restoration.run_analysis()
def run_with_base_class():
client = IncoreClient(pyglobals.INCORE_API_DEV_URL)
epf_rest = EpfRestoration(client)
restorationsvc = RestorationService(client)
mapping_set = MappingSet(
restorationsvc.get_mapping(
"61f302e6e3a03e465500b3eb")) # new format of mapping
epf_rest.load_remote_input_dataset('epfs', '6189c103d5b02930aa3efc35')
epf_rest.set_input_dataset('dfr3_mapping_set', mapping_set)
epf_rest.set_parameter("result_name", "epf_restoration.csv")
epf_rest.set_parameter("restoration_key", "Restoration ID Code")
epf_rest.set_parameter("end_time", 100.0)
epf_rest.set_parameter("time_interval", 1.0)
epf_rest.set_parameter("pf_interval", 0.01)
epf_rest.run_analysis()
# test utility function
epf_rest_util = EpfRestorationUtil(epf_rest)
functionality = epf_rest_util.get_percentage_func(
guid="60748fbd-67c3-4f8d-beb9-26685a53d3c5",
damage_state="DS_0",
time=2.0)
time = epf_rest_util.get_restoration_time(
guid="60748fbd-67c3-4f8d-beb9-26685a53d3c5",
damage_state="DS_1",
pf=0.81)
print(functionality, time)
def run_with_base_class():
client = IncoreClient(pyglobals.INCORE_API_DEV_URL)
wf_rest = WaterFacilityRestoration(client)
# Load restoration mapping
restorationsvc = RestorationService(client)
mapping_set = MappingSet(
restorationsvc.get_mapping(
"61f075ee903e515036cee0a5")) # new format of mapping
wf_rest.load_remote_input_dataset(
"water_facilities", "5a284f2ac7d30d13bc081e52") # water facility
wf_rest.set_input_dataset('dfr3_mapping_set', mapping_set)
wf_rest.set_parameter("result_name", "wf_restoration")
wf_rest.set_parameter("restoration_key", "Restoration ID Code")
wf_rest.set_parameter("end_time", 100.0)
wf_rest.set_parameter("time_interval", 1.0)
wf_rest.set_parameter("pf_interval", 0.05)
wf_rest.run_analysis()
# test utility function
wf_util = WaterFacilityRestorationUtil(wf_rest)
functionality = wf_util.get_percentage_func(
guid="e1bce78d-00a1-4605-95f3-3776ff907f73",
damage_state="DS_0",
time=2.0)
time = wf_util.get_restoration_time(
guid="e1bce78d-00a1-4605-95f3-3776ff907f73",
damage_state="DS_1",
pf=0.81)
print(functionality, time)
def waterfacility_restoration(self, mapping_set, restoration_key, end_time, time_interval,
pf_interval):
"""Gets applicable restoration curve set and calculates restoration time and functionality
Args:
mapping_set (class): Restoration Mapping Set
restoration_key (str): Restoration Key to determine which curve to use. E.g. Restoration ID Code
end_time (float): User specified end repair time
time_interval (float): Increment interval of repair time. Default to 1 (1 day)
pf_interval (float): Increment interval of percentage of functionality. Default 0.1 (10%)
Returns:
time_results (list): Given Percentage of functionality, the change of repair time
pf_results (list): Given Repair time, change of the percentage of functionality
"""
time_results = []
pf_results = []
for mapping in mapping_set.mappings:
# parse rules to get inventory class. e.g. treatment plan, tank, pump etc
if isinstance(mapping.rules, list):
inventory_class = RestorationService.extract_inventory_class_legacy(mapping.rules)
elif isinstance(mapping.rules, dict):
inventory_class = RestorationService.extract_inventory_class(mapping.rules)
else:
raise ValueError("Unsupported mapping rules!")
# get restoration curves
# if it's string:id; then need to fetch it from remote and cast to restorationcurveset object
restoration_curve_set = mapping.entry[restoration_key]
if isinstance(restoration_curve_set, str):
restoration_curve_set = RestorationCurveSet(self.restorationsvc.get_dfr3_set(restoration_curve_set))
# given time calculate pf
time = np.arange(0, end_time + time_interval, time_interval)
for t in time:
pf_results.append({
"inventory_class": inventory_class,
"time": t,
**restoration_curve_set.calculate_restoration_rates(time=t)
})
# given pf calculate time
pf = np.arange(0, 1 + pf_interval, pf_interval)
for p in pf:
new_dict = {}
t_res = restoration_curve_set.calculate_inverse_restoration_rates(time=p)
for key, value in t_res.items():
new_dict.update({"time_" + key: value})
time_results.append({
"inventory_class": inventory_class,
"percentage_of_functionality": p,
**new_dict
})
return pf_results, time_results
def run_with_base_class():
client = IncoreClient(pyglobals.INCORE_API_DEV_URL)
epf_rest = ElectricPowerFacilityRestoration(client)
restorationsvc = RestorationService(client)
mapping_set = MappingSet(
restorationsvc.get_mapping(
"61f302e6e3a03e465500b3eb")) # new format of mapping
epf_rest.set_input_dataset('dfr3_mapping_set', mapping_set)
epf_rest.set_parameter("result_name", "epf_restoration.csv")
epf_rest.set_parameter("restoration_key", "Restoration ID Code")
epf_rest.set_parameter("end_time", 100.0)
epf_rest.set_parameter("time_interval", 1.0)
epf_rest.set_parameter("pf_interval", 0.01)
epf_rest.run_analysis()
def pytest_sessionstart(session):
"""
Called after the Session object has been created and
before performing collection and entering the run test loop.
"""
try:
with open(
os.path.join(os.path.dirname(__file__), "pyincore/.incorepw"),
'r') as f:
cred = f.read().splitlines()
except EnvironmentError:
assert False
credentials = jwt.decode(cred[0], cred[1])
monkeypatch = MonkeyPatch()
monkeypatch.setattr("builtins.input", lambda x: credentials["username"])
monkeypatch.setattr("getpass.getpass", lambda y: credentials["password"])
client = IncoreClient(service_url=pyglobals.INCORE_API_DEV_URL,
token_file_name=".incrtesttoken")
pytest.client = client
pytest.datasvc = DataService(client)
pytest.fragilitysvc = FragilityService(client)
pytest.repairsvc = RepairService(client)
pytest.restorationsvc = RestorationService(client)
pytest.hazardsvc = HazardService(client)
pytest.spacesvc = SpaceService(client)
print(
f"Successfully initialized Incore client and services. Using {pyglobals.INCORE_API_DEV_URL}"
)
def run_with_base_class():
client = IncoreClient(pyglobals.INCORE_API_DEV_URL)
wf_rest = WaterFacilityRestoration(client)
# Load restoration mapping
restorationsvc = RestorationService(client)
mapping_set = MappingSet(
restorationsvc.get_mapping(
"61f075ee903e515036cee0a5")) # new format of mapping
wf_rest.set_input_dataset('dfr3_mapping_set', mapping_set)
wf_rest.set_parameter("result_name", "wf_restoration")
wf_rest.set_parameter("restoration_key", "Restoration ID Code")
wf_rest.set_parameter("end_time", 100.0)
wf_rest.set_parameter("time_interval", 1.0)
wf_rest.set_parameter("pf_interval", 0.05)
wf_rest.run_analysis()
def __init__(self, incore_client):
self.restorationsvc = RestorationService(incore_client)
super(WaterFacilityRestoration, self).__init__(incore_client)
class WaterFacilityRestoration(BaseAnalysis):
"""Computes water facility restoration for an earthquake, tsunami, tornado, or hurricane exposure.
"""
def __init__(self, incore_client):
self.restorationsvc = RestorationService(incore_client)
super(WaterFacilityRestoration, self).__init__(incore_client)
def run(self):
"""Performs Water facility restoration analysis by using the parameters from the spec
and creates an output dataset in csv format
Returns:
bool: True if successful, False otherwise
"""
mapping_set = self.get_input_dataset("dfr3_mapping_set")
restoration_key = self.get_parameter("restoration_key")
if restoration_key is None:
restoration_key = "Restoration ID Code"
end_time = self.get_parameter("end_time")
if end_time is None:
end_time = 365.0
time_interval = self.get_parameter("time_interval")
if time_interval is None:
time_interval = 1
pf_interval = self.get_parameter("pf_interval")
if pf_interval is None:
pf_interval = 0.05
(pf_results, time_results) = self.waterfacility_restoration(mapping_set, restoration_key, end_time,
time_interval, pf_interval)
self.set_result_csv_data("pf_results", time_results, name="percentage_of_functionality_" +
self.get_parameter("result_name"))
self.set_result_csv_data("time_results", pf_results, name="reptime_" + self.get_parameter("result_name"))
return True
def waterfacility_restoration(self, mapping_set, restoration_key, end_time, time_interval,
pf_interval):
"""Gets applicable restoration curve set and calculates restoration time and functionality
Args:
mapping_set (class): Restoration Mapping Set
restoration_key (str): Restoration Key to determine which curve to use. E.g. Restoration ID Code
end_time (float): User specified end repair time
time_interval (float): Increment interval of repair time. Default to 1 (1 day)
pf_interval (float): Increment interval of percentage of functionality. Default 0.1 (10%)
Returns:
time_results (list): Given Percentage of functionality, the change of repair time
pf_results (list): Given Repair time, change of the percentage of functionality
"""
time_results = []
pf_results = []
for mapping in mapping_set.mappings:
# parse rules to get inventory class. e.g. treatment plan, tank, pump etc
if isinstance(mapping.rules, list):
inventory_class = RestorationService.extract_inventory_class_legacy(mapping.rules)
elif isinstance(mapping.rules, dict):
inventory_class = RestorationService.extract_inventory_class(mapping.rules)
else:
raise ValueError("Unsupported mapping rules!")
# get restoration curves
# if it's string:id; then need to fetch it from remote and cast to restorationcurveset object
restoration_curve_set = mapping.entry[restoration_key]
if isinstance(restoration_curve_set, str):
restoration_curve_set = RestorationCurveSet(self.restorationsvc.get_dfr3_set(restoration_curve_set))
# given time calculate pf
time = np.arange(0, end_time + time_interval, time_interval)
for t in time:
pf_results.append({
"inventory_class": inventory_class,
"time": t,
**restoration_curve_set.calculate_restoration_rates(time=t)
})
# given pf calculate time
pf = np.arange(0, 1 + pf_interval, pf_interval)
for p in pf:
new_dict = {}
t_res = restoration_curve_set.calculate_inverse_restoration_rates(time=p)
for key, value in t_res.items():
new_dict.update({"time_" + key: value})
time_results.append({
"inventory_class": inventory_class,
"percentage_of_functionality": p,
**new_dict
})
return pf_results, time_results
def get_spec(self):
return {
'name': 'water-facility-restoration',
'description': 'water facility restoration analysis',
'input_parameters': [
{
'id': 'restoration_key',
'required': False,
'description': 'restoration key to use in mapping dataset',
'type': str
},
{
'id': 'result_name',
'required': True,
'description': 'result dataset name',
'type': str
},
{
'id': 'end_time',
'required': False,
'description': 'end time in days. Default to 365.',
'type': float
},
{
'id': 'time_interval',
'required': False,
'description': 'incremental interval for time in days. Default to 1',
'type': float
},
{
'id': 'pf_interval',
'required': False,
'description': 'incremental interval for percentage of functionality. Default to 0.05',
'type': float
}
],
'input_datasets': [
{
'id': 'dfr3_mapping_set',
'required': True,
'description': 'DFR3 Mapping Set Object',
'type': ['incore:dfr3MappingSet'],
}
],
'output_datasets': [
{
'id': 'pf_results',
'parent_type': '',
'description': 'A csv file recording functionality change with time for each class and limit '
'state.',
'type': 'incore:waterFacilityRestorationFunc'
},
{
'id': 'time_results',
'parent_type': '',
'description': 'A csv file recording repair time at certain functionality recovery for each class '
'and limit state.',
'type': 'incore:waterFacilityRestorationTime'
},
]
}
class WaterFacilityRestoration(BaseAnalysis):
"""Computes water facility restoration for an earthquake, tsunami, tornado, or hurricane exposure.
"""
def __init__(self, incore_client):
self.restorationsvc = RestorationService(incore_client)
super(WaterFacilityRestoration, self).__init__(incore_client)
def run(self):
"""Performs Water facility restoration analysis by using the parameters from the spec
and creates an output dataset in csv format
Returns:
bool: True if successful, False otherwise
"""
inventory_list = list(
self.get_input_dataset("water_facilities").get_inventory_reader())
mapping_set = self.get_input_dataset("dfr3_mapping_set")
restoration_key = self.get_parameter("restoration_key")
if restoration_key is None:
restoration_key = "Restoration ID Code"
end_time = self.get_parameter("end_time")
if end_time is None:
end_time = 365.0
time_interval = self.get_parameter("time_interval")
if time_interval is None:
time_interval = 1
pf_interval = self.get_parameter("pf_interval")
if pf_interval is None:
pf_interval = 0.05
(inventory_restoration_map, pf_results,
time_results) = self.waterfacility_restoration(
inventory_list, mapping_set, restoration_key, end_time,
time_interval, pf_interval)
self.set_result_csv_data("inventory_restoration_map",
inventory_restoration_map,
name="inventory_restoration_map_" +
self.get_parameter("result_name"))
self.set_result_csv_data("pf_results",
time_results,
name="percentage_of_functionality_" +
self.get_parameter("result_name"))
self.set_result_csv_data("time_results",
pf_results,
name="reptime_" +
self.get_parameter("result_name"))
return True
def waterfacility_restoration(self, inventory_list, mapping_set,
restoration_key, end_time, time_interval,
pf_interval):
"""Gets applicable restoration curve set and calculates restoration time and functionality
Args:
inventory_list (list): Multiple water facilities from input inventory set.
mapping_set (class): Restoration Mapping Set
restoration_key (str): Restoration Key to determine which curve to use. E.g. Restoration ID Code
end_time (float): User specified end repair time
time_interval (float): Increment interval of repair time. Default to 1 (1 day)
pf_interval (float): Increment interval of percentage of functionality. Default 0.1 (10%)
Returns:
inventory_restoration_map (list): A map between inventory and restoration being applied
time_results (list): Given Percentage of functionality, the change of repair time
pf_results (list): Given Repair time, change of the percentage of functionality
"""
# Obtain the restoration id for each water facility
inventory_restoration_map = []
restoration_sets = self.restorationsvc.match_inventory(
self.get_input_dataset("dfr3_mapping_set"), inventory_list,
restoration_key)
for inventory in inventory_list:
if inventory["id"] in restoration_sets.keys():
restoration_set_id = restoration_sets[inventory["id"]].id
else:
restoration_set_id = None
inventory_restoration_map.append({
"guid":
inventory['properties']['guid'],
"restoration_id":
restoration_set_id
})
time_results = []
pf_results = []
for mapping in mapping_set.mappings:
# get restoration curves
# if it's string:id; then need to fetch it from remote and cast to restorationcurveset object
restoration_curve_set = mapping.entry[restoration_key]
if isinstance(restoration_curve_set, str):
restoration_curve_set = RestorationCurveSet(
self.restorationsvc.get_dfr3_set(restoration_curve_set))
# given time calculate pf
time = np.arange(0, end_time + time_interval, time_interval)
for t in time:
pf_results.append({
"restoration_id":
restoration_curve_set.id,
"time":
t,
**restoration_curve_set.calculate_restoration_rates(time=t)
})
# given pf calculate time
pf = np.arange(0, 1 + pf_interval, pf_interval)
for p in pf:
new_dict = {}
t_res = restoration_curve_set.calculate_inverse_restoration_rates(
time=p)
for key, value in t_res.items():
new_dict.update({"time_" + key: value})
time_results.append({
"restoration_id": restoration_curve_set.id,
"percentage_of_functionality": p,
**new_dict
})
return inventory_restoration_map, pf_results, time_results
def get_spec(self):
return {
'name':
'water-facility-restoration',
'description':
'water facility restoration analysis',
'input_parameters': [{
'id': 'restoration_key',
'required': False,
'description': 'restoration key to use in mapping dataset',
'type': str
}, {
'id': 'result_name',
'required': True,
'description': 'result dataset name',
'type': str
}, {
'id': 'end_time',
'required': False,
'description': 'end time in days. Default to 365.',
'type': float
}, {
'id': 'time_interval',
'required': False,
'description':
'incremental interval for time in days. Default to 1',
'type': float
}, {
'id': 'pf_interval',
'required': False,
'description':
'incremental interval for percentage of functionality. Default to 0.05',
'type': float
}],
'input_datasets': [{
'id': 'water_facilities',
'required': True,
'description': 'Water Facility Inventory',
'type': ['ergo:waterFacilityTopo'],
}, {
'id': 'dfr3_mapping_set',
'required': True,
'description': 'DFR3 Mapping Set Object',
'type': ['incore:dfr3MappingSet'],
}],
'output_datasets': [
{
'id':
"inventory_restoration_map",
'parent_type':
'',
'description':
'A csv file recording the mapping relationship between GUID and restoration id '
'applicable.',
'type':
'incore:inventoryRestorationMap'
},
{
'id':
'pf_results',
'parent_type':
'',
'description':
'A csv file recording functionality change with time for each class and limit '
'state.',
'type':
'incore:waterFacilityRestorationFunc'
},
{
'id':
'time_results',
'parent_type':
'',
'description':
'A csv file recording repair time at certain functionality recovery for each class '
'and limit state.',
'type':
'incore:waterFacilityRestorationTime'
},
]
}
def __init__(self, incore_client):
self.restorationsvc = RestorationService(incore_client)
super(EpfRestoration, self).__init__(incore_client)
class PipelineRestoration(BaseAnalysis):
"""
Args:
incore_client (IncoreClient): Service authentication.
"""
def __init__(self, incore_client):
self.restorationsvc = RestorationService(incore_client)
super(PipelineRestoration, self).__init__(incore_client)
def get_spec(self):
"""Get specifications of the Pipeline Restoration analysis.
Returns:
obj: A JSON object of specifications of the pipeline restoration analysis.
"""
return {
'name':
'pipeline-restoration',
'description':
'calculate the restoration times for damaged pipelines',
'input_parameters': [
{
'id': 'result_name',
'required': True,
'description': 'name of the result csv dataset',
'type': str
},
{
'id': 'num_cpu',
'required': False,
'description':
'If using parallel execution, the number of cpus to request',
'type': int
},
{
'id': 'num_available_workers',
'required': True,
'description':
'Number of available workers to work on the repairs',
'type': int
},
{
'id': 'restoration_key',
'required': False,
'description': 'restoration key to use in mapping dataset',
'type': str
},
],
'input_datasets': [{
'id':
'pipeline',
'required':
True,
'description':
'Pipeline Inventory',
'type': ['ergo:buriedPipelineTopology', 'ergo:pipeline'],
}, {
'id':
'pipeline_damage',
'required':
True,
'description':
'pipeline damage results with repairs',
'type': ['ergo:pipelineDamageVer2', 'ergo:pipelineDamageVer3']
}, {
'id': 'dfr3_mapping_set',
'required': True,
'description': 'DFR3 Mapping Set Object',
'type': ['incore:dfr3MappingSet'],
}],
'output_datasets': [{
'id': 'pipeline_restoration',
'description': 'CSV file of pipeline restoration times',
'type': 'incore:pipelineRestorationVer1'
}]
}
def run(self):
"""Executes pipeline restoration analysis."""
pipelines_df = self.get_input_dataset(
"pipeline").get_dataframe_from_shapefile()
pipeline_dmg = self.get_input_dataset(
"pipeline_damage").get_csv_reader()
pipelines_dmg_df = pd.DataFrame(list(pipeline_dmg))
damage_result = pipelines_dmg_df.merge(pipelines_df, on='guid')
damage_result = damage_result.to_dict(orient='records')
user_defined_cpu = 1
if not self.get_parameter("num_cpu") is None and self.get_parameter(
"num_cpu") > 0:
user_defined_cpu = self.get_parameter("num_cpu")
num_workers = AnalysisUtil.determine_parallelism_locally(
self, len(damage_result), user_defined_cpu)
avg_bulk_input_size = int(len(damage_result) / num_workers)
inventory_args = []
count = 0
inventory_list = damage_result
while count < len(inventory_list):
inventory_args.append(inventory_list[count:count +
avg_bulk_input_size])
count += avg_bulk_input_size
restoration_results = self.pipeline_restoration_concurrent_future(
self.pipeline_restoration_bulk_input, num_workers, inventory_args)
self.set_result_csv_data("pipeline_restoration",
restoration_results,
name=self.get_parameter("result_name"))
return True
def pipeline_restoration_concurrent_future(self, function_name,
parallelism, *args):
"""Utilizes concurrent.future module.
Args:
function_name (function): The function to be parallelized.
parallelism (int): Number of workers in parallelization.
*args: All the arguments in order to pass into parameter function_name.
Returns:
list: A list of dictionary with restoration details
"""
res_output = []
with concurrent.futures.ProcessPoolExecutor(
max_workers=parallelism) as executor:
for res_ret in executor.map(function_name, *args):
res_output.extend(res_ret)
return res_output
def pipeline_restoration_bulk_input(self, damage):
"""Run analysis for pipeline restoration calculation
Args:
damage (obj): An output of pipeline damage with repair rate
Returns:
restoration_results (list): A list of dictionary restoration times and inventory details
"""
restoration_results = []
num_available_workers = self.get_parameter("num_available_workers")
restoration_key = self.get_parameter("restoration_key")
if restoration_key is None:
restoration_key = "Restoration ID Code"
restoration_sets = self.restorationsvc.match_list_of_dicts(
self.get_input_dataset("dfr3_mapping_set"), damage,
restoration_key)
for dmg in damage:
res = self.restoration_time(dmg, num_available_workers,
restoration_sets[dmg['guid']])
restoration_results.append(res)
return restoration_results
@staticmethod
def restoration_time(dmg, num_available_workers, restoration_set):
"""Calculates restoration time for a single pipeline.
Args:
dmg (obj): Pipeline damage analysis output for a single entry.
num_available_workers (int): Number of available workers working on the repairs.
restoration_set(obj): Restoration curve(s) to be be used
Returns:
dict: A dictionary with id/guid and restoration time, along with some inventory metadata
"""
res_result = collections.OrderedDict()
if 'guid' in dmg.keys():
res_result['guid'] = dmg['guid']
else:
res_result['guid'] = 'NA'
res_result[
'repair_time'] = restoration_set.calculate_restoration_rates(
**{
"break_rate": float(dmg['breakrate']),
"leak_rate": float(dmg['leakrate']),
"pipe_length": dmg['length'],
"num_workers": num_available_workers
})['RT']
return res_result