def main():
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
root_dir = os.path.join(data_path, 'Waterways', 'viwa', 'html')
r = re.compile('LuongHangHoaThongQua">([^<]*)')
viwa_ports_tons = []
for root, dirs, files in os.walk(root_dir):
for file in files:
if file.endswith(".html") and 'detail' not in file:
input_file = os.path.join(root, file)
port_no = re.findall('\d+', file)
with open(input_file, 'r') as fh:
for line in fh:
m = r.search(line)
if m:
if m.group(1).isdigit():
viwa_ports_tons.append(
(int(port_no[0]), float(m.group(1))))
else:
viwa_ports_tons.append((int(port_no[0]), 0))
viwa_ports_tons_df = pd.DataFrame(viwa_ports_tons,
columns=['cangbenid', 'tons'])
viwa_ports_tons_df = viwa_ports_tons_df[viwa_ports_tons_df['tons'] > 0]
print(len(viwa_ports_tons_df.index))
print(viwa_ports_tons_df['cangbenid'].values.tolist())
shp_file = os.path.join(data_path, 'Waterways', 'viwa', 'cang-ports.shp')
ports = gpd.read_file(shp_file)
ports.columns = map(str.lower, ports.columns)
cols = ports.columns.values.tolist()
print(len(ports.index))
# print (ports['cangbenid'].values.tolist())
ports_with_geom = []
id_list = []
for iter_, values in ports.iterrows():
if values['geometry'] and values['cangbenid'] not in id_list:
if int(values['cangbenid']
) in viwa_ports_tons_df['cangbenid'].values.tolist():
print('match')
tons = viwa_ports_tons_df.loc[
viwa_ports_tons_df['cangbenid'] ==
int(values['cangbenid']), 'tons'].sum()
ports_with_geom.append(tuple(list(values) + [tons]))
else:
ports_with_geom.append(tuple(list(values) + [0]))
id_list.append(values['cangbenid'])
ports_df = pd.DataFrame(ports_with_geom, columns=cols + ['tons'])
ports = gpd.GeoDataFrame(ports_df, crs='epsg:4326')
# print (len(ports.index))
print(ports)
ports.to_file(
os.path.join(data_path, 'Waterways', 'viwa_select', 'iwt_ports.shp'))
def create_Ratmarg(self, Table):
model = self.m
try:
data_path = load_config()['paths']['data']
RatMarg = pd.read_csv(os.path.join(
data_path, 'input_data', 'Ratmarg_{}.csv'.format(self.name)),
index_col=[0],
header=0)
if self.EORA is True and (set(list(RatMarg.index.values)) != set(
list(self.countries + ['ROW']))):
RatMarg = ratmarg_IO(Table, self.EORA)
elif (set(list(RatMarg.index.values)) != set(list(
self.countries))):
RatMarg = ratmarg_IO(Table, self.EORA)
except:
RatMarg = ratmarg_IO(Table, self.EORA)
Ratmarginal = {(r, k): v
for r, kv in RatMarg.iterrows()
for k, v in kv.to_dict().items()}
model.Ratmarg = Param(model.R,
model.S,
initialize=Ratmarginal,
doc='Rationing marginal',
mutable=True)
self.Ratmarg = model.Ratmarg
def create_Ratmarg(self, Table):
"""Estimate the marginal values of the rationing variable
Parameters
----------
Table
the **io_basic** class object
Set *self*.Ratmarg - Pyomo Parameter instance for the marginal value of rationing
"""
model = self.m
try:
data_path = load_config()['paths']['data']
RatMarg = pd.read_csv(os.path.join(
data_path, 'input_data', 'Ratmarg_{}.csv'.format(self.name)),
index_col=[0],
header=0)
if (set(list(RatMarg.index.values)) != set(list(self.regions))):
RatMarg = ratmarg_IO(Table)
except:
RatMarg = ratmarg_IO(Table)
Ratmarginal = {(r, k): v
for r, kv in RatMarg.iterrows()
for k, v in kv.to_dict().items()}
model.Ratmarg = Param(model.R,
model.S,
initialize=Ratmarginal,
doc='Rationing marginal',
mutable=True)
self.Ratmarg = model.Ratmarg
def ratmarg_SUT(table_in,EORA=False):
data_path = load_config()['paths']['data']
table_in.prep_data()
load_db_SUT(table_in)
'''
RUN SCRIPT WITH DISRUPTION
'''
setdir = os.path.join(data_path,'gams_runs')
ws = GamsWorkspace(setdir)
ws.get_working_directory()
gamsfile_in = os.path.join(setdir,"obtain_marg_value_SUT.gms")
gamsfile = os.path.join(setdir,"obtain_marg_value_SUT_{}.gms".format(table_in.name))
copyfile(gamsfile_in, gamsfile)
str_ctry = ','.join(table_in.countries)
str_fd = 'FinalD' #','.join(list(table_in.FD_labels['FD'].unique()))
with open(gamsfile, 'r') as file:
# read a list of lines into data
data = file.readlines()
gdx_file = "%s.gdx" % table_in.name
data[30] = '$GDXIN '+gdx_file+'\n'
str_ind = ','.join(table_in.sectors)
data[40] ='ind(col) list of industries /'+str_ind+'/\n'
data[37] = '/'+str_ctry+'/\n'
data[39] = '/'+str_ctry+'/\n'
data[43] = '/'+str_fd+'/\n'
with open(gamsfile, 'w') as file:
file.writelines( data )
gamsfile_run = gamsfile.replace("..\\..\\gams_runs\\", "")
t1 = ws.add_job_from_file(gamsfile_run)
t1.run()
Ratmarg = []
index_ = []
for rec in t1.out_db["Ratmarg"]:
index_.append((rec.keys[0],rec.keys[1]))
Ratmarg.append(rec.get_value())
index_ = pd.MultiIndex.from_tuples(index_, names=('CNTRY', 'IND'))
Ratmarginal = pd.DataFrame(Ratmarg, index=index_).unstack()
Ratmarginal.columns = Ratmarginal.columns.droplevel()
Ratmarginal.to_csv(os.path.join(data_path,'input_data','Ratmarg_{}.csv'.format(table_in.name)))
return Ratmarginal
def main():
"""Pre-process CVTS
"""
incoming_root = load_config()['paths']['incoming_data']
data_root = load_config()['paths']['data']
output_root = load_config()['paths']['output']
dir_raw_roads = os.path.join(data_root, 'post_processed_networks')
dir_raw_cvts = os.path.join(incoming_root,'20170801')
dir_inter_reduse = os.path.join(output_root, 'transport cvts analysis', 'intermediate', 'reduse')
dir_results_routes = os.path.join(output_root, 'transport cvts analysis', 'results', 'routes')
dir_results_routes_collected = os.path.join(
output_root, 'transport cvts analysis', 'results', 'routes_collected')
dir_results_traffic_count = os.path.join(output_root, 'transport cvts analysis', 'results', 'traffic_count')
Reduse dataset with 70 percent (processing 4s/100mb)
print('Reduce dataset size')
reduse_dataset(dir_raw_cvts, dir_inter_reduse)
# Read road network in memory
print('Read road network')
geojson_road_network = read_shapefile(dir_raw_roads, 'road_edges.shp')
# Generate routes by mapping gps points on the road network
print('Generate routes')
find_routes(geojson_road_network, dir_inter_reduse, dir_results_routes)
# add traffic attribute to route network (count id's in routes)
print('Add traffic count attribute to road network')
geojson_road_network = add_traffic_count_to_road_network(
geojson_road_network, dir_results_routes, dir_results_traffic_count)
write_shapefile(geojson_road_network, dir_results_traffic_count, 'road_network.shp')
# add routes to single file
create_single_route_file(dir_results_routes, dir_results_routes_collected)
def create_disruption(input_file,
output_dir,
min_rice=True,
single_point=True):
# Define current directory and data directory
data_path = load_config()['paths']['data']
# read national IO
comm_des = df_com_to_ind(pd.read_excel(os.path.join(
data_path, 'OD_data', 'national_scale_od_matrix.xlsx'),
sheet_name='total'),
min_rice=min_rice)
# read vietnam failure results
vnm_failure_all_failure = pd.read_csv(input_file, index_col=[0])
if single_point == False:
mapper = dict(
zip(vnm_failure_all_failure.index.unique(), [
'multi_{}'.format(n) for n in np.arange(
1,
len(vnm_failure_all_failure.index.unique()) + 1, 1)
]))
vnm_failure_all_failure.index = vnm_failure_all_failure.index.map(
lambda x: mapper[x])
pd.DataFrame.from_dict(mapper, orient='index').to_csv(
os.path.join(output_dir, 'mapper.csv'))
event_dict = {}
for id_, scenario in vnm_failure_all_failure.groupby(level=0):
if id_ == 'multi_102':
break
disruption = df_com_to_ind(scenario,
min_rice=min_rice).div(comm_des).dropna(
axis=0, how='all').fillna(0)
disruption = pd.DataFrame(disruption.stack(0))
disruption.columns = ['value']
disruption = disruption.loc[disruption.value > 0]
disruption['value'] = 1 - (disruption['value'] * 0.2)
event_dict[id_] = disruption['value'].to_dict()
return event_dict
def main():
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
truck_unit_wt = [5.0, 20.0]
# provinces to consider
province_list = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa']
province_terrian = ['mountain', 'flat', 'flat']
growth_scenarios = [(5, 'low'), (6.5, 'forecast'), (10, 'high')]
base_year = 2016
types = ['min', 'max']
path_types = ['min_edge_path', 'max_edge_path']
rev_types = ['min_netrev', 'max_netrev']
tons_types = ['min_croptons', 'max_croptons']
dist_types = ['min_distance', 'max_distance']
time_types = ['min_time', 'max_time']
cost_types = ['min_gcost', 'max_gcost']
vechicle_types = ['min_vehicle_nums', 'max_vehicle_nums']
flow_paths_data = os.path.join(
output_path, 'flow_mapping_paths',
'province_roads_district_center_flow_paths.xlsx')
fail_scenarios_data = os.path.join(
output_path, 'hazard_scenarios',
'province_roads_hazard_intersections.xlsx')
rd_prop_file = os.path.join(data_path, 'Roads', 'road_properties',
'road_properties.xlsx')
cols = [
'origin', 'destination', 'min_edge_path', 'max_edge_path',
'min_netrev', 'max_netrev', 'min_croptons', 'max_croptons',
'min_distance', 'max_distance', 'min_time', 'max_time', 'min_gcost',
'max_gcost'
]
'''
Path OD flow disruptions
'''
for prn in range(len(province_list)):
# for prn in range(0,1):
province = province_list[prn]
# set all paths for all input files we are going to use
province_name = province.replace(' ', '').lower()
for tr_wt in truck_unit_wt:
flow_output_excel = os.path.join(
output_path, 'failure_results',
'multiple_edge_failures_totals_{0}_{1}_tons_projections.xlsx'.
format(province_name, int(tr_wt)))
excl_wrtr_1 = pd.ExcelWriter(flow_output_excel)
for grth in growth_scenarios:
edge_fail_ranges = []
for t in range(len(types)):
df_path = os.path.join(
output_path, 'failure_results',
'multiple_edge_failures_all_path_impacts_{0}_{1}_{2}_tons.csv'
.format(province_name, types[t], int(tr_wt)))
df = pd.read_csv(df_path).fillna(0)
df = df[[
'edge_id', 'econ_value', 'tons', 'old_cost',
'new_cost', 'no_access'
]]
df = df.drop_duplicates([
'edge_id', 'econ_value', 'tons', 'old_cost',
'new_cost', 'no_access'
])
# df = df.groupby(['edge_id', 'no_access'])['econ_value','tons','old_cost','new_cost'].
edge_impact = df[['edge_id']]
# print (edge_impact)
cols = []
for year in range(2016, 2050):
edge_impact['{0}_econ_value_{1}'.format(
types[t], year)] = math.pow(
(1 + grth[0] / 100),
year - base_year) * df['econ_value']
edge_impact['{0}_tons_{1}'.format(
types[t], year)] = math.pow(
(1 + grth[0] / 100),
year - base_year) * df['tons']
edge_impact['{0}_econ_loss_{1}'.format(
types[t], year
)] = df['no_access'] * edge_impact[
'{0}_econ_value_{1}'.format(types[t], year)] + (
1 - df['no_access']) * np.maximum(
1,
np.ceil(edge_impact['{0}_tons_{1}'.format(
types[t], year)] / tr_wt)) * (
df['new_cost'] - df['old_cost'])
cols += [
'{0}_econ_value_{1}'.format(types[t], year),
'{0}_tons_{1}'.format(types[t], year),
'{0}_econ_loss_{1}'.format(types[t], year)
]
edge_impact = edge_impact.groupby(
['edge_id'])[cols].sum().reset_index()
edge_fail_ranges.append(edge_impact)
print(
'Done with province {0} {1} tons {2} with growth rate {3}'
.format(province_name, tr_wt, types[t], grth))
edge_impact = edge_fail_ranges[0]
edge_impact = pd.merge(edge_impact,
edge_fail_ranges[1],
how='left',
on=['edge_id']).fillna(0)
for year in range(2016, 2050):
edge_impact['swap'] = edge_impact.apply(
lambda x: swap_min_max(
x, 'min_econ_value_{0}'.format(year),
'max_econ_value_{0}'.format(year)),
axis=1)
edge_impact[[
'min_econ_value_{0}'.format(year),
'max_econ_value_{0}'.format(year)
]] = edge_impact['swap'].apply(pd.Series)
edge_impact.drop('swap', axis=1, inplace=True)
edge_impact['swap'] = edge_impact.apply(
lambda x: swap_min_max(x, 'min_tons_{0}'.format(year),
'max_tons_{0}'.format(year)),
axis=1)
edge_impact[[
'min_tons_{0}'.format(year),
'max_tons_{0}'.format(year)
]] = edge_impact['swap'].apply(pd.Series)
edge_impact.drop('swap', axis=1, inplace=True)
edge_impact['swap'] = edge_impact.apply(
lambda x: swap_min_max(
x, 'min_econ_loss_{0}'.format(year),
'max_econ_loss_{0}'.format(year)),
axis=1)
edge_impact[[
'min_econ_loss_{0}'.format(year),
'max_econ_loss_{0}'.format(year)
]] = edge_impact['swap'].apply(pd.Series)
edge_impact.drop('swap', axis=1, inplace=True)
edge_impact.to_excel(excl_wrtr_1, grth[1], index=False)
excl_wrtr_1.save()
def run_mrio_disaggregate(notrade=False,
min_rice=True,
own_production_ratio=0.8):
"""This function will disaggregate the (single-region) national Input-Output table to a provincial multiregional Input-Output table
Parameters
- notrade - Boolean to specify whether we should include trade in the disaggregation. This should be set to **True** in the first step of the disaggregation. The default is set to **False**
- min_rice - Boolean to determine whether you want to use the minimal rice value or the maximum rice value from the flow analysis. The default is set to **True**
- own_production_ratio - Specify how much supply and demand is locally supplied and used, and how much is imported/exported. The default is set to **0.8**
Outputs
- .csv file containing the new multiregional Input-Output table.
- pandas DataFrame with a multiregional Input-Output table
"""
data_path = load_config()['paths']['data']
# load provincial shapefile
provinces = load_provincial_stats(data_path)
provinces.name_eng = provinces.name_eng.apply(
lambda x: x.replace(' ', '_').replace('-', '_'))
# estimate gross value added
provinces['raw_gva'] = estimate_gva(provinces, in_million=True)
# prepare proxies for settings_trade
create_proxies(data_path,
notrade=notrade,
own_production_ratio=own_production_ratio,
min_rice=min_rice)
# run mrio_disaggregate
if notrade == False:
p = subprocess.Popen(['mrio_disaggregate', 'settings_trade.yml'],
cwd=os.path.join(data_path, 'IO_analysis',
'MRIO_TABLE'))
p.wait()
else:
p = subprocess.Popen(['mrio_disaggregate', 'settings_notrade.yml'],
cwd=os.path.join(data_path, 'IO_analysis',
'MRIO_TABLE'))
p.wait()
# get reordered mrio with new region classification
Xin = load_output(data_path, provinces, notrade=notrade)
# convert to numpy matrix
X0 = Xin.as_matrix()
# get sum of rows and columns
u = X0.sum(axis=1)
v = X0.sum(axis=0)
# and only keep T
v[:(len(u) - 3)] = u[:-3]
# apply RAS method to rebalance the table
X1 = ras_method(X0, u, v, eps=5e-5)
# copy new balanced table into dataframe
Xin.iloc[:, :] = X1
# add indices to it
index = list(Xin.index)
index[567], index[568], index[569] = ('total',
'tax_sub'), ('total',
'import_'), ('total',
'valueA')
Xin.index = pd.MultiIndex.from_tuples(index, names=['region', 'sector'])
# save outpout
if notrade == True:
Xin.to_csv(
os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE',
'notrade_trade.csv'))
else:
Xin.to_csv(
os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE',
'IO_VIETNAM.csv'))
if notrade == False:
Region_sum = Xin.groupby(Xin.columns.get_level_values(0),
axis='columns').sum().groupby(
Xin.index.get_level_values(0),
axis='index').sum()
Region_sum.to_csv(
os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE',
'region_trade.csv'))
return Xin
def main():
"""Intersect networks with hazards
1. Specify the paths from where you to read and write:
- Input data
- Intermediate calcuations data
- Output results
2. Supply input data and parameters
- Names of the three Provinces - List of string types
- Paths of the mode files - List of tuples of strings
- Names of modes - List of strings
- Names of output modes - List of strings
- Condition 'Yes' or 'No' is the users wants to process results
3. Give the paths to the input data files:
- Hazard directory
"""
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
# Supply input data and parameters
provinces = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa']
modes_file_paths = [('Roads', 'national_roads'),
('Railways', 'national_rail'),
('Airports', 'airnetwork'), ('Waterways', 'waterways'),
('Waterways', 'waterways')]
modes = ['road', 'rail', 'air', 'inland', 'coastal']
out_modes = [
'national_roads', 'national_rail', 'air_ports', 'inland_ports',
'sea_ports'
]
province_results = 'Yes'
national_results = 'Yes'
# Give the paths to the input data files
hazard_dir = os.path.join(data_path, 'Hazard_data')
# Specify the output files and paths to be created
output_dir = os.path.join(output_path,
'networks_hazards_intersection_shapefiles')
if not os.path.exists(output_dir):
os.mkdir(output_dir)
# Start province roads and all hazards intersections
if province_results == 'Yes':
for province in provinces:
province_name = province.replace(' ', '').lower()
road_shp_path = os.path.join(
data_path, 'Roads', '{}_roads'.format(province_name),
'vietbando_{}_edges.shp'.format(province_name))
road_shp = 'vietbando_{}_edges.shp'.format(province_name)
output_dir = os.path.join(
output_path, 'networks_hazards_intersection_shapefiles',
'{}_roads_hazards_intersections'.format(province_name))
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
print('* Starting {} roads and all hazards intersections'.format(
province))
intersect_networks_and_all_hazards(hazard_dir,
road_shp_path,
road_shp,
output_dir,
network_type='edges')
if national_results == 'Yes':
for m in range(len(modes)):
mode_data_path = os.path.join(data_path, modes_file_paths[m][0],
modes_file_paths[m][1])
if modes[m] in ['road', 'rail']:
for mode_file in os.listdir(mode_data_path):
try:
if mode_file.endswith(
".shp") and 'edges' in mode_file.lower().strip(
):
edges_in = os.path.join(mode_data_path, mode_file)
edges_name = mode_file
except:
return ('Network edge file necessary')
output_dir = os.path.join(
output_path, 'networks_hazards_intersection_shapefiles',
'{}_hazard_intersections'.format(out_modes[m]))
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
print('* Starting national {} and all hazards intersections'.
format(modes[m]))
intersect_networks_and_all_hazards(hazard_dir,
edges_in,
edges_name,
output_dir,
network_type='edges')
elif modes[m] in ['air', 'inland', 'coastal']:
for mode_file in os.listdir(mode_data_path):
try:
if mode_file.endswith(
".shp") and 'nodes' in mode_file.lower().strip(
):
nodes_in = os.path.join(mode_data_path, mode_file)
nodes_name = mode_file
except:
return ('Network node file necessary')
output_dir = os.path.join(
output_path, 'networks_hazards_intersection_shapefiles',
'{}_hazard_intersections'.format(out_modes[m]))
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
print('* Starting national {} and all hazards intersections'.
format(modes[m]))
intersect_networks_and_all_hazards(hazard_dir,
nodes_in,
nodes_name,
output_dir,
network_type='nodes')
def main():
data_path,calc_path,output_path = load_config()['paths']['data'],load_config()['paths']['calc'],load_config()['paths']['output']
# provinces to consider
province_list = ['Lao Cai','Binh Dinh','Thanh Hoa']
province_terrian = ['mountain','flat','flat']
# shp_output_path = os.path.join(output_path,'flow_mapping_shapefiles')
# flow_output_excel = os.path.join(output_path,'flow_mapping_paths','province_roads_district_center_flow_paths.xlsx')
# excl_wrtr = pd.ExcelWriter(flow_output_excel)
fail_scenarios_data = os.path.join(output_path,'hazard_scenarios','province_roads_hazard_intersections.xlsx')
rd_prop_file = os.path.join(data_path,'Roads','road_properties','road_properties.xlsx')
'''
Path OD flow disruptions
'''
# for prn in range(len(province_list)):
for prn in range(0,1):
province = province_list[prn]
# set all paths for all input files we are going to use
province_name = province.replace(' ','').lower()
all_edge_fail_scenarios = pd.read_excel(fail_scenarios_data,sheet_name = province_name)
all_edges = list(set(all_edge_fail_scenarios['edge_id'].values.tolist()))
all_edge_fail_scenarios['road_cond'] = 'unknown'
all_edge_fail_scenarios['asset_type'] = 'unknown'
all_edge_fail_scenarios['width'] = 0
# print ('done with file reading')
'''
First do single edge failures
'''
edges_in = os.path.join(data_path,'Roads','{}_roads'.format(province_name),'vietbando_{}_edges.shp'.format(province_name))
edges = province_shapefile_to_dataframe(edges_in,province_terrian[prn],rd_prop_file)
edge_attr = list(zip(edges['edge_id'].values.tolist(),edges['road_cond'].values.tolist(),edges['asset_type'].values.tolist(),edges['width'].values.tolist()))
# print (edge_attr)
edge_attr = [e for e in edge_attr if e[0] in all_edges]
for e in edge_attr:
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'road_cond'] = e[1]
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'asset_type'] = e[2]
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'width'] = e[3]
# all_edge_fail_scenarios['attributes'] = all_edge_fail_scenarios.apply(lambda x: assign_failed_edge_attributes(x,edge_attr),axis = 1)
# all_edge_fail_scenarios['asset_type','width'] = all_edge_fail_scenarios['attributes'].apply(pd.Series)
# all_edge_fail_scenarios.drop('attributes',axis=1,inplace=True)
single_edge_path = os.path.join(output_path,'failure_results','single_edge_failures_totals_{0}.csv'.format(province_name))
edge_impacts = pd.read_csv(single_edge_path)
edge_impacts_attr = list(zip(edge_impacts['edge_id'].values.tolist(),edge_impacts['min_econ_loss'].values.tolist(),edge_impacts['max_econ_loss'].values.tolist()))
all_edge_fail_scenarios['min_econ_loss'] = 0
all_edge_fail_scenarios['max_econ_loss'] = 0
edge_impacts_attr = [e for e in edge_impacts_attr if e[0] in all_edges]
for e in edge_impacts_attr:
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'min_econ_loss'] = e[1]
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'max_econ_loss'] = e[2]
# all_edge_fail_scenarios['attributes'] = all_edge_fail_scenarios.apply(lambda x: assign_failed_edge_attributes(x,edge_impacts_attr),axis = 1)
# all_edge_fail_scenarios['min_econ_loss','max_econ_loss'] = all_edge_fail_scenarios['attributes'].apply(pd.Series)
# all_edge_fail_scenarios.drop('attributes',axis=1,inplace=True)
df_path = os.path.join(output_path,'hazard_scenarios','roads_hazard_intersections_{}.csv'.format(province_name))
all_edge_fail_scenarios.to_csv(df_path,index = False)
def mrio_to_excel(Xin, min_rice=True):
"""Save the newly created multiregional Input-Output table to Excel, in the format required for the MRIA calculation.
Parameters
- Xin - pandas DataFrame of the new multiregional Input-Output table
- min_rice - Boolean to determine whether you want to use the minimal rice value or the maximum rice value from the flow analysis. The default is set to **True**
Outputs
- .xlsx file with the multiregional Input-Output table
"""
data_path = load_config()['paths']['data']
Xnew = Xin.copy()
# prepare exoort and finalD data
Exports = pd.DataFrame(
Xnew.iloc[:, Xnew.columns.get_level_values(1) == 'col3'].sum(axis=1),
columns=['Exports'])
Exports.columns = pd.MultiIndex.from_tuples(
list(zip(['Total'], ['Export'])))
FinalD_ToT = Xnew.iloc[:, ((Xnew.columns.get_level_values(1) == 'col1') |
(Xnew.columns.get_level_values(1) == 'col2'))]
FinalD_ToT = FinalD_ToT.groupby(level=0, axis=1).sum()
FinalD_ToT.columns = pd.MultiIndex.from_tuples(
list(zip(FinalD_ToT.columns,
len(FinalD_ToT.columns) * ['FinDem'])))
Xnew.drop(['col1', 'col2', 'col3'], axis=1, level=1, inplace=True)
Xnew = pd.concat([Xnew, FinalD_ToT, Exports], axis=1)
# write to excel
if min_rice == True:
writer = pd.ExcelWriter(
os.path.join(data_path, 'input_data', 'IO_VIETNAM_MIN.xlsx'))
else:
writer = pd.ExcelWriter(
os.path.join(data_path, 'input_data', 'IO_VIETNAM_MAX.xlsx'))
# write T
df_T = Xnew.iloc[:567, :567]
df_T.columns = df_T.columns.droplevel()
df_labels_T = pd.DataFrame(df_T.reset_index()[['region', 'sector']])
df_T.reset_index(inplace=True, drop=True)
df_T.to_excel(writer, 'T', index=False, header=False)
df_labels_T.to_excel(writer, 'labels_T', index=False, header=False)
# write FD
df_FD = Xnew.iloc[:567, 567:630]
df_labels_FD = pd.DataFrame(list(df_FD.columns))
df_FD.columns = df_FD.columns.droplevel()
df_FD.reset_index(inplace=True, drop=True)
df_FD.to_excel(writer, 'FD', index=False, header=False)
df_labels_FD.to_excel(writer, 'labels_FD', index=False, header=False)
# write ExpROW
df_ExpROW = Exports[:567]
df_labels_ExpROW = pd.DataFrame(list(
df_ExpROW.columns.get_level_values(1)))
df_ExpROW.reset_index(inplace=True, drop=True)
df_ExpROW.columns = df_ExpROW.columns.droplevel()
df_ExpROW.to_excel(writer, 'ExpROW', index=False, header=False)
df_labels_ExpROW.reset_index(inplace=True, drop=True)
df_labels_ExpROW.columns = ['Export']
df_labels_ExpROW.to_excel(writer,
'labels_ExpROW',
index=False,
header=False)
# write VA
df_VA = pd.DataFrame(Xnew.iloc[567:, :].T[('total', 'tax_sub')] +
Xnew.iloc[567:, :].T[('total', 'valueA')],
columns=['VA'])
df_VA['imports'] = Xnew.iloc[567:, :].T[('total', 'import_')]
df_VA.reset_index(inplace=True, drop=True)
df_VA.to_excel(writer, 'VA', index=False, header=False)
df_labels_VA = pd.DataFrame(['Import', 'VA']).T
df_labels_VA.to_excel(writer, 'labels_VA', index=False, header=False)
# save excel
writer.save()
def create_disruption(input_file,
output_dir,
min_rice=True,
single_point=True):
"""Create disruption file for the economic analysis.
The input for this disruption file is the outcome of the flow analysis. This function
translate the failure in transport flows into impacts to the economic sectors.
Parameters
----------
input_file : str
name of the path to a flow failure scenario file
output_dir : str
name for output directory for mapper file.
min_rice : bool, optional
determine whether you want to use the minimal rice value or the maximum rice value from
the flow analysis. This MUST match the value used when creating the MRIO table. The
default is **True**.
single_point : bool, optional
determine whether you are converting a single-point or multi-point failure analaysis to
a disruption file. The default is **True**.
Returns
-------
event_dict
Dictionary of all unique failure events, in terms of percentage disruption per sector
in each directly affected region due to the flow failure.
"""
# Define current directory and data directory
data_path = load_config()['paths']['data']
# read national IO
comm_des = df_com_to_ind(pd.read_excel(os.path.join(
data_path, 'OD_data', 'national_scale_od_matrix.xlsx'),
sheet_name='total'),
min_rice=min_rice)
# read vietnam failure results
vnm_failure_all_failure = pd.read_csv(input_file, index_col=[0])
if single_point == False:
mapper = dict(
zip(vnm_failure_all_failure.index.unique(), [
'multi_{}'.format(n) for n in np.arange(
1,
len(vnm_failure_all_failure.index.unique()) + 1, 1)
]))
vnm_failure_all_failure.index = vnm_failure_all_failure.index.map(
lambda x: mapper[x])
pd.DataFrame.from_dict(mapper, orient='index').to_csv(
os.path.join(output_dir, 'mapper.csv'))
event_dict = {}
for id_, scenario in vnm_failure_all_failure.groupby(level=0):
if id_ == 'multi_102':
break
disruption = df_com_to_ind(scenario,
min_rice=min_rice).div(comm_des).dropna(
axis=0, how='all').fillna(0)
disruption = pd.DataFrame(disruption.stack(0))
disruption.columns = ['value']
disruption = disruption.loc[disruption.value > 0]
disruption['value'] = 1 - (disruption['value'] * 0.2)
event_dict[id_] = disruption['value'].to_dict()
return event_dict
def load_db_SUT(table_in,RoW=None):
data_path = load_config()['paths']['data']
'''CREATE GAMS WORKSPACE'''
ws = GamsWorkspace(os.path.join(data_path,'gams_runs'))
''' CREATE INPUT FILES GAMS GDX '''
db = ws.add_database()
#set regions
reg = db.add_set("reg",1,"Regions")
for r in (table_in.countries):
reg.add_record(r)
#set rowcol
rowcol = db.add_set("rowcol",1,"All rows and columns")
industries = list(table_in.sectors)
products = list(table_in.products)
final_demand = ['FinalD']
Import_lab = ['Import']
Export_lab = ['Export']
VA_lab = ['VA']
rowcol_input = industries + final_demand + Export_lab + products + VA_lab
for r in (rowcol_input):
rowcol.add_record(r)
#set row
row = db.add_set("row",1,"All rows")
row_input = products + VA_lab + Import_lab
for r in (row_input):
row.add_record(r)
#set col
col = db.add_set("col",1,"All columns")
col_input = industries + final_demand
for r in (col_input):
col.add_record(r)
#set industries
industries_ = db.add_set("ind",1,"Industries")
for r in industries:
industries_.add_record(r)
#set Use table
use_m = db.add_parameter("REG_USE2013", 4, "Interaction matrix")
for k, v in table_in.Use.items():
use_m.add_record(k).value = v
#set Supply table
sup_m = db.add_parameter("REG_SUP2013", 4, "Interaction matrix")
for k, v in table_in.Sup.items():
sup_m.add_record(k).value = v
#set export ROW
exp = db.add_parameter("ExpROW_ini", 3, "Exports to ROW")
for k, v in table_in.ExpROW.items():
exp.add_record(k).value = v
#set export ROW
imp = db.add_parameter("ImpROW_ini", 3, "Imports from ROW")
for k, v in table_in.ImpROW.items():
imp.add_record(k).value = v
#set ValueA
val = db.add_parameter("ValueA_ini", 3, "Value Added")
for k, v in table_in.ValueA.items():
val.add_record(k).value = v
# And save to GDX file
db.export(os.path.join(data_path,"gams_runs","{}.gdx".format(table_in.name)))
def main():
data_path = load_config()['paths']['data']
hazard_intersections_path = load_config()['paths']['output']
# modes = ['road','rail','air','inland','coastal']
modes = ['road']
out_modes = [
'national_roads', 'national_rail', 'air_ports', 'inland_ports',
'sea_ports'
]
bnds = [3, 4, 5]
thresholds = [1, 2, 3, 4, 999]
commune_shp = os.path.join(data_path, 'Vietnam_boundaries',
'who_boundaries', 'who_communes.shp')
hazard_description_file = os.path.join(
data_path, 'Hazard_data', 'hazard_data_folder_data_info.xlsx')
hazard_df = pd.read_excel(hazard_description_file,
sheet_name='file_contents')
hazard_files = hazard_df['file_name'].values.tolist()
data_excel = os.path.join(
hazard_intersections_path, 'hazard_scenarios',
'national_scale_hazard_intersections_roads.xlsx')
excel_writer = pd.ExcelWriter(data_excel)
for m in range(len(modes)):
# set all paths for all input files we are going to use
data_dict = []
intersection_dir = os.path.join(
hazard_intersections_path,
'{}_hazard_intersections'.format(out_modes[m]))
for root, dirs, files in os.walk(intersection_dir):
for file in files:
if file.endswith(".shp"):
hazard_dict = {}
hazard_dict['sector'] = modes[m]
hazard_shp = os.path.join(root, file)
hz_file = [h for h in hazard_files if h in file][0]
hazard_dict['hazard_type'] = hazard_df.loc[
hazard_df.file_name == hz_file].hazard_type.values[0]
hazard_dict['model'] = hazard_df.loc[
hazard_df.file_name == hz_file].model.values[0]
hazard_dict['year'] = hazard_df.loc[hazard_df.file_name ==
hz_file].year.values[0]
hazard_dict['climate_scenario'] = hazard_df.loc[
hazard_df.file_name ==
hz_file].climate_scenario.values[0]
hazard_dict['probability'] = hazard_df.loc[
hazard_df.file_name == hz_file].probability.values[0]
band_type = hazard_df.loc[hazard_df.file_name ==
hz_file].banded.values[0]
if str(band_type) == 'True':
hazard_dict['band_num'] = [
b for b in bnds if '{}_band'.format(b) in file
][0]
band_names = hazard_df.loc[
hazard_df.file_name ==
hz_file].bands.values[0].split(',')
hazard_dict['band_name'] = [
b for b in band_names
if str(hazard_dict['band_num']) in b
][0]
hazard_dict['min_val'] = 0
hazard_dict['max_val'] = 0
else:
hazard_dict['band_num'] = 0
hazard_dict['band_name'] = 'none'
hazard_thrs = [
(thresholds[t], thresholds[t + 1])
for t in range(len(thresholds) - 1)
if '{0}m-{1}m'.format(thresholds[t], thresholds[
t + 1]) in file
][0]
hazard_dict['min_val'] = hazard_thrs[0]
hazard_dict['max_val'] = hazard_thrs[1]
data_dict = spatial_scenario_selection_national(
hazard_shp, commune_shp, hazard_dict, data_dict,
modes[m])
print('Done with', file)
data_df = pd.DataFrame(data_dict)
data_df_cols = data_df.columns.values.tolist()
if 'length' in data_df_cols:
selected_cols = [cols for cols in data_df_cols if cols != 'length']
data_df = data_df.groupby(
selected_cols)['length'].sum().reset_index()
data_df.to_excel(excel_writer, modes[m], index=False)
excel_writer.save()
del data_df
'Economic_failure_results',
'summarized'
)
'''Create output folders'''
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
sums.to_csv(os.path.join(data_path,
'Results',
'Economic_failure_results',
'summarized',
'{}_summarized.csv'.format(os.path.basename(os.path.splitext(input_file)[0]))))
return pd.concat(collect_outputs),sums
if __name__ == '__main__':
data_path = load_config()['paths']['data']
# input_file = os.path.join(data_path,'Results','Failure_results','single_edge_failures_totals_national_road_max.csv')
# get_all_input_files = [os.path.join(data_path,'Results','Failure_results',x) for x in os.listdir(os.path.join(data_path,'Results','Failure_results')) if x.endswith(".csv")]
get_all_input_files = [os.path.join(data_path,'Results','Failure_results','roads',x) for x in os.listdir(os.path.join(data_path,'Results','Failure_results','roads')) if x.endswith(".csv")]
# with Pool(int(cpu_count())-2) as pool:
# pool.map(estimate_losses,get_all_input_files,chunksize=1)
for gi in get_all_input_files:
estimate_losses(gi)
def main():
'''
Create the database connection
'''
conf = load_config()
try:
conn = psycopg2.connect(**conf['database'])
except:
print("I am unable to connect to the database")
curs = conn.cursor()
engine = create_engine(
'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
{**conf['database']}))
'''
Step 2: Create the OD proprotions for the differnet modes
'''
'''
First get the modal shares
'''
modes = ['road', 'rail', 'air', 'water']
ind_cols = [
'sugar', 'wood', 'steel', 'constructi', 'cement', 'fertilizer', 'coal',
'petroluem', 'manufactur', 'fishery', 'meat'
]
crop_cols = [
'rice', 'cash', 'cass', 'teas', 'maiz', 'rubb', 'swpo', 'acof', 'rcof',
'pepp'
]
commodities = ind_cols + crop_cols
province_id_list = []
sql_query = "select od_id,name_eng from province_level_stats"
curs.execute(sql_query)
read_layer = curs.fetchall()
for row in read_layer:
province_id_list.append((row[0], row[1]))
pth_dict = {}
pth_idx = 0
pth_list = []
pth_k_list = []
for com in commodities:
for m in modes:
ifile = 'network_od_flows_' + com + m + '.csv'
od_flows = pd.read_csv(ifile).fillna(0)
od_flows = od_flows[od_flows['Tonnage'] > 0.5]
if od_flows.empty is False:
flow_node_edge = od_flows.groupby([
'node_path', 'edge_path', 'Origin_region',
'Destination_region'
])['Tonnage'].sum().reset_index()
pth_idx, pth_list, pth_k_list, pth_dict = get_node_edge_path_flows(
flow_node_edge, province_id_list, com, pth_idx, pth_list,
pth_k_list, pth_dict, 0.5)
del od_flows
print('Done with', ifile)
pth_tuple_list = []
for key, values in pth_dict.items():
pth_tuple = [0] * (3 + len(commodities))
pth_tuple[0] = key
pth_tuple[1] = pth_dict[key]['node_path']
pth_tuple[2] = pth_dict[key]['edge_path']
values_keys = pth_dict[key].keys()
for i in range(len(commodities)):
if commodities[i] in values_keys:
pth_tuple[i + 3] = pth_dict[key][commodities[i]]
# pth_tuple.append(sum(pth_tuple[4:]))
pth_tuple_list.append(tuple(pth_tuple))
excel_writer = pd.ExcelWriter('vnm_path_flows.xlsx')
df = pd.DataFrame(pth_tuple_list,
columns=['path_index', 'node_path', 'edge_path'] +
commodities)
df.to_excel(excel_writer, 'path_flows', index=False)
excel_writer.save()
def main():
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
'''
cols = ['band_name','band_num','climate_scenario','commune_id','commune_name','district_id','district_name',
'edge_id','hazard_type','max_val','min_val','model','probability','province_id','province_name','sector',
'year','length','road_cond','asset_type','width','min_econ_loss','max_econ_loss']
'''
start_year = 2016
end_year = 2050
truck_unit_wt = [5.0, 20.0]
discount_rate = 12.0
total_discount_ratio = []
for year in range(start_year, end_year):
# total_discount_ratio += 1.0/math.pow(1.0 + 1.0*discount_rate/100.0,year - start_year)
total_discount_ratio.append(
1.0 /
math.pow(1.0 + 1.0 * discount_rate / 100.0, year - start_year))
# total_discount_ratio = sum(total_discount_ratio_list)
df_path = os.path.join(data_path, 'Adaptation_options',
'adaptation_options.xlsx')
adaptation_df = pd.read_excel(df_path, sheet_name='adapt_options')
adaptation_df['total_discount_ratio'] = sum(total_discount_ratio)
min_maintain_discount_ratio_list = []
max_maintain_discount_ratio_list = []
for iter_, row in adaptation_df.iterrows():
min_maintain_schedule = row['maintenance_times_min']
max_maintain_schedule = row['maintenance_times_max']
min_maintain_discount_ratio = 0
max_maintain_discount_ratio = 0
max_maintain_discount_years = np.arange(start_year, end_year,
min_maintain_schedule)
min_maintain_discount_years = np.arange(start_year, end_year,
max_maintain_schedule)
for year in max_maintain_discount_years[1:]:
max_maintain_discount_ratio += 1.0 / math.pow(
1.0 + 1.0 * discount_rate / 100.0, year - start_year)
for year in min_maintain_discount_years[1:]:
min_maintain_discount_ratio += 1.0 / math.pow(
1.0 + 1.0 * discount_rate / 100.0, year - start_year)
min_maintain_discount_ratio_list.append(min_maintain_discount_ratio)
max_maintain_discount_ratio_list.append(max_maintain_discount_ratio)
adaptation_df[
'min_maintain_discount_ratio'] = min_maintain_discount_ratio_list
adaptation_df[
'max_maintain_discount_ratio'] = max_maintain_discount_ratio_list
adaptation_df['min_benefit'] = adaptation_df[
'rehab_cost_min'] * adaptation_df['total_discount_ratio']
adaptation_df['max_benefit'] = adaptation_df[
'rehab_cost_max'] * adaptation_df['total_discount_ratio']
adaptation_df['min_cost'] = adaptation_df['adapt_cost_min'] * adaptation_df[
'total_discount_ratio'] + adaptation_df[
'maintain_cost_min'] * adaptation_df['min_maintain_discount_ratio']
adaptation_df['max_cost'] = adaptation_df['adapt_cost_max'] * adaptation_df[
'total_discount_ratio'] + adaptation_df[
'maintain_cost_max'] * adaptation_df['max_maintain_discount_ratio']
print(adaptation_df)
'''
Add new strategy
cols = ['strategy','asset_type','asset_cond','location','height_m','adapt_cost_min','adapt_cost_max',
'maintain_cost_min','maintain_cost_max','rehab_cost_min','rehab_cost_max','height_incr_min',
'height_incr_max','maintenance_times_min','maintenance_times_max','cost_unit']
'''
cols = [
'strategy', 'asset_type', 'asset_cond', 'location', 'height_m',
'cost_unit', 'min_cost', 'max_cost', 'min_benefit', 'max_benefit'
]
adaptation_options = adaptation_df[cols]
ad_opt = []
st_desc = ['road change', 'roads', 'unpaved-paved', 'all', 0, '$/m2']
st_min_cost = adaptation_df.loc[adaptation_df['asset_cond'] == 'paved',
'min_cost'].sum()
st_max_cost = adaptation_df.loc[adaptation_df['asset_cond'] == 'paved',
'max_cost'].sum()
st_min_benefit = adaptation_df.loc[
adaptation_df['asset_cond'] == 'unpaved',
'rehab_cost_min'].sum() * total_discount_ratio[0] + adaptation_df.loc[
adaptation_df['asset_cond'] == 'paved',
'rehab_cost_min'].sum() * sum(total_discount_ratio[1:])
st_max_benefit = adaptation_df.loc[
adaptation_df['asset_cond'] == 'unpaved',
'rehab_cost_max'].sum() * total_discount_ratio[0] + adaptation_df.loc[
adaptation_df['asset_cond'] == 'paved',
'rehab_cost_max'].sum() * sum(total_discount_ratio[1:])
ad_opt.append(st_desc +
[st_min_cost, st_max_cost, st_min_benefit, st_max_benefit])
new_ht = 6
st_desc = [
'dyke building', 'dyke', 'rural', 'sea', new_ht, 'million USD/km'
]
st_min_cost = adaptation_df.loc[
adaptation_df['height_m'] == 4,
'min_cost'].sum() + (new_ht - 4) * adaptation_df.loc[
adaptation_df['height_m'] == 4,
'height_incr_min'].sum() * sum(total_discount_ratio)
st_max_cost = adaptation_df.loc[
adaptation_df['height_m'] == 4,
'max_cost'].sum() + (new_ht - 4) * adaptation_df.loc[
adaptation_df['height_m'] == 4,
'height_incr_max'].sum() * sum(total_discount_ratio)
st_min_benefit = adaptation_df.loc[adaptation_df['height_m'] == 4,
'min_benefit'].sum()
st_max_benefit = adaptation_df.loc[adaptation_df['height_m'] == 4,
'max_benefit'].sum()
ad_opt.append(st_desc +
[st_min_cost, st_max_cost, st_min_benefit, st_max_benefit])
ad_opt_df = pd.DataFrame(ad_opt, columns=cols)
adaptation_options = adaptation_options.append(ad_opt_df,
ignore_index=True)
print(adaptation_options)
cols = [
'band_num', 'climate_scenario', 'edge_id', 'hazard_type', 'max_val',
'min_val', 'model', 'probability', 'year', 'exposure_length'
]
# index_cols = ['edge_id','hazard_type','model','climate_scenario','year','road_cond','asset_type','width','road_length']
selection_criteria = [
'commune_id', 'hazard_type', 'model', 'climate_scenario', 'year'
]
filter_cols = ['edge_id', 'exposed_length'] + selection_criteria
# provinces to consider
province_list = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa']
province_terrian = ['mountain', 'flat', 'flat']
growth_scenarios = [(5, 'low'), (6.5, 'forecast'), (10, 'high')]
base_year = 2016
types = ['min', 'max']
fail_scenarios_data = os.path.join(
output_path, 'hazard_scenarios',
'province_roads_hazard_intersections.xlsx')
rd_prop_file = os.path.join(data_path, 'Roads', 'road_properties',
'road_properties.xlsx')
duration_max = [10, 15, 20, 25, 30]
length_thr = 100.0
for prn in range(len(province_list)):
# for prn in range(1,3):
province = province_list[prn]
# set all paths for all input files we are going to use
province_name = province.replace(' ', '').lower()
all_edge_fail_scenarios = pd.read_excel(fail_scenarios_data,
sheet_name=province_name)
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['probability'] ==
'none', 'probability'] = 1.0
all_edge_fail_scenarios['probability'] = pd.to_numeric(
all_edge_fail_scenarios['probability'])
all_edge_fail_scenarios.rename(columns={'length': 'exposure_length'},
inplace=True)
all_edge_fail_scenarios = all_edge_fail_scenarios[cols]
all_edge_fail_scenarios = all_edge_fail_scenarios.drop_duplicates(
subset=cols, keep=False)
all_edges = list(
set(all_edge_fail_scenarios['edge_id'].values.tolist()))
all_edge_fail_scenarios['road_cond'] = 'unknown'
all_edge_fail_scenarios['asset_type'] = 'unknown'
all_edge_fail_scenarios['width'] = 0
all_edge_fail_scenarios['road_length'] = 0
edges_in = os.path.join(data_path, 'Roads',
'{}_roads'.format(province_name),
'vietbando_{}_edges.shp'.format(province_name))
edges = province_shapefile_to_dataframe(edges_in,
province_terrian[prn],
rd_prop_file)
edge_attr = list(
zip(edges['edge_id'].values.tolist(),
edges['road_cond'].values.tolist(),
edges['asset_type'].values.tolist(),
edges['width'].values.tolist(),
edges['length'].values.tolist()))
# print (edge_attr)
edge_attr = [e for e in edge_attr if e[0] in all_edges]
for e in edge_attr:
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] ==
e[0], 'road_cond'] = e[1]
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] ==
e[0], 'asset_type'] = e[2]
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] ==
e[0], 'width'] = e[3]
all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] ==
e[0], 'road_length'] = 1000.0 * e[4]
# all_edge_fail_scenarios['percent_exposure'] = 100.0*all_edge_fail_scenarios['exposure_length']/all_edge_fail_scenarios['road_length']
# df_path = os.path.join(output_path,'hazard_scenarios','roads_hazard_intersections_{}.csv'.format(province_name))
# all_edge_fail_scenarios.to_csv(df_path,index = False)
# all_edge_fail_scenarios = all_edge_fail_scenarios.set_index(index_cols)
# scenarios = list(set(all_edge_fail_scenarios.index.values.tolist()))
all_edge_fail_scenarios = all_edge_fail_scenarios.set_index(
selection_criteria)
scenarios = list(set(all_edge_fail_scenarios.index.values.tolist()))
multiple_ef_list = []
for criteria in criteria_set:
if len(all_edge_fail_scenarios.loc[criteria,
'edge_id'].index) == 1:
efail = [
all_edge_fail_scenarios.loc[criteria, 'edge_id'].item()
]
else:
efail = list(
set(all_edge_fail_scenarios.loc[
criteria, 'edge_id'].values.tolist()))
flength = all_edge_fail_scenarios.loc[criteria, 'length'].sum()
pflength = flength / all_edge_fail_scenarios.loc[
criteria, 'road_length'].sum()
criteria_dict = {
**(dict(list(zip(selection_criteria, criteria)))),
**{
'exposed_length': flength
}
}
multiple_ef_list.append((efail, criteria_dict))
for tr_wt in truck_unit_wt:
flow_output_excel = os.path.join(
output_path, 'failure_results',
'multiple_edge_failures_totals_{0}_{1}_tons_projections.xlsx'.
format(province_name, int(tr_wt)))
adapt_output_excel = os.path.join(
output_path, 'failure_results',
'multiple_edge_failures_scenarios_{0}_{1}_tons_adapt_options.xlsx'
.format(province_name, int(tr_wt)))
excl_wrtr = pd.ExcelWriter(adapt_output_excel)
for grth in growth_scenarios:
loss_time_series = pd.read_excel(flow_output_excel,
sheet_name=grth[1])
for t in range(len(types)):
loss_cols = []
for year in range(start_year, end_year):
# col = '{0}_econ_loss_{1}'.format(types[t],year)
loss_cols.append('{0}_econ_loss_{1}'.format(
types[t], year))
# print (cols)
# loss_time_series = loss_time_series[['edge_id']+cols]
# print (loss_time_series)
loss_time_series['{}_total_loss'.format(
types[t])] = loss_time_series[loss_cols].sum(axis=1)
loss_time_series = loss_time_series[[
'edge_id', 'min_econ_loss_{}'.format(base_year),
'max_econ_loss_{}'.format(base_year), 'min_total_loss',
'max_total_loss'
]]
loss_time_series = loss_time_series[
loss_time_series['max_total_loss'] > 0]
print('done with loss estimation')
for dur in range(len(duration_max)):
scenarios_list = []
for sc in scenarios:
edge = sc[0]
road_cond = sc[-4]
width = sc[-2]
# print (road_cond,width)
sub_df = all_edge_fail_scenarios.loc[sc]
if len(sub_df.index) == 1:
min_exposure_len = sub_df.loc[sc,
'exposure_length']
max_exposure_len = sub_df.loc[sc,
'exposure_length']
min_height = sub_df.loc[sc, 'min_val']
max_height = sub_df.loc[sc, 'max_val']
min_band_num = sub_df.loc[sc, 'band_num']
max_band_num = sub_df.loc[sc, 'band_num']
min_per = sub_df.loc[sc, 'percent_exposure']
max_per = sub_df.loc[sc, 'percent_exposure']
min_dur = 0.01 * duration_max[dur] * sub_df.loc[
sc, 'percent_exposure']
if sub_df.loc[sc, 'exposure_length'] < length_thr:
max_dur = 0.01 * duration_max[
dur] * sub_df.loc[sc, 'percent_exposure']
risk_wt = 0.01 * duration_max[dur] * sub_df.loc[
sc, 'percent_exposure'] * sub_df.loc[
sc, 'probability']
else:
max_dur = duration_max[dur]
risk_wt = duration_max[dur] * sub_df.loc[
sc, 'probability']
else:
prob = list(
set(sub_df['probability'].values.tolist()))
min_height = max(sub_df['min_val'].values.tolist())
max_height = max(sub_df['max_val'].values.tolist())
min_band_num = min(
sub_df['band_num'].values.tolist())
max_band_num = max(
sub_df['band_num'].values.tolist())
sub_df = sub_df.set_index('probability')
exposure_len = []
per = []
risk_wt = 0
for pr in prob:
# per_exp = sub_df.loc[pr,'percent_exposure'].sum()
if sub_df.loc[
pr, 'percent_exposure'].sum() > 100.0:
exposure_len.append(
100.0 *
sub_df.loc[pr,
'exposure_length'].sum() /
sub_df.loc[pr,
'percent_exposure'].sum())
per.append(100.0)
risk_wt += 1.0 * duration_max[dur] * pr
else:
exposure_len.append(
sub_df.loc[pr,
'exposure_length'].sum())
if sub_df.loc[pr, 'exposure_length'].sum(
) < length_thr:
per.append(sub_df.loc[
pr, 'percent_exposure'].sum())
risk_wt += 0.01 * duration_max[
dur] * sub_df.loc[
pr,
'percent_exposure'].sum() * pr
else:
per.append(100.0)
risk_wt += duration_max[dur] * pr
max_exposure_len = max(exposure_len)
min_exposure_len = min(exposure_len)
min_per = min(per)
max_per = max(per)
min_dur = 0.01 * duration_max[dur] * min_per
max_dur = 0.01 * duration_max[dur] * max_per
sc_list = list(sc) + [
min_band_num, max_band_num, min_height, max_height,
min_per, max_per, min_dur, max_dur,
min_exposure_len, max_exposure_len
]
# scenarios_list.append(list(sc) + [min_band_num,max_band_num,min_height,max_height,min_per,max_per,min_dur,max_dur,min_exposure_len,max_exposure_len,risk_wt])
if edge in loss_time_series['edge_id'].values.tolist():
min_daily_loss = loss_time_series.loc[
loss_time_series['edge_id'] == edge,
'min_econ_loss_{}'.format(base_year)].sum()
max_daily_loss = loss_time_series.loc[
loss_time_series['edge_id'] == edge,
'max_econ_loss_{}'.format(base_year)].sum()
edge_options = []
min_edge_ead = sum(
total_discount_ratio
) * risk_wt * loss_time_series.loc[
loss_time_series['edge_id'] == edge,
'min_total_loss'].sum()
max_edge_ead = sum(
total_discount_ratio
) * risk_wt * loss_time_series.loc[
loss_time_series['edge_id'] == edge,
'max_total_loss'].sum()
if max_height > 4:
max_height = 6
if max_height > 0:
edge_options = net_present_value(
adaptation_options,
'height_m', [max_height],
min_edge_ead,
max_edge_ead,
edge_options,
edge_width=1.0,
edge_length=1000.0 * max_exposure_len)
if road_cond == 'unpaved':
edge_options = net_present_value(
adaptation_options,
'asset_cond', ['unpaved', 'unpaved-paved'],
min_edge_ead,
max_edge_ead,
edge_options,
edge_width=width,
edge_length=max_exposure_len)
if road_cond == 'paved':
edge_options = net_present_value(
adaptation_options,
'asset_cond', ['paved'],
min_edge_ead,
max_edge_ead,
edge_options,
edge_width=width,
edge_length=max_exposure_len)
else:
min_daily_loss = 0
max_daily_loss = 0
min_edge_ead = 0
max_edge_ead = 0
edge_options = [{
'strategy': 'none',
'min_npv': 0,
'max_npv': 0,
'min_bc_ratio': 0,
'max_bc_ratio': 0
}]
for options in edge_options:
scenarios_list.append(sc_list + [
min_daily_loss, max_daily_loss, min_edge_ead,
max_edge_ead, options['strategy'],
options['min_npv'], options['max_npv'],
options['min_bc_ratio'],
options['max_bc_ratio']
])
new_cols = [
'min_band', 'max_band', 'min_height', 'max_height',
'min_exposure_percent', 'max_exposure_percent',
'min_duration', 'max_duration', 'min_exposure_length',
'max_exposure_length', 'min_daily_loss',
'max_daily_loss', 'min_npv_nooption',
'max_npv_nooption', 'adapt_strategy', 'min_npv',
'max_npv', 'min_bc_ratio', 'max_bc_ratio'
]
scenarios_df = pd.DataFrame(scenarios_list,
columns=index_cols + new_cols)
# df_path = os.path.join(output_path,'hazard_scenarios','roads_hazard_intersections_{}_risks.csv'.format(province_name))
# scenarios_df.to_csv(df_path,index = False)
scenarios_df.to_excel(excl_wrtr,
grth[1] + '_' +
str(duration_max[dur]),
index=False)
excl_wrtr.save()
print(
'Done with {0} in {1} tons {2} growth scenario {3} days'
.format(province_name, tr_wt, grth[1],
duration_max[dur]))
def main():
"""Process hazard data
1. Specify the paths from where to read and write:
- Input data
- Hazard data
2. Supply input data and parameters
- Thresholds of flood hazards
- Values of bands to be selected
- Color code of multi-band rasters
- Specific file names that might require some specific operations
"""
data_path = load_config()['paths']['data']
root_dir = os.path.join(data_path, 'Hazard_data')
thresholds = [1, 2, 3, 4, 999]
band_vals_1 = [3, 4]
band_vals_2 = [4, 5]
color_codes_1 = [(255, 190, 190), (245, 0, 0), (255, 0, 0)]
color_codes_2 = [(255, 170, 0), (255, 128, 0)]
specific_files = ['LSZ_NgheAn_to_PhuYen.tif']
f_all = []
for root, dirs, files in os.walk(root_dir):
for file in files:
if file.endswith(".tif") or file.endswith(".tiff"):
band_nums, crs, unique_data_values = raster_projections_and_databands(
os.path.join(root, file))
print(file, crs, unique_data_values)
if 'epsg' in crs:
crs_split = crs.split(':')
s_crs = [int(c) for c in crs_split
if c.isdigit() is True][0]
else:
s_crs = 32648
if not unique_data_values:
# threshold based datasets
for t in range(len(thresholds) - 1):
thr_1 = thresholds[t]
thr_2 = thresholds[t + 1]
in_file = os.path.join(root, file)
tmp_1 = os.path.join(
root,
file.split(".tif")[0] + '_mask.tiff')
tmp_2 = os.path.join(
root,
file.split(".tif")[0] + '_mask.shp')
out_file = os.path.join(
root,
file.split(".tif")[0] +
'_{0}m-{1}m_threshold.shp'.format(thr_1, thr_2))
convert_geotiff_to_vector_with_threshold(
thr_1, thr_2, in_file, s_crs, tmp_1, tmp_2,
out_file)
elif band_nums == 1:
# code value based dataset
if file in specific_files:
code_vals = band_vals_1
else:
code_vals = band_vals_2
for c in code_vals:
in_file = os.path.join(root, file)
tmp_1 = os.path.join(
root,
file.split(".tif")[0] + '_mask.tiff')
tmp_2 = os.path.join(
root,
file.split(".tif")[0] + '_mask.shp')
out_file = os.path.join(
root,
file.split(".tif")[0] + '_{}_band.shp'.format(c))
convert_geotiff_to_vector_with_threshold(
c, c + 1, in_file, s_crs, tmp_1, tmp_2, out_file)
if band_nums == 3:
# multi-band color datasets
# remove nodata values from the bands in the raster
raster_rewrite(os.path.join(root, file),
os.path.join(root, 'test.tif'), 0)
for dv in unique_data_values:
if dv in color_codes_1:
thr = 5
bc = dv
in_file = os.path.join(root, file)
tmp_1 = os.path.join(
root,
file.split(".tif")[0] + '_mask.tiff')
tmp_2 = os.path.join(
root,
file.split(".tif")[0] + '_mask.shp')
out_file = os.path.join(
root,
file.split(".tif")[0] +
'_{}_band.shp'.format(thr))
convert_geotiff_to_vector_with_multibands(
bc, in_file, s_crs, tmp_1, tmp_2, out_file)
elif dv in color_codes_2:
thr = 4
bc = dv
in_file = os.path.join(root, file)
tmp_1 = os.path.join(
root,
file.split(".tif")[0] + '_mask.tiff')
tmp_2 = os.path.join(
root,
file.split(".tif")[0] + '_mask.shp')
out_file = os.path.join(
root,
file.split(".tif")[0] +
'_{}_band.shp'.format(thr))
convert_geotiff_to_vector_with_multibands(
bc, in_file, s_crs, tmp_1, tmp_2, out_file)
def load_db_IO(table_in,EORA=False,RoW=None):
data_path = load_config()['paths']['data']
'''CREATE GAMS WORKSPACE'''
ws = GamsWorkspace(os.path.join(data_path,'gams_runs'))
''' CREATE INPUT FILES GAMS GDX '''
db = ws.add_database()
#set regions
reg = db.add_set("reg",1,"Regions")
if EORA is True:
for r in (table_in.countries+['ROW']):
reg.add_record(r)
else:
for r in (table_in.countries):
reg.add_record(r)
#set rowcol
rowcol = db.add_set("rowcol",1,"All rows and columns")
if EORA is True:
industries = list(table_in.sectors) + ['Total']
final_demand = list(table_in.FD_labels['FD'].unique())
else:
industries = list(table_in.sectors)
final_demand = list(table_in.FD_labels['tfd'].unique())
Import_lab = ['Import']
Export_lab = ['Export']
VA_lab = ['VA']
rowcol_input = industries + final_demand + VA_lab + Import_lab + Export_lab
for r in (rowcol_input):
rowcol.add_record(r)
#set row
row = db.add_set("row",1,"All rows")
row_input = industries + VA_lab + Import_lab
for r in (row_input):
row.add_record(r)
#set col
col = db.add_set("col",1,"All columns")
col_input = industries + final_demand
for r in (col_input):
col.add_record(r)
#set industries
industries_ = db.add_set("S",1,"Industries")
for r in industries:
industries_.add_record(r)
#set FinalD
fd_ = GamsParameter(db,"FinDem_ini", 4, "FinDem")
for k, v in table_in.FinalD.items():
fd_.add_record(k).value = v
#set interaction matrix of intermediate demand
z_m = db.add_parameter("Z_matrix_ini", 4, "Interaction matrix")
for k, v in table_in.Z_matrix.items():
z_m.add_record(k).value = v
#set interaction matrix of intermediate demand
a_m = db.add_parameter("A_matrix_ini", 4, "A matrix")
for k, v in table_in.A_matrix.items():
a_m.add_record(k).value = v
if EORA is not True:
#set Export ROW
exp = db.add_parameter("ExpROW_ini", 3, "Exports to ROW")
for k, v in table_in.ExpROW.items():
exp.add_record(k).value = v
#set ValueA
val = db.add_parameter("ValueA_ini", 3, "Value Added")
for k, v in table_in.ValueA.items():
val.add_record(k).value = v
# And save to GDX file
db.export(os.path.join(data_path,"gams_runs","{}.gdx".format(table_in.name)))
def main():
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
# drop_cols = ['gid','node_id']
# viwa_file = os.path.join(data_path,'Waterways','viwa_select','iwt_ports.shp')
# viwa_in = gpd.read_file(viwa_file)
# for d in drop_cols:
# if d in viwa_in.columns.values.tolist():
# viwa_in.drop(d,axis=1,inplace=True)
# viwa_in['port_type'] = 'inland'
# seaport_file = os.path.join(data_path,'Waterways','inlandandseaports','vietnam_seaport_nodes.shp')
# seaport_in = gpd.read_file(seaport_file)
# for d in drop_cols:
# if d in seaport_in.columns.values.tolist():
# seaport_in.drop(d,axis=1,inplace=True)
# seaport_in['port_type'] = 'sea'
# all_ports = pd.concat([viwa_in,seaport_in], axis=0, sort = 'False', ignore_index=True).fillna(0)
# all_ports = gpd.GeoDataFrame(all_ports,crs='epsg:4326')
# print (all_ports)
# all_ports.to_file(os.path.join(data_path,'Waterways','waterways','ports.shp'))
# edges_file = os.path.join(data_path,'Waterways','waterways','wateredges.shp')
# edges_in = gpd.read_file(edges_file)
# drop_cols = ['eid','edge_id','node_f_id','node_t_id']
# for d in drop_cols:
# if d in edges_in.columns.values.tolist():
# edges_in.drop(d,axis=1,inplace=True)
# edges_in.to_file(os.path.join(data_path,'Waterways','waterways','port_routes.shp'))
# imp_sea_ports = {'class_1A':[135,146,147,140],'class_1':[149,150,169,175,136,137,158,163,153,151,139,171,172,170]}
# port_ids = []
# imp_ports_ids = [135,146,147,140,149,150,169,175,136,137,158,163,153,151,139,171,172,170]
# for im_sp in imp_ports_ids:
# if im_sp in [135,146,147,140]:
# port_ids.append(('class_1A','watern_{}'.format(im_sp)))
# elif im_sp in [149,150,169,175,136,137,158,163,153,151,139,171,172,170]:
# port_ids.append(('class_1','watern_{}'.format(im_sp)))
# print (port_ids)
# ports_file = os.path.join(data_path,'Waterways','waterways','ports_nodes.shp')
# ports_in = gpd.read_file(ports_file).fillna(0)
# ports_in['port_class'] = 'none'
# for p in port_ids:
# ports_in.loc[ports_in['NODE_ID'] == p[1],'port_class'] = p[0]
# ports_in.loc[ports_in['NODE_ID'] == p[1],'TONS'] = 5000000
# ports_in.loc[ports_in['TONS'] == 0, 'TONS'] = 1
# # print (ports_in)
# ports_in.to_file(ports_file)
air_file = os.path.join(data_path, 'Airports', 'airnetwork',
'airportedges.shp')
air_in = gpd.read_file(air_file)
air_in['g_id'] = [
int(x.split('_')[1]) for x in air_in['edge_id'].values.tolist()
]
air_in.rename(columns={'node_f_id': 'from_node'}, inplace=True)
air_in.rename(columns={'node_t_id': 'to_node'}, inplace=True)
air_in = air_in[[
'edge_id', 'g_id', 'from_node', 'to_node', 'speed', 'geometry'
]]
air_in.to_file(
os.path.join(data_path, 'Airports', 'airnetwork', 'airportedges.shp'))
def main():
curdir = os.getcwd()
conf = load_config()
try:
conn = psycopg2.connect(**conf['database'])
except:
print("I am unable to connect to the database")
curs = conn.cursor()
engine = create_engine(
'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
{**conf['database']}))
layers_ids = [('roads_edge_flood', 'edge_id', 'road2009edges', 'geom'),
('rail_edge_flood', 'edge_id', 'railnetworkedges', 'geom'),
('roads_node_flood', 'node_id', 'road2009nodes', 'geom'),
('rail_node_flood', 'node_id', 'railnetworknodes', 'geom'),
('port_flood', 'node_id', 'seaport_nodes', 'geom'),
('airport_flood', 'node_id', 'airport_nodes', 'geom'),
('provinces_flooding', 'name_eng', 'province_level_stats',
'geom')]
regional_table = 'province_level_stats'
mode_edge_tables = ['rail_edge_flood', 'roads_edge_flood']
edge_id = 'edge_id'
regional_name = 'name_eng'
col_names = [
'level13_vt_100_mask_1', 'level14_vt_100_mask_1',
'level15_vt_100_mask_1', 'level16_vt_100_mask_1'
]
excel_writer = pd.ExcelWriter('vnm_road_rail_flood_list.xlsx')
for c in col_names:
fl_rg_list = []
for m in mode_edge_tables:
edge_region_list = []
sql_query = '''SELECT A.{0},B.{1} FROM {2} as A, {3} as B where st_intersects(A.geom,B.geom) is True and A.{4} > 0
'''.format(edge_id, regional_name, m, regional_table, c)
cur.execute(sql_query)
read_layer = cur.fetchall()
for row in read_layer:
edge_region_list.append((row[0], row[1]))
tedge_regions = [r[1] for r in edge_region_list]
tedge_regions = sorted(list(set(tedge_regions)))
for rg in tedge_regions:
ed = [r[0] for r in edge_region_list if r[1] == rg]
fl_rg_list.append((rg, {'flood_edge': ed}))
fl_rg_list = sorted(fl_rg_list, key=lambda x: x[0])
df = pd.DataFrame(fl_rg_list, columns=['region', 'flood_list'])
df.to_excel(excel_writer, c, index=False)
excel_writer.save()
conf = load_config()
try:
conn = psycopg2.connect(**conf['database'])
except:
print("I am unable to connect to the database")
curs = conn.cursor()
engine = create_engine(
'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
{**conf['database']}))
layers_ids = [('roads_edge_flood', 'edge_id', 'road2009edges', 'geom'),
('rail_edge_flood', 'edge_id', 'railnetworkedges', 'geom'),
('roads_node_flood', 'node_id', 'road2009nodes', 'geom'),
('rail_node_flood', 'node_id', 'railnetworknodes', 'geom'),
('port_flood', 'node_id', 'seaport_nodes', 'geom'),
('airport_flood', 'node_id', 'airport_nodes', 'geom'),
('provinces_flooding', 'name_eng', 'province_level_stats',
'geom')]
regional_table = 'province_level_stats'
mode_edge_tables = ['rail_edge_flood', 'roads_edge_flood']
edge_id = 'edge_id'
regional_name = 'name_eng'
col_names = [
'level13_vt_100_mask_1', 'level14_vt_100_mask_1',
'level15_vt_100_mask_1', 'level16_vt_100_mask_1'
]
excel_writer = pd.ExcelWriter('vnm_road_rail_flood_list.xlsx')
for c in col_names:
fl_rg_list = []
for m in mode_edge_tables:
edge_region_list = []
sql_query = '''SELECT A.{0},B.{1} FROM {2} as A, {3} as B where st_intersects(A.geom,B.geom) is True and A.{4} > 0
'''.format(edge_id, regional_name, m, regional_table, c)
cur.execute(sql_query)
read_layer = cur.fetchall()
for row in read_layer:
edge_region_list.append((row[0], row[1]))
tedge_regions = [r[1] for r in edge_region_list]
tedge_regions = sorted(list(set(tedge_regions)))
for rg in tedge_regions:
ed = [r[0] for r in edge_region_list if r[1] == rg]
fl_rg_list.append((rg, {'flood_edge': ed}))
fl_rg_list = sorted(fl_rg_list, key=lambda x: x[0])
df = pd.DataFrame(fl_rg_list, columns=['region', 'flood_list'])
df.to_excel(excel_writer, c, index=False)
excel_writer.save()
def main():
"""Process results
1. Specify the paths from where you to read and write:
- Input data
- Intermediate calcuations data
- Output results
2. Supply input data and parameters
- Names of the three Provinces - List of string types
- Names of modes - List of strings
- Names of output modes - List of strings
- Names of hazard bands - List of integers
- Names of hazard thresholds - List of integers
- Condition 'Yes' or 'No' is the users wants to process results
3. Give the paths to the input data files:
- Commune boundary and stats data shapefile
- Hazard datasets description Excel file
- String name of sheet in hazard datasets description Excel file
"""
config = load_config()
output_path = config['paths']['output']
data_path = config['paths']['data']
# Supply input data and parameters
provinces = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa']
start_year = 2016
length_thr = 500.0
cols = [
'band_num', 'climate_scenario', 'edge_id', 'hazard_type', 'max_val',
'min_val', 'model', 'probability', 'year', 'length'
]
index_cols = [
'edge_id', 'hazard_type', 'model', 'climate_scenario', 'year', 'level',
'terrain', 'surface', 'road_class', 'road_cond', 'asset_type', 'width',
'road_length'
]
# Give the paths to the input data files
network_data_excel = os.path.join(data_path, 'post_processed_networks')
fail_scenarios_data = os.path.join(output_path, 'hazard_scenarios')
# Specify the output files and paths to be created
output_dir = os.path.join(output_path, 'hazard_scenarios')
# Process province scale results
for province in provinces:
# Load mode network DataFrame
province_name = province.replace(' ', '').lower()
print('* Loading {} network DataFrame'.format(province))
G_df = pd.read_excel(os.path.join(network_data_excel,
'province_roads_edges.xlsx'),
sheet_name=province_name,
encoding='utf-8')
G_df = G_df[[
'edge_id', 'level', 'terrain', 'surface', 'road_class',
'road_cond', 'asset_type', 'width', 'length'
]]
# Load failure scenarios
print('* Loading {} failure scenarios'.format(province))
hazard_scenarios = pd.read_excel(os.path.join(
fail_scenarios_data, 'province_roads_hazard_intersections.xlsx'),
sheet_name=province_name)
hazard_scenarios = hazard_scenarios.drop_duplicates(subset=cols,
keep='first')
all_edge_fail_scenarios = combine_hazards_and_network_attributes_and_impacts(
hazard_scenarios, G_df)
print('* Creating {} hazard-network scenarios'.format(province))
scenarios_df = create_hazard_scenarios_for_adaptation(
all_edge_fail_scenarios, index_cols, length_thr)
df_path = os.path.join(
output_dir,
'roads_hazard_intersections_{}_risks.csv'.format(province_name))
scenarios_df.to_csv(df_path, index=False)
del scenarios_df
gdf_edges['netrev'] = 0
for path in save_paths:
gdf_edges.loc[gdf_edges['edge_id'].isin(path[2]), 'netrev'] += path[3]
if save_edges == True:
gdf_edges.to_file(
os.path.join(
output_path,
'weighted_edges_district_center_flows_{}.shp'.format(
region_name)))
return gdf_edges
if __name__ == '__main__':
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
# provinces to consider
province_list = ['Thanh Hoa', 'Binh Dinh', 'Lao Cai']
# province_list = ['Thanh Hoa']
district_committe_names = [
'district_people_committee_points_thanh_hoa.shp',
'district_province_peoples_committee_point_binh_dinh.shp',
'district_people_committee_points_lao_cai.shp'
]
shp_output_path = os.path.join(output_path, 'flow_mapping_shapefiles')
flow_output_excel = os.path.join(
output_path, 'flow_mapping_paths',
'province_roads_district_center_flow_paths.xlsx')
def main():
'''
Create the database connection
'''
conf = load_config()
try:
conn = psycopg2.connect(**conf['database'])
except:
print("I am unable to connect to the database")
curs = conn.cursor()
engine = create_engine(
'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
{**conf['database']}))
od_data_file = os.path.join(conf['paths']['data'], 'od_data',
'OD_transport_data_2008_v2.xlsx')
'''
Step 2: Create the OD proprotions for the differnet modes
'''
'''
First get the modal shares
'''
modes = ['road', 'rail', 'air', 'water']
mode_cols = ['road', 'rail', 'air', 'inland', 'coastal']
new_mode_cols = ['o', 'd', 'road', 'rail', 'air', 'water']
mode_table = [
'airport_nodes', 'waternodes', 'railnetworknodes', 'road2009nodes'
]
mode_edge_tables = [
'airport_edges', 'wateredges', 'railnetworkedges', 'road2009edges'
]
mode_flow_tables = []
for mo in mode_edge_tables:
fl_table = mo + '_flows'
mode_flow_tables.append(fl_table)
'''
Get the modal shares
'''
od_data_modes = pd.read_excel(od_data_file, sheet_name='mode').fillna(0)
# od_data_modes.columns = map(str.lower, od_data_modes.columns)
o_id_col = 'o'
d_id_col = 'd'
od_data_modes['total'] = od_data_modes[mode_cols].sum(axis=1)
for m in mode_cols:
od_data_modes[m] = od_data_modes[m] / od_data_modes['total'].replace(
np.inf, 0)
od_data_modes['water'] = od_data_modes['inland'] + od_data_modes['coastal']
od_data_modes = od_data_modes.fillna(0)
# od_data_modes.to_csv('mode_frac.csv',index = False)
od_fracs = od_data_modes[new_mode_cols]
od_data_com = pd.read_excel(od_data_file, sheet_name='goods').fillna(0)
ind_cols = [
'sugar', 'wood', 'steel', 'constructi', 'cement', 'fertilizer', 'coal',
'petroluem', 'manufactur', 'fishery', 'meat'
]
od_fracs = pd.merge(od_fracs, od_data_com, how='left', on=['o', 'd'])
del od_data_com, od_data_modes
od_fracs = od_fracs.fillna(0)
# od_fracs.to_csv('od_fracs.csv')
for ind in ind_cols:
'''
Step 2 assign the crop to the closest transport mode node
'''
# mode_table = ['road2009nodes','railwaynetworknodes','airport_nodes','waternodes']
# mode_edge_tables = ['road2009edges','railwaynetworkedges','airport_edges','wateredges']
# modes = ['road','rail','air','water']
modes = ['air', 'water', 'rail', 'road']
mode_id = 'node_id'
od_id = 'od_id'
pop_id = 'population'
o_id_col = 'o'
d_id_col = 'd'
'''
Get the network
'''
eid = 'edge_id'
nfid = 'node_f_id'
ntid = 'node_t_id'
spid = 'speed'
gmid = 'geom'
o_id_col = 'o'
d_id_col = 'd'
'''
Get the node edge flows
'''
excel_writer = pd.ExcelWriter('vietnam_flow_stats_' + ind + '.xlsx')
for m in range(len(mode_table)):
od_nodes_regions = []
sql_query = '''select {0}, {1}, 100*{2}/(sum({3}) over (Partition by {4})) from {5}
'''.format(mode_id, od_id, pop_id, pop_id, od_id, mode_table[m])
curs.execute(sql_query)
read_layer = curs.fetchall()
if read_layer:
for row in read_layer:
n = row[0]
r = row[1]
p = float(row[2])
if p > 0:
od_nodes_regions.append((n, r, p))
all_net_dict = {
'edge': [],
'from_node': [],
'to_node': [],
'distance': [],
'speed': [],
'travel_cost': []
}
all_net_dict = tnc.create_network_dictionary(
all_net_dict, mode_edge_tables[m], eid, nfid, ntid, spid,
'geom', curs, conn)
od_net = tnc.create_igraph_topology(all_net_dict)
'''
Get the OD flows
'''
net_dict = {
'Origin_id': [],
'Destination_id': [],
'Origin_region': [],
'Destination_region': [],
'Tonnage': [],
'edge_path': [],
'node_path': []
}
ofile = 'network_od_flows_' + ind + modes[m] + '.csv'
output_file = open(ofile, 'w')
wr = csv.writer(output_file,
delimiter=',',
quoting=csv.QUOTE_MINIMAL)
wr.writerow(net_dict.keys())
ind_mode = modes[m] + '_' + ind
od_fracs[ind_mode] = od_fracs[modes[m]] * od_fracs[ind]
od_flows = list(
zip(od_fracs[o_id_col].values.tolist(),
od_fracs[d_id_col].values.tolist(),
od_fracs[ind_mode].values.tolist()))
origins = list(set(od_fracs[o_id_col].values.tolist()))
destinations = list(set(od_fracs[d_id_col].values.tolist()))
dflows = []
# print (od_flows)
for o in origins:
for d in destinations:
fval = [
fl for (org, des, fl) in od_flows
if org == o and des == d
]
if len(fval) == 1 and fval[0] > 0:
o_matches = [(item[0], item[2])
for item in od_nodes_regions
if item[1] == o]
if len(o_matches) > 0:
for o_vals in o_matches:
o_val = 1.0 * fval[0] * (1.0 * o_vals[1] / 100)
o_node = o_vals[0]
d_matches = [(item[0], item[2])
for item in od_nodes_regions
if item[1] == d]
if len(d_matches) > 0:
for d_vals in d_matches:
od_val = 1.0 * o_val * (
1.0 * d_vals[1] / 100)
d_node = d_vals[0]
if od_val > 0 and o_node != d_node:
# od_net = tnc.add_igraph_costs(od_net,t_val,0)
orgn_node = od_net.vs[
'node'].index(o_node)
dest_node = od_net.vs[
'node'].index(d_node)
# n_pth = od_net.get_shortest_paths(orgn_node,to = dest_node, weights = 'travel_cost', mode = 'OUT', output='vpath')[0]
e_pth = od_net.get_shortest_paths(
orgn_node,
to=dest_node,
weights='travel_cost',
mode='OUT',
output='epath')[0]
# n_list = [od_net.vs[n]['node'] for n in n_pth]
e_list = [
od_net.es[n]['edge']
for n in e_pth
]
# cst = sum([od_net.es[n]['cost'] for n in e_pth])
net_dict = {
'Origin_id': o_node,
'Destination_id': d_node,
'Origin_region': o,
'Destination_region': d,
'Tonnage': od_val,
'edge_path': e_list,
'node_path': [o_node, d_node]
}
wr.writerow(net_dict.values())
dflows.append(
(str([o_node, d_node]),
str(e_list), od_val))
print(o, d, fval, modes[m], ind)
node_table = modes[m] + '_node_flows'
edge_table = modes[m] + '_edge_flows'
# dom_flows = pd.read_csv(ofile).fillna(0)
dom_flows = pd.DataFrame(
dflows, columns=['node_path', 'edge_path', 'Tonnage'])
flow_node_edge = dom_flows.groupby(
['node_path', 'edge_path'])['Tonnage'].sum().reset_index()
n_dict = {}
e_dict = {}
n_dict, e_dict = get_node_edge_flows(flow_node_edge, n_dict,
e_dict)
node_list = get_id_flows(n_dict)
df = pd.DataFrame(node_list, columns=['node_id', ind])
df.to_excel(excel_writer, node_table, index=False)
excel_writer.save()
edge_list = get_id_flows(e_dict)
df = pd.DataFrame(edge_list, columns=['edge_id', ind])
df.to_excel(excel_writer, edge_table, index=False)
excel_writer.save()
if df.empty:
add_zeros_columns_to_table_psycopg2(mode_flow_tables[m], [ind],
['double precision'], conn)
else:
df.to_sql('dummy_flows',
engine,
if_exists='replace',
schema='public',
index=False)
add_columns_to_table_psycopg2(mode_flow_tables[m],
'dummy_flows', [ind],
['double precision'], 'edge_id',
conn)
curs.close()
conn.close()
def main():
'''
Create the database connection
'''
conf = load_config()
try:
conn = psycopg2.connect(**conf['database'])
except:
print("I am unable to connect to the database")
curs = conn.cursor()
engine = create_engine(
'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
{**conf['database']}))
crop_data_path = os.path.join(conf['paths']['data'], 'crop_data')
od_data_file = os.path.join(conf['paths']['data'], 'od_data',
'OD_transport_data_2008_v2.xlsx')
'''
Step 2: Create the OD proprotions for the differnet modes
'''
'''
First get the modal shares
'''
modes = ['road', 'rail', 'air', 'water']
mode_cols = ['road', 'rail', 'air', 'inland', 'coastal']
new_mode_cols = ['o', 'd', 'road', 'rail', 'air', 'water']
crop_names = [
'rice', 'cash', 'cass', 'teas', 'maiz', 'rubb', 'swpo', 'acof', 'rcof',
'pepp'
]
mode_table = [
'airport_nodes', 'waternodes', 'railnetworknodes', 'road2009nodes'
]
mode_edge_tables = [
'airport_edges', 'wateredges', 'railnetworkedges', 'road2009edges'
]
mode_flow_tables = []
for mo in mode_edge_tables:
fl_table = mo + '_flows'
mode_flow_tables.append(fl_table)
drop_postgres_table_psycopg2(fl_table, conn)
with conn.cursor() as cur:
sql_query = "create table {0} as select edge_id,geom from {1}".format(
fl_table, mo)
cur.execute(sql_query)
conn.commit()
'''
Get the modal shares
'''
od_data_modes = pd.read_excel(od_data_file, sheet_name='mode').fillna(0)
# od_data_modes.columns = map(str.lower, od_data_modes.columns)
o_id_col = 'o'
d_id_col = 'd'
od_data_modes['total'] = od_data_modes[mode_cols].sum(axis=1)
for m in mode_cols:
od_data_modes[m] = od_data_modes[m] / od_data_modes['total'].replace(
np.inf, 0)
od_data_modes['water'] = od_data_modes['inland'] + od_data_modes['coastal']
od_data_modes = od_data_modes.fillna(0)
# od_data_modes.to_csv('mode_frac.csv',index = False)
od_fracs = od_data_modes[new_mode_cols]
od_data_com = pd.read_excel(od_data_file, sheet_name='goods').fillna(0)
crop_cols = ['rice', 'indust-cro']
for cr in crop_cols:
od_data_com_sums = od_data_com.groupby(['o', 'd']).agg({cr: 'sum'})
od_com_frac = od_data_com_sums.groupby(
level=0).apply(lambda x: x / float(x.sum()))
od_com_frac = od_com_frac.reset_index(level=['o', 'd'])
od_fracs = pd.merge(od_fracs, od_com_frac, how='left', on=['o', 'd'])
del od_data_com, od_data_com_sums, od_com_frac
od_fracs = od_fracs.fillna(0)
# od_fracs.to_csv('od_fracs.csv')
for file in os.listdir(crop_data_path):
if file.endswith(".tif") and 'spam_p' in file.lower().strip():
fpath = os.path.join(crop_data_path, file)
crop_name = [
cr for cr in crop_names if cr in file.lower().strip()
][0]
raster_in = fpath
outCSVName = 'crop_concentrations.csv'
crop_table = crop_name + '_production'
'''Clip to region and convert to points'''
os.system('gdal2xyz.py -csv ' + raster_in + ' ' + outCSVName)
'''Load points and convert to geodataframe with coordinates'''
load_points = pd.read_csv(outCSVName,
header=None,
names=['x', 'y', 'crop_prod'],
index_col=None)
load_points = load_points[load_points['crop_prod'] > 0]
# load_points.to_csv('crop_concentrations.csv', index = False)
geometry = [Point(xy) for xy in zip(load_points.x, load_points.y)]
load_points = load_points.drop(['x', 'y'], axis=1)
crs = {'init': 'epsg:4326'}
points_gdp = gpd.GeoDataFrame(load_points,
crs=crs,
geometry=geometry)
points_gdp['geom'] = points_gdp['geometry'].apply(
lambda x: WKTElement(x.wkt, srid=4326))
#drop the geometry column as it is now duplicative
points_gdp.drop('geometry', 1, inplace=True)
# points_gdp = points_gdp.rename(columns={'geometry':'geom'}).set_geometry('geom')
del load_points
print('created geopandas dataframe from the points')
points_gdp.to_sql(crop_table,
engine,
if_exists='replace',
schema='public',
index=True,
dtype={'geom': Geometry('POINT', srid=4326)})
del points_gdp
'''
Add gid field to crop table
'''
with conn.cursor() as cur:
sql_query = "alter table {0} add column gid serial".format(
crop_table)
cur.execute(sql_query)
conn.commit()
print('Done with loading crop table to database')
'''
Step 1
Assign the regions closest to the crop nodes
'''
nd_table = crop_table
regional_table = 'province_level_stats'
dummy_table = 'dummy_table'
nd_id = 'gid'
nd_gm = 'geom'
regional_gm = 'geom'
nd_attr = ['gid']
regional_attr = ['name_eng', 'provinceid', 'od_id']
regional_attr_type = ['character varying', 'integer', 'integer']
drop_postgres_table_psycopg2(dummy_table, conn)
# nodes_polygons_nearest_psycopg2(dummy_table,nd_table,regional_table,nd_attr,regional_attr,nd_gm,regional_gm,cur,conn)
nodes_polygons_within_nearest_psycopg2(dummy_table, nd_table,
regional_table, nd_attr,
nd_id, regional_attr, nd_gm,
regional_gm, conn)
print('Done with assigning attributes to the crop table')
add_columns_to_table_psycopg2(nd_table, dummy_table, regional_attr,
regional_attr_type, nd_id, conn)
print('Done with adding columns to the crop table')
'''
Step 2 assign the crop to the closest transport mode node
'''
# mode_table = ['road2009nodes','railwaynetworknodes','airport_nodes','waternodes']
# mode_edge_tables = ['road2009edges','railwaynetworkedges','airport_edges','wateredges']
# modes = ['road','rail','air','water']
modes = ['air', 'water', 'rail', 'road']
mode_id = 'node_id'
crop_id = 'gid'
mode_crop_m = 'od_id'
crop_mode_m = 'od_id'
crop_prod = 'crop_prod'
od_id = 'od_id'
od_id_type = 'integer'
o_id_col = 'o'
d_id_col = 'd'
'''
Get the network
'''
eid = 'edge_id'
nfid = 'node_f_id'
ntid = 'node_t_id'
spid = 'speed'
gmid = 'geom'
o_id_col = 'o'
d_id_col = 'd'
'''
Get the node edge flows
'''
excel_writer = pd.ExcelWriter('vietnam_flow_stats_' + crop_name +
'.xlsx')
for m in range(len(mode_table)):
drop_postgres_table_psycopg2(dummy_table, conn)
nodes_polygons_aggregations(dummy_table, mode_table[m],
crop_table, mode_id, crop_id,
mode_crop_m, crop_mode_m,
crop_prod, nd_gm, regional_gm,
conn)
add_columns_to_table_psycopg2(dummy_table, mode_table[m],
[od_id], [od_id_type], mode_id,
conn)
od_nodes_regions = []
with conn.cursor() as cur:
sql_query = '''select {0}, {1}, {2}, {3}/(sum({4}) over (Partition by {5})) from {6}
'''.format(mode_id, od_id, crop_prod, crop_prod, crop_prod, od_id,
dummy_table)
cur.execute(sql_query)
read_layer = cur.fetchall()
if read_layer:
for row in read_layer:
n = row[0]
r = row[1]
c = float(row[2])
p = float(row[3])
if p > 0:
od_nodes_regions.append((n, r, c, p))
all_net_dict = {
'edge': [],
'from_node': [],
'to_node': [],
'distance': [],
'speed': [],
'travel_cost': []
}
all_net_dict = tnc.create_network_dictionary(
all_net_dict, mode_edge_tables[m], eid, nfid, ntid, spid,
'geom', curs, conn)
od_net = tnc.create_igraph_topology(all_net_dict)
'''
Get the OD flows
'''
net_dict = {
'Origin_id': [],
'Destination_id': [],
'Origin_region': [],
'Destination_region': [],
'Tonnage': [],
'edge_path': [],
'node_path': []
}
ofile = 'network_od_flows_' + crop_name + modes[m] + '.csv'
output_file = open(ofile, 'w')
wr = csv.writer(output_file,
delimiter=',',
quoting=csv.QUOTE_MINIMAL)
wr.writerow(net_dict.keys())
crop_mode = modes[m] + '_' + crop_name
if crop_name in ('rice', 'cereal', 'wheat'):
od_fracs[crop_mode] = od_fracs[modes[m]] * od_fracs['rice']
else:
od_fracs[crop_mode] = od_fracs[
modes[m]] * od_fracs['indust-cro']
od_flows = list(
zip(od_fracs[o_id_col].values.tolist(),
od_fracs[d_id_col].values.tolist(),
od_fracs[crop_mode].values.tolist()))
origins = list(set(od_fracs[o_id_col].values.tolist()))
destinations = list(set(od_fracs[d_id_col].values.tolist()))
dflows = []
# print (od_flows)
for o in origins:
for d in destinations:
fval = [
fl for (org, des, fl) in od_flows
if org == o and des == d
]
if len(fval) == 1 and fval[0] > 0:
o_matches = [(item[0], item[2])
for item in od_nodes_regions
if item[1] == o]
if len(o_matches) > 0:
for o_vals in o_matches:
o_val = 1.0 * fval[0] * o_vals[1]
o_node = o_vals[0]
d_matches = [(item[0], item[3])
for item in od_nodes_regions
if item[1] == d]
if len(d_matches) > 0:
for d_vals in d_matches:
od_val = 1.0 * o_val * d_vals[
1] / 365
d_node = d_vals[0]
if od_val > 0 and o_node != d_node:
# od_net = tnc.add_igraph_costs(od_net,t_val,0)
orgn_node = od_net.vs[
'node'].index(o_node)
dest_node = od_net.vs[
'node'].index(d_node)
# n_pth = od_net.get_shortest_paths(orgn_node,to = dest_node, weights = 'travel_cost', mode = 'OUT', output='vpath')[0]
e_pth = od_net.get_shortest_paths(
orgn_node,
to=dest_node,
weights='travel_cost',
mode='OUT',
output='epath')[0]
# n_list = [od_net.vs[n]['node'] for n in n_pth]
e_list = [
od_net.es[n]['edge']
for n in e_pth
]
# cst = sum([od_net.es[n]['cost'] for n in e_pth])
net_dict = {
'Origin_id': o_node,
'Destination_id': d_node,
'Origin_region': o,
'Destination_region': d,
'Tonnage': od_val,
'edge_path': e_list,
'node_path':
[o_node, d_node]
}
wr.writerow(net_dict.values())
dflows.append(
(str([o_node, d_node]),
str(e_list), od_val))
print(o, d, fval, modes[m], crop_name)
node_table = modes[m] + '_node_flows'
edge_table = modes[m] + '_edge_flows'
# dom_flows = pd.read_csv(ofile).fillna(0)
dom_flows = pd.DataFrame(
dflows, columns=['node_path', 'edge_path', 'Tonnage'])
flow_node_edge = dom_flows.groupby(
['node_path', 'edge_path'])['Tonnage'].sum().reset_index()
n_dict = {}
e_dict = {}
n_dict, e_dict = get_node_edge_flows(flow_node_edge, n_dict,
e_dict)
node_list = get_id_flows(n_dict)
df = pd.DataFrame(node_list, columns=['node_id', crop_name])
df.to_excel(excel_writer, node_table, index=False)
excel_writer.save()
edge_list = get_id_flows(e_dict)
df = pd.DataFrame(edge_list, columns=['edge_id', crop_name])
df.to_excel(excel_writer, edge_table, index=False)
excel_writer.save()
if df.empty:
add_zeros_columns_to_table_psycopg2(
mode_flow_tables[m], [crop_name], ['double precision'],
conn)
else:
df.to_sql('dummy_flows',
engine,
if_exists='replace',
schema='public',
index=False)
add_columns_to_table_psycopg2(mode_flow_tables[m],
'dummy_flows', [crop_name],
['double precision'],
'edge_id', conn)
curs.close()
conn.close()
r = requests.get(
'http://hatang.viwa.gov.vn/BanDo/_ChiTietCangBen',
params={
'id': port_id
}
)
with open(output_path, 'w', encoding='utf-8') as fh:
fh.write(r.text)
except requests.exceptions.ConnectionError:
print("Error", port_id)
# also request details
output_path = os.path.join(output_dir, 'cangben-{}-detail.html'.format(port_id))
if not os.path.exists(output_path):
print(port_id, "details")
try:
r = requests.get(
'http://cangben.viwa.gov.vn/trangchu/cangben/{}'.format(port_id)
)
with open(output_path, 'w', encoding='utf-8') as fh:
fh.write(r.text)
except requests.exceptions.ConnectionError:
print("Error", port_id)
if __name__ == '__main__':
conf = load_config()
PORT_LIST_PATH = os.path.join(conf['paths']['data'], 'Waterways', 'viwa', 'datacangben-all.json')
OUTPUT_DIR = os.path.join(conf['paths']['data'], 'Waterways', 'viwa', 'html')
main(PORT_LIST_PATH, OUTPUT_DIR)
def main():
curdir = os.getcwd()
conf = load_config()
try:
conn = psycopg2.connect(**conf['database'])
except:
print("I am unable to connect to the database")
cur = conn.cursor()
engine = create_engine(
'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
{**conf['database']}))
shp_file_path = os.path.join(conf['paths']['data'], 'Hazards_data',
'coastal_typhoon_floods')
'''
Get the road flooding results
'''
layers_ids = [('roads_edge_flood', 'edge_id', 'road2009edges', 'geom'),
('rail_edge_flood', 'edge_id', 'railnetworkedges', 'geom'),
('roads_node_flood', 'node_id', 'road2009nodes', 'geom'),
('rail_node_flood', 'node_id', 'railnetworknodes', 'geom'),
('port_flood', 'node_id', 'seaport_nodes', 'geom'),
('airport_flood', 'node_id', 'airport_nodes', 'geom'),
('provinces_flooding', 'name_eng', 'province_level_stats',
'geom')]
flood_list = []
excel_writer = pd.ExcelWriter('vnm_flood_results.xlsx')
for l in layers_ids:
sql_query = "drop table if exists %s" % (l[0])
cur.execute(sql_query)
conn.commit()
sql_query = "create table %s as select %s,%s from %s" % (l[0], l[1],
l[3], l[2])
cur.execute(sql_query)
conn.commit()
for file in os.listdir(shp_file_path):
if file.endswith(".shp") and 'mask_1' in file:
# print (file)
# print(os.path.join(shp_file_path, file))
fpath = os.path.join(shp_file_path, file)
flds, geom_type, cr, bds, n_f = vector_details(fpath)
fname = file.split(".shp")
fname = fname[0]
flood_list.append(fname)
cr_split = cr.split(':')
print(cr_split)
fr_srid = [c for c in cr_split if c.isdigit() is True]
if fr_srid:
print(fr_srid)
elif 'wgs84' in cr.lower().strip():
fr_srid = '4326'
command = "shp2pgsql -I -s {0}:4326 -d {1} {2} | psql -U {3} -d {4}".format(
fr_srid, fpath, fname, conf['database']['user'],
conf['database']['database'])
# sp.call(command.split())
sp.call(command, shell=True)
sql_query = "drop table if exists dummy_flood"
cur.execute(sql_query)
conn.commit()
if l[0] in ['roads_edge_flood', 'rail_edge_flood']:
sql_query = '''create table dummy_flood as select A.%s,
sum(st_length(st_intersection(A.%s::geography,B.geom::geography)))/1000 as %s
from %s as A, %s as B where st_intersects(A.%s,B.geom) is True and st_isvalid(A.%s) and st_isvalid(B.geom)
group by A.%s''' % (l[1], l[3], fname, l[2], fname, l[3], l[3], l[1])
cl_type = 'double precision'
elif l[0] in [
'roads_node_flood', 'rail_node_flood', 'port_flood',
'airport_flood'
]:
sql_query = '''create table dummy_flood as select A.%s, 1 as %s
from %s as A, %s as B where st_within(A.%s,B.geom) is True
''' % (l[1], fname, l[2], fname, l[3])
cl_type = 'integer'
else:
sql_query = '''create table dummy_flood as select A.%s,
sum(st_area(st_intersection(A.%s::geography,B.geom::geography)))/1000000 as %s
from %s as A, %s as B where st_intersects(A.%s,B.geom) is True and st_isvalid(A.%s) and st_isvalid(B.geom)
group by A.%s''' % (l[1], l[3], fname, l[2], fname, l[3], l[3], l[1])
cl_type = 'double precision'
cur.execute(sql_query)
conn.commit()
add_column_to_table(l[0], 'dummy_flood', fname, l[1], cl_type,
cur, conn)
'''
Write everthing to a pandas dataframe
'''
df = pd.read_sql_table(l[0], engine)
df.to_excel(excel_writer, l[0], index=False)
excel_writer.save()
'''
Do some diagnostics
'''
cur.close()
conn.close()
import csv
from pathlib import Path
import time
import fiona
from collections import OrderedDict, defaultdict
from rtree import index
from shapely.geometry import mapping, shape, Point, LineString, MultiLineString
from vtra.utils import load_config
# C Incoming data/20170801.zip -> data/raw/cvts/20170801
# D Work Processes/Vietnam/data/Roads/ -> data/raw/cvts/Roads
data_root = load_config()['paths']['data']
dir_raw_cvts = os.path.join(data_root, 'Cvts', 'raw', '20170801')
dir_raw_roads = os.path.join(data_root, 'Roads', 'national_roads')
dir_inter_reduse = os.path.join(data_root, 'Cvts', 'intermediate', 'reduse')
dir_inter_clip = os.path.join(data_root, 'Cvts', 'intermediate', 'clip')
dir_results_routes = os.path.join(data_root, 'Cvts', 'results', 'routes')
dir_results_routes_collected = os.path.join(data_root, 'Cvts', 'results',
'routes_collected')
dir_results_traffic_count = os.path.join(data_root, 'Cvts', 'results',
'traffic_count')
def reduse_dataset(source_dir, dest_dir):
# Reduce the size of the dataset to speed up further processing
# and output this in an intermediate mirror dataset
def estimate_losses(input_file):
print('{} started!'.format(input_file))
data_path = load_config()['paths']['data']
''' Set booleans'''
if 'min' in input_file:
min_rice = True
elif 'max' in input_file:
min_rice = False
if 'single' in input_file:
single_point = True
elif 'multiple' in input_file:
single_point = False
''' Specify file path '''
if min_rice == True:
filepath = os.path.join(data_path,'input_data','IO_VIETNAM_MIN.xlsx')
else:
filepath = os.path.join(data_path,'input_data','IO_VIETNAM_MAX.xlsx')
'''Create data input'''
DATA = io_basic('Vietnam',filepath,2010)
DATA.prep_data()
'''Run model and create some output'''
output = pd.DataFrame()
'''Specify disruption'''
output_dir = os.path.join(
data_path,
'Results',
'Economic_Failure_Results',
os.path.basename(os.path.splitext(input_file)[0])
)
'''Create output folders'''
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
event_dict = create_disruption(input_file,output_dir,min_rice=min_rice,single_point=single_point)
# print (event_dict.keys())
collect_outputs = {}
for iter_,event in enumerate(list(event_dict.keys())):
if np.average(1 - np.array(list(event_dict[event].values()))) < 0.001:
print('Event {} will cause no impacts'.format(event))
continue
print('Event {} started!'.format(event))
try:
disr_dict_fd = {}
disr_dict_sup = event_dict[event]
'''Get direct losses '''
disrupt = pd.DataFrame.from_dict(disr_dict_sup,orient='index')
disrupt.reset_index(inplace=True)
disrupt[['region', 'sector']] = disrupt['index'].apply(pd.Series)
'''Create model'''
MRIA_RUN = MRIA(DATA.name,DATA.countries,DATA.sectors,EORA=False,list_fd_cats=['FinDem'])
'''Define sets and alias'''
# CREATE SETS
MRIA_RUN.create_sets()
# CREATE ALIAS
MRIA_RUN.create_alias()
''' Define tables and parameters'''
MRIA_RUN.baseline_data(DATA,disr_dict_sup,disr_dict_fd)
MRIA_RUN.impact_data(DATA,disr_dict_sup,disr_dict_fd)
'''Get base line values'''
output['x_in'] = pd.Series(MRIA_RUN.X.get_values())*43
output.index.names = ['region','sector']
'''Get direct losses '''
disrupt = pd.DataFrame.from_dict(disr_dict_sup,orient='index')
disrupt.reset_index(inplace=True)
disrupt[['region', 'sector']] = disrupt['index'].apply(pd.Series)
disrupt.drop('index',axis=1,inplace=True)
disrupt = 1- disrupt.groupby(['region', 'sector']).sum()
disrupt.columns = ['shock']
output['dir_losses'] = (disrupt['shock']*output['x_in']).fillna(0)*-1
MRIA_RUN.run_impactmodel()
output['x_out'] = pd.Series(MRIA_RUN.X.get_values())*43
output['total_losses'] = (output['x_out'] - output['x_in'])
output['ind_losses'] = (output['total_losses'] - output['dir_losses'])
output = output/365
output = output.drop(['x_in','x_out'],axis=1)
output.to_csv(os.path.join(output_dir,'{}.csv'.format(event)))
prov_impact = output.groupby(level=0,axis=0).sum()
collect_outputs[event] = prov_impact
except Exception as e:
print('Failed to finish {} because of {}!'.format(event,e))
if collect_outputs:
'''Specify disruption'''
output_dir = os.path.join(data_path,
'Results',
'Economic_failure_results',
'provincial'
)
'''Create output folders'''
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
pd.concat(collect_outputs).to_csv(os.path.join(
data_path,
'Results',
'Economic_failure_results',
'provincial',
'{}_provincial.csv'.format(os.path.basename(os.path.splitext(input_file)[0]))))
get_sums = {}
for event in collect_outputs:
get_sums[event] = collect_outputs[event]['total_losses'].sum()
sums = pd.DataFrame.from_dict(get_sums,orient='index')
sums.columns = ['total_losses']
'''Specify disruption'''
output_dir = os.path.join(data_path,
'Results',
'Economic_failure_results',
'summarized'
)
'''Create output folders'''
if os.path.exists(output_dir) == False:
os.mkdir(output_dir)
sums.to_csv(os.path.join(data_path,
'Results',
'Economic_failure_results',
'summarized',
'{}_summarized.csv'.format(os.path.basename(os.path.splitext(input_file)[0]))))
return pd.concat(collect_outputs),sums
def main():
data_path, calc_path, output_path = load_config(
)['paths']['data'], load_config()['paths']['calc'], load_config(
)['paths']['output']
truck_unit_wt = [5.0, 20.0]
# provinces to consider
province_list = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa']
province_terrian = ['mountain', 'flat', 'flat']
types = ['min', 'max']
path_types = ['min_edge_path', 'max_edge_path']
rev_types = ['min_netrev', 'max_netrev']
tons_types = ['min_croptons', 'max_croptons']
dist_types = ['min_distance', 'max_distance']
time_types = ['min_time', 'max_time']
cost_types = ['min_gcost', 'max_gcost']
vechicle_types = ['min_vehicle_nums', 'max_vehicle_nums']
flow_paths_data = os.path.join(
output_path, 'flow_mapping_paths',
'province_roads_district_center_flow_paths.xlsx')
fail_scenarios_data = os.path.join(
output_path, 'hazard_scenarios',
'province_roads_hazard_intersections.xlsx')
rd_prop_file = os.path.join(data_path, 'Roads', 'road_properties',
'road_properties.xlsx')
cols = [
'origin', 'destination', 'min_edge_path', 'max_edge_path',
'min_netrev', 'max_netrev', 'min_croptons', 'max_croptons',
'min_distance', 'max_distance', 'min_time', 'max_time', 'min_gcost',
'max_gcost'
]
selection_criteria = [
'commune_id', 'hazard_type', 'model', 'climate_scenario', 'year',
'probability'
]
filter_cols = ['edge_id', 'exposed_length'] + selection_criteria
'''
Path OD flow disruptions
'''
for prn in range(len(province_list)):
# for prn in range(0,1):
province = province_list[prn]
# set all paths for all input files we are going to use
province_name = province.replace(' ', '').lower()
'''
First do multiple edge failures
'''
edges_in = os.path.join(data_path, 'Roads',
'{}_roads'.format(province_name),
'vietbando_{}_edges.shp'.format(province_name))
# G = province_shapefile_to_network(edges_in,province_terrian[prn],rd_prop_file)
G_df = province_shapefile_to_dataframe(edges_in, province_terrian[prn],
rd_prop_file)
fail_df = pd.read_excel(fail_scenarios_data, sheet_name=province_name)
fail_df = fail_df.set_index(selection_criteria)
criteria_set = list(set(fail_df.index.values.tolist()))
multiple_ef_list = []
for criteria in criteria_set:
if len(fail_df.loc[criteria, 'edge_id'].index) == 1:
efail = [fail_df.loc[criteria, 'edge_id'].item()]
else:
efail = list(
set(fail_df.loc[criteria, 'edge_id'].values.tolist()))
flength = fail_df.loc[criteria, 'length'].sum()
criteria_dict = {
**(dict(list(zip(selection_criteria, criteria)))),
**{
'exposed_length': flength
}
}
multiple_ef_list.append((efail, criteria_dict))
# print (multiple_ef_list)
# print (len(multiple_ef_list))
for tr_wt in truck_unit_wt:
flow_df = pd.read_excel(flow_paths_data,
sheet_name=province_name +
'_{}_tons'.format(int(tr_wt)))
edge_fail_ranges = []
for t in range(len(types)):
# for t in range(1,2):
ef_list = []
e_list = []
ef_sc_list = []
for edge_fail in multiple_ef_list:
if edge_fail[0] not in e_list:
ef_dict = igraph_scenario_edge_failures(
G_df, edge_fail[0], flow_df, tr_wt, path_types[t],
rev_types[t], tons_types[t], cost_types[t],
dist_types[t], time_types[t], vechicle_types[t])
if ef_dict:
ef_list += ef_dict
e_list.append(edge_fail[0])
ef_sc_list += [{
**ef,
**edge_fail[1]
} for ef in ef_dict]
elif ef_list:
ef_sc_list += [{
**ef,
**edge_fail[1]
} for ef in ef_list
if ef['edge_id'] == str(edge_fail[0])]
print('Done with province {0} edge {1} type {2}'.format(
province_name, edge_fail[1], types[t]))
df = pd.DataFrame(ef_sc_list)
df_path = os.path.join(
output_path, 'failure_results',
'multiple_edge_failures_all_path_impacts_{0}_{1}_{2}_tons.csv'
.format(province_name, types[t], int(tr_wt)))
df.to_csv(df_path, index=False)
df['econ_loss'] = df['no_access'] * df['econ_value'] + (
1 - df['no_access']) * df['vehicle_nums'] * (
df['new_cost'] - df['old_cost'])
egde_impact = df[filter_cols +
['econ_value', 'tons', 'econ_loss']]
egde_impact = egde_impact.groupby(filter_cols)[
'econ_value', 'tons', 'econ_loss'].sum().reset_index()
egde_impact.rename(columns={
'econ_value':
'{}_econ_value'.format(types[t]),
'tons':
'{}_tons'.format(types[t]),
'econ_loss':
'{}_econ_loss'.format(types[t])
},
inplace=True)
edge_fail_ranges.append(egde_impact)
egde_impact = edge_fail_ranges[0]
egde_impact = pd.merge(egde_impact,
edge_fail_ranges[1],
how='left',
on=filter_cols)
df_path = os.path.join(
output_path, 'failure_results',
'multiple_edge_failures_totals_{0}_{1}_tons.csv'.format(
province_name, int(tr_wt)))
egde_impact.to_csv(df_path, index=False)
