def main(): data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] root_dir = os.path.join(data_path, 'Waterways', 'viwa', 'html') r = re.compile('LuongHangHoaThongQua">([^<]*)') viwa_ports_tons = [] for root, dirs, files in os.walk(root_dir): for file in files: if file.endswith(".html") and 'detail' not in file: input_file = os.path.join(root, file) port_no = re.findall('\d+', file) with open(input_file, 'r') as fh: for line in fh: m = r.search(line) if m: if m.group(1).isdigit(): viwa_ports_tons.append( (int(port_no[0]), float(m.group(1)))) else: viwa_ports_tons.append((int(port_no[0]), 0)) viwa_ports_tons_df = pd.DataFrame(viwa_ports_tons, columns=['cangbenid', 'tons']) viwa_ports_tons_df = viwa_ports_tons_df[viwa_ports_tons_df['tons'] > 0] print(len(viwa_ports_tons_df.index)) print(viwa_ports_tons_df['cangbenid'].values.tolist()) shp_file = os.path.join(data_path, 'Waterways', 'viwa', 'cang-ports.shp') ports = gpd.read_file(shp_file) ports.columns = map(str.lower, ports.columns) cols = ports.columns.values.tolist() print(len(ports.index)) # print (ports['cangbenid'].values.tolist()) ports_with_geom = [] id_list = [] for iter_, values in ports.iterrows(): if values['geometry'] and values['cangbenid'] not in id_list: if int(values['cangbenid'] ) in viwa_ports_tons_df['cangbenid'].values.tolist(): print('match') tons = viwa_ports_tons_df.loc[ viwa_ports_tons_df['cangbenid'] == int(values['cangbenid']), 'tons'].sum() ports_with_geom.append(tuple(list(values) + [tons])) else: ports_with_geom.append(tuple(list(values) + [0])) id_list.append(values['cangbenid']) ports_df = pd.DataFrame(ports_with_geom, columns=cols + ['tons']) ports = gpd.GeoDataFrame(ports_df, crs='epsg:4326') # print (len(ports.index)) print(ports) ports.to_file( os.path.join(data_path, 'Waterways', 'viwa_select', 'iwt_ports.shp'))
def create_Ratmarg(self, Table): model = self.m try: data_path = load_config()['paths']['data'] RatMarg = pd.read_csv(os.path.join( data_path, 'input_data', 'Ratmarg_{}.csv'.format(self.name)), index_col=[0], header=0) if self.EORA is True and (set(list(RatMarg.index.values)) != set( list(self.countries + ['ROW']))): RatMarg = ratmarg_IO(Table, self.EORA) elif (set(list(RatMarg.index.values)) != set(list( self.countries))): RatMarg = ratmarg_IO(Table, self.EORA) except: RatMarg = ratmarg_IO(Table, self.EORA) Ratmarginal = {(r, k): v for r, kv in RatMarg.iterrows() for k, v in kv.to_dict().items()} model.Ratmarg = Param(model.R, model.S, initialize=Ratmarginal, doc='Rationing marginal', mutable=True) self.Ratmarg = model.Ratmarg
def create_Ratmarg(self, Table): """Estimate the marginal values of the rationing variable Parameters ---------- Table the **io_basic** class object Set *self*.Ratmarg - Pyomo Parameter instance for the marginal value of rationing """ model = self.m try: data_path = load_config()['paths']['data'] RatMarg = pd.read_csv(os.path.join( data_path, 'input_data', 'Ratmarg_{}.csv'.format(self.name)), index_col=[0], header=0) if (set(list(RatMarg.index.values)) != set(list(self.regions))): RatMarg = ratmarg_IO(Table) except: RatMarg = ratmarg_IO(Table) Ratmarginal = {(r, k): v for r, kv in RatMarg.iterrows() for k, v in kv.to_dict().items()} model.Ratmarg = Param(model.R, model.S, initialize=Ratmarginal, doc='Rationing marginal', mutable=True) self.Ratmarg = model.Ratmarg
def ratmarg_SUT(table_in,EORA=False): data_path = load_config()['paths']['data'] table_in.prep_data() load_db_SUT(table_in) ''' RUN SCRIPT WITH DISRUPTION ''' setdir = os.path.join(data_path,'gams_runs') ws = GamsWorkspace(setdir) ws.get_working_directory() gamsfile_in = os.path.join(setdir,"obtain_marg_value_SUT.gms") gamsfile = os.path.join(setdir,"obtain_marg_value_SUT_{}.gms".format(table_in.name)) copyfile(gamsfile_in, gamsfile) str_ctry = ','.join(table_in.countries) str_fd = 'FinalD' #','.join(list(table_in.FD_labels['FD'].unique())) with open(gamsfile, 'r') as file: # read a list of lines into data data = file.readlines() gdx_file = "%s.gdx" % table_in.name data[30] = '$GDXIN '+gdx_file+'\n' str_ind = ','.join(table_in.sectors) data[40] ='ind(col) list of industries /'+str_ind+'/\n' data[37] = '/'+str_ctry+'/\n' data[39] = '/'+str_ctry+'/\n' data[43] = '/'+str_fd+'/\n' with open(gamsfile, 'w') as file: file.writelines( data ) gamsfile_run = gamsfile.replace("..\\..\\gams_runs\\", "") t1 = ws.add_job_from_file(gamsfile_run) t1.run() Ratmarg = [] index_ = [] for rec in t1.out_db["Ratmarg"]: index_.append((rec.keys[0],rec.keys[1])) Ratmarg.append(rec.get_value()) index_ = pd.MultiIndex.from_tuples(index_, names=('CNTRY', 'IND')) Ratmarginal = pd.DataFrame(Ratmarg, index=index_).unstack() Ratmarginal.columns = Ratmarginal.columns.droplevel() Ratmarginal.to_csv(os.path.join(data_path,'input_data','Ratmarg_{}.csv'.format(table_in.name))) return Ratmarginal
def main(): """Pre-process CVTS """ incoming_root = load_config()['paths']['incoming_data'] data_root = load_config()['paths']['data'] output_root = load_config()['paths']['output'] dir_raw_roads = os.path.join(data_root, 'post_processed_networks') dir_raw_cvts = os.path.join(incoming_root,'20170801') dir_inter_reduse = os.path.join(output_root, 'transport cvts analysis', 'intermediate', 'reduse') dir_results_routes = os.path.join(output_root, 'transport cvts analysis', 'results', 'routes') dir_results_routes_collected = os.path.join( output_root, 'transport cvts analysis', 'results', 'routes_collected') dir_results_traffic_count = os.path.join(output_root, 'transport cvts analysis', 'results', 'traffic_count') Reduse dataset with 70 percent (processing 4s/100mb) print('Reduce dataset size') reduse_dataset(dir_raw_cvts, dir_inter_reduse) # Read road network in memory print('Read road network') geojson_road_network = read_shapefile(dir_raw_roads, 'road_edges.shp') # Generate routes by mapping gps points on the road network print('Generate routes') find_routes(geojson_road_network, dir_inter_reduse, dir_results_routes) # add traffic attribute to route network (count id's in routes) print('Add traffic count attribute to road network') geojson_road_network = add_traffic_count_to_road_network( geojson_road_network, dir_results_routes, dir_results_traffic_count) write_shapefile(geojson_road_network, dir_results_traffic_count, 'road_network.shp') # add routes to single file create_single_route_file(dir_results_routes, dir_results_routes_collected)
def create_disruption(input_file, output_dir, min_rice=True, single_point=True): # Define current directory and data directory data_path = load_config()['paths']['data'] # read national IO comm_des = df_com_to_ind(pd.read_excel(os.path.join( data_path, 'OD_data', 'national_scale_od_matrix.xlsx'), sheet_name='total'), min_rice=min_rice) # read vietnam failure results vnm_failure_all_failure = pd.read_csv(input_file, index_col=[0]) if single_point == False: mapper = dict( zip(vnm_failure_all_failure.index.unique(), [ 'multi_{}'.format(n) for n in np.arange( 1, len(vnm_failure_all_failure.index.unique()) + 1, 1) ])) vnm_failure_all_failure.index = vnm_failure_all_failure.index.map( lambda x: mapper[x]) pd.DataFrame.from_dict(mapper, orient='index').to_csv( os.path.join(output_dir, 'mapper.csv')) event_dict = {} for id_, scenario in vnm_failure_all_failure.groupby(level=0): if id_ == 'multi_102': break disruption = df_com_to_ind(scenario, min_rice=min_rice).div(comm_des).dropna( axis=0, how='all').fillna(0) disruption = pd.DataFrame(disruption.stack(0)) disruption.columns = ['value'] disruption = disruption.loc[disruption.value > 0] disruption['value'] = 1 - (disruption['value'] * 0.2) event_dict[id_] = disruption['value'].to_dict() return event_dict
def main(): data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] truck_unit_wt = [5.0, 20.0] # provinces to consider province_list = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa'] province_terrian = ['mountain', 'flat', 'flat'] growth_scenarios = [(5, 'low'), (6.5, 'forecast'), (10, 'high')] base_year = 2016 types = ['min', 'max'] path_types = ['min_edge_path', 'max_edge_path'] rev_types = ['min_netrev', 'max_netrev'] tons_types = ['min_croptons', 'max_croptons'] dist_types = ['min_distance', 'max_distance'] time_types = ['min_time', 'max_time'] cost_types = ['min_gcost', 'max_gcost'] vechicle_types = ['min_vehicle_nums', 'max_vehicle_nums'] flow_paths_data = os.path.join( output_path, 'flow_mapping_paths', 'province_roads_district_center_flow_paths.xlsx') fail_scenarios_data = os.path.join( output_path, 'hazard_scenarios', 'province_roads_hazard_intersections.xlsx') rd_prop_file = os.path.join(data_path, 'Roads', 'road_properties', 'road_properties.xlsx') cols = [ 'origin', 'destination', 'min_edge_path', 'max_edge_path', 'min_netrev', 'max_netrev', 'min_croptons', 'max_croptons', 'min_distance', 'max_distance', 'min_time', 'max_time', 'min_gcost', 'max_gcost' ] ''' Path OD flow disruptions ''' for prn in range(len(province_list)): # for prn in range(0,1): province = province_list[prn] # set all paths for all input files we are going to use province_name = province.replace(' ', '').lower() for tr_wt in truck_unit_wt: flow_output_excel = os.path.join( output_path, 'failure_results', 'multiple_edge_failures_totals_{0}_{1}_tons_projections.xlsx'. format(province_name, int(tr_wt))) excl_wrtr_1 = pd.ExcelWriter(flow_output_excel) for grth in growth_scenarios: edge_fail_ranges = [] for t in range(len(types)): df_path = os.path.join( output_path, 'failure_results', 'multiple_edge_failures_all_path_impacts_{0}_{1}_{2}_tons.csv' .format(province_name, types[t], int(tr_wt))) df = pd.read_csv(df_path).fillna(0) df = df[[ 'edge_id', 'econ_value', 'tons', 'old_cost', 'new_cost', 'no_access' ]] df = df.drop_duplicates([ 'edge_id', 'econ_value', 'tons', 'old_cost', 'new_cost', 'no_access' ]) # df = df.groupby(['edge_id', 'no_access'])['econ_value','tons','old_cost','new_cost']. edge_impact = df[['edge_id']] # print (edge_impact) cols = [] for year in range(2016, 2050): edge_impact['{0}_econ_value_{1}'.format( types[t], year)] = math.pow( (1 + grth[0] / 100), year - base_year) * df['econ_value'] edge_impact['{0}_tons_{1}'.format( types[t], year)] = math.pow( (1 + grth[0] / 100), year - base_year) * df['tons'] edge_impact['{0}_econ_loss_{1}'.format( types[t], year )] = df['no_access'] * edge_impact[ '{0}_econ_value_{1}'.format(types[t], year)] + ( 1 - df['no_access']) * np.maximum( 1, np.ceil(edge_impact['{0}_tons_{1}'.format( types[t], year)] / tr_wt)) * ( df['new_cost'] - df['old_cost']) cols += [ '{0}_econ_value_{1}'.format(types[t], year), '{0}_tons_{1}'.format(types[t], year), '{0}_econ_loss_{1}'.format(types[t], year) ] edge_impact = edge_impact.groupby( ['edge_id'])[cols].sum().reset_index() edge_fail_ranges.append(edge_impact) print( 'Done with province {0} {1} tons {2} with growth rate {3}' .format(province_name, tr_wt, types[t], grth)) edge_impact = edge_fail_ranges[0] edge_impact = pd.merge(edge_impact, edge_fail_ranges[1], how='left', on=['edge_id']).fillna(0) for year in range(2016, 2050): edge_impact['swap'] = edge_impact.apply( lambda x: swap_min_max( x, 'min_econ_value_{0}'.format(year), 'max_econ_value_{0}'.format(year)), axis=1) edge_impact[[ 'min_econ_value_{0}'.format(year), 'max_econ_value_{0}'.format(year) ]] = edge_impact['swap'].apply(pd.Series) edge_impact.drop('swap', axis=1, inplace=True) edge_impact['swap'] = edge_impact.apply( lambda x: swap_min_max(x, 'min_tons_{0}'.format(year), 'max_tons_{0}'.format(year)), axis=1) edge_impact[[ 'min_tons_{0}'.format(year), 'max_tons_{0}'.format(year) ]] = edge_impact['swap'].apply(pd.Series) edge_impact.drop('swap', axis=1, inplace=True) edge_impact['swap'] = edge_impact.apply( lambda x: swap_min_max( x, 'min_econ_loss_{0}'.format(year), 'max_econ_loss_{0}'.format(year)), axis=1) edge_impact[[ 'min_econ_loss_{0}'.format(year), 'max_econ_loss_{0}'.format(year) ]] = edge_impact['swap'].apply(pd.Series) edge_impact.drop('swap', axis=1, inplace=True) edge_impact.to_excel(excl_wrtr_1, grth[1], index=False) excl_wrtr_1.save()
def run_mrio_disaggregate(notrade=False, min_rice=True, own_production_ratio=0.8): """This function will disaggregate the (single-region) national Input-Output table to a provincial multiregional Input-Output table Parameters - notrade - Boolean to specify whether we should include trade in the disaggregation. This should be set to **True** in the first step of the disaggregation. The default is set to **False** - min_rice - Boolean to determine whether you want to use the minimal rice value or the maximum rice value from the flow analysis. The default is set to **True** - own_production_ratio - Specify how much supply and demand is locally supplied and used, and how much is imported/exported. The default is set to **0.8** Outputs - .csv file containing the new multiregional Input-Output table. - pandas DataFrame with a multiregional Input-Output table """ data_path = load_config()['paths']['data'] # load provincial shapefile provinces = load_provincial_stats(data_path) provinces.name_eng = provinces.name_eng.apply( lambda x: x.replace(' ', '_').replace('-', '_')) # estimate gross value added provinces['raw_gva'] = estimate_gva(provinces, in_million=True) # prepare proxies for settings_trade create_proxies(data_path, notrade=notrade, own_production_ratio=own_production_ratio, min_rice=min_rice) # run mrio_disaggregate if notrade == False: p = subprocess.Popen(['mrio_disaggregate', 'settings_trade.yml'], cwd=os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE')) p.wait() else: p = subprocess.Popen(['mrio_disaggregate', 'settings_notrade.yml'], cwd=os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE')) p.wait() # get reordered mrio with new region classification Xin = load_output(data_path, provinces, notrade=notrade) # convert to numpy matrix X0 = Xin.as_matrix() # get sum of rows and columns u = X0.sum(axis=1) v = X0.sum(axis=0) # and only keep T v[:(len(u) - 3)] = u[:-3] # apply RAS method to rebalance the table X1 = ras_method(X0, u, v, eps=5e-5) # copy new balanced table into dataframe Xin.iloc[:, :] = X1 # add indices to it index = list(Xin.index) index[567], index[568], index[569] = ('total', 'tax_sub'), ('total', 'import_'), ('total', 'valueA') Xin.index = pd.MultiIndex.from_tuples(index, names=['region', 'sector']) # save outpout if notrade == True: Xin.to_csv( os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE', 'notrade_trade.csv')) else: Xin.to_csv( os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE', 'IO_VIETNAM.csv')) if notrade == False: Region_sum = Xin.groupby(Xin.columns.get_level_values(0), axis='columns').sum().groupby( Xin.index.get_level_values(0), axis='index').sum() Region_sum.to_csv( os.path.join(data_path, 'IO_analysis', 'MRIO_TABLE', 'region_trade.csv')) return Xin
def main(): """Intersect networks with hazards 1. Specify the paths from where you to read and write: - Input data - Intermediate calcuations data - Output results 2. Supply input data and parameters - Names of the three Provinces - List of string types - Paths of the mode files - List of tuples of strings - Names of modes - List of strings - Names of output modes - List of strings - Condition 'Yes' or 'No' is the users wants to process results 3. Give the paths to the input data files: - Hazard directory """ data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] # Supply input data and parameters provinces = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa'] modes_file_paths = [('Roads', 'national_roads'), ('Railways', 'national_rail'), ('Airports', 'airnetwork'), ('Waterways', 'waterways'), ('Waterways', 'waterways')] modes = ['road', 'rail', 'air', 'inland', 'coastal'] out_modes = [ 'national_roads', 'national_rail', 'air_ports', 'inland_ports', 'sea_ports' ] province_results = 'Yes' national_results = 'Yes' # Give the paths to the input data files hazard_dir = os.path.join(data_path, 'Hazard_data') # Specify the output files and paths to be created output_dir = os.path.join(output_path, 'networks_hazards_intersection_shapefiles') if not os.path.exists(output_dir): os.mkdir(output_dir) # Start province roads and all hazards intersections if province_results == 'Yes': for province in provinces: province_name = province.replace(' ', '').lower() road_shp_path = os.path.join( data_path, 'Roads', '{}_roads'.format(province_name), 'vietbando_{}_edges.shp'.format(province_name)) road_shp = 'vietbando_{}_edges.shp'.format(province_name) output_dir = os.path.join( output_path, 'networks_hazards_intersection_shapefiles', '{}_roads_hazards_intersections'.format(province_name)) if os.path.exists(output_dir) == False: os.mkdir(output_dir) print('* Starting {} roads and all hazards intersections'.format( province)) intersect_networks_and_all_hazards(hazard_dir, road_shp_path, road_shp, output_dir, network_type='edges') if national_results == 'Yes': for m in range(len(modes)): mode_data_path = os.path.join(data_path, modes_file_paths[m][0], modes_file_paths[m][1]) if modes[m] in ['road', 'rail']: for mode_file in os.listdir(mode_data_path): try: if mode_file.endswith( ".shp") and 'edges' in mode_file.lower().strip( ): edges_in = os.path.join(mode_data_path, mode_file) edges_name = mode_file except: return ('Network edge file necessary') output_dir = os.path.join( output_path, 'networks_hazards_intersection_shapefiles', '{}_hazard_intersections'.format(out_modes[m])) if os.path.exists(output_dir) == False: os.mkdir(output_dir) print('* Starting national {} and all hazards intersections'. format(modes[m])) intersect_networks_and_all_hazards(hazard_dir, edges_in, edges_name, output_dir, network_type='edges') elif modes[m] in ['air', 'inland', 'coastal']: for mode_file in os.listdir(mode_data_path): try: if mode_file.endswith( ".shp") and 'nodes' in mode_file.lower().strip( ): nodes_in = os.path.join(mode_data_path, mode_file) nodes_name = mode_file except: return ('Network node file necessary') output_dir = os.path.join( output_path, 'networks_hazards_intersection_shapefiles', '{}_hazard_intersections'.format(out_modes[m])) if os.path.exists(output_dir) == False: os.mkdir(output_dir) print('* Starting national {} and all hazards intersections'. format(modes[m])) intersect_networks_and_all_hazards(hazard_dir, nodes_in, nodes_name, output_dir, network_type='nodes')
def main(): data_path,calc_path,output_path = load_config()['paths']['data'],load_config()['paths']['calc'],load_config()['paths']['output'] # provinces to consider province_list = ['Lao Cai','Binh Dinh','Thanh Hoa'] province_terrian = ['mountain','flat','flat'] # shp_output_path = os.path.join(output_path,'flow_mapping_shapefiles') # flow_output_excel = os.path.join(output_path,'flow_mapping_paths','province_roads_district_center_flow_paths.xlsx') # excl_wrtr = pd.ExcelWriter(flow_output_excel) fail_scenarios_data = os.path.join(output_path,'hazard_scenarios','province_roads_hazard_intersections.xlsx') rd_prop_file = os.path.join(data_path,'Roads','road_properties','road_properties.xlsx') ''' Path OD flow disruptions ''' # for prn in range(len(province_list)): for prn in range(0,1): province = province_list[prn] # set all paths for all input files we are going to use province_name = province.replace(' ','').lower() all_edge_fail_scenarios = pd.read_excel(fail_scenarios_data,sheet_name = province_name) all_edges = list(set(all_edge_fail_scenarios['edge_id'].values.tolist())) all_edge_fail_scenarios['road_cond'] = 'unknown' all_edge_fail_scenarios['asset_type'] = 'unknown' all_edge_fail_scenarios['width'] = 0 # print ('done with file reading') ''' First do single edge failures ''' edges_in = os.path.join(data_path,'Roads','{}_roads'.format(province_name),'vietbando_{}_edges.shp'.format(province_name)) edges = province_shapefile_to_dataframe(edges_in,province_terrian[prn],rd_prop_file) edge_attr = list(zip(edges['edge_id'].values.tolist(),edges['road_cond'].values.tolist(),edges['asset_type'].values.tolist(),edges['width'].values.tolist())) # print (edge_attr) edge_attr = [e for e in edge_attr if e[0] in all_edges] for e in edge_attr: all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'road_cond'] = e[1] all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'asset_type'] = e[2] all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'width'] = e[3] # all_edge_fail_scenarios['attributes'] = all_edge_fail_scenarios.apply(lambda x: assign_failed_edge_attributes(x,edge_attr),axis = 1) # all_edge_fail_scenarios['asset_type','width'] = all_edge_fail_scenarios['attributes'].apply(pd.Series) # all_edge_fail_scenarios.drop('attributes',axis=1,inplace=True) single_edge_path = os.path.join(output_path,'failure_results','single_edge_failures_totals_{0}.csv'.format(province_name)) edge_impacts = pd.read_csv(single_edge_path) edge_impacts_attr = list(zip(edge_impacts['edge_id'].values.tolist(),edge_impacts['min_econ_loss'].values.tolist(),edge_impacts['max_econ_loss'].values.tolist())) all_edge_fail_scenarios['min_econ_loss'] = 0 all_edge_fail_scenarios['max_econ_loss'] = 0 edge_impacts_attr = [e for e in edge_impacts_attr if e[0] in all_edges] for e in edge_impacts_attr: all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'min_econ_loss'] = e[1] all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'max_econ_loss'] = e[2] # all_edge_fail_scenarios['attributes'] = all_edge_fail_scenarios.apply(lambda x: assign_failed_edge_attributes(x,edge_impacts_attr),axis = 1) # all_edge_fail_scenarios['min_econ_loss','max_econ_loss'] = all_edge_fail_scenarios['attributes'].apply(pd.Series) # all_edge_fail_scenarios.drop('attributes',axis=1,inplace=True) df_path = os.path.join(output_path,'hazard_scenarios','roads_hazard_intersections_{}.csv'.format(province_name)) all_edge_fail_scenarios.to_csv(df_path,index = False)
def mrio_to_excel(Xin, min_rice=True): """Save the newly created multiregional Input-Output table to Excel, in the format required for the MRIA calculation. Parameters - Xin - pandas DataFrame of the new multiregional Input-Output table - min_rice - Boolean to determine whether you want to use the minimal rice value or the maximum rice value from the flow analysis. The default is set to **True** Outputs - .xlsx file with the multiregional Input-Output table """ data_path = load_config()['paths']['data'] Xnew = Xin.copy() # prepare exoort and finalD data Exports = pd.DataFrame( Xnew.iloc[:, Xnew.columns.get_level_values(1) == 'col3'].sum(axis=1), columns=['Exports']) Exports.columns = pd.MultiIndex.from_tuples( list(zip(['Total'], ['Export']))) FinalD_ToT = Xnew.iloc[:, ((Xnew.columns.get_level_values(1) == 'col1') | (Xnew.columns.get_level_values(1) == 'col2'))] FinalD_ToT = FinalD_ToT.groupby(level=0, axis=1).sum() FinalD_ToT.columns = pd.MultiIndex.from_tuples( list(zip(FinalD_ToT.columns, len(FinalD_ToT.columns) * ['FinDem']))) Xnew.drop(['col1', 'col2', 'col3'], axis=1, level=1, inplace=True) Xnew = pd.concat([Xnew, FinalD_ToT, Exports], axis=1) # write to excel if min_rice == True: writer = pd.ExcelWriter( os.path.join(data_path, 'input_data', 'IO_VIETNAM_MIN.xlsx')) else: writer = pd.ExcelWriter( os.path.join(data_path, 'input_data', 'IO_VIETNAM_MAX.xlsx')) # write T df_T = Xnew.iloc[:567, :567] df_T.columns = df_T.columns.droplevel() df_labels_T = pd.DataFrame(df_T.reset_index()[['region', 'sector']]) df_T.reset_index(inplace=True, drop=True) df_T.to_excel(writer, 'T', index=False, header=False) df_labels_T.to_excel(writer, 'labels_T', index=False, header=False) # write FD df_FD = Xnew.iloc[:567, 567:630] df_labels_FD = pd.DataFrame(list(df_FD.columns)) df_FD.columns = df_FD.columns.droplevel() df_FD.reset_index(inplace=True, drop=True) df_FD.to_excel(writer, 'FD', index=False, header=False) df_labels_FD.to_excel(writer, 'labels_FD', index=False, header=False) # write ExpROW df_ExpROW = Exports[:567] df_labels_ExpROW = pd.DataFrame(list( df_ExpROW.columns.get_level_values(1))) df_ExpROW.reset_index(inplace=True, drop=True) df_ExpROW.columns = df_ExpROW.columns.droplevel() df_ExpROW.to_excel(writer, 'ExpROW', index=False, header=False) df_labels_ExpROW.reset_index(inplace=True, drop=True) df_labels_ExpROW.columns = ['Export'] df_labels_ExpROW.to_excel(writer, 'labels_ExpROW', index=False, header=False) # write VA df_VA = pd.DataFrame(Xnew.iloc[567:, :].T[('total', 'tax_sub')] + Xnew.iloc[567:, :].T[('total', 'valueA')], columns=['VA']) df_VA['imports'] = Xnew.iloc[567:, :].T[('total', 'import_')] df_VA.reset_index(inplace=True, drop=True) df_VA.to_excel(writer, 'VA', index=False, header=False) df_labels_VA = pd.DataFrame(['Import', 'VA']).T df_labels_VA.to_excel(writer, 'labels_VA', index=False, header=False) # save excel writer.save()
def create_disruption(input_file, output_dir, min_rice=True, single_point=True): """Create disruption file for the economic analysis. The input for this disruption file is the outcome of the flow analysis. This function translate the failure in transport flows into impacts to the economic sectors. Parameters ---------- input_file : str name of the path to a flow failure scenario file output_dir : str name for output directory for mapper file. min_rice : bool, optional determine whether you want to use the minimal rice value or the maximum rice value from the flow analysis. This MUST match the value used when creating the MRIO table. The default is **True**. single_point : bool, optional determine whether you are converting a single-point or multi-point failure analaysis to a disruption file. The default is **True**. Returns ------- event_dict Dictionary of all unique failure events, in terms of percentage disruption per sector in each directly affected region due to the flow failure. """ # Define current directory and data directory data_path = load_config()['paths']['data'] # read national IO comm_des = df_com_to_ind(pd.read_excel(os.path.join( data_path, 'OD_data', 'national_scale_od_matrix.xlsx'), sheet_name='total'), min_rice=min_rice) # read vietnam failure results vnm_failure_all_failure = pd.read_csv(input_file, index_col=[0]) if single_point == False: mapper = dict( zip(vnm_failure_all_failure.index.unique(), [ 'multi_{}'.format(n) for n in np.arange( 1, len(vnm_failure_all_failure.index.unique()) + 1, 1) ])) vnm_failure_all_failure.index = vnm_failure_all_failure.index.map( lambda x: mapper[x]) pd.DataFrame.from_dict(mapper, orient='index').to_csv( os.path.join(output_dir, 'mapper.csv')) event_dict = {} for id_, scenario in vnm_failure_all_failure.groupby(level=0): if id_ == 'multi_102': break disruption = df_com_to_ind(scenario, min_rice=min_rice).div(comm_des).dropna( axis=0, how='all').fillna(0) disruption = pd.DataFrame(disruption.stack(0)) disruption.columns = ['value'] disruption = disruption.loc[disruption.value > 0] disruption['value'] = 1 - (disruption['value'] * 0.2) event_dict[id_] = disruption['value'].to_dict() return event_dict
def load_db_SUT(table_in,RoW=None): data_path = load_config()['paths']['data'] '''CREATE GAMS WORKSPACE''' ws = GamsWorkspace(os.path.join(data_path,'gams_runs')) ''' CREATE INPUT FILES GAMS GDX ''' db = ws.add_database() #set regions reg = db.add_set("reg",1,"Regions") for r in (table_in.countries): reg.add_record(r) #set rowcol rowcol = db.add_set("rowcol",1,"All rows and columns") industries = list(table_in.sectors) products = list(table_in.products) final_demand = ['FinalD'] Import_lab = ['Import'] Export_lab = ['Export'] VA_lab = ['VA'] rowcol_input = industries + final_demand + Export_lab + products + VA_lab for r in (rowcol_input): rowcol.add_record(r) #set row row = db.add_set("row",1,"All rows") row_input = products + VA_lab + Import_lab for r in (row_input): row.add_record(r) #set col col = db.add_set("col",1,"All columns") col_input = industries + final_demand for r in (col_input): col.add_record(r) #set industries industries_ = db.add_set("ind",1,"Industries") for r in industries: industries_.add_record(r) #set Use table use_m = db.add_parameter("REG_USE2013", 4, "Interaction matrix") for k, v in table_in.Use.items(): use_m.add_record(k).value = v #set Supply table sup_m = db.add_parameter("REG_SUP2013", 4, "Interaction matrix") for k, v in table_in.Sup.items(): sup_m.add_record(k).value = v #set export ROW exp = db.add_parameter("ExpROW_ini", 3, "Exports to ROW") for k, v in table_in.ExpROW.items(): exp.add_record(k).value = v #set export ROW imp = db.add_parameter("ImpROW_ini", 3, "Imports from ROW") for k, v in table_in.ImpROW.items(): imp.add_record(k).value = v #set ValueA val = db.add_parameter("ValueA_ini", 3, "Value Added") for k, v in table_in.ValueA.items(): val.add_record(k).value = v # And save to GDX file db.export(os.path.join(data_path,"gams_runs","{}.gdx".format(table_in.name)))
def main(): data_path = load_config()['paths']['data'] hazard_intersections_path = load_config()['paths']['output'] # modes = ['road','rail','air','inland','coastal'] modes = ['road'] out_modes = [ 'national_roads', 'national_rail', 'air_ports', 'inland_ports', 'sea_ports' ] bnds = [3, 4, 5] thresholds = [1, 2, 3, 4, 999] commune_shp = os.path.join(data_path, 'Vietnam_boundaries', 'who_boundaries', 'who_communes.shp') hazard_description_file = os.path.join( data_path, 'Hazard_data', 'hazard_data_folder_data_info.xlsx') hazard_df = pd.read_excel(hazard_description_file, sheet_name='file_contents') hazard_files = hazard_df['file_name'].values.tolist() data_excel = os.path.join( hazard_intersections_path, 'hazard_scenarios', 'national_scale_hazard_intersections_roads.xlsx') excel_writer = pd.ExcelWriter(data_excel) for m in range(len(modes)): # set all paths for all input files we are going to use data_dict = [] intersection_dir = os.path.join( hazard_intersections_path, '{}_hazard_intersections'.format(out_modes[m])) for root, dirs, files in os.walk(intersection_dir): for file in files: if file.endswith(".shp"): hazard_dict = {} hazard_dict['sector'] = modes[m] hazard_shp = os.path.join(root, file) hz_file = [h for h in hazard_files if h in file][0] hazard_dict['hazard_type'] = hazard_df.loc[ hazard_df.file_name == hz_file].hazard_type.values[0] hazard_dict['model'] = hazard_df.loc[ hazard_df.file_name == hz_file].model.values[0] hazard_dict['year'] = hazard_df.loc[hazard_df.file_name == hz_file].year.values[0] hazard_dict['climate_scenario'] = hazard_df.loc[ hazard_df.file_name == hz_file].climate_scenario.values[0] hazard_dict['probability'] = hazard_df.loc[ hazard_df.file_name == hz_file].probability.values[0] band_type = hazard_df.loc[hazard_df.file_name == hz_file].banded.values[0] if str(band_type) == 'True': hazard_dict['band_num'] = [ b for b in bnds if '{}_band'.format(b) in file ][0] band_names = hazard_df.loc[ hazard_df.file_name == hz_file].bands.values[0].split(',') hazard_dict['band_name'] = [ b for b in band_names if str(hazard_dict['band_num']) in b ][0] hazard_dict['min_val'] = 0 hazard_dict['max_val'] = 0 else: hazard_dict['band_num'] = 0 hazard_dict['band_name'] = 'none' hazard_thrs = [ (thresholds[t], thresholds[t + 1]) for t in range(len(thresholds) - 1) if '{0}m-{1}m'.format(thresholds[t], thresholds[ t + 1]) in file ][0] hazard_dict['min_val'] = hazard_thrs[0] hazard_dict['max_val'] = hazard_thrs[1] data_dict = spatial_scenario_selection_national( hazard_shp, commune_shp, hazard_dict, data_dict, modes[m]) print('Done with', file) data_df = pd.DataFrame(data_dict) data_df_cols = data_df.columns.values.tolist() if 'length' in data_df_cols: selected_cols = [cols for cols in data_df_cols if cols != 'length'] data_df = data_df.groupby( selected_cols)['length'].sum().reset_index() data_df.to_excel(excel_writer, modes[m], index=False) excel_writer.save() del data_df
'Economic_failure_results', 'summarized' ) '''Create output folders''' if os.path.exists(output_dir) == False: os.mkdir(output_dir) sums.to_csv(os.path.join(data_path, 'Results', 'Economic_failure_results', 'summarized', '{}_summarized.csv'.format(os.path.basename(os.path.splitext(input_file)[0])))) return pd.concat(collect_outputs),sums if __name__ == '__main__': data_path = load_config()['paths']['data'] # input_file = os.path.join(data_path,'Results','Failure_results','single_edge_failures_totals_national_road_max.csv') # get_all_input_files = [os.path.join(data_path,'Results','Failure_results',x) for x in os.listdir(os.path.join(data_path,'Results','Failure_results')) if x.endswith(".csv")] get_all_input_files = [os.path.join(data_path,'Results','Failure_results','roads',x) for x in os.listdir(os.path.join(data_path,'Results','Failure_results','roads')) if x.endswith(".csv")] # with Pool(int(cpu_count())-2) as pool: # pool.map(estimate_losses,get_all_input_files,chunksize=1) for gi in get_all_input_files: estimate_losses(gi)
def main(): ''' Create the database connection ''' conf = load_config() try: conn = psycopg2.connect(**conf['database']) except: print("I am unable to connect to the database") curs = conn.cursor() engine = create_engine( 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( {**conf['database']})) ''' Step 2: Create the OD proprotions for the differnet modes ''' ''' First get the modal shares ''' modes = ['road', 'rail', 'air', 'water'] ind_cols = [ 'sugar', 'wood', 'steel', 'constructi', 'cement', 'fertilizer', 'coal', 'petroluem', 'manufactur', 'fishery', 'meat' ] crop_cols = [ 'rice', 'cash', 'cass', 'teas', 'maiz', 'rubb', 'swpo', 'acof', 'rcof', 'pepp' ] commodities = ind_cols + crop_cols province_id_list = [] sql_query = "select od_id,name_eng from province_level_stats" curs.execute(sql_query) read_layer = curs.fetchall() for row in read_layer: province_id_list.append((row[0], row[1])) pth_dict = {} pth_idx = 0 pth_list = [] pth_k_list = [] for com in commodities: for m in modes: ifile = 'network_od_flows_' + com + m + '.csv' od_flows = pd.read_csv(ifile).fillna(0) od_flows = od_flows[od_flows['Tonnage'] > 0.5] if od_flows.empty is False: flow_node_edge = od_flows.groupby([ 'node_path', 'edge_path', 'Origin_region', 'Destination_region' ])['Tonnage'].sum().reset_index() pth_idx, pth_list, pth_k_list, pth_dict = get_node_edge_path_flows( flow_node_edge, province_id_list, com, pth_idx, pth_list, pth_k_list, pth_dict, 0.5) del od_flows print('Done with', ifile) pth_tuple_list = [] for key, values in pth_dict.items(): pth_tuple = [0] * (3 + len(commodities)) pth_tuple[0] = key pth_tuple[1] = pth_dict[key]['node_path'] pth_tuple[2] = pth_dict[key]['edge_path'] values_keys = pth_dict[key].keys() for i in range(len(commodities)): if commodities[i] in values_keys: pth_tuple[i + 3] = pth_dict[key][commodities[i]] # pth_tuple.append(sum(pth_tuple[4:])) pth_tuple_list.append(tuple(pth_tuple)) excel_writer = pd.ExcelWriter('vnm_path_flows.xlsx') df = pd.DataFrame(pth_tuple_list, columns=['path_index', 'node_path', 'edge_path'] + commodities) df.to_excel(excel_writer, 'path_flows', index=False) excel_writer.save()
def main(): data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] ''' cols = ['band_name','band_num','climate_scenario','commune_id','commune_name','district_id','district_name', 'edge_id','hazard_type','max_val','min_val','model','probability','province_id','province_name','sector', 'year','length','road_cond','asset_type','width','min_econ_loss','max_econ_loss'] ''' start_year = 2016 end_year = 2050 truck_unit_wt = [5.0, 20.0] discount_rate = 12.0 total_discount_ratio = [] for year in range(start_year, end_year): # total_discount_ratio += 1.0/math.pow(1.0 + 1.0*discount_rate/100.0,year - start_year) total_discount_ratio.append( 1.0 / math.pow(1.0 + 1.0 * discount_rate / 100.0, year - start_year)) # total_discount_ratio = sum(total_discount_ratio_list) df_path = os.path.join(data_path, 'Adaptation_options', 'adaptation_options.xlsx') adaptation_df = pd.read_excel(df_path, sheet_name='adapt_options') adaptation_df['total_discount_ratio'] = sum(total_discount_ratio) min_maintain_discount_ratio_list = [] max_maintain_discount_ratio_list = [] for iter_, row in adaptation_df.iterrows(): min_maintain_schedule = row['maintenance_times_min'] max_maintain_schedule = row['maintenance_times_max'] min_maintain_discount_ratio = 0 max_maintain_discount_ratio = 0 max_maintain_discount_years = np.arange(start_year, end_year, min_maintain_schedule) min_maintain_discount_years = np.arange(start_year, end_year, max_maintain_schedule) for year in max_maintain_discount_years[1:]: max_maintain_discount_ratio += 1.0 / math.pow( 1.0 + 1.0 * discount_rate / 100.0, year - start_year) for year in min_maintain_discount_years[1:]: min_maintain_discount_ratio += 1.0 / math.pow( 1.0 + 1.0 * discount_rate / 100.0, year - start_year) min_maintain_discount_ratio_list.append(min_maintain_discount_ratio) max_maintain_discount_ratio_list.append(max_maintain_discount_ratio) adaptation_df[ 'min_maintain_discount_ratio'] = min_maintain_discount_ratio_list adaptation_df[ 'max_maintain_discount_ratio'] = max_maintain_discount_ratio_list adaptation_df['min_benefit'] = adaptation_df[ 'rehab_cost_min'] * adaptation_df['total_discount_ratio'] adaptation_df['max_benefit'] = adaptation_df[ 'rehab_cost_max'] * adaptation_df['total_discount_ratio'] adaptation_df['min_cost'] = adaptation_df['adapt_cost_min'] * adaptation_df[ 'total_discount_ratio'] + adaptation_df[ 'maintain_cost_min'] * adaptation_df['min_maintain_discount_ratio'] adaptation_df['max_cost'] = adaptation_df['adapt_cost_max'] * adaptation_df[ 'total_discount_ratio'] + adaptation_df[ 'maintain_cost_max'] * adaptation_df['max_maintain_discount_ratio'] print(adaptation_df) ''' Add new strategy cols = ['strategy','asset_type','asset_cond','location','height_m','adapt_cost_min','adapt_cost_max', 'maintain_cost_min','maintain_cost_max','rehab_cost_min','rehab_cost_max','height_incr_min', 'height_incr_max','maintenance_times_min','maintenance_times_max','cost_unit'] ''' cols = [ 'strategy', 'asset_type', 'asset_cond', 'location', 'height_m', 'cost_unit', 'min_cost', 'max_cost', 'min_benefit', 'max_benefit' ] adaptation_options = adaptation_df[cols] ad_opt = [] st_desc = ['road change', 'roads', 'unpaved-paved', 'all', 0, '$/m2'] st_min_cost = adaptation_df.loc[adaptation_df['asset_cond'] == 'paved', 'min_cost'].sum() st_max_cost = adaptation_df.loc[adaptation_df['asset_cond'] == 'paved', 'max_cost'].sum() st_min_benefit = adaptation_df.loc[ adaptation_df['asset_cond'] == 'unpaved', 'rehab_cost_min'].sum() * total_discount_ratio[0] + adaptation_df.loc[ adaptation_df['asset_cond'] == 'paved', 'rehab_cost_min'].sum() * sum(total_discount_ratio[1:]) st_max_benefit = adaptation_df.loc[ adaptation_df['asset_cond'] == 'unpaved', 'rehab_cost_max'].sum() * total_discount_ratio[0] + adaptation_df.loc[ adaptation_df['asset_cond'] == 'paved', 'rehab_cost_max'].sum() * sum(total_discount_ratio[1:]) ad_opt.append(st_desc + [st_min_cost, st_max_cost, st_min_benefit, st_max_benefit]) new_ht = 6 st_desc = [ 'dyke building', 'dyke', 'rural', 'sea', new_ht, 'million USD/km' ] st_min_cost = adaptation_df.loc[ adaptation_df['height_m'] == 4, 'min_cost'].sum() + (new_ht - 4) * adaptation_df.loc[ adaptation_df['height_m'] == 4, 'height_incr_min'].sum() * sum(total_discount_ratio) st_max_cost = adaptation_df.loc[ adaptation_df['height_m'] == 4, 'max_cost'].sum() + (new_ht - 4) * adaptation_df.loc[ adaptation_df['height_m'] == 4, 'height_incr_max'].sum() * sum(total_discount_ratio) st_min_benefit = adaptation_df.loc[adaptation_df['height_m'] == 4, 'min_benefit'].sum() st_max_benefit = adaptation_df.loc[adaptation_df['height_m'] == 4, 'max_benefit'].sum() ad_opt.append(st_desc + [st_min_cost, st_max_cost, st_min_benefit, st_max_benefit]) ad_opt_df = pd.DataFrame(ad_opt, columns=cols) adaptation_options = adaptation_options.append(ad_opt_df, ignore_index=True) print(adaptation_options) cols = [ 'band_num', 'climate_scenario', 'edge_id', 'hazard_type', 'max_val', 'min_val', 'model', 'probability', 'year', 'exposure_length' ] # index_cols = ['edge_id','hazard_type','model','climate_scenario','year','road_cond','asset_type','width','road_length'] selection_criteria = [ 'commune_id', 'hazard_type', 'model', 'climate_scenario', 'year' ] filter_cols = ['edge_id', 'exposed_length'] + selection_criteria # provinces to consider province_list = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa'] province_terrian = ['mountain', 'flat', 'flat'] growth_scenarios = [(5, 'low'), (6.5, 'forecast'), (10, 'high')] base_year = 2016 types = ['min', 'max'] fail_scenarios_data = os.path.join( output_path, 'hazard_scenarios', 'province_roads_hazard_intersections.xlsx') rd_prop_file = os.path.join(data_path, 'Roads', 'road_properties', 'road_properties.xlsx') duration_max = [10, 15, 20, 25, 30] length_thr = 100.0 for prn in range(len(province_list)): # for prn in range(1,3): province = province_list[prn] # set all paths for all input files we are going to use province_name = province.replace(' ', '').lower() all_edge_fail_scenarios = pd.read_excel(fail_scenarios_data, sheet_name=province_name) all_edge_fail_scenarios.loc[all_edge_fail_scenarios['probability'] == 'none', 'probability'] = 1.0 all_edge_fail_scenarios['probability'] = pd.to_numeric( all_edge_fail_scenarios['probability']) all_edge_fail_scenarios.rename(columns={'length': 'exposure_length'}, inplace=True) all_edge_fail_scenarios = all_edge_fail_scenarios[cols] all_edge_fail_scenarios = all_edge_fail_scenarios.drop_duplicates( subset=cols, keep=False) all_edges = list( set(all_edge_fail_scenarios['edge_id'].values.tolist())) all_edge_fail_scenarios['road_cond'] = 'unknown' all_edge_fail_scenarios['asset_type'] = 'unknown' all_edge_fail_scenarios['width'] = 0 all_edge_fail_scenarios['road_length'] = 0 edges_in = os.path.join(data_path, 'Roads', '{}_roads'.format(province_name), 'vietbando_{}_edges.shp'.format(province_name)) edges = province_shapefile_to_dataframe(edges_in, province_terrian[prn], rd_prop_file) edge_attr = list( zip(edges['edge_id'].values.tolist(), edges['road_cond'].values.tolist(), edges['asset_type'].values.tolist(), edges['width'].values.tolist(), edges['length'].values.tolist())) # print (edge_attr) edge_attr = [e for e in edge_attr if e[0] in all_edges] for e in edge_attr: all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'road_cond'] = e[1] all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'asset_type'] = e[2] all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'width'] = e[3] all_edge_fail_scenarios.loc[all_edge_fail_scenarios['edge_id'] == e[0], 'road_length'] = 1000.0 * e[4] # all_edge_fail_scenarios['percent_exposure'] = 100.0*all_edge_fail_scenarios['exposure_length']/all_edge_fail_scenarios['road_length'] # df_path = os.path.join(output_path,'hazard_scenarios','roads_hazard_intersections_{}.csv'.format(province_name)) # all_edge_fail_scenarios.to_csv(df_path,index = False) # all_edge_fail_scenarios = all_edge_fail_scenarios.set_index(index_cols) # scenarios = list(set(all_edge_fail_scenarios.index.values.tolist())) all_edge_fail_scenarios = all_edge_fail_scenarios.set_index( selection_criteria) scenarios = list(set(all_edge_fail_scenarios.index.values.tolist())) multiple_ef_list = [] for criteria in criteria_set: if len(all_edge_fail_scenarios.loc[criteria, 'edge_id'].index) == 1: efail = [ all_edge_fail_scenarios.loc[criteria, 'edge_id'].item() ] else: efail = list( set(all_edge_fail_scenarios.loc[ criteria, 'edge_id'].values.tolist())) flength = all_edge_fail_scenarios.loc[criteria, 'length'].sum() pflength = flength / all_edge_fail_scenarios.loc[ criteria, 'road_length'].sum() criteria_dict = { **(dict(list(zip(selection_criteria, criteria)))), **{ 'exposed_length': flength } } multiple_ef_list.append((efail, criteria_dict)) for tr_wt in truck_unit_wt: flow_output_excel = os.path.join( output_path, 'failure_results', 'multiple_edge_failures_totals_{0}_{1}_tons_projections.xlsx'. format(province_name, int(tr_wt))) adapt_output_excel = os.path.join( output_path, 'failure_results', 'multiple_edge_failures_scenarios_{0}_{1}_tons_adapt_options.xlsx' .format(province_name, int(tr_wt))) excl_wrtr = pd.ExcelWriter(adapt_output_excel) for grth in growth_scenarios: loss_time_series = pd.read_excel(flow_output_excel, sheet_name=grth[1]) for t in range(len(types)): loss_cols = [] for year in range(start_year, end_year): # col = '{0}_econ_loss_{1}'.format(types[t],year) loss_cols.append('{0}_econ_loss_{1}'.format( types[t], year)) # print (cols) # loss_time_series = loss_time_series[['edge_id']+cols] # print (loss_time_series) loss_time_series['{}_total_loss'.format( types[t])] = loss_time_series[loss_cols].sum(axis=1) loss_time_series = loss_time_series[[ 'edge_id', 'min_econ_loss_{}'.format(base_year), 'max_econ_loss_{}'.format(base_year), 'min_total_loss', 'max_total_loss' ]] loss_time_series = loss_time_series[ loss_time_series['max_total_loss'] > 0] print('done with loss estimation') for dur in range(len(duration_max)): scenarios_list = [] for sc in scenarios: edge = sc[0] road_cond = sc[-4] width = sc[-2] # print (road_cond,width) sub_df = all_edge_fail_scenarios.loc[sc] if len(sub_df.index) == 1: min_exposure_len = sub_df.loc[sc, 'exposure_length'] max_exposure_len = sub_df.loc[sc, 'exposure_length'] min_height = sub_df.loc[sc, 'min_val'] max_height = sub_df.loc[sc, 'max_val'] min_band_num = sub_df.loc[sc, 'band_num'] max_band_num = sub_df.loc[sc, 'band_num'] min_per = sub_df.loc[sc, 'percent_exposure'] max_per = sub_df.loc[sc, 'percent_exposure'] min_dur = 0.01 * duration_max[dur] * sub_df.loc[ sc, 'percent_exposure'] if sub_df.loc[sc, 'exposure_length'] < length_thr: max_dur = 0.01 * duration_max[ dur] * sub_df.loc[sc, 'percent_exposure'] risk_wt = 0.01 * duration_max[dur] * sub_df.loc[ sc, 'percent_exposure'] * sub_df.loc[ sc, 'probability'] else: max_dur = duration_max[dur] risk_wt = duration_max[dur] * sub_df.loc[ sc, 'probability'] else: prob = list( set(sub_df['probability'].values.tolist())) min_height = max(sub_df['min_val'].values.tolist()) max_height = max(sub_df['max_val'].values.tolist()) min_band_num = min( sub_df['band_num'].values.tolist()) max_band_num = max( sub_df['band_num'].values.tolist()) sub_df = sub_df.set_index('probability') exposure_len = [] per = [] risk_wt = 0 for pr in prob: # per_exp = sub_df.loc[pr,'percent_exposure'].sum() if sub_df.loc[ pr, 'percent_exposure'].sum() > 100.0: exposure_len.append( 100.0 * sub_df.loc[pr, 'exposure_length'].sum() / sub_df.loc[pr, 'percent_exposure'].sum()) per.append(100.0) risk_wt += 1.0 * duration_max[dur] * pr else: exposure_len.append( sub_df.loc[pr, 'exposure_length'].sum()) if sub_df.loc[pr, 'exposure_length'].sum( ) < length_thr: per.append(sub_df.loc[ pr, 'percent_exposure'].sum()) risk_wt += 0.01 * duration_max[ dur] * sub_df.loc[ pr, 'percent_exposure'].sum() * pr else: per.append(100.0) risk_wt += duration_max[dur] * pr max_exposure_len = max(exposure_len) min_exposure_len = min(exposure_len) min_per = min(per) max_per = max(per) min_dur = 0.01 * duration_max[dur] * min_per max_dur = 0.01 * duration_max[dur] * max_per sc_list = list(sc) + [ min_band_num, max_band_num, min_height, max_height, min_per, max_per, min_dur, max_dur, min_exposure_len, max_exposure_len ] # scenarios_list.append(list(sc) + [min_band_num,max_band_num,min_height,max_height,min_per,max_per,min_dur,max_dur,min_exposure_len,max_exposure_len,risk_wt]) if edge in loss_time_series['edge_id'].values.tolist(): min_daily_loss = loss_time_series.loc[ loss_time_series['edge_id'] == edge, 'min_econ_loss_{}'.format(base_year)].sum() max_daily_loss = loss_time_series.loc[ loss_time_series['edge_id'] == edge, 'max_econ_loss_{}'.format(base_year)].sum() edge_options = [] min_edge_ead = sum( total_discount_ratio ) * risk_wt * loss_time_series.loc[ loss_time_series['edge_id'] == edge, 'min_total_loss'].sum() max_edge_ead = sum( total_discount_ratio ) * risk_wt * loss_time_series.loc[ loss_time_series['edge_id'] == edge, 'max_total_loss'].sum() if max_height > 4: max_height = 6 if max_height > 0: edge_options = net_present_value( adaptation_options, 'height_m', [max_height], min_edge_ead, max_edge_ead, edge_options, edge_width=1.0, edge_length=1000.0 * max_exposure_len) if road_cond == 'unpaved': edge_options = net_present_value( adaptation_options, 'asset_cond', ['unpaved', 'unpaved-paved'], min_edge_ead, max_edge_ead, edge_options, edge_width=width, edge_length=max_exposure_len) if road_cond == 'paved': edge_options = net_present_value( adaptation_options, 'asset_cond', ['paved'], min_edge_ead, max_edge_ead, edge_options, edge_width=width, edge_length=max_exposure_len) else: min_daily_loss = 0 max_daily_loss = 0 min_edge_ead = 0 max_edge_ead = 0 edge_options = [{ 'strategy': 'none', 'min_npv': 0, 'max_npv': 0, 'min_bc_ratio': 0, 'max_bc_ratio': 0 }] for options in edge_options: scenarios_list.append(sc_list + [ min_daily_loss, max_daily_loss, min_edge_ead, max_edge_ead, options['strategy'], options['min_npv'], options['max_npv'], options['min_bc_ratio'], options['max_bc_ratio'] ]) new_cols = [ 'min_band', 'max_band', 'min_height', 'max_height', 'min_exposure_percent', 'max_exposure_percent', 'min_duration', 'max_duration', 'min_exposure_length', 'max_exposure_length', 'min_daily_loss', 'max_daily_loss', 'min_npv_nooption', 'max_npv_nooption', 'adapt_strategy', 'min_npv', 'max_npv', 'min_bc_ratio', 'max_bc_ratio' ] scenarios_df = pd.DataFrame(scenarios_list, columns=index_cols + new_cols) # df_path = os.path.join(output_path,'hazard_scenarios','roads_hazard_intersections_{}_risks.csv'.format(province_name)) # scenarios_df.to_csv(df_path,index = False) scenarios_df.to_excel(excl_wrtr, grth[1] + '_' + str(duration_max[dur]), index=False) excl_wrtr.save() print( 'Done with {0} in {1} tons {2} growth scenario {3} days' .format(province_name, tr_wt, grth[1], duration_max[dur]))
def main(): """Process hazard data 1. Specify the paths from where to read and write: - Input data - Hazard data 2. Supply input data and parameters - Thresholds of flood hazards - Values of bands to be selected - Color code of multi-band rasters - Specific file names that might require some specific operations """ data_path = load_config()['paths']['data'] root_dir = os.path.join(data_path, 'Hazard_data') thresholds = [1, 2, 3, 4, 999] band_vals_1 = [3, 4] band_vals_2 = [4, 5] color_codes_1 = [(255, 190, 190), (245, 0, 0), (255, 0, 0)] color_codes_2 = [(255, 170, 0), (255, 128, 0)] specific_files = ['LSZ_NgheAn_to_PhuYen.tif'] f_all = [] for root, dirs, files in os.walk(root_dir): for file in files: if file.endswith(".tif") or file.endswith(".tiff"): band_nums, crs, unique_data_values = raster_projections_and_databands( os.path.join(root, file)) print(file, crs, unique_data_values) if 'epsg' in crs: crs_split = crs.split(':') s_crs = [int(c) for c in crs_split if c.isdigit() is True][0] else: s_crs = 32648 if not unique_data_values: # threshold based datasets for t in range(len(thresholds) - 1): thr_1 = thresholds[t] thr_2 = thresholds[t + 1] in_file = os.path.join(root, file) tmp_1 = os.path.join( root, file.split(".tif")[0] + '_mask.tiff') tmp_2 = os.path.join( root, file.split(".tif")[0] + '_mask.shp') out_file = os.path.join( root, file.split(".tif")[0] + '_{0}m-{1}m_threshold.shp'.format(thr_1, thr_2)) convert_geotiff_to_vector_with_threshold( thr_1, thr_2, in_file, s_crs, tmp_1, tmp_2, out_file) elif band_nums == 1: # code value based dataset if file in specific_files: code_vals = band_vals_1 else: code_vals = band_vals_2 for c in code_vals: in_file = os.path.join(root, file) tmp_1 = os.path.join( root, file.split(".tif")[0] + '_mask.tiff') tmp_2 = os.path.join( root, file.split(".tif")[0] + '_mask.shp') out_file = os.path.join( root, file.split(".tif")[0] + '_{}_band.shp'.format(c)) convert_geotiff_to_vector_with_threshold( c, c + 1, in_file, s_crs, tmp_1, tmp_2, out_file) if band_nums == 3: # multi-band color datasets # remove nodata values from the bands in the raster raster_rewrite(os.path.join(root, file), os.path.join(root, 'test.tif'), 0) for dv in unique_data_values: if dv in color_codes_1: thr = 5 bc = dv in_file = os.path.join(root, file) tmp_1 = os.path.join( root, file.split(".tif")[0] + '_mask.tiff') tmp_2 = os.path.join( root, file.split(".tif")[0] + '_mask.shp') out_file = os.path.join( root, file.split(".tif")[0] + '_{}_band.shp'.format(thr)) convert_geotiff_to_vector_with_multibands( bc, in_file, s_crs, tmp_1, tmp_2, out_file) elif dv in color_codes_2: thr = 4 bc = dv in_file = os.path.join(root, file) tmp_1 = os.path.join( root, file.split(".tif")[0] + '_mask.tiff') tmp_2 = os.path.join( root, file.split(".tif")[0] + '_mask.shp') out_file = os.path.join( root, file.split(".tif")[0] + '_{}_band.shp'.format(thr)) convert_geotiff_to_vector_with_multibands( bc, in_file, s_crs, tmp_1, tmp_2, out_file)
def load_db_IO(table_in,EORA=False,RoW=None): data_path = load_config()['paths']['data'] '''CREATE GAMS WORKSPACE''' ws = GamsWorkspace(os.path.join(data_path,'gams_runs')) ''' CREATE INPUT FILES GAMS GDX ''' db = ws.add_database() #set regions reg = db.add_set("reg",1,"Regions") if EORA is True: for r in (table_in.countries+['ROW']): reg.add_record(r) else: for r in (table_in.countries): reg.add_record(r) #set rowcol rowcol = db.add_set("rowcol",1,"All rows and columns") if EORA is True: industries = list(table_in.sectors) + ['Total'] final_demand = list(table_in.FD_labels['FD'].unique()) else: industries = list(table_in.sectors) final_demand = list(table_in.FD_labels['tfd'].unique()) Import_lab = ['Import'] Export_lab = ['Export'] VA_lab = ['VA'] rowcol_input = industries + final_demand + VA_lab + Import_lab + Export_lab for r in (rowcol_input): rowcol.add_record(r) #set row row = db.add_set("row",1,"All rows") row_input = industries + VA_lab + Import_lab for r in (row_input): row.add_record(r) #set col col = db.add_set("col",1,"All columns") col_input = industries + final_demand for r in (col_input): col.add_record(r) #set industries industries_ = db.add_set("S",1,"Industries") for r in industries: industries_.add_record(r) #set FinalD fd_ = GamsParameter(db,"FinDem_ini", 4, "FinDem") for k, v in table_in.FinalD.items(): fd_.add_record(k).value = v #set interaction matrix of intermediate demand z_m = db.add_parameter("Z_matrix_ini", 4, "Interaction matrix") for k, v in table_in.Z_matrix.items(): z_m.add_record(k).value = v #set interaction matrix of intermediate demand a_m = db.add_parameter("A_matrix_ini", 4, "A matrix") for k, v in table_in.A_matrix.items(): a_m.add_record(k).value = v if EORA is not True: #set Export ROW exp = db.add_parameter("ExpROW_ini", 3, "Exports to ROW") for k, v in table_in.ExpROW.items(): exp.add_record(k).value = v #set ValueA val = db.add_parameter("ValueA_ini", 3, "Value Added") for k, v in table_in.ValueA.items(): val.add_record(k).value = v # And save to GDX file db.export(os.path.join(data_path,"gams_runs","{}.gdx".format(table_in.name)))
def main(): data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] # drop_cols = ['gid','node_id'] # viwa_file = os.path.join(data_path,'Waterways','viwa_select','iwt_ports.shp') # viwa_in = gpd.read_file(viwa_file) # for d in drop_cols: # if d in viwa_in.columns.values.tolist(): # viwa_in.drop(d,axis=1,inplace=True) # viwa_in['port_type'] = 'inland' # seaport_file = os.path.join(data_path,'Waterways','inlandandseaports','vietnam_seaport_nodes.shp') # seaport_in = gpd.read_file(seaport_file) # for d in drop_cols: # if d in seaport_in.columns.values.tolist(): # seaport_in.drop(d,axis=1,inplace=True) # seaport_in['port_type'] = 'sea' # all_ports = pd.concat([viwa_in,seaport_in], axis=0, sort = 'False', ignore_index=True).fillna(0) # all_ports = gpd.GeoDataFrame(all_ports,crs='epsg:4326') # print (all_ports) # all_ports.to_file(os.path.join(data_path,'Waterways','waterways','ports.shp')) # edges_file = os.path.join(data_path,'Waterways','waterways','wateredges.shp') # edges_in = gpd.read_file(edges_file) # drop_cols = ['eid','edge_id','node_f_id','node_t_id'] # for d in drop_cols: # if d in edges_in.columns.values.tolist(): # edges_in.drop(d,axis=1,inplace=True) # edges_in.to_file(os.path.join(data_path,'Waterways','waterways','port_routes.shp')) # imp_sea_ports = {'class_1A':[135,146,147,140],'class_1':[149,150,169,175,136,137,158,163,153,151,139,171,172,170]} # port_ids = [] # imp_ports_ids = [135,146,147,140,149,150,169,175,136,137,158,163,153,151,139,171,172,170] # for im_sp in imp_ports_ids: # if im_sp in [135,146,147,140]: # port_ids.append(('class_1A','watern_{}'.format(im_sp))) # elif im_sp in [149,150,169,175,136,137,158,163,153,151,139,171,172,170]: # port_ids.append(('class_1','watern_{}'.format(im_sp))) # print (port_ids) # ports_file = os.path.join(data_path,'Waterways','waterways','ports_nodes.shp') # ports_in = gpd.read_file(ports_file).fillna(0) # ports_in['port_class'] = 'none' # for p in port_ids: # ports_in.loc[ports_in['NODE_ID'] == p[1],'port_class'] = p[0] # ports_in.loc[ports_in['NODE_ID'] == p[1],'TONS'] = 5000000 # ports_in.loc[ports_in['TONS'] == 0, 'TONS'] = 1 # # print (ports_in) # ports_in.to_file(ports_file) air_file = os.path.join(data_path, 'Airports', 'airnetwork', 'airportedges.shp') air_in = gpd.read_file(air_file) air_in['g_id'] = [ int(x.split('_')[1]) for x in air_in['edge_id'].values.tolist() ] air_in.rename(columns={'node_f_id': 'from_node'}, inplace=True) air_in.rename(columns={'node_t_id': 'to_node'}, inplace=True) air_in = air_in[[ 'edge_id', 'g_id', 'from_node', 'to_node', 'speed', 'geometry' ]] air_in.to_file( os.path.join(data_path, 'Airports', 'airnetwork', 'airportedges.shp'))
def main(): curdir = os.getcwd() conf = load_config() try: conn = psycopg2.connect(**conf['database']) except: print("I am unable to connect to the database") curs = conn.cursor() engine = create_engine( 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( {**conf['database']})) layers_ids = [('roads_edge_flood', 'edge_id', 'road2009edges', 'geom'), ('rail_edge_flood', 'edge_id', 'railnetworkedges', 'geom'), ('roads_node_flood', 'node_id', 'road2009nodes', 'geom'), ('rail_node_flood', 'node_id', 'railnetworknodes', 'geom'), ('port_flood', 'node_id', 'seaport_nodes', 'geom'), ('airport_flood', 'node_id', 'airport_nodes', 'geom'), ('provinces_flooding', 'name_eng', 'province_level_stats', 'geom')] regional_table = 'province_level_stats' mode_edge_tables = ['rail_edge_flood', 'roads_edge_flood'] edge_id = 'edge_id' regional_name = 'name_eng' col_names = [ 'level13_vt_100_mask_1', 'level14_vt_100_mask_1', 'level15_vt_100_mask_1', 'level16_vt_100_mask_1' ] excel_writer = pd.ExcelWriter('vnm_road_rail_flood_list.xlsx') for c in col_names: fl_rg_list = [] for m in mode_edge_tables: edge_region_list = [] sql_query = '''SELECT A.{0},B.{1} FROM {2} as A, {3} as B where st_intersects(A.geom,B.geom) is True and A.{4} > 0 '''.format(edge_id, regional_name, m, regional_table, c) cur.execute(sql_query) read_layer = cur.fetchall() for row in read_layer: edge_region_list.append((row[0], row[1])) tedge_regions = [r[1] for r in edge_region_list] tedge_regions = sorted(list(set(tedge_regions))) for rg in tedge_regions: ed = [r[0] for r in edge_region_list if r[1] == rg] fl_rg_list.append((rg, {'flood_edge': ed})) fl_rg_list = sorted(fl_rg_list, key=lambda x: x[0]) df = pd.DataFrame(fl_rg_list, columns=['region', 'flood_list']) df.to_excel(excel_writer, c, index=False) excel_writer.save() conf = load_config() try: conn = psycopg2.connect(**conf['database']) except: print("I am unable to connect to the database") curs = conn.cursor() engine = create_engine( 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( {**conf['database']})) layers_ids = [('roads_edge_flood', 'edge_id', 'road2009edges', 'geom'), ('rail_edge_flood', 'edge_id', 'railnetworkedges', 'geom'), ('roads_node_flood', 'node_id', 'road2009nodes', 'geom'), ('rail_node_flood', 'node_id', 'railnetworknodes', 'geom'), ('port_flood', 'node_id', 'seaport_nodes', 'geom'), ('airport_flood', 'node_id', 'airport_nodes', 'geom'), ('provinces_flooding', 'name_eng', 'province_level_stats', 'geom')] regional_table = 'province_level_stats' mode_edge_tables = ['rail_edge_flood', 'roads_edge_flood'] edge_id = 'edge_id' regional_name = 'name_eng' col_names = [ 'level13_vt_100_mask_1', 'level14_vt_100_mask_1', 'level15_vt_100_mask_1', 'level16_vt_100_mask_1' ] excel_writer = pd.ExcelWriter('vnm_road_rail_flood_list.xlsx') for c in col_names: fl_rg_list = [] for m in mode_edge_tables: edge_region_list = [] sql_query = '''SELECT A.{0},B.{1} FROM {2} as A, {3} as B where st_intersects(A.geom,B.geom) is True and A.{4} > 0 '''.format(edge_id, regional_name, m, regional_table, c) cur.execute(sql_query) read_layer = cur.fetchall() for row in read_layer: edge_region_list.append((row[0], row[1])) tedge_regions = [r[1] for r in edge_region_list] tedge_regions = sorted(list(set(tedge_regions))) for rg in tedge_regions: ed = [r[0] for r in edge_region_list if r[1] == rg] fl_rg_list.append((rg, {'flood_edge': ed})) fl_rg_list = sorted(fl_rg_list, key=lambda x: x[0]) df = pd.DataFrame(fl_rg_list, columns=['region', 'flood_list']) df.to_excel(excel_writer, c, index=False) excel_writer.save()
def main(): """Process results 1. Specify the paths from where you to read and write: - Input data - Intermediate calcuations data - Output results 2. Supply input data and parameters - Names of the three Provinces - List of string types - Names of modes - List of strings - Names of output modes - List of strings - Names of hazard bands - List of integers - Names of hazard thresholds - List of integers - Condition 'Yes' or 'No' is the users wants to process results 3. Give the paths to the input data files: - Commune boundary and stats data shapefile - Hazard datasets description Excel file - String name of sheet in hazard datasets description Excel file """ config = load_config() output_path = config['paths']['output'] data_path = config['paths']['data'] # Supply input data and parameters provinces = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa'] start_year = 2016 length_thr = 500.0 cols = [ 'band_num', 'climate_scenario', 'edge_id', 'hazard_type', 'max_val', 'min_val', 'model', 'probability', 'year', 'length' ] index_cols = [ 'edge_id', 'hazard_type', 'model', 'climate_scenario', 'year', 'level', 'terrain', 'surface', 'road_class', 'road_cond', 'asset_type', 'width', 'road_length' ] # Give the paths to the input data files network_data_excel = os.path.join(data_path, 'post_processed_networks') fail_scenarios_data = os.path.join(output_path, 'hazard_scenarios') # Specify the output files and paths to be created output_dir = os.path.join(output_path, 'hazard_scenarios') # Process province scale results for province in provinces: # Load mode network DataFrame province_name = province.replace(' ', '').lower() print('* Loading {} network DataFrame'.format(province)) G_df = pd.read_excel(os.path.join(network_data_excel, 'province_roads_edges.xlsx'), sheet_name=province_name, encoding='utf-8') G_df = G_df[[ 'edge_id', 'level', 'terrain', 'surface', 'road_class', 'road_cond', 'asset_type', 'width', 'length' ]] # Load failure scenarios print('* Loading {} failure scenarios'.format(province)) hazard_scenarios = pd.read_excel(os.path.join( fail_scenarios_data, 'province_roads_hazard_intersections.xlsx'), sheet_name=province_name) hazard_scenarios = hazard_scenarios.drop_duplicates(subset=cols, keep='first') all_edge_fail_scenarios = combine_hazards_and_network_attributes_and_impacts( hazard_scenarios, G_df) print('* Creating {} hazard-network scenarios'.format(province)) scenarios_df = create_hazard_scenarios_for_adaptation( all_edge_fail_scenarios, index_cols, length_thr) df_path = os.path.join( output_dir, 'roads_hazard_intersections_{}_risks.csv'.format(province_name)) scenarios_df.to_csv(df_path, index=False) del scenarios_df
gdf_edges['netrev'] = 0 for path in save_paths: gdf_edges.loc[gdf_edges['edge_id'].isin(path[2]), 'netrev'] += path[3] if save_edges == True: gdf_edges.to_file( os.path.join( output_path, 'weighted_edges_district_center_flows_{}.shp'.format( region_name))) return gdf_edges if __name__ == '__main__': data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] # provinces to consider province_list = ['Thanh Hoa', 'Binh Dinh', 'Lao Cai'] # province_list = ['Thanh Hoa'] district_committe_names = [ 'district_people_committee_points_thanh_hoa.shp', 'district_province_peoples_committee_point_binh_dinh.shp', 'district_people_committee_points_lao_cai.shp' ] shp_output_path = os.path.join(output_path, 'flow_mapping_shapefiles') flow_output_excel = os.path.join( output_path, 'flow_mapping_paths', 'province_roads_district_center_flow_paths.xlsx')
def main(): ''' Create the database connection ''' conf = load_config() try: conn = psycopg2.connect(**conf['database']) except: print("I am unable to connect to the database") curs = conn.cursor() engine = create_engine( 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( {**conf['database']})) od_data_file = os.path.join(conf['paths']['data'], 'od_data', 'OD_transport_data_2008_v2.xlsx') ''' Step 2: Create the OD proprotions for the differnet modes ''' ''' First get the modal shares ''' modes = ['road', 'rail', 'air', 'water'] mode_cols = ['road', 'rail', 'air', 'inland', 'coastal'] new_mode_cols = ['o', 'd', 'road', 'rail', 'air', 'water'] mode_table = [ 'airport_nodes', 'waternodes', 'railnetworknodes', 'road2009nodes' ] mode_edge_tables = [ 'airport_edges', 'wateredges', 'railnetworkedges', 'road2009edges' ] mode_flow_tables = [] for mo in mode_edge_tables: fl_table = mo + '_flows' mode_flow_tables.append(fl_table) ''' Get the modal shares ''' od_data_modes = pd.read_excel(od_data_file, sheet_name='mode').fillna(0) # od_data_modes.columns = map(str.lower, od_data_modes.columns) o_id_col = 'o' d_id_col = 'd' od_data_modes['total'] = od_data_modes[mode_cols].sum(axis=1) for m in mode_cols: od_data_modes[m] = od_data_modes[m] / od_data_modes['total'].replace( np.inf, 0) od_data_modes['water'] = od_data_modes['inland'] + od_data_modes['coastal'] od_data_modes = od_data_modes.fillna(0) # od_data_modes.to_csv('mode_frac.csv',index = False) od_fracs = od_data_modes[new_mode_cols] od_data_com = pd.read_excel(od_data_file, sheet_name='goods').fillna(0) ind_cols = [ 'sugar', 'wood', 'steel', 'constructi', 'cement', 'fertilizer', 'coal', 'petroluem', 'manufactur', 'fishery', 'meat' ] od_fracs = pd.merge(od_fracs, od_data_com, how='left', on=['o', 'd']) del od_data_com, od_data_modes od_fracs = od_fracs.fillna(0) # od_fracs.to_csv('od_fracs.csv') for ind in ind_cols: ''' Step 2 assign the crop to the closest transport mode node ''' # mode_table = ['road2009nodes','railwaynetworknodes','airport_nodes','waternodes'] # mode_edge_tables = ['road2009edges','railwaynetworkedges','airport_edges','wateredges'] # modes = ['road','rail','air','water'] modes = ['air', 'water', 'rail', 'road'] mode_id = 'node_id' od_id = 'od_id' pop_id = 'population' o_id_col = 'o' d_id_col = 'd' ''' Get the network ''' eid = 'edge_id' nfid = 'node_f_id' ntid = 'node_t_id' spid = 'speed' gmid = 'geom' o_id_col = 'o' d_id_col = 'd' ''' Get the node edge flows ''' excel_writer = pd.ExcelWriter('vietnam_flow_stats_' + ind + '.xlsx') for m in range(len(mode_table)): od_nodes_regions = [] sql_query = '''select {0}, {1}, 100*{2}/(sum({3}) over (Partition by {4})) from {5} '''.format(mode_id, od_id, pop_id, pop_id, od_id, mode_table[m]) curs.execute(sql_query) read_layer = curs.fetchall() if read_layer: for row in read_layer: n = row[0] r = row[1] p = float(row[2]) if p > 0: od_nodes_regions.append((n, r, p)) all_net_dict = { 'edge': [], 'from_node': [], 'to_node': [], 'distance': [], 'speed': [], 'travel_cost': [] } all_net_dict = tnc.create_network_dictionary( all_net_dict, mode_edge_tables[m], eid, nfid, ntid, spid, 'geom', curs, conn) od_net = tnc.create_igraph_topology(all_net_dict) ''' Get the OD flows ''' net_dict = { 'Origin_id': [], 'Destination_id': [], 'Origin_region': [], 'Destination_region': [], 'Tonnage': [], 'edge_path': [], 'node_path': [] } ofile = 'network_od_flows_' + ind + modes[m] + '.csv' output_file = open(ofile, 'w') wr = csv.writer(output_file, delimiter=',', quoting=csv.QUOTE_MINIMAL) wr.writerow(net_dict.keys()) ind_mode = modes[m] + '_' + ind od_fracs[ind_mode] = od_fracs[modes[m]] * od_fracs[ind] od_flows = list( zip(od_fracs[o_id_col].values.tolist(), od_fracs[d_id_col].values.tolist(), od_fracs[ind_mode].values.tolist())) origins = list(set(od_fracs[o_id_col].values.tolist())) destinations = list(set(od_fracs[d_id_col].values.tolist())) dflows = [] # print (od_flows) for o in origins: for d in destinations: fval = [ fl for (org, des, fl) in od_flows if org == o and des == d ] if len(fval) == 1 and fval[0] > 0: o_matches = [(item[0], item[2]) for item in od_nodes_regions if item[1] == o] if len(o_matches) > 0: for o_vals in o_matches: o_val = 1.0 * fval[0] * (1.0 * o_vals[1] / 100) o_node = o_vals[0] d_matches = [(item[0], item[2]) for item in od_nodes_regions if item[1] == d] if len(d_matches) > 0: for d_vals in d_matches: od_val = 1.0 * o_val * ( 1.0 * d_vals[1] / 100) d_node = d_vals[0] if od_val > 0 and o_node != d_node: # od_net = tnc.add_igraph_costs(od_net,t_val,0) orgn_node = od_net.vs[ 'node'].index(o_node) dest_node = od_net.vs[ 'node'].index(d_node) # n_pth = od_net.get_shortest_paths(orgn_node,to = dest_node, weights = 'travel_cost', mode = 'OUT', output='vpath')[0] e_pth = od_net.get_shortest_paths( orgn_node, to=dest_node, weights='travel_cost', mode='OUT', output='epath')[0] # n_list = [od_net.vs[n]['node'] for n in n_pth] e_list = [ od_net.es[n]['edge'] for n in e_pth ] # cst = sum([od_net.es[n]['cost'] for n in e_pth]) net_dict = { 'Origin_id': o_node, 'Destination_id': d_node, 'Origin_region': o, 'Destination_region': d, 'Tonnage': od_val, 'edge_path': e_list, 'node_path': [o_node, d_node] } wr.writerow(net_dict.values()) dflows.append( (str([o_node, d_node]), str(e_list), od_val)) print(o, d, fval, modes[m], ind) node_table = modes[m] + '_node_flows' edge_table = modes[m] + '_edge_flows' # dom_flows = pd.read_csv(ofile).fillna(0) dom_flows = pd.DataFrame( dflows, columns=['node_path', 'edge_path', 'Tonnage']) flow_node_edge = dom_flows.groupby( ['node_path', 'edge_path'])['Tonnage'].sum().reset_index() n_dict = {} e_dict = {} n_dict, e_dict = get_node_edge_flows(flow_node_edge, n_dict, e_dict) node_list = get_id_flows(n_dict) df = pd.DataFrame(node_list, columns=['node_id', ind]) df.to_excel(excel_writer, node_table, index=False) excel_writer.save() edge_list = get_id_flows(e_dict) df = pd.DataFrame(edge_list, columns=['edge_id', ind]) df.to_excel(excel_writer, edge_table, index=False) excel_writer.save() if df.empty: add_zeros_columns_to_table_psycopg2(mode_flow_tables[m], [ind], ['double precision'], conn) else: df.to_sql('dummy_flows', engine, if_exists='replace', schema='public', index=False) add_columns_to_table_psycopg2(mode_flow_tables[m], 'dummy_flows', [ind], ['double precision'], 'edge_id', conn) curs.close() conn.close()
def main(): ''' Create the database connection ''' conf = load_config() try: conn = psycopg2.connect(**conf['database']) except: print("I am unable to connect to the database") curs = conn.cursor() engine = create_engine( 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( {**conf['database']})) crop_data_path = os.path.join(conf['paths']['data'], 'crop_data') od_data_file = os.path.join(conf['paths']['data'], 'od_data', 'OD_transport_data_2008_v2.xlsx') ''' Step 2: Create the OD proprotions for the differnet modes ''' ''' First get the modal shares ''' modes = ['road', 'rail', 'air', 'water'] mode_cols = ['road', 'rail', 'air', 'inland', 'coastal'] new_mode_cols = ['o', 'd', 'road', 'rail', 'air', 'water'] crop_names = [ 'rice', 'cash', 'cass', 'teas', 'maiz', 'rubb', 'swpo', 'acof', 'rcof', 'pepp' ] mode_table = [ 'airport_nodes', 'waternodes', 'railnetworknodes', 'road2009nodes' ] mode_edge_tables = [ 'airport_edges', 'wateredges', 'railnetworkedges', 'road2009edges' ] mode_flow_tables = [] for mo in mode_edge_tables: fl_table = mo + '_flows' mode_flow_tables.append(fl_table) drop_postgres_table_psycopg2(fl_table, conn) with conn.cursor() as cur: sql_query = "create table {0} as select edge_id,geom from {1}".format( fl_table, mo) cur.execute(sql_query) conn.commit() ''' Get the modal shares ''' od_data_modes = pd.read_excel(od_data_file, sheet_name='mode').fillna(0) # od_data_modes.columns = map(str.lower, od_data_modes.columns) o_id_col = 'o' d_id_col = 'd' od_data_modes['total'] = od_data_modes[mode_cols].sum(axis=1) for m in mode_cols: od_data_modes[m] = od_data_modes[m] / od_data_modes['total'].replace( np.inf, 0) od_data_modes['water'] = od_data_modes['inland'] + od_data_modes['coastal'] od_data_modes = od_data_modes.fillna(0) # od_data_modes.to_csv('mode_frac.csv',index = False) od_fracs = od_data_modes[new_mode_cols] od_data_com = pd.read_excel(od_data_file, sheet_name='goods').fillna(0) crop_cols = ['rice', 'indust-cro'] for cr in crop_cols: od_data_com_sums = od_data_com.groupby(['o', 'd']).agg({cr: 'sum'}) od_com_frac = od_data_com_sums.groupby( level=0).apply(lambda x: x / float(x.sum())) od_com_frac = od_com_frac.reset_index(level=['o', 'd']) od_fracs = pd.merge(od_fracs, od_com_frac, how='left', on=['o', 'd']) del od_data_com, od_data_com_sums, od_com_frac od_fracs = od_fracs.fillna(0) # od_fracs.to_csv('od_fracs.csv') for file in os.listdir(crop_data_path): if file.endswith(".tif") and 'spam_p' in file.lower().strip(): fpath = os.path.join(crop_data_path, file) crop_name = [ cr for cr in crop_names if cr in file.lower().strip() ][0] raster_in = fpath outCSVName = 'crop_concentrations.csv' crop_table = crop_name + '_production' '''Clip to region and convert to points''' os.system('gdal2xyz.py -csv ' + raster_in + ' ' + outCSVName) '''Load points and convert to geodataframe with coordinates''' load_points = pd.read_csv(outCSVName, header=None, names=['x', 'y', 'crop_prod'], index_col=None) load_points = load_points[load_points['crop_prod'] > 0] # load_points.to_csv('crop_concentrations.csv', index = False) geometry = [Point(xy) for xy in zip(load_points.x, load_points.y)] load_points = load_points.drop(['x', 'y'], axis=1) crs = {'init': 'epsg:4326'} points_gdp = gpd.GeoDataFrame(load_points, crs=crs, geometry=geometry) points_gdp['geom'] = points_gdp['geometry'].apply( lambda x: WKTElement(x.wkt, srid=4326)) #drop the geometry column as it is now duplicative points_gdp.drop('geometry', 1, inplace=True) # points_gdp = points_gdp.rename(columns={'geometry':'geom'}).set_geometry('geom') del load_points print('created geopandas dataframe from the points') points_gdp.to_sql(crop_table, engine, if_exists='replace', schema='public', index=True, dtype={'geom': Geometry('POINT', srid=4326)}) del points_gdp ''' Add gid field to crop table ''' with conn.cursor() as cur: sql_query = "alter table {0} add column gid serial".format( crop_table) cur.execute(sql_query) conn.commit() print('Done with loading crop table to database') ''' Step 1 Assign the regions closest to the crop nodes ''' nd_table = crop_table regional_table = 'province_level_stats' dummy_table = 'dummy_table' nd_id = 'gid' nd_gm = 'geom' regional_gm = 'geom' nd_attr = ['gid'] regional_attr = ['name_eng', 'provinceid', 'od_id'] regional_attr_type = ['character varying', 'integer', 'integer'] drop_postgres_table_psycopg2(dummy_table, conn) # nodes_polygons_nearest_psycopg2(dummy_table,nd_table,regional_table,nd_attr,regional_attr,nd_gm,regional_gm,cur,conn) nodes_polygons_within_nearest_psycopg2(dummy_table, nd_table, regional_table, nd_attr, nd_id, regional_attr, nd_gm, regional_gm, conn) print('Done with assigning attributes to the crop table') add_columns_to_table_psycopg2(nd_table, dummy_table, regional_attr, regional_attr_type, nd_id, conn) print('Done with adding columns to the crop table') ''' Step 2 assign the crop to the closest transport mode node ''' # mode_table = ['road2009nodes','railwaynetworknodes','airport_nodes','waternodes'] # mode_edge_tables = ['road2009edges','railwaynetworkedges','airport_edges','wateredges'] # modes = ['road','rail','air','water'] modes = ['air', 'water', 'rail', 'road'] mode_id = 'node_id' crop_id = 'gid' mode_crop_m = 'od_id' crop_mode_m = 'od_id' crop_prod = 'crop_prod' od_id = 'od_id' od_id_type = 'integer' o_id_col = 'o' d_id_col = 'd' ''' Get the network ''' eid = 'edge_id' nfid = 'node_f_id' ntid = 'node_t_id' spid = 'speed' gmid = 'geom' o_id_col = 'o' d_id_col = 'd' ''' Get the node edge flows ''' excel_writer = pd.ExcelWriter('vietnam_flow_stats_' + crop_name + '.xlsx') for m in range(len(mode_table)): drop_postgres_table_psycopg2(dummy_table, conn) nodes_polygons_aggregations(dummy_table, mode_table[m], crop_table, mode_id, crop_id, mode_crop_m, crop_mode_m, crop_prod, nd_gm, regional_gm, conn) add_columns_to_table_psycopg2(dummy_table, mode_table[m], [od_id], [od_id_type], mode_id, conn) od_nodes_regions = [] with conn.cursor() as cur: sql_query = '''select {0}, {1}, {2}, {3}/(sum({4}) over (Partition by {5})) from {6} '''.format(mode_id, od_id, crop_prod, crop_prod, crop_prod, od_id, dummy_table) cur.execute(sql_query) read_layer = cur.fetchall() if read_layer: for row in read_layer: n = row[0] r = row[1] c = float(row[2]) p = float(row[3]) if p > 0: od_nodes_regions.append((n, r, c, p)) all_net_dict = { 'edge': [], 'from_node': [], 'to_node': [], 'distance': [], 'speed': [], 'travel_cost': [] } all_net_dict = tnc.create_network_dictionary( all_net_dict, mode_edge_tables[m], eid, nfid, ntid, spid, 'geom', curs, conn) od_net = tnc.create_igraph_topology(all_net_dict) ''' Get the OD flows ''' net_dict = { 'Origin_id': [], 'Destination_id': [], 'Origin_region': [], 'Destination_region': [], 'Tonnage': [], 'edge_path': [], 'node_path': [] } ofile = 'network_od_flows_' + crop_name + modes[m] + '.csv' output_file = open(ofile, 'w') wr = csv.writer(output_file, delimiter=',', quoting=csv.QUOTE_MINIMAL) wr.writerow(net_dict.keys()) crop_mode = modes[m] + '_' + crop_name if crop_name in ('rice', 'cereal', 'wheat'): od_fracs[crop_mode] = od_fracs[modes[m]] * od_fracs['rice'] else: od_fracs[crop_mode] = od_fracs[ modes[m]] * od_fracs['indust-cro'] od_flows = list( zip(od_fracs[o_id_col].values.tolist(), od_fracs[d_id_col].values.tolist(), od_fracs[crop_mode].values.tolist())) origins = list(set(od_fracs[o_id_col].values.tolist())) destinations = list(set(od_fracs[d_id_col].values.tolist())) dflows = [] # print (od_flows) for o in origins: for d in destinations: fval = [ fl for (org, des, fl) in od_flows if org == o and des == d ] if len(fval) == 1 and fval[0] > 0: o_matches = [(item[0], item[2]) for item in od_nodes_regions if item[1] == o] if len(o_matches) > 0: for o_vals in o_matches: o_val = 1.0 * fval[0] * o_vals[1] o_node = o_vals[0] d_matches = [(item[0], item[3]) for item in od_nodes_regions if item[1] == d] if len(d_matches) > 0: for d_vals in d_matches: od_val = 1.0 * o_val * d_vals[ 1] / 365 d_node = d_vals[0] if od_val > 0 and o_node != d_node: # od_net = tnc.add_igraph_costs(od_net,t_val,0) orgn_node = od_net.vs[ 'node'].index(o_node) dest_node = od_net.vs[ 'node'].index(d_node) # n_pth = od_net.get_shortest_paths(orgn_node,to = dest_node, weights = 'travel_cost', mode = 'OUT', output='vpath')[0] e_pth = od_net.get_shortest_paths( orgn_node, to=dest_node, weights='travel_cost', mode='OUT', output='epath')[0] # n_list = [od_net.vs[n]['node'] for n in n_pth] e_list = [ od_net.es[n]['edge'] for n in e_pth ] # cst = sum([od_net.es[n]['cost'] for n in e_pth]) net_dict = { 'Origin_id': o_node, 'Destination_id': d_node, 'Origin_region': o, 'Destination_region': d, 'Tonnage': od_val, 'edge_path': e_list, 'node_path': [o_node, d_node] } wr.writerow(net_dict.values()) dflows.append( (str([o_node, d_node]), str(e_list), od_val)) print(o, d, fval, modes[m], crop_name) node_table = modes[m] + '_node_flows' edge_table = modes[m] + '_edge_flows' # dom_flows = pd.read_csv(ofile).fillna(0) dom_flows = pd.DataFrame( dflows, columns=['node_path', 'edge_path', 'Tonnage']) flow_node_edge = dom_flows.groupby( ['node_path', 'edge_path'])['Tonnage'].sum().reset_index() n_dict = {} e_dict = {} n_dict, e_dict = get_node_edge_flows(flow_node_edge, n_dict, e_dict) node_list = get_id_flows(n_dict) df = pd.DataFrame(node_list, columns=['node_id', crop_name]) df.to_excel(excel_writer, node_table, index=False) excel_writer.save() edge_list = get_id_flows(e_dict) df = pd.DataFrame(edge_list, columns=['edge_id', crop_name]) df.to_excel(excel_writer, edge_table, index=False) excel_writer.save() if df.empty: add_zeros_columns_to_table_psycopg2( mode_flow_tables[m], [crop_name], ['double precision'], conn) else: df.to_sql('dummy_flows', engine, if_exists='replace', schema='public', index=False) add_columns_to_table_psycopg2(mode_flow_tables[m], 'dummy_flows', [crop_name], ['double precision'], 'edge_id', conn) curs.close() conn.close()
r = requests.get( 'http://hatang.viwa.gov.vn/BanDo/_ChiTietCangBen', params={ 'id': port_id } ) with open(output_path, 'w', encoding='utf-8') as fh: fh.write(r.text) except requests.exceptions.ConnectionError: print("Error", port_id) # also request details output_path = os.path.join(output_dir, 'cangben-{}-detail.html'.format(port_id)) if not os.path.exists(output_path): print(port_id, "details") try: r = requests.get( 'http://cangben.viwa.gov.vn/trangchu/cangben/{}'.format(port_id) ) with open(output_path, 'w', encoding='utf-8') as fh: fh.write(r.text) except requests.exceptions.ConnectionError: print("Error", port_id) if __name__ == '__main__': conf = load_config() PORT_LIST_PATH = os.path.join(conf['paths']['data'], 'Waterways', 'viwa', 'datacangben-all.json') OUTPUT_DIR = os.path.join(conf['paths']['data'], 'Waterways', 'viwa', 'html') main(PORT_LIST_PATH, OUTPUT_DIR)
def main(): curdir = os.getcwd() conf = load_config() try: conn = psycopg2.connect(**conf['database']) except: print("I am unable to connect to the database") cur = conn.cursor() engine = create_engine( 'postgresql://{user}:{password}@{host}:{port}/{database}'.format( {**conf['database']})) shp_file_path = os.path.join(conf['paths']['data'], 'Hazards_data', 'coastal_typhoon_floods') ''' Get the road flooding results ''' layers_ids = [('roads_edge_flood', 'edge_id', 'road2009edges', 'geom'), ('rail_edge_flood', 'edge_id', 'railnetworkedges', 'geom'), ('roads_node_flood', 'node_id', 'road2009nodes', 'geom'), ('rail_node_flood', 'node_id', 'railnetworknodes', 'geom'), ('port_flood', 'node_id', 'seaport_nodes', 'geom'), ('airport_flood', 'node_id', 'airport_nodes', 'geom'), ('provinces_flooding', 'name_eng', 'province_level_stats', 'geom')] flood_list = [] excel_writer = pd.ExcelWriter('vnm_flood_results.xlsx') for l in layers_ids: sql_query = "drop table if exists %s" % (l[0]) cur.execute(sql_query) conn.commit() sql_query = "create table %s as select %s,%s from %s" % (l[0], l[1], l[3], l[2]) cur.execute(sql_query) conn.commit() for file in os.listdir(shp_file_path): if file.endswith(".shp") and 'mask_1' in file: # print (file) # print(os.path.join(shp_file_path, file)) fpath = os.path.join(shp_file_path, file) flds, geom_type, cr, bds, n_f = vector_details(fpath) fname = file.split(".shp") fname = fname[0] flood_list.append(fname) cr_split = cr.split(':') print(cr_split) fr_srid = [c for c in cr_split if c.isdigit() is True] if fr_srid: print(fr_srid) elif 'wgs84' in cr.lower().strip(): fr_srid = '4326' command = "shp2pgsql -I -s {0}:4326 -d {1} {2} | psql -U {3} -d {4}".format( fr_srid, fpath, fname, conf['database']['user'], conf['database']['database']) # sp.call(command.split()) sp.call(command, shell=True) sql_query = "drop table if exists dummy_flood" cur.execute(sql_query) conn.commit() if l[0] in ['roads_edge_flood', 'rail_edge_flood']: sql_query = '''create table dummy_flood as select A.%s, sum(st_length(st_intersection(A.%s::geography,B.geom::geography)))/1000 as %s from %s as A, %s as B where st_intersects(A.%s,B.geom) is True and st_isvalid(A.%s) and st_isvalid(B.geom) group by A.%s''' % (l[1], l[3], fname, l[2], fname, l[3], l[3], l[1]) cl_type = 'double precision' elif l[0] in [ 'roads_node_flood', 'rail_node_flood', 'port_flood', 'airport_flood' ]: sql_query = '''create table dummy_flood as select A.%s, 1 as %s from %s as A, %s as B where st_within(A.%s,B.geom) is True ''' % (l[1], fname, l[2], fname, l[3]) cl_type = 'integer' else: sql_query = '''create table dummy_flood as select A.%s, sum(st_area(st_intersection(A.%s::geography,B.geom::geography)))/1000000 as %s from %s as A, %s as B where st_intersects(A.%s,B.geom) is True and st_isvalid(A.%s) and st_isvalid(B.geom) group by A.%s''' % (l[1], l[3], fname, l[2], fname, l[3], l[3], l[1]) cl_type = 'double precision' cur.execute(sql_query) conn.commit() add_column_to_table(l[0], 'dummy_flood', fname, l[1], cl_type, cur, conn) ''' Write everthing to a pandas dataframe ''' df = pd.read_sql_table(l[0], engine) df.to_excel(excel_writer, l[0], index=False) excel_writer.save() ''' Do some diagnostics ''' cur.close() conn.close()
import csv from pathlib import Path import time import fiona from collections import OrderedDict, defaultdict from rtree import index from shapely.geometry import mapping, shape, Point, LineString, MultiLineString from vtra.utils import load_config # C Incoming data/20170801.zip -> data/raw/cvts/20170801 # D Work Processes/Vietnam/data/Roads/ -> data/raw/cvts/Roads data_root = load_config()['paths']['data'] dir_raw_cvts = os.path.join(data_root, 'Cvts', 'raw', '20170801') dir_raw_roads = os.path.join(data_root, 'Roads', 'national_roads') dir_inter_reduse = os.path.join(data_root, 'Cvts', 'intermediate', 'reduse') dir_inter_clip = os.path.join(data_root, 'Cvts', 'intermediate', 'clip') dir_results_routes = os.path.join(data_root, 'Cvts', 'results', 'routes') dir_results_routes_collected = os.path.join(data_root, 'Cvts', 'results', 'routes_collected') dir_results_traffic_count = os.path.join(data_root, 'Cvts', 'results', 'traffic_count') def reduse_dataset(source_dir, dest_dir): # Reduce the size of the dataset to speed up further processing # and output this in an intermediate mirror dataset
def estimate_losses(input_file): print('{} started!'.format(input_file)) data_path = load_config()['paths']['data'] ''' Set booleans''' if 'min' in input_file: min_rice = True elif 'max' in input_file: min_rice = False if 'single' in input_file: single_point = True elif 'multiple' in input_file: single_point = False ''' Specify file path ''' if min_rice == True: filepath = os.path.join(data_path,'input_data','IO_VIETNAM_MIN.xlsx') else: filepath = os.path.join(data_path,'input_data','IO_VIETNAM_MAX.xlsx') '''Create data input''' DATA = io_basic('Vietnam',filepath,2010) DATA.prep_data() '''Run model and create some output''' output = pd.DataFrame() '''Specify disruption''' output_dir = os.path.join( data_path, 'Results', 'Economic_Failure_Results', os.path.basename(os.path.splitext(input_file)[0]) ) '''Create output folders''' if os.path.exists(output_dir) == False: os.mkdir(output_dir) event_dict = create_disruption(input_file,output_dir,min_rice=min_rice,single_point=single_point) # print (event_dict.keys()) collect_outputs = {} for iter_,event in enumerate(list(event_dict.keys())): if np.average(1 - np.array(list(event_dict[event].values()))) < 0.001: print('Event {} will cause no impacts'.format(event)) continue print('Event {} started!'.format(event)) try: disr_dict_fd = {} disr_dict_sup = event_dict[event] '''Get direct losses ''' disrupt = pd.DataFrame.from_dict(disr_dict_sup,orient='index') disrupt.reset_index(inplace=True) disrupt[['region', 'sector']] = disrupt['index'].apply(pd.Series) '''Create model''' MRIA_RUN = MRIA(DATA.name,DATA.countries,DATA.sectors,EORA=False,list_fd_cats=['FinDem']) '''Define sets and alias''' # CREATE SETS MRIA_RUN.create_sets() # CREATE ALIAS MRIA_RUN.create_alias() ''' Define tables and parameters''' MRIA_RUN.baseline_data(DATA,disr_dict_sup,disr_dict_fd) MRIA_RUN.impact_data(DATA,disr_dict_sup,disr_dict_fd) '''Get base line values''' output['x_in'] = pd.Series(MRIA_RUN.X.get_values())*43 output.index.names = ['region','sector'] '''Get direct losses ''' disrupt = pd.DataFrame.from_dict(disr_dict_sup,orient='index') disrupt.reset_index(inplace=True) disrupt[['region', 'sector']] = disrupt['index'].apply(pd.Series) disrupt.drop('index',axis=1,inplace=True) disrupt = 1- disrupt.groupby(['region', 'sector']).sum() disrupt.columns = ['shock'] output['dir_losses'] = (disrupt['shock']*output['x_in']).fillna(0)*-1 MRIA_RUN.run_impactmodel() output['x_out'] = pd.Series(MRIA_RUN.X.get_values())*43 output['total_losses'] = (output['x_out'] - output['x_in']) output['ind_losses'] = (output['total_losses'] - output['dir_losses']) output = output/365 output = output.drop(['x_in','x_out'],axis=1) output.to_csv(os.path.join(output_dir,'{}.csv'.format(event))) prov_impact = output.groupby(level=0,axis=0).sum() collect_outputs[event] = prov_impact except Exception as e: print('Failed to finish {} because of {}!'.format(event,e)) if collect_outputs: '''Specify disruption''' output_dir = os.path.join(data_path, 'Results', 'Economic_failure_results', 'provincial' ) '''Create output folders''' if os.path.exists(output_dir) == False: os.mkdir(output_dir) pd.concat(collect_outputs).to_csv(os.path.join( data_path, 'Results', 'Economic_failure_results', 'provincial', '{}_provincial.csv'.format(os.path.basename(os.path.splitext(input_file)[0])))) get_sums = {} for event in collect_outputs: get_sums[event] = collect_outputs[event]['total_losses'].sum() sums = pd.DataFrame.from_dict(get_sums,orient='index') sums.columns = ['total_losses'] '''Specify disruption''' output_dir = os.path.join(data_path, 'Results', 'Economic_failure_results', 'summarized' ) '''Create output folders''' if os.path.exists(output_dir) == False: os.mkdir(output_dir) sums.to_csv(os.path.join(data_path, 'Results', 'Economic_failure_results', 'summarized', '{}_summarized.csv'.format(os.path.basename(os.path.splitext(input_file)[0])))) return pd.concat(collect_outputs),sums
def main(): data_path, calc_path, output_path = load_config( )['paths']['data'], load_config()['paths']['calc'], load_config( )['paths']['output'] truck_unit_wt = [5.0, 20.0] # provinces to consider province_list = ['Lao Cai', 'Binh Dinh', 'Thanh Hoa'] province_terrian = ['mountain', 'flat', 'flat'] types = ['min', 'max'] path_types = ['min_edge_path', 'max_edge_path'] rev_types = ['min_netrev', 'max_netrev'] tons_types = ['min_croptons', 'max_croptons'] dist_types = ['min_distance', 'max_distance'] time_types = ['min_time', 'max_time'] cost_types = ['min_gcost', 'max_gcost'] vechicle_types = ['min_vehicle_nums', 'max_vehicle_nums'] flow_paths_data = os.path.join( output_path, 'flow_mapping_paths', 'province_roads_district_center_flow_paths.xlsx') fail_scenarios_data = os.path.join( output_path, 'hazard_scenarios', 'province_roads_hazard_intersections.xlsx') rd_prop_file = os.path.join(data_path, 'Roads', 'road_properties', 'road_properties.xlsx') cols = [ 'origin', 'destination', 'min_edge_path', 'max_edge_path', 'min_netrev', 'max_netrev', 'min_croptons', 'max_croptons', 'min_distance', 'max_distance', 'min_time', 'max_time', 'min_gcost', 'max_gcost' ] selection_criteria = [ 'commune_id', 'hazard_type', 'model', 'climate_scenario', 'year', 'probability' ] filter_cols = ['edge_id', 'exposed_length'] + selection_criteria ''' Path OD flow disruptions ''' for prn in range(len(province_list)): # for prn in range(0,1): province = province_list[prn] # set all paths for all input files we are going to use province_name = province.replace(' ', '').lower() ''' First do multiple edge failures ''' edges_in = os.path.join(data_path, 'Roads', '{}_roads'.format(province_name), 'vietbando_{}_edges.shp'.format(province_name)) # G = province_shapefile_to_network(edges_in,province_terrian[prn],rd_prop_file) G_df = province_shapefile_to_dataframe(edges_in, province_terrian[prn], rd_prop_file) fail_df = pd.read_excel(fail_scenarios_data, sheet_name=province_name) fail_df = fail_df.set_index(selection_criteria) criteria_set = list(set(fail_df.index.values.tolist())) multiple_ef_list = [] for criteria in criteria_set: if len(fail_df.loc[criteria, 'edge_id'].index) == 1: efail = [fail_df.loc[criteria, 'edge_id'].item()] else: efail = list( set(fail_df.loc[criteria, 'edge_id'].values.tolist())) flength = fail_df.loc[criteria, 'length'].sum() criteria_dict = { **(dict(list(zip(selection_criteria, criteria)))), **{ 'exposed_length': flength } } multiple_ef_list.append((efail, criteria_dict)) # print (multiple_ef_list) # print (len(multiple_ef_list)) for tr_wt in truck_unit_wt: flow_df = pd.read_excel(flow_paths_data, sheet_name=province_name + '_{}_tons'.format(int(tr_wt))) edge_fail_ranges = [] for t in range(len(types)): # for t in range(1,2): ef_list = [] e_list = [] ef_sc_list = [] for edge_fail in multiple_ef_list: if edge_fail[0] not in e_list: ef_dict = igraph_scenario_edge_failures( G_df, edge_fail[0], flow_df, tr_wt, path_types[t], rev_types[t], tons_types[t], cost_types[t], dist_types[t], time_types[t], vechicle_types[t]) if ef_dict: ef_list += ef_dict e_list.append(edge_fail[0]) ef_sc_list += [{ **ef, **edge_fail[1] } for ef in ef_dict] elif ef_list: ef_sc_list += [{ **ef, **edge_fail[1] } for ef in ef_list if ef['edge_id'] == str(edge_fail[0])] print('Done with province {0} edge {1} type {2}'.format( province_name, edge_fail[1], types[t])) df = pd.DataFrame(ef_sc_list) df_path = os.path.join( output_path, 'failure_results', 'multiple_edge_failures_all_path_impacts_{0}_{1}_{2}_tons.csv' .format(province_name, types[t], int(tr_wt))) df.to_csv(df_path, index=False) df['econ_loss'] = df['no_access'] * df['econ_value'] + ( 1 - df['no_access']) * df['vehicle_nums'] * ( df['new_cost'] - df['old_cost']) egde_impact = df[filter_cols + ['econ_value', 'tons', 'econ_loss']] egde_impact = egde_impact.groupby(filter_cols)[ 'econ_value', 'tons', 'econ_loss'].sum().reset_index() egde_impact.rename(columns={ 'econ_value': '{}_econ_value'.format(types[t]), 'tons': '{}_tons'.format(types[t]), 'econ_loss': '{}_econ_loss'.format(types[t]) }, inplace=True) edge_fail_ranges.append(egde_impact) egde_impact = edge_fail_ranges[0] egde_impact = pd.merge(egde_impact, edge_fail_ranges[1], how='left', on=filter_cols) df_path = os.path.join( output_path, 'failure_results', 'multiple_edge_failures_totals_{0}_{1}_tons.csv'.format( province_name, int(tr_wt))) egde_impact.to_csv(df_path, index=False)