def network_connections(data, network_data): """ Extract upstream and downstream connections between segments in network Args: data (DataFrame): Network parameter dataset, prepared network_data (dict): network metadata Returns: conn (dict): downstream connections rconn (dict): upstream connections """ # extract downstream connections conn = nhd_network.extract_connections( data, network_data["columns"]["downstream"]) # extract upstream connections rconn = nhd_network.reverse_network(conn) return conn, rconn
def qlat_destination_compute( data_native, data_merged, merged_segments, pruned_segments, network_data ): # build a list of all segments that need crosswalking if bool(list(pruned_segments)): segments = merged_segments + list(pruned_segments) else: segments = merged_segments # compute connections using native network data conn = nhd_network.extract_connections( data_native, network_data["columns"]["downstream"] ) rconn = nhd_network.reverse_network(conn) # initialize a dictionary to store qlat destination nodes for pruned/merged segments qlat_destinations = {} for idx in segments: # find the segment to recieve qlats from the pruned or merged segment if conn[idx]: ds_idx = conn[idx] while bool(ds_idx[0] in data_merged.index) == False: ds_idx = conn[ds_idx[0]] elif rconn[idx]: ds_idx = rconn[idx] while bool(ds_idx[0] in data_merged.index) == False: us_idx = conn[ds_idx[0]] else: ds_idx = [] # update the qlat destination dict qlat_destinations[str(idx)] = str(ds_idx) return qlat_destinations
def main(): args = _handle_args() nts = 144 debuglevel = -1 * args.debuglevel verbose = args.verbose showtiming = args.showtiming supernetwork = args.supernetwork break_network_at_waterbodies = args.break_network_at_waterbodies write_output = args.write_output assume_short_ts = args.assume_short_ts test_folder = pathlib.Path(root, "test") geo_input_folder = test_folder.joinpath("input", "geo") # TODO: Make these commandline args """##NHD Subset (Brazos/Lower Colorado)""" # supernetwork = 'Brazos_LowerColorado_Named_Streams' # supernetwork = 'Brazos_LowerColorado_ge5' # supernetwork = 'Pocono_TEST1' """##NHD CONUS order 5 and greater""" # supernetwork = 'CONUS_ge5' """These are large -- be careful""" # supernetwork = 'Mainstems_CONUS' # supernetwork = 'CONUS_FULL_RES_v20' # supernetwork = 'CONUS_Named_Streams' #create a subset of the full resolution by reading the GNIS field # supernetwork = 'CONUS_Named_combined' #process the Named streams through the Full-Res paths to join the many hanging reaches if verbose: print("creating supernetwork connections set") if showtiming: start_time = time.time() # STEP 1 network_data = nnu.set_supernetwork_data( supernetwork=args.supernetwork, geo_input_folder=geo_input_folder, verbose=False, debuglevel=debuglevel, ) cols = network_data["columns"] data = nhd_io.read(network_data["geo_file_path"]) data = data[list(cols.values())] data = data.set_index(cols["key"]) if "mask_file_path" in network_data: data_mask = nhd_io.read_mask( network_data["mask_file_path"], layer_string=network_data["mask_layer_string"], ) data = data.filter(data_mask.iloc[:, network_data["mask_key"]], axis=0) data = data.sort_index() data = nhd_io.replace_downstreams(data, cols["downstream"], 0) if args.ql: qlats = nhd_io.read_qlat(args.ql) else: qlats = constant_qlats(data, nts, 10.0) connections = nhd_network.extract_connections(data, cols["downstream"]) wbodies = nhd_network.extract_waterbodies( data, cols["waterbody"], network_data["waterbody_null_code"] ) if verbose: print("supernetwork connections set complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) # STEP 2 if showtiming: start_time = time.time() if verbose: print("organizing connections into reaches ...") rconn = nhd_network.reverse_network(connections) subnets = nhd_network.reachable_network(rconn) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) subreaches[tw] = nhd_network.dfs_decomposition(net, path_func) if verbose: print("reach organization complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) if showtiming: start_time = time.time() data["dt"] = 300.0 data = data.rename(columns=nnu.reverse_dict(cols)) data = data.astype("float32") # datasub = data[['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']] parallelcompute = False if parallelcompute: with Parallel(n_jobs=-1, backend="threading") as parallel: jobs = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = data.loc[ r, ["dt", "bw", "tw", "twcc", "dx", "n", "ncc", "cs", "s0"] ].sort_index() qlat_sub = qlats.loc[r].sort_index() jobs.append( delayed(mc_reach.compute_network)( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values, ) ) results = parallel(jobs) else: results = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = data.loc[ r, ["dt", "bw", "tw", "twcc", "dx", "n", "ncc", "cs", "s0"] ].sort_index() qlat_sub = qlats.loc[r].sort_index() results.append( mc_reach.compute_network( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values, ) ) fdv_columns = pd.MultiIndex.from_product( [range(nts), ["q", "v", "d"]] ).to_flat_index() flowveldepth = pd.concat( [pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in results], copy=False ) flowveldepth = flowveldepth.sort_index() flowveldepth.to_csv(f"{args.supernetwork}.csv") print(flowveldepth) if verbose: print("ordered reach computation complete") if showtiming: print("... in %s seconds." % (time.time() - start_time))
def main(): global connections global networks global flowdepthvel verbose = True debuglevel = 0 showtiming = True test_folder = os.path.join(root, r"test") geo_input_folder = os.path.join(test_folder, r"input", r"geo") # TODO: Make these commandline args """##NHD Subset (Brazos/Lower Colorado)""" # supernetwork = 'Brazos_LowerColorado_ge5' supernetwork = "Pocono_TEST1" """##NHD CONUS order 5 and greater""" # supernetwork = 'CONUS_ge5' """These are large -- be careful""" # supernetwork = 'Mainstems_CONUS' # supernetwork = 'CONUS_FULL_RES_v20' # supernetwork = 'CONUS_Named_Streams' #create a subset of the full resolution by reading the GNIS field # supernetwork = 'CONUS_Named_combined' #process the Named streams through the Full-Res paths to join the many hanging reaches if verbose: print("creating supernetwork connections set") if showtiming: start_time = time.time() # STEP 1 supernetwork_data, supernetwork_values = nnu.set_networks( supernetwork=supernetwork, geo_input_folder=geo_input_folder, verbose=False # , verbose = verbose , debuglevel=debuglevel, ) if verbose: print("supernetwork connections set complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) # STEP 2 if showtiming: start_time = time.time() if verbose: print("organizing connections into reaches ...") networks = nru.compose_networks( supernetwork_values, verbose=False # , verbose = verbose , debuglevel=debuglevel, showtiming=showtiming, ) if verbose: print("reach organization complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) if showtiming: start_time = time.time() connections = supernetwork_values[0] connections, conn_data = separate_data( connections, { "key": supernetwork_data["key_col"], "length": supernetwork_data["length_col"], "bottomwidth": supernetwork_data["bottomwidth_col"], "topwidth": supernetwork_data["topwidth_col"], "manningn": supernetwork_data["manningn_col"], "ChSlp": supernetwork_data["ChSlp_col"], "slope": supernetwork_data["slope_col"], "topwidthcc": supernetwork_data["topwidthcc_col"], "manningncc": supernetwork_data["manningncc_col"], }, ) conn_data = conn_data[conn_data[:, 0].argsort()] # Index Map: # flow_prev, depth_prev, vel_prev, qlat_prev # flow_curr, depth_curr, vel_curr, qlat_curr flowdepthvel = np.zeros((len(connections), 8)) RN = dict(nhd_network.reverse_network(connections)) for ts in range(1440): for n in nhd_network.kahn_toposort(connections): process_edge(n, RN, flowdepthvel, conn_data) with np.printoptions(precision=5, suppress=True, linewidth=120): print(flowdepthvel) sorted_conns = sorted(connections.keys()) print(sorted_conns, all(conn_data[:, 0] == sorted_conns)) # parallelcompute = False # if not parallelcompute: # if verbose: # print("executing computation on ordered reaches ...") # # for terminal_segment, network in networks.items(): # compute_network( # network, # conn_data, # supernetwork_data, # connections, # flowdepthvel, # verbose=False, # debuglevel=debuglevel, # ) # print(f"{terminal_segment}") # if showtiming: # print("... in %s seconds." % (time.time() - start_time)) # # else: # if verbose: # print(f"executing parallel computation on ordered reaches .... ") # # for terminal_segment, network in networks.items(): # # print(terminal_segment, network) # # print(tuple(([x for x in networks.keys()][i], [x for x in networks.values()][i]) for i in range(len(networks)))) # nslist = ( # [ # network, # conn_data, # TODO: This should probably be global... # connections, # flowdepthvel, # False, # debuglevel, # ] # for terminal_segment, network in networks.items() # ) # with multiprocessing.Pool() as pool: # results = pool.starmap(compute_network, nslist) if verbose: print("ordered reach computation complete") if showtiming: print("... in %s seconds." % (time.time() - start_time))
def main(): global data_values verbose = True debuglevel = 0 showtiming = True nts = 1440 test_folder = os.path.join(root, r"test") geo_input_folder = os.path.join(test_folder, r"input", r"geo") # TODO: Make these commandline args """##NHD Subset (Brazos/Lower Colorado)""" # supernetwork = 'Brazos_LowerColorado_ge5' #supernetwork = "Pocono_TEST2" """##NHD CONUS order 5 and greater""" # supernetwork = 'CONUS_ge5' """These are large -- be careful""" supernetwork = 'Mainstems_CONUS' # supernetwork = 'CONUS_FULL_RES_v20' # supernetwork = 'CONUS_Named_Streams' #create a subset of the full resolution by reading the GNIS field # supernetwork = 'CONUS_Named_combined' #process the Named streams through the Full-Res paths to join the many hanging reaches if verbose: print("creating supernetwork connections set") if showtiming: start_time = time.time() # STEP 1 network_data = nnu.set_supernetwork_data( supernetwork=supernetwork, geo_input_folder=geo_input_folder ) cols = [v for c, v in network_data.items() if c.endswith("_col")] data = nhd_io.read(network_data["geo_file_path"]) data = data[cols] data = data.set_index(network_data["key_col"]) if "mask_file_path" in network_data: data_mask = nhd_io.read_mask( network_data["mask_file_path"], layer_string=network_data["mask_layer_string"], ) data = data.filter(data_mask.iloc[:, network_data["mask_key"]], axis=0) data = data.sort_index() data = replace_downstreams(data, network_data['downstream_col'], 0) if supernetwork == "Pocono_TEST2": qlats = pd.read_csv('../../test/input/geo/PoconoSampleData2/Pocono_ql_testsamp1_nwm_mc.txt', index_col='ntt') qlats = qlats.drop(columns=['nt']) qlats.columns = qlats.columns.astype(int) qlats = qlats.sort_index(axis='columns').sort_index(axis='index') qlats = qlats.drop(columns=qlats.columns.difference(data.index)).T qlats = qlats.astype('float32') else: qlats = constant_qlats(data, nts, 10.0) connections = nhd_network.extract_connections(data, network_data["downstream_col"]) rconn = nhd_network.reverse_network(connections) # rconn_annoated = translate_network_to_index(rconn, data.index) if verbose: print("supernetwork connections set complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) # STEP 2 if showtiming: start_time = time.time() if verbose: print("organizing connections into reaches ...") subnets = nhd_network.reachable_network(rconn) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) reach = nhd_network.dfs_decomposition( nhd_network.reverse_network(net), path_func ) subreaches[tw] = reach if verbose: print("reach organization complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) if showtiming: start_time = time.time() # flowdepthvel = {node: {'flow': {'prev': 0, 'curr': 0} # , 'depth': {'prev': 0, 'curr': 0} # , 'vel': {'prev': 0, 'curr': 0} # , 'qlat': {'prev': 0, 'curr': 0}} for node in nhd_network.nodes(connections)} parallelcompute = False # Data column ordering is very important as we directly lookup values. # The column order *must* be: # 0: bw, 1: tw, 2: twcc, 3: dx, 4: n_manning 5: n_manning_cc, 6: cs, 7: s0, 8: qlat data['dt'] = 300.0 data = data.rename(columns={'Length': 'dx', 'TopWdth': 'tw', 'TopWdthCC': 'twcc', 'BtmWdth': 'bw', 'nCC': 'ncc', 'So': 's0', 'ChSlp': 'cs'}) data = data.astype('float32') #datasub = data[['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']] #qlats = qlats.loc[:, :nts] compute_start = time.time() if parallelcompute: if verbose: print("executing computation on ordered reaches ...") with Parallel( n_jobs=-1, pre_dispatch="all", backend="threading", verbose=5 ) as parallel: jobs = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) #assert r[-1] == tw # always be True #assert len(data.index.intersection(r)) == len(r) data_sub = data.loc[r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']].sort_index() qlat_sub = qlats.loc[r].sort_index() jobs.append( delayed(mc_reach.compute_network)( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values ) ) random.shuffle(jobs) rets = parallel(jobs) #for findex, fdv in rets: # flowdepthvel[findex] = fdv else: rets = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) #assert r[-1] == tw # always be True #assert len(data.index.intersection(r)) == len(r) data_sub = data.loc[r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']].sort_index() #TODO: Do the data_sub = data.loc as a preprocessing step, then dump the pointer to data qlat_sub = qlats.loc[r].sort_index() rets.append( mc_reach.compute_network( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values)) # TODO: rets could be dumped to files #findex, fdv = mc_reach.compute_network(ts, reach, subnets[tw], data_idx, data_values, qlat_values) #flowdepthvel[findex] = fdv if verbose: print( f"tailwater: {tw} completed", end = "", ) # NOTE: Mississippi River tailwater is {22811611,}: if showtiming: print( f"... in {time.time()-compute_start} seconds ({twi}/{len(subreaches)})" ) print("Computation time: ", time.time() - compute_start) fdv_columns = pd.MultiIndex.from_product([range(nts), ['q', 'v', 'd']], names=['timestep', 'qvd']) flowveldepth = pd.concat([pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in rets]) print(flowveldepth)
def main(): args = _handle_args() #The following 2 values are currently hard coded for this test domain nts = 720 # number of timestep = 1140 * 60(model timestep) = 86400 = day dt_mc = 300.0 # time interval for MC #Currently tested on the Sugar Creek domain ngen_network_df = nhd_io.read_geopandas( os.path.join(next_gen_input_folder, args.supernetwork)) #Create dictionary mapping each connection ID ngen_network_dict = dict(zip(ngen_network_df.ID, ngen_network_df.toID)) def node_key_func(x): return int(x[4:]) #Extract the ID integer values waterbody_connections = { node_key_func(k): node_key_func(v) for k, v in ngen_network_dict.items() } #Convert dictionary connections to data frame and make ID column the index waterbody_df = pd.DataFrame.from_dict(waterbody_connections, orient='index', columns=['to']) #Sort ID index column waterbody_df = waterbody_df.sort_index() waterbody_df = nhd_io.replace_downstreams(waterbody_df, "to", 0) connections = nhd_network.extract_connections(waterbody_df, "to") #Read and convert catchment lateral flows to format that can be processed by compute_network qlats = next_gen_io.read_catchment_lateral_flows(next_gen_input_folder) rconn = nhd_network.reverse_network(connections) subnets = nhd_network.reachable_network(rconn, check_disjoint=False) waterbody_df['dt'] = 300.0 #Setting all below to 1.0 until we can get the appropriate parameters waterbody_df['bw'] = 1.0 waterbody_df['tw'] = 1.0 waterbody_df['twcc'] = 1.0 waterbody_df['dx'] = 1.0 waterbody_df['n'] = 1.0 waterbody_df['ncc'] = 1.0 waterbody_df['cs'] = 1.0 waterbody_df['s0'] = 1.0 #Set types as float32 waterbody_df = waterbody_df.astype({ "dt": "float32", "bw": "float32", "tw": "float32", "twcc": "float32", "dx": "float32", "n": "float32", "ncc": "float32", "cs": "float32", "s0": "float32" }) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) subreaches[tw] = nhd_network.dfs_decomposition(net, path_func) results = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = waterbody_df.loc[ r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0' ]].sort_index() qlat_sub = qlats.loc[r].sort_index() results.append( mc_reach.compute_network(nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values)) fdv_columns = pd.MultiIndex.from_product([range(nts), ['q', 'v', 'd']]).to_flat_index() flowveldepth = pd.concat( [pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in results], copy=False) flowveldepth = flowveldepth.sort_index() outfile_base_name = (args.supernetwork).split(".")[0] flowveldepth.to_csv(f"{outfile_base_name}_mc_results.csv") print(flowveldepth)