def main(): args = _handle_args() next_gen_input_folder = test_folder.joinpath("input", "next_gen") if args.input: next_gen_input_folder = pathlib.Path(args.input) # The following 2 values are currently hard coded for this test domain nts = 720 # number of timestep = 1140 * 60(model timestep) = 86400 = day dt_mc = 300.0 # time interval for MC # Currently tested on the Sugar Creek domain ngen_network_df = nhd_io.read_geopandas(args.supernetwork) if args.subset: ngen_network_df = ngen_network_df[ ngen_network_df['realized_catchment'].isin(args.subset)] # Create dictionary mapping each connection ID ngen_network_dict = dict(zip(ngen_network_df.id, ngen_network_df.toid)) #ngen_network_dict = dict(zip(ngen_network_df.ID, ngen_network_df.toID)) def node_key_func(x): return int(x[3:]) # Extract the ID integer values waterbody_connections = { node_key_func(k): node_key_func(v) for k, v in ngen_network_dict.items() } # Convert dictionary connections to data frame and make ID column the index waterbody_df = pd.DataFrame.from_dict(waterbody_connections, orient='index', columns=['to']) # Sort ID index column waterbody_df = waterbody_df.sort_index() waterbody_df = nhd_io.replace_downstreams(waterbody_df, "to", 0) connections = nhd_network.extract_connections(waterbody_df, "to") # Read and convert catchment lateral flows to format that can be processed by compute_network qlats = next_gen_io.read_catchment_lateral_flows(next_gen_input_folder) print(qlats) rconn = nhd_network.reverse_network(connections) subnets = nhd_network.reachable_network(rconn, check_disjoint=False) # read the routelink file nhd_routelink = nhd_io.read_netcdf("data/RouteLink_NHDPLUS.nc") nhd_routelink['dt'] = 300.0 nhd_routelink.set_index("link", inplace=True) routelink_cols = { "downstream": "to", "dx": "Length", "n": "n", "ncc": "nCC", "s0": "So", "bw": "BtmWdth", "tw": "TopWdth", "twcc": "TopWdthCC", "waterbody": "NHDWaterbodyComID", "musk": "MusK", "musx": "MusX", "cs": "ChSlp", } routelink_cols = dict([(value, key) for key, value in routelink_cols.items()]) nhd_routelink.rename(columns=routelink_cols, inplace=True) with open(next_gen_input_folder / 'coarse/crosswalk.json') as f: crosswalk_data = json.load(f) waterbody_df['comid'] = waterbody_df.apply( lambda x: crosswalk_data['cat-' + str(x.name)]['outlet_COMID'], axis=1) waterbody_df = waterbody_df.join(nhd_routelink, on='comid', how='left') del nhd_routelink # initial conditions, assume to be zero # TO DO: Allow optional reading of initial conditions from WRF q0 = pd.DataFrame(0, index=waterbody_df.index, columns=["qu0", "qd0", "h0"], dtype="float32") #Set types as float32 waterbody_df = waterbody_df.astype({ "dt": "float32", "bw": "float32", "tw": "float32", "twcc": "float32", "dx": "float32", "n": "float32", "ncc": "float32", "cs": "float32", "s0": "float32" }) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) subreaches[tw] = nhd_network.dfs_decomposition(net, path_func) results = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = waterbody_df.loc[ r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0' ]].sort_index() #data_sub = waterbody_df.loc[r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']] qlat_sub = qlats.loc[r].sort_index() q0_sub = q0.loc[r].sort_index() results.append( mc_reach.compute_network(nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values, q0_sub.values)) fdv_columns = pd.MultiIndex.from_product([range(nts), ['q', 'v', 'd']]).to_flat_index() flowveldepth = pd.concat( [pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in results], copy=False) flowveldepth = flowveldepth.sort_index() outfile_base_name = (args.supernetwork).split(".")[0] flowveldepth.to_csv(f"{outfile_base_name}_mc_results.csv") print(flowveldepth)
def main(): global data_values verbose = True debuglevel = 0 showtiming = True nts = 1440 test_folder = os.path.join(root, r"test") geo_input_folder = os.path.join(test_folder, r"input", r"geo") # TODO: Make these commandline args """##NHD Subset (Brazos/Lower Colorado)""" # supernetwork = 'Brazos_LowerColorado_ge5' #supernetwork = "Pocono_TEST2" """##NHD CONUS order 5 and greater""" # supernetwork = 'CONUS_ge5' """These are large -- be careful""" supernetwork = 'Mainstems_CONUS' # supernetwork = 'CONUS_FULL_RES_v20' # supernetwork = 'CONUS_Named_Streams' #create a subset of the full resolution by reading the GNIS field # supernetwork = 'CONUS_Named_combined' #process the Named streams through the Full-Res paths to join the many hanging reaches if verbose: print("creating supernetwork connections set") if showtiming: start_time = time.time() # STEP 1 network_data = nnu.set_supernetwork_data( supernetwork=supernetwork, geo_input_folder=geo_input_folder ) cols = [v for c, v in network_data.items() if c.endswith("_col")] data = nhd_io.read(network_data["geo_file_path"]) data = data[cols] data = data.set_index(network_data["key_col"]) if "mask_file_path" in network_data: data_mask = nhd_io.read_mask( network_data["mask_file_path"], layer_string=network_data["mask_layer_string"], ) data = data.filter(data_mask.iloc[:, network_data["mask_key"]], axis=0) data = data.sort_index() data = replace_downstreams(data, network_data['downstream_col'], 0) if supernetwork == "Pocono_TEST2": qlats = pd.read_csv('../../test/input/geo/PoconoSampleData2/Pocono_ql_testsamp1_nwm_mc.txt', index_col='ntt') qlats = qlats.drop(columns=['nt']) qlats.columns = qlats.columns.astype(int) qlats = qlats.sort_index(axis='columns').sort_index(axis='index') qlats = qlats.drop(columns=qlats.columns.difference(data.index)).T qlats = qlats.astype('float32') else: qlats = constant_qlats(data, nts, 10.0) connections = nhd_network.extract_connections(data, network_data["downstream_col"]) rconn = nhd_network.reverse_network(connections) # rconn_annoated = translate_network_to_index(rconn, data.index) if verbose: print("supernetwork connections set complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) # STEP 2 if showtiming: start_time = time.time() if verbose: print("organizing connections into reaches ...") subnets = nhd_network.reachable_network(rconn) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) reach = nhd_network.dfs_decomposition( nhd_network.reverse_network(net), path_func ) subreaches[tw] = reach if verbose: print("reach organization complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) if showtiming: start_time = time.time() # flowdepthvel = {node: {'flow': {'prev': 0, 'curr': 0} # , 'depth': {'prev': 0, 'curr': 0} # , 'vel': {'prev': 0, 'curr': 0} # , 'qlat': {'prev': 0, 'curr': 0}} for node in nhd_network.nodes(connections)} parallelcompute = False # Data column ordering is very important as we directly lookup values. # The column order *must* be: # 0: bw, 1: tw, 2: twcc, 3: dx, 4: n_manning 5: n_manning_cc, 6: cs, 7: s0, 8: qlat data['dt'] = 300.0 data = data.rename(columns={'Length': 'dx', 'TopWdth': 'tw', 'TopWdthCC': 'twcc', 'BtmWdth': 'bw', 'nCC': 'ncc', 'So': 's0', 'ChSlp': 'cs'}) data = data.astype('float32') #datasub = data[['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']] #qlats = qlats.loc[:, :nts] compute_start = time.time() if parallelcompute: if verbose: print("executing computation on ordered reaches ...") with Parallel( n_jobs=-1, pre_dispatch="all", backend="threading", verbose=5 ) as parallel: jobs = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) #assert r[-1] == tw # always be True #assert len(data.index.intersection(r)) == len(r) data_sub = data.loc[r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']].sort_index() qlat_sub = qlats.loc[r].sort_index() jobs.append( delayed(mc_reach.compute_network)( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values ) ) random.shuffle(jobs) rets = parallel(jobs) #for findex, fdv in rets: # flowdepthvel[findex] = fdv else: rets = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) #assert r[-1] == tw # always be True #assert len(data.index.intersection(r)) == len(r) data_sub = data.loc[r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']].sort_index() #TODO: Do the data_sub = data.loc as a preprocessing step, then dump the pointer to data qlat_sub = qlats.loc[r].sort_index() rets.append( mc_reach.compute_network( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values)) # TODO: rets could be dumped to files #findex, fdv = mc_reach.compute_network(ts, reach, subnets[tw], data_idx, data_values, qlat_values) #flowdepthvel[findex] = fdv if verbose: print( f"tailwater: {tw} completed", end = "", ) # NOTE: Mississippi River tailwater is {22811611,}: if showtiming: print( f"... in {time.time()-compute_start} seconds ({twi}/{len(subreaches)})" ) print("Computation time: ", time.time() - compute_start) fdv_columns = pd.MultiIndex.from_product([range(nts), ['q', 'v', 'd']], names=['timestep', 'qvd']) flowveldepth = pd.concat([pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in rets]) print(flowveldepth)
def main(): args = _handle_args() nts = 144 debuglevel = -1 * args.debuglevel verbose = args.verbose showtiming = args.showtiming supernetwork = args.supernetwork break_network_at_waterbodies = args.break_network_at_waterbodies write_output = args.write_output assume_short_ts = args.assume_short_ts test_folder = pathlib.Path(root, "test") geo_input_folder = test_folder.joinpath("input", "geo") # TODO: Make these commandline args """##NHD Subset (Brazos/Lower Colorado)""" # supernetwork = 'Brazos_LowerColorado_Named_Streams' # supernetwork = 'Brazos_LowerColorado_ge5' # supernetwork = 'Pocono_TEST1' """##NHD CONUS order 5 and greater""" # supernetwork = 'CONUS_ge5' """These are large -- be careful""" # supernetwork = 'Mainstems_CONUS' # supernetwork = 'CONUS_FULL_RES_v20' # supernetwork = 'CONUS_Named_Streams' #create a subset of the full resolution by reading the GNIS field # supernetwork = 'CONUS_Named_combined' #process the Named streams through the Full-Res paths to join the many hanging reaches if verbose: print("creating supernetwork connections set") if showtiming: start_time = time.time() # STEP 1 network_data = nnu.set_supernetwork_data( supernetwork=args.supernetwork, geo_input_folder=geo_input_folder, verbose=False, debuglevel=debuglevel, ) cols = network_data["columns"] data = nhd_io.read(network_data["geo_file_path"]) data = data[list(cols.values())] data = data.set_index(cols["key"]) if "mask_file_path" in network_data: data_mask = nhd_io.read_mask( network_data["mask_file_path"], layer_string=network_data["mask_layer_string"], ) data = data.filter(data_mask.iloc[:, network_data["mask_key"]], axis=0) data = data.sort_index() data = nhd_io.replace_downstreams(data, cols["downstream"], 0) if args.ql: qlats = nhd_io.read_qlat(args.ql) else: qlats = constant_qlats(data, nts, 10.0) connections = nhd_network.extract_connections(data, cols["downstream"]) wbodies = nhd_network.extract_waterbodies( data, cols["waterbody"], network_data["waterbody_null_code"] ) if verbose: print("supernetwork connections set complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) # STEP 2 if showtiming: start_time = time.time() if verbose: print("organizing connections into reaches ...") rconn = nhd_network.reverse_network(connections) subnets = nhd_network.reachable_network(rconn) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) subreaches[tw] = nhd_network.dfs_decomposition(net, path_func) if verbose: print("reach organization complete") if showtiming: print("... in %s seconds." % (time.time() - start_time)) if showtiming: start_time = time.time() data["dt"] = 300.0 data = data.rename(columns=nnu.reverse_dict(cols)) data = data.astype("float32") # datasub = data[['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0']] parallelcompute = False if parallelcompute: with Parallel(n_jobs=-1, backend="threading") as parallel: jobs = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = data.loc[ r, ["dt", "bw", "tw", "twcc", "dx", "n", "ncc", "cs", "s0"] ].sort_index() qlat_sub = qlats.loc[r].sort_index() jobs.append( delayed(mc_reach.compute_network)( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values, ) ) results = parallel(jobs) else: results = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = data.loc[ r, ["dt", "bw", "tw", "twcc", "dx", "n", "ncc", "cs", "s0"] ].sort_index() qlat_sub = qlats.loc[r].sort_index() results.append( mc_reach.compute_network( nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values, ) ) fdv_columns = pd.MultiIndex.from_product( [range(nts), ["q", "v", "d"]] ).to_flat_index() flowveldepth = pd.concat( [pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in results], copy=False ) flowveldepth = flowveldepth.sort_index() flowveldepth.to_csv(f"{args.supernetwork}.csv") print(flowveldepth) if verbose: print("ordered reach computation complete") if showtiming: print("... in %s seconds." % (time.time() - start_time))
def main(): args = _handle_args() #The following 2 values are currently hard coded for this test domain nts = 720 # number of timestep = 1140 * 60(model timestep) = 86400 = day dt_mc = 300.0 # time interval for MC #Currently tested on the Sugar Creek domain ngen_network_df = nhd_io.read_geopandas( os.path.join(next_gen_input_folder, args.supernetwork)) #Create dictionary mapping each connection ID ngen_network_dict = dict(zip(ngen_network_df.ID, ngen_network_df.toID)) def node_key_func(x): return int(x[4:]) #Extract the ID integer values waterbody_connections = { node_key_func(k): node_key_func(v) for k, v in ngen_network_dict.items() } #Convert dictionary connections to data frame and make ID column the index waterbody_df = pd.DataFrame.from_dict(waterbody_connections, orient='index', columns=['to']) #Sort ID index column waterbody_df = waterbody_df.sort_index() waterbody_df = nhd_io.replace_downstreams(waterbody_df, "to", 0) connections = nhd_network.extract_connections(waterbody_df, "to") #Read and convert catchment lateral flows to format that can be processed by compute_network qlats = next_gen_io.read_catchment_lateral_flows(next_gen_input_folder) rconn = nhd_network.reverse_network(connections) subnets = nhd_network.reachable_network(rconn, check_disjoint=False) waterbody_df['dt'] = 300.0 #Setting all below to 1.0 until we can get the appropriate parameters waterbody_df['bw'] = 1.0 waterbody_df['tw'] = 1.0 waterbody_df['twcc'] = 1.0 waterbody_df['dx'] = 1.0 waterbody_df['n'] = 1.0 waterbody_df['ncc'] = 1.0 waterbody_df['cs'] = 1.0 waterbody_df['s0'] = 1.0 #Set types as float32 waterbody_df = waterbody_df.astype({ "dt": "float32", "bw": "float32", "tw": "float32", "twcc": "float32", "dx": "float32", "n": "float32", "ncc": "float32", "cs": "float32", "s0": "float32" }) subreaches = {} for tw, net in subnets.items(): path_func = partial(nhd_network.split_at_junction, net) subreaches[tw] = nhd_network.dfs_decomposition(net, path_func) results = [] for twi, (tw, reach) in enumerate(subreaches.items(), 1): r = list(chain.from_iterable(reach)) data_sub = waterbody_df.loc[ r, ['dt', 'bw', 'tw', 'twcc', 'dx', 'n', 'ncc', 'cs', 's0' ]].sort_index() qlat_sub = qlats.loc[r].sort_index() results.append( mc_reach.compute_network(nts, reach, subnets[tw], data_sub.index.values, data_sub.columns.values, data_sub.values, qlat_sub.values)) fdv_columns = pd.MultiIndex.from_product([range(nts), ['q', 'v', 'd']]).to_flat_index() flowveldepth = pd.concat( [pd.DataFrame(d, index=i, columns=fdv_columns) for i, d in results], copy=False) flowveldepth = flowveldepth.sort_index() outfile_base_name = (args.supernetwork).split(".")[0] flowveldepth.to_csv(f"{outfile_base_name}_mc_results.csv") print(flowveldepth)