def original_delineation(cfg): """Original Delineation by calling TauDEM functions""" # Check directories UtilClass.mkdir(cfg.workspace) UtilClass.mkdir(cfg.dirs.log) bin_dir = cfg.seims_bin mpi_bin = cfg.mpi_bin np = cfg.np TauDEMWorkflow.watershed_delineation(bin_dir, mpi_bin, np, cfg.dem, cfg.outlet_file, cfg.d8acc_threshold, cfg.d8down_method, cfg.taudems, cfg.logs.delineation)
def mask_origin_delineated_data(cfg): """Mask the original delineated data by Subbasin raster.""" subbasin_tau_file = cfg.taudems.subbsn geodata2dbdir = cfg.dirs.geodata2db UtilClass.mkdir(geodata2dbdir) mask_file = cfg.spatials.mask RasterUtilClass.get_mask_from_raster(subbasin_tau_file, mask_file) # Total 12 raster files original_files = [ cfg.taudems.subbsn, cfg.taudems.d8flow, cfg.taudems.stream_raster, cfg.taudems.slp, cfg.taudems.filldem, cfg.taudems.d8acc, cfg.taudems.stream_order, cfg.taudems.dinf, cfg.taudems.dinf_d8dir, cfg.taudems.dinf_slp, cfg.taudems.dinf_weight, cfg.taudems.dist2stream_d8 ] # output masked files output_files = [ cfg.taudems.subbsn_m, cfg.taudems.d8flow_m, cfg.taudems.stream_m, cfg.spatials.slope, cfg.spatials.filldem, cfg.spatials.d8acc, cfg.spatials.stream_order, cfg.spatials.dinf, cfg.spatials.dinf_d8dir, cfg.spatials.dinf_slp, cfg.spatials.dinf_weight, cfg.spatials.dist2stream_d8 ] default_values = [] for i in range(len(original_files)): default_values.append(DEFAULT_NODATA) # other input rasters need to be masked # soil and landuse FileClass.check_file_exists(cfg.soil) FileClass.check_file_exists(cfg.landuse) original_files.append(cfg.soil) output_files.append(cfg.spatials.soil_type) default_values.append(cfg.default_soil) original_files.append(cfg.landuse) output_files.append(cfg.spatials.landuse) default_values.append(cfg.default_landuse) # management fields if cfg.mgt_field is not None and FileClass.is_file_exists( cfg.mgt_field): original_files.append(cfg.mgt_field) output_files.append(cfg.spatials.mgt_field) default_values.append(DEFAULT_NODATA) config_file = cfg.logs.mask_cfg # run mask operation print("Mask original delineated data by Subbasin raster...") SpatialDelineation.mask_raster_cpp(cfg.seims_bin, mask_file, original_files, output_files, default_values, config_file)
def post_process_of_delineated_data(cfg): """Do some necessary transfer for subbasin, stream, and flow direction raster.""" # inputs stream_net_file = cfg.taudems.streamnet_shp subbasin_file = cfg.taudems.subbsn_m flow_dir_file_tau = cfg.taudems.d8flow_m stream_raster_file = cfg.taudems.stream_m # outputs # -- shapefile shp_dir = cfg.dirs.geoshp UtilClass.mkdir(shp_dir) # ---- outlet, copy from DirNameUtils.TauDEM FileClass.copy_files(cfg.taudems.outlet_m, cfg.vecs.outlet) # ---- reaches output_reach_file = cfg.vecs.reach # ---- subbasins subbasin_vector_file = cfg.vecs.subbsn # -- raster file output_subbasin_file = cfg.spatials.subbsn output_flow_dir_file = cfg.spatials.d8flow output_stream_link_file = cfg.spatials.stream_link output_hillslope_file = cfg.spatials.hillslope id_map = StreamnetUtil.serialize_streamnet(stream_net_file, output_reach_file) RasterUtilClass.raster_reclassify(subbasin_file, id_map, output_subbasin_file, GDT_Int32) StreamnetUtil.assign_stream_id_raster(stream_raster_file, output_subbasin_file, output_stream_link_file) # Convert D8 encoding rule to ArcGIS if cfg.is_TauDEM: shutil_copy(flow_dir_file_tau, output_flow_dir_file) else: D8Util.convert_code(flow_dir_file_tau, output_flow_dir_file) # convert raster to shapefile (for subbasin and basin) print "Generating subbasin vector..." VectorUtilClass.raster2shp(output_subbasin_file, subbasin_vector_file, "subbasin", FieldNames.subbasin_id) mask_file = cfg.spatials.mask basin_vector = cfg.vecs.bsn print "Generating basin vector..." VectorUtilClass.raster2shp(mask_file, basin_vector, "basin", FieldNames.basin) # delineate hillslope DelineateHillslope.downstream_method_whitebox(output_stream_link_file, flow_dir_file_tau, output_hillslope_file)
def spatial_rasters(cfg, subbasin_num): """Import spatial raster data.""" UtilClass.mkdir(cfg.dirs.import2db) if not cfg.cluster: # changed by LJ, SubbasinID is 0 means the whole basin! subbasin_num = 0 start_id = 0 subbasin_file = cfg.spatials.mask else: start_id = 1 subbasin_file = cfg.spatials.subbsn for i in range(start_id, subbasin_num + 1): subdir = cfg.dirs.import2db + SEP + str(i) UtilClass.rmmkdir(subdir) str_cmd = '"%s/import_raster" %s %s %s %s %s %d %s' % ( cfg.seims_bin, subbasin_file, cfg.dirs.geodata2db, cfg.spatial_db, DBTableNames.gridfs_spatial, cfg.hostname, cfg.port, cfg.dirs.import2db) # print (str_cmd) UtilClass.run_command(str_cmd)
def generate_reach_table(cfg, maindb): """generate reaches table""" area_dic, dx = ImportReaches2Mongo.get_subbasin_cell_count( cfg.spatials.subbsn) (downStreamDic, downstreamUpOrderDic, upstreamDownOrderDic, depthDic, slopeDic, widthDic, lenDic) = ImportReaches2Mongo.down_stream(cfg.vecs.reach, cfg.is_TauDEM) # for k in downStreamDic: # print (k, downStreamDic[k]) g = nx.DiGraph() for k in downStreamDic: if downStreamDic[k] > 0: g.add_edge(k, downStreamDic[k]) ns = g.nodes() # construct the METIS input file UtilClass.mkdir(cfg.dirs.metis) metis_input = r'%s/metis.txt' % cfg.dirs.metis f = open(metis_input, 'w') f.write(str(len(ns)) + "\t" + str(len(g.edges())) + "\t" + "010\t1\n") for node in ns: if node <= 0: continue f.write(str(area_dic[node]) + "\t") for e in g.out_edges(node): if e[1] > 0: f.write(str(e[1]) + "\t") for e in g.in_edges(node): if e[0] > 0: f.write(str(e[0]) + "\t") f.write("\n") f.close() # execute metis nlist = [ 1, ] if cfg.cluster: a = [1, 2, 3, 6] a2 = [12 * pow(2, i) for i in range(8)] a.extend(a2) b = [1, 3] b2 = [i / 2 for i in a2] b.extend(b2) c = [1, 2] c2 = [i / 3 for i in a2] c.extend(c2) d = [1] d2 = [i / 6 for i in a2] d.extend(d2) e = [i / 12 for i in a2] nlist = a + b + c + d + e nlist.extend(range(1, 129)) # nlist.extend([576, 288, 512, 258, 172]) nlist = list(set(nlist)) nlist.sort() # nlist should be less than the number of subbasin, otherwise it will make nonsense. # by LJ nlist = [x for x in nlist if x <= max(ns)] # interpolation among different stream orders min_manning = 0.035 max_manning = 0.075 min_order = 1 max_order = 1 for k, up_order in upstreamDownOrderDic.items(): if up_order > max_order: max_order = up_order dic_manning = dict() a = (max_manning - min_manning) / (max_order - min_order) for tmpid in downStreamDic.keys(): dic_manning[tmpid] = max_manning - a * ( upstreamDownOrderDic[tmpid] - min_order) def import_reach_info(n, down_stream_dic, gdic_k, gdic_p): """import reach info""" for tmpid2 in down_stream_dic: dic = dict() dic[ImportReaches2Mongo._SUBBASIN] = tmpid2 dic[ImportReaches2Mongo._DOWNSTREAM] = down_stream_dic[tmpid2] dic[ImportReaches2Mongo. _UPDOWN_ORDER] = upstreamDownOrderDic[tmpid2] dic[ImportReaches2Mongo. _DOWNUP_ORDER] = downstreamUpOrderDic[tmpid2] dic[ImportReaches2Mongo._MANNING] = dic_manning[tmpid2] dic[ImportReaches2Mongo._SLOPE] = slopeDic[tmpid2] dic[ImportReaches2Mongo._V0] = sqrt(slopeDic[tmpid2]) * \ pow(depthDic[tmpid2], 2. / 3.) / dic[ ImportReaches2Mongo._MANNING] dic[ImportReaches2Mongo._NUMCELLS] = area_dic[tmpid2] if n == 1: dic[ImportReaches2Mongo._GROUP] = n else: dic[ImportReaches2Mongo._KMETIS] = gdic_k[tmpid2] dic[ImportReaches2Mongo._PMETIS] = gdic_p[tmpid2] dic[ImportReaches2Mongo._GROUPDIVIDED] = n dic[ImportReaches2Mongo._WIDTH] = widthDic[tmpid2] dic[ImportReaches2Mongo._LENGTH] = lenDic[tmpid2] dic[ImportReaches2Mongo._DEPTH] = depthDic[tmpid2] dic[ImportReaches2Mongo._AREA] = area_dic[tmpid2] * dx * dx dic[ImportReaches2Mongo._SIDESLP] = 2. dic[ImportReaches2Mongo._BC1] = 0.55 dic[ImportReaches2Mongo._BC2] = 1.1 dic[ImportReaches2Mongo._BC3] = 0.21 dic[ImportReaches2Mongo._BC4] = 0.35 dic[ImportReaches2Mongo._RK1] = 1.71 dic[ImportReaches2Mongo._RK2] = 50 dic[ImportReaches2Mongo._RK3] = 0.36 dic[ImportReaches2Mongo._RK4] = 2 dic[ImportReaches2Mongo._RS1] = 1 dic[ImportReaches2Mongo._RS2] = 0.05 dic[ImportReaches2Mongo._RS3] = 0.5 dic[ImportReaches2Mongo._RS4] = 0.05 dic[ImportReaches2Mongo._RS5] = 0.05 dic[ImportReaches2Mongo._COVER] = 0.1 dic[ImportReaches2Mongo._EROD] = 0.1 dic[ImportReaches2Mongo._DISOX] = 10 dic[ImportReaches2Mongo._BOD] = 10 dic[ImportReaches2Mongo._ALGAE] = 0 # 10 dic[ImportReaches2Mongo._ORGN] = 0 # 10 dic[ImportReaches2Mongo._NH4] = 0 # 1 # 8. dic[ImportReaches2Mongo._NO2] = 0 # 0. dic[ImportReaches2Mongo._NO3] = 0 # 1 # 8. dic[ImportReaches2Mongo._ORGP] = 0 # 10. dic[ImportReaches2Mongo._SOLP] = 0 # 0.1 # 0.5 dic[ImportReaches2Mongo._GWNO3] = 0 # 10. dic[ImportReaches2Mongo._GWSOLP] = 0 # 10. cur_filter = {ImportReaches2Mongo._SUBBASIN: tmpid2} maindb[ImportReaches2Mongo._TAB_REACH].find_one_and_replace( cur_filter, dic, upsert=True) for n in nlist: print('divide number: ', n) if n == 1: import_reach_info(n, downStreamDic, {}, {}) continue # for cluster, based on kmetis str_command = '"%s/gpmetis" %s %d' % (cfg.seims_bin, metis_input, n) result = UtilClass.run_command(str_command) f_metis_output = open( '%s/kmetisResult%d.txt' % (cfg.dirs.metis, n), 'w') for line in result: f_metis_output.write(line) f_metis_output.close() metis_output = "%s.part.%d" % (metis_input, n) f = open(metis_output) lines = f.readlines() group_kmetis = [int(item) for item in lines] f.close() adjust_group_result(g, area_dic, group_kmetis, n) # pmetis str_command = '"%s/gpmetis" -ptype=rb %s %d' % (cfg.seims_bin, metis_input, n) result = UtilClass.run_command(str_command) f_metis_output = open( '%s/pmetisResult%d.txt' % (cfg.dirs.metis, n), 'w') for line in result: f_metis_output.write(line) f_metis_output.close() f = open(metis_output) lines = f.readlines() group_pmetis = [int(item) for item in lines] f.close() adjust_group_result(g, area_dic, group_pmetis, n) group_dic_k, group_dic_p = \ ImportReaches2Mongo.add_group_field(cfg.vecs.reach, ImportReaches2Mongo._LINKNO, n, group_kmetis, group_pmetis, ns) group_dic_k, group_dic_p = \ ImportReaches2Mongo.add_group_field(cfg.vecs.subbsn, ImportReaches2Mongo._SUBBASIN, n, group_kmetis, group_pmetis, ns) import_reach_info(n, downStreamDic, group_dic_k, group_dic_p) maindb[ImportReaches2Mongo._TAB_REACH].create_index([ (ImportReaches2Mongo._SUBBASIN, ASCENDING), (ImportReaches2Mongo._GROUPDIVIDED, ASCENDING) ])