def grid_layering(cfg, n_subbasins):
"""Invoke grid layering program."""
layering_dir = cfg.dirs.layerinfo
UtilClass.rmmkdir(layering_dir)
if not cfg.cluster:
n_subbasins = 0
str_cmd = '"%s/grid_layering" %s %d %s %s %s %d' % (
cfg.seims_bin, cfg.hostname, cfg.port, layering_dir,
cfg.spatial_db, DBTableNames.gridfs_spatial, n_subbasins)
# print (str_cmd)
UtilClass.run_command(str_cmd)
def iuh(cfg, n_subbasins):
"""Invoke IUH program"""
if not cfg.cluster:
n_subbasins = 0
if cfg.gen_iuh:
dt = 24
str_cmd = '"%s/iuh" %s %d %s %s %s %d' % (
cfg.seims_bin, cfg.hostname, cfg.port, cfg.spatial_db,
DBTableNames.gridfs_spatial, dt, n_subbasins)
# print (str_cmd)
UtilClass.run_command(str_cmd)
def original_delineation(cfg):
"""Original Delineation by calling TauDEM functions"""
# Check directories
UtilClass.mkdir(cfg.workspace)
UtilClass.mkdir(cfg.dirs.log)
bin_dir = cfg.seims_bin
mpi_bin = cfg.mpi_bin
np = cfg.np
TauDEMWorkflow.watershed_delineation(bin_dir, mpi_bin, np, cfg.dem,
cfg.outlet_file,
cfg.d8acc_threshold,
cfg.d8down_method, cfg.taudems,
cfg.logs.delineation)
def mask_origin_delineated_data(cfg):
"""Mask the original delineated data by Subbasin raster."""
subbasin_tau_file = cfg.taudems.subbsn
geodata2dbdir = cfg.dirs.geodata2db
UtilClass.mkdir(geodata2dbdir)
mask_file = cfg.spatials.mask
RasterUtilClass.get_mask_from_raster(subbasin_tau_file, mask_file)
# Total 12 raster files
original_files = [
cfg.taudems.subbsn, cfg.taudems.d8flow, cfg.taudems.stream_raster,
cfg.taudems.slp, cfg.taudems.filldem, cfg.taudems.d8acc,
cfg.taudems.stream_order, cfg.taudems.dinf, cfg.taudems.dinf_d8dir,
cfg.taudems.dinf_slp, cfg.taudems.dinf_weight,
cfg.taudems.dist2stream_d8
]
# output masked files
output_files = [
cfg.taudems.subbsn_m, cfg.taudems.d8flow_m, cfg.taudems.stream_m,
cfg.spatials.slope, cfg.spatials.filldem, cfg.spatials.d8acc,
cfg.spatials.stream_order, cfg.spatials.dinf,
cfg.spatials.dinf_d8dir, cfg.spatials.dinf_slp,
cfg.spatials.dinf_weight, cfg.spatials.dist2stream_d8
]
default_values = []
for i in range(len(original_files)):
default_values.append(DEFAULT_NODATA)
# other input rasters need to be masked
# soil and landuse
FileClass.check_file_exists(cfg.soil)
FileClass.check_file_exists(cfg.landuse)
original_files.append(cfg.soil)
output_files.append(cfg.spatials.soil_type)
default_values.append(cfg.default_soil)
original_files.append(cfg.landuse)
output_files.append(cfg.spatials.landuse)
default_values.append(cfg.default_landuse)
# management fields
if cfg.mgt_field is not None and FileClass.is_file_exists(
cfg.mgt_field):
original_files.append(cfg.mgt_field)
output_files.append(cfg.spatials.mgt_field)
default_values.append(DEFAULT_NODATA)
config_file = cfg.logs.mask_cfg
# run mask operation
print("Mask original delineated data by Subbasin raster...")
SpatialDelineation.mask_raster_cpp(cfg.seims_bin, mask_file,
original_files, output_files,
default_values, config_file)
def post_process_of_delineated_data(cfg):
"""Do some necessary transfer for subbasin, stream, and flow direction raster."""
# inputs
stream_net_file = cfg.taudems.streamnet_shp
subbasin_file = cfg.taudems.subbsn_m
flow_dir_file_tau = cfg.taudems.d8flow_m
stream_raster_file = cfg.taudems.stream_m
# outputs
# -- shapefile
shp_dir = cfg.dirs.geoshp
UtilClass.mkdir(shp_dir)
# ---- outlet, copy from DirNameUtils.TauDEM
FileClass.copy_files(cfg.taudems.outlet_m, cfg.vecs.outlet)
# ---- reaches
output_reach_file = cfg.vecs.reach
# ---- subbasins
subbasin_vector_file = cfg.vecs.subbsn
# -- raster file
output_subbasin_file = cfg.spatials.subbsn
output_flow_dir_file = cfg.spatials.d8flow
output_stream_link_file = cfg.spatials.stream_link
output_hillslope_file = cfg.spatials.hillslope
id_map = StreamnetUtil.serialize_streamnet(stream_net_file,
output_reach_file)
RasterUtilClass.raster_reclassify(subbasin_file, id_map,
output_subbasin_file, GDT_Int32)
StreamnetUtil.assign_stream_id_raster(stream_raster_file,
output_subbasin_file,
output_stream_link_file)
# Convert D8 encoding rule to ArcGIS
if cfg.is_TauDEM:
shutil_copy(flow_dir_file_tau, output_flow_dir_file)
else:
D8Util.convert_code(flow_dir_file_tau, output_flow_dir_file)
# convert raster to shapefile (for subbasin and basin)
print "Generating subbasin vector..."
VectorUtilClass.raster2shp(output_subbasin_file, subbasin_vector_file,
"subbasin", FieldNames.subbasin_id)
mask_file = cfg.spatials.mask
basin_vector = cfg.vecs.bsn
print "Generating basin vector..."
VectorUtilClass.raster2shp(mask_file, basin_vector, "basin",
FieldNames.basin)
# delineate hillslope
DelineateHillslope.downstream_method_whitebox(output_stream_link_file,
flow_dir_file_tau,
output_hillslope_file)
def mask_raster_cpp(bin_dir, maskfile, originalfiles, outputfiles,
default_values, configfile):
"""Call mask_raster program (cpp version) to mask raster"""
# write mask configuration file
n = len(originalfiles)
# write mask config file
f = open(configfile, 'w')
f.write(maskfile + "\n")
f.write("%d\n" % (n, ))
for i in range(n):
s = "%s\t%d\t%s\n" % (originalfiles[i], default_values[i],
outputfiles[i])
f.write(s)
f.close()
# run command
UtilClass.run_command('"%s/mask_raster" %s' % (bin_dir, configfile))
def reclassify_landuse_parameters(bin_dir, config_file, dst_dir,
landuse_file, lookup_dir,
landuse_attr_list, default_landuse_id):
"""
Reclassify landuse parameters by lookup table.
TODO(LJ): this function should be replaced by replaceByDict() function!
"""
# prepare reclassify configuration file
f_reclass_lu = open(config_file, 'w')
f_reclass_lu.write("%s\t%d\n" % (landuse_file, default_landuse_id))
f_reclass_lu.write("%s\n" % lookup_dir)
f_reclass_lu.write(dst_dir + "\n")
n = len(landuse_attr_list)
f_reclass_lu.write("%d\n" % n)
f_reclass_lu.write("\n".join(landuse_attr_list))
f_reclass_lu.close()
s = '"%s/reclassify" %s' % (bin_dir, config_file)
UtilClass.run_command(s)
def export_landuse_lookup_files_from_mongodb(cfg, maindb):
"""export landuse lookup tables to txt file from MongoDB."""
lookup_dir = cfg.dirs.lookup
property_namelist = ModelParamDataUtils.landuse_fields
property_map = {}
property_namelist.append('USLE_P')
query_result = maindb['LANDUSELOOKUP'].find()
if query_result is None:
raise RuntimeError(
"LanduseLoop Collection is not existed or empty!")
count = 0
for row in query_result:
# print row
value_map = dict()
for i, p_name in enumerate(property_namelist):
if StringClass.string_match(p_name, "USLE_P"):
# Currently, USLE_P is set as 1 for all landuse.
value_map[p_name] = 1
else:
if StringClass.string_match(p_name, "Manning"):
value_map[p_name] = row.get(p_name) * 10
else:
value_map[p_name] = row.get(p_name)
count += 1
property_map[count] = value_map
n = len(property_map)
UtilClass.rmmkdir(lookup_dir)
for propertyName in property_namelist:
f = open("%s/%s.txt" % (
lookup_dir,
propertyName,
), 'w')
f.write("%d\n" % n)
for prop_id in property_map:
s = "%d %f\n" % (prop_id, property_map[prop_id][propertyName])
f.write(s)
f.close()
def spatial_rasters(cfg, subbasin_num):
"""Import spatial raster data."""
UtilClass.mkdir(cfg.dirs.import2db)
if not cfg.cluster: # changed by LJ, SubbasinID is 0 means the whole basin!
subbasin_num = 0
start_id = 0
subbasin_file = cfg.spatials.mask
else:
start_id = 1
subbasin_file = cfg.spatials.subbsn
for i in range(start_id, subbasin_num + 1):
subdir = cfg.dirs.import2db + SEP + str(i)
UtilClass.rmmkdir(subdir)
str_cmd = '"%s/import_raster" %s %s %s %s %s %d %s' % (
cfg.seims_bin, subbasin_file, cfg.dirs.geodata2db, cfg.spatial_db,
DBTableNames.gridfs_spatial, cfg.hostname, cfg.port,
cfg.dirs.import2db)
# print (str_cmd)
UtilClass.run_command(str_cmd)
def generate_reach_table(cfg, maindb):
"""generate reaches table"""
area_dic, dx = ImportReaches2Mongo.get_subbasin_cell_count(
cfg.spatials.subbsn)
(downStreamDic, downstreamUpOrderDic, upstreamDownOrderDic, depthDic,
slopeDic, widthDic,
lenDic) = ImportReaches2Mongo.down_stream(cfg.vecs.reach,
cfg.is_TauDEM)
# for k in downStreamDic:
# print (k, downStreamDic[k])
g = nx.DiGraph()
for k in downStreamDic:
if downStreamDic[k] > 0:
g.add_edge(k, downStreamDic[k])
ns = g.nodes()
# construct the METIS input file
UtilClass.mkdir(cfg.dirs.metis)
metis_input = r'%s/metis.txt' % cfg.dirs.metis
f = open(metis_input, 'w')
f.write(str(len(ns)) + "\t" + str(len(g.edges())) + "\t" + "010\t1\n")
for node in ns:
if node <= 0:
continue
f.write(str(area_dic[node]) + "\t")
for e in g.out_edges(node):
if e[1] > 0:
f.write(str(e[1]) + "\t")
for e in g.in_edges(node):
if e[0] > 0:
f.write(str(e[0]) + "\t")
f.write("\n")
f.close()
# execute metis
nlist = [
1,
]
if cfg.cluster:
a = [1, 2, 3, 6]
a2 = [12 * pow(2, i) for i in range(8)]
a.extend(a2)
b = [1, 3]
b2 = [i / 2 for i in a2]
b.extend(b2)
c = [1, 2]
c2 = [i / 3 for i in a2]
c.extend(c2)
d = [1]
d2 = [i / 6 for i in a2]
d.extend(d2)
e = [i / 12 for i in a2]
nlist = a + b + c + d + e
nlist.extend(range(1, 129))
# nlist.extend([576, 288, 512, 258, 172])
nlist = list(set(nlist))
nlist.sort()
# nlist should be less than the number of subbasin, otherwise it will make nonsense.
# by LJ
nlist = [x for x in nlist if x <= max(ns)]
# interpolation among different stream orders
min_manning = 0.035
max_manning = 0.075
min_order = 1
max_order = 1
for k, up_order in upstreamDownOrderDic.items():
if up_order > max_order:
max_order = up_order
dic_manning = dict()
a = (max_manning - min_manning) / (max_order - min_order)
for tmpid in downStreamDic.keys():
dic_manning[tmpid] = max_manning - a * (
upstreamDownOrderDic[tmpid] - min_order)
def import_reach_info(n, down_stream_dic, gdic_k, gdic_p):
"""import reach info"""
for tmpid2 in down_stream_dic:
dic = dict()
dic[ImportReaches2Mongo._SUBBASIN] = tmpid2
dic[ImportReaches2Mongo._DOWNSTREAM] = down_stream_dic[tmpid2]
dic[ImportReaches2Mongo.
_UPDOWN_ORDER] = upstreamDownOrderDic[tmpid2]
dic[ImportReaches2Mongo.
_DOWNUP_ORDER] = downstreamUpOrderDic[tmpid2]
dic[ImportReaches2Mongo._MANNING] = dic_manning[tmpid2]
dic[ImportReaches2Mongo._SLOPE] = slopeDic[tmpid2]
dic[ImportReaches2Mongo._V0] = sqrt(slopeDic[tmpid2]) * \
pow(depthDic[tmpid2], 2. / 3.) / dic[
ImportReaches2Mongo._MANNING]
dic[ImportReaches2Mongo._NUMCELLS] = area_dic[tmpid2]
if n == 1:
dic[ImportReaches2Mongo._GROUP] = n
else:
dic[ImportReaches2Mongo._KMETIS] = gdic_k[tmpid2]
dic[ImportReaches2Mongo._PMETIS] = gdic_p[tmpid2]
dic[ImportReaches2Mongo._GROUPDIVIDED] = n
dic[ImportReaches2Mongo._WIDTH] = widthDic[tmpid2]
dic[ImportReaches2Mongo._LENGTH] = lenDic[tmpid2]
dic[ImportReaches2Mongo._DEPTH] = depthDic[tmpid2]
dic[ImportReaches2Mongo._AREA] = area_dic[tmpid2] * dx * dx
dic[ImportReaches2Mongo._SIDESLP] = 2.
dic[ImportReaches2Mongo._BC1] = 0.55
dic[ImportReaches2Mongo._BC2] = 1.1
dic[ImportReaches2Mongo._BC3] = 0.21
dic[ImportReaches2Mongo._BC4] = 0.35
dic[ImportReaches2Mongo._RK1] = 1.71
dic[ImportReaches2Mongo._RK2] = 50
dic[ImportReaches2Mongo._RK3] = 0.36
dic[ImportReaches2Mongo._RK4] = 2
dic[ImportReaches2Mongo._RS1] = 1
dic[ImportReaches2Mongo._RS2] = 0.05
dic[ImportReaches2Mongo._RS3] = 0.5
dic[ImportReaches2Mongo._RS4] = 0.05
dic[ImportReaches2Mongo._RS5] = 0.05
dic[ImportReaches2Mongo._COVER] = 0.1
dic[ImportReaches2Mongo._EROD] = 0.1
dic[ImportReaches2Mongo._DISOX] = 10
dic[ImportReaches2Mongo._BOD] = 10
dic[ImportReaches2Mongo._ALGAE] = 0 # 10
dic[ImportReaches2Mongo._ORGN] = 0 # 10
dic[ImportReaches2Mongo._NH4] = 0 # 1 # 8.
dic[ImportReaches2Mongo._NO2] = 0 # 0.
dic[ImportReaches2Mongo._NO3] = 0 # 1 # 8.
dic[ImportReaches2Mongo._ORGP] = 0 # 10.
dic[ImportReaches2Mongo._SOLP] = 0 # 0.1 # 0.5
dic[ImportReaches2Mongo._GWNO3] = 0 # 10.
dic[ImportReaches2Mongo._GWSOLP] = 0 # 10.
cur_filter = {ImportReaches2Mongo._SUBBASIN: tmpid2}
maindb[ImportReaches2Mongo._TAB_REACH].find_one_and_replace(
cur_filter, dic, upsert=True)
for n in nlist:
print('divide number: ', n)
if n == 1:
import_reach_info(n, downStreamDic, {}, {})
continue
# for cluster, based on kmetis
str_command = '"%s/gpmetis" %s %d' % (cfg.seims_bin, metis_input,
n)
result = UtilClass.run_command(str_command)
f_metis_output = open(
'%s/kmetisResult%d.txt' % (cfg.dirs.metis, n), 'w')
for line in result:
f_metis_output.write(line)
f_metis_output.close()
metis_output = "%s.part.%d" % (metis_input, n)
f = open(metis_output)
lines = f.readlines()
group_kmetis = [int(item) for item in lines]
f.close()
adjust_group_result(g, area_dic, group_kmetis, n)
# pmetis
str_command = '"%s/gpmetis" -ptype=rb %s %d' % (cfg.seims_bin,
metis_input, n)
result = UtilClass.run_command(str_command)
f_metis_output = open(
'%s/pmetisResult%d.txt' % (cfg.dirs.metis, n), 'w')
for line in result:
f_metis_output.write(line)
f_metis_output.close()
f = open(metis_output)
lines = f.readlines()
group_pmetis = [int(item) for item in lines]
f.close()
adjust_group_result(g, area_dic, group_pmetis, n)
group_dic_k, group_dic_p = \
ImportReaches2Mongo.add_group_field(cfg.vecs.reach, ImportReaches2Mongo._LINKNO,
n, group_kmetis, group_pmetis, ns)
group_dic_k, group_dic_p = \
ImportReaches2Mongo.add_group_field(cfg.vecs.subbsn, ImportReaches2Mongo._SUBBASIN,
n, group_kmetis, group_pmetis, ns)
import_reach_info(n, downStreamDic, group_dic_k, group_dic_p)
maindb[ImportReaches2Mongo._TAB_REACH].create_index([
(ImportReaches2Mongo._SUBBASIN, ASCENDING),
(ImportReaches2Mongo._GROUPDIVIDED, ASCENDING)
])