def grid_layering(cfg, n_subbasins):
"""Invoke grid layering program."""
layering_dir = cfg.dirs.layerinfo
UtilClass.rmmkdir(layering_dir)
if not cfg.cluster:
n_subbasins = 0
str_cmd = '"%s/grid_layering" %s %d %s %s %s %d' % (
cfg.seims_bin, cfg.hostname, cfg.port, layering_dir,
cfg.spatial_db, DBTableNames.gridfs_spatial, n_subbasins)
# print (str_cmd)
UtilClass.run_command(str_cmd)
def iuh(cfg, n_subbasins):
"""Invoke IUH program"""
if not cfg.cluster:
n_subbasins = 0
if cfg.gen_iuh:
dt = 24
str_cmd = '"%s/iuh" %s %d %s %s %s %d' % (
cfg.seims_bin, cfg.hostname, cfg.port, cfg.spatial_db,
DBTableNames.gridfs_spatial, dt, n_subbasins)
# print (str_cmd)
UtilClass.run_command(str_cmd)
def mask_raster_cpp(bin_dir, maskfile, originalfiles, outputfiles,
default_values, configfile):
"""Call mask_raster program (cpp version) to mask raster"""
# write mask configuration file
n = len(originalfiles)
# write mask config file
f = open(configfile, 'w')
f.write(maskfile + "\n")
f.write("%d\n" % (n, ))
for i in range(n):
s = "%s\t%d\t%s\n" % (originalfiles[i], default_values[i],
outputfiles[i])
f.write(s)
f.close()
# run command
UtilClass.run_command('"%s/mask_raster" %s' % (bin_dir, configfile))
def reclassify_landuse_parameters(bin_dir, config_file, dst_dir,
landuse_file, lookup_dir,
landuse_attr_list, default_landuse_id):
"""
Reclassify landuse parameters by lookup table.
TODO(LJ): this function should be replaced by replaceByDict() function!
"""
# prepare reclassify configuration file
f_reclass_lu = open(config_file, 'w')
f_reclass_lu.write("%s\t%d\n" % (landuse_file, default_landuse_id))
f_reclass_lu.write("%s\n" % lookup_dir)
f_reclass_lu.write(dst_dir + "\n")
n = len(landuse_attr_list)
f_reclass_lu.write("%d\n" % n)
f_reclass_lu.write("\n".join(landuse_attr_list))
f_reclass_lu.close()
s = '"%s/reclassify" %s' % (bin_dir, config_file)
UtilClass.run_command(s)
def spatial_rasters(cfg, subbasin_num):
"""Import spatial raster data."""
UtilClass.mkdir(cfg.dirs.import2db)
if not cfg.cluster: # changed by LJ, SubbasinID is 0 means the whole basin!
subbasin_num = 0
start_id = 0
subbasin_file = cfg.spatials.mask
else:
start_id = 1
subbasin_file = cfg.spatials.subbsn
for i in range(start_id, subbasin_num + 1):
subdir = cfg.dirs.import2db + SEP + str(i)
UtilClass.rmmkdir(subdir)
str_cmd = '"%s/import_raster" %s %s %s %s %s %d %s' % (
cfg.seims_bin, subbasin_file, cfg.dirs.geodata2db, cfg.spatial_db,
DBTableNames.gridfs_spatial, cfg.hostname, cfg.port,
cfg.dirs.import2db)
# print (str_cmd)
UtilClass.run_command(str_cmd)
def generate_reach_table(cfg, maindb):
"""generate reaches table"""
area_dic, dx = ImportReaches2Mongo.get_subbasin_cell_count(
cfg.spatials.subbsn)
(downStreamDic, downstreamUpOrderDic, upstreamDownOrderDic, depthDic,
slopeDic, widthDic,
lenDic) = ImportReaches2Mongo.down_stream(cfg.vecs.reach,
cfg.is_TauDEM)
# for k in downStreamDic:
# print (k, downStreamDic[k])
g = nx.DiGraph()
for k in downStreamDic:
if downStreamDic[k] > 0:
g.add_edge(k, downStreamDic[k])
ns = g.nodes()
# construct the METIS input file
UtilClass.mkdir(cfg.dirs.metis)
metis_input = r'%s/metis.txt' % cfg.dirs.metis
f = open(metis_input, 'w')
f.write(str(len(ns)) + "\t" + str(len(g.edges())) + "\t" + "010\t1\n")
for node in ns:
if node <= 0:
continue
f.write(str(area_dic[node]) + "\t")
for e in g.out_edges(node):
if e[1] > 0:
f.write(str(e[1]) + "\t")
for e in g.in_edges(node):
if e[0] > 0:
f.write(str(e[0]) + "\t")
f.write("\n")
f.close()
# execute metis
nlist = [
1,
]
if cfg.cluster:
a = [1, 2, 3, 6]
a2 = [12 * pow(2, i) for i in range(8)]
a.extend(a2)
b = [1, 3]
b2 = [i / 2 for i in a2]
b.extend(b2)
c = [1, 2]
c2 = [i / 3 for i in a2]
c.extend(c2)
d = [1]
d2 = [i / 6 for i in a2]
d.extend(d2)
e = [i / 12 for i in a2]
nlist = a + b + c + d + e
nlist.extend(range(1, 129))
# nlist.extend([576, 288, 512, 258, 172])
nlist = list(set(nlist))
nlist.sort()
# nlist should be less than the number of subbasin, otherwise it will make nonsense.
# by LJ
nlist = [x for x in nlist if x <= max(ns)]
# interpolation among different stream orders
min_manning = 0.035
max_manning = 0.075
min_order = 1
max_order = 1
for k, up_order in upstreamDownOrderDic.items():
if up_order > max_order:
max_order = up_order
dic_manning = dict()
a = (max_manning - min_manning) / (max_order - min_order)
for tmpid in downStreamDic.keys():
dic_manning[tmpid] = max_manning - a * (
upstreamDownOrderDic[tmpid] - min_order)
def import_reach_info(n, down_stream_dic, gdic_k, gdic_p):
"""import reach info"""
for tmpid2 in down_stream_dic:
dic = dict()
dic[ImportReaches2Mongo._SUBBASIN] = tmpid2
dic[ImportReaches2Mongo._DOWNSTREAM] = down_stream_dic[tmpid2]
dic[ImportReaches2Mongo.
_UPDOWN_ORDER] = upstreamDownOrderDic[tmpid2]
dic[ImportReaches2Mongo.
_DOWNUP_ORDER] = downstreamUpOrderDic[tmpid2]
dic[ImportReaches2Mongo._MANNING] = dic_manning[tmpid2]
dic[ImportReaches2Mongo._SLOPE] = slopeDic[tmpid2]
dic[ImportReaches2Mongo._V0] = sqrt(slopeDic[tmpid2]) * \
pow(depthDic[tmpid2], 2. / 3.) / dic[
ImportReaches2Mongo._MANNING]
dic[ImportReaches2Mongo._NUMCELLS] = area_dic[tmpid2]
if n == 1:
dic[ImportReaches2Mongo._GROUP] = n
else:
dic[ImportReaches2Mongo._KMETIS] = gdic_k[tmpid2]
dic[ImportReaches2Mongo._PMETIS] = gdic_p[tmpid2]
dic[ImportReaches2Mongo._GROUPDIVIDED] = n
dic[ImportReaches2Mongo._WIDTH] = widthDic[tmpid2]
dic[ImportReaches2Mongo._LENGTH] = lenDic[tmpid2]
dic[ImportReaches2Mongo._DEPTH] = depthDic[tmpid2]
dic[ImportReaches2Mongo._AREA] = area_dic[tmpid2] * dx * dx
dic[ImportReaches2Mongo._SIDESLP] = 2.
dic[ImportReaches2Mongo._BC1] = 0.55
dic[ImportReaches2Mongo._BC2] = 1.1
dic[ImportReaches2Mongo._BC3] = 0.21
dic[ImportReaches2Mongo._BC4] = 0.35
dic[ImportReaches2Mongo._RK1] = 1.71
dic[ImportReaches2Mongo._RK2] = 50
dic[ImportReaches2Mongo._RK3] = 0.36
dic[ImportReaches2Mongo._RK4] = 2
dic[ImportReaches2Mongo._RS1] = 1
dic[ImportReaches2Mongo._RS2] = 0.05
dic[ImportReaches2Mongo._RS3] = 0.5
dic[ImportReaches2Mongo._RS4] = 0.05
dic[ImportReaches2Mongo._RS5] = 0.05
dic[ImportReaches2Mongo._COVER] = 0.1
dic[ImportReaches2Mongo._EROD] = 0.1
dic[ImportReaches2Mongo._DISOX] = 10
dic[ImportReaches2Mongo._BOD] = 10
dic[ImportReaches2Mongo._ALGAE] = 0 # 10
dic[ImportReaches2Mongo._ORGN] = 0 # 10
dic[ImportReaches2Mongo._NH4] = 0 # 1 # 8.
dic[ImportReaches2Mongo._NO2] = 0 # 0.
dic[ImportReaches2Mongo._NO3] = 0 # 1 # 8.
dic[ImportReaches2Mongo._ORGP] = 0 # 10.
dic[ImportReaches2Mongo._SOLP] = 0 # 0.1 # 0.5
dic[ImportReaches2Mongo._GWNO3] = 0 # 10.
dic[ImportReaches2Mongo._GWSOLP] = 0 # 10.
cur_filter = {ImportReaches2Mongo._SUBBASIN: tmpid2}
maindb[ImportReaches2Mongo._TAB_REACH].find_one_and_replace(
cur_filter, dic, upsert=True)
for n in nlist:
print('divide number: ', n)
if n == 1:
import_reach_info(n, downStreamDic, {}, {})
continue
# for cluster, based on kmetis
str_command = '"%s/gpmetis" %s %d' % (cfg.seims_bin, metis_input,
n)
result = UtilClass.run_command(str_command)
f_metis_output = open(
'%s/kmetisResult%d.txt' % (cfg.dirs.metis, n), 'w')
for line in result:
f_metis_output.write(line)
f_metis_output.close()
metis_output = "%s.part.%d" % (metis_input, n)
f = open(metis_output)
lines = f.readlines()
group_kmetis = [int(item) for item in lines]
f.close()
adjust_group_result(g, area_dic, group_kmetis, n)
# pmetis
str_command = '"%s/gpmetis" -ptype=rb %s %d' % (cfg.seims_bin,
metis_input, n)
result = UtilClass.run_command(str_command)
f_metis_output = open(
'%s/pmetisResult%d.txt' % (cfg.dirs.metis, n), 'w')
for line in result:
f_metis_output.write(line)
f_metis_output.close()
f = open(metis_output)
lines = f.readlines()
group_pmetis = [int(item) for item in lines]
f.close()
adjust_group_result(g, area_dic, group_pmetis, n)
group_dic_k, group_dic_p = \
ImportReaches2Mongo.add_group_field(cfg.vecs.reach, ImportReaches2Mongo._LINKNO,
n, group_kmetis, group_pmetis, ns)
group_dic_k, group_dic_p = \
ImportReaches2Mongo.add_group_field(cfg.vecs.subbsn, ImportReaches2Mongo._SUBBASIN,
n, group_kmetis, group_pmetis, ns)
import_reach_info(n, downStreamDic, group_dic_k, group_dic_p)
maindb[ImportReaches2Mongo._TAB_REACH].create_index([
(ImportReaches2Mongo._SUBBASIN, ASCENDING),
(ImportReaches2Mongo._GROUPDIVIDED, ASCENDING)
])