def convert_code(in_file, out_file, in_alg='taudem', out_alg='arcgis', datatype=None):
"""
convert D8 flow direction code from one algorithm to another.
Args:
in_file: input raster file path
out_file: output raster file path
in_alg: available algorithms are in FlowModelConst.d8_dirs. "taudem" is the default
out_alg: same as in_alg. "arcgis" is the default
datatype: default is None and use the datatype of the in_file
"""
FileClass.check_file_exists(in_file)
in_alg = in_alg.lower()
out_alg = out_alg.lower()
if in_alg not in FlowModelConst.d8_dirs or out_alg not in FlowModelConst.d8_dirs:
raise RuntimeError('The input algorithm name should one of %s' %
', '.join(list(FlowModelConst.d8_dirs.keys())))
convert_dict = dict()
in_code = FlowModelConst.d8_dirs.get(in_alg)
out_code = FlowModelConst.d8_dirs.get(out_alg)
assert len(in_code) == len(out_code)
for i, tmp_in_code in enumerate(in_code):
convert_dict[tmp_in_code] = out_code[i]
if datatype is not None and datatype in GDALDataType:
RasterUtilClass.raster_reclassify(in_file, convert_dict, out_file, datatype)
else:
RasterUtilClass.raster_reclassify(in_file, convert_dict, out_file)
def calculate_environment(self):
if not self.modelrun: # no evaluate done
self.economy = self.worst_econ
self.environment = self.worst_env
return
rfile = self.modelout_dir + os.path.sep + self.bmps_info['ENVEVAL']
if not FileClass.is_file_exists(rfile):
time.sleep(5) # sleep 5 seconds wait for the ouput
if not FileClass.is_file_exists(rfile):
print('WARNING: Although SEIMS model runs successfully, the desired output: %s'
' cannot be found!' % rfile)
self.economy = self.worst_econ
self.environment = self.worst_env
return
base_amount = self.bmps_info['BASE_ENV']
if StringClass.string_match(rfile.split('.')[-1], 'tif'): # Raster data
rr = RasterUtilClass.read_raster(rfile)
soil_erosion_amount = rr.get_sum() / self.timerange # unit: year
# reduction rate of soil erosion
self.environment = (base_amount - soil_erosion_amount) / base_amount
elif StringClass.string_match(rfile.split('.')[-1], 'txt'): # Time series data
sed_sum = read_simulation_from_txt(self.modelout_dir) # TODO, fix it later, lj
self.environment = (base_amount - sed_sum) / base_amount
else:
self.economy = self.worst_econ
self.environment = self.worst_env
return
def add_group_field(shp_file, subbasin_field_name, group_metis_dict):
"""add group information to subbasin ESRI shapefile
Args:
shp_file: Subbasin Shapefile
subbasin_field_name: field name of subbasin
group_metis_dict: returned by func`metis_partition`
"""
if not group_metis_dict:
return
ds_reach = ogr_Open(shp_file, update=True)
layer_reach = ds_reach.GetLayer(0)
layer_def = layer_reach.GetLayerDefn()
icode = layer_def.GetFieldIndex(subbasin_field_name)
igrp = layer_def.GetFieldIndex(ImportReaches2Mongo._GROUP)
ikgrp = layer_def.GetFieldIndex(ImportReaches2Mongo._KMETIS)
ipgrp = layer_def.GetFieldIndex(ImportReaches2Mongo._PMETIS)
if igrp < 0:
new_field = ogr_FieldDefn(ImportReaches2Mongo._GROUP, OFTInteger)
layer_reach.CreateField(new_field)
if ikgrp < 0:
new_field = ogr_FieldDefn(ImportReaches2Mongo._KMETIS, OFTInteger)
layer_reach.CreateField(new_field)
if ipgrp < 0:
new_field = ogr_FieldDefn(ImportReaches2Mongo._PMETIS, OFTInteger)
layer_reach.CreateField(new_field)
ftmap = dict()
layer_reach.ResetReading()
ft = layer_reach.GetNextFeature()
while ft is not None:
tmpid = ft.GetFieldAsInteger(icode)
ftmap[tmpid] = ft
ft = layer_reach.GetNextFeature()
groups = group_metis_dict[1]['group']
for i, n in enumerate(groups):
for node, d in group_metis_dict.items():
ftmap[node].SetField(ImportReaches2Mongo._GROUP, n)
ftmap[node].SetField(ImportReaches2Mongo._KMETIS, d['kmetis'][i])
ftmap[node].SetField(ImportReaches2Mongo._PMETIS, d['pmetis'][i])
layer_reach.SetFeature(ftmap[node])
# copy the reach file to new file
prefix = os.path.splitext(shp_file)[0]
dstfile = prefix + "_" + str(n) + ".shp"
FileClass.copy_files(shp_file, dstfile)
layer_reach.SyncToDisk()
ds_reach.Destroy()
del ds_reach
def write_gtiff_file(f_name, n_rows, n_cols, data, geotransform, srs, nodata_value,
gdal_type=GDT_Float32):
"""Output Raster to GeoTiff format file.
Args:
f_name: output gtiff file name.
n_rows: Row count.
n_cols: Col count.
data: 2D array data.
geotransform: geographic transformation.
srs: coordinate system.
nodata_value: nodata value.
gdal_type (:obj:`pygeoc.raster.GDALDataType`): output raster data type,
GDT_Float32 as default.
"""
UtilClass.mkdir(os.path.dirname(FileClass.get_file_fullpath(f_name)))
driver = gdal_GetDriverByName(str('GTiff'))
try:
ds = driver.Create(f_name, n_cols, n_rows, 1, gdal_type)
except Exception:
print('Cannot create output file %s' % f_name)
return
ds.SetGeoTransform(geotransform)
try:
ds.SetProjection(srs.ExportToWkt())
except AttributeError or Exception:
ds.SetProjection(srs)
ds.GetRasterBand(1).SetNoDataValue(nodata_value)
# if data contains numpy.nan, then replaced by nodata_value
if isinstance(data, numpy.ndarray) and data.dtype in [numpy.dtype('int'),
numpy.dtype('float')]:
data = numpy.where(numpy.isnan(data), nodata_value, data)
ds.GetRasterBand(1).WriteArray(data)
ds = None
def main():
from preprocess.config import parse_ini_configuration
seims_cfg = parse_ini_configuration()
client = ConnectMongoDB(seims_cfg.hostname, seims_cfg.port)
conn = client.get_conn()
db_model = conn[seims_cfg.spatial_db]
spatial_gfs = GridFS(db_model, DBTableNames.gridfs_spatial)
csv_path = r'C:\z_data\zhongTianShe\model_data_seims\field_scale_params'
csv_files = FileClass.get_full_filename_by_suffixes(csv_path, ['.csv'])
field_count = 7419
prefix = 9999
# Create mask file
mask_name = '%d_MASK' % prefix
mask_array = [[1] * field_count]
import_array_to_mongodb(spatial_gfs, mask_array, mask_name)
# Create spatial parameters
for csv_file in csv_files:
print('Import %s...' % csv_file)
param_arrays = read_field_arrays_from_csv(csv_file)
for key, value in list(param_arrays.items()):
import_array_to_mongodb(spatial_gfs, value, '%d_%s' % (prefix, key))
def write_asc_file(filename, data, xsize, ysize, geotransform, nodata_value):
"""Output Raster to ASCII file.
Args:
filename: output ASCII filename.
data: 2D array data.
xsize: Col count.
ysize: Row count.
geotransform: geographic transformation.
nodata_value: nodata_flow value.
"""
UtilClass.mkdir(os.path.dirname(FileClass.get_file_fullpath(filename)))
header = 'NCOLS %d\n' \
'NROWS %d\n' \
'XLLCENTER %f\n' \
'YLLCENTER %f\n' \
'CELLSIZE %f\n' \
'NODATA_VALUE %f' % (xsize, ysize, geotransform[0] + 0.5 * geotransform[1],
geotransform[3] - (ysize - 0.5) * geotransform[1],
geotransform[1], nodata_value)
with open(filename, 'w', encoding='utf-8') as f:
f.write(header)
for i in range(0, ysize):
for j in range(0, xsize):
f.write('%s\t' % repr(data[i][j]))
f.write('\n')
f.close()
def workflow(cfg, maindb, climdb):
"""
This function mainly to import measurement data to MongoDB
data type may include Q (discharge, m3/s), SED (mg/L), tn (mg/L), tp (mg/L), etc.
the required parameters that defined in configuration file (*.ini)
"""
if not cfg.use_observed:
return False
c_list = climdb.collection_names()
if not StringClass.string_in_list(DBTableNames.observes, c_list):
climdb.create_collection(DBTableNames.observes)
else:
climdb.drop_collection(DBTableNames.observes)
if not StringClass.string_in_list(DBTableNames.sites, c_list):
climdb.create_collection(DBTableNames.sites)
if not StringClass.string_in_list(DBTableNames.var_desc, c_list):
climdb.create_collection(DBTableNames.var_desc)
file_list = FileClass.get_full_filename_by_suffixes(cfg.observe_dir, ['.txt'])
meas_file_list = []
site_loc = []
for fl in file_list:
if StringClass.is_substring('observed_', fl):
meas_file_list.append(fl)
else:
site_loc.append(fl)
ImportObservedData.data_from_txt(maindb, climdb, meas_file_list, site_loc,
cfg.spatials.subbsn)
return True
def get_psa_config():
"""Parse arguments.
Returns:
cf: ConfigParse object of *.ini file
mtd: Parameters sensitivity method name, currently, 'morris' and 'fast' are supported.
"""
# define input arguments
parser = argparse.ArgumentParser(description="Execute parameters sensitivity analysis.")
parser.add_argument('-ini', type=str, help="Full path of configuration file")
# add mutually group
psa_group = parser.add_mutually_exclusive_group()
psa_group.add_argument('-morris', action='store_true', help='Run Morris Screening method')
psa_group.add_argument('-fast', action='store_true', help='Run FAST variant-based method')
# parse arguments
args = parser.parse_args()
ini_file = args.ini
psa_mtd = 'morris' # Default
if args.fast:
psa_mtd = 'fast'
elif args.morris:
psa_mtd = 'morris'
if not FileClass.is_file_exists(ini_file):
raise ImportError('Configuration file is not existed: %s' % ini_file)
cf = ConfigParser()
cf.read(ini_file)
return cf, psa_mtd
def output_hillslope(method_id):
"""Output hillslope according different stream cell value method."""
for (tmp_row, tmp_col) in stream_coors:
tmp_hillslp_ids = DelineateHillslope.cal_hs_codes(max_id,
stream_data[tmp_row][tmp_col])
if 0 < method_id < 3:
hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[method_id]
# is head stream cell?
if (tmp_row, tmp_col) in headstream_coors:
hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[0]
elif method_id == 3:
hillslope_mtx[tmp_row][tmp_col] = DEFAULT_NODATA
# Output to raster file
hillslope_out_new = hillslope_out
dirpath = os.path.dirname(hillslope_out_new) + os.path.sep
corename = FileClass.get_core_name_without_suffix(hillslope_out_new)
if method_id == 1:
hillslope_out_new = dirpath + corename + '_right.tif'
elif method_id == 2:
hillslope_out_new = dirpath + corename + '_left.tif'
elif method_id == 3:
hillslope_out_new = dirpath + corename + '_nodata.tif'
RasterUtilClass.write_gtiff_file(hillslope_out_new, nrows, ncols,
hillslope_mtx,
geotrans, srs, DEFAULT_NODATA, datatype)
def calculate_sensitivity(self):
"""Calculate Morris elementary effects.
It is worth to be noticed that evaluate_models() allows to return
several output variables, hence we should calculate each of them separately.
"""
if not self.psa_si:
if FileClass.is_file_exists(self.cfg.outfiles.psa_si_json):
with open(self.cfg.outfiles.psa_si_json, 'r') as f:
self.psa_si = UtilClass.decode_strs_in_dict(json.load(f))
return
if not self.objnames:
if FileClass.is_file_exists('%s/objnames.pickle' % self.cfg.psa_outpath):
with open('%s/objnames.pickle' % self.cfg.psa_outpath, 'r') as f:
self.objnames = pickle.load(f)
if self.output_values is None or len(self.output_values) == 0:
self.evaluate_models()
if self.param_values is None or len(self.param_values) == 0:
self.generate_samples()
if not self.param_defs:
self.read_param_ranges()
row, col = self.output_values.shape
assert (row == self.run_count)
for i in range(col):
print(self.objnames[i])
if self.cfg.method == 'morris':
tmp_Si = morris_alz(self.param_defs,
self.param_values,
self.output_values[:, i],
conf_level=0.95, print_to_console=True,
num_levels=self.cfg.morris.num_levels,
grid_jump=self.cfg.morris.grid_jump)
elif self.cfg.method == 'fast':
tmp_Si = fast_alz(self.param_defs, self.output_values[:, i],
print_to_console=True)
else:
raise ValueError('%s method is not supported now!' % self.cfg.method)
self.psa_si[i] = tmp_Si
# print(self.psa_si)
# Save as json, which can be loaded by json.load()
json_data = json.dumps(self.psa_si, indent=4, cls=SpecialJsonEncoder)
with open(self.cfg.outfiles.psa_si_json, 'w') as f:
f.write(json_data)
self.output_psa_si()
def pitremove(np, dem, filleddem, workingdir=None, mpiexedir=None, exedir=None, log_file=None,
runtime_file=None, hostfile=None):
"""Run pit remove using the flooding approach """
fname = TauDEM.func_name('pitremove')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-z': dem}, workingdir,
None,
{'-fel': filleddem},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def dinfflowdir(np, filleddem, flowangle, slope, workingdir=None, mpiexedir=None, exedir=None,
log_file=None, runtime_file=None, hostfile=None):
"""Run Dinf flow direction"""
fname = TauDEM.func_name('dinfflowdir')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-fel': filleddem}, workingdir,
None,
{'-ang': flowangle, '-slp': slope},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def gridnet(np, pfile, plenfile, tlenfile, gordfile, outlet=None, workingdir=None,
mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Run gridnet"""
fname = TauDEM.func_name('gridnet')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-p': pfile, '-o': outlet}, workingdir,
None,
{'-plen': plenfile, '-tlen': tlenfile, '-gord': gordfile},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def threshold(np, acc, stream_raster, threshold=100., workingdir=None,
mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Run threshold for stream raster"""
fname = TauDEM.func_name('threshold')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-ssa': acc}, workingdir,
{'-thresh': threshold},
{'-src': stream_raster},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def peukerdouglas(np, fel, streamSkeleton, workingdir=None, mpiexedir=None, exedir=None,
log_file=None, runtime_file=None, hostfile=None):
"""Run peuker-douglas function"""
fname = TauDEM.func_name('peukerdouglas')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-fel': fel}, workingdir,
None,
{'-ss': streamSkeleton},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def read_crop_lookup_table(crop_lookup_file):
"""read crop lookup table"""
FileClass.check_file_exists(crop_lookup_file)
data_items = read_data_items_from_txt(crop_lookup_file)
attr_dic = dict()
fields = data_items[0]
n = len(fields)
for i in range(n):
attr_dic[fields[i]] = dict()
for items in data_items[1:]:
cur_id = int(items[0])
for i in range(n):
dic = attr_dic[fields[i]]
try:
dic[cur_id] = float(items[i])
except ValueError:
dic[cur_id] = items[i]
return attr_dic
def raster2shp(rasterfile, vectorshp, layername=None, fieldname=None,
band_num=1, mask='default'):
"""Convert raster to ESRI shapefile"""
FileClass.remove_files(vectorshp)
FileClass.check_file_exists(rasterfile)
# this allows GDAL to throw Python Exceptions
gdal.UseExceptions()
src_ds = gdal.Open(rasterfile)
if src_ds is None:
print('Unable to open %s' % rasterfile)
sys.exit(1)
try:
srcband = src_ds.GetRasterBand(band_num)
except RuntimeError as e:
# for example, try GetRasterBand(10)
print('Band ( %i ) not found, %s' % (band_num, e))
sys.exit(1)
if mask == 'default':
maskband = srcband.GetMaskBand()
elif mask is None or mask.upper() == 'NONE':
maskband = None
else:
mask_ds = gdal.Open(mask)
maskband = mask_ds.GetRasterBand(1)
# create output datasource
if layername is None:
layername = FileClass.get_core_name_without_suffix(rasterfile)
drv = ogr_GetDriverByName(str('ESRI Shapefile'))
dst_ds = drv.CreateDataSource(vectorshp)
srs = None
if src_ds.GetProjection() != '':
srs = osr_SpatialReference()
srs.ImportFromWkt(src_ds.GetProjection())
dst_layer = dst_ds.CreateLayer(str(layername), srs=srs)
if fieldname is None:
fieldname = layername.upper()
fd = ogr_FieldDefn(str(fieldname), OFTInteger)
dst_layer.CreateField(fd)
dst_field = 0
result = gdal.Polygonize(srcband, maskband, dst_layer, dst_field,
['8CONNECTED=8'], callback=None)
return result
def post_process_of_delineated_data(cfg):
"""Do some necessary transfer for subbasin, stream, and flow direction raster."""
# inputs
stream_net_file = cfg.taudems.streamnet_shp
subbasin_file = cfg.taudems.subbsn_m
flow_dir_file_tau = cfg.taudems.d8flow_m
stream_raster_file = cfg.taudems.stream_m
# outputs
# -- shapefile
shp_dir = cfg.dirs.geoshp
UtilClass.mkdir(shp_dir)
# ---- outlet, copy from DirNameUtils.TauDEM
FileClass.copy_files(cfg.taudems.outlet_m, cfg.vecs.outlet)
# ---- reaches
output_reach_file = cfg.vecs.reach
# ---- subbasins
subbasin_vector_file = cfg.vecs.subbsn
# -- raster file
output_subbasin_file = cfg.spatials.subbsn
output_flow_dir_file = cfg.spatials.d8flow
output_stream_link_file = cfg.spatials.stream_link
output_hillslope_file = cfg.spatials.hillslope
id_map = StreamnetUtil.serialize_streamnet(stream_net_file, output_reach_file)
RasterUtilClass.raster_reclassify(subbasin_file, id_map, output_subbasin_file, GDT_Int32)
StreamnetUtil.assign_stream_id_raster(stream_raster_file, output_subbasin_file,
output_stream_link_file)
# Convert D8 encoding rule to ArcGIS
D8Util.convert_code(flow_dir_file_tau, output_flow_dir_file)
# convert raster to shapefile (for subbasin and basin)
print('Generating subbasin vector...')
VectorUtilClass.raster2shp(output_subbasin_file, subbasin_vector_file, 'subbasin',
FieldNames.subbasin_id)
mask_file = cfg.spatials.mask
basin_vector = cfg.vecs.bsn
print('Generating basin vector...')
VectorUtilClass.raster2shp(mask_file, basin_vector, 'basin', FieldNames.basin)
# delineate hillslope
DelineateHillslope.downstream_method_whitebox(output_stream_link_file, flow_dir_file_tau,
output_hillslope_file)
def __init__(self, cf, method='morris'):
"""Initialization."""
self.method = method
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf)
# 2. Common settings of parameters sensitivity analysis
if 'PSA_Settings' not in cf.sections():
raise ValueError("[PSA_Settings] section MUST be existed in *.ini file.")
self.evaluate_params = list()
if cf.has_option('PSA_Settings', 'evaluateparam'):
eva_str = cf.get('PSA_Settings', 'evaluateparam')
self.evaluate_params = StringClass.split_string(eva_str, ',')
else:
self.evaluate_params = ['Q'] # Default
self.param_range_def = 'morris_param_rng.def' # Default
if cf.has_option('PSA_Settings', 'paramrngdef'):
self.param_range_def = cf.get('PSA_Settings', 'paramrngdef')
self.param_range_def = self.model.model_dir + os.path.sep + self.param_range_def
if not FileClass.is_file_exists(self.param_range_def):
raise IOError('Ranges of parameters MUST be provided!')
if not (cf.has_option('PSA_Settings', 'psa_time_start') and
cf.has_option('PSA_Settings', 'psa_time_end')):
raise ValueError("Start and end time of PSA MUST be specified in [PSA_Settings].")
try:
# UTCTIME
tstart = cf.get('PSA_Settings', 'psa_time_start')
tend = cf.get('PSA_Settings', 'psa_time_end')
self.psa_stime = StringClass.get_datetime(tstart)
self.psa_etime = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be"YYYY-MM-DD HH:MM:SS".')
if self.psa_stime >= self.psa_etime:
raise ValueError("Wrong time settings in [PSA_Settings]!")
# 3. Parameters settings for specific sensitivity analysis methods
self.morris = None
self.fast = None
if self.method == 'fast':
self.fast = FASTConfig(cf)
self.psa_outpath = '%s/PSA-FAST-N%dM%d' % (self.model.model_dir,
self.fast.N, self.fast.M)
elif self.method == 'morris':
self.morris = MorrisConfig(cf)
self.psa_outpath = '%s/PSA-Morris-N%dL%dJ%d' % (self.model.model_dir,
self.morris.N,
self.morris.num_levels,
self.morris.grid_jump)
# Do not remove psa_outpath if already existed
UtilClass.mkdir(self.psa_outpath)
self.outfiles = PSAOutputs(self.psa_outpath)
def __init__(self, bin_dir='', model_dir='', nthread=4, lyrmtd=0,
host='127.0.0.1', port=27017, scenario_id=-1, calibration_id=-1,
version='OMP', nprocess=1, mpi_bin='', hosts_opt='-f', hostfile='',
**kwargs): # Allow any other keyword arguments
# Derived from input arguments
args_dict = dict()
if 'args_dict' in kwargs: # Preferred to use 'args_dict' if existed.
args_dict = kwargs['args_dict']
bin_dir = args_dict['bin_dir'] if 'bin_dir' in args_dict else bin_dir
model_dir = args_dict['model_dir'] if 'model_dir' in args_dict else model_dir
self.version = args_dict['version'] if 'version' in args_dict else version
suffix = '.exe' if sysstr == 'Windows' else ''
if self.version == 'MPI':
self.seims_exec = bin_dir + os.path.sep + 'seims_mpi' + suffix
else:
self.seims_exec = bin_dir + os.path.sep + 'seims_omp' + suffix
if not FileClass.is_file_exists(self.seims_exec): # If not support OpenMP, use `seims`!
self.seims_exec = bin_dir + os.path.sep + 'seims' + suffix
self.seims_exec = os.path.abspath(self.seims_exec)
self.model_dir = os.path.abspath(model_dir)
self.nthread = args_dict['nthread'] if 'nthread' in args_dict else nthread
self.lyrmtd = args_dict['lyrmtd'] if 'lyrmtd' in args_dict else lyrmtd
self.host = args_dict['host'] if 'host' in args_dict else host
self.port = args_dict['port'] if 'port' in args_dict else port
self.scenario_id = args_dict['scenario_id'] if 'scenario_id' in args_dict else scenario_id
self.calibration_id = args_dict[
'calibration_id'] if 'calibration_id' in args_dict else calibration_id
self.nprocess = args_dict['nprocess'] if 'nprocess' in args_dict else nprocess
self.mpi_bin = args_dict['mpi_bin'] if 'mpi_bin' in args_dict else mpi_bin
self.hosts_opt = args_dict['hosts_opt'] if 'hosts_opt' in args_dict else hosts_opt
self.hostfile = args_dict['hostfile'] if 'hostfile' in args_dict else hostfile
# Concatenate executable command
self.cmd = self.Command
self.run_success = False
self.output_dir = self.OutputDirectory
# Read model data from MongoDB
self.db_name = os.path.split(self.model_dir)[1]
self.outlet_id = self.OutletID
self.start_time, self.end_time = self.SimulatedPeriod
# Data maybe used after model run
self.timespan = dict()
self.obs_vars = list() # Observation types at the outlet
self.obs_value = dict() # Observation value, key: DATETIME, value: value list of obs_vars
self.sim_vars = list() # Simulation types at the outlet, which is part of obs_vars
self.sim_value = dict() # Simulation value, same as obs_value
# The format of sim_obs_dict:
# {VarName: {'UTCDATETIME': [t1, t2, ..., tn],
# 'Obs': [o1, o2, ..., on],
# 'Sim': [s1, s2, ..., sn]},
# ...
# }
self.sim_obs_dict = dict()
def d8hdisttostrm(np, p, src, dist, thresh, workingdir=None,
mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Run D8 horizontal distance down to stream.
"""
fname = TauDEM.func_name('d8hdisttostrm')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-p': p, '-src': src},
workingdir,
{'-thresh': thresh},
{'-dist': dist},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def moveoutletstostrm(np, flowdir, streamRaster, outlet, modifiedOutlet,
workingdir=None, mpiexedir=None,
exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Run move the given outlets to stream"""
fname = TauDEM.func_name('moveoutletstostrm')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-p': flowdir, '-src': streamRaster, '-o': outlet},
workingdir,
None,
{'-om': modifiedOutlet},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def convert2geojson(jsonfile, src_srs, dst_srs, src_file):
"""convert shapefile to geojson file"""
if os.path.exists(jsonfile):
os.remove(jsonfile)
if sysstr == 'Windows':
exepath = '"%s/Lib/site-packages/osgeo/ogr2ogr"' % sys.exec_prefix
else:
exepath = FileClass.get_executable_fullpath('ogr2ogr')
# os.system(s)
s = '%s -f GeoJSON -s_srs "%s" -t_srs %s %s %s' % (
exepath, src_srs, dst_srs, jsonfile, src_file)
UtilClass.run_command(s)
def streamnet(np, filleddem, flowdir, acc, streamRaster, modifiedOutlet,
streamOrder, chNetwork, chCoord, streamNet, subbasin, workingdir=None,
mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Run streamnet"""
fname = TauDEM.func_name('streamnet')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-fel': filleddem, '-p': flowdir, '-ad8': acc, '-src': streamRaster,
'-o': modifiedOutlet}, workingdir,
None,
{'-ord': streamOrder, '-tree': chNetwork, '-coord': chCoord,
'-net': streamNet, '-w': subbasin},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def areadinf(np, angfile, sca, outlet=None, wg=None, edgecontaimination=False,
workingdir=None, mpiexedir=None, exedir=None,
log_file=None, runtime_file=None, hostfile=None):
"""Run Accumulate area according to Dinf flow direction"""
# -nc means do not consider edge contaimination
if edgecontaimination:
in_params = {'-nc': None}
else:
in_params = None
fname = TauDEM.func_name('areadinf')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-ang': angfile, '-o': outlet, '-wg': wg}, workingdir,
in_params,
{'-sca': sca},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def dinfdistdown(np, ang, fel, slp, src, statsm, distm, edgecontamination, wg, dist,
workingdir=None, mpiexedir=None, exedir=None,
log_file=None, runtime_file=None, hostfile=None):
"""Run D-inf distance down to stream"""
in_params = {'-m': '%s %s' % (TauDEM.convertstatsmethod(statsm),
TauDEM.convertdistmethod(distm))}
if StringClass.string_match(edgecontamination, 'false') or edgecontamination is False:
in_params['-nc'] = None
fname = TauDEM.func_name('dinfdistdown')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-fel': fel, '-slp': slp, '-ang': ang, '-src': src, '-wg': wg},
workingdir,
in_params,
{'-dd': dist},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def dropanalysis(np, fel, p, ad8, ssa, outlet, minthresh, maxthresh, numthresh,
logspace, drp, workingdir=None,
mpiexedir=None, exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Drop analysis for optimal threshold for extracting stream."""
parstr = '%f %f %f' % (minthresh, maxthresh, numthresh)
if logspace == 'false':
parstr += ' 1'
else:
parstr += ' 0'
fname = TauDEM.func_name('dropanalysis')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-fel': fel, '-p': p, '-ad8': ad8, '-ssa': ssa, '-o': outlet},
workingdir,
{'-par': parstr},
{'-drp': drp},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def connectdown(np, p, acc, outlet, wtsd=None, workingdir=None, mpiexedir=None,
exedir=None, log_file=None, runtime_file=None, hostfile=None):
"""Reads an ad8 contributing area file,
identifies the location of the largest ad8 value as the outlet of the largest watershed"""
# If watershed is not specified, use acc to generate a mask layer.
if wtsd is None or not os.path.isfile(wtsd):
p, workingdir = TauDEM.check_infile_and_wp(p, workingdir)
wtsd = workingdir + os.sep + 'wtsd_default.tif'
RasterUtilClass.get_mask_from_raster(p, wtsd, True)
fname = TauDEM.func_name('connectdown')
return TauDEM.run(FileClass.get_executable_fullpath(fname, exedir),
{'-p': p, '-ad8': acc, '-w': wtsd},
workingdir,
None,
{'-o': outlet},
{'mpipath': mpiexedir, 'hostfile': hostfile, 'n': np},
{'logfile': log_file, 'runtimefile': runtime_file})
def ParamDefs(self):
"""Read cali_param_rng.def file
name,lower_bound,upper_bound
e.g.,
Param1,0,1
Param2,0.5,1.2
Param3,-1.0,1.0
Returns:
a dictionary containing:
- names - the names of the parameters
- bounds - a list of lists of lower and upper bounds
- num_vars - a scalar indicating the number of variables
(the length of names)
"""
# read param_defs.json if already existed
if self.param_defs:
return self.param_defs
# read param_range_def file and output to json file
client = ConnectMongoDB(self.cfg.model.host, self.cfg.model.port)
conn = client.get_conn()
db = conn[self.cfg.model.db_name]
collection = db['PARAMETERS']
names = list()
bounds = list()
num_vars = 0
if not FileClass.is_file_exists(self.cfg.param_range_def):
raise ValueError('Parameters definition file: %s is not'
' existed!' % self.cfg.param_range_def)
items = read_data_items_from_txt(self.cfg.param_range_def)
for item in items:
if len(item) < 3:
continue
# find parameter name, print warning message if not existed
cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True)
if not cursor.count():
print('WARNING: parameter %s is not existed!' % item[0])
continue
num_vars += 1
names.append(item[0])
bounds.append([float(item[1]), float(item[2])])
self.param_defs = {'names': names, 'bounds': bounds, 'num_vars': num_vars}
return self.param_defs
def mask_origin_delineated_data(cfg):
"""Mask the original delineated data by Subbasin raster."""
subbasin_tau_file = cfg.taudems.subbsn
geodata2dbdir = cfg.dirs.geodata2db
UtilClass.mkdir(geodata2dbdir)
mask_file = cfg.spatials.mask
RasterUtilClass.get_mask_from_raster(subbasin_tau_file, mask_file)
# Total 12 raster files
original_files = [cfg.taudems.subbsn, cfg.taudems.d8flow, cfg.taudems.stream_raster,
cfg.taudems.slp, cfg.taudems.filldem, cfg.taudems.d8acc,
cfg.taudems.stream_order, cfg.taudems.dinf, cfg.taudems.dinf_d8dir,
cfg.taudems.dinf_slp, cfg.taudems.dinf_weight,
cfg.taudems.dist2stream_d8]
# output masked files
output_files = [cfg.taudems.subbsn_m, cfg.taudems.d8flow_m, cfg.taudems.stream_m,
cfg.spatials.slope, cfg.spatials.filldem, cfg.spatials.d8acc,
cfg.spatials.stream_order, cfg.spatials.dinf, cfg.spatials.dinf_d8dir,
cfg.spatials.dinf_slp, cfg.spatials.dinf_weight,
cfg.spatials.dist2stream_d8]
default_values = list()
for i in range(len(original_files)):
default_values.append(DEFAULT_NODATA)
# other input rasters need to be masked
# soil and landuse
FileClass.check_file_exists(cfg.soil)
FileClass.check_file_exists(cfg.landuse)
original_files.append(cfg.soil)
output_files.append(cfg.spatials.soil_type)
default_values.append(cfg.default_soil)
original_files.append(cfg.landuse)
output_files.append(cfg.spatials.landuse)
default_values.append(cfg.default_landuse)
# Additional raster file
for k, v in cfg.additional_rs.items():
org_v = v
if not FileClass.is_file_exists(org_v):
v = cfg.spatial_dir + os.path.sep + org_v
if not FileClass.is_file_exists(v):
print('WARNING: The additional file %s MUST be located in '
'SPATIAL_DATA_DIR, or provided as full file path!' % k)
continue
original_files.append(v)
output_files.append(cfg.dirs.geodata2db + os.path.sep + k + '.tif')
default_values.append(DEFAULT_NODATA)
config_file = cfg.logs.mask_cfg
# run mask operation
print('Mask original delineated data by Subbasin raster...')
SpatialDelineation.mask_raster_cpp(cfg.seims_bin, mask_file, original_files,
output_files, default_values, config_file)
def interpolate_observed_data_to_regular_interval(in_file,
time_interval,
start_time,
end_time,
eliminate_zero=False,
time_sys_output='UTCTIME',
day_divided_hour=0):
"""
Interpolate not regular observed data to regular time interval data.
Args:
in_file: input data file, the basic format is as follows:
line 1: #<time_system> [<time_zone>], e.g., #LOCALTIME 8, #UTCTIME
line 2: DATETIME,field1,field2,...
line 3: YYYY-mm-dd HH:MM:SS,field1_value,field2_value,...
line 4: ...
...
Field name can be PCP, FLOW, SED
the unit is mm/h, m3/s, g/L (i.e., kg/m3), respectively.
time_interval: time interval, unit is minute, e.g., daily output is 1440
start_time: start time, the format must be 'YYYY-mm-dd HH:MM:SS', and the time system
is based on time_sys.
end_time: end time, see also start_time.
eliminate_zero: Boolean flag. If true, the time interval without original records will
not be output.
time_sys_output: time system of output time_system, the format must be
'<time_system> [<time_zone>]', e.g.,
'LOCALTIME'
'LOCALTIME 8'
'UTCTIME' (default)
day_divided_hour: If the time_interval is equal to N*1440, this parameter should be
carefully specified. The value must range from 0 to 23. e.g.,
day_divided_hour ==> day ranges (all expressed as 2013-02-03)
0 ==> 2013-02-03 00:00:00 to 2013-02-03 23:59:59 (default)
8 ==> 2013-02-03 08:00:00 to 2013-02-04 07:59:59
20 ==> 2013-02-03 20:00:00 to 2013-02-04 19:59:59
Returns:
The output data files are located in the same directory with the input file.
The nomenclature is: <field name>_<time system>_<time interval>_<nonzero>, e.g.,
pcp_utctime_1440_nonzero.txt, flow_localtime_60.txt
"""
FileClass.check_file_exists(in_file)
time_sys_input, time_zone_input = HydroClimateUtilClass.get_time_system_from_data_file(
in_file)
data_items = read_data_items_from_txt(in_file)
flds = data_items[0][:]
data_items.remove(flds)
if not 0 <= day_divided_hour <= 23:
raise ValueError('Day divided hour must range from 0 to 23!')
try:
date_idx = flds.index('DATETIME')
flds.remove('DATETIME')
except ValueError:
raise ValueError('DATETIME must be one of the fields!')
# available field
available_flds = ['FLOW', 'SED', 'PCP']
def check_avaiable_field(cur_fld):
"""Check if the given field name is supported."""
support_flag = False
for fff in available_flds:
if fff.lower() in cur_fld.lower():
support_flag = True
break
return support_flag
ord_data = OrderedDict()
time_zone_output = time.timezone / -3600
if time_sys_output.lower().find('local') >= 0:
tmpstrs = StringClass.split_string(time_sys_output, [' '])
if len(tmpstrs) == 2 and MathClass.isnumerical(tmpstrs[1]):
time_zone_output = int(tmpstrs[1])
time_sys_output = 'LOCALTIME'
else:
time_sys_output = 'UTCTIME'
time_zone_output = 0
for item in data_items:
org_datetime = StringClass.get_datetime(item[date_idx])
if time_sys_input == 'LOCALTIME':
org_datetime -= timedelta(hours=time_zone_input)
# now, org_datetime is UTC time.
if time_sys_output == 'LOCALTIME':
org_datetime += timedelta(hours=time_zone_output)
# now, org_datetime is consistent with the output time system
ord_data[org_datetime] = list()
for i, v in enumerate(item):
if i == date_idx:
continue
if MathClass.isnumerical(v):
ord_data[org_datetime].append(float(v))
else:
ord_data[org_datetime].append(v)
# print(ord_data)
itp_data = OrderedDict()
out_time_delta = timedelta(minutes=time_interval)
sdatetime = StringClass.get_datetime(start_time)
edatetime = StringClass.get_datetime(end_time)
item_dtime = sdatetime
if time_interval % 1440 == 0:
item_dtime = sdatetime.replace(hour=0, minute=0, second=0) + \
timedelta(minutes=day_divided_hour * 60)
while item_dtime <= edatetime:
# print(item_dtime)
# if item_dtime.month == 12 and item_dtime.day == 31:
# print("debug")
sdt = item_dtime # start datetime of records
edt = item_dtime + out_time_delta # end datetime of records
# get original data items
org_items = list()
pre_dt = list(ord_data.keys())[0]
pre_added = False
for i, v in list(ord_data.items()):
if sdt <= i < edt:
if not pre_added and pre_dt < sdt < i and sdt - pre_dt < out_time_delta:
# only add one item that less than sdt.
org_items.append([pre_dt] + ord_data.get(pre_dt))
pre_added = True
org_items.append([i] + v)
if i > edt:
break
pre_dt = i
if len(org_items) > 0:
org_items.append([edt]) # Just add end time for compute convenient
if org_items[0][0] < sdt:
org_items[0][
0] = sdt # set the begin datetime of current time interval
# if eliminate time interval without original records
# initial interpolated list
itp_data[item_dtime] = [0.] * len(flds)
if len(org_items) == 0:
if eliminate_zero:
itp_data.popitem()
item_dtime += out_time_delta
continue
# core interpolation code
flow_idx = -1
for v_idx, v_name in enumerate(flds):
if not check_avaiable_field(v_name):
continue
if 'SED' in v_name.upper(): # FLOW must be existed
for v_idx2, v_name2 in enumerate(flds):
if 'FLOW' in v_name2.upper():
flow_idx = v_idx2
break
if flow_idx < 0:
raise RuntimeError(
'To interpolate SED, FLOW must be provided!')
for v_idx, v_name in enumerate(flds):
if not check_avaiable_field(v_name):
continue
itp_value = 0.
itp_auxiliary_value = 0.
for org_item_idx, org_item_dtv in enumerate(org_items):
if org_item_idx == 0:
continue
org_item_dt = org_item_dtv[0]
pre_item_dtv = org_items[org_item_idx - 1]
pre_item_dt = pre_item_dtv[0]
tmp_delta_dt = org_item_dt - pre_item_dt
tmp_delta_secs = tmp_delta_dt.days * 86400 + tmp_delta_dt.seconds
if 'SED' in v_name.upper():
itp_value += pre_item_dtv[v_idx + 1] * pre_item_dtv[flow_idx + 1] * \
tmp_delta_secs
itp_auxiliary_value += pre_item_dtv[flow_idx +
1] * tmp_delta_secs
else:
itp_value += pre_item_dtv[v_idx + 1] * tmp_delta_secs
if 'SED' in v_name.upper():
if MathClass.floatequal(itp_auxiliary_value, 0.):
itp_value = 0.
print('WARNING: Flow is 0 for %s, please check!' %
item_dtime.strftime('%Y-%m-%d %H:%M:%S'))
itp_value /= itp_auxiliary_value
elif 'FLOW' in v_name.upper():
itp_value /= (out_time_delta.days * 86400 +
out_time_delta.seconds)
elif 'PCP' in v_name.upper(
): # the input is mm/h, and output is mm
itp_value /= 3600.
itp_data[item_dtime][v_idx] = round(itp_value, 4)
item_dtime += out_time_delta
# for i, v in itp_data.items():
# print(i, v)
# output to files
work_path = os.path.dirname(in_file)
header_str = '#' + time_sys_output
if time_sys_output == 'LOCALTIME':
header_str = header_str + ' ' + str(time_zone_output)
for idx, fld in enumerate(flds):
if not check_avaiable_field(fld):
continue
file_name = fld + '_' + time_sys_output + '_' + str(time_interval)
if eliminate_zero:
file_name += '_nonzero'
file_name += '.txt'
out_file = work_path + os.path.sep + file_name
with open(out_file, 'w') as f:
f.write(header_str + '\n')
f.write('DATETIME,' + fld + '\n')
for i, v in list(itp_data.items()):
cur_line = i.strftime('%Y-%m-%d %H:%M:%S') + ',' + str(
v[idx]) + '\n'
f.write(cur_line)
def scenario_from_texts(cfg, main_db, scenario_db):
"""Import BMPs Scenario data to MongoDB
Args:
cfg: SEIMS configuration object
main_db: climate database
scenario_db: scenario database
Returns:
False if failed, otherwise True.
"""
if not cfg.use_scernario:
return False
print('Import BMP Scenario Data... ')
bmp_files = FileClass.get_filename_by_suffixes(cfg.scenario_dir,
['.txt', '.csv'])
bmp_tabs = list()
bmp_tabs_path = list()
for f in bmp_files:
bmp_tabs.append(f.split('.')[0])
bmp_tabs_path.append(cfg.scenario_dir + os.path.sep + f)
# initialize if collection not existed
c_list = scenario_db.collection_names()
for item in bmp_tabs:
if not StringClass.string_in_list(item.upper(), c_list):
scenario_db.create_collection(item.upper())
else:
scenario_db.drop_collection(item.upper())
# Read subbasin.tif and dist2Stream.tif
subbasin_r = RasterUtilClass.read_raster(cfg.spatials.subbsn)
dist2stream_r = RasterUtilClass.read_raster(
cfg.spatials.dist2stream_d8)
# End reading
for j, bmp_txt in enumerate(bmp_tabs_path):
bmp_tab_name = bmp_tabs[j]
data_array = read_data_items_from_txt(bmp_txt)
field_array = data_array[0]
data_array = data_array[1:]
for item in data_array:
dic = dict()
for i, field_name in enumerate(field_array):
if MathClass.isnumerical(item[i]):
v = float(item[i])
if v % 1. == 0.:
v = int(v)
dic[field_name.upper()] = v
else:
dic[field_name.upper()] = str(item[i]).upper()
if StringClass.string_in_list(ImportScenario2Mongo._LocalX, list(dic.keys())) and \
StringClass.string_in_list(ImportScenario2Mongo._LocalY, list(dic.keys())):
subbsn_id = subbasin_r.get_value_by_xy(
dic[ImportScenario2Mongo._LocalX.upper()],
dic[ImportScenario2Mongo._LocalY.upper()])
distance = dist2stream_r.get_value_by_xy(
dic[ImportScenario2Mongo._LocalX.upper()],
dic[ImportScenario2Mongo._LocalY.upper()])
if subbsn_id is not None and distance is not None:
dic[ImportScenario2Mongo._SUBBASINID] = int(subbsn_id)
dic[ImportScenario2Mongo._DISTDOWN] = float(distance)
scenario_db[bmp_tab_name.upper()].find_one_and_replace(
dic, dic, upsert=True)
else:
scenario_db[bmp_tab_name.upper()].find_one_and_replace(
dic, dic, upsert=True)
# print('BMP tables are imported.')
# Write BMP database name into Model workflow database
c_list = main_db.collection_names()
if not StringClass.string_in_list(DBTableNames.main_scenario, c_list):
main_db.create_collection(DBTableNames.main_scenario)
bmp_info_dic = dict()
bmp_info_dic[ImportScenario2Mongo._FLD_DB] = cfg.bmp_scenario_db
main_db[DBTableNames.main_scenario].find_one_and_replace(bmp_info_dic,
bmp_info_dic,
upsert=True)
return True
def __init__(
self,
bin_dir='', # type: AnyStr # The directory of SEIMS binary
model_dir='', # type: AnyStr # The directory of SEIMS model
nthread=4, # type: int # Thread number for OpenMP
lyrmtd=0, # type: int # Layering method, can be 0 (UP_DOWN) or 1 (DOWN_UP)
host='127.0.0.1', # type: AnyStr # MongoDB host address, default is `localhost`
port=27017, # type: int # MongoDB port, default is 27017
scenario_id=-1, # type: int # Scenario ID defined in `<model>_Scenario` database
calibration_id=-1, # type: int # Calibration ID used for model auto-calibration
version='OMP', # type: AnyStr # SEIMS version, can be `MPI` or `OMP` (default)
nprocess=1, # type: int # Process number for MPI
mpi_bin='', # type: AnyStr # Full path of MPI executable file, e.g., './mpirun`
hosts_opt='-f', # type: AnyStr # Option for assigning hosts,
# e.g., `-f`, `-hostfile`, `-machine`, `-machinefile`
hostfile='', # type: AnyStr # File containing host names,
# or file mapping process numbers to machines
simu_stime=None, # type: Optional[datetime, AnyStr] # Start time of simulation
simu_etime=None, # type: Optional[datetime, AnyStr] # End time of simulation
args_dict=None # type: Dict[AnyStr, Optional[AnyStr, datetime, int]]
):
# type: (...) -> None
# Derived from input arguments
if args_dict is None: # Preferred to use 'args_dict' if existed.
args_dict = dict()
bin_dir = args_dict['bin_dir'] if 'bin_dir' in args_dict else bin_dir
model_dir = args_dict[
'model_dir'] if 'model_dir' in args_dict else model_dir
self.version = args_dict[
'version'] if 'version' in args_dict else version
suffix = '.exe' if sysstr == 'Windows' else ''
if self.version == 'MPI':
self.seims_exec = '%s/seims_mpi%s' % (bin_dir, suffix)
else:
self.seims_exec = '%s/seims_omp%s' % (bin_dir, suffix)
if not FileClass.is_file_exists(
self.seims_exec): # If not support OpenMP, use `seims`!
self.seims_exec = '%s/seims%s' % (bin_dir, suffix)
self.seims_exec = os.path.abspath(self.seims_exec)
self.model_dir = os.path.abspath(model_dir)
self.nthread = args_dict[
'nthread'] if 'nthread' in args_dict else nthread
self.lyrmtd = args_dict['lyrmtd'] if 'lyrmtd' in args_dict else lyrmtd
self.host = args_dict['host'] if 'host' in args_dict else host
self.port = args_dict['port'] if 'port' in args_dict else port
self.scenario_id = args_dict[
'scenario_id'] if 'scenario_id' in args_dict else scenario_id
self.calibration_id = args_dict['calibration_id'] \
if 'calibration_id' in args_dict else calibration_id
self.nprocess = args_dict[
'nprocess'] if 'nprocess' in args_dict else nprocess
self.mpi_bin = args_dict[
'mpi_bin'] if 'mpi_bin' in args_dict else mpi_bin
self.hosts_opt = args_dict[
'hosts_opt'] if 'hosts_opt' in args_dict else hosts_opt
self.hostfile = args_dict[
'hostfile'] if 'hostfile' in args_dict else hostfile
self.simu_stime = args_dict[
'simu_stime'] if 'simu_stime' in args_dict else simu_stime
self.simu_etime = args_dict[
'simu_etime'] if 'simu_etime' in args_dict else simu_etime
if is_string(
self.simu_stime) and not isinstance(self.simu_stime, datetime):
self.simu_stime = StringClass.get_datetime(self.simu_stime)
if is_string(
self.simu_etime) and not isinstance(self.simu_etime, datetime):
self.simu_etime = StringClass.get_datetime(self.simu_etime)
# Concatenate executable command
self.cmd = self.Command
self.run_success = False
self.output_dir = self.OutputDirectory
# Read model data from MongoDB
self.db_name = os.path.split(self.model_dir)[1]
self.outlet_id = self.OutletID
self.start_time, self.end_time = self.SimulatedPeriod # Note: Times period in FILE_IN.
self.output_items = dict() # type: Dict[AnyStr, Union[List[AnyStr]]]
# Data maybe used after model run
self.timespan = dict(
) # type: Dict[AnyStr, Dict[AnyStr, Union[float, Dict[AnyStr, float]]]]
self.obs_vars = list(
) # type: List[AnyStr] # Observation types at the outlet
self.obs_value = dict(
) # type: Dict[datetime, List[float]] # Observation value
self.sim_vars = list(
) # type: List[AnyStr] # Simulation types, part of `obs_vars`
self.sim_value = dict(
) # type: Dict[datetime, List[float]] # Simulation value
# The format of sim_obs_dict:
# {VarName: {'UTCDATETIME': [t1, t2, ..., tn],
# 'Obs': [o1, o2, ..., on],
# 'Sim': [s1, s2, ..., sn]},
# ...
# }
self.sim_obs_dict = dict(
) # type: Dict[AnyStr, Dict[AnyStr, Union[float, List[Union[datetime, float]]]]]
self.runtime = 0.
self.runlogs = list() # type: List[AnyStr]
def run(function_name, in_files, wp=None, in_params=None, out_files=None, mpi_params=None,
log_params=None):
"""
Run TauDEM function.
- 1. The command will not execute if any input file does not exist.
- 2. An error will be detected after running the TauDEM command if
any output file does not exist;
Args:
function_name (str): Full path of TauDEM function.
in_files (dict, required): Dict of pairs of parameter id (string) and file path
(string or list) for input files, e.g.::
{'-z': '/full/path/to/dem.tif'}
wp (str, optional): Workspace for outputs. If not specified, the directory of the
first input file in ``in_files`` will be used.
in_params (dict, optional): Dict of pairs of parameter id (string) and value
(or None for a flag parameter without a value) for input parameters, e.g.::
{'-nc': None}
{'-thresh': threshold}
{'-m': 'ave' 's', '-nc': None}
out_files (dict, optional): Dict of pairs of parameter id (string) and file
path (string or list) for output files, e.g.::
{'-fel': 'filleddem.tif'}
{'-maxS': ['harden.tif', 'maxsimi.tif']}
mpi_params (dict, optional): Dict of pairs of parameter id (string) and value or
path for MPI setting, e.g.::
{'mpipath':'/soft/bin','hostfile':'/soft/bin/cluster.node','n':4}
{'mpipath':'/soft/bin', 'n':4}
{'n':4}
log_params (dict, optional): Dict of pairs of parameter id (string) and value or
path for runtime and log output parameters. e.g.::
{'logfile': '/home/user/log.txt',
'runtimefile': '/home/user/runtime.txt'}
Returns:
True if TauDEM run successfully, otherwise False.
"""
# Check input files
if in_files is None:
TauDEM.error('Input files parameter is required!')
if not isinstance(in_files, dict):
TauDEM.error('The input files parameter must be a dict!')
for (pid, infile) in iteritems(in_files):
if infile is None:
continue
if isinstance(infile, list) or isinstance(infile, tuple):
for idx, inf in enumerate(infile):
if inf is None:
continue
inf, wp = TauDEM.check_infile_and_wp(inf, wp)
in_files[pid][idx] = inf
continue
if os.path.exists(infile):
infile, wp = TauDEM.check_infile_and_wp(infile, wp)
in_files[pid] = os.path.abspath(infile)
else:
# For more flexible input files extension.
# e.g., -inputtags 1 <path/to/tag1.tif> 2 <path/to/tag2.tif> ...
# in such unpredictable circumstance, we cannot check the existance of
# input files, so the developer will check it in other place.
if len(StringClass.split_string(infile, ' ')) > 1:
continue
else: # the infile still should be a existing file, so check in workspace
if wp is None:
TauDEM.error('Workspace should not be None!')
infile = wp + os.sep + infile
if not os.path.exists(infile):
TauDEM.error('Input files parameter %s: %s is not existed!' %
(pid, infile))
in_files[pid] = os.path.abspath(infile)
# Make workspace dir if not existed
UtilClass.mkdir(wp)
# Check the log parameter
log_file = None
runtime_file = None
if log_params is not None:
if not isinstance(log_params, dict):
TauDEM.error('The log parameter must be a dict!')
if 'logfile' in log_params and log_params['logfile'] is not None:
log_file = log_params['logfile']
# If log_file is just a file name, then save it in the default workspace.
if os.sep not in log_file:
log_file = wp + os.sep + log_file
log_file = os.path.abspath(log_file)
if 'runtimefile' in log_params and log_params['runtimefile'] is not None:
runtime_file = log_params['runtimefile']
# If log_file is just a file name, then save it in the default workspace.
if os.sep not in runtime_file:
runtime_file = wp + os.sep + runtime_file
runtime_file = os.path.abspath(runtime_file)
# remove out_files to avoid any file IO related error
new_out_files = list()
if out_files is not None:
if not isinstance(out_files, dict):
TauDEM.error('The output files parameter must be a dict!')
for (pid, out_file) in iteritems(out_files):
if out_file is None:
continue
if isinstance(out_file, list) or isinstance(out_file, tuple):
for idx, outf in enumerate(out_file):
if outf is None:
continue
outf = FileClass.get_file_fullpath(outf, wp)
FileClass.remove_files(outf)
out_files[pid][idx] = outf
new_out_files.append(outf)
else:
out_file = FileClass.get_file_fullpath(out_file, wp)
FileClass.remove_files(out_file)
out_files[pid] = out_file
new_out_files.append(out_file)
# concatenate command line
commands = list()
# MPI header
if mpi_params is not None:
if not isinstance(mpi_params, dict):
TauDEM.error('The MPI settings parameter must be a dict!')
if 'mpipath' in mpi_params and mpi_params['mpipath'] is not None:
commands.append(mpi_params['mpipath'] + os.sep + 'mpiexec')
else:
commands.append('mpiexec')
if 'hostfile' in mpi_params and mpi_params['hostfile'] is not None \
and not StringClass.string_match(mpi_params['hostfile'], 'none') \
and os.path.isfile(mpi_params['hostfile']):
commands.append('-f')
commands.append(mpi_params['hostfile'])
if 'n' in mpi_params and mpi_params['n'] > 1:
commands.append('-n')
commands.append(str(mpi_params['n']))
else: # If number of processor is less equal than 1, then do not call mpiexec.
commands = []
# append TauDEM function name, which can be full path or just one name
commands.append(function_name)
# append input files
for (pid, infile) in iteritems(in_files):
if infile is None:
continue
if pid[0] != '-':
pid = '-' + pid
commands.append(pid)
if isinstance(infile, list) or isinstance(infile, tuple):
commands.append(' '.join(tmpf for tmpf in infile))
else:
commands.append(infile)
# append input parameters
if in_params is not None:
if not isinstance(in_params, dict):
TauDEM.error('The input parameters must be a dict!')
for (pid, v) in iteritems(in_params):
if pid[0] != '-':
pid = '-' + pid
commands.append(pid)
# allow for parameter which is an flag without value
if v != '' and v is not None:
if MathClass.isnumerical(v):
commands.append(str(v))
else:
commands.append(v)
# append output parameters
if out_files is not None:
for (pid, outfile) in iteritems(out_files):
if outfile is None:
continue
if pid[0] != '-':
pid = '-' + pid
commands.append(pid)
if isinstance(outfile, list) or isinstance(outfile, tuple):
commands.append(' '.join(tmpf for tmpf in outfile))
else:
commands.append(outfile)
# run command
runmsg = UtilClass.run_command(commands)
TauDEM.log(runmsg, log_file)
TauDEM.output_runtime_to_log(function_name, runmsg, runtime_file)
# Check out_files, raise RuntimeError if not exist.
for of in new_out_files:
if not os.path.exists(of):
TauDEM.error('%s failed, and the %s was not generated!' % (function_name, of))
return False
return True
def read_param_ranges(self):
"""Read param_rng.def file
name,lower_bound,upper_bound,group,dist
(group and dist are optional)
e.g.,
Param1,0,1[,Group1][,dist1]
Param2,0,1[,Group2][,dist2]
Param3,0,1[,Group3][,dist3]
Returns:
a dictionary containing:
- names - the names of the parameters
- bounds - a list of lists of lower and upper bounds
- num_vars - a scalar indicating the number of variables
(the length of names)
- groups - a list of group names (strings) for each variable
- dists - a list of distributions for the problem,
None if not specified or all uniform
"""
# read param_defs.json if already existed
if not self.param_defs:
if FileClass.is_file_exists(self.cfg.outfiles.param_defs_json):
with open(self.cfg.outfiles.param_defs_json, 'r') as f:
self.param_defs = UtilClass.decode_strs_in_dict(
json.load(f))
return
# read param_range_def file and output to json file
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
collection = db['PARAMETERS']
names = list()
bounds = list()
groups = list()
dists = list()
num_vars = 0
items = read_data_items_from_txt(self.cfg.param_range_def)
for item in items:
if len(item) < 3:
continue
# find parameter name, print warning message if not existed
cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True)
if not cursor.count():
print('WARNING: parameter %s is not existed!' % item[0])
continue
num_vars += 1
names.append(item[0])
bounds.append([float(item[1]), float(item[2])])
# If the fourth column does not contain a group name, use
# the parameter name
if len(item) >= 4:
groups.append(item[3])
else:
groups.append(item[0])
if len(item) >= 5:
dists.append(item[4])
else:
dists.append('unif')
if groups == names:
groups = None
elif len(set(groups)) == 1:
raise ValueError(
'Only one group defined, results will not bemeaningful')
# setting dists to none if all are uniform
# because non-uniform scaling is not needed
if all([d == 'unif' for d in dists]):
dists = None
self.param_defs = {
'names': names,
'bounds': bounds,
'num_vars': num_vars,
'groups': groups,
'dists': dists
}
# Save as json, which can be loaded by json.load()
json_data = json.dumps(self.param_defs,
indent=4,
cls=SpecialJsonEncoder)
with open(self.cfg.outfiles.param_defs_json, 'w') as f:
f.write(json_data)
def evaluate_models(self):
"""Run SEIMS for objective output variables, and write out.
"""
if self.output_values is None or len(self.output_values) == 0:
if FileClass.is_file_exists(self.cfg.outfiles.output_values_txt):
self.output_values = numpy.loadtxt(
self.cfg.outfiles.output_values_txt)
return
assert (self.run_count > 0)
# model configurations
model_cfg_dict = self.model.ConfigDict
# Parameters to be evaluated
input_eva_vars = self.cfg.evaluate_params
# split tasks if needed
task_num = self.run_count // 480 # In our cluster, the largest workers number is 96.
if task_num == 0:
split_seqs = [range(self.run_count)]
else:
split_seqs = numpy.array_split(numpy.arange(self.run_count),
task_num + 1)
split_seqs = [a.tolist() for a in split_seqs]
# Loop partitioned tasks
run_model_stime = time.time()
exec_times = list() # execute time of all model runs
for idx, cali_seqs in enumerate(split_seqs):
cur_out_file = '%s/outputs_%d.txt' % (
self.cfg.outfiles.output_values_dir, idx)
if FileClass.is_file_exists(cur_out_file):
continue
model_cfg_dict_list = list()
for i, caliid in enumerate(cali_seqs):
tmpcfg = deepcopy(model_cfg_dict)
tmpcfg['calibration_id'] = caliid
model_cfg_dict_list.append(tmpcfg)
try: # parallel on multiprocessor or clusters using SCOOP
from scoop import futures
output_models = list(
futures.map(create_run_model, model_cfg_dict_list))
except ImportError or ImportWarning: # serial
output_models = list(map(create_run_model,
model_cfg_dict_list))
time.sleep(
0.1
) # Wait a moment in case of unpredictable file system error
# Read observation data from MongoDB only once
if len(
output_models
) < 1: # Although this is not gonna happen, just for insurance.
continue
obs_vars, obs_data_dict = output_models[0].ReadOutletObservations(
input_eva_vars)
if (len(obs_vars)) < 1: # Make sure the observation data exists.
continue
# Loop the executed models
eva_values = list()
for imod, mod_obj in enumerate(output_models):
# Read executable timespan of each model run
exec_times.append(mod_obj.GetTimespan())
# Set observation data since there is no need to read from MongoDB.
if imod != 0:
mod_obj.SetOutletObservations(obs_vars, obs_data_dict)
# Read simulation
mod_obj.ReadTimeseriesSimulations(self.cfg.psa_stime,
self.cfg.psa_etime)
# Calculate NSE, R2, RMSE, PBIAS, RSR, ln(NSE), NSE1, and NSE3
self.objnames, obj_values = mod_obj.CalcTimeseriesStatistics(
mod_obj.sim_obs_dict)
eva_values.append(obj_values)
# delete model output directory for saving storage
rmtree(mod_obj.output_dir)
if not isinstance(eva_values, numpy.ndarray):
eva_values = numpy.array(eva_values)
numpy.savetxt(cur_out_file, eva_values, delimiter=' ', fmt='%.4f')
# Save as pickle data for further usage. DO not save all models which maybe very large!
cur_model_out_file = '%s/models_%d.pickle' % (
self.cfg.outfiles.output_values_dir, idx)
with open(cur_model_out_file, 'wb') as f:
pickle.dump(output_models, f)
exec_times = numpy.array(exec_times)
numpy.savetxt('%s/exec_time_allmodelruns.txt' % self.cfg.psa_outpath,
exec_times,
delimiter=' ',
fmt='%.4f')
print('Running time of all SEIMS models:\n'
'\tIO\tCOMP\tSIMU\tRUNTIME\n'
'MAX\t%s\n'
'MIN\t%s\n'
'AVG\t%s\n'
'SUM\t%s\n' %
('\t'.join('%.3f' % v for v in exec_times.max(0)), '\t'.join(
'%.3f' % v for v in exec_times.min(0)), '\t'.join(
'%.3f' % v for v in exec_times.mean(0)), '\t'.join(
'%.3f' % v for v in exec_times.sum(0))))
print('Running time of executing SEIMS models: %.2fs' %
(time.time() - run_model_stime))
# Save objective names as pickle data for further usgae
with open('%s/objnames.pickle' % self.cfg.psa_outpath, 'wb') as f:
pickle.dump(self.objnames, f)
# load the first part of output values
self.output_values = numpy.loadtxt('%s/outputs_0.txt' %
self.cfg.outfiles.output_values_dir)
if task_num == 0:
import shutil
shutil.move(
'%s/outputs_0.txt' % self.cfg.outfiles.output_values_dir,
self.cfg.outfiles.output_values_txt)
shutil.rmtree(self.cfg.outfiles.output_values_dir)
return
for idx in range(1, task_num + 1):
tmp_outputs = numpy.loadtxt(
'%s/outputs_%d.txt' %
(self.cfg.outfiles.output_values_dir, idx))
self.output_values = numpy.concatenate(
(self.output_values, tmp_outputs))
numpy.savetxt(self.cfg.outfiles.output_values_txt,
self.output_values,
delimiter=' ',
fmt='%.4f')
def downstream_method_whitebox(stream_raster,
flow_dir_raster,
hillslope_out,
d8alg="taudem",
stream_value_method=-1):
"""Algorithm modified from Whitebox GAT v3.4.0.
source code: https://github.com/jblindsay/whitebox-geospatial-analysis-tools/
blob/master/HydroTools/src/plugins/Hillslopes.java
Args:
stream_raster: Stream cell value greater than 0 is identified by stream
The input stream are recommended sequenced as 1, 2, 3...
flow_dir_raster: D8 flow direction in TauDEM code
hillslope_out: With the sequenced stream IDs, the output hillslope will be numbered:
- Header hillslope: MaxStreamID + (current_id - 1) * 3 + 1
- Right hillslope: MaxStreamID + (current_id - 1) * 3 + 2
- Left hillslope: MaxStreamID + (current_id - 1) * 3 + 3
d8alg: Currently, "TauDEM", "ArcGIS", and "Whitebox" are supported.
stream_value_method: stream value assigned method, depend on this parameter,
the output hillslope will be appended as follows:
-1 - all the four files will be output.
0 - keep stream link code, which has the default file name
1 - Set to the value of right hillslope and head hillslope, <name>_r.tif
2 - Set to the value of left hillslope and head hillslope, <name>_l.tif
3 - Set stream cell to NoData, <name>_n.tif
"""
print('Delineating hillslopes (header, left, and right hillslope)...')
streamr = RasterUtilClass.read_raster(stream_raster)
stream_data = streamr.data
stream_nodata = streamr.noDataValue
geotrans = streamr.geotrans
srs = streamr.srs
nrows = streamr.nRows
ncols = streamr.nCols
datatype = streamr.dataType
flowd8r = RasterUtilClass.read_raster(flow_dir_raster)
flowd8_data = flowd8r.data
flowd8_nodata = flowd8r.noDataValue
if flowd8r.nRows != nrows or flowd8r.nCols != ncols:
raise ValueError("The input extent of D8 flow direction is not "
"consistent with stream data!")
# definition of utility functions
def inflow_stream_number(vrow, vcol, flowmodel="taudem"):
"""
Count the inflow stream cell number and coordinates of all inflow cells
Args:
vrow: row number
vcol: col number
flowmodel: D8 flow direction algorithm.
Returns:
neighb_stream_cell_num: inflow cells number that is stream
cell_coors: inflow cell coordinates, the size() is equal or greater
than neighb_stream_cell_num
"""
neighb_stream_cell_num = 0
cell_coors = []
for c in range(8):
newrow = vrow + FlowModelConst.ccw_drow[c]
newcol = vcol + FlowModelConst.ccw_dcol[c]
if newrow < 0 or newrow >= nrows or newcol < 0 or newcol >= ncols:
continue
if flowd8_data[newrow][
newcol] == FlowModelConst.d8_inflows.get(flowmodel)[c]:
cell_coors.append((newrow, newcol))
if stream_data[newrow][newcol] > 0 \
and stream_data[newrow][newcol] != stream_nodata:
neighb_stream_cell_num += 1
return neighb_stream_cell_num, cell_coors
def assign_sequenced_stream_ids(c_id, vrow, vcol, flowmodel="taudem"):
"""set sequenced stream IDs"""
in_strm_num, in_coors = inflow_stream_number(vrow, vcol, flowmodel)
if in_strm_num == 0:
# it's a headwater location so start a downstream flowpath
c_id += 1
tmp_row = vrow
tmp_col = vcol
sequenced_stream_d[tmp_row][tmp_col] = c_id
searched_flag = True
while searched_flag:
# find the downslope neighbour
tmpflowd8 = flowd8_data[tmp_row][tmp_col]
if tmpflowd8 < 0 or tmpflowd8 == flowd8_nodata:
if stream_data[tmp_row][tmp_col] > 0 \
and stream_data[tmp_row][tmp_col] != stream_nodata:
# it is a valid stream cell and probably just has no downslope
# neighbour (e.g. at the edge of the grid)
sequenced_stream_d[tmp_row][tmp_col] = c_id
break
tmp_row, tmp_col = D8Util.downstream_index(
tmpflowd8, tmp_row, tmp_col, flowmodel)
if tmp_row < 0 or tmp_row >= nrows or tmp_col < 0 or tmp_col >= ncols:
break
if stream_data[tmp_row][tmp_col] <= 0:
searched_flag = False # it is not a stream cell
else:
if sequenced_stream_d[tmp_row][tmp_col] > 0:
# run into a larger stream, end the downstream search
break
# is it a confluence (conjunction node)
in_strm_num, in_coors = inflow_stream_number(
tmp_row, tmp_col, flowmodel)
if in_strm_num >= 2:
c_id += 1
sequenced_stream_d[tmp_row][tmp_col] = c_id
return c_id
def assign_hillslope_code_of_neighbors(vrow, vcol, flowmodel="taudem"):
"""set hillslope code for neighbors of current stream cell."""
stream_coors.append((vrow, vcol))
in_strm_num, in_coors = inflow_stream_number(vrow, vcol, flowmodel)
strm_id = stream_data[vrow][vcol]
# print('Assign hillslope code for stream cell, r: %d, c: %d, ID: %d' % (vrow, vcol,
# int(strm_id)))
# set hillslope IDs
hillslp_ids = DelineateHillslope.cal_hs_codes(max_id, strm_id)
cur_d8_value = flowd8_data[vrow][vcol]
if in_strm_num == 0: # it is a one-order stream head
headstream_coors.append((vrow, vcol))
for (in_nostrm_row, in_nostrm_col) in in_coors:
hillslope_mtx[in_nostrm_row][in_nostrm_col] = hillslp_ids[
0]
else: # search the 3*3 neighbors by clockwise and counterclockwise separately
if cur_d8_value <= 0 or cur_d8_value == flowd8_nodata:
return
dirv = int(cur_d8_value) # direction code
d_idx = FlowModelConst.d8_dirs.get(flowmodel).index(
dirv) # direction index
# look to the right side, i.e. clockwise
d_idx_r = d_idx
while True and len(in_coors) > 0:
d_idx_r -= 1
if d_idx_r > 7:
d_idx_r = 0
if d_idx_r < 0:
d_idx_r = 7
tmp_row = vrow + FlowModelConst.ccw_drow[d_idx_r]
tmp_col = vcol + FlowModelConst.ccw_dcol[d_idx_r]
if (tmp_row, tmp_col
) not in in_coors: # not inflow to this cell
continue
tmpstream = stream_data[tmp_row][tmp_col]
in_coors.remove((tmp_row, tmp_col))
if tmpstream <= 0 or tmpstream == stream_nodata:
hillslope_mtx[tmp_row][tmp_col] = hillslp_ids[
1] # right hillslope
else: # encounter another in flow stream
break
# look to the left side, i.e. counterclockwise
d_idx_l = d_idx
while True and len(in_coors) > 0:
d_idx_l += 1
if d_idx_l > 7:
d_idx_l = 0
if d_idx_l < 0:
d_idx_l = 7
tmp_row = vrow + FlowModelConst.ccw_drow[d_idx_l]
tmp_col = vcol + FlowModelConst.ccw_dcol[d_idx_l]
if (tmp_row, tmp_col
) not in in_coors: # not inflow to this cell
continue
tmpstream = stream_data[tmp_row][tmp_col]
in_coors.remove((tmp_row, tmp_col))
if tmpstream <= 0 or tmpstream == stream_nodata:
hillslope_mtx[tmp_row][tmp_col] = hillslp_ids[
2] # left hillslope
else: # encounter another in flow stream
break
# if any inflow cells existed?
if len(in_coors) > 0:
for (tmp_row, tmp_col) in in_coors:
tmpstream = stream_data[tmp_row][tmp_col]
if tmpstream <= 0 or tmpstream == stream_nodata:
hillslope_mtx[tmp_row][tmp_col] = hillslp_ids[0]
# add the current cell as head stream
headstream_coors.append((vrow, vcol))
def output_hillslope(method_id):
"""Output hillslope according different stream cell value method."""
for (tmp_row, tmp_col) in stream_coors:
tmp_hillslp_ids = DelineateHillslope.cal_hs_codes(
max_id, stream_data[tmp_row][tmp_col])
if 0 < method_id < 3:
hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[
method_id]
# is head stream cell?
if (tmp_row, tmp_col) in headstream_coors:
hillslope_mtx[tmp_row][tmp_col] = tmp_hillslp_ids[0]
elif method_id == 3:
hillslope_mtx[tmp_row][tmp_col] = DEFAULT_NODATA
# Output to raster file
hillslope_out_new = hillslope_out
dirpath = os.path.dirname(hillslope_out_new) + os.path.sep
corename = FileClass.get_core_name_without_suffix(
hillslope_out_new)
if method_id == 1:
hillslope_out_new = dirpath + corename + '_right.tif'
elif method_id == 2:
hillslope_out_new = dirpath + corename + '_left.tif'
elif method_id == 3:
hillslope_out_new = dirpath + corename + '_nodata.tif'
RasterUtilClass.write_gtiff_file(hillslope_out_new, nrows, ncols,
hillslope_mtx, geotrans, srs,
DEFAULT_NODATA, datatype)
# 1. assign a unique id to each link in the stream network if needed
assign_stream_id = False
tmp = numpy.where((stream_data > 0) & (stream_data != stream_nodata),
stream_data, numpy.nan)
max_id = int(numpy.nanmax(tmp)) # i.e., stream link number
min_id = int(numpy.nanmin(tmp))
for i in range(min_id, max_id + 1):
if i not in tmp:
assign_stream_id = True
break
if max_id == min_id:
assign_stream_id = True
current_id = 0
if assign_stream_id:
# calculate and output sequenced stream raster
sequenced_stream_d = numpy.ones((nrows, ncols)) * DEFAULT_NODATA
for row in range(nrows):
for col in range(ncols):
# if the cell is not a stream, or has been assigned an ID
if stream_data[row][col] <= 0 or stream_data[row][col] == stream_nodata \
or sequenced_stream_d[row][col] > 0:
continue
current_id = assign_sequenced_stream_ids(
current_id, row, col, d8alg)
stream_data = numpy.copy(sequenced_stream_d)
stream_nodata = DEFAULT_NODATA
stream_core = FileClass.get_core_name_without_suffix(stream_raster)
stream_seq_file = os.path.dirname(
stream_raster) + os.path.sep + stream_core + '_seq.tif'
RasterUtilClass.write_gtiff_file(stream_seq_file, nrows, ncols,
sequenced_stream_d, geotrans, srs,
DEFAULT_NODATA, datatype)
max_id = current_id
# 2. assign hillslope code according to the 3*3 neighbors of stream cells
hillslope_mtx = numpy.copy(stream_data)
hillslope_mtx[stream_data == stream_nodata] = DEFAULT_NODATA
headstream_coors = [] # head stream cells
stream_coors = [] # all stream cells, include head stream cells.
for row in range(nrows):
for col in range(ncols):
# if not a stream cell, or hillslope code has been assigned
if stream_data[row][col] <= 0 or stream_data[row][col] == stream_nodata \
or hillslope_mtx[row][col] < 0:
continue
assign_hillslope_code_of_neighbors(row, col, d8alg)
# 3. From each cell, search downstream for not assigned hillslope
for row in range(nrows):
for col in range(ncols):
if hillslope_mtx[row][col] > 0 or flowd8_data[row][
col] == flowd8_nodata:
continue
flag = False
tmprow = row
tmpcol = col
tmpcoors = [(row, col)]
hillslp_id = DEFAULT_NODATA
while not flag:
# find it's downslope neighbour
curflowdir = flowd8_data[tmprow][tmpcol]
if curflowdir <= 0 or curflowdir == flowd8_nodata:
break
curflowdir = int(curflowdir)
tmprow, tmpcol = D8Util.downstream_index(
curflowdir, tmprow, tmpcol, d8alg)
if tmprow < 0 or tmprow >= nrows or tmpcol < 0 or tmpcol >= ncols:
break
# if the new cell already has a hillslope value, use that
if hillslope_mtx[tmprow][tmpcol] > 0:
hillslp_id = hillslope_mtx[tmprow][tmpcol]
flag = True
if not flag:
tmpcoors.append((tmprow, tmpcol))
# set the source cells
for (crow, ccol) in tmpcoors:
hillslope_mtx[crow][ccol] = hillslp_id
# 4. reassign stream cell's value according to stream_value_method, and output
if stream_value_method < 0: # output
output_hillslope(0)
output_hillslope(1)
output_hillslope(2)
output_hillslope(3)
else:
output_hillslope(stream_value_method)
def serialize_streamnet(streamnet_file, output_reach_file):
"""Eliminate reach with zero length and return the reach ID map.
Args:
streamnet_file: original stream net ESRI shapefile
output_reach_file: serialized stream net, ESRI shapefile
Returns:
id pairs {origin: newly assigned}
"""
FileClass.copy_files(streamnet_file, output_reach_file)
ds_reach = ogr_Open(output_reach_file, update=True)
layer_reach = ds_reach.GetLayer(0)
layer_def = layer_reach.GetLayerDefn()
i_link = layer_def.GetFieldIndex(FLD_LINKNO)
i_link_downslope = layer_def.GetFieldIndex(FLD_DSLINKNO)
i_len = layer_def.GetFieldIndex(REACH_LENGTH)
old_id_list = []
# there are some reaches with zero length.
# this program will remove these zero-length reaches
# output_dic is used to store the downstream reaches of these zero-length
# reaches
output_dic = {}
ft = layer_reach.GetNextFeature()
while ft is not None:
link_id = ft.GetFieldAsInteger(i_link)
reach_len = ft.GetFieldAsDouble(i_len)
if link_id not in old_id_list:
if reach_len < DELTA:
downstream_id = ft.GetFieldAsInteger(i_link_downslope)
output_dic[link_id] = downstream_id
else:
old_id_list.append(link_id)
ft = layer_reach.GetNextFeature()
old_id_list.sort()
id_map = {}
for i, old_id in enumerate(old_id_list):
id_map[old_id] = i + 1
# print(id_map)
# change old ID to new ID
layer_reach.ResetReading()
ft = layer_reach.GetNextFeature()
while ft is not None:
link_id = ft.GetFieldAsInteger(i_link)
if link_id not in id_map:
layer_reach.DeleteFeature(ft.GetFID())
ft = layer_reach.GetNextFeature()
continue
ds_id = ft.GetFieldAsInteger(i_link_downslope)
ds_id = output_dic.get(ds_id, ds_id)
ds_id = output_dic.get(ds_id, ds_id)
ft.SetField(FLD_LINKNO, id_map[link_id])
if ds_id in id_map:
ft.SetField(FLD_DSLINKNO, id_map[ds_id])
else:
# print(ds_id)
ft.SetField(FLD_DSLINKNO, -1)
layer_reach.SetFeature(ft)
ft = layer_reach.GetNextFeature()
ds_reach.ExecuteSQL("REPACK reach")
layer_reach.SyncToDisk()
ds_reach.Destroy()
del ds_reach
return id_map
def __init__(self, cfg_parser, bin_dir=None, proc_num=-1, rawdem=None, root_dir=None):
"""
Initialize an AutoFuzSlpPosConfig object
Args:
cfg_parser: ConfigParser object
bin_dir: Executable binaries path
proc_num: thread (or process) number used for MPI
rawdem: DEM of study area
root_dir: workspace path
"""
# Part I Initialize attributes
# 1.1. input parameters
self.cf = cfg_parser
self.bin_dir = bin_dir
self.proc = proc_num
self.root_dir = root_dir
# 1.2. Required inputs
self.dem = rawdem
# 1.3. Executable Flags (Set default flags first)
self.flag_preprocess = True
self.flag_selecttyploc = True
self.flag_auto_typlocparams = True
self.flag_fuzzyinference = True
self.flag_auto_inferenceparams = True
self.flag_log = True
# 1.4. Optional inputs
self.mpi_dir = None
self.hostfile = None
self.outlet = None
self.valley = None
self.ridge = None
self.regional_attr = None
# 1.5. Optional DTA related parameters
self.flow_model = 1
self.rpi_method = 1
self.dist_exp = 8
self.max_move_dist = 50
self.numthresh = 20
self.d8_stream_thresh = 0
self.d8_down_method = 'Surface'
self.d8_stream_tag = 1
self.d8_up_method = 'Surface'
self.dinf_stream_thresh = 0
self.dinf_down_stat = 'Average'
self.dinf_down_method = 'Surface'
self.dinf_dist_down_wg = None
self.propthresh = 0.0
self.dinf_up_stat = 'Average'
self.dinf_up_method = 'Surface'
# 1.6. Slope position types, tags, typical location extract value ranges, and inference
# parameters.
self.slppostype = list() # From top to bottom on hillslope.
self.slppostag = list() # The same sequence with self.slppostype
self.selectedtopo = dict() # Topographic attributes used for AutoFuzSlpPos
self.extractrange = dict()
self.param4typloc = dict()
self.infshape = dict()
self.inferparam = dict()
# 1.7. derived attributes
self.ws = None
self.log = None
self.topoparam = None
self.slpposresult = None
self.pretaudem = None
self.singleslpposconf = dict()
# Part II Set default settings, if cfg_parser is None, the program still can be executed.
# default slope position settings
self.slppostype = ['rdg', 'shd', 'bks', 'fts', 'vly']
self.slppostag = [1, 2, 4, 8, 16]
self.selectedtopolist = ['rpi', 'profc', 'slp', 'elev']
self.selectedtopo = dict()
self.extractrange = {'rdg': {'rpi': [0.99, 1.0]},
'shd': {'rpi': [0.9, 0.95]},
'bks': {'rpi': [0.5, 0.6]},
'fts': {'rpi': [0.15, 0.2]},
'vly': {'rpi': [0., 0.1]}}
self._DEFAULT_PARAM_TYPLOC = [10, 0.1, 0.3, 1, 0.1, 1, 50, 4.0]
for slppos in self.slppostype:
self.param4typloc[slppos] = self._DEFAULT_PARAM_TYPLOC[:]
self.infshape = {'rdg': {'rpi': 'S', 'profc': 'S', 'slp': 'Z', 'elev': 'SN'},
'shd': {'rpi': 'B', 'profc': 'S', 'slp': 'B', 'elev': 'N'},
'bks': {'rpi': 'B', 'profc': 'B', 'slp': 'S', 'elev': 'N'},
'fts': {'rpi': 'B', 'profc': 'ZB', 'slp': 'ZB', 'elev': 'N'},
'vly': {'rpi': 'Z', 'profc': 'Z', 'slp': 'Z', 'elev': 'N'}}
if self.cf is None:
if self.dem is not None and self.bin_dir is not None:
if self.root_dir is None:
self.root_dir = os.path.dirname(self.dem)
if self.mpi_dir is None:
mpipath = FileClass.get_executable_fullpath('mpiexec')
self.mpi_dir = os.path.dirname(mpipath)
if self.mpi_dir is None:
raise RuntimeError('Can not find mpiexec, make sure you have it installed!')
else:
raise RuntimeError("You MUST select one of ini file or dem and bin!")
if self.root_dir is not None:
self.ws = CreateWorkspace(self.root_dir)
self.log = LogNames(self.ws.log_dir)
self.topoparam = TopoAttrNames(self.ws)
self.slpposresult = FuzSlpPosFiles(self.ws)
self.pretaudem = PreProcessAttrNames(self.ws.pre_dir, self.flow_model)
for attr in self.selectedtopolist:
self.selectedtopo[attr] = self.topoparam.get_attr_file(attr)
for slppos in self.slppostype:
self.singleslpposconf[slppos] = SingleSlpPosFiles(self.ws, slppos)
# Part III Parse the *.ini configuration file if existed
# Be careful, bin_dir, root_dir, proc_num, and rawdem related attributes
# that have already set MUST not be changed in the following procedures.
if self.cf is not None:
# Parse and check validation of all available inputs
# define the section names in the *.ini configuration file
_require = 'REQUIRED'
_flag = 'EXECUTABLE_FLAGS'
_optdta = 'OPTIONAL_DTA'
_opt = 'OPTIONAL'
_opttyploc = 'OPTIONAL_TYPLOC'
_optfuzinf = 'OPTIONAL_FUZINF'
self.read_required_section(_require)
self.read_flag_section(_flag)
self.read_optionaldta_section(_optdta)
self.read_optional_section(_opt)
self.read_optiontyploc_section(_opttyploc)
self.read_optionfuzinf_section(_optfuzinf)
def __init__(self,
bin_dir='',
model_dir='',
nthread=4,
lyrmtd=0,
host='127.0.0.1',
port=27017,
scenario_id=-1,
calibration_id=-1,
version='OMP',
nprocess=1,
mpi_bin='',
hosts_opt='-f',
hostfile='',
**kwargs): # Allow any other keyword arguments
# Derived from input arguments
args_dict = dict()
if 'args_dict' in kwargs: # Preferred to use 'args_dict' if existed.
args_dict = kwargs['args_dict']
bin_dir = args_dict['bin_dir'] if 'bin_dir' in args_dict else bin_dir
model_dir = args_dict[
'model_dir'] if 'model_dir' in args_dict else model_dir
self.version = args_dict[
'version'] if 'version' in args_dict else version
suffix = '.exe' if sysstr == 'Windows' else ''
if self.version == 'MPI':
self.seims_exec = bin_dir + os.path.sep + 'seims_mpi' + suffix
else:
self.seims_exec = bin_dir + os.path.sep + 'seims_omp' + suffix
if not FileClass.is_file_exists(
self.seims_exec): # If not support OpenMP, use `seims`!
self.seims_exec = bin_dir + os.path.sep + 'seims' + suffix
self.seims_exec = os.path.abspath(self.seims_exec)
self.model_dir = os.path.abspath(model_dir)
self.nthread = args_dict[
'nthread'] if 'nthread' in args_dict else nthread
self.lyrmtd = args_dict['lyrmtd'] if 'lyrmtd' in args_dict else lyrmtd
self.host = args_dict['host'] if 'host' in args_dict else host
self.port = args_dict['port'] if 'port' in args_dict else port
self.scenario_id = args_dict[
'scenario_id'] if 'scenario_id' in args_dict else scenario_id
self.calibration_id = args_dict[
'calibration_id'] if 'calibration_id' in args_dict else calibration_id
self.nprocess = args_dict[
'nprocess'] if 'nprocess' in args_dict else nprocess
self.mpi_bin = args_dict[
'mpi_bin'] if 'mpi_bin' in args_dict else mpi_bin
self.hosts_opt = args_dict[
'hosts_opt'] if 'hosts_opt' in args_dict else hosts_opt
self.hostfile = args_dict[
'hostfile'] if 'hostfile' in args_dict else hostfile
# Concatenate executable command
self.cmd = self.Command
self.run_success = False
self.output_dir = self.OutputDirectory
# Read model data from MongoDB
self.db_name = os.path.split(self.model_dir)[1]
self.outlet_id = self.OutletID
self.start_time, self.end_time = self.SimulatedPeriod
# Data maybe used after model run
self.timespan = dict()
self.obs_vars = list() # Observation types at the outlet
self.obs_value = dict(
) # Observation value, key: DATETIME, value: value list of obs_vars
self.sim_vars = list(
) # Simulation types at the outlet, which is part of obs_vars
self.sim_value = dict() # Simulation value, same as obs_value
# The format of sim_obs_dict:
# {VarName: {'UTCDATETIME': [t1, t2, ..., tn],
# 'Obs': [o1, o2, ..., on],
# 'Sim': [s1, s2, ..., sn]},
# ...
# }
self.sim_obs_dict = dict()
def __init__(self, cf):
# Default arguments
self.host = '127.0.0.1' # localhost by default
self.port = 27017
self.bin_dir = ''
self.model_dir = ''
self.db_name = ''
self.version = 'OMP'
self.mpi_bin = None
self.hosts_opt = None
self.hostfile = None
self.nprocess = 1
self.nthread = 1
self.lyrmtd = 1
self.scenario_id = 0
self.calibration_id = -1
self.config_dict = dict()
if 'SEIMS_Model' not in cf.sections():
raise ValueError(
"[SEIMS_Model] section MUST be existed in *.ini file.")
self.host = cf.get('SEIMS_Model', 'hostname')
self.port = cf.getint('SEIMS_Model', 'port')
if not StringClass.is_valid_ip_addr(self.host):
raise ValueError('HOSTNAME defined in [SEIMS_Model] is illegal!')
self.bin_dir = cf.get('SEIMS_Model', 'bin_dir')
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.bin_dir)):
raise IOError('Please Check Directories defined in [SEIMS_Model]. '
'BIN_DIR and MODEL_DIR are required!')
self.db_name = os.path.split(self.model_dir)[1]
if cf.has_option('SEIMS_Model', 'version'):
self.version = cf.get('SEIMS_Model', 'version')
if cf.has_option('SEIMS_Model',
'mpi_bin'): # full path of the executable MPI program
self.mpi_bin = cf.get('SEIMS_Model', 'mpi_bin')
if cf.has_option('SEIMS_Model', 'hostopt'):
self.hosts_opt = cf.get('SEIMS_Model', 'hostopt')
if cf.has_option('SEIMS_Model', 'hostfile'):
self.hostfile = cf.get('SEIMS_Model', 'hostfile')
if cf.has_option('SEIMS_Model', 'processnum'):
self.nprocess = cf.getint('SEIMS_Model', 'processnum')
if cf.has_option('SEIMS_Model', 'threadsnum'):
self.nthread = cf.getint('SEIMS_Model', 'threadsnum')
if cf.has_option('SEIMS_Model', 'layeringmethod'):
self.lyrmtd = cf.getint('SEIMS_Model', 'layeringmethod')
if cf.has_option('SEIMS_Model', 'scenarioid'):
self.scenario_id = cf.getint('SEIMS_Model', 'scenarioid')
if cf.has_option('SEIMS_Model', 'calibrationid'):
self.calibration_id = cf.getint('SEIMS_Model', 'calibrationid')
if not (cf.has_option('SEIMS_Model', 'sim_time_start')
and cf.has_option('SEIMS_Model', 'sim_time_end')):
raise ValueError(
"Start and end time MUST be specified in [SEIMS_Model].")
try:
# UTCTIME
tstart = cf.get('SEIMS_Model', 'sim_time_start')
tend = cf.get('SEIMS_Model', 'sim_time_end')
self.time_start = StringClass.get_datetime(tstart)
self.time_end = StringClass.get_datetime(tend)
except ValueError:
raise ValueError('The time format MUST be "YYYY-MM-DD HH:MM:SS".')
if self.time_start >= self.time_end:
raise ValueError("Wrong time settings in [SEIMS_Model]!")
# Running time counted by time.time() of Python, in case of failed of GetTimespan()
self.runtime = 0.
def __init__(self, cf):
"""Initialization."""
# 1. Directories
self.base_dir = None
self.clim_dir = None
self.spatial_dir = None
self.observe_dir = None
self.scenario_dir = None
self.model_dir = None
self.txt_db_dir = None
self.preproc_script_dir = None
self.seims_bin = None
self.mpi_bin = None
self.workspace = None
# 1.1. Directory determined flags
self.use_observed = True
self.use_scernario = True
# 2. MongoDB configuration and database, collation, GridFS names
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.climate_db = ''
self.bmp_scenario_db = ''
self.spatial_db = ''
# 3. Switch for building SEIMS. These switches should be removed! By lj.
# self.gen_cn = True
# self.gen_runoff_coef = True
# self.gen_crop = True
# self.gen_iuh = True
# 4. Climate inputs
self.hydro_climate_vars = None
self.prec_sites = None
self.prec_data = None
self.Meteo_sites = None
self.Meteo_data = None
self.thiessen_field = 'ID'
# 5. Spatial inputs
self.prec_sites_thiessen = None
self.meteo_sites_thiessen = None
self.dem = None
self.outlet_file = None
self.landuse = None
self.landcover_init_param = None
self.soil = None
self.soil_property = None
self.fields_partition = False
self.fields_partition_thresh = list()
self.additional_rs = dict()
# 6. Option parameters
self.d8acc_threshold = 0
self.np = 4
self.d8down_method = 's'
self.dorm_hr = -1.
self.temp_base = 0.
self.imper_perc_in_urban = 0.
self.default_landuse = -1
self.default_soil = -1
# 1. Directories
if 'PATH' in cf.sections():
self.base_dir = cf.get('PATH', 'base_data_dir')
self.clim_dir = cf.get('PATH', 'climate_data_dir')
self.spatial_dir = cf.get('PATH', 'spatial_data_dir')
self.observe_dir = cf.get('PATH', 'measurement_data_dir')
self.scenario_dir = cf.get('PATH', 'bmp_data_dir')
self.model_dir = cf.get('PATH', 'model_dir')
self.txt_db_dir = cf.get('PATH', 'txt_db_dir')
self.preproc_script_dir = cf.get('PATH', 'preproc_script_dir')
self.seims_bin = cf.get('PATH', 'cpp_program_dir')
self.mpi_bin = cf.get('PATH', 'mpiexec_dir')
self.workspace = cf.get('PATH', 'working_dir')
else:
raise ValueError('[PATH] section MUST be existed in *.ini file.')
if not (FileClass.is_dir_exists(self.base_dir)
and FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.txt_db_dir)
and FileClass.is_dir_exists(self.preproc_script_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError(
'Please Check Directories defined in [PATH]. '
'BASE_DATA_DIR, MODEL_DIR, TXT_DB_DIR, PREPROC_SCRIPT_DIR, '
'and CPP_PROGRAM_DIR are required!')
if not FileClass.is_dir_exists(self.mpi_bin):
self.mpi_bin = None
if not FileClass.is_dir_exists(self.workspace):
try: # first try to make dirs
UtilClass.mkdir(self.workspace)
# os.mkdir(self.workspace)
except OSError as exc:
self.workspace = self.model_dir + os.path.sep + 'preprocess_output'
print('WARNING: Make WORKING_DIR failed: %s. '
'Use the default: %s' % (exc.message, self.workspace))
if not os.path.exists(self.workspace):
UtilClass.mkdir(self.workspace)
self.dirs = DirNameUtils(self.workspace)
self.logs = LogNameUtils(self.dirs.log)
self.vecs = VectorNameUtils(self.dirs.geoshp)
self.taudems = TauDEMFilesUtils(self.dirs.taudem)
self.spatials = SpatialNamesUtils(self.dirs.geodata2db)
self.modelcfgs = ModelCfgUtils(self.model_dir)
self.paramcfgs = ModelParamDataUtils(self.preproc_script_dir +
os.path.sep + 'database')
if not FileClass.is_dir_exists(self.clim_dir):
print(
'The CLIMATE_DATA_DIR is not existed, try the default folder name "climate".'
)
self.clim_dir = self.base_dir + os.path.sep + 'climate'
if not FileClass.is_dir_exists(self.clim_dir):
raise IOError(
'Directories named "climate" MUST BE located in [base_dir]!'
)
if not FileClass.is_dir_exists(self.spatial_dir):
print(
'The SPATIAL_DATA_DIR is not existed, try the default folder name "spatial".'
)
self.spatial_dir = self.base_dir + os.path.sep + 'spatial'
raise IOError(
'Directories named "spatial" MUST BE located in [base_dir]!')
if not FileClass.is_dir_exists(self.observe_dir):
self.observe_dir = None
self.use_observed = False
if not FileClass.is_dir_exists(self.scenario_dir):
self.scenario_dir = None
self.use_scernario = False
# 2. MongoDB related
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.climate_db = cf.get('MONGODB', 'climatedbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. Model related switch. The SWITCH section should be removed! By lj.
# by default, OpenMP version and daily (longterm) mode will be built
# if 'SWITCH' in cf.sections():
# self.gen_cn = cf.getboolean('SWITCH', 'gencn')
# self.gen_runoff_coef = cf.getboolean('SWITCH', 'genrunoffcoef')
# self.gen_crop = cf.getboolean('SWITCH', 'gencrop')
#
# if self.storm_mode:
# self.gen_iuh = False
# self.climate_db = ModelNameUtils.standardize_climate_dbname(self.climate_db)
# 4. Climate Input
if 'CLIMATE' in cf.sections():
self.hydro_climate_vars = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'hydroclimatevarfile')
self.prec_sites = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'precsitefile')
self.prec_data = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'precdatafile')
self.Meteo_sites = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'meteositefile')
self.Meteo_data = self.clim_dir + os.path.sep + cf.get(
'CLIMATE', 'meteodatafile')
self.thiessen_field = cf.get('CLIMATE', 'thiessenidfield')
else:
raise ValueError(
'Climate input file names MUST be provided in [CLIMATE]!')
# 5. Spatial Input
if 'SPATIAL' in cf.sections():
self.prec_sites_thiessen = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'precsitesthiessen')
self.meteo_sites_thiessen = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'meteositesthiessen')
self.dem = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'dem')
self.outlet_file = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'outlet_file')
if not os.path.exists(self.outlet_file):
self.outlet_file = None
self.landuse = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'landusefile')
self.landcover_init_param = self.txt_db_dir + os.path.sep + cf.get(
'SPATIAL', 'landcoverinitfile')
self.soil = self.spatial_dir + os.path.sep + cf.get(
'SPATIAL', 'soilseqnfile')
self.soil_property = self.txt_db_dir + os.path.sep + cf.get(
'SPATIAL', 'soilseqntext')
if cf.has_option('SPATIAL', 'additionalfile'):
additional_dict_str = cf.get('SPATIAL', 'additionalfile')
tmpdict = json.loads(additional_dict_str)
tmpdict = {
str(k): (str(v) if isinstance(v, str) else v)
for k, v in list(tmpdict.items())
}
for k, v in list(tmpdict.items()):
# Existence check has been moved to mask_origin_delineated_data()
# in sp_delineation.py
self.additional_rs[k] = v
# Field partition
if cf.has_option('SPATIAL', 'field_partition_thresh'):
ths = cf.get('SPATIAL', 'field_partition_thresh')
thsv = StringClass.extract_numeric_values_from_string(ths)
if thsv is not None:
self.fields_partition_thresh = [int(v) for v in thsv]
self.fields_partition = True
else:
raise ValueError(
'Spatial input file names MUST be provided in [SPATIAL]!')
# 6. Option parameters
if 'OPTIONAL_PARAMETERS' in cf.sections():
self.d8acc_threshold = cf.getfloat('OPTIONAL_PARAMETERS',
'd8accthreshold')
self.np = cf.getint('OPTIONAL_PARAMETERS', 'np')
self.d8down_method = cf.get('OPTIONAL_PARAMETERS', 'd8downmethod')
if StringClass.string_match(self.d8down_method, 'surface'):
self.d8down_method = 's'
elif StringClass.string_match(self.d8down_method, 'horizontal'):
self.d8down_method = 'h'
elif StringClass.string_match(self.d8down_method, 'pythagoras'):
self.d8down_method = 'p'
elif StringClass.string_match(self.d8down_method, 'vertical'):
self.d8down_method = 'v'
else:
self.d8down_method = self.d8down_method.lower()
if self.d8down_method not in ['s', 'h', 'p', 'v']:
self.d8down_method = 'h'
self.dorm_hr = cf.getfloat('OPTIONAL_PARAMETERS', 'dorm_hr')
self.temp_base = cf.getfloat('OPTIONAL_PARAMETERS', 't_base')
self.imper_perc_in_urban = cf.getfloat(
'OPTIONAL_PARAMETERS', 'imperviouspercinurbancell')
self.default_landuse = cf.getint('OPTIONAL_PARAMETERS',
'defaultlanduse')
self.default_soil = cf.getint('OPTIONAL_PARAMETERS', 'defaultsoil')
def __init__(self, cf):
"""Initialization."""
# 1. NSGA-II related parameters
self.nsga2_ngens = 1
self.nsga2_npop = 4
self.nsga2_rcross = 0.75
self.nsga2_pmut = 0.05
self.nsga2_rmut = 0.1
self.nsga2_rsel = 0.8
if 'NSGA2' in cf.sections():
self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum')
self.nsga2_npop = cf.getint('NSGA2', 'populationsize')
self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate')
self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc')
self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate')
self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate')
else:
raise ValueError('[NSGA2] section MUST be existed in *.ini file.')
if self.nsga2_npop % 4 != 0:
raise ValueError('PopulationSize must be a multiple of 4.')
# 2. MongoDB
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.spatial_db = ''
self.bmp_scenario_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. SEIMS_Model
self.model_dir = ''
self.seims_bin = ''
self.seims_nthread = 1
self.seims_lyrmethod = 0
if 'SEIMS_Model' in cf.sections():
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
self.seims_bin = cf.get('SEIMS_Model', 'bin_dir')
self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum')
self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod')
else:
raise ValueError(
"[SEIMS_Model] section MUST be existed in *.ini file.")
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError('Please Check Directories defined in [PATH]. '
'BIN_DIR and MODEL_DIR are required!')
# 4. Application specific setting section [BMPs]
self.bmps_info = dict()
self.bmps_rule = False
self.rule_method = 1
self.bmps_retain = dict()
self.export_sce_txt = False
self.export_sce_tif = False
if 'BMPs' in cf.sections():
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule')
if cf.has_option('BMPs', 'rule_method'):
self.rule_method = cf.getint('BMPs', 'rule_method')
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(
self.bmps_retain)
if cf.has_option('BMPs', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('BMPs',
'export_scenario_txt')
if cf.has_option('BMPs', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('BMPs',
'export_scenario_tif')
else:
raise ValueError("[BMPs] section MUST be existed for specific SA.")
self.bmps_info = json.loads(bmpsinfostr)
self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info)
# 5. Application specific setting section [Effectiveness]
self.worst_econ = 0
self.worst_env = 0
self.runtime_years = 0
if 'Effectiveness' in cf.sections():
self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy')
self.worst_env = cf.getfloat('Effectiveness', 'worst_environment')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
# 6. define gene_values
fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop)
fn += '_rule' if self.bmps_rule else '_random'
self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn
self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.nsga2_dir)
UtilClass.rmmkdir(self.scenario_dir)
self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt'
self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt'
self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log'
self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'
def __init__(
self,
tag_names, # type: List[Tuple[int, AnyStr]]
slpposf, # type: AnyStr
reach_shp, # type: AnyStr
hillslpf, # type: AnyStr
landusef # type: AnyStr
):
# type: (...) -> None
"""Initialization.
Args:
tag_names: [tag(integer), name(str)], tag should be ascending from up to bottom.
slpposf: Crisp classification of slope position full filename.
reach_shp: Reach shapefile used to extract the up-down relationships of subbasins
hillslpf: Delineated hillslope file by sd_hillslope.py.
landusef: Landuse, used to statistics areas of each landuse types within
slope position units
Attributes:
slppos_tags(OrderedDict): {tag: name}
subbsin_tree: up-down stream relationships of subbasins.
{subbsnID: {'upstream': [], 'downstream': []}}
units_updwon: Output json data of slope position units.
{"slppos_1": {id:{"downslope": [ids], "upslope": [ids], "landuse": {luID: area}
"hillslope": [hillslpID], "subbasin": [subbsnID], "area": area
}
}
"slppos_2": ...
}
"""
# Check the file existence
FileClass.check_file_exists(slpposf)
FileClass.check_file_exists(reach_shp)
FileClass.check_file_exists(hillslpf)
FileClass.check_file_exists(landusef)
# Set inputs
self.ws = os.path.dirname(slpposf)
tag_names = sorted(tag_names, key=lambda x: x[0])
# initialize slope position dict with up-down relationships
self.slppos_tags = OrderedDict(
) # type: Dict[int, Dict[AnyStr, Union[int, AnyStr]]]
for idx, tagname in enumerate(tag_names):
tag, name = tagname
if len(tag_names) > 1:
if idx == 0:
self.slppos_tags[int(tag)] = {
'name': name,
'upslope': -1,
'downslope': tag_names[idx + 1][0]
}
elif idx == len(tag_names) - 1:
self.slppos_tags[int(tag)] = {
'name': name,
'upslope': tag_names[idx - 1][0],
'downslope': -1
}
else:
self.slppos_tags[int(tag)] = {
'name': name,
'upslope': tag_names[idx - 1][0],
'downslope': tag_names[idx + 1][0]
}
else:
self.slppos_tags[int(tag)] = {
'name': name,
'upslope': -1,
'downslope': -1
}
self.reach = reach_shp
# read raster data and check the extent based on hillslope.
hillslpr = RasterUtilClass.read_raster(hillslpf)
self.data_hillslp = hillslpr.data
self.nrows = hillslpr.nRows
self.ncols = hillslpr.nCols
self.dx = hillslpr.dx
self.nodata_hillslp = hillslpr.noDataValue
self.geotrans = hillslpr.geotrans
self.srs = hillslpr.srs
self.datatype = hillslpr.dataType
slpposr = RasterUtilClass.read_raster(slpposf)
if slpposr.nRows != self.nrows or slpposr.nCols != self.ncols:
raise ValueError(
'The slopeposition raster MUST have the same dimensions'
' with hillslope!')
self.data_slppos = slpposr.data
self.nodata_slppos = slpposr.noDataValue
landuser = RasterUtilClass.read_raster(landusef)
if landuser.nRows != self.nrows or landuser.nCols != self.ncols:
raise ValueError(
'The landuser raster MUST have the same dimensions'
' with hillslope!')
self.data_landuse = landuser.data
self.nodata_landuse = landuser.noDataValue
# Set intermediate data
self.subbsin_num = -1
self.subbsin_tree = dict(
) # type: Dict[int, int] # {subbsnID: dst_subbsnID}
self.units_updwon = OrderedDict(
) # type: Dict[AnyStr, Dict[int, Dict[AnyStr, Union[List[float], AnyStr]]]]
for tag in self.slppos_tags:
self.units_updwon[self.slppos_tags.get(tag).get('name')] = dict()
self.slppos_ids = numpy.ones((self.nrows, self.ncols)) * DEFAULT_NODATA
self.hierarchy_units = dict(
) # type: Dict[int, Dict[int, Dict[AnyStr, int]]]
# Set gene_values of outputs
self.outf_units_origin = self.ws + os.path.sep + 'slppos_units_origin_uniqueid.tif'
self.outshp_units_origin = self.ws + os.path.sep + 'origin_uniqueid.shp'
self.json_units_origin = self.ws + os.path.sep + 'original_updown.json'
self.outf_units_merged = self.ws + os.path.sep + 'slppos_units.tif'
self.outshp_units_merged = self.ws + os.path.sep + 'slppos_units_merged.shp'
self.json_units_merged = self.ws + os.path.sep + 'updown.json'
def pre_processing(cfg):
start_t = time.time()
if not cfg.flag_preprocess:
return 0
single_basin = False
if cfg.outlet is not None:
single_basin = True
pretaudem_done = check_watershed_delineation_results(cfg)
if cfg.valley is None or not FileClass.is_file_exists(cfg.valley) or not pretaudem_done:
cfg.valley = cfg.pretaudem.stream_raster
# Watershed delineation based on D8 flow model.
TauDEMWorkflow.watershed_delineation(cfg.proc, cfg.dem, cfg.outlet, cfg.d8_stream_thresh,
single_basin,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
logfile=cfg.log.preproc, runtime_file=cfg.log.runtime,
hostfile=cfg.hostfile)
# use outlet_m or not
outlet_use = None
if single_basin:
outlet_use = cfg.pretaudem.outlet_m
log_status = open(cfg.log.preproc, 'a', encoding='utf-8')
log_status.write('Calculating RPI(Relative Position Index)...\n')
log_status.flush()
if cfg.flow_model == 1: # Dinf model, extract stream using the D8 threshold
if cfg.valley is None or not FileClass.is_file_exists(cfg.valley):
if cfg.d8_stream_thresh <= 0:
with open(cfg.pretaudem.drptxt, 'r', encoding='utf-8') as drpf:
temp_contents = drpf.read()
(beg, cfg.d8_stream_thresh) = temp_contents.rsplit(' ', 1)
print(cfg.d8_stream_thresh)
TauDEMExtension.areadinf(cfg.proc, cfg.pretaudem.dinf,
cfg.pretaudem.dinfacc_weight, outlet_use,
cfg.pretaudem.stream_pd, 'false',
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
TauDEMExtension.threshold(cfg.proc, cfg.pretaudem.dinfacc_weight,
cfg.pretaudem.stream_dinf, float(cfg.d8_stream_thresh),
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
cfg.valley = cfg.pretaudem.stream_dinf
# calculate Height Above the Nearest Drainage (HAND)
TauDEMExtension.dinfdistdown(cfg.proc, cfg.pretaudem.dinf, cfg.pretaudem.filldem,
cfg.pretaudem.dinf_slp, cfg.valley,
cfg.dinf_down_stat, 'v', 'false',
cfg.dinf_dist_down_wg, cfg.pretaudem.dist2stream_v,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
else:
# calculate Height Above the Nearest Drainage (HAND)
TauDEMExtension.d8distdowntostream(cfg.proc, cfg.pretaudem.d8flow,
cfg.pretaudem.filldem, cfg.valley,
cfg.pretaudem.dist2stream_v, 'v', 1,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
if cfg.rpi_method == 1: # calculate RPI based on hydrological proximity measures (Default).
if cfg.flow_model == 0: # D8 model
TauDEMExtension.d8distdowntostream(cfg.proc, cfg.pretaudem.d8flow,
cfg.pretaudem.filldem, cfg.valley,
cfg.pretaudem.dist2stream, cfg.d8_down_method, 1,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
TauDEMExtension.d8distuptoridge(cfg.proc, cfg.pretaudem.d8flow,
cfg.pretaudem.filldem, cfg.ridge,
cfg.pretaudem.distup2rdg, cfg.d8_up_method,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
elif cfg.flow_model == 1: # Dinf model
# Dinf distance down
TauDEMExtension.dinfdistdown(cfg.proc, cfg.pretaudem.dinf, cfg.pretaudem.filldem,
cfg.pretaudem.dinf_slp, cfg.valley,
cfg.dinf_down_stat, cfg.dinf_down_method, 'false',
cfg.dinf_dist_down_wg, cfg.pretaudem.dist2stream,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
TauDEMExtension.dinfdistuptoridge(cfg.proc, cfg.pretaudem.dinf,
cfg.pretaudem.filldem, cfg.pretaudem.dinf_slp,
cfg.propthresh, cfg.pretaudem.distup2rdg,
cfg.dinf_up_stat, cfg.dinf_up_method, 'false',
cfg.ridge,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
TauDEMExtension.simplecalculator(cfg.proc, cfg.pretaudem.dist2stream,
cfg.pretaudem.distup2rdg, cfg.pretaudem.rpi_hydro, 4,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
if cfg.rpi_method == 0: # calculate RPI based on Skidmore's method
if cfg.ridge is None or not FileClass.is_file_exists(cfg.ridge):
cfg.ridge = cfg.pretaudem.rdgsrc
angfile = cfg.pretaudem.d8flow
elevfile = cfg.pretaudem.dist2stream_v
if cfg.flow_model == 1: # D-inf model
angfile = cfg.pretaudem.dinf
elevfile = cfg.pretaudem.dist2stream_v
TauDEMExtension.extractridge(cfg.proc, angfile, elevfile, cfg.ridge,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
TauDEMExtension.rpiskidmore(cfg.proc, cfg.valley, cfg.ridge,
cfg.pretaudem.rpi_skidmore, 1, 1,
cfg.pretaudem.dist2stream_ed, cfg.pretaudem.dist2rdg_ed,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
log_status.write('Calculating Horizontal Curvature and Profile Curvature...\n')
TauDEMExtension.curvature(cfg.proc, cfg.pretaudem.filldem,
cfg.topoparam.profc, cfg.topoparam.horizc,
None, None, None, None, None,
cfg.ws.pre_dir, cfg.mpi_dir, cfg.bin_dir,
cfg.log.preproc, cfg.log.runtime, cfg.hostfile)
if cfg.flow_model == 0:
slope_rad_to_deg(cfg.pretaudem.slp, cfg.topoparam.slope)
elif cfg.flow_model == 1:
slope_rad_to_deg(cfg.pretaudem.dinf_slp, cfg.topoparam.slope)
if cfg.rpi_method == 1:
copy2(cfg.pretaudem.rpi_hydro, cfg.topoparam.rpi)
else:
copy2(cfg.pretaudem.rpi_skidmore, cfg.topoparam.rpi)
copy2(cfg.pretaudem.dist2stream_v, cfg.topoparam.hand)
copy2(cfg.pretaudem.filldem, cfg.topoparam.elev)
if single_basin: # clip RPI
RasterUtilClass.mask_raster(cfg.topoparam.rpi, cfg.pretaudem.subbsn, cfg.topoparam.rpi)
log_status.write('Preprocessing succeed!\n')
end_t = time.time()
cost = (end_t - start_t) / 60.
log_status.write('Time consuming: %.2f min.\n' % cost)
log_status.close()
with open(cfg.log.runtime, 'a', encoding='utf-8') as logf:
logf.write('Preprocessing Time-consuming: %s\n' % repr(cost))
return cost
def main(landusef, unitsf, jsonout):
"""Construct common spatial units data in JSON file format."""
# Check the file existence
FileClass.check_file_exists(landusef)
FileClass.check_file_exists(unitsf)
# read raster data and check the extent based on landuse.
landuser = RasterUtilClass.read_raster(landusef)
data_landuse = landuser.data
nrows = landuser.nRows
ncols = landuser.nCols
dx = landuser.dx
nodata_landuse = landuser.noDataValue
fieldr = RasterUtilClass.read_raster(unitsf)
if fieldr.nRows != nrows or fieldr.nCols != ncols:
raise ValueError('The spatial units raster MUST have the same dimensions'
' with landuse!')
data_units = fieldr.data
nodata_units = fieldr.noDataValue
units_info = dict() # type: Dict[AnyStr, Dict[Union[int, AnyStr], Dict[AnyStr, Union[int, float, List[Union[int,float]], AnyStr, Dict[int, float]]]]]
units_info.setdefault('units', dict())
units_info.setdefault('overview', dict())
units_ids = list() # type: List[int]
for m in range(nrows):
for n in range(ncols):
cur_lu = int(data_landuse[m][n])
cur_unit = int(data_units[m][n])
if cur_unit == nodata_units or cur_lu == nodata_landuse or cur_lu <= 0:
continue
if cur_unit not in units_ids:
units_ids.append(cur_unit)
if cur_unit not in units_info['units']:
units_info['units'].setdefault(cur_unit, {'landuse': dict(),
'primarylanduse': 0,
'area': 0.})
if cur_lu not in units_info['units'][cur_unit]['landuse']:
units_info['units'][cur_unit]['landuse'][cur_lu] = 1
else:
units_info['units'][cur_unit]['landuse'][cur_lu] += 1
for k, v in viewitems(units_info['units']):
area_field = 0.
area_max = 0.
area_max_lu = 0
for luid, luarea in viewitems(v['landuse']):
v['landuse'][luid] = luarea * dx * dx * 1.e-6
area_field += v['landuse'][luid]
if v['landuse'][luid] > area_max:
area_max = v['landuse'][luid]
area_max_lu = luid
v['area'] = area_field
v['primarylanduse'] = area_max_lu
units_info['overview'].setdefault('all_units', len(units_ids))
# save to json
json_data = json.dumps(units_info, indent=4)
with open(json_out, 'w', encoding='utf-8') as f:
f.write('%s' % json_data)
def watershed_delineation(np, dem, outlet_file=None, thresh=0, singlebasin=False,
workingdir=None, mpi_bin=None, bin_dir=None,
logfile=None, runtime_file=None, hostfile=None):
"""Watershed Delineation."""
# 1. Check directories
if not os.path.exists(dem):
TauDEM.error('DEM: %s is not existed!' % dem)
dem = os.path.abspath(dem)
if workingdir is None:
workingdir = os.path.dirname(dem)
namecfg = TauDEMFilesUtils(workingdir)
workingdir = namecfg.workspace
UtilClass.mkdir(workingdir)
# 2. Check log file
if logfile is not None and FileClass.is_file_exists(logfile):
os.remove(logfile)
# 3. Get predefined intermediate file names
filled_dem = namecfg.filldem
flow_dir = namecfg.d8flow
slope = namecfg.slp
flow_dir_dinf = namecfg.dinf
slope_dinf = namecfg.dinf_slp
dir_code_dinf = namecfg.dinf_d8dir
weight_dinf = namecfg.dinf_weight
acc = namecfg.d8acc
stream_raster = namecfg.stream_raster
default_outlet = namecfg.outlet_pre
modified_outlet = namecfg.outlet_m
stream_skeleton = namecfg.stream_pd
acc_with_weight = namecfg.d8acc_weight
stream_order = namecfg.stream_order
ch_network = namecfg.channel_net
ch_coord = namecfg.channel_coord
stream_net = namecfg.streamnet_shp
subbasin = namecfg.subbsn
dist2_stream_d8 = namecfg.dist2stream_d8
# 4. perform calculation
UtilClass.writelog(logfile, "[Output] %d..., %s" % (10, "pitremove DEM..."), 'a')
TauDEM.pitremove(np, dem, filled_dem, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" %
(20, "Calculating D8 and Dinf flow direction..."), 'a')
TauDEM.d8flowdir(np, filled_dem, flow_dir, slope, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
TauDEM.dinfflowdir(np, filled_dem, flow_dir_dinf, slope_dinf, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
DinfUtil.output_compressed_dinf(flow_dir_dinf, dir_code_dinf, weight_dinf)
UtilClass.writelog(logfile, "[Output] %d..., %s" % (30, "D8 flow accumulation..."), 'a')
TauDEM.aread8(np, flow_dir, acc, None, None, False, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" %
(40, "Generating stream raster initially..."), 'a')
min_accum, max_accum, mean_accum, std_accum = RasterUtilClass.raster_statistics(acc)
TauDEM.threshold(np, acc, stream_raster, mean_accum, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" % (50, "Moving outlet to stream..."), 'a')
if outlet_file is None:
outlet_file = default_outlet
TauDEM.connectdown(np, flow_dir, acc, outlet_file, wtsd=None,
workingdir=workingdir, mpiexedir=mpi_bin, exedir=bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
TauDEM.moveoutletstostrm(np, flow_dir, stream_raster, outlet_file,
modified_outlet, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" %
(60, "Generating stream skeleton..."), 'a')
TauDEM.peukerdouglas(np, filled_dem, stream_skeleton, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" %
(70, "Flow accumulation with outlet..."), 'a')
tmp_outlet = None
if singlebasin:
tmp_outlet = modified_outlet
TauDEM.aread8(np, flow_dir, acc_with_weight, tmp_outlet, stream_skeleton, False,
workingdir, mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
if thresh <= 0: # find the optimal threshold using dropanalysis function
UtilClass.writelog(logfile, "[Output] %d..., %s" %
(75, "Drop analysis to select optimal threshold..."), 'a')
min_accum, max_accum, mean_accum, std_accum = \
RasterUtilClass.raster_statistics(acc_with_weight)
if mean_accum - std_accum < 0:
minthresh = mean_accum
else:
minthresh = mean_accum - std_accum
maxthresh = mean_accum + std_accum
numthresh = 20
logspace = 'true'
drp_file = namecfg.drptxt
TauDEM.dropanalysis(np, filled_dem, flow_dir, acc_with_weight,
acc_with_weight, modified_outlet, minthresh, maxthresh,
numthresh, logspace, drp_file, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
if not FileClass.is_file_exists(drp_file):
raise RuntimeError("Dropanalysis failed and drp.txt was not created!")
with open(drp_file, 'r', encoding='utf-8') as drpf:
temp_contents = drpf.read()
(beg, thresh) = temp_contents.rsplit(' ', 1)
print(thresh)
UtilClass.writelog(logfile, "[Output] %d..., %s" % (80, "Generating stream raster..."), 'a')
TauDEM.threshold(np, acc_with_weight, stream_raster, float(thresh),
workingdir, mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" % (90, "Generating stream net..."), 'a')
TauDEM.streamnet(np, filled_dem, flow_dir, acc_with_weight, stream_raster,
modified_outlet, stream_order, ch_network,
ch_coord, stream_net, subbasin, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d..., %s" %
(95, "Calculating distance to stream (D8)..."), 'a')
TauDEM.d8hdisttostrm(np, flow_dir, stream_raster, dist2_stream_d8, 1,
workingdir, mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, "[Output] %d.., %s" %
(100, "Original subbasin delineation is finished!"), 'a')
def watershed_delineation(np, dem, outlet_file=None, thresh=0, singlebasin=False,
workingdir=None, mpi_bin=None, bin_dir=None,
logfile=None, runtime_file=None, hostfile=None,
avoid_redo=False):
"""Watershed Delineation based on D8 flow direction.
Args:
np: process number for MPI
dem: DEM path
outlet_file: predefined outlet shapefile path
thresh: predefined threshold for extracting stream from accumulated flow direction
singlebasin: when set True, only extract subbasins that drains into predefined outlets
workingdir: directory that store outputs
mpi_bin: directory of MPI executable binary, e.g., mpiexec, mpirun
bin_dir: directory of TauDEM and other executable binaries
logfile: log file path
runtime_file: runtime file path
hostfile: host list file path for MPI
avoid_redo: avoid executing some functions that do not depend on input arguments
when repeatedly invoke this function
"""
# 1. Check directories
if not os.path.exists(dem):
TauDEM.error('DEM: %s is not existed!' % dem)
dem = os.path.abspath(dem)
if workingdir is None or workingdir is '':
workingdir = os.path.dirname(dem)
nc = TauDEMFilesUtils(workingdir) # predefined names
workingdir = nc.workspace
UtilClass.mkdir(workingdir)
# 2. Check log file
if logfile is not None and FileClass.is_file_exists(logfile):
os.remove(logfile)
# 3. perform calculation
# Filling DEM
if not (avoid_redo and FileClass.is_file_exists(nc.filldem)):
UtilClass.writelog(logfile, '[Output] %s' % 'remove pit...', 'a')
TauDEM.pitremove(np, dem, nc.filldem, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
# Flow direction based on D8 algorithm
if not (avoid_redo and FileClass.is_file_exists(nc.d8flow)):
UtilClass.writelog(logfile, '[Output] %s' % 'D8 flow direction...', 'a')
TauDEM.d8flowdir(np, nc.filldem, nc.d8flow, nc.slp, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
# Flow accumulation without stream skeleton as weight
if not (avoid_redo and FileClass.is_file_exists(nc.d8acc)):
UtilClass.writelog(logfile, '[Output] %s' % 'D8 flow accumulation...', 'a')
TauDEM.aread8(np, nc.d8flow, nc.d8acc, None, None, False, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
# Initial stream network using mean accumulation as threshold
UtilClass.writelog(logfile, '[Output] %s' % 'Generating stream raster initially...', 'a')
min_accum, max_accum, mean_accum, std_accum = RasterUtilClass.raster_statistics(nc.d8acc)
TauDEM.threshold(np, nc.d8acc, nc.stream_raster, mean_accum, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
# Outlets position initialization and adjustment
UtilClass.writelog(logfile, '[Output] %s' % 'Moving outlet to stream...', 'a')
if outlet_file is None: # if not given, take cell with maximum accumulation as outlet
outlet_file = nc.outlet_pre
TauDEM.connectdown(np, nc.d8flow, nc.d8acc, outlet_file, nc.outlet_m, wtsd=None,
workingdir=workingdir, mpiexedir=mpi_bin, exedir=bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
TauDEM.moveoutletstostrm(np, nc.d8flow, nc.stream_raster, outlet_file,
nc.outlet_m, workingdir=workingdir,
mpiexedir=mpi_bin, exedir=bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
# Stream skeleton by peuker-douglas algorithm
UtilClass.writelog(logfile, '[Output] %s' % 'Generating stream skeleton ...', 'a')
TauDEM.peukerdouglas(np, nc.filldem, nc.stream_pd, workingdir,
mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
# Weighted flow acculation with outlet
UtilClass.writelog(logfile, '[Output] %s' % 'Flow accumulation with outlet...', 'a')
tmp_outlet = None
if singlebasin:
tmp_outlet = nc.outlet_m
TauDEM.aread8(np, nc.d8flow, nc.d8acc_weight, tmp_outlet, nc.stream_pd, False,
workingdir, mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
# Determine threshold by input argument or dropanalysis function
if thresh <= 0: # find the optimal threshold using dropanalysis function
UtilClass.writelog(logfile, '[Output] %s' %
'Drop analysis to select optimal threshold...', 'a')
min_accum, max_accum, mean_accum, std_accum = \
RasterUtilClass.raster_statistics(nc.d8acc_weight)
if mean_accum - std_accum < 0:
minthresh = mean_accum
else:
minthresh = mean_accum - std_accum
maxthresh = mean_accum + std_accum
TauDEM.dropanalysis(np, nc.filldem, nc.d8flow, nc.d8acc_weight,
nc.d8acc_weight, nc.outlet_m, minthresh, maxthresh,
20, 'true', nc.drptxt, workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
if not FileClass.is_file_exists(nc.drptxt):
# raise RuntimeError('Dropanalysis failed and drp.txt was not created!')
UtilClass.writelog(logfile, '[Output] %s' %
'dropanalysis failed!', 'a')
thresh = 0.5 * (maxthresh - minthresh) + minthresh
else:
with open(nc.drptxt, 'r', encoding='utf-8') as drpf:
temp_contents = drpf.read()
(beg, thresh) = temp_contents.rsplit(' ', 1)
thresh = float(thresh)
UtilClass.writelog(logfile, '[Output] %s: %f' %
('Selected optimal threshold: ', thresh), 'a')
# Final stream network
UtilClass.writelog(logfile, '[Output] %s' % 'Generating stream raster...', 'a')
TauDEM.threshold(np, nc.d8acc_weight, nc.stream_raster, thresh,
workingdir, mpi_bin, bin_dir, log_file=logfile,
runtime_file=runtime_file, hostfile=hostfile)
UtilClass.writelog(logfile, '[Output] %s' % 'Generating stream net...', 'a')
TauDEM.streamnet(np, nc.filldem, nc.d8flow, nc.d8acc_weight, nc.stream_raster,
nc.outlet_m, nc.stream_order, nc.channel_net,
nc.channel_coord, nc.streamnet_shp, nc.subbsn,
workingdir, mpi_bin, bin_dir,
log_file=logfile, runtime_file=runtime_file, hostfile=hostfile)
# Serialize IDs of subbasins and the corresponding streams
UtilClass.writelog(logfile, '[Output] %s' % 'Serialize subbasin&stream IDs...', 'a')
id_map = StreamnetUtil.serialize_streamnet(nc.streamnet_shp, nc.streamnet_m)
RasterUtilClass.raster_reclassify(nc.subbsn, id_map, nc.subbsn_m, GDT_Int32)
StreamnetUtil.assign_stream_id_raster(nc.stream_raster, nc.subbsn_m, nc.stream_m)
# convert raster to shapefile (for subbasin and basin)
UtilClass.writelog(logfile, '[Output] %s' % 'Generating subbasin vector...', 'a')
VectorUtilClass.raster2shp(nc.subbsn_m, nc.subbsn_shp, 'subbasin', 'SUBBASINID')
# Finish the workflow
UtilClass.writelog(logfile, '[Output] %s' %
'Original subbasin delineation is finished!', 'a')
