def post_process_of_delineated_data(cfg):
"""Do some necessary transfer for subbasin, stream, and flow direction raster."""
# inputs
stream_net_file = cfg.taudems.streamnet_shp
subbasin_file = cfg.taudems.subbsn_m
flow_dir_file_tau = cfg.taudems.d8flow_m
stream_raster_file = cfg.taudems.stream_m
# outputs
# -- shapefile
shp_dir = cfg.dirs.geoshp
UtilClass.mkdir(shp_dir)
# ---- outlet, copy from DirNameUtils.TauDEM
FileClass.copy_files(cfg.taudems.outlet_m, cfg.vecs.outlet)
# ---- reaches
output_reach_file = cfg.vecs.reach
# ---- subbasins
subbasin_vector_file = cfg.vecs.subbsn
# -- raster file
output_subbasin_file = cfg.spatials.subbsn
output_flow_dir_file = cfg.spatials.d8flow
output_stream_link_file = cfg.spatials.stream_link
output_hillslope_file = cfg.spatials.hillslope
id_map = StreamnetUtil.serialize_streamnet(stream_net_file,
output_reach_file)
RasterUtilClass.raster_reclassify(subbasin_file, id_map,
output_subbasin_file, GDT_Int32)
StreamnetUtil.assign_stream_id_raster(stream_raster_file,
output_subbasin_file,
output_stream_link_file)
# Convert D8 encoding rule to ArcGIS
D8Util.convert_code(flow_dir_file_tau, output_flow_dir_file)
# convert raster to shapefile (for subbasin and basin)
print('Generating subbasin vector...')
VectorUtilClass.raster2shp(output_subbasin_file, subbasin_vector_file,
'subbasin', FieldNames.subbasin_id)
mask_file = cfg.spatials.mask
basin_vector = cfg.vecs.bsn
print('Generating basin vector...')
VectorUtilClass.raster2shp(mask_file, basin_vector, 'basin',
FieldNames.basin)
# delineate hillslope
DelineateHillslope.downstream_method_whitebox(output_stream_link_file,
flow_dir_file_tau,
output_hillslope_file)
def post_process_of_delineated_data(cfg):
"""Do some necessary transfer for subbasin, stream, and flow direction raster."""
# inputs
stream_net_file = cfg.taudems.streamnet_shp
subbasin_file = cfg.taudems.subbsn_m
flow_dir_file_tau = cfg.taudems.d8flow_m
stream_raster_file = cfg.taudems.stream_m
# outputs
# -- shapefile
shp_dir = cfg.dirs.geoshp
UtilClass.mkdir(shp_dir)
# ---- outlet, copy from DirNameUtils.TauDEM
FileClass.copy_files(cfg.taudems.outlet_m, cfg.vecs.outlet)
# ---- reaches
output_reach_file = cfg.vecs.reach
# ---- subbasins
subbasin_vector_file = cfg.vecs.subbsn
# -- raster file
output_subbasin_file = cfg.spatials.subbsn
output_flow_dir_file = cfg.spatials.d8flow
output_stream_link_file = cfg.spatials.stream_link
output_hillslope_file = cfg.spatials.hillslope
id_map = StreamnetUtil.serialize_streamnet(stream_net_file, output_reach_file)
RasterUtilClass.raster_reclassify(subbasin_file, id_map, output_subbasin_file, GDT_Int32)
StreamnetUtil.assign_stream_id_raster(stream_raster_file, output_subbasin_file,
output_stream_link_file)
# Convert D8 encoding rule to ArcGIS
D8Util.convert_code(flow_dir_file_tau, output_flow_dir_file)
# convert raster to shapefile (for subbasin and basin)
print('Generating subbasin vector...')
VectorUtilClass.raster2shp(output_subbasin_file, subbasin_vector_file, 'subbasin',
FieldNames.subbasin_id)
mask_file = cfg.spatials.mask
basin_vector = cfg.vecs.bsn
print('Generating basin vector...')
VectorUtilClass.raster2shp(mask_file, basin_vector, 'basin', FieldNames.basin)
# delineate hillslope
DelineateHillslope.downstream_method_whitebox(output_stream_link_file, flow_dir_file_tau,
output_hillslope_file)
def assign_uniqueid_slppos_units(self):
"""Get unique ID by multiply slope position value and hillslope ID"""
for m in range(self.nrows):
for n in range(self.ncols):
if self.data_slppos[m][n] == self.nodata_slppos or \
self.data_hillslp[m][n] == self.nodata_hillslp:
continue
cur_slppos = int(self.data_slppos[m][n])
cur_spname = self.slppos_tags.get(cur_slppos).get('name')
cur_hs = int(self.data_hillslp[m][n])
if cur_hs <= self.subbsin_num: # which is stream
continue
cur_subbsn = DelineateHillslope.get_subbasin_from_hillslope_id(
cur_hs, self.subbsin_num)
if cur_subbsn <= 0:
continue
if cur_slppos not in self.slppos_tags:
continue
cur_unit = cur_hs * cur_slppos
self.slppos_ids[m][n] = cur_unit
# calculate theoretical downslope and upslope unit ID (check later)
down_id = self.slppos_tags.get(cur_slppos).get(
'downslope') * cur_hs
up_id = self.slppos_tags.get(cur_slppos).get(
'upslope') * cur_hs
if down_id < 0:
down_id = -1
if up_id < 0:
up_id = -1
if cur_unit not in self.units_updwon.get(cur_spname):
self.units_updwon.get(cur_spname)[cur_unit] = {
'area': 1,
'downslope': down_id,
'upslope': up_id,
'hillslope': [cur_hs],
'subbasin': [cur_subbsn],
'landuse': dict()
}
else:
self.units_updwon.get(cur_spname)[cur_unit]['area'] += 1
# add landuse statistics
cur_lu = self.data_landuse[m][n]
if cur_lu != self.nodata_landuse and cur_lu > 0:
cur_lu = int(cur_lu)
if cur_lu not in self.units_updwon.get(
cur_spname)[cur_unit]['landuse']:
self.units_updwon.get(
cur_spname)[cur_unit]['landuse'][cur_lu] = 1
else:
self.units_updwon.get(
cur_spname)[cur_unit]['landuse'][cur_lu] += 1
def assign_uniqueid_slppos_units(self):
"""Get unique ID by multiply slope position value and hillslope ID"""
for m in range(self.nrows):
for n in range(self.ncols):
if self.data_slppos[m][n] == self.nodata_slppos or \
self.data_hillslp[m][n] == self.nodata_hillslp:
continue
cur_slppos = int(self.data_slppos[m][n])
cur_spname = self.slppos_tags.get(cur_slppos).get('name')
cur_hs = int(self.data_hillslp[m][n])
if cur_hs <= self.subbsin_num: # which is stream
continue
cur_subbsn = DelineateHillslope.get_subbasin_from_hillslope_id(cur_hs,
self.subbsin_num)
if cur_subbsn <= 0:
continue
if cur_slppos not in self.slppos_tags:
continue
cur_unit = cur_hs * cur_slppos
self.slppos_ids[m][n] = cur_unit
# calculate theoretical downslope and upslope unit ID (check later)
down_id = self.slppos_tags.get(cur_slppos).get('downslope') * cur_hs
up_id = self.slppos_tags.get(cur_slppos).get('upslope') * cur_hs
if down_id < 0:
down_id = -1
if up_id < 0:
up_id = -1
if cur_unit not in self.units_updwon.get(cur_spname):
self.units_updwon.get(cur_spname)[cur_unit] = {'area': 1,
'downslope': down_id,
'upslope': up_id,
'hillslope': [cur_hs],
'subbasin': [cur_subbsn],
'landuse': dict()}
else:
self.units_updwon.get(cur_spname)[cur_unit]['area'] += 1
# add landuse statistics
cur_lu = self.data_landuse[m][n]
if cur_lu != self.nodata_landuse and cur_lu > 0:
cur_lu = int(cur_lu)
if cur_lu not in self.units_updwon.get(cur_spname)[cur_unit]['landuse']:
self.units_updwon.get(cur_spname)[cur_unit]['landuse'][cur_lu] = 1
else:
self.units_updwon.get(cur_spname)[cur_unit]['landuse'][cur_lu] += 1
def merge_slopeposition_units(self):
"""Merge hillslope/subbasin with incomplete slope position sequences"""
all_units_ids = list()
for sp in self.slppos_tags:
name = self.slppos_tags.get(sp).get('name')
all_units_ids += list(self.units_updwon.get(name).keys())
# loop each slope position units
hillslp_elim = dict()
for sp, spdict in self.slppos_tags.items():
name = spdict.get('name')
for unitid, unitdict in self.units_updwon.get(name).items():
flag = True # complete sequences or not
if spdict.get('upslope') > 0 > unitdict.get('upslope'):
flag = False
if spdict.get('downslope') > 0 > unitdict.get('downslope'):
flag = False
cur_hillslp_id = unitdict.get('hillslope')[0]
if not flag:
if cur_hillslp_id not in hillslp_elim:
hillslp_elim[cur_hillslp_id] = [unitid]
if cur_hillslp_id in hillslp_elim and unitid not in hillslp_elim[
cur_hillslp_id]:
hillslp_elim[cur_hillslp_id].append(unitid)
print('Hillslope with incomplete slope position sequences: %s' %
hillslp_elim.__str__())
# Basic procedure:
# 1. if header is incomplete, search left and right;
# if left/right is incomplete, search header and right/left;
# 2. if still not incomplete, check the subbasin,
# 3. if still not satisfied, merge the left and right to the left or right of downstream
# (or upstream for outlet subbasin). The header always with the left hillslope.
hillslope_merge_order = {0: [1, 2], 1: [0, 2], 2: [0, 1]}
hillslopes_pair = dict() # TODO. ljzhu.
units_pair = dict(
) # unit id pairs, key is merged unit, value is destination unit.
for hillid, unitids in hillslp_elim.items():
subbsnid = DelineateHillslope.get_subbasin_from_hillslope_id(
hillid, self.subbsin_num)
hillids = DelineateHillslope.cal_hs_codes(self.subbsin_num,
subbsnid)
idx = hillids.index(hillid)
# slope positions that existed in current hillslope
sp_seq = list()
for _id in unitids:
_id //= hillid
if _id not in sp_seq:
sp_seq.append(_id)
satisfied = False
cur_units_pair = dict()
for curidx in hillslope_merge_order[idx]:
dst_hillid = hillids[curidx]
for _id in sp_seq:
dst_unit = _id * dst_hillid
if dst_unit in all_units_ids:
cur_units_pair[_id * hillid] = dst_unit
for sp in self.slppos_tags:
_uid = sp * dst_hillid
if _uid in all_units_ids and sp not in sp_seq:
sp_seq.append(sp)
if SlopePositionUnits.check_slppos_sequence(
sp_seq, self.slppos_tags):
satisfied = True
break
if satisfied:
units_pair.update(cur_units_pair)
else: # means the current subbasin should be merged to downstream
downstream_merge_idx = {0: 1, 1: 1, 2: 2}
cur_units_pair = dict()
dst_subbsn = self.subbsin_tree.get(subbsnid)
if dst_subbsn < 0: # outlet subbasin
for k, v in self.subbsin_tree.items():
if v == subbsnid:
dst_subbsn = k
break
# although dst_subbsn may not less than 0, just make sure that!
if dst_subbsn > 0:
dst_hillids = DelineateHillslope.cal_hs_codes(
self.subbsin_num, dst_subbsn)
dst_downstream_hs = dst_hillids[downstream_merge_idx[idx]]
for _id in sp_seq:
dst_unit = _id * dst_downstream_hs
if dst_unit in all_units_ids:
cur_units_pair[_id * hillid] = dst_unit
units_pair.update(cur_units_pair)
print('Slope position unit merge-destination pairs: %s' %
units_pair.__str__())
# begin to merge
for srcid, dstid in units_pair.items():
# replace values in slope position units raster
for k, v in units_pair.items():
self.slppos_ids[self.slppos_ids == k] = v
for sp in self.slppos_tags:
name = self.slppos_tags.get(sp).get('name')
if srcid in list(self.units_updwon.get(
name).keys()): # dstid MUST be here too.
srcdict = self.units_updwon.get(name)[srcid]
self.units_updwon[name][dstid]['area'] += srcdict['area']
for luid in srcdict['landuse']:
if luid not in self.units_updwon.get(
name)[dstid]['landuse']:
self.units_updwon[name][dstid]['landuse'][luid] = \
srcdict['landuse'][luid]
else:
self.units_updwon[name][dstid]['landuse'][luid] += \
srcdict['landuse'][luid]
if srcdict['hillslope'][0] not in self.units_updwon[name][
dstid]['hillslope']:
self.units_updwon[name][dstid]['hillslope'].append(
srcdict['hillslope'][0])
if srcdict['subbasin'][0] not in self.units_updwon[name][
dstid]['subbasin']:
self.units_updwon[name][dstid]['subbasin'].append(
srcdict['subbasin'][0])
self.units_updwon.get(name).pop(srcid)
break
# rewrite the overview
units_count = 0
for sp in self.slppos_tags:
name = self.slppos_tags.get(sp).get('name')
cur_units_count = len(list(self.units_updwon.get(name).keys()))
self.units_updwon.get('overview')[name] = cur_units_count
units_count += cur_units_count
self.units_updwon.get('overview')['all_units'] = units_count
def merge_slopeposition_units(self):
"""Merge hillslope/subbasin with incomplete slope position sequences"""
all_units_ids = list()
for sp in self.slppos_tags:
name = self.slppos_tags.get(sp).get('name')
all_units_ids += list(self.units_updwon.get(name).keys())
# loop each slope position units
hillslp_elim = dict()
for sp, spdict in self.slppos_tags.items():
name = spdict.get('name')
for unitid, unitdict in self.units_updwon.get(name).items():
flag = True # complete sequences or not
if spdict.get('upslope') > 0 > unitdict.get('upslope'):
flag = False
if spdict.get('downslope') > 0 > unitdict.get('downslope'):
flag = False
cur_hillslp_id = unitdict.get('hillslope')[0]
if not flag:
if cur_hillslp_id not in hillslp_elim:
hillslp_elim[cur_hillslp_id] = [unitid]
if cur_hillslp_id in hillslp_elim and unitid not in hillslp_elim[cur_hillslp_id]:
hillslp_elim[cur_hillslp_id].append(unitid)
print('Hillslope with incomplete slope position sequences: ' + hillslp_elim.__str__())
# Basic procedure:
# 1. if header is incomplete, search left and right;
# if left/right is incomplete, search header and right/left;
# 2. if still not incomplete, check the subbasin,
# 3. if still not satisfied, merge the left and right to the left or right of downstream
# (or upstream for outlet subbasin). The header always with the left hillslope.
hillslope_merge_order = {0: [1, 2], 1: [0, 2], 2: [0, 1]}
units_pair = dict() # unit id pairs, key is merged unit, value is destination unit.
for hillid, unitids in hillslp_elim.items():
subbsnid = DelineateHillslope.get_subbasin_from_hillslope_id(hillid, self.subbsin_num)
hillids = DelineateHillslope.cal_hs_codes(self.subbsin_num, subbsnid)
idx = hillids.index(hillid)
# slope positions that existed in current hillslope
sp_seq = list()
for _id in unitids:
_id /= hillid
if _id not in sp_seq:
sp_seq.append(_id)
satisfied = False
cur_units_pair = dict()
for curidx in hillslope_merge_order[idx]:
dst_hillid = hillids[curidx]
for _id in sp_seq:
dst_unit = _id * dst_hillid
if dst_unit in all_units_ids:
cur_units_pair[_id * hillid] = dst_unit
for sp in self.slppos_tags:
_uid = sp * dst_hillid
if _uid in all_units_ids and sp not in sp_seq:
sp_seq.append(sp)
if SlopePositionUnits.check_slppos_sequence(sp_seq, self.slppos_tags):
satisfied = True
break
if satisfied:
units_pair.update(cur_units_pair)
else: # means the current subbasin should be merged to downstream
downstream_merge_idx = {0: 1, 1: 1, 2: 2}
cur_units_pair = dict()
dst_subbsn = self.subbsin_tree.get(subbsnid)
if dst_subbsn < 0: # outlet subbasin
for k, v in self.subbsin_tree.items():
if v == subbsnid:
dst_subbsn = k
break
if dst_subbsn > 0: # although dst_subbsn may not less than 0, just make sure that!
dst_hillids = DelineateHillslope.cal_hs_codes(self.subbsin_num, dst_subbsn)
dst_downstream_hs = dst_hillids[downstream_merge_idx[idx]]
for _id in sp_seq:
dst_unit = _id * dst_downstream_hs
if dst_unit in all_units_ids:
cur_units_pair[_id * hillid] = dst_unit
units_pair.update(cur_units_pair)
print('Slope position unit merge-destination pairs: ' + units_pair.__str__())
# begin to merge
for srcid, dstid in units_pair.items():
# replace values in slope position units raster
for k, v in units_pair.items():
self.slppos_ids[self.slppos_ids == k] = v
for sp in self.slppos_tags:
name = self.slppos_tags.get(sp).get('name')
if srcid in list(self.units_updwon.get(name).keys()): # dstid MUST be here too.
srcdict = self.units_updwon.get(name)[srcid]
self.units_updwon[name][dstid]['area'] += srcdict['area']
for luid in srcdict['landuse']:
if luid not in self.units_updwon.get(name)[dstid]['landuse']:
self.units_updwon[name][dstid]['landuse'][luid] = \
srcdict['landuse'][luid]
else:
self.units_updwon[name][dstid]['landuse'][luid] += \
srcdict['landuse'][luid]
if srcdict['hillslope'][0] not in self.units_updwon[name][dstid]['hillslope']:
self.units_updwon[name][dstid]['hillslope'].append(srcdict['hillslope'][0])
if srcdict['subbasin'][0] not in self.units_updwon[name][dstid]['subbasin']:
self.units_updwon[name][dstid]['subbasin'].append(srcdict['subbasin'][0])
self.units_updwon.get(name).pop(srcid)
break
# rewrite the overview
units_count = 0
for sp in self.slppos_tags:
name = self.slppos_tags.get(sp).get('name')
cur_units_count = len(list(self.units_updwon.get(name).keys()))
self.units_updwon.get('overview')[name] = cur_units_count
units_count += cur_units_count
self.units_updwon.get('overview')['all_units'] = units_count