def __init__(self, cf):
# type: (ConfigParser) -> None
"""Initialization."""
SAConfig.__init__(self, cf) # initialize base class first
# 1. Check the required key and values
requiredkeys = [
'COLLECTION', 'DISTRIBUTION', 'SUBSCENARIO', 'ENVEVAL', 'BASE_ENV',
'UNITJSON'
]
for k in requiredkeys:
if k not in self.bmps_info:
raise ValueError(
'%s: MUST be provided in BMPs_cfg_units or BMPs_info!' % k)
# 2. Slope position units information
units_json = self.bmps_info.get('UNITJSON')
unitsf = self.model.model_dir + os.sep + units_json
if not FileClass.is_file_exists(unitsf):
raise Exception('UNITJSON file %s is not existed!' % unitsf)
with open(unitsf, 'r', encoding='utf-8') as updownfo:
self.units_infos = json.load(updownfo)
self.units_infos = UtilClass.decode_strs_in_dict(self.units_infos)
if 'overview' not in self.units_infos:
raise ValueError('overview MUST be existed in the UNITJSON file.')
if 'all_units' not in self.units_infos['overview']:
raise ValueError(
'all_units MUST be existed in overview dict of UNITJSON.')
self.units_num = self.units_infos['overview']['all_units'] # type: int
# 3. Collection name and subscenario IDs
self.bmps_coll = self.bmps_info.get('COLLECTION') # type: str
self.bmps_subids = self.bmps_info.get('SUBSCENARIO') # type: List[int]
# 4. Construct the dict of gene index to unit ID, and unit ID to gene index
self.unit_to_gene = OrderedDict() # type: OrderedDict[int, int]
self.gene_to_unit = dict() # type: Dict[int, int]
# 5. Construct the upstream-downstream units of each unit if necessary
self.updown_units = dict() # type: Dict[int, Dict[AnyStr, List[int]]]
def main_manual(sceid, gene_values):
"""Test of set scenario manually."""
cf = get_config_parser()
base_cfg = SAConfig(cf) # type: SAConfig
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS
cfg = SASlpPosConfig(cf)
elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD
cfg = SAConnFieldConfig(cf)
else: # Common spatial units, e.g., HRU and EXPLICITHRU
cfg = SACommUnitConfig(cf)
cfg.construct_indexes_units_gene()
sce = SUScenario(cfg)
sce.set_unique_id(sceid)
sce.initialize(input_genes=gene_values)
sce.boundary_adjustment()
sce.decoding()
sce.export_to_mongodb()
sce.execute_seims_model()
sce.export_sce_tif = True
sce.export_scenario_to_gtiff(sce.model.OutputDirectory + os.sep +
'scenario_%d.tif' % sceid)
sce.calculate_economy()
sce.calculate_environment()
print('Scenario %d: %s\n' %
(sceid, ', '.join(repr(v) for v in sce.gene_values)))
print('Effectiveness:\n\teconomy: %f\n\tenvironment: %f\n' %
(sce.economy, sce.environment))
sce.clean(delete_scenario=True, delete_spatial_gfs=True)
def main_single():
"""Test of single evaluation of scenario."""
cf = get_config_parser()
base_cfg = SAConfig(cf) # type: SAConfig
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS
cfg = SASlpPosConfig(cf)
elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD
cfg = SAConnFieldConfig(cf)
else: # Common spatial units, e.g., HRU and EXPLICITHRU
cfg = SACommUnitConfig(cf)
cfg.construct_indexes_units_gene()
sce = SUScenario(cfg)
sce.initialize()
sce.boundary_adjustment()
sceid = sce.set_unique_id()
print(sceid, sce.gene_values.__str__())
sce.decoding()
sce.export_to_mongodb()
sce.execute_seims_model()
sce.calculate_economy()
sce.calculate_environment()
print('Scenario %d: %s\n' %
(sceid, ', '.join(repr(v) for v in sce.gene_values)))
print('Effectiveness:\n\teconomy: %f\n\tenvironment: %f\n' %
(sce.economy, sce.environment))
def main_multiple(eval_num):
# type: (int) -> None
"""Test of multiple evaluations of scenarios."""
cf = get_config_parser()
base_cfg = SAConfig(cf) # type: SAConfig
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS
cfg = SASlpPosConfig(cf)
elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD
cfg = SAConnFieldConfig(cf)
else: # Common spatial units, e.g., HRU and EXPLICITHRU
cfg = SACommUnitConfig(cf)
cfg.construct_indexes_units_gene()
cost = list()
for _ in range(eval_num):
sce = SUScenario(cfg)
sce.initialize()
sceid = sce.set_unique_id()
print(sceid, sce.gene_values.__str__())
sce.calculate_economy()
cost.append(sce.economy)
print(max(cost), min(cost), sum(cost) / len(cost))
def main():
wtsd_name = get_watershed_name(
'Specify watershed name to run scenario analysis.')
if wtsd_name not in list(DEMO_MODELS.keys()):
print('%s is not one of the available demo watershed: %s' %
(wtsd_name, ','.join(list(DEMO_MODELS.keys()))))
exit(-1)
cur_path = UtilClass.current_path(lambda: 0)
SEIMS_path = os.path.abspath(cur_path + '../../..')
model_paths = ModelPaths(SEIMS_path, wtsd_name, DEMO_MODELS[wtsd_name])
cf = write_scenario_analysis_config_file(model_paths,
'scenario_analysis.ini')
base_cfg = SAConfig(cf) # type: SAConfig
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS
cfg = SASlpPosConfig(cf)
elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD
cfg = SAConnFieldConfig(cf)
else: # Common spatial units, e.g., HRU and EXPLICITHRU
cfg = SACommUnitConfig(cf)
cfg.construct_indexes_units_gene()
sce = SUScenario(cfg)
scoop_log('### START TO SCENARIOS OPTIMIZING ###')
start_t = time.time()
fpop, fstats = sa_nsga2.main(sce)
fpop.sort(key=lambda x: x.fitness.values)
scoop_log(fstats)
with open(cfg.opt.logbookfile, 'w', encoding='utf-8') as f:
# In case of 'TypeError: write() argument 1 must be unicode, not str' in Python2.7
# when using unicode_literals, please use '%s' to concatenate string!
f.write('%s' % fstats.__str__())
end_t = time.time()
scoop_log('Running time: %.2fs' % (end_t - start_t))
def main_test_crossover_mutate(gen_num, cx_rate, mut_perc, mut_rate):
# type: (int, float, float, float) -> None
"""Test mutate function."""
from deap import base
from scenario_analysis import BMPS_CFG_UNITS, BMPS_CFG_METHODS
from scenario_analysis.config import SAConfig
from scenario_analysis.spatialunits.config import SASlpPosConfig, SAConnFieldConfig, \
SACommUnitConfig
from scenario_analysis.spatialunits.scenario import SUScenario
cf = get_config_parser()
base_cfg = SAConfig(cf) # type: SAConfig
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS
cfg = SASlpPosConfig(cf)
elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD
cfg = SAConnFieldConfig(cf)
else: # Common spatial units, e.g., HRU and EXPLICITHRU
cfg = SACommUnitConfig(cf)
cfg.construct_indexes_units_gene()
# Initialize gene values for individual 1 and 2
sce1 = SUScenario(cfg)
ind1 = sce1.initialize()
sceid1 = sce1.set_unique_id()
print('Scenario1-ID: %d\n Initial genes: %s' % (sceid1, ind1.__str__()))
sce2 = SUScenario(cfg)
ind2 = sce2.initialize()
sceid2 = sce2.set_unique_id()
print('Scenario2-ID: %d\n Initial genes: %s' % (sceid2, ind2.__str__()))
# Clone old individuals
toolbox = base.Toolbox()
for gen_id in list(range(gen_num)):
# Calculate initial economic benefit
inicost1 = sce1.calculate_economy()
inicost2 = sce2.calculate_economy()
print('## Generation %d ##' % gen_id)
# Crossover
print('Crossover:')
old_ind1 = toolbox.clone(ind1)
old_ind2 = toolbox.clone(ind2)
if random.random() <= cx_rate:
if base_cfg.bmps_cfg_method == BMPS_CFG_METHODS[
3]: # SLPPOS method
crossover_slppos(ind1, ind2, sce1.cfg.hillslp_genes_num)
elif base_cfg.bmps_cfg_method == BMPS_CFG_METHODS[
2]: # UPDOWN method
crossover_updown(sce1.cfg.updown_units, sce1.cfg.gene_to_unit,
sce1.cfg.unit_to_gene, ind1, ind2)
else:
crossover_rdm(ind1, ind2)
if not check_individual_diff(
old_ind1, ind1) and not check_individual_diff(old_ind2, ind2):
print(' No crossover occurred on Scenario1 and Scenario2!')
else:
print(' Crossover genes:\n Scenario1: %s\n'
' Scenario2: %s' % (ind1.__str__(), ind2.__str__()))
# Calculate economic benefit after crossover
setattr(sce1, 'gene_values', ind1)
cxcost1 = sce1.calculate_economy()
setattr(sce2, 'gene_values', ind2)
cxcost2 = sce2.calculate_economy()
# Mutate
print('Mutate:')
old_ind1 = toolbox.clone(ind1)
old_ind2 = toolbox.clone(ind2)
if base_cfg.bmps_cfg_method == BMPS_CFG_METHODS[0]:
possible_gene_values = list(sce1.bmps_params.keys())
if 0 not in possible_gene_values:
possible_gene_values.append(0)
mutate_rdm(possible_gene_values,
ind1,
perc=mut_perc,
indpb=mut_rate)
mutate_rdm(possible_gene_values,
ind2,
perc=mut_perc,
indpb=mut_rate)
else:
tagnames = None
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]:
tagnames = sce1.cfg.slppos_tagnames
mutate_rule(sce1.cfg.units_infos,
sce1.cfg.gene_to_unit,
sce1.cfg.unit_to_gene,
sce1.suit_bmps['LANDUSE'],
ind1,
mut_perc,
mut_rate,
unit=base_cfg.bmps_cfg_unit,
method=base_cfg.bmps_cfg_method,
bmpgrades=sce1.bmps_grade,
tagnames=tagnames,
thresholds=sce1.cfg.boundary_adaptive_threshs)
mutate_rule(sce2.cfg.units_infos,
sce2.cfg.gene_to_unit,
sce2.cfg.unit_to_gene,
sce2.suit_bmps['LANDUSE'],
ind2,
mut_perc,
mut_rate,
unit=base_cfg.bmps_cfg_unit,
method=base_cfg.bmps_cfg_method,
bmpgrades=sce2.bmps_grade,
tagnames=tagnames,
thresholds=sce2.cfg.boundary_adaptive_threshs)
if not check_individual_diff(old_ind1, ind1):
print(' No mutation occurred on Scenario1!')
else:
print(' Mutated genes of Scenario1: %s' % ind1.__str__())
if not check_individual_diff(old_ind2, ind2):
print(' No mutation occurred on Scenario2!')
else:
print(' Mutated genes of Scenario2: %s' % ind2.__str__())
# Calculate economic benefit after mutate
setattr(sce1, 'gene_values', ind1)
mutcost1 = sce1.calculate_economy()
setattr(sce2, 'gene_values', ind2)
mutcost2 = sce2.calculate_economy()
print('Initial cost: \n'
' Scenario1: %.3f, Scenario2: %.3f\n'
'Crossover:\n'
' Scenario1: %.3f, Scenario2: %.3f\n'
'Mutation:\n'
' Scenario1: %.3f, Scenario2: %.3f\n' %
(inicost1, inicost2, cxcost1, cxcost2, mutcost1, mutcost2))
idx += 1
spidx += 1
spname = self.slppos_tagnames[spidx][1]
uid = next_uid
next_uid = self.units_infos[spname][next_uid]['downslope']
assert (idx == self.units_num)
# Trace upslope and append their unit IDs
for cuid in self.updown_units:
self.updown_units[cuid]['all_upslope'] = trace_upslope_units(
cuid, self.updown_units)[:]
if __name__ == '__main__':
cf = get_config_parser()
base_cfg = SAConfig(cf) # type: SAConfig
if base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[3]: # SLPPOS
cfg = SASlpPosConfig(cf)
elif base_cfg.bmps_cfg_unit == BMPS_CFG_UNITS[2]: # CONNFIELD
cfg = SAConnFieldConfig(cf)
else: # Common spatial units, e.g., HRU and EXPLICITHRU
cfg = SACommUnitConfig(cf)
cfg.construct_indexes_units_gene()
# test the picklable of SASPUConfig class.
import pickle
s = pickle.dumps(cfg)
new_cfg = pickle.loads(
s) # type: Union[SASlpPosConfig, SAConnFieldConfig, SACommUnitConfig]
print('BMPs configuration units number: %d ' % new_cfg.units_num)
def __init__(self, cf):
"""Initialization."""
SAConfig.__init__(self, cf) # initialize base class first
# Handling self.bmps_info for specific application
# 1. Check the required key and values
requiredkeys = ['COLLECTION', 'DISTRIBUTION', 'SUBSCENARIO', 'UPDOWNJSON',
'ENVEVAL', 'BASE_ENV']
for k in requiredkeys:
if k not in self.bmps_info:
raise ValueError('[%s]: MUST be provided!' % k)
# 2. Slope position units information
updownf = self.bmps_info.get('UPDOWNJSON')
FileClass.check_file_exists(updownf)
with open(updownf, 'r') as updownfo:
self.units_infos = json.load(updownfo)
self.units_infos = UtilClass.decode_strs_in_dict(self.units_infos)
# 3. Get slope position sequence
sptags = cf.get('BMPs', 'slppos_tag_name')
self.slppos_tags = json.loads(sptags)
self.slppos_tags = UtilClass.decode_strs_in_dict(self.slppos_tags)
self.slppos_tagnames = sorted(list(self.slppos_tags.items()), key=operator.itemgetter(0))
self.slppos_unit_num = self.units_infos['overview']['all_units']
self.slppos_to_gene = OrderedDict()
self.gene_to_slppos = dict()
# method 1: (deprecated)
# gene index: 0, 1, 2, ..., n
# slppos unit: rdg1, rdg2,..., bks1, bks2,..., vly1, vly2...
# idx = 0
# for tag, sp in self.slppos_tagnames:
# for uid in self.units_infos[sp]:
# self.gene_to_slppos[idx] = uid
# self.slppos_to_gene[uid] = idx
# idx += 1
# method 2:
# gene index: 0, 1, 2, ..., n
# slppos unit: rdg1, bks2, vly1,..., rdgn, bksn, vlyn
idx = 0
spname = self.slppos_tagnames[0][1]
for uid, udict in self.units_infos[spname].items():
spidx = 0
self.gene_to_slppos[idx] = uid
self.slppos_to_gene[uid] = idx
idx += 1
next_uid = udict['downslope']
while next_uid > 0:
self.gene_to_slppos[idx] = next_uid
self.slppos_to_gene[next_uid] = idx
idx += 1
spidx += 1
spname = self.slppos_tagnames[spidx][1]
next_uid = self.units_infos[spname][next_uid]['downslope']
assert (idx == self.slppos_unit_num)
# 4. SubScenario IDs and parameters read from MongoDB
self.bmps_subids = self.bmps_info.get('SUBSCENARIO')
self.bmps_coll = self.bmps_info.get('COLLECTION')
self.bmps_params = dict()
self.slppos_suit_bmps = dict()
self.read_bmp_parameters()
self.get_suitable_bmps_for_slppos()
A list contains BMPs identifier of each gene location.
"""
# Create configuration rate for each location randomly, 0.25 ~ 0.75
cr = random.randint(40, 60) / 100.
# cr = 0.75
if self.rules:
self.rule_based_config(cr)
else:
self.random_based_config(cr)
if len(self.gene_values) == self.gene_num > 0:
return self.gene_values
else:
raise RuntimeError(
'Initialize Scenario failed, please check the inherited scenario'
' class, especially the overwritten rule_based_config and'
' random_based_config!')
if __name__ == '__main__':
cf = get_config_parser()
cfg = SAConfig(cf)
sceobj = Scenario(cfg)
# test the picklable of Scenario class.
import pickle
s = pickle.dumps(sceobj)
# print(s)
new_cfg = pickle.loads(s)
print(new_cfg.bin_dir)
