def calibration_objectives(cali_obj, ind):
"""Evaluate the objectives of given individual.
"""
cali_obj.ID = ind.id
model_args = cali_obj.model.ConfigDict
model_args.setdefault('calibration_id', -1)
model_args['calibration_id'] = ind.id
model_obj = MainSEIMS(args_dict=model_args)
# Set observation data to model_obj, no need to query database
model_obj.SetOutletObservations(ind.obs.vars, ind.obs.data)
# Execute model
model_obj.SetMongoClient()
model_obj.run()
time.sleep(0.1) # Wait a moment in case of unpredictable file system error
# read simulation data of the entire simulation period (include calibration and validation)
if model_obj.ReadTimeseriesSimulations():
ind.sim.vars = model_obj.sim_vars[:]
ind.sim.data = deepcopy(model_obj.sim_value)
else:
model_obj.clean(calibration_id=ind.id)
model_obj.UnsetMongoClient()
return ind
# Calculate NSE, R2, RMSE, PBIAS, and RSR, etc. of calibration period
ind.cali.vars, ind.cali.data = model_obj.ExtractSimData(
cali_obj.cfg.cali_stime, cali_obj.cfg.cali_etime)
ind.cali.sim_obs_data = model_obj.ExtractSimObsData(
cali_obj.cfg.cali_stime, cali_obj.cfg.cali_etime)
ind.cali.objnames, \
ind.cali.objvalues = model_obj.CalcTimeseriesStatistics(ind.cali.sim_obs_data,
cali_obj.cfg.cali_stime,
cali_obj.cfg.cali_etime)
if ind.cali.objnames and ind.cali.objvalues:
ind.cali.valid = True
# Calculate NSE, R2, RMSE, PBIAS, and RSR, etc. of validation period
if cali_obj.cfg.calc_validation:
ind.vali.vars, ind.vali.data = model_obj.ExtractSimData(
cali_obj.cfg.vali_stime, cali_obj.cfg.vali_etime)
ind.vali.sim_obs_data = model_obj.ExtractSimObsData(
cali_obj.cfg.vali_stime, cali_obj.cfg.vali_etime)
ind.vali.objnames, \
ind.vali.objvalues = model_obj.CalcTimeseriesStatistics(ind.vali.sim_obs_data,
cali_obj.cfg.vali_stime,
cali_obj.cfg.vali_etime)
if ind.vali.objnames and ind.vali.objvalues:
ind.vali.valid = True
# Get timespan
ind.io_time, ind.comp_time, ind.simu_time, ind.runtime = model_obj.GetTimespan(
)
# delete model output directory for saving storage
model_obj.clean(calibration_id=ind.id)
model_obj.UnsetMongoClient()
return ind
def main():
cur_path = UtilClass.current_path(lambda: 0)
SEIMS_path = os.path.abspath(cur_path + '../../..')
model_paths = ModelPaths(SEIMS_path, 'dianbu2', 'demo_dianbu2_model')
scenario_id = 0
seims_obj = MainSEIMS(model_paths.bin_dir, model_paths.model_dir, scenario_id=scenario_id)
seims_obj.run()
def main():
cur_path = UtilClass.current_path(lambda: 0)
SEIMS_path = os.path.abspath(cur_path + '../../..')
model_paths = ModelPaths(SEIMS_path, 'dianbu2', 'demo_dianbu2_model')
scenario_id = 0
seims_obj = MainSEIMS(model_paths.bin_dir,
model_paths.model_dir,
scenario_id=scenario_id)
seims_obj.run()
def calibration_objectives(cali_obj, ind):
"""Evaluate the objectives of given individual.
"""
cali_obj.ID = ind.id
model_args = cali_obj.model.ConfigDict
model_args.setdefault('calibration_id', -1)
model_args['calibration_id'] = ind.id
model_obj = MainSEIMS(args_dict=model_args)
# Set observation data to model_obj, no need to query database
model_obj.SetOutletObservations(ind.obs.vars, ind.obs.data)
# Execute model
model_obj.run()
time.sleep(0.1) # Wait a moment in case of unpredictable file system error
# read simulation data of the entire simulation period (include calibration and validation)
if model_obj.ReadTimeseriesSimulations():
ind.sim.vars = model_obj.sim_vars[:]
ind.sim.data = deepcopy(model_obj.sim_value)
else:
return ind
# Calculate NSE, R2, RMSE, PBIAS, and RSR, etc. of calibration period
ind.cali.vars, ind.cali.data = model_obj.ExtractSimData(cali_obj.cfg.cali_stime,
cali_obj.cfg.cali_etime)
ind.cali.sim_obs_data = model_obj.ExtractSimObsData(cali_obj.cfg.cali_stime,
cali_obj.cfg.cali_etime)
ind.cali.objnames, \
ind.cali.objvalues = model_obj.CalcTimeseriesStatistics(ind.cali.sim_obs_data,
cali_obj.cfg.cali_stime,
cali_obj.cfg.cali_etime)
if ind.cali.objnames and ind.cali.objvalues:
ind.cali.valid = True
# Calculate NSE, R2, RMSE, PBIAS, and RSR, etc. of validation period
if cali_obj.cfg.calc_validation:
ind.vali.vars, ind.vali.data = model_obj.ExtractSimData(cali_obj.cfg.vali_stime,
cali_obj.cfg.vali_etime)
ind.vali.sim_obs_data = model_obj.ExtractSimObsData(cali_obj.cfg.vali_stime,
cali_obj.cfg.vali_etime)
ind.vali.objnames, \
ind.vali.objvalues = model_obj.CalcTimeseriesStatistics(ind.vali.sim_obs_data,
cali_obj.cfg.vali_stime,
cali_obj.cfg.vali_etime)
if ind.vali.objnames and ind.vali.objvalues:
ind.vali.valid = True
# Get timespan
ind.io_time, ind.comp_time, ind.simu_time, ind.runtime = model_obj.GetTimespan()
# delete model output directory for saving storage
shutil.rmtree(model_obj.output_dir)
return ind
def main():
wtsd_name = get_watershed_name(
'Specify watershed name to run SEIMS-based model.')
if wtsd_name not in list(DEMO_MODELS.keys()):
print('%s is not one of the available demo watershed: %s' %
(wtsd_name, ','.join(list(DEMO_MODELS.keys()))))
exit(-1)
cur_path = UtilClass.current_path(lambda: 0)
SEIMS_path = os.path.abspath(cur_path + '../../..')
model_paths = ModelPaths(SEIMS_path, wtsd_name, DEMO_MODELS[wtsd_name])
scenario_id = 0
seims_obj = MainSEIMS(model_paths.bin_dir,
model_paths.model_dir,
scenario_id=scenario_id)
seims_obj.run()
def main():
wtsd_name = get_watershed_name(
'Specify watershed name to run SEIMS-based model.')
if wtsd_name not in list(DEMO_MODELS.keys()):
print('%s is not one of the available demo watershed: %s' %
(wtsd_name, ','.join(list(DEMO_MODELS.keys()))))
exit(-1)
cur_path = UtilClass.current_path(lambda: 0)
SEIMS_path = os.path.abspath(cur_path + '../../..')
model_paths = ModelPaths(SEIMS_path, wtsd_name, DEMO_MODELS[wtsd_name])
runmodel_cfg = write_runmodel_config_file(model_paths, 'runmodel.ini')
seims_obj = MainSEIMS(args_dict=runmodel_cfg.ConfigDict)
seims_obj.run()
for l in seims_obj.runlogs:
print(l)
def execute_seims_model(self):
"""Run SEIMS for evaluating environmental effectiveness.
If execution fails, the `self.economy` and `self.environment` will be set the worst values.
"""
print_message('Scenario ID: %d, running SEIMS model...' % self.ID)
seims_obj = MainSEIMS(self.bin_dir, self.model_dir, self.nthread,
self.lyrmethod, self.hostname, self.port, self.ID)
self.modelrun = seims_obj.run()
return self.modelrun
def execute_seims_model(self):
"""Run SEIMS for evaluating environmental effectiveness.
If execution fails, the `self.economy` and `self.environment` will be set the worst values.
"""
print_message('Scenario ID: %d, running SEIMS model...' % self.ID)
seims_obj = MainSEIMS(self.bin_dir, self.model_dir, self.nthread,
self.lyrmethod, self.hostname, self.port,
self.ID)
self.modelrun = seims_obj.run()
return self.modelrun
class Scenario(object):
"""Base class of Scenario Analysis.
Attributes:
ID(integer): Unique ID in BMPScenario database -> BMP_SCENARIOS collection
eval_timerange(float): Simulation time range, read from MongoDB, the unit is year.
economy(float): Economical effectiveness, e.g., income minus expenses
environment(float): Environmental effectiveness, e.g., reduction rate of soil erosion
gene_num(integer): The number of genes of one chromosome, i.e., an individual
gene_values(list): BMP identifiers on each location of gene. The length is gen_num.
bmp_items(dict): BMP configuration items that can be imported to MongoDB directly.
The key is `bson.objectid.ObjectId`, the value is scenario item dict.
rules(boolean): Config BMPs randomly or rule-based.
modelrun(boolean): Has SEIMS model run successfully?
"""
def __init__(self, cfg):
# type: (SAConfig) -> None
"""Initialize."""
self.ID = -1
self.eval_timerange = 1. # unit: year
self.economy = 0.
self.environment = 0.
self.worst_econ = cfg.worst_econ
self.worst_env = cfg.worst_env
self.gene_num = 0
self.gene_values = list() # type: List[int]
self.bmp_items = dict()
self.rule_mtd = cfg.bmps_cfg_method
self.bmps_info = cfg.bmps_info
self.bmps_retain = cfg.bmps_retain
self.eval_info = cfg.eval_info
self.export_sce_txt = cfg.export_sce_txt
self.export_sce_tif = cfg.export_sce_tif
self.scenario_dir = cfg.scenario_dir # predefined directories to store scenarios related
# SEIMS-based model related
self.modelcfg = cfg.model
self.modelcfg_dict = self.modelcfg.ConfigDict
self.model = MainSEIMS(args_dict=self.modelcfg_dict)
self.model.SetMongoClient()
self.model.ReadMongoDBData()
self.scenario_db = self.model.ScenarioDBName
self.model.ResetSimulationPeriod() # Reset the simulation period
# Reset the starttime and endtime of the desired outputs according to evaluation period
if ModelCfgFields.output_id in self.eval_info:
self.model.ResetOutputsPeriod(
self.eval_info[ModelCfgFields.output_id], cfg.eval_stime,
cfg.eval_etime)
else:
print(
'Warning: No OUTPUTID is defined in BMPs_info. Please make sure the '
'STARTTIME and ENDTIME of ENVEVAL are consistent with Evaluation period!'
)
self.model.UnsetMongoClient() # Unset in time!
# (Re)Calculate timerange in the unit of year
dlt = cfg.eval_etime - cfg.eval_stime + timedelta(seconds=1)
self.eval_timerange = (dlt.days * 86400. + dlt.seconds) / 86400. / 365.
self.modelout_dir = None # determined in `execute_seims_model` based on unique scenario ID
self.modelrun = False # indicate whether the model has been executed
def set_unique_id(self, given_id=None):
# type: (Optional[int]) -> int
"""Set unique ID."""
if given_id is None:
self.ID = next(generate_uniqueid())
else:
self.ID = given_id
# Update scenario ID for self.modelcfg and self.model
self.model.scenario_id = self.ID
self.modelcfg.scenario_id = self.ID
self.modelcfg_dict[
'scenario_id'] = self.ID if self.modelcfg_dict else 0
return self.ID
def rule_based_config(self, method, conf_rate):
# type: (float, str) -> None
"""Config available BMPs to each gene of the chromosome by rule-based method.
Virtual function that should be overridden in inherited Scenario class.
"""
pass
def random_based_config(self, conf_rate):
# type: (float) -> None
"""Config available BMPs to each gene of the chromosome by random-based method.
Virtual function that should be overridden in inherited Scenario class.
"""
pass
def decoding(self):
"""Decoding gene_values to bmp_items
This function should be overridden.
"""
pass
def export_to_mongodb(self):
"""Export current scenario to MongoDB.
Delete the same ScenarioID if existed.
"""
# client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port)
# conn = client.get_conn()
conn = MongoDBObj.client
db = conn[self.scenario_db]
collection = db[DBTableNames.scenarios]
try:
# find ScenarioID, remove if existed.
if collection.find({
'ID': self.ID
}, no_cursor_timeout=True).count():
collection.remove({'ID': self.ID})
except NetworkTimeout or Exception:
# In case of unexpected raise
pass
for objid, bmp_item in viewitems(self.bmp_items):
bmp_item['_id'] = ObjectId()
collection.insert_one(bmp_item)
# client.close()
def export_scenario_to_txt(self):
"""Export current scenario information to text file.
This function is better be called after `calculate_environment` and `calculate_environment`
or in static method, e.g., `scenario_effectiveness`.
"""
if not self.export_sce_txt:
return
ofile = self.scenario_dir + os.path.sep + 'Scenario_%d.txt' % self.ID
with open(ofile, 'w', encoding='utf-8') as outfile:
outfile.write('Scenario ID: %d\n' % self.ID)
outfile.write('Gene number: %d\n' % self.gene_num)
outfile.write('Gene values: %s\n' % ', '.join(
(repr(v) for v in self.gene_values)))
outfile.write('Scenario items:\n')
if len(self.bmp_items) > 0:
header = list()
for obj, item in viewitems(self.bmp_items):
header = list(item.keys())
break
outfile.write('\t'.join(header))
outfile.write('\n')
for obj, item in viewitems(self.bmp_items):
outfile.write('\t'.join(
str(v) for v in list(item.values())))
outfile.write('\n')
outfile.write(
'Effectiveness:\n\teconomy: %f\n\tenvironment: %f\n' %
(self.economy, self.environment))
def export_scenario_to_gtiff(self):
"""Export the areal BMPs to gtiff for further analysis.
This function should be overridden in inherited class.
"""
pass
def import_from_mongodb(self, sid):
"""Import a specified Scenario (`sid`) from MongoDB.
This function should be overridden in inherited class.
Returns:
True if succeed, otherwise False.
"""
pass
def import_from_txt(self, sid):
"""Import a specified Scenario (`sid`) from text file.
This function should be overridden in inherited class.
Returns:
True if succeed, otherwise False.
"""
pass
def calculate_economy(self):
"""Calculate economical effectiveness, which is application specified."""
pass
def calculate_environment(self):
"""Calculate environment effectiveness, which is application specified."""
pass
def clean(self,
scenario_id=None,
calibration_id=None,
delete_scenario=False,
delete_spatial_gfs=False):
"""Clean the intermediate data."""
# model clean
self.model.SetMongoClient()
self.model.clean(scenario_id=scenario_id,
calibration_id=calibration_id,
delete_scenario=delete_scenario,
delete_spatial_gfs=delete_spatial_gfs)
self.model.UnsetMongoClient()
def execute_seims_model(self):
"""Run SEIMS for evaluating environmental effectiveness.
If execution fails, the `self.economy` and `self.environment` will be set the worst values.
"""
scoop_log('Scenario ID: %d, running SEIMS model...' % self.ID)
self.model.scenario_id = self.ID
self.modelout_dir = self.model.OutputDirectory
self.model.SetMongoClient()
self.model.run()
self.model.UnsetMongoClient()
self.modelrun = True
return self.model.run_success
def initialize(self, input_genes=None):
# type: (Optional[List]) -> List
"""Initialize a scenario.
Returns:
A list contains BMPs identifier of each gene location.
"""
pass