def __init__(self, cf):
# type: (ConfigParser) -> None
"""Initialization."""
SAConfig.__init__(self, cf) # initialize base class first
# 1. Check the required key and values
requiredkeys = [
'COLLECTION', 'DISTRIBUTION', 'SUBSCENARIO', 'ENVEVAL', 'BASE_ENV',
'UNITJSON'
]
for k in requiredkeys:
if k not in self.bmps_info:
raise ValueError(
'%s: MUST be provided in BMPs_cfg_units or BMPs_info!' % k)
# 2. Slope position units information
units_json = self.bmps_info.get('UNITJSON')
unitsf = self.model.model_dir + os.sep + units_json
if not FileClass.is_file_exists(unitsf):
raise Exception('UNITJSON file %s is not existed!' % unitsf)
with open(unitsf, 'r', encoding='utf-8') as updownfo:
self.units_infos = json.load(updownfo)
self.units_infos = UtilClass.decode_strs_in_dict(self.units_infos)
if 'overview' not in self.units_infos:
raise ValueError('overview MUST be existed in the UNITJSON file.')
if 'all_units' not in self.units_infos['overview']:
raise ValueError(
'all_units MUST be existed in overview dict of UNITJSON.')
self.units_num = self.units_infos['overview']['all_units'] # type: int
# 3. Collection name and subscenario IDs
self.bmps_coll = self.bmps_info.get('COLLECTION') # type: str
self.bmps_subids = self.bmps_info.get('SUBSCENARIO') # type: List[int]
# 4. Construct the dict of gene index to unit ID, and unit ID to gene index
self.unit_to_gene = OrderedDict() # type: OrderedDict[int, int]
self.gene_to_unit = dict() # type: Dict[int, int]
# 5. Construct the upstream-downstream units of each unit if necessary
self.updown_units = dict() # type: Dict[int, Dict[AnyStr, List[int]]]
def calculate_sensitivity(self):
"""Calculate Morris elementary effects.
It is worth to be noticed that evaluate_models() allows to return
several output variables, hence we should calculate each of them separately.
"""
if not self.psa_si:
if FileClass.is_file_exists(self.cfg.outfiles.psa_si_json):
with open(self.cfg.outfiles.psa_si_json, 'r',
encoding='utf-8') as f:
self.psa_si = UtilClass.decode_strs_in_dict(json.load(f))
return
if not self.objnames:
if FileClass.is_file_exists('%s/objnames.pickle' %
self.cfg.psa_outpath):
with open('%s/objnames.pickle' % self.cfg.psa_outpath,
'r',
encoding='utf-8') as f:
self.objnames = pickle.load(f)
if self.output_values is None or len(self.output_values) == 0:
self.evaluate_models()
if self.param_values is None or len(self.param_values) == 0:
self.generate_samples()
if not self.param_defs:
self.read_param_ranges()
row, col = self.output_values.shape
assert (row == self.run_count)
for i in range(col):
print(self.objnames[i])
if self.cfg.method == 'morris':
tmp_Si = morris_alz(self.param_defs,
self.param_values,
self.output_values[:, i],
conf_level=0.95,
print_to_console=True,
num_levels=self.cfg.morris.num_levels)
elif self.cfg.method == 'fast':
tmp_Si = fast_alz(self.param_defs,
self.output_values[:, i],
print_to_console=True)
else:
raise ValueError('%s method is not supported now!' %
self.cfg.method)
self.psa_si[i] = tmp_Si
# print(self.psa_si)
# Save as json, which can be loaded by json.load()
json_data = json.dumps(self.psa_si, indent=4, cls=SpecialJsonEncoder)
with open(self.cfg.outfiles.psa_si_json, 'w', encoding='utf-8') as f:
f.write('%s' % json_data)
self.output_psa_si()
def read_bmp_parameters(self):
"""Read BMP configuration from MongoDB.
Each BMP is stored in Collection as one item identified by 'SUBSCENARIO' field,
so the `self.bmps_params` is dict with BMP_ID ('SUBSCENARIO') as key.
"""
# client = ConnectMongoDB(self.modelcfg.host, self.modelcfg.port)
# conn = client.get_conn()
conn = MongoDBObj.client
scenariodb = conn[self.scenario_db]
bmpcoll = scenariodb[self.cfg.bmps_coll]
findbmps = bmpcoll.find({}, no_cursor_timeout=True)
for fb in findbmps:
fb = UtilClass.decode_strs_in_dict(fb)
if 'SUBSCENARIO' not in fb:
continue
curid = fb['SUBSCENARIO']
if curid not in self.cfg.bmps_subids:
continue
if curid not in self.bmps_params:
self.bmps_params[curid] = dict()
for k, v in fb.items():
if k == 'SUBSCENARIO':
continue
elif k == 'LANDUSE':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = None
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
elif k == 'SLPPOS':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = list(self.cfg.slppos_tags.keys())
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
else:
self.bmps_params[curid][k] = v
def calculate_sensitivity(self):
"""Calculate Morris elementary effects.
It is worth to be noticed that evaluate_models() allows to return
several output variables, hence we should calculate each of them separately.
"""
if not self.psa_si:
if FileClass.is_file_exists(self.cfg.outfiles.psa_si_json):
with open(self.cfg.outfiles.psa_si_json, 'r') as f:
self.psa_si = UtilClass.decode_strs_in_dict(json.load(f))
return
if not self.objnames:
if FileClass.is_file_exists('%s/objnames.pickle' % self.cfg.psa_outpath):
with open('%s/objnames.pickle' % self.cfg.psa_outpath, 'r') as f:
self.objnames = pickle.load(f)
if self.output_values is None or len(self.output_values) == 0:
self.evaluate_models()
if self.param_values is None or len(self.param_values) == 0:
self.generate_samples()
if not self.param_defs:
self.read_param_ranges()
row, col = self.output_values.shape
assert (row == self.run_count)
for i in range(col):
print(self.objnames[i])
if self.cfg.method == 'morris':
tmp_Si = morris_alz(self.param_defs,
self.param_values,
self.output_values[:, i],
conf_level=0.95, print_to_console=True,
num_levels=self.cfg.morris.num_levels,
grid_jump=self.cfg.morris.grid_jump)
elif self.cfg.method == 'fast':
tmp_Si = fast_alz(self.param_defs, self.output_values[:, i],
print_to_console=True)
else:
raise ValueError('%s method is not supported now!' % self.cfg.method)
self.psa_si[i] = tmp_Si
# print(self.psa_si)
# Save as json, which can be loaded by json.load()
json_data = json.dumps(self.psa_si, indent=4, cls=SpecialJsonEncoder)
with open(self.cfg.outfiles.psa_si_json, 'w') as f:
f.write(json_data)
self.output_psa_si()
def read_bmp_parameters(self):
"""Read BMP configuration from MongoDB."""
client = ConnectMongoDB(self.hostname, self.port)
conn = client.get_conn()
scenariodb = conn[self.bmp_scenario_db]
bmpcoll = scenariodb[self.bmps_coll]
findbmps = bmpcoll.find({}, no_cursor_timeout=True)
for fb in findbmps:
fb = UtilClass.decode_strs_in_dict(fb)
if 'SUBSCENARIO' not in fb:
continue
curid = fb['SUBSCENARIO']
if curid not in self.bmps_subids:
continue
if curid not in self.bmps_params:
self.bmps_params[curid] = dict()
for k, v in fb.items():
if k == 'SUBSCENARIO':
continue
elif k == 'LANDUSE':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = None
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
elif k == 'SLPPOS':
if isinstance(v, int):
v = [v]
elif v == 'ALL' or v == '':
v = list(self.slppos_tags.keys())
else:
v = StringClass.extract_numeric_values_from_string(v)
v = [int(abs(nv)) for nv in v]
self.bmps_params[curid][k] = v[:]
else:
self.bmps_params[curid][k] = v
client.close()
def read_param_ranges(self):
"""Read param_rng.def file
name,lower_bound,upper_bound,group,dist
(group and dist are optional)
e.g.,
Param1,0,1[,Group1][,dist1]
Param2,0,1[,Group2][,dist2]
Param3,0,1[,Group3][,dist3]
Returns:
a dictionary containing:
- names - the names of the parameters
- bounds - a list of lists of lower and upper bounds
- num_vars - a scalar indicating the number of variables
(the length of names)
- groups - a list of group names (strings) for each variable
- dists - a list of distributions for the problem,
None if not specified or all uniform
"""
# read param_defs.json if already existed
if not self.param_defs:
if FileClass.is_file_exists(self.cfg.outfiles.param_defs_json):
with open(self.cfg.outfiles.param_defs_json, 'r', encoding='utf-8') as f:
self.param_defs = UtilClass.decode_strs_in_dict(json.load(f))
return
# read param_range_def file and output to json file
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
collection = db['PARAMETERS']
names = list()
bounds = list()
groups = list()
dists = list()
num_vars = 0
items = read_data_items_from_txt(self.cfg.param_range_def)
for item in items:
if len(item) < 3:
continue
# find parameter name, print warning message if not existed
cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True)
if not cursor.count():
print('WARNING: parameter %s is not existed!' % item[0])
continue
num_vars += 1
names.append(item[0])
bounds.append([float(item[1]), float(item[2])])
# If the fourth column does not contain a group name, use
# the parameter name
if len(item) >= 4:
groups.append(item[3])
else:
groups.append(item[0])
if len(item) >= 5:
dists.append(item[4])
else:
dists.append('unif')
if groups == names:
groups = None
elif len(set(groups)) == 1:
raise ValueError('Only one group defined, results will not bemeaningful')
# setting dists to none if all are uniform
# because non-uniform scaling is not needed
if all([d == 'unif' for d in dists]):
dists = None
self.param_defs = {'names': names, 'bounds': bounds,
'num_vars': num_vars, 'groups': groups, 'dists': dists}
# Save as json, which can be loaded by json.load()
json_data = json.dumps(self.param_defs, indent=4, cls=SpecialJsonEncoder)
with open(self.cfg.outfiles.param_defs_json, 'w', encoding='utf-8') as f:
f.write('%s' % json_data)
def __init__(self, cf, method='nsga2'):
# type: (ConfigParser, str) -> None
"""Initialization."""
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf) # type: ParseSEIMSConfig
# 2. Common settings of BMPs scenario
self.eval_stime = None # type: Optional[datetime]
self.eval_etime = None # type: Optional[datetime]
self.worst_econ = 0.
self.worst_env = 0.
self.runtime_years = 0.
self.export_sce_txt = False
self.export_sce_tif = False
if 'Scenario_Common' not in cf.sections():
raise ValueError(
'[Scenario_Common] section MUST be existed in *.ini file.')
self.eval_stime = parse_datetime_from_ini(cf, 'Scenario_Common',
'eval_time_start')
self.eval_etime = parse_datetime_from_ini(cf, 'Scenario_Common',
'eval_time_end')
self.worst_econ = cf.getfloat('Scenario_Common', 'worst_economy')
self.worst_env = cf.getfloat('Scenario_Common', 'worst_environment')
self.runtime_years = cf.getfloat('Scenario_Common', 'runtime_years')
if cf.has_option('Scenario_Common', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('Scenario_Common',
'export_scenario_txt')
if cf.has_option('Scenario_Common', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('Scenario_Common',
'export_scenario_tif')
# 3. Application specific setting section [BMPs]
# Selected BMPs, the key is BMPID, and value is the BMP information dict
self.bmps_info = dict(
) # type: Dict[int, Dict[AnyStr, Union[int, float, AnyStr, List[Union[int, float, AnyStr]]]]]
# BMPs to be constant for generated scenarios during optimization, same format with bmps_info
self.bmps_retain = dict(
) # type: Dict[int, Dict[AnyStr, Union[int, float, AnyStr, List[Union[int, float, AnyStr]]]]]
self.eval_info = dict(
) # type: Dict[AnyStr, Union[int, float, AnyStr]]
self.bmps_cfg_unit = 'CONNFIELD' # type: AnyStr
self.bmps_cfg_method = 'RAND' # type: AnyStr
if 'BMPs' not in cf.sections():
raise ValueError(
'[BMPs] section MUST be existed for specific scenario analysis.'
)
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_info = UtilClass.decode_strs_in_dict(json.loads(bmpsinfostr))
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(self.bmps_retain)
evalinfostr = cf.get('BMPs', 'eval_info')
self.eval_info = UtilClass.decode_strs_in_dict(json.loads(evalinfostr))
bmpscfgunitstr = cf.get('BMPs', 'bmps_cfg_units')
bmpscfgunitdict = UtilClass.decode_strs_in_dict(
json.loads(bmpscfgunitstr))
for unitname, unitcfg in viewitems(bmpscfgunitdict):
self.bmps_cfg_unit = unitname
if self.bmps_cfg_unit not in BMPS_CFG_UNITS:
raise ValueError('BMPs configuration unit MUST be '
'one of %s' % BMPS_CFG_UNITS.__str__())
if not isinstance(unitcfg, dict):
raise ValueError(
'The value of BMPs configuration unit MUST be dict value!')
for cfgname, cfgvalue in viewitems(unitcfg):
for bmpid, bmpdict in viewitems(self.bmps_info):
if cfgname in bmpdict:
continue
self.bmps_info[bmpid][cfgname] = cfgvalue
break
if cf.has_option('BMPs', 'bmps_cfg_method'):
self.bmps_cfg_method = cf.get('BMPs', 'bmps_cfg_method')
if self.bmps_cfg_method not in BMPS_CFG_METHODS:
print('BMPs configuration method MUST be one of %s' %
BMPS_CFG_METHODS.__str__())
self.bmps_cfg_method = 'RAND'
# Check the validation of configuration unit and method
if self.bmps_cfg_method not in BMPS_CFG_PAIR.get(self.bmps_cfg_unit):
raise ValueError('BMPs configuration method %s '
'is not supported on unit %s' %
(self.bmps_cfg_method, self.bmps_cfg_unit))
# Optimize boundary of BMP configuration unit
self.boundary_adaptive = False
self.boundary_adaptive_threshs = None
if cf.has_option('BMPs', 'bmps_cfg_units_opt'):
self.boundary_adaptive = cf.getboolean('BMPs',
'bmps_cfg_units_opt')
if cf.has_option('BMPs', 'boundary_adaptive_threshold'):
tstr = cf.get('BMPs', 'boundary_adaptive_threshold')
self.boundary_adaptive_threshs = StringClass.extract_numeric_values_from_string(
tstr)
if 0 not in self.boundary_adaptive_threshs:
self.boundary_adaptive_threshs.append(
0) # 0 means no adjustment of boundary
for tmp_thresh in self.boundary_adaptive_threshs:
if -1 * tmp_thresh not in self.boundary_adaptive_threshs:
self.boundary_adaptive_threshs.append(-1 * tmp_thresh)
# 4. Parameters settings for specific optimization algorithm
self.opt_mtd = method
self.opt = None # type: Union[ParseNSGA2Config, None]
if self.opt_mtd == 'nsga2':
self.opt = ParseNSGA2Config(
cf, self.model.model_dir,
'SA_NSGA2_%s_%s' % (self.bmps_cfg_unit, self.bmps_cfg_method))
# Using the existed population derived from previous scenario optimization
self.initial_byinput = cf.getboolean(self.opt_mtd.upper(), 'inputpopulation') if \
cf.has_option(self.opt_mtd.upper(), 'inputpopulation') else False
self.input_pareto_file = None
self.input_pareto_gen = -1
if cf.has_option(self.opt_mtd.upper(), 'paretofrontsfile'):
self.input_pareto_file = cf.get(self.opt_mtd.upper(),
'paretofrontsfile')
if cf.has_option(self.opt_mtd.upper(), 'generationselected'):
self.input_pareto_gen = cf.getint(self.opt_mtd.upper(),
'generationselected')
self.scenario_dir = self.opt.out_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.scenario_dir)
# 5. (Optional) Plot settings for matplotlib
self.plot_cfg = PlotConfig(cf)
def read_param_ranges(self):
"""Read param_rng.def file
name,lower_bound,upper_bound,group,dist
(group and dist are optional)
e.g.,
Param1,0,1[,Group1][,dist1]
Param2,0,1[,Group2][,dist2]
Param3,0,1[,Group3][,dist3]
Returns:
a dictionary containing:
- names - the names of the parameters
- bounds - a list of lists of lower and upper bounds
- num_vars - a scalar indicating the number of variables
(the length of names)
- groups - a list of group names (strings) for each variable
- dists - a list of distributions for the problem,
None if not specified or all uniform
"""
# read param_defs.json if already existed
if not self.param_defs:
if FileClass.is_file_exists(self.cfg.outfiles.param_defs_json):
with open(self.cfg.outfiles.param_defs_json, 'r') as f:
self.param_defs = UtilClass.decode_strs_in_dict(json.load(f))
return
# read param_range_def file and output to json file
client = ConnectMongoDB(self.model.host, self.model.port)
conn = client.get_conn()
db = conn[self.model.db_name]
collection = db['PARAMETERS']
names = list()
bounds = list()
groups = list()
dists = list()
num_vars = 0
items = read_data_items_from_txt(self.cfg.param_range_def)
for item in items:
if len(item) < 3:
continue
# find parameter name, print warning message if not existed
cursor = collection.find({'NAME': item[0]}, no_cursor_timeout=True)
if not cursor.count():
print('WARNING: parameter %s is not existed!' % item[0])
continue
num_vars += 1
names.append(item[0])
bounds.append([float(item[1]), float(item[2])])
# If the fourth column does not contain a group name, use
# the parameter name
if len(item) >= 4:
groups.append(item[3])
else:
groups.append(item[0])
if len(item) >= 5:
dists.append(item[4])
else:
dists.append('unif')
if groups == names:
groups = None
elif len(set(groups)) == 1:
raise ValueError('Only one group defined, results will not bemeaningful')
# setting dists to none if all are uniform
# because non-uniform scaling is not needed
if all([d == 'unif' for d in dists]):
dists = None
self.param_defs = {'names': names, 'bounds': bounds,
'num_vars': num_vars, 'groups': groups, 'dists': dists}
# Save as json, which can be loaded by json.load()
json_data = json.dumps(self.param_defs, indent=4, cls=SpecialJsonEncoder)
with open(self.cfg.outfiles.param_defs_json, 'w') as f:
f.write(json_data)
def __init__(self, cf):
"""Initialization."""
SAConfig.__init__(self, cf) # initialize base class first
# Handling self.bmps_info for specific application
# 1. Check the required key and values
requiredkeys = ['COLLECTION', 'DISTRIBUTION', 'SUBSCENARIO', 'UPDOWNJSON',
'ENVEVAL', 'BASE_ENV']
for k in requiredkeys:
if k not in self.bmps_info:
raise ValueError('[%s]: MUST be provided!' % k)
# 2. Slope position units information
updownf = self.bmps_info.get('UPDOWNJSON')
FileClass.check_file_exists(updownf)
with open(updownf, 'r') as updownfo:
self.units_infos = json.load(updownfo)
self.units_infos = UtilClass.decode_strs_in_dict(self.units_infos)
# 3. Get slope position sequence
sptags = cf.get('BMPs', 'slppos_tag_name')
self.slppos_tags = json.loads(sptags)
self.slppos_tags = UtilClass.decode_strs_in_dict(self.slppos_tags)
self.slppos_tagnames = sorted(list(self.slppos_tags.items()), key=operator.itemgetter(0))
self.slppos_unit_num = self.units_infos['overview']['all_units']
self.slppos_to_gene = OrderedDict()
self.gene_to_slppos = dict()
# method 1: (deprecated)
# gene index: 0, 1, 2, ..., n
# slppos unit: rdg1, rdg2,..., bks1, bks2,..., vly1, vly2...
# idx = 0
# for tag, sp in self.slppos_tagnames:
# for uid in self.units_infos[sp]:
# self.gene_to_slppos[idx] = uid
# self.slppos_to_gene[uid] = idx
# idx += 1
# method 2:
# gene index: 0, 1, 2, ..., n
# slppos unit: rdg1, bks2, vly1,..., rdgn, bksn, vlyn
idx = 0
spname = self.slppos_tagnames[0][1]
for uid, udict in self.units_infos[spname].items():
spidx = 0
self.gene_to_slppos[idx] = uid
self.slppos_to_gene[uid] = idx
idx += 1
next_uid = udict['downslope']
while next_uid > 0:
self.gene_to_slppos[idx] = next_uid
self.slppos_to_gene[next_uid] = idx
idx += 1
spidx += 1
spname = self.slppos_tagnames[spidx][1]
next_uid = self.units_infos[spname][next_uid]['downslope']
assert (idx == self.slppos_unit_num)
# 4. SubScenario IDs and parameters read from MongoDB
self.bmps_subids = self.bmps_info.get('SUBSCENARIO')
self.bmps_coll = self.bmps_info.get('COLLECTION')
self.bmps_params = dict()
self.slppos_suit_bmps = dict()
self.read_bmp_parameters()
self.get_suitable_bmps_for_slppos()
def __init__(self, cf, method='nsga2'):
# type: (ConfigParser, str) -> None
"""Initialization."""
# 1. SEIMS model related
self.model = ParseSEIMSConfig(cf)
# 2. Common settings of BMPs scenario
self.eval_stime = None
self.eval_etime = None
self.worst_econ = 0.
self.worst_env = 0.
self.runtime_years = 0.
self.export_sce_txt = False
self.export_sce_tif = False
if 'Scenario_Common' not in cf.sections():
raise ValueError(
'[Scenario_Common] section MUST be existed in *.ini file.')
self.eval_stime = parse_datetime_from_ini(cf, 'Scenario_Common',
'eval_time_start')
self.eval_etime = parse_datetime_from_ini(cf, 'Scenario_Common',
'eval_time_end')
self.worst_econ = cf.getfloat('Scenario_Common', 'worst_economy')
self.worst_env = cf.getfloat('Scenario_Common', 'worst_environment')
self.runtime_years = cf.getfloat('Scenario_Common', 'runtime_years')
if cf.has_option('Scenario_Common', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('Scenario_Common',
'export_scenario_txt')
if cf.has_option('Scenario_Common', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('Scenario_Common',
'export_scenario_tif')
# 3. Application specific setting section [BMPs]
self.bmps_info = dict() # BMP to be evaluated, JSON format
self.bmps_retain = dict() # BMPs to be constant, JSON format
self.bmps_cfg_unit = 'SLPPOS'
self.bmps_cfg_method = 'RDM'
if 'BMPs' not in cf.sections():
raise ValueError(
'[BMPs] section MUST be existed for specific scenario analysis.'
)
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_info = UtilClass.decode_strs_in_dict(json.loads(bmpsinfostr))
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(self.bmps_retain)
bmpscfgunitstr = cf.get('BMPs', 'bmps_cfg_units')
bmpscfgunitdict = UtilClass.decode_strs_in_dict(
json.loads(bmpscfgunitstr))
for unitname, unitcfg in viewitems(bmpscfgunitdict):
self.bmps_cfg_unit = unitname
if self.bmps_cfg_unit not in BMPS_CFG_UNITS:
raise ValueError('BMPs configuration unit MUST be '
'one of %s' % BMPS_CFG_UNITS.__str__())
if not isinstance(unitcfg, dict):
raise ValueError(
'The value of BMPs configuration unit MUST be dict value!')
for cfgname, cfgvalue in viewitems(unitcfg):
self.bmps_info[cfgname] = cfgvalue
break
if cf.has_option('BMPs', 'bmps_cfg_method'):
self.bmps_cfg_method = cf.get('BMPs', 'bmps_cfg_method')
if self.bmps_cfg_method not in BMPS_CFG_METHODS:
print('BMPs configuration method MUST be one of %s' %
BMPS_CFG_METHODS.__str__())
self.bmps_cfg_method = 'RDM'
# Check the validation of configuration unit and method
if self.bmps_cfg_method not in BMPS_CFG_PAIR.get(self.bmps_cfg_unit):
raise ValueError('BMPs configuration method %s '
'is not supported on unit %s' %
(self.bmps_cfg_method, self.bmps_cfg_unit))
# 4. Parameters settings for specific optimization algorithm
self.opt_mtd = method
self.opt = None # type: Union[ParseNSGA2Config]
if self.opt_mtd == 'nsga2':
self.opt = ParseNSGA2Config(
cf, self.model.model_dir,
'SA_NSGA2_%s_%s' % (self.bmps_cfg_unit, self.bmps_cfg_method))
self.scenario_dir = self.opt.out_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.scenario_dir)
def __init__(self, cf):
"""Initialization."""
# 1. NSGA-II related parameters
self.nsga2_ngens = 1
self.nsga2_npop = 4
self.nsga2_rcross = 0.75
self.nsga2_pmut = 0.05
self.nsga2_rmut = 0.1
self.nsga2_rsel = 0.8
if 'NSGA2' in cf.sections():
self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum')
self.nsga2_npop = cf.getint('NSGA2', 'populationsize')
self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate')
self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc')
self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate')
self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate')
else:
raise ValueError('[NSGA2] section MUST be existed in *.ini file.')
if self.nsga2_npop % 4 != 0:
raise ValueError('PopulationSize must be a multiple of 4.')
# 2. MongoDB
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.spatial_db = ''
self.bmp_scenario_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
else:
raise ValueError(
'[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. SEIMS_Model
self.model_dir = ''
self.seims_bin = ''
self.seims_nthread = 1
self.seims_lyrmethod = 0
if 'SEIMS_Model' in cf.sections():
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
self.seims_bin = cf.get('SEIMS_Model', 'bin_dir')
self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum')
self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod')
else:
raise ValueError(
"[SEIMS_Model] section MUST be existed in *.ini file.")
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError('Please Check Directories defined in [PATH]. '
'BIN_DIR and MODEL_DIR are required!')
# 4. Application specific setting section [BMPs]
self.bmps_info = dict()
self.bmps_rule = False
self.rule_method = 1
self.bmps_retain = dict()
self.export_sce_txt = False
self.export_sce_tif = False
if 'BMPs' in cf.sections():
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule')
if cf.has_option('BMPs', 'rule_method'):
self.rule_method = cf.getint('BMPs', 'rule_method')
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(
self.bmps_retain)
if cf.has_option('BMPs', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('BMPs',
'export_scenario_txt')
if cf.has_option('BMPs', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('BMPs',
'export_scenario_tif')
else:
raise ValueError("[BMPs] section MUST be existed for specific SA.")
self.bmps_info = json.loads(bmpsinfostr)
self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info)
# 5. Application specific setting section [Effectiveness]
self.worst_econ = 0
self.worst_env = 0
self.runtime_years = 0
if 'Effectiveness' in cf.sections():
self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy')
self.worst_env = cf.getfloat('Effectiveness', 'worst_environment')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
# 6. define gene_values
fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop)
fn += '_rule' if self.bmps_rule else '_random'
self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn
self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.nsga2_dir)
UtilClass.rmmkdir(self.scenario_dir)
self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt'
self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt'
self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log'
self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'
def __init__(self, cf):
"""Initialization."""
# 1. NSGA-II related parameters
self.nsga2_ngens = 1
self.nsga2_npop = 4
self.nsga2_rcross = 0.75
self.nsga2_pmut = 0.05
self.nsga2_rmut = 0.1
self.nsga2_rsel = 0.8
if 'NSGA2' in cf.sections():
self.nsga2_ngens = cf.getint('NSGA2', 'generationsnum')
self.nsga2_npop = cf.getint('NSGA2', 'populationsize')
self.nsga2_rcross = cf.getfloat('NSGA2', 'crossoverrate')
self.nsga2_pmut = cf.getfloat('NSGA2', 'maxmutateperc')
self.nsga2_rmut = cf.getfloat('NSGA2', 'mutaterate')
self.nsga2_rsel = cf.getfloat('NSGA2', 'selectrate')
else:
raise ValueError('[NSGA2] section MUST be existed in *.ini file.')
if self.nsga2_npop % 4 != 0:
raise ValueError('PopulationSize must be a multiple of 4.')
# 2. MongoDB
self.hostname = '127.0.0.1' # localhost by default
self.port = 27017
self.spatial_db = ''
self.bmp_scenario_db = ''
if 'MONGODB' in cf.sections():
self.hostname = cf.get('MONGODB', 'hostname')
self.port = cf.getint('MONGODB', 'port')
self.spatial_db = cf.get('MONGODB', 'spatialdbname')
self.bmp_scenario_db = cf.get('MONGODB', 'bmpscenariodbname')
else:
raise ValueError('[MONGODB] section MUST be existed in *.ini file.')
if not StringClass.is_valid_ip_addr(self.hostname):
raise ValueError('HOSTNAME illegal defined in [MONGODB]!')
# 3. SEIMS_Model
self.model_dir = ''
self.seims_bin = ''
self.seims_nthread = 1
self.seims_lyrmethod = 0
if 'SEIMS_Model' in cf.sections():
self.model_dir = cf.get('SEIMS_Model', 'model_dir')
self.seims_bin = cf.get('SEIMS_Model', 'bin_dir')
self.seims_nthread = cf.getint('SEIMS_Model', 'threadsnum')
self.seims_lyrmethod = cf.getint('SEIMS_Model', 'layeringmethod')
else:
raise ValueError("[SEIMS_Model] section MUST be existed in *.ini file.")
if not (FileClass.is_dir_exists(self.model_dir)
and FileClass.is_dir_exists(self.seims_bin)):
raise IOError('Please Check Directories defined in [PATH]. '
'BIN_DIR and MODEL_DIR are required!')
# 4. Application specific setting section [BMPs]
self.bmps_info = dict()
self.bmps_rule = False
self.rule_method = 1
self.bmps_retain = dict()
self.export_sce_txt = False
self.export_sce_tif = False
if 'BMPs' in cf.sections():
bmpsinfostr = cf.get('BMPs', 'bmps_info')
self.bmps_rule = cf.getboolean('BMPs', 'bmps_rule')
if cf.has_option('BMPs', 'rule_method'):
self.rule_method = cf.getint('BMPs', 'rule_method')
if cf.has_option('BMPs', 'bmps_retain'):
bmpsretainstr = cf.get('BMPs', 'bmps_retain')
self.bmps_retain = json.loads(bmpsretainstr)
self.bmps_retain = UtilClass.decode_strs_in_dict(self.bmps_retain)
if cf.has_option('BMPs', 'export_scenario_txt'):
self.export_sce_txt = cf.getboolean('BMPs', 'export_scenario_txt')
if cf.has_option('BMPs', 'export_scenario_tif'):
self.export_sce_tif = cf.getboolean('BMPs', 'export_scenario_tif')
else:
raise ValueError("[BMPs] section MUST be existed for specific SA.")
self.bmps_info = json.loads(bmpsinfostr)
self.bmps_info = UtilClass.decode_strs_in_dict(self.bmps_info)
# 5. Application specific setting section [Effectiveness]
self.worst_econ = 0
self.worst_env = 0
self.runtime_years = 0
if 'Effectiveness' in cf.sections():
self.worst_econ = cf.getfloat('Effectiveness', 'worst_economy')
self.worst_env = cf.getfloat('Effectiveness', 'worst_environment')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
self.runtime_years = cf.getfloat('Effectiveness', 'runtime_years')
# 6. define gene_values
fn = 'Gen_%d_Pop_%d' % (self.nsga2_ngens, self.nsga2_npop)
fn += '_rule' if self.bmps_rule else '_random'
self.nsga2_dir = self.model_dir + os.path.sep + 'NSGA2_OUTPUT' + os.path.sep + fn
self.scenario_dir = self.nsga2_dir + os.path.sep + 'Scenarios'
UtilClass.rmmkdir(self.nsga2_dir)
UtilClass.rmmkdir(self.scenario_dir)
self.hypervlog = self.nsga2_dir + os.path.sep + 'hypervolume.txt'
self.scenariolog = self.nsga2_dir + os.path.sep + 'scenarios_info.txt'
self.logfile = self.nsga2_dir + os.path.sep + 'runtime.log'
self.logbookfile = self.nsga2_dir + os.path.sep + 'logbook.txt'