"waterdemand.consumption_re.something_like_sum_demand",
)
)
print "Create a model object"
years = range(2001, 2003)
# single
model = RegressionModel()
print "Estimate coefficients - single"
coefficients, other_est_results = model.estimate(specification, consumption,
outcome_attribute="waterdemand.%s.sum_demand" % consumption_type, # if outcome_attribute is opus_core.func.ln(), the simulation results need to take exp()
index=index_est,
procedure="opus_core.estimate_linear_regression",
data_objects=dataset_pool.datasets_in_pool())
"""Simulate over the set of years."""
for year in years:
print "\nSimulate water demand %s" % year
SimulationState().set_current_time(year)
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset_pool.remove_all_datasets()
gridcells = dataset_pool.get_dataset("gridcell")
#create a ConsumptionDataset instance out of gridcells - simulate water demand for every gridcell
resources = Resources({'data':{
"grid_id":gridcells.get_id_attribute(),
"year":year * ones(gridcells.size()),
"month":months[0] * ones(gridcells.size()),
def _run_year(self,
year,
models,
simulation_state,
debuglevel,
resources,
write_datasets_to_cache_at_end_of_year,
cleanup_datasets=True):
"""
Assumes that all datasets resides in the cache directory in binary format.
"""
try:
import wingdbstub
except:
pass
self.vardict = {}
log_file_name = os.path.join(simulation_state.get_cache_directory(),
"year_%s_log.txt" % year)
logger.enable_file_logging(log_file_name, 'w')
try:
logger.start_block('Simulate year %s' % year)
try:
base_year = resources['base_year']
if year == base_year:
year_for_base_year_cache = year # case of estimation
else:
year_for_base_year_cache = year - 1
cache_storage = AttributeCache().get_flt_storage_for_year(
year_for_base_year_cache)
self.vardict['cache_storage'] = cache_storage
base_cache_storage = AttributeCache().get_flt_storage_for_year(
base_year)
self.vardict['base_cache_storage'] = base_cache_storage
simulation_state.set_flush_datasets(
resources.get("flush_variables", False))
SessionConfiguration()["simulation_year"] = year
SessionConfiguration()["debuglevel"] = debuglevel
datasets_to_preload_in_year = resources.get(
'datasets_to_preload_in_year', {})
if datasets_to_preload_in_year.get(year, None) is not None:
datasets_to_preload = datasets_to_preload_in_year[year]
else:
datasets_to_preload = resources.get(
'datasets_to_preload', {})
for dataset_name in datasets_to_preload:
SessionConfiguration().get_dataset_from_pool(dataset_name)
models_configuration = resources.get('models_configuration',
{})
dataset_pool = SessionConfiguration().get_dataset_pool()
datasets = {}
for dataset_name, its_dataset in dataset_pool.datasets_in_pool(
).iteritems():
self.vardict[dataset_name] = its_dataset
datasets[dataset_name] = its_dataset
exec '%s=its_dataset' % dataset_name
# This is needed. It resides in locals()
# and is passed on to models as they run.
### TODO: There has got to be a better way!
model_resources = Resources(datasets)
n_models, model_group_members_to_run = self.get_number_of_models_and_model_group_members_to_run(
models, models_configuration)
self.run_year_namespace = locals()
#==========
# Run the models.
#==========
model_number = -1
for model_entry in models:
# list 'models' can be in the form:
# [{'model_name_1': {'group_members': ['residential', 'commercial']}},
# {'model_name_2': {'group_members': [{'residential': ['estimate','run']},
# 'commercial']}},
# {'model_name_3': ['estimate', 'run']},
# 'model_name_4',
# {'model_name_5': {'group_members': 'all'}}
# ]
# get list of methods to be processed evtl. for each group member
if isinstance(model_entry, dict):
model_name, value = model_entry.items()[0]
if not isinstance(value, dict): # is a model group
processes = value
if not isinstance(processes, list):
processes = [processes]
else: # in the form 'model_name_4' in the comment above
model_name = model_entry
processes = ["run"]
group_member = None
model_group = model_group_members_to_run[model_name][1]
last_member = max(
1,
len(model_group_members_to_run[model_name][0].keys()))
for imember in range(last_member):
controller_config = models_configuration[model_name][
"controller"]
model_configuration = models_configuration[model_name]
if model_group_members_to_run[model_name][0].keys():
group_member_name = model_group_members_to_run[
model_name][0].keys()[imember]
group_member = ModelGroupMember(
model_group, group_member_name)
processes = model_group_members_to_run[model_name][
0][group_member_name]
member_model_name = "%s_%s" % (group_member_name,
model_name)
if member_model_name in models_configuration.keys(
):
model_configuration = models_configuration[
member_model_name]
if "controller" in model_configuration.keys():
controller_config = model_configuration[
"controller"]
datasets_to_preload_for_this_model = controller_config.get(
'_model_structure_dependencies_',
{}).get('dataset', [])
for dataset_name in datasets_to_preload_for_this_model:
try:
if not dataset_pool.has_dataset(
dataset_name) or (
dataset_name
not in datasets.keys()):
ds = dataset_pool.get_dataset(dataset_name)
self.vardict[dataset_name] = ds
datasets[dataset_name] = ds
exec '%s=ds' % dataset_name
except:
logger.log_warning(
'Failed to load dataset %s.' %
dataset_name)
# import part
if "import" in controller_config.keys():
import_config = controller_config["import"]
for import_module in import_config.keys():
exec("from %s import %s" %
(import_module,
import_config[import_module]))
# gui_import_replacements part
# This is a temporary hack -- replicates the functionality of the "import" section
# for use with the GUI. The contents of this part of the config is a dictionary.
# Keys are names of models (not used here). Values are 2 element pairs.
# The first element is a name and the second is a value. Bind the name to the value.
if "gui_import_replacements" in controller_config.keys(
):
import_replacement_config = controller_config[
"gui_import_replacements"]
for model_name in import_replacement_config.keys():
pair = import_replacement_config[model_name]
temp = pair[1]
exec("%s = temp") % pair[0]
# init part
model = self.do_init(locals())
# estimate and/or run part
for process in processes:
model_number = model_number + 1
# write status file
model.set_model_system_status_parameters(
year, n_models, model_number,
resources.get('status_file_for_gui', None))
model.write_status_for_gui()
# prepare part
exec(self.do_prepare(locals()))
processmodel_config = controller_config[process]
if "output" in processmodel_config.keys():
outputvar = processmodel_config["output"]
else:
outputvar = "process_output"
self.vardict[outputvar] = self.do_process(locals())
exec outputvar + '=self.vardict[outputvar]'
# check command file from gui, if the simulation should be stopped or paused
self.do_commands_from_gui(
resources.get('command_file_for_gui', None))
# capture namespace for interactive estimation
self.run_year_namespace = locals()
self.flush_datasets_after_model(resources)
del model
collect()
# Write all datasets to cache.
if write_datasets_to_cache_at_end_of_year:
logger.start_block(
'Writing datasets to cache for year %s' % year)
try:
for dataset_name, its_dataset in SessionConfiguration(
).get_dataset_pool().datasets_in_pool().iteritems():
self.flush_dataset(its_dataset)
finally:
logger.end_block()
finally:
logger.end_block()
finally:
logger.disable_file_logging(log_file_name)
if cleanup_datasets:
SessionConfiguration().delete_datasets()
def _run_year(
self,
year,
models,
simulation_state,
debuglevel,
resources,
write_datasets_to_cache_at_end_of_year,
cleanup_datasets=True,
):
"""
Assumes that all datasets resides in the cache directory in binary format.
"""
try:
import wingdbstub
except:
pass
self.vardict = {}
log_file_name = os.path.join(simulation_state.get_cache_directory(), "year_%s_log.txt" % year)
logger.enable_file_logging(log_file_name, "w")
try:
logger.start_block("Simulate year %s" % year)
try:
base_year = resources["base_year"]
if year == base_year:
year_for_base_year_cache = year # case of estimation
else:
year_for_base_year_cache = year - 1
cache_storage = AttributeCache().get_flt_storage_for_year(year_for_base_year_cache)
self.vardict["cache_storage"] = cache_storage
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
self.vardict["base_cache_storage"] = base_cache_storage
simulation_state.set_flush_datasets(resources.get("flush_variables", False))
SessionConfiguration()["simulation_year"] = year
SessionConfiguration()["debuglevel"] = debuglevel
datasets_to_preload_in_year = resources.get("datasets_to_preload_in_year", {})
if datasets_to_preload_in_year.get(year, None) is not None:
datasets_to_preload = datasets_to_preload_in_year[year]
else:
datasets_to_preload = resources.get("datasets_to_preload", {})
for dataset_name in datasets_to_preload:
SessionConfiguration().get_dataset_from_pool(dataset_name)
models_configuration = resources.get("models_configuration", {})
dataset_pool = SessionConfiguration().get_dataset_pool()
datasets = {}
for dataset_name, its_dataset in dataset_pool.datasets_in_pool().iteritems():
self.vardict[dataset_name] = its_dataset
datasets[dataset_name] = its_dataset
exec "%s=its_dataset" % dataset_name
# This is needed. It resides in locals()
# and is passed on to models as they run.
### TODO: There has got to be a better way!
model_resources = Resources(datasets)
n_models, model_group_members_to_run = self.get_number_of_models_and_model_group_members_to_run(
models, models_configuration
)
self.run_year_namespace = locals()
# ==========
# Run the models.
# ==========
model_number = -1
for model_entry in models:
# list 'models' can be in the form:
# [{'model_name_1': {'group_members': ['residential', 'commercial']}},
# {'model_name_2': {'group_members': [{'residential': ['estimate','run']},
# 'commercial']}},
# {'model_name_3': ['estimate', 'run']},
# 'model_name_4',
# {'model_name_5': {'group_members': 'all'}}
# ]
# get list of methods to be processed evtl. for each group member
if isinstance(model_entry, dict):
model_name, value = model_entry.items()[0]
if not isinstance(value, dict): # is a model group
processes = value
if not isinstance(processes, list):
processes = [processes]
else: # in the form 'model_name_4' in the comment above
model_name = model_entry
processes = ["run"]
group_member = None
model_group = model_group_members_to_run[model_name][1]
last_member = max(1, len(model_group_members_to_run[model_name][0].keys()))
for imember in range(last_member):
controller_config = models_configuration[model_name]["controller"]
model_configuration = models_configuration[model_name]
if model_group_members_to_run[model_name][0].keys():
group_member_name = model_group_members_to_run[model_name][0].keys()[imember]
group_member = ModelGroupMember(model_group, group_member_name)
processes = model_group_members_to_run[model_name][0][group_member_name]
member_model_name = "%s_%s" % (group_member_name, model_name)
if member_model_name in models_configuration.keys():
model_configuration = models_configuration[member_model_name]
if "controller" in model_configuration.keys():
controller_config = model_configuration["controller"]
datasets_to_preload_for_this_model = controller_config.get(
"_model_structure_dependencies_", {}
).get("dataset", [])
for dataset_name in datasets_to_preload_for_this_model:
try:
if not dataset_pool.has_dataset(dataset_name) or (dataset_name not in datasets.keys()):
ds = dataset_pool.get_dataset(dataset_name)
self.vardict[dataset_name] = ds
datasets[dataset_name] = ds
exec "%s=ds" % dataset_name
except:
logger.log_warning("Failed to load dataset %s." % dataset_name)
# import part
if "import" in controller_config.keys():
import_config = controller_config["import"]
for import_module in import_config.keys():
exec ("from %s import %s" % (import_module, import_config[import_module]))
# gui_import_replacements part
# This is a temporary hack -- replicates the functionality of the "import" section
# for use with the GUI. The contents of this part of the config is a dictionary.
# Keys are names of models (not used here). Values are 2 element pairs.
# The first element is a name and the second is a value. Bind the name to the value.
if "gui_import_replacements" in controller_config.keys():
import_replacement_config = controller_config["gui_import_replacements"]
for model_name in import_replacement_config.keys():
pair = import_replacement_config[model_name]
temp = pair[1]
exec ("%s = temp") % pair[0]
# init part
model = self.do_init(locals())
# estimate and/or run part
for process in processes:
model_number = model_number + 1
# write status file
model.set_model_system_status_parameters(
year, n_models, model_number, resources.get("status_file_for_gui", None)
)
model.write_status_for_gui()
# prepare part
exec (self.do_prepare(locals()))
processmodel_config = controller_config[process]
if "output" in processmodel_config.keys():
outputvar = processmodel_config["output"]
else:
outputvar = "process_output"
self.vardict[outputvar] = self.do_process(locals())
exec outputvar + "=self.vardict[outputvar]"
# check command file from gui, if the simulation should be stopped or paused
self.do_commands_from_gui(resources.get("command_file_for_gui", None))
# capture namespace for interactive estimation
self.run_year_namespace = locals()
self.flush_datasets_after_model(resources)
del model
collect()
# Write all datasets to cache.
if write_datasets_to_cache_at_end_of_year:
logger.start_block("Writing datasets to cache for year %s" % year)
try:
for dataset_name, its_dataset in (
SessionConfiguration().get_dataset_pool().datasets_in_pool().iteritems()
):
self.flush_dataset(its_dataset)
finally:
logger.end_block()
finally:
logger.end_block()
finally:
logger.disable_file_logging(log_file_name)
if cleanup_datasets:
SessionConfiguration().delete_datasets()
"waterdemand.consumption_re.something_like_sum_demand",
)
)
print "Create a model object"
years = range(2001, 2003)
# single
model = RegressionModel()
print "Estimate coefficients - single"
coefficients, other_est_results = model.estimate(specification, consumption,
outcome_attribute="waterdemand.%s.sum_demand" % consumption_type, # if outcome_attribute is opus_core.func.ln(), the simulation results need to take exp()
index=index_est,
procedure="opus_core.estimate_linear_regression",
data_objects=dataset_pool.datasets_in_pool())
"""Simulate over the set of years."""
for year in years:
print "\nSimulate water demand %s" % year
SimulationState().set_current_time(year)
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset_pool.remove_all_datasets()
gridcells = dataset_pool.get_dataset("gridcell")
#create a ConsumptionDataset instance out of gridcells - simulate water demand for every gridcell
resources = Resources({'data':{
"grid_id":gridcells.get_id_attribute(),
"year":year * ones(gridcells.size()),
"month":months[0] * ones(gridcells.size()),
