class Estimator(ModelExplorer):
def __init__(self, config=None, save_estimation_results=False):
if 'cache_directory' not in config or config['cache_directory'] is None:
raise KeyError("The cache directory must be specified in the "
"given configuration, giving the filesystem path to the cache "
"directory containing the data with which to estimate. Please "
"check that your configuration contains the 'cache_directory' "
"entry and that it is not None.")
self.simulation_state = SimulationState(new_instance=True, start_time=config.get('base_year', 0))
self.simulation_state.set_cache_directory(config['cache_directory'])
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
self.config = Resources(config)
self.save_estimation_results = save_estimation_results
self.debuglevel = self.config.get("debuglevel", 4)
self.model_system = ModelSystem()
self.agents_index_for_prediction = None
models = self.config.get('models',[])
self.model_name = None
if "model_name" in config.keys():
self.model_name = config["model_name"]
else:
for model in models:
if isinstance(model, dict):
model_name = model.keys()[0]
if (model[model_name] == "estimate") or (isinstance(model[model_name], list)
and ("estimate" in model[model_name])):
self.model_name = model_name
break
estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {})
if len(estimate_config_changes) > 0:
change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}})
estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}')
estimate_config = Resources({})
try:
estimate_config = eval(estimate_config_str)
except:
pass
estimate_config.merge(estimate_config_changes)
self.config.merge(change)
self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config
def estimate(self, out_storage=None):
self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False)
self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
self.log_results()
def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None):
"""specification_module_name is name of a module that contains a dictionary called
'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
is expected to have an entry of this name. If 'submodels' is given (list or a number),
the restimation is done only for those submodels.
"""
if specification_module_name is not None:
exec("import " + specification_module_name)
eval("reload (" + specification_module_name + ")")
exec("specification_dict =" + specification_module_name + ".specification")
if type is not None:
specification_dict = specification_dict[type]
if submodels is not None: #remove all submodels but the given ones from specification
submodels_to_be_deleted = specification_dict.keys()
if not isinstance(submodels, list):
submodels = [submodels]
for sm in submodels:
if sm not in submodels_to_be_deleted:
raise ValueError, "Submodel %s not in the specification." % sm
submodels_to_be_deleted.remove(sm)
if "_definition_" in submodels_to_be_deleted:
submodels_to_be_deleted.remove("_definition_")
for sm in submodels_to_be_deleted:
del specification_dict[sm]
self.specification = EquationSpecification(specification_dict=specification_dict)
new_namespace = self.model_system.run_year_namespace
keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
new_namespace[keys_coeff_spec["specification"]] = self.specification
self.coefficients, coeff_dict_dummy = self.model_system.do_process(new_namespace)
## update run_year_namespce since it's not been updated by do_process
self.model_system.run_year_namespace = new_namespace
self.model_system.run_year_namespace[keys_coeff_spec["coefficients"]] = self.coefficients
## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
#self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def predict(self, predicted_choice_id_name, agents_index=None):
""" Run prediction. Currently makes sense only for choice models."""
# Create temporary configuration where all words 'estimate' are replaced by 'run'
tmp_config = Resources(self.config)
if self.agents_index_for_prediction is None:
self.agents_index_for_prediction = self.get_agent_set_index().copy()
if agents_index is None:
agents_index = self.agents_index_for_prediction
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}"
### save specification and coefficients to cache (no matter the save_estimation_results flag)
### so that the prepare_for_run method could load specification and coefficients from there
#output_configuration = self.config['output_configuration']
#del self.config['output_configuration']
#self.save_results()
#self.config['output_configuration'] = output_configuration
#self.model_system.run_year_namespace["coefficients"] = self.coefficients
#del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
try:
run_year_namespace = copy.copy(self.model_system.run_year_namespace)
except:
logger.log_error("The estimate() method must be run first")
return False
try:
agents = self.get_agent_set()
choice_id_name = self.get_choice_set().get_id_name()[0]
# save current locations of agents
current_choices = agents.get_attribute(choice_id_name).copy()
dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1
#agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset choices for all agents
agents.modify_attribute(name=choice_id_name, data=dummy_data, index=agents_index) #reset choices for agents in agents_index
run_year_namespace["process"] = "run"
run_year_namespace["coeff_est"] = self.coefficients
run_year_namespace["agents_index"] = agents_index
run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run']
new_choices = self.model_system.do_process(run_year_namespace)
#self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
#new_choices = agents.get_attribute(choice_id_name).copy()
agents.modify_attribute(name=choice_id_name, data=current_choices)
dummy_data[agents_index] = new_choices
if predicted_choice_id_name not in agents.get_known_attribute_names():
agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data)
else:
agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data)
logger.log_status("Predictions saved into attribute " + predicted_choice_id_name)
return True
except Exception, e:
logger.log_error("Error encountered in prediction: %s" % e)
logger.log_stack_trace()
return False
class Estimator(ModelExplorer):
def __init__(self, config=None, save_estimation_results=False):
if 'cache_directory' not in config or config['cache_directory'] is None:
raise KeyError("The cache directory must be specified in the "
"given configuration, giving the filesystem path to the cache "
"directory containing the data with which to estimate. Please "
"check that your configuration contains the 'cache_directory' "
"entry and that it is not None.")
self.simulation_state = SimulationState(new_instance=True, start_time=config.get('base_year', 0))
self.simulation_state.set_cache_directory(config['cache_directory'])
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
self.config = Resources(config)
self.save_estimation_results = save_estimation_results
self.debuglevel = self.config.get("debuglevel", 4)
self.model_system = ModelSystem()
self.agents_index_for_prediction = None
models = self.config.get('models',[])
self.model_name = None
if "model_name" in config.keys():
self.model_name = config["model_name"]
else:
for model in models:
if isinstance(model, dict):
model_name = model.keys()[0]
if (model[model_name] == "estimate") or (isinstance(model[model_name], list)
and ("estimate" in model[model_name])):
self.model_name = model_name
break
estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {})
if len(estimate_config_changes) > 0:
change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}})
estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}')
estimate_config = Resources({})
try:
estimate_config = eval(estimate_config_str)
except:
pass
estimate_config.merge(estimate_config_changes)
self.config.merge(change)
self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config
def estimate(self, out_storage=None):
self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False)
self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
self.log_results()
def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None):
"""specification_module_name is name of a module that contains a dictionary called
'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
is expected to have an entry of this name. If 'submodels' is given (list or a number),
the restimation is done only for those submodels.
"""
if specification_module_name is not None:
exec("import " + specification_module_name)
eval("reload (" + specification_module_name + ")")
exec("specification_dict =" + specification_module_name + ".specification")
if type is not None:
specification_dict = specification_dict[type]
if submodels is not None: #remove all submodels but the given ones from specification
submodels_to_be_deleted = specification_dict.keys()
if not isinstance(submodels, list):
submodels = [submodels]
for sm in submodels:
if sm not in submodels_to_be_deleted:
raise ValueError, "Submodel %s not in the specification." % sm
submodels_to_be_deleted.remove(sm)
if "_definition_" in submodels_to_be_deleted:
submodels_to_be_deleted.remove("_definition_")
for sm in submodels_to_be_deleted:
del specification_dict[sm]
self.specification = EquationSpecification(specification_dict=specification_dict)
new_namespace = self.model_system.run_year_namespace
keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
new_namespace[keys_coeff_spec["specification"]] = self.specification
self.coefficients, coeff_dict_dummy = self.model_system.do_process(new_namespace)
## update run_year_namespce since it's not been updated by do_process
self.model_system.run_year_namespace = new_namespace
self.model_system.run_year_namespace[keys_coeff_spec["coefficients"]] = self.coefficients
## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
#self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def predict(self, predicted_choice_id_name, agents_index=None):
""" Run prediction. Currently makes sense only for choice models."""
# Create temporary configuration where all words 'estimate' are replaced by 'run'
tmp_config = Resources(self.config)
if self.agents_index_for_prediction is None:
self.agents_index_for_prediction = self.get_agent_set_index().copy()
if agents_index is None:
agents_index = self.agents_index_for_prediction
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}"
### save specification and coefficients to cache (no matter the save_estimation_results flag)
### so that the prepare_for_run method could load specification and coefficients from there
#output_configuration = self.config['output_configuration']
#del self.config['output_configuration']
#self.save_results()
#self.config['output_configuration'] = output_configuration
#self.model_system.run_year_namespace["coefficients"] = self.coefficients
#del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
try:
run_year_namespace = copy.copy(self.model_system.run_year_namespace)
except:
logger.log_error("The estimate() method must be run first")
return False
try:
agents = self.get_agent_set()
choice_id_name = self.get_choice_set().get_id_name()[0]
# save current locations of agents
current_choices = agents.get_attribute(choice_id_name).copy()
dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1
#agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset choices for all agents
agents.modify_attribute(name=choice_id_name, data=dummy_data, index=agents_index) #reset choices for agents in agents_index
run_year_namespace["process"] = "run"
run_year_namespace["coeff_est"] = self.coefficients
run_year_namespace["agents_index"] = agents_index
run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run']
new_choices = self.model_system.do_process(run_year_namespace)
#self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
#new_choices = agents.get_attribute(choice_id_name).copy()
agents.modify_attribute(name=choice_id_name, data=current_choices)
dummy_data[agents_index] = new_choices
if predicted_choice_id_name not in agents.get_known_attribute_names():
agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data)
else:
agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data)
logger.log_status("Predictions saved into attribute " + predicted_choice_id_name)
return True
except Exception, e:
logger.log_error("Error encountered in prediction: %s" % e)
logger.log_stack_trace()
return False
