def run_simulation(self, simulation_instance=None):
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
logger.log_status("Data cache in %s" %
self.simulation_state.get_cache_directory())
def run_simulation(self, simulation_instance=None):
logger.start_block('Simulation on database %s'
% self.config['scenario_database_configuration'].database_name)
try:
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
finally:
logger.end_block()
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def run_simulation(self, simulation_instance=None):
logger.start_block(
'Simulation on database %s' %
self.config['scenario_database_configuration'].database_name)
try:
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
finally:
logger.end_block()
logger.log_status("Data cache in %s" %
self.simulation_state.get_cache_directory())
config['years'] = [options.year, options.year]
config['seed'] = options.seed
config["datasets_to_cache_after_each_model"]=[]
config['flush_variables'] = False
config = Resources(config)
cache_directory = config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy
simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory)
config['cache_directory'] = cache_directory
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
model_system = ModelSystem()
model_system.run(config, write_datasets_to_cache_at_end_of_year=False)
m = model_system.run_year_namespace["model"]
mi = m.model_interaction
submodels = m.model_interaction.get_submodels()
assert len(submodels) == 1
submodel = submodels[0]
data = m.get_all_data(submodel)
sampling_prob = mi.interaction_dataset['__sampling_probability'] \
if '__sampling_probability' in mi.interaction_dataset.get_known_attribute_names() \
else None
model_data.append({'data':data,
'model_name':model_name,
'sampling_probability': sampling_prob,
'choice_type':options.choice_type[h][i]
class Estimator(GenericModelExplorer):
def __init__(self, config=None, save_estimation_results=False):
if 'cache_directory' not in config or config['cache_directory'] is None:
raise KeyError(
"The cache directory must be specified in the "
"given configuration, giving the filesystem path to the cache "
"directory containing the data with which to estimate. Please "
"check that your configuration contains the 'cache_directory' "
"entry and that it is not None.")
self.simulation_state = SimulationState(new_instance=True)
self.simulation_state.set_cache_directory(config['cache_directory'])
SessionConfiguration(
new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
self.config = Resources(config)
self.save_estimation_results = save_estimation_results
self.debuglevel = self.config.get("debuglevel", 4)
self.model_system = ModelSystem()
self.agents_index_for_prediction = None
models = self.config.get('models', [])
self.model_name = None
if "model_name" in config.keys():
self.model_name = config["model_name"]
else:
for model in models:
if isinstance(model, dict):
model_name = model.keys()[0]
if (model[model_name] == "estimate") or (
isinstance(model[model_name], list) and
("estimate" in model[model_name])):
self.model_name = model_name
break
estimate_config_changes = self.config.get(
'config_changes_for_estimation', {}).get('estimate_config', {})
if len(estimate_config_changes) > 0:
change = Resources({
'models_configuration': {
self.model_name: {
'controller': {
'init': {
'arguments': {}
}
}
}
}
})
estimate_config_str = self.config['models_configuration'].get(
self.model_name,
{}).get('controller',
{}).get('init',
{}).get('arguments',
{}).get('estimate_config', '{}')
estimate_config = Resources({})
try:
estimate_config = eval(estimate_config_str)
except:
pass
estimate_config.merge(estimate_config_changes)
self.config.merge(change)
self.config['models_configuration'][
self.model_name]['controller']['init']['arguments'][
'estimate_config'] = 'Resources(%s)' % estimate_config
def estimate(self, out_storage=None):
self.model_system.run(self.config,
write_datasets_to_cache_at_end_of_year=False)
self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def reestimate(self,
specification_module_name=None,
specification_dict=None,
out_storage=None,
type=None,
submodels=None):
"""specification_module_name is name of a module that contains a dictionary called
'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
is expected to have an entry of this name. If 'submodels' is given (list or a number),
the restimation is done only for those submodels.
"""
if specification_module_name is not None:
exec("import " + specification_module_name)
eval("reload (" + specification_module_name + ")")
exec("specification_dict =" + specification_module_name +
".specification")
if type is not None:
specification_dict = specification_dict[type]
if submodels is not None: #remove all submodels but the given ones from specification
submodels_to_be_deleted = specification_dict.keys()
if not isinstance(submodels, list):
submodels = [submodels]
for sm in submodels:
if sm not in submodels_to_be_deleted:
raise ValueError, "Submodel %s not in the specification." % sm
submodels_to_be_deleted.remove(sm)
if "_definition_" in submodels_to_be_deleted:
submodels_to_be_deleted.remove("_definition_")
for sm in submodels_to_be_deleted:
del specification_dict[sm]
self.specification = EquationSpecification(
specification_dict=specification_dict)
new_namespace = self.model_system.run_year_namespace
keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
new_namespace[keys_coeff_spec["specification"]] = self.specification
self.coefficients, coeff_dict_dummy = self.model_system.do_process(
new_namespace)
## update run_year_namespce since it's not been updated by do_process
self.model_system.run_year_namespace = new_namespace
self.model_system.run_year_namespace[
keys_coeff_spec["coefficients"]] = self.coefficients
## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
#self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def predict(self, predicted_choice_id_name, agents_index=None):
""" Run prediction. Currently makes sense only for choice models."""
# Create temporary configuration where all words 'estimate' are replaced by 'run'
tmp_config = Resources(self.config)
if self.agents_index_for_prediction is None:
self.agents_index_for_prediction = self.get_agent_set_index().copy(
)
if agents_index is None:
agents_index = self.agents_index_for_prediction
tmp_config['models_configuration'][self.model_name]['controller'][
'run']['arguments']['coefficients'] = "coeff_est"
tmp_config['models_configuration'][self.model_name]['controller'][
'run']['arguments']['agents_index'] = "agents_index"
tmp_config['models_configuration'][self.model_name]['controller'][
'run']['arguments']['chunk_specification'] = "{'nchunks':1}"
### save specification and coefficients to cache (no matter the save_estimation_results flag)
### so that the prepare_for_run method could load specification and coefficients from there
#output_configuration = self.config['output_configuration']
#del self.config['output_configuration']
#self.save_results()
#self.config['output_configuration'] = output_configuration
#self.model_system.run_year_namespace["coefficients"] = self.coefficients
#del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
try:
run_year_namespace = copy.copy(
self.model_system.run_year_namespace)
except:
logger.log_error("The estimate() method must be run first")
return False
try:
agents = self.get_agent_set()
choice_id_name = self.get_choice_set().get_id_name()[0]
# save current locations of agents
current_choices = agents.get_attribute(choice_id_name).copy()
dummy_data = zeros(current_choices.size,
dtype=current_choices.dtype) - 1
agents.modify_attribute(name=choice_id_name,
data=dummy_data) #reset all choices
run_year_namespace["process"] = "run"
run_year_namespace["coeff_est"] = self.coefficients
run_year_namespace["agents_index"] = agents_index
run_year_namespace["processmodel_config"] = tmp_config[
'models_configuration'][self.model_name]['controller']['run']
new_choices = self.model_system.do_process(run_year_namespace)
#self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
#new_choices = agents.get_attribute(choice_id_name).copy()
agents.modify_attribute(name=choice_id_name, data=current_choices)
dummy_data[agents_index] = new_choices
if predicted_choice_id_name not in agents.get_known_attribute_names(
):
agents.add_primary_attribute(name=predicted_choice_id_name,
data=dummy_data)
else:
agents.modify_attribute(name=predicted_choice_id_name,
data=dummy_data)
logger.log_status("Predictions saved into attribute " +
predicted_choice_id_name)
return True
except Exception, e:
logger.log_error("Error encountered in prediction: %s" % e)
logger.log_stack_trace()
return False
class Estimator(GenericModelExplorer):
def __init__(self, config=None, save_estimation_results=False):
if 'cache_directory' not in config or config['cache_directory'] is None:
raise KeyError("The cache directory must be specified in the "
"given configuration, giving the filesystem path to the cache "
"directory containing the data with which to estimate. Please "
"check that your configuration contains the 'cache_directory' "
"entry and that it is not None.")
self.simulation_state = SimulationState(new_instance=True)
self.simulation_state.set_cache_directory(config['cache_directory'])
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
self.config = Resources(config)
self.save_estimation_results = save_estimation_results
self.debuglevel = self.config.get("debuglevel", 4)
self.model_system = ModelSystem()
self.agents_index_for_prediction = None
models = self.config.get('models',[])
self.model_name = None
if "model_name" in config.keys():
self.model_name = config["model_name"]
else:
for model in models:
if isinstance(model, dict):
model_name = model.keys()[0]
if (model[model_name] == "estimate") or (isinstance(model[model_name], list)
and ("estimate" in model[model_name])):
self.model_name = model_name
break
estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {})
if len(estimate_config_changes) > 0:
change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}})
estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}')
estimate_config = Resources({})
try:
estimate_config = eval(estimate_config_str)
except:
pass
estimate_config.merge(estimate_config_changes)
self.config.merge(change)
self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config
def estimate(self, out_storage=None):
self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False)
self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None):
"""specification_module_name is name of a module that contains a dictionary called
'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
is expected to have an entry of this name. If 'submodels' is given (list or a number),
the restimation is done only for those submodels.
"""
if specification_module_name is not None:
exec("import " + specification_module_name)
eval("reload (" + specification_module_name + ")")
exec("specification_dict =" + specification_module_name + ".specification")
if type is not None:
specification_dict = specification_dict[type]
if submodels is not None: #remove all submodels but the given ones from specification
submodels_to_be_deleted = specification_dict.keys()
if not isinstance(submodels, list):
submodels = [submodels]
for sm in submodels:
if sm not in submodels_to_be_deleted:
raise ValueError, "Submodel %s not in the specification." % sm
submodels_to_be_deleted.remove(sm)
if "_definition_" in submodels_to_be_deleted:
submodels_to_be_deleted.remove("_definition_")
for sm in submodels_to_be_deleted:
del specification_dict[sm]
self.specification = EquationSpecification(specification_dict=specification_dict)
new_namespace = self.model_system.run_year_namespace
keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
new_namespace[keys_coeff_spec["specification"]] = self.specification
self.coefficients, coeff_dict_dummy = self.model_system.do_process(new_namespace)
## update run_year_namespce since it's not been updated by do_process
self.model_system.run_year_namespace = new_namespace
self.model_system.run_year_namespace[keys_coeff_spec["coefficients"]] = self.coefficients
## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
#self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def predict(self, predicted_choice_id_name, agents_index=None):
""" Run prediction. Currently makes sense only for choice models."""
# Create temporary configuration where all words 'estimate' are replaced by 'run'
tmp_config = Resources(self.config)
if self.agents_index_for_prediction is None:
self.agents_index_for_prediction = self.get_agent_set_index().copy()
if agents_index is None:
agents_index = self.agents_index_for_prediction
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}"
### save specification and coefficients to cache (no matter the save_estimation_results flag)
### so that the prepare_for_run method could load specification and coefficients from there
#output_configuration = self.config['output_configuration']
#del self.config['output_configuration']
#self.save_results()
#self.config['output_configuration'] = output_configuration
#self.model_system.run_year_namespace["coefficients"] = self.coefficients
#del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
try:
run_year_namespace = copy.copy(self.model_system.run_year_namespace)
except:
logger.log_error("The estimate() method must be run first")
return False
try:
agents = self.get_agent_set()
choice_id_name = self.get_choice_set().get_id_name()[0]
# save current locations of agents
current_choices = agents.get_attribute(choice_id_name).copy()
dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1
agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset all choices
run_year_namespace["process"] = "run"
run_year_namespace["coeff_est"] = self.coefficients
run_year_namespace["agents_index"] = agents_index
run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run']
new_choices = self.model_system.do_process(run_year_namespace)
#self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
#new_choices = agents.get_attribute(choice_id_name).copy()
agents.modify_attribute(name=choice_id_name, data=current_choices)
dummy_data[agents_index] = new_choices
if predicted_choice_id_name not in agents.get_known_attribute_names():
agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data)
else:
agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data)
logger.log_status("Predictions saved into attribute " + predicted_choice_id_name)
return True
except Exception, e:
logger.log_error("Error encountered in prediction: %s" % e)
logger.log_stack_trace()
return False
def run_simulation(self, simulation_instance=None):
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
class ModelExplorer(GenericModelExplorer):
def __init__(self,
model,
year,
scenario_name=None,
model_group=None,
configuration=None,
xml_configuration=None,
cache_directory=None):
self.model_group = model_group
self.explored_model = model
if configuration is None:
if xml_configuration is None:
raise StandardError, "Either dictionary based or XML based configuration must be given."
config = xml_configuration.get_run_configuration(scenario_name)
else:
config = Configuration(configuration)
if model is not None:
dependent_models = config['models_configuration'][model][
'controller'].get('dependencies', [])
config['models'] = dependent_models
if model_group is None:
config['models'] = config['models'] + [{model: ["run"]}]
else:
config['models'] = config['models'] + [{
model: {
"group_members": [{
model_group: ["run"]
}]
}
}]
else:
config['models'] = []
config['years'] = [year, year]
config["datasets_to_cache_after_each_model"] = []
config['flush_variables'] = False
self.config = Resources(config)
self.xml_configuration = xml_configuration
if cache_directory is None:
cache_directory = config[
'creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy
self.simulation_state = SimulationState(new_instance=True,
base_cache_dir=cache_directory)
self.config['cache_directory'] = cache_directory
SessionConfiguration(
new_instance=True,
package_order=self.config['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
def run(self):
self.model_system = ModelSystem()
self.model_system.run(self.config,
write_datasets_to_cache_at_end_of_year=False)
logger.log_status("Data cache in %s" %
self.simulation_state.get_cache_directory())
def get_agents_for_simulation(self):
return self.get_active_agent_set()
def get_model_name(self):
return (self.explored_model, self.model_group)
def get_specification(self):
return self.get_model().get_specified_coefficients().specification
def get_probabilities(self, submodel=-2):
"""Return a tuple of probabilities and choices, see ChoiceModel.get_probabilities_and_choices.
Works only for the ChoiceModel class.
"""
model = self.get_model()
if isinstance(model, ChoiceModel):
return model.get_probabilities_and_choices(submodel)
print '\nMethod is implemented only for ChoiceModels.\n'
def export_probabilities(self, submodel=-2, filename='./choice_model.txt'):
"""Export probabilities and choices into a file. Works only for the ChoiceModel class"""
model = self.get_model()
if isinstance(model, ChoiceModel):
model.export_probabilities(submodel, file_name=filename)
else:
print '\nMethod is implemented only for ChoiceModels.\n'
class ModelExplorer(GenericModelExplorer):
def __init__(self, model, year, scenario_name=None, model_group=None, configuration=None, xml_configuration=None,
cache_directory=None):
self.model_group = model_group
self.explored_model = model
if configuration is None:
if xml_configuration is None:
raise StandardError, "Either dictionary based or XML based configuration must be given."
config = xml_configuration.get_run_configuration(scenario_name)
else:
config = Configuration(configuration)
if model is not None:
dependent_models = config['models_configuration'][model]['controller'].get('dependencies', [])
config['models'] = dependent_models
if model_group is None:
config['models'] = config['models'] + [{model: ["run"]}]
else:
config['models'] = config['models'] + [{model: {"group_members": [{model_group: ["run"]}]}}]
else:
config['models'] = []
config['years'] = [year, year]
config["datasets_to_cache_after_each_model"]=[]
config['flush_variables'] = False
self.config = Resources(config)
self.xml_configuration = xml_configuration
if cache_directory is None:
cache_directory = config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy
self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory)
self.config['cache_directory'] = cache_directory
SessionConfiguration(new_instance=True,
package_order=self.config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
def run(self):
self.model_system = ModelSystem()
self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False)
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def get_agents_for_simulation(self):
return self.get_active_agent_set()
def get_model_name(self):
return (self.explored_model, self.model_group)
def get_specification(self):
return self.get_model().get_specified_coefficients().specification
def get_probabilities(self, submodel=-2):
"""Return a tuple of probabilities and choices, see ChoiceModel.get_probabilities_and_choices.
Works only for the ChoiceModel class.
"""
model = self.get_model()
if isinstance(model, ChoiceModel):
return model.get_probabilities_and_choices(submodel)
print '\nMethod is implemented only for ChoiceModels.\n'
def export_probabilities(self, submodel=-2, filename='./choice_model.txt'):
"""Export probabilities and choices into a file. Works only for the ChoiceModel class"""
model = self.get_model()
if isinstance(model, ChoiceModel):
model.export_probabilities(submodel, file_name=filename)
else:
print '\nMethod is implemented only for ChoiceModels.\n'