def run_simulation(self, simulation_instance=None): if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) #simulation_instance.run_multiprocess(self.config, is_run_subset=True) logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def run_simulation(self, simulation_instance=None): logger.start_block('Simulation on database %s' % self.config['scenario_database_configuration'].database_name) try: if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) #simulation_instance.run_multiprocess(self.config, is_run_subset=True) finally: logger.end_block() logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def run_simulation(self, simulation_instance=None): logger.start_block( 'Simulation on database %s' % self.config['scenario_database_configuration'].database_name) try: if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) #simulation_instance.run_multiprocess(self.config, is_run_subset=True) finally: logger.end_block() logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
config['years'] = [options.year, options.year] config['seed'] = options.seed config["datasets_to_cache_after_each_model"]=[] config['flush_variables'] = False config = Resources(config) cache_directory = config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory) config['cache_directory'] = cache_directory SessionConfiguration(new_instance=True, package_order=config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) model_system = ModelSystem() model_system.run(config, write_datasets_to_cache_at_end_of_year=False) m = model_system.run_year_namespace["model"] mi = m.model_interaction submodels = m.model_interaction.get_submodels() assert len(submodels) == 1 submodel = submodels[0] data = m.get_all_data(submodel) sampling_prob = mi.interaction_dataset['__sampling_probability'] \ if '__sampling_probability' in mi.interaction_dataset.get_known_attribute_names() \ else None model_data.append({'data':data, 'model_name':model_name, 'sampling_probability': sampling_prob, 'choice_type':options.choice_type[h][i]
class Estimator(GenericModelExplorer): def __init__(self, config=None, save_estimation_results=False): if 'cache_directory' not in config or config['cache_directory'] is None: raise KeyError( "The cache directory must be specified in the " "given configuration, giving the filesystem path to the cache " "directory containing the data with which to estimate. Please " "check that your configuration contains the 'cache_directory' " "entry and that it is not None.") self.simulation_state = SimulationState(new_instance=True) self.simulation_state.set_cache_directory(config['cache_directory']) SessionConfiguration( new_instance=True, package_order=config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) self.config = Resources(config) self.save_estimation_results = save_estimation_results self.debuglevel = self.config.get("debuglevel", 4) self.model_system = ModelSystem() self.agents_index_for_prediction = None models = self.config.get('models', []) self.model_name = None if "model_name" in config.keys(): self.model_name = config["model_name"] else: for model in models: if isinstance(model, dict): model_name = model.keys()[0] if (model[model_name] == "estimate") or ( isinstance(model[model_name], list) and ("estimate" in model[model_name])): self.model_name = model_name break estimate_config_changes = self.config.get( 'config_changes_for_estimation', {}).get('estimate_config', {}) if len(estimate_config_changes) > 0: change = Resources({ 'models_configuration': { self.model_name: { 'controller': { 'init': { 'arguments': {} } } } } }) estimate_config_str = self.config['models_configuration'].get( self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}') estimate_config = Resources({}) try: estimate_config = eval(estimate_config_str) except: pass estimate_config.merge(estimate_config_changes) self.config.merge(change) self.config['models_configuration'][ self.model_name]['controller']['init']['arguments'][ 'estimate_config'] = 'Resources(%s)' % estimate_config def estimate(self, out_storage=None): self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False) self.extract_coefficients_and_specification() if self.save_estimation_results: self.save_results(out_storage=out_storage) def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None): """specification_module_name is name of a module that contains a dictionary called 'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object. 'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary is expected to have an entry of this name. If 'submodels' is given (list or a number), the restimation is done only for those submodels. """ if specification_module_name is not None: exec("import " + specification_module_name) eval("reload (" + specification_module_name + ")") exec("specification_dict =" + specification_module_name + ".specification") if type is not None: specification_dict = specification_dict[type] if submodels is not None: #remove all submodels but the given ones from specification submodels_to_be_deleted = specification_dict.keys() if not isinstance(submodels, list): submodels = [submodels] for sm in submodels: if sm not in submodels_to_be_deleted: raise ValueError, "Submodel %s not in the specification." % sm submodels_to_be_deleted.remove(sm) if "_definition_" in submodels_to_be_deleted: submodels_to_be_deleted.remove("_definition_") for sm in submodels_to_be_deleted: del specification_dict[sm] self.specification = EquationSpecification( specification_dict=specification_dict) new_namespace = self.model_system.run_year_namespace keys_coeff_spec = self.get_keys_for_coefficients_and_specification() new_namespace[keys_coeff_spec["specification"]] = self.specification self.coefficients, coeff_dict_dummy = self.model_system.do_process( new_namespace) ## update run_year_namespce since it's not been updated by do_process self.model_system.run_year_namespace = new_namespace self.model_system.run_year_namespace[ keys_coeff_spec["coefficients"]] = self.coefficients ## this gets coeff and spec from run_year_namespce and is only updated in _run_year method #self.extract_coefficients_and_specification() if self.save_estimation_results: self.save_results(out_storage=out_storage) def predict(self, predicted_choice_id_name, agents_index=None): """ Run prediction. Currently makes sense only for choice models.""" # Create temporary configuration where all words 'estimate' are replaced by 'run' tmp_config = Resources(self.config) if self.agents_index_for_prediction is None: self.agents_index_for_prediction = self.get_agent_set_index().copy( ) if agents_index is None: agents_index = self.agents_index_for_prediction tmp_config['models_configuration'][self.model_name]['controller'][ 'run']['arguments']['coefficients'] = "coeff_est" tmp_config['models_configuration'][self.model_name]['controller'][ 'run']['arguments']['agents_index'] = "agents_index" tmp_config['models_configuration'][self.model_name]['controller'][ 'run']['arguments']['chunk_specification'] = "{'nchunks':1}" ### save specification and coefficients to cache (no matter the save_estimation_results flag) ### so that the prepare_for_run method could load specification and coefficients from there #output_configuration = self.config['output_configuration'] #del self.config['output_configuration'] #self.save_results() #self.config['output_configuration'] = output_configuration #self.model_system.run_year_namespace["coefficients"] = self.coefficients #del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run'] try: run_year_namespace = copy.copy( self.model_system.run_year_namespace) except: logger.log_error("The estimate() method must be run first") return False try: agents = self.get_agent_set() choice_id_name = self.get_choice_set().get_id_name()[0] # save current locations of agents current_choices = agents.get_attribute(choice_id_name).copy() dummy_data = zeros(current_choices.size, dtype=current_choices.dtype) - 1 agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset all choices run_year_namespace["process"] = "run" run_year_namespace["coeff_est"] = self.coefficients run_year_namespace["agents_index"] = agents_index run_year_namespace["processmodel_config"] = tmp_config[ 'models_configuration'][self.model_name]['controller']['run'] new_choices = self.model_system.do_process(run_year_namespace) #self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False) #new_choices = agents.get_attribute(choice_id_name).copy() agents.modify_attribute(name=choice_id_name, data=current_choices) dummy_data[agents_index] = new_choices if predicted_choice_id_name not in agents.get_known_attribute_names( ): agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data) else: agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data) logger.log_status("Predictions saved into attribute " + predicted_choice_id_name) return True except Exception, e: logger.log_error("Error encountered in prediction: %s" % e) logger.log_stack_trace() return False
class Estimator(GenericModelExplorer): def __init__(self, config=None, save_estimation_results=False): if 'cache_directory' not in config or config['cache_directory'] is None: raise KeyError("The cache directory must be specified in the " "given configuration, giving the filesystem path to the cache " "directory containing the data with which to estimate. Please " "check that your configuration contains the 'cache_directory' " "entry and that it is not None.") self.simulation_state = SimulationState(new_instance=True) self.simulation_state.set_cache_directory(config['cache_directory']) SessionConfiguration(new_instance=True, package_order=config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) self.config = Resources(config) self.save_estimation_results = save_estimation_results self.debuglevel = self.config.get("debuglevel", 4) self.model_system = ModelSystem() self.agents_index_for_prediction = None models = self.config.get('models',[]) self.model_name = None if "model_name" in config.keys(): self.model_name = config["model_name"] else: for model in models: if isinstance(model, dict): model_name = model.keys()[0] if (model[model_name] == "estimate") or (isinstance(model[model_name], list) and ("estimate" in model[model_name])): self.model_name = model_name break estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {}) if len(estimate_config_changes) > 0: change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}}) estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}') estimate_config = Resources({}) try: estimate_config = eval(estimate_config_str) except: pass estimate_config.merge(estimate_config_changes) self.config.merge(change) self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config def estimate(self, out_storage=None): self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False) self.extract_coefficients_and_specification() if self.save_estimation_results: self.save_results(out_storage=out_storage) def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None): """specification_module_name is name of a module that contains a dictionary called 'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object. 'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary is expected to have an entry of this name. If 'submodels' is given (list or a number), the restimation is done only for those submodels. """ if specification_module_name is not None: exec("import " + specification_module_name) eval("reload (" + specification_module_name + ")") exec("specification_dict =" + specification_module_name + ".specification") if type is not None: specification_dict = specification_dict[type] if submodels is not None: #remove all submodels but the given ones from specification submodels_to_be_deleted = specification_dict.keys() if not isinstance(submodels, list): submodels = [submodels] for sm in submodels: if sm not in submodels_to_be_deleted: raise ValueError, "Submodel %s not in the specification." % sm submodels_to_be_deleted.remove(sm) if "_definition_" in submodels_to_be_deleted: submodels_to_be_deleted.remove("_definition_") for sm in submodels_to_be_deleted: del specification_dict[sm] self.specification = EquationSpecification(specification_dict=specification_dict) new_namespace = self.model_system.run_year_namespace keys_coeff_spec = self.get_keys_for_coefficients_and_specification() new_namespace[keys_coeff_spec["specification"]] = self.specification self.coefficients, coeff_dict_dummy = self.model_system.do_process(new_namespace) ## update run_year_namespce since it's not been updated by do_process self.model_system.run_year_namespace = new_namespace self.model_system.run_year_namespace[keys_coeff_spec["coefficients"]] = self.coefficients ## this gets coeff and spec from run_year_namespce and is only updated in _run_year method #self.extract_coefficients_and_specification() if self.save_estimation_results: self.save_results(out_storage=out_storage) def predict(self, predicted_choice_id_name, agents_index=None): """ Run prediction. Currently makes sense only for choice models.""" # Create temporary configuration where all words 'estimate' are replaced by 'run' tmp_config = Resources(self.config) if self.agents_index_for_prediction is None: self.agents_index_for_prediction = self.get_agent_set_index().copy() if agents_index is None: agents_index = self.agents_index_for_prediction tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est" tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index" tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}" ### save specification and coefficients to cache (no matter the save_estimation_results flag) ### so that the prepare_for_run method could load specification and coefficients from there #output_configuration = self.config['output_configuration'] #del self.config['output_configuration'] #self.save_results() #self.config['output_configuration'] = output_configuration #self.model_system.run_year_namespace["coefficients"] = self.coefficients #del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run'] try: run_year_namespace = copy.copy(self.model_system.run_year_namespace) except: logger.log_error("The estimate() method must be run first") return False try: agents = self.get_agent_set() choice_id_name = self.get_choice_set().get_id_name()[0] # save current locations of agents current_choices = agents.get_attribute(choice_id_name).copy() dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1 agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset all choices run_year_namespace["process"] = "run" run_year_namespace["coeff_est"] = self.coefficients run_year_namespace["agents_index"] = agents_index run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run'] new_choices = self.model_system.do_process(run_year_namespace) #self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False) #new_choices = agents.get_attribute(choice_id_name).copy() agents.modify_attribute(name=choice_id_name, data=current_choices) dummy_data[agents_index] = new_choices if predicted_choice_id_name not in agents.get_known_attribute_names(): agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data) else: agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data) logger.log_status("Predictions saved into attribute " + predicted_choice_id_name) return True except Exception, e: logger.log_error("Error encountered in prediction: %s" % e) logger.log_stack_trace() return False
class ModelExplorer(GenericModelExplorer): def __init__(self, model, year, scenario_name=None, model_group=None, configuration=None, xml_configuration=None, cache_directory=None): self.model_group = model_group self.explored_model = model if configuration is None: if xml_configuration is None: raise StandardError, "Either dictionary based or XML based configuration must be given." config = xml_configuration.get_run_configuration(scenario_name) else: config = Configuration(configuration) if model is not None: dependent_models = config['models_configuration'][model][ 'controller'].get('dependencies', []) config['models'] = dependent_models if model_group is None: config['models'] = config['models'] + [{model: ["run"]}] else: config['models'] = config['models'] + [{ model: { "group_members": [{ model_group: ["run"] }] } }] else: config['models'] = [] config['years'] = [year, year] config["datasets_to_cache_after_each_model"] = [] config['flush_variables'] = False self.config = Resources(config) self.xml_configuration = xml_configuration if cache_directory is None: cache_directory = config[ 'creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory) self.config['cache_directory'] = cache_directory SessionConfiguration( new_instance=True, package_order=self.config['dataset_pool_configuration']. package_order, in_storage=AttributeCache()) def run(self): self.model_system = ModelSystem() self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False) logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory()) def get_agents_for_simulation(self): return self.get_active_agent_set() def get_model_name(self): return (self.explored_model, self.model_group) def get_specification(self): return self.get_model().get_specified_coefficients().specification def get_probabilities(self, submodel=-2): """Return a tuple of probabilities and choices, see ChoiceModel.get_probabilities_and_choices. Works only for the ChoiceModel class. """ model = self.get_model() if isinstance(model, ChoiceModel): return model.get_probabilities_and_choices(submodel) print '\nMethod is implemented only for ChoiceModels.\n' def export_probabilities(self, submodel=-2, filename='./choice_model.txt'): """Export probabilities and choices into a file. Works only for the ChoiceModel class""" model = self.get_model() if isinstance(model, ChoiceModel): model.export_probabilities(submodel, file_name=filename) else: print '\nMethod is implemented only for ChoiceModels.\n'
class ModelExplorer(GenericModelExplorer): def __init__(self, model, year, scenario_name=None, model_group=None, configuration=None, xml_configuration=None, cache_directory=None): self.model_group = model_group self.explored_model = model if configuration is None: if xml_configuration is None: raise StandardError, "Either dictionary based or XML based configuration must be given." config = xml_configuration.get_run_configuration(scenario_name) else: config = Configuration(configuration) if model is not None: dependent_models = config['models_configuration'][model]['controller'].get('dependencies', []) config['models'] = dependent_models if model_group is None: config['models'] = config['models'] + [{model: ["run"]}] else: config['models'] = config['models'] + [{model: {"group_members": [{model_group: ["run"]}]}}] else: config['models'] = [] config['years'] = [year, year] config["datasets_to_cache_after_each_model"]=[] config['flush_variables'] = False self.config = Resources(config) self.xml_configuration = xml_configuration if cache_directory is None: cache_directory = config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory) self.config['cache_directory'] = cache_directory SessionConfiguration(new_instance=True, package_order=self.config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) def run(self): self.model_system = ModelSystem() self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False) logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory()) def get_agents_for_simulation(self): return self.get_active_agent_set() def get_model_name(self): return (self.explored_model, self.model_group) def get_specification(self): return self.get_model().get_specified_coefficients().specification def get_probabilities(self, submodel=-2): """Return a tuple of probabilities and choices, see ChoiceModel.get_probabilities_and_choices. Works only for the ChoiceModel class. """ model = self.get_model() if isinstance(model, ChoiceModel): return model.get_probabilities_and_choices(submodel) print '\nMethod is implemented only for ChoiceModels.\n' def export_probabilities(self, submodel=-2, filename='./choice_model.txt'): """Export probabilities and choices into a file. Works only for the ChoiceModel class""" model = self.get_model() if isinstance(model, ChoiceModel): model.export_probabilities(submodel, file_name=filename) else: print '\nMethod is implemented only for ChoiceModels.\n'