示例#1
0
 def run_simulation(self, simulation_instance=None):
     if simulation_instance is None:
         simulation_instance = ModelSystem()
     simulation_instance.run(self.config)
     #simulation_instance.run_multiprocess(self.config, is_run_subset=True)
     logger.log_status("Data cache in %s" %
                       self.simulation_state.get_cache_directory())
 def run_simulation(self, simulation_instance=None):
     logger.start_block("Simulation on database %s" % self.config["scenario_database_configuration"].database_name)
     try:
         if simulation_instance is None:
             simulation_instance = ModelSystem()
         simulation_instance.run(self.config)
         # simulation_instance.run_multiprocess(self.config, is_run_subset=True)
     finally:
         logger.end_block()
     logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
 def run_simulation(self, simulation_instance=None):
     logger.start_block('Simulation on database %s' 
         % self.config['scenario_database_configuration'].database_name)
     try:
         if simulation_instance is None:
             simulation_instance = ModelSystem()
         simulation_instance.run(self.config)
         #simulation_instance.run_multiprocess(self.config, is_run_subset=True)
     finally:
         logger.end_block()
     logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
示例#4
0
 def run_simulation(self, simulation_instance=None):
     if simulation_instance is None:
         simulation_instance = ModelSystem()
     simulation_instance.run(self.config)
     #simulation_instance.run_multiprocess(self.config, is_run_subset=True)
     logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
示例#5
0
class ModelExplorer(object):
    def __init__(self, model, year, scenario_name=None, model_group=None, configuration=None, xml_configuration=None, 
                 cache_directory=None):
        self.model_group = model_group
        self.explored_model = model
 
        if configuration is None:
            if xml_configuration is None:
                raise StandardError, "Either dictionary based or XML based configuration must be given."
            config = xml_configuration.get_run_configuration(scenario_name)
        else:
            config = Configuration(configuration)
            
        self.scenario_models = config['models']
        if config.get('models_in_year', None) is not None and config['models_in_year'].get(year, None) is not None:
            del config['models_in_year'][year]
        if model is not None:
            dependent_models = config['models_configuration'][model]['controller'].get('dependencies', [])
            config['models'] = dependent_models
            if model_group is None:
                config['models'] = config['models'] + [{model: ["run"]}]
            else:
                config['models'] = config['models'] + [{model: {"group_members": [{model_group: ["run"]}]}}]
        else:
            config['models'] = []
            
        config['years'] = [year, year]
        config["datasets_to_cache_after_each_model"]=[]
        config['flush_variables'] = False
        
        self.config = Resources(config)
        self.xml_configuration = xml_configuration
        
        if cache_directory is None:
            cache_directory = config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy
        self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory, 
                                                start_time=config.get('base_year', 0))
        self.config['cache_directory'] = cache_directory
        
        SessionConfiguration(new_instance=True,
                             package_order=self.config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        
    def run(self):
        self.model_system = ModelSystem()
        self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False,
                              cleanup_datasets=False)
        logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
        
    def get_agents_for_simulation(self):
        return self.get_active_agent_set()
        
    def get_model_name(self):
        return (self.explored_model, self.model_group)
        
    def get_specification(self):
        return self.get_model().get_specified_coefficients().specification
    
    def get_probabilities(self, submodel=-2):
        """Return a tuple of probabilities and choices, see ChoiceModel.get_probabilities_and_choices.
        Works only for the ChoiceModel class.
        """
        model = self.get_model()
        #if isinstance(model, ChoiceModel):
        return model.get_probabilities_and_choices(submodel)
        #print '\nMethod is implemented only for ChoiceModels.\n'

    def export_probabilities(self, submodel=-2, filename='./choice_model.txt'):
        """Export probabilities and choices into a file. Works only for the ChoiceModel class"""
        
        model = self.get_model()
        #if isinstance(model, ChoiceModel):
        model.export_probabilities(submodel, file_name=filename)
        #else:
        #    print '\nMethod is implemented only for ChoiceModels.\n'
            
    def get_model(self):
        """Return a model object."""
        return self.model_system.run_year_namespace["model"]
    
    def get_dataset(self, dataset_name):
        """Return a Dataset object of the given name."""
        ds = self.model_system.run_year_namespace.get(dataset_name, None)
        if ds is None:
            if dataset_name not in self.model_system.run_year_namespace["datasets"].keys():
                ds = self.get_dataset_pool().get_dataset(dataset_name)
            else:
                ds = self.model_system.run_year_namespace["datasets"][dataset_name]
        return ds
        
    def get_data(self, coefficient, submodel=-2):
        """Calls method get_data of the Model object. Should return a data array for the 
        given coefficient and submodel. Can be used only on in models that are estimable."""
        return self.get_model().get_data(coefficient, submodel)

    def get_coefficient_names(self, submodel=-2):
        """Calls method get_coefficient_names of the Model object which should return
           coefficient names for the given submodel. Can be used only on in models that are estimable."""
        return self.get_model().get_coefficient_names(submodel)
    
    def get_coefficients(self, submodel=-2):
        """Return an object of class SpecifiedCoefficientsFor1Submodel giving the model coefficients. 
        Can be used only on in models that are estimable."""
        return SpecifiedCoefficientsFor1Submodel(self.get_model().get_specified_coefficients(), submodel)

    def get_data_as_dataset(self, submodel=-2, **kwargs):
        """Calls method get_data_as_dataset of the Model object which should return
        an object of class Dataset containing model data. 
        Works only for ChoiceModel (returns InteractionDataset), 
        and for RegressionModel (returns Dataset). 
        """
        return self.get_model().get_data_as_dataset(submodel, **kwargs)
                
    def get_choice_set(self): 
        """Return a Dataset of choices. Works only for the ChoiceModel class.
        """
        return self.get_model().model_interaction.interaction_dataset.get_dataset(2)
    
    def get_choice_set_index(self):
        """Return an array of indices of choices. Works only for the ChoiceModel class.
        """
        return self.get_model().model_interaction.interaction_dataset.get_index(2)
        
    def get_choice_set_index_for_submodel(self, submodel):
        """Return an array of indices of choices for the given submodel. 
        Works only for the ChoiceModel class.
        """
        index = self.get_choice_set_index()
        return take (index, indices=self.get_model().observations_mapping[submodel], axis=0)
    
    def get_active_choice_set(self, submodel=None):
        """Return choice set as seen by agents in the model.
        Works only for the ChoiceModel class.
        """
        if submodel is None:
            choices = self.get_choice_set_index()
        else:
            choices = self.get_choice_set_index_for_submodel(submodel)
        choices = unique(choices.flatten())
        ds = self.get_choice_set()
        return DatasetSubset(ds, choices)
                             
    def get_agent_set(self):
        """Return a Dataset of all agents.
        """
        return self.get_model().get_agent_set()
        
    def get_agent_set_index(self):
        """Return an array of indices of agents active in the model. 
        """
        return self.get_model().get_agent_set_index()
        
    def get_agent_set_index_for_submodel(self, submodel):
        """Return an array of indices of agents for the given submodel. 
        """
        return self.get_model().get_agent_set_index_for_submodel(submodel)
    
    def get_active_agent_set(self, submodel=None):
        """Return agent set that make choices in the model.
        Works only for the ChoiceModel class.
        """
        agents = self.get_agent_set()
        if submodel is None:
            index = self.get_agent_set_index()
        else:
            index = self.get_agent_set_index_for_submodel(submodel)
        return DatasetSubset(agents, index)
    
    def agent_summary(self, submodel=None):
        ds = self.get_active_agent_set(submodel=submodel)
        ds.summary()
        
    def choice_summary(self, submodel=None):
        ds = self.get_active_choice_set(submodel=submodel)
        ds.summary()
       
    def data_summary(self, **kwargs):
        ds = self.get_data_as_dataset(**kwargs)
        ds.summary()
        
    def _get_before_after_dataset_from_attribute(self, var_name, storage, **kwargs):
        dataset_name = var_name.get_dataset_name()
        ds = self.get_dataset(dataset_name)
        ds.compute_variables([var_name], dataset_pool=self.get_dataset_pool())
        ds.copy_attribute_by_reload(var_name, storage=storage, **kwargs)
        return ds
    
    def get_before_after_attribute(self, attribute_name):
        """Return a dictionary with elements 'before' (contains an array of the given attribute
        that is reloaded from the cache) and 'after' (contains an array of the given attribute 
        with the current values).
        """
        from opus_core.store.attribute_cache import AttributeCache
        var_name = VariableName(attribute_name)
        storage = AttributeCache(self.simulation_state.get_cache_directory())
        ds = self._get_before_after_dataset_from_attribute(var_name, storage=storage,
                   package_order=self.get_dataset_pool().get_package_order())       
        return {'after': ds[var_name.get_alias()],
                'before': ds.get_attribute('%s_reload__' % var_name.get_alias())}
        
    def summary_before_after(self, attribute_name):
        """Print summary of the given attribute 'before' (values
        reloaded from the cache) and 'after' (current values).
        """
        from opus_core.store.attribute_cache import AttributeCache
        var_name = VariableName(attribute_name)
        storage = AttributeCache(self.simulation_state.get_cache_directory())
        ds = self._get_before_after_dataset_from_attribute(var_name, storage=storage, 
                   package_order=self.get_dataset_pool().get_package_order())
        print ''
        print 'Before model run:'
        print '================='
        ds.summary(names=['%s_reload__' % var_name.get_alias()])
        print ''
        print 'After model run:'
        print '================='
        #ds.summary(names=[var_name.get_alias()])
        ds.summary(names=[var_name.get_alias()])
        
    def model_dependencies(self, model=None, group=None):
        """Prints out all dependencies for the model."""
        from opus_core.variables.dependency_query import DependencyChart
        if model is None: # current model
            model, group = self.get_model_name()
            spec = self.get_specification()
        else:
            spec = None
        if model == 'all': # print dependencies for all models
            for thismodel in self.scenario_models:
                thisgroups = None
                if isinstance(thismodel, dict):
                    thisgroups = thismodel[thismodel.keys()[0]].get('group_members', None)
                    thismodel = thismodel.keys()[0]
                if not isinstance(thisgroups, list):
                    thisgroups = [thisgroups]                
                for group in thisgroups:
                    chart = DependencyChart(self.xml_configuration, model=thismodel, model_group=group)
                    chart.print_model_dependencies()
        else:
            chart = DependencyChart(self.xml_configuration, model=model, model_group=group, 
                                specification=spec)
            chart.print_model_dependencies()
        
    def variable_dependencies(self, name):
        """Prints out dependencies of this variable. 'name' can be either an alias from 
        the model specification or an expression."""
        from opus_core.variables.dependency_query import DependencyChart
        varname = None
        allvars = self.get_specification().get_variable_names()
        for ivar in range(len(allvars)):
            thisvar = allvars[ivar]
            if not isinstance(thisvar, VariableName):
                thisvar = VariableName(thisvar)
            if name == thisvar.get_alias():
                varname = thisvar
                break
        if varname is None:
            varname = VariableName(name)
        chart = DependencyChart(self.xml_configuration)
        chart.print_dependencies(varname.get_expression())
              
    def compute_expression(self, attribute_name):
        """Compute any expression and return its values."""
        var_name = VariableName(attribute_name)
        dataset_name = var_name.get_dataset_name()
        ds = self.get_dataset(dataset_name)
        return ds.compute_variables([var_name], dataset_pool=self.get_dataset_pool())
        
    def get_dataset_pool(self):
        return self.model_system.run_year_namespace["dataset_pool"]
    
    def plot_histogram_before_after(self, attribute_name, bins=None):
        """Plot histograms of values returned by the method get_before_after_attribute."""
        from opus_core.plot_functions import create_histogram, show_plots
        from matplotlib.pylab import figure
        values = self.get_before_after_attribute(attribute_name)
        alias = VariableName(attribute_name).get_alias()
        fig = figure()
        fig.add_subplot(121)
        create_histogram(values['before'], main='%s (before)' % alias, bins=bins)
        fig.add_subplot(122)
        create_histogram(values['after'], main='%s (after)' % alias, bins=bins)
        show_plots()
        
    def get_correlation(self, submodel=-2):
        """Return an array of correlations between all variables of the model data (for given submodel).
        Works only for ChoiceModel and RegressionModel"""
        ds = self.get_data_as_dataset(submodel)
        attrs = [attr for attr in ds.get_known_attribute_names() if attr not in ds.get_id_name()]
        return ds.correlation_matrix(attrs)
        
    def plot_correlation(self, submodel=-2, useR=False, **kwargs):
        """Plot correlations between all variables of the model data (for given submodel).
        Works only for ChoiceModel and RegressionModel"""
        ds = self.get_data_as_dataset(submodel)
        attrs = [attr for attr in ds.get_known_attribute_names() if attr not in ds.get_id_name()]
        ds.correlation_image(attrs, useR=useR, **kwargs)
        
    def plot_choice_set(self, agents_index=None, aggregate_to=None, matplotlib=True, **kwargs):
        """Plot map of the sampled choice set. 
        agents_index can be given to restrict the set of agents to which the choice set belongs to. 
        aggregate_to is a name of a dataset which the choice set should be aggregated to.
        If matplotlib is False, mapnik is used (and required). 
        Additional arguments are passed to plot_map or plot_map_matplotlib.
        E.g. (choice set are buildings, aggregated to zones, for the first agent)
        er.plot_choice_set(aggregate_to='zone', matplotlib=False, project_name='psrc_parcel', 
                            file='choice_set0.png', agents_index=0)
        """
        choice_set = self.get_choice_set()
        if agents_index is None:
            flatten_choice_index = self.get_choice_set_index().ravel()
        else:
            flatten_choice_index = self.get_choice_set_index()[agents_index,:].ravel()
        if aggregate_to is not None:
            ds_aggr = self.get_dataset(aggregate_to)
            result = ds_aggr.sum_over_ids(choice_set[ds_aggr.get_id_name()[0]][flatten_choice_index], 
                                               ones(flatten_choice_index.size))
            ds = ds_aggr
        else:
            result = choice_set.sum_over_ids(choice_set.get_id_attribute()[flatten_choice_index], 
                                             ones(flatten_choice_index.size))
            ds = choice_set
        dummy_attribute_name = '__sampled_choice_set__'
        ds.add_attribute(name=dummy_attribute_name, data=result)
        if matplotlib:
            coord_syst = None
            if ds.get_coordinate_system() is None and hasattr(ds, 'compute_coordinate_system'):
                coord_syst = ds.compute_coordinate_system(dataset_pool=self.get_dataset_pool())
            ds.plot_map_matplotlib(dummy_attribute_name, background=-1, coordinate_system=coord_syst, **kwargs)
        else:
            ds.plot_map(dummy_attribute_name, background=-1, **kwargs)
        ds.delete_one_attribute(dummy_attribute_name)
        
    def plot_choice_set_attribute(self, name, agents_index=None, aggregate_to=None, function='sum', 
                                  matplotlib=True, **kwargs):
        """Plot map of the given attribute for the sampled choice set.
        agents_index can be given to restrict the set of agents to which the choice set belongs to. 
        aggregate_to is a name of a dataset which the choice set should be aggregated to.
        function defines the aggregating function (e.g. sum, mean, median, etc.)
        If matplotlib is False, mapnik is used (and required). 
        Additional arguments are passed to plot_map or plot_map_matplotlib.
        E.g. er.plot_choice_set_attribute('residential_units', aggregate_to='zone', matplotlib=False, 
                                    project_name='psrc_parcel', file='choice_resunits.png')
        """
        choice_set = self.get_choice_set()
        if agents_index is None:
            flatten_choice_index = self.get_choice_set_index().ravel()
        else:
            flatten_choice_index = self.get_choice_set_index()[agents_index,:].ravel()
        filter_var = ones(choice_set.size(), dtype='int16')
        filter_var[unique(flatten_choice_index)] = 0
        filter_idx = where(filter_var)[0]
        if aggregate_to is not None:
            ds_aggr = self.get_dataset(aggregate_to)
            result = ds_aggr.aggregate_over_ids(choice_set[ds_aggr.get_id_name()[0]][flatten_choice_index], 
                                                     what=choice_set[name][flatten_choice_index], function=function)
            filter = ds_aggr.sum_over_ids(choice_set[ds_aggr.get_id_name()[0]][filter_idx], 
                                                     ones(filter_idx.size))
            filter = filter > 0
            ds = ds_aggr
        else:
            result = choice_set.aggregate_over_ids(choice_set.get_id_attribute()[flatten_choice_index], 
                                                   what=choice_set[name][flatten_choice_index], function=function)
            filter = filter_var
            ds = choice_set
        dummy_attribute_name = '__sampled_choice_set_attribute__'
        ds.add_attribute(name=dummy_attribute_name, data=result)
        dummy_filter_name = '__sampled_choice_set_filter__'
        ds.add_attribute(name=dummy_filter_name, data=filter)
        if matplotlib:
            coord_syst = None
            if ds.get_coordinate_system() is None and hasattr(ds, 'compute_coordinate_system'):
                coord_syst = ds.compute_coordinate_system(dataset_pool=self.get_dataset_pool())
            ds.plot_map_matplotlib(dummy_attribute_name, filter=dummy_filter_name, coordinate_system=coord_syst, **kwargs)
        else:
            ds.plot_map(dummy_attribute_name, filter=dummy_filter_name, **kwargs)
        ds.delete_one_attribute(dummy_attribute_name)
        ds.delete_one_attribute(dummy_filter_name)
                   
    def plot_coefficients(self, submodel=-2, exclude_constant=True, eqidx=0, plot=True, 
                          store_values_to_file=None):
        """ Plot a barchart of coefficient values. This can be used in a regression model, 
        when coefficients are standardized 
        (i.e. using the estimation module opus_core.estimate_linear_regression_standardized).
        store_values_to_file can be a file name where the values are stored.
        """
        coef = self.get_coefficients(submodel)
        values = coef.get_coefficient_values()
        names = coef.get_coefficient_names()
        sd = coef.get_standard_errors()
        idx=ones(names.shape[1], dtype="bool")
        if exclude_constant:
            pos = coef.get_constants_positions()
            if pos.size > 0:               
                idx[pos]=0
        if store_values_to_file is not None:
            n = idx.sum()
            result = concatenate((reshape(names[eqidx, idx], (n,1)), 
                                 reshape(values[eqidx, idx], (n,1)),
                                 reshape(sd[eqidx, idx], (n,1))), axis=1)
            write_to_text_file(store_values_to_file, array(['coefficient_name', 'estimate', 'standard_error']), 
                               delimiter='\t')
            write_table_to_text_file(store_values_to_file, result, delimiter='\t', mode='a')
        if plot:
            plot_barchart(values[eqidx, idx], labels = names[eqidx, idx], errors=sd[eqidx, idx])
        else:
            return {'names': names[eqidx, idx], 'values': values[eqidx, idx], 'errors': sd[eqidx, idx]}
        
    def create_latex_tables(self, directory, other_info_keys=None):
        from opus_core.latex_table_creator import LatexTableCreator
        LTC = LatexTableCreator()
        LTC.create_latex_table_for_coefficients_for_model(
            self.get_model().get_specified_coefficients().coefficients, self.explored_model, directory, 
                                other_info_keys=other_info_keys)
        LTC.create_latex_table_for_specifications_for_model(
            self.get_model().get_specified_coefficients().specification, self.explored_model, directory)
示例#6
0
class ModelExplorer(object):
    def __init__(self, model, year, scenario_name=None, model_group=None, configuration=None, xml_configuration=None, 
                 cache_directory=None):
        self.model_group = model_group
        self.explored_model = model
 
        if configuration is None:
            if xml_configuration is None:
                raise StandardError, "Either dictionary based or XML based configuration must be given."
            config = xml_configuration.get_run_configuration(scenario_name)
        else:
            config = Configuration(configuration)
            
        self.scenario_models = config['models']
        if config.get('models_in_year', None) is not None and config['models_in_year'].get(year, None) is not None:
            del config['models_in_year'][year]
        if model is not None:
            dependent_models = config['models_configuration'][model]['controller'].get('dependencies', [])
            config['models'] = dependent_models
            if model_group is None:
                config['models'] = config['models'] + [{model: ["run"]}]
            else:
                config['models'] = config['models'] + [{model: {"group_members": [{model_group: ["run"]}]}}]
        else:
            config['models'] = []
            
        config['years'] = [year, year]
        config["datasets_to_cache_after_each_model"]=[]
        config['flush_variables'] = False
        
        self.config = Resources(config)
        self.xml_configuration = xml_configuration
        
        if cache_directory is None:
            cache_directory = config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy
        self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory, 
                                                start_time=config.get('base_year', 0))
        self.config['cache_directory'] = cache_directory
        
        SessionConfiguration(new_instance=True,
                             package_order=self.config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        
    def run(self):
        self.model_system = ModelSystem()
        self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False,
                              cleanup_datasets=False)
        logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
        
    def get_agents_for_simulation(self):
        return self.get_active_agent_set()
        
    def get_model_name(self):
        return (self.explored_model, self.model_group)
        
    def get_specification(self):
        return self.get_model().get_specified_coefficients().specification
    
    def get_probabilities(self, submodel=-2):
        """Return a tuple of probabilities and choices, see ChoiceModel.get_probabilities_and_choices.
        Works only for the ChoiceModel class.
        """
        model = self.get_model()
        #if isinstance(model, ChoiceModel):
        return model.get_probabilities_and_choices(submodel)
        #print '\nMethod is implemented only for ChoiceModels.\n'

    def export_probabilities(self, submodel=-2, filename='./choice_model.txt'):
        """Export probabilities and choices into a file. Works only for the ChoiceModel class"""
        
        model = self.get_model()
        #if isinstance(model, ChoiceModel):
        model.export_probabilities(submodel, file_name=filename)
        #else:
        #    print '\nMethod is implemented only for ChoiceModels.\n'
            
    def get_model(self):
        """Return a model object."""
        return self.model_system.run_year_namespace["model"]
    
    def get_dataset(self, dataset_name):
        """Return a Dataset object of the given name."""
        ds = self.model_system.run_year_namespace.get(dataset_name, None)
        if ds is None:
            if dataset_name not in self.model_system.run_year_namespace["datasets"].keys():
                ds = self.get_dataset_pool().get_dataset(dataset_name)
            else:
                ds = self.model_system.run_year_namespace["datasets"][dataset_name]
        return ds
        
    def get_data(self, coefficient, submodel=-2):
        """Calls method get_data of the Model object. Should return a data array for the 
        given coefficient and submodel. Can be used only on in models that are estimable."""
        return self.get_model().get_data(coefficient, submodel)

    def get_coefficient_names(self, submodel=-2):
        """Calls method get_coefficient_names of the Model object which should return
           coefficient names for the given submodel. Can be used only on in models that are estimable."""
        return self.get_model().get_coefficient_names(submodel)
    
    def get_coefficients(self, submodel=-2):
        """Return an object of class SpecifiedCoefficientsFor1Submodel giving the model coefficients. 
        Can be used only on in models that are estimable."""
        return SpecifiedCoefficientsFor1Submodel(self.get_model().get_specified_coefficients(), submodel)

    def get_data_as_dataset(self, submodel=-2, **kwargs):
        """Calls method get_data_as_dataset of the Model object which should return
        an object of class Dataset containing model data. 
        Works only for ChoiceModel (returns InteractionDataset), 
        and for RegressionModel (returns Dataset). 
        """
        return self.get_model().get_data_as_dataset(submodel, **kwargs)
                
    def get_choice_set(self): 
        """Return a Dataset of choices. Works only for the ChoiceModel class.
        """
        return self.get_model().model_interaction.interaction_dataset.get_dataset(2)
    
    def get_choice_set_index(self):
        """Return an array of indices of choices. Works only for the ChoiceModel class.
        """
        return self.get_model().model_interaction.interaction_dataset.get_index(2)
        
    def get_choice_set_index_for_submodel(self, submodel):
        """Return an array of indices of choices for the given submodel. 
        Works only for the ChoiceModel class.
        """
        index = self.get_choice_set_index()
        return take (index, indices=self.get_model().observations_mapping[submodel], axis=0)
    
    def get_active_choice_set(self, submodel=None):
        """Return choice set as seen by agents in the model.
        Works only for the ChoiceModel class.
        """
        if submodel is None:
            choices = self.get_choice_set_index()
        else:
            choices = self.get_choice_set_index_for_submodel(submodel)
        choices = unique(choices.flatten())
        ds = self.get_choice_set()
        return DatasetSubset(ds, choices)
                             
    def get_agent_set(self):
        """Return a Dataset of all agents. Works only for the ChoiceModel class.
        """
        return self.get_model().model_interaction.interaction_dataset.get_dataset(1)
        
    def get_agent_set_index(self):
        """Return an array of indices of agents that are the choosers. 
        Works only for the ChoiceModel class.
        """
        return self.get_model().model_interaction.interaction_dataset.get_index(1)
        
    def get_agent_set_index_for_submodel(self, submodel):
        """Return an array of indices of agents for the given submodel that are the choosers. 
        Works only for the ChoiceModel class.
        """
        model = self.get_model()
        return model.model_interaction.interaction_dataset.get_index(1)[model.observations_mapping[submodel]]
    
    def get_active_agent_set(self, submodel=None):
        """Return agent set that make choices in the model.
        Works only for the ChoiceModel class.
        """
        agents = self.get_agent_set()
        if submodel is None:
            index = self.get_agent_set_index()
        else:
            index = self.get_agent_set_index_for_submodel(submodel)
        return DatasetSubset(agents, index)
    
    def agent_summary(self, submodel=None):
        ds = self.get_active_agent_set(submodel=submodel)
        ds.summary()
        
    def choice_summary(self, submodel=None):
        ds = self.get_active_choice_set(submodel=submodel)
        ds.summary()
       
    def data_summary(self, **kwargs):
        ds = self.get_data_as_dataset(**kwargs)
        ds.summary()
        
    def _get_before_after_dataset_from_attribute(self, var_name, storage, **kwargs):
        dataset_name = var_name.get_dataset_name()
        ds = self.get_dataset(dataset_name)
        ds.compute_variables([var_name], dataset_pool=self.get_dataset_pool())
        ds.copy_attribute_by_reload(var_name, storage=storage, **kwargs)
        return ds
    
    def get_before_after_attribute(self, attribute_name):
        """Return a dictionary with elements 'before' (contains an array of the given attribute
        that is reloaded from the cache) and 'after' (contains an array of the given attribute 
        with the current values).
        """
        from opus_core.store.attribute_cache import AttributeCache
        var_name = VariableName(attribute_name)
        storage = AttributeCache(self.simulation_state.get_cache_directory())
        ds = self._get_before_after_dataset_from_attribute(var_name, storage=storage,
                   package_order=self.get_dataset_pool().get_package_order())       
        return {'after': ds[var_name.get_alias()],
                'before': ds.get_attribute('%s_reload__' % var_name.get_alias())}
        
    def summary_before_after(self, attribute_name):
        """Print summary of the given attribute 'before' (values
        reloaded from the cache) and 'after' (current values).
        """
        from opus_core.store.attribute_cache import AttributeCache
        var_name = VariableName(attribute_name)
        storage = AttributeCache(self.simulation_state.get_cache_directory())
        ds = self._get_before_after_dataset_from_attribute(var_name, storage=storage, 
                   package_order=self.get_dataset_pool().get_package_order())
        print ''
        print 'Before model run:'
        print '================='
        ds.summary(names=['%s_reload__' % var_name.get_alias()])
        print ''
        print 'After model run:'
        print '================='
        #ds.summary(names=[var_name.get_alias()])
        ds.summary(names=[var_name.get_alias()])
        
    def model_dependencies(self, model=None, group=None):
        """Prints out all dependencies for the model."""
        from opus_core.variables.dependency_query import DependencyChart
        if model is None: # current model
            model, group = self.get_model_name()
            spec = self.get_specification()
        else:
            spec = None
        if model == 'all': # print dependencies for all models
            for thismodel in self.scenario_models:
                thisgroups = None
                if isinstance(thismodel, dict):
                    thisgroups = thismodel[thismodel.keys()[0]].get('group_members', None)
                    thismodel = thismodel.keys()[0]
                if not isinstance(thisgroups, list):
                    thisgroups = [thisgroups]                
                for group in thisgroups:
                    chart = DependencyChart(self.xml_configuration, model=thismodel, model_group=group)
                    chart.print_model_dependencies()
        else:
            chart = DependencyChart(self.xml_configuration, model=model, model_group=group, 
                                specification=spec)
            chart.print_model_dependencies()
        
    def variable_dependencies(self, name):
        """Prints out dependencies of this variable. 'name' can be either an alias from 
        the model specification or an expression."""
        from opus_core.variables.dependency_query import DependencyChart
        varname = None
        allvars = self.get_specification().get_variable_names()
        for ivar in range(len(allvars)):
            thisvar = allvars[ivar]
            if not isinstance(thisvar, VariableName):
                thisvar = VariableName(thisvar)
            if name == thisvar.get_alias():
                varname = thisvar
                break
        if varname is None:
            varname = VariableName(name)
        chart = DependencyChart(self.xml_configuration)
        chart.print_dependencies(varname.get_expression())
              
    def compute_expression(self, attribute_name):
        """Compute any expression and return its values."""
        var_name = VariableName(attribute_name)
        dataset_name = var_name.get_dataset_name()
        ds = self.get_dataset(dataset_name)
        return ds.compute_variables([var_name], dataset_pool=self.get_dataset_pool())
        
    def get_dataset_pool(self):
        return self.model_system.run_year_namespace["dataset_pool"]
    
    def plot_histogram_before_after(self, attribute_name, bins=None):
        """Plot histograms of values returned by the method get_before_after_attribute."""
        from opus_core.plot_functions import create_histogram, show_plots
        from matplotlib.pylab import figure
        values = self.get_before_after_attribute(attribute_name)
        alias = VariableName(attribute_name).get_alias()
        fig = figure()
        fig.add_subplot(121)
        create_histogram(values['before'], main='%s (before)' % alias, bins=bins)
        fig.add_subplot(122)
        create_histogram(values['after'], main='%s (after)' % alias, bins=bins)
        show_plots()
        
    def get_correlation(self, submodel=-2):
        """Return an array of correlations between all variables of the model data (for given submodel).
        Works only for ChoiceModel and RegressionModel"""
        ds = self.get_data_as_dataset(submodel)
        attrs = [attr for attr in ds.get_known_attribute_names() if attr not in ds.get_id_name()]
        return ds.correlation_matrix(attrs)
        
    def plot_correlation(self, submodel=-2, useR=False, **kwargs):
        """Plot correlations between all variables of the model data (for given submodel).
        Works only for ChoiceModel and RegressionModel"""
        ds = self.get_data_as_dataset(submodel)
        attrs = [attr for attr in ds.get_known_attribute_names() if attr not in ds.get_id_name()]
        ds.correlation_image(attrs, useR=useR, **kwargs)
        
    def plot_choice_set(self, agents_index=None, aggregate_to=None, matplotlib=True, **kwargs):
        """Plot map of the sampled choice set. 
        agents_index can be given to restrict the set of agents to which the choice set belongs to. 
        aggregate_to is a name of a dataset which the choice set should be aggregated to.
        If matplotlib is False, mapnik is used (and required). 
        Additional arguments are passed to plot_map or plot_map_matplotlib.
        E.g. (choice set are buildings, aggregated to zones, for the first agent)
        er.plot_choice_set(aggregate_to='zone', matplotlib=False, project_name='psrc_parcel', 
                            file='choice_set0.png', agents_index=0)
        """
        choice_set = self.get_choice_set()
        if agents_index is None:
            flatten_choice_index = self.get_choice_set_index().ravel()
        else:
            flatten_choice_index = self.get_choice_set_index()[agents_index,:].ravel()
        if aggregate_to is not None:
            ds_aggr = self.get_dataset(aggregate_to)
            result = ds_aggr.sum_over_ids(choice_set[ds_aggr.get_id_name()[0]][flatten_choice_index], 
                                               ones(flatten_choice_index.size))
            ds = ds_aggr
        else:
            result = choice_set.sum_over_ids(choice_set.get_id_attribute()[flatten_choice_index], 
                                             ones(flatten_choice_index.size))
            ds = choice_set
        dummy_attribute_name = '__sampled_choice_set__'
        ds.add_attribute(name=dummy_attribute_name, data=result)
        if matplotlib:
            coord_syst = None
            if ds.get_coordinate_system() is None and hasattr(ds, 'compute_coordinate_system'):
                coord_syst = ds.compute_coordinate_system(dataset_pool=self.get_dataset_pool())
            ds.plot_map_matplotlib(dummy_attribute_name, background=-1, coordinate_system=coord_syst, **kwargs)
        else:
            ds.plot_map(dummy_attribute_name, background=-1, **kwargs)
        ds.delete_one_attribute(dummy_attribute_name)
        
    def plot_choice_set_attribute(self, name, agents_index=None, aggregate_to=None, function='sum', 
                                  matplotlib=True, **kwargs):
        """Plot map of the given attribute for the sampled choice set.
        agents_index can be given to restrict the set of agents to which the choice set belongs to. 
        aggregate_to is a name of a dataset which the choice set should be aggregated to.
        function defines the aggregating function (e.g. sum, mean, median, etc.)
        If matplotlib is False, mapnik is used (and required). 
        Additional arguments are passed to plot_map or plot_map_matplotlib.
        E.g. er.plot_choice_set_attribute('residential_units', aggregate_to='zone', matplotlib=False, 
                                    project_name='psrc_parcel', file='choice_resunits.png')
        """
        choice_set = self.get_choice_set()
        if agents_index is None:
            flatten_choice_index = self.get_choice_set_index().ravel()
        else:
            flatten_choice_index = self.get_choice_set_index()[agents_index,:].ravel()
        filter_var = ones(choice_set.size(), dtype='int16')
        filter_var[unique(flatten_choice_index)] = 0
        filter_idx = where(filter_var)[0]
        if aggregate_to is not None:
            ds_aggr = self.get_dataset(aggregate_to)
            result = ds_aggr.aggregate_over_ids(choice_set[ds_aggr.get_id_name()[0]][flatten_choice_index], 
                                                     what=choice_set[name][flatten_choice_index], function=function)
            filter = ds_aggr.sum_over_ids(choice_set[ds_aggr.get_id_name()[0]][filter_idx], 
                                                     ones(filter_idx.size))
            filter = filter > 0
            ds = ds_aggr
        else:
            result = choice_set.aggregate_over_ids(choice_set.get_id_attribute()[flatten_choice_index], 
                                                   what=choice_set[name][flatten_choice_index], function=function)
            filter = filter_var
            ds = choice_set
        dummy_attribute_name = '__sampled_choice_set_attribute__'
        ds.add_attribute(name=dummy_attribute_name, data=result)
        dummy_filter_name = '__sampled_choice_set_filter__'
        ds.add_attribute(name=dummy_filter_name, data=filter)
        if matplotlib:
            coord_syst = None
            if ds.get_coordinate_system() is None and hasattr(ds, 'compute_coordinate_system'):
                coord_syst = ds.compute_coordinate_system(dataset_pool=self.get_dataset_pool())
            ds.plot_map_matplotlib(dummy_attribute_name, filter=dummy_filter_name, coordinate_system=coord_syst, **kwargs)
        else:
            ds.plot_map(dummy_attribute_name, filter=dummy_filter_name, **kwargs)
        ds.delete_one_attribute(dummy_attribute_name)
        ds.delete_one_attribute(dummy_filter_name)
                   
    def plot_coefficients(self, submodel=-2, exclude_constant=True, eqidx=0, plot=True, 
                          store_values_to_file=None):
        """ Plot a barchart of coefficient values. This can be used in a regression model, 
        when coefficients are standardized 
        (i.e. using the estimation module opus_core.estimate_linear_regression_standardized).
        store_values_to_file can be a file name where the values are stored.
        """
        coef = self.get_coefficients(submodel)
        values = coef.get_coefficient_values()
        names = coef.get_coefficient_names()
        sd = coef.get_standard_errors()
        idx=ones(names.shape[1], dtype="bool")
        if exclude_constant:
            pos = coef.get_constants_positions()
            if pos.size > 0:               
                idx[pos]=0
        if store_values_to_file is not None:
            n = idx.sum()
            result = concatenate((reshape(names[eqidx, idx], (n,1)), 
                                 reshape(values[eqidx, idx], (n,1)),
                                 reshape(sd[eqidx, idx], (n,1))), axis=1)
            write_to_text_file(store_values_to_file, array(['coefficient_name', 'estimate', 'standard_error']), 
                               delimiter='\t')
            write_table_to_text_file(store_values_to_file, result, delimiter='\t', mode='a')
        if plot:
            plot_barchart(values[eqidx, idx], labels = names[eqidx, idx], errors=sd[eqidx, idx])
        else:
            return {'names': names[eqidx, idx], 'values': values[eqidx, idx], 'errors': sd[eqidx, idx]}
        
    def create_latex_tables(self, directory, other_info_keys=None):
        from opus_core.latex_table_creator import LatexTableCreator
        LTC = LatexTableCreator()
        LTC.create_latex_table_for_coefficients_for_model(
            self.get_model().get_specified_coefficients().coefficients, self.explored_model, directory, 
                                other_info_keys=other_info_keys)
        LTC.create_latex_table_for_specifications_for_model(
            self.get_model().get_specified_coefficients().specification, self.explored_model, directory)
示例#7
0
class Estimator(ModelExplorer):
    def __init__(self, config=None, save_estimation_results=False):
        if 'cache_directory' not in config or config['cache_directory'] is None:
            raise KeyError("The cache directory must be specified in the "
                "given configuration, giving the filesystem path to the cache "
                "directory containing the data with which to estimate. Please "
                "check that your configuration contains the 'cache_directory' "
                "entry and that it is not None.")

        self.simulation_state = SimulationState(new_instance=True, start_time=config.get('base_year', 0))
        self.simulation_state.set_cache_directory(config['cache_directory'])

        SessionConfiguration(new_instance=True,
                             package_order=config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        self.config = Resources(config)
        self.save_estimation_results = save_estimation_results
        self.debuglevel = self.config.get("debuglevel", 4)
        self.model_system = ModelSystem()
        self.agents_index_for_prediction = None
        
        models = self.config.get('models',[])

        self.model_name = None
        if "model_name" in config.keys():
            self.model_name = config["model_name"]
        else:
            for model in models:
                if isinstance(model, dict):
                    model_name = model.keys()[0]
                    if (model[model_name] == "estimate") or (isinstance(model[model_name], list)
                        and ("estimate" in model[model_name])):
                            self.model_name = model_name
                            break
        estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {})
        if len(estimate_config_changes) > 0:
            change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}})
            estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}')
            estimate_config = Resources({})
            try:
                estimate_config = eval(estimate_config_str)
            except:
                pass
 
            estimate_config.merge(estimate_config_changes)
            self.config.merge(change)
            self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config

    def estimate(self, out_storage=None):
        self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False)
        self.extract_coefficients_and_specification()
        
        if self.save_estimation_results:
            self.save_results(out_storage=out_storage)
            self.log_results()

    def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None):
        """specification_module_name is name of a module that contains a dictionary called
        'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
        'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
        is expected to have an entry of this name. If 'submodels' is given (list or a number),
        the restimation is done only for those submodels.
        """
        if specification_module_name is not None:
            exec("import " + specification_module_name)
            eval("reload (" + specification_module_name + ")")
            exec("specification_dict =" + specification_module_name + ".specification")
            
        if type is not None:
            specification_dict = specification_dict[type]
        if submodels is not None: #remove all submodels but the given ones from specification
            submodels_to_be_deleted = specification_dict.keys()
            if not isinstance(submodels, list):
                submodels = [submodels]
            for sm in submodels:
                if sm not in submodels_to_be_deleted:
                    raise ValueError, "Submodel %s not in the specification." % sm
                submodels_to_be_deleted.remove(sm)
                if "_definition_" in submodels_to_be_deleted:
                    submodels_to_be_deleted.remove("_definition_")
            for sm in submodels_to_be_deleted:
                del specification_dict[sm]
        self.specification = EquationSpecification(specification_dict=specification_dict)
        new_namespace = self.model_system.run_year_namespace
        keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
        new_namespace[keys_coeff_spec["specification"]] = self.specification
        self.coefficients, coeff_dict_dummy = self.model_system.do_process(new_namespace)
        ## update run_year_namespce since it's not been updated by do_process
        self.model_system.run_year_namespace = new_namespace
        self.model_system.run_year_namespace[keys_coeff_spec["coefficients"]] = self.coefficients
        
        ## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
        #self.extract_coefficients_and_specification()  
        if self.save_estimation_results:
            self.save_results(out_storage=out_storage)

    def predict(self, predicted_choice_id_name, agents_index=None):
        """ Run prediction. Currently makes sense only for choice models."""
        # Create temporary configuration where all words 'estimate' are replaced by 'run'
        tmp_config = Resources(self.config)
        
        if self.agents_index_for_prediction is None:
            self.agents_index_for_prediction = self.get_agent_set_index().copy()
            
        if agents_index is None:
            agents_index = self.agents_index_for_prediction
        
        tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est"
        tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index"
        tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}"

        ### save specification and coefficients to cache (no matter the save_estimation_results flag)
        ### so that the prepare_for_run method could load specification and coefficients from there
        #output_configuration = self.config['output_configuration']
        #del self.config['output_configuration']
        #self.save_results()
        
        #self.config['output_configuration'] = output_configuration
        
        #self.model_system.run_year_namespace["coefficients"] = self.coefficients
        #del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
        
        try:
            run_year_namespace = copy.copy(self.model_system.run_year_namespace)
        except:
            logger.log_error("The estimate() method must be run first")
            return False
        
        try:
            agents = self.get_agent_set()
            choice_id_name = self.get_choice_set().get_id_name()[0]
            # save current locations of agents
            current_choices = agents.get_attribute(choice_id_name).copy()
            dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1
            #agents.modify_attribute(name=choice_id_name, data=dummy_data)  #reset choices for all agents
            agents.modify_attribute(name=choice_id_name, data=dummy_data, index=agents_index)  #reset choices for agents in agents_index
            
            run_year_namespace["process"] = "run"
            run_year_namespace["coeff_est"] = self.coefficients
            run_year_namespace["agents_index"] = agents_index
            run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run']
            new_choices = self.model_system.do_process(run_year_namespace)
            
            #self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
            #new_choices = agents.get_attribute(choice_id_name).copy()
            agents.modify_attribute(name=choice_id_name, data=current_choices)
            dummy_data[agents_index] = new_choices
            if predicted_choice_id_name not in agents.get_known_attribute_names():
                agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data)
            else:
                agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data)
            logger.log_status("Predictions saved into attribute " + predicted_choice_id_name)
            return True
        except Exception, e:
            logger.log_error("Error encountered in prediction: %s" % e)
            logger.log_stack_trace()
        
        return False
示例#8
0
class Estimator(ModelExplorer):
    def __init__(self, config=None, save_estimation_results=False):
        if 'cache_directory' not in config or config['cache_directory'] is None:
            raise KeyError("The cache directory must be specified in the "
                "given configuration, giving the filesystem path to the cache "
                "directory containing the data with which to estimate. Please "
                "check that your configuration contains the 'cache_directory' "
                "entry and that it is not None.")

        self.simulation_state = SimulationState(new_instance=True, start_time=config.get('base_year', 0))
        self.simulation_state.set_cache_directory(config['cache_directory'])

        SessionConfiguration(new_instance=True,
                             package_order=config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        self.config = Resources(config)
        self.save_estimation_results = save_estimation_results
        self.debuglevel = self.config.get("debuglevel", 4)
        self.model_system = ModelSystem()
        self.agents_index_for_prediction = None
        
        models = self.config.get('models',[])

        self.model_name = None
        if "model_name" in config.keys():
            self.model_name = config["model_name"]
        else:
            for model in models:
                if isinstance(model, dict):
                    model_name = model.keys()[0]
                    if (model[model_name] == "estimate") or (isinstance(model[model_name], list)
                        and ("estimate" in model[model_name])):
                            self.model_name = model_name
                            break
        estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {})
        if len(estimate_config_changes) > 0:
            change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}})
            estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}')
            estimate_config = Resources({})
            try:
                estimate_config = eval(estimate_config_str)
            except:
                pass
 
            estimate_config.merge(estimate_config_changes)
            self.config.merge(change)
            self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config

    def estimate(self, out_storage=None):
        self.model_system.run(self.config, write_datasets_to_cache_at_end_of_year=False)
        self.extract_coefficients_and_specification()
        
        if self.save_estimation_results:
            self.save_results(out_storage=out_storage)
            self.log_results()

    def reestimate(self, specification_module_name=None, specification_dict=None, out_storage=None, type=None, submodels=None):
        """specification_module_name is name of a module that contains a dictionary called
        'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
        'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
        is expected to have an entry of this name. If 'submodels' is given (list or a number),
        the restimation is done only for those submodels.
        """
        if specification_module_name is not None:
            exec("import " + specification_module_name)
            eval("reload (" + specification_module_name + ")")
            exec("specification_dict =" + specification_module_name + ".specification")
            
        if type is not None:
            specification_dict = specification_dict[type]
        if submodels is not None: #remove all submodels but the given ones from specification
            submodels_to_be_deleted = specification_dict.keys()
            if not isinstance(submodels, list):
                submodels = [submodels]
            for sm in submodels:
                if sm not in submodels_to_be_deleted:
                    raise ValueError, "Submodel %s not in the specification." % sm
                submodels_to_be_deleted.remove(sm)
                if "_definition_" in submodels_to_be_deleted:
                    submodels_to_be_deleted.remove("_definition_")
            for sm in submodels_to_be_deleted:
                del specification_dict[sm]
        self.specification = EquationSpecification(specification_dict=specification_dict)
        new_namespace = self.model_system.run_year_namespace
        keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
        new_namespace[keys_coeff_spec["specification"]] = self.specification
        self.coefficients, coeff_dict_dummy = self.model_system.do_process(new_namespace)
        ## update run_year_namespce since it's not been updated by do_process
        self.model_system.run_year_namespace = new_namespace
        self.model_system.run_year_namespace[keys_coeff_spec["coefficients"]] = self.coefficients
        
        ## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
        #self.extract_coefficients_and_specification()  
        if self.save_estimation_results:
            self.save_results(out_storage=out_storage)

    def predict(self, predicted_choice_id_name, agents_index=None):
        """ Run prediction. Currently makes sense only for choice models."""
        # Create temporary configuration where all words 'estimate' are replaced by 'run'
        tmp_config = Resources(self.config)
        
        if self.agents_index_for_prediction is None:
            self.agents_index_for_prediction = self.get_agent_set_index().copy()
            
        if agents_index is None:
            agents_index = self.agents_index_for_prediction
        
        tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est"
        tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index"
        tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}"

        ### save specification and coefficients to cache (no matter the save_estimation_results flag)
        ### so that the prepare_for_run method could load specification and coefficients from there
        #output_configuration = self.config['output_configuration']
        #del self.config['output_configuration']
        #self.save_results()
        
        #self.config['output_configuration'] = output_configuration
        
        #self.model_system.run_year_namespace["coefficients"] = self.coefficients
        #del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
        
        try:
            run_year_namespace = copy.copy(self.model_system.run_year_namespace)
        except:
            logger.log_error("The estimate() method must be run first")
            return False
        
        try:
            agents = self.get_agent_set()
            choice_id_name = self.get_choice_set().get_id_name()[0]
            # save current locations of agents
            current_choices = agents.get_attribute(choice_id_name).copy()
            dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1
            #agents.modify_attribute(name=choice_id_name, data=dummy_data)  #reset choices for all agents
            agents.modify_attribute(name=choice_id_name, data=dummy_data, index=agents_index)  #reset choices for agents in agents_index
            
            run_year_namespace["process"] = "run"
            run_year_namespace["coeff_est"] = self.coefficients
            run_year_namespace["agents_index"] = agents_index
            run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run']
            new_choices = self.model_system.do_process(run_year_namespace)
            
            #self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
            #new_choices = agents.get_attribute(choice_id_name).copy()
            agents.modify_attribute(name=choice_id_name, data=current_choices)
            dummy_data[agents_index] = new_choices
            if predicted_choice_id_name not in agents.get_known_attribute_names():
                agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data)
            else:
                agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data)
            logger.log_status("Predictions saved into attribute " + predicted_choice_id_name)
            return True
        except Exception, e:
            logger.log_error("Error encountered in prediction: %s" % e)
            logger.log_stack_trace()
        
        return False