def run_chunk(self, index, dataset, specification, coefficients):
self.specified_coefficients = SpecifiedCoefficients().create(
coefficients, specification, neqs=1)
compute_resources = Resources({"debug": self.debug})
submodels = self.specified_coefficients.get_submodels()
self.get_status_for_gui().update_pieces_using_submodels(
submodels=submodels, leave_pieces=2)
self.map_agents_to_submodels(submodels,
self.submodel_string,
dataset,
index,
dataset_pool=self.dataset_pool,
resources=compute_resources)
variables = self.specified_coefficients.get_full_variable_names_without_constants(
)
self.debug.print_debug("Compute variables ...", 4)
self.increment_current_status_piece()
dataset.compute_variables(variables,
dataset_pool=self.dataset_pool,
resources=compute_resources)
data = {}
coef = {}
outcome = self.initial_values[index].copy()
for submodel in submodels:
coef[submodel] = SpecifiedCoefficientsFor1Submodel(
self.specified_coefficients, submodel)
self.coefficient_names[submodel] = coef[
submodel].get_coefficient_names_without_constant()[0, :]
self.debug.print_debug(
"Compute regression for submodel " + str(submodel), 4)
self.increment_current_status_piece()
self.data[submodel] = dataset.create_regression_data(
coef[submodel],
index=index[self.observations_mapping[submodel]])
nan_index = where(isnan(self.data[submodel]))[1]
inf_index = where(isinf(self.data[submodel]))[1]
vnames = asarray(coef[submodel].get_variable_names())
if nan_index.size > 0:
nan_var_index = unique(nan_index)
self.data[submodel] = nan_to_num(self.data[submodel])
logger.log_warning(
"NaN(Not A Number) is returned from variable %s; it is replaced with %s."
% (vnames[nan_var_index], nan_to_num(nan)))
#raise ValueError, "NaN(Not A Number) is returned from variable %s; check the model specification table and/or attribute values used in the computation for the variable." % vnames[nan_var_index]
if inf_index.size > 0:
inf_var_index = unique(inf_index)
self.data[submodel] = nan_to_num(self.data[submodel])
logger.log_warning(
"Inf is returned from variable %s; it is replaced with %s."
% (vnames[inf_var_index], nan_to_num(inf)))
#raise ValueError, "Inf is returned from variable %s; check the model specification table and/or attribute values used in the computation for the variable." % vnames[inf_var_index]
if (self.data[submodel].shape[0] >
0) and (self.data[submodel].size >
0): # observations for this submodel available
outcome[self.observations_mapping[submodel]] = \
self.regression.run(self.data[submodel], coef[submodel].get_coefficient_values()[0,:],
resources=self.run_config).astype(outcome.dtype)
return outcome
def estimate(self,
specification,
dataset,
outcome_attribute,
index=None,
procedure=None,
data_objects=None,
estimate_config=None,
debuglevel=0):
"""'specification' is of type EquationSpecification,
'dataset' is of type Dataset,
'outcome_attribute' - string that determines the dependent variable,
'index' are indices of individuals in dataset for which
the model runs. If it is None, the whole dataset is considered.
'procedure' - name of the estimation procedure. If it is None,
there should be an entry "estimation" in 'estimate_config' that determines the procedure. The class
must have a method 'run' that takes as arguments 'data', 'regression_procedure' and 'resources'.
It returns a dictionary with entries 'estimators', 'standard_errors' and 't_values' (all 1D numpy arrays).
'data_objects' is a dictionary where each key is the name of an data object
('zone', ...) and its value is an object of class Dataset.
'estimate_config' is of type Resources, it gives additional arguments for the estimation procedure.
'debuglevel' overwrites the class 'debuglevel'.
"""
#import wingdbstub
self.debug.flag = debuglevel
if estimate_config == None:
estimate_config = Resources()
if not isinstance(estimate_config, Resources) and isinstance(
estimate_config, dict):
estimate_config = Resources(estimate_config)
self.estimate_config = estimate_config.merge_with_defaults(
self.estimate_config)
if data_objects is not None:
self.dataset_pool.add_datasets_if_not_included(data_objects)
self.procedure = procedure
if self.procedure == None:
self.procedure = self.estimate_config.get("estimation", None)
if self.procedure is not None:
self.procedure = ModelComponentCreator().get_model_component(
self.procedure)
else:
logger.log_warning(
"No estimation procedure given, or problems with loading the corresponding module."
)
compute_resources = Resources({"debug": self.debug})
if dataset.size() <= 0: # no data loaded yet
dataset.get_id_attribute()
if index == None:
index = arange(dataset.size())
if not isinstance(index, ndarray):
index = array(index)
estimation_size_agents = self.estimate_config.get(
"estimation_size_agents",
None) # should be a proportion of the agent_set
if estimation_size_agents == None:
estimation_size_agents = 1.0
else:
estimation_size_agents = max(min(estimation_size_agents, 1.0),
0.0) # between 0 and 1
if estimation_size_agents < 1.0:
self.debug.print_debug("Sampling agents for estimation ...", 3)
estimation_idx = sample_noreplace(
arange(index.size), int(index.size * estimation_size_agents))
else:
estimation_idx = arange(index.size)
estimation_idx = index[estimation_idx]
self.debug.print_debug(
"Number of observations for estimation: " +
str(estimation_idx.size), 2)
if estimation_idx.size <= 0:
self.debug.print_debug("Nothing to be done.", 2)
return (None, None)
coefficients = create_coefficient_from_specification(specification)
specified_coefficients = SpecifiedCoefficients().create(coefficients,
specification,
neqs=1)
submodels = specified_coefficients.get_submodels()
self.get_status_for_gui().update_pieces_using_submodels(
submodels=submodels, leave_pieces=2)
self.map_agents_to_submodels(
submodels,
self.submodel_string,
dataset,
estimation_idx,
dataset_pool=self.dataset_pool,
resources=compute_resources,
submodel_size_max=self.estimate_config.get('submodel_size_max',
None))
variables = specified_coefficients.get_full_variable_names_without_constants(
)
self.debug.print_debug("Compute variables ...", 4)
self.increment_current_status_piece()
dataset.compute_variables(variables,
dataset_pool=self.dataset_pool,
resources=compute_resources)
coef = {}
estimated_coef = {}
self.outcome = {}
dataset.compute_variables([outcome_attribute],
dataset_pool=self.dataset_pool,
resources=compute_resources)
regression_resources = Resources(estimate_config)
regression_resources.merge({"debug": self.debug})
outcome_variable_name = VariableName(outcome_attribute)
for submodel in submodels:
coef[submodel] = SpecifiedCoefficientsFor1Submodel(
specified_coefficients, submodel)
self.increment_current_status_piece()
logger.log_status("Estimate regression for submodel " +
str(submodel),
tags=["estimate"],
verbosity_level=2)
logger.log_status("Number of observations: " +
str(self.observations_mapping[submodel].size),
tags=["estimate"],
verbosity_level=2)
self.data[
submodel] = dataset.create_regression_data_for_estimation(
coef[submodel],
index=estimation_idx[self.observations_mapping[submodel]])
self.coefficient_names[submodel] = coef[
submodel].get_coefficient_names_without_constant()[0, :]
if (self.data[submodel].shape[0] > 0
) and (self.data[submodel].size > 0) and (
self.procedure
is not None): # observations for this submodel available
self.outcome[submodel] = dataset.get_attribute_by_index(
outcome_variable_name.get_alias(),
estimation_idx[self.observations_mapping[submodel]])
regression_resources.merge({"outcome": self.outcome[submodel]})
regression_resources.merge({
"coefficient_names":
self.coefficient_names[submodel].tolist(),
"constant_position":
coef[submodel].get_constants_positions()
})
estimated_coef[submodel] = self.procedure.run(
self.data[submodel],
self.regression,
resources=regression_resources)
if "estimators" in estimated_coef[submodel].keys():
coef[submodel].set_coefficient_values(
estimated_coef[submodel]["estimators"])
if "standard_errors" in estimated_coef[submodel].keys():
coef[submodel].set_standard_errors(
estimated_coef[submodel]["standard_errors"])
if "other_measures" in estimated_coef[submodel].keys():
for measure in estimated_coef[submodel][
"other_measures"].keys():
coef[submodel].set_measure(
measure, estimated_coef[submodel]["other_measures"]
[measure])
if "other_info" in estimated_coef[submodel].keys():
for info in estimated_coef[submodel]["other_info"]:
coef[submodel].set_other_info(
info, estimated_coef[submodel]["other_info"][info])
coefficients.fill_coefficients(coef)
self.save_predicted_values_and_errors(specification,
coefficients,
dataset,
outcome_variable_name,
index=index,
data_objects=data_objects)
return (coefficients, estimated_coef)
agent_set.get_attribute(submodel_string_short_name),
"submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients,
specification,
neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants(
)
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc', 'urbansim', 'opus_core'],
storage=AttributeCache())
choice_set = dataset_pool.get_dataset(choice_set_name)
if choice_filter_index is None: