def __init__(self):
Configuration.__init__(self, config)
self.merge(my_configuration)
if not my_configuration[
'creating_baseyear_cache_configuration'].cache_from_database:
del self[
"input_configuration"] # don't bother with the database if everything is in cache
def __init__(
self,
model,
year,
scenario_name=None,
model_group=None,
configuration=None,
xml_configuration=None,
cache_directory=None,
):
self.model_group = model_group
self.explored_model = model
if configuration is None:
if xml_configuration is None:
raise StandardError, "Either dictionary based or XML based configuration must be given."
config = xml_configuration.get_run_configuration(scenario_name)
else:
config = Configuration(configuration)
self.scenario_models = config["models"]
if config.get("models_in_year", None) is not None and config["models_in_year"].get(year, None) is not None:
del config["models_in_year"][year]
if model is not None:
dependent_models = config["models_configuration"][model]["controller"].get("dependencies", [])
config["models"] = dependent_models
if model_group is None:
config["models"] = config["models"] + [{model: ["run"]}]
else:
config["models"] = config["models"] + [{model: {"group_members": [{model_group: ["run"]}]}}]
else:
config["models"] = []
config["years"] = [year, year]
config["datasets_to_cache_after_each_model"] = []
config["flush_variables"] = False
self.config = Resources(config)
self.xml_configuration = xml_configuration
if cache_directory is None:
cache_directory = config["creating_baseyear_cache_configuration"].baseyear_cache.existing_cache_to_copy
self.simulation_state = SimulationState(
new_instance=True, base_cache_dir=cache_directory, start_time=config.get("base_year", 0)
)
self.config["cache_directory"] = cache_directory
SessionConfiguration(
new_instance=True,
package_order=self.config["dataset_pool_configuration"].package_order,
in_storage=AttributeCache(),
)
def test_defaults(self):
creator = BuildingTransitionModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.building_transition_model':
'BuildingTransitionModel'
},
'init': {
'arguments': {
'debuglevel': 'debuglevel'
},
'name': 'BuildingTransitionModel'
},
'run': {
'arguments': {
'building_categories': """{
'residential': array([1,2,3,5,10,20]),
'commercial': 1000*array([1, 2, 5, 10]),
'industrial': 1000*array([1,2,5,10])
}""",
'building_set': 'building',
'building_types_table': 'building_type',
'dataset_pool': 'dataset_pool',
'location_set': 'gridcell',
'vacancy_table': 'target_vacancy',
'year': 'year',
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def execute(self):
# Names of intermediate objects used to get data between steps
# in this model process.
_characteristics = 'characteristics'
_control_totals = 'control_totals'
return Configuration({
'import': {
'urbansim.models.%s' % self._model_name:
'HouseholdTransitionModel'
},
'init': {
'arguments': {
'debuglevel': self.debuglevel,
'location_id_name': "'%s'" % self.location_id_name
},
'name': 'HouseholdTransitionModel'
},
'prepare_for_run': {
'arguments': {
'storage': 'base_cache_storage'
},
'name': 'prepare_for_run',
'output': '(%s, %s)' % (_control_totals, _characteristics)
},
'run': {
'arguments': {
'characteristics': _characteristics,
'control_totals': _control_totals,
'household_set': self.household_set,
'year': 'year'
}
}
})
def test_defaults(self):
creator = EmploymentRelocationModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.employment_relocation_model_creator': 'EmploymentRelocationModelCreator'
},
'init': {
'arguments': {'debuglevel': 'debuglevel',
'location_id_name': "'grid_id'",
'probabilities': "'urbansim.employment_relocation_probabilities'"},
'name': 'EmploymentRelocationModelCreator().get_model',
},
'prepare_for_run': {
'arguments': {
'rate_storage': 'base_cache_storage',
'rate_table': "'annual_relocation_rates_for_jobs'",
'what': "'jobs'"
},
'name': 'prepare_for_run',
'output': 'erm_resources'
},
'run': {
'arguments': {
'resources': 'erm_resources',
'agent_set': 'job'
},
'output': 'erm_index'
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def create_latex_tables_for_model(self, config, model_name, dir):
"""Write to directory dir a file containing a LaTeX table describing this model's
coefficients, and a file containing a LaTeX table describing this model's specification.
Files will be named named <model_name>_coefficients.tex and <model_name>_specification.tex.
"""
config = Configuration(config)
model_system = ModelSystem()
input_db, output_db = model_system._get_database_connections(config)
sql_storage = StorageFactory().get_storage('sql_storage',
storage_location=input_db)
#TODO: only do the next stuff if this model has coefficients
if 'controller' not in config['models_configuration'][model_name]:
return
if 'prepare_for_run' not in config['models_configuration'][model_name][
'controller']:
return
if 'coefficients' not in config['models_configuration'][model_name][
'controller']['prepare_for_run']['output']:
return
specification_table_name = config['models_configuration'][
model_name].get('specification_table', None)
coefficents_table_name = config['models_configuration'][
model_name].get('coefficients_table', None)
(specification, coefficients) = prepare_specification_and_coefficients(
specification_storage=sql_storage,
specification_table=specification_table_name,
coefficients_storage=sql_storage,
coefficients_table=coefficents_table_name)
self.create_latex_table_for_coefficients_for_model(
coefficients, model_name, dir)
self.create_latex_table_for_specifications_for_model(
specification, model_name, dir)
def test_with_arguments(self):
creator = EventsCoordinatorConfigurationCreator(
location_set = 'location_set',
development_type_set = 'development_type_set',
input_events = 'input_events',
output_changed_indices = 'output_changed_indices',
output_processed_development_event_indices = 'output_processed_development_event_indices',
)
expected = Configuration({
'import': {
'urbansim.models.events_coordinator_and_storing': 'EventsCoordinatorAndStoring'
},
'init': {'name': 'EventsCoordinatorAndStoring'},
'run': {
'arguments': {
'current_year': 'year',
'development_event_set': 'input_events',
'development_type_set': 'development_type_set',
'location_set': 'location_set',
'model_configuration': 'model_configuration'
},
'output': '(output_changed_indices, output_processed_development_event_indices)'
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def execute(self):
# Names of intermediate objects used to get data between steps
# in this model process.
_resources = 'hrm_resources'
return Configuration({
'import': {
'urbansim.models.%s_creator' % self._model_name:
'HouseholdRelocationModelCreator'
},
'init': {
'arguments': {
'debuglevel': self.debuglevel,
'location_id_name': "'%s'" % self.location_id_name,
'probabilities': get_string_or_None(self.probabilities),
},
'name': 'HouseholdRelocationModelCreator().get_model'
},
'prepare_for_run': {
'arguments': {
'rate_storage': 'base_cache_storage',
'rate_table': get_string_or_None(self.rate_table),
'what': "'%s'" % self.what,
},
'name': 'prepare_for_run',
'output': _resources,
},
'run': {
'arguments': {
'agent_set': self.agent_set,
'resources': _resources,
},
'output': self.output_index,
}
})
def test_with_arguments(self):
creator = DevelopmentProjectTypeConfigurationCreator(
categories=[9, 9, 9, 9],
developable_maximum_unit_variable_full_name=
'developable_maximum_unit_variable_full_name',
developable_minimum_unit_variable_full_name=
'developable_minimum_unit_variable_full_name',
residential=False,
units='units',
)
expected = Configuration({
'categories':
array([9, 9, 9, 9]),
'developable_maximum_unit_variable_full_name':
'developable_maximum_unit_variable_full_name',
'developable_minimum_unit_variable_full_name':
'developable_minimum_unit_variable_full_name',
'residential':
False,
'units':
'units'
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def execute(self):
return Configuration({
'import': {
'urbansim.models.%s' % self._model_name:
'BuildingTransitionModel'
},
'init': {
'arguments': {
'debuglevel': self.debuglevel
},
'name': 'BuildingTransitionModel'
},
'run': {
'arguments': {
'building_categories': ### TODO: Construct this list from the development_project_type_configuration info
"""{
'residential': array([1,2,3,5,10,20]),
'commercial': 1000*array([1, 2, 5, 10]),
'industrial': 1000*array([1,2,5,10])
}""",
'building_set': self.building_set,
'building_types_table': self.building_types_table,
'dataset_pool': 'dataset_pool',
'location_set': self.location_set,
'vacancy_table': self.vacancy_table,
'year': 'year',
}
}
})
def test_defaults(self):
creator = HouseholdTransitionModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.household_transition_model':
'HouseholdTransitionModel'
},
'init': {
'arguments': {
'debuglevel': 'debuglevel',
'location_id_name': "'grid_id'"
},
'name': 'HouseholdTransitionModel'
},
'prepare_for_run': {
'arguments': {
'storage': 'base_cache_storage'
},
'name': 'prepare_for_run',
'output': '(control_totals, characteristics)'
},
'run': {
'arguments': {
'characteristics': 'characteristics',
'control_totals': 'control_totals',
'household_set': 'household',
'year': 'year'
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_with_arguments(self):
creator = BuildingRelocationModelConfigurationCreator(
agent_set='agent_set',
location_id_name='location_id_name',
output_index='output_index',
)
expected = Configuration({
'import': {
'urbansim.models.agent_relocation_model':
'AgentRelocationModel'
},
'init': {
'arguments': {
'location_id_name': 'location_id_name',
},
'name': 'AgentRelocationModel'
},
'run': {
'arguments': {
'agent_set': 'agent_set'
},
'output': 'output_index'
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_with_arguments(self):
creator = DistributeUnplacedJobsModelConfigurationCreator(
debuglevel=9999,
agent_set='agent_set',
location_set='location_set',
agents_filter='job.sector_id==10')
expected = Configuration({
'import': {
'urbansim.models.distribute_unplaced_jobs_model':
'DistributeUnplacedJobsModel'
},
'init': {
'arguments': {
'debuglevel': 9999,
'filter': None,
'dataset_pool': 'dataset_pool'
},
'name': 'DistributeUnplacedJobsModel'
},
'run': {
'arguments': {
'agent_set': 'agent_set',
'data_objects': 'datasets',
'location_set': 'location_set',
'agents_filter': "'job.sector_id==10'"
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def get_resources_for_run_id_from_history(self,
run_id,
filter_by_status=False):
"""Returns the resources for this run_id, as stored in the run_activity table.
"""
run_activity = self.services_db.get_table('run_activity')
if filter_by_status:
whereclause = and_(run_activity.c.status == 'started',
run_activity.c.run_id == int(run_id))
else:
whereclause = run_activity.c.run_id == int(run_id)
query = select(columns=[run_activity.c.resources],
whereclause=whereclause)
run_resources = self.services_db.execute(query).fetchone()
if not run_resources:
raise StandardError(
"run_id %s doesn't exist on server %s" %
(run_id, self.services_db.get_connection_string(scrub=True)))
try:
r = self._unpickle(run_resources[0])
config = Configuration(r)
except:
logger.log_error(
'Could not create the configuration file for run %i' % run_id)
raise
return config
def test_with_arguments(self):
creator = DevelopmentEventTransitionModelConfigurationCreator(
output_events='output_events',
input_projects='input_projects',
)
expected = Configuration({
'import': {
'urbansim_zone.models.development_event_transition_model':
'DevelopmentEventTransitionModel'
},
'init': {
'name': 'DevelopmentEventTransitionModel'
},
'run': {
'arguments': {
'projects': 'input_projects',
'year': 'year'
},
'output': 'output_events',
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_defaults(self):
creator = GovernmentalRegionalEmploymentLocationChoiceModelConfigurationCreator(
)
expected = Configuration({
'import': {
'washtenaw.models.regional_scaling_jobs_model':
'RegionalScalingJobsModel'
},
'init': {
'arguments': {
'debuglevel': 'debuglevel',
'filter': None,
'dataset_pool': 'dataset_pool'
},
'name': 'RegionalScalingJobsModel'
},
'run': {
'arguments': {
'agent_set': 'job',
'agents_index': 'erm_index',
'data_objects': 'datasets',
'location_set': 'gridcell',
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_defaults(self):
creator = DevelopmentEventTransitionModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.development_event_transition_model':
'DevelopmentEventTransitionModel'
},
'init': {
'name': 'DevelopmentEventTransitionModel'
},
'prepare_for_run': {
'arguments': {
'dev_projects': 'dptm_results',
'model_configuration': 'model_configuration'
},
'name': 'prepare_for_run',
'output': '(all_project_types, all_project_units)'
},
'run': {
'arguments': {
'debuglevel': 'debuglevel',
'projects': 'dptm_results',
'types': 'all_project_types',
'units': 'all_project_units',
'year': 'year'
},
'output': 'development_events'
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_with_arguments(self):
creator = DevelopmentProjectTransitionModelConfigurationCreator(
debuglevel = 9999,
location_set = 'location_set',
history_table = 'history_table',
vacancy_table = 'vacancy_table',
output_results = 'output_results',
vacancy_variables = {'commercial': 'gridcell.my_commercial_vacant_var',
'industrial': 'gridcell.my_industrial_vacant_var'}
)
expected = Configuration({
'import': {
'urbansim.models.development_project_transition_model': 'DevelopmentProjectTransitionModel'
},
'init': {
'arguments': {'debuglevel': 9999},
'name': 'DevelopmentProjectTransitionModel'
},
'run': {
'arguments': {
'history_table': 'history_table',
'location_set': 'location_set',
'model_configuration': 'model_configuration',
'resources': 'model_resources',
'vacancy_table': 'vacancy_table',
'year': 'year',
'resources': "{'industrial_vacant_variable': 'gridcell.my_industrial_vacant_var','commercial_vacant_variable': 'gridcell.my_commercial_vacant_var',}"
},
'output': 'output_results'
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def execute(self):
# Names of intermediate objects used to get data between steps
# in this model process.
_control_totals = 'control_totals'
return Configuration({
'import': {
'urbansim.models.%s' % self._model_name:
'EmploymentTransitionModel'
},
'init': {
'arguments': {
'debuglevel': self.debuglevel,
'location_id_name': "'%s'" % self.location_id_name,
},
'name': 'EmploymentTransitionModel'
},
'prepare_for_run': {
'arguments': {
'storage': 'base_cache_storage'
},
'name': 'prepare_for_run',
'output': _control_totals,
},
'run': {
'arguments': {
'control_totals': _control_totals,
'job_building_types': self.job_building_types,
'job_set': self.job_set,
'year': 'year'
}
}
})
def test_defaults(self):
creator = DevelopmentProjectTransitionModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.development_project_transition_model': 'DevelopmentProjectTransitionModel'
},
'init': {
'arguments': {'debuglevel': 4},
'name': 'DevelopmentProjectTransitionModel'
},
'run': {
'arguments': {
'history_table': 'development_event_history',
'location_set': 'gridcell',
'model_configuration': 'model_configuration',
'resources': 'model_resources',
'vacancy_table': 'target_vacancy',
'year': 'year',
'resources': None
},
'output': 'dptm_results'
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_with_arguments(self):
creator = AddProjectsToBuildingsConfigurationCreator(
input_projects='input_projects',
units_names={
'residential': 'my_units',
'commercial': 'job_spaces'
})
expected = Configuration({
'import': {
'urbansim_zone.models.add_projects_to_buildings':
'AddProjectsToBuildings'
},
'init': {
'name': 'AddProjectsToBuildings'
},
'run': {
'arguments': {
'projects':
'input_projects',
'building_set':
'building',
'building_type_set':
'building_type',
'location_id_name':
"'zone_id'",
'units_names':
"{'residential': 'my_units', 'commercial': 'job_spaces'}"
},
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_defaults(self):
creator = EmploymentTransitionModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.employment_transition_model':
'EmploymentTransitionModel'
},
'init': {
'arguments': {
'debuglevel': 'debuglevel',
'location_id_name': "'grid_id'"
},
'name': 'EmploymentTransitionModel'
},
'prepare_for_run': {
'arguments': {
'storage': 'base_cache_storage'
},
'name': 'prepare_for_run',
'output': 'control_totals'
},
'run': {
'arguments': {
'control_totals': 'control_totals',
'job_building_types': 'job_building_type',
'job_set': 'job',
'year': 'year'
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_with_arguments(self):
creator = GovernmentalEmploymentLocationChoiceModelConfigurationCreator(
agent_set='agent_set',
location_set='location_set',
debuglevel=9999,
input_index='input_index',
)
expected = Configuration({
'import': {
'urbansim.models.scaling_jobs_model': 'ScalingJobsModel'
},
'init': {
'arguments': {
'debuglevel': 9999,
'filter': None,
'dataset_pool': 'dataset_pool'
},
'name': 'ScalingJobsModel',
},
'run': {
'arguments': {
'agent_set': 'agent_set',
'agents_index': 'input_index',
'data_objects': 'datasets',
'location_set': 'location_set',
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def test_defaults(self):
creator = DistributeUnplacedJobsModelConfigurationCreator()
expected = Configuration({
'import': {
'urbansim.models.distribute_unplaced_jobs_model':
'DistributeUnplacedJobsModel'
},
'init': {
'arguments': {
'debuglevel': 'debuglevel',
'filter': None,
'dataset_pool': 'dataset_pool'
},
'name': 'DistributeUnplacedJobsModel'
},
'run': {
'arguments': {
'agent_set': 'job',
'data_objects': 'datasets',
'location_set': 'gridcell',
'agents_filter': None
}
}
})
result = creator.execute()
self.assertDictsEqual(result, expected)
def update_config_for_multiple_runs(self, config):
models_to_update = config.get('models_with_sampled_coefficients', [])
if 'models_in_year' not in config.keys():
config['models_in_year'] = {}
if config['models_in_year'].get(config['base_year'] + 1, None) is None:
config['models_in_year'][config['base_year'] +
1] = config.get('models')
for umodel in models_to_update:
try:
i = config['models_in_year'][config['base_year'] +
1].index(umodel)
new_model_name = '%s_sampled_coef' % umodel
config['models_in_year'][config['base_year'] +
1][i] = new_model_name
except:
pass
config["models_configuration"][new_model_name] = Configuration(
config["models_configuration"][umodel])
config["models_configuration"][new_model_name]["controller"][
"prepare_for_run"]["arguments"]["sample_coefficients"] = True
config["models_configuration"][new_model_name]["controller"][
"prepare_for_run"]["arguments"]["distribution"] = "'normal'"
config["models_configuration"][new_model_name]["controller"][
"prepare_for_run"]["arguments"][
"cache_storage"] = "base_cache_storage"
def execute(self):
# Names of intermediate objects used to get data between steps
# in this model process.
_types = 'all_project_types'
_units = 'all_project_units'
return Configuration({
'import': {
'urbansim.models.%s' % self._model_name:
'DevelopmentEventTransitionModel'
},
'init': {
'name': 'DevelopmentEventTransitionModel'
},
'prepare_for_run': {
'arguments': {
'dev_projects': self.input_projects,
'model_configuration': 'model_configuration'
},
'name': 'prepare_for_run',
'output': '(%s, %s)' % (_types, _units)
},
'run': {
'arguments': {
'debuglevel': self.debuglevel,
'projects': self.input_projects,
'types': _types,
'units': _units,
'year': 'year'
},
'output': self.output_events,
}
})
def execute(self):
# Names of intermediate objects used to get data between steps
# in this model process.
_coefficients = 'coefficients'
_specification = 'specification'
return Configuration({
'estimate': {
'arguments': {
'agent_set': self.agent_set,
'data_objects': 'datasets',
'debuglevel': self.debuglevel,
'procedure': "'%s'" % self.estimation_procedure,
'specification': _specification,
},
'output': '(%s, _)' % _coefficients
},
'import': {
'urbansim.models.%s' % self._model_name:
'AutoOwnershipChoiceModel'
},
'init': {
'arguments': {
'choice_attribute_name':
"'%s'" % self.choice_attribute_name,
'choice_set': list(self.choice_set),
},
'name': 'AutoOwnershipChoiceModel'
},
'prepare_for_estimate': {
'arguments': {
'specification_storage': 'base_cache_storage',
'specification_table': "'%s'" % self.specification_table
},
'name': 'prepare_for_estimate',
'output': '(%s)' % _specification
},
'prepare_for_run': {
'arguments': {
'coefficients_storage': 'base_cache_storage',
'coefficients_table': "'%s'" % self.coefficients_table,
'specification_storage': 'base_cache_storage',
'specification_table': "'%s'" % self.specification_table,
},
'name': 'prepare_for_run',
'output': '(%s, %s)' % (_specification, _coefficients)
},
'run': {
'arguments': {
'agent_set': self.agent_set,
'chunk_specification':
"{'records_per_chunk':%s}" % self.records_per_chunk,
'coefficients': _coefficients,
'data_objects': 'datasets',
'specification': _specification,
}
}
})
def test_regression_model_with_constant_variation(self):
"""Estimate the model and run it on the same data as the estimation. The result should be equal to the original data.
If there is a change in the explanatory variables, the result should not be equal.
"""
storage = StorageFactory().get_storage('dict_storage')
table_name = 'dataset_table'
data = {
"attr1": array([30, 0, 90, 100, 65, 50]),
"attr2": array([2002, 1968, 1880, 1921, 1956, 1989]),
"attr3": array([0.5, 0.1, 0.3, 0.9, 0.2, 0.8]),
"outcome": array([20, 40, 15, 5, 40, 30], dtype="int32"),
"id": array([1, 2, 3, 4, 5, 6])
}
storage.write_table(table_name=table_name, table_data=data)
dataset = Dataset(in_storage=storage,
in_table_name=table_name,
id_name="id")
specification = EquationSpecification(variables=("attr1", "attr2",
"attr3", "constant"),
coefficients=("b1", "b2", "b3",
"constant"))
model = RegressionModelWithAdditionInitialResiduals(
outcome_attribute="outcome")
coef, dummy = model.estimate(
specification,
dataset,
outcome_attribute="outcome",
procedure="opus_core.estimate_linear_regression")
result = model.run(specification, coef, dataset)
# if estimated and run on the same data, it should give the original outcome
self.assertEqual(ma.allequal(result, data["outcome"]), True)
# if some values changed it shoudn't be the same for those elements
dataset.set_values_of_one_attribute("attr1", array([32, 10]),
arange(2))
result2 = model.run(specification, coef, dataset)
self.assertEqual(ma.allequal(result2[0:2], data["outcome"][0:2]),
False)
self.assertEqual(ma.allequal(result2[2:], data["outcome"][2:]), True)
# check if exclusion of missing values is working
dataset.set_values_of_one_attribute("outcome", array([0, 0]),
array([2, 4]))
dataset.delete_one_attribute("_init_error_outcome")
model.run(specification,
coef,
dataset,
run_config=Configuration(
{'exclude_missing_values_from_initial_error': True}))
initial_error = dataset.get_attribute("_init_error_outcome")
self.assertEqual(ma.allequal(initial_error[array([2, 4])], 0), True)
self.assertEqual(ma.allequal(initial_error[array([0, 1, 3, 4, 5])], 0),
False)
def execute(self):
development_project_types = {}
for development_project_type, config in self.development_project_type_set.iteritems(
):
development_project_types[
development_project_type] = config.execute()
return Configuration(development_project_types)
def execute(self):
# Names of intermediate objects used to get data between steps
# in this model process.
_coefficients = 'coefficients'
_specification = 'specification'
_index = 'index'
return Configuration({
'estimate': {
'arguments': {
'data_objects': 'datasets',
'dataset': self.dataset,
'debuglevel': self.debuglevel,
'index': _index,
'specification': _specification,
'procedure': "'%s'" % self.estimation_procedure
},
'output': '(%s, _)' % _coefficients
},
'import': {
'urbansim.models.%s' % self._model_name:
'ResidentialLandShareModel'
},
'init': {
'name': 'ResidentialLandShareModel'
},
'prepare_for_estimate': {
'arguments': {
'dataset': self.dataset,
'specification_storage': 'base_cache_storage',
'specification_table': "'%s'" % self.specification_table
},
'name': 'prepare_for_estimate',
'output': '(%s, %s)' % (_specification, _index)
},
'prepare_for_run': {
'arguments': {
'coefficients_storage': 'base_cache_storage',
'coefficients_table': "'%s'" % self.coefficients_table,
'specification_storage': 'base_cache_storage',
'specification_table': "'%s'" % self.specification_table
},
'name': 'prepare_for_run',
'output': '(%s, %s)' % (_specification, _coefficients)
},
'run': {
'arguments': {
'coefficients': _coefficients,
'data_objects': 'datasets',
'dataset': self.dataset,
'debuglevel': self.debuglevel,
'index': self.input_changed_indices,
'specification': _specification
}
}
})
def __new__(cls, *args, **kwargs):
an_instance = object.__new__(cls)
default_config = Configuration(an_instance.get_configuration())["init"]
config = None
if "model_configuration" in kwargs.keys():
config = kwargs["model_configuration"]
del kwargs["model_configuration"]
an_instance.model_configuration = default_config.merge_defaults_with_arguments_and_config(
config, **kwargs)
if 'run' in map(lambda (name, obj): name, getmembers(an_instance, isroutine)):
run_method = an_instance.run
def config_run_method (*req_args, **opt_args):
default_config = Configuration(an_instance.get_configuration())["run"]
an_instance.model_configuration = default_config.merge_defaults_with_arguments_and_config(
an_instance.model_configuration, **opt_args)
results = run_method(*req_args, **opt_args)
return results
an_instance.run = config_run_method
def __init__(self):
Configuration.__init__(self)
self['residential_regional_development_project_location_choice_model'] = {
'controller': RegionalDevelopmentProjectLocationChoiceModelConfigurationCreator(
project_type = 'residential',
coefficients_table = 'residential_development_location_choice_model_coefficients',
specification_table = 'residential_development_location_choice_model_specification',
#submodel_string = None
).execute(),
}
self['commercial_regional_development_project_location_choice_model'] = {
'controller': RegionalDevelopmentProjectLocationChoiceModelConfigurationCreator(
project_type = 'commercial',
coefficients_table = 'commercial_development_location_choice_model_coefficients',
specification_table = 'commercial_development_location_choice_model_specification',
#submodel_string = None
).execute(),
}
self['industrial_regional_development_project_location_choice_model'] = {
'controller': RegionalDevelopmentProjectLocationChoiceModelConfigurationCreator(
project_type = 'industrial',
coefficients_table = 'industrial_development_location_choice_model_coefficients',
specification_table = 'industrial_development_location_choice_model_specification',
#submodel_string = None
).execute(),
}
self['regional_development_project_transition_model'] = {
'controller': RegionalDevelopmentProjectTransitionModelConfigurationCreator(
output_results = 'dptm_results',
).execute(),
}
self['regional_household_transition_model'] = {
'controller': RegionalHouseholdTransitionModelConfigurationCreator().execute(),
}
self['regional_employment_transition_model'] = {
'controller': RegionalEmploymentTransitionModelConfigurationCreator().execute(),
}
self['regional_employment_location_choice_model'] = {
'controller': RegionalEmploymentLocationChoiceModelConfigurationCreator(
).execute(),
}
self['home_based_regional_employment_location_choice_model'] = {
'controller': HomeBasedRegionalEmploymentLocationChoiceModelConfigurationCreator(
).execute(),
}
self['governmental_regional_employment_location_choice_model'] = {
'controller': GovernmentalRegionalEmploymentLocationChoiceModelConfigurationCreator(
).execute(),
}
self['regional_household_location_choice_model'] = {
'controller': RegionalHouseholdLocationChoiceModelConfigurationCreator(
).execute(),
}
self['regional_household_relocation_model'] = {
'controller': RegionalHouseholdRelocationModelConfigurationCreator(
).execute(),
}
self['regional_employment_relocation_model'] = {
'controller': RegionalEmploymentRelocationModelConfigurationCreator(
).execute(),
}
self['regional_distribute_unplaced_jobs_model'] = {
'controller': RegionalDistributeUnplacedJobsModelConfigurationCreator(
).execute(),
}
self['jobs_event_model'] = {
'controller': JobsEventModelConfigurationCreator(
).execute(),
}
self['households_event_model'] = {
'controller': HouseholdsEventModelConfigurationCreator(
).execute(),
}
#if __name__ == '__main__':
# ControllerConfiguration()
def __init__(self, data={}):
"""The argument 'data' is a python native dictionary (class 'dict').
"""
Configuration.__init__(self, data)
def __init__(self):
Configuration.__init__(self, data = {
'scenario_database_configuration':ScenarioDatabaseConfiguration(
database_name = "washtenaw_class",
),
'datasets_to_cache_after_each_model':[],
'low_memory_mode':False,
'creating_baseyear_cache_configuration':CreatingBaseyearCacheConfiguration(
unroll_gridcells = True,
cache_from_database = False,
baseyear_cache = BaseyearCacheConfiguration(
existing_cache_to_copy = '/urbansim_cache/workshop/cache_source',
#years_to_cache = range(1996,2001)
),
tables_to_cache = [
'annual_employment_control_totals',
'annual_household_control_totals',
'buildings',
'building_types',
'development_event_history',
'gridcells',
'households',
'job_building_types',
'jobs',
'travel_data',
'zones',
'counties',
'commercial_development_location_choice_model_coefficients',
'commercial_development_location_choice_model_specification',
'commercial_employment_location_choice_model_coefficients',
'commercial_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_specification',
'industrial_development_location_choice_model_coefficients',
'industrial_development_location_choice_model_specification',
'residential_development_location_choice_model_coefficients',
'residential_development_location_choice_model_specification',
'fazes',
'urbansim_constants',
'household_location_choice_model_coefficients',
'household_location_choice_model_specification',
'land_price_model_coefficients',
'land_price_model_specification',
'residential_land_share_model_coefficients',
'residential_land_share_model_specification',
'plan_type_group_definitions',
'plan_type_groups',
'large_areas',
'household_characteristics_for_ht',
'development_types',
'development_type_group_definitions',
'development_constraints',
'annual_relocation_rates_for_households',
'annual_relocation_rates_for_jobs',
'base_year',
'cities',
'development_events',
'development_type_groups',
'employment_adhoc_sector_group_definitions',
'employment_adhoc_sector_groups',
'employment_sectors',
'plan_types',
'race_names',
'target_vacancies',
'jobs_for_estimation',
'households_for_estimation',
'development_events_exogenous',
'job_building_types'
],
tables_to_cache_nchunks = {'gridcells':1},
tables_to_copy_to_previous_years = {
'development_type_group_definitions':1995,
'development_type_groups':1995,
'development_types':1995,
'development_constraints':1995,
'urbansim_constants':1995,
},
),
'dataset_pool_configuration': DatasetPoolConfiguration(
package_order=['washtenaw', 'urbansim', 'opus_core'],
),
'base_year': 2000,
'years': (2000,2000),
})
def __init__(self):
Configuration.__init__(self, config)
self.merge(my_configuration)
if not my_configuration['creating_baseyear_cache_configuration'].cache_from_database:
del self["input_configuration"] # don't bother with the database if everything is in cache
def __getitem__(self, key):
"""First check for key in my configuration; then check in Session configuration"""
try:
return Configuration.__getitem__(self, key)
except KeyError:
return SessionConfiguration().get_dataset_from_pool(key)
# Opus/UrbanSim urban simulation software.
# Copyright (C) 2010-2011 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE
from urbansim.configs.base_config_zone import run_configuration as config
from urbansim.configs.estimation_base_config import estimation_configuration
from opus_core.configuration import Configuration
run_configuration = Configuration(config)
run_configuration.merge(estimation_configuration)
run_configuration["models_configuration"]["prepare_datasets_for_aggregation"]["controller"]["run"]["arguments"] = {
"datasets_variables": "{household: ['urbansim.household.zone_id'], " +
"job: ['urbansim.job.zone_id'], " +
"building: ['urbansim.building.zone_id'], " +
"zone: ['urbansim.zone.industrial_sqft_per_job', " +
"'urbansim.zone.commercial_sqft_per_job', " +
"'is_near_arterial = zone.aggregate(urbansim.gridcell.is_near_arterial, function=aggregate)', " +
"'is_near_highway = zone.aggregate(urbansim.gridcell.is_near_highway, function=aggregate)', " +
"'acres_of_land = zone.aggregate(urbansim.gridcell.acres_of_land)', " +
"'urbansim.zone.avg_val_per_unit_commercial', " +
"'urbansim.zone.avg_val_per_unit_industrial', " +
"'urbansim.zone.avg_val_per_unit_governmental', " +
"'urbansim.zone.avg_val_per_unit_vacant_land', " +
"'urbansim.zone.avg_val_per_unit_residential', ]" +
"}",
"data_objects": "datasets"
}
run_configuration["models_configuration"]["household_location_choice_model"]["controller"]["run"]["arguments"]["maximum_runs"]=1
run_configuration["models_configuration"]["employment_location_choice_model"]["controller"]["run"]["arguments"]["maximum_runs"]=1
def __init__(self):
Configuration.__init__(self)
self['models'] = [
'real_estate_price_model',
'development_project_transition_model',
'commercial_development_project_location_choice_model',
'industrial_development_project_location_choice_model',
'residential_development_project_location_choice_model',
'add_projects_to_buildings',
'household_transition_model',
'employment_transition_model',
'household_relocation_model',
'household_location_choice_model',
'employment_relocation_model',
{ 'employment_location_choice_model': { 'group_members': '_all_'}},
'distribute_unplaced_jobs_model',
]
self['model_system'] = 'urbansim.model_coordinators.model_system'
my_controller_configuration = {
'real_estate_price_model': RealEstatePriceModelConfigurationCreator(
dataset='building',
outcome_attribute = 'ln_unit_price=ln(building.avg_value)',
submodel_string = 'building_type_id',
filter_variable = None,
).execute(),
'residential_development_project_location_choice_model': DevelopmentProjectLocationChoiceModelConfigurationCreator(
project_type = 'residential',
coefficients_table = 'residential_development_location_choice_model_coefficients',
specification_table = 'residential_development_location_choice_model_specification',
units = 'residential_units',
sampler = None,
capacity_string = 'urbansim_zone.zone.developable_residential_units',
).execute(),
'commercial_development_project_location_choice_model': DevelopmentProjectLocationChoiceModelConfigurationCreator(
project_type = 'commercial',
coefficients_table = 'commercial_development_location_choice_model_coefficients',
specification_table = 'commercial_development_location_choice_model_specification',
units = 'commercial_job_spaces',
sampler = None,
).execute(),
'industrial_development_project_location_choice_model': DevelopmentProjectLocationChoiceModelConfigurationCreator(
project_type = 'industrial',
coefficients_table = 'industrial_development_location_choice_model_coefficients',
specification_table = 'industrial_development_location_choice_model_specification',
units = 'industrial_job_spaces',
sampler = None,
).execute(),
'development_project_transition_model': DevelopmentProjectTransitionModelConfigurationCreator(
vacancy_variables = {"commercial": "urbansim_zone.zone.number_of_vacant_commercial_jobs",
"industrial": "urbansim_zone.zone.number_of_vacant_industrial_jobs"
},
output_results = 'dptm_results',
).execute(),
'add_projects_to_buildings': AddProjectsToBuildingsConfigurationCreator().execute(),
'employment_transition_model':
EmploymentTransitionModelConfigurationCreator(
location_id_name="zone_id",
).execute(),
'household_transition_model':
HouseholdTransitionModelConfigurationCreator(
location_id_name="zone_id",
).execute(),
'employment_relocation_model':
EmploymentRelocationModelConfigurationCreator(
location_id_name = 'zone_id',
output_index = 'erm_index').execute(),
'household_relocation_model': HouseholdRelocationModelConfigurationCreator(
location_id_name = 'zone_id',
output_index = 'hrm_index',
).execute(),
'household_location_choice_model': HouseholdLocationChoiceModelConfigurationCreator(
location_set = "zone",
capacity_string = 'urbansim_zone.zone.vacant_residential_units',
estimation_weight_string = 'urbansim_zone.zone.vacant_residential_units',
portion_to_unplace = 1/3.0,
nchunks = 3,
number_of_units_string = None,
input_index = 'hrm_index',
).execute(),
'employment_location_choice_model':
EmploymentLocationChoiceModelConfigurationCreator(
location_set = "zone",
input_index = 'erm_index',
agents_for_estimation_table = "jobs_for_estimation",
estimation_weight_string = None,
number_of_units_string = None,
portion_to_unplace = 0,
capacity_string = "urbansim_zone.zone.number_of_vacant_SSS_jobs",
).execute(),
'home_based_employment_location_choice_model':
EmploymentLocationChoiceModelConfigurationCreator(
location_set = "zone",
input_index = 'erm_index',
estimation_weight_string = "urbansim.zone.number_of_households",
agents_for_estimation_table = None, # will take standard jobs table
estimation_size_agents = 0.5,
number_of_units_string = None,
portion_to_unplace = 0,
capacity_string = "urbansim.zone.number_of_households",
).execute(),
'governmental_employment_location_choice_model':
GovernmentalEmploymentLocationChoiceModelConfigurationCreator(
input_index = 'erm_index',
location_set = 'zone'
).execute(),
'distribute_unplaced_jobs_model': DistributeUnplacedJobsModelConfigurationCreator(
location_set = 'zone'
).execute(),
}
self['models_configuration'] = {}
for model in my_controller_configuration.keys():
if model not in self["models_configuration"].keys():
self["models_configuration"][model] = {}
self['models_configuration'][model]['controller'] = my_controller_configuration[model]
self["datasets_to_preload"] = {
'building':{},
'zone':{},
'household':{},
'job': {},
'job_building_type':{},
'target_vacancy':{},
'development_event_history':{},
'building_type': {}
}
def __init__(self):
Configuration.__init__(self, self.__my_configuration())
def config_run_method (*req_args, **opt_args):
default_config = Configuration(an_instance.get_configuration())["run"]
an_instance.model_configuration = default_config.merge_defaults_with_arguments_and_config(
an_instance.model_configuration, **opt_args)
results = run_method(*req_args, **opt_args)
return results
def get_configuration(self):
run_configuration = Configuration(config)
local_configuration = self.get_local_configuration()
run_configuration.merge(local_configuration)
return run_configuration
def __init__(self):
Configuration.__init__(
self,
data={
"scenario_database_configuration": ScenarioDatabaseConfiguration(database_name="washtenaw_class"),
"datasets_to_cache_after_each_model": [],
"low_memory_mode": False,
"creating_baseyear_cache_configuration": CreatingBaseyearCacheConfiguration(
unroll_gridcells=True,
cache_from_database=False,
baseyear_cache=BaseyearCacheConfiguration(
existing_cache_to_copy="/urbansim_cache/workshop/cache_source",
# years_to_cache = range(1996,2001)
),
tables_to_cache=[
"annual_employment_control_totals",
"annual_household_control_totals",
"buildings",
"building_types",
"development_event_history",
"gridcells",
"households",
"job_building_types",
"jobs",
"travel_data",
"zones",
"counties",
"commercial_development_location_choice_model_coefficients",
"commercial_development_location_choice_model_specification",
"commercial_employment_location_choice_model_coefficients",
"commercial_employment_location_choice_model_specification",
"home_based_employment_location_choice_model_specification",
"home_based_employment_location_choice_model_coefficients",
"industrial_employment_location_choice_model_coefficients",
"industrial_employment_location_choice_model_specification",
"industrial_development_location_choice_model_coefficients",
"industrial_development_location_choice_model_specification",
"residential_development_location_choice_model_coefficients",
"residential_development_location_choice_model_specification",
"fazes",
"urbansim_constants",
"household_location_choice_model_coefficients",
"household_location_choice_model_specification",
"land_price_model_coefficients",
"land_price_model_specification",
"residential_land_share_model_coefficients",
"residential_land_share_model_specification",
"plan_type_group_definitions",
"plan_type_groups",
"large_areas",
"household_characteristics_for_ht",
"development_types",
"development_type_group_definitions",
"development_constraints",
"annual_relocation_rates_for_households",
"annual_relocation_rates_for_jobs",
"base_year",
"cities",
"development_events",
"development_type_groups",
"employment_adhoc_sector_group_definitions",
"employment_adhoc_sector_groups",
"employment_sectors",
"plan_types",
"race_names",
"target_vacancies",
"jobs_for_estimation",
"households_for_estimation",
"development_events_exogenous",
"job_building_types",
],
tables_to_cache_nchunks={"gridcells": 1},
tables_to_copy_to_previous_years={
"development_type_group_definitions": 1995,
"development_type_groups": 1995,
"development_types": 1995,
"development_constraints": 1995,
"urbansim_constants": 1995,
},
),
"dataset_pool_configuration": DatasetPoolConfiguration(
package_order=["washtenaw", "urbansim", "opus_core"]
),
"base_year": 2000,
"years": (2000, 2000),
},
)
def __init__(self):
Configuration.__init__(self)
estimation_changes = self._get_estimation_changes()
self.merge(estimation_changes)
def __init__(self, model, specification_module=None,
xml_configuration=None, model_group=None, configuration={},
save_estimation_results=False):
"""
If 'specification_module' is given, it contains the specification defined as a dictionary in a module.
Alternatively, the specification can be passed in an xml format in the 'xml_configuration' argument
(which should be an instance of XMLConfiguration).
If both of those arguments are None, the specification is taken from the cache.
'configuration' is an Opus configuration.
It can contain an entry 'config_changes_for_estimation' which is a dictionary
where keys are model names and values are controller changes for that model.
If 'configuration' is None, it is taken from 'xml_configuration'.
If xml_configuration is used, and if it has a non-empty expression library, the dictionary representing
the expression library is added to the configuration under the key 'expression_library'.
If save_estimation_results is True, the estimation results are saved in the output configuration
(if given in 'configuration') and in the cache.
"""
self.specification_module = specification_module
self.xml_configuration = xml_configuration
self.model_group = model_group
self.estimated_model = model
self.explored_model = model
if configuration is None:
if self.xml_configuration is None:
raise StandardError, "Either dictionary based or XML based configuration must be given."
config = self.xml_configuration.get_estimation_configuration(model, model_group)
else:
config = Configuration(configuration)
config_changes = config.get('config_changes_for_estimation', {})
specification_dict=None
if self.xml_configuration is not None:
specification_dict = self.xml_configuration.get_estimation_specification(model, model_group)
if model_group is None:
if model in config_changes.keys():
config.merge(config_changes[model])
else:
config['models'] = [{model: ["estimate"]}]
if specification_module is not None:
config = update_controller_by_specification_from_module(
config, model, specification_module)
elif specification_dict is not None:
config = update_controller_by_specification_from_dict(config, model, specification_dict)
else:
if model in config_changes.keys():
if model_group in config_changes[model]:
config.merge(config_changes[model][model_group])
else:
config.merge(config_changes[model])
else:
config['models'] = [{model: {"group_members": [{model_group: ["estimate"]}]}}]
if (specification_module is not None) or (specification_dict is not None):
if '%s_%s' % (model_group, model) in config["models_configuration"].keys():
model_name_in_configuration = '%s_%s' % (model_group, model)
else:
model_name_in_configuration = model
if specification_module is not None:
config = update_controller_by_specification_from_module(config, model_name_in_configuration, specification_module)
config["models_configuration"][model_name_in_configuration]["controller"]["prepare_for_estimate"]["arguments"]["specification_dict"] = "spec['%s']" % model_group
else:
config = update_controller_by_specification_from_dict(config, model, specification_dict[model_group])
if model.startswith('%s_' % model_group):
config['model_name_for_coefficients'] = model
else:
config['model_name_for_coefficients'] = '%s_%s' % (model_group, model)
Estimator.__init__(self, config, save_estimation_results=save_estimation_results)
def __init__(self):
my_config = self._get_initial_config()
Configuration.__init__(self, my_config)