def prepare_specification_and_coefficients(specification_storage=None,
specification_table=None,
coefficients_storage=None,
coefficients_table=None,
sample_coefficients=False,
cache_storage=None,
**kwargs):
""" Load specification and coefficients from given tables in given storages.
If 'sample_coefficients' is True, coefficients are sampled from given distribution. In such a case,
either an argument 'distribution' should be given (equal either 'normal' or 'uniform'),
or argument distribution_dictionary should be given containing details about sampling specific coefficients
(see docstring for method sample_values in opus_core/coefficients.py).
If coefficients are sampled, the new values are flushed into cache.
"""
specification = None
if specification_storage is not None and specification_table is not None:
specification = EquationSpecification(in_storage=specification_storage)
specification.load(in_table_name=specification_table)
coefficients = None
if (coefficients_storage is not None) and (coefficients_table is not None):
coefficients = Coefficients(in_storage=coefficients_storage)
coefficients.load(in_table_name=coefficients_table)
if sample_coefficients:
coefficients = coefficients.sample_values(**kwargs)
coefficients.flush_coefficients(coefficients_table, cache_storage)
return (specification, coefficients)
def prepare_specification_and_coefficients(specification_storage=None,
specification_table=None,
coefficients_storage=None,
coefficients_table=None,
sample_coefficients=False,
cache_storage=None,
**kwargs):
""" Load specification and coefficients from given tables in given storages.
If 'sample_coefficients' is True, coefficients are sampled from given distribution. In such a case,
either an argument 'distribution' should be given (equal either 'normal' or 'uniform'),
or argument distribution_dictionary should be given containing details about sampling specific coefficients
(see docstring for method sample_values in opus_core/coefficients.py).
If coefficients are sampled, the new values are flushed into cache.
"""
specification = None
if specification_storage is not None and specification_table is not None:
specification = EquationSpecification(in_storage=specification_storage)
specification.load(in_table_name=specification_table)
coefficients = None
if (coefficients_storage is not None) and (coefficients_table is not None):
coefficients = Coefficients(in_storage=coefficients_storage)
coefficients.load(in_table_name=coefficients_table)
if sample_coefficients:
coefficients = coefficients.sample_values(**kwargs)
coefficients.flush_coefficients(coefficients_table, cache_storage)
return (specification, coefficients)
def test_estimation_without_procedure(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"id": array([1, 2, 3, 4]),
"attr1": array([4, 7, 2, 1]),
"attr2": array([6.8, 2.6, 0, 1]),
"submodel_id": array([1, 2, 2, 1]),
"outcome": array([0, 1, 0, 1])
})
ds = Dataset(in_storage=storage, in_table_name='dataset', id_name="id")
specification = EquationSpecification(
variables=array(["constant", "attr2", "attr1"]),
coefficients=array(["constant", "ba2", "ba1"]))
model = RegressionModel()
model.estimate(specification, ds, "outcome")
data_attr1 = model.get_data("ba1")
self.assert_(ma.allequal(ds.get_attribute("attr1"), data_attr1),
msg="Error in getting data from regression model")
specification_2subm = EquationSpecification(
variables=array(["constant", "attr2", "constant", "attr1"]),
coefficients=array(["constant", "ba2", "constant", "ba1"]),
submodels=array([1, 1, 2, 2]))
model = RegressionModel(submodel_string="submodel_id")
model.estimate(specification_2subm, ds, "outcome")
data_attr1 = model.get_data("ba1", 1)
self.assert_(
data_attr1 == None,
msg=
"Error in getting data from regression model with multiple submodels."
)
data_attr1 = model.get_data("ba1", 2)
self.assert_(
ma.allequal(ds.get_attribute("attr1")[1:3], data_attr1),
msg=
"Error in getting data from regression model with multiple submodels."
)
d = model.get_data_as_dataset(2)
self.assert_(
ma.allequal(
ds.get_attribute("attr1")[1:3], d.get_attribute("ba1")),
msg=
"Error in getting data from regression model with multiple submodels."
)
def test_residential_land_share_model(self):
storage = StorageFactory().get_storage('dict_storage')
gridcell_set_table_name = 'gridcell_set'
storage.write_table(table_name=gridcell_set_table_name,
table_data={
"residential_units": array([1000, 0, 10, 500]),
"development_type_id": array([8, 17, 4, 8]),
"grid_id": array([1, 2, 3, 4])
})
gridcell_set = GridcellDataset(in_storage=storage,
in_table_name=gridcell_set_table_name)
specification = EquationSpecification(
variables=("urbansim.gridcell.devtype_8",
"gridcell.residential_units", "constant"),
coefficients=("DEV8", "units", "constant"))
coefficients = Coefficients(names=("constant", "DEV8", "units"),
values=(-0.4, 1.6, 0.01))
ResidentialLandShareModel(debuglevel=3).run(specification,
coefficients, gridcell_set)
result = gridcell_set.get_attribute("fraction_residential_land")
self.assertEqual(
ma.allclose(result,
array([0.9999863, 0.4013123, 0.4255575, 0.9979747]),
rtol=1e-3), True)
def skip_test_bma(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"id":
arange(100) + 1,
"attr1":
concatenate((random.randint(0, 10, 50),
random.randint(20, 40, 50))),
"attr2":
random.ranf(100),
"outcome":
array(50 * [0] + 50 * [1])
})
ds = Dataset(in_storage=storage, in_table_name='dataset', id_name="id")
specification = EquationSpecification(
variables=array(["constant", "attr2", "attr1"]),
coefficients=array(["constant", "ba2", "ba1"]))
filename = 'bma_output.pdf'
model = RegressionModel(
estimate_config={'bma_imageplot_filename': filename})
model.estimate(specification,
ds,
"outcome",
procedure="opus_core.bma_for_linear_regression_r")
def get_specification_for_estimation(specification_dict=None, specification_storage=None,
specification_table = None):
from opus_core.equation_specification import EquationSpecification
if specification_dict is not None:
return EquationSpecification(specification_dict=specification_dict)
(specification, dummy) = prepare_specification_and_coefficients(specification_storage, specification_table)
return specification
def test_unplaced_agents_decrease_available_space(self):
"""Using the household location choice model, create a set of available spaces and
2000 unplaced agents (along with 5000 placed agents). Run the model, and check that
the unplaced agents were placed, and the number of available spaces has decreased"""
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data={
'grid_id': array(2000 * [0] + 5000 * [1]),
'household_id': arange(7000) + 1
})
storage.write_table(table_name='gridcells',
table_data={
'residential_units': array(50 * [10000]),
'grid_id': arange(50) + 1
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
coefficients = Coefficients(names=("dummy", ), values=(0.1, ))
specification = EquationSpecification(
variables=("gridcell.residential_units", ),
coefficients=("dummy", ))
"""need to specify to the household location choice model exactly which households are moving,
because by default it assumes all current households want to move, but in this test,
the 5000 households already in gridcell #1 shouldn't move.
here, we specify that only the unplaced households should be moved."""
agents_index = where(households.get_attribute("grid_id") == 0)[0]
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
agents_index=agents_index,
debuglevel=1)
gridcells.compute_variables(
["urbansim.gridcell.vacant_residential_units"],
resources=Resources({"household": households}))
vacancies = gridcells.get_attribute("vacant_residential_units")
"""since there were 5000 households already in gridcell #1, and gridcell #1 has
10000 residential units, there should be no more than 5000 vacant residential units
in gridcell #1 after running this model"""
self.assertEqual(vacancies[0] <= 5000, True,
"Error: %d" % (vacancies[0], ))
"""there should be exactly 430000 vacant residential units after the model run,
because there were originally 50 gridcells with 10000 residential units each,
and a total of 7000 units are occupied after the run"""
self.assertEqual(
sum(vacancies) == 50 * 10000 - 7000, True,
"Error: %d" % (sum(vacancies)))
def test_estimation_with_restricted_submodel_size(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"id":
array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"attr1":
array([4, 7, 2, 1, 5, 4, 5, 3, 2, 1]),
"attr2":
array([
6.8,
2.6,
0,
1,
0,
4.3,
2.1,
6,
8,
7,
]),
"submodel_id":
array([1, 2, 2, 1, 1, 1, 2, 1, 1, 1]),
"outcome":
array([0, 1, 0, 1, 1, 1, 0, 0, 1, 1])
})
ds = Dataset(in_storage=storage, in_table_name='dataset', id_name="id")
specification_2subm = EquationSpecification(
variables=array(["constant", "attr2", "constant", "attr1"]),
coefficients=array(["constant", "ba2", "constant", "ba1"]),
submodels=array([1, 1, 2, 2]))
model = RegressionModel(submodel_string="submodel_id",
estimate_config={'submodel_size_max': 5})
model.estimate(specification_2subm,
ds,
"outcome",
procedure='opus_core.estimate_linear_regression')
data_attr1 = model.get_data("ba1", 1)
self.assert_(
data_attr1 == None,
msg=
"Error in getting data from regression model with multiple submodels."
)
data_attr1 = model.get_data("ba2", 1)
self.assert_(data_attr1.size == 5,
msg="Error in sub-sampling data in regression model.")
data_attr1 = model.get_data("ba1", 2)
self.assert_(
ma.allequal(
ds.get_attribute("attr1")[array([1, 2, 6])], data_attr1),
msg=
"Error in getting data from regression model with multiple submodels."
)
def test_regression_model_with_constant_variation(self):
"""Estimate the model and run it on the same data as the estimation. The result should be equal to the original data.
If there is a change in the explanatory variables, the result should not be equal.
"""
storage = StorageFactory().get_storage('dict_storage')
table_name = 'dataset_table'
data = {
"attr1": array([30, 0, 90, 100, 65, 50]),
"attr2": array([2002, 1968, 1880, 1921, 1956, 1989]),
"attr3": array([0.5, 0.1, 0.3, 0.9, 0.2, 0.8]),
"outcome": array([20, 40, 15, 5, 40, 30], dtype="int32"),
"id": array([1, 2, 3, 4, 5, 6])
}
storage.write_table(table_name=table_name, table_data=data)
dataset = Dataset(in_storage=storage,
in_table_name=table_name,
id_name="id")
specification = EquationSpecification(variables=("attr1", "attr2",
"attr3", "constant"),
coefficients=("b1", "b2", "b3",
"constant"))
model = RegressionModelWithAdditionInitialResiduals(
outcome_attribute="outcome")
coef, dummy = model.estimate(
specification,
dataset,
outcome_attribute="outcome",
procedure="opus_core.estimate_linear_regression")
result = model.run(specification, coef, dataset)
# if estimated and run on the same data, it should give the original outcome
self.assertEqual(ma.allequal(result, data["outcome"]), True)
# if some values changed it shoudn't be the same for those elements
dataset.set_values_of_one_attribute("attr1", array([32, 10]),
arange(2))
result2 = model.run(specification, coef, dataset)
self.assertEqual(ma.allequal(result2[0:2], data["outcome"][0:2]),
False)
self.assertEqual(ma.allequal(result2[2:], data["outcome"][2:]), True)
# check if exclusion of missing values is working
dataset.set_values_of_one_attribute("outcome", array([0, 0]),
array([2, 4]))
dataset.delete_one_attribute("_init_error_outcome")
model.run(specification,
coef,
dataset,
run_config=Configuration(
{'exclude_missing_values_from_initial_error': True}))
initial_error = dataset.get_attribute("_init_error_outcome")
self.assertEqual(ma.allequal(initial_error[array([2, 4])], 0), True)
self.assertEqual(ma.allequal(initial_error[array([0, 1, 3, 4, 5])], 0),
False)
def create_simple_specification_from_dictionary():
"""Corresponds to the specification in 'create_simple_regression_specification'."""
spec = {
-2: [ # use always -2, if there are no submodels
('constant', 'b_0'), ('dataset.some_primary_attribute', 'b_1'),
('my_variable = log(package.dataset.variable1*package.dataset.variable2)',
'b_2')
]
}
return EquationSpecification(specification_dict=spec)
def create_specification_with_multiple_submodels():
"""It has 2 submodels and a fixed value for cofficient 'bias'."""
spec = {
1: [
'constant', ('dataset.some_primary_attribute', 'b_1'),
('my_variable = log(package.dataset.variable1*package.dataset.variable2)',
'b_2')
],
5: ['constant', 'package.dataset.variable1', ('bias', 'bias', 1)]
}
return EquationSpecification(specification_dict=spec)
def create_specification_with_definition():
"""All variables are defined in a preamble. They are then further referenced only by aliases.
"""
spec = {
'_definition_':
[('dataset.some_primary_attribute', 'b_1'),
('my_variable = log(package.dataset.variable1*package.dataset.variable2)',
'b_2'), 'package.dataset.variable1',
'var2 = log(package.dataset.variable2*1000)', ('bias', 'bias', 1)],
1: ['constant', 'some_primary_attribute', 'my_variable', 'var2'],
5: ['constant', 'variable1', 'bias']
}
return EquationSpecification(specification_dict=spec)
def test_agents_do_not_go_to_inferior_locations(self):
"""100 gridcells - 99 with attractiveness 2000, 1 with attractiveness 1, no capacity restrictions
10,000 households
We set the coefficient value for attracitiveness to 1.
"""
storage = StorageFactory().get_storage('dict_storage')
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(10000) + 1,
'grid_id': array(10000 * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(100) + 1,
'attractiveness': array(99 * [2000] + [1])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("attractcoef", ), values=(1, ))
specification = EquationSpecification(
variables=("gridcell.attractiveness", ),
coefficients=("attractcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute_by_id("number_of_households", 100)
# nobody should choose gridcell 100
self.assertEqual(result, 0, "Error: %s is not equal to 0" % (result, ))
def create_simple_regression_specification():
"""
y ~ b_0 + b_1*x_1 + b_2*x_2
b_{i} are coefficients, x_1 is a variable 'dataset.some_primary_attribute',
x_2 is an expression 'my_variable = log(package.dataset.variable1*package.dataset.variable2)'
"""
specification = EquationSpecification(
#'constant' is a reserved word in Opus' specification for an intercept
variables=
('constant', 'dataset.some_primary_attribute',
'my_variable = log(package.dataset.variable1*package.dataset.variable2)'
),
coefficients=('b_0', 'b_1', 'b_2'))
return specification
def reestimate(self,
specification_module_name=None,
specification_dict=None,
out_storage=None,
type=None,
submodels=None):
"""specification_module_name is name of a module that contains a dictionary called
'specification'. If it is not given, the argument specification_dict must be given which is a dictionary object.
'type' is the name of model member, such as 'commercial', 'residential'. The specification dictionary
is expected to have an entry of this name. If 'submodels' is given (list or a number),
the restimation is done only for those submodels.
"""
if specification_module_name is not None:
exec("import " + specification_module_name)
eval("reload (" + specification_module_name + ")")
exec("specification_dict =" + specification_module_name +
".specification")
if type is not None:
specification_dict = specification_dict[type]
if submodels is not None: #remove all submodels but the given ones from specification
submodels_to_be_deleted = specification_dict.keys()
if not isinstance(submodels, list):
submodels = [submodels]
for sm in submodels:
if sm not in submodels_to_be_deleted:
raise ValueError, "Submodel %s not in the specification." % sm
submodels_to_be_deleted.remove(sm)
if "_definition_" in submodels_to_be_deleted:
submodels_to_be_deleted.remove("_definition_")
for sm in submodels_to_be_deleted:
del specification_dict[sm]
self.specification = EquationSpecification(
specification_dict=specification_dict)
new_namespace = self.model_system.run_year_namespace
keys_coeff_spec = self.get_keys_for_coefficients_and_specification()
new_namespace[keys_coeff_spec["specification"]] = self.specification
self.coefficients, coeff_dict_dummy = self.model_system.do_process(
new_namespace)
## update run_year_namespce since it's not been updated by do_process
self.model_system.run_year_namespace = new_namespace
self.model_system.run_year_namespace[
keys_coeff_spec["coefficients"]] = self.coefficients
## this gets coeff and spec from run_year_namespce and is only updated in _run_year method
#self.extract_coefficients_and_specification()
if self.save_estimation_results:
self.save_results(out_storage=out_storage)
def xtest_gracefully_handle_empty_choice_sets(self):
storage = StorageFactory().get_storage('dict_storage')
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(10000) + 1,
'grid_id': array(100 * range(100)) + 1
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(100) + 1,
'residential_units': array(100 * [100])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("dummy", ), values=(0, ))
specification = EquationSpecification(
variables=("gridcell.residential_units", ),
coefficients=("dummy", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute_by_id("number_of_households", 100)
# nobody should choose gridcell 100
self.assertEqual(ma.allclose(result.sum(), 0, rtol=0), True,
"Error: %s is not equal to 0" % (result.sum(), ))
def test_do_nothing_if_no_agents(self):
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(table_name=households_table_name,
table_data={
"household_id": arange(10000) + 1,
"grid_id": array(10000 * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
gridcells_table_name = 'gridcells'
storage.write_table(table_name=gridcells_table_name,
table_data={
"grid_id": arange(100) + 1,
"cost": array(50 * [100] + 50 * [1000])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
agents_index=array([], dtype='int32'),
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute("number_of_households")
# check the individual gridcells
self.assertEqual(ma.allclose(result, zeros((100, )), rtol=0), True)
def setUp(self):
# create a dataset
self.dataset = Dataset(data={
"car_time":
array([
52.9, 4.10, 4.10, 56.20, 51.80, 0.20, 27.60, 89.90, 41.50,
95.00, 99.10, 18.50, 82.00, 8.60, 22.50, 51.40, 81.00, 51.00,
62.20, 95.10, 41.60
]),
"bus_time":
array([
4.4, 28.50, 86.90, 31.60, 20.20, 91.20, 79.70, 2.20, 24.50,
43.50, 8.40, 84.00, 38.00, 1.60, 74.10, 83.80, 19.20, 85.00,
90.10, 22.20, 91.50
]),
"choice":
array([
2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1
]),
"id":
arange(21) + 1
},
id_name="id",
dataset_name="person")
choices = Dataset(data={
"choice": array([1, 2]),
"names": array(["car", "bus"])
},
id_name="choice",
dataset_name="transport")
self.choicemodel = ChoiceModel(
choice_set=choices,
utilities="opus_core.linear_utilities",
probabilities="opus_core.mnl_probabilities",
choices="opus_core.random_choices")
self.specification = EquationSpecification(
coefficients=("beta1", "beta2", "beta2"),
variables=("constant", "biogeme.person_transport.time",
"biogeme.person_transport.time"),
equations=(1, 1, 2))
def test_do_nothing_if_empty_set(self):
storage = StorageFactory().get_storage('dict_storage')
gridcell_set_table_name = 'gridcell_set'
storage.write_table(table_name=gridcell_set_table_name,
table_data={"grid_id": array([], dtype='int32')})
gridcell_set = GridcellDataset(in_storage=storage,
in_table_name=gridcell_set_table_name)
specification = EquationSpecification(
variables=("percent_residential_within_walking_distance",
"gridcell_year_built", "constant"),
coefficients=("PRWWD", "YB", "constant"))
coefficients = Coefficients(names=("constant", "PRWWD", "YB"),
values=(10.0, -0.0025, 0.0001))
lp = HousingPriceModel(debuglevel=4)
lp.filter_attribute = None
result = lp.run(specification, coefficients, gridcell_set)
self.assertEqual(result.size, 0)
def run(self, nbs, jobs, data_objects, dbcon, variables=(), coefficients=(), submodels=(), \
debuglevel=0):
t1 = time()
l = len(variables)
print variables, coefficients, submodels
specification = EquationSpecification(variables=variables,
coefficients=coefficients,
submodels=submodels)
# storage = StorageFactory().get_storage(type='mysql_storage', storage_location=dbcon)
# specification = EquationSpecification(storage=storage)
# specification.load(place="employment_non_home_based_location_choice_model_specification")
# coefficients = Coefficients(storage=storage)
# coefficients.load(place="employment_commercial_location_choice_model_coefficients")
elcm = EmploymentLocationChoiceModelCreator().get_model(
location_set=nbs, sample_size_locations=10
) # choice set size (includes current location)
##save estimation results
con = OpusDatabase(hostname=DB_settings.db_host_name,
username=DB_settings.db_user_name,
password=DB_settings.db_password,
database_name=This_Settings.outputdb)
estimation_set = JobDataset(in_storage=StorageFactory().get_storage(
'sql_storage', storage_location=dbcon),
in_place="jobs_for_estimation")
result = elcm.estimate(specification,
agent_set=estimation_set,
data_objects=data_objects,
debuglevel=debuglevel)
#save estimation results
# save_object(specification, 'employment_location_choice_model_specification', type='mysql_storage', base=con)
# save_object(result[0], 'employment_location_choice_model_coefficients', type='mysql_storage', base=con)
print "Simulation done. " + str(time() - t1) + " s"
def test_correct_land_value(self):
#TODO: need to remove this when fixed LP correction only working when year >= 2002
from opus_core.simulation_state import SimulationState
SimulationState().set_current_time(2002)
storage = StorageFactory().get_storage('dict_storage')
gridcell_set_table_name = 'gridcell_set'
storage.write_table(
table_name=gridcell_set_table_name,
table_data={
"percent_residential_within_walking_distance":array([30, 0, 90, 100]),
"gridcell_year_built":array([2002, 1968, 1880, 1921]),
"fraction_residential_land":array([0.5, 0.1, 0.3, 0.9]),
"residential_land_value":array([0, 0, 0, 0]),
"residential_land_value_lag1":array([15000, 0, 7500, 0]),
"nonresidential_land_value":array([0, 0, 0, 0]),
"nonresidential_land_value_lag1":array([15000, 0, 17500, 0]),
"development_type_id":array( [2, 1, 1, 1]),
"development_type_id_lag2":array( [1, 1, 1, 1]),
"grid_id": array([1,2,3,4])
}
)
gridcell_set = GridcellDataset(in_storage=storage, in_table_name=gridcell_set_table_name)
specification = EquationSpecification(variables=(
"percent_residential_within_walking_distance",
"gridcell_year_built", "constant"),
coefficients=("PRWWD", "YB", "constant"))
coefficients = Coefficients(names=("constant", "PRWWD", "YB"), values=(10.0, -0.0025, 0.0001))
lp = LandPriceModel(filter=None, debuglevel=3)
lp.run(specification, coefficients, gridcell_set)
correctmodel = CorrectLandValue()
correctmodel.run(gridcell_set)
result1 = gridcell_set.get_attribute("residential_land_value")
result2 = gridcell_set.get_attribute("nonresidential_land_value")
self.assertEqual(ma.allclose(result1, array([15000.0, 2681.723, 6367.914, 18708.617]), rtol=1e-3), True)
self.assertEqual(ma.allclose(result2, array([15000.0, 24135.510, 14858.466, 2078.735]), rtol=1e-3), True)
def test_housing_price_model(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='gridcells',
table_data={
'percent_residential_within_walking_distance':
array([30, 0, 90, 100]),
'gridcell_year_built':
array([2002, 1968, 1880, 1921]),
'housing_price':
array([0, 0, 0, 0]).astype(float32),
'development_type_id':
array([1, 1, 1, 1]),
'grid_id':
array([1, 2, 3, 4])
})
gridcell_set = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
specification = EquationSpecification(
variables=('percent_residential_within_walking_distance',
'gridcell_year_built', 'constant'),
coefficients=('PRWWD', 'YB', 'constant'))
coefficients = Coefficients(names=("constant", "PRWWD", "YB"),
values=(10.0, -0.25, 0.1))
lp = HousingPriceModel(debuglevel=3)
lp.filter_attribute = None
lp.run(specification, coefficients, gridcell_set)
result = gridcell_set.get_attribute("housing_price")
self.assertEqual(
ma.allclose(result, array([202.7, 206.8, 175.5, 177.1]),
rtol=1e-3), True)
# Models
########
#HLCM
# locations from gridcellset.tab
locations= GridcellDataset(in_storage = StorageFactory().get_storage('tab_storage', storage_location = "."),
in_table_name = "gridcellset", id_name="location")
locations.summary()
seed(1)
coefficients = Coefficients(names=("costcoef", ), values=(-0.01,))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set = locations,
sampler=None,
utilities="opus_core.linear_utilities",
probabilities="opus_core.mnl_probabilities",
choices="opus_core.random_choices_from_index",
compute_capacity_flag=False)
agents.get_attribute("location")
results = hlcm.run(specification, coefficients, agents)
agents.get_attribute("location")
hlcm.upc_sequence.plot_choice_histograms(
capacity=locations.get_attribute("vacant_units"))
hlcm.upc_sequence.show_plots()
def run_ALCM(niter):
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(table_name=households_table_name,
table_data={
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids
})
gridcells_table_name = 'gridcells'
storage.write_table(table_name=gridcells_table_name,
table_data={
'grid_id': arange(ngcs) + 1,
'cost':
array(ngcs_attr * [100] + ngcs_noattr * [1000])
})
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
gridcells = GridcellDataset(in_storage=storage,
in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=('costcoef', ), values=(-0.001, ))
specification = EquationSpecification(variables=('gridcell.cost', ),
coefficients=('costcoef', ))
logger.be_quiet()
result = zeros((niter, ngcs))
for iter in range(niter):
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices='opus_core.random_choices_from_index',
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(['urbansim.gridcell.number_of_households'],
resources=Resources(
{'household': households}))
result_more_attractive = gridcells.get_attribute_by_id(
'number_of_households',
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
'number_of_households', arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute='grid_id',
values=hh_grid_ids)
gridcells.delete_one_attribute('number_of_households')
result[iter, :] = concatenate(
(result_more_attractive, result_less_attractive))
#print result #, result_more_attractive.sum(), result_less_attractive.sum()
return result
from opus_core.variables.variable_name import VariableName
from opus_core.variables.attribute_type import AttributeType
submodel_string_short_name = VariableName(submodel_string).get_short_name()
if submodel_string_short_name in agent_set.get_known_attribute_names():
agent_set.add_attribute(
agent_set.get_attribute(submodel_string_short_name),
"submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients,
specification,
neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants(
)
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
def estimate(self, spec_var=None, spec_py=None,
submodel_string = "workers",
agent_sample_rate=0.005, alt_sample_size=None):
"""
"""
CLOSE = 0.001
sampler = "opus_core.samplers.weighted_sampler"
if alt_sample_size==None:
sampler = None
date_time_str=strftime("%Y_%m_%d__%H_%M", localtime())
agent_sample_rate_str = "__ASR_" + str(agent_sample_rate)
alt_sample_size_str = "_ALT_" + str(alt_sample_size)
info_file = date_time_str + agent_sample_rate_str + alt_sample_size_str + "__info.txt"
logger.enable_file_logging(date_time_str + agent_sample_rate_str + alt_sample_size_str + "__run.txt")
logger.enable_memory_logging()
logger.log_status("Constrained Estimation with agent sample rate of %s and alternatvie sample size %s\n" % \
(agent_sample_rate, alt_sample_size))
t1 = time()
SimulationState().set_current_time(2000)
self.nbs = SessionConfiguration().get_dataset_from_pool("neighborhood")
self.hhs = SessionConfiguration().get_dataset_from_pool('household')
depts, lambda_value = compute_lambda(self.nbs)
supply, vacancy_rate = compute_supply_and_vacancy_rate(self.nbs, depts, lambda_value)
self.nbs.set_values_of_one_attribute("supply", supply)
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset_pool.add_datasets_if_not_included({'vacancy_rate': vacancy_rate,
'sample_rate':agent_sample_rate
})
SessionConfiguration()["CLOSE"] = CLOSE
SessionConfiguration()['info_file'] = info_file
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(in_table_name="household_location_choice_model_specification")
#submodel_string = "workers"
seed(71) # was: seed(71,110)
self.model_name = "household_location_choice_model"
model = HouseholdLocationChoiceModelCreator().get_model(location_set=self.nbs,
submodel_string=submodel_string,
sampler = sampler,
estimation_size_agents = agent_sample_rate * 100/20,
# proportion of the agent set that should be used for the estimation,
#
sample_size_locations = alt_sample_size, # choice set size (includes current location)
compute_capacity_flag = True,
probabilities = "opus_core.mnl_probabilities",
choices = "urbansim.lottery_choices",
run_config = Resources({"capacity_string":"supply"}),
estimate_config = Resources({"capacity_string":"supply","compute_capacity_flag":True}))
#TODO: since households_for_estimation currently is the same as households, create_households_for_estimation
#becomes unnecesarry
#agent_set, agents_index_for_estimation = create_households_for_estimation(self.hhs, self.in_con)
agent_set = self.hhs; agents_index_for_estimation = arange(self.hhs.size())
self.result = model.estimate(self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops" ) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time()-t1) + " s")
def test_agents_go_to_attractive_locations(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
nhhs = 1000
ngcs = 50
ngcs_attr = ngcs/2
ngcs_noattr = ngcs - ngcs_attr
#print ngcs_attr, ngcs_noattr
hh_grid_ids = array([-1]*nhhs)
household_data = {"household_id": arange(nhhs)+1, "grid_id": hh_grid_ids}
gridcell_data = {"grid_id": arange(ngcs)+1, "cost":array(ngcs_attr*[100]+ngcs_noattr*[1000])}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'households', table_data = household_data)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
storage.write_table(table_name = 'gridcells', table_data = gridcell_data)
gridcells = HouseholdDataset(in_storage=storage, in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001,))
specification = EquationSpecification(variables=("gridcell.cost", ), coefficients=("costcoef", ))
logger.be_quiet()
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index",
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1,
chunk_specification={'nchunks':1})
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_more_attractive = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1)
result_less_attractive = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1))
households.set_values_of_one_attribute(attribute="grid_id", values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result_less_attractive[0]=result_less_attractive[0] + self.wrong_number
result = concatenate((result_more_attractive, result_less_attractive))
#print standard_deviation(result[ngcs_attr:(ngcs-1)])
return result
expected_results = array(ngcs_attr*[nhhs*0.71/float(ngcs_attr)] + ngcs_noattr*[nhhs*0.29/float(ngcs_noattr)])
#print expected_results
R = 1000
#r = [2, 5, 10, 50, 100, 1000]
r = [2, 5, 10, 15, 20]
#r=[20]
levels = [0.05, 0.01]
#levels = [0.01]
#wrong_numbers = [8, 10, 12, 14, 16, 18, 21, 23]
#wrong_numbers = [2, 4, 6, 8, 10, 12, 14, 16]
#wrong_numbers = [3, 6, 9, 12, 15, 18]
wrong_numbers = [12, 15, 18]
for wn in wrong_numbers:
self.wrong_number = wn
print "Wrong number =", self.wrong_number
power = zeros((len(r), len(levels)))
for ir in range(len(r)):
for il in range(len(levels)):
print "r =", r[ir],", level =",levels[il]
seed(1)
for iR in range(R):
try:
self.run_stochastic_test(__file__, run_model, expected_results,
r[ir], significance_level=levels[il])
except:
power[ir,il]=power[ir,il]+1
print "Power:",power[ir,il]/float(R)
print power/float(R)
submodel_string = test_settings[model]['submodel_string']
from opus_core.variables.variable_name import VariableName
from opus_core.variables.attribute_type import AttributeType
submodel_string_short_name = VariableName(submodel_string).get_short_name()
if submodel_string_short_name in agent_set.get_known_attribute_names():
agent_set.add_attribute(agent_set.get_attribute(submodel_string_short_name), "submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients, specification, neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants()
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc','urbansim','opus_core'],
storage=AttributeCache())
def estimate(self, spec_var=None, spec_py=None,
movers_index = None,
submodel_string = "",
alt_sample_size=None,
sampler = "opus_core.samplers.weighted_sampler",
weight_string = "supply",
aggregate_demand = False,
submarket_definition = ('zone', 'building_type_id'),
sample_size_from_each_stratum = 50
):
"""
"""
t1 = time()
SimulationState().set_current_time(2000)
dataset_pool=SessionConfiguration().get_dataset_pool()
buildings = dataset_pool.get_dataset("building")
agent_set = dataset_pool.get_dataset('household')
#buildings.load_dataset()
submarket_geography = dataset_pool.get_dataset(submarket_definition[0])
intermediates = '[]'
if submarket_geography.dataset_name == 'zone':
intermediates = '[parcel]'
elif submarket_geography.dataset_name == 'faz':
intermediates = '[zone, parcel]'
elif submarket_geography.dataset_name == 'large_area':
intermediates = '[faz, zone, parcel]'
submarket_id_expression = 'building.disaggregate(%s.%s, intermediates=%s) * 100' % \
(submarket_geography.dataset_name, submarket_geography.get_id_name()[0],
intermediates)
submarket_variables = ['%s=numpy.ceil(submarket.submarket_id / 100)' % submarket_geography.get_id_name()[0]]
if submarket_definition[1] == 'residential_building_type_id':
set_residential_building_types(dataset_pool.get_dataset("building_type"), dataset_pool.get_dataset("building"))
if submarket_definition[1] != '':
submarket_id_expression = submarket_id_expression + ' + building.%s' % submarket_definition[1]
submarket_variables.append(submarket_definition[1] + '=submarket.submarket_id % 100' )
submarkets = define_submarket(buildings,
submarket_id_expression,
#"urbansim_parcel.building.zone_id*100 + building.residential_building_type_id",
#"building.disaggregate(faz.large_area_id, intermediates=[zone, parcel]) * 100 + building.residential_building_type_id",
compute_variables=submarket_variables + [
"residential_units=submarket.aggregate(building.residential_units)",
"number_of_buildings_with_non_zero_units=submarket.aggregate(building.residential_units > 0 )",
"number_of_surveyed_households=submarket.aggregate(household.household_id > 5000000, intermediates=[building])",
],
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'submarket.supply > 0',
#"psrc_parcel.building.large_area_id*100 + building.residential_building_type_id",
#compute_variables=['residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id / 100)']
#"psrc_parcel.building.large_area_id",
#compute_variables=[#'residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id)']
)
dataset_pool.add_datasets_if_not_included({'submarket':submarkets})
compute_lambda_and_supply(buildings, agent_set, movers_index, submarkets)
submarket_filter = 'submarket.supply > 0'
if submarket_filter is not None:
from numpy import logical_not
submarkets.remove_elements(index= where( logical_not(submarkets.compute_variables(submarket_filter)) )[0])
submarkets.touch_attribute(submarkets.get_id_name()[0])
buildings.touch_attribute(submarkets.get_id_name()[0])
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(in_table_name="household_location_choice_model_specification")
self.model_name = "household_location_choice_model"
agent_set, agents_index_for_estimation = get_households_for_estimation(agent_set,
AttributeCache(),
"households_for_estimation",
exclude_condition="household.disaggregate(submarket.submarket_id, intermediates=[building])<=0",
)
agent_set.compute_variables("submarket_id=household.disaggregate(building.submarket_id)")
agent_sample_rate = agents_index_for_estimation.size / float(movers_index.size)
dataset_pool.add_datasets_if_not_included({'sample_rate': agent_sample_rate })
if aggregate_demand:
location_set = buildings
aggregate_dataset = 'submarket'
#weight_string = 'inv_submarket_supply = 1.0 / (building.disaggregate(submarket.number_of_agents(building))).astype(float32) * (building.disaggregate(submarket.submarket_id) > 0)'
#weight_string = 'submarket_supply = (building.disaggregate(submarket.supply) > 0).astype(int32)'
#weight_string = 'submarket_supply = building.disaggregate(submarket.supply) * (building.disaggregate(submarket.submarket_id) > 0).astype(float32)'
else:
location_set = submarkets
aggregate_dataset = None
#weight_string = 'supply'
model = HouseholdLocationChoiceModelCreator().get_model(location_set=location_set,
#location_set=submarkets,
#filter = 'building.disaggregate(submarket.submarket_id) > 0',
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'building.disaggregate(numpy.logical_and(submarket.number_of_buildings_with_non_zero_units > 5000, submarket.number_of_surveyed_households > 0))',
submodel_string=submodel_string,
sampler = sampler,
#estimation_size_agents = agent_sample_rate * 100/20,
# proportion of the agent set that should be used for the estimation
sample_size_locations = alt_sample_size,
#sample_proportion_locations = 1.0/1000,
# choice set size (includes current location)
compute_capacity_flag = True,
probabilities = "opus_core.mnl_probabilities",
choices = "urbansim.lottery_choices",
#run_config = Resources({"capacity_string":"supply"}),
estimate_config = Resources({"capacity_string":"supply",
"weights_for_estimation_string":weight_string,
"aggregate_to_dataset":aggregate_dataset,
"stratum": "building.disaggregate(submarket.submarket_id)",
"sample_size_from_each_stratum": sample_size_from_each_stratum,
#"index2":where(submarkets.compute_variables('submarket.number_of_surveyed_households > 0'))[0],
#"sample_rate": 1.0/5000,
#"sample_size_from_chosen_stratum": 0,
"include_chosen_choice": True
}))
# was dataset_pool.add_datasets_if_not_included({'sample_rate':agent_sample_rate})
self.result = model.estimate(self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops" ) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time()-t1) + " s")
def test_agents_equally_distributed_across_attractive_locations(self):
"""Create 5000 unplaced households and 50 gridcells with equal attractiveness.
Theoretically, after running the location_choice_model, there should be
100 houesholds in each gridcell since they're equally attractive, but due to random
sampling there will be a little deviance. The test also checks, if the aggregated probabilities,
i.e. housing demand, are equally distributed.
"""
nhhs = 5000
household_data = {
"household_id": arange(nhhs) + 1,
"grid_id": array(nhhs * [-1])
}
gridcell_data = {"grid_id": arange(50) + 1, "cost": array(50 * [1000])}
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
def run_model1():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification, coefficients, agent_set=households)
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
return gridcells.get_attribute("number_of_households")
expected_results = array(50 * [nhhs / 50])
self.run_stochastic_test(__file__, run_model1, expected_results, 10)
def run_model2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
run_config=Resources(
{"demand_string": "gridcell.housing_demand"}))
return gridcells.get_attribute("housing_demand")
#check aggregated demand
expected_results = array(50 * [nhhs / 50])
self.run_stochastic_test(__file__, run_model2, expected_results, 5)
def test_agents_go_to_attractive_locations(self):
"""10 gridcells - 5 with cost 100, 5 with cost 1000, no capacity restrictions
100 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
household_data = {
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids
}
gridcell_data = {
'grid_id': arange(ngcs) + 1,
'cost': array(ngcs_attr * [100] + ngcs_noattr * [1000])
}
storage.write_table(table_name='households', table_data=household_data)
storage.write_table(table_name='gridcells', table_data=gridcell_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute="grid_id",
values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result = concatenate(
(result_more_attractive, result_less_attractive))
return result
expected_results = array(ngcs_attr * [nhhs * 0.71 / float(ngcs_attr)] +
ngcs_noattr *
[nhhs * 0.29 / float(ngcs_noattr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
def run_model_2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
return array(
[result_more_attractive.sum(),
result_less_attractive.sum()])
expected_results = array([nhhs * 0.71, nhhs * 0.29])
self.run_stochastic_test(__file__, run_model_2, expected_results, 10)
def test_agents_placed_in_appropriate_types(self):
"""Create 1000 unplaced industrial jobs and 1 commercial job. Allocate 50 commercial
gridcells with enough space for 10 commercial jobs per gridcell. After running the
EmploymentLocationChoiceModel, the 1 commercial job should be placed,
but the 100 industrial jobs should remain unplaced
"""
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='job_building_types',
table_data={
'id': array([2, 1]),
'name': array(['commercial', 'industrial'])
})
job_building_types = JobBuildingTypeDataset(
in_storage=storage, in_table_name='job_building_types')
storage.write_table(table_name='jobs',
table_data={
'job_id': arange(1001) + 1,
'grid_id': array([0] * 1001),
'building_type': array([1] * 1000 + [2])
})
jobs = JobDataset(in_storage=storage, in_table_name='jobs')
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(50) + 1,
'commercial_sqft': array([1000] * 50),
'commercial_sqft_per_job': array([100] * 50)
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
coefficients = Coefficients(names=("dummy", ), values=(0.1, ))
specification = EquationSpecification(
variables=("gridcell.commercial_sqft", ), coefficients=("dummy", ))
compute_resources = Resources({
"job": jobs,
"job_building_type": job_building_types
})
agents_index = where(jobs.get_attribute("grid_id") == 0)
unplace_jobs = DatasetSubset(jobs, agents_index)
agents_index = where(
unplace_jobs.get_attribute("building_type") == 2)[0]
gridcells.compute_variables(
["urbansim.gridcell.number_of_commercial_jobs"],
resources=compute_resources)
commercial_jobs = gridcells.get_attribute("number_of_commercial_jobs")
gridcells.compute_variables(
["urbansim.gridcell.number_of_industrial_jobs"],
resources=compute_resources)
industrial_jobs = gridcells.get_attribute("number_of_industrial_jobs")
model_group = ModelGroup(job_building_types, "name")
elcm = EmploymentLocationChoiceModel(
ModelGroupMember(model_group, "commercial"),
location_set=gridcells,
agents_grouping_attribute="job.building_type",
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
elcm.run(specification,
coefficients,
agent_set=jobs,
agents_index=agents_index,
debuglevel=1)
gridcells.compute_variables(
["urbansim.gridcell.number_of_commercial_jobs"],
resources=compute_resources)
commercial_jobs = gridcells.get_attribute("number_of_commercial_jobs")
gridcells.compute_variables(
["urbansim.gridcell.number_of_industrial_jobs"],
resources=compute_resources)
industrial_jobs = gridcells.get_attribute("number_of_industrial_jobs")
self.assertEqual(
commercial_jobs.sum() == 1, True,
"Error, there should only be a total of 1 commercial job")
self.assertEqual(
industrial_jobs.sum() == 0, True,
"Error, there should be no industrial jobs because there's no space for them"
)
#content_coefficients+='{0:40s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])#'{0:30s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])
content_coefficients+='{0:40s} {1:10s} {2:10s} {3:10s} {4:10s}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])
else:
#content_coefficients+='{0:40s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], sub_ids[i], t_stats[i]) #'{0:30s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], sub_ids[i], t_stats[i])
content_coefficients+='{0:40s} {1:10s} {2:10s} {3:10f} {4:10s}\n'.format(names[i], estimates[i], std_errors[i], sub_ids[i], t_stats[i])
content_coefficients+='\n\r'
# now do the same for the specification ...
model = model.replace('coefficients', 'specification')
content_specification+='\n\r'
content_specification+='Model: %s\n\r' %model
# get model specification ...
specification = EquationSpecification(in_storage=storage)
specification.load(in_table_name=model)
# store specification directly as csv
#from opus_core.store.csv_storage import csv_storage
#out_path = os.path.join(path, model)
#out_storage = csv_storage(storage_location = out_path)
#specification.write(out_storage=out_storage, out_table_name=model) # writes out specifications as csv file
# ... prepare for printing ...
names_spec = specification.get_coefficient_names()
submodels_spec = specification.get_submodels()
long_var_names_spec = specification.get_long_variable_names()
# ... finally print out all available data
content_specification+='{0:40s} {1:10s} {2:20s}\n\r'.format('coefficient_name', 'submodel_id', 'variable_name')
def test_stochastic_test_case(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
storage = StorageFactory().get_storage('dict_storage')
nhouseholds = 10000
ngrids = 10
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(nhouseholds) + 1,
'grid_id': array(nhouseholds * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id':
arange(ngrids) + 1,
'cost':
array(ngrids / 2 * [100] + ngrids / 2 * [1000])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False, \
choices = "opus_core.random_choices_from_index", sampler=None)
#sample_size_locations = 30)
# check the individual gridcells
# This is a stochastic model, so it may legitimately fail occassionally.
success = []
def inner_loop():
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngrids / 2) + 1)
return result_more_attractive
expected_results = array(ngrids / 2 *
[nhouseholds * 0.71 / (ngrids / 2)])
self.run_stochastic_test(__file__,
inner_loop,
expected_results,
10,
type="pearson",
transformation=None)
# Make sure it fails when expected distribution is different from actual.
expected_results = array(ngrids / 2 *
[nhouseholds * 0.61 / (ngrids / 2)])
try:
self.run_stochastic_test(__file__,
inner_loop,
expected_results,
10,
type="poisson",
transformation=None)
except AssertionError:
pass
########
#HLCM
# locations from gridcellset.tab
locations = GridcellDataset(in_storage=StorageFactory().get_storage(
'tab_storage', storage_location="."),
in_table_name="gridcellset",
id_name="location")
locations.summary()
seed(1)
coefficients = Coefficients(names=("costcoef", ), values=(-0.01, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=locations,
sampler=None,
utilities="opus_core.linear_utilities",
probabilities="opus_core.mnl_probabilities",
choices="opus_core.random_choices_from_index",
compute_capacity_flag=False)
agents.get_attribute("location")
results = hlcm.run(specification, coefficients, agents)
agents.get_attribute("location")
hlcm.upc_sequence.plot_choice_histograms(
capacity=locations.get_attribute("vacant_units"))
hlcm.upc_sequence.show_plots()
def test_place_agents_to_correct_areas(self):
"""10 gridcells - 5 in area 1, 5 in area 2, with equal cost, no capacity restrictions
100 households - 70 live in area 1, 30 live in area 2.
We set the coefficient value for cost -0.001.
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs/2
hh_grid_ids = array(nhhs*[-1])
lareas = array(ngcs_attr*[1] + ngcs_attr*[2])
hh_lareas = array(70*[1] + 30*[2])
household_data = {
'household_id': arange(nhhs)+1,
'grid_id': hh_grid_ids,
'faz_id': hh_lareas
}
gridcell_data = {
'grid_id': arange(ngcs)+1,
'cost':array(ngcs*[100]),
'faz_id': lareas
}
storage.write_table(table_name = 'households', table_data = household_data)
storage.write_table(table_name = 'gridcells', table_data = gridcell_data)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
gridcells = GridcellDataset(in_storage=storage, in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001,))
specification = EquationSpecification(variables=("gridcell.cost", ), coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
households = HouseholdDataset(in_storage=storage, in_table_name='households')
hlcm = SubareaHouseholdLocationChoiceModel(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index", sample_size_locations = 4, subarea_id_name="faz_id")
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1)
result_area2 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1))
gridcells.delete_one_attribute("number_of_households")
result = concatenate((result_area1, result_area2))
return result
expected_results = array(ngcs_attr*[nhhs*0.7/float(ngcs_attr)] + ngcs_attr*[nhhs*0.3/float(ngcs_attr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
# check the exact sum
hlcm = SubareaHouseholdLocationChoiceModel(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index", sample_size_locations = 4, subarea_id_name="faz_id")
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1)
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1).sum()
result_area2 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1)).sum()
results = array([result_area1, result_area2])
expected_results = array([70, 30])
self.assertEqual(ma.allequal(expected_results, results), True, "Error, should_be: %s, but result: %s" % (
expected_results, results))
