def run(self, base_directory, urbansim_cache_directory, years):
""" run the simulation
base_directory: directory contains all years folder of lccm.
urbansim_cache_directory: directory contains all years folder of urbansim cache.
years: lists of year to run."""
model = LandCoverChangeModel(self.possible_lcts,
submodel_string=self.lct_attribute,
choice_attribute_name=self.lct_attribute,
debuglevel=4)
coefficients = Coefficients()
storage = StorageFactory().get_storage('tab_storage',
storage_location=os.path.join(
self.package_path, 'data'))
coefficients.load(in_storage=storage,
in_table_name="land_cover_change_model_coefficients")
specification = EquationSpecification(in_storage=storage)
specification.load(
in_table_name="land_cover_change_model_specification")
specification.set_variable_prefix("biocomplexity.land_cover.")
constants = Constants()
simulation_state = SimulationState()
simulation_state.set_cache_directory(urbansim_cache_directory)
attribute_cache = AttributeCache()
index = arange(100000)
for year in years:
simulation_state.set_current_time(year)
#land_cover_path = os.path.join(base_directory, str(year))
land_cover_path = base_directory
land_covers = LandCoverDataset(
in_storage=StorageFactory().get_storage(
'flt_storage', storage_location=land_cover_path),
out_storage=StorageFactory().get_storage(
'flt_storage', storage_location=land_cover_path),
debuglevel=4)
land_covers.subset_by_index(index)
#land_covers.load_dataset()
gridcells = GridcellDataset(in_storage=attribute_cache,
debuglevel=4)
agents_index = None
model.run(specification,
coefficients,
land_covers,
data_objects={
"gridcell": gridcells,
"constants": constants,
"flush_variables": True
},
chunk_specification={'nchunks': 1})
land_covers.flush_dataset()
del gridcells
del land_covers
def test_residential_land_share_model(self):
storage = StorageFactory().get_storage('dict_storage')
gridcell_set_table_name = 'gridcell_set'
storage.write_table(table_name=gridcell_set_table_name,
table_data={
"residential_units": array([1000, 0, 10, 500]),
"development_type_id": array([8, 17, 4, 8]),
"grid_id": array([1, 2, 3, 4])
})
gridcell_set = GridcellDataset(in_storage=storage,
in_table_name=gridcell_set_table_name)
specification = EquationSpecification(
variables=("urbansim.gridcell.devtype_8",
"gridcell.residential_units", "constant"),
coefficients=("DEV8", "units", "constant"))
coefficients = Coefficients(names=("constant", "DEV8", "units"),
values=(-0.4, 1.6, 0.01))
ResidentialLandShareModel(debuglevel=3).run(specification,
coefficients, gridcell_set)
result = gridcell_set.get_attribute("fraction_residential_land")
self.assertEqual(
ma.allclose(result,
array([0.9999863, 0.4013123, 0.4255575, 0.9979747]),
rtol=1e-3), True)
def test_unplaced_agents_decrease_available_space(self):
"""Using the household location choice model, create a set of available spaces and
2000 unplaced agents (along with 5000 placed agents). Run the model, and check that
the unplaced agents were placed, and the number of available spaces has decreased"""
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data={
'grid_id': array(2000 * [0] + 5000 * [1]),
'household_id': arange(7000) + 1
})
storage.write_table(table_name='gridcells',
table_data={
'residential_units': array(50 * [10000]),
'grid_id': arange(50) + 1
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
coefficients = Coefficients(names=("dummy", ), values=(0.1, ))
specification = EquationSpecification(
variables=("gridcell.residential_units", ),
coefficients=("dummy", ))
"""need to specify to the household location choice model exactly which households are moving,
because by default it assumes all current households want to move, but in this test,
the 5000 households already in gridcell #1 shouldn't move.
here, we specify that only the unplaced households should be moved."""
agents_index = where(households.get_attribute("grid_id") == 0)[0]
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
agents_index=agents_index,
debuglevel=1)
gridcells.compute_variables(
["urbansim.gridcell.vacant_residential_units"],
resources=Resources({"household": households}))
vacancies = gridcells.get_attribute("vacant_residential_units")
"""since there were 5000 households already in gridcell #1, and gridcell #1 has
10000 residential units, there should be no more than 5000 vacant residential units
in gridcell #1 after running this model"""
self.assertEqual(vacancies[0] <= 5000, True,
"Error: %d" % (vacancies[0], ))
"""there should be exactly 430000 vacant residential units after the model run,
because there were originally 50 gridcells with 10000 residential units each,
and a total of 7000 units are occupied after the run"""
self.assertEqual(
sum(vacancies) == 50 * 10000 - 7000, True,
"Error: %d" % (sum(vacancies)))
def prepare_specification_and_coefficients(specification_storage=None,
specification_table=None,
coefficients_storage=None,
coefficients_table=None,
sample_coefficients=False,
cache_storage=None,
**kwargs):
""" Load specification and coefficients from given tables in given storages.
If 'sample_coefficients' is True, coefficients are sampled from given distribution. In such a case,
either an argument 'distribution' should be given (equal either 'normal' or 'uniform'),
or argument distribution_dictionary should be given containing details about sampling specific coefficients
(see docstring for method sample_values in opus_core/coefficients.py).
If coefficients are sampled, the new values are flushed into cache.
"""
specification = None
if specification_storage is not None and specification_table is not None:
specification = EquationSpecification(in_storage=specification_storage)
specification.load(in_table_name=specification_table)
coefficients = None
if (coefficients_storage is not None) and (coefficients_table is not None):
coefficients = Coefficients(in_storage=coefficients_storage)
coefficients.load(in_table_name=coefficients_table)
if sample_coefficients:
coefficients = coefficients.sample_values(**kwargs)
coefficients.flush_coefficients(coefficients_table, cache_storage)
return (specification, coefficients)
def prepare_specification_and_coefficients(specification_storage=None,
specification_table=None,
coefficients_storage=None,
coefficients_table=None,
sample_coefficients=False,
cache_storage=None,
**kwargs):
""" Load specification and coefficients from given tables in given storages.
If 'sample_coefficients' is True, coefficients are sampled from given distribution. In such a case,
either an argument 'distribution' should be given (equal either 'normal' or 'uniform'),
or argument distribution_dictionary should be given containing details about sampling specific coefficients
(see docstring for method sample_values in opus_core/coefficients.py).
If coefficients are sampled, the new values are flushed into cache.
"""
specification = None
if specification_storage is not None and specification_table is not None:
specification = EquationSpecification(in_storage=specification_storage)
specification.load(in_table_name=specification_table)
coefficients = None
if (coefficients_storage is not None) and (coefficients_table is not None):
coefficients = Coefficients(in_storage=coefficients_storage)
coefficients.load(in_table_name=coefficients_table)
if sample_coefficients:
coefficients = coefficients.sample_values(**kwargs)
coefficients.flush_coefficients(coefficients_table, cache_storage)
return (specification, coefficients)
def run(self, base_directory, urbansim_cache_directory, years):
""" run the simulation
base_directory: directory contains all years folder of lccm.
urbansim_cache_directory: directory contains all years folder of urbansim cache.
years: lists of year to run."""
model = LandCoverChangeModel(self.possible_lcts, submodel_string=self.lct_attribute,
choice_attribute_name= self.lct_attribute, debuglevel=4)
coefficients = Coefficients()
storage = StorageFactory().get_storage('tab_storage',
storage_location=os.path.join(self.package_path, 'data'))
coefficients.load(in_storage=storage, in_table_name="land_cover_change_model_coefficients")
specification = EquationSpecification(in_storage=storage)
specification.load(in_table_name="land_cover_change_model_specification")
specification.set_variable_prefix("biocomplexity.land_cover.")
constants = Constants()
simulation_state = SimulationState()
simulation_state.set_cache_directory(urbansim_cache_directory)
attribute_cache = AttributeCache()
index = arange(100000)
for year in years:
simulation_state.set_current_time(year)
#land_cover_path = os.path.join(base_directory, str(year))
land_cover_path = base_directory
land_covers = LandCoverDataset(in_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
out_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
debuglevel=4)
land_covers.subset_by_index(index)
#land_covers.load_dataset()
gridcells = GridcellDataset(in_storage=attribute_cache, debuglevel=4)
agents_index = None
model.run(specification, coefficients, land_covers, data_objects={"gridcell":gridcells,
"constants":constants, "flush_variables":True},
chunk_specification = {'nchunks':1}
)
land_covers.flush_dataset()
del gridcells
del land_covers
def test_agents_do_not_go_to_inferior_locations(self):
"""100 gridcells - 99 with attractiveness 2000, 1 with attractiveness 1, no capacity restrictions
10,000 households
We set the coefficient value for attracitiveness to 1.
"""
storage = StorageFactory().get_storage('dict_storage')
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(10000) + 1,
'grid_id': array(10000 * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(100) + 1,
'attractiveness': array(99 * [2000] + [1])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("attractcoef", ), values=(1, ))
specification = EquationSpecification(
variables=("gridcell.attractiveness", ),
coefficients=("attractcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute_by_id("number_of_households", 100)
# nobody should choose gridcell 100
self.assertEqual(result, 0, "Error: %s is not equal to 0" % (result, ))
def xtest_gracefully_handle_empty_choice_sets(self):
storage = StorageFactory().get_storage('dict_storage')
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(10000) + 1,
'grid_id': array(100 * range(100)) + 1
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(100) + 1,
'residential_units': array(100 * [100])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("dummy", ), values=(0, ))
specification = EquationSpecification(
variables=("gridcell.residential_units", ),
coefficients=("dummy", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute_by_id("number_of_households", 100)
# nobody should choose gridcell 100
self.assertEqual(ma.allclose(result.sum(), 0, rtol=0), True,
"Error: %s is not equal to 0" % (result.sum(), ))
def test_do_nothing_if_no_agents(self):
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(table_name=households_table_name,
table_data={
"household_id": arange(10000) + 1,
"grid_id": array(10000 * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
gridcells_table_name = 'gridcells'
storage.write_table(table_name=gridcells_table_name,
table_data={
"grid_id": arange(100) + 1,
"cost": array(50 * [100] + 50 * [1000])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
agents_index=array([], dtype='int32'),
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute("number_of_households")
# check the individual gridcells
self.assertEqual(ma.allclose(result, zeros((100, )), rtol=0), True)
def test_do_nothing_if_empty_set(self):
storage = StorageFactory().get_storage('dict_storage')
gridcell_set_table_name = 'gridcell_set'
storage.write_table(table_name=gridcell_set_table_name,
table_data={"grid_id": array([], dtype='int32')})
gridcell_set = GridcellDataset(in_storage=storage,
in_table_name=gridcell_set_table_name)
specification = EquationSpecification(
variables=("percent_residential_within_walking_distance",
"gridcell_year_built", "constant"),
coefficients=("PRWWD", "YB", "constant"))
coefficients = Coefficients(names=("constant", "PRWWD", "YB"),
values=(10.0, -0.0025, 0.0001))
lp = HousingPriceModel(debuglevel=4)
lp.filter_attribute = None
result = lp.run(specification, coefficients, gridcell_set)
self.assertEqual(result.size, 0)
def test_correct_land_value(self):
#TODO: need to remove this when fixed LP correction only working when year >= 2002
from opus_core.simulation_state import SimulationState
SimulationState().set_current_time(2002)
storage = StorageFactory().get_storage('dict_storage')
gridcell_set_table_name = 'gridcell_set'
storage.write_table(
table_name=gridcell_set_table_name,
table_data={
"percent_residential_within_walking_distance":array([30, 0, 90, 100]),
"gridcell_year_built":array([2002, 1968, 1880, 1921]),
"fraction_residential_land":array([0.5, 0.1, 0.3, 0.9]),
"residential_land_value":array([0, 0, 0, 0]),
"residential_land_value_lag1":array([15000, 0, 7500, 0]),
"nonresidential_land_value":array([0, 0, 0, 0]),
"nonresidential_land_value_lag1":array([15000, 0, 17500, 0]),
"development_type_id":array( [2, 1, 1, 1]),
"development_type_id_lag2":array( [1, 1, 1, 1]),
"grid_id": array([1,2,3,4])
}
)
gridcell_set = GridcellDataset(in_storage=storage, in_table_name=gridcell_set_table_name)
specification = EquationSpecification(variables=(
"percent_residential_within_walking_distance",
"gridcell_year_built", "constant"),
coefficients=("PRWWD", "YB", "constant"))
coefficients = Coefficients(names=("constant", "PRWWD", "YB"), values=(10.0, -0.0025, 0.0001))
lp = LandPriceModel(filter=None, debuglevel=3)
lp.run(specification, coefficients, gridcell_set)
correctmodel = CorrectLandValue()
correctmodel.run(gridcell_set)
result1 = gridcell_set.get_attribute("residential_land_value")
result2 = gridcell_set.get_attribute("nonresidential_land_value")
self.assertEqual(ma.allclose(result1, array([15000.0, 2681.723, 6367.914, 18708.617]), rtol=1e-3), True)
self.assertEqual(ma.allclose(result2, array([15000.0, 24135.510, 14858.466, 2078.735]), rtol=1e-3), True)
def test_housing_price_model(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='gridcells',
table_data={
'percent_residential_within_walking_distance':
array([30, 0, 90, 100]),
'gridcell_year_built':
array([2002, 1968, 1880, 1921]),
'housing_price':
array([0, 0, 0, 0]).astype(float32),
'development_type_id':
array([1, 1, 1, 1]),
'grid_id':
array([1, 2, 3, 4])
})
gridcell_set = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
specification = EquationSpecification(
variables=('percent_residential_within_walking_distance',
'gridcell_year_built', 'constant'),
coefficients=('PRWWD', 'YB', 'constant'))
coefficients = Coefficients(names=("constant", "PRWWD", "YB"),
values=(10.0, -0.25, 0.1))
lp = HousingPriceModel(debuglevel=3)
lp.filter_attribute = None
lp.run(specification, coefficients, gridcell_set)
result = gridcell_set.get_attribute("housing_price")
self.assertEqual(
ma.allclose(result, array([202.7, 206.8, 175.5, 177.1]),
rtol=1e-3), True)
def test_agents_go_to_attractive_locations(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
nhhs = 1000
ngcs = 50
ngcs_attr = ngcs/2
ngcs_noattr = ngcs - ngcs_attr
#print ngcs_attr, ngcs_noattr
hh_grid_ids = array([-1]*nhhs)
household_data = {"household_id": arange(nhhs)+1, "grid_id": hh_grid_ids}
gridcell_data = {"grid_id": arange(ngcs)+1, "cost":array(ngcs_attr*[100]+ngcs_noattr*[1000])}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'households', table_data = household_data)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
storage.write_table(table_name = 'gridcells', table_data = gridcell_data)
gridcells = HouseholdDataset(in_storage=storage, in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001,))
specification = EquationSpecification(variables=("gridcell.cost", ), coefficients=("costcoef", ))
logger.be_quiet()
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index",
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1,
chunk_specification={'nchunks':1})
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_more_attractive = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1)
result_less_attractive = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1))
households.set_values_of_one_attribute(attribute="grid_id", values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result_less_attractive[0]=result_less_attractive[0] + self.wrong_number
result = concatenate((result_more_attractive, result_less_attractive))
#print standard_deviation(result[ngcs_attr:(ngcs-1)])
return result
expected_results = array(ngcs_attr*[nhhs*0.71/float(ngcs_attr)] + ngcs_noattr*[nhhs*0.29/float(ngcs_noattr)])
#print expected_results
R = 1000
#r = [2, 5, 10, 50, 100, 1000]
r = [2, 5, 10, 15, 20]
#r=[20]
levels = [0.05, 0.01]
#levels = [0.01]
#wrong_numbers = [8, 10, 12, 14, 16, 18, 21, 23]
#wrong_numbers = [2, 4, 6, 8, 10, 12, 14, 16]
#wrong_numbers = [3, 6, 9, 12, 15, 18]
wrong_numbers = [12, 15, 18]
for wn in wrong_numbers:
self.wrong_number = wn
print "Wrong number =", self.wrong_number
power = zeros((len(r), len(levels)))
for ir in range(len(r)):
for il in range(len(levels)):
print "r =", r[ir],", level =",levels[il]
seed(1)
for iR in range(R):
try:
self.run_stochastic_test(__file__, run_model, expected_results,
r[ir], significance_level=levels[il])
except:
power[ir,il]=power[ir,il]+1
print "Power:",power[ir,il]/float(R)
print power/float(R)
from opus_core.variables.variable_name import VariableName
from opus_core.variables.attribute_type import AttributeType
submodel_string_short_name = VariableName(submodel_string).get_short_name()
if submodel_string_short_name in agent_set.get_known_attribute_names():
agent_set.add_attribute(agent_set.get_attribute(submodel_string_short_name), "submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients, specification, neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants()
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc','urbansim','opus_core'],
storage=AttributeCache())
choice_set = dataset_pool.get_dataset(choice_set_name)
if choice_filter_index is None:
def test_agents_go_to_attractive_locations(self):
"""10 gridcells - 5 with cost 100, 5 with cost 1000, no capacity restrictions
100 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
household_data = {
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids
}
gridcell_data = {
'grid_id': arange(ngcs) + 1,
'cost': array(ngcs_attr * [100] + ngcs_noattr * [1000])
}
storage.write_table(table_name='households', table_data=household_data)
storage.write_table(table_name='gridcells', table_data=gridcell_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute="grid_id",
values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result = concatenate(
(result_more_attractive, result_less_attractive))
return result
expected_results = array(ngcs_attr * [nhhs * 0.71 / float(ngcs_attr)] +
ngcs_noattr *
[nhhs * 0.29 / float(ngcs_noattr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
def run_model_2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
return array(
[result_more_attractive.sum(),
result_less_attractive.sum()])
expected_results = array([nhhs * 0.71, nhhs * 0.29])
self.run_stochastic_test(__file__, run_model_2, expected_results, 10)
def test_agents_equally_distributed_across_attractive_locations(self):
"""Create 5000 unplaced households and 50 gridcells with equal attractiveness.
Theoretically, after running the location_choice_model, there should be
100 houesholds in each gridcell since they're equally attractive, but due to random
sampling there will be a little deviance. The test also checks, if the aggregated probabilities,
i.e. housing demand, are equally distributed.
"""
nhhs = 5000
household_data = {
"household_id": arange(nhhs) + 1,
"grid_id": array(nhhs * [-1])
}
gridcell_data = {"grid_id": arange(50) + 1, "cost": array(50 * [1000])}
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
def run_model1():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification, coefficients, agent_set=households)
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
return gridcells.get_attribute("number_of_households")
expected_results = array(50 * [nhhs / 50])
self.run_stochastic_test(__file__, run_model1, expected_results, 10)
def run_model2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
run_config=Resources(
{"demand_string": "gridcell.housing_demand"}))
return gridcells.get_attribute("housing_demand")
#check aggregated demand
expected_results = array(50 * [nhhs / 50])
self.run_stochastic_test(__file__, run_model2, expected_results, 5)
def test_agents_placed_in_appropriate_types(self):
"""Create 1000 unplaced industrial jobs and 1 commercial job. Allocate 50 commercial
gridcells with enough space for 10 commercial jobs per gridcell. After running the
EmploymentLocationChoiceModel, the 1 commercial job should be placed,
but the 100 industrial jobs should remain unplaced
"""
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='job_building_types',
table_data={
'id': array([2, 1]),
'name': array(['commercial', 'industrial'])
})
job_building_types = JobBuildingTypeDataset(
in_storage=storage, in_table_name='job_building_types')
storage.write_table(table_name='jobs',
table_data={
'job_id': arange(1001) + 1,
'grid_id': array([0] * 1001),
'building_type': array([1] * 1000 + [2])
})
jobs = JobDataset(in_storage=storage, in_table_name='jobs')
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(50) + 1,
'commercial_sqft': array([1000] * 50),
'commercial_sqft_per_job': array([100] * 50)
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
coefficients = Coefficients(names=("dummy", ), values=(0.1, ))
specification = EquationSpecification(
variables=("gridcell.commercial_sqft", ), coefficients=("dummy", ))
compute_resources = Resources({
"job": jobs,
"job_building_type": job_building_types
})
agents_index = where(jobs.get_attribute("grid_id") == 0)
unplace_jobs = DatasetSubset(jobs, agents_index)
agents_index = where(
unplace_jobs.get_attribute("building_type") == 2)[0]
gridcells.compute_variables(
["urbansim.gridcell.number_of_commercial_jobs"],
resources=compute_resources)
commercial_jobs = gridcells.get_attribute("number_of_commercial_jobs")
gridcells.compute_variables(
["urbansim.gridcell.number_of_industrial_jobs"],
resources=compute_resources)
industrial_jobs = gridcells.get_attribute("number_of_industrial_jobs")
model_group = ModelGroup(job_building_types, "name")
elcm = EmploymentLocationChoiceModel(
ModelGroupMember(model_group, "commercial"),
location_set=gridcells,
agents_grouping_attribute="job.building_type",
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
elcm.run(specification,
coefficients,
agent_set=jobs,
agents_index=agents_index,
debuglevel=1)
gridcells.compute_variables(
["urbansim.gridcell.number_of_commercial_jobs"],
resources=compute_resources)
commercial_jobs = gridcells.get_attribute("number_of_commercial_jobs")
gridcells.compute_variables(
["urbansim.gridcell.number_of_industrial_jobs"],
resources=compute_resources)
industrial_jobs = gridcells.get_attribute("number_of_industrial_jobs")
self.assertEqual(
commercial_jobs.sum() == 1, True,
"Error, there should only be a total of 1 commercial job")
self.assertEqual(
industrial_jobs.sum() == 0, True,
"Error, there should be no industrial jobs because there's no space for them"
)
def run_ALCM(niter):
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(table_name=households_table_name,
table_data={
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids
})
gridcells_table_name = 'gridcells'
storage.write_table(table_name=gridcells_table_name,
table_data={
'grid_id': arange(ngcs) + 1,
'cost':
array(ngcs_attr * [100] + ngcs_noattr * [1000])
})
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
gridcells = GridcellDataset(in_storage=storage,
in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=('costcoef', ), values=(-0.001, ))
specification = EquationSpecification(variables=('gridcell.cost', ),
coefficients=('costcoef', ))
logger.be_quiet()
result = zeros((niter, ngcs))
for iter in range(niter):
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices='opus_core.random_choices_from_index',
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(['urbansim.gridcell.number_of_households'],
resources=Resources(
{'household': households}))
result_more_attractive = gridcells.get_attribute_by_id(
'number_of_households',
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
'number_of_households', arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute='grid_id',
values=hh_grid_ids)
gridcells.delete_one_attribute('number_of_households')
result[iter, :] = concatenate(
(result_more_attractive, result_less_attractive))
#print result #, result_more_attractive.sum(), result_less_attractive.sum()
return result
households, procedure="opus_core.bhhh_mnl_estimation")
#
# Uncomment to output mycoef.tab
#
#coefficients.write(out_storage=storage, out_table_name="mycoef")
from numpy.random import seed
seed(1)
choices = choicemodel.run(
specification, coefficients, households, debuglevel=1)
households.modify_attribute(name="choice_id", data=choices)
from opus_core.coefficients import Coefficients
coefficients = Coefficients(
names=array(["beta01", "beta12", "beta03", "beta13"]),
values=array([0.5, 0.2, -5.0, 1.3]))
# LCM
locations = Dataset(in_storage = storage,
in_table_name = 'locations',
id_name='location',
dataset_name='gridcell')
coefficients = Coefficients(names=("costcoef", ), values=(-0.01,))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
from urbansim.models.household_location_choice_model import HouseholdLocationChoiceModel
hlcm = HouseholdLocationChoiceModel(
def run(self, base_directory, urbansim_cache_directory, years, output_directory, temp_folder,
coefficients_name, specification_name, convert_flt=True, convert_input=False):
""" run the simulation
base_directory: directory contains all years folder of lccm.
urbansim_cache_directory: directory contains all years folder of urbansim cache.
years: lists of year to run."""
model = LandCoverChangeModel(self.possible_lcts, submodel_string=self.lct_attribute,
choice_attribute_name=self.lct_attribute, debuglevel=4)
coefficients = Coefficients()
storage = StorageFactory().get_storage('tab_storage',
storage_location=os.path.join(self.package_path, 'data'))
coefficients.load(in_storage=storage, in_table_name=coefficients_name)
specification = EquationSpecification(in_storage=storage)
specification.load(in_table_name=specification_name)
specification.set_variable_prefix("biocomplexity.land_cover.")
constants = Constants()
simulation_state = SimulationState()
simulation_state.set_cache_directory(urbansim_cache_directory)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=['biocomplexity', 'urbansim', 'opus_core'],
in_storage=AttributeCache())
ncols = LccmConfiguration.ncols
if temp_folder is None:
self.temp_land_cover_dir = tempfile.mkdtemp()
else:
self.temp_land_cover_dir = temp_folder
for year in years:
land_cover_path = self._generate_input_land_cover(year, base_directory, urbansim_cache_directory,
years, output_directory, convert_flt, convert_input)
#max_size = 174338406 (orig) - act. int: 19019944 (37632028 incl NoData)
max_size = self._get_max_index(land_cover_path) # 1st instance of lc_dataset - but looks like a 'lite' version
offset = min(LccmConfiguration.offset, max_size)
s = 0
t = offset
while (s < t and t <= max_size):
logger.log_status("Offset: ", s, t)
index = arange(s,t)
land_cover_cache_path=os.path.join(urbansim_cache_directory,str(year),'land_covers')
self._clean_up_land_cover_cache(land_cover_cache_path)
simulation_state.set_current_time(year)
# 2nd instance of lc_dataset
land_covers = LandCoverDataset(in_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
out_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
debuglevel=4)
land_covers.subset_by_index(index)
# land_covers.load_dataset()
gridcells = GridcellDataset(in_storage=attribute_cache, debuglevel=4)
agents_index = None
model.run(specification, coefficients, land_covers, data_objects={"gridcell":gridcells,
"constants":constants, "flush_variables":True},
chunk_specification = {'nchunks':5}) ## chunk size set here
land_covers.flush_dataset()
del gridcells
del land_covers
# self._generate_output_flt(year, urbansim_cache_directory, output_directory, convert_flt)
self._generate_output_flt2(year, urbansim_cache_directory, output_directory, convert_flt)
if t >= max_size: break
s = max(t-10*ncols,s)
t = min(t+offset-10*ncols,max_size)
# clean up temp storage after done simulation
shutil.rmtree(self.temp_land_cover_dir)
def test_place_agents_to_correct_areas(self):
"""10 gridcells - 5 in area 1, 5 in area 2, with equal cost, no capacity restrictions
100 households - 70 live in area 1, 30 live in area 2.
We set the coefficient value for cost -0.001.
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs/2
hh_grid_ids = array(nhhs*[-1])
lareas = array(ngcs_attr*[1] + ngcs_attr*[2])
hh_lareas = array(70*[1] + 30*[2])
household_data = {
'household_id': arange(nhhs)+1,
'grid_id': hh_grid_ids,
'faz_id': hh_lareas
}
gridcell_data = {
'grid_id': arange(ngcs)+1,
'cost':array(ngcs*[100]),
'faz_id': lareas
}
storage.write_table(table_name = 'households', table_data = household_data)
storage.write_table(table_name = 'gridcells', table_data = gridcell_data)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
gridcells = GridcellDataset(in_storage=storage, in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001,))
specification = EquationSpecification(variables=("gridcell.cost", ), coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
households = HouseholdDataset(in_storage=storage, in_table_name='households')
hlcm = SubareaHouseholdLocationChoiceModel(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index", sample_size_locations = 4, subarea_id_name="faz_id")
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1)
result_area2 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1))
gridcells.delete_one_attribute("number_of_households")
result = concatenate((result_area1, result_area2))
return result
expected_results = array(ngcs_attr*[nhhs*0.7/float(ngcs_attr)] + ngcs_attr*[nhhs*0.3/float(ngcs_attr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
# check the exact sum
hlcm = SubareaHouseholdLocationChoiceModel(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index", sample_size_locations = 4, subarea_id_name="faz_id")
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1)
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1).sum()
result_area2 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1)).sum()
results = array([result_area1, result_area2])
expected_results = array([70, 30])
self.assertEqual(ma.allequal(expected_results, results), True, "Error, should_be: %s, but result: %s" % (
expected_results, results))
# Models
########
#HLCM
# locations from gridcellset.tab
locations = GridcellDataset(in_storage=StorageFactory().get_storage(
'tab_storage', storage_location="."),
in_table_name="gridcellset",
id_name="location")
locations.summary()
seed(1)
coefficients = Coefficients(names=("costcoef", ), values=(-0.01, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=locations,
sampler=None,
utilities="opus_core.linear_utilities",
probabilities="opus_core.mnl_probabilities",
choices="opus_core.random_choices_from_index",
compute_capacity_flag=False)
agents.get_attribute("location")
results = hlcm.run(specification, coefficients, agents)
agents.get_attribute("location")
def test_stochastic_test_case(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
storage = StorageFactory().get_storage('dict_storage')
nhouseholds = 10000
ngrids = 10
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(nhouseholds) + 1,
'grid_id': array(nhouseholds * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id':
arange(ngrids) + 1,
'cost':
array(ngrids / 2 * [100] + ngrids / 2 * [1000])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False, \
choices = "opus_core.random_choices_from_index", sampler=None)
#sample_size_locations = 30)
# check the individual gridcells
# This is a stochastic model, so it may legitimately fail occassionally.
success = []
def inner_loop():
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngrids / 2) + 1)
return result_more_attractive
expected_results = array(ngrids / 2 *
[nhouseholds * 0.71 / (ngrids / 2)])
self.run_stochastic_test(__file__,
inner_loop,
expected_results,
10,
type="pearson",
transformation=None)
# Make sure it fails when expected distribution is different from actual.
expected_results = array(ngrids / 2 *
[nhouseholds * 0.61 / (ngrids / 2)])
try:
self.run_stochastic_test(__file__,
inner_loop,
expected_results,
10,
type="poisson",
transformation=None)
except AssertionError:
pass
# coefficients of following models are printed out
models = ['real_estate_price_model_coefficients', 'household_location_choice_model_coefficients', 'non_home_based_employment_location_choice_model_coefficients', 'home_based_employment_location_choice_model_coefficients', 'work_at_home_choice_model_coefficients', 'workplace_choice_model_for_resident_coefficients' ]
content_coefficients = '' # contains whole content for coefficients
content_specification = '' # contains whole content for specifications
# go trough all models ...
for m in range(len(models)):
model = models[m]
content_coefficients+='\n\r'
content_coefficients+='Model: %s\n\r' %model
# ... get their coefficients ...
coefficients = Coefficients(in_storage=storage)
coefficients.load(resources=None, in_storage=storage, in_table_name=model)
# ... prepare for printing ...
names = coefficients.names
estimates = coefficients.values
std_errors = coefficients.standard_errors
sub_ids = coefficients.submodels
t_stats = coefficients.other_measures['t_statistic']
# ... finally print out all available data
content_coefficients+='{0:40s} {1:10s} {2:10s} {3:10s} {4:10s}\n\r'.format('coefficient_name', 'estimate', 'error', 'submodel_id', 't_statistic')
for i in range(len(names)):
if len(sub_ids) <= 0:
#content_coefficients+='{0:40s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])#'{0:30s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])
content_coefficients+='{0:40s} {1:10s} {2:10s} {3:10s} {4:10s}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])
locations_psrc.compute_variables("urbansim.gridcell.ln_total_land_value")
locations_psrc.plot_map("ln_total_land_value")
# Models
########
#HLCM
# locations from gridcellset.tab
locations= GridcellDataset(in_storage = StorageFactory().get_storage('tab_storage', storage_location = "."),
in_table_name = "gridcellset", id_name="location")
locations.summary()
seed(1)
coefficients = Coefficients(names=("costcoef", ), values=(-0.01,))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set = locations,
sampler=None,
utilities="opus_core.linear_utilities",
probabilities="opus_core.mnl_probabilities",
choices="opus_core.random_choices_from_index",
compute_capacity_flag=False)
agents.get_attribute("location")
results = hlcm.run(specification, coefficients, agents)
agents.get_attribute("location")
submodel_string_short_name = VariableName(submodel_string).get_short_name()
if submodel_string_short_name in agent_set.get_known_attribute_names():
agent_set.add_attribute(
agent_set.get_attribute(submodel_string_short_name),
"submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients,
specification,
neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants(
)
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc', 'urbansim', 'opus_core'],
storage=AttributeCache())