def test_demolish_buildings_on_a_parcel(self):
"""test demolish buildings, create new buildings, and convert an existing building
"""
scheduled_events_data = {
"year":
array([2000, 2000, 2000, 2000, 2000]),
"action":
array(["remove", "add", "set_value", "set_value", "set_value"]),
"amount":
array([4, 2, 8, 7, 150]),
"attribute":
array([
"", "", "residential_units", "non_residential_sqft",
"price_per_unit"
]),
"building_id":
array([3, -1, 5, 5, 5]),
"parcel_id":
array([-1, 1, -1, -1, -1]),
"residential_units":
array([-1, 2, -1, -1, -1]),
"non_residential_sqft":
array([-1, 1, -1, -1, -1]),
"price_per_unit":
array([-1, 99, -1, -1, -1]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='events',
table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage,
in_table_name='events',
id_name=[])
model = ScheduledEventsModel(self.buildings,
scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
results = self.buildings.size()
should_be = 9
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
index = self.buildings.get_attribute("building_id") > 8
results = array([
self.buildings.get_attribute("parcel_id")[index],
self.buildings.get_attribute("residential_units")[index],
self.buildings.get_attribute("non_residential_sqft")[index],
self.buildings.get_attribute("price_per_unit")[index]
])
should_be = array([[1, 1], [2, 2], [1, 1], [99, 99]])
self.assertTrue(
allclose(should_be, results),
"Error, should_be: %s, but result: %s" % (should_be, results))
index = where(self.buildings.get_attribute("building_id") == 5)
results = self.buildings.get_multiple_attributes([
"parcel_id", "residential_units", "non_residential_sqft",
"price_per_unit"
])[index]
should_be = array([4, 8, 7, 150])
self.assertTrue(
allclose(should_be, results),
"Error, should_be: %s, but result: %s" % (should_be, results))
def DELtest_add_and_remove_agents_from_geography_other_than_location_set(
self):
"""this has been included in the above test
"""
scheduled_events_data = {
"year": array([2000, 2000, 2000, 2000, 2000]),
"action": array(["remove", "remove", "add", "add", "target"]),
"amount": array([1, 1, 4, 3, 7]),
"sector_id": array([13, 13, -1, 11, 12]),
"building_id": array([-1, -1, -1, 8, -1]),
"raz_id": array([3, 4, 5, -1, -1]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='events',
table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage,
in_table_name='events',
id_name=[])
model = ScheduledEventsModel(self.jobs,
scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
#check that there are indeed 50000 total households after running the model
results = self.jobs.size()
should_be = 17
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
#examine each action in turn:
results = logical_and(
self.jobs.get_attribute("sector_id") == 13,
self.jobs.get_attribute("raz_id") == 3).sum()
should_be = 2 - 1
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
results = logical_and(
self.jobs.get_attribute("sector_id") == 13,
self.jobs.get_attribute("raz_id") == 4).sum()
should_be = 1 - 1
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
results = (self.jobs.get_attribute("raz_id") == 5).sum()
should_be = 2 + 4
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
results = logical_and(
self.jobs.get_attribute("sector_id") == 11,
self.jobs.get_attribute("building_id") == 8).sum()
should_be = 0 + 3
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
results = (self.jobs.get_attribute("sector_id") == 12).sum()
should_be = 7
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
def test_modify_dataset_attribute(self):
"""
"""
scheduled_events_data = {
"year":
array([2000, 2000, 2000, 2000, 2001, 2001]),
"action":
array([
"set_value", "subtract_value", "add_value", "multiply_value",
"subtract_value", "multiply_value"
]),
"amount":
array([4, 2, 3, 1.1, 1, 0.9]),
"attribute":
array([
"residential_units", "non_residential_sqft",
"non_residential_sqft", "price_per_unit",
"non_residential_sqft", "price_per_unit"
]),
"building_id":
array([3, 3, 5, -1, 3, -1]),
"parcel_id":
array([-1, -1, -1, 5, -1, 5]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='events',
table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage,
in_table_name='events',
id_name=[])
model = ScheduledEventsModel(self.buildings,
scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
results = self.buildings.size()
should_be = 8
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
#examine each action in turn:
index = self.buildings.get_attribute("building_id") == 3
results = (self.buildings.get_attribute("residential_units")[index],
self.buildings.get_attribute("non_residential_sqft")[index])
should_be = (4, 1)
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
index = self.buildings.get_attribute("building_id") == 5
results = self.buildings.get_attribute("non_residential_sqft")[index]
should_be = 1 + 3
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
results = self.buildings.get_attribute("price_per_unit")
should_be = array([50, 21, 32, 15, 60, 90, 100 * 1.1, 200 * 1.1])
self.assertTrue(
allclose(should_be, results),
"Error, should_be: %s, but result: %s" % (should_be, results))
model.run(year=2001)
index = self.buildings.get_attribute("building_id") == 3
results = (self.buildings.get_attribute("residential_units")[index],
self.buildings.get_attribute("non_residential_sqft")[index])
should_be = (4, 0)
self.assertEqual(
should_be, results,
"Error, should_be: %s, but result: %s" % (should_be, results))
results = self.buildings.get_attribute("price_per_unit")
should_be = array(
[50, 21, 32, 15, 60, 90, 100 * 1.1 * 0.9, 200 * 1.1 * 0.9])
self.assertTrue(
allclose(should_be, results),
"Error, should_be: %s, but result: %s" % (should_be, results))
raise StandardError, "cache directory argument is required (-c or --cache-directory)"
if (options.start_year is None) and (options.refinements_directory is
None):
parser.print_usage()
raise StandardError, "Either start year (argument -s or --start-year) or refinements directory (argument --refinements-directory) must be given."
start_year = options.start_year
end_year = options.end_year
package_order = [
package.strip() for package in options.package_order.split(",")
]
refinements = None
refinements_storage = None
if options.refinements_directory is not None:
refinements_storage = StorageFactory().get_storage(
'flt_storage', storage_location=options.refinements_directory)
refinements = DatasetFactory().search_for_dataset(
'refinement',
package_order,
arguments={'in_storage': refinements_storage})
years = refinements.get_attribute('year')
if start_year is None: start_year = years.min()
if end_year is None: end_year = years.max()
simulation_state = SimulationState()
simulation_state.set_cache_directory(options.cache_directory)
simulation_state.set_current_time(start_year)
attribute_cache = AttributeCache()
dataset_pool = SessionConfiguration(
new_instance=True,
package_order=package_order,
def test_place_agents_to_correct_areas(self):
"""10 gridcells - 5 in area 1, 5 in area 2, with equal cost, no capacity restrictions
100 households - 70 live in area 1, 30 live in area 2.
We set the coefficient value for cost -0.001.
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
hh_grid_ids = array(nhhs * [-1])
lareas = array(ngcs_attr * [1] + ngcs_attr * [2])
hh_lareas = array(70 * [1] + 30 * [2])
household_data = {
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids,
'large_area_id': hh_lareas
}
gridcell_data = {
'grid_id': arange(ngcs) + 1,
'cost': array(ngcs * [100]),
'large_area_id': lareas
}
storage.write_table(table_name='households', table_data=household_data)
storage.write_table(table_name='gridcells', table_data=gridcell_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
hlcm = RegionalHouseholdLocationChoiceModel(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=4)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_area1 = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_area2 = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
gridcells.delete_one_attribute("number_of_households")
result = concatenate((result_area1, result_area2))
return result
expected_results = array(ngcs_attr * [nhhs * 0.7 / float(ngcs_attr)] +
ngcs_attr * [nhhs * 0.3 / float(ngcs_attr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
# check the exact sum
hlcm = RegionalHouseholdLocationChoiceModel(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=4)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households",
arange(ngcs_attr) +
1).sum()
result_area2 = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1)).sum()
results = array([result_area1, result_area2])
expected_results = array([70, 30])
self.assertEqual(
ma.allequal(expected_results, results), True,
"Error, should_be: %s, but result: %s" %
(expected_results, results))
from opus_core.storage_factory import StorageFactory
from opus_core.database_management.configurations.scenario_database_configuration import ScenarioDatabaseConfiguration
from opus_core.database_management.configurations.estimation_database_configuration import EstimationDatabaseConfiguration
from urbansim.configs.base_configuration import AbstractUrbansimConfiguration
from urbansim.configurations.creating_baseyear_cache_configuration import CreatingBaseyearCacheConfiguration
from opus_core.database_management.database_server import DatabaseServer
config = AbstractUrbansimConfiguration()
db_server = DatabaseServer(ScenarioDatabaseConfiguration())
db = db_server.get_database('PSRC_2000_baseyear_old_elcm_coeff')
config_changes = {
'description':'baseline with skims using old coefficients',
'in_storage':StorageFactory().get_storage('sql_storage',
storage_location = db
),
'cache_directory':None, ### TODO: Set this cache_directory to something useful.
'creating_baseyear_cache_configuration':CreatingBaseyearCacheConfiguration(
cache_directory_root = 'd:/urbansim_cache',
cache_from_database = True,
cache_scenario_database = 'urbansim.model_coordinators.cache_scenario_database',
tables_to_cache = [
'annual_employment_control_totals',
'annual_household_control_totals',
'buildings',
'building_types',
'development_event_history',
'gridcells',
'households',
'job_building_types',
def test_same_distribution_after_household_addition(self):
"""Using the control_totals and no marginal characteristics,
add households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000, 2000]),
"total_number_of_households": array([20000, 30000]),
"large_area_id": array([1, 2])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set',
table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set',
table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage,
in_table_name='hct_set',
what="household")
storage.write_table(
table_name='hc_set',
table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name='hc_set')
model = RegionalHouseholdTransitionModel()
model.run(year=2000,
household_set=hh_set,
control_totals=hct_set,
characteristics=hc_set)
#check that there are 20000 (area 1) and 30000 (area 2) total households after running the model
areas = hh_set.get_attribute("large_area_id")
results = array([0, 0])
for iarea in [0, 1]:
results[iarea] = where(areas == [1, 2][iarea])[0].size
should_be = [20000, 30000]
self.assertEqual(
ma.allclose(should_be, results, rtol=1e-1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of unplaced households is exactly the number of new households created
results = where(hh_set.get_attribute("grid_id") <= 0)[0].size
should_be = [17000]
self.assertEqual(
ma.allclose(should_be, results, rtol=1e-1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group and each area is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = array([
# area 1
3000.0 / 16500.0 * 20000.0,
1000.0 / 16500.0 * 20000.0,
1500.0 / 16500.0 * 20000.0,
2000.0 / 16500.0 * 20000.0,
1000.0 / 16500.0 * 20000.0,
2500.0 / 16500.0 * 20000.0,
1500.0 / 16500.0 * 20000.0,
4000.0 / 16500.0 * 20000.0,
# area 2
3000.0 / 16500.0 * 30000.0,
1000.0 / 16500.0 * 30000.0,
1500.0 / 16500.0 * 30000.0,
2000.0 / 16500.0 * 30000.0,
1000.0 / 16500.0 * 30000.0,
2500.0 / 16500.0 * 30000.0,
1500.0 / 16500.0 * 30000.0,
4000.0 / 16500.0 * 30000.0
])
self.assertEqual(
ma.allclose(results, should_be, rtol=0.1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
# check the types of the attributes
self.assertEqual(
hh_set.get_attribute("age_of_head").dtype, int32,
"Error in data type of the new household set. Should be: int32, is: %s"
% str(hh_set.get_attribute("age_of_head").dtype))
self.assertEqual(
hh_set.get_attribute("income").dtype, int32,
"Error in data type of the new household set. Should be: int32, is: %s"
% str(hh_set.get_attribute("income").dtype))
self.assertEqual(
hh_set.get_attribute("persons").dtype, int8,
"Error in data type of the new household set. Should be: int8, is: %s"
% str(hh_set.get_attribute("persons").dtype))
def test_agents_go_to_attractive_locations(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
nhhs = 1000
ngcs = 50
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
household_data = {
"household_id": arange(nhhs) + 1,
"grid_id": hh_grid_ids
}
gridcell_data = {
"grid_id": arange(ngcs) + 1,
"cost": array(ngcs_attr * [100] + ngcs_noattr * [1000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = HouseholdDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ),
values=(-0.001, ))
specification = EquationSpecification(
variables=("gridcell.cost", ), coefficients=("costcoef", ))
logger.be_quiet()
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute="grid_id",
values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result = concatenate(
(result_more_attractive, result_less_attractive))
#print result #, result_more_attractive.sum(), result_less_attractive.sum()
return result
expected_results = array(
ngcs_attr * [nhhs * 0.71 / float(ngcs_attr)] +
ngcs_noattr * [nhhs * 0.29 / float(ngcs_noattr)])
#print expected_results
R = 1000
#r = [2, 5, 10, 50, 100, 1000]
r = [2, 5, 10, 15, 20]
#r=[5]
levels = [0.05, 0.01]
#levels = [0.05]
power = zeros((len(r), len(levels)))
for ir in range(len(r)):
for il in range(len(levels)):
print "r=", r[ir], ", level=", levels[il]
seed(1)
for iR in range(R):
try:
self.run_stochastic_test(
__file__,
run_model,
expected_results,
r[ir],
significance_level=levels[il])
except:
power[ir, il] = power[ir, il] + 1
print "Power: ", power[ir, il] / float(R)
print power / float(R)
# Copyright (C) 2005-2009 University of Washington
# See opus_core/LICENSE
# Test of the installation
from opus_core.opus_package_info import package as opus_core_package
opus_core_package().info()
from urbansim.opus_package_info import package as urbansim_package
urbansim_package().info()
# Working with datasets
import os
import urbansim
us_path = urbansim.__path__[0]
from opus_core.storage_factory import StorageFactory
storage = StorageFactory().get_storage('tab_storage',
storage_location = os.path.join(us_path, "data/tutorial"))
from opus_core.datasets.dataset import Dataset
households = Dataset(in_storage = storage,
in_table_name = 'households',
id_name='household_id',
dataset_name='household')
households.get_attribute_names()
households.get_id_attribute()
households.size()
households.get_attribute("income")
households.get_attribute_names()
households.load_dataset()
households.get_attribute_names()
#households.plot_histogram("income", bins = 10)
#households.r_histogram("income")
except:
pass
#create table on mysql server
cursor.execute('CREATE TABLE target_vacancies '
'( \
target_vacancy_rate float, \
year int, \
total_spaces varchar(50), \
occupied_spaces varchar(50), \
building_group_id tinyint)') #this variable needs to be defined in aliases.py
#building_type_id tinyint)')
conn.commit()
us_path=r'C:\opus\data\sanfrancisco\baseyear_2009_scenario_baseline\2009'
storage = StorageFactory().get_storage('flt_storage',storage_location = us_path)
#read classification from local cache
bt = Dataset(in_storage = storage,in_table_name = 'building_types',id_name='building_type_id',dataset_name='building_type')
btclass= Dataset(in_storage = storage,in_table_name = 'building_type_classification',id_name='class_id',dataset_name='building_type_classification')
targetvacancies = Dataset(in_storage = storage,in_table_name = 'target_vacancies',id_name='target_vacancies_id',dataset_name='target_vacancies')
#===============================================================================
# y=a*sin(bx+c)+d
# treat vacancy like a sine function. Constants (b, c) are crafted so there are 64 months between each peak
# last peak was late 2007.
# amp (a) denotes the amplitude of each cycle, while the base (d) signifies the center of the Function.
#http://www.nber.org/cycles.html
#===============================================================================
def xtest_power_HTM_controlling_with_marginal_characteristics(self):
nhhs = 5000
ngroups = 4
nhhsg = int(nhhs / ngroups)
nhhslg = nhhs - (ngroups - 1) * nhhsg
should_nhhs = nhhs - 2000
logger.be_quiet()
household_data = {
"age_of_head":
array(nhhsg / 2 * [18] + (nhhsg - nhhsg / 2) * [35] +
nhhsg / 2 * [30] + (nhhsg - nhhsg / 2) * [40] +
nhhsg / 2 * [38] + (nhhsg - nhhsg / 2) * [65] +
nhhslg / 2 * [50] + (nhhslg - nhhslg / 2) * [80]),
"income":
array(nhhsg * [500] + nhhsg * [2000] + nhhsg * [7000] +
nhhslg * [15000]),
"household_id":
arange(nhhs) + 1
}
household_characteristics_for_ht_data = {
"characteristic": array(4 * ["income"] + 4 * ["age_of_head"]),
"min": array([0, 1001, 5001, 10001, 0, 31, 41, 61]),
"max": array([1000, 5000, 10000, -1, 30, 40, 60, -1])
}
annual_household_control_totals_data = {
"year": array([2000]),
"total_number_of_households": array([should_nhhs])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='hc_set',
table_data=household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name='hc_set')
storage.write_table(
table_name='hct_set',
table_data=annual_household_control_totals_data)
hct_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name='hct_set')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdCharacteristicDataset(
in_storage=storage, in_table_name='households')
income = households.get_attribute("income")
age = households.get_attribute("age_of_head")
idx1 = where(income <= 1000)[0]
idx2 = where(logical_and(income <= 5000, income > 1000))[0]
idx3 = where(logical_and(income <= 10000, income > 5000))[0]
idx4 = where(income > 10000)[0]
expected_results = array([
age[idx1].mean(), age[idx2].mean(), age[idx3].mean(),
age[idx4].mean()
])
def run_model():
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdCharacteristicDataset(
in_storage=storage, in_table_name='households')
model = HouseholdTransitionModel()
model.run(year=2000,
household_set=households,
control_totals=hct_set,
characteristics=hc_set)
income = households.get_attribute("income")
age = households.get_attribute("age_of_head")
idx1 = where(income <= 1000)[0]
idx2 = where(logical_and(income <= 5000, income > 1000))[0]
idx3 = where(logical_and(income <= 10000, income > 5000))[0]
idx4 = where(income > 10000)[0]
results = array([
age[idx1].mean(), age[idx2].mean(), age[idx3].mean(),
age[idx4].mean()
])
results[-1] = results[-1] + self.wrong_number
#print results
return results
#print expected_results
R = 1000
#r = [2, 5, 10, 50, 100, 1000]
#r = [2, 5, 10, 15, 20]
r = [2, 5]
levels = [0.05, 0.01]
#levels = [0.05]
#wrong_numbers = [0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25, 2.5]
wrong_numbers = [1]
for wn in wrong_numbers:
self.wrong_number = wn
print "Wrong number = ", self.wrong_number
power = zeros((len(r), len(levels)))
for ir in range(len(r)):
for il in range(len(levels)):
print "r=", r[ir], ", level=", levels[il]
seed(1)
for iR in range(R):
try:
self.run_stochastic_test(
__file__,
run_model,
expected_results,
r[ir],
significance_level=levels[il],
transformation=None)
except:
power[ir, il] = power[ir, il] + 1
print "Power: ", power[ir, il] / float(R)
print power / float(R)
def setUp(self):
self.storage = StorageFactory().get_storage('dict_storage')
self.storage.write_table(table_name='development_event_history',
table_data={
"zone_id":
arange(1, 100 + 1),
"scheduled_year":
array(100 * [1999]),
"building_type_id":
array(30 * [1] + 35 * [2] + 35 * [4]),
"residential_units":
array(65 * [0] + 35 * [50]),
"non_residential_sqft":
array(65 * [500] + 35 * [0]),
})
# create 10 zones, each with 200 residential units and space for 100 commercial jobs,
# and 100 industrial jobs
self.storage.write_table(table_name='zones',
table_data={
"zone_id": arange(1, 10 + 1),
})
self.storage.write_table(
table_name='buildings',
table_data={
"building_id":
arange(1, 31), # 1 building per building_type and zone
"zone_id":
array([
1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7,
7, 8, 8, 8, 9, 9, 9, 10, 10, 10
]),
"building_type_id":
array(10 * [1, 2, 4]),
"residential_units":
array(10 * [0, 0, 200]),
"non_residential_sqft":
array(10 * [10, 100, 0])
})
self.storage.write_table(table_name='building_types',
table_data={
"building_type_id":
arange(1, 5),
"building_type_name":
array([
'commercial', 'industrial',
'governmental', 'residential'
]),
"is_residential":
array([0, 0, 0, 1])
})
# create 1000 households, 100 in each of the 10 zones.
# there will initially be 100 vacant residential units in each zone.
self.storage.write_table(table_name='households',
table_data={
"household_id": arange(1, 1000 + 1),
"zone_id": array(100 * range(1, 11))
})
# create 250 commercial jobs and distribute them equally across the 10 zones,
# 25 commercial jobs/zone
self.storage.write_table(table_name='jobs',
table_data={
"job_id": arange(1, 250 + 1),
"building_id": array(25 * range(1, 11)),
"home_based": array(250 * [0]),
})
self.storage.write_table(table_name="building_sqft_per_job",
table_data={
"zone_id":
array([1, 1, 1, 2, 2, 2, 3, 3]),
"building_type_id":
array([1, 2, 3, 1, 2, 3, 1, 3]),
"building_sqft_per_job":
array([100, 50, 200, 80, 60, 500, 20,
10]),
})
self.dataset_pool = DatasetPool(
package_order=['urbansim_zone', 'urbansim_parcel', "urbansim"],
storage=self.storage)
self.compute_resources = Resources({})
def __init__(self):
config = AbstractUrbansimConfiguration()
db_server = DatabaseServer(ScenarioDatabaseConfiguration())
db = db_server.get_database('PSRC_2000_baseyear')
config_changes = {
'description':
'baseline with no travel model',
'in_storage':
StorageFactory().get_storage('sql_storage', storage_location=db),
'cache_directory':
None, ### TODO: Set this cache_directory to something useful.
'creating_baseyear_cache_configuration':
CreatingBaseyearCacheConfiguration(
cache_directory_root='c:/urbansim_cache',
cache_from_database=True,
cache_scenario_database=
'urbansim.model_coordinators.cache_scenario_database',
tables_to_cache=[
'annual_employment_control_totals',
'annual_household_control_totals', 'buildings',
'building_types', 'development_event_history', 'gridcells',
'households', 'job_building_types', 'jobs', 'travel_data',
'zones', 'counties',
'commercial_development_location_choice_model_coefficients',
'commercial_development_location_choice_model_specification',
'commercial_employment_location_choice_model_coefficients',
'commercial_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_specification',
'industrial_development_location_choice_model_coefficients',
'industrial_development_location_choice_model_specification',
'residential_development_location_choice_model_coefficients',
'residential_development_location_choice_model_specification',
'fazes', 'urbansim_constants',
'household_location_choice_model_coefficients',
'household_location_choice_model_specification',
'land_price_model_coefficients',
'land_price_model_specification',
'residential_land_share_model_coefficients',
'residential_land_share_model_specification',
'plan_type_group_definitions', 'plan_type_groups',
'large_areas', 'household_characteristics_for_ht',
'development_types', 'development_type_group_definitions',
'development_constraints',
'annual_relocation_rates_for_households',
'annual_relocation_rates_for_jobs', 'base_year', 'cities',
'development_events', 'development_type_groups',
'employment_adhoc_sector_group_definitions',
'employment_adhoc_sector_groups', 'employment_events',
'employment_sectors', 'land_use_events', 'plan_types',
'race_names', 'target_vacancies', 'jobs_for_estimation',
'households_for_estimation',
'development_events_exogenous', 'job_building_types'
],
tables_to_cache_nchunks={'gridcells': 1},
tables_to_copy_to_previous_years={
'development_type_groups': 1995,
'development_types': 1995,
'development_type_group_definitions': 1995,
'urbansim_constants': 1995,
},
),
'scenario_database_configuration':
ScenarioDatabaseConfiguration(
database_name='PSRC_2000_baseyear', ),
'base_year':
2000,
'years': (2001, 2030),
}
config.merge(config_changes)
self.merge(config)
def __init__(self):
config = AbstractUrbansimConfiguration()
db_server = DatabaseServer(ScenarioDatabaseConfiguration())
db = db_server.get_database('PSRC_2000_baseyear')
config_changes = {
'description':
'baseline with travel model',
'in_storage':
StorageFactory().get_storage('sql_storage', storage_location=db),
'cache_directory':
None, ### TODO: Set this cache_directory to something useful.
'creating_baseyear_cache_configuration':
CreatingBaseyearCacheConfiguration(
cache_directory_root='d:/urbansim_cache',
cache_from_database=True,
baseyear_cache=BaseyearCacheConfiguration(
r'D:\urbansim_cache\run_1417.2006_12_08_01_50'),
cache_scenario_database=
'urbansim.model_coordinators.cache_scenario_database',
tables_to_cache=[
'annual_employment_control_totals',
'annual_household_control_totals', 'buildings',
'building_types', 'development_event_history', 'gridcells',
'households', 'job_building_types', 'jobs', 'travel_data',
'zones', 'counties',
'commercial_development_location_choice_model_coefficients',
'commercial_development_location_choice_model_specification',
'commercial_employment_location_choice_model_coefficients',
'commercial_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_specification',
'industrial_development_location_choice_model_coefficients',
'industrial_development_location_choice_model_specification',
'residential_development_location_choice_model_coefficients',
'residential_development_location_choice_model_specification',
'fazes', 'urbansim_constants',
'household_location_choice_model_coefficients',
'household_location_choice_model_specification',
'land_price_model_coefficients',
'land_price_model_specification',
'residential_land_share_model_coefficients',
'residential_land_share_model_specification',
'plan_type_group_definitions', 'plan_type_groups',
'large_areas', 'household_characteristics_for_ht',
'development_types', 'development_type_group_definitions',
'development_constraints',
'annual_relocation_rates_for_households',
'annual_relocation_rates_for_jobs', 'base_year', 'cities',
'development_events', 'development_type_groups',
'employment_adhoc_sector_group_definitions',
'employment_adhoc_sector_groups', 'employment_events',
'employment_sectors', 'land_use_events', 'plan_types',
'race_names', 'target_vacancies', 'jobs_for_estimation',
'households_for_estimation',
'development_events_exogenous', 'job_building_types'
],
tables_to_cache_nchunks={'gridcells': 4},
tables_to_copy_to_previous_years={
'development_type_groups': 1995,
'development_types': 1995,
'development_type_group_definitions': 1995,
},
),
'models': [], # run no urbansim models
'scenario_database_configuration':
ScenarioDatabaseConfiguration(
database_name='PSRC_2000_baseyear', ),
'base_year':
2000,
'years': (2001, 2001),
'unroll_gridcells':
False
}
config.merge(config_changes)
travel_model_config = create_travel_model_configuration(
'baseline_travel_model_psrc_fresh',
mode='get_emme2_data_after_run',
years_to_run={2001: '2000_06'},
emme2_batch_file='QUICKRUN.BAT')
config['travel_model_configuration'] = travel_model_config
self.merge(config)
2000,
'tables_to_cache': [
'gridcells',
# 'households',
# 'jobs',
]
})
#CacheScenarioDatabase().run(gridcell_config)
# step 2 cache water demand data by
dbcon = ScenarioDatabase(database_name="water_demand_seattle2")
print "Create Storage object."
from opus_core.storage_factory import StorageFactory
storage = StorageFactory().get_storage(type="mysql_storage",
storage_location=dbcon)
from waterdemand.datasets.consumption_dataset import ConsumptionDataset
consumption_types = ['wrmr', 'wcsr', 'wrsr'] #'wcmr'
for consumption_type in consumption_types:
consumption = ConsumptionDataset(in_storage=storage,
in_table_name=consumption_type +
'_grid')
for year in range(1990, 2001):
print "%s %s" % (consumption_type, year)
year_index = where(consumption.get_attribute("billyear") == year)
out_storage = StorageFactory().get_storage(
type="flt_storage",
storage_location=os.path.join(cache_directory, str(year)))
def _visualize(self, args, cache_directory=None):
self.visualizations = []
indicators_to_visualize = {}
interface = IndicatorFrameworkInterface(self.project)
not_visualized = []
#get common years
years = set([])
for indicator in self.indicators:
years |= set(indicator['years'])
source_data_objs = {}
for indicator in self.indicators:
indicator_name = indicator['indicator_name']
source_data_name = indicator['source_data_name']
dataset_name = indicator['dataset_name']
if (self.indicator_type == 'mapnik_map'
or self.indicator_type == 'mapnik_animated_map'
) and dataset_name not in self.spatial_datasets:
not_visualized.append(indicator)
continue
if source_data_name not in source_data_objs:
source_data = interface.get_source_data(
source_data_name=source_data_name,
years=list(years),
cache_directory=cache_directory)
source_data_objs[source_data_name] = source_data
else:
source_data = source_data_objs[source_data_name]
indicator = interface.get_indicator(indicator_name=indicator_name,
dataset_name=dataset_name)
computed_indicator = interface.get_computed_indicator(
indicator=indicator,
source_data=source_data,
dataset_name=dataset_name)
computed_indicator.gui_indicator_name = indicator_name
#####################
#hack to get plausible primary keys...
cache_directory = source_data.cache_directory
_storage_location = os.path.join(cache_directory, 'indicators',
'_stored_data',
repr(source_data.years[0]))
storage = StorageFactory().get_storage(
type='flt_storage', storage_location=_storage_location)
cols = storage.get_column_names(table_name=dataset_name)
primary_keys = [col for col in cols if col.find('_id') != -1]
computed_indicator.primary_keys = primary_keys
##################
name = computed_indicator.get_file_name(
suppress_extension_addition=True)
indicators_to_visualize[name] = computed_indicator
viz_args = {}
if self.indicator_type == 'mapnik_map':
viz_type = self.indicator_type
elif self.indicator_type == 'mapnik_animated_map':
viz_type = self.indicator_type
elif self.indicator_type == 'matplotlib_chart':
viz_type = self.indicator_type
elif self.indicator_type == 'tab':
viz_type = 'table'
if 'output_style' not in viz_args:
viz_args['output_style'] = Table.ALL
viz_args['output_type'] = 'tab'
elif self.indicator_type == 'table_esri':
viz_type = 'table'
if 'output_style' not in viz_args:
viz_args['output_style'] = Table.ALL
viz_args['output_type'] = 'esri'
elif self.indicator_type == 'table_per_year':
viz_type = 'table'
if 'output_style' not in viz_args:
viz_args['output_style'] = Table.PER_YEAR
viz_args['output_type'] = 'tab'
elif self.indicator_type == 'table_per_attribute':
viz_type = 'table'
if 'output_style' not in viz_args:
viz_args['output_style'] = Table.PER_ATTRIBUTE
viz_args['output_type'] = 'tab'
viz_args.update(self.kwargs)
viz_args.update(args)
# try:
# import pydevd;pydevd.settrace()
# except:
# pass
viz_factory = VisualizationFactory()
self.visualizations = viz_factory.visualize(
indicators_to_visualize=indicators_to_visualize.keys(),
computed_indicators=indicators_to_visualize,
visualization_type=viz_type,
**viz_args)
if self.visualizations is None:
self.visualizations = []
return not_visualized
def on_pb_urbancanvas_clicked(self):
run_name = self.current_run
indicator_name = self.current_indicator
indicator_dataset = self.current_indicator_dataset
if indicator_dataset != 'parcel':
MessageBox.information(
mainwindow=self,
text=
'Not a parcel variable. Only parcel variables can be sent to UrbanCanvas'
)
else:
start_year = int(self.current_year)
end_year = start_year
if run_name is None or indicator_name is None or start_year is None:
return
key = (run_name, indicator_name, start_year)
self.pb_urbancanvas.setText('Sending to UrbanCanvas...')
indicator_nodes = get_available_indicator_nodes(self.project)
dataset = None
for indicator_node in indicator_nodes:
ind_dataset, name = get_variable_dataset_and_name(
indicator_node)
if name == indicator_name and ind_dataset == indicator_dataset:
dataset = ind_dataset
break
if dataset is None:
raise Exception('Could not find dataset for indicator %s' %
indicator_name)
table_params = {
'name': None,
'output_type': 'tab',
'indicators': [indicator_name],
}
expression_library = self.project.xml_config.get_expression_library(
)
expression = expression_library[(dataset, name)]
logger.log_note(expression)
base_year = end_year
project_name = self.project.name
opus_data_path = self.project.xml_config.get_opus_data_path()
logger.log_note(base_year)
logger.log_note(project_name)
logger.log_note(opus_data_path)
interface = IndicatorFrameworkInterface(self.project)
source_data = interface.get_source_data(source_data_name=run_name,
years=[
end_year,
])
cache = os.path.join(source_data.cache_directory, str(end_year))
logger.log_note(cache)
storage = StorageFactory().get_storage('flt_storage',
storage_location=cache)
dataset_pool = DatasetPool(storage=storage,
package_order=[
project_name, 'urbansim_parcel',
'urbansim', 'opus_core'
])
parcels = dataset_pool.get_dataset('parcel')
parcel_ids = pd.Series(parcels.get_attribute('parcel_id'))
values = pd.Series(
parcels.compute_variables(
[expression], dataset_pool=dataset_pool).astype('float'))
parcels = pd.DataFrame({
"parcel_id": parcel_ids,
"vl_values": values
})
parcels.set_index(keys='parcel_id', inplace=True)
#parcels["vl_values"][parcels["vl_values"]==0] = np.nan
parcels = parcels[parcels["vl_values"] > 0]
os.chdir(os.path.join(opus_data_path, project_name))
np.savez('results_browser_indicator',
parcel_id=parcels.vl_values.index.values.astype('int32'),
values=parcels.vl_values.values.astype('int32'))
##############UNCOMMENT IF WEBSERVICE IS DESIRED
# parcels.save('variable_library.pkl') ##I believe 'save' was just deprectated in pandas- its now to_pickle or some such thing... change this later
# web_service_path = os.path.join(os.getenv("OPUS_HOME"),'src',project_name,'scripts','web_service.py')
# logger.log_note(web_service_path)
# p = subprocess.Popen([sys.executable,web_service_path])
# MessageBox.information(mainwindow = self, text = 'Click OK when done viewing in UrbanCanvas')
# p.kill()
# self.pb_urbancanvas.setText('View in UrbanCanvas')
MessageBox.information(
mainwindow=self,
text=
'Variable exported to the project data directory for viewing in UrbanCanvas'
)
self.pb_urbancanvas.setText('View in UrbanCanvas')
def setUp( self ):
SimulationState().set_current_time(2000)
self.storage = StorageFactory().get_storage('dict_storage')
self.storage.write_table(
table_name='development_event_history',
table_data={
"zone_id":arange( 1, 100+1 ),
"scheduled_year":array( 100*[1999] ),
"building_type_id": array(30*[1] + 35*[2] + 35*[4]),
"residential_units":array( 65*[0] + 35*[50] ),
"non_residential_sqft":array( 65*[500] + 35*[0] )
}
)
self.storage.write_table(
table_name='zones',
table_data={
"zone_id": arange( 1, 10+1 ),
}
)
self.storage.write_table(
table_name='buildings',
table_data={
"building_id": arange(1,31), # 1 building per building_type and zone
"zone_id": array( [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10,10,10] ),
"building_type_id": array(10*[1,2,4]),
"residential_units": array(10*[0, 0, 100]),
"non_residential_sqft": array(10*[100,150,0])
}
)
self.storage.write_table(
table_name='building_types',
table_data={
"building_type_id": arange(1,5),
"building_type_name": array(['commercial', 'industrial','governmental', 'residential']),
"is_residential": array([0,0,0,1])
}
)
# create 1000 households, 100 in each of the 10 zones.
# there will initially be 100 vacant residential units in each zone.
self.storage.write_table(
table_name='households',
table_data={
"household_id":arange( 1, 1000+1 ),
"building_id": array(100 * range(3,31,3))
}
)
# create 250 commercial jobs
self.storage.write_table(
table_name='jobs',
table_data={
"job_id":arange( 1, 250+1 ),
"building_id":array( 20*range(1,31,3) + 5*range(2,32,3) ),
"home_based":array( 250*[0] ),
}
)
self.storage.write_table(
table_name = "building_sqft_per_job",
table_data = {
"building_id": array([1, 2, 4, 5, 7, 8,10, 11,13,14,16,17,19,20,22,23,25,26,28, 29]),
"zone_id": array([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10]),
"building_type_id": array([1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2]),
"building_sqft_per_job":array([10, 5, 8, 6, 20,10,10,20,10,20,10,20,10,20,10,20,10,20,10, 20]),
}
)
self.dataset_pool = DatasetPool(package_order=['urbansim_zone', 'urbansim_parcel', "urbansim"],
storage=self.storage)
self.compute_resources = Resources({})
def estimate(self,
spec_var=None,
spec_py=None,
submodel_string="workers",
agent_sample_rate=0.005,
alt_sample_size=None):
"""
"""
CLOSE = 0.001
sampler = "opus_core.samplers.weighted_sampler"
if alt_sample_size == None:
sampler = None
date_time_str = strftime("%Y_%m_%d__%H_%M", localtime())
agent_sample_rate_str = "__ASR_" + str(agent_sample_rate)
alt_sample_size_str = "_ALT_" + str(alt_sample_size)
info_file = date_time_str + agent_sample_rate_str + alt_sample_size_str + "__info.txt"
logger.enable_file_logging(date_time_str + agent_sample_rate_str +
alt_sample_size_str + "__run.txt")
logger.enable_memory_logging()
logger.log_status("Constrained Estimation with agent sample rate of %s and alternatvie sample size %s\n" % \
(agent_sample_rate, alt_sample_size))
t1 = time()
SimulationState().set_current_time(2000)
self.nbs = SessionConfiguration().get_dataset_from_pool("neighborhood")
self.hhs = SessionConfiguration().get_dataset_from_pool('household')
depts, lambda_value = compute_lambda(self.nbs)
supply, vacancy_rate = compute_supply_and_vacancy_rate(
self.nbs, depts, lambda_value)
self.nbs.set_values_of_one_attribute("supply", supply)
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset_pool.add_datasets_if_not_included({
'vacancy_rate':
vacancy_rate,
'sample_rate':
agent_sample_rate
})
SessionConfiguration()["CLOSE"] = CLOSE
SessionConfiguration()['info_file'] = info_file
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(
in_table_name="household_location_choice_model_specification")
#submodel_string = "workers"
seed(71) # was: seed(71,110)
self.model_name = "household_location_choice_model"
model = HouseholdLocationChoiceModelCreator().get_model(
location_set=self.nbs,
submodel_string=submodel_string,
sampler=sampler,
estimation_size_agents=agent_sample_rate * 100 / 20,
# proportion of the agent set that should be used for the estimation,
#
sample_size_locations=
alt_sample_size, # choice set size (includes current location)
compute_capacity_flag=True,
probabilities="opus_core.mnl_probabilities",
choices="urbansim.lottery_choices",
run_config=Resources({"capacity_string": "supply"}),
estimate_config=Resources({
"capacity_string": "supply",
"compute_capacity_flag": True
}))
#TODO: since households_for_estimation currently is the same as households, create_households_for_estimation
#becomes unnecesarry
#agent_set, agents_index_for_estimation = create_households_for_estimation(self.hhs, self.in_con)
agent_set = self.hhs
agents_index_for_estimation = arange(self.hhs.size())
self.result = model.estimate(
self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops"
) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time() - t1) + " s")
def run(self, realestate_dataset,
year=None,
occupied_spaces_variable="occupied_units",
total_spaces_variable="total_units",
target_attribute_name='target_vacancy_rate',
sample_from_dataset = None,
sample_filter="",
reset_attribute_value={},
year_built = 'year_built',
dataset_pool=None,
append_to_realestate_dataset = False,
table_name = "development_projects",
dataset_name = "development_project",
id_name = [],
**kwargs):
"""
sample_filter attribute/variable indicates which records in the dataset are eligible in the sampling for removal or cloning
append_to_realestate_dataset - whether to append the new dataset to realestate_dataset
"""
if self.target_vancy_dataset is None:
raise RuntimeError, "target_vacancy_rate dataset is unspecified."
if not sample_from_dataset:
sample_from_dataset = realestate_dataset
#if dataset_pool is None:
# dataset_pool = SessionConfiguration().get_dataset_pool()
if year is None:
year = SimulationState().get_current_time()
this_year_index = where(self.target_vancy_dataset.get_attribute('year')==year)[0]
target_vacancy_for_this_year = DatasetSubset(self.target_vancy_dataset, this_year_index)
column_names = list(set( self.target_vancy_dataset.get_known_attribute_names() ) - set( [ target_attribute_name, occupied_spaces_variable, total_spaces_variable, 'year', '_hidden_id_'] ))
column_names.sort(reverse=True)
column_values = dict([ (name, target_vacancy_for_this_year.get_attribute(name)) for name in column_names + [target_attribute_name]])
independent_variables = list(set([re.sub('_max$', '', re.sub('_min$', '', col)) for col in column_names]))
dataset_known_attributes = realestate_dataset.get_known_attribute_names()
for variable in independent_variables:
if variable not in dataset_known_attributes:
realestate_dataset.compute_one_variable_with_unknown_package(variable, dataset_pool=dataset_pool)
sample_from_dataset.compute_one_variable_with_unknown_package(variable, dataset_pool=dataset_pool)
dataset_known_attributes = realestate_dataset.get_known_attribute_names() #update after compute
if sample_filter:
short_name = VariableName(sample_filter).get_alias()
if short_name not in dataset_known_attributes:
filter_indicator = sample_from_dataset.compute_variables(sample_filter, dataset_pool=dataset_pool)
else:
filter_indicator = sample_from_dataset.get_attribute(short_name)
else:
filter_indicator = 1
sampled_index = array([], dtype=int32)
#log header
if PrettyTable is not None:
status_log = PrettyTable()
status_log.set_field_names(column_names + ["actual", "target", "difference", "action"])
else:
logger.log_status("\t".join(column_names + ["actual", "target", "difference", "action"]))
error_log = ''
for index in range(target_vacancy_for_this_year.size()):
this_sampled_index = array([], dtype=int32)
indicator = ones( realestate_dataset.size(), dtype='bool' )
sample_indicator = ones( sample_from_dataset.size(), dtype='bool' )
criterion = {} # for logging
for attribute in independent_variables:
if attribute in dataset_known_attributes:
dataset_attribute = realestate_dataset.get_attribute(attribute)
sample_attribute = sample_from_dataset.get_attribute(attribute)
else:
raise ValueError, "attribute %s used in target vacancy dataset can not be found in dataset %s" % (attribute, realestate_dataset.get_dataset_name())
if attribute + '_min' in column_names:
amin = target_vacancy_for_this_year.get_attribute(attribute+'_min')[index]
criterion.update({attribute + '_min':amin})
if amin != -1:
indicator *= dataset_attribute >= amin
sample_indicator *= sample_attribute >= amin
if attribute + '_max' in column_names:
amax = target_vacancy_for_this_year.get_attribute(attribute+'_max')[index]
criterion.update({attribute + '_max':amax})
if amax != -1:
indicator *= dataset_attribute <= amax
sample_indicator *= sample_attribute <= amax
if attribute in column_names:
aval = column_values[attribute][index]
criterion.update({attribute:aval})
if aval == -1:
continue
elif aval == -2: ##treat -2 in control totals column as complement set, i.e. all other values not already specified in this column
indicator *= logical_not(ismember(dataset_attribute, column_values[attribute]))
sample_indicator *= logical_not(ismember(sample_attribute, column_values[attribute]))
else:
indicator *= dataset_attribute == aval
sample_indicator *= sample_attribute == aval
this_total_spaces_variable, this_occupied_spaces_variable = total_spaces_variable, occupied_spaces_variable
## total/occupied_spaces_variable can be specified either as a universal name for all realestate
## or in targe_vacancy_rate dataset for each vacancy category
if occupied_spaces_variable in target_vacancy_for_this_year.get_known_attribute_names():
this_occupied_spaces_variable = target_vacancy_for_this_year.get_attribute(occupied_spaces_variable)[index]
if total_spaces_variable in target_vacancy_for_this_year.get_known_attribute_names():
this_total_spaces_variable = target_vacancy_for_this_year.get_attribute(total_spaces_variable)[index]
logger.be_quiet() #temporarily disable logging
realestate_dataset.compute_one_variable_with_unknown_package(this_occupied_spaces_variable, dataset_pool=dataset_pool)
realestate_dataset.compute_one_variable_with_unknown_package(this_total_spaces_variable, dataset_pool=dataset_pool)
sample_from_dataset.compute_one_variable_with_unknown_package(this_total_spaces_variable, dataset_pool=dataset_pool)
logger.talk()
actual_num = (indicator * realestate_dataset.get_attribute(this_total_spaces_variable)).sum()
target_num = int(round( (indicator * realestate_dataset.get_attribute(this_occupied_spaces_variable)).sum() /\
(1 - target_vacancy_for_this_year.get_attribute(target_attribute_name)[index])
))
diff = target_num - actual_num
if diff > 0:
total_spaces_in_sample_dataset = sample_from_dataset.get_attribute(this_total_spaces_variable)
legit_index = where(logical_and(sample_indicator, filter_indicator) * total_spaces_in_sample_dataset > 0)[0]
if legit_index.size > 0:
mean_size = total_spaces_in_sample_dataset[legit_index].mean()
num_of_projects_to_sample = int( diff / mean_size )
while total_spaces_in_sample_dataset[this_sampled_index].sum() < diff:
lucky_index = sample_replace(legit_index, num_of_projects_to_sample)
this_sampled_index = concatenate((this_sampled_index, lucky_index))
this_sampled_index = this_sampled_index[0:(1+searchsorted(cumsum(total_spaces_in_sample_dataset[this_sampled_index]), diff))]
sampled_index = concatenate((sampled_index, this_sampled_index))
else:
error_log += "There is nothing to sample from %s and no new development will happen for " % sample_from_dataset.get_dataset_name() + \
','.join([col+"="+str(criterion[col]) for col in column_names]) + '\n'
#if diff < 0: #TODO demolition; not yet supported
##log status
action = "0"
if this_sampled_index.size > 0:
action_num = total_spaces_in_sample_dataset[this_sampled_index].sum()
if diff > 0: action = "+" + str(action_num)
if diff < 0: action = "-" + str(action_num)
cat = [ str(criterion[col]) for col in column_names]
cat += [str(actual_num), str(target_num), str(diff), action]
if PrettyTable is not None:
status_log.add_row(cat)
else:
logger.log_status("\t".join(cat))
if PrettyTable is not None:
logger.log_status("\n" + status_log.get_string())
if error_log:
logger.log_error(error_log)
result_data = {}
result_dataset = None
index = array([], dtype='int32')
if sampled_index.size > 0:
### ideally duplicate_rows() is all needed to add newly cloned rows
### to be more cautious, copy the data to be cloned, remove elements, then append the cloned data
##realestate_dataset.duplicate_rows(sampled_index)
result_data.setdefault(year_built, resize(year, sampled_index.size).astype('int32'))
for attribute in sample_from_dataset.get_primary_attribute_names():
if reset_attribute_value.has_key(attribute):
result_data[attribute] = resize(array(reset_attribute_value[attribute]), sampled_index.size)
else:
result_data[attribute] = sample_from_dataset.get_attribute_by_index(attribute, sampled_index)
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name=table_name, table_data=result_data)
result_dataset = Dataset(id_name = id_name,
in_storage = storage,
in_table_name = table_name,
dataset_name = dataset_name
)
index = arange(result_dataset.size())
if append_to_realestate_dataset:
if len(result_data) > 0:
index = realestate_dataset.add_elements(result_data, require_all_attributes=False,
change_ids_if_not_unique=True)
result_dataset = realestate_dataset
return (result_dataset, index)
def test_same_distribution_after_household_subtraction(self):
"""Using the control_totals and no marginal characteristics,
subtract households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000, 2000]),
"total_number_of_households": array([8000, 12000]),
"large_area_id": array([1, 2])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set',
table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set',
table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage,
in_table_name='hct_set',
what="household")
storage.write_table(
table_name='hc_set',
table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name='hc_set')
model = RegionalHouseholdTransitionModel()
model.run(year=2000,
household_set=hh_set,
control_totals=hct_set,
characteristics=hc_set)
#check that there are indeed 8000 (area 1) and 12000 (area 2) total households after running the model
areas = hh_set.get_attribute("large_area_id")
results = array([0, 0])
for iarea in [0, 1]:
results[iarea] = where(areas == [1, 2][iarea])[0].size
should_be = [8000, 12000]
self.assertEqual(
ma.allclose(should_be, results, rtol=1e-1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = array([ # area 1
3000.0 / 16500.0 * 8000.0,
1000.0 / 16500.0 * 8000.0,
1500.0 / 16500.0 * 8000.0,
2000.0 / 16500.0 * 8000.0,
1000.0 / 16500.0 * 8000.0,
2500.0 / 16500.0 * 8000.0,
1500.0 / 16500.0 * 8000.0,
4000.0 / 16500.0 * 8000.0,
# area 2
3000.0 / 16500.0 * 12000.0,
1000.0 / 16500.0 * 12000.0,
1500.0 / 16500.0 * 12000.0,
2000.0 / 16500.0 * 12000.0,
1000.0 / 16500.0 * 12000.0,
2500.0 / 16500.0 * 12000.0,
1500.0 / 16500.0 * 12000.0,
4000.0 / 16500.0 * 12000.0
])
self.assertEqual(
ma.allclose(results, should_be, rtol=0.1), True,
"Error, should_be: %s,\n but result: %s" % (should_be, results))
def estimate(self,
spec_var=None,
spec_py=None,
movers_index=None,
submodel_string="",
alt_sample_size=None,
sampler="opus_core.samplers.weighted_sampler",
weight_string="supply",
aggregate_demand=False,
submarket_definition=('zone', 'building_type_id'),
sample_size_from_each_stratum=50):
"""
"""
t1 = time()
SimulationState().set_current_time(2000)
dataset_pool = SessionConfiguration().get_dataset_pool()
buildings = dataset_pool.get_dataset("building")
agent_set = dataset_pool.get_dataset('household')
#buildings.load_dataset()
submarket_geography = dataset_pool.get_dataset(submarket_definition[0])
intermediates = '[]'
if submarket_geography.dataset_name == 'zone':
intermediates = '[parcel]'
elif submarket_geography.dataset_name == 'faz':
intermediates = '[zone, parcel]'
elif submarket_geography.dataset_name == 'large_area':
intermediates = '[faz, zone, parcel]'
submarket_id_expression = 'building.disaggregate(%s.%s, intermediates=%s) * 100' % \
(submarket_geography.dataset_name, submarket_geography.get_id_name()[0],
intermediates)
submarket_variables = [
'%s=numpy.ceil(submarket.submarket_id / 100)' %
submarket_geography.get_id_name()[0]
]
if submarket_definition[1] == 'residential_building_type_id':
set_residential_building_types(
dataset_pool.get_dataset("building_type"),
dataset_pool.get_dataset("building"))
if submarket_definition[1] != '':
submarket_id_expression = submarket_id_expression + ' + building.%s' % submarket_definition[
1]
submarket_variables.append(submarket_definition[1] +
'=submarket.submarket_id % 100')
submarkets = define_submarket(
buildings,
submarket_id_expression,
#"urbansim_parcel.building.zone_id*100 + building.residential_building_type_id",
#"building.disaggregate(faz.large_area_id, intermediates=[zone, parcel]) * 100 + building.residential_building_type_id",
compute_variables=submarket_variables + [
"residential_units=submarket.aggregate(building.residential_units)",
"number_of_buildings_with_non_zero_units=submarket.aggregate(building.residential_units > 0 )",
"number_of_surveyed_households=submarket.aggregate(household.household_id > 5000000, intermediates=[building])",
],
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'submarket.supply > 0',
#"psrc_parcel.building.large_area_id*100 + building.residential_building_type_id",
#compute_variables=['residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id / 100)']
#"psrc_parcel.building.large_area_id",
#compute_variables=[#'residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id)']
)
dataset_pool.add_datasets_if_not_included({'submarket': submarkets})
compute_lambda_and_supply(buildings, agent_set, movers_index,
submarkets)
submarket_filter = 'submarket.supply > 0'
if submarket_filter is not None:
from numpy import logical_not
submarkets.remove_elements(index=where(
logical_not(submarkets.compute_variables(submarket_filter)))
[0])
submarkets.touch_attribute(submarkets.get_id_name()[0])
buildings.touch_attribute(submarkets.get_id_name()[0])
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(
in_table_name="household_location_choice_model_specification")
self.model_name = "household_location_choice_model"
agent_set, agents_index_for_estimation = get_households_for_estimation(
agent_set,
AttributeCache(),
"households_for_estimation",
exclude_condition=
"household.disaggregate(submarket.submarket_id, intermediates=[building])<=0",
)
agent_set.compute_variables(
"submarket_id=household.disaggregate(building.submarket_id)")
agent_sample_rate = agents_index_for_estimation.size / float(
movers_index.size)
dataset_pool.add_datasets_if_not_included(
{'sample_rate': agent_sample_rate})
if aggregate_demand:
location_set = buildings
aggregate_dataset = 'submarket'
#weight_string = 'inv_submarket_supply = 1.0 / (building.disaggregate(submarket.number_of_agents(building))).astype(float32) * (building.disaggregate(submarket.submarket_id) > 0)'
#weight_string = 'submarket_supply = (building.disaggregate(submarket.supply) > 0).astype(int32)'
#weight_string = 'submarket_supply = building.disaggregate(submarket.supply) * (building.disaggregate(submarket.submarket_id) > 0).astype(float32)'
else:
location_set = submarkets
aggregate_dataset = None
#weight_string = 'supply'
model = HouseholdLocationChoiceModelCreator().get_model(
location_set=location_set,
#location_set=submarkets,
#filter = 'building.disaggregate(submarket.submarket_id) > 0',
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'building.disaggregate(numpy.logical_and(submarket.number_of_buildings_with_non_zero_units > 5000, submarket.number_of_surveyed_households > 0))',
submodel_string=submodel_string,
sampler=sampler,
#estimation_size_agents = agent_sample_rate * 100/20,
# proportion of the agent set that should be used for the estimation
sample_size_locations=alt_sample_size,
#sample_proportion_locations = 1.0/1000,
# choice set size (includes current location)
compute_capacity_flag=True,
probabilities="opus_core.mnl_probabilities",
choices="urbansim.lottery_choices",
#run_config = Resources({"capacity_string":"supply"}),
estimate_config=Resources({
"capacity_string": "supply",
"weights_for_estimation_string": weight_string,
"aggregate_to_dataset": aggregate_dataset,
"stratum": "building.disaggregate(submarket.submarket_id)",
"sample_size_from_each_stratum": sample_size_from_each_stratum,
#"index2":where(submarkets.compute_variables('submarket.number_of_surveyed_households > 0'))[0],
#"sample_rate": 1.0/5000,
#"sample_size_from_chosen_stratum": 0,
"include_chosen_choice": True
}))
# was dataset_pool.add_datasets_if_not_included({'sample_rate':agent_sample_rate})
self.result = model.estimate(
self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops"
) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time() - t1) + " s")
