def test_same_distribution_after_household_subtraction(self):
"""Using the control_totals and no marginal characteristics,
subtract households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000]),
"total_number_of_households": array([20000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household", id_name="year")
storage.write_table(table_name='hc_set', table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = HouseholdTransitionModel()
model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 20000 total households after running the model
results = hh_set.size()
should_be = [20000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = [6000.0/33000.0*20000.0, 2000.0/33000.0*20000.0, 3000.0/33000.0*20000.0, 4000.0/33000.0*20000.0,
2000.0/33000.0*20000.0, 5000.0/33000.0*20000.0, 3000.0/33000.0*20000.0, 8000.0/33000.0*20000.0]
self.assertEqual(ma.allclose(results, should_be, rtol=0.05),
True, "Error, should_be: %s,\n but result: %s" % (should_be, results))
def test_unplaced_agents_decrease_available_space(self):
"""Using the household location choice model, create a set of available spaces and
2000 unplaced agents (along with 5000 placed agents). Run the model, and check that
the unplaced agents were placed, and the number of available spaces has decreased"""
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data={
'grid_id': array(2000 * [0] + 5000 * [1]),
'household_id': arange(7000) + 1
})
storage.write_table(table_name='gridcells',
table_data={
'residential_units': array(50 * [10000]),
'grid_id': arange(50) + 1
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
coefficients = Coefficients(names=("dummy", ), values=(0.1, ))
specification = EquationSpecification(
variables=("gridcell.residential_units", ),
coefficients=("dummy", ))
"""need to specify to the household location choice model exactly which households are moving,
because by default it assumes all current households want to move, but in this test,
the 5000 households already in gridcell #1 shouldn't move.
here, we specify that only the unplaced households should be moved."""
agents_index = where(households.get_attribute("grid_id") == 0)[0]
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
agents_index=agents_index,
debuglevel=1)
gridcells.compute_variables(
["urbansim.gridcell.vacant_residential_units"],
resources=Resources({"household": households}))
vacancies = gridcells.get_attribute("vacant_residential_units")
"""since there were 5000 households already in gridcell #1, and gridcell #1 has
10000 residential units, there should be no more than 5000 vacant residential units
in gridcell #1 after running this model"""
self.assertEqual(vacancies[0] <= 5000, True,
"Error: %d" % (vacancies[0], ))
"""there should be exactly 430000 vacant residential units after the model run,
because there were originally 50 gridcells with 10000 residential units each,
and a total of 7000 units are occupied after the run"""
self.assertEqual(
sum(vacancies) == 50 * 10000 - 7000, True,
"Error: %d" % (sum(vacancies)))
def test_controlling_with_three_marginal_characteristics(self):
"""Controlling with all three possible marginal characteristics in this example, age_of_head, income, and persons,
this would partition the 8 groups into the same 8 groups, and with a control total specified for each group, we must
ensure that the control totals for each group exactly meet the specifications.
"""
#IMPORTANT: marginal characteristics grouping indices have to start at 0!
annual_household_control_totals_data = {
"year": array(8*[2000]),
#"age_of_head": array(4*[0] + 4*[1]),
"age_of_head_min": array([ 0, 0, 0, 0, 50, 50, 50, 50]),
"age_of_head_max": array([49,49,49,49,100,100,100,100]),
#"income": array(2*[0] + 2*[1] + 2*[0] + 2*[1]),
"income_min": array([ 0, 0,40000,40000, 0, 0,40000,40000]),
"income_max": array([39999,39999, -1, -1,39999,39999, -1, -1]),
#"persons": array([0,1,0,1,0,1,0,1]),
"persons_min": array([0, 3,0, 3,0, 3,0, 3]),
"persons_max": array([2,-1,2,-1,2,-1,2,-1]),
"total_number_of_households": array([4000, 5000, 1000, 3000, 0, 6000, 3000, 8000])
}
##size of columns was not even, removed last element of min and max
#household_characteristics_for_ht_data = {
#"characteristic": array(2*['age_of_head'] + 2*['income'] + 2*['persons']),
#"min": array([0, 50, 0, 40000, 0, 3]),
#"max": array([49, 100, 39999, -1, 2, -1])
#}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household', id_name=[])
#storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
#hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
# unplace some households
where10 = where(hh_set.get_attribute("grid_id")<>10)[0]
hh_set.modify_attribute(name="grid_id", data=zeros(where10.size), index=where10)
model = TransitionModel(hh_set, control_total_dataset=hct_set)
model.run(year=2000, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
#check that there are indeed 33000 total households after running the model
results = hh_set.size()
should_be = [30000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of households in each group exactly match the control totals specified
results = self.get_count_all_groups(hh_set)
should_be = [4000, 5000, 1000, 3000, 0, 6000, 3000, 8000]
self.assertEqual(ma.allclose(results, should_be),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def run_ALCM(niter):
nhhs = 100
ngcs = 10
ngcs_attr = ngcs/2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs*[-1])
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(
table_name = households_table_name,
table_data = {
'household_id': arange(nhhs)+1,
'grid_id': hh_grid_ids
}
)
gridcells_table_name = 'gridcells'
storage.write_table(
table_name = gridcells_table_name,
table_data = {
'grid_id': arange(ngcs)+1,
'cost':array(ngcs_attr*[100]+ngcs_noattr*[1000])
}
)
households = HouseholdDataset(in_storage=storage, in_table_name=households_table_name)
gridcells = GridcellDataset(in_storage=storage, in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=('costcoef', ), values=(-0.001,))
specification = EquationSpecification(variables=('gridcell.cost', ), coefficients=('costcoef', ))
logger.be_quiet()
result = zeros((niter,ngcs))
for iter in range(niter):
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False,
choices = 'opus_core.random_choices_from_index',
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1,
chunk_specification={'nchunks':1})
# get results
gridcells.compute_variables(['urbansim.gridcell.number_of_households'],
resources=Resources({'household':households}))
result_more_attractive = gridcells.get_attribute_by_id('number_of_households', arange(ngcs_attr)+1)
result_less_attractive = gridcells.get_attribute_by_id('number_of_households', arange(ngcs_attr+1, ngcs+1))
households.set_values_of_one_attribute(attribute='grid_id', values=hh_grid_ids)
gridcells.delete_one_attribute('number_of_households')
result[iter,:] = concatenate((result_more_attractive, result_less_attractive))
#print result #, result_more_attractive.sum(), result_less_attractive.sum()
return result
def create_households_for_estimation(agent_set, dbcon):
estimation_set = HouseholdDataset(in_storage=StorageFactory().get_storage('mysql_storage', storage_location=dbcon),
in_table_name="households_for_estimation")
agent_set.unload_primary_attributes()
agent_set.load_dataset(attributes='*')
estimation_set.load_dataset(attributes=agent_set.get_primary_attribute_names())
for attr in agent_set.get_attribute_names():
agent_set.attribute_boxes[attr].set_data(concatenate((estimation_set.attribute_boxes[attr].get_data(), agent_set.attribute_boxes[attr].get_data())))
agent_set._update_id_mapping()
agent_set.update_size()
return (agent_set, arange(estimation_set.size()))
def test_controlling_with_one_marginal_characteristic(self):
"""Using the age_of_head as a marginal characteristic, which would partition the 8 groups into two larger groups
(those with age_of_head < 40 and >= 40), ensure that the control totals are met and that the distribution within
each large group is the same before and after running the model
"""
#IMPORTANT: marginal characteristics grouping indices have to start at 0!
#i.e. below, there is one marg. char. "age_of_head". here we indicate that the first "large group" (groups 1-4),
#consisting of those groups with age_of_head < 40 should total 25000 households after running this model for one year,
#and the second large group, those groups with age_of_head > 40, should total 15000 households
annual_household_control_totals_data = {
"year": array([2000, 2000]),
"age_of_head": array([0,1]),
"total_number_of_households": array([25000, 15000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household', id_name=['year' ,'age_of_head'])
storage.write_table(table_name='hc_set', table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
storage.write_table(table_name='prs_set', table_data=self.person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
model = HouseholdTransitionModel()
model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 40000 total households after running the model
results = hh_set.size()
should_be = [40000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the total number of households within first four groups increased by 10000
#and that the total number of households within last four groups decreased by 3000
results = self.get_count_all_groups(hh_set)
should_be = [25000, 15000]
self.assertEqual(ma.allclose([sum(results[0:4]), sum(results[4:8])], should_be, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households within groups 1-4 and 5-8 are the same before and after
#running the model, respectively
should_be = [6000.0/15000.0*25000.0, 2000.0/15000.0*25000.0, 3000.0/15000.0*25000.0, 4000.0/15000.0*25000.0,
2000.0/18000.0*15000.0, 5000.0/18000.0*15000.0, 3000.0/18000.0*15000.0, 8000.0/18000.0*15000.0]
self.assertEqual(ma.allclose(results, should_be, rtol=0.05),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_same_distribution_after_household_subtraction(self):
"""Using the control_totals and no marginal characteristics,
subtract households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000]),
"total_number_of_households": array([20000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set',
table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set',
table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage,
in_table_name='hct_set',
what="household",
id_name="year")
storage.write_table(
table_name='hc_set',
table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name='hc_set')
model = HouseholdTransitionModel()
model.run(year=2000,
household_set=hh_set,
control_totals=hct_set,
characteristics=hc_set)
#check that there are indeed 20000 total households after running the model
results = hh_set.size()
should_be = [20000]
self.assertEqual(
ma.allclose(should_be, results, rtol=1e-1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = [
6000.0 / 33000.0 * 20000.0, 2000.0 / 33000.0 * 20000.0,
3000.0 / 33000.0 * 20000.0, 4000.0 / 33000.0 * 20000.0,
2000.0 / 33000.0 * 20000.0, 5000.0 / 33000.0 * 20000.0,
3000.0 / 33000.0 * 20000.0, 8000.0 / 33000.0 * 20000.0
]
self.assertEqual(
ma.allclose(results, should_be, rtol=0.05), True,
"Error, should_be: %s,\n but result: %s" % (should_be, results))
def test_unplaced_agents_decrease_available_space(self):
"""Using the household location choice model, create a set of available spaces and
2000 unplaced agents (along with 5000 placed agents). Run the model, and check that
the unplaced agents were placed, and the number of available spaces has decreased"""
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data = {
'grid_id': array(2000*[0] + 5000*[1]),
'household_id': arange(7000)+1
}
)
storage.write_table(table_name='gridcells',
table_data= {
'residential_units':array(50*[10000]),
'grid_id': arange(50)+1
}
)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
gridcells = GridcellDataset(in_storage=storage, in_table_name='gridcells')
coefficients = Coefficients(names=("dummy",), values=(0.1,))
specification = EquationSpecification(variables=("gridcell.residential_units",), coefficients=("dummy",))
"""need to specify to the household location choice model exactly which households are moving,
because by default it assumes all current households want to move, but in this test,
the 5000 households already in gridcell #1 shouldn't move.
here, we specify that only the unplaced households should be moved."""
agents_index = where(households.get_attribute("grid_id") == 0)[0]
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells,
choices = "opus_core.random_choices_from_index", sample_size_locations = 30)
hlcm.run(specification, coefficients, agent_set=households, agents_index=agents_index, debuglevel=1)
gridcells.compute_variables(["urbansim.gridcell.vacant_residential_units"],
resources=Resources({"household":households}))
vacancies = gridcells.get_attribute("vacant_residential_units")
"""since there were 5000 households already in gridcell #1, and gridcell #1 has
10000 residential units, there should be no more than 5000 vacant residential units
in gridcell #1 after running this model"""
self.assertEqual(vacancies[0] <= 5000,
True, "Error: %d" % (vacancies[0],))
"""there should be exactly 430000 vacant residential units after the model run,
because there were originally 50 gridcells with 10000 residential units each,
and a total of 7000 units are occupied after the run"""
self.assertEqual(sum(vacancies) == 50 * 10000 - 7000,
True, "Error: %d" % (sum(vacancies)))
def create_households_for_estimation(agent_set, dbcon):
estimation_set = HouseholdDataset(
in_storage=StorageFactory().get_storage('mysql_storage',
storage_location=dbcon),
in_table_name="households_for_estimation")
agent_set.unload_primary_attributes()
agent_set.load_dataset(attributes='*')
estimation_set.load_dataset(
attributes=agent_set.get_primary_attribute_names())
for attr in agent_set.get_attribute_names():
agent_set.attribute_boxes[attr].set_data(
concatenate((estimation_set.attribute_boxes[attr].get_data(),
agent_set.attribute_boxes[attr].get_data())))
agent_set._update_id_mapping()
agent_set.update_size()
return (agent_set, arange(estimation_set.size()))
def run_model():
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
hlcm = RegionalHouseholdLocationChoiceModel(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=4)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_area1 = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_area2 = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
gridcells.delete_one_attribute("number_of_households")
result = concatenate((result_area1, result_area2))
return result
def run_model_2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
return array(
[result_more_attractive.sum(),
result_less_attractive.sum()])
def test_same_distribution_after_household_subtraction(self):
"""Using the control_totals and no marginal characteristics,
subtract households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000, 2000]),
"total_number_of_households": array([8000, 12000]),
"faz_id": array([1,2])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'hh_set', table_data = self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household")
storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
# storage.write_table(table_name='prs_set', table_data=self.person_data)
# prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
model = SubareaHouseholdTransitionModel(subarea_id_name="faz_id")
# model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 8000 (area 1) and 12000 (area 2) total households after running the model
areas = hh_set.get_attribute("faz_id")
results = array([0,0])
for iarea in [0,1]:
results[iarea] = where(areas == [1,2][iarea])[0].size
should_be = [8000, 12000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = array([# area 1
3000.0/16500.0*8000.0, 1000.0/16500.0*8000.0, 1500.0/16500.0*8000.0, 2000.0/16500.0*8000.0,
1000.0/16500.0*8000.0, 2500.0/16500.0*8000.0, 1500.0/16500.0*8000.0, 4000.0/16500.0*8000.0,
# area 2
3000.0/16500.0*12000.0, 1000.0/16500.0*12000.0, 1500.0/16500.0*12000.0, 2000.0/16500.0*12000.0,
1000.0/16500.0*12000.0, 2500.0/16500.0*12000.0, 1500.0/16500.0*12000.0, 4000.0/16500.0*12000.0])
self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
True, "Error, should_be: %s,\n but result: %s" % (should_be, results))
def test_same_distribution_after_household_subtraction(self):
"""Using the control_totals and no marginal characteristics,
subtract households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000, 2000]),
"total_number_of_households": array([8000, 12000]),
"faz_id": array([1,2])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'hh_set', table_data = self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household")
storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
# storage.write_table(table_name='prs_set', table_data=self.person_data)
# prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
model = SubareaHouseholdTransitionModel(subarea_id_name="faz_id")
# model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 8000 (area 1) and 12000 (area 2) total households after running the model
areas = hh_set.get_attribute("faz_id")
results = array([0,0])
for iarea in [0,1]:
results[iarea] = where(areas == [1,2][iarea])[0].size
should_be = [8000, 12000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = array([# area 1
3000.0/16500.0*8000.0, 1000.0/16500.0*8000.0, 1500.0/16500.0*8000.0, 2000.0/16500.0*8000.0,
1000.0/16500.0*8000.0, 2500.0/16500.0*8000.0, 1500.0/16500.0*8000.0, 4000.0/16500.0*8000.0,
# area 2
3000.0/16500.0*12000.0, 1000.0/16500.0*12000.0, 1500.0/16500.0*12000.0, 2000.0/16500.0*12000.0,
1000.0/16500.0*12000.0, 2500.0/16500.0*12000.0, 1500.0/16500.0*12000.0, 4000.0/16500.0*12000.0])
self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
True, "Error, should_be: %s,\n but result: %s" % (should_be, results))
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
edges_table_name = 'edges'
storage.write_table(
table_name=edges_table_name,
table_data={
"source": array([10, 20, 10, 30]),
"target": array([2, 3, 3, 1]),
"cost": array([12, 1, 15, 17])
},
)
parcels_table_name = 'parcels'
storage.write_table(
table_name=parcels_table_name,
table_data={"parcel_id": array([1, 2, 3])},
)
households_table_name = 'households'
storage.write_table(
table_name=households_table_name,
table_data={
"household_id": array([1, 2, 3, 4]),
"parcel_id": array([3, 1, 1, 2]),
"work_place_parcel_id": array([10, 30, 30, 20])
},
)
parcels = ParcelDataset(in_storage=storage,
in_table_name=parcels_table_name)
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
household_x_parcels = HouseholdXParcelDataset(dataset1=households,
dataset2=parcels)
dataset_pool = DatasetPool(
package_order=['transit_accessibility', 'psrc', 'urbansim'],
storage=storage)
values = household_x_parcels.compute_variables(
self.variable_name, dataset_pool=dataset_pool)
default_value = travel_time_hbw_transit_from_work_to_home.default_value
should_be = array([[default_value, 12, 15],
[17, default_value, default_value],
[17, default_value, default_value],
[default_value, default_value, 1]])
self.assert_(ma.allclose(values, should_be, rtol=1e-10),
"Error in " + self.variable_name)
def test_same_distribution_after_household_addition(self):
"""Using the control_totals and no marginal characteristics,
add households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000, 2000]),
"total_number_of_households": array([20000, 30000]),
"large_area_id": array([1,2])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'hh_set', table_data = self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household")
storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = RegionalHouseholdTransitionModel()
model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are 20000 (area 1) and 30000 (area 2) total households after running the model
areas = hh_set.get_attribute("large_area_id")
results = array([0,0])
for iarea in [0,1]:
results[iarea] = where(areas == [1,2][iarea])[0].size
should_be = [20000, 30000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of unplaced households is exactly the number of new households created
results = where(hh_set.get_attribute("grid_id")<=0)[0].size
should_be = [17000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group and each area is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = array([
# area 1
3000.0/16500.0*20000.0, 1000.0/16500.0*20000.0, 1500.0/16500.0*20000.0, 2000.0/16500.0*20000.0,
1000.0/16500.0*20000.0, 2500.0/16500.0*20000.0, 1500.0/16500.0*20000.0, 4000.0/16500.0*20000.0,
# area 2
3000.0/16500.0*30000.0, 1000.0/16500.0*30000.0, 1500.0/16500.0*30000.0, 2000.0/16500.0*30000.0,
1000.0/16500.0*30000.0, 2500.0/16500.0*30000.0, 1500.0/16500.0*30000.0, 4000.0/16500.0*30000.0])
self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# check the types of the attributes
self.assertEqual(hh_set.get_attribute("age_of_head").dtype, int32,
"Error in data type of the new household set. Should be: int32, is: %s" % str(hh_set.get_attribute("age_of_head").dtype))
self.assertEqual(hh_set.get_attribute("income").dtype, int32,
"Error in data type of the new household set. Should be: int32, is: %s" % str(hh_set.get_attribute("income").dtype))
self.assertEqual(hh_set.get_attribute("persons").dtype, int8,
"Error in data type of the new household set. Should be: int8, is: %s" % str(hh_set.get_attribute("persons").dtype))
def __init__(self):
in_storage = StorageFactory().get_storage(
'sql_storage',
hostname=settings.get_db_host_name(),
username=settings.get_db_user_name(),
password=settings.get_db_password(),
database_name=settings.db)
hhs = HouseholdDataset(in_storage=in_storage, nchunks=3)
print "Read and Write HouseholdDataset."
out_storage = StorageFactory().build_storage_for_dataset(
type='flt_storage', storage_location=settings.dir)
ReadWriteADataset(hhs,
out_storage=out_storage,
out_table_name=settings.hhsubdir)
def test_agents_do_not_go_to_inferior_locations(self):
"""100 gridcells - 99 with attractiveness 2000, 1 with attractiveness 1, no capacity restrictions
10,000 households
We set the coefficient value for attracitiveness to 1.
"""
storage = StorageFactory().get_storage('dict_storage')
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(10000) + 1,
'grid_id': array(10000 * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(100) + 1,
'attractiveness': array(99 * [2000] + [1])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("attractcoef", ), values=(1, ))
specification = EquationSpecification(
variables=("gridcell.attractiveness", ),
coefficients=("attractcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute_by_id("number_of_households", 100)
# nobody should choose gridcell 100
self.assertEqual(result, 0, "Error: %s is not equal to 0" % (result, ))
def test_same_distribution_after_household_addition(self):
"""Using the control_totals and no marginal characteristics,
add households and ensure that the distribution within each group stays the same
"""
annual_household_control_totals_data = {
"year": array([2000]),
"total_number_of_households": array([50000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='prs_set', table_data=self.person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household", id_name="year")
storage.write_table(table_name='hc_set', table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = HouseholdTransitionModel()
model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 50000 total households after running the model
results = hh_set.size()
should_be = [50000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of unplaced households is exactly the number of new households created
results = where(hh_set.get_attribute("building_id")<=0)[0].size
should_be = [17000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the distribution of households in each group is the same as before running the model
results = self.get_count_all_groups(hh_set)
should_be = array([6000.0/33000.0*50000.0, 2000.0/33000.0*50000.0, 3000.0/33000.0*50000.0, 4000.0/33000.0*50000.0,
2000.0/33000.0*50000.0, 5000.0/33000.0*50000.0, 3000.0/33000.0*50000.0, 8000.0/33000.0*50000.0])
self.assertEqual(ma.allclose(results, should_be, rtol=0.05),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# check the types of the attributes
self.assertEqual(hh_set.get_attribute("age_of_head").dtype, int32,
"Error in data type of the new household set. Should be: int32, is: %s" % str(hh_set.get_attribute("age_of_head").dtype))
self.assertEqual(hh_set.get_attribute("income").dtype, int32,
"Error in data type of the new household set. Should be: int32, is: %s" % str(hh_set.get_attribute("income").dtype))
self.assertEqual(hh_set.get_attribute("persons").dtype, int8,
"Error in data type of the new household set. Should be: int8, is: %s" % str(hh_set.get_attribute("persons").dtype))
def test_do_nothing_if_no_agents(self):
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(table_name=households_table_name,
table_data={
"household_id": arange(10000) + 1,
"grid_id": array(10000 * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
gridcells_table_name = 'gridcells'
storage.write_table(table_name=gridcells_table_name,
table_data={
"grid_id": arange(100) + 1,
"cost": array(50 * [100] + 50 * [1000])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
agents_index=array([], dtype='int32'),
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute("number_of_households")
# check the individual gridcells
self.assertEqual(ma.allclose(result, zeros((100, )), rtol=0), True)
def xtest_gracefully_handle_empty_choice_sets(self):
storage = StorageFactory().get_storage('dict_storage')
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(10000) + 1,
'grid_id': array(100 * range(100)) + 1
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id': arange(100) + 1,
'residential_units': array(100 * [100])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("dummy", ), values=(0, ))
specification = EquationSpecification(
variables=("gridcell.residential_units", ),
coefficients=("dummy", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources(
{"household": households}))
result = gridcells.get_attribute_by_id("number_of_households", 100)
# nobody should choose gridcell 100
self.assertEqual(ma.allclose(result.sum(), 0, rtol=0), True,
"Error: %s is not equal to 0" % (result.sum(), ))
def test_controlling_with_three_marginal_characteristics(self):
"""Controlling with all three possible marginal characteristics in this example, age_of_head, income, and persons,
this would partition the 8 groups into the same 8 groups, and with a control total specified for each group, we must
ensure that the control totals for each group exactly meet the specifications.
"""
#IMPORTANT: marginal characteristics grouping indices have to start at 0!
annual_household_control_totals_data = {
"year": array(8*[2000]),
"age_of_head": array(4*[0] + 4*[1]),
"income": array(2*[0] + 2*[1] + 2*[0] + 2*[1]),
"persons": array([0,1,0,1,0,1,0,1]),
"total_number_of_households": array([4000, 5000, 1000, 3000, 0, 6000, 3000, 8000])
}
#size of columns was not even, removed last element of min and max
household_characteristics_for_ht_data = {
"characteristic": array(2*['age_of_head'] + 2*['income'] + 2*['persons']),
"min": array([0, 50, 0, 40000, 0, 3]),
"max": array([49, 100, 39999, -1, 2, -1])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household', id_name=['year' ,'age_of_head', 'income', 'persons'])
storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
# unplace some households
where10 = where(hh_set.get_attribute("building_id")<>10)[0]
hh_set.modify_attribute(name="building_id", data=zeros(where10.size), index=where10)
storage.write_table(table_name='prs_set', table_data=self.person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
model = HouseholdTransitionModel()
model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 33000 total households after running the model
results = hh_set.size()
should_be = [30000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of households in each group exactly match the control totals specified
results = self.get_count_all_groups(hh_set)
should_be = [4000, 5000, 1000, 3000, 0, 6000, 3000, 8000]
self.assertEqual(ma.allclose(results, should_be),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_person_dataset(self):
households_data = {
"household_id":arange(4)+1,
"building_id": array([3,6,1,2], dtype=int32),
"persons": array([1,2,2,4], dtype=int32)
}
household_characteristics_for_ht_data = {
"characteristic": array(2*['persons']),
"min": array([1, 3]),
"max": array([2,-1])
}
person_data = {
"person_id": arange(9)+1,
"household_id": array([1,2,2,3,3,4,4,4,4]),
"job_id": array([30, 50, 0, 1, 23, 54, 78, 2, 6]),
}
annual_household_control_totals_data = {
"year": array(2*[2000]),
"persons": array([0,1]),
"total_number_of_households": array([0, 4])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='prs_set', table_data=person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household", id_name=["year", "persons"])
storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = HouseholdTransitionModel(debuglevel=3)
model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
# The run should remove the first three households and first 5 persons and add 3 copies of the last household, i.e. 12 persons
self.assertEqual(prs_set.size(), 16, "Error in size of the person_set. Should be 16, is %s." % prs_set.size())
self.assertEqual(ma.allequal(prs_set.get_attribute('household_id'), array([4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7])), True,
"Error in assigning household_id to new persons.")
def __init__(self,
variables=(),
coefficients=(),
submodels=(),
debuglevel=0):
# neighborhoods, jobs and households are loaded from disk ('flt_storage'), other objects from mysql. Change it as you need.
self.nbs = NeighborhoodDataset(in_storage=StorageFactory().get_storage('flt_storage', storage_location=This_Settings.dir),
in_place=This_Settings.nbsubdir, \
out_storage=StorageFactory().get_storage('flt_storage', storage_location = This_Settings.outputdir),
debuglevel=debuglevel)
self.jobs = JobDataset(in_base=This_Settings.dir,
in_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=This_Settings.dir),
out_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=This_Settings.outputdir),
in_place=This_Settings.jobsubdir,
debuglevel=debuglevel)
Con = OpusDatabase(hostname=DB_settings.db_host_name,
username=DB_settings.db_user_name,
password=DB_settings.db_password,
database_name=This_Settings.db)
self.hhs = HouseholdDataset(
in_base=This_Settings.dir,
in_storage=StorageFactory().get_storage(
'flt_storage', storage_location=This_Settings.dir),
out_storage=StorageFactory().get_storage(
'flt_storage', storage_location=This_Settings.outputdir),
in_place=This_Settings.hhsubdir,
debuglevel=debuglevel)
self.resources = Resources({"household": self.hhs, "job": self.jobs})
#simulate
Paris_simulation().run(nbs=self.nbs, jobs=self.jobs, \
data_objects=self.resources,\
dbcon=Con, variables=variables, coefficients=coefficients, submodels=submodels, debuglevel=debuglevel)
Con.close_connection()
def run_model2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification,
coefficients,
agent_set=households,
run_config=Resources(
{"demand_string": "gridcell.housing_demand"}))
return gridcells.get_attribute("housing_demand")
def run_model1():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=30)
hlcm.run(specification, coefficients, agent_set=households)
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
return gridcells.get_attribute("number_of_households")
def run_HTM(niter):
nhhs = 5000
ngroups = 4
nhhsg = int(nhhs / ngroups)
nhhslg = nhhs - (ngroups - 1) * nhhsg
should_nhhs = nhhs - 2000
storage = StorageFactory().get_storage('dict_storage')
hc_set_table_name = 'hc_set'
storage.write_table(
table_name=hc_set_table_name,
table_data={
'characteristic': array(4 * ['income'] + 4 * ['age_of_head']),
'min': array([0, 1001, 5001, 10001, 0, 31, 41, 61]),
'max': array([1000, 5000, 10000, -1, 30, 40, 60, -1])
},
)
hct_set_table_name = 'hct_set'
storage.write_table(
table_name=hct_set_table_name,
table_data={
'year': array([2000]),
'total_number_of_households': array([should_nhhs])
},
)
households_table_name = 'households'
storage.write_table(
table_name=households_table_name,
table_data={
'age_of_head':
array(nhhsg / 2 * [18] + (nhhsg - nhhsg / 2) * [35] +
nhhsg / 2 * [30] + (nhhsg - nhhsg / 2) * [40] +
nhhsg / 2 * [38] + (nhhsg - nhhsg / 2) * [65] +
nhhslg / 2 * [50] + (nhhslg - nhhslg / 2) * [80]),
'income':
array(nhhsg * [500] + nhhsg * [2000] + nhhsg * [7000] +
nhhslg * [15000]),
'household_id':
arange(nhhs) + 1
},
)
hc_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name=hc_set_table_name)
hct_set = ControlTotalDataset(in_storage=storage,
in_table_name=hct_set_table_name,
what='household',
id_name=['year'])
logger.be_quiet()
result = zeros((niter, 4))
for iter in range(niter):
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
model = HouseholdTransitionModel()
model.run(year=2000,
household_set=households,
control_totals=hct_set,
characteristics=hc_set)
income = households.get_attribute('income')
age = households.get_attribute('age_of_head')
idx1 = where(income <= 1000)[0]
idx2 = where(logical_and(income <= 5000, income > 1000))[0]
idx3 = where(logical_and(income <= 10000, income > 5000))[0]
idx4 = where(income > 10000)[0]
result[iter, :] = array([
age[idx1].mean(), age[idx2].mean(), age[idx3].mean(),
age[idx4].mean()
])
return result
def test_controlling_age_of_head(self):
""" Controls for one marginal characteristics, namely age_of_head.
"""
annual_household_control_totals_data = {
"year": array([2000, 2000, 2000, 2001, 2001, 2001, 2002, 2002, 2002]),
"age_of_head": array([0,1,2,0,1,2, 0,1,2]),
"total_number_of_households": array([25013, 21513, 18227, # 2000
10055, 15003, 17999, # 2001
15678, 14001, 20432]) # 2002
}
household_characteristics_for_ht_data = {
"characteristic": array(3*['age_of_head']),
"min": array([0, 35, 65]),
"max": array([34, 64, -1])
}
households_data = {
"household_id":arange(15000)+1,
"building_id": array(15000*[1]),
"age_of_head": array(1000*[25] + 1000*[28] + 2000*[32] + 1000*[34] +
2000*[35] + 1000*[40] + 1000*[54]+ 1000*[62] +
1000*[65] + 1000*[68] + 2000*[71] + 1000*[98]),
"persons": array(1000*[2] + 2000*[3] + 1000*[1] + 1000*[6] + 1000*[1] + 1000*[4] +
3000*[1]+ 5000*[5], dtype=int8)
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household',
id_name=['year' ,'age_of_head'])
storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
storage.write_table(table_name='prs_set', table_data=self.person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
model = HouseholdTransitionModel(debuglevel=3)
# this run should add households in all four categories
model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[0:3]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(hc_set.size(), dtype=int32)
results[0] = where(hh_set.get_attribute('age_of_head') <= hc_set.get_attribute("max")[0], 1,0).sum()
for i in range(1, hc_set.size()-1):
results[i] = logical_and(where(hh_set.get_attribute('age_of_head') >= hc_set.get_attribute("min")[i], 1,0),
where(hh_set.get_attribute('age_of_head') <= hc_set.get_attribute("max")[i], 1,0)).sum()
results[hc_set.size()-1] = where(hh_set.get_attribute('age_of_head') >= hc_set.get_attribute("min")[hc_set.size()-1], 1,0).sum()
should_be = hct_set.get_attribute("total_number_of_households")[0:3]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should remove households in all four categories
model.run(year=2001, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[3:6]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(hc_set.size(), dtype=int32)
results[0] = where(hh_set.get_attribute('age_of_head') <= hc_set.get_attribute("max")[0], 1,0).sum()
for i in range(1, hc_set.size()-1):
results[i] = logical_and(where(hh_set.get_attribute('age_of_head') >= hc_set.get_attribute("min")[i], 1,0),
where(hh_set.get_attribute('age_of_head') <= hc_set.get_attribute("max")[i], 1,0)).sum()
results[hc_set.size()-1] = where(hh_set.get_attribute('age_of_head') >= hc_set.get_attribute("min")[hc_set.size()-1], 1,0).sum()
should_be = hct_set.get_attribute("total_number_of_households")[3:6]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should add and remove households
model.run(year=2002, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[6:9]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(hc_set.size(), dtype=int32)
results[0] = where(hh_set.get_attribute('age_of_head') <= hc_set.get_attribute("max")[0], 1,0).sum()
for i in range(1, hc_set.size()-1):
results[i] = logical_and(where(hh_set.get_attribute('age_of_head') >= hc_set.get_attribute("min")[i], 1,0),
where(hh_set.get_attribute('age_of_head') <= hc_set.get_attribute("max")[i], 1,0)).sum()
results[hc_set.size()-1] = where(hh_set.get_attribute('age_of_head') >= hc_set.get_attribute("min")[hc_set.size()-1], 1,0).sum()
should_be = hct_set.get_attribute("total_number_of_households")[6:9]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_controlling_income(self):
""" Controls for one marginal characteristics, namely income.
"""
annual_household_control_totals_data = {
"year": array([2000, 2000, 2000, 2000, 2001, 2001, 2001, 2001, 2002, 2002, 2002, 2002]),
"income": array([0,1,2,3,0,1,2,3, 0,1,2,3]),
"total_number_of_households": array([25013, 21513, 18227, 18493, # 2000
10055, 15003, 17999, 17654, # 2001
15678, 14001, 20432, 14500]) # 2002
}
household_characteristics_for_ht_data = {
"characteristic": array(4*['income']),
"min": array([0, 40000, 120000, 70000]), # category 120000 has index 3 and category 70000 has index 2
"max": array([39999, 69999, -1, 119999]) # (testing row invariance)
}
hc_sorted_index = array([0,1,3,2])
households_data = {
"household_id":arange(20000)+1,
"building_id": array(19950*[1] + 50*[0]),
"income": array(1000*[1000] + 1000*[10000] + 2000*[20000] + 1000*[35000] + 2000*[45000] +
1000*[50000] + 2000*[67000]+ 2000*[90000] + 1000*[100005] + 2000*[110003] +
1000*[120000] + 1000*[200000] + 2000*[500000] + 1000*[630000]),
"persons": array(3000*[2] + 2000*[3] + 1000*[1] + 1000*[6] + 1000*[1] + 1000*[4] +
3000*[1]+ 8000*[5], dtype=int8)
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household', id_name=['year' ,'income'])
storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
storage.write_table(table_name='prs_set', table_data=self.person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
model = HouseholdTransitionModel(debuglevel=3)
# this run should add households in all four categories
model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
results = hh_set.size()
should_be = [83246]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(hc_set.size(), dtype=int32)
results[0] = where(hh_set.get_attribute('income') <=
hc_set.get_attribute("max")[hc_sorted_index[0]], 1,0).sum()
for i in range(1, hc_set.size()-1):
results[i] = logical_and(where(hh_set.get_attribute('income') >=
hc_set.get_attribute("min")[hc_sorted_index[i]], 1,0),
where(hh_set.get_attribute('income') <=
hc_set.get_attribute("max")[hc_sorted_index[i]], 1,0)).sum()
results[-1] = where(hh_set.get_attribute('income') >= hc_set.get_attribute("min")[hc_sorted_index[-1]], 1,0).sum()
should_be = hct_set.get_attribute("total_number_of_households")[0:4]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should remove households in all four categories
model.run(year=2001, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[4:8]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(hc_set.size(), dtype=int32)
results[0] = where(hh_set.get_attribute('income') <=
hc_set.get_attribute("max")[hc_sorted_index[0]], 1,0).sum()
for i in range(1, hc_set.size()-1):
results[i] = logical_and(where(hh_set.get_attribute('income') >=
hc_set.get_attribute("min")[hc_sorted_index[i]], 1,0),
where(hh_set.get_attribute('income') <=
hc_set.get_attribute("max")[hc_sorted_index[i]], 1,0)).sum()
results[-1] = where(hh_set.get_attribute('income') >= hc_set.get_attribute("min")[hc_sorted_index[-1]], 1,0).sum()
should_be = hct_set.get_attribute("total_number_of_households")[4:8]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should add and remove households
model.run(year=2002, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[8:13]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(hc_set.size(), dtype=int32)
results[0] = where(hh_set.get_attribute('income') <= hc_set.get_attribute("max")[hc_sorted_index[0]], 1,0).sum()
for i in range(1, hc_set.size()-1):
results[i] = logical_and(where(hh_set.get_attribute('income') >=
hc_set.get_attribute("min")[hc_sorted_index[i]], 1,0),
where(hh_set.get_attribute('income') <=
hc_set.get_attribute("max")[hc_sorted_index[i]], 1,0)).sum()
results[-1] = where(hh_set.get_attribute('income') >= hc_set.get_attribute("min")[hc_sorted_index[-1]], 1,0).sum()
should_be = hct_set.get_attribute("total_number_of_households")[8:13]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_controlling_income(self):
""" Controls for one marginal characteristics, namely income.
"""
annual_household_control_totals_data = {
"year": array([2000, 2000, 2000, 2000, 2001, 2001, 2001, 2001, 2002, 2002, 2002, 2002]),
#"income": array([0,1,2,3,0,1,2,3, 0,1,2,3]),
"income_min": array([ 0,40000, 70000,120000, 0,40000, 70000,120000, 0,40000, 70000,120000]),
"income_max": array([39999,69999,119999, -1, 39999,69999,119999, -1, 39999,69999,119999, -1]),
"total_number_of_households": array([25013, 21513, 18227, 18493, # 2000
10055, 15003, 17999, 17654, # 2001
15678, 14001, 20432, 14500]) # 2002
}
#household_characteristics_for_ht_data = {
#"characteristic": array(4*['income']),
#"min": array([0, 40000, 120000, 70000]), # category 120000 has index 3 and category 70000 has index 2
#"max": array([39999, 69999, -1, 119999]) # (testing row invariance)
#}
#hc_sorted_index = array([0,1,3,2])
households_data = {
"household_id":arange(20000)+1,
"grid_id": array(19950*[1] + 50*[0]),
"income": array(1000*[1000] + 1000*[10000] + 2000*[20000] + 1000*[35000] + 2000*[45000] +
1000*[50000] + 2000*[67000]+ 2000*[90000] + 1000*[100005] + 2000*[110003] +
1000*[120000] + 1000*[200000] + 2000*[500000] + 1000*[630000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household', id_name=[])
#storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
#hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = TransitionModel(hh_set, control_total_dataset=hct_set)
model.run(year=2000, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
results = hh_set.size()
should_be = [83246]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
cats = 4
results = zeros(cats, dtype=int32)
results[0] = (hh_set.get_attribute('income') <= hct_set.get_attribute("income_max")[0]).sum()
for i in range(1, cats-1):
results[i] = logical_and(hh_set.get_attribute('income') >= hct_set.get_attribute("income_min")[i],
hh_set.get_attribute('income') <= hct_set.get_attribute("income_max")[i]).sum()
results[-1] = (hh_set.get_attribute('income') >= hct_set.get_attribute("income_min")[i+1]).sum()
should_be = hct_set.get_attribute("total_number_of_households")[0:4]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should remove households in all four categories
#model.run(year=2001, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2001, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[4:8]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(cats, dtype=int32)
results[0] = (hh_set.get_attribute('income') <= hct_set.get_attribute("income_max")[4]).sum()
for i in range(1, cats-1):
results[i] = logical_and(hh_set.get_attribute('income') >= hct_set.get_attribute("income_min")[i+4],
hh_set.get_attribute('income') <= hct_set.get_attribute("income_max")[i+4]).sum()
results[-1] = (hh_set.get_attribute('income') >= hct_set.get_attribute("income_min")[i+5]).sum()
should_be = hct_set.get_attribute("total_number_of_households")[4:8]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should add and remove households
#model.run(year=2002, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2002, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[8:12]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(cats, dtype=int32)
results[0] = (hh_set.get_attribute('income') <= hct_set.get_attribute("income_max")[8]).sum()
for i in range(1, cats-1):
results[i] = logical_and(hh_set.get_attribute('income') >= hct_set.get_attribute("income_min")[i+8],
hh_set.get_attribute('income') <= hct_set.get_attribute("income_max")[i+8]).sum()
results[-1] = (hh_set.get_attribute('income') >= hct_set.get_attribute("income_min")[i+9]).sum()
should_be = hct_set.get_attribute("total_number_of_households")[8:12]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_controlling_age_of_head(self):
""" Controls for one marginal characteristics, namely age_of_head.
"""
annual_household_control_totals_data = {
"year": array([2000, 2000, 2000, 2001, 2001, 2001, 2002, 2002, 2002]),
#"age_of_head": array([0,1,2,0,1,2, 0,1,2]),
"age_of_head_min": array([ 0,35,65, 0,35,65, 0,35,65]),
"age_of_head_max": array([34,64,-1, 34,64,-1, 34,64,-1]),
"total_number_of_households": array([25013, 21513, 18227, # 2000
10055, 15003, 17999, # 2001
15678, 14001, 20432]) # 2002
}
#household_characteristics_for_ht_data = {
#"characteristic": array(3*['age_of_head']),
#"min": array([0, 35, 65]),
#"max": array([34, 64, -1])
#}
households_data = {
"household_id":arange(15000)+1,
"grid_id": array(15000*[1]),
"age_of_head": array(1000*[25] + 1000*[28] + 2000*[32] + 1000*[34] +
2000*[35] + 1000*[40] + 1000*[54]+ 1000*[62] +
1000*[65] + 1000*[68] + 2000*[71] + 1000*[98])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set', table_data=households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set', table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household',
id_name=[])
#storage.write_table(table_name='hc_set', table_data=household_characteristics_for_ht_data)
#hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = TransitionModel(hh_set, control_total_dataset=hct_set)
model.run(year=2000, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[0:3]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
cats = 3
results = zeros(cats, dtype=int32)
results[0] = (hh_set.get_attribute('age_of_head') <= hct_set.get_attribute("age_of_head_max")[0]).sum()
for i in range(1, cats-1):
results[i] = logical_and(hh_set.get_attribute('age_of_head') >= hct_set.get_attribute("age_of_head_min")[i],
hh_set.get_attribute('age_of_head') <= hct_set.get_attribute("age_of_head_max")[i]).sum()
results[-1] = (hh_set.get_attribute('age_of_head') >= hct_set.get_attribute("age_of_head_min")[i+1]).sum()
should_be = hct_set.get_attribute("total_number_of_households")[0:3]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should remove households in all four categories
#model.run(year=2001, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2001, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[3:6]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(cats, dtype=int32)
results[0] = (hh_set.get_attribute('age_of_head') <= hct_set.get_attribute("age_of_head_max")[0]).sum()
for i in range(1, cats-1):
results[i] = logical_and(hh_set.get_attribute('age_of_head') >= hct_set.get_attribute("age_of_head_min")[i+3],
hh_set.get_attribute('age_of_head') <= hct_set.get_attribute("age_of_head_max")[i+3]).sum()
results[-1] = (hh_set.get_attribute('age_of_head') >= hct_set.get_attribute("age_of_head_min")[i+4]).sum()
should_be = hct_set.get_attribute("total_number_of_households")[3:6]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
# this run should add and remove households
#model.run(year=2002, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2002, target_attribute_name="total_number_of_households", reset_dataset_attribute_value={'grid_id':-1})
results = hh_set.size()
should_be = [(hct_set.get_attribute("total_number_of_households")[6:9]).sum()]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
results = zeros(cats, dtype=int32)
results[0] = where(hh_set.get_attribute('age_of_head') <= hct_set.get_attribute("age_of_head_max")[0], 1,0).sum()
for i in range(1, cats-1):
results[i] = logical_and(hh_set.get_attribute('age_of_head') >= hct_set.get_attribute("age_of_head_min")[i+6],
hh_set.get_attribute('age_of_head') <= hct_set.get_attribute("age_of_head_max")[i+6]).sum()
results[-1] = (hh_set.get_attribute('age_of_head') >= hct_set.get_attribute("age_of_head_min")[i+7]).sum()
should_be = hct_set.get_attribute("total_number_of_households")[6:9]
self.assertEqual(ma.allclose(results, should_be, rtol=1e-6),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_controlling_with_one_marginal_characteristic(self):
"""Using the age_of_head as a marginal characteristic, which would partition the 8 groups into two larger groups
(those with age_of_head < 40 and >= 40), ensure that the control totals are met and that the distribution within
each large group is the same before and after running the model
"""
#IMPORTANT: marginal characteristics grouping indices have to start at 0!
#i.e. below, there is one marg. char. "age_of_head". here we indicate that the first "large group" (groups 1-4),
#consisting of those groups with age_of_head < 40 should total 25000 households after running this model for one year,
#and the second large group, those groups with age_of_head > 40, should total 15000 households
annual_household_control_totals_data = {
"year": array([2000, 2000, 2000, 2000]),
"age_of_head": array([0, 1, 0, 1]),
"total_number_of_households": array([20000, 10000, 5000, 5000]),
"faz_id": array([1, 1, 2, 2] )
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'hh_set', table_data = self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household')
# storage.write_table(table_name='prs_set', table_data=self.person_data)
# prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')
model = SubareaHouseholdTransitionModel(subarea_id_name="faz_id")
# model.run(year=2000, person_set=prs_set, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)
#check that there are indeed 40000 total households after running the model
areas = hh_set.get_attribute("faz_id")
results = array([0,0])
for iarea in [0,1]:
results[iarea] = where(areas == [1,2][iarea])[0].size
should_be = [30000, 10000]
self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of households within the groups correspond to the control totals
results = self.get_count_all_groups(hh_set)
should_be = [20000, 10000, 5000, 5000]
idx1 = arange(0,4)
idx2 = arange(4,8)
idx3 = arange(8,12)
idx4 = arange(12,16)
self.assertEqual(ma.allclose([results[idx1].sum(), results[idx2].sum(), results[idx3].sum(), results[idx4].sum()], should_be, rtol=1e-1),
True, "Error, should_be: %s, but result: %s" % (should_be,
array([results[idx1].sum(), results[idx2].sum(), results[idx3].sum(), results[idx4].sum()])))
#check that the distribution of households within the groups are the same before and after
#running the model, respectively
should_be = [# area 1
3000.0/7500.0*20000.0, 1000.0/7500.0*20000.0, 1500.0/7500.0*20000.0, 2000.0/7500.0*20000.0,
1000.0/9000.0*10000.0, 2500.0/9000.0*10000.0, 1500.0/9000.0*10000.0, 4000.0/9000.0*10000.0,
# area 2
3000.0/7500.0*5000.0, 1000.0/7500.0*5000.0, 1500.0/7500.0*5000.0, 2000.0/7500.0*5000.0,
1000.0/9000.0*5000.0, 2500.0/9000.0*5000.0, 1500.0/9000.0*5000.0, 4000.0/9000.0*5000.0]
self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
True, "Error, should_be: %s, but result: %s" % (should_be, results))
def get_resources(self, data_dictionary, dataset):
"""Create resources for computing a variable. """
resources = Resources()
for key in data_dictionary.keys():
if key in self.datasets:
data = data_dictionary[key]
storage = StorageFactory().get_storage('dict_storage')
if self.id_names[key] not in data_dictionary[key].keys(
) and not isinstance(self.id_names[key], list):
data[self.id_names[key]] = arange(
1,
len(data_dictionary[key][data_dictionary[key].keys()
[0]]) + 1) # add id array
id_name = self.id_names[key]
storage.write_table(table_name='data', table_data=data)
if key == "gridcell":
gc = GridcellDataset(in_storage=storage,
in_table_name='data')
# add relative_x and relative_y
gc.get_id_attribute()
n = int(ceil(sqrt(gc.size())))
if "relative_x" not in data.keys():
x = (indices((n, n)) + 1)[1].ravel()
gc.add_attribute(x[0:gc.size()],
"relative_x",
metadata=1)
if "relative_y" not in data.keys():
y = (indices((n, n)) + 1)[0].ravel()
gc.add_attribute(y[0:gc.size()],
"relative_y",
metadata=1)
resources.merge({key: gc})
elif key == "household":
resources.merge({
key:
HouseholdDataset(in_storage=storage,
in_table_name='data')
})
elif key == "development_project":
resources.merge({
key:
DevelopmentProjectDataset(in_storage=storage,
in_table_name='data')
})
elif key == "development_event":
resources.merge({
key:
DevelopmentEventDataset(in_storage=storage,
in_table_name='data')
})
elif key == "neighborhood":
resources.merge({
key:
NeighborhoodDataset(in_storage=storage,
in_table_name='data')
})
elif key == "job":
resources.merge({
key:
JobDataset(in_storage=storage, in_table_name='data')
})
elif key == "zone":
resources.merge({
key:
ZoneDataset(in_storage=storage, in_table_name='data')
})
elif key == "travel_data":
resources.merge({
key:
TravelDataDataset(in_storage=storage,
in_table_name='data')
})
elif key == "faz":
resources.merge({
key:
FazDataset(in_storage=storage, in_table_name='data')
})
elif key == "fazdistrict":
resources.merge({
key:
FazdistrictDataset(in_storage=storage,
in_table_name='data')
})
elif key == "race":
resources.merge({
key:
RaceDataset(in_storage=storage, in_table_name='data')
})
elif key == "county":
resources.merge({
key:
CountyDataset(in_storage=storage, in_table_name='data')
})
elif key == "large_area":
resources.merge({
key:
LargeAreaDataset(in_storage=storage,
in_table_name='data')
})
elif key == "development_group":
resources.merge({
key:
DevelopmentGroupDataset(in_storage=storage,
in_table_name='data')
})
elif key == "employment_sector_group":
resources.merge({
key:
EmploymentSectorGroupDataset(in_storage=storage,
in_table_name='data')
})
elif key == "plan_type_group":
resources.merge({
key:
PlanTypeGroupDataset(in_storage=storage,
in_table_name='data')
})
elif key == "building":
resources.merge({
key:
BuildingDataset(in_storage=storage,
in_table_name='data')
})
else:
resources.merge({key: data_dictionary[key]})
if dataset in self.interactions:
if dataset == "household_x_gridcell":
resources.merge({
"dataset":
HouseholdXGridcellDataset(dataset1=resources["household"],
dataset2=resources["gridcell"])
})
if dataset == "job_x_gridcell":
resources.merge({
"dataset":
JobXGridcellDataset(dataset1=resources["job"],
dataset2=resources["gridcell"])
})
if dataset == "household_x_zone":
resources.merge({
"dataset":
HouseholdXZoneDataset(dataset1=resources["household"],
dataset2=resources["zone"])
})
if dataset == "household_x_neighborhood":
resources.merge({
"dataset":
HouseholdXNeighborhoodDataset(
dataset1=resources["household"],
dataset2=resources["neighborhood"])
})
if dataset == "development_project_x_gridcell":
resources.merge({
"dataset":
DevelopmentProjectXGridcellDataset(
dataset1=resources["development_project"],
dataset2=resources["gridcell"])
})
else:
resources.merge({"dataset": resources[dataset]})
resources.merge({"check_variables": '*', "debug": 4})
return resources
def test_stochastic_test_case(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
storage = StorageFactory().get_storage('dict_storage')
nhouseholds = 10000
ngrids = 10
#create households
storage.write_table(table_name='households',
table_data={
'household_id': arange(nhouseholds) + 1,
'grid_id': array(nhouseholds * [-1])
})
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
# create gridcells
storage.write_table(table_name='gridcells',
table_data={
'grid_id':
arange(ngrids) + 1,
'cost':
array(ngrids / 2 * [100] + ngrids / 2 * [1000])
})
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# run the model
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False, \
choices = "opus_core.random_choices_from_index", sampler=None)
#sample_size_locations = 30)
# check the individual gridcells
# This is a stochastic model, so it may legitimately fail occassionally.
success = []
def inner_loop():
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngrids / 2) + 1)
return result_more_attractive
expected_results = array(ngrids / 2 *
[nhouseholds * 0.71 / (ngrids / 2)])
self.run_stochastic_test(__file__,
inner_loop,
expected_results,
10,
type="pearson",
transformation=None)
# Make sure it fails when expected distribution is different from actual.
expected_results = array(ngrids / 2 *
[nhouseholds * 0.61 / (ngrids / 2)])
try:
self.run_stochastic_test(__file__,
inner_loop,
expected_results,
10,
type="poisson",
transformation=None)
except AssertionError:
pass
from opus_core.session_configuration import SessionConfiguration
from urbansim.datasets.household_dataset import HouseholdDataset
from urbansim.datasets.gridcell_dataset import GridcellDataset
from urbansim.datasets.zone_dataset import ZoneDataset
from urbansim.datasets.job_dataset import JobDataset
from urbansim.models.household_location_choice_model_creator import HouseholdLocationChoiceModelCreator
from opus_core.database_management.database_server import DatabaseServer
from opus_core.database_management.configurations.scenario_database_configuration import ScenarioDatabaseConfiguration
# Datasets
##########
# agents from householdset.tab
agents = HouseholdDataset(in_storage=StorageFactory().get_storage(
'tab_storage', storage_location='.'),
in_table_name="householdset",
id_name="agent_id")
agents.summary()
agents.get_attribute("income")
agents.plot_histogram("income", bins=10)
agents.r_histogram("income")
agents.r_scatter("income", "persons")
# gridcells from PSRC
locations_psrc = GridcellDataset(in_storage=StorageFactory().get_storage(
'flt_storage', storage_location="/home/hana/bandera/urbansim/data/GPSRC"),
in_table_name="gc")
locations_psrc.summary()
locations_psrc.plot_histogram("distance_to_highway", bins=15)
locations_psrc.r_image("distance_to_highway")
def test_place_agents_to_correct_areas(self):
"""10 gridcells - 5 in area 1, 5 in area 2, with equal cost, no capacity restrictions
100 households - 70 live in area 1, 30 live in area 2.
We set the coefficient value for cost -0.001.
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs/2
hh_grid_ids = array(nhhs*[-1])
lareas = array(ngcs_attr*[1] + ngcs_attr*[2])
hh_lareas = array(70*[1] + 30*[2])
household_data = {
'household_id': arange(nhhs)+1,
'grid_id': hh_grid_ids,
'faz_id': hh_lareas
}
gridcell_data = {
'grid_id': arange(ngcs)+1,
'cost':array(ngcs*[100]),
'faz_id': lareas
}
storage.write_table(table_name = 'households', table_data = household_data)
storage.write_table(table_name = 'gridcells', table_data = gridcell_data)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
gridcells = GridcellDataset(in_storage=storage, in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001,))
specification = EquationSpecification(variables=("gridcell.cost", ), coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
households = HouseholdDataset(in_storage=storage, in_table_name='households')
hlcm = SubareaHouseholdLocationChoiceModel(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index", sample_size_locations = 4, subarea_id_name="faz_id")
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1)
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1)
result_area2 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1))
gridcells.delete_one_attribute("number_of_households")
result = concatenate((result_area1, result_area2))
return result
expected_results = array(ngcs_attr*[nhhs*0.7/float(ngcs_attr)] + ngcs_attr*[nhhs*0.3/float(ngcs_attr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
# check the exact sum
hlcm = SubareaHouseholdLocationChoiceModel(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index", sample_size_locations = 4, subarea_id_name="faz_id")
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1)
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_area1 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1).sum()
result_area2 = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1)).sum()
results = array([result_area1, result_area2])
expected_results = array([70, 30])
self.assertEqual(ma.allequal(expected_results, results), True, "Error, should_be: %s, but result: %s" % (
expected_results, results))
def setUp( self ):
"""here, we simulate 50 residential units
and 5000 commercial, industrial, and governmental sqft added to each of the gridcells in previous years.
"""
### TODO: do not redefine these constants.
self.comc = 1
self.indc = 3
self.govc = 2
self.sfhc = 4
self.mfhc = 5
storage = StorageFactory().get_storage('dict_storage')
gridcells_table_name = 'gridcells'
# create 100 gridcells, each with 200 residential units and space for 100 commercial jobs,
# 100 industrial jobs, and residential, industrial, and commercial value at $500,000 each
storage.write_table(
table_name=gridcells_table_name,
table_data={
"grid_id": arange( 1, 100+1 ),
"commercial_sqft_per_job":array( 100*[100] ),
"industrial_sqft_per_job":array( 100*[100] ),
"single_family_improvement_value":array( 100*[500000] ),
"commercial_improvement_value":array( 100*[500000] ),
"industrial_improvement_value":array( 100*[500000] )
}
)
self.gridcells = GridcellDataset(in_storage=storage, in_table_name=gridcells_table_name)
buildings_table_name = 'buildings'
# 2000 buildings (1000 with 20 residential units each, 500 with 20 commercial job and 500 with 20 industrial job each)
storage.write_table(
table_name=buildings_table_name,
table_data={
"building_id":arange( 1, 2000+1 ), # 2000 buildings
"grid_id":array( 20*range( 1, 100+1 ), dtype=int32 ), # spread evenly across 100 gridcells
"building_type_id":array(1000*[self.sfhc] +
500*[self.comc] +
500*[self.indc], dtype=int8),
"sqft": array(1000*[0] +
500*[2000] +
500*[2000], dtype=int32),
"residential_units": array(1000*[20] +
500* [0] +
500* [0], dtype=int32),
"improvement_value": array(1000*[50] +
500* [50] +
500* [50], dtype=float32),
"year_built": array(1000*[1940] +
500* [1940] +
500* [1940], dtype=int32)
}
)
self.buildings = BuildingDataset(in_storage=storage, in_table_name=buildings_table_name)
households_table_name = 'households'
# create 10000 households, 100 in each of the 100 gridcells.
# there will initially be 100 vacant residential units in each gridcell then.
storage.write_table(
table_name=households_table_name,
table_data={
"household_id":arange( 1, 10000+1 ),
"grid_id":array( 100*range( 1, 100+1 ), dtype=int32 )
}
)
self.households = HouseholdDataset(in_storage=storage, in_table_name=households_table_name)
building_types_table_name = 'building_types'
storage.write_table(
table_name=building_types_table_name,
table_data={
"building_type_id":array([self.govc,self.comc,self.indc, self.sfhc, self.mfhc], dtype=int8),
"name": array(["governmental", "commercial", "industrial", "single_family","multiple_family"]),
"units": array(["governmental_sqft", "commercial_sqft", "industrial_sqft", "residential_units", "residential_units"]),
"is_residential": array([0,0,0,1,1], dtype='?')
}
)
self.building_types = BuildingTypeDataset(in_storage=storage, in_table_name=building_types_table_name)
job_building_types_table_name = 'job_building_types'
storage.write_table(
table_name=job_building_types_table_name,
table_data={
"id":array([self.govc,self.comc,self.indc, self.sfhc, self.mfhc], dtype=int8),
"name": array(["governmental", "commercial", "industrial", "single_family","multiple_family"])
}
)
self.job_building_types = JobBuildingTypeDataset(in_storage=storage, in_table_name=job_building_types_table_name)
jobs_table_name = 'jobs'
# create 2500 commercial jobs and distribute them equally across the 100 gridcells,
# 25 commercial buildings/gridcell
storage.write_table(
table_name=jobs_table_name,
table_data={
"job_id":arange( 1, 2500+1 ),
"grid_id":array( 25*range( 1, 100+1 ), dtype=int32 ),
"sector_id":array( 2500*[1], dtype=int32 ),
"building_type":array(2500*[self.comc], dtype=int8)
}
)
self.jobs = JobDataset(in_storage=storage, in_table_name=jobs_table_name)
self.dataset_pool = DatasetPool()
self.dataset_pool.add_datasets_if_not_included({
"household":self.households,
"job":self.jobs,
"building":self.buildings,
"building_type": self.building_types,
"job_building_type": self.job_building_types})
self.building_categories = {'commercial': array([1000,5000]),
'industrial': array([500,800,1000])}
def test_agents_go_to_attractive_locations(self):
"""10 gridcells - 5 with cost 100, 5 with cost 1000, no capacity restrictions
100 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
storage = StorageFactory().get_storage('dict_storage')
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
household_data = {
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids
}
gridcell_data = {
'grid_id': arange(ngcs) + 1,
'cost': array(ngcs_attr * [100] + ngcs_noattr * [1000])
}
storage.write_table(table_name='households', table_data=household_data)
storage.write_table(table_name='gridcells', table_data=gridcell_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001, ))
specification = EquationSpecification(variables=("gridcell.cost", ),
coefficients=("costcoef", ))
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute="grid_id",
values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result = concatenate(
(result_more_attractive, result_less_attractive))
return result
expected_results = array(ngcs_attr * [nhhs * 0.71 / float(ngcs_attr)] +
ngcs_noattr *
[nhhs * 0.29 / float(ngcs_noattr)])
self.run_stochastic_test(__file__, run_model, expected_results, 10)
def run_model_2():
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=household_data)
households = HouseholdDataset(in_storage=storage,
in_table_name='households')
storage.write_table(table_name='gridcells',
table_data=gridcell_data)
gridcells = GridcellDataset(in_storage=storage,
in_table_name='gridcells')
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices="opus_core.random_choices_from_index",
sample_size_locations=8)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1)
# get results
gridcells.compute_variables(
["urbansim.gridcell.number_of_households"],
resources=Resources({"household": households}))
result_more_attractive = gridcells.get_attribute_by_id(
"number_of_households",
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
"number_of_households", arange(ngcs_attr + 1, ngcs + 1))
return array(
[result_more_attractive.sum(),
result_less_attractive.sum()])
expected_results = array([nhhs * 0.71, nhhs * 0.29])
self.run_stochastic_test(__file__, run_model_2, expected_results, 10)
def test_controlling_with_one_marginal_characteristic(self):
"""Using the age_of_head as a marginal characteristic, which would partition the 8 groups into two larger groups
(those with age_of_head < 40 and >= 40), ensure that the control totals are met and that the distribution within
each large group is the same before and after running the model
"""
#IMPORTANT: marginal characteristics grouping indices have to start at 0!
#i.e. below, there is one marg. char. "age_of_head". here we indicate that the first "large group" (groups 1-4),
#consisting of those groups with age_of_head < 40 should total 25000 households after running this model for one year,
#and the second large group, those groups with age_of_head > 40, should total 15000 households
annual_household_control_totals_data = {
"year": array([2000, 2000, 2000, 2000]),
"age_of_head": array([0, 1, 0, 1]),
"total_number_of_households": array([20000, 10000, 5000, 5000]),
"faz_id": array([1, 1, 2, 2])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='hh_set',
table_data=self.households_data)
hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')
storage.write_table(table_name='hct_set',
table_data=annual_household_control_totals_data)
hct_set = ControlTotalDataset(in_storage=storage,
in_table_name='hct_set',
what='household')
storage.write_table(table_name='prs_set', table_data=self.person_data)
prs_set = PersonDataset(in_storage=storage, in_table_name='prs_set')
storage.write_table(
table_name='hc_set',
table_data=self.household_characteristics_for_ht_data)
hc_set = HouseholdCharacteristicDataset(in_storage=storage,
in_table_name='hc_set')
model = SubareaHouseholdTransitionModel(subarea_id_name="faz_id")
model.run(year=2000,
person_set=prs_set,
household_set=hh_set,
control_totals=hct_set,
characteristics=hc_set)
#check that there are indeed 40000 total households after running the model
areas = hh_set.get_attribute("faz_id")
results = array([0, 0])
for iarea in [0, 1]:
results[iarea] = where(areas == [1, 2][iarea])[0].size
should_be = [30000, 10000]
self.assertEqual(
ma.allclose(should_be, results, rtol=1e-1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
#check that the number of households within the groups correspond to the control totals
results = self.get_count_all_groups(hh_set)
should_be = [20000, 10000, 5000, 5000]
idx1 = arange(0, 4)
idx2 = arange(4, 8)
idx3 = arange(8, 12)
idx4 = arange(12, 16)
self.assertEqual(
ma.allclose([
results[idx1].sum(), results[idx2].sum(), results[idx3].sum(),
results[idx4].sum()
],
should_be,
rtol=1e-1), True,
"Error, should_be: %s, but result: %s" %
(should_be,
array([
results[idx1].sum(), results[idx2].sum(), results[idx3].sum(),
results[idx4].sum()
])))
#check that the distribution of households within the groups are the same before and after
#running the model, respectively
should_be = [ # area 1
3000.0 / 7500.0 * 20000.0,
1000.0 / 7500.0 * 20000.0,
1500.0 / 7500.0 * 20000.0,
2000.0 / 7500.0 * 20000.0,
1000.0 / 9000.0 * 10000.0,
2500.0 / 9000.0 * 10000.0,
1500.0 / 9000.0 * 10000.0,
4000.0 / 9000.0 * 10000.0,
# area 2
3000.0 / 7500.0 * 5000.0,
1000.0 / 7500.0 * 5000.0,
1500.0 / 7500.0 * 5000.0,
2000.0 / 7500.0 * 5000.0,
1000.0 / 9000.0 * 5000.0,
2500.0 / 9000.0 * 5000.0,
1500.0 / 9000.0 * 5000.0,
4000.0 / 9000.0 * 5000.0
]
self.assertEqual(
ma.allclose(results, should_be, rtol=0.1), True,
"Error, should_be: %s, but result: %s" % (should_be, results))
def run_ALCM(niter):
nhhs = 100
ngcs = 10
ngcs_attr = ngcs / 2
ngcs_noattr = ngcs - ngcs_attr
hh_grid_ids = array(nhhs * [-1])
storage = StorageFactory().get_storage('dict_storage')
households_table_name = 'households'
storage.write_table(table_name=households_table_name,
table_data={
'household_id': arange(nhhs) + 1,
'grid_id': hh_grid_ids
})
gridcells_table_name = 'gridcells'
storage.write_table(table_name=gridcells_table_name,
table_data={
'grid_id': arange(ngcs) + 1,
'cost':
array(ngcs_attr * [100] + ngcs_noattr * [1000])
})
households = HouseholdDataset(in_storage=storage,
in_table_name=households_table_name)
gridcells = GridcellDataset(in_storage=storage,
in_table_name=gridcells_table_name)
# create coefficients and specification
coefficients = Coefficients(names=('costcoef', ), values=(-0.001, ))
specification = EquationSpecification(variables=('gridcell.cost', ),
coefficients=('costcoef', ))
logger.be_quiet()
result = zeros((niter, ngcs))
for iter in range(niter):
hlcm = HouseholdLocationChoiceModelCreator().get_model(
location_set=gridcells,
compute_capacity_flag=False,
choices='opus_core.random_choices_from_index',
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification,
coefficients,
agent_set=households,
debuglevel=1,
chunk_specification={'nchunks': 1})
# get results
gridcells.compute_variables(['urbansim.gridcell.number_of_households'],
resources=Resources(
{'household': households}))
result_more_attractive = gridcells.get_attribute_by_id(
'number_of_households',
arange(ngcs_attr) + 1)
result_less_attractive = gridcells.get_attribute_by_id(
'number_of_households', arange(ngcs_attr + 1, ngcs + 1))
households.set_values_of_one_attribute(attribute='grid_id',
values=hh_grid_ids)
gridcells.delete_one_attribute('number_of_households')
result[iter, :] = concatenate(
(result_more_attractive, result_less_attractive))
#print result #, result_more_attractive.sum(), result_less_attractive.sum()
return result
def test_agents_go_to_attractive_locations(self):
"""100 gridcells - 50 with cost 100, 50 with cost 1000, no capacity restrictions
10,000 households
We set the coefficient value for cost -0.001. This leads to probability
proportion 0.71 (less costly gridcells) to 0.29 (expensive gridcells)
(derived from the logit formula)
"""
nhhs = 1000
ngcs = 50
ngcs_attr = ngcs/2
ngcs_noattr = ngcs - ngcs_attr
#print ngcs_attr, ngcs_noattr
hh_grid_ids = array([-1]*nhhs)
household_data = {"household_id": arange(nhhs)+1, "grid_id": hh_grid_ids}
gridcell_data = {"grid_id": arange(ngcs)+1, "cost":array(ngcs_attr*[100]+ngcs_noattr*[1000])}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name = 'households', table_data = household_data)
households = HouseholdDataset(in_storage=storage, in_table_name='households')
storage.write_table(table_name = 'gridcells', table_data = gridcell_data)
gridcells = HouseholdDataset(in_storage=storage, in_table_name='gridcells')
# create coefficients and specification
coefficients = Coefficients(names=("costcoef", ), values=(-0.001,))
specification = EquationSpecification(variables=("gridcell.cost", ), coefficients=("costcoef", ))
logger.be_quiet()
# check the individual gridcells
def run_model():
hlcm = HouseholdLocationChoiceModelCreator().get_model(location_set=gridcells, compute_capacity_flag=False,
choices = "opus_core.random_choices_from_index",
sampler=None,
#sample_size_locations = 30
)
hlcm.run(specification, coefficients, agent_set=households, debuglevel=1,
chunk_specification={'nchunks':1})
# get results
gridcells.compute_variables(["urbansim.gridcell.number_of_households"],
resources=Resources({"household":households}))
result_more_attractive = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr)+1)
result_less_attractive = gridcells.get_attribute_by_id("number_of_households", arange(ngcs_attr+1, ngcs+1))
households.set_values_of_one_attribute(attribute="grid_id", values=hh_grid_ids)
gridcells.delete_one_attribute("number_of_households")
result_less_attractive[0]=result_less_attractive[0] + self.wrong_number
result = concatenate((result_more_attractive, result_less_attractive))
#print standard_deviation(result[ngcs_attr:(ngcs-1)])
return result
expected_results = array(ngcs_attr*[nhhs*0.71/float(ngcs_attr)] + ngcs_noattr*[nhhs*0.29/float(ngcs_noattr)])
#print expected_results
R = 1000
#r = [2, 5, 10, 50, 100, 1000]
r = [2, 5, 10, 15, 20]
#r=[20]
levels = [0.05, 0.01]
#levels = [0.01]
#wrong_numbers = [8, 10, 12, 14, 16, 18, 21, 23]
#wrong_numbers = [2, 4, 6, 8, 10, 12, 14, 16]
#wrong_numbers = [3, 6, 9, 12, 15, 18]
wrong_numbers = [12, 15, 18]
for wn in wrong_numbers:
self.wrong_number = wn
print "Wrong number =", self.wrong_number
power = zeros((len(r), len(levels)))
for ir in range(len(r)):
for il in range(len(levels)):
print "r =", r[ir],", level =",levels[il]
seed(1)
for iR in range(R):
try:
self.run_stochastic_test(__file__, run_model, expected_results,
r[ir], significance_level=levels[il])
except:
power[ir,il]=power[ir,il]+1
print "Power:",power[ir,il]/float(R)
print power/float(R)
from opus_core.session_configuration import SessionConfiguration
from urbansim.datasets.household_dataset import HouseholdDataset
from urbansim.datasets.gridcell_dataset import GridcellDataset
from urbansim.datasets.zone_dataset import ZoneDataset
from urbansim.datasets.job_dataset import JobDataset
from urbansim.models.household_location_choice_model_creator import HouseholdLocationChoiceModelCreator
from opus_core.database_management.database_server import DatabaseServer
from opus_core.database_management.configurations.scenario_database_configuration import ScenarioDatabaseConfiguration
# Datasets
##########
# agents from householdset.tab
agents = HouseholdDataset(in_storage = StorageFactory().get_storage('tab_storage', storage_location='.'),
in_table_name = "householdset", id_name="agent_id")
agents.summary()
agents.get_attribute("income")
agents.plot_histogram("income", bins = 10)
agents.r_histogram("income")
agents.r_scatter("income", "persons")
# gridcells from PSRC
locations_psrc = GridcellDataset(in_storage = StorageFactory().get_storage('flt_storage',
storage_location = "/home/hana/bandera/urbansim/data/GPSRC"),
in_table_name = "gc")
locations_psrc.summary()
locations_psrc.plot_histogram("distance_to_highway", bins = 15)
locations_psrc.r_image("distance_to_highway")
locations_psrc.plot_map("distance_to_highway")
def run_HTM(niter):
nhhs = 5000
ngroups = 4
nhhsg = int(nhhs/ngroups)
nhhslg = nhhs-(ngroups-1)*nhhsg
should_nhhs = nhhs-2000
storage = StorageFactory().get_storage('dict_storage')
hc_set_table_name = 'hc_set'
storage.write_table(
table_name = hc_set_table_name,
table_data = {
'characteristic': array(4*['income']+4*['age_of_head']),
'min':array([0,1001,5001, 10001, 0, 31, 41, 61]),
'max':array([1000, 5000, 10000,-1, 30, 40, 60, -1])
},
)
hct_set_table_name = 'hct_set'
storage.write_table(
table_name = hct_set_table_name,
table_data = {
'year':array([2000]),
'total_number_of_households':array([should_nhhs])
},
)
households_table_name = 'households'
storage.write_table(
table_name = households_table_name,
table_data = {
'age_of_head': array(nhhsg/2*[18]+(nhhsg-nhhsg/2)*[35] +
nhhsg/2*[30] + (nhhsg-nhhsg/2)*[40] +
nhhsg/2*[38] + (nhhsg-nhhsg/2)*[65] +
nhhslg/2*[50] + (nhhslg-nhhslg/2)*[80]
),
'income': array(nhhsg*[500] + nhhsg*[2000] +
nhhsg*[7000] + nhhslg*[15000]
),
'household_id':arange(nhhs)+1
},
)
hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name=hc_set_table_name)
hct_set = ControlTotalDataset(
in_storage = storage,
in_table_name = hct_set_table_name,
what = 'household',
id_name = ['year']
)
logger.be_quiet()
result = zeros((niter,4))
for iter in range(niter):
households = HouseholdDataset(in_storage=storage, in_table_name=households_table_name)
model = HouseholdTransitionModel()
model.run(year=2000, household_set=households, control_totals=hct_set, characteristics=hc_set)
income = households.get_attribute('income')
age = households.get_attribute('age_of_head')
idx1 = where(income <= 1000)[0]
idx2 = where(logical_and(income <= 5000, income > 1000))[0]
idx3 = where(logical_and(income <= 10000, income > 5000))[0]
idx4 = where(income > 10000)[0]
result[iter,:] = array([age[idx1].mean(), age[idx2].mean(), age[idx3].mean(), age[idx4].mean()])
return result
