def __init__(self, config):
if 'estimation_database_configuration' in config:
db_server = DatabaseServer(config['estimation_database_configuration'])
db = db_server.get_database(config['estimation_database_configuration'].database_name)
out_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=db)
else:
out_storage = StorageFactory().get_storage(type='flt_storage',
storage_location=os.path.join(config['cache_directory'], str(config['base_year']+1)))
simulation_state = SimulationState()
simulation_state.set_cache_directory(config['cache_directory'])
simulation_state.set_current_time(config['base_year'])
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=attribute_cache)
if not os.path.exists(os.path.join(config['cache_directory'], str(config['base_year']))):
#raise RuntimeError, "datasets uncached; run prepare_estimation_data.py first"
CacheScenarioDatabase().run(config, unroll_gridcells=False)
for dataset_name in config['datasets_to_preload']:
SessionConfiguration().get_dataset_from_pool(dataset_name)
households = SessionConfiguration().get_dataset_from_pool("household")
household_ids = households.get_id_attribute()
workers = households.get_attribute("workers")
hh_ids = []
member_ids = []
is_worker = []
job_ids = []
for i in range(households.size()):
if workers[i] > 0:
hh_ids += [household_ids[i]] * workers[i]
member_ids += range(1, workers[i]+1)
is_worker += [1] * workers[i]
job_ids += [-1] * workers[i]
in_storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
in_storage.write_table(
table_name=persons_table_name,
table_data={
'person_id':arange(len(hh_ids))+1,
'household_id':array(hh_ids),
'member_id':array(member_ids),
'is_worker':array(is_worker),
'job_id':array(job_ids),
},
)
persons = PersonDataset(in_storage=in_storage, in_table_name=persons_table_name)
persons.write_dataset(out_storage=out_storage, out_table_name=persons_table_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5]),
'household_id': array([1, 1, 3, 3, 3]),
'member_id': array([1, 2, 1, 2, 3]),
'bususe': array([1, 1, 2, 2, 1])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
data_dictionary = {
'person':persons
},
dataset = 'person'
)
should_be = array([1, 1, 0, 0, 1])
self.assert_(ma.allclose(values, should_be, rtol=1e-7),
'Error in ' + self.variable_name)
def test_my_inputs(self):
persons_storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
persons_storage.write_table(
table_name=persons_table_name,
table_data={
'person_id':array([1, 2, 3, 4, 5, 6]),
'household_id':array([1, 1, 2, 3, 3, 3]),
'member_id':array([1, 2, 1, 1, 2, 3]),
'worker1':array([1, 0, 0, 0, 0, 1]),
},
)
persons = PersonDataset(in_storage=persons_storage, in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name,
data_dictionary = {
'person':persons,
'household':{
'household_id':array([1, 2, 3])
}
},
dataset = 'household'
)
should_be = array([1, 0, 1])
self.assertEqual(ma.allclose(values, should_be, rtol=1e-7), \
True, msg = "Error in " + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id':array([1, 2, 3, 4, 5]),
'household_id':array([1, 1, 3, 3, 3]),
'member_id':array([1,2,1,2,3]),
'work_place_county':array(['033','061','035','033','033'])
},
)
persons = PersonDataset(in_storage=storage, in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name,
data_dictionary = {
'person':persons
},
dataset = 'person'
)
should_be = array([True, False, False, True, True])
self.assert_(ma.allequal(values, should_be),
'Error in ' + self.variable_name)
def test(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5, 6, 7]),
'household_id':array([1, 1, 3, 3, 3, 2, 2]),
'member_id':array([1, 2, 1, 2, 3, 1, 2]),
'work_nonhome_based':array([1, 0, 1, 0, 1, 1, 0]),
},
)
persons = PersonDataset(in_storage=storage, in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
{"person":persons, \
"zone":{
"zone_id":array([1, 2, 3])
},
"household":{ \
"household_id":array([1, 2, 3]),
"zone_id":array([1, 3, 1]),
}}, \
dataset = "zone")
should_be = array([1,0,1])
self.assertEqual(ma.allclose(values, should_be, rtol=1e-20), \
True, msg = "Error in " + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
"person_id": array([1, 2, 3, 4, 5]),
"household_id": array([1, 1, 3, 3, 3]),
"member_id": array([1, 2, 1, 2, 3]),
"job_id": array([-1, 1, 3, 0, 5]),
"work_nonhome_based": array([1, 0, 1, 0, 0])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
{
"person":persons },
dataset = "person")
should_be = array([1, 0, 0, 0, 0])
self.assertEqual(ma.allclose(values, should_be, rtol=1e-7),
True,
msg="Error in " + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5, 6]),
'household_id': array([1, 1, 2, 3, 3, 3]),
'member_id': array([1, 2, 1, 1, 2, 3]),
'worker1': array([1, 0, 1, 0, 0, 1]),
'job_zone_id': array([1, 3, 3, 2, 1, 3])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(
self.variable_name,
data_dictionary={
'gridcell': {
'gridcell_id': array([1, 2, 3, 4]),
'zone_id': array([1, 1, 3, 2])
},
'household': {
'household_id': array([1, 2, 3, 4, 5]),
'zone_id': array([3, 1, 1, 1, 2]),
'cars': array([0, 2, 2, 0, 1]),
'number_of_nonhome_based_workers': array([1, 3, 1, 1, 1])
},
'person': persons,
'travel_data': {
'from_zone_id':
array([3, 3, 1, 1, 1, 2, 2, 3, 2]),
'to_zone_id':
array([1, 3, 1, 3, 2, 1, 3, 2, 2]),
'am_total_transit_time_walk':
array([1.1, 2.2, 3.3, 4.4, 0.5, 0.7, 8.7, 7.8, 1.0])
}
},
dataset='household_x_gridcell',
)
default_value = workerDDD_SSS_if_household_SSS.default_value
should_be = array(
[[3.3, 3.3, 1.1, 0.7], [4.4, 4.4, 2.2, 8.7], [0, 0, 0, 0],
[default_value, default_value, default_value, default_value],
[0, 0, 0, 0]])
self.assert_(ma.allclose(values, should_be, rtol=1e-3),
'Error in ' + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
edges_table_name = 'edges'
storage.write_table(
table_name=edges_table_name,
table_data={
'source': array([1, 2, 1, 3]),
'target': array([2, 3, 3, 1]),
'cost': array([12, 1, 15, 17])
},
)
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5, 6]),
'household_id': array([1, 1, 3, 3, 4, 5]),
'member_id': array([1, 2, 1, 2, 4, 1]),
'home_parcel_id': array([3, 1, 1, 2, 2, 3]),
'work_place_parcel_id': array([1, 3, 3, 1, 4, 2])
},
)
edges = EdgeDataset(in_storage=storage,
in_table_name=edges_table_name)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name,
data_dictionary={
'person':
persons,
'edge': edges
},
dataset='person')
default_value = travel_time_hbw_transit_from_work_to_home.default_value
should_be = array([13, 17, 17, 12, default_value, 1])
values = [value[0] for value in values]
self.assert_(ma.allclose(values, should_be, rtol=1e-3),
'Error in ' + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5, 6]),
'household_id': array([1, 1, 2, 3, 3, 3]),
'member_id': array([1, 2, 1, 1, 2, 3]),
'worker1': array([1, 0, 1, 0, 0, 1]),
'work_place_zone_id': array([1, 3, 3, 2, 1, 3])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
data_dictionary = {
'household':{
'household_id':array([1,2,3,4,5]),
'zone_id':array([3, 1, 1, 1, 2]),
},
'zone':{
'zone_id':array([1, 3, 2]),
},
'person':persons,
'travel_data':{
'from_zone_id':array([3,3,1,1,1,2,2,3,2]),
'to_zone_id':array([1,3,1,3,2,1,3,2,2]),
'am_single_vehicle_to_work_travel_time':array([1.1, 2.2, 3.3, 4.4, 0.5, 0.7, 8.7, 7.8, 1.0]),
}
},
dataset = 'household_x_zone'
)
default_value = workerDDD_travel_time_hbw_am_drive_alone_from_home_to_work.default_value
should_be = array([[3.3, 1.1, 0.7], [4.4, 2.2, 8.7], [4.4, 2.2, 8.7],
[default_value, default_value, default_value],
[default_value, default_value, default_value]])
self.assert_(ma.allclose(values, should_be, rtol=1e-3),
'Error in ' + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5, 6]),
'household_id': array([1, 1, 2, 3, 3, 3]),
'member_id': array([1, 2, 1, 1, 2, 3]),
'worker1': array([1, 0, 1, 0, 0, 1]),
'work_place_parcel_id': array([1, 2, 3, 5, 4, 4])
},
)
parcels_table_name = 'parcels'
storage.write_table(
table_name=parcels_table_name,
table_data={
'parcel_id': array([1, 2, 3, 4, 5]),
'county': array(['033', '061', '035', '033', '033'])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
parcels = ParcelDataset(in_storage=storage,
in_table_name=parcels_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
{ "person":persons,
"household":{
"household_id":array([1, 2, 3])},
"parcel":parcels},
dataset = "household" )
should_be = array([1, 0, 1])
self.assertEqual(ma.allclose(values, should_be, rtol=1e-7), \
True, msg = "Error in " + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id':array([1, 2, 3, 4, 5]),
'household_id':array([1, 1, 3, 3, 3]),
'member_id':array([1,2,1,2,3]),
'work_place_parcel_id':array([1,1,3,2,4]),
},
)
parcels_table_name = 'jobs'
storage.write_table(
table_name=parcels_table_name,
table_data={
'parcel_id':array([1,2,3,4]),
'county':array(['033','035','033','053']),
},
)
persons = PersonDataset(in_storage=storage, in_table_name=persons_table_name)
parcels = ParcelDataset(in_storage=storage, in_table_name=parcels_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
data_dictionary = {
'parcel':parcels,
'person':persons
},
dataset = 'person'
)
should_be = array(['033', '033', '033', '035', '053'])
for i in range(values.size):
self.assert_(values[i] == should_be[i],
'Error in ' + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id':
array([1, 2, 3, 4, 5, 6]),
'household_id':
array([1, 1, 2, 3, 3, 3]),
'member_id':
array([1, 2, 1, 1, 2, 3]),
'worker1':
array([1, 0, 1, 0, 0, 1]),
'var_name_hbw_am_drive_alone_from_home_to_work':
array([5.3, 2, 999, 0, 1.1, 7])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name, \
data_dictionary = {
'person':persons,
'household':{
'household_id':array([1, 2, 3])
}
},
dataset = 'household'
)
should_be = array([5.3, 999, 7])
self.assert_(ma.allclose(values, should_be, rtol=1e-7),
'Error in ' + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': array([1, 2, 3, 4, 5]),
'household_id': array([1, 1, 3, 3, 3]),
'member_id': array([1, 2, 1, 2, 3]),
'home_zone_id': array([3, 1, 1, 2, 3]),
'work_place_zone_id': array([1, 3, 3, 1, 2])
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
values = VariableTestToolbox().compute_variable(
self.variable_name,
data_dictionary={
'person': persons,
'travel_data': {
'from_zone_id':
array([3, 3, 1, 1, 1, 2, 2, 3, 2]),
'to_zone_id':
array([1, 3, 1, 3, 2, 1, 3, 2, 2]),
'am_single_vehicle_to_work_travel_time':
array([1.1, 2.2, 3.3, 4.4, 0.5, 0.7, 8.7, 7.8, 1.0])
}
},
dataset='person')
should_be = array([1.1, 4.4, 4.4, 0.7, 7.8])
self.assert_(ma.allclose(values, should_be, rtol=1e-2),
'Error in ' + self.variable_name)
def test_my_inputs(self):
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
'person_id':array([1, 2, 3, 4, 5]),
'household_id':array([1, 1, 3, 3, 3]),
'member_id':array([1,2,1,2,3]),
'work_place_parcel_id':array([1,1,3,2,5])
},
)
parcels_table_name = 'jobs'
storage.write_table(
table_name=parcels_table_name,
table_data={
'parcel_id':array([1,2,3,4]),
'zone_id': array([5,6,7,8])
},
)
persons = PersonDataset(in_storage=storage, in_table_name=persons_table_name)
parcels = ParcelDataset(in_storage=storage, in_table_name=parcels_table_name)
values = VariableTestToolbox().compute_variable(self.variable_name,
data_dictionary = {
'parcel':parcels,
'person':persons
},
dataset = 'person'
)
should_be = array([5, 5, 7, 6, -1])
self.assert_(ma.allclose(values, should_be, rtol=1e-7),
'Error in ' + self.variable_name)
def __init__(self, config):
if 'estimation_database_configuration' in config:
db_server = DatabaseServer(
config['estimation_database_configuration'])
db = db_server.get_database(
config['estimation_database_configuration'].database_name)
out_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=db)
else:
out_storage = StorageFactory().get_storage(
type='flt_storage',
storage_location=os.path.join(config['cache_directory'],
str(config['base_year'] + 1)))
simulation_state = SimulationState()
simulation_state.set_cache_directory(config['cache_directory'])
simulation_state.set_current_time(config['base_year'])
attribute_cache = AttributeCache()
SessionConfiguration(
new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=attribute_cache)
if not os.path.exists(
os.path.join(config['cache_directory'], str(
config['base_year']))):
#raise RuntimeError, "datasets uncached; run prepare_estimation_data.py first"
CacheScenarioDatabase().run(config, unroll_gridcells=False)
for dataset_name in config['datasets_to_preload']:
SessionConfiguration().get_dataset_from_pool(dataset_name)
households = SessionConfiguration().get_dataset_from_pool("household")
household_ids = households.get_id_attribute()
workers = households.get_attribute("workers")
hh_ids = []
member_ids = []
is_worker = []
job_ids = []
for i in range(households.size()):
if workers[i] > 0:
hh_ids += [household_ids[i]] * workers[i]
member_ids += range(1, workers[i] + 1)
is_worker += [1] * workers[i]
job_ids += [-1] * workers[i]
in_storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
in_storage.write_table(
table_name=persons_table_name,
table_data={
'person_id': arange(len(hh_ids)) + 1,
'household_id': array(hh_ids),
'member_id': array(member_ids),
'is_worker': array(is_worker),
'job_id': array(job_ids),
},
)
persons = PersonDataset(in_storage=in_storage,
in_table_name=persons_table_name)
persons.write_dataset(out_storage=out_storage,
out_table_name=persons_table_name)
def run(self,
person_set,
household_set=None,
job_set=None,
expand_person_set=True,
resources=None):
"""main_dataset is either household or job,
if job_set is not None, it will set job_id of
person/worker whose job 'disappears' to -1.
if household_set is not None, it will expand personset
to include persons in households that just 'immigrate'
by household_transition_model, as well as eliminate
persons whose household has emmigrated;
"""
if person_set is not None and job_set is not None:
id_name = job_set.get_id_name()[0]
person_job_id_array = person_set.get_attribute(id_name)
person_attr_index = job_set.try_get_id_index(
person_job_id_array, return_value_if_not_found=-9)
index = where(person_attr_index == -9)[0]
person_set.set_values_of_one_attribute("job_id",
resize(
array([-1],
dtype="int32"),
index.size),
index=index)
if household_set is not None:
if person_set is None:
storage = StorageFactory().get_storage('dict_storage')
persons_table_name = 'persons'
storage.write_table(
table_name=persons_table_name,
table_data={
"person_id": array([], dtype='int32'),
"household_id": array([], dtype='int32'),
"member_id": array([], dtype='int32'),
"work_nonhome_based": array([], dtype='int32')
},
)
persons = PersonDataset(in_storage=storage,
in_table_name=persons_table_name)
id_name = household_set.get_id_name()[0]
person_hh_id_array = person_set.get_attribute(id_name)
if expand_person_set:
hh_ids = []
member_ids = []
job_ids = []
work_nonhome_based = []
#because immigrant households are assigned household_id larger than previous max id
max_hh_id = person_hh_id_array.max()
household_ids = household_set.get_id_attribute()
hh_indices = where(household_ids > max_hh_id)[0]
workers = household_set.get_attribute("workers")
for index in hh_indices:
household_id = household_ids[index]
if workers[index] > 0:
hh_ids += [household_id] * workers[index]
member_ids += range(1, workers[index] + 1)
job_ids += [-1] * workers[index]
work_nonhome_based += [-1] * workers[index]
#TODO: make this code works more robustly: above code depends on Household transition model
#assigning new households with ids larger than max_id. Code below is extremely SLOW.
#unique_persons_hh_ids = unique(person_hh_id_array)
#workers = main_dataset.get_attribute("workers")
#hh_indices = arange(main_dataset.size())
#for index in hh_indices:
#household_id = id_array[index]
#if household_id not in unique_persons_hh_ids:
#if workers[index] > 0:
#hh_ids += [household_id] * workers[index]
#member_ids += range(1, workers[index]+1)
#job_ids += [-1] * workers[index]
new_persons = {
"person_id":
arange(len(hh_ids)) + person_set.get_id_attribute().max() +
1,
"household_id":
array(hh_ids),
"member_id":
array(member_ids),
"job_id":
array(job_ids),
"work_nonhome_based":
array(work_nonhome_based)
}
person_set.add_elements(new_persons,
require_all_attributes=False)
#delete a person from person set if its household_id is not in households
person_attr_index = household_set.try_get_id_index(
person_hh_id_array, return_value_if_not_found=-9)
eliminate_index = where(person_attr_index == -9)[0]
person_set.remove_elements(eliminate_index)
return person_set