def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
esri_data_path = params_dict["esri_data_path"]
esri_table_name = params_dict["esri_table_name"]
opus_data_directory = params_dict["opus_data_directory"]
opus_data_year = params_dict["opus_data_year"]
input_storage = esri_storage(storage_location=esri_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache())
if esri_table_name == "ALL":
logCB("Sending all tables to OPUS storage...\n")
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(dataset_name=i, in_storage=input_storage, out_storage=output_storage)
else:
logCB("Exporting table '%s' to OPUS storage located at %s...\n" % (esri_table_name, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=esri_table_name, in_storage=input_storage, out_storage=output_storage
)
logCB("Finished exporting table '%s'\n" % (esri_table_name))
def _compute_variable_for_prior_year(self,
dataset,
full_name,
time,
resources=None):
"""Create a new dataset for this variable, compute the variable, and then return
the values for this variable."""
calling_dataset_pool = SessionConfiguration().get_dataset_pool()
calling_time = SimulationState().get_current_time()
SimulationState().set_current_time(time)
try:
# Get an empty dataset pool with same search paths.
my_dataset_pool = DatasetPool(
package_order=calling_dataset_pool.get_package_order(),
storage=AttributeCache())
ds = dataset.empty_dataset_like_me(in_storage=AttributeCache())
# Don't pass any datasets via resources, since they may be from a different time.
my_resources = Resources(resources)
for key in my_resources:
if isinstance(key, Dataset):
del my_resources[key]
ds.compute_variables(full_name,
my_dataset_pool,
resources=my_resources)
values = ds.get_attribute(full_name)
return values
finally:
SimulationState().set_current_time(calling_time)
def compare_travel_data_sets(self):
# get copied travel data csv
copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp')
if not os.path.exists(copied_travel_data_location):
raise StandardError('Travel data not found: %s' % copied_travel_data_location)
logger.log_status('Get copied travel data: %s' % copied_travel_data_location)
# convert travel data csv into travel data set matrix
in_storage = csv_storage(storage_location = copied_travel_data_location)
table_name = "travel_data"
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# get exsisting travel data set and convert it also into travel data set matrix
year = self.run_config['base_year']+2
attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory'])
cache_storage = attribute_cache.get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
from numpy import savetxt # for debugging
savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f")
savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f")
# compare both data set matices
compare = travel_data_attribute_mat == existing_travel_data_attribute_mat
# return result
return compare.all()
def test(self):
opus_core_path = OpusPackage().get_opus_core_path()
dbf_directory = os.path.join(opus_core_path, 'tests', 'data',
'dbf')
table_name = 'test_logical'
cache_directory = self._temp_dir
year = 1000
exporter = ExportDbfTableToCacheCommand(
dbf_directory=dbf_directory,
table_name=table_name,
cache_directory=cache_directory,
year=year,
)
exporter.execute()
attribute_cache = AttributeCache(cache_directory=cache_directory)
old_time = SimulationState().get_current_time()
SimulationState().set_current_time(year)
values = attribute_cache.load_table(table_name)
self.assertEqual(set(['keyid', 'works']), set(values.keys()))
self.assert_(ma.allequal(array([1, 2, 3, 4, 5]), values['keyid']))
self.assert_(ma.allequal(array([1, 1, -1, 0, 0]), values['works']))
SimulationState().set_current_time(old_time)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
csv_data_path = params_dict['csv_data_path']
table_name = params_dict['csv_table_name']
input_storage = csv_storage(storage_location = csv_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
else:
lst = [table_name]
for i in lst:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage)
logCB("Successfully exported all tables.")
def opusRun(progressCB, logCB, config):
tm_config = config["travel_model_configuration"]
for key, val in tm_config.iteritems():
tm_config[str(key)] = str(val)
opus_data_directory = tm_config["travel_data_dir"]
opus_data_year = tm_config["year_dir"]
csv_data_path = tm_config["travel_data_path"]
table_name = tm_config["travel_data_table_name"]
input_storage = csv_storage(storage_location=csv_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache())
if table_name == "ALL":
logCB("caching all tables...\n")
lst = input_storage.get_table_names()
else:
lst = [table_name]
for i in lst:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(dataset_name=i, in_storage=input_storage, out_storage=output_storage)
logCB("Successfully exported all tables.")
class QCDataSizeTests(opus_unittest.OpusTestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp_attribute_cache')
self.table_name = 'test_table'
self.storage = AttributeCache(self.temp_dir)
def tearDown(self):
if os.path.exists(self.temp_dir):
rmtree(self.temp_dir)
def test_detect(self):
# create cache where a table has attributes of different length,
# namely size 2 in 1980 and size 3 in 1979
SimulationState().set_current_time(1980)
table_data = {'int_column': np.array([100, 70], dtype="int32"),
'bool_column': np.array([False, True])}
# file name will be e.g. 'int_column.li4' for a little-endian machine
self.storage.write_table(self.table_name, table_data)
SimulationState().set_current_time(1979)
table_data = {'flt_column': np.array([10, 70, 5.7], dtype="float32")}
self.storage.write_table(self.table_name, table_data)
res = DatasetSizeModel(self.temp_dir).run()
SimulationState().set_current_time(2000)
self.assertEqual(res.sum(), 1)
# reset time to the original one
self.assertEqual(SimulationState().get_current_time(), 2000)
def test(self):
opus_core_path = OpusPackage().get_opus_core_path()
dbf_directory = os.path.join(
opus_core_path, 'tests', 'data', 'dbf')
table_name = 'test_logical'
cache_directory = self._temp_dir
year = 1000
exporter = ExportDbfTableToCacheCommand(
dbf_directory = dbf_directory,
table_name = table_name,
cache_directory = cache_directory,
year = year,
)
exporter.execute()
attribute_cache = AttributeCache(cache_directory=cache_directory)
old_time = SimulationState().get_current_time()
SimulationState().set_current_time(year)
values = attribute_cache.load_table(table_name)
self.assertEqual(set(['keyid', 'works']), set(values.keys()))
self.assert_(ma.allequal(array([1,2,3,4,5]), values['keyid']))
self.assert_(ma.allequal(array([1,1,-1,0,0]), values['works']))
SimulationState().set_current_time(old_time)
def compare_travel_data_sets(self):
# get copied travel data csv
copied_travel_data_location = os.path.join(self.destination, "opus_matsim", "tmp")
if not os.path.exists(copied_travel_data_location):
raise StandardError("Travel data not found: %s" % copied_travel_data_location)
logger.log_status("Get copied travel data: %s" % copied_travel_data_location)
# convert travel data csv into travel data set matrix
in_storage = csv_storage(storage_location=copied_travel_data_location)
table_name = "travel_data"
travel_data_attribute = "single_vehicle_to_work_travel_cost"
travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name)
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# get exsisting travel data set and convert it also into travel data set matrix
year = self.run_config["base_year"] + 2
attribute_cache = AttributeCache(cache_directory=self.run_config["cache_directory"])
cache_storage = attribute_cache.get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name)
existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(
travel_data_attribute, fill=999
)
from numpy import savetxt # for debugging
savetxt(os.path.join(self.destination, "origin_travel_data.txt"), travel_data_attribute_mat, fmt="%f")
savetxt(os.path.join(self.destination, "existing_travel_data"), existing_travel_data_attribute_mat, fmt="%f")
# compare both data set matices
compare = travel_data_attribute_mat == existing_travel_data_attribute_mat
# return result
return compare.all()
def test(self):
# Set up a test cache
storage = AttributeCache(cache_directory=self._temp_dir)
SimulationState().set_current_time(2000)
table_name = 'foo'
values = {
'attribute1': array([1, 2, 3], dtype=int32),
'attribute2': array([4, 5, 6], dtype=int32),
}
storage.write_table(table_name, values)
table_dir = os.path.join(self._temp_dir, '2000', table_name)
self.assert_(os.path.exists(table_dir))
actual = set(os.listdir(table_dir))
expected = set([
'attribute1.%(endian)si4' % replacements,
'attribute2.%(endian)si4' % replacements
])
self.assertEqual(expected, actual)
exporter = ExportCacheToDbfTableCommand(
cache_directory=self._temp_dir,
year='2000',
table_name=table_name,
dbf_directory=self._temp_dir,
decimalcount=4,
)
exporter.execute()
out_storage = dbf_storage(self._temp_dir)
db = _dbf_class(out_storage._get_file_path_for_table(table_name))
length = max([len(values[key]) for key in values.keys()])
i = 0
field_type = {}
for name, type in [
field.fieldInfo()[:2] for field in db.header.fields
]:
field_type[name] = type
for rec in db:
for key in values.keys():
if field_type[key.upper()] is 'F':
self.assertAlmostEqual(values[key][i], rec[key], 4)
else:
self.assertEqual(values[key][i], rec[key])
i = i + 1
self.assertEquals(
length,
i,
msg="More values expected than the dbf file contains")
db.close()
def save_results(self, out_storage=None, model_name=None):
if self.specification is None or self.coefficients is None:
raise ValueError, "model specification or coefficient is None"
#invalid = self.coefficients.is_invalid()
if False:
logger.log_warning('Invalid coefficients. Not saving results!')
return
if model_name is None:
model_name = self.config.get('model_name_for_coefficients', None)
if model_name is None:
if self.model_name is not None:
model_name = self.model_name
else:
raise ValueError, "model_name unspecified"
out_storage_available = True
if out_storage:
pass
elif 'estimation_database_configuration' in self.config:
try:
db_server = DatabaseServer(self.config['estimation_database_configuration'])
database_name = self.config["estimation_database_configuration"].database_name
if not db_server.has_database(database_name):
db_server.create_database(database_name)
output_db = db_server.get_database(database_name)
out_storage = StorageFactory().get_storage(
type='sql_storage',
storage_location=output_db)
except:
logger.log_warning("Problem with connecting database given by 'estimation_database_configuration'.")
out_storage_available = False
else:
logger.log_warning("No estimation_database_configuration given.")
out_storage_available = False
# the original model name of development_project_lcm is too long as a mysql db table name, truncate it
if model_name.rfind("_development_project_location_choice_model") >=0:
model_name = model_name.replace('_project', '')
specification_table = '%s_specification' % model_name
coefficients_table = '%s_coefficients' % model_name
if out_storage_available:
logger.start_block("Writing specification and coefficients into storage given by 'estimation_database_configuration'")
self.specification.write(out_storage=out_storage, out_table_name=specification_table)
self.coefficients.write(out_storage=out_storage, out_table_name=coefficients_table)
logger.end_block()
logger.start_block("Writing specification and coefficients into %s" % AttributeCache().get_storage_location())
self.specification.write(out_storage=AttributeCache(), out_table_name=specification_table)
self.coefficients.write(out_storage=AttributeCache(), out_table_name=coefficients_table)
logger.end_block()
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_directory = paths.prepend_opus_home_if_relative(
opus_data_directory)
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
overwrite = params_dict['overwrite']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name,
create_if_doesnt_exist=False)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
lst = input_storage.get_table_names()
else:
lst = re.split(' +', table_name.strip())
tables = len(lst)
lst_out = create_list_string(lst, ', ')
logCB('caching tables:\n%s\n' % lst_out)
for j, i in enumerate(lst, start=1):
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
overwrite=overwrite,
)
progressCB(100 * j / tables)
logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_directory = paths.prepend_opus_home_if_relative(opus_data_directory)
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
overwrite = params_dict['overwrite']
dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration = dbs_config)
opusdb = server.get_database(database_name=database_name, create_if_doesnt_exist=False)
input_storage = sql_storage(storage_location = opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
lst = input_storage.get_table_names()
else:
lst = re.split(' +', table_name.strip())
tables = len(lst)
lst_out = create_list_string(lst, ', ')
logCB('caching tables:\n%s\n' % lst_out)
for j, i in enumerate(lst, start=1):
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
try:
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
overwrite = overwrite,
)
except:
logCB('Error in exporting %s.' % i)
progressCB(100 * j / tables)
logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output csv data path
csv_data_path = params_dict['csv_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
execute_after_export = params_dict['execute_after_export']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
output_storage = csv_storage(storage_location = csv_data_path)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
logCB("Successfully exported all datasets.")
if execute_after_export:
file_name_list = [output_storage._get_file_path_for_table(i)
for i in opus_table_name_list]
subprocess.Popen([execute_after_export] + file_name_list)
def setUp(self):
building_data = {
'building_id': array([1, 2, 3, 4, 5, 6, 7, 8]),
'parcel_id': array([1, 2, 2, 3, 4, 4, 5, 5]),
'non_residential_sqft': \
array([6, 2, 3, 6, 1, 2, 5, 0]),
'residential_units': \
array([0, 0, 0, 0, 0, 0, 1, 1]),
'price_per_unit': \
array([50,21,32,15,60,90,100,200])
}
parcel_data = {
'parcel_id': array([1, 2, 3, 4, 5]),
'generic_land_use_type_id': array([6, 6, 3, 4, 1]),
'raz_id': array([3, 4, 5, 5, 6])
}
job_data = {
'job_id': array([1, 2, 3, 4, 5, 6, 7, 8]),
'building_id': array([1, 1, 2, 3, 6, 1, 6, 4]),
#'parcel_id': array([ 1, 1, 2, 2, 4, 1, 4, 3]),
#'raz_id': array([ 3, 3, 4, 4, 5, 3, 5, 5]),
'sector_id': array([13, 12, 13, 12, 13, 13, 12, 13]),
'dummy_id': array([1, 2, 3, 4, 5, 6, 7, 8]),
}
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
self.attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=self.attribute_cache).get_dataset_pool()
#storage = StorageFactory().get_storage('flt_storage', storage_location=self.tmp_dir)
self.attribute_cache.write_table(table_name='buildings',
table_data=building_data)
self.attribute_cache.write_table(table_name='parcels',
table_data=parcel_data)
# self.attribute_cache.write_table(table_name = 'households', table_data = household_data)
self.attribute_cache.write_table(table_name='jobs',
table_data=job_data)
# self.attribute_cache.write_table(table_name = 'persons', table_data = person_data)
# self.attribute_cache.write_table(table_name = 'refinements', table_data = refinement_data)
#self.dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim', 'opus_core'])
# self.refinement = self.dataset_pool.get_dataset('refinement')
self.jobs = self.dataset_pool.get_dataset('job')
# self.persons = self.dataset_pool.get_dataset('person')
# self.hhs = self.dataset_pool.get_dataset('household')
self.buildings = self.dataset_pool.get_dataset('building')
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
database_name = params_dict['database_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
opus_table_name = params_dict['opus_table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
#input_storage = sql_storage(storage_location = opusdb)
input_storage = attribute_cache.get_flt_storage_for_year(year)
#output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
if opus_data_year == 'ALL':
opusdb = server.get_database(database_name=database_name + "_" +
str(year))
output_storage = sql_storage(storage_location=opusdb)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = re.split(' +', opus_table_name.strip())
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
class DatasetSizeModel(Model):
"""Checks if all datasets after collapsing over all years have attributes of the same size."""
def __init__(self, directory=None):
if directory is None:
directory = SimulationState().get_cache_directory()
self.cache = AttributeCache(directory)
def run(self):
year_orig = SimulationState().get_current_time()
years = self.years_in_cache()
SimulationState().set_current_time(years[0])
storages = {}
for year in years:
storages[year] = flt_storage(os.path.join(self.cache.get_storage_location(), '%s' % year))
tables = self.cache.get_table_names()
counts = pd.Series(np.zeros(len(tables), dtype="int32"), index=tables)
for table in tables:
columns = self.cache._get_column_names_and_years(table)
values = []
names = []
colyears = []
for col, year in columns:
if col in names:
continue
data = storages[year].load_table(table, column_names=col)
values.append(data[col].size)
names.append(col)
colyears.append(year)
values = np.array(values)
if(all(values == values[0])):
continue # all attributes have the same size
# there is an inconsistency in attributes length
names = np.array(names)
colyears = np.array(colyears)
uc = np.unique(values, return_counts=True)
imax = np.argmax(uc[1])
idx = np.where(values <> uc[0][imax])[0]
df = pd.DataFrame({"column": names[idx], "year": colyears[idx], "size": values[idx]})
df = df.append(pd.DataFrame({"column": np.array(["all other columns"]), "year": np.array([years[0]]), "size": np.array([uc[0][imax]])}))
logger.log_status("Inconsistency in table ", table, ":\n", df)
counts[table] = df.shape[0] - 1
SimulationState().set_current_time(year_orig)
logger.log_status("Model total:", counts.sum(), ' size inconsistencies found.')
return counts
def years_in_cache(self):
return self.cache._get_sorted_list_of_years(start_with_current_year=False)
def setup_environment(cache_directory, year, package_order, additional_datasets={}):
gc.collect()
ss = SimulationState(new_instance=True)
ss.set_cache_directory(cache_directory)
ss.set_current_time(year)
ac = AttributeCache()
storage = ac.get_flt_storage_for_year(year)
sc = SessionConfiguration(new_instance=True,
package_order=package_order,
in_storage=ac)
logger.log_status("Setup environment for year %s. Use cache directory %s." % (year, storage.get_storage_location()))
dp = sc.get_dataset_pool()
for name, ds in additional_datasets.iteritems():
dp.replace_dataset(name, ds)
return dp
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output esri data path
esri_data_path = params_dict['esri_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
if esri_is_avail:
output_storage = esri_storage(storage_location=esri_data_path)
else:
output_storage = None
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
def _make_all_indicators(self, indicators, source_data):
computed_indicators = {}
indicators_by_dataset = {}
for year in source_data.years:
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=self.package_order,
in_storage=AttributeCache())
for name, indicator in indicators.items():
dataset_name = indicator.dataset_name
if dataset_name not in indicators_by_dataset:
indicators_by_dataset[dataset_name] = [(name, indicator)]
else:
indicators_by_dataset[dataset_name].append(
(name, indicator))
for dataset_name, indicators_in_dataset in indicators_by_dataset.items(
):
dataset = SessionConfiguration().get_dataset_from_pool(
dataset_name)
self._make_indicators_for_dataset(
dataset=dataset,
indicators_in_dataset=indicators_in_dataset,
source_data=source_data,
computed_indicators=computed_indicators,
year=year)
self.computed_indicators[source_data.name] = computed_indicators
return computed_indicators
def __init__(self, config):
ss = SimulationState(new_instance=True)
ss.set_current_time(config['base_year'])
ss.set_cache_directory(config['cache_directory'])
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
#if not os.path.exists(config['cache_directory']): ## if cache exists, it will automatically skip
cacher = CreateBaseyearCache()
cache_dir = cacher.run(config)
if 'estimation_database_configuration' in config:
db_server = DatabaseServer(config['estimation_database_configuration'])
db = db_server.get_database(config['estimation_database_configuration'].database_name)
out_storage = StorageFactory().get_storage(
'sql_storage',
storage_location = db)
else:
output_cache = os.path.join(config['cache_directory'], str(config['base_year']+1))
out_storage = StorageFactory().get_storage('flt_storage', storage_location=output_cache)
dataset_pool = SessionConfiguration().get_dataset_pool()
households = dataset_pool.get_dataset("household")
buildings = dataset_pool.get_dataset("building")
zones = dataset_pool.get_dataset("zone")
zone_ids = zones.get_id_attribute()
capacity_attribute_name = "residential_units" #_of_use_id_%s" % id
capacity_variable_name = "%s=sanfrancisco.zone.aggregate_%s_from_building" % \
(capacity_attribute_name, capacity_attribute_name)
buildings.compute_variables("sanfrancisco.building.zone_id", dataset_pool=dataset_pool)
zones.compute_variables(capacity_variable_name, dataset_pool=dataset_pool)
building_zone_id = buildings.get_attribute('zone_id')
# is_household_unplace = datasets['household'].get_attribute("building_id") <= 0
is_household_unplaced = 1 #all households are unplaced
household_building_id = zeros(households.size(), dtype='int32')-1 #datasets['household'].get_attribute("building_id")
for zone_id in zone_ids:
capacity = zones.get_attribute_by_id(capacity_attribute_name, zone_id)
is_household_in_this_zone = (households.get_attribute('zone_id') == zone_id)
is_unplaced_household_in_this_zone = is_household_in_this_zone * is_household_unplaced
is_building_in_this_zone = (building_zone_id == zone_id)
# if not is_household_in_this_zone.sum() <= capacity:
if capacity == 0 or is_household_in_this_zone.sum()==0:
print "WARNING: zone %s has %s households but only %s units" % (zone_id, is_household_in_this_zone.sum(), capacity)
continue
prob = buildings.get_attribute(capacity_attribute_name) * is_building_in_this_zone / array(capacity, dtype=float64)
r = random(sum(is_unplaced_household_in_this_zone))
prob_cumsum = ncumsum(prob)
index_to_bldg = searchsorted(prob_cumsum, r)
household_building_id[where(is_unplaced_household_in_this_zone)] = buildings.get_attribute_by_index('building_id', index_to_bldg)
# import pdb;pdb.set_trace()
households.set_values_of_one_attribute('building_id', household_building_id)
households.write_dataset(out_table_name='households', out_storage=out_storage)
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
# print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them"
# print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id"
# solved 3dec08 by hana
input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim",
"tmp")
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def get_needed_matrices_from_emme2(self,
year,
cache_directory,
bank_dir,
matrix_variable_map,
matrices_created=False):
"""Copies the specified emme/2 matrices into the specified travel_data variable names.
"""
logger.start_block('Getting matricies from emme2')
try:
zone_set = SessionConfiguration().get_dataset_from_pool('zone')
zone_set.load_dataset()
travel_data_set = self.get_travel_data_from_emme2(
zone_set, bank_dir, matrix_variable_map, matrices_created)
finally:
logger.end_block()
logger.start_block('Writing data to cache')
try:
next_year = year + 1
out_storage = AttributeCache().get_flt_storage_for_year(next_year)
travel_data_set.write_dataset(attributes='*',
out_storage=out_storage,
out_table_name='travel_data')
finally:
logger.end_block()
def setUp(self):
run_configuration = TestCacheConfiguration()
SimulationState(new_instance=True)
SessionConfiguration(run_configuration,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = run_configuration['base_year']
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Use the test cache.
opus_core_path = package().get_opus_core_path()
test_cache_path = os.path.join(opus_core_path, 'data', 'test_cache')
new_cache_path = os.path.join(self.temp_dir, 'cache')
copytree(test_cache_path, new_cache_path)
# Make sure the copied files are writable.
for (dirpath, dirnames, filenames) in os.walk(new_cache_path):
for file_name in filenames:
full_path = os.path.join(dirpath, file_name)
os.chmod(full_path, S_IWRITE | S_IREAD)
SimulationState().set_cache_directory(new_cache_path)
SimulationState().set_current_time(self.base_year)
self.config = Resources(run_configuration)
cache_directory = SimulationState().get_cache_directory()
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
def unroll_gridcells_to_cache_from_buildings(self, gridcells, buildings,
cache_directory, base_year):
"""Populate the cache with the unrolled gridcells info derived
from the buildings table.
"""
logger.start_block('Unrolling gridcell data from buildings')
try:
storage = AttributeCache().get_flt_storage_for_year(base_year)
urbansim_constant = SessionConfiguration().get_dataset_from_pool('urbansim_constant')
print "recent_years = %s" % urbansim_constant['recent_years']
recent_years = urbansim_constant['recent_years']
roller = RollbackGridcellsFromBuildings()
for year in range(base_year, base_year-recent_years-1, -1):
logger.start_block('Unrolling gridcells into year %d' % (year-1))
try:
roller.unroll_gridcells_for_one_year(gridcells,
buildings,
year,
dataset_pool=SessionConfiguration().get_dataset_pool())
flt_directory = os.path.join(cache_directory, str(year-1))
flt_storage = StorageFactory().get_storage(
type='flt_storage', subdir='store',
storage_location=flt_directory)
gridcells.write_dataset(out_storage=flt_storage)
finally:
logger.end_block()
finally:
logger.end_block()
def test_at_year_2000(self):
cache_dir = os.path.join(self.temp_dir, 'cache')
data = {
self._id_name: array([1, 2, 3]),
'population': array([10, 20, 30]),
}
self._write_data_to_year(data, cache_dir, 2000)
attribute_cache = AttributeCache(cache_directory=cache_dir)
SimulationState(new_instance=True,
base_cache_dir=self.temp_dir)
SimulationState().set_cache_directory(cache_dir)
SessionConfiguration(new_instance=True,
in_storage=attribute_cache)
SimulationState().set_current_time(2000)
dataset_pool_2000 = DatasetPool(package_order=['urbansim'],
storage=attribute_cache)
dataset = dataset_pool_2000.get_dataset(self._dataset_name)
variable_name = '%s.%s.percent_population_difference_from_2000' % (
self._package_name, self._dataset_name)
dataset.compute_variables([variable_name],
dataset_pool=dataset_pool_2000)
pop_2000 = dataset.get_attribute(variable_name)
self.assert_(ma.allequal(pop_2000, array([0, 0, 0])))
def prepare_for_simulation(self, run_configuration, cache_directory=None):
self.config = Resources(run_configuration)
self.simulation_state = SimulationState(new_instance=True,
base_cache_dir=cache_directory)
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config[
'cache_directory'] = self.simulation_state.get_cache_directory(
)
SessionConfiguration(
new_instance=True,
package_order=self.config['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
ForkProcess().fork_new_process(
self.config['creating_baseyear_cache_configuration'].
cache_scenario_database, self.config)
# Create output database (normally done by run manager)
if 'estimation_database_configuration' in self.config:
db_server = DatabaseServer(
self.config['estimation_database_configuration'])
if not db_server.has_database(
self.config['estimation_database_configuration'].
database_name):
db_server.create_database(
self.config['estimation_database_configuration'].
database_name)
def _do_run_simple_test_run(caller, temp_dir, config, end_year=None):
"""Runs model system with a single model (for speed).
Sets the .resources property of the caller before starting the run.
"""
runs_manager = RunManager(config)
run_configuration = _get_run_config(temp_dir=temp_dir)
insert_auto_generated_cache_directory_if_needed(run_configuration)
run_configuration[
'creating_baseyear_cache_configuration'].cache_directory_root = temp_dir
run_configuration['models'] = ['land_price_model']
if end_year is not None:
run_configuration['years'] = (run_configuration['years'][0], end_year)
SessionConfiguration(
new_instance=True,
package_order=run_configuration['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
insert_auto_generated_cache_directory_if_needed(run_configuration)
caller.resources = run_configuration
runs_manager.setup_new_run(
cache_directory=run_configuration['cache_directory'],
configuration=run_configuration)
runs_manager.run_run(run_configuration)
def prepare_for_simulation(self, config, cache_directory=None):
self.config = Resources(config)
base_cache_dir = self.config[
'creating_baseyear_cache_configuration'].cache_directory_root
self.simulation_state = SimulationState(new_instance=True,
base_cache_dir=base_cache_dir,
start_time=self.config.get(
'base_year', 0))
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config[
'cache_directory'] = self.simulation_state.get_cache_directory(
)
SessionConfiguration(
new_instance=True,
package_order=self.config['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
if config['creating_baseyear_cache_configuration'].cache_from_database:
ForkProcess().fork_new_process(
self.config['creating_baseyear_cache_configuration'].
cache_scenario_database, self.config)
else:
CacheFltData().run(self.config)
def _set_cache_directory(self, cache_directory):
if cache_directory != SimulationState().get_cache_directory():
SimulationState().set_cache_directory(cache_directory)
SessionConfiguration(
new_instance=True,
package_order=self.dataset_pool_configuration.package_order,
in_storage=AttributeCache())
def test(self):
urbansim_constant = UrbansimConstantDataset(
in_storage=AttributeCache())
self.assertEqual(urbansim_constant['absolute_min_year'], 1800)
self.assertAlmostEqual(urbansim_constant['acres'],
150 * 150 * 0.0002471, 6)
self.assert_(urbansim_constant["walking_distance_footprint"].ndim == 2)
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
input_directory = os.path.join(config['root'], 'opus_matsim', 'tmp')
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def _check_dataset_methods_on_dataset_view(self, ds, years_to_merge):
self.assert_(ds is not None)
ds.load_dataset(attributes='*',
in_table_name='tests',
in_storage=AttributeCache())
id = ds.get_attribute('id')
attr1 = ds.get_attribute('attr1')
# Does compute_variables work?
ds.compute_variables(['opus_core.test.attr1_times_2'])
attr1_times_2 = ds.get_attribute('attr1_times_2')
# Are values as expected?
self.assert_(ma.allequal(attr1 * 2, attr1_times_2))
# Does results have expected number of elements?
self.assertEqual(len(years_to_merge) * 3, len(attr1_times_2))
# Does _compute_if_needed work?
ds._compute_if_needed(
'opus_core.test.attr2_times_2',
dataset_pool=SessionConfiguration().get_dataset_pool())
attr2_times_2 = ds.get_attribute('attr2_times_2')
attr2 = ds.get_attribute('attr2')
self.assert_(ma.allequal(attr2 * 2, attr2_times_2))
def test_compute_a_variable(self):
"""Return merged dataset for this set of years."""
test_data = {
1000: {
'tests': {
'id': array([1, 2, 3]),
'attr1': array([10, 20, 30]),
},
},
1001: {
'tests': {
'id': array([1, 2, 3]),
'attr1': array([40, 50, 60]),
},
},
}
cache_creator = CreateTestAttributeCache()
cache_creator.create_attribute_cache_with_data(self.temp_dir,
test_data)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=['opus_core'],
in_storage=attribute_cache)
ds = MultipleYearDatasetView(
name_of_dataset_to_merge='test',
in_table_name='tests',
years_to_merge=[1000, 1001],
attribute_cache=attribute_cache,
)
ds.compute_variables(['opus_core.test.attr1_times_2'])
def setUp(self):
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp_test_indicators')
storage = StorageFactory().get_storage('dict_storage')
tables_data = {
'large_areas': {
"large_area_id":array([1, 2, 3])
},
'fazes': {
"faz_id":array([1,2,3,4]),
"large_area_id":array([1,2,3,3]),
},
'zones': {
"zone_id":array([1,2,3,4,5]),
"faz_id":array([1,2,2,3,4]),
},
'gridcells': {
"grid_id":array([1,2,3,4,5,6,7,8,9]),
"zone_id":array([1,1,1,2,2,3,3,4,5]),
"total_land_value":array([10, 11, 12, 13, 14, 15, 16, 17, 18]),
"is_in_plan_type_group_residential":array([1, 1, 1, 1, 0, 0, 1, 0, 1])
},
'urbansim_constants':{
'acres': array([1.5]),
},
}
self.year = 1000
SimulationState().set_current_time(self.year)
exporter = ExportStorage()
for table_name, table_data in tables_data.iteritems():
storage.write_table(table_name=table_name, table_data=table_data)
exporter.export_dataset(dataset_name=table_name,
in_storage=storage,
out_storage=AttributeCache(cache_directory=self.temp_dir))
def prepare_for_running_macro(parser):
from opus_core.file_utilities import get_resources_from_file
parser.add_option("-r",
"--resources",
dest="resources_file_name",
action="store",
type="string",
help="Name of file containing resources")
parser.add_option("-y",
"--year",
dest="year",
action="store",
type="int",
help="Year in which to 'run' the travel model")
parser.add_option(
"-o",
"--output-file",
dest="output_file",
action="store",
type="string",
default=None,
help=
"Output log file. If not given, it is written into urbansim cache directory."
)
(options, args) = parser.parse_args()
r = get_resources_from_file(options.resources_file_name)
resources = Resources(get_resources_from_file(options.resources_file_name))
SessionConfiguration(
new_instance=True,
package_order=resources['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
return (resources, options)
def test(self):
# Set up a test cache
storage = AttributeCache(cache_directory=self._temp_dir)
SimulationState().set_current_time(2000)
table_name = 'foo'
values = {
'attribute1': array([1,2,3], dtype=int32),
'attribute2': array([4,5,6], dtype=int32),
}
storage.write_table(table_name, values)
table_dir = os.path.join(self._temp_dir, '2000', table_name)
self.assert_(os.path.exists(table_dir))
actual = set(os.listdir(table_dir))
expected = set(['attribute1.%(endian)si4' % replacements, 'attribute2.%(endian)si4' % replacements])
self.assertEqual(expected, actual)
exporter = ExportCacheToDbfTableCommand(
cache_directory = self._temp_dir,
year = '2000',
table_name = table_name,
dbf_directory = self._temp_dir,
decimalcount = 4,
)
exporter.execute()
out_storage = dbf_storage(self._temp_dir)
db = _dbf_class(out_storage._get_file_path_for_table(table_name))
length = max([len(values[key]) for key in values.keys()])
i = 0
field_type = {}
for name, type in [field.fieldInfo()[:2] for field in db.header.fields]:
field_type[name] = type
for rec in db:
for key in values.keys():
if field_type[key.upper()] is 'F':
self.assertAlmostEqual(values[key][i], rec[key], 4)
else:
self.assertEqual(values[key][i], rec[key])
i = i + 1
self.assertEquals(length, i, msg="More values expected than the dbf file contains")
db.close()
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output esri data path
esri_data_path = params_dict['esri_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
if esri_is_avail:
output_storage = esri_storage(storage_location = esri_data_path)
else:
output_storage = None
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
def setUp(self):
household_data = {
'household_id': array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
'blockgroup_id': array([1, 1, 1, 1, 2, 2, 2, 2, 2, 3]),
}
fraction_data = {
'fraction_id':
array([1, 2, 3, 4, 5, 6,
6]), #unused, but necessary to use dataset_pool to get data
'blockgroup_id': array([1, 1, 1, 2, 2, 2, 3]),
'zone_id': array([1, 2, 3, 3, 4, 5, 6]),
'fraction': array([0.25, 0.25, 0.5, 0.2, 0.4, 0.4, 1.0])
}
blockgroup_data = {
#unused by the model, for result verification only
'blockgroup_id': array([1, 2, 3]),
}
zone_data = {
#unused by the model, for result verification only
'zone_id': array([1, 2, 3, 4, 5, 6]),
}
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=attribute_cache).get_dataset_pool()
#storage = StorageFactory().get_storage('flt_storage', storage_location=self.tmp_dir)
attribute_cache.write_table(table_name='households',
table_data=household_data)
attribute_cache.write_table(table_name='fractions',
table_data=fraction_data)
attribute_cache.write_table(table_name='blockgroups',
table_data=blockgroup_data)
attribute_cache.write_table(table_name='zones', table_data=zone_data)
#self.dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim', 'opus_core'])
self.household = self.dataset_pool.get_dataset('household')
self.fraction = self.dataset_pool.get_dataset('fraction')
self.blockgroup = self.dataset_pool.get_dataset('blockgroup')
self.zone = self.dataset_pool.get_dataset('zone')
def run(self, location_set, development_event_set, *args, **kwargs):
changed_indices, processed_development_event_indices = \
EventsCoordinator.run(self, location_set,
development_event_set, *args, **kwargs)
if development_event_set is not None:
subset = DatasetSubset(development_event_set,
processed_development_event_indices)
subset.write_dataset(out_storage=AttributeCache())
return (changed_indices, processed_development_event_indices)
def run(self, directory=None, check_size=True):
"""
"directory" is the cache to be compared to the reference. It should not include the year
as the model checks all years.
Set "check_sizes" to False if no size check of the datasets is required.
"""
if directory is None:
directory = SimulationState().get_cache_directory()
self.cache = AttributeCache(directory)
year_orig = SimulationState().get_current_time()
years = self.years_in_cache()
SimulationState().set_current_time(years[0])
storages = {}
for year in years:
storages[year] = flt_storage(os.path.join(self.cache.get_storage_location(), '%s' % year))
df = pd.DataFrame(columns=["Table", "Less-than-ref", "More-than-ref", "Year", "Size", "Size-ref"])
tables = self.cache.get_table_names()
for table in tables:
columns_list = self.cache.get_column_names(table)
columns = Set(columns_list)
ref_columns_list = self.reference_storage.get_column_names(table, lowercase=True)
ref_columns = Set(ref_columns_list)
more = columns.difference(ref_columns)
less = ref_columns.difference(columns)
samesize = True
if check_size:
table_size = self.cache.load_table(table, columns_list[0])[columns_list[0]].size
reftable_size = self.reference_storage.load_table(table, ref_columns_list[0])[ref_columns_list[0]].size
if table_size <> reftable_size:
samesize = False
if len(more) == 0 and len(less) == 0 and samesize:
continue
df.loc[df.shape[0]] = [table, ', '.join(less), ', '.join(more), '', 0, 0]
if len(more) == 0 and samesize:
continue
# if there are columns in the "more" column, write out the corresponding years
columns_and_years = self.cache._get_column_names_and_years(table)
more_years = []
for col, year in columns_and_years:
if col in more:
more_years.append(year)
df.loc[df.shape[0]-1, "Year"] = ', '.join(np.unique(np.array(more_years).astype("str")))
if not samesize: # there is difference in table sizes
df.loc[df.shape[0]-1, "Size"] = table_size
df.loc[df.shape[0]-1, "Size-ref"] = reftable_size
if not check_size or (df['Size'].sum()==0 and df['Size-ref'].sum()==0):
# remove the size columns if not used
del df['Size']
del df['Size-ref']
if df.shape[0] > 0:
logger.log_status("Differences in data structure relative to %s:" % self.reference_storage.get_storage_location())
logger.log_status(df)
else:
logger.log_status("Data structure corresponds to the one in %s" % self.reference_storage.get_storage_location())
return df
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
database_name = params_dict['database_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
opus_table_name = params_dict['opus_table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration = dbs_config)
opusdb = server.get_database(database_name=database_name)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
#input_storage = sql_storage(storage_location = opusdb)
input_storage = attribute_cache.get_flt_storage_for_year(year)
#output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
output_storage = sql_storage(storage_location = opusdb)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration = dbs_config)
opusdb = server.get_database(database_name=database_name)
input_storage = sql_storage(storage_location = opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
else:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(table_name, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name = table_name,
in_storage = input_storage,
out_storage = output_storage)
def import_travel_model_data(config, year):
cache_directory = config['cache_directory']
simulation_state = SimulationState()
simulation_state.set_current_time(year)
simulation_state.set_cache_directory(cache_directory)
out_store = AttributeCache().get_flt_storage_for_year(year+1)
out_store_loc = out_store.get_storage_location()
tm_config = config['travel_model_configuration']
data_to_import = tm_config['tm_to_urbansim_variable_mapping']
base_dir = mtc_common.tm_get_base_dir(config)
data_dir = tm_config[year]['data_dir']
for dataset_name, skim_file in data_to_import.iteritems():
skim_file = os.path.join(base_dir, data_dir, skim_file)
data = read_csv(skim_file, header=0)
with block("Caching {} to {}".format(dataset_name, out_store_loc)):
logger.log_status("Source file {}".format(skim_file))
opus_ds = to_opus_dataset(data, out_store, dataset_name)
def create_attribute_cache_with_data(self, cache_dir, data):
"""Populate the cache_dir with the given datasets for the given years.
data is a dictionary with year as key.
The value of each year is a dictionary with dataset name as key.
The value for each dataset name is a dictionary with attribute name as key.
The value for each attribute name is a numpy array of values.
cache_dir must exist.
"""
SimulationState().set_cache_directory(cache_dir)
attr_cache = AttributeCache()
for year, datasets in data.iteritems():
year_dir = os.path.join(cache_dir, str(year))
if not os.path.exists(year_dir):
os.makedirs(year_dir)
SimulationState().set_current_time(year)
flt_storage = attr_cache.get_flt_storage_for_year(year)
for dataset_name, attributes in datasets.iteritems():
flt_storage.write_table(table_name=dataset_name, table_data=attributes)
def run(self, table_names, out_storage=None, table_name_pattern=None, cache_directory=None, year=None, **kwargs):
"""
export specified tables to database
table_name_pattern: For example '{table_name}_{scenario_name}_{year}'
"""
if not hasattr(self, "out_storage"):
if out_storage is None:
raise ValueError, "Either out_storage argument needs to be specified or " + "prepare_for_run called before run method to create a valid out_storage."
else:
self.out_storage = out_storage
sim_state = SimulationState()
if sim_state.get_current_time() == 0:
sim_state.set_current_time(9999)
if cache_directory is None:
cache_directory = sim_state.get_cache_directory()
attr_cache = AttributeCache(cache_directory=cache_directory)
if year is None:
years = attr_cache._get_sorted_list_of_years()
else:
assert isinstance(year, int)
years = [year]
for table_name in table_names:
kwargs["table_name"] = table_name
for year in years:
kwargs["year"] = year
out_table_name = table_name_pattern.format(**kwargs)
in_storage = attr_cache.get_flt_storage_for_year(year)
# cache_path = os.path.join(cache_directory, str(year))
# in_storage = flt_storage(storage_location=cache_path)
# TODO drop_table(table_name) if table_name exists
ExportStorage().export_dataset(
table_name, in_storage=in_storage, out_storage=self.out_storage, out_dataset_name=out_table_name
)
self.post_run(kwargs["scenario_name"], years)
def __init__(self, output_dir=None, year=None):
''' Constructor
'''
# get working path as an anchor e.g. to determine the config file location.
self.working_path = test_path.__path__[0]
print "Working path: %s" % self.working_path
# get config file location
self.config_file = os.path.join( self.working_path, 'configs', 'seattle_parcel_travel_cost_test.xml')
# get seattle_parcel configuration
config = XMLConfiguration( self.config_file ).get_run_configuration( "Seattle_baseline" )
self.input_storage = None
# get first simulation year
self.year = year
if self.year == None:
self.year = config['base_year']
base_year_data_path = paths.get_opus_data_path_path('seattle_parcel', 'base_year_data')
attribute_cache = AttributeCache(cache_directory=base_year_data_path)
self.input_storage = attribute_cache.get_flt_storage_for_year(self.year)
else:
attribute_cache = AttributeCache().get_flt_storage_for_year(self.year)
self.input_storage = attribute_cache
# get output dir path
output_directory = output_dir
if output_directory == None:
# set deafult
output_directory = paths.get_opus_home_path('opus_matsim', 'tmp')
if not os.path.exists( output_directory ):
try: os.mkdir( output_directory )
except: pass
# init
self.csv_data_path = output_directory # os.path.join(output_directory, 'travel_data_dir')
def setUp(self):
building_data = {
'building_id': array([1, 2, 3, 4, 5, 6, 7, 8]),
'parcel_id': array([1, 2, 2, 3, 4, 4, 5, 5]),
'non_residential_sqft': \
array([6, 2, 3, 6, 1, 2, 5, 0]),
'residential_units': \
array([0, 0, 0, 0, 0, 0, 1, 1]),
'price_per_unit': \
array([50,21,32,15,60,90,100,200])
}
parcel_data = {
'parcel_id': array([1, 2, 3, 4, 5]),
'generic_land_use_type_id': array([6, 6, 3, 4, 1]),
'raz_id': array([3, 4, 5, 5, 6])
}
job_data = {
'job_id': array([ 1, 2, 3, 4, 5, 6, 7, 8]),
'building_id': array([ 1, 1, 2, 3, 6, 1, 6, 4]),
#'parcel_id': array([ 1, 1, 2, 2, 4, 1, 4, 3]),
#'raz_id': array([ 3, 3, 4, 4, 5, 3, 5, 5]),
'sector_id': array([13,12,13,12,13,13,12,13]),
'dummy_id': array([ 1, 2, 3, 4, 5, 6, 7, 8]),
}
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
self.attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=self.attribute_cache).get_dataset_pool()
#storage = StorageFactory().get_storage('flt_storage', storage_location=self.tmp_dir)
self.attribute_cache.write_table(table_name = 'buildings', table_data = building_data)
self.attribute_cache.write_table(table_name = 'parcels', table_data = parcel_data)
# self.attribute_cache.write_table(table_name = 'households', table_data = household_data)
self.attribute_cache.write_table(table_name = 'jobs', table_data = job_data)
# self.attribute_cache.write_table(table_name = 'persons', table_data = person_data)
# self.attribute_cache.write_table(table_name = 'refinements', table_data = refinement_data)
#self.dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim', 'opus_core'])
# self.refinement = self.dataset_pool.get_dataset('refinement')
self.jobs = self.dataset_pool.get_dataset('job')
# self.persons = self.dataset_pool.get_dataset('person')
# self.hhs = self.dataset_pool.get_dataset('household')
self.buildings = self.dataset_pool.get_dataset('building')
def setUp(self):
household_data = {
'household_id': array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
'blockgroup_id': array([1, 1, 1, 1, 2, 2, 2, 2, 2, 3]),
}
fraction_data = {
'fraction_id': array([1, 2, 3, 4, 5, 6, 6]), #unused, but necessary to use dataset_pool to get data
'blockgroup_id': array([1, 1, 1, 2, 2, 2, 3]),
'zone_id': array([1, 2, 3, 3, 4, 5, 6]),
'fraction': array([0.25, 0.25, 0.5, 0.2, 0.4, 0.4, 1.0])
}
blockgroup_data = {
#unused by the model, for result verification only
'blockgroup_id': array([1, 2, 3]),
}
zone_data = {
#unused by the model, for result verification only
'zone_id': array([1, 2, 3, 4, 5, 6]),
}
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=attribute_cache).get_dataset_pool()
#storage = StorageFactory().get_storage('flt_storage', storage_location=self.tmp_dir)
attribute_cache.write_table(table_name = 'households', table_data = household_data)
attribute_cache.write_table(table_name = 'fractions', table_data = fraction_data)
attribute_cache.write_table(table_name = 'blockgroups', table_data = blockgroup_data)
attribute_cache.write_table(table_name = 'zones', table_data = zone_data)
#self.dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim', 'opus_core'])
self.household = self.dataset_pool.get_dataset('household')
self.fraction = self.dataset_pool.get_dataset('fraction')
self.blockgroup = self.dataset_pool.get_dataset('blockgroup')
self.zone = self.dataset_pool.get_dataset('zone')
class MergeCache(Model):
"""Merge multiple years of one cache directory into a single one that can be used
for example for a warm start."""
def __init__(self, directory):
self.storage = AttributeCache(directory)
def run(self, year, cleanup_settings={}):
SimulationState().set_current_time(year)
tables = self.storage.get_table_names()
# cleanup
for table in tables:
tabdata = self.storage.load_table(table)
if table in cleanup_settings.keys():
for attr in cleanup_settings[table]:
if attr in tabdata.keys():
logger.log_status('Deleting attribute %s in %s.' % (attr, table))
del tabdata[attr]
self.storage.write_table(table, tabdata)
logger.log_status('Deleting all computed tables.')
self.storage.delete_computed_tables()
logger.log_status('Cache directory merged into %s' % year)
cache_year is None):
parser.print_help()
sys.exit(1)
if table_name is None:
table_name = 'ALL'
dbserverconfig = EstimationDatabaseConfiguration(
database_configuration = options.database_configuration
)
opusdb = OpusDatabase(dbserverconfig, db_name)
input_storage = sql_storage(storage_location = opusdb)
attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
SimulationState().set_current_time(cache_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logger.log_status('Caching all tables in database...')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
def setUp(self):
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp_attribute_cache')
self.table_name = 'test_table'
self.storage = AttributeCache(self.temp_dir)
def setUp(self, attribute_cache=True):
hh_data = {
'household_id': array([1, 2, 3, 4]),
'building_id': array([11, 22, 33, 22]),
'size': array([4, 3, 2, 1]),
'income': array([51, 52, 53, 54])*1000,
'keep': array([4.1, 4.2, 4.3, 4.4]),
}
p_data = {
'person_id': array([ 1, 2, 3, 5, 6, 7, 8, 9, 10]),
'household_id': array([ 1, 1, 1, 2, 2, 3, 3, 3, 4]),
'age': array([75, 71, 29, 56, 16, 22, 20, 96, 88]),
}
if attribute_cache:
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
self.attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=self.attribute_cache
).get_dataset_pool()
self.attribute_cache.write_table(table_name='households',
table_data=hh_data)
self.attribute_cache.write_table(table_name='persons',
table_data=p_data)
self.hh_ds = self.dataset_pool.get_dataset('household')
self.p_ds = self.dataset_pool.get_dataset('person')
else:
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=hh_data)
self.hh_ds = Dataset(in_storage=storage,
in_table_name='households',
dataset_name='household')
storage.write_table(table_name='persons',
table_data=p_data)
self.p_ds = Dataset(in_storage=storage,
in_table_name='persons',
dataset_name='person')
self.dmgh_data_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
self.dmgh_data_file = os.path.join(self.dmgh_data_dir,
'demographic_data.h5')
out_fh = h5py.File(self.dmgh_data_file, 'w')
n_hhs = 5
hh_dtype = {'names':['year', 'household_id', 'income', 'head_person_id'],
'formats':['i4', 'i4', 'f8', 'i4']}
hhdata = out_fh.create_dataset('household', shape=(n_hhs, ), dtype=hh_dtype,
compression='gzip', compression_opts=9)
hhs = [(2000, 5, 65000.0, 9),
(2000, 1, 61000.0, 3),
(2000, 2, 62000.0, 4),
(2000, 3, 63000.0, 7),
(2001, 1, 71000.0, 3)]
hhdata[:] = array(hhs, dtype=hh_dtype)
n_ps = 16
ps_dtype = {'names':['year', 'person_id', 'household_id', 'age'],
'formats':['i4', 'i4', 'i4', 'i4']}
psdata = out_fh.create_dataset('person', shape=(n_ps, ), dtype=ps_dtype,
compression='gzip', compression_opts=9)
ps = [(2000, 1, 1, 76),
(2000, 2, 1, 72),
(2000, 3, 1, 30),
(2000, 4, 2, -1),
(2000, 5, 2, 57),
(2000, 6, 2, 17),
(2000, 9, 5, 67),
(2000,10, 5, 71),
(2000, 7, 3, 23),
(2000, 8, 3, 21),
(2000,81, 3, 2),
(2001, 1, 1, 77),
(2001, 2, 1, 73),
(2001, 3, 1, 31),
(2001, 4, 1, 35),
(2001,31, 1, 1)]
psdata[:] = array(ps, dtype=ps_dtype)
dataset_names = ['household', 'person']
for dataset_name in dataset_names:
for year in unique(out_fh[dataset_name][:, 'year']):
year_str = str(year)
group = out_fh.get(year_str, None)
if group is None:
group = out_fh.create_group(year_str)
is_year = out_fh[dataset_name][:, 'year'] == year
group.create_dataset(dataset_name, data=out_fh[dataset_name][is_year])
del out_fh[dataset_name]
out_fh.close()
def setUp(self):
building_data = {
'building_id': array([1, 2, 3, 4, 5, 6, 7, 8]),
'parcel_id': array([1, 2, 2, 3, 4, 4, 5, 5]),
'non_residential_sqft': \
array([6, 2, 3, 6, 1, 2, 5, 0]),
'residential_units': \
array([0, 0, 0, 0, 0, 0, 1, 1])
}
parcel_data = {
'parcel_id': array([1, 2, 3, 4, 5]),
'generic_land_use_type_id': array([6, 6, 3, 4, 1]),
'raz_id': array([3, 4, 5, 5, 6])
}
job_data = {
'job_id': array([ 1, 2, 3, 4, 5, 6, 7, 8]),
'building_id': array([ 1, 1, 2, 3, 6, 1, 6, 4]),
'sector_id': array([13,12,13,12,13,13,12,13]),
'dummy_id': array([ 1, 2, 3, 4, 5, 6, 7, 8])
}
household_data = {
'household_id': array([1, 2]),
'building_id': array([7, 8]),
'persons': array([3, 4]),
}
person_data = {
'person_id': array([ 1, 2, 3, 4, 5, 6, 7]),
'household_id': array([ 1, 1, 1, 2, 2, 2, 2]),
'job_id': array([ 2, 1, -1, -1, 3, 4, 7])
}
refinement_data = {
'refinement_id': arange(1, 8),
'year': array([2021,2021,2021,2022, 2023, 2024, 2024]),
'transaction_id':array([1, 1, 1, 2, 3, 1, 1]),
'action': array(['subtract', 'subtract', 'add', 'target', 'add', 'add', 'set_value']),
'amount': array([2, 1, 4, 7, 1, 1, -1]),
'agent_dataset': array(['job',
'job',
'job',
'household',
'household',
'household',
'person'
]),
'agent_expression': array(['job.sector_id==13',
'job.sector_id==13',
'',
'household.household_id>0',
'household.persons>5',
'household.persons==3',
'person.job_id'
]),
'location_expression': array(['urbansim.building.raz_id==3',
'urbansim.building.raz_id==4',
'(urbansim.building.raz_id==5) * (building.disaggregate(parcel.generic_land_use_type_id)==4)',
'urbansim.building.raz_id==6',
'urbansim.building.raz_id==6',
'urbansim.building.raz_id==6',
'household.refinement_id==6'
]),
'location_capacity_attribute':array(['',
'non_residential_sqft',
'non_residential_sqft',
'residential_units',
'residential_units',
'',
''
])
}
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=attribute_cache).get_dataset_pool()
#storage = StorageFactory().get_storage('flt_storage', storage_location=self.tmp_dir)
attribute_cache.write_table(table_name = 'buildings', table_data = building_data)
attribute_cache.write_table(table_name = 'parcels', table_data = parcel_data)
attribute_cache.write_table(table_name = 'households', table_data = household_data)
attribute_cache.write_table(table_name = 'jobs', table_data = job_data)
attribute_cache.write_table(table_name = 'persons', table_data = person_data)
attribute_cache.write_table(table_name = 'refinements', table_data = refinement_data)
#self.dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim', 'opus_core'])
self.refinement = self.dataset_pool.get_dataset('refinement')
self.jobs = self.dataset_pool.get_dataset('job')
self.persons = self.dataset_pool.get_dataset('person')
self.hhs = self.dataset_pool.get_dataset('household')
self.buildings = self.dataset_pool.get_dataset('building')
class Tests(opus_unittest.OpusTestCase):
"""unittest"""
def setUp(self, attribute_cache=True):
hh_data = {
'household_id': array([1, 2, 3, 4]),
'building_id': array([11, 22, 33, 22]),
'size': array([4, 3, 2, 1]),
'income': array([51, 52, 53, 54])*1000,
'keep': array([4.1, 4.2, 4.3, 4.4]),
}
p_data = {
'person_id': array([ 1, 2, 3, 5, 6, 7, 8, 9, 10]),
'household_id': array([ 1, 1, 1, 2, 2, 3, 3, 3, 4]),
'age': array([75, 71, 29, 56, 16, 22, 20, 96, 88]),
}
if attribute_cache:
self.tmp_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
SimulationState().set_cache_directory(self.tmp_dir)
self.attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=self.attribute_cache
).get_dataset_pool()
self.attribute_cache.write_table(table_name='households',
table_data=hh_data)
self.attribute_cache.write_table(table_name='persons',
table_data=p_data)
self.hh_ds = self.dataset_pool.get_dataset('household')
self.p_ds = self.dataset_pool.get_dataset('person')
else:
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data=hh_data)
self.hh_ds = Dataset(in_storage=storage,
in_table_name='households',
dataset_name='household')
storage.write_table(table_name='persons',
table_data=p_data)
self.p_ds = Dataset(in_storage=storage,
in_table_name='persons',
dataset_name='person')
self.dmgh_data_dir = tempfile.mkdtemp(prefix='urbansim_tmp')
self.dmgh_data_file = os.path.join(self.dmgh_data_dir,
'demographic_data.h5')
out_fh = h5py.File(self.dmgh_data_file, 'w')
n_hhs = 5
hh_dtype = {'names':['year', 'household_id', 'income', 'head_person_id'],
'formats':['i4', 'i4', 'f8', 'i4']}
hhdata = out_fh.create_dataset('household', shape=(n_hhs, ), dtype=hh_dtype,
compression='gzip', compression_opts=9)
hhs = [(2000, 5, 65000.0, 9),
(2000, 1, 61000.0, 3),
(2000, 2, 62000.0, 4),
(2000, 3, 63000.0, 7),
(2001, 1, 71000.0, 3)]
hhdata[:] = array(hhs, dtype=hh_dtype)
n_ps = 16
ps_dtype = {'names':['year', 'person_id', 'household_id', 'age'],
'formats':['i4', 'i4', 'i4', 'i4']}
psdata = out_fh.create_dataset('person', shape=(n_ps, ), dtype=ps_dtype,
compression='gzip', compression_opts=9)
ps = [(2000, 1, 1, 76),
(2000, 2, 1, 72),
(2000, 3, 1, 30),
(2000, 4, 2, -1),
(2000, 5, 2, 57),
(2000, 6, 2, 17),
(2000, 9, 5, 67),
(2000,10, 5, 71),
(2000, 7, 3, 23),
(2000, 8, 3, 21),
(2000,81, 3, 2),
(2001, 1, 1, 77),
(2001, 2, 1, 73),
(2001, 3, 1, 31),
(2001, 4, 1, 35),
(2001,31, 1, 1)]
psdata[:] = array(ps, dtype=ps_dtype)
dataset_names = ['household', 'person']
for dataset_name in dataset_names:
for year in unique(out_fh[dataset_name][:, 'year']):
year_str = str(year)
group = out_fh.get(year_str, None)
if group is None:
group = out_fh.create_group(year_str)
is_year = out_fh[dataset_name][:, 'year'] == year
group.create_dataset(dataset_name, data=out_fh[dataset_name][is_year])
del out_fh[dataset_name]
out_fh.close()
def tearDown(self):
shutil.rmtree(self.dmgh_data_dir)
if hasattr(self, 'tmp_dir'):
shutil.rmtree(self.tmp_dir)
def test_run1(self):
model = ExternalDemographicModel()
attrs_mapping = {'income': "household.income",
'size': "household.number_of_agents(person)",
'age_of_head': "household.aggregate(person.age * (person.disaggregate(household.head_person_id)==person.person_id))",
}
attrs_mapping_p = { 'household_id': 'person.household_id',
'age': 'person.age',
'age_months' : 'age * 12', }
model.run(self.dmgh_data_file, self.hh_ds, self.p_ds,
year=2000,
keep_attributes=['building_id', 'keep'],
keep_attributes_p=[],
demographic_attributes=attrs_mapping,
demographic_attributes_p=attrs_mapping_p,
dataset_pool=self.dataset_pool)
new_hh_ds = self.dataset_pool.get_dataset('household')
self.assert_(allclose(new_hh_ds['household_id'], array([ 5, 1, 2, 3])))
self.assert_(allclose(new_hh_ds['building_id'], array([-1, 11, 22, 33])))
self.assert_(allclose(new_hh_ds['keep'], array([-1,4.1, 4.2, 4.3])))
self.assert_(allclose(new_hh_ds['income'], array([65, 61, 62, 63])*1000.0))
self.assert_(allclose(new_hh_ds['size'], array([ 2, 3, 3, 3])))
self.assert_(allclose(new_hh_ds['age_of_head'], array([67, 30, -1, 23])))
new_p_ds = self.dataset_pool.get_dataset('person')
print('array([' + ', '.join([str(i) for i in new_p_ds['person_id']]) + '])')
print('array([' + ', '.join([str(i) for i in new_p_ds['household_id']]) + '])')
print('array([' + ', '.join([str(i) for i in new_p_ds['age']]) + '])')
print('array([' + ', '.join([str(i) for i in new_p_ds['age_months']]) + '])')
self.assert_(allclose(new_p_ds['person_id'], array([1, 2, 3, 4, 5, 6, 9, 10, 7, 8, 81])))
self.assert_(allclose(new_p_ds['household_id'], array([1, 1, 1, 2, 2, 2, 5, 5, 3, 3, 3])))
self.assert_(allclose(new_p_ds['age'], array([76, 72, 30, -1, 57, 17, 67, 71, 23, 21, 2])))
self.assert_(allclose(new_p_ds['age_months'], array([912, 864, 360, -12, 684, 204, 804, 852, 276, 252, 24])))
self.assert_((new_p_ds['age'] * 12 == new_p_ds['age_months']).all(), 'age_months computed correctly')
model.run(self.dmgh_data_file, self.hh_ds, self.p_ds,
year=2001,
keep_attributes=['building_id', 'keep'],
keep_attributes_p=[],
demographic_attributes=attrs_mapping,
demographic_attributes_p=attrs_mapping_p,
dataset_pool=self.dataset_pool)
new_hh_ds = self.dataset_pool.get_dataset('household')
self.assert_(allclose(new_hh_ds['household_id'], array([ 1])))
self.assert_(allclose(new_hh_ds['building_id'], array([11])))
self.assert_(allclose(new_hh_ds['keep'], array([4.1])))
self.assert_(allclose(new_hh_ds['income'], array([71])*1000.0))
self.assert_(allclose(new_hh_ds['size'], array([ 5])))
self.assert_(allclose(new_hh_ds['age_of_head'], array([31])))
new_p_ds = self.dataset_pool.get_dataset('person')
print('array([' + ', '.join([str(i) for i in new_p_ds['person_id']]) + '])')
print('array([' + ', '.join([str(i) for i in new_p_ds['household_id']]) + '])')
print('array([' + ', '.join([str(i) for i in new_p_ds['age']]) + '])')
print('array([' + ', '.join([str(i) for i in new_p_ds['age_months']]) + '])')
self.assert_(allclose(new_p_ds['person_id'], array([1, 2, 3, 4, 31])))
self.assert_(allclose(new_p_ds['household_id'], array([1, 1, 1, 1, 1])))
self.assert_(allclose(new_p_ds['age'], array([77, 73, 31, 35, 1])))
self.assert_(allclose(new_p_ds['age_months'], array([924, 876, 372, 420, 12])))
self.assert_((new_p_ds['age'] * 12 == new_p_ds['age_months']).all(), 'age_months computed correctly')
def __init__(self, directory=None):
if directory is None:
directory = SimulationState().get_cache_directory()
self.cache = AttributeCache(directory)
class Tests(opus_unittest.OpusTestCase):
def setUp(self):
building_data = {
"building_id": array([1, 2, 3, 4, 5, 6, 7, 8]),
"parcel_id": array([1, 2, 2, 3, 4, 4, 5, 5]),
"non_residential_sqft": array([6, 2, 3, 6, 1, 2, 5, 0]),
"residential_units": array([0, 0, 0, 0, 0, 0, 1, 1]),
"price_per_unit": array([50, 21, 32, 15, 60, 90, 100, 200]),
}
parcel_data = {
"parcel_id": array([1, 2, 3, 4, 5]),
"generic_land_use_type_id": array([6, 6, 3, 4, 1]),
"raz_id": array([3, 4, 5, 5, 6]),
}
job_data = {
"job_id": array([1, 2, 3, 4, 5, 6, 7, 8]),
"building_id": array([1, 1, 2, 3, 6, 1, 6, 4]),
#'parcel_id': array([ 1, 1, 2, 2, 4, 1, 4, 3]),
#'raz_id': array([ 3, 3, 4, 4, 5, 3, 5, 5]),
"sector_id": array([13, 12, 13, 12, 13, 13, 12, 13]),
"dummy_id": array([1, 2, 3, 4, 5, 6, 7, 8]),
}
self.tmp_dir = tempfile.mkdtemp(prefix="urbansim_tmp")
SimulationState().set_cache_directory(self.tmp_dir)
self.attribute_cache = AttributeCache()
self.dataset_pool = SessionConfiguration(
new_instance=True, package_order=["urbansim", "opus_core"], in_storage=self.attribute_cache
).get_dataset_pool()
# storage = StorageFactory().get_storage('flt_storage', storage_location=self.tmp_dir)
self.attribute_cache.write_table(table_name="buildings", table_data=building_data)
self.attribute_cache.write_table(table_name="parcels", table_data=parcel_data)
# self.attribute_cache.write_table(table_name = 'households', table_data = household_data)
self.attribute_cache.write_table(table_name="jobs", table_data=job_data)
# self.attribute_cache.write_table(table_name = 'persons', table_data = person_data)
# self.attribute_cache.write_table(table_name = 'refinements', table_data = refinement_data)
# self.dataset_pool = DatasetPool(storage = storage, package_order = ['urbansim_parcel', 'urbansim', 'opus_core'])
# self.refinement = self.dataset_pool.get_dataset('refinement')
self.jobs = self.dataset_pool.get_dataset("job")
# self.persons = self.dataset_pool.get_dataset('person')
# self.hhs = self.dataset_pool.get_dataset('household')
self.buildings = self.dataset_pool.get_dataset("building")
# self.buildings.compute_variables('raz_id=building.disaggregate(parcel.raz_id)', self.dataset_pool)
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def test_add_and_remove_agents(self):
"""
"""
scheduled_events_data = {
"year": array([2000, 2000, 2000, 2000, 2000]),
"action": array(["remove", "remove", "add", "add", "target"]),
"amount": array([1, 1, 4, 3, 7]),
"sector_id": array([13, 12, -1, 11, 12]),
"building_id": array([-1, -1, -1, 8, -1]),
"raz_id": array([3, 5, 5, -1, -1]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="events", table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage, in_table_name="events", id_name=[])
model = ScheduledEventsModel(self.jobs, scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
# check that there are indeed 50000 total households after running the model
results = self.jobs.size()
should_be = 18
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
# examine each action in turn:
results = logical_and(self.jobs.get_attribute("sector_id") == 13, self.jobs.get_attribute("raz_id") == 3).sum()
should_be = 2 - 1
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = logical_and(self.jobs.get_attribute("sector_id") == 12, self.jobs.get_attribute("raz_id") == 5).sum()
should_be = 1 - 1
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = (self.jobs.get_attribute("raz_id") == 5).sum()
should_be = 3 - 1 + 4
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = logical_and(
self.jobs.get_attribute("sector_id") == 11, self.jobs.get_attribute("building_id") == 8
).sum()
should_be = 0 + 3
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = (self.jobs.get_attribute("sector_id") == 12).sum()
should_be = 7
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
def DELtest_add_and_remove_agents_from_geography_other_than_location_set(self):
"""this has been included in the above test
"""
scheduled_events_data = {
"year": array([2000, 2000, 2000, 2000, 2000]),
"action": array(["remove", "remove", "add", "add", "target"]),
"amount": array([1, 1, 4, 3, 7]),
"sector_id": array([13, 13, -1, 11, 12]),
"building_id": array([-1, -1, -1, 8, -1]),
"raz_id": array([3, 4, 5, -1, -1]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="events", table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage, in_table_name="events", id_name=[])
model = ScheduledEventsModel(self.jobs, scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
# check that there are indeed 50000 total households after running the model
results = self.jobs.size()
should_be = 17
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
# examine each action in turn:
results = logical_and(self.jobs.get_attribute("sector_id") == 13, self.jobs.get_attribute("raz_id") == 3).sum()
should_be = 2 - 1
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = logical_and(self.jobs.get_attribute("sector_id") == 13, self.jobs.get_attribute("raz_id") == 4).sum()
should_be = 1 - 1
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = (self.jobs.get_attribute("raz_id") == 5).sum()
should_be = 2 + 4
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = logical_and(
self.jobs.get_attribute("sector_id") == 11, self.jobs.get_attribute("building_id") == 8
).sum()
should_be = 0 + 3
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = (self.jobs.get_attribute("sector_id") == 12).sum()
should_be = 7
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
def test_modify_dataset_attribute(self):
"""
"""
scheduled_events_data = {
"year": array([2000, 2000, 2000, 2000, 2001, 2001]),
"action": array(
["set_value", "subtract_value", "add_value", "multiply_value", "subtract_value", "multiply_value"]
),
"amount": array([4, 2, 3, 1.1, 1, 0.9]),
"attribute": array(
[
"residential_units",
"non_residential_sqft",
"non_residential_sqft",
"price_per_unit",
"non_residential_sqft",
"price_per_unit",
]
),
"building_id": array([3, 3, 5, -1, 3, -1]),
"parcel_id": array([-1, -1, -1, 5, -1, 5]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="events", table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage, in_table_name="events", id_name=[])
model = ScheduledEventsModel(self.buildings, scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
results = self.buildings.size()
should_be = 8
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
# examine each action in turn:
index = self.buildings.get_attribute("building_id") == 3
results = (
self.buildings.get_attribute("residential_units")[index],
self.buildings.get_attribute("non_residential_sqft")[index],
)
should_be = (4, 1)
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
index = self.buildings.get_attribute("building_id") == 5
results = self.buildings.get_attribute("non_residential_sqft")[index]
should_be = 1 + 3
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = self.buildings.get_attribute("price_per_unit")
should_be = array([50, 21, 32, 15, 60, 90, 100 * 1.1, 200 * 1.1])
self.assertTrue(allclose(should_be, results), "Error, should_be: %s, but result: %s" % (should_be, results))
model.run(year=2001)
index = self.buildings.get_attribute("building_id") == 3
results = (
self.buildings.get_attribute("residential_units")[index],
self.buildings.get_attribute("non_residential_sqft")[index],
)
should_be = (4, 0)
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
results = self.buildings.get_attribute("price_per_unit")
should_be = array([50, 21, 32, 15, 60, 90, 100 * 1.1 * 0.9, 200 * 1.1 * 0.9])
self.assertTrue(allclose(should_be, results), "Error, should_be: %s, but result: %s" % (should_be, results))
def test_demolish_buildings_on_a_parcel(self):
"""test demolish buildings, create new buildings, and convert an existing building
"""
scheduled_events_data = {
"year": array([2000, 2000, 2000, 2000, 2000]),
"action": array(["remove", "add", "set_value", "set_value", "set_value"]),
"amount": array([4, 2, 8, 7, 150]),
"attribute": array(["", "", "residential_units", "non_residential_sqft", "price_per_unit"]),
"building_id": array([3, -1, 5, 5, 5]),
"parcel_id": array([-1, 1, -1, -1, -1]),
"residential_units": array([-1, 2, -1, -1, -1]),
"non_residential_sqft": array([-1, 1, -1, -1, -1]),
"price_per_unit": array([-1, 99, -1, -1, -1]),
}
# self.attribute_cache.write_table(table_name = 'scheduled_events', table_data = scheduled_events_data)
# events_dataset = self.dataset_pool.get_dataset('scheduled_event')
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="events", table_data=scheduled_events_data)
events_dataset = Dataset(in_storage=storage, in_table_name="events", id_name=[])
model = ScheduledEventsModel(self.buildings, scheduled_events_dataset=events_dataset)
model.run(year=2000, dataset_pool=self.dataset_pool)
results = self.buildings.size()
should_be = 9
self.assertEqual(should_be, results, "Error, should_be: %s, but result: %s" % (should_be, results))
index = self.buildings.get_attribute("building_id") > 8
results = array(
[
self.buildings.get_attribute("parcel_id")[index],
self.buildings.get_attribute("residential_units")[index],
self.buildings.get_attribute("non_residential_sqft")[index],
self.buildings.get_attribute("price_per_unit")[index],
]
)
should_be = array([[1, 1], [2, 2], [1, 1], [99, 99]])
self.assertTrue(allclose(should_be, results), "Error, should_be: %s, but result: %s" % (should_be, results))
index = where(self.buildings.get_attribute("building_id") == 5)
results = self.buildings.get_multiple_attributes(
["parcel_id", "residential_units", "non_residential_sqft", "price_per_unit"]
)[index]
should_be = array([4, 8, 7, 150])
self.assertTrue(allclose(should_be, results), "Error, should_be: %s, but result: %s" % (should_be, results))
def __init__(self, directory):
self.storage = AttributeCache(directory)