def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
csv_data_path = params_dict['csv_data_path']
table_name = params_dict['csv_table_name']
input_storage = csv_storage(storage_location = csv_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
else:
lst = [table_name]
for i in lst:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage)
logCB("Successfully exported all tables.")
def test_run(self):
print "Entering test run"
logger.log_status("Loading travel data: %s" % self.travel_data_source)
# get travel data as an attribute marix
in_storage = csv_storage(storage_location=self.travel_data_source_dir)
table_name = "travel_data"
travel_data_attribute = "single_vehicle_to_work_travel_cost"
travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name)
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=0)
# determine location to store and read attribute matrix
location1 = os.path.join(self.tempDir, "attrib_matrix1.txt")
location2 = os.path.join(self.tempDir, "attrib_matrix2.txt")
# store attribute matrix
savetxt(location1, travel_data_attribute_mat, fmt="%i")
savetxt(location2, travel_data_attribute_mat, fmt="%i")
# read attribute matrix
matrix1 = genfromtxt(location1)
matrix2 = genfromtxt(location2)
# compare both matices
result = matrix1 == matrix2
self.assertTrue(result.all())
print "Leaving test run"
def init(self, year, config):
self.input_directory = paths.get_opus_home_path( "opus_matsim", "tmp" )
logger.log_status("input_directory: " + self.input_directory )
self.in_storage = csv_storage(storage_location = self.input_directory)
self.cache_storage = AttributeCache().get_flt_storage_for_year(year)
self.cache_directory = config['cache_directory']
self.delete_travel_data_columns = ['am_bike_to_work_travel_time',
'am_biking_person_trips',
#'am_pk_period_drive_alone_vehicle_trips',
'am_total_transit_time_walk',
'am_transit_person_trip_table',
#'am_walk_time_in_minutes',
'am_walking_person_trips',
'am_double_vehicle_to_work_travel_time',
'am_threeplus_vehicle_to_work_travel_time',
'logsum_hbw_am_income_1',
'logsum_hbw_am_income_2',
'logsum_hbw_am_income_3',
'logsum_hbw_am_income_4',
'md_vehicle_miles_traveled',
'nweubk',
'nweuda',
'nweus2',
'nweus3',
'nweutw',
'nweuwk',
'pm_ev_ni_vehicle_miles_traveled',
'single_vehicle_to_work_travel_distance']
self.travel_data_table_name = "travel_data"
self.zone_table_name = "zones"
def compare_travel_data_sets(self):
# get copied travel data csv
copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp')
if not os.path.exists(copied_travel_data_location):
raise StandardError('Travel data not found: %s' % copied_travel_data_location)
logger.log_status('Get copied travel data: %s' % copied_travel_data_location)
# convert travel data csv into travel data set matrix
in_storage = csv_storage(storage_location = copied_travel_data_location)
table_name = "travel_data"
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# get exsisting travel data set and convert it also into travel data set matrix
year = self.run_config['base_year']+2
attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory'])
cache_storage = attribute_cache.get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
from numpy import savetxt # for debugging
savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f")
savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f")
# compare both data set matices
compare = travel_data_attribute_mat == existing_travel_data_attribute_mat
# return result
return compare.all()
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
input_directory = os.path.join(config['root'], 'opus_matsim', 'tmp')
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
# print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them"
# print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id"
# solved 3dec08 by hana
input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim",
"tmp")
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def opusRun(progressCB, logCB, config):
tm_config = config["travel_model_configuration"]
for key, val in tm_config.iteritems():
tm_config[str(key)] = str(val)
opus_data_directory = tm_config["travel_data_dir"]
opus_data_year = tm_config["year_dir"]
csv_data_path = tm_config["travel_data_path"]
table_name = tm_config["travel_data_table_name"]
input_storage = csv_storage(storage_location=csv_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache())
if table_name == "ALL":
logCB("caching all tables...\n")
lst = input_storage.get_table_names()
else:
lst = [table_name]
for i in lst:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(dataset_name=i, in_storage=input_storage, out_storage=output_storage)
logCB("Successfully exported all tables.")
def test_run(self):
print "Entering test run"
logger.log_status('Loading travel data: %s' % self.travel_data_source)
# get travel data as an attribute marix
in_storage = csv_storage(storage_location=self.travel_data_source_dir)
table_name = "travel_data"
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(
travel_data_attribute, fill=0)
# determine location to store and read attribute matrix
location1 = os.path.join(self.tempDir, 'attrib_matrix1.txt')
location2 = os.path.join(self.tempDir, 'attrib_matrix2.txt')
# store attribute matrix
savetxt(location1, travel_data_attribute_mat, fmt="%i")
savetxt(location2, travel_data_attribute_mat, fmt="%i")
# read attribute matrix
matrix1 = genfromtxt(location1)
matrix2 = genfromtxt(location2)
# compare both matices
result = (matrix1 == matrix2)
self.assertTrue(result.all())
print "Leaving test run"
def compare_travel_data_sets(self):
# get copied travel data csv
copied_travel_data_location = os.path.join(self.destination, "opus_matsim", "tmp")
if not os.path.exists(copied_travel_data_location):
raise StandardError("Travel data not found: %s" % copied_travel_data_location)
logger.log_status("Get copied travel data: %s" % copied_travel_data_location)
# convert travel data csv into travel data set matrix
in_storage = csv_storage(storage_location=copied_travel_data_location)
table_name = "travel_data"
travel_data_attribute = "single_vehicle_to_work_travel_cost"
travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name)
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# get exsisting travel data set and convert it also into travel data set matrix
year = self.run_config["base_year"] + 2
attribute_cache = AttributeCache(cache_directory=self.run_config["cache_directory"])
cache_storage = attribute_cache.get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name)
existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(
travel_data_attribute, fill=999
)
from numpy import savetxt # for debugging
savetxt(os.path.join(self.destination, "origin_travel_data.txt"), travel_data_attribute_mat, fmt="%f")
savetxt(os.path.join(self.destination, "existing_travel_data"), existing_travel_data_attribute_mat, fmt="%f")
# compare both data set matices
compare = travel_data_attribute_mat == existing_travel_data_attribute_mat
# return result
return compare.all()
def init(self, year, config):
self.input_directory = os.path.join( opus_matsim.__path__[0], 'tests', 'testdata' )
logger.log_status("input_directory: " + self.input_directory )
self.in_storage = csv_storage(storage_location = self.input_directory)
self.cache_storage = AttributeCache().get_flt_storage_for_year(year)
self.cache_directory = config['cache_directory']
self.travel_data_table_name = "travel_data"
self.zone_table_name = "zones"
def export(self):
''' Run export process
'''
output_storage = csv_storage(storage_location = self.csv_data_path)
ExportStorage().export_dataset(
dataset_name = 'travel_data',
in_storage = self.input_storage,
out_storage = output_storage,
)
def export(self):
''' Run export process
'''
output_storage = csv_storage(storage_location = self.csv_data_path)
ExportStorage().export_dataset(
dataset_name = 'travel_data',
in_storage = self.input_storage,
out_storage = output_storage,
)
def get_travel_data_from_travel_model(self, config, year, zone_set):
""" Integrates modified travel times and pre-computed travel costs
into the UrbanSim cache.
"""
logger.log_status('Starting GetTestTravelDataIntoCache.get_travel_data...')
# get sensitivity test path asan anchor to determine the location of the MATSim travel_data file (see below).
test_dir_path = test_dir.__path__[0]
# for debugging
try: #tnicolai
import pydevd
pydevd.settrace()
except: pass
# get the exsisting travel data from the current year
logger.log_status('Loading travel data from UrbanSim cache (year:%i)' % year)
table_name = "travel_data"
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
###### modifyed travel time travel data
logger.log_status('Integrating modifyed travel times in year %i for next simulation year.')
input_directory = os.path.join( os.environ['OPUS_HOME'], "opus_matsim", "tmp" )
logger.log_status("input_directory: " + input_directory )
# location of the modified travel time travel_data
in_storage = csv_storage(storage_location = input_directory)
# create travel data set (travel times)
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
# join the modifyed travel times with the travel data set of the current year
existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY)
##### pre-calcualted MATSim travel data (travel costs)
# logger.log_status('Integrating pre-calculated travel costs (MATSim) in year %i for next simulation year.')
# input_directory = os.path.join( test_dir_path, 'data', 'travel_cost')
# logger.log_status("input_directory: " + input_directory )
# # check source file
# if not os.path.exists( input_directory ):
# print 'File not found! %s' % input_directory
# sys.exit()
# location of pre-calculated MATSim travel costs
# in_storage = csv_storage(storage_location = input_directory)
# create travel data set (travel costs)
# travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
# join travel data set from pre-calcualted MATSim results
# existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output csv data path
csv_data_path = params_dict['csv_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
execute_after_export = params_dict['execute_after_export']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
output_storage = csv_storage(storage_location = csv_data_path)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
logCB("Successfully exported all datasets.")
if execute_after_export:
file_name_list = [output_storage._get_file_path_for_table(i)
for i in opus_table_name_list]
subprocess.Popen([execute_after_export] + file_name_list)
def test_run(self):
print "Entering test run"
path = paths.get_opus_home_path('opus_matsim', 'tmp')
# check if travel data exsits
travel_data = os.path.join( path, "travel_data.csv" )
if not os.path.exists(travel_data):
print "Travel Data not found!!!"
sys.exit()
in_storage = csv_storage(storage_location = path)
table_name = "travel_data"
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id)
l = numpy.atleast_1d(origin_zones).tolist()
origin_list = set(l) # removes duplicates and sorts the list in ascending order
# destination_list = len(origin_list) * self.cbd # creates a list that contains the zone id of the cbd an has the same length as "origin_list"
# set high travel costs for all origin to cbd pairs
for id in origin_list:
travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.high_travel_cost, id, self.cbd)
# adjust cbd to cbd
travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, 0.0, self.cbd, self.cbd)
# adjust prefered zone to cbd
travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.low_travel_cost, self.preferential_zone, self.cbd)
w = travel_data_set.get_index_by_origin_and_destination_ids(110, 129)
x = travel_data_set.get_index_by_origin_and_destination_ids(129, 129)
y = travel_data_set.get_index_by_origin_and_destination_ids(20, 129)
z = travel_data_set.get_index_by_origin_and_destination_ids(20, 20)
print w
print x
print y
print z
origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id)
destination_zones = travel_data_set.get_attribute_as_column(self.destination_zone_id)
my_travel_data_attr_mat = travel_data_set.get_attribute_as_matrix('travel_data.single_vehicle_to_work_travel_cost',
fill=999)
my_travel_data_attr_mat[origin_zones, destination_zones] = 1.03
cbd_ids = where(origin_zones == 129)
print "Leaving test run"
def test_run(self):
print "Entering test run"
path = os.path.join(os.environ['OPUS_HOME'], 'opus_matsim/tmp')
# check if travel data exsits
travel_data = os.path.join( path, "travel_data.csv" )
if not os.path.exists(travel_data):
print "Travel Data not found!!!"
sys.exit()
in_storage = csv_storage(storage_location = path)
table_name = "travel_data"
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id)
l = numpy.atleast_1d(origin_zones).tolist()
origin_list = set(l) # removes duplicates and sorts the list in ascending order
# destination_list = len(origin_list) * self.cbd # creates a list that contains the zone id of the cbd an has the same length as "origin_list"
# set high travel costs for all origin to cbd pairs
for id in origin_list:
travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.high_travel_cost, id, self.cbd)
# adjust cbd to cbd
travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, 0.0, self.cbd, self.cbd)
# adjust prefered zone to cbd
travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.low_travel_cost, self.preferential_zone, self.cbd)
w = travel_data_set.get_index_by_origin_and_destination_ids(110, 129)
x = travel_data_set.get_index_by_origin_and_destination_ids(129, 129)
y = travel_data_set.get_index_by_origin_and_destination_ids(20, 129)
z = travel_data_set.get_index_by_origin_and_destination_ids(20, 20)
print w
print x
print y
print z
origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id)
destination_zones = travel_data_set.get_attribute_as_column(self.destination_zone_id)
my_travel_data_attr_mat = travel_data_set.get_attribute_as_matrix('travel_data.single_vehicle_to_work_travel_cost',
fill=999)
my_travel_data_attr_mat[origin_zones, destination_zones] = 1.03
cbd_ids = where(origin_zones == 129)
print "Leaving test run"
def init(self, year, config):
config_obj_v3 = MATSimConfigObjectV3(config, year)
self.input_directory = config_obj_v3.matsim4urbansim_matsim_temp_path
# self.input_directory = paths.get_opus_home_path( matsim4opus, matsim_temp )
logger.log_status("input_directory: " + self.input_directory )
self.in_storage = csv_storage(storage_location = self.input_directory)
self.cache_storage = AttributeCache().get_flt_storage_for_year(year)
self.cache_directory = config['cache_directory']
# tnicolai: experimental code
self.matsim_controler = self.__get_matsim_controler_section(config)
# this deletes travel model attributes from cache that are not updated/handled by the travel model (MATSim)
self.delete_travel_data_columns = ['am_bike_to_work_travel_time',
'am_biking_person_trips',
#'am_pk_period_drive_alone_vehicle_trips',
'am_total_transit_time_walk',
'am_transit_person_trip_table',
#'am_walk_time_in_minutes',
'am_walking_person_trips',
'am_double_vehicle_to_work_travel_time',
'am_threeplus_vehicle_to_work_travel_time',
'logsum_hbw_am_income_1',
'logsum_hbw_am_income_2',
'logsum_hbw_am_income_3',
'logsum_hbw_am_income_4',
'md_vehicle_miles_traveled',
'nweubk',
'nweuda',
'nweus2',
'nweus3',
'nweutw',
'nweuwk',
'pm_ev_ni_vehicle_miles_traveled',
'single_vehicle_to_work_travel_distance']
self.travel_data_table_name = "travel_data"
self.parcel_table_name = "parcels"
self.zone_table_name = "zones"
self.person_data_table = "persons"
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
input_directory = os.path.join( config['root'], 'opus_matsim', 'tmp' )
logger.log_status("input_directory: " + input_directory )
in_storage = csv_storage(storage_location = input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def test_run(self):
print "Entering test run"
# This test checks if the pre-computed MATSim travel data
# is loaded correctly into UrbanSim. Therefore the UrbanSim
# travel data matrix is converted into a numpy array and two
# predefined values are checked whether they match with MATSim data.
#
# Here an example:
# my_list = [[1,2,3],
# [4,5,6],
# [7,8,9]]
#
# my_list[0][1] should be = 2
# my_list[2][2] should be = 9
table_name = 'travel_data'
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
# location of pre-calculated MATSim travel costs
in_storage = csv_storage(storage_location = self.input_directory)
# create travel data set (travel costs)
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=31)
# converting from numpy array into a 2d list
travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist()
# get two values for validation
value1 = int(travel_list[1][1]) # should be = 0
value2 = int(travel_list[2][1]) # should be = 120
logger.log_status('First validation value should be 0. Current value is %i' % value1)
logger.log_status('Second validation value should be 120. Current value is %i' % value2)
self.assertTrue( value1 == 0 )
self.assertTrue( value2 == 120 )
# self.dump_travel_list(travel_list) # for debugging
print "Leaving test run"
def test_run(self):
print "Entering test run"
# This test loads an exising travel data as a TravelDataSet (numpy array)
# and accesses single (pre-known) values to validate the conversion process
# (numpy array into standard python list).
#
# Here an example:
# my_list = [[1,2,3],
# [4,5,6],
# [7,8,9]]
#
# my_list[0][1] should be = 2
# my_list[2][2] should be = 9
table_name = 'travel_data'
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
# location of pre-calculated MATSim travel costs
in_storage = csv_storage(storage_location = self.input_directory)
# create travel data set (travel costs)
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=31)
# converting from numpy array into a 2d list
travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist()
# get two values for validation
value1 = int(travel_list[1][1]) # should be = 0
value2 = int(travel_list[2][1]) # should be = 120
logger.log_status('First validation value should be 0. Current value is %i' % value1)
logger.log_status('Second validation value should be 120. Current value is %i' % value2)
self.assertTrue( value1 == 0 )
self.assertTrue( value2 == 120 )
# self.dump_travel_list(travel_list) # for debugging
print "Leaving test run"
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
# print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them"
# print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id"
# solved 3dec08 by hana
input_directory = os.path.join( os.environ['OPUS_HOME'], "opus_matsim", "tmp" )
logger.log_status("input_directory: " + input_directory )
in_storage = csv_storage(storage_location = input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def get_travel_cost_matrix(self):
''' Returns the pre-calculated MATSim travel costs as 2d list
'''
# get sensitivity test path
test_dir_path = test_dir.__path__[0]
input_directory = os.path.join( test_dir_path, 'data', 'travel_cost')
print "input_directory: %s" % input_directory
# check source file
if not os.path.exists( input_directory ):
print 'File not found! %s' % input_directory
sys.exit()
table_name = 'travel_data'
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
# location of pre-calculated MATSim travel costs
in_storage = csv_storage(storage_location = input_directory)
# create travel data set (travel costs)
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# travel cost matris as 2d array
travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist()
return travel_list
def get_travel_cost_matrix(self):
''' Returns the pre-calculated MATSim travel costs as 2d list
'''
# get sensitivity test path
test_dir_path = test_dir.__path__[0]
input_directory = os.path.join( test_dir_path, 'data', 'travel_cost')
print "input_directory: %s" % input_directory
# check source file
if not os.path.exists( input_directory ):
print 'File not found! %s' % input_directory
sys.exit()
table_name = 'travel_data'
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
# location of pre-calculated MATSim travel costs
in_storage = csv_storage(storage_location = input_directory)
# create travel data set (travel costs)
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# travel cost matris as 2d array
travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist()
return travel_list
# Opus/UrbanSim urban simulation software.
# Copyright (C) 2012 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE
from opus_core.store.csv_storage import csv_storage
from opus_core.store.dbf_storage import dbf_storage
import os, sys
if len(sys.argv) not in (2,3):
print "Usage: %s csv_file [dbf_file]" % sys.argv[0]
sys.exit(0)
csv_file = sys.argv[1]
csv_file = os.path.normpath(csv_file)
csv_path, csv_name = os.path.split(csv_file)
csv_table, csv_ext = os.path.splitext(csv_name)
if len(sys.argv) == 2:
dbf_path, dbf_table = csv_path, csv_table
elif len(sys.argv) == 3:
dbf_file = sys.argv[2]
dbf_file = os.path.normpath(dbf_file)
dbf_path, dbf_name = os.path.split(dbf_file)
dbf_table, dbf_ext = os.path.splitext(dbf_name)
csv_store = csv_storage(storage_location=csv_path)
dbf_store = dbf_storage(storage_location=dbf_path)
data = csv_store.load_table(csv_table)
dbf_store.write_table(dbf_table, data)
parser = OptionParser()
parser.add_option('-c', '--cache_path', dest='cache_path', type='string',
help='The filesystem path to the cache to export (required)')
parser.add_option('-o', '--output_directory', dest='output_directory',
type='string', help='The filesystem path of the database to which '
'output will be written (required)')
parser.add_option('-t', '--table_name', dest='table_name',
type='string', help='Name of table to be exported (optional). Used if only one table should be exported.')
(options, args) = parser.parse_args()
cache_path = options.cache_path
output_directory = options.output_directory
table_name = options.table_name
if None in (cache_path, output_directory):
parser.print_help()
sys.exit(1)
in_storage = flt_storage(storage_location=cache_path)
out_storage = csv_storage(storage_location=output_directory)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
if table_name is not None:
ExportStorage().export_dataset(table_name, in_storage=in_storage, out_storage=out_storage)
else:
ExportStorage().export(in_storage=in_storage, out_storage=out_storage)
def get_travel_data_from_travel_model(self, config, year, zone_set):
""" Integrates modified travel times and pre-computed travel costs
into the UrbanSim cache.
"""
logger.log_status(
'Starting GetTestTravelDataIntoCache.get_travel_data...')
# get sensitivity test path asan anchor to determine the location of the MATSim travel_data file (see below).
test_dir_path = test_dir.__path__[0]
# for debugging
try: #tnicolai
import pydevd
pydevd.settrace()
except:
pass
# get the exsisting travel data from the current year
logger.log_status('Loading travel data from UrbanSim cache (year:%i)' %
year)
table_name = "travel_data"
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
###### modifyed travel time travel data
logger.log_status(
'Integrating modifyed travel times in year %i for next simulation year.'
)
input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim",
"tmp")
logger.log_status("input_directory: " + input_directory)
# location of the modified travel time travel_data
in_storage = csv_storage(storage_location=input_directory)
# create travel data set (travel times)
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
# join the modifyed travel times with the travel data set of the current year
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
##### pre-calcualted MATSim travel data (travel costs)
# logger.log_status('Integrating pre-calculated travel costs (MATSim) in year %i for next simulation year.')
# input_directory = os.path.join( test_dir_path, 'data', 'travel_cost')
# logger.log_status("input_directory: " + input_directory )
# # check source file
# if not os.path.exists( input_directory ):
# print 'File not found! %s' % input_directory
# sys.exit()
# location of pre-calculated MATSim travel costs
# in_storage = csv_storage(storage_location = input_directory)
# create travel data set (travel costs)
# travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
# join travel data set from pre-calcualted MATSim results
# existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def run(self, config, year, storage_type='sql'):
"""
"""
tm_config = config['travel_model_configuration']
database_server_config = tm_config.get("database_server_configuration", 'simtravel_database_server')
database_name = tm_config.get("database_name", 'mag_zone_baseyear')
cache_directory = config['cache_directory']
simulation_state = SimulationState()
simulation_state.set_cache_directory(cache_directory)
simulation_state.set_current_time(year)
attribute_cache = AttributeCache()
dataset_pool = SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=attribute_cache).get_dataset_pool()
if storage_type == 'sql':
db_server = DatabaseServer(DatabaseConfiguration(
database_name = database_name,
database_configuration = database_server_config
)
)
if not db_server.has_database(database_name):
print "Db doesn't exist creating one"
db_server.create_database(database_name)
db = db_server.get_database(database_name)
output_storage = sql_storage(storage_location = db)
elif storage_type == 'csv':
csv_directory = os.path.join(cache_directory, 'csv', str(year))
output_storage = csv_storage(storage_location=csv_directory)
else:
raise ValueError, "Unsupported output storage type {}".format(storage_type)
logger.start_block('Compute and export data to openAMOS...')
hh = dataset_pool.get_dataset('household')
hh_recs = dataset_pool.get_dataset('households_recs')
#hh_recs.add_attribute(0,"htaz1")
#hh_recs.flush_dataset()
#syn_hh = dataset_pool.get_dataset('synthetic_household')
hh_variables = ['houseid=household.household_id',
"hhsize=household.number_of_agents(person)",
"one=(household.household_id>0).astype('i')",
"inclt35k=(household.income<35000).astype('i')",
"incge35k=(household.income>=35000).astype('i')",
"incge50k=(household.income>=50000).astype('i')",
"incge75k=(household.income>=75000).astype('i')",
"incge100k=(household.income>=100000).astype('i')",
"inc35t50=((household.income>=35000) & (household.income<50000)).astype('i')",
"inc50t75=((household.income>=50000) & (household.income<75000)).astype('i')",
"inc75t100=((household.income>=75000) & (household.income<100000)).astype('i')",
'htaz = ((houseid>0)*(household.disaggregate(building.zone_id)-100) + (houseid<=0)*0)',
#'htaz = ((houseid>0) & (htaz1>100))*(htaz1-100)+((houseid>0) & (htaz1==-1))*1122',
"withchild = (household.aggregate(person.age<18)>0).astype('i')",
"noc = household.aggregate(person.age<18)",
"numadlt = household.aggregate(person.age>=18)",
"hinc=household.income",
"wif=household.workers",
#"wif=household.aggregate(mag_zone.person.is_employed)",
'numwrkr=household.workers',
#'numwrkr=household.aggregate(mag_zone.person.is_employed)',
'nwrkcnt=household.number_of_agents(person) - household.workers',
#'nwrkcnt=household.number_of_agents(person) - household.aggregate(mag_zone.person.is_employed)',
'yrbuilt=mag_zone.household.yrbuilt',
'mag_zone.household.sparent',
'mag_zone.household.rur',
'mag_zone.household.urb',
'zonetid = household.disaggregate(building.zone_id)',
]
self.prepare_attributes(hh, hh_variables)
attrs_to_export = hh_recs.get_known_attribute_names()
self.write_dataset(hh, attrs_to_export, output_storage)
dataset_pool._remove_dataset(hh.dataset_name)
persons = dataset_pool.get_dataset('person')
persons.out_table_name_default = 'persons'
# Recoding invalid work and school locations to some random valid values
persons_recs = dataset_pool.get_dataset('persons_recs')
persons_recs.add_attribute(persons['person_id'],"personuniqueid")
persons_recs.add_attribute(persons['marriage_status'],"marstat")
persons_recs.add_attribute(persons['student_status'],"schstat")
"""
persons_recs.add_attribute(persons['wtaz0'],"htaz_act")
persons_recs.add_attribute(0,"wtaz_rec")
persons_recs.add_attribute(0,"wtaz_rec1")
persons_recs.add_attribute(0,"wtaz_rec2")
persons_recs.add_attribute(0,"wtaz1_1")
persons_recs.add_attribute(0,"wtaz1_2")
persons_recs.add_attribute(0,"wtaz1_3")
#persons_recs.add_attribute(persons['student_status'],"schstat")
"""
persons_recs.add_attribute(0,"wtaz1")
persons_recs.add_attribute(0,"htaz")
persons_recs.add_attribute(0,"schtaz1")
persons_recs.flush_dataset()
#syn_persons = dataset_pool.get_dataset('synthetic_person')
persons_variables = ['personid=mag_zone.person.member_id',
'personuniqueid=person.person_id',
'houseid=person.household_id',
"one=(person.person_id>0).astype('i')",
'trvtime=mag_zone.person.travel_time_from_home_to_work',
'timetowk=mag_zone.person.travel_time_from_home_to_work',
#'mag_zone.person.tmtowrk',
#'tmtowrk=person.disaggregate(synthetic_person.tmtowrk)',
"ag5t10=((person.age>=5) & (person.age<=10)).astype('i')",
"ag11t14=((person.age>=11) & (person.age<=14)).astype('i')",
"ag15t17=((person.age>=15) & (person.age<=17)).astype('i')",
"ag18t24=((person.age>=18) & (person.age<=24)).astype('i')",
"ag25t34=((person.age>=25) & (person.age<=34)).astype('i')",
"ag35t44=((person.age>=35) & (person.age<=44)).astype('i')",
"ag45t54=((person.age>=45) & (person.age<=54)).astype('i')",
"ag55t64=((person.age>=55) & (person.age<=64)).astype('i')",
"agge65=(person.age>=65).astype('i')",
"ag12t17=((person.age>=12) & (person.age<=17)).astype('i')",
"ag5t14=((person.age>=5) & (person.age<=14)).astype('i')",
"agge15=(person.age>=15).astype('i')",
"wrkr=(person.employment_status==1).astype('i')",
"isemploy=(person.employment_status==1).astype('i')",
"fulltim=(mag_zone.person.full_time==1).astype('i')",
'parttim=mag_zone.person.part_time',
'htaz = ((houseid>0)*(person.disaggregate(building.zone_id, intermediates=[household])-100) + (houseid<=0)*0)',
'wtaz1=(person.wtaz <= 0)*0 + (person.wtaz > 0)*(person.wtaz-100)',
"presch = ((person.age < 5)&(houseid>0)).astype('i')",
"mag_zone.person.schstat",
'schtaz1 = (person.schtaz <= 0)*0 + (person.schtaz > 0)*(person.schtaz-100)',
'marstat = person.marriage_status',
'enroll = person.student_status',
'grade = person.student_status & person.education',
'educ = person.education',
"male = (person.sex==1).astype('i')",
"female = (person.sex==2).astype('i')",
"coled = (person.education >= 10).astype('i')",
'race1 = person.race',
"white = (person.race == 1).astype('i')",
'person.hispanic'
]
self.prepare_attributes(persons, persons_variables)
attrs_to_export = persons_recs.get_known_attribute_names()
self.write_dataset(persons, attrs_to_export, output_storage)
dataset_pool._remove_dataset(persons.dataset_name)
zones = dataset_pool.get_dataset('zone')
zones_variables = [
"retail_employment=zone.aggregate(mag_zone.job.sector_group=='retail')",
"public_employment=zone.aggregate(mag_zone.job.sector_group=='public')",
"office_employment=zone.aggregate(mag_zone.job.sector_group=='office')",
"industrial_employment=zone.aggregate(mag_zone.job.sector_group=='individual')",
"other_employment=zone.aggregate(mag_zone.job.sector_group=='other')",
"retail_employment_density=zone.aggregate(mag_zone.job.sector_group=='retail')/zone.acres",
"public_employment_density=zone.aggregate(mag_zone.job.sector_group=='public')/zone.acres",
"office_employment_density=zone.aggregate(mag_zone.job.sector_group=='office')/zone.acres",
"industrial_employment_density=zone.aggregate(mag_zone.job.sector_group=='individual')/zone.acres",
"other_employment_density=zone.aggregate(mag_zone.job.sector_group=='other')/zone.acres",
"total_area=zone.acres",
"lowest_income=zone.aggregate(household.income < scoreatpercentile(household.income, 20))",
"low_income=zone.aggregate(household.income < scoreatpercentile(household.income, 40))",
"high_income=zone.aggregate(household.income > scoreatpercentile(household.income, 80))",
#"institutional_population=zone.disaggregate(locations.institutional_population)",
#"groupquarter_households=zone.disaggregate(locations.groupquarter_households)",
"residential_households=zone.number_of_agents(household)",
"locationid=zone.zone_id",
]
locations = dataset_pool['locations']
self.prepare_attributes(zones, zones_variables, dataset2=locations)
attrs_to_export = locations.get_known_attribute_names()
self.write_dataset(locations, attrs_to_export, output_storage)
dataset_pool._remove_dataset(locations.dataset_name)
#raw_input("check location block")
logger.end_block()
"The attribute cache year into which to write the output (required).")
(options, args) = parser.parse_args()
csv_directory = options.csv_directory
attribute_cache_directory = options.attribute_cache_directory
table_name = options.table_name
cache_year = options.cache_year
if (csv_directory is None or attribute_cache_directory is None
or table_name is None or cache_year is None):
parser.print_help()
sys.exit(1)
input_storage = csv_storage(storage_location=csv_directory)
attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
SimulationState().set_current_time(cache_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
logger.start_block(
"Exporting table '%s' to year %s of cache located at %s..." %
(table_name, cache_year, attribute_cache_directory))
try:
ExportStorage().export_dataset(
dataset_name=table_name,
in_storage=input_storage,
(options, args) = parser.parse_args()
csv_directory = options.csv_directory
attribute_cache_directory = options.attribute_cache_directory
table_name = options.table_name
cache_year = options.cache_year
if (csv_directory is None or
attribute_cache_directory is None or
table_name is None or
cache_year is None):
parser.print_help()
sys.exit(1)
input_storage = csv_storage(storage_location = csv_directory)
attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
SimulationState().set_current_time(cache_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
logger.start_block("Exporting table '%s' to year %s of cache located at %s..." %
(table_name, cache_year, attribute_cache_directory))
try:
ExportStorage().export_dataset(
dataset_name = table_name,
in_storage = input_storage,
out_storage = output_storage,
def run(self, year=None, years_to_run=[], configuration=None):
if year not in years_to_run or self.data_to_export == None:
return
cache_directory = configuration['cache_directory']
simulation_state = SimulationState()
simulation_state.set_cache_directory(cache_directory)
simulation_state.set_current_time(year)
attribute_cache = AttributeCache()
package_order=configuration['dataset_pool_configuration'].package_order
dataset_pool = SessionConfiguration(new_instance=True,
package_order=package_order,
in_storage=attribute_cache
).get_dataset_pool()
out_dir = os.path.join(cache_directory, "mtc_data")
out_storage = csv_storage(storage_location=out_dir)
# Adjust the age distribution per ABAG/MTC's specifications
age_control_dir = os.path.join(paths.OPUS_DATA_PATH, configuration['project_name'], "ageControl")
age_control_storage = csv_storage(storage_location=age_control_dir)
age_control_files = os.listdir(age_control_dir)
years = np.array(map(lambda x:
int(os.path.basename(x).replace("tazData", "").replace(".csv", "")),
glob.glob(os.path.join(age_control_dir, "tazData*.csv"))))
closest_year = years[np.argmin(np.abs(years - [year]*len(years)))]
if closest_year != year:
logger.log_warning("Could not find age control data for " + str(year) +
". Choosing nearest year " + str(closest_year) + ".")
age_control_table = age_control_storage.load_table("tazData" + str(closest_year), lowercase=False)
# Calculate the ABAG shares of person by age
age_categories = ['AGE0004', 'AGE0519', 'AGE2044', 'AGE4564', 'AGE65P']
age_category_sums = dict((k, age_control_table[k].sum()) for k in age_categories)
total = sum(age_category_sums.values())
abag_age_category_shares = dict((k, age_category_sums[k]/total) for k in age_categories)
for data_fname, variable_mapping in self.data_to_export.iteritems():
if not flip_urbansim_to_tm_variable_mappling:
col_names = variable_mapping.values()
variables_aliases = ["=".join(mapping[::-1]) for mapping in \
variable_mapping.iteritems()]
else:
col_names = variable_mapping.keys()
variables_aliases = ["=".join(mapping) for mapping in \
variable_mapping.iteritems()]
dataset_name = VariableName(variables_aliases[0]).get_dataset_name()
dataset = dataset_pool.get_dataset(dataset_name)
dataset.compute_variables(variables_aliases)
if data_fname == "ABAGData":
logger.log_status("Adjusting ABAGData to match age controls")
age_category_sums = dict((k, dataset[k].sum()) for k in age_categories)
total = sum(age_category_sums.values())
us_age_category_shares = dict((k, age_category_sums[k]/total) for k in age_categories)
adjustments = dict((k, abag_age_category_shares[k]/us_age_category_shares[k]) for k in age_categories)
diff = np.zeros(dataset.n)
for k in age_categories:
before = dataset[k]
dataset[k] = np.array(map(lambda v : round(v*adjustments[k]), dataset.get_attribute(k)))
diff += (dataset[k] - before)
dataset["TOTPOP"] += diff
dataset["HHPOP"] += diff
logger.log_status("NOTE: Adjusted total population by %d (%2.3f%%) due to rounding error." %
(int(diff.sum()), diff.sum()*100/total))
org_fname = os.path.join(out_dir, "%s.computed.csv" % data_fname)
new_fname = os.path.join(out_dir, "%s%s.csv" % (year,data_fname))
block_msg = "Writing {} for travel model to {}".format(data_fname,
new_fname)
with block(block_msg):
dataset.write_dataset(attributes=col_names,
out_storage=out_storage,
out_table_name=data_fname)
#rename & process header
shutil.move(org_fname, new_fname)
os.system("sed 's/:[a-z][0-9]//g' -i %s" % new_fname)
