def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] csv_data_path = params_dict['csv_data_path'] table_name = params_dict['csv_table_name'] input_storage = csv_storage(storage_location = csv_data_path) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logCB('caching all tables...\n') lst = input_storage.get_table_names() else: lst = [table_name] for i in lst: logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory)) ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage) logCB("Successfully exported all tables.")
def test_run(self): print "Entering test run" logger.log_status("Loading travel data: %s" % self.travel_data_source) # get travel data as an attribute marix in_storage = csv_storage(storage_location=self.travel_data_source_dir) table_name = "travel_data" travel_data_attribute = "single_vehicle_to_work_travel_cost" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=0) # determine location to store and read attribute matrix location1 = os.path.join(self.tempDir, "attrib_matrix1.txt") location2 = os.path.join(self.tempDir, "attrib_matrix2.txt") # store attribute matrix savetxt(location1, travel_data_attribute_mat, fmt="%i") savetxt(location2, travel_data_attribute_mat, fmt="%i") # read attribute matrix matrix1 = genfromtxt(location1) matrix2 = genfromtxt(location2) # compare both matices result = matrix1 == matrix2 self.assertTrue(result.all()) print "Leaving test run"
def init(self, year, config): self.input_directory = paths.get_opus_home_path( "opus_matsim", "tmp" ) logger.log_status("input_directory: " + self.input_directory ) self.in_storage = csv_storage(storage_location = self.input_directory) self.cache_storage = AttributeCache().get_flt_storage_for_year(year) self.cache_directory = config['cache_directory'] self.delete_travel_data_columns = ['am_bike_to_work_travel_time', 'am_biking_person_trips', #'am_pk_period_drive_alone_vehicle_trips', 'am_total_transit_time_walk', 'am_transit_person_trip_table', #'am_walk_time_in_minutes', 'am_walking_person_trips', 'am_double_vehicle_to_work_travel_time', 'am_threeplus_vehicle_to_work_travel_time', 'logsum_hbw_am_income_1', 'logsum_hbw_am_income_2', 'logsum_hbw_am_income_3', 'logsum_hbw_am_income_4', 'md_vehicle_miles_traveled', 'nweubk', 'nweuda', 'nweus2', 'nweus3', 'nweutw', 'nweuwk', 'pm_ev_ni_vehicle_miles_traveled', 'single_vehicle_to_work_travel_distance'] self.travel_data_table_name = "travel_data" self.zone_table_name = "zones"
def compare_travel_data_sets(self): # get copied travel data csv copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp') if not os.path.exists(copied_travel_data_location): raise StandardError('Travel data not found: %s' % copied_travel_data_location) logger.log_status('Get copied travel data: %s' % copied_travel_data_location) # convert travel data csv into travel data set matrix in_storage = csv_storage(storage_location = copied_travel_data_location) table_name = "travel_data" travel_data_attribute = 'single_vehicle_to_work_travel_cost' travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) # get exsisting travel data set and convert it also into travel data set matrix year = self.run_config['base_year']+2 attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory']) cache_storage = attribute_cache.get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name ) existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) from numpy import savetxt # for debugging savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f") savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f") # compare both data set matices compare = travel_data_attribute_mat == existing_travel_data_attribute_mat # return result return compare.all()
def get_travel_data_from_travel_model(self, config, year, zone_set): """ """ logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...') input_directory = os.path.join(config['root'], 'opus_matsim', 'tmp') logger.log_status("input_directory: " + input_directory) in_storage = csv_storage(storage_location=input_directory) table_name = "travel_data" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) ##TODO:This may not work or may be wrong after the id_names of travel_data ##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang) existing_travel_data_set.join( travel_data_set, travel_data_set.get_non_id_primary_attribute_names(), metadata=AttributeType.PRIMARY) return existing_travel_data_set
def get_travel_data_from_travel_model(self, config, year, zone_set): """ """ logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...') # print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them" # print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id" # solved 3dec08 by hana input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim", "tmp") logger.log_status("input_directory: " + input_directory) in_storage = csv_storage(storage_location=input_directory) table_name = "travel_data" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) ##TODO:This may not work or may be wrong after the id_names of travel_data ##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang) existing_travel_data_set.join( travel_data_set, travel_data_set.get_non_id_primary_attribute_names(), metadata=AttributeType.PRIMARY) return existing_travel_data_set
def opusRun(progressCB, logCB, config): tm_config = config["travel_model_configuration"] for key, val in tm_config.iteritems(): tm_config[str(key)] = str(val) opus_data_directory = tm_config["travel_data_dir"] opus_data_year = tm_config["year_dir"] csv_data_path = tm_config["travel_data_path"] table_name = tm_config["travel_data_table_name"] input_storage = csv_storage(storage_location=csv_data_path) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == "ALL": logCB("caching all tables...\n") lst = input_storage.get_table_names() else: lst = [table_name] for i in lst: logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory)) ExportStorage().export_dataset(dataset_name=i, in_storage=input_storage, out_storage=output_storage) logCB("Successfully exported all tables.")
def test_run(self): print "Entering test run" logger.log_status('Loading travel data: %s' % self.travel_data_source) # get travel data as an attribute marix in_storage = csv_storage(storage_location=self.travel_data_source_dir) table_name = "travel_data" travel_data_attribute = 'single_vehicle_to_work_travel_cost' travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix( travel_data_attribute, fill=0) # determine location to store and read attribute matrix location1 = os.path.join(self.tempDir, 'attrib_matrix1.txt') location2 = os.path.join(self.tempDir, 'attrib_matrix2.txt') # store attribute matrix savetxt(location1, travel_data_attribute_mat, fmt="%i") savetxt(location2, travel_data_attribute_mat, fmt="%i") # read attribute matrix matrix1 = genfromtxt(location1) matrix2 = genfromtxt(location2) # compare both matices result = (matrix1 == matrix2) self.assertTrue(result.all()) print "Leaving test run"
def compare_travel_data_sets(self): # get copied travel data csv copied_travel_data_location = os.path.join(self.destination, "opus_matsim", "tmp") if not os.path.exists(copied_travel_data_location): raise StandardError("Travel data not found: %s" % copied_travel_data_location) logger.log_status("Get copied travel data: %s" % copied_travel_data_location) # convert travel data csv into travel data set matrix in_storage = csv_storage(storage_location=copied_travel_data_location) table_name = "travel_data" travel_data_attribute = "single_vehicle_to_work_travel_cost" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) # get exsisting travel data set and convert it also into travel data set matrix year = self.run_config["base_year"] + 2 attribute_cache = AttributeCache(cache_directory=self.run_config["cache_directory"]) cache_storage = attribute_cache.get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix( travel_data_attribute, fill=999 ) from numpy import savetxt # for debugging savetxt(os.path.join(self.destination, "origin_travel_data.txt"), travel_data_attribute_mat, fmt="%f") savetxt(os.path.join(self.destination, "existing_travel_data"), existing_travel_data_attribute_mat, fmt="%f") # compare both data set matices compare = travel_data_attribute_mat == existing_travel_data_attribute_mat # return result return compare.all()
def init(self, year, config): self.input_directory = os.path.join( opus_matsim.__path__[0], 'tests', 'testdata' ) logger.log_status("input_directory: " + self.input_directory ) self.in_storage = csv_storage(storage_location = self.input_directory) self.cache_storage = AttributeCache().get_flt_storage_for_year(year) self.cache_directory = config['cache_directory'] self.travel_data_table_name = "travel_data" self.zone_table_name = "zones"
def export(self): ''' Run export process ''' output_storage = csv_storage(storage_location = self.csv_data_path) ExportStorage().export_dataset( dataset_name = 'travel_data', in_storage = self.input_storage, out_storage = output_storage, )
def get_travel_data_from_travel_model(self, config, year, zone_set): """ Integrates modified travel times and pre-computed travel costs into the UrbanSim cache. """ logger.log_status('Starting GetTestTravelDataIntoCache.get_travel_data...') # get sensitivity test path asan anchor to determine the location of the MATSim travel_data file (see below). test_dir_path = test_dir.__path__[0] # for debugging try: #tnicolai import pydevd pydevd.settrace() except: pass # get the exsisting travel data from the current year logger.log_status('Loading travel data from UrbanSim cache (year:%i)' % year) table_name = "travel_data" cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name ) ###### modifyed travel time travel data logger.log_status('Integrating modifyed travel times in year %i for next simulation year.') input_directory = os.path.join( os.environ['OPUS_HOME'], "opus_matsim", "tmp" ) logger.log_status("input_directory: " + input_directory ) # location of the modified travel time travel_data in_storage = csv_storage(storage_location = input_directory) # create travel data set (travel times) travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) # join the modifyed travel times with the travel data set of the current year existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY) ##### pre-calcualted MATSim travel data (travel costs) # logger.log_status('Integrating pre-calculated travel costs (MATSim) in year %i for next simulation year.') # input_directory = os.path.join( test_dir_path, 'data', 'travel_cost') # logger.log_status("input_directory: " + input_directory ) # # check source file # if not os.path.exists( input_directory ): # print 'File not found! %s' % input_directory # sys.exit() # location of pre-calculated MATSim travel costs # in_storage = csv_storage(storage_location = input_directory) # create travel data set (travel costs) # travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) # join travel data set from pre-calcualted MATSim results # existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY) return existing_travel_data_set
def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) # Output csv data path csv_data_path = params_dict['csv_data_path'] # Data clasification - Database (must be specified) opus_data_directory = params_dict['opus_data_directory'] # Data clasification - Dataset (explicit or ALL) opus_data_year = params_dict['opus_data_year'] # Data clasification - Array (explicit or ALL) opus_table_name = params_dict['opus_table_name'] execute_after_export = params_dict['execute_after_export'] attribute_cache = AttributeCache(cache_directory=opus_data_directory) attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4] if opus_data_year != 'ALL': attribute_cache_years = [opus_data_year] for year in attribute_cache_years: input_storage = attribute_cache.get_flt_storage_for_year(year) output_storage = csv_storage(storage_location = csv_data_path) SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if opus_table_name != 'ALL': opus_table_name_list = [opus_table_name] else: opus_table_name_list = input_storage.get_table_names() for i in opus_table_name_list: logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory)) ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage, ) logCB("Successfully exported all datasets.") if execute_after_export: file_name_list = [output_storage._get_file_path_for_table(i) for i in opus_table_name_list] subprocess.Popen([execute_after_export] + file_name_list)
def test_run(self): print "Entering test run" path = paths.get_opus_home_path('opus_matsim', 'tmp') # check if travel data exsits travel_data = os.path.join( path, "travel_data.csv" ) if not os.path.exists(travel_data): print "Travel Data not found!!!" sys.exit() in_storage = csv_storage(storage_location = path) table_name = "travel_data" travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id) l = numpy.atleast_1d(origin_zones).tolist() origin_list = set(l) # removes duplicates and sorts the list in ascending order # destination_list = len(origin_list) * self.cbd # creates a list that contains the zone id of the cbd an has the same length as "origin_list" # set high travel costs for all origin to cbd pairs for id in origin_list: travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.high_travel_cost, id, self.cbd) # adjust cbd to cbd travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, 0.0, self.cbd, self.cbd) # adjust prefered zone to cbd travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.low_travel_cost, self.preferential_zone, self.cbd) w = travel_data_set.get_index_by_origin_and_destination_ids(110, 129) x = travel_data_set.get_index_by_origin_and_destination_ids(129, 129) y = travel_data_set.get_index_by_origin_and_destination_ids(20, 129) z = travel_data_set.get_index_by_origin_and_destination_ids(20, 20) print w print x print y print z origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id) destination_zones = travel_data_set.get_attribute_as_column(self.destination_zone_id) my_travel_data_attr_mat = travel_data_set.get_attribute_as_matrix('travel_data.single_vehicle_to_work_travel_cost', fill=999) my_travel_data_attr_mat[origin_zones, destination_zones] = 1.03 cbd_ids = where(origin_zones == 129) print "Leaving test run"
def test_run(self): print "Entering test run" path = os.path.join(os.environ['OPUS_HOME'], 'opus_matsim/tmp') # check if travel data exsits travel_data = os.path.join( path, "travel_data.csv" ) if not os.path.exists(travel_data): print "Travel Data not found!!!" sys.exit() in_storage = csv_storage(storage_location = path) table_name = "travel_data" travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id) l = numpy.atleast_1d(origin_zones).tolist() origin_list = set(l) # removes duplicates and sorts the list in ascending order # destination_list = len(origin_list) * self.cbd # creates a list that contains the zone id of the cbd an has the same length as "origin_list" # set high travel costs for all origin to cbd pairs for id in origin_list: travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.high_travel_cost, id, self.cbd) # adjust cbd to cbd travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, 0.0, self.cbd, self.cbd) # adjust prefered zone to cbd travel_data_set.set_values_of_one_attribute_with_od_pairs(self.travel_data_attribute, self.low_travel_cost, self.preferential_zone, self.cbd) w = travel_data_set.get_index_by_origin_and_destination_ids(110, 129) x = travel_data_set.get_index_by_origin_and_destination_ids(129, 129) y = travel_data_set.get_index_by_origin_and_destination_ids(20, 129) z = travel_data_set.get_index_by_origin_and_destination_ids(20, 20) print w print x print y print z origin_zones = travel_data_set.get_attribute_as_column(self.origin_zone_id) destination_zones = travel_data_set.get_attribute_as_column(self.destination_zone_id) my_travel_data_attr_mat = travel_data_set.get_attribute_as_matrix('travel_data.single_vehicle_to_work_travel_cost', fill=999) my_travel_data_attr_mat[origin_zones, destination_zones] = 1.03 cbd_ids = where(origin_zones == 129) print "Leaving test run"
def init(self, year, config): config_obj_v3 = MATSimConfigObjectV3(config, year) self.input_directory = config_obj_v3.matsim4urbansim_matsim_temp_path # self.input_directory = paths.get_opus_home_path( matsim4opus, matsim_temp ) logger.log_status("input_directory: " + self.input_directory ) self.in_storage = csv_storage(storage_location = self.input_directory) self.cache_storage = AttributeCache().get_flt_storage_for_year(year) self.cache_directory = config['cache_directory'] # tnicolai: experimental code self.matsim_controler = self.__get_matsim_controler_section(config) # this deletes travel model attributes from cache that are not updated/handled by the travel model (MATSim) self.delete_travel_data_columns = ['am_bike_to_work_travel_time', 'am_biking_person_trips', #'am_pk_period_drive_alone_vehicle_trips', 'am_total_transit_time_walk', 'am_transit_person_trip_table', #'am_walk_time_in_minutes', 'am_walking_person_trips', 'am_double_vehicle_to_work_travel_time', 'am_threeplus_vehicle_to_work_travel_time', 'logsum_hbw_am_income_1', 'logsum_hbw_am_income_2', 'logsum_hbw_am_income_3', 'logsum_hbw_am_income_4', 'md_vehicle_miles_traveled', 'nweubk', 'nweuda', 'nweus2', 'nweus3', 'nweutw', 'nweuwk', 'pm_ev_ni_vehicle_miles_traveled', 'single_vehicle_to_work_travel_distance'] self.travel_data_table_name = "travel_data" self.parcel_table_name = "parcels" self.zone_table_name = "zones" self.person_data_table = "persons"
def get_travel_data_from_travel_model(self, config, year, zone_set): """ """ logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...') input_directory = os.path.join( config['root'], 'opus_matsim', 'tmp' ) logger.log_status("input_directory: " + input_directory ) in_storage = csv_storage(storage_location = input_directory) table_name = "travel_data" travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name ) ##TODO:This may not work or may be wrong after the id_names of travel_data ##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang) existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY) return existing_travel_data_set
def test_run(self): print "Entering test run" # This test checks if the pre-computed MATSim travel data # is loaded correctly into UrbanSim. Therefore the UrbanSim # travel data matrix is converted into a numpy array and two # predefined values are checked whether they match with MATSim data. # # Here an example: # my_list = [[1,2,3], # [4,5,6], # [7,8,9]] # # my_list[0][1] should be = 2 # my_list[2][2] should be = 9 table_name = 'travel_data' travel_data_attribute = 'single_vehicle_to_work_travel_cost' # location of pre-calculated MATSim travel costs in_storage = csv_storage(storage_location = self.input_directory) # create travel data set (travel costs) travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=31) # converting from numpy array into a 2d list travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist() # get two values for validation value1 = int(travel_list[1][1]) # should be = 0 value2 = int(travel_list[2][1]) # should be = 120 logger.log_status('First validation value should be 0. Current value is %i' % value1) logger.log_status('Second validation value should be 120. Current value is %i' % value2) self.assertTrue( value1 == 0 ) self.assertTrue( value2 == 120 ) # self.dump_travel_list(travel_list) # for debugging print "Leaving test run"
def test_run(self): print "Entering test run" # This test loads an exising travel data as a TravelDataSet (numpy array) # and accesses single (pre-known) values to validate the conversion process # (numpy array into standard python list). # # Here an example: # my_list = [[1,2,3], # [4,5,6], # [7,8,9]] # # my_list[0][1] should be = 2 # my_list[2][2] should be = 9 table_name = 'travel_data' travel_data_attribute = 'single_vehicle_to_work_travel_cost' # location of pre-calculated MATSim travel costs in_storage = csv_storage(storage_location = self.input_directory) # create travel data set (travel costs) travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=31) # converting from numpy array into a 2d list travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist() # get two values for validation value1 = int(travel_list[1][1]) # should be = 0 value2 = int(travel_list[2][1]) # should be = 120 logger.log_status('First validation value should be 0. Current value is %i' % value1) logger.log_status('Second validation value should be 120. Current value is %i' % value2) self.assertTrue( value1 == 0 ) self.assertTrue( value2 == 120 ) # self.dump_travel_list(travel_list) # for debugging print "Leaving test run"
def get_travel_data_from_travel_model(self, config, year, zone_set): """ """ logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...') # print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them" # print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id" # solved 3dec08 by hana input_directory = os.path.join( os.environ['OPUS_HOME'], "opus_matsim", "tmp" ) logger.log_status("input_directory: " + input_directory ) in_storage = csv_storage(storage_location = input_directory) table_name = "travel_data" travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name ) ##TODO:This may not work or may be wrong after the id_names of travel_data ##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang) existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY) return existing_travel_data_set
def get_travel_cost_matrix(self): ''' Returns the pre-calculated MATSim travel costs as 2d list ''' # get sensitivity test path test_dir_path = test_dir.__path__[0] input_directory = os.path.join( test_dir_path, 'data', 'travel_cost') print "input_directory: %s" % input_directory # check source file if not os.path.exists( input_directory ): print 'File not found! %s' % input_directory sys.exit() table_name = 'travel_data' travel_data_attribute = 'single_vehicle_to_work_travel_cost' # location of pre-calculated MATSim travel costs in_storage = csv_storage(storage_location = input_directory) # create travel data set (travel costs) travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) # travel cost matris as 2d array travel_list = numpy.atleast_2d(travel_data_attribute_mat).tolist() return travel_list
# Opus/UrbanSim urban simulation software. # Copyright (C) 2012 University of California, Berkeley, 2005-2009 University of Washington # See opus_core/LICENSE from opus_core.store.csv_storage import csv_storage from opus_core.store.dbf_storage import dbf_storage import os, sys if len(sys.argv) not in (2,3): print "Usage: %s csv_file [dbf_file]" % sys.argv[0] sys.exit(0) csv_file = sys.argv[1] csv_file = os.path.normpath(csv_file) csv_path, csv_name = os.path.split(csv_file) csv_table, csv_ext = os.path.splitext(csv_name) if len(sys.argv) == 2: dbf_path, dbf_table = csv_path, csv_table elif len(sys.argv) == 3: dbf_file = sys.argv[2] dbf_file = os.path.normpath(dbf_file) dbf_path, dbf_name = os.path.split(dbf_file) dbf_table, dbf_ext = os.path.splitext(dbf_name) csv_store = csv_storage(storage_location=csv_path) dbf_store = dbf_storage(storage_location=dbf_path) data = csv_store.load_table(csv_table) dbf_store.write_table(dbf_table, data)
parser = OptionParser() parser.add_option('-c', '--cache_path', dest='cache_path', type='string', help='The filesystem path to the cache to export (required)') parser.add_option('-o', '--output_directory', dest='output_directory', type='string', help='The filesystem path of the database to which ' 'output will be written (required)') parser.add_option('-t', '--table_name', dest='table_name', type='string', help='Name of table to be exported (optional). Used if only one table should be exported.') (options, args) = parser.parse_args() cache_path = options.cache_path output_directory = options.output_directory table_name = options.table_name if None in (cache_path, output_directory): parser.print_help() sys.exit(1) in_storage = flt_storage(storage_location=cache_path) out_storage = csv_storage(storage_location=output_directory) if not os.path.exists(output_directory): os.makedirs(output_directory) if table_name is not None: ExportStorage().export_dataset(table_name, in_storage=in_storage, out_storage=out_storage) else: ExportStorage().export(in_storage=in_storage, out_storage=out_storage)
def get_travel_data_from_travel_model(self, config, year, zone_set): """ Integrates modified travel times and pre-computed travel costs into the UrbanSim cache. """ logger.log_status( 'Starting GetTestTravelDataIntoCache.get_travel_data...') # get sensitivity test path asan anchor to determine the location of the MATSim travel_data file (see below). test_dir_path = test_dir.__path__[0] # for debugging try: #tnicolai import pydevd pydevd.settrace() except: pass # get the exsisting travel data from the current year logger.log_status('Loading travel data from UrbanSim cache (year:%i)' % year) table_name = "travel_data" cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) ###### modifyed travel time travel data logger.log_status( 'Integrating modifyed travel times in year %i for next simulation year.' ) input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim", "tmp") logger.log_status("input_directory: " + input_directory) # location of the modified travel time travel_data in_storage = csv_storage(storage_location=input_directory) # create travel data set (travel times) travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) # join the modifyed travel times with the travel data set of the current year existing_travel_data_set.join( travel_data_set, travel_data_set.get_non_id_primary_attribute_names(), metadata=AttributeType.PRIMARY) ##### pre-calcualted MATSim travel data (travel costs) # logger.log_status('Integrating pre-calculated travel costs (MATSim) in year %i for next simulation year.') # input_directory = os.path.join( test_dir_path, 'data', 'travel_cost') # logger.log_status("input_directory: " + input_directory ) # # check source file # if not os.path.exists( input_directory ): # print 'File not found! %s' % input_directory # sys.exit() # location of pre-calculated MATSim travel costs # in_storage = csv_storage(storage_location = input_directory) # create travel data set (travel costs) # travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) # join travel data set from pre-calcualted MATSim results # existing_travel_data_set.join(travel_data_set, travel_data_set.get_non_id_primary_attribute_names(),metadata=AttributeType.PRIMARY) return existing_travel_data_set
def run(self, config, year, storage_type='sql'): """ """ tm_config = config['travel_model_configuration'] database_server_config = tm_config.get("database_server_configuration", 'simtravel_database_server') database_name = tm_config.get("database_name", 'mag_zone_baseyear') cache_directory = config['cache_directory'] simulation_state = SimulationState() simulation_state.set_cache_directory(cache_directory) simulation_state.set_current_time(year) attribute_cache = AttributeCache() dataset_pool = SessionConfiguration(new_instance=True, package_order=config['dataset_pool_configuration'].package_order, in_storage=attribute_cache).get_dataset_pool() if storage_type == 'sql': db_server = DatabaseServer(DatabaseConfiguration( database_name = database_name, database_configuration = database_server_config ) ) if not db_server.has_database(database_name): print "Db doesn't exist creating one" db_server.create_database(database_name) db = db_server.get_database(database_name) output_storage = sql_storage(storage_location = db) elif storage_type == 'csv': csv_directory = os.path.join(cache_directory, 'csv', str(year)) output_storage = csv_storage(storage_location=csv_directory) else: raise ValueError, "Unsupported output storage type {}".format(storage_type) logger.start_block('Compute and export data to openAMOS...') hh = dataset_pool.get_dataset('household') hh_recs = dataset_pool.get_dataset('households_recs') #hh_recs.add_attribute(0,"htaz1") #hh_recs.flush_dataset() #syn_hh = dataset_pool.get_dataset('synthetic_household') hh_variables = ['houseid=household.household_id', "hhsize=household.number_of_agents(person)", "one=(household.household_id>0).astype('i')", "inclt35k=(household.income<35000).astype('i')", "incge35k=(household.income>=35000).astype('i')", "incge50k=(household.income>=50000).astype('i')", "incge75k=(household.income>=75000).astype('i')", "incge100k=(household.income>=100000).astype('i')", "inc35t50=((household.income>=35000) & (household.income<50000)).astype('i')", "inc50t75=((household.income>=50000) & (household.income<75000)).astype('i')", "inc75t100=((household.income>=75000) & (household.income<100000)).astype('i')", 'htaz = ((houseid>0)*(household.disaggregate(building.zone_id)-100) + (houseid<=0)*0)', #'htaz = ((houseid>0) & (htaz1>100))*(htaz1-100)+((houseid>0) & (htaz1==-1))*1122', "withchild = (household.aggregate(person.age<18)>0).astype('i')", "noc = household.aggregate(person.age<18)", "numadlt = household.aggregate(person.age>=18)", "hinc=household.income", "wif=household.workers", #"wif=household.aggregate(mag_zone.person.is_employed)", 'numwrkr=household.workers', #'numwrkr=household.aggregate(mag_zone.person.is_employed)', 'nwrkcnt=household.number_of_agents(person) - household.workers', #'nwrkcnt=household.number_of_agents(person) - household.aggregate(mag_zone.person.is_employed)', 'yrbuilt=mag_zone.household.yrbuilt', 'mag_zone.household.sparent', 'mag_zone.household.rur', 'mag_zone.household.urb', 'zonetid = household.disaggregate(building.zone_id)', ] self.prepare_attributes(hh, hh_variables) attrs_to_export = hh_recs.get_known_attribute_names() self.write_dataset(hh, attrs_to_export, output_storage) dataset_pool._remove_dataset(hh.dataset_name) persons = dataset_pool.get_dataset('person') persons.out_table_name_default = 'persons' # Recoding invalid work and school locations to some random valid values persons_recs = dataset_pool.get_dataset('persons_recs') persons_recs.add_attribute(persons['person_id'],"personuniqueid") persons_recs.add_attribute(persons['marriage_status'],"marstat") persons_recs.add_attribute(persons['student_status'],"schstat") """ persons_recs.add_attribute(persons['wtaz0'],"htaz_act") persons_recs.add_attribute(0,"wtaz_rec") persons_recs.add_attribute(0,"wtaz_rec1") persons_recs.add_attribute(0,"wtaz_rec2") persons_recs.add_attribute(0,"wtaz1_1") persons_recs.add_attribute(0,"wtaz1_2") persons_recs.add_attribute(0,"wtaz1_3") #persons_recs.add_attribute(persons['student_status'],"schstat") """ persons_recs.add_attribute(0,"wtaz1") persons_recs.add_attribute(0,"htaz") persons_recs.add_attribute(0,"schtaz1") persons_recs.flush_dataset() #syn_persons = dataset_pool.get_dataset('synthetic_person') persons_variables = ['personid=mag_zone.person.member_id', 'personuniqueid=person.person_id', 'houseid=person.household_id', "one=(person.person_id>0).astype('i')", 'trvtime=mag_zone.person.travel_time_from_home_to_work', 'timetowk=mag_zone.person.travel_time_from_home_to_work', #'mag_zone.person.tmtowrk', #'tmtowrk=person.disaggregate(synthetic_person.tmtowrk)', "ag5t10=((person.age>=5) & (person.age<=10)).astype('i')", "ag11t14=((person.age>=11) & (person.age<=14)).astype('i')", "ag15t17=((person.age>=15) & (person.age<=17)).astype('i')", "ag18t24=((person.age>=18) & (person.age<=24)).astype('i')", "ag25t34=((person.age>=25) & (person.age<=34)).astype('i')", "ag35t44=((person.age>=35) & (person.age<=44)).astype('i')", "ag45t54=((person.age>=45) & (person.age<=54)).astype('i')", "ag55t64=((person.age>=55) & (person.age<=64)).astype('i')", "agge65=(person.age>=65).astype('i')", "ag12t17=((person.age>=12) & (person.age<=17)).astype('i')", "ag5t14=((person.age>=5) & (person.age<=14)).astype('i')", "agge15=(person.age>=15).astype('i')", "wrkr=(person.employment_status==1).astype('i')", "isemploy=(person.employment_status==1).astype('i')", "fulltim=(mag_zone.person.full_time==1).astype('i')", 'parttim=mag_zone.person.part_time', 'htaz = ((houseid>0)*(person.disaggregate(building.zone_id, intermediates=[household])-100) + (houseid<=0)*0)', 'wtaz1=(person.wtaz <= 0)*0 + (person.wtaz > 0)*(person.wtaz-100)', "presch = ((person.age < 5)&(houseid>0)).astype('i')", "mag_zone.person.schstat", 'schtaz1 = (person.schtaz <= 0)*0 + (person.schtaz > 0)*(person.schtaz-100)', 'marstat = person.marriage_status', 'enroll = person.student_status', 'grade = person.student_status & person.education', 'educ = person.education', "male = (person.sex==1).astype('i')", "female = (person.sex==2).astype('i')", "coled = (person.education >= 10).astype('i')", 'race1 = person.race', "white = (person.race == 1).astype('i')", 'person.hispanic' ] self.prepare_attributes(persons, persons_variables) attrs_to_export = persons_recs.get_known_attribute_names() self.write_dataset(persons, attrs_to_export, output_storage) dataset_pool._remove_dataset(persons.dataset_name) zones = dataset_pool.get_dataset('zone') zones_variables = [ "retail_employment=zone.aggregate(mag_zone.job.sector_group=='retail')", "public_employment=zone.aggregate(mag_zone.job.sector_group=='public')", "office_employment=zone.aggregate(mag_zone.job.sector_group=='office')", "industrial_employment=zone.aggregate(mag_zone.job.sector_group=='individual')", "other_employment=zone.aggregate(mag_zone.job.sector_group=='other')", "retail_employment_density=zone.aggregate(mag_zone.job.sector_group=='retail')/zone.acres", "public_employment_density=zone.aggregate(mag_zone.job.sector_group=='public')/zone.acres", "office_employment_density=zone.aggregate(mag_zone.job.sector_group=='office')/zone.acres", "industrial_employment_density=zone.aggregate(mag_zone.job.sector_group=='individual')/zone.acres", "other_employment_density=zone.aggregate(mag_zone.job.sector_group=='other')/zone.acres", "total_area=zone.acres", "lowest_income=zone.aggregate(household.income < scoreatpercentile(household.income, 20))", "low_income=zone.aggregate(household.income < scoreatpercentile(household.income, 40))", "high_income=zone.aggregate(household.income > scoreatpercentile(household.income, 80))", #"institutional_population=zone.disaggregate(locations.institutional_population)", #"groupquarter_households=zone.disaggregate(locations.groupquarter_households)", "residential_households=zone.number_of_agents(household)", "locationid=zone.zone_id", ] locations = dataset_pool['locations'] self.prepare_attributes(zones, zones_variables, dataset2=locations) attrs_to_export = locations.get_known_attribute_names() self.write_dataset(locations, attrs_to_export, output_storage) dataset_pool._remove_dataset(locations.dataset_name) #raw_input("check location block") logger.end_block()
"The attribute cache year into which to write the output (required).") (options, args) = parser.parse_args() csv_directory = options.csv_directory attribute_cache_directory = options.attribute_cache_directory table_name = options.table_name cache_year = options.cache_year if (csv_directory is None or attribute_cache_directory is None or table_name is None or cache_year is None): parser.print_help() sys.exit(1) input_storage = csv_storage(storage_location=csv_directory) attribute_cache = AttributeCache(cache_directory=attribute_cache_directory) output_storage = attribute_cache.get_flt_storage_for_year(cache_year) SimulationState().set_current_time(cache_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) logger.start_block( "Exporting table '%s' to year %s of cache located at %s..." % (table_name, cache_year, attribute_cache_directory)) try: ExportStorage().export_dataset( dataset_name=table_name, in_storage=input_storage,
(options, args) = parser.parse_args() csv_directory = options.csv_directory attribute_cache_directory = options.attribute_cache_directory table_name = options.table_name cache_year = options.cache_year if (csv_directory is None or attribute_cache_directory is None or table_name is None or cache_year is None): parser.print_help() sys.exit(1) input_storage = csv_storage(storage_location = csv_directory) attribute_cache = AttributeCache(cache_directory=attribute_cache_directory) output_storage = attribute_cache.get_flt_storage_for_year(cache_year) SimulationState().set_current_time(cache_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) logger.start_block("Exporting table '%s' to year %s of cache located at %s..." % (table_name, cache_year, attribute_cache_directory)) try: ExportStorage().export_dataset( dataset_name = table_name, in_storage = input_storage, out_storage = output_storage,
def run(self, year=None, years_to_run=[], configuration=None): if year not in years_to_run or self.data_to_export == None: return cache_directory = configuration['cache_directory'] simulation_state = SimulationState() simulation_state.set_cache_directory(cache_directory) simulation_state.set_current_time(year) attribute_cache = AttributeCache() package_order=configuration['dataset_pool_configuration'].package_order dataset_pool = SessionConfiguration(new_instance=True, package_order=package_order, in_storage=attribute_cache ).get_dataset_pool() out_dir = os.path.join(cache_directory, "mtc_data") out_storage = csv_storage(storage_location=out_dir) # Adjust the age distribution per ABAG/MTC's specifications age_control_dir = os.path.join(paths.OPUS_DATA_PATH, configuration['project_name'], "ageControl") age_control_storage = csv_storage(storage_location=age_control_dir) age_control_files = os.listdir(age_control_dir) years = np.array(map(lambda x: int(os.path.basename(x).replace("tazData", "").replace(".csv", "")), glob.glob(os.path.join(age_control_dir, "tazData*.csv")))) closest_year = years[np.argmin(np.abs(years - [year]*len(years)))] if closest_year != year: logger.log_warning("Could not find age control data for " + str(year) + ". Choosing nearest year " + str(closest_year) + ".") age_control_table = age_control_storage.load_table("tazData" + str(closest_year), lowercase=False) # Calculate the ABAG shares of person by age age_categories = ['AGE0004', 'AGE0519', 'AGE2044', 'AGE4564', 'AGE65P'] age_category_sums = dict((k, age_control_table[k].sum()) for k in age_categories) total = sum(age_category_sums.values()) abag_age_category_shares = dict((k, age_category_sums[k]/total) for k in age_categories) for data_fname, variable_mapping in self.data_to_export.iteritems(): if not flip_urbansim_to_tm_variable_mappling: col_names = variable_mapping.values() variables_aliases = ["=".join(mapping[::-1]) for mapping in \ variable_mapping.iteritems()] else: col_names = variable_mapping.keys() variables_aliases = ["=".join(mapping) for mapping in \ variable_mapping.iteritems()] dataset_name = VariableName(variables_aliases[0]).get_dataset_name() dataset = dataset_pool.get_dataset(dataset_name) dataset.compute_variables(variables_aliases) if data_fname == "ABAGData": logger.log_status("Adjusting ABAGData to match age controls") age_category_sums = dict((k, dataset[k].sum()) for k in age_categories) total = sum(age_category_sums.values()) us_age_category_shares = dict((k, age_category_sums[k]/total) for k in age_categories) adjustments = dict((k, abag_age_category_shares[k]/us_age_category_shares[k]) for k in age_categories) diff = np.zeros(dataset.n) for k in age_categories: before = dataset[k] dataset[k] = np.array(map(lambda v : round(v*adjustments[k]), dataset.get_attribute(k))) diff += (dataset[k] - before) dataset["TOTPOP"] += diff dataset["HHPOP"] += diff logger.log_status("NOTE: Adjusted total population by %d (%2.3f%%) due to rounding error." % (int(diff.sum()), diff.sum()*100/total)) org_fname = os.path.join(out_dir, "%s.computed.csv" % data_fname) new_fname = os.path.join(out_dir, "%s%s.csv" % (year,data_fname)) block_msg = "Writing {} for travel model to {}".format(data_fname, new_fname) with block(block_msg): dataset.write_dataset(attributes=col_names, out_storage=out_storage, out_table_name=data_fname) #rename & process header shutil.move(org_fname, new_fname) os.system("sed 's/:[a-z][0-9]//g' -i %s" % new_fname)