def prepare_for_run(self, database_configuration, database_name): ## sql protocol, hostname, username and password are set in ## $OPUS_HOME/settings/database_server_setting.xml db_config = DatabaseConfiguration(database_name=database_name, database_configuration=database_configuration) db_server = DatabaseServer(db_config) if not db_server.has_database(database_name): db_server.create_database(database_name) db = db_server.get_database(database_name) self.out_storage = sql_storage(storage_location=db) return self.out_storage
def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_directory = paths.prepend_opus_home_if_relative(opus_data_directory) opus_data_year = params_dict['opus_data_year'] database_name = params_dict['database_name'] table_name = params_dict['table_name'] database_server_connection = params_dict['database_server_connection'] overwrite = params_dict['overwrite'] dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration = dbs_config) opusdb = server.get_database(database_name=database_name, create_if_doesnt_exist=False) input_storage = sql_storage(storage_location = opusdb) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': lst = input_storage.get_table_names() else: lst = re.split(' +', table_name.strip()) tables = len(lst) lst_out = create_list_string(lst, ', ') logCB('caching tables:\n%s\n' % lst_out) for j, i in enumerate(lst, start=1): logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory)) try: ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage, overwrite = overwrite, ) except: logCB('Error in exporting %s.' % i) progressCB(100 * j / tables) logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB, logCB, params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_directory = paths.prepend_opus_home_if_relative( opus_data_directory) opus_data_year = params_dict['opus_data_year'] database_name = params_dict['database_name'] table_name = params_dict['table_name'] database_server_connection = params_dict['database_server_connection'] overwrite = params_dict['overwrite'] dbs_config = DatabaseServerConfiguration( database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration=dbs_config) opusdb = server.get_database(database_name=database_name, create_if_doesnt_exist=False) input_storage = sql_storage(storage_location=opusdb) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': lst = input_storage.get_table_names() else: lst = re.split(' +', table_name.strip()) tables = len(lst) lst_out = create_list_string(lst, ', ') logCB('caching tables:\n%s\n' % lst_out) for j, i in enumerate(lst, start=1): logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory)) ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage, out_storage=output_storage, overwrite=overwrite, ) progressCB(100 * j / tables) logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB, logCB, params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) database_name = params_dict['database_name'] opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] opus_table_name = params_dict['opus_table_name'] database_server_connection = params_dict['database_server_connection'] dbs_config = DatabaseServerConfiguration( database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration=dbs_config) opusdb = server.get_database(database_name=database_name) attribute_cache = AttributeCache(cache_directory=opus_data_directory) attribute_cache_years = [ int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4 ] if opus_data_year != 'ALL': attribute_cache_years = [opus_data_year] for year in attribute_cache_years: #input_storage = sql_storage(storage_location = opusdb) input_storage = attribute_cache.get_flt_storage_for_year(year) #output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) if opus_data_year == 'ALL': opusdb = server.get_database(database_name=database_name + "_" + str(year)) output_storage = sql_storage(storage_location=opusdb) SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if opus_table_name != 'ALL': opus_table_name_list = re.split(' +', opus_table_name.strip()) else: opus_table_name_list = input_storage.get_table_names() for i in opus_table_name_list: logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory)) ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage, out_storage=output_storage, )
def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) database_name = params_dict['database_name'] opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] opus_table_name = params_dict['opus_table_name'] database_server_connection = params_dict['database_server_connection'] dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration = dbs_config) opusdb = server.get_database(database_name=database_name) attribute_cache = AttributeCache(cache_directory=opus_data_directory) attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4] if opus_data_year != 'ALL': attribute_cache_years = [opus_data_year] for year in attribute_cache_years: #input_storage = sql_storage(storage_location = opusdb) input_storage = attribute_cache.get_flt_storage_for_year(year) #output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) output_storage = sql_storage(storage_location = opusdb) SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if opus_table_name != 'ALL': opus_table_name_list = [opus_table_name] else: opus_table_name_list = input_storage.get_table_names() for i in opus_table_name_list: logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory)) ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage, )
def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] database_name = params_dict['database_name'] table_name = params_dict['table_name'] database_server_connection = params_dict['database_server_connection'] dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration = dbs_config) opusdb = server.get_database(database_name=database_name) input_storage = sql_storage(storage_location = opusdb) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logCB('caching all tables...\n') lst = input_storage.get_table_names() for i in lst: ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage, ) else: logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (table_name, opus_data_year, opus_data_directory)) ExportStorage().export_dataset( dataset_name = table_name, in_storage = input_storage, out_storage = output_storage)
def opusRun(progressCB, logCB, params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] database_name = params_dict['database_name'] table_name = params_dict['table_name'] database_server_connection = params_dict['database_server_connection'] dbs_config = DatabaseServerConfiguration( database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration=dbs_config) opusdb = server.get_database(database_name=database_name) input_storage = sql_storage(storage_location=opusdb) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logCB('caching all tables...\n') lst = input_storage.get_table_names() for i in lst: ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage, out_storage=output_storage, ) else: logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (table_name, opus_data_year, opus_data_directory)) ExportStorage().export_dataset(dataset_name=table_name, in_storage=input_storage, out_storage=output_storage)
if (db_name is None or attribute_cache_directory is None or cache_year is None): parser.print_help() sys.exit(1) if table_name is None: table_name = 'ALL' dbserverconfig = EstimationDatabaseConfiguration( database_configuration = options.database_configuration ) opusdb = OpusDatabase(dbserverconfig, db_name) input_storage = sql_storage(storage_location = opusdb) attribute_cache = AttributeCache(cache_directory=attribute_cache_directory) output_storage = attribute_cache.get_flt_storage_for_year(cache_year) SimulationState().set_current_time(cache_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logger.log_status('Caching all tables in database...') lst = input_storage.get_table_names() for i in lst: ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage,
cache_year = options.cache_year if (db_name is None or attribute_cache_directory is None or cache_year is None): parser.print_help() sys.exit(1) if table_name is None: table_name = 'ALL' dbserverconfig = EstimationDatabaseConfiguration( database_configuration=options.database_configuration) opusdb = OpusDatabase(dbserverconfig, db_name) input_storage = sql_storage(storage_location=opusdb) attribute_cache = AttributeCache(cache_directory=attribute_cache_directory) output_storage = attribute_cache.get_flt_storage_for_year(cache_year) SimulationState().set_current_time(cache_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logger.log_status('Caching all tables in database...') lst = input_storage.get_table_names() for i in lst: ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage,
def run(self, config, year, storage_type='sql'): """ """ tm_config = config['travel_model_configuration'] database_server_config = tm_config.get("database_server_configuration", 'simtravel_database_server') database_name = tm_config.get("database_name", 'mag_zone_baseyear') cache_directory = config['cache_directory'] simulation_state = SimulationState() simulation_state.set_cache_directory(cache_directory) simulation_state.set_current_time(year) attribute_cache = AttributeCache() dataset_pool = SessionConfiguration(new_instance=True, package_order=config['dataset_pool_configuration'].package_order, in_storage=attribute_cache).get_dataset_pool() if storage_type == 'sql': db_server = DatabaseServer(DatabaseConfiguration( database_name = database_name, database_configuration = database_server_config ) ) if not db_server.has_database(database_name): print "Db doesn't exist creating one" db_server.create_database(database_name) db = db_server.get_database(database_name) output_storage = sql_storage(storage_location = db) elif storage_type == 'csv': csv_directory = os.path.join(cache_directory, 'csv', str(year)) output_storage = csv_storage(storage_location=csv_directory) else: raise ValueError, "Unsupported output storage type {}".format(storage_type) logger.start_block('Compute and export data to openAMOS...') hh = dataset_pool.get_dataset('household') hh_recs = dataset_pool.get_dataset('households_recs') #hh_recs.add_attribute(0,"htaz1") #hh_recs.flush_dataset() #syn_hh = dataset_pool.get_dataset('synthetic_household') hh_variables = ['houseid=household.household_id', "hhsize=household.number_of_agents(person)", "one=(household.household_id>0).astype('i')", "inclt35k=(household.income<35000).astype('i')", "incge35k=(household.income>=35000).astype('i')", "incge50k=(household.income>=50000).astype('i')", "incge75k=(household.income>=75000).astype('i')", "incge100k=(household.income>=100000).astype('i')", "inc35t50=((household.income>=35000) & (household.income<50000)).astype('i')", "inc50t75=((household.income>=50000) & (household.income<75000)).astype('i')", "inc75t100=((household.income>=75000) & (household.income<100000)).astype('i')", 'htaz = ((houseid>0)*(household.disaggregate(building.zone_id)-100) + (houseid<=0)*0)', #'htaz = ((houseid>0) & (htaz1>100))*(htaz1-100)+((houseid>0) & (htaz1==-1))*1122', "withchild = (household.aggregate(person.age<18)>0).astype('i')", "noc = household.aggregate(person.age<18)", "numadlt = household.aggregate(person.age>=18)", "hinc=household.income", "wif=household.workers", #"wif=household.aggregate(mag_zone.person.is_employed)", 'numwrkr=household.workers', #'numwrkr=household.aggregate(mag_zone.person.is_employed)', 'nwrkcnt=household.number_of_agents(person) - household.workers', #'nwrkcnt=household.number_of_agents(person) - household.aggregate(mag_zone.person.is_employed)', 'yrbuilt=mag_zone.household.yrbuilt', 'mag_zone.household.sparent', 'mag_zone.household.rur', 'mag_zone.household.urb', 'zonetid = household.disaggregate(building.zone_id)', ] self.prepare_attributes(hh, hh_variables) attrs_to_export = hh_recs.get_known_attribute_names() self.write_dataset(hh, attrs_to_export, output_storage) dataset_pool._remove_dataset(hh.dataset_name) persons = dataset_pool.get_dataset('person') persons.out_table_name_default = 'persons' # Recoding invalid work and school locations to some random valid values persons_recs = dataset_pool.get_dataset('persons_recs') persons_recs.add_attribute(persons['person_id'],"personuniqueid") persons_recs.add_attribute(persons['marriage_status'],"marstat") persons_recs.add_attribute(persons['student_status'],"schstat") """ persons_recs.add_attribute(persons['wtaz0'],"htaz_act") persons_recs.add_attribute(0,"wtaz_rec") persons_recs.add_attribute(0,"wtaz_rec1") persons_recs.add_attribute(0,"wtaz_rec2") persons_recs.add_attribute(0,"wtaz1_1") persons_recs.add_attribute(0,"wtaz1_2") persons_recs.add_attribute(0,"wtaz1_3") #persons_recs.add_attribute(persons['student_status'],"schstat") """ persons_recs.add_attribute(0,"wtaz1") persons_recs.add_attribute(0,"htaz") persons_recs.add_attribute(0,"schtaz1") persons_recs.flush_dataset() #syn_persons = dataset_pool.get_dataset('synthetic_person') persons_variables = ['personid=mag_zone.person.member_id', 'personuniqueid=person.person_id', 'houseid=person.household_id', "one=(person.person_id>0).astype('i')", 'trvtime=mag_zone.person.travel_time_from_home_to_work', 'timetowk=mag_zone.person.travel_time_from_home_to_work', #'mag_zone.person.tmtowrk', #'tmtowrk=person.disaggregate(synthetic_person.tmtowrk)', "ag5t10=((person.age>=5) & (person.age<=10)).astype('i')", "ag11t14=((person.age>=11) & (person.age<=14)).astype('i')", "ag15t17=((person.age>=15) & (person.age<=17)).astype('i')", "ag18t24=((person.age>=18) & (person.age<=24)).astype('i')", "ag25t34=((person.age>=25) & (person.age<=34)).astype('i')", "ag35t44=((person.age>=35) & (person.age<=44)).astype('i')", "ag45t54=((person.age>=45) & (person.age<=54)).astype('i')", "ag55t64=((person.age>=55) & (person.age<=64)).astype('i')", "agge65=(person.age>=65).astype('i')", "ag12t17=((person.age>=12) & (person.age<=17)).astype('i')", "ag5t14=((person.age>=5) & (person.age<=14)).astype('i')", "agge15=(person.age>=15).astype('i')", "wrkr=(person.employment_status==1).astype('i')", "isemploy=(person.employment_status==1).astype('i')", "fulltim=(mag_zone.person.full_time==1).astype('i')", 'parttim=mag_zone.person.part_time', 'htaz = ((houseid>0)*(person.disaggregate(building.zone_id, intermediates=[household])-100) + (houseid<=0)*0)', 'wtaz1=(person.wtaz <= 0)*0 + (person.wtaz > 0)*(person.wtaz-100)', "presch = ((person.age < 5)&(houseid>0)).astype('i')", "mag_zone.person.schstat", 'schtaz1 = (person.schtaz <= 0)*0 + (person.schtaz > 0)*(person.schtaz-100)', 'marstat = person.marriage_status', 'enroll = person.student_status', 'grade = person.student_status & person.education', 'educ = person.education', "male = (person.sex==1).astype('i')", "female = (person.sex==2).astype('i')", "coled = (person.education >= 10).astype('i')", 'race1 = person.race', "white = (person.race == 1).astype('i')", 'person.hispanic' ] self.prepare_attributes(persons, persons_variables) attrs_to_export = persons_recs.get_known_attribute_names() self.write_dataset(persons, attrs_to_export, output_storage) dataset_pool._remove_dataset(persons.dataset_name) zones = dataset_pool.get_dataset('zone') zones_variables = [ "retail_employment=zone.aggregate(mag_zone.job.sector_group=='retail')", "public_employment=zone.aggregate(mag_zone.job.sector_group=='public')", "office_employment=zone.aggregate(mag_zone.job.sector_group=='office')", "industrial_employment=zone.aggregate(mag_zone.job.sector_group=='individual')", "other_employment=zone.aggregate(mag_zone.job.sector_group=='other')", "retail_employment_density=zone.aggregate(mag_zone.job.sector_group=='retail')/zone.acres", "public_employment_density=zone.aggregate(mag_zone.job.sector_group=='public')/zone.acres", "office_employment_density=zone.aggregate(mag_zone.job.sector_group=='office')/zone.acres", "industrial_employment_density=zone.aggregate(mag_zone.job.sector_group=='individual')/zone.acres", "other_employment_density=zone.aggregate(mag_zone.job.sector_group=='other')/zone.acres", "total_area=zone.acres", "lowest_income=zone.aggregate(household.income < scoreatpercentile(household.income, 20))", "low_income=zone.aggregate(household.income < scoreatpercentile(household.income, 40))", "high_income=zone.aggregate(household.income > scoreatpercentile(household.income, 80))", #"institutional_population=zone.disaggregate(locations.institutional_population)", #"groupquarter_households=zone.disaggregate(locations.groupquarter_households)", "residential_households=zone.number_of_agents(household)", "locationid=zone.zone_id", ] locations = dataset_pool['locations'] self.prepare_attributes(zones, zones_variables, dataset2=locations) attrs_to_export = locations.get_known_attribute_names() self.write_dataset(locations, attrs_to_export, output_storage) dataset_pool._remove_dataset(locations.dataset_name) #raw_input("check location block") logger.end_block()
sys.exit(1) table_name = options.table_name logger.log_status('Initializing database...') db_server = DatabaseServer( EstimationDatabaseConfiguration( database_name=database_name, database_configuration=options.database_configuration)) if not db_server.has_database( database_name): # if only one table should be exported, db_server.create_database(database_name) # the database can exist db = db_server.get_database(database_name) input_storage = flt_storage(storage_location=cache_path) output_storage = sql_storage(storage_location=db) logger.start_block('Exporting cache to sql...') try: if table_name is None: ExportStorage().export(in_storage=input_storage, out_storage=output_storage) else: db.drop_table(table_name) ExportStorage().export_dataset(table_name, in_storage=input_storage, out_storage=output_storage) finally: logger.end_block()
cache_path = options.cache_path database_name = options.database_name if database_name is None or cache_path is None: parser.print_help() sys.exit(1) table_name = options.table_name logger.log_status('Initializing database...') db_server = DatabaseServer(EstimationDatabaseConfiguration( database_name = database_name, database_configuration = options.database_configuration ) ) if not db_server.has_database(database_name): # if only one table should be exported, db_server.create_database(database_name) # the database can exist db = db_server.get_database(database_name) input_storage = flt_storage(storage_location = cache_path) output_storage = sql_storage( storage_location = db) with logger.block('Exporting cache to sql...'): if table_name is None: ExportStorage().export(in_storage=input_storage, out_storage=output_storage) else: db.drop_table(table_name) ExportStorage().export_dataset(table_name, in_storage=input_storage, out_storage=output_storage)