def prepare_for_run(self, database_configuration, database_name):
## sql protocol, hostname, username and password are set in
## $OPUS_HOME/settings/database_server_setting.xml
db_config = DatabaseConfiguration(database_name=database_name, database_configuration=database_configuration)
db_server = DatabaseServer(db_config)
if not db_server.has_database(database_name):
db_server.create_database(database_name)
db = db_server.get_database(database_name)
self.out_storage = sql_storage(storage_location=db)
return self.out_storage
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_directory = paths.prepend_opus_home_if_relative(opus_data_directory)
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
overwrite = params_dict['overwrite']
dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration = dbs_config)
opusdb = server.get_database(database_name=database_name, create_if_doesnt_exist=False)
input_storage = sql_storage(storage_location = opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
lst = input_storage.get_table_names()
else:
lst = re.split(' +', table_name.strip())
tables = len(lst)
lst_out = create_list_string(lst, ', ')
logCB('caching tables:\n%s\n' % lst_out)
for j, i in enumerate(lst, start=1):
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
try:
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
overwrite = overwrite,
)
except:
logCB('Error in exporting %s.' % i)
progressCB(100 * j / tables)
logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_directory = paths.prepend_opus_home_if_relative(
opus_data_directory)
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
overwrite = params_dict['overwrite']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name,
create_if_doesnt_exist=False)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
lst = input_storage.get_table_names()
else:
lst = re.split(' +', table_name.strip())
tables = len(lst)
lst_out = create_list_string(lst, ', ')
logCB('caching tables:\n%s\n' % lst_out)
for j, i in enumerate(lst, start=1):
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
overwrite=overwrite,
)
progressCB(100 * j / tables)
logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
database_name = params_dict['database_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
opus_table_name = params_dict['opus_table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
#input_storage = sql_storage(storage_location = opusdb)
input_storage = attribute_cache.get_flt_storage_for_year(year)
#output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
if opus_data_year == 'ALL':
opusdb = server.get_database(database_name=database_name + "_" +
str(year))
output_storage = sql_storage(storage_location=opusdb)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = re.split(' +', opus_table_name.strip())
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
database_name = params_dict['database_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
opus_table_name = params_dict['opus_table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration = dbs_config)
opusdb = server.get_database(database_name=database_name)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
#input_storage = sql_storage(storage_location = opusdb)
input_storage = attribute_cache.get_flt_storage_for_year(year)
#output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
output_storage = sql_storage(storage_location = opusdb)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration = dbs_config)
opusdb = server.get_database(database_name=database_name)
input_storage = sql_storage(storage_location = opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
else:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(table_name, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name = table_name,
in_storage = input_storage,
out_storage = output_storage)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
else:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(table_name, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(dataset_name=table_name,
in_storage=input_storage,
out_storage=output_storage)
if (db_name is None or
attribute_cache_directory is None or
cache_year is None):
parser.print_help()
sys.exit(1)
if table_name is None:
table_name = 'ALL'
dbserverconfig = EstimationDatabaseConfiguration(
database_configuration = options.database_configuration
)
opusdb = OpusDatabase(dbserverconfig, db_name)
input_storage = sql_storage(storage_location = opusdb)
attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
SimulationState().set_current_time(cache_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logger.log_status('Caching all tables in database...')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
cache_year = options.cache_year
if (db_name is None or attribute_cache_directory is None
or cache_year is None):
parser.print_help()
sys.exit(1)
if table_name is None:
table_name = 'ALL'
dbserverconfig = EstimationDatabaseConfiguration(
database_configuration=options.database_configuration)
opusdb = OpusDatabase(dbserverconfig, db_name)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
SimulationState().set_current_time(cache_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logger.log_status('Caching all tables in database...')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
def run(self, config, year, storage_type='sql'):
"""
"""
tm_config = config['travel_model_configuration']
database_server_config = tm_config.get("database_server_configuration", 'simtravel_database_server')
database_name = tm_config.get("database_name", 'mag_zone_baseyear')
cache_directory = config['cache_directory']
simulation_state = SimulationState()
simulation_state.set_cache_directory(cache_directory)
simulation_state.set_current_time(year)
attribute_cache = AttributeCache()
dataset_pool = SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=attribute_cache).get_dataset_pool()
if storage_type == 'sql':
db_server = DatabaseServer(DatabaseConfiguration(
database_name = database_name,
database_configuration = database_server_config
)
)
if not db_server.has_database(database_name):
print "Db doesn't exist creating one"
db_server.create_database(database_name)
db = db_server.get_database(database_name)
output_storage = sql_storage(storage_location = db)
elif storage_type == 'csv':
csv_directory = os.path.join(cache_directory, 'csv', str(year))
output_storage = csv_storage(storage_location=csv_directory)
else:
raise ValueError, "Unsupported output storage type {}".format(storage_type)
logger.start_block('Compute and export data to openAMOS...')
hh = dataset_pool.get_dataset('household')
hh_recs = dataset_pool.get_dataset('households_recs')
#hh_recs.add_attribute(0,"htaz1")
#hh_recs.flush_dataset()
#syn_hh = dataset_pool.get_dataset('synthetic_household')
hh_variables = ['houseid=household.household_id',
"hhsize=household.number_of_agents(person)",
"one=(household.household_id>0).astype('i')",
"inclt35k=(household.income<35000).astype('i')",
"incge35k=(household.income>=35000).astype('i')",
"incge50k=(household.income>=50000).astype('i')",
"incge75k=(household.income>=75000).astype('i')",
"incge100k=(household.income>=100000).astype('i')",
"inc35t50=((household.income>=35000) & (household.income<50000)).astype('i')",
"inc50t75=((household.income>=50000) & (household.income<75000)).astype('i')",
"inc75t100=((household.income>=75000) & (household.income<100000)).astype('i')",
'htaz = ((houseid>0)*(household.disaggregate(building.zone_id)-100) + (houseid<=0)*0)',
#'htaz = ((houseid>0) & (htaz1>100))*(htaz1-100)+((houseid>0) & (htaz1==-1))*1122',
"withchild = (household.aggregate(person.age<18)>0).astype('i')",
"noc = household.aggregate(person.age<18)",
"numadlt = household.aggregate(person.age>=18)",
"hinc=household.income",
"wif=household.workers",
#"wif=household.aggregate(mag_zone.person.is_employed)",
'numwrkr=household.workers',
#'numwrkr=household.aggregate(mag_zone.person.is_employed)',
'nwrkcnt=household.number_of_agents(person) - household.workers',
#'nwrkcnt=household.number_of_agents(person) - household.aggregate(mag_zone.person.is_employed)',
'yrbuilt=mag_zone.household.yrbuilt',
'mag_zone.household.sparent',
'mag_zone.household.rur',
'mag_zone.household.urb',
'zonetid = household.disaggregate(building.zone_id)',
]
self.prepare_attributes(hh, hh_variables)
attrs_to_export = hh_recs.get_known_attribute_names()
self.write_dataset(hh, attrs_to_export, output_storage)
dataset_pool._remove_dataset(hh.dataset_name)
persons = dataset_pool.get_dataset('person')
persons.out_table_name_default = 'persons'
# Recoding invalid work and school locations to some random valid values
persons_recs = dataset_pool.get_dataset('persons_recs')
persons_recs.add_attribute(persons['person_id'],"personuniqueid")
persons_recs.add_attribute(persons['marriage_status'],"marstat")
persons_recs.add_attribute(persons['student_status'],"schstat")
"""
persons_recs.add_attribute(persons['wtaz0'],"htaz_act")
persons_recs.add_attribute(0,"wtaz_rec")
persons_recs.add_attribute(0,"wtaz_rec1")
persons_recs.add_attribute(0,"wtaz_rec2")
persons_recs.add_attribute(0,"wtaz1_1")
persons_recs.add_attribute(0,"wtaz1_2")
persons_recs.add_attribute(0,"wtaz1_3")
#persons_recs.add_attribute(persons['student_status'],"schstat")
"""
persons_recs.add_attribute(0,"wtaz1")
persons_recs.add_attribute(0,"htaz")
persons_recs.add_attribute(0,"schtaz1")
persons_recs.flush_dataset()
#syn_persons = dataset_pool.get_dataset('synthetic_person')
persons_variables = ['personid=mag_zone.person.member_id',
'personuniqueid=person.person_id',
'houseid=person.household_id',
"one=(person.person_id>0).astype('i')",
'trvtime=mag_zone.person.travel_time_from_home_to_work',
'timetowk=mag_zone.person.travel_time_from_home_to_work',
#'mag_zone.person.tmtowrk',
#'tmtowrk=person.disaggregate(synthetic_person.tmtowrk)',
"ag5t10=((person.age>=5) & (person.age<=10)).astype('i')",
"ag11t14=((person.age>=11) & (person.age<=14)).astype('i')",
"ag15t17=((person.age>=15) & (person.age<=17)).astype('i')",
"ag18t24=((person.age>=18) & (person.age<=24)).astype('i')",
"ag25t34=((person.age>=25) & (person.age<=34)).astype('i')",
"ag35t44=((person.age>=35) & (person.age<=44)).astype('i')",
"ag45t54=((person.age>=45) & (person.age<=54)).astype('i')",
"ag55t64=((person.age>=55) & (person.age<=64)).astype('i')",
"agge65=(person.age>=65).astype('i')",
"ag12t17=((person.age>=12) & (person.age<=17)).astype('i')",
"ag5t14=((person.age>=5) & (person.age<=14)).astype('i')",
"agge15=(person.age>=15).astype('i')",
"wrkr=(person.employment_status==1).astype('i')",
"isemploy=(person.employment_status==1).astype('i')",
"fulltim=(mag_zone.person.full_time==1).astype('i')",
'parttim=mag_zone.person.part_time',
'htaz = ((houseid>0)*(person.disaggregate(building.zone_id, intermediates=[household])-100) + (houseid<=0)*0)',
'wtaz1=(person.wtaz <= 0)*0 + (person.wtaz > 0)*(person.wtaz-100)',
"presch = ((person.age < 5)&(houseid>0)).astype('i')",
"mag_zone.person.schstat",
'schtaz1 = (person.schtaz <= 0)*0 + (person.schtaz > 0)*(person.schtaz-100)',
'marstat = person.marriage_status',
'enroll = person.student_status',
'grade = person.student_status & person.education',
'educ = person.education',
"male = (person.sex==1).astype('i')",
"female = (person.sex==2).astype('i')",
"coled = (person.education >= 10).astype('i')",
'race1 = person.race',
"white = (person.race == 1).astype('i')",
'person.hispanic'
]
self.prepare_attributes(persons, persons_variables)
attrs_to_export = persons_recs.get_known_attribute_names()
self.write_dataset(persons, attrs_to_export, output_storage)
dataset_pool._remove_dataset(persons.dataset_name)
zones = dataset_pool.get_dataset('zone')
zones_variables = [
"retail_employment=zone.aggregate(mag_zone.job.sector_group=='retail')",
"public_employment=zone.aggregate(mag_zone.job.sector_group=='public')",
"office_employment=zone.aggregate(mag_zone.job.sector_group=='office')",
"industrial_employment=zone.aggregate(mag_zone.job.sector_group=='individual')",
"other_employment=zone.aggregate(mag_zone.job.sector_group=='other')",
"retail_employment_density=zone.aggregate(mag_zone.job.sector_group=='retail')/zone.acres",
"public_employment_density=zone.aggregate(mag_zone.job.sector_group=='public')/zone.acres",
"office_employment_density=zone.aggregate(mag_zone.job.sector_group=='office')/zone.acres",
"industrial_employment_density=zone.aggregate(mag_zone.job.sector_group=='individual')/zone.acres",
"other_employment_density=zone.aggregate(mag_zone.job.sector_group=='other')/zone.acres",
"total_area=zone.acres",
"lowest_income=zone.aggregate(household.income < scoreatpercentile(household.income, 20))",
"low_income=zone.aggregate(household.income < scoreatpercentile(household.income, 40))",
"high_income=zone.aggregate(household.income > scoreatpercentile(household.income, 80))",
#"institutional_population=zone.disaggregate(locations.institutional_population)",
#"groupquarter_households=zone.disaggregate(locations.groupquarter_households)",
"residential_households=zone.number_of_agents(household)",
"locationid=zone.zone_id",
]
locations = dataset_pool['locations']
self.prepare_attributes(zones, zones_variables, dataset2=locations)
attrs_to_export = locations.get_known_attribute_names()
self.write_dataset(locations, attrs_to_export, output_storage)
dataset_pool._remove_dataset(locations.dataset_name)
#raw_input("check location block")
logger.end_block()
sys.exit(1)
table_name = options.table_name
logger.log_status('Initializing database...')
db_server = DatabaseServer(
EstimationDatabaseConfiguration(
database_name=database_name,
database_configuration=options.database_configuration))
if not db_server.has_database(
database_name): # if only one table should be exported,
db_server.create_database(database_name) # the database can exist
db = db_server.get_database(database_name)
input_storage = flt_storage(storage_location=cache_path)
output_storage = sql_storage(storage_location=db)
logger.start_block('Exporting cache to sql...')
try:
if table_name is None:
ExportStorage().export(in_storage=input_storage,
out_storage=output_storage)
else:
db.drop_table(table_name)
ExportStorage().export_dataset(table_name,
in_storage=input_storage,
out_storage=output_storage)
finally:
logger.end_block()
cache_path = options.cache_path
database_name = options.database_name
if database_name is None or cache_path is None:
parser.print_help()
sys.exit(1)
table_name = options.table_name
logger.log_status('Initializing database...')
db_server = DatabaseServer(EstimationDatabaseConfiguration(
database_name = database_name,
database_configuration = options.database_configuration
)
)
if not db_server.has_database(database_name): # if only one table should be exported,
db_server.create_database(database_name) # the database can exist
db = db_server.get_database(database_name)
input_storage = flt_storage(storage_location = cache_path)
output_storage = sql_storage(
storage_location = db)
with logger.block('Exporting cache to sql...'):
if table_name is None:
ExportStorage().export(in_storage=input_storage, out_storage=output_storage)
else:
db.drop_table(table_name)
ExportStorage().export_dataset(table_name, in_storage=input_storage, out_storage=output_storage)
