def setUp(self):
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp_test_indicators')
storage = StorageFactory().get_storage('dict_storage')
tables_data = {
'large_areas': {
"large_area_id":array([1, 2, 3])
},
'fazes': {
"faz_id":array([1,2,3,4]),
"large_area_id":array([1,2,3,3]),
},
'zones': {
"zone_id":array([1,2,3,4,5]),
"faz_id":array([1,2,2,3,4]),
},
'gridcells': {
"grid_id":array([1,2,3,4,5,6,7,8,9]),
"zone_id":array([1,1,1,2,2,3,3,4,5]),
"total_land_value":array([10, 11, 12, 13, 14, 15, 16, 17, 18]),
"is_in_plan_type_group_residential":array([1, 1, 1, 1, 0, 0, 1, 0, 1])
},
'urbansim_constants':{
'acres': array([1.5]),
},
}
self.year = 1000
SimulationState().set_current_time(self.year)
exporter = ExportStorage()
for table_name, table_data in tables_data.iteritems():
storage.write_table(table_name=table_name, table_data=table_data)
exporter.export_dataset(dataset_name=table_name,
in_storage=storage,
out_storage=AttributeCache(cache_directory=self.temp_dir))
def setUp(self):
self.temp_dir = tempfile.mkdtemp(prefix="opus_tmp_test_indicators")
storage = StorageFactory().get_storage("dict_storage")
tables_data = {
"large_areas": {"large_area_id": array([1, 2, 3])},
"fazes": {"faz_id": array([1, 2, 3, 4]), "large_area_id": array([1, 2, 3, 3])},
"zones": {"zone_id": array([1, 2, 3, 4, 5]), "faz_id": array([1, 2, 2, 3, 4])},
"gridcells": {
"grid_id": array([1, 2, 3, 4, 5, 6, 7, 8, 9]),
"zone_id": array([1, 1, 1, 2, 2, 3, 3, 4, 5]),
"total_land_value": array([10, 11, 12, 13, 14, 15, 16, 17, 18]),
"is_in_plan_type_group_residential": array([1, 1, 1, 1, 0, 0, 1, 0, 1]),
},
"urbansim_constants": {"acres": array([1.5])},
}
self.year = 1000
SimulationState().set_current_time(self.year)
exporter = ExportStorage()
for table_name, table_data in tables_data.iteritems():
storage.write_table(table_name=table_name, table_data=table_data)
exporter.export_dataset(
dataset_name=table_name, in_storage=storage, out_storage=AttributeCache(cache_directory=self.temp_dir)
)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
csv_data_path = params_dict['csv_data_path']
table_name = params_dict['csv_table_name']
input_storage = csv_storage(storage_location = csv_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
else:
lst = [table_name]
for i in lst:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage)
logCB("Successfully exported all tables.")
def run (self):
time = -1
latest = ""
directoryname = 'data/vibe_gridcell/runs/'
if self.isParcel is True:
directoryname = 'data/vibe_parcel/'
for filename in os.listdir(os.path.join(os.environ['OPUS_HOME'], directoryname)):
print filename
if time == -1:
time = os.path.getmtime(os.path.join(os.environ['OPUS_HOME'], directoryname, filename))
latest = filename
if os.path.getmtime(os.path.join(os.environ['OPUS_HOME'], directoryname, filename)) > time:
time = os.path.getmtime(os.path.join(os.environ['OPUS_HOME'], directoryname, filename))
latest = filename
config = DatabaseServerConfiguration(host_name = 'localhost',
user_name = 'urbansim',
password = '******',
protocol = 'mysql')
db_server = DatabaseServer(config)
for i in range(1981, 1980+int(self.YearsToRun)):
newdir = latest + '/' + str(i)
flt_directory_in = os.path.join(os.environ['OPUS_HOME'], directoryname ,newdir)
input_storage = flt_storage(storage_location = flt_directory_in)
db = db_server.get_database('ress_'+str(i))
output_storage = StorageFactory().get_storage('sql_storage', storage_location = db)
ExportStorage().export(in_storage=input_storage, out_storage=output_storage)
def export(self):
''' Run export process
'''
output_storage = csv_storage(storage_location = self.csv_data_path)
ExportStorage().export_dataset(
dataset_name = 'travel_data',
in_storage = self.input_storage,
out_storage = output_storage,
)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
else:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(table_name, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(dataset_name=table_name,
in_storage=input_storage,
out_storage=output_storage)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_directory = paths.prepend_opus_home_if_relative(
opus_data_directory)
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
overwrite = params_dict['overwrite']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name,
create_if_doesnt_exist=False)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
lst = input_storage.get_table_names()
else:
lst = re.split(' +', table_name.strip())
tables = len(lst)
lst_out = create_list_string(lst, ', ')
logCB('caching tables:\n%s\n' % lst_out)
for j, i in enumerate(lst, start=1):
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
overwrite=overwrite,
)
progressCB(100 * j / tables)
logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output csv data path
csv_data_path = params_dict['csv_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
execute_after_export = params_dict['execute_after_export']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
output_storage = csv_storage(storage_location = csv_data_path)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
logCB("Successfully exported all datasets.")
if execute_after_export:
file_name_list = [output_storage._get_file_path_for_table(i)
for i in opus_table_name_list]
subprocess.Popen([execute_after_export] + file_name_list)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
esri_data_path = params_dict['esri_data_path']
esri_table_name = params_dict['esri_table_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
input_storage = esri_storage(storage_location=esri_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if esri_table_name == 'ALL':
logCB("Sending all tables to OPUS storage...\n")
lst = input_storage.get_table_names()
for i in lst:
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
else:
logCB("Exporting table '%s' to OPUS storage located at %s...\n" %
(esri_table_name, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=esri_table_name,
in_storage=input_storage,
out_storage=output_storage,
)
logCB("Finished exporting table '%s'\n" % (esri_table_name))
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
database_name = params_dict['database_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
opus_table_name = params_dict['opus_table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
#input_storage = sql_storage(storage_location = opusdb)
input_storage = attribute_cache.get_flt_storage_for_year(year)
#output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
if opus_data_year == 'ALL':
opusdb = server.get_database(database_name=database_name + "_" +
str(year))
output_storage = sql_storage(storage_location=opusdb)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = re.split(' +', opus_table_name.strip())
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output esri data path
esri_data_path = params_dict['esri_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
if esri_is_avail:
output_storage = esri_storage(storage_location=esri_data_path)
else:
output_storage = None
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
attribute_cache_directory = options.attribute_cache_directory
table_name = options.table_name
cache_year = options.cache_year
if (tab_directory is None or attribute_cache_directory is None
or table_name is None or cache_year is None):
parser.print_help()
sys.exit(1)
input_storage = tab_storage(storage_location=tab_directory)
attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
SimulationState().set_current_time(cache_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
logger.start_block(
"Exporting table '%s' to year %s of cache located at %s..." %
(table_name, cache_year, attribute_cache_directory))
try:
ExportStorage().export_dataset(
dataset_name=table_name,
in_storage=input_storage,
out_storage=output_storage,
)
finally:
logger.end_block()
def _get_exporter(self):
"""hook for unit tests"""
if self._exporter is None:
# Create default exporter object.
self._exporter = ExportStorage()
return self._exporter
parser = OptionParser()
parser.add_option('-c', '--cache_path', dest='cache_path', type='string',
help='The filesystem path to the cache to export (required)')
parser.add_option('-o', '--output_directory', dest='output_directory',
type='string', help='The filesystem path of the database to which '
'output will be written (required)')
parser.add_option('-t', '--table_name', dest='table_name',
type='string', help='Name of table to be exported (optional). Used if only one table should be exported.')
(options, args) = parser.parse_args()
cache_path = options.cache_path
output_directory = options.output_directory
table_name = options.table_name
if None in (cache_path, output_directory):
parser.print_help()
sys.exit(1)
in_storage = flt_storage(storage_location = cache_path)
out_storage = tab_storage(storage_location = output_directory)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
if table_name is not None:
ExportStorage().export_dataset(table_name, in_storage=in_storage, out_storage=out_storage)
else:
ExportStorage().export(in_storage=in_storage, out_storage=out_storage)
)
parser.add_option(
'--compression',
dest='compression',
type='string',
default=None,
help=
'Compression type. Available: gzip, lzf. Default is no compression.')
(options, args) = parser.parse_args()
cache_path = options.cache_path
output_file = options.output_file
table_name = options.table_name
if None in (cache_path, output_file):
parser.print_help()
sys.exit(1)
in_storage = flt_storage(storage_location=cache_path)
out_storage = hdf5g_storage(storage_location=output_file)
if table_name is not None:
ExportStorage().export_dataset(table_name,
in_storage=in_storage,
out_storage=out_storage,
compression=options.compression)
else:
ExportStorage().export(in_storage=in_storage,
out_storage=out_storage,
compression=options.compression)