def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
csv_data_path = params_dict['csv_data_path']
table_name = params_dict['csv_table_name']
input_storage = csv_storage(storage_location = csv_data_path)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
logCB('caching all tables...\n')
lst = input_storage.get_table_names()
else:
lst = [table_name]
for i in lst:
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage)
logCB("Successfully exported all tables.")
def _compute_variable_for_prior_year(self,
dataset,
full_name,
time,
resources=None):
"""Create a new dataset for this variable, compute the variable, and then return
the values for this variable."""
calling_dataset_pool = SessionConfiguration().get_dataset_pool()
calling_time = SimulationState().get_current_time()
SimulationState().set_current_time(time)
try:
# Get an empty dataset pool with same search paths.
my_dataset_pool = DatasetPool(
package_order=calling_dataset_pool.get_package_order(),
storage=AttributeCache())
ds = dataset.empty_dataset_like_me(in_storage=AttributeCache())
# Don't pass any datasets via resources, since they may be from a different time.
my_resources = Resources(resources)
for key in my_resources:
if isinstance(key, Dataset):
del my_resources[key]
ds.compute_variables(full_name,
my_dataset_pool,
resources=my_resources)
values = ds.get_attribute(full_name)
return values
finally:
SimulationState().set_current_time(calling_time)
def save_results(self, out_storage=None, model_name=None):
if self.specification is None or self.coefficients is None:
raise ValueError, "model specification or coefficient is None"
#invalid = self.coefficients.is_invalid()
if False:
logger.log_warning('Invalid coefficients. Not saving results!')
return
if model_name is None:
model_name = self.config.get('model_name_for_coefficients', None)
if model_name is None:
if self.model_name is not None:
model_name = self.model_name
else:
raise ValueError, "model_name unspecified"
out_storage_available = True
if out_storage:
pass
elif 'estimation_database_configuration' in self.config:
try:
db_server = DatabaseServer(self.config['estimation_database_configuration'])
database_name = self.config["estimation_database_configuration"].database_name
if not db_server.has_database(database_name):
db_server.create_database(database_name)
output_db = db_server.get_database(database_name)
out_storage = StorageFactory().get_storage(
type='sql_storage',
storage_location=output_db)
except:
logger.log_warning("Problem with connecting database given by 'estimation_database_configuration'.")
out_storage_available = False
else:
logger.log_warning("No estimation_database_configuration given.")
out_storage_available = False
# the original model name of development_project_lcm is too long as a mysql db table name, truncate it
if model_name.rfind("_development_project_location_choice_model") >=0:
model_name = model_name.replace('_project', '')
specification_table = '%s_specification' % model_name
coefficients_table = '%s_coefficients' % model_name
if out_storage_available:
logger.start_block("Writing specification and coefficients into storage given by 'estimation_database_configuration'")
self.specification.write(out_storage=out_storage, out_table_name=specification_table)
self.coefficients.write(out_storage=out_storage, out_table_name=coefficients_table)
logger.end_block()
logger.start_block("Writing specification and coefficients into %s" % AttributeCache().get_storage_location())
self.specification.write(out_storage=AttributeCache(), out_table_name=specification_table)
self.coefficients.write(out_storage=AttributeCache(), out_table_name=coefficients_table)
logger.end_block()
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
opus_data_directory = params_dict['opus_data_directory']
opus_data_directory = paths.prepend_opus_home_if_relative(
opus_data_directory)
opus_data_year = params_dict['opus_data_year']
database_name = params_dict['database_name']
table_name = params_dict['table_name']
database_server_connection = params_dict['database_server_connection']
overwrite = params_dict['overwrite']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name,
create_if_doesnt_exist=False)
input_storage = sql_storage(storage_location=opusdb)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
SimulationState().set_current_time(opus_data_year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if table_name == 'ALL':
lst = input_storage.get_table_names()
else:
lst = re.split(' +', table_name.strip())
tables = len(lst)
lst_out = create_list_string(lst, ', ')
logCB('caching tables:\n%s\n' % lst_out)
for j, i in enumerate(lst, start=1):
logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
(i, opus_data_year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
overwrite=overwrite,
)
progressCB(100 * j / tables)
logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB,logCB,params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output csv data path
csv_data_path = params_dict['csv_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
execute_after_export = params_dict['execute_after_export']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
output_storage = csv_storage(storage_location = csv_data_path)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory))
ExportStorage().export_dataset(
dataset_name = i,
in_storage = input_storage,
out_storage = output_storage,
)
logCB("Successfully exported all datasets.")
if execute_after_export:
file_name_list = [output_storage._get_file_path_for_table(i)
for i in opus_table_name_list]
subprocess.Popen([execute_after_export] + file_name_list)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
database_name = params_dict['database_name']
opus_data_directory = params_dict['opus_data_directory']
opus_data_year = params_dict['opus_data_year']
opus_table_name = params_dict['opus_table_name']
database_server_connection = params_dict['database_server_connection']
dbs_config = DatabaseServerConfiguration(
database_configuration=database_server_connection)
server = DatabaseServer(database_server_configuration=dbs_config)
opusdb = server.get_database(database_name=database_name)
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
#input_storage = sql_storage(storage_location = opusdb)
input_storage = attribute_cache.get_flt_storage_for_year(year)
#output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
if opus_data_year == 'ALL':
opusdb = server.get_database(database_name=database_name + "_" +
str(year))
output_storage = sql_storage(storage_location=opusdb)
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = re.split(' +', opus_table_name.strip())
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
def opusRun(progressCB, logCB, params):
params_dict = {}
for key, val in params.iteritems():
params_dict[str(key)] = str(val)
# Output esri data path
esri_data_path = params_dict['esri_data_path']
# Data clasification - Database (must be specified)
opus_data_directory = params_dict['opus_data_directory']
# Data clasification - Dataset (explicit or ALL)
opus_data_year = params_dict['opus_data_year']
# Data clasification - Array (explicit or ALL)
opus_table_name = params_dict['opus_table_name']
attribute_cache = AttributeCache(cache_directory=opus_data_directory)
attribute_cache_years = [
int(year) for year in os.listdir(opus_data_directory)
if year.isdigit() and len(year) == 4
]
if opus_data_year != 'ALL':
attribute_cache_years = [opus_data_year]
for year in attribute_cache_years:
input_storage = attribute_cache.get_flt_storage_for_year(year)
if esri_is_avail:
output_storage = esri_storage(storage_location=esri_data_path)
else:
output_storage = None
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=[],
in_storage=AttributeCache())
if opus_table_name != 'ALL':
opus_table_name_list = [opus_table_name]
else:
opus_table_name_list = input_storage.get_table_names()
for i in opus_table_name_list:
logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory))
ExportStorage().export_dataset(
dataset_name=i,
in_storage=input_storage,
out_storage=output_storage,
)
def setUp(self):
run_configuration = TestCacheConfiguration()
SimulationState(new_instance=True)
SessionConfiguration(run_configuration,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = run_configuration['base_year']
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Use the test cache.
opus_core_path = package().get_opus_core_path()
test_cache_path = os.path.join(opus_core_path, 'data', 'test_cache')
new_cache_path = os.path.join(self.temp_dir, 'cache')
copytree(test_cache_path, new_cache_path)
# Make sure the copied files are writable.
for (dirpath, dirnames, filenames) in os.walk(new_cache_path):
for file_name in filenames:
full_path = os.path.join(dirpath, file_name)
os.chmod(full_path, S_IWRITE | S_IREAD)
SimulationState().set_cache_directory(new_cache_path)
SimulationState().set_current_time(self.base_year)
self.config = Resources(run_configuration)
cache_directory = SimulationState().get_cache_directory()
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
def _make_all_indicators(self, indicators, source_data):
computed_indicators = {}
indicators_by_dataset = {}
for year in source_data.years:
SimulationState().set_current_time(year)
SessionConfiguration(new_instance=True,
package_order=self.package_order,
in_storage=AttributeCache())
for name, indicator in indicators.items():
dataset_name = indicator.dataset_name
if dataset_name not in indicators_by_dataset:
indicators_by_dataset[dataset_name] = [(name, indicator)]
else:
indicators_by_dataset[dataset_name].append(
(name, indicator))
for dataset_name, indicators_in_dataset in indicators_by_dataset.items(
):
dataset = SessionConfiguration().get_dataset_from_pool(
dataset_name)
self._make_indicators_for_dataset(
dataset=dataset,
indicators_in_dataset=indicators_in_dataset,
source_data=source_data,
computed_indicators=computed_indicators,
year=year)
self.computed_indicators[source_data.name] = computed_indicators
return computed_indicators
def get_needed_matrices_from_emme2(self,
year,
cache_directory,
bank_dir,
matrix_variable_map,
matrices_created=False):
"""Copies the specified emme/2 matrices into the specified travel_data variable names.
"""
logger.start_block('Getting matricies from emme2')
try:
zone_set = SessionConfiguration().get_dataset_from_pool('zone')
zone_set.load_dataset()
travel_data_set = self.get_travel_data_from_emme2(
zone_set, bank_dir, matrix_variable_map, matrices_created)
finally:
logger.end_block()
logger.start_block('Writing data to cache')
try:
next_year = year + 1
out_storage = AttributeCache().get_flt_storage_for_year(next_year)
travel_data_set.write_dataset(attributes='*',
out_storage=out_storage,
out_table_name='travel_data')
finally:
logger.end_block()
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
# print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them"
# print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id"
# solved 3dec08 by hana
input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim",
"tmp")
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def setUp(self):
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp_test_indicators')
storage = StorageFactory().get_storage('dict_storage')
tables_data = {
'large_areas': {
"large_area_id":array([1, 2, 3])
},
'fazes': {
"faz_id":array([1,2,3,4]),
"large_area_id":array([1,2,3,3]),
},
'zones': {
"zone_id":array([1,2,3,4,5]),
"faz_id":array([1,2,2,3,4]),
},
'gridcells': {
"grid_id":array([1,2,3,4,5,6,7,8,9]),
"zone_id":array([1,1,1,2,2,3,3,4,5]),
"total_land_value":array([10, 11, 12, 13, 14, 15, 16, 17, 18]),
"is_in_plan_type_group_residential":array([1, 1, 1, 1, 0, 0, 1, 0, 1])
},
'urbansim_constants':{
'acres': array([1.5]),
},
}
self.year = 1000
SimulationState().set_current_time(self.year)
exporter = ExportStorage()
for table_name, table_data in tables_data.iteritems():
storage.write_table(table_name=table_name, table_data=table_data)
exporter.export_dataset(dataset_name=table_name,
in_storage=storage,
out_storage=AttributeCache(cache_directory=self.temp_dir))
def test_compute_a_variable(self):
"""Return merged dataset for this set of years."""
test_data = {
1000: {
'tests': {
'id': array([1, 2, 3]),
'attr1': array([10, 20, 30]),
},
},
1001: {
'tests': {
'id': array([1, 2, 3]),
'attr1': array([40, 50, 60]),
},
},
}
cache_creator = CreateTestAttributeCache()
cache_creator.create_attribute_cache_with_data(self.temp_dir,
test_data)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=['opus_core'],
in_storage=attribute_cache)
ds = MultipleYearDatasetView(
name_of_dataset_to_merge='test',
in_table_name='tests',
years_to_merge=[1000, 1001],
attribute_cache=attribute_cache,
)
ds.compute_variables(['opus_core.test.attr1_times_2'])
def prepare_for_running_macro(parser):
from opus_core.file_utilities import get_resources_from_file
parser.add_option("-r",
"--resources",
dest="resources_file_name",
action="store",
type="string",
help="Name of file containing resources")
parser.add_option("-y",
"--year",
dest="year",
action="store",
type="int",
help="Year in which to 'run' the travel model")
parser.add_option(
"-o",
"--output-file",
dest="output_file",
action="store",
type="string",
default=None,
help=
"Output log file. If not given, it is written into urbansim cache directory."
)
(options, args) = parser.parse_args()
r = get_resources_from_file(options.resources_file_name)
resources = Resources(get_resources_from_file(options.resources_file_name))
SessionConfiguration(
new_instance=True,
package_order=resources['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
return (resources, options)
def _do_run_simple_test_run(caller, temp_dir, config, end_year=None):
"""Runs model system with a single model (for speed).
Sets the .resources property of the caller before starting the run.
"""
runs_manager = RunManager(config)
run_configuration = _get_run_config(temp_dir=temp_dir)
insert_auto_generated_cache_directory_if_needed(run_configuration)
run_configuration[
'creating_baseyear_cache_configuration'].cache_directory_root = temp_dir
run_configuration['models'] = ['land_price_model']
if end_year is not None:
run_configuration['years'] = (run_configuration['years'][0], end_year)
SessionConfiguration(
new_instance=True,
package_order=run_configuration['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
insert_auto_generated_cache_directory_if_needed(run_configuration)
caller.resources = run_configuration
runs_manager.setup_new_run(
cache_directory=run_configuration['cache_directory'],
configuration=run_configuration)
runs_manager.run_run(run_configuration)
def unroll_gridcells_to_cache_from_buildings(self, gridcells, buildings,
cache_directory, base_year):
"""Populate the cache with the unrolled gridcells info derived
from the buildings table.
"""
logger.start_block('Unrolling gridcell data from buildings')
try:
storage = AttributeCache().get_flt_storage_for_year(base_year)
urbansim_constant = SessionConfiguration().get_dataset_from_pool('urbansim_constant')
print "recent_years = %s" % urbansim_constant['recent_years']
recent_years = urbansim_constant['recent_years']
roller = RollbackGridcellsFromBuildings()
for year in range(base_year, base_year-recent_years-1, -1):
logger.start_block('Unrolling gridcells into year %d' % (year-1))
try:
roller.unroll_gridcells_for_one_year(gridcells,
buildings,
year,
dataset_pool=SessionConfiguration().get_dataset_pool())
flt_directory = os.path.join(cache_directory, str(year-1))
flt_storage = StorageFactory().get_storage(
type='flt_storage', subdir='store',
storage_location=flt_directory)
gridcells.write_dataset(out_storage=flt_storage)
finally:
logger.end_block()
finally:
logger.end_block()
def test_at_year_2000(self):
cache_dir = os.path.join(self.temp_dir, 'cache')
data = {
self._id_name: array([1, 2, 3]),
'population': array([10, 20, 30]),
}
self._write_data_to_year(data, cache_dir, 2000)
attribute_cache = AttributeCache(cache_directory=cache_dir)
SimulationState(new_instance=True,
base_cache_dir=self.temp_dir)
SimulationState().set_cache_directory(cache_dir)
SessionConfiguration(new_instance=True,
in_storage=attribute_cache)
SimulationState().set_current_time(2000)
dataset_pool_2000 = DatasetPool(package_order=['urbansim'],
storage=attribute_cache)
dataset = dataset_pool_2000.get_dataset(self._dataset_name)
variable_name = '%s.%s.percent_population_difference_from_2000' % (
self._package_name, self._dataset_name)
dataset.compute_variables([variable_name],
dataset_pool=dataset_pool_2000)
pop_2000 = dataset.get_attribute(variable_name)
self.assert_(ma.allequal(pop_2000, array([0, 0, 0])))
def prepare_for_simulation(self, config, cache_directory=None):
self.config = Resources(config)
base_cache_dir = self.config[
'creating_baseyear_cache_configuration'].cache_directory_root
self.simulation_state = SimulationState(new_instance=True,
base_cache_dir=base_cache_dir,
start_time=self.config.get(
'base_year', 0))
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config[
'cache_directory'] = self.simulation_state.get_cache_directory(
)
SessionConfiguration(
new_instance=True,
package_order=self.config['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
if config['creating_baseyear_cache_configuration'].cache_from_database:
ForkProcess().fork_new_process(
self.config['creating_baseyear_cache_configuration'].
cache_scenario_database, self.config)
else:
CacheFltData().run(self.config)
def test(self):
opus_core_path = OpusPackage().get_opus_core_path()
dbf_directory = os.path.join(opus_core_path, 'tests', 'data',
'dbf')
table_name = 'test_logical'
cache_directory = self._temp_dir
year = 1000
exporter = ExportDbfTableToCacheCommand(
dbf_directory=dbf_directory,
table_name=table_name,
cache_directory=cache_directory,
year=year,
)
exporter.execute()
attribute_cache = AttributeCache(cache_directory=cache_directory)
old_time = SimulationState().get_current_time()
SimulationState().set_current_time(year)
values = attribute_cache.load_table(table_name)
self.assertEqual(set(['keyid', 'works']), set(values.keys()))
self.assert_(ma.allequal(array([1, 2, 3, 4, 5]), values['keyid']))
self.assert_(ma.allequal(array([1, 1, -1, 0, 0]), values['works']))
SimulationState().set_current_time(old_time)
def test(self):
urbansim_constant = UrbansimConstantDataset(
in_storage=AttributeCache())
self.assertEqual(urbansim_constant['absolute_min_year'], 1800)
self.assertAlmostEqual(urbansim_constant['acres'],
150 * 150 * 0.0002471, 6)
self.assert_(urbansim_constant["walking_distance_footprint"].ndim == 2)
def prepare_for_simulation(self, run_configuration, cache_directory=None):
self.config = Resources(run_configuration)
self.simulation_state = SimulationState(new_instance=True,
base_cache_dir=cache_directory)
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config[
'cache_directory'] = self.simulation_state.get_cache_directory(
)
SessionConfiguration(
new_instance=True,
package_order=self.config['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
ForkProcess().fork_new_process(
self.config['creating_baseyear_cache_configuration'].
cache_scenario_database, self.config)
# Create output database (normally done by run manager)
if 'estimation_database_configuration' in self.config:
db_server = DatabaseServer(
self.config['estimation_database_configuration'])
if not db_server.has_database(
self.config['estimation_database_configuration'].
database_name):
db_server.create_database(
self.config['estimation_database_configuration'].
database_name)
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
input_directory = os.path.join(config['root'], 'opus_matsim', 'tmp')
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def _set_cache_directory(self, cache_directory):
if cache_directory != SimulationState().get_cache_directory():
SimulationState().set_cache_directory(cache_directory)
SessionConfiguration(
new_instance=True,
package_order=self.dataset_pool_configuration.package_order,
in_storage=AttributeCache())
def compare_travel_data_sets(self):
# get copied travel data csv
copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp')
if not os.path.exists(copied_travel_data_location):
raise StandardError('Travel data not found: %s' % copied_travel_data_location)
logger.log_status('Get copied travel data: %s' % copied_travel_data_location)
# convert travel data csv into travel data set matrix
in_storage = csv_storage(storage_location = copied_travel_data_location)
table_name = "travel_data"
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# get exsisting travel data set and convert it also into travel data set matrix
year = self.run_config['base_year']+2
attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory'])
cache_storage = attribute_cache.get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
from numpy import savetxt # for debugging
savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f")
savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f")
# compare both data set matices
compare = travel_data_attribute_mat == existing_travel_data_attribute_mat
# return result
return compare.all()
def _check_dataset_methods_on_dataset_view(self, ds, years_to_merge):
self.assert_(ds is not None)
ds.load_dataset(attributes='*',
in_table_name='tests',
in_storage=AttributeCache())
id = ds.get_attribute('id')
attr1 = ds.get_attribute('attr1')
# Does compute_variables work?
ds.compute_variables(['opus_core.test.attr1_times_2'])
attr1_times_2 = ds.get_attribute('attr1_times_2')
# Are values as expected?
self.assert_(ma.allequal(attr1 * 2, attr1_times_2))
# Does results have expected number of elements?
self.assertEqual(len(years_to_merge) * 3, len(attr1_times_2))
# Does _compute_if_needed work?
ds._compute_if_needed(
'opus_core.test.attr2_times_2',
dataset_pool=SessionConfiguration().get_dataset_pool())
attr2_times_2 = ds.get_attribute('attr2_times_2')
attr2 = ds.get_attribute('attr2')
self.assert_(ma.allequal(attr2 * 2, attr2_times_2))
def __init__(self, config):
ss = SimulationState(new_instance=True)
ss.set_current_time(config['base_year'])
ss.set_cache_directory(config['cache_directory'])
SessionConfiguration(new_instance=True,
package_order=config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
#if not os.path.exists(config['cache_directory']): ## if cache exists, it will automatically skip
cacher = CreateBaseyearCache()
cache_dir = cacher.run(config)
if 'estimation_database_configuration' in config:
db_server = DatabaseServer(config['estimation_database_configuration'])
db = db_server.get_database(config['estimation_database_configuration'].database_name)
out_storage = StorageFactory().get_storage(
'sql_storage',
storage_location = db)
else:
output_cache = os.path.join(config['cache_directory'], str(config['base_year']+1))
out_storage = StorageFactory().get_storage('flt_storage', storage_location=output_cache)
dataset_pool = SessionConfiguration().get_dataset_pool()
households = dataset_pool.get_dataset("household")
buildings = dataset_pool.get_dataset("building")
zones = dataset_pool.get_dataset("zone")
zone_ids = zones.get_id_attribute()
capacity_attribute_name = "residential_units" #_of_use_id_%s" % id
capacity_variable_name = "%s=sanfrancisco.zone.aggregate_%s_from_building" % \
(capacity_attribute_name, capacity_attribute_name)
buildings.compute_variables("sanfrancisco.building.zone_id", dataset_pool=dataset_pool)
zones.compute_variables(capacity_variable_name, dataset_pool=dataset_pool)
building_zone_id = buildings.get_attribute('zone_id')
# is_household_unplace = datasets['household'].get_attribute("building_id") <= 0
is_household_unplaced = 1 #all households are unplaced
household_building_id = zeros(households.size(), dtype='int32')-1 #datasets['household'].get_attribute("building_id")
for zone_id in zone_ids:
capacity = zones.get_attribute_by_id(capacity_attribute_name, zone_id)
is_household_in_this_zone = (households.get_attribute('zone_id') == zone_id)
is_unplaced_household_in_this_zone = is_household_in_this_zone * is_household_unplaced
is_building_in_this_zone = (building_zone_id == zone_id)
# if not is_household_in_this_zone.sum() <= capacity:
if capacity == 0 or is_household_in_this_zone.sum()==0:
print "WARNING: zone %s has %s households but only %s units" % (zone_id, is_household_in_this_zone.sum(), capacity)
continue
prob = buildings.get_attribute(capacity_attribute_name) * is_building_in_this_zone / array(capacity, dtype=float64)
r = random(sum(is_unplaced_household_in_this_zone))
prob_cumsum = ncumsum(prob)
index_to_bldg = searchsorted(prob_cumsum, r)
household_building_id[where(is_unplaced_household_in_this_zone)] = buildings.get_attribute_by_index('building_id', index_to_bldg)
# import pdb;pdb.set_trace()
households.set_values_of_one_attribute('building_id', household_building_id)
households.write_dataset(out_table_name='households', out_storage=out_storage)
def run(self, location_set, development_event_set, *args, **kwargs):
changed_indices, processed_development_event_indices = \
EventsCoordinator.run(self, location_set,
development_event_set, *args, **kwargs)
if development_event_set is not None:
subset = DatasetSubset(development_event_set,
processed_development_event_indices)
subset.write_dataset(out_storage=AttributeCache())
return (changed_indices, processed_development_event_indices)
def flush_datasets_after_model(self, resources):
if resources.get('flush_variables', False):
AttributeCache().delete_computed_tables()
# this will also delete computed attributes
datasets_to_cache = SessionConfiguration().get_dataset_pool(
).datasets_in_pool().keys()
else:
datasets_to_cache = resources.get(
"datasets_to_cache_after_each_model", [])
self.flush_datasets(datasets_to_cache, after_model=True)
def test(self):
# Set up a test cache
storage = AttributeCache(cache_directory=self._temp_dir)
SimulationState().set_current_time(2000)
table_name = 'foo'
values = {
'attribute1': array([1, 2, 3], dtype=int32),
'attribute2': array([4, 5, 6], dtype=int32),
}
storage.write_table(table_name, values)
table_dir = os.path.join(self._temp_dir, '2000', table_name)
self.assert_(os.path.exists(table_dir))
actual = set(os.listdir(table_dir))
expected = set([
'attribute1.%(endian)si4' % replacements,
'attribute2.%(endian)si4' % replacements
])
self.assertEqual(expected, actual)
exporter = ExportCacheToDbfTableCommand(
cache_directory=self._temp_dir,
year='2000',
table_name=table_name,
dbf_directory=self._temp_dir,
decimalcount=4,
)
exporter.execute()
out_storage = dbf_storage(self._temp_dir)
db = _dbf_class(out_storage._get_file_path_for_table(table_name))
length = max([len(values[key]) for key in values.keys()])
i = 0
field_type = {}
for name, type in [
field.fieldInfo()[:2] for field in db.header.fields
]:
field_type[name] = type
for rec in db:
for key in values.keys():
if field_type[key.upper()] is 'F':
self.assertAlmostEqual(values[key][i], rec[key], 4)
else:
self.assertEqual(values[key][i], rec[key])
i = i + 1
self.assertEquals(
length,
i,
msg="More values expected than the dbf file contains")
db.close()
def __init__(self, package_name, attribute_name, lag_offset, dataset_name,
index_name):
self.package_name = package_name
self.attribute_name = attribute_name
self.simulation_state = SimulationState()
self.attribute_cache = AttributeCache()
self.lag_offset = lag_offset
self.dataset_name = dataset_name
self.index_name = index_name
self.lag_index_name = "%s_lag%d" % (index_name, lag_offset)
Variable.__init__(self)