def run(self, config, year):
""" This class simulates a MATSim run. Therefore it copies
real travel data into the OPUS_HOME and modifies the
entries in the following runs.
"""
logger.start_block("Starting RunDummyTravelTimeTravelModel.run(...)")
self.config = config
# get travel model parameter from the opus dictionary
self.travel_model_configuration = config['travel_model_configuration']
self.first_year = 2001 # TODO make configurable (bayeyear + 1)
# set output directory for travel data
self.travel_data_dir = paths.get_opus_home_path( "opus_matsim", "tmp" )
# for debugging
#try: #tnicolai
# import pydevd
# pydevd.settrace()
#except: pass
# set travel data for test simulation
if year == self.first_year:
logger.log_status('Exporting travel_data from base_year_cache to %s' % self.travel_data_dir)
self.export_travel_data(None)
logger.log_status("Modifying travel data.")
self.modify_travel_data()
logger.log_status("Finished modifying...")
else:
logger.log_status("Travel data was modified before. Nothing to do...")
logger.end_block()
def _write_workplaces_to_files(self, person_set, tm_input_files):
home_zones = person_set.get_attribute("zone_id")
job_zones = person_set.get_attribute("job_zone_id")
igroup = 0
for tm_file in tm_input_files:
logger.start_block("Writing to emme2 input file: " +
tm_input_files[igroup])
try:
newfile = open(tm_input_files[igroup], 'w')
try:
newfile.write(r"""c prepared: %s
t matrices
m matrix=mf9%s default=incr
""" % (time.strftime("%c", time.localtime(time.time())), igroup + 1))
line_template = " %3d %3d 1 \n"
person_idx = where(
logical_and(
person_set.get_attribute(
"is_placed_non_home_based_worker_with_job"),
person_set.get_attribute("income_group_%s" %
(igroup + 1))))[0]
for i in person_idx:
newfile.write(line_template %
(home_zones[i], job_zones[i]))
finally:
newfile.close()
finally:
logger.end_block()
igroup += 1
return tm_input_files
def cache_database_table(self, table_name, base_year, database, in_storage, config):
"""Copy this table from input database into attribute cache.
"""
logger.start_block('Caching table %s' % table_name)
try:
#TODO: why is the config being modified...seems like its kind of useless here...
config['storage_location'] = os.path.join(config['cache_directory'], str(base_year), table_name)
if not os.path.exists(config['storage_location']):
flt_storage = StorageFactory().get_storage(
type='flt_storage',
subdir='store',
storage_location=config['storage_location'])
table = database.get_table(table_name)
id_name = [primary_key.name.lower() for primary_key in table.primary_key]
dataset = Dataset(resources=config,
in_storage=in_storage,
out_storage=flt_storage,
in_table_name=table_name,
id_name = id_name)
nchunks = config['creating_baseyear_cache_configuration'].tables_to_cache_nchunks.get(table_name, 1)
current_time = SimulationState().get_current_time()
SimulationState().set_current_time(base_year)
dataset.load_dataset(nchunks=nchunks, flush_after_each_chunk=True)
SimulationState().set_current_time(current_time)
else:
logger.log_status(config['storage_location'] + " already exits; skip caching " + table_name)
finally:
logger.end_block()
def unroll_gridcells_to_cache_from_buildings(self, gridcells, buildings,
cache_directory, base_year):
"""Populate the cache with the unrolled gridcells info derived
from the buildings table.
"""
logger.start_block('Unrolling gridcell data from buildings')
try:
storage = AttributeCache().get_flt_storage_for_year(base_year)
urbansim_constant = SessionConfiguration().get_dataset_from_pool('urbansim_constant')
print "recent_years = %s" % urbansim_constant['recent_years']
recent_years = urbansim_constant['recent_years']
roller = RollbackGridcellsFromBuildings()
for year in range(base_year, base_year-recent_years-1, -1):
logger.start_block('Unrolling gridcells into year %d' % (year-1))
try:
roller.unroll_gridcells_for_one_year(gridcells,
buildings,
year,
dataset_pool=SessionConfiguration().get_dataset_pool())
flt_directory = os.path.join(cache_directory, str(year-1))
flt_storage = StorageFactory().get_storage(
type='flt_storage', subdir='store',
storage_location=flt_directory)
gridcells.write_dataset(out_storage=flt_storage)
finally:
logger.end_block()
finally:
logger.end_block()
def _put_one_matrix_into_travel_data_set(self, travel_data_set, max_zone_id, matrix_name, attribute_name, bank_path,
matrices_created=False):
"""
Adds to the given travel_data_set the data for the given matrix
that is in the emme/2 data bank.
"""
logger.start_block('Copying data for matrix %s into variable %s' %
(matrix_name, attribute_name))
try:
if not matrices_created:
self._get_matrix_into_data_file(matrix_name, max_zone_id, bank_path)
file_name = "_one_matrix.txt"
else:
file_name = "%s_one_matrix.txt" % matrix_name
file_contents = self._get_emme2_data_from_file(join(bank_path, file_name))
travel_data_set.add_primary_attribute(data=zeros(travel_data_set.size(), dtype=float32),
name=attribute_name)
odv = array([line.split() for line in file_contents], dtype=float32)
if odv.size == 0:
logger.log_error("Skipped exporting travel_data attribute %s: No data is exported from EMME matrix." % attribute_name)
else:
travel_data_set.set_values_of_one_attribute_with_od_pairs(attribute=attribute_name,
values=odv[:,2],
O=odv[:,0].astype('int32'),
D=odv[:,1].astype('int32')
)
finally:
logger.end_block()
def preprocess_datasets(self):
logger.start_block('Pre-processing buildings')
self.compute_building_variables()
# consolidate buildings of the same type on the same parcel
parcels = self.dataset_pool.get_dataset('parcel')
number_of_buildings = parcels.compute_variables("number_of_buildings = parcel.number_of_agents(building)",
dataset_pool=self.dataset_pool)
multiple_bldg_parcels = where(number_of_buildings > 1)[0]
bldgs_to_remove = zeros(self.input_buildings.size(), dtype='bool8')
consolidated = array([0, 0])
logger.log_status("Original total: %s buildings" % self.input_buildings.size())
for i in multiple_bldg_parcels:
bidx = where(self.input_buildings['parcel_id'] == parcels["parcel_id"][i])[0]
bts = unique(self.input_buildings["building_type_id"][bidx])
if bts.size == 1:
cons_idx = bidx[0]
for attr in ["non_residential_sqft", "land_area", "residential_units"]:
self.input_buildings[attr][cons_idx] = self.input_buildings[attr][bidx].sum()
unitattr = self.get_units(cons_idx)
totsqft = self.input_buildings[unitattr][bidx] * self.input_buildings["sqft_per_unit"][bidx]
if self.input_buildings[unitattr][cons_idx] > 0:
self.input_buildings["sqft_per_unit"][cons_idx] = round(totsqft.sum()/self.input_buildings[unitattr][cons_idx].astype("float32"))
bldgs_to_remove[bidx[1:]] = True
consolidated = consolidated + array([1, bidx.size])
continue
# bldgs_to_remove[bidx] = True
# self.parcels_not_processed = self.parcels_not_processed + [parcels["parcel_id"][i]]
self.input_buildings.subset_by_index(where(logical_not(bldgs_to_remove))[0])
if "template_id" in self.input_buildings.get_known_attribute_names():
self.original_templates = self.input_buildings["template_id"].copy()
self.input_buildings.add_attribute(zeros(self.input_buildings.size(), dtype="int32"), name="template_id",
metadata=1)
logger.log_status("%s buildings consolidated into %s." % (consolidated[1], consolidated[0]))
logger.log_status("%s parcels were not processed." % len(self.parcels_not_processed))
logger.end_block()
def unroll_gridcells_to_cache_from_buildings(self, gridcells, buildings,
cache_directory, base_year):
"""Populate the cache with the unrolled gridcells info derived
from the buildings table.
"""
logger.start_block('Unrolling gridcell data from buildings')
try:
storage = AttributeCache().get_flt_storage_for_year(base_year)
urbansim_constant = SessionConfiguration().get_dataset_from_pool('urbansim_constant')
print "recent_years = %s" % urbansim_constant['recent_years']
recent_years = urbansim_constant['recent_years']
roller = RollbackGridcellsFromBuildings()
for year in range(base_year, base_year-recent_years-1, -1):
logger.start_block('Unrolling gridcells into year %d' % (year-1))
try:
roller.unroll_gridcells_for_one_year(gridcells,
buildings,
year,
dataset_pool=SessionConfiguration().get_dataset_pool())
flt_directory = os.path.join(cache_directory, str(year-1))
flt_storage = StorageFactory().get_storage(
type='flt_storage', subdir='store',
storage_location=flt_directory)
gridcells.write_dataset(out_storage=flt_storage)
finally:
logger.end_block()
finally:
logger.end_block()
def get_needed_matrices_from_emme2(self,
year,
cache_directory,
bank_dir,
matrix_variable_map,
matrices_created=False):
"""Copies the specified emme/2 matrices into the specified travel_data variable names.
"""
logger.start_block('Getting matricies from emme2')
try:
zone_set = SessionConfiguration().get_dataset_from_pool('zone')
zone_set.load_dataset()
travel_data_set = self.get_travel_data_from_emme2(
zone_set, bank_dir, matrix_variable_map, matrices_created)
finally:
logger.end_block()
logger.start_block('Writing data to cache')
try:
next_year = year + 1
out_storage = AttributeCache().get_flt_storage_for_year(next_year)
travel_data_set.write_dataset(attributes='*',
out_storage=out_storage,
out_table_name='travel_data')
finally:
logger.end_block()
def run(self, config, year):
""" This class simulates a MATSim run. Therefore it copies
real travel data into the OPUS_HOME and modifies the
travel cost entries.
"""
logger.start_block("Starting ModifyTravelCosts.run(...)")
self.config = config
# get travel model parameter from the opus dictionary
self.travel_model_configuration = config['travel_model_configuration']
self.base_year = self.travel_model_configuration['base_year']
# for debugging
#try: #tnicolai
# import pydevd
# pydevd.settrace()
#except: pass
# set travel data for test simulation
if year == self.base_year + 1:
logger.log_status(
"Prepare copying pre-calculated MATSim travel data to OPUS_HOME tmp directory."
)
self.copy_pre_calculated_MATSim_travel_costs()
logger.log_status("Modifying travel costs.")
self.modify_travel_costs() # comment out for base scenario
logger.log_status(
"Finished modifying...") # comment out for base scenario
# use modified travel data for all following runs
else:
logger.log_status(
"Travel data was modified before. So there is nothing to do..."
)
logger.end_block()
def _generate_input_land_cover(self, current_year, base_directory,
urbansim_cache_directory, years, output_directory,
convert_flt, convert_input):
if current_year == years[0]:
if not convert_input:
return base_directory
else:
package_dir_path = package().get_package_path()
command = os.path.join(package_dir_path, "tools", "lc_convert.py")
status = os.system(command + ' %s -i "%s" -o "%s"' % ('input data', base_directory, self.temp_land_cover_dir))
assert(status == 0, "generate input failed")
return self.temp_land_cover_dir
previous_year = self._get_previous_year(current_year, years)
if not convert_flt:
logger.start_block("Copy data from %s to temp land cover folder" % urbansim_cache_directory)
try:
self._copy_invariants_to_temp_land_cover_dir(os.path.join(urbansim_cache_directory, str(previous_year)))
finally:
logger.end_block()
return self.temp_land_cover_dir
# package_dir_path = package().get_package_path()
# command = os.path.join(package_dir_path, "tools", "lc_convert.py")
flt_directory_in = os.path.join(output_directory, str(previous_year))
flt_directory_out = self.temp_land_cover_dir
LCCMInputConvert()._convert_lccm_input(flt_directory_in, flt_directory_out)
# status = os.system(command + ' %d -i "%s" -o "%s"' % (previous_year, flt_directory_in, flt_directory_out))
# assert(status == 0, "generate input failed")
return self.temp_land_cover_dir
def run_emme2_macro(self,
macro_path,
bank_path,
scenario_number=-1,
output_file=None,
append_to_output=True):
"""
Runs this emme/2 macro in the bank specified.
"""
logger.start_block('Running emme2 macro %s in bank at %s' %
(macro_path, bank_path))
# generate a random file name
temp_macro_file_name = tempfile.NamedTemporaryFile().name
prior_cwd = os.getcwd()
if output_file is None:
out = ""
else:
out = "> %s" % output_file
if append_to_output:
out = " >%s" % out
try:
os.chdir(bank_path)
shutil.copy(macro_path, temp_macro_file_name)
cmd = "%s 000 -m %s" % (self.emme_cmd, temp_macro_file_name)
if scenario_number != -1:
cmd = "%s %s" % (cmd, scenario_number)
cmd = "%s%s" % (cmd, out)
logger.log_status(cmd)
if os.system(cmd):
raise StandardError("Problem with simulation")
finally:
os.remove(temp_macro_file_name)
os.chdir(prior_cwd)
logger.end_block()
def run(self, config, year):
""" This class simulates a MATSim run. Therefore it copies
real travel data into the OPUS_HOME and modifies the
travel cost entries.
"""
logger.start_block("Starting ModifyTravelCosts.run(...)")
self.config = config
# get travel model parameter from the opus dictionary
self.travel_model_configuration = config['travel_model_configuration']
self.base_year = self.travel_model_configuration['base_year']
# for debugging
#try: #tnicolai
# import pydevd
# pydevd.settrace()
#except: pass
# set travel data for test simulation
if year == self.base_year+1:
logger.log_status("Prepare copying pre-calculated MATSim travel data to OPUS_HOME tmp directory.")
self.copy_pre_calculated_MATSim_travel_costs()
logger.log_status("Modifying travel costs.")
self.modify_travel_costs() # comment out for base scenario
logger.log_status("Finished modifying...") # comment out for base scenario
# use modified travel data for all following runs
else:
logger.log_status("Travel data was modified before. So there is nothing to do...")
logger.end_block()
def _call_input_file_writer(self, year, datasets, in_table_names, out_table_names, variable_names, dataset_pool):
current_year_tm_dir = self.get_daysim_dir(year)
file_config = self.config['travel_model_configuration'].get('daysim_file', {})
file_format = file_config.get('format', 'tab')
if(file_format == 'hdf5g'):
current_year_tm_dir = os.path.join(current_year_tm_dir, file_config.get('name', 'daysim_inputs.hdf5'))
meta_data = self.config['travel_model_configuration'].get('meta_data', {})
storage = StorageFactory().get_storage('%s_storage' % file_format, storage_location = current_year_tm_dir)
kwargs = {}
mode={file_format: Storage.OVERWRITE}
if file_format == 'csv' or file_format == 'tab' or file_format == 'tsv' or file_format == 'dat':
kwargs['append_type_info'] = False
if file_format.startswith('hdf5'):
kwargs['compression'] = file_config.get('hdf5_compression', None)
logger.start_block('Writing Daysim inputs.')
for dataset_name, dataset in datasets.iteritems():
ds_meta = meta_data.get(in_table_names[dataset_name], {})
if file_format.startswith('hdf5'):
kwargs['column_meta'] = ds_meta
attr_vals = {}
for attr in variable_names[dataset_name]:
attr_vals[attr] = dataset[attr]
storage.write_table(table_name = out_table_names[dataset_name], table_data = attr_vals, mode = mode[file_format], **kwargs)
mode['hdf5g'] = Storage.APPEND
logger.end_block()
logger.log_status('Daysim inputs written into %s' % current_year_tm_dir)
return out_table_names.values()
def run_emme2_macro(self, macro_path, bank_path, scenario_number=-1, output_file=None, append_to_output=True):
"""
Runs this emme/2 macro in the bank specified.
"""
logger.start_block('Running emme2 macro %s in bank at %s' %
(macro_path, bank_path))
# generate a random file name
temp_macro_file_name = tempfile.NamedTemporaryFile().name
prior_cwd = os.getcwd()
if output_file is None:
out = ""
else:
out = "> %s" % output_file
if append_to_output:
out = " >%s" % out
try:
os.chdir(bank_path)
shutil.copy(macro_path, temp_macro_file_name)
cmd = "%s 000 -m %s" % (self.emme_cmd, temp_macro_file_name)
if scenario_number != -1:
cmd = "%s %s" % (cmd, scenario_number)
cmd = "%s%s" % (cmd, out)
logger.log_status(cmd)
if os.system(cmd):
raise StandardError("Problem with simulation")
finally:
os.remove(temp_macro_file_name)
os.chdir(prior_cwd)
logger.end_block()
def _run_each_year_as_separate_process(self,
iyear,
year,
seed=None,
resources=None,
profiler_name=None,
log_file=None):
logger.start_block('Running simulation for year %d in new process' %
year)
resources['years'] = (year, year)
resources['seed'] = seed,
if profiler_name is not None:
# add year to the profile name
resources["profile_filename"] = "%s_%s" % (profiler_name, year)
optional_args = []
if log_file:
optional_args += ['--log-file-name', os.path.split(log_file)[-1]]
success = False
try:
logger.disable_file_logging(log_file)
success = self._fork_new_process(
'opus_core.model_coordinators.model_system',
resources,
optional_args=optional_args)
logger.enable_file_logging(log_file, verbose=False)
finally:
logger.end_block()
return success
def _run_each_year_as_separate_process(
self,
start_year,
end_year,
seed_array,
resources,
log_file_name='run_multiprocess.log'):
log_file = os.path.join(resources['cache_directory'], log_file_name)
profiler_name = resources.get("profile_filename", None)
iyear = 0
for year in range(start_year, end_year + 1):
logger.start_block(
'Running simulation for year %d in new process' % year)
try:
resources['years'] = (year, year)
resources['seed'] = seed_array[iyear],
logger.disable_file_logging(log_file)
if profiler_name is not None:
resources["profile_filename"] = "%s_%s" % (
profiler_name, year) # add year to the profile name
self._fork_new_process(
'opus_core.model_coordinators.model_system',
resources,
optional_args=['--log-file-name', log_file_name])
logger.enable_file_logging(log_file, verbose=False)
finally:
logger.end_block()
iyear += 1
self._notify_stopped()
def run(self, config, year):
"""Running MATSim. A lot of paths are relative; the base path is ${OPUS_HOME}/opus_matsim. As long as ${OPUS_HOME}
is correctly set and the matsim tarfile was unpacked in OPUS_HOME, this should work out of the box. There may eventually
be problems with the java version.
"""
try:
import pydevd
pydevd.settrace()
except: pass
logger.start_block("Starting RunTravelModel.run(...)")
self.setUp( config )
config_obj = MATSimConfigObject(config, year, self.matsim_config_full)
config_obj.marschall()
cmd = """cd %(opus_home)s/opus_matsim ; java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s""" % {
'opus_home': os.environ['OPUS_HOME'],
'vmargs': "-Xmx2000m",
'classpath': "libs/log4j/log4j/1.2.15/log4j-1.2.15.jar:libs/jfree/jfreechart/1.0.7/jfreechart-1.0.7.jar:libs/jfree/jcommon/1.0.9/jcommon-1.0.9.jar:classesMATSim:classesToronto:classesTNicolai:classesKai:classesEntry", # 'classpath': "classes:jar/MATSim.jar",
'javaclass': "playground.run.Matsim4Urbansim",
'matsim_config_file': self.matsim_config_full }
logger.log_status('Running command %s' % cmd )
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "Matsim Run failed. Code returned by cmd was %d" % (cmd_result)
logger.log_error(error_msg)
logger.log_error("Note that currently (dec/08), paths in the matsim config files are relative to the opus_matsim root,")
logger.log_error(" which is one level 'down' from OPUS_HOME.")
raise StandardError(error_msg)
logger.end_block()
def log_results(self):
procedure = self.model_system.run_year_namespace["model"].procedure
if not hasattr(procedure, 'print_results'):
logger.log_warning("Estimation procedure %s doesn't have a print_results() method, " % procedure + \
"which is needed to log estimation results.")
return
tmp_config = Resources(self.config)
outputvar = tmp_config['models_configuration'][self.model_name]['controller']['estimate']['arguments']['output']
results = self.model_system.vardict.get(outputvar, "process_output")[1]
storage_location = AttributeCache().get_storage_location()
log_file_name = "estimate_models.log" ## one file for all estimation results
logger.enable_file_logging( os.path.join(storage_location, log_file_name),
mode = 'a') ##appending instead of overwriting
logger.start_block("%s Estimation Results" % self.model_name)
for submodel, submodel_results in results.items():
logger.log_status( "Submodel %s" % submodel)
if submodel_results == {}:
logger.log_warning("No estimation results for submodel %s" % submodel)
else:
try:
procedure.print_results(submodel_results)
except:
logger.log_warning("Problems in printing results for submodel %s" % submodel)
logger.end_block()
logger.disable_file_logging()
def run(self, config, year):
""" This class simulates a MATSim run copying a manipulated
travel data into opus home tmp directory
"""
logger.start_block("Starting CopyDummyTravelData.run(...)")
self.config = config
# get travel model parameter from the opus dictionary
self.travel_model_configuration = config['travel_model_configuration']
self.base_year = self.travel_model_configuration['base_year']
# for debugging
#try: #tnicolai
# import pydevd
# pydevd.settrace()
#except: pass
# set travel data for test simulation
if year == self.base_year+1:
logger.log_status("Prepare copying pre-calculated MATSim travel data to OPUS_HOME tmp directory.")
self.copy_dummy_travel_data()
self.modify_workplace_accessibility()
# use modified travel data for all following runs
else:
logger.log_status("Travel data is already copied in the first iteration.")
logger.end_block()
def _run_each_year_as_separate_process(
self, iyear, year, seed=None, resources=None, profiler_name=None, log_file=None
):
logger.start_block("Running simulation for year %d in new process" % year)
resources["years"] = (year, year)
resources["seed"] = (seed,)
if profiler_name is not None:
# add year to the profile name
resources["profile_filename"] = "%s_%s" % (profiler_name, year)
optional_args = []
if log_file:
optional_args += ["--log-file-name", os.path.split(log_file)[-1]]
success = False
try:
logger.disable_file_logging(log_file)
success = self._fork_new_process(
"opus_core.model_coordinators.model_system", resources, optional_args=optional_args
)
logger.enable_file_logging(log_file, verbose=False)
finally:
logger.end_block()
return success
def _run_each_year_as_separate_process(self, start_year, end_year, seed_array, resources, log_file_name='run_multiprocess.log'):
skip_first_year_of_urbansim = resources.get('skip_urbansim', False)
log_file = os.path.join(resources['cache_directory'], log_file_name)
profiler_name = resources.get("profile_filename", None)
iyear = 0
for year in range(start_year, end_year+1):
if (year <> start_year) or ((year == start_year) and (not skip_first_year_of_urbansim)):
logger.start_block('Running UrbanSim for year %d in new process' % year)
try:
resources['years'] = (year, year)
resources['seed'] = seed_array[iyear],
logger.disable_file_logging(log_file)
if profiler_name is not None:
resources["profile_filename"] = "%s_%s" % (profiler_name, year) # add year to the profile name
self._fork_new_process(
'urbansim.model_coordinators.model_system', resources, optional_args=['--log-file-name', log_file_name])
logger.enable_file_logging(log_file, verbose=False)
finally:
logger.end_block()
if ('travel_model_configuration' in resources) and (not resources.get('skip_travel_model', False)):
# tnicolai add start year to travel model config
tmc = resources['travel_model_configuration']
tmc['start_year'] = start_year # end tnicolai
self._run_travel_models_in_separate_processes(resources['travel_model_configuration'], year, resources)
if 'post_year_configuration' in resources:
self._run_travel_models_in_separate_processes(resources['post_year_configuration'], year, resources)
iyear +=1
self._notify_stopped()
def run(self, config, year):
"""Running MATSim. A lot of paths are relative; the base path is ${OPUS_HOME}/opus_matsim. As long as ${OPUS_HOME}
is correctly set and the matsim tar-file was unpacked in OPUS_HOME, this should work out of the box. There may eventually
be problems with the java version.
"""
logger.start_block("Starting RunTravelModel.run(...)")
# try: # tnicolai :for debugging
# import pydevd
# pydevd.settrace()
# except: pass
config_obj_v3 = MATSimConfigObjectV3(config, year)
self.matsim_config_full = config_obj_v3.marschall()
# check for test parameter
tmc = config["travel_model_configuration"]
if tmc["matsim4urbansim"].get("test_parameter") != None:
self.test_parameter = tmc["matsim4urbansim"].get("test_parameter")
# change to directory opus_matsim
os.chdir(config_obj_v3.matsim4opus_path)
# int cmd
cmd = ""
# calling travel model with cmd command
if sys.platform.lower() == "win32":
# reserve memory for java
xmx = "-Xmx1500m" # Windows can't reserve more than 1500m
logger.log_note("Note that Java for Windows can't reserve more than 1500 MB of memory to run MATSim!!!")
cmd = """java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s %(test_parameter)s""" % {
"vmargs": xmx,
"classpath": "jar/matsim.jar;jar/contrib/matsim4urbansim.jar",
"javaclass": "org.matsim.contrib.matsim4urbansim.run.MATSim4UrbanSimZone",
"matsim_config_file": self.matsim_config_full,
"test_parameter": self.test_parameter,
}
else:
# reserve memory for java
xmx = "-Xmx4000m"
cmd = """java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s %(test_parameter)s""" % {
"vmargs": xmx,
"classpath": "jar/matsim.jar:jar/contrib/matsim4urbansim.jar",
"javaclass": "org.matsim.contrib.matsim4urbansim.run.MATSim4UrbanSimZone",
"matsim_config_file": self.matsim_config_full,
"test_parameter": self.test_parameter,
}
logger.log_status("Running command %s" % cmd)
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "MATSim Run failed. Code returned by cmd was %d" % (cmd_result)
logger.log_error(error_msg)
logger.log_error("Note that paths in the matsim config files are relative to the matsim4opus root,")
logger.log_error("which is one level 'down' from OPUS_HOME.")
raise StandardError(error_msg)
logger.end_block()
def logged_run_method (*req_args, **opt_args):
logger.start_block("Running %s" % an_instance.full_name(), tags=["model", "model-run"],
verbosity_level=1)
try:
results = run_method(*req_args, **opt_args)
finally:
logger.end_block()
return results
def logged_estimate_method (*req_args, **opt_args):
logger.start_block("Estimating %s" % an_instance.full_name(), tags=["model", "model-estimate"],
verbosity_level=1)
try:
results = estimate_method(*req_args, **opt_args)
finally:
logger.end_block()
return results
def logged_run_method (*req_args, **opt_args):
logger.start_block("Running %s" % an_instance.full_name(), tags=["model", "model-run"],
verbosity_level=1)
try:
results = run_method(*req_args, **opt_args)
finally:
logger.end_block()
return results
def logged_method(*req_args, **opt_args):
logger.start_block(name=an_instance.name(), verbose=False)
try:
results = compute_method(*req_args, **opt_args)
an_instance._do_flush_dependent_variables_if_required()
finally:
logger.end_block()
return results
def logged_estimate_method (*req_args, **opt_args):
logger.start_block("Estimating %s" % an_instance.full_name(), tags=["model", "model-estimate"],
verbosity_level=1)
try:
results = estimate_method(*req_args, **opt_args)
finally:
logger.end_block()
return results
def export_dataset(self, dataset_name, in_storage, out_storage):
logger.start_block('Exporting dataset %s' % dataset_name)
try:
values_from_storage = in_storage.load_table(dataset_name)
out_storage.write_table(dataset_name, values_from_storage)
finally:
logger.end_block()
def tearDown(self):
logger.log_status('entering tearDown')
logger.log_status('Removing loaded/stored file: %s ...' %self.xsd_destination)
if os.path.exists(self.xsd_destination):
os.remove(self.xsd_destination)
logger.log_status('... removing finished.')
logger.log_status('leaving tearDown')
logger.end_block('Finished XSDLoadTest')
def export_dataset(self, dataset_name, in_storage, out_storage):
logger.start_block('Exporting dataset %s' % dataset_name)
try:
values_from_storage = in_storage.load_table(dataset_name)
out_storage.write_table(dataset_name, values_from_storage)
finally:
logger.end_block()
def logged_method (*req_args, **opt_args):
logger.start_block(name=an_instance.name(), verbose=False)
try:
results = compute_method(*req_args, **opt_args)
an_instance._do_flush_dependent_variables_if_required()
finally:
logger.end_block()
return results
def run(self, refinement_dataset=None, current_year=None,
action_order=['subtract', 'add', 'multiple', 'set_value', 'convert', 'demolish', 'delete'],
dataset_pool=None):
"""
"""
if refinement_dataset is None:
refinement_dataset = dataset_pool.get_dataset('refinement')
self.id_names = (refinement_dataset.get_id_name()[0], 'transaction_id')
if current_year is None:
current_year = SimulationState().get_current_time()
#refinements_this_year = copy.deepcopy(refinement_dataset)
refinements_this_year = refinement_dataset
this_year_index = where(refinement_dataset.get_attribute('year')==current_year)[0]
all_years_index = where(refinement_dataset.get_attribute('year')==-1)[0]
refinements_this_year.subset_by_index(concatenate( (this_year_index, all_years_index) ),
flush_attributes_if_not_loaded=False)
transactions = refinements_this_year.get_attribute('transaction_id')
actions = refinements_this_year.get_attribute('action')
for this_transaction in sort( unique(transactions) ):
#transaction_list = [] # list of each action in this transaction
agents_pool = [] # index to agents to keep track agent within 1 transaction
logger.start_block("Transaction %i" % this_transaction)
for action_type in action_order:
action_function = getattr(self, '_' + action_type)
for refinement_index in where( logical_and(transactions==this_transaction, actions == action_type))[0]:
this_refinement = refinements_this_year.get_data_element(refinement_index)
## get agent_dataset and location_dataset if specified
agent_dataset_name = this_refinement.agent_dataset
agent_dataset = dataset_pool.get_dataset( agent_dataset_name )
location_dataset = None
logger.log_status("Action: %s\nAmount: %s\nAttribute: %s\nFilter: %s" % \
(action_type, this_refinement.amount, this_refinement.agent_attribute,
this_refinement.agent_filter
) )
action_function( agents_pool, this_refinement.amount,
agent_dataset, location_dataset,
this_refinement,
dataset_pool )
agent_dataset.flush_dataset()
dataset_pool._remove_dataset(agent_dataset.get_dataset_name())
if location_dataset is not None:
location_dataset.flush_dataset()
dataset_pool._remove_dataset(location_dataset.get_dataset_name())
## delete agents still in agents_pool at the end of the transaction
#agent_dataset.remove_elements( array(agents_pool) )
# dataset_pool.flush_loaded_datasets()
# dataset_pool.remove_all_datasets()
logger.end_block()
def _run_PopSyn(self, tm_config, year, run_dir):
""" Gets the inputs all setup and then runs the Population Synthesizer.
"""
# update sfzones.csv with the landusemodel output
dest_dir = os.path.join(run_dir, self.SUBDIR_POPULATION_SYNTHESIZER)
if not os.path.exists(dest_dir): os.makedirs(dest_dir)
# copy over inputs files
dest_inputsdir = os.path.join(dest_dir, "inputs")
if not os.path.exists(dest_inputsdir): os.makedirs(dest_inputsdir)
# copy over the tazdata from the TazDataProcessor step
shutil.copy2(os.path.join(run_dir, self.SUBDIR_TAZ_DATA_PROCESSOR, "tazdata.dbf"), dest_inputsdir)
# copy over controls files
dest_controlsdir = os.path.join(dest_dir, "controls")
if os.path.exists(dest_controlsdir): shutil.rmtree(dest_controlsdir)
src_controlsdir = os.path.join(tm_config['popsyn_srcdir'], "controls")
shutil.copytree(src_controlsdir, dest_controlsdir)
self._updateConfigPaths(os.path.join(dest_controlsdir, "hhSubmodels.properties"),
[ [r"(tazdata.file\s*=\s*)(\S*)", r"\1inputs\tazdata.dbf"],
[r"(tazdata.out.file\s*=\s*)(\S*)", r"\1inputs\tazdata_converted.csv"],
[r"(\S\s*=\s*)(inputs/)(\S*)", "%s%s%s" % (r"\1",os.path.join(tm_config['popsyn_srcdir'],"inputs"),r"\\\3")]
])
self._updateConfigPaths(os.path.join(dest_controlsdir, "arc.properties"),
[ [r"(Forecast.TazFile\s*=\s*)(\S*)", r"\1inputs\tazdata_converted.csv"],
[r"(\S\s*=\s*)(./inputs/)(\S*)", "%s%s%s" % (r"\1",os.path.join(tm_config['popsyn_srcdir'],"inputs"),r"\\\3")]
])
# make the outputs directory
dest_outputsdir = os.path.join(dest_dir, "outputs")
if not os.path.exists(dest_outputsdir): os.makedirs(dest_outputsdir)
for dir in ["intermediate", "syntheticPop", "validation"]:
newdir = os.path.join(dest_outputsdir, dir)
if not os.path.exists(newdir): os.makedirs(newdir)
# copy over the batch
batfile = "runPopSyn.bat"
shutil.copy2(os.path.join(tm_config['popsyn_srcdir'], batfile), dest_dir)
# run the Population Synthesizer
sfsampfile = os.path.join(dest_outputsdir, "syntheticPop", "sfsamp.txt")
if os.path.exists(sfsampfile):
logger.log_status("Synthesized population file %s exists -- skipping creation!" % sfsampfile)
else:
cmd = os.path.join(dest_dir, batfile)
logger.start_block("Running [%s]" % (cmd))
popsynproc = subprocess.Popen( cmd, cwd = dest_dir, stdout=subprocess.PIPE )
for line in popsynproc.stdout:
logger.log_status(line.strip('\r\n'))
popsynret = popsynproc.wait()
logger.log_status("Returned %d" % (popsynret))
if popsynret != 0: raise StandardError, "Population Synthesizer exited with bad return code"
# put the output files in place
shutil.copy2(sfsampfile, os.path.join(run_dir, self.SUBDIR_LANDUSE_INPUTS))
logger.end_block()
def run(self, refinement_dataset=None, current_year=None,
action_order=['subtract', 'add', 'multiple', 'set_value', 'convert', 'demolish', 'delete'],
dataset_pool=None):
"""
"""
if refinement_dataset is None:
refinement_dataset = dataset_pool.get_dataset('refinement')
self.id_names = (refinement_dataset.get_id_name()[0], 'transaction_id')
if current_year is None:
current_year = SimulationState().get_current_time()
#refinements_this_year = copy.deepcopy(refinement_dataset)
refinements_this_year = refinement_dataset
this_year_index = where(refinement_dataset.get_attribute('year')==current_year)[0]
all_years_index = where(refinement_dataset.get_attribute('year')==-1)[0]
refinements_this_year.subset_by_index(concatenate( (this_year_index, all_years_index) ),
flush_attributes_if_not_loaded=False)
transactions = refinements_this_year.get_attribute('transaction_id')
actions = refinements_this_year.get_attribute('action')
for this_transaction in sort( unique(transactions) ):
#transaction_list = [] # list of each action in this transaction
agents_pool = [] # index to agents to keep track agent within 1 transaction
logger.start_block("Transaction %i" % this_transaction)
for action_type in action_order:
action_function = getattr(self, '_' + action_type)
for refinement_index in where( logical_and(transactions==this_transaction, actions == action_type))[0]:
this_refinement = refinements_this_year.get_data_element(refinement_index)
## get agent_dataset and location_dataset if specified
agent_dataset_name = this_refinement.agent_dataset
agent_dataset = dataset_pool.get_dataset( agent_dataset_name )
location_dataset = None
logger.log_status("Action: %s\nAmount: %s\nAttribute: %s\nFilter: %s" % \
(action_type, this_refinement.amount, this_refinement.agent_attribute,
this_refinement.agent_filter
) )
action_function( agents_pool, this_refinement.amount,
agent_dataset, location_dataset,
this_refinement,
dataset_pool )
agent_dataset.flush_dataset()
dataset_pool._remove_dataset(agent_dataset.get_dataset_name())
if location_dataset is not None:
location_dataset.flush_dataset()
dataset_pool._remove_dataset(location_dataset.get_dataset_name())
## delete agents still in agents_pool at the end of the transaction
#agent_dataset.remove_elements( array(agents_pool) )
# dataset_pool.flush_loaded_datasets()
# dataset_pool.remove_all_datasets()
logger.end_block()
def run(self,
chunk_specification,
dataset,
dataset_index=None,
result_array_type=float32,
**kwargs):
""" 'chunk_specification' - determines number of chunks to use when computing over
the dataset set.
'dataset' - an object of class Dataset that is to be chunked.
'dataset_index' - index of individuals in dataset to be chunked.
'result_array_type' - type of the resulting array. Can be any numerical type of numpy array.
**kwargs - keyword arguments.
The method chunks dataset_index in the desired number of chunks (minimum is 1) and for each chunk it calls the method
'run_chunk'. The order of the individuals entering the chunking is determined by the method 'get_agents_order'.
"""
if dataset_index == None:
dataset_index = arange(dataset.size())
if not isinstance(dataset_index, ndarray):
dataset_index = array(dataset_index)
logger.log_status("Total number of individuals: %s" %
dataset_index.size)
result_array = zeros(dataset_index.size, dtype=result_array_type)
if dataset_index.size <= 0:
logger.log_status("Nothing to be done.")
return result_array
all_indexed_individuals = DatasetSubset(dataset, dataset_index)
ordered_agent_indices = self.get_agents_order(
all_indexed_individuals) # set order of individuals in chunks
# TODO: Remove next six lines after we inherit chunk specification as a text string.
if (chunk_specification is None):
chunk_specification = {'nchunks': 1}
chunker = ChunkSpecification(chunk_specification)
self.number_of_chunks = chunker.nchunks(dataset_index)
chunksize = int(
ceil(all_indexed_individuals.size() /
float(self.number_of_chunks)))
for ichunk in range(self.number_of_chunks):
logger.start_block("%s chunk %d out of %d." %
(self.model_short_name,
(ichunk + 1), self.number_of_chunks))
self.index_of_current_chunk = ichunk
try:
chunk_agent_indices = ordered_agent_indices[arange(
(ichunk * chunksize),
min((ichunk + 1) * chunksize,
all_indexed_individuals.size()))]
logger.log_status("Number of agents in this chunk: %s" %
chunk_agent_indices.size)
result_array[chunk_agent_indices] = self.run_chunk(
dataset_index[chunk_agent_indices], dataset,
**kwargs).astype(result_array_type)
finally:
logger.end_block()
return result_array
def tearDown(self):
logger.log_status('entering tearDown')
logger.log_status('Removing loaded/stored file: %s ...' %
self.xsd_destination)
if os.path.exists(self.xsd_destination):
os.remove(self.xsd_destination)
logger.log_status('... removing finished.')
logger.log_status('leaving tearDown')
logger.end_block('Finished XSDLoadTest')
def _write_to_xls_file(self, data, header, filepath):
logger.start_block("Writing to transcad input file")
workbook = xlwt.Workbook()
worksheet = workbook.add_sheet('data')
[worksheet.write(0, c, cname) for c, cname in enumerate(header)]
[worksheet.write(rc[0]+1, rc[1], v) for rc, v in ndenumerate(data)]
workbook.save(filepath)
logger.end_block()
def run(self, myconfig={}, year=2001):
"""Runs the travel model, using appropriate info from config.
"""
tm_config = myconfig["travel_model_configuration"]
# verify that the base directory exists and there's a runmodel.bat in it, or we're a nogo
base_dir = tm_config['travel_model_base_directory']
if not os.path.exists(base_dir):
raise StandardError, \
"Travel model base directory '%s' must exist with a standard %s" % (base_dir, self.RUN_BATCH)
src_runmodelfile = os.path.join(base_dir, self.RUN_BATCH)
if not os.path.exists(src_runmodelfile):
raise StandardError, \
"Travel model base directory '%s' must exist with a standard %s" % (base_dir, self.RUN_BATCH)
run_dir = os.path.join(base_dir, tm_config[year]['year_dir'])
if not os.path.exists(run_dir):
os.makedirs(run_dir)
# if the final skims are already there, then abort -- this has run already
if os.path.exists(os.path.join(run_dir, "finalTRNWLWAM.h5")):
logger.log_status("Final skims found in %s, skipping travel model" % run_dir)
return
runinputs_dir = os.path.join(run_dir, self.SUBDIR_LANDUSE_INPUTS)
if not os.path.exists(runinputs_dir):
os.makedirs(runinputs_dir)
# run the TAZ data processor and the Population Synthesizers
self._run_TazDataProcessor(tm_config, year, run_dir)
self._run_PopSyn(tm_config, year, run_dir)
# make the run
shutil.copy2(src_runmodelfile, run_dir)
self._updateConfigPaths( os.path.join(run_dir, self.RUN_BATCH),
[ [r"(set LANDUSE\s*=\s*)(\S*)", r"\1.\%s" % (self.SUBDIR_LANDUSE_INPUTS)],
[r"(?P<set>set (BASEDEMAND|MTCDEMAND|NETWORKS)\s*=\s*\S*)(\d\d\d\d)[ ]*$", r"\g<set>" + str(year)]
], doubleBackslash=False)
# dispatcher to taraval, where models start
outfile = open( os.path.join(run_dir, self.RUN_DISPATCH), 'w')
outfile.write("%s\n" % (self.RUN_BATCH))
outfile.close()
# run it!
cmd = self.DISPATCH + " " + self.RUN_DISPATCH + " " + self.CLUSTER_MACHINE
logger.start_block("Running TRAVEL MODDEL [%s]" % (cmd))
tmproc = subprocess.Popen( cmd, cwd = run_dir, stdout=subprocess.PIPE )
for line in tmproc.stdout:
logger.log_status(line.strip('\r\n'))
tmret = tmproc.wait()
logger.log_status("Returned %d" % (tmret))
# TODO - why does it return 1 when it seems ok?!?!
if tmret != 0 and tmret != 1: raise StandardError, "%s exited with bad return code" % (cmd)
logger.end_block()
def _write_to_txt_file(self, data, header, input_file, delimiter='\t'):
logger.start_block("Writing to transcad input file")
newfile = open(input_file, 'w')
newfile.write(delimiter.join(header) + "\n")
rows, cols = data.shape
for n in range(rows):
newfile.write(delimiter.join([str(x) for x in data[n,]]) + "\n")
newfile.close()
logger.end_block()
def run_simulation(self, simulation_instance=None):
logger.start_block("Simulation on database %s" % self.config["scenario_database_configuration"].database_name)
try:
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
# simulation_instance.run_multiprocess(self.config, is_run_subset=True)
finally:
logger.end_block()
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def run(self, config, year):
"""Running MATSim. A lot of paths are relative; the base path is ${OPUS_HOME}/opus_matsim. As long as ${OPUS_HOME}
is correctly set and the matsim tarfile was unpacked in OPUS_HOME, this should work out of the box. There may eventually
be problems with the java version.
"""
logger.start_block("Starting RunTravelModel.run(...)")
# tnicolai :for debugging
# try:
# import pydevd
# pydevd.settrace()
# except: pass
self.__setUp(config)
config_obj = MATSimConfigObject(config, year, self.matsim_config_full)
config_obj.marschall()
# tnicolai: original call
# cmd = """cd %(opus_home)s/opus_matsim ; java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s""" % {
# 'opus_home': paths.OPUS_HOME,
# 'vmargs': "-Xmx2000m",
# 'classpath': "libs/log4j/log4j/1.2.15/log4j-1.2.15.jar:libs/jfree/jfreechart/1.0.7/jfreechart-1.0.7.jar:libs/jfree/jcommon/1.0.9/jcommon-1.0.9.jar:classesMATSim:classesToronto:classesTNicolai:classesKai:classesEntry", # 'classpath': "classes:jar/MATSim.jar",
# 'javaclass': "playground.tnicolai.urbansim.cupum.MATSim4UrbansimCUPUM",
# 'matsim_config_file': self.matsim_config_full,
# 'test_parameter': self.test_parameter }
# tnicolai : test for matsim jar execution ...
cmd = (
"""cd %(opus_home)s/opus_matsim ; java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s %(test_parameter)s"""
% {
"opus_home": paths.OPUS_HOME,
"vmargs": "-Xmx8000m", # set to 8GB on math cluster and 2GB on Notebook
"classpath": "jar/matsim4urbansim.jar",
"javaclass": "playground.tnicolai.urbansim.cupum.MATSim4UrbansimCUPUM",
"matsim_config_file": self.matsim_config_full,
"test_parameter": self.test_parameter,
}
)
logger.log_status("Running command %s" % cmd)
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "Matsim Run failed. Code returned by cmd was %d" % (cmd_result)
logger.log_error(error_msg)
logger.log_error(
"Note that currently (dec/08), paths in the matsim config files are relative to the opus_matsim root,"
)
logger.log_error(" which is one level 'down' from OPUS_HOME.")
raise StandardError(error_msg)
logger.end_block()
def plot_data(self, data_array, main, delimiter=','):
#def _write_to_txt_file(self, data, header, input_file, delimiter='\t'):
logger.start_block("Writing data to file " + main)
newfile = open(main + '.csv', 'w')
#newfile.write(delimiter.join(header) + "\n")
rows, cols = data_array.shape
for n in range(rows):
newfile.write(delimiter.join([str(x) for x in data_array[n,]]) + "\n")
newfile.close()
logger.end_block()
def run_simulation(self, simulation_instance=None):
logger.start_block('Simulation on database %s'
% self.config['scenario_database_configuration'].database_name)
try:
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
finally:
logger.end_block()
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def run(self):
logger.start_block()
insert_auto_generated_cache_directory_if_needed(self.config)
run_manager = RunManager(ServicesDatabaseConfiguration())
run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
run_manager.run_run(self.config, run_as_multiprocess = True )
logger.end_block()
def save_results(self, out_storage=None, model_name=None):
if self.specification is None or self.coefficients is None:
raise ValueError, "model specification or coefficient is None"
#invalid = self.coefficients.is_invalid()
if False:
logger.log_warning('Invalid coefficients. Not saving results!')
return
if model_name is None:
model_name = self.config.get('model_name_for_coefficients', None)
if model_name is None:
if self.model_name is not None:
model_name = self.model_name
else:
raise ValueError, "model_name unspecified"
out_storage_available = True
if out_storage:
pass
elif 'estimation_database_configuration' in self.config:
try:
db_server = DatabaseServer(self.config['estimation_database_configuration'])
database_name = self.config["estimation_database_configuration"].database_name
if not db_server.has_database(database_name):
db_server.create_database(database_name)
output_db = db_server.get_database(database_name)
out_storage = StorageFactory().get_storage(
type='sql_storage',
storage_location=output_db)
except:
logger.log_warning("Problem with connecting database given by 'estimation_database_configuration'.")
out_storage_available = False
else:
logger.log_warning("No estimation_database_configuration given.")
out_storage_available = False
# the original model name of development_project_lcm is too long as a mysql db table name, truncate it
if model_name.rfind("_development_project_location_choice_model") >=0:
model_name = model_name.replace('_project', '')
specification_table = '%s_specification' % model_name
coefficients_table = '%s_coefficients' % model_name
if out_storage_available:
logger.start_block("Writing specification and coefficients into storage given by 'estimation_database_configuration'")
self.specification.write(out_storage=out_storage, out_table_name=specification_table)
self.coefficients.write(out_storage=out_storage, out_table_name=coefficients_table)
logger.end_block()
logger.start_block("Writing specification and coefficients into %s" % AttributeCache().get_storage_location())
self.specification.write(out_storage=AttributeCache(), out_table_name=specification_table)
self.coefficients.write(out_storage=AttributeCache(), out_table_name=coefficients_table)
logger.end_block()
def save_results(self, out_storage=None, model_name=None):
if self.specification is None or self.coefficients is None:
raise ValueError, "model specification or coefficient is None"
#invalid = self.coefficients.is_invalid()
if False:
logger.log_warning('Invalid coefficients. Not saving results!')
return
if model_name is None:
model_name = self.config.get('model_name_for_coefficients', None)
if model_name is None:
if self.model_name is not None:
model_name = self.model_name
else:
raise ValueError, "model_name unspecified"
out_storage_available = True
if out_storage:
pass
elif 'estimation_database_configuration' in self.config:
try:
db_server = DatabaseServer(self.config['estimation_database_configuration'])
database_name = self.config["estimation_database_configuration"].database_name
if not db_server.has_database(database_name):
db_server.create_database(database_name)
output_db = db_server.get_database(database_name)
out_storage = StorageFactory().get_storage(
type='sql_storage',
storage_location=output_db)
except:
logger.log_warning("Problem with connecting database given by 'estimation_database_configuration'.")
out_storage_available = False
else:
logger.log_warning("No estimation_database_configuration given.")
out_storage_available = False
# the original model name of development_project_lcm is too long as a mysql db table name, truncate it
if model_name.rfind("_development_project_location_choice_model") >=0:
model_name = model_name.replace('_project', '')
specification_table = '%s_specification' % model_name
coefficients_table = '%s_coefficients' % model_name
if out_storage_available:
logger.start_block("Writing specification and coefficients into storage given by 'estimation_database_configuration'")
self.specification.write(out_storage=out_storage, out_table_name=specification_table)
self.coefficients.write(out_storage=out_storage, out_table_name=coefficients_table)
logger.end_block()
logger.start_block("Writing specification and coefficients into %s" % AttributeCache().get_storage_location())
self.specification.write(out_storage=AttributeCache(), out_table_name=specification_table)
self.coefficients.write(out_storage=AttributeCache(), out_table_name=coefficients_table)
logger.end_block()
def run(self, config, year):
"""Running MATSim. A lot of paths are relative; the base path is ${OPUS_HOME}/opus_matsim. As long as ${OPUS_HOME}
is correctly set and the matsim tarfile was unpacked in OPUS_HOME, this should work out of the box. There may eventually
be problems with the java version.
"""
logger.start_block("Starting RunTravelModel.run(...)")
travel_model_configuration = config['travel_model_configuration']
# The default config file name
matsim_config_filename = travel_model_configuration[
'matsim_config_filename']
# over-written if there is a specific config file name for the year:
if travel_model_configuration[year]['matsim_config_filename']:
matsim_config_filename = travel_model_configuration[year][
'matsim_config_filename']
cmd = """cd %(opus_home)s/opus_matsim ; java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s --year=%(year)i --samplingRate=%(sampling_rate)f""" % {
'opus_home':
os.environ['OPUS_HOME'],
'vmargs':
"-Xmx2000m",
'classpath':
"classes:jar/MATSim.jar",
'javaclass':
"playground.run.Matsim4Urbansim",
'matsim_config_file':
os.path.join(os.environ['OPUS_HOME'], "opus_matsim",
matsim_config_filename),
'sampling_rate':
config['travel_model_configuration']['sampling_rate'],
'year':
year
}
logger.log_status('Running command %s' % cmd)
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "Matsim Run failed. Code returned by cmd was %d" % (
cmd_result)
logger.log_error(error_msg)
logger.log_error(
"Note that currently (dec/08), paths in the matsim config files are relative to the opus_matsim root,"
)
logger.log_error(" which is one level 'down' from OPUS_HOME.")
raise StandardError(error_msg)
logger.end_block()
def create_file_cache_directories(directory,
prefix='',
file_name='cache_directories'):
logger.start_block('Creating file %s in %s' % (file_name, directory))
all_dirs = os.listdir(directory)
all_dirs = [x for x in all_dirs if x.startswith(prefix)]
if not prefix.startswith('.'):
all_dirs = [x for x in all_dirs if not x.startswith('.')]
for i in range(len(all_dirs)):
all_dirs[i] = os.path.join(directory, all_dirs[i])
write_to_text_file(os.path.join(directory, file_name), all_dirs)
logger.end_block()
def _write_to_txt_file(self, data, header, input_file, delimiter=','):
logger.start_block("Writing to travel_model input file %s" %
input_file)
newfile = open(input_file, 'w')
newfile.write(delimiter.join(header).replace("tm.", "") + "\n")
for row in range(len(data[0])):
for col in range(len(header)):
newfile.write(str(data[col][row]))
if col < len(header) - 1:
newfile.write(",")
newfile.write("\n")
newfile.close()
logger.end_block()
def _print_table(self, table_name):
"""Provide debugging info to figure out why the above test is failing, sometimes."""
try:
results = self.db.GetResultsFromQuery('select * from %s' %
table_name)
logger.start_block('Contents of table %s' % table_name)
try:
for row in results:
logger.log_status(row)
finally:
logger.end_block()
except:
logger.log_status('Error accessing table %s' % table_name)
logger.log_stack_trace()
def _create_db_from_chain_via_python(self, from_database_configuration,
to_database_configuration,
tables_to_copy):
db_server_from = DatabaseServer(from_database_configuration)
db_server_to = DatabaseServer(to_database_configuration)
db_server_to.drop_database(to_database_configuration.database_name)
db_server_to.create_database(to_database_configuration.database_name)
database_out = db_server_to.get_database(
to_database_configuration.database_name)
scenario_db_manager = ScenarioDatabaseManager(
server_configuration=from_database_configuration,
base_scenario_database_name=from_database_configuration.
database_name)
table_mapping = scenario_db_manager.get_database_to_table_mapping()
cross_db_operations = CrossDatabaseOperations()
#by default, copy all tables
if tables_to_copy == []:
tables_to_copy = sum(table_mapping.values(),
[]) # flat a list of lists
elif 'scenario_information' not in tables_to_copy:
tables_to_copy.append('scenario_information')
for database_name, tables in table_mapping.items():
database_in = db_server_from.get_database(database_name)
for table in tables:
if table not in tables_to_copy:
continue
logger.start_block("Copying table '%s' from database '%s'" %
(table, database_name))
try:
cross_db_operations.copy_table(table_to_copy=table,
database_in=database_in,
database_out=database_out,
use_chunking=True)
finally:
logger.end_block()
database_in.close()
self._fix_scenario_information_table(database_out)
database_out.close()
db_server_from.close()
db_server_to.close()
def _run_each_year_as_separate_process(
self,
start_year,
end_year,
seed_array,
resources,
log_file_name='run_multiprocess.log'):
skip_first_year_of_urbansim = resources.get('skip_urbansim', False)
log_file = os.path.join(resources['cache_directory'], log_file_name)
profiler_name = resources.get("profile_filename", None)
iyear = 0
for year in range(start_year, end_year + 1):
if (year <> start_year) or ((year == start_year) and
(not skip_first_year_of_urbansim)):
logger.start_block(
'Running UrbanSim for year %d in new process' % year)
try:
resources['years'] = (year, year)
resources['seed'] = seed_array[iyear],
logger.disable_file_logging(log_file)
if profiler_name is not None:
resources["profile_filename"] = "%s_%s" % (
profiler_name, year
) # add year to the profile name
self._fork_new_process(
'urbansim.model_coordinators.model_system',
resources,
optional_args=['--log-file-name', log_file_name])
logger.enable_file_logging(log_file, verbose=False)
finally:
logger.end_block()
if ('travel_model_configuration' in resources) and (
not resources.get('skip_travel_model', False)):
# tnicolai add start year to travel model config
tmc = resources['travel_model_configuration']
tmc['start_year'] = start_year # end tnicolai
self._run_travel_models_in_separate_processes(
resources['travel_model_configuration'], year, resources)
if 'post_year_configuration' in resources:
self._run_travel_models_in_separate_processes(
resources['post_year_configuration'], year, resources)
iyear += 1
self._notify_stopped()
def export(self, in_storage, out_storage):
dataset_names = in_storage.get_table_names()
logger.start_block('Exporting tables')
logger.log_status("Reading tables from '%s'" %
in_storage.get_storage_location())
if not dataset_names:
logger.log_warning('This location has no tables to export!')
logger.log_warning(
'Did you specify a location containing the data for a single year?'
)
else:
for dataset_name in dataset_names:
self.export_dataset(dataset_name, in_storage, out_storage)
logger.end_block()
def copy_scenario_database(self,
from_database_configuration,
to_database_configuration,
tables_to_copy = []):
logger.start_block("Copying tables from database chain starting at '%s' on '%s'\nto database '%s' on '%s'"
% (from_database_configuration.database_name,
from_database_configuration.host_name,
to_database_configuration.database_name,
to_database_configuration.host_name))
try:
self._create_db_from_chain_via_python(
from_database_configuration = from_database_configuration,
to_database_configuration = to_database_configuration,
tables_to_copy = tables_to_copy)
finally:
logger.end_block()
def run_estimation(self,
estimation_config,
model_name,
save_estimation_results=True):
config = Baseline()
config.merge(estimation_config)
config['config_changes_for_estimation'] = ConfigChangesForEstimation()
logger.start_block('Estimating %s' % model_name)
try:
estimator = UrbansimEstimationRunner(
models[model_name][0],
specification_module=models[model_name][1],
model_group=models[model_name][2],
configuration=config,
save_estimation_results=save_estimation_results)
estimator.estimate()
finally:
logger.end_block()
def cache_database_table(self, table_name, base_year, database, in_storage,
config):
"""Copy this table from input database into attribute cache.
"""
logger.start_block('Caching table %s' % table_name)
try:
#TODO: why is the config being modified...seems like its kind of useless here...
config['storage_location'] = os.path.join(
config['cache_directory'], str(base_year), table_name)
if not os.path.exists(config['storage_location']):
flt_storage = StorageFactory().get_storage(
type='flt_storage',
subdir='store',
storage_location=config['storage_location'])
table = database.get_table(table_name)
id_name = [
primary_key.name.lower()
for primary_key in table.primary_key
]
dataset = Dataset(resources=config,
in_storage=in_storage,
out_storage=flt_storage,
in_table_name=table_name,
id_name=id_name)
nchunks = config[
'creating_baseyear_cache_configuration'].tables_to_cache_nchunks.get(
table_name, 1)
current_time = SimulationState().get_current_time()
SimulationState().set_current_time(base_year)
dataset.load_dataset(nchunks=nchunks,
flush_after_each_chunk=True)
SimulationState().set_current_time(current_time)
else:
logger.log_status(config['storage_location'] +
" already exits; skip caching " + table_name)
finally:
logger.end_block()
def run(self, config, year):
"""Running MATSim. A lot of paths are relative; the base path is ${OPUS_HOME}/opus_matsim. As long as ${OPUS_HOME}
is correctly set and the matsim tarfile was unpacked in OPUS_HOME, this should work out of the box. There may eventually
be problems with the java version.
"""
try:
import pydevd
pydevd.settrace()
except:
pass
logger.start_block("Starting RunTravelModel.run(...)")
self.setUp(config)
config_obj = MATSimConfigObject(config, year, self.matsim_config_full)
config_obj.marschall()
cmd = """cd %(opus_home)s/opus_matsim ; java %(vmargs)s -cp %(classpath)s %(javaclass)s %(matsim_config_file)s""" % {
'opus_home': os.environ['OPUS_HOME'],
'vmargs': "-Xmx2000m",
'classpath':
"libs/log4j/log4j/1.2.15/log4j-1.2.15.jar:libs/jfree/jfreechart/1.0.7/jfreechart-1.0.7.jar:libs/jfree/jcommon/1.0.9/jcommon-1.0.9.jar:classesMATSim:classesToronto:classesTNicolai:classesKai:classesEntry", # 'classpath': "classes:jar/MATSim.jar",
'javaclass': "playground.run.Matsim4Urbansim",
'matsim_config_file': self.matsim_config_full
}
logger.log_status('Running command %s' % cmd)
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "Matsim Run failed. Code returned by cmd was %d" % (
cmd_result)
logger.log_error(error_msg)
logger.log_error(
"Note that currently (dec/08), paths in the matsim config files are relative to the opus_matsim root,"
)
logger.log_error(" which is one level 'down' from OPUS_HOME.")
raise StandardError(error_msg)
logger.end_block()
def _write_to_file(self, zone_set, variables_list, tm_input_file):
logger.start_block("Writing to emme2 input file: " + tm_input_file)
try:
newfile = open(tm_input_file, 'w')
"""write travel model input file into a particular file format emme2 can read"""
try:
newfile.write(r"""c from
c prepared: %s
t matrices
m matrix="hhemp"
""" % time.strftime("%c", time.localtime(time.time())))
line_template = "%4d %3d: %8.2f \n"
for taz_id in zone_set.get_id_attribute():
for i in range(101, 125):
newfile.write(line_template % (taz_id, i, self._get_value_for_zone(taz_id, zone_set, variables_list[i-101])))
finally:
newfile.close()
finally:
logger.end_block()
return tm_input_file
def _put_one_matrix_into_travel_data_set(self,
travel_data_set,
max_zone_id,
matrix_name,
attribute_name,
bank_path,
matrices_created=False):
"""
Adds to the given travel_data_set the data for the given matrix
that is in the emme/2 data bank.
"""
logger.start_block('Copying data for matrix %s into variable %s' %
(matrix_name, attribute_name))
try:
if not matrices_created:
self._get_matrix_into_data_file(matrix_name, max_zone_id,
bank_path)
file_name = "_one_matrix.txt"
else:
file_name = "%s_one_matrix.txt" % matrix_name
file_contents = self._get_emme2_data_from_file(
join(bank_path, file_name))
travel_data_set.add_primary_attribute(data=zeros(
travel_data_set.size(), dtype=float32),
name=attribute_name)
odv = array([line.split() for line in file_contents],
dtype=float32)
if odv.size == 0:
logger.log_error(
"Skipped exporting travel_data attribute %s: No data is exported from EMME matrix."
% attribute_name)
else:
travel_data_set.set_values_of_one_attribute_with_od_pairs(
attribute=attribute_name,
values=odv[:, 2],
O=odv[:, 0].astype('int32'),
D=odv[:, 1].astype('int32'))
finally:
logger.end_block()