def prepare_for_run(self, configuration_path=None, config=None, run_id=None, prepare_cache=True, run_name = None):
"""Configuration is given either as an opus path (configuration_path) or as a Configuration object (config)."""
if run_id is not None:
self.run_id = run_id
config = self.get_run_manager().get_resources_for_run_id_from_history(run_id=self.run_id)
self.remote_communication_path = '%s/%s' % (self.remote_communication_path_root, self.run_id)
else:
if configuration_path is not None:
opus_path = configuration_path
config = get_config_from_opus_path(opus_path)
else:
if config is None:
raise StandardError, "Either configuration_path, config or run_id must be given."
insert_auto_generated_cache_directory_if_needed(config)
self.run_id = self.get_run_manager()._get_new_run_id()
head, tail = os.path.split(config['cache_directory'])
config['cache_directory'] = '%s/run_%s.%s' % (head, self.run_id, tail)
self.remote_communication_path = '%s/%s' % (self.remote_communication_path_root, self.run_id)
if not self.skip_urbansim and prepare_cache:
self.prepare_cache_and_communication_path(config)
self.get_run_manager().add_row_to_history(self.run_id, config, "started", run_name = None)
#check that run_id must exist
results = self.get_run_manager().services_db.GetResultsFromQuery(
"SELECT * from run_activity WHERE run_id = %s " % self.run_id)
if not len(results) > 1:
raise StandardError, "run_id %s doesn't exist in run_activity table." % self.run_id
self.set_local_output_path()
return config
def prepare_for_run(self,
configuration_path=None,
config=None,
run_id=None):
"""Configuration is given either as an opus path (configuration_path) or as a Configuration object (config)."""
run_manager = self.get_run_manager()
if run_id is not None:
config = run_manager.get_resources_for_run_id_from_history(
run_id=run_id)
else:
if configuration_path is not None:
config = get_config_from_opus_path(configuration_path)
elif config is None:
raise StandardError, "Either configuration_path, config or run_id must be given."
insert_auto_generated_cache_directory_if_needed(config)
run_manager.setup_new_run(
cache_directory=config['cache_directory'],
configuration=config)
run_id = run_manager.run_id
config['cache_directory'] = pathname2url(
run_manager.get_current_cache_directory())
## pathname2url converts '\' or '\\' to '/'; it is necessary when this script is invoked from a nt os
run_manager.add_row_to_history(run_id, config, "started")
#verify run_id has been added to services db
results = run_manager.services_db.GetResultsFromQuery(
"SELECT * from run_activity WHERE run_id = %s " % run_id)
if not len(results) > 1:
raise StandardError, "run_id %s doesn't exist in run_activity table." % run_id
return run_id, config
def prepare_for_run(self, configuration_path=None, config=None, run_id=None, prepare_cache=True, run_name = None):
"""Configuration is given either as an opus path (configuration_path) or as a Configuration object (config)."""
if run_id is not None:
self.run_id = run_id
config = self.get_run_manager().get_resources_for_run_id_from_history(run_id=self.run_id)
self.remote_communication_path = '%s/%s' % (self.remote_communication_path_root, self.run_id)
else:
if configuration_path is not None:
opus_path = configuration_path
config = get_config_from_opus_path(opus_path)
else:
if config is None:
raise StandardError, "Either configuration_path, config or run_id must be given."
insert_auto_generated_cache_directory_if_needed(config)
self.run_id = self.get_run_manager()._get_new_run_id()
head, tail = os.path.split(config['cache_directory'])
config['cache_directory'] = '%s/run_%s.%s' % (head, self.run_id, tail)
self.remote_communication_path = '%s/%s' % (self.remote_communication_path_root, self.run_id)
if not self.skip_urbansim and prepare_cache:
self.prepare_cache_and_communication_path(config)
self.get_run_manager().add_row_to_history(self.run_id, config, "started", run_name = None)
#check that run_id must exist
results = self.get_run_manager().services_db.GetResultsFromQuery(
"SELECT * from run_activity WHERE run_id = %s " % self.run_id)
if not len(results) > 1:
raise StandardError, "run_id %s doesn't exist in run_activity table." % self.run_id
self.set_local_output_path()
return config
def prepare_for_run(self, configuration_path=None, config=None, run_id=None):
"""Configuration is given either as an opus path (configuration_path) or as a Configuration object (config)."""
run_manager = self.get_run_manager()
if run_id is not None:
config = run_manager.get_resources_for_run_id_from_history(run_id=run_id)
else:
if configuration_path is not None:
config = get_config_from_opus_path(configuration_path)
elif config is None:
raise StandardError, "Either configuration_path, config or run_id must be given."
insert_auto_generated_cache_directory_if_needed(config)
run_manager.setup_new_run(cache_directory = config['cache_directory'],
configuration = config)
run_id = run_manager.run_id
config['cache_directory'] = pathname2url(run_manager.get_current_cache_directory())
## pathname2url converts '\' or '\\' to '/'; it is necessary when this script is invoked from a nt os
run_manager.add_row_to_history(run_id, config, "started")
#verify run_id has been added to services db
results = run_manager.services_db.GetResultsFromQuery(
"SELECT * from run_activity WHERE run_id = %s " % run_id)
if not len(results) > 1:
raise StandardError, "run_id %s doesn't exist in run_activity table." % run_id
return run_id, config
def main():
option_group = EstimationOptionGroup()
parser = option_group.parser
(options, args) = parser.parse_args()
if options.model_name is None:
raise StandardError, "Model name (argument -m) must be given."
if (options.configuration_path is None) and (options.xml_configuration is
None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.specification is None) and (options.xml_configuration is None):
logger.log_warning(
"No specification given (arguments -s or -x). Specification taken from the cache."
)
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
estimator = EstimationRunner(model=options.model_name,
specification_module=options.specification,
xml_configuration=xconfig,
model_group=options.model_group,
configuration=config,
save_estimation_results=options.save_results)
estimator.estimate()
return estimator
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs):
self.run_ids_dict = {} # dict of run_id and finished year
if run_id_file is None:
config = get_config_from_opus_path(configuration_path)
self.number_of_runs = config.get("number_of_runs", 1)
root_seed = config.get("seed", None)
seed(root_seed)
# generate different seed for each run (each seed contains 1 number)
seed_array = randint(1, 2**30, self.number_of_runs)
for irun in range(self.number_of_runs):
config['seed'] = (seed_array[irun], )
RemoteRun.prepare_for_run(self,
config=config,
prepare_cache=False)
self.run_ids_dict[self.run_id] = (0, 'NA')
self.run_id_file = self.default_run_id_file
else:
self.read_run_id_file(run_id_file)
self.run_id_file = run_id_file
self.get_run_manager()
self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime())
logger.log_status("run_id_file: %s" % self.run_id_file)
self.write_into_run_id_file()
return None
def main():
option_group = EstimationOptionGroup()
parser = option_group.parser
(options, args) = parser.parse_args()
if options.model_name is None:
raise StandardError, "Model name (argument -m) must be given."
if (options.configuration_path is None) and (options.xml_configuration is None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.specification is None) and (options.xml_configuration is None):
logger.log_warning("No specification given (arguments -s or -x). Specification taken from the cache.")
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
estimator = EstimationRunner(model=options.model_name,
specification_module=options.specification,
xml_configuration=xconfig,
model_group=options.model_group,
configuration=config,
save_estimation_results=options.save_results)
estimator.estimate()
return estimator
def prepare_run_manager(option_group=None):
if option_group is None:
option_group = StartRunOptionGroup()
parser = option_group.parser
options, args = option_group.parse()
run_manager = RunManager(
option_group.get_services_database_configuration(options))
if options.pickled_resource_file is not None:
f = file(options.pickled_resource_file, 'r')
try:
config = pickle.load(f)
finally:
f.close()
elif options.configuration_path is not None:
opus_path = options.configuration_path
try:
config = get_config_from_opus_path(opus_path)
except ImportError:
# TODO: Once all fully-specified configurations are stored as classes,
# get rid of this use.
import_stmt = 'from %s import run_configuration as config' % opus_path
exec(import_stmt)
insert_auto_generated_cache_directory_if_needed(config)
elif options.xml_configuration is not None:
if options.scenario_name is None:
parser.print_help()
sys.exit(1)
config = XMLConfiguration(
options.xml_configuration).get_run_configuration(
options.scenario_name)
insert_auto_generated_cache_directory_if_needed(config)
else:
parser.print_help()
sys.exit(1)
if options.existing_cache_to_copy is not None:
config[
'creating_baseyear_cache_configuration'].cache_from_database = False
config[
'creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration(
existing_cache_to_copy=options.existing_cache_to_copy, )
if options.years_to_cache is not None:
config[
'creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(
options.years_to_cache)
if options.profile_filename is not None:
config["profile_filename"] = options.profile_filename
run_manager.setup_new_run(cache_directory=config['cache_directory'],
configuration=config)
return options, config, run_manager
def prepare_run_manager(option_group=None):
if option_group is None:
option_group = StartRunOptionGroup()
parser = option_group.parser
options, args = option_group.parse()
run_manager = RunManager(option_group.get_services_database_configuration(options))
if options.pickled_resource_file is not None:
f = file(options.pickled_resource_file, 'r')
try:
config = pickle.load(f)
finally:
f.close()
elif options.configuration_path is not None:
opus_path = options.configuration_path
try:
config = get_config_from_opus_path(opus_path)
except ImportError:
# TODO: Once all fully-specified configurations are stored as classes,
# get rid of this use.
import_stmt = 'from %s import run_configuration as config' % opus_path
exec(import_stmt)
insert_auto_generated_cache_directory_if_needed(config)
elif options.xml_configuration is not None:
if options.scenario_name is None:
parser.print_help()
sys.exit(1)
config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name)
insert_auto_generated_cache_directory_if_needed(config)
else:
parser.print_help()
sys.exit(1)
if options.existing_cache_to_copy is not None:
config['creating_baseyear_cache_configuration'].cache_from_database = False
config['creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration(
existing_cache_to_copy = options.existing_cache_to_copy,
)
if options.years_to_cache is not None:
config['creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(options.years_to_cache)
if options.profile_filename is not None:
config["profile_filename"] = options.profile_filename
run_manager.setup_new_run(cache_directory = config['cache_directory'],
configuration = config)
return options, config, run_manager
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs):
self.run_ids_dict = {} # dict of run_id and finished year
if run_id_file is None:
config = get_config_from_opus_path(configuration_path)
self.number_of_runs = config.get("number_of_runs", 1)
root_seed = config.get("seed", None)
seed(root_seed)
# generate different seed for each run (each seed contains 1 number)
seed_array = randint(1,2**30, self.number_of_runs)
for irun in range(self.number_of_runs):
config['seed']= (seed_array[irun],)
RemoteRun.prepare_for_run(self, config=config, prepare_cache=False)
self.run_ids_dict[self.run_id] = (0, 'NA')
self.run_id_file = self.default_run_id_file
else:
self.read_run_id_file(run_id_file)
self.run_id_file = run_id_file
self.get_run_manager()
self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime())
logger.log_status("run_id_file: %s" % self.run_id_file)
self.write_into_run_id_file()
return None
def main():
import sys
option_group = ModelExplorerOptionGroup()
parser = option_group.parser
(options, args) = parser.parse_args()
if options.year is None:
raise StandardError, "Year (argument -y) must be given."
if options.cache_directory is None:
raise StandardError, "Cache directory (argument -d) must be given."
if (options.configuration_path is None) and (options.xml_configuration is None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.scenario_name is None) and (options.xml_configuration is not None):
raise StandardError, "No scenario given (argument -s). Must be specified if option -x is used."
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
if options.cache_directory == 'BASE':
cache_directory = None
else:
cache_directory = options.cache_directory
explorer = ModelExplorer(model=options.model_name, year=int(options.year),
scenario_name=options.scenario_name,
model_group=options.model_group,
configuration=config,
xml_configuration=xconfig,
cache_directory=cache_directory)
explorer.run()
return explorer
)
parser.add_option(
"--database-configuration",
dest="database_configuration",
default=None,
action="store",
help=
"Name of the scenario database server configuration in database_server_configurations.xml."
)
(options, args) = parser.parse_args()
if options.configuration_path is not None:
opus_path = options.configuration_path
try:
config = get_config_from_opus_path(opus_path)
except ImportError:
import_stmt = 'from %s import run_configuration as config' % opus_path
exec(import_stmt)
elif options.xml_configuration is not None:
if options.scenario_name is None:
parser.print_help()
sys.exit(1)
config = XMLConfiguration(
options.xml_configuration).get_run_configuration(
options.scenario_name)
else:
parser.print_help()
sys.exit(1)
if options.cache_directory is not None:
parser = option_group.parser
(options, args) = parser.parse_args()
run_manager = RunManager(option_group.get_services_database_configuration(options))
if options.pickled_resource_file is not None:
f = file(options.pickled_resource_file, 'r')
try:
config = pickle.load(f)
finally:
f.close()
elif options.configuration_path is not None:
opus_path = options.configuration_path
try:
config = get_config_from_opus_path(opus_path)
except ImportError:
# TODO: Once all fully-specified configurations are stored as classes,
# get rid of this use.
import_stmt = 'from %s import run_configuration as config' % opus_path
exec(import_stmt)
insert_auto_generated_cache_directory_if_needed(config)
elif options.xml_configuration is not None:
if options.scenario_name is None:
parser.print_help()
sys.exit(1)
config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name)
insert_auto_generated_cache_directory_if_needed(config)
else:
parser.print_help()
sys.exit(1)
parser = option_group.parser
(options, args) = parser.parse_args()
if options.model_name is None:
raise StandardError, "Model name (argument -m) must be given."
if (options.configuration_path is None) and (options.xml_configuration is None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.specification is None) and (options.xml_configuration is None):
logger.log_warning("No specification given (arguments -s or -x). Specification taken from the cache.")
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
estimator = EstimationRunner(model=options.model_name,
specification_module=options.specification,
xml_configuration=xconfig,
model_group=options.model_group,
configuration=config,
save_estimation_results=options.save_results)
estimator.estimate()
er = estimator
ds = er.get_data_as_dataset()
ds.summary()
ds.get_attribute("ln_residential_units").shape
estimator.create_prediction_success_table(summarize_by="fazdistrict_id=building.disaggregate(faz.fazdistrict_id, intermediates=[zone, parcel])")
if options.cache_directory is None:
raise StandardError, "Cache directory (argument -d) must be given."
if (options.configuration_path is None) and (options.xml_configuration is
None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.scenario_name is None) and (options.xml_configuration
is not None):
raise StandardError, "No scenario given (argument -s). Must be specified if option -x is used."
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
if options.cache_directory == 'BASE':
cache_directory = None
else:
cache_directory = options.cache_directory
explorer = ModelExplorer(model=options.model_name,
year=int(options.year),
scenario_name=options.scenario_name,
model_group=options.model_group,
configuration=config,
xml_configuration=xconfig,
cache_directory=cache_directory)
explorer.run()
ex = explorer
from opus_core.file_utilities import get_resources_from_file
parser = OptionParser()
parser.add_option("-r", "--resources", dest="resources_file_name", action="store", type="string",
help="Name of file containing resources", default=None)
parser.add_option("-y", "--year", dest="year", action="store", type="int",
help="Year in which to 'run' the travel model")
#these options were added by SFCTA
parser.add_option("-c", "--alt-config", dest="alt_config", action="store", type="string",
help="opus path to config if pickled resources not available", default=None)
parser.add_option("-d", "--alt-cache-dir", dest="alt_cache_dir", action="store", type="string",
help="cache directory if pickled resources not available", default=None)
(options, args) = parser.parse_args()
#options here added by SFCTA
if (options.alt_config is not None) and (options.alt_cache_dir is not None):
resources=get_config_from_opus_path(options.alt_config)
resources['cache_directory']=options.alt_cache_dir
elif options.resources_file_name is not None:
r = get_resources_from_file(options.resources_file_name)
resources = Resources(get_resources_from_file(options.resources_file_name))
else:
parser.print_help()
sys.exit(1)
SessionConfiguration(new_instance=True,
package_order=resources['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
# logger.enable_memory_logging()
GetTravelModelDataIntoCache().run(resources, options.year)
help="opus path to config if pickled resources not available",
default=None)
parser.add_option(
"-d",
"--alt-cache-dir",
dest="alt_cache_dir",
action="store",
type="string",
help="cache directory if pickled resources not available",
default=None)
(options, args) = parser.parse_args()
#options here added by SFCTA
if (options.alt_config is not None) and (options.alt_cache_dir
is not None):
resources = get_config_from_opus_path(options.alt_config)
resources['cache_directory'] = options.alt_cache_dir
elif options.resources_file_name is not None:
r = get_resources_from_file(options.resources_file_name)
resources = Resources(
get_resources_from_file(options.resources_file_name))
else:
parser.print_help()
sys.exit(1)
SessionConfiguration(
new_instance=True,
package_order=resources['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
# logger.enable_memory_logging()
