def prepare_for_run(self, configuration_path=None, config=None, run_id=None, prepare_cache=True, run_name = None): """Configuration is given either as an opus path (configuration_path) or as a Configuration object (config).""" if run_id is not None: self.run_id = run_id config = self.get_run_manager().get_resources_for_run_id_from_history(run_id=self.run_id) self.remote_communication_path = '%s/%s' % (self.remote_communication_path_root, self.run_id) else: if configuration_path is not None: opus_path = configuration_path config = get_config_from_opus_path(opus_path) else: if config is None: raise StandardError, "Either configuration_path, config or run_id must be given." insert_auto_generated_cache_directory_if_needed(config) self.run_id = self.get_run_manager()._get_new_run_id() head, tail = os.path.split(config['cache_directory']) config['cache_directory'] = '%s/run_%s.%s' % (head, self.run_id, tail) self.remote_communication_path = '%s/%s' % (self.remote_communication_path_root, self.run_id) if not self.skip_urbansim and prepare_cache: self.prepare_cache_and_communication_path(config) self.get_run_manager().add_row_to_history(self.run_id, config, "started", run_name = None) #check that run_id must exist results = self.get_run_manager().services_db.GetResultsFromQuery( "SELECT * from run_activity WHERE run_id = %s " % self.run_id) if not len(results) > 1: raise StandardError, "run_id %s doesn't exist in run_activity table." % self.run_id self.set_local_output_path() return config
def prepare_for_run(self, configuration_path=None, config=None, run_id=None): """Configuration is given either as an opus path (configuration_path) or as a Configuration object (config).""" run_manager = self.get_run_manager() if run_id is not None: config = run_manager.get_resources_for_run_id_from_history( run_id=run_id) else: if configuration_path is not None: config = get_config_from_opus_path(configuration_path) elif config is None: raise StandardError, "Either configuration_path, config or run_id must be given." insert_auto_generated_cache_directory_if_needed(config) run_manager.setup_new_run( cache_directory=config['cache_directory'], configuration=config) run_id = run_manager.run_id config['cache_directory'] = pathname2url( run_manager.get_current_cache_directory()) ## pathname2url converts '\' or '\\' to '/'; it is necessary when this script is invoked from a nt os run_manager.add_row_to_history(run_id, config, "started") #verify run_id has been added to services db results = run_manager.services_db.GetResultsFromQuery( "SELECT * from run_activity WHERE run_id = %s " % run_id) if not len(results) > 1: raise StandardError, "run_id %s doesn't exist in run_activity table." % run_id return run_id, config
def prepare_for_run(self, configuration_path=None, config=None, run_id=None): """Configuration is given either as an opus path (configuration_path) or as a Configuration object (config).""" run_manager = self.get_run_manager() if run_id is not None: config = run_manager.get_resources_for_run_id_from_history(run_id=run_id) else: if configuration_path is not None: config = get_config_from_opus_path(configuration_path) elif config is None: raise StandardError, "Either configuration_path, config or run_id must be given." insert_auto_generated_cache_directory_if_needed(config) run_manager.setup_new_run(cache_directory = config['cache_directory'], configuration = config) run_id = run_manager.run_id config['cache_directory'] = pathname2url(run_manager.get_current_cache_directory()) ## pathname2url converts '\' or '\\' to '/'; it is necessary when this script is invoked from a nt os run_manager.add_row_to_history(run_id, config, "started") #verify run_id has been added to services db results = run_manager.services_db.GetResultsFromQuery( "SELECT * from run_activity WHERE run_id = %s " % run_id) if not len(results) > 1: raise StandardError, "run_id %s doesn't exist in run_activity table." % run_id return run_id, config
def main(): option_group = EstimationOptionGroup() parser = option_group.parser (options, args) = parser.parse_args() if options.model_name is None: raise StandardError, "Model name (argument -m) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.specification is None) and (options.xml_configuration is None): logger.log_warning( "No specification given (arguments -s or -x). Specification taken from the cache." ) if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) estimator = EstimationRunner(model=options.model_name, specification_module=options.specification, xml_configuration=xconfig, model_group=options.model_group, configuration=config, save_estimation_results=options.save_results) estimator.estimate() return estimator
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs): self.run_ids_dict = {} # dict of run_id and finished year if run_id_file is None: config = get_config_from_opus_path(configuration_path) self.number_of_runs = config.get("number_of_runs", 1) root_seed = config.get("seed", None) seed(root_seed) # generate different seed for each run (each seed contains 1 number) seed_array = randint(1, 2**30, self.number_of_runs) for irun in range(self.number_of_runs): config['seed'] = (seed_array[irun], ) RemoteRun.prepare_for_run(self, config=config, prepare_cache=False) self.run_ids_dict[self.run_id] = (0, 'NA') self.run_id_file = self.default_run_id_file else: self.read_run_id_file(run_id_file) self.run_id_file = run_id_file self.get_run_manager() self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime()) logger.log_status("run_id_file: %s" % self.run_id_file) self.write_into_run_id_file() return None
def main(): option_group = EstimationOptionGroup() parser = option_group.parser (options, args) = parser.parse_args() if options.model_name is None: raise StandardError, "Model name (argument -m) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.specification is None) and (options.xml_configuration is None): logger.log_warning("No specification given (arguments -s or -x). Specification taken from the cache.") if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) estimator = EstimationRunner(model=options.model_name, specification_module=options.specification, xml_configuration=xconfig, model_group=options.model_group, configuration=config, save_estimation_results=options.save_results) estimator.estimate() return estimator
def prepare_run_manager(option_group=None): if option_group is None: option_group = StartRunOptionGroup() parser = option_group.parser options, args = option_group.parse() run_manager = RunManager( option_group.get_services_database_configuration(options)) if options.pickled_resource_file is not None: f = file(options.pickled_resource_file, 'r') try: config = pickle.load(f) finally: f.close() elif options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) insert_auto_generated_cache_directory_if_needed(config) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration( options.xml_configuration).get_run_configuration( options.scenario_name) insert_auto_generated_cache_directory_if_needed(config) else: parser.print_help() sys.exit(1) if options.existing_cache_to_copy is not None: config[ 'creating_baseyear_cache_configuration'].cache_from_database = False config[ 'creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration( existing_cache_to_copy=options.existing_cache_to_copy, ) if options.years_to_cache is not None: config[ 'creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval( options.years_to_cache) if options.profile_filename is not None: config["profile_filename"] = options.profile_filename run_manager.setup_new_run(cache_directory=config['cache_directory'], configuration=config) return options, config, run_manager
def prepare_run_manager(option_group=None): if option_group is None: option_group = StartRunOptionGroup() parser = option_group.parser options, args = option_group.parse() run_manager = RunManager(option_group.get_services_database_configuration(options)) if options.pickled_resource_file is not None: f = file(options.pickled_resource_file, 'r') try: config = pickle.load(f) finally: f.close() elif options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) insert_auto_generated_cache_directory_if_needed(config) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name) insert_auto_generated_cache_directory_if_needed(config) else: parser.print_help() sys.exit(1) if options.existing_cache_to_copy is not None: config['creating_baseyear_cache_configuration'].cache_from_database = False config['creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration( existing_cache_to_copy = options.existing_cache_to_copy, ) if options.years_to_cache is not None: config['creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(options.years_to_cache) if options.profile_filename is not None: config["profile_filename"] = options.profile_filename run_manager.setup_new_run(cache_directory = config['cache_directory'], configuration = config) return options, config, run_manager
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs): self.run_ids_dict = {} # dict of run_id and finished year if run_id_file is None: config = get_config_from_opus_path(configuration_path) self.number_of_runs = config.get("number_of_runs", 1) root_seed = config.get("seed", None) seed(root_seed) # generate different seed for each run (each seed contains 1 number) seed_array = randint(1,2**30, self.number_of_runs) for irun in range(self.number_of_runs): config['seed']= (seed_array[irun],) RemoteRun.prepare_for_run(self, config=config, prepare_cache=False) self.run_ids_dict[self.run_id] = (0, 'NA') self.run_id_file = self.default_run_id_file else: self.read_run_id_file(run_id_file) self.run_id_file = run_id_file self.get_run_manager() self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime()) logger.log_status("run_id_file: %s" % self.run_id_file) self.write_into_run_id_file() return None
def main(): import sys option_group = ModelExplorerOptionGroup() parser = option_group.parser (options, args) = parser.parse_args() if options.year is None: raise StandardError, "Year (argument -y) must be given." if options.cache_directory is None: raise StandardError, "Cache directory (argument -d) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.scenario_name is None) and (options.xml_configuration is not None): raise StandardError, "No scenario given (argument -s). Must be specified if option -x is used." if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) if options.cache_directory == 'BASE': cache_directory = None else: cache_directory = options.cache_directory explorer = ModelExplorer(model=options.model_name, year=int(options.year), scenario_name=options.scenario_name, model_group=options.model_group, configuration=config, xml_configuration=xconfig, cache_directory=cache_directory) explorer.run() return explorer
) parser.add_option( "--database-configuration", dest="database_configuration", default=None, action="store", help= "Name of the scenario database server configuration in database_server_configurations.xml." ) (options, args) = parser.parse_args() if options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration( options.xml_configuration).get_run_configuration( options.scenario_name) else: parser.print_help() sys.exit(1) if options.cache_directory is not None:
parser = option_group.parser (options, args) = parser.parse_args() run_manager = RunManager(option_group.get_services_database_configuration(options)) if options.pickled_resource_file is not None: f = file(options.pickled_resource_file, 'r') try: config = pickle.load(f) finally: f.close() elif options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) insert_auto_generated_cache_directory_if_needed(config) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name) insert_auto_generated_cache_directory_if_needed(config) else: parser.print_help() sys.exit(1)
parser = option_group.parser (options, args) = parser.parse_args() if options.model_name is None: raise StandardError, "Model name (argument -m) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.specification is None) and (options.xml_configuration is None): logger.log_warning("No specification given (arguments -s or -x). Specification taken from the cache.") if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) estimator = EstimationRunner(model=options.model_name, specification_module=options.specification, xml_configuration=xconfig, model_group=options.model_group, configuration=config, save_estimation_results=options.save_results) estimator.estimate() er = estimator ds = er.get_data_as_dataset() ds.summary() ds.get_attribute("ln_residential_units").shape estimator.create_prediction_success_table(summarize_by="fazdistrict_id=building.disaggregate(faz.fazdistrict_id, intermediates=[zone, parcel])")
if options.cache_directory is None: raise StandardError, "Cache directory (argument -d) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.scenario_name is None) and (options.xml_configuration is not None): raise StandardError, "No scenario given (argument -s). Must be specified if option -x is used." if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) if options.cache_directory == 'BASE': cache_directory = None else: cache_directory = options.cache_directory explorer = ModelExplorer(model=options.model_name, year=int(options.year), scenario_name=options.scenario_name, model_group=options.model_group, configuration=config, xml_configuration=xconfig, cache_directory=cache_directory) explorer.run() ex = explorer
from opus_core.file_utilities import get_resources_from_file parser = OptionParser() parser.add_option("-r", "--resources", dest="resources_file_name", action="store", type="string", help="Name of file containing resources", default=None) parser.add_option("-y", "--year", dest="year", action="store", type="int", help="Year in which to 'run' the travel model") #these options were added by SFCTA parser.add_option("-c", "--alt-config", dest="alt_config", action="store", type="string", help="opus path to config if pickled resources not available", default=None) parser.add_option("-d", "--alt-cache-dir", dest="alt_cache_dir", action="store", type="string", help="cache directory if pickled resources not available", default=None) (options, args) = parser.parse_args() #options here added by SFCTA if (options.alt_config is not None) and (options.alt_cache_dir is not None): resources=get_config_from_opus_path(options.alt_config) resources['cache_directory']=options.alt_cache_dir elif options.resources_file_name is not None: r = get_resources_from_file(options.resources_file_name) resources = Resources(get_resources_from_file(options.resources_file_name)) else: parser.print_help() sys.exit(1) SessionConfiguration(new_instance=True, package_order=resources['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) # logger.enable_memory_logging() GetTravelModelDataIntoCache().run(resources, options.year)
help="opus path to config if pickled resources not available", default=None) parser.add_option( "-d", "--alt-cache-dir", dest="alt_cache_dir", action="store", type="string", help="cache directory if pickled resources not available", default=None) (options, args) = parser.parse_args() #options here added by SFCTA if (options.alt_config is not None) and (options.alt_cache_dir is not None): resources = get_config_from_opus_path(options.alt_config) resources['cache_directory'] = options.alt_cache_dir elif options.resources_file_name is not None: r = get_resources_from_file(options.resources_file_name) resources = Resources( get_resources_from_file(options.resources_file_name)) else: parser.print_help() sys.exit(1) SessionConfiguration( new_instance=True, package_order=resources['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) # logger.enable_memory_logging()