def run(args): """ Run the models. Specify a project folder using the '--working_dir' option, or point to the config, data, and output folders directly with '--config', '--data', and '--output'. Both '--config' and '--data' can be specified multiple times. Directories listed first take precedence. """ from activitysim import abm # register injectables tracing.config_logger(basic=True) handle_standard_args(args) # possibly update injectables tracing.config_logger( basic=False) # update using possibly new logging configs config.filter_warnings() logging.captureWarnings(capture=True) log_settings() t0 = tracing.print_elapsed_time() # If you provide a resume_after argument to pipeline.run # the pipeline manager will attempt to load checkpointed tables from the checkpoint store # and resume pipeline processing on the next submodel step after the specified checkpoint resume_after = config.setting('resume_after', None) # cleanup if not resuming if not resume_after: cleanup_output_files() elif config.setting('cleanup_trace_files_on_resume', False): tracing.delete_trace_files() if config.setting('multiprocess', False): logger.info('run multiprocess simulation') from activitysim.core import mp_tasks run_list = mp_tasks.get_run_list() injectables = {k: inject.get_injectable(k) for k in INJECTABLES} mp_tasks.run_multiprocess(run_list, injectables) else: logger.info('run single process simulation') pipeline.run(models=config.setting('models'), resume_after=resume_after) pipeline.close_pipeline() chunk.log_write_hwm() tracing.print_elapsed_time('all models', t0)
def run(): config.handle_standard_args() # specify None for a pseudo random base seed # inject.add_injectable('rng_base_seed', 0) tracing.config_logger() config.filter_warnings() tracing.delete_csv_files() # If you provide a resume_after argument to pipeline.run # the pipeline manager will attempt to load checkpointed tables from the checkpoint store # and resume pipeline processing on the next submodel step after the specified checkpoint resume_after = setting('resume_after', None) if resume_after: print("resume_after", resume_after) pipeline.run(models=setting('models'), resume_after=resume_after) # tables will no longer be available after pipeline is closed pipeline.close_pipeline()
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.set_option.html # pd.set_option('display.max_columns', 50) data_dir = "E:/projects/clients/ASIM/data/mtc_tm1" data_dir = '/Users/jeff.doyle/work/activitysim-data/mtc_tm1/data' data_dir = '../example/data' # inject.add_injectable('data_dir', '/Users/jeff.doyle/work/activitysim-data/mtc_tm1/data') inject.add_injectable('data_dir', ['ancillary_data', data_dir]) # inject.add_injectable('data_dir', ['ancillary_data', '../activitysim/abm/test/data']) inject.add_injectable('configs_dir', ['configs', '../example/configs']) injectables = config.handle_standard_args() tracing.config_logger() config.filter_warnings() log_settings(injectables) t0 = tracing.print_elapsed_time() # cleanup if not resuming if not config.setting('resume_after', False): cleanup_output_files() run_list = mp_tasks.get_run_list() if run_list['multiprocess']: # do this after config.handle_standard_args, as command line args may override injectables injectables = list( set(injectables) | set(['data_dir', 'configs_dir', 'output_dir']))
def run(args): """ Run the models. Specify a project folder using the '--working_dir' option, or point to the config, data, and output folders directly with '--config', '--data', and '--output'. Both '--config' and '--data' can be specified multiple times. Directories listed first take precedence. returns: int: sys.exit exit code """ # register abm steps and other abm-specific injectables # by default, assume we are running activitysim.abm # other callers (e.g. populationsim) will have to arrange to register their own steps and injectables # (presumably) in a custom run_simulation.py instead of using the 'activitysim run' command if not inject.is_injectable('preload_injectables'): from activitysim import abm # register abm steps and other abm-specific injectables tracing.config_logger(basic=True) handle_standard_args(args) # possibly update injectables # legacy support for run_list setting nested 'models' and 'resume_after' settings if config.setting('run_list'): warnings.warn( "Support for 'run_list' settings group will be removed.\n" "The run_list.steps setting is renamed 'models'.\n" "The run_list.resume_after setting is renamed 'resume_after'.\n" "Specify both 'models' and 'resume_after' directly in settings config file.", FutureWarning) run_list = config.setting('run_list') if 'steps' in run_list: assert not config.setting('models'), \ f"Don't expect 'steps' in run_list and 'models' as stand-alone setting!" config.override_setting('models', run_list['steps']) if 'resume_after' in run_list: assert not config.setting('resume_after'), \ f"Don't expect 'resume_after' both in run_list and as stand-alone setting!" config.override_setting('resume_after', run_list['resume_after']) # If you provide a resume_after argument to pipeline.run # the pipeline manager will attempt to load checkpointed tables from the checkpoint store # and resume pipeline processing on the next submodel step after the specified checkpoint resume_after = config.setting('resume_after', None) # cleanup if not resuming if not resume_after: cleanup_output_files() elif config.setting('cleanup_trace_files_on_resume', False): tracing.delete_trace_files() tracing.config_logger( basic=False) # update using possibly new logging configs config.filter_warnings() logging.captureWarnings(capture=True) # directories for k in ['configs_dir', 'settings_file_name', 'data_dir', 'output_dir']: logger.info('SETTING %s: %s' % (k, inject.get_injectable(k, None))) log_settings = inject.get_injectable('log_settings', {}) for k in log_settings: logger.info('SETTING %s: %s' % (k, config.setting(k))) t0 = tracing.print_elapsed_time() if config.setting('multiprocess', False): logger.info('run multiprocess simulation') from activitysim.core import mp_tasks run_list = mp_tasks.get_run_list() injectables = {k: inject.get_injectable(k) for k in INJECTABLES} mp_tasks.run_multiprocess(run_list, injectables) assert not pipeline.is_open() if config.setting('cleanup_pipeline_after_run', False): pipeline.cleanup_pipeline() else: logger.info('run single process simulation') pipeline.run(models=config.setting('models'), resume_after=resume_after) if config.setting('cleanup_pipeline_after_run', False): pipeline.cleanup_pipeline( ) # has side effect of closing open pipeline else: pipeline.close_pipeline() chunk.log_write_hwm() tracing.print_elapsed_time('all models', t0) return 0
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.set_option.html # pd.set_option('display.max_columns', 50) data_dir = "E:/projects/clients/ASIM/data/mtc_tm1" data_dir = '/Users/jeff.doyle/work/activitysim-data/mtc_tm1/data' data_dir = '../example/data' # inject.add_injectable('data_dir', '/Users/jeff.doyle/work/activitysim-data/mtc_tm1/data') inject.add_injectable('data_dir', ['ancillary_data', data_dir]) # inject.add_injectable('data_dir', ['ancillary_data', '../activitysim/abm/test/data']) inject.add_injectable('configs_dir', ['configs', '../example/configs']) injectables = config.handle_standard_args() tracing.config_logger() config.filter_warnings() log_settings(injectables) t0 = tracing.print_elapsed_time() # cleanup if not resuming if not config.setting('resume_after', False): cleanup_output_files() run_list = mp_tasks.get_run_list() if run_list['multiprocess']: # do this after config.handle_standard_args, as command line args may override injectables injectables = list(set(injectables) | set(['data_dir', 'configs_dir', 'output_dir'])) injectables = {k: inject.get_injectable(k) for k in injectables}
def run(args): """ Run the models. Specify a project folder using the '--working_dir' option, or point to the config, data, and output folders directly with '--config', '--data', and '--output'. Both '--config' and '--data' can be specified multiple times. Directories listed first take precedence. returns: int: sys.exit exit code """ # register abm steps and other abm-specific injectables # by default, assume we are running activitysim.abm # other callers (e.g. populationsim) will have to arrange to register their own steps and injectables # (presumably) in a custom run_simulation.py instead of using the 'activitysim run' command if not inject.is_injectable('preload_injectables'): from activitysim import abm # register abm steps and other abm-specific injectables tracing.config_logger(basic=True) handle_standard_args(args) # possibly update injectables # legacy support for run_list setting nested 'models' and 'resume_after' settings if config.setting('run_list'): warnings.warn( "Support for 'run_list' settings group will be removed.\n" "The run_list.steps setting is renamed 'models'.\n" "The run_list.resume_after setting is renamed 'resume_after'.\n" "Specify both 'models' and 'resume_after' directly in settings config file.", FutureWarning) run_list = config.setting('run_list') if 'steps' in run_list: assert not config.setting('models'), \ f"Don't expect 'steps' in run_list and 'models' as stand-alone setting!" config.override_setting('models', run_list['steps']) if 'resume_after' in run_list: assert not config.setting('resume_after'), \ f"Don't expect 'resume_after' both in run_list and as stand-alone setting!" config.override_setting('resume_after', run_list['resume_after']) # If you provide a resume_after argument to pipeline.run # the pipeline manager will attempt to load checkpointed tables from the checkpoint store # and resume pipeline processing on the next submodel step after the specified checkpoint resume_after = config.setting('resume_after', None) # cleanup if not resuming if not resume_after: cleanup_output_files() elif config.setting('cleanup_trace_files_on_resume', False): tracing.delete_trace_files() tracing.config_logger( basic=False) # update using possibly new logging configs config.filter_warnings() logging.captureWarnings(capture=True) # directories for k in ['configs_dir', 'settings_file_name', 'data_dir', 'output_dir']: logger.info('SETTING %s: %s' % (k, inject.get_injectable(k, None))) log_settings = inject.get_injectable('log_settings', {}) for k in log_settings: logger.info('SETTING %s: %s' % (k, config.setting(k))) # OMP_NUM_THREADS: openmp # OPENBLAS_NUM_THREADS: openblas # MKL_NUM_THREADS: mkl for env in ['MKL_NUM_THREADS', 'OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS']: logger.info(f"ENV {env}: {os.getenv(env)}") np_info_keys = [ 'atlas_blas_info', 'atlas_blas_threads_info', 'atlas_info', 'atlas_threads_info', 'blas_info', 'blas_mkl_info', 'blas_opt_info', 'lapack_info', 'lapack_mkl_info', 'lapack_opt_info', 'mkl_info' ] for cfg_key in np_info_keys: info = np.__config__.get_info(cfg_key) if info: for info_key in ['libraries']: if info_key in info: logger.info( f"NUMPY {cfg_key} {info_key}: {info[info_key]}") t0 = tracing.print_elapsed_time() try: if config.setting('multiprocess', False): logger.info('run multiprocess simulation') from activitysim.core import mp_tasks injectables = {k: inject.get_injectable(k) for k in INJECTABLES} mp_tasks.run_multiprocess(injectables) assert not pipeline.is_open() if config.setting('cleanup_pipeline_after_run', False): pipeline.cleanup_pipeline() else: logger.info('run single process simulation') pipeline.run(models=config.setting('models'), resume_after=resume_after) if config.setting('cleanup_pipeline_after_run', False): pipeline.cleanup_pipeline( ) # has side effect of closing open pipeline else: pipeline.close_pipeline() mem.log_global_hwm() # main process except Exception: # log time until error and the error traceback tracing.print_elapsed_time('all models until this error', t0) logger.exception('activitysim run encountered an unrecoverable error') raise chunk.consolidate_logs() mem.consolidate_logs() tracing.print_elapsed_time('all models', t0) return 0