def test_mp_run():
mp_configs_dir = os.path.join(os.path.dirname(__file__), 'configs_mp')
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
inject.add_injectable('configs_dir', [mp_configs_dir, configs_dir])
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
tracing.config_logger()
run_list = mp_tasks.get_run_list()
mp_tasks.print_run_list(run_list)
# do this after config.handle_standard_args, as command line args may override injectables
injectables = ['data_dir', 'configs_dir', 'output_dir']
injectables = {k: inject.get_injectable(k) for k in injectables}
# pipeline.run(models=run_list['models'], resume_after=run_list['resume_after'])
mp_tasks.run_multiprocess(run_list, injectables)
pipeline.open_pipeline('_')
regress_mini_auto()
pipeline.close_pipeline()
def test_mp_run():
mp_configs_dir = os.path.join(os.path.dirname(__file__), 'configs_mp')
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
inject.add_injectable('configs_dir', [mp_configs_dir, configs_dir])
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
tracing.config_logger()
run_list = mp_tasks.get_run_list()
mp_tasks.print_run_list(run_list)
# do this after config.handle_standard_args, as command line args may override injectables
injectables = ['data_dir', 'configs_dir', 'output_dir']
injectables = {k: inject.get_injectable(k) for k in injectables}
# pipeline.run(models=run_list['models'], resume_after=run_list['resume_after'])
mp_tasks.run_multiprocess(run_list, injectables)
pipeline.open_pipeline('_')
regress_mini_auto()
pipeline.close_pipeline()
def run(run_list, injectables=None):
if run_list['multiprocess']:
logger.info("run multiprocess simulation")
mp_tasks.run_multiprocess(run_list, injectables)
else:
logger.info("run single process simulation")
pipeline.run(models=run_list['models'], resume_after=run_list['resume_after'])
pipeline.close_pipeline()
mem.log_global_hwm()
def run(run_list, injectables=None):
if run_list['multiprocess']:
logger.info("run multiprocess simulation")
mp_tasks.run_multiprocess(run_list, injectables)
else:
logger.info("run single process simulation")
pipeline.run(models=run_list['models'], resume_after=run_list['resume_after'])
pipeline.close_pipeline()
chunk.log_write_hwm()
def run(args):
"""
Run the models. Specify a project folder using the '--working_dir' option,
or point to the config, data, and output folders directly with
'--config', '--data', and '--output'. Both '--config' and '--data' can be
specified multiple times. Directories listed first take precedence.
"""
from activitysim import abm # register injectables
tracing.config_logger(basic=True)
handle_standard_args(args) # possibly update injectables
tracing.config_logger(
basic=False) # update using possibly new logging configs
config.filter_warnings()
logging.captureWarnings(capture=True)
log_settings()
t0 = tracing.print_elapsed_time()
# If you provide a resume_after argument to pipeline.run
# the pipeline manager will attempt to load checkpointed tables from the checkpoint store
# and resume pipeline processing on the next submodel step after the specified checkpoint
resume_after = config.setting('resume_after', None)
# cleanup if not resuming
if not resume_after:
cleanup_output_files()
elif config.setting('cleanup_trace_files_on_resume', False):
tracing.delete_trace_files()
if config.setting('multiprocess', False):
logger.info('run multiprocess simulation')
from activitysim.core import mp_tasks
run_list = mp_tasks.get_run_list()
injectables = {k: inject.get_injectable(k) for k in INJECTABLES}
mp_tasks.run_multiprocess(run_list, injectables)
else:
logger.info('run single process simulation')
pipeline.run(models=config.setting('models'),
resume_after=resume_after)
pipeline.close_pipeline()
chunk.log_write_hwm()
tracing.print_elapsed_time('all models', t0)
def test_mp_run():
mp_configs_dir = os.path.join(os.path.dirname(__file__), 'configs_mp')
setup_dirs(ancillary_configs_dir=mp_configs_dir)
run_list = mp_tasks.get_run_list()
mp_tasks.print_run_list(run_list)
# do this after config.handle_standard_args, as command line args may override injectables
injectables = ['data_dir', 'configs_dir', 'output_dir']
injectables = {k: inject.get_injectable(k) for k in injectables}
# pipeline.run(models=run_list['models'], resume_after=run_list['resume_after'])
mp_tasks.run_multiprocess(run_list, injectables)
pipeline.open_pipeline('_')
regress_mini_auto()
pipeline.close_pipeline()
def test_mp_run():
setup_dirs()
# Debugging ----------------------
run_list = mp_tasks.get_run_list()
mp_tasks.print_run_list(run_list)
# --------------------------------
# do this after config.handle_standard_args, as command line args
# may override injectables
injectables = ["data_dir", "configs_dir", "output_dir"]
injectables = {k: inject.get_injectable(k) for k in injectables}
mp_tasks.run_multiprocess(injectables)
pipeline.open_pipeline("_")
regress()
pipeline.close_pipeline()
def test_mp_run():
configs_dir = [example_path('configs_3_zone'), example_path('configs')]
data_dir = example_path('data_3')
setup_dirs(configs_dir, data_dir)
inject.add_injectable('settings_file_name', 'settings_mp.yaml')
run_list = mp_tasks.get_run_list()
mp_tasks.print_run_list(run_list)
# do this after config.handle_standard_args, as command line args may override injectables
injectables = [
'data_dir', 'configs_dir', 'output_dir', 'settings_file_name'
]
injectables = {k: inject.get_injectable(k) for k in injectables}
mp_tasks.run_multiprocess(run_list, injectables)
pipeline.open_pipeline('_')
regress_3_zone()
pipeline.close_pipeline()
def run(args):
"""
Run the models. Specify a project folder using the '--working_dir' option,
or point to the config, data, and output folders directly with
'--config', '--data', and '--output'. Both '--config' and '--data' can be
specified multiple times. Directories listed first take precedence.
returns:
int: sys.exit exit code
"""
# register abm steps and other abm-specific injectables
# by default, assume we are running activitysim.abm
# other callers (e.g. populationsim) will have to arrange to register their own steps and injectables
# (presumably) in a custom run_simulation.py instead of using the 'activitysim run' command
if not inject.is_injectable('preload_injectables'):
from activitysim import abm # register abm steps and other abm-specific injectables
tracing.config_logger(basic=True)
handle_standard_args(args) # possibly update injectables
# legacy support for run_list setting nested 'models' and 'resume_after' settings
if config.setting('run_list'):
warnings.warn(
"Support for 'run_list' settings group will be removed.\n"
"The run_list.steps setting is renamed 'models'.\n"
"The run_list.resume_after setting is renamed 'resume_after'.\n"
"Specify both 'models' and 'resume_after' directly in settings config file.",
FutureWarning)
run_list = config.setting('run_list')
if 'steps' in run_list:
assert not config.setting('models'), \
f"Don't expect 'steps' in run_list and 'models' as stand-alone setting!"
config.override_setting('models', run_list['steps'])
if 'resume_after' in run_list:
assert not config.setting('resume_after'), \
f"Don't expect 'resume_after' both in run_list and as stand-alone setting!"
config.override_setting('resume_after', run_list['resume_after'])
# If you provide a resume_after argument to pipeline.run
# the pipeline manager will attempt to load checkpointed tables from the checkpoint store
# and resume pipeline processing on the next submodel step after the specified checkpoint
resume_after = config.setting('resume_after', None)
# cleanup if not resuming
if not resume_after:
cleanup_output_files()
elif config.setting('cleanup_trace_files_on_resume', False):
tracing.delete_trace_files()
tracing.config_logger(
basic=False) # update using possibly new logging configs
config.filter_warnings()
logging.captureWarnings(capture=True)
# directories
for k in ['configs_dir', 'settings_file_name', 'data_dir', 'output_dir']:
logger.info('SETTING %s: %s' % (k, inject.get_injectable(k, None)))
log_settings = inject.get_injectable('log_settings', {})
for k in log_settings:
logger.info('SETTING %s: %s' % (k, config.setting(k)))
t0 = tracing.print_elapsed_time()
if config.setting('multiprocess', False):
logger.info('run multiprocess simulation')
from activitysim.core import mp_tasks
run_list = mp_tasks.get_run_list()
injectables = {k: inject.get_injectable(k) for k in INJECTABLES}
mp_tasks.run_multiprocess(run_list, injectables)
assert not pipeline.is_open()
if config.setting('cleanup_pipeline_after_run', False):
pipeline.cleanup_pipeline()
else:
logger.info('run single process simulation')
pipeline.run(models=config.setting('models'),
resume_after=resume_after)
if config.setting('cleanup_pipeline_after_run', False):
pipeline.cleanup_pipeline(
) # has side effect of closing open pipeline
else:
pipeline.close_pipeline()
chunk.log_write_hwm()
tracing.print_elapsed_time('all models', t0)
return 0
def run(args):
"""
Run the models. Specify a project folder using the '--working_dir' option,
or point to the config, data, and output folders directly with
'--config', '--data', and '--output'. Both '--config' and '--data' can be
specified multiple times. Directories listed first take precedence.
returns:
int: sys.exit exit code
"""
# register abm steps and other abm-specific injectables
# by default, assume we are running activitysim.abm
# other callers (e.g. populationsim) will have to arrange to register their own steps and injectables
# (presumably) in a custom run_simulation.py instead of using the 'activitysim run' command
if not inject.is_injectable('preload_injectables'):
from activitysim import abm # register abm steps and other abm-specific injectables
tracing.config_logger(basic=True)
handle_standard_args(args) # possibly update injectables
# legacy support for run_list setting nested 'models' and 'resume_after' settings
if config.setting('run_list'):
warnings.warn(
"Support for 'run_list' settings group will be removed.\n"
"The run_list.steps setting is renamed 'models'.\n"
"The run_list.resume_after setting is renamed 'resume_after'.\n"
"Specify both 'models' and 'resume_after' directly in settings config file.",
FutureWarning)
run_list = config.setting('run_list')
if 'steps' in run_list:
assert not config.setting('models'), \
f"Don't expect 'steps' in run_list and 'models' as stand-alone setting!"
config.override_setting('models', run_list['steps'])
if 'resume_after' in run_list:
assert not config.setting('resume_after'), \
f"Don't expect 'resume_after' both in run_list and as stand-alone setting!"
config.override_setting('resume_after', run_list['resume_after'])
# If you provide a resume_after argument to pipeline.run
# the pipeline manager will attempt to load checkpointed tables from the checkpoint store
# and resume pipeline processing on the next submodel step after the specified checkpoint
resume_after = config.setting('resume_after', None)
# cleanup if not resuming
if not resume_after:
cleanup_output_files()
elif config.setting('cleanup_trace_files_on_resume', False):
tracing.delete_trace_files()
tracing.config_logger(
basic=False) # update using possibly new logging configs
config.filter_warnings()
logging.captureWarnings(capture=True)
# directories
for k in ['configs_dir', 'settings_file_name', 'data_dir', 'output_dir']:
logger.info('SETTING %s: %s' % (k, inject.get_injectable(k, None)))
log_settings = inject.get_injectable('log_settings', {})
for k in log_settings:
logger.info('SETTING %s: %s' % (k, config.setting(k)))
# OMP_NUM_THREADS: openmp
# OPENBLAS_NUM_THREADS: openblas
# MKL_NUM_THREADS: mkl
for env in ['MKL_NUM_THREADS', 'OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS']:
logger.info(f"ENV {env}: {os.getenv(env)}")
np_info_keys = [
'atlas_blas_info', 'atlas_blas_threads_info', 'atlas_info',
'atlas_threads_info', 'blas_info', 'blas_mkl_info', 'blas_opt_info',
'lapack_info', 'lapack_mkl_info', 'lapack_opt_info', 'mkl_info'
]
for cfg_key in np_info_keys:
info = np.__config__.get_info(cfg_key)
if info:
for info_key in ['libraries']:
if info_key in info:
logger.info(
f"NUMPY {cfg_key} {info_key}: {info[info_key]}")
t0 = tracing.print_elapsed_time()
try:
if config.setting('multiprocess', False):
logger.info('run multiprocess simulation')
from activitysim.core import mp_tasks
injectables = {k: inject.get_injectable(k) for k in INJECTABLES}
mp_tasks.run_multiprocess(injectables)
assert not pipeline.is_open()
if config.setting('cleanup_pipeline_after_run', False):
pipeline.cleanup_pipeline()
else:
logger.info('run single process simulation')
pipeline.run(models=config.setting('models'),
resume_after=resume_after)
if config.setting('cleanup_pipeline_after_run', False):
pipeline.cleanup_pipeline(
) # has side effect of closing open pipeline
else:
pipeline.close_pipeline()
mem.log_global_hwm() # main process
except Exception:
# log time until error and the error traceback
tracing.print_elapsed_time('all models until this error', t0)
logger.exception('activitysim run encountered an unrecoverable error')
raise
chunk.consolidate_logs()
mem.consolidate_logs()
tracing.print_elapsed_time('all models', t0)
return 0