def preload_injectables():
logger.info("preload_injectables")
# inject.add_step('track_skim_usage', track_skim_usage)
inject.add_step('write_data_dictionary', write_data_dictionary)
inject.add_step('write_tables', write_tables)
def preload_injectables():
"""
preload bulky injectables up front - stuff that isn't inserted into the pipeline
"""
logger.info("preload_injectables")
inject.add_step('track_skim_usage', track_skim_usage)
inject.add_step('write_data_dictionary', write_data_dictionary)
inject.add_step('write_tables', write_tables)
table_list = config.setting('input_table_list')
# default ActivitySim table names and indices
if table_list is None:
logger.warn("No 'input_table_list' found in settings. This will be a "
"required setting in upcoming versions of ActivitySim.")
new_settings = inject.get_injectable('settings')
new_settings['input_table_list'] = DEFAULT_TABLE_LIST
inject.add_injectable('settings', new_settings)
t0 = tracing.print_elapsed_time()
# FIXME - still want to do this?
# if inject.get_injectable('skim_dict', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_dict", t0, debug=True)
#
# if inject.get_injectable('skim_stack', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_stack", t0, debug=True)
return True
def test_pipeline_checkpoint_drop():
inject.add_step('step1', steps.step1)
inject.add_step('step2', steps.step2)
inject.add_step('step3', steps.step3)
inject.add_step('step_add_col', steps.step_add_col)
inject.add_step('step_forget_tab', steps.step_forget_tab)
_MODELS = [
'step1',
'_step2',
'_step_add_col.table_name=table2;column_name=c2',
'_step_forget_tab.table_name=table2',
'step3',
'step_forget_tab.table_name=table3',
]
pipeline.run(models=_MODELS, resume_after=None)
checkpoints = pipeline.get_checkpoints()
print("checkpoints\n", checkpoints)
pipeline.get_table("table1")
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("table2")
assert "never checkpointed" in str(excinfo.value)
# can't get a dropped table from current checkpoint
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("table3")
assert "was dropped" in str(excinfo.value)
# ensure that we can still get table3 from a checkpoint at which it existed
pipeline.get_table("table3", checkpoint_name="step3")
pipeline.close_pipeline()
close_handlers()
def test_pipeline_run():
inject.add_step('step1', steps.step1)
inject.add_step('step2', steps.step2)
inject.add_step('step3', steps.step3)
inject.add_step('step_add_col', steps.step_add_col)
inject.dump_state()
_MODELS = [
'step1', 'step2', 'step3',
'step_add_col.table_name=table2;column_name=c2'
]
pipeline.run(models=_MODELS, resume_after=None)
checkpoints = pipeline.get_checkpoints()
print("checkpoints\n", checkpoints)
c2 = pipeline.get_table("table2").c2
# get table from
pipeline.get_table("table1", checkpoint_name="step3")
# try to get a table from a step before it was checkpointed
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("table2", checkpoint_name="step1")
assert "not in checkpoint 'step1'" in str(excinfo.value)
# try to get a non-existant table
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("bogus")
assert "never checkpointed" in str(excinfo.value)
# try to get an existing table from a non-existant checkpoint
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("table1", checkpoint_name="bogus")
assert "not in checkpoints" in str(excinfo.value)
pipeline.close_pipeline()
close_handlers()
def preload_injectables():
"""
preload bulky injectables up front - stuff that isn't inserted into the pipeline
"""
logger.info("preload_injectables")
inject.add_step('track_skim_usage', track_skim_usage)
inject.add_step('write_data_dictionary', write_data_dictionary)
inject.add_step('write_tables', write_tables)
table_list = config.setting('input_table_list')
# default ActivitySim table names and indices
if table_list is None:
logger.warning(
"No 'input_table_list' found in settings. This will be a "
"required setting in upcoming versions of ActivitySim.")
new_settings = inject.get_injectable('settings')
new_settings['input_table_list'] = DEFAULT_TABLE_LIST
inject.add_injectable('settings', new_settings)
# FIXME undocumented feature
if config.setting('write_raw_tables'):
# write raw input tables as csv (before annotation)
csv_dir = config.output_file_path('raw_tables')
if not os.path.exists(csv_dir):
os.makedirs(csv_dir) # make directory if needed
table_names = [t['tablename'] for t in table_list]
for t in table_names:
df = inject.get_table(t).to_frame()
if t == 'households':
df.drop(columns='chunk_id', inplace=True)
df.to_csv(os.path.join(csv_dir, '%s.csv' % t), index=True)
t0 = tracing.print_elapsed_time()
# FIXME - still want to do this?
# if inject.get_injectable('skim_dict', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_dict", t0, debug=True)
#
# if inject.get_injectable('skim_stack', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_stack", t0, debug=True)
return True
def preload_injectables():
"""
preload bulky injectables up front - stuff that isn't inserted into the pipeline
"""
logger.info("preload_injectables")
inject.add_step('track_skim_usage', track_skim_usage)
inject.add_step('write_data_dictionary', write_data_dictionary)
inject.add_step('write_tables', write_tables)
t0 = tracing.print_elapsed_time()
# FIXME - still want to do this?
# if inject.get_injectable('skim_dict', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_dict", t0, debug=True)
#
# if inject.get_injectable('skim_stack', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_stack", t0, debug=True)
return True
def preload_injectables():
"""
preload bulky injectables up front - stuff that isn't inserted into the pipeline
"""
logger.info("preload_injectables")
inject.add_step('track_skim_usage', track_skim_usage)
inject.add_step('write_data_dictionary', write_data_dictionary)
inject.add_step('write_tables', write_tables)
t0 = tracing.print_elapsed_time()
# FIXME - still want to do this?
# if inject.get_injectable('skim_dict', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_dict", t0, debug=True)
#
# if inject.get_injectable('skim_stack', None) is not None:
# t0 = tracing.print_elapsed_time("preload skim_stack", t0, debug=True)
return True
def preload_injectables():
_inject.add_step('write_data_dictionary', write_data_dictionary)
_inject.add_step('write_tables', write_tables)