def setup_dirs(ancillary_configs_dir=None, data_dir=None):
# ancillary_configs_dir is used by run_mp to test multiprocess
test_pipeline_configs_dir = os.path.join(os.path.dirname(__file__),
'configs')
example_configs_dir = example_path('configs')
configs_dir = [test_pipeline_configs_dir, example_configs_dir]
if ancillary_configs_dir is not None:
configs_dir = [ancillary_configs_dir] + configs_dir
inject.add_injectable('configs_dir', configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable('output_dir', output_dir)
if not data_dir:
data_dir = example_path('data')
inject.add_injectable('data_dir', data_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('omx')
def setup_working_dir(example_name, inherit=False):
os.chdir(example_dir(example_name))
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('log')
tracing.delete_output_files('h5')
if inherit:
data_dir = inject.get_injectable('data_dir')
example_data_dir = os.path.join(example_dir('example'), 'data')
inject.add_injectable('data_dir', [data_dir, example_data_dir],
cache=True)
def setup_dirs(configs_dir, data_dir):
print(f"configs_dir: {configs_dir}")
inject.add_injectable('configs_dir', configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable('output_dir', output_dir)
print(f"data_dir: {data_dir}")
inject.add_injectable('data_dir', data_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('omx')
def test_full_run2():
configs_dir = os.path.join(os.path.dirname(__file__), 'configs2')
inject.add_injectable("configs_dir", configs_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data2')
inject.add_injectable("data_dir", data_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
_MODELS = [
'input_pre_processor',
'setup_data_structures',
'initial_seed_balancing',
'meta_control_factoring',
'final_seed_balancing',
'integerize_final_seed_weights',
'sub_balancing.geography = DISTRICT',
'sub_balancing.geography = TRACT',
'sub_balancing.geography=TAZ',
'expand_households',
'summarize',
'write_tables'
]
pipeline.run(models=_MODELS, resume_after=None)
assert isinstance(pipeline.get_table('expanded_household_ids'), pd.DataFrame)
# output tables list action: include
assert os.path.exists(os.path.join(output_dir, 'expanded_household_ids.csv'))
assert os.path.exists(os.path.join(output_dir, 'summary_DISTRICT.csv'))
assert not os.path.exists(os.path.join(output_dir, 'summary_TAZ.csv'))
# tables will no longer be available after pipeline is closed
pipeline.close_pipeline()
inject.clear_cache()
def setup_dirs(configs_dir):
inject.add_injectable("configs_dir", configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
def setup_dirs():
configs_dir = os.path.join(os.path.dirname(__file__), "configs")
mp_configs_dir = os.path.join(os.path.dirname(__file__), "configs_mp")
inject.add_injectable("configs_dir", [mp_configs_dir, configs_dir])
output_dir = os.path.join(os.path.dirname(__file__), "output")
inject.add_injectable("output_dir", output_dir)
data_dir = os.path.join(os.path.dirname(__file__), "data")
inject.add_injectable("data_dir", data_dir)
tracing.config_logger()
tracing.delete_output_files("csv")
tracing.delete_output_files("txt")
tracing.delete_output_files("yaml")
def cleanup_output_files():
active_log_files = \
[h.baseFilename for h in logger.root.handlers if isinstance(h, logging.FileHandler)]
tracing.delete_output_files('log', ignore=active_log_files)
tracing.delete_output_files('h5')
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('prof')
def cleanup_output_files():
tracing.delete_trace_files()
tracing.delete_output_files('h5')
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('prof')
tracing.delete_output_files('omx')
def cleanup_output_files():
active_log_files = \
[h.baseFilename for h in logger.root.handlers if isinstance(h, logging.FileHandler)]
tracing.delete_output_files('log', ignore=active_log_files)
tracing.delete_output_files('h5')
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('prof')
