def setup_dirs(ancillary_configs_dir=None, data_dir=None): # ancillary_configs_dir is used by run_mp to test multiprocess test_pipeline_configs_dir = os.path.join(os.path.dirname(__file__), 'configs') example_configs_dir = example_path('configs') configs_dir = [test_pipeline_configs_dir, example_configs_dir] if ancillary_configs_dir is not None: configs_dir = [ancillary_configs_dir] + configs_dir inject.add_injectable('configs_dir', configs_dir) output_dir = os.path.join(os.path.dirname(__file__), 'output') inject.add_injectable('output_dir', output_dir) if not data_dir: data_dir = example_path('data') inject.add_injectable('data_dir', data_dir) inject.clear_cache() tracing.config_logger() tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') tracing.delete_output_files('omx')
def setup_working_dir(example_name, inherit=False): os.chdir(example_dir(example_name)) tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('log') tracing.delete_output_files('h5') if inherit: data_dir = inject.get_injectable('data_dir') example_data_dir = os.path.join(example_dir('example'), 'data') inject.add_injectable('data_dir', [data_dir, example_data_dir], cache=True)
def setup_dirs(configs_dir, data_dir): print(f"configs_dir: {configs_dir}") inject.add_injectable('configs_dir', configs_dir) output_dir = os.path.join(os.path.dirname(__file__), 'output') inject.add_injectable('output_dir', output_dir) print(f"data_dir: {data_dir}") inject.add_injectable('data_dir', data_dir) inject.clear_cache() tracing.config_logger() tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') tracing.delete_output_files('omx')
def test_full_run2(): configs_dir = os.path.join(os.path.dirname(__file__), 'configs2') inject.add_injectable("configs_dir", configs_dir) data_dir = os.path.join(os.path.dirname(__file__), 'data2') inject.add_injectable("data_dir", data_dir) output_dir = os.path.join(os.path.dirname(__file__), 'output') inject.add_injectable("output_dir", output_dir) inject.clear_cache() tracing.config_logger() tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') _MODELS = [ 'input_pre_processor', 'setup_data_structures', 'initial_seed_balancing', 'meta_control_factoring', 'final_seed_balancing', 'integerize_final_seed_weights', 'sub_balancing.geography = DISTRICT', 'sub_balancing.geography = TRACT', 'sub_balancing.geography=TAZ', 'expand_households', 'summarize', 'write_tables' ] pipeline.run(models=_MODELS, resume_after=None) assert isinstance(pipeline.get_table('expanded_household_ids'), pd.DataFrame) # output tables list action: include assert os.path.exists(os.path.join(output_dir, 'expanded_household_ids.csv')) assert os.path.exists(os.path.join(output_dir, 'summary_DISTRICT.csv')) assert not os.path.exists(os.path.join(output_dir, 'summary_TAZ.csv')) # tables will no longer be available after pipeline is closed pipeline.close_pipeline() inject.clear_cache()
def setup_dirs(configs_dir): inject.add_injectable("configs_dir", configs_dir) output_dir = os.path.join(os.path.dirname(__file__), 'output') inject.add_injectable("output_dir", output_dir) data_dir = os.path.join(os.path.dirname(__file__), 'data') inject.add_injectable("data_dir", data_dir) inject.clear_cache() tracing.config_logger() tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml')
def setup_dirs(): configs_dir = os.path.join(os.path.dirname(__file__), "configs") mp_configs_dir = os.path.join(os.path.dirname(__file__), "configs_mp") inject.add_injectable("configs_dir", [mp_configs_dir, configs_dir]) output_dir = os.path.join(os.path.dirname(__file__), "output") inject.add_injectable("output_dir", output_dir) data_dir = os.path.join(os.path.dirname(__file__), "data") inject.add_injectable("data_dir", data_dir) tracing.config_logger() tracing.delete_output_files("csv") tracing.delete_output_files("txt") tracing.delete_output_files("yaml")
def cleanup_output_files(): active_log_files = \ [h.baseFilename for h in logger.root.handlers if isinstance(h, logging.FileHandler)] tracing.delete_output_files('log', ignore=active_log_files) tracing.delete_output_files('h5') tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') tracing.delete_output_files('prof')
def cleanup_output_files(): tracing.delete_trace_files() tracing.delete_output_files('h5') tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') tracing.delete_output_files('prof') tracing.delete_output_files('omx')
def cleanup_output_files(): active_log_files = \ [h.baseFilename for h in logger.root.handlers if isinstance(h, logging.FileHandler)] tracing.delete_output_files('log', ignore=active_log_files) tracing.delete_output_files('h5') tracing.delete_output_files('csv') tracing.delete_output_files('txt') tracing.delete_output_files('yaml') tracing.delete_output_files('prof')