def test_misc():
inject.clear_cache()
with pytest.raises(RuntimeError) as excinfo:
inject.get_injectable("configs_dir")
assert "directory does not exist" in str(excinfo.value)
with pytest.raises(RuntimeError) as excinfo:
inject.get_injectable("data_dir")
assert "directory does not exist" in str(excinfo.value)
with pytest.raises(RuntimeError) as excinfo:
inject.get_injectable("output_dir")
assert "directory does not exist" in str(excinfo.value)
configs_dir = os.path.join(os.path.dirname(__file__), 'configs_test_misc')
inject.add_injectable("configs_dir", configs_dir)
settings = inject.get_injectable("settings")
assert isinstance(settings, dict)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
# default values if not specified in settings
assert inject.get_injectable("chunk_size") == 0
def test_misc():
inject.clear_cache()
with pytest.raises(RuntimeError) as excinfo:
inject.get_injectable("configs_dir")
assert "directory does not exist" in str(excinfo.value)
with pytest.raises(RuntimeError) as excinfo:
inject.get_injectable("data_dir")
assert "directory does not exist" in str(excinfo.value)
with pytest.raises(RuntimeError) as excinfo:
inject.get_injectable("output_dir")
assert "directory does not exist" in str(excinfo.value)
configs_dir = os.path.join(os.path.dirname(__file__), 'configs_test_misc')
inject.add_injectable("configs_dir", configs_dir)
settings = inject.get_injectable("settings")
assert isinstance(settings, dict)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
# default values if not specified in settings
assert inject.get_injectable("chunk_size") == 0
def setup_dirs(ancillary_configs_dir=None, data_dir=None):
# ancillary_configs_dir is used by run_mp to test multiprocess
test_pipeline_configs_dir = os.path.join(os.path.dirname(__file__),
'configs')
example_configs_dir = example_path('configs')
configs_dir = [test_pipeline_configs_dir, example_configs_dir]
if ancillary_configs_dir is not None:
configs_dir = [ancillary_configs_dir] + configs_dir
inject.add_injectable('configs_dir', configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable('output_dir', output_dir)
if not data_dir:
data_dir = example_path('data')
inject.add_injectable('data_dir', data_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('omx')
def test_full_run2_repop_replace():
# Note: tests are run in alphabetical order.
# This tests expects to find the pipeline h5 file from
# test_full_run1 in the output folder
_MODELS = [
'input_pre_processor.table_list=repop_input_table_list;repop',
'repop_setup_data_structures',
'initial_seed_balancing.final=true;repop',
'integerize_final_seed_weights.repop',
'repop_balancing',
'expand_households.repop;replace',
'write_synthetic_population.repop',
'write_tables.repop',
]
pipeline.run(models=_MODELS, resume_after='summarize')
expanded_household_ids = pipeline.get_table('expanded_household_ids')
assert isinstance(expanded_household_ids, pd.DataFrame)
taz_hh_counts = expanded_household_ids.groupby('TAZ').size()
assert len(taz_hh_counts) == TAZ_COUNT
assert taz_hh_counts.loc[100] == TAZ_100_HH_REPOP_COUNT
# tables will no longer be available after pipeline is closed
pipeline.close_pipeline()
inject.clear_cache()
def test_full_run1():
_MODELS = [
'input_pre_processor',
'setup_data_structures',
'initial_seed_balancing',
'meta_control_factoring',
'final_seed_balancing',
'integerize_final_seed_weights',
'sub_balancing.geography=TRACT',
'sub_balancing.geography=TAZ',
'expand_households',
'summarize',
'write_tables',
'write_synthetic_population',
]
pipeline.run(models=_MODELS, resume_after=None)
expanded_household_ids = pipeline.get_table('expanded_household_ids')
assert isinstance(expanded_household_ids, pd.DataFrame)
taz_hh_counts = expanded_household_ids.groupby('TAZ').size()
assert len(taz_hh_counts) == TAZ_COUNT
assert taz_hh_counts.loc[100] == TAZ_100_HH_COUNT
# output_tables action: skip
output_dir = inject.get_injectable('output_dir')
assert not os.path.exists(os.path.join(output_dir, 'households.csv'))
assert os.path.exists(os.path.join(output_dir, 'summary_DISTRICT_1.csv'))
# tables will no longer be available after pipeline is closed
pipeline.close_pipeline()
inject.clear_cache()
def test_mini_pipeline_run2():
# the important thing here is that we should get
# exactly the same results as for test_mini_pipeline_run
# when we restart pipeline
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
setup_dirs(configs_dir)
inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE)
# should be able to get this BEFORE pipeline is opened
checkpoints_df = pipeline.get_checkpoints()
prev_checkpoint_count = len(checkpoints_df.index)
# print "checkpoints_df\n", checkpoints_df[['checkpoint_name']]
assert prev_checkpoint_count == 8
pipeline.open_pipeline('auto_ownership_simulate')
regress_mini_auto()
# try to run a model already in pipeline
with pytest.raises(RuntimeError) as excinfo:
pipeline.run_model('auto_ownership_simulate')
assert "run model 'auto_ownership_simulate' more than once" in str(
excinfo.value)
# and these new ones
pipeline.run_model('cdap_simulate')
pipeline.run_model('mandatory_tour_frequency')
regress_mini_mtf()
# should be able to get this before pipeline is closed (from existing open store)
checkpoints_df = pipeline.get_checkpoints()
assert len(checkpoints_df.index) == prev_checkpoint_count
# - write list of override_hh_ids to override_hh_ids.csv in data for use in next test
num_hh_ids = 10
hh_ids = pipeline.get_table("households").head(num_hh_ids).index.values
hh_ids = pd.DataFrame({'household_id': hh_ids})
data_dir = inject.get_injectable('data_dir')
hh_ids.to_csv(os.path.join(data_dir, 'override_hh_ids.csv'),
index=False,
header=True)
pipeline.close_pipeline()
inject.clear_cache()
close_handlers()
def test_zero_chunk_size():
settings = inject_settings(chunk_size=0)
inject.clear_cache()
tracing.config_logger()
MODELS = settings['models']
pipeline.run(models=MODELS, resume_after='aggregate_od_processor')
pipeline.close_pipeline()
def test_rng_access():
setup_dirs()
inject.add_injectable('rng_base_seed', 0)
pipeline.open_pipeline()
rng = pipeline.get_rn_generator()
assert isinstance(rng, random.Random)
pipeline.close_pipeline()
inject.clear_cache()
def test_mini_pipeline_run2():
# the important thing here is that we should get
# exactly the same results as for test_mini_pipeline_run
# when we restart pipeline
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
setup_dirs(configs_dir)
inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE)
# should be able to get this BEFORE pipeline is opened
checkpoints_df = pipeline.get_checkpoints()
prev_checkpoint_count = len(checkpoints_df.index)
# print "checkpoints_df\n", checkpoints_df[['checkpoint_name']]
assert prev_checkpoint_count == 8
pipeline.open_pipeline('auto_ownership_simulate')
regress_mini_auto()
# try to run a model already in pipeline
with pytest.raises(RuntimeError) as excinfo:
pipeline.run_model('auto_ownership_simulate')
assert "run model 'auto_ownership_simulate' more than once" in str(excinfo.value)
# and these new ones
pipeline.run_model('cdap_simulate')
pipeline.run_model('mandatory_tour_frequency')
regress_mini_mtf()
# should be able to get this before pipeline is closed (from existing open store)
checkpoints_df = pipeline.get_checkpoints()
assert len(checkpoints_df.index) == prev_checkpoint_count
# - write list of override_hh_ids to override_hh_ids.csv in data for use in next test
num_hh_ids = 10
hh_ids = pipeline.get_table("households").head(num_hh_ids).index.values
hh_ids = pd.DataFrame({'household_id': hh_ids})
data_dir = inject.get_injectable('data_dir')
hh_ids.to_csv(os.path.join(data_dir, 'override_hh_ids.csv'), index=False, header=True)
pipeline.close_pipeline()
inject.clear_cache()
close_handlers()
def test_run_4step():
settings = inject_settings(chunk_size=None,
trace_hh_id=None,
trace_od=None)
inject.clear_cache()
tracing.config_logger()
MODELS = settings['models']
pipeline.run(models=MODELS, resume_after=None)
pipeline.close_pipeline()
def run_abm(models,
resume_after=None,
chunk_size=None,
trace_hh_id=None,
trace_od=None):
settings = inject_settings(chunk_size=chunk_size,
trace_hh_id=trace_hh_id,
trace_od=trace_od)
inject.clear_cache()
tracing.config_logger()
pipeline.run(models=models, resume_after=resume_after)
def test_rng_access():
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
setup_dirs(configs_dir)
inject.add_injectable('rng_base_seed', 0)
pipeline.open_pipeline()
rng = pipeline.get_rn_generator()
assert isinstance(rng, random.Random)
pipeline.close_pipeline()
inject.clear_cache()
def test_full_run2():
configs_dir = os.path.join(os.path.dirname(__file__), 'configs2')
inject.add_injectable("configs_dir", configs_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data2')
inject.add_injectable("data_dir", data_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
_MODELS = [
'input_pre_processor',
'setup_data_structures',
'initial_seed_balancing',
'meta_control_factoring',
'final_seed_balancing',
'integerize_final_seed_weights',
'sub_balancing.geography = DISTRICT',
'sub_balancing.geography = TRACT',
'sub_balancing.geography=TAZ',
'expand_households',
'summarize',
'write_tables'
]
pipeline.run(models=_MODELS, resume_after=None)
assert isinstance(pipeline.get_table('expanded_household_ids'), pd.DataFrame)
# output tables list action: include
assert os.path.exists(os.path.join(output_dir, 'expanded_household_ids.csv'))
assert os.path.exists(os.path.join(output_dir, 'summary_DISTRICT.csv'))
assert not os.path.exists(os.path.join(output_dir, 'summary_TAZ.csv'))
# tables will no longer be available after pipeline is closed
pipeline.close_pipeline()
inject.clear_cache()
def setup_dirs(configs_dir):
inject.add_injectable("configs_dir", configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
def test_mini_pipeline_run():
setup_dirs()
inject_settings(households_sample_size=HOUSEHOLDS_SAMPLE_SIZE,
write_skim_cache=True
)
_MODELS = [
'initialize_landuse',
'compute_accessibility',
'initialize_households',
'school_location',
'workplace_location',
'auto_ownership_simulate'
]
pipeline.run(models=_MODELS, resume_after=None)
regress_mini_auto()
pipeline.run_model('cdap_simulate')
pipeline.run_model('mandatory_tour_frequency')
regress_mini_mtf()
regress_mini_location_choice_logsums()
# try to get a non-existant table
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("bogus")
assert "never checkpointed" in str(excinfo.value)
# try to get an existing table from a non-existant checkpoint
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("households", checkpoint_name="bogus")
assert "not in checkpoints" in str(excinfo.value)
# should create optional workplace_location_sample table
workplace_location_sample_df = pipeline.get_table("workplace_location_sample")
assert 'mode_choice_logsum' in workplace_location_sample_df
pipeline.close_pipeline()
inject.clear_cache()
close_handlers()
def setup_function():
inject.reinject_decorated_tables()
inject.remove_injectable('skim_dict')
inject.remove_injectable('skim_stack')
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
inject.add_injectable("configs_dir", configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
inject.add_injectable("data_dir", data_dir)
inject.clear_cache()
tracing.config_logger()
def setup_dirs(configs_dir, data_dir):
print(f"configs_dir: {configs_dir}")
inject.add_injectable('configs_dir', configs_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable('output_dir', output_dir)
print(f"data_dir: {data_dir}")
inject.add_injectable('data_dir', data_dir)
inject.clear_cache()
tracing.config_logger()
tracing.delete_output_files('csv')
tracing.delete_output_files('txt')
tracing.delete_output_files('yaml')
tracing.delete_output_files('omx')
def test_mini_pipeline_run():
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
setup_dirs(configs_dir)
inject_settings(configs_dir,
households_sample_size=HOUSEHOLDS_SAMPLE_SIZE,
# use_shadow_pricing=True
)
_MODELS = [
'initialize_landuse',
'compute_accessibility',
'initialize_households',
'school_location',
'workplace_location',
'auto_ownership_simulate'
]
pipeline.run(models=_MODELS, resume_after=None)
regress_mini_auto()
pipeline.run_model('cdap_simulate')
pipeline.run_model('mandatory_tour_frequency')
regress_mini_mtf()
# try to get a non-existant table
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("bogus")
assert "never checkpointed" in str(excinfo.value)
# try to get an existing table from a non-existant checkpoint
with pytest.raises(RuntimeError) as excinfo:
pipeline.get_table("households", checkpoint_name="bogus")
assert "not in checkpoints" in str(excinfo.value)
pipeline.close_pipeline()
inject.clear_cache()
close_handlers()
def test_mini_pipeline_run3():
# test that hh_ids setting overrides household sampling
setup_dirs()
inject_settings(hh_ids='override_hh_ids.csv')
households = inject.get_table('households').to_frame()
override_hh_ids = pd.read_csv(config.data_file_path('override_hh_ids.csv'))
print("\noverride_hh_ids\n%s" % override_hh_ids)
print("\nhouseholds\n%s" % households.index)
assert households.shape[0] == override_hh_ids.shape[0]
assert households.index.isin(override_hh_ids.household_id).all()
inject.clear_cache()
close_handlers()
def test_load_cached_accessibility():
inject.clear_cache()
inject.reinject_decorated_tables()
data_dir = [
os.path.join(os.path.dirname(__file__), 'data'),
example_path('data')
]
setup_dirs(data_dir=data_dir)
#
# add OPTIONAL ceched table accessibility to input_table_list
# activitysim.abm.tables.land_use.accessibility() will load this table if listed here
# presumably independently calculated outside activitysim or a cached copy created during a previous run
#
settings = config.read_settings_file('settings.yaml', mandatory=True)
input_table_list = settings.get('input_table_list')
input_table_list.append({
'tablename': 'accessibility',
'filename': 'cached_accessibility.csv',
'index_col': 'zone_id'
})
inject_settings(households_sample_size=HOUSEHOLDS_SAMPLE_SIZE,
input_table_list=input_table_list)
_MODELS = [
'initialize_landuse',
# 'compute_accessibility', # we load accessibility table ordinarily created by compute_accessibility
'initialize_households',
]
pipeline.run(models=_MODELS, resume_after=None)
accessibility_df = pipeline.get_table("accessibility")
assert 'auPkRetail' in accessibility_df
pipeline.close_pipeline()
inject.clear_cache()
close_handlers()
def test_mini_pipeline_run3():
# test that hh_ids setting overrides household sampling
configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
setup_dirs(configs_dir)
inject_settings(configs_dir, hh_ids='override_hh_ids.csv')
households = inject.get_table('households').to_frame()
override_hh_ids = pd.read_csv(config.data_file_path('override_hh_ids.csv'))
print("\noverride_hh_ids\n", override_hh_ids)
print("\nhouseholds\n", households.index)
assert households.shape[0] == override_hh_ids.shape[0]
assert households.index.isin(override_hh_ids.household_id).all()
inject.clear_cache()
close_handlers()
def test_weighting():
configs_dir = os.path.join(os.path.dirname(__file__), '..', '..',
'example_survey_weighting', 'configs')
inject.add_injectable("configs_dir", configs_dir)
data_dir = os.path.join(os.path.dirname(__file__), '..', '..',
'example_survey_weighting', 'data')
inject.add_injectable("data_dir", data_dir)
output_dir = os.path.join(os.path.dirname(__file__), 'output')
inject.add_injectable("output_dir", output_dir)
inject.clear_cache()
tracing.config_logger()
_MODELS = [
'input_pre_processor', 'setup_data_structures',
'initial_seed_balancing', 'meta_control_factoring',
'final_seed_balancing', 'summarize', 'write_tables'
]
pipeline.run(models=_MODELS, resume_after=None)
summary_hh_weights = pipeline.get_table('summary_hh_weights')
total_summary_hh_weights = summary_hh_weights[
'SUBREGCluster_balanced_weight'].sum()
seed_households = pd.read_csv(os.path.join(data_dir,
'seed_households.csv'))
total_seed_households_weights = seed_households['HHweight'].sum()
assert abs(total_summary_hh_weights - total_seed_households_weights) < 1
# tables will no longer be available after pipeline is closed
pipeline.close_pipeline()
inject.clear_cache()
def teardown_function(func):
inject.clear_cache()
inject.reinject_decorated_tables()
def teardown_function(func):
inject.clear_cache()
def teardown_function(func):
inject.clear_cache()
inject.reinject_decorated_tables()
