def test_restart_experiment(experiment_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment = experiment_class(
config=sample_config(),
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
)
experiment.run()
evaluations = num_linked_evaluations(db_engine)
assert evaluations > 0
experiment = experiment_class(
config=sample_config(),
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
replace=False,
)
experiment.make_entity_date_table = mock.Mock()
experiment.run()
assert not experiment.make_entity_date_table.called
def shared_db_engine_with_source_data(shared_db_engine):
"""A successfully-run experiment. Its database schemas and project storage can be queried.
Returns: (triage.experiments.SingleThreadedExperiment)
"""
populate_source_data(shared_db_engine)
yield shared_db_engine
def test_filepaths_and_queries_give_same_hashes(experiment_class):
with testing.postgresql.Postgresql() as postgresql, TemporaryDirectory(
) as temp_dir, mock.patch("triage.util.conf.open",
side_effect=open_side_effect) as mock_file:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
query_config = sample_config(query_source="query")
file_config = sample_config(query_source="filepath")
experiment_with_queries = experiment_class(
config=query_config,
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
)
experiment_with_filepaths = experiment_class(
config=file_config,
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
)
assert (experiment_with_queries.experiment_hash ==
experiment_with_filepaths.experiment_hash)
assert (experiment_with_queries.cohort_table_name ==
experiment_with_filepaths.cohort_table_name)
assert (experiment_with_queries.labels_table_name ==
experiment_with_filepaths.labels_table_name)
def test_experiment_validator():
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
populate_source_data(db_engine)
with mock.patch("triage.util.conf.open",
side_effect=open_side_effect) as mock_file:
ExperimentValidator(db_engine).run(sample_config("query"))
ExperimentValidator(db_engine).run(sample_config("filepath"))
def test_profiling(db_engine):
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
project_path = os.path.join(temp_dir, "inspections")
SingleThreadedExperiment(config=sample_config(),
db_engine=db_engine,
project_path=project_path,
profile=True).run()
assert len(os.listdir(os.path.join(project_path,
"profiling_stats"))) == 1
def prepare_experiment(config):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment = SingleThreadedExperiment(config=config,
db_engine=db_engine,
project_path=os.path.join(
temp_dir, 'inspections'),
cleanup=False)
yield experiment
def test_cleanup_timeout(_clean_up_mock, experiment_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment = experiment_class(
config=sample_config(),
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
cleanup_timeout=0.02, # Set short timeout
)
with pytest.raises(TimeoutError):
experiment()
def test_profiling(db_engine):
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir, mock.patch(
"triage.util.conf.open",
side_effect=open_side_effect) as mock_file:
project_path = os.path.join(temp_dir, "inspections")
SingleThreadedExperiment(
config=sample_config(),
db_engine=db_engine,
project_path=project_path,
profile=True,
).run()
assert len(os.listdir(os.path.join(project_path,
"profiling_stats"))) == 1
def test_validate_default(experiment_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment = experiment_class(
config=sample_config(),
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
)
experiment.validate = mock.MagicMock()
experiment.run()
experiment.validate.assert_called_once()
def finished_experiment(shared_db_engine, shared_project_storage):
"""A successfully-run experiment. Its database schemas and project storage can be queried.
Returns: (triage.experiments.SingleThreadedExperiment)
"""
populate_source_data(shared_db_engine)
base_config = sample_config()
experiment = SingleThreadedExperiment(
base_config,
db_engine=shared_db_engine,
project_path=shared_project_storage.project_path
)
experiment.run()
return experiment
def finished_experiment(shared_db_engine, shared_project_storage):
"""A successfully-run experiment. Its database schemas and project storage can be queried.
Returns: (triage.experiments.SingleThreadedExperiment)
"""
populate_source_data(shared_db_engine)
base_config = sample_config()
with mock.patch("triage.util.conf.open", side_effect=open_side_effect) as mock_file:
experiment = SingleThreadedExperiment(
base_config,
db_engine=shared_db_engine,
project_path=shared_project_storage.project_path,
)
experiment.run()
return experiment
def test_validate_default(experiment_class):
with testing.postgresql.Postgresql() as postgresql, TemporaryDirectory(
) as temp_dir, mock.patch("triage.util.conf.open",
side_effect=open_side_effect) as mock_file:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
experiment = experiment_class(
config=sample_config(),
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=True,
)
experiment.validate = mock.MagicMock()
experiment.run()
experiment.validate.assert_called_once()
def prepare_experiment(config):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
with mock.patch(
"triage.util.conf.open", side_effect=open_side_effect
) as mock_file:
experiment = SingleThreadedExperiment(
config=config,
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
cleanup=False,
partial_run=True,
)
yield experiment
def test_fill_timechop_config_missing():
remove_keys = [
'model_update_frequency',
'training_as_of_date_frequencies',
'test_as_of_date_frequencies',
'max_training_histories',
'test_durations',
'feature_start_time',
'feature_end_time',
'label_start_time',
'label_end_time',
'training_label_timespans',
'test_label_timespans'
]
# ensure redundant keys properly raise errors
config = sample_config()
config['temporal_config']['label_timespans'] = '1y'
with pytest.raises(KeyError):
timechop_config = fill_timechop_config_missing(config, None)
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
populate_source_data(db_engine)
config = sample_config()
for key in remove_keys:
config['temporal_config'].pop(key)
config['temporal_config']['label_timespans'] = '1y'
timechop_config = fill_timechop_config_missing(config, db_engine)
assert timechop_config['model_update_frequency'] == '100y'
assert timechop_config['training_as_of_date_frequencies'] == '100y'
assert timechop_config['test_as_of_date_frequencies'] == '100y'
assert timechop_config['max_training_histories'] == '0d'
assert timechop_config['test_durations'] == '0d'
assert timechop_config['training_label_timespans'] == '1y'
assert timechop_config['test_label_timespans'] == '1y'
assert 'label_timespans' not in timechop_config.keys()
assert timechop_config['feature_start_time'] == '2010-10-01'
assert timechop_config['feature_end_time'] == '2013-10-01'
assert timechop_config['label_start_time'] == '2010-10-01'
assert timechop_config['label_end_time'] == '2013-10-01'
def model_evaluator(shared_db_engine, shared_project_storage):
"""Returns an instantiated ModelEvaluator available at module scope"""
populate_source_data(shared_db_engine)
base_config = sample_config()
# We need to have an ensemble model to test ModelEvaluator correctly
# so we can't use the finished_experiment fixture"""
base_config['grid_config'] = {
'sklearn.ensemble.ExtraTreesClassifier': {
'n_estimators': [10],
'criterion': ['gini'],
'max_depth': [1],
'max_features': ['sqrt'],
'min_samples_split': [2],
}
}
SingleThreadedExperiment(
base_config,
db_engine=shared_db_engine,
project_path=shared_project_storage.project_path).run()
return ModelEvaluator(1, 1, shared_db_engine)
def model_evaluator(shared_db_engine, shared_project_storage):
"""Returns an instantiated ModelEvaluator available at module scope"""
populate_source_data(shared_db_engine)
base_config = sample_config()
# We need to have an ensemble model to test ModelEvaluator correctly
# so we can't use the finished_experiment fixture"""
base_config["grid_config"] = {
"sklearn.ensemble.ExtraTreesClassifier": {
"n_estimators": [10],
"criterion": ["gini"],
"max_depth": [1],
"max_features": ["sqrt"],
"min_samples_split": [2],
}
}
with mock.patch("triage.util.conf.open",
side_effect=open_side_effect) as mock_file:
SingleThreadedExperiment(
base_config,
db_engine=shared_db_engine,
project_path=shared_project_storage.project_path,
).run()
return ModelEvaluator(1, 1, shared_db_engine)
def test_simple_experiment(experiment_class, matrix_storage_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment_class(
config=sample_config(),
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
matrix_storage_class=matrix_storage_class,
cleanup=True,
).run()
# assert
# 1. that model groups entries are present
num_mgs = len([
row for row in db_engine.execute(
"select * from model_metadata.model_groups")
])
assert num_mgs > 0
# 2. that model entries are present, and linked to model groups
num_models = len([
row for row in db_engine.execute("""
select * from model_metadata.model_groups
join model_metadata.models using (model_group_id)
where model_comment = 'test2-final-final'
""")
])
assert num_models > 0
# 3. predictions, linked to models for both training and testing predictions
for set_type in ("train", "test"):
num_predictions = len([
row for row in db_engine.execute("""
select * from {}_results.predictions
join model_metadata.models using (model_id)""".format(
set_type, set_type))
])
assert num_predictions > 0
# 4. evaluations linked to predictions linked to models, for training and testing
for set_type in ("train", "test"):
num_evaluations = len([
row for row in db_engine.execute("""
select * from {}_results.evaluations e
join model_metadata.models using (model_id)
join {}_results.predictions p on (
e.model_id = p.model_id and
e.evaluation_start_time <= p.as_of_date and
e.evaluation_end_time >= p.as_of_date)
""".format(set_type, set_type, set_type))
])
assert num_evaluations > 0
# 5. subset evaluations linked to subsets and predictions linked to
# models, for training and testing
for set_type in ("train", "test"):
num_evaluations = len([
row for row in db_engine.execute("""
select e.model_id, e.subset_hash from {}_results.evaluations e
join model_metadata.models using (model_id)
join model_metadata.subsets using (subset_hash)
join {}_results.predictions p on (
e.model_id = p.model_id and
e.evaluation_start_time <= p.as_of_date and
e.evaluation_end_time >= p.as_of_date)
group by e.model_id, e.subset_hash
""".format(set_type, set_type))
])
# 4 model groups trained/tested on 2 splits, with 1 metric + parameter
assert num_evaluations == 8
# 6. experiment
num_experiments = len([
row for row in db_engine.execute(
"select * from model_metadata.experiments")
])
assert num_experiments == 1
# 7. that models are linked to experiments
num_models_with_experiment = len([
row for row in db_engine.execute("""
select * from model_metadata.experiments
join model_metadata.experiment_models using (experiment_hash)
join model_metadata.models using (model_hash)
""")
])
assert num_models == num_models_with_experiment
# 8. that models have the train end date and label timespan
results = [(model["train_end_time"], model["training_label_timespan"])
for model in db_engine.execute(
"select * from model_metadata.models")]
assert sorted(set(results)) == [
(datetime(2012, 6, 1), timedelta(180)),
(datetime(2013, 6, 1), timedelta(180)),
]
# 9. that the right number of individual importances are present
individual_importances = [
row for row in db_engine.execute("""
select * from test_results.individual_importances
join model_metadata.models using (model_id)
""")
]
assert len(
individual_importances) == num_predictions * 2 # only 2 features
# 10. Checking the proper matrices created and stored
matrices = [
row for row in db_engine.execute("""
select matrix_type, num_observations from model_metadata.matrices"""
)
]
types = [i[0] for i in matrices]
counts = [i[1] for i in matrices]
assert types.count("train") == 2
assert types.count("test") == 2
for i in counts:
assert i > 0
assert len(matrices) == 4
# 11. Checking that all matrices are associated with the experiment
linked_matrices = list(
db_engine.execute("""select * from model_metadata.matrices
join model_metadata.experiment_matrices using (matrix_uuid)
join model_metadata.experiments using (experiment_hash)"""))
assert len(linked_matrices) == len(matrices)
def test_baselines_with_missing_features(experiment_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
populate_source_data(db_engine)
# set up the config with the baseline model and feature group mixing
config = sample_config()
config["grid_config"] = {
"triage.component.catwalk.baselines.rankers.PercentileRankOneFeature":
{
"feature":
["entity_features_entity_id_1year_cat_sightings_count"]
}
}
config["feature_group_definition"] = {
"tables": [
"entity_features_aggregation_imputed",
"zip_code_features_aggregation_imputed",
]
}
config["feature_group_strategies"] = ["leave-one-in"]
with TemporaryDirectory() as temp_dir:
experiment_class(
config=config,
db_engine=db_engine,
project_path=os.path.join(temp_dir, "inspections"),
).run()
# assert
# 1. that model groups entries are present
num_mgs = len([
row for row in db_engine.execute(
"select * from model_metadata.model_groups")
])
assert num_mgs > 0
# 2. that model entries are present, and linked to model groups
num_models = len([
row for row in db_engine.execute("""
select * from model_metadata.model_groups
join model_metadata.models using (model_group_id)
where model_comment = 'test2-final-final'
""")
])
assert num_models > 0
# 3. predictions, linked to models
num_predictions = len([
row for row in db_engine.execute("""
select * from test_results.predictions
join model_metadata.models using (model_id)""")
])
assert num_predictions > 0
# 4. evaluations linked to predictions linked to models
num_evaluations = len([
row for row in db_engine.execute("""
select * from test_results.evaluations e
join model_metadata.models using (model_id)
join test_results.predictions p on (
e.model_id = p.model_id and
e.evaluation_start_time <= p.as_of_date and
e.evaluation_end_time >= p.as_of_date)
""")
])
assert num_evaluations > 0
# 5. experiment
num_experiments = len([
row for row in db_engine.execute(
"select * from model_metadata.experiments")
])
assert num_experiments == 1
# 6. that models are linked to experiments
num_models_with_experiment = len([
row for row in db_engine.execute("""
select * from model_metadata.experiments
join model_metadata.experiment_models using (experiment_hash)
join model_metadata.models using (model_hash)
""")
])
assert num_models == num_models_with_experiment
# 7. that models have the train end date and label timespan
results = [(model["train_end_time"], model["training_label_timespan"])
for model in db_engine.execute(
"select * from model_metadata.models")]
assert sorted(set(results)) == [
(datetime(2012, 6, 1), timedelta(180)),
(datetime(2013, 6, 1), timedelta(180)),
]
# 8. that the right number of individual importances are present
individual_importances = [
row for row in db_engine.execute("""
select * from test_results.individual_importances
join model_metadata.models using (model_id)
""")
]
assert len(
individual_importances) == num_predictions * 2 # only 2 features
def test_simple_experiment(experiment_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment_class(config=sample_config(),
db_engine=db_engine,
model_storage_class=FSModelStorageEngine,
project_path=os.path.join(temp_dir,
'inspections')).run()
# assert
# 1. that model groups entries are present
num_mgs = len([
row
for row in db_engine.execute('select * from results.model_groups')
])
assert num_mgs > 0
# 2. that model entries are present, and linked to model groups
num_models = len([
row for row in db_engine.execute('''
select * from results.model_groups
join results.models using (model_group_id)
where model_comment = 'test2-final-final'
''')
])
assert num_models > 0
# 3. predictions, linked to models
num_predictions = len([
row for row in db_engine.execute('''
select * from results.predictions
join results.models using (model_id)''')
])
assert num_predictions > 0
# 4. evaluations linked to predictions linked to models
num_evaluations = len([
row for row in db_engine.execute('''
select * from results.evaluations e
join results.models using (model_id)
join results.predictions p on (
e.model_id = p.model_id and
e.evaluation_start_time <= p.as_of_date and
e.evaluation_end_time >= p.as_of_date)
''')
])
assert num_evaluations > 0
# 5. experiment
num_experiments = len([
row
for row in db_engine.execute('select * from results.experiments')
])
assert num_experiments == 1
# 6. that models are linked to experiments
num_models_with_experiment = len([
row for row in db_engine.execute('''
select * from results.experiments
join results.models using (experiment_hash)
''')
])
assert num_models == num_models_with_experiment
# 7. that models have the train end date and label timespan
results = [
(model['train_end_time'], model['training_label_timespan'])
for model in db_engine.execute('select * from results.models')
]
assert sorted(set(results)) == [
(datetime(2012, 6, 1), timedelta(180)),
(datetime(2013, 6, 1), timedelta(180)),
]
# 8. that the right number of individual importances are present
individual_importances = [
row for row in db_engine.execute('''
select * from results.individual_importances
join results.models using (model_id)
''')
]
assert len(
individual_importances) == num_predictions * 2 # only 2 features
def test_experiment_validator():
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
populate_source_data(db_engine)
ExperimentValidator(db_engine).run(sample_config())
def test_simple_experiment(experiment_class):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
experiment_class(config=sample_config(),
db_engine=db_engine,
model_storage_class=FSModelStorageEngine,
project_path=os.path.join(temp_dir,
'inspections')).run()
# assert
# 1. that model groups entries are present
num_mgs = len([
row for row in db_engine.execute(
'select * from model_metadata.model_groups')
])
assert num_mgs > 0
# 2. that model entries are present, and linked to model groups
num_models = len([
row for row in db_engine.execute('''
select * from model_metadata.model_groups
join model_metadata.models using (model_group_id)
where model_comment = 'test2-final-final'
''')
])
assert num_models > 0
# 3. predictions, linked to models for both training and testing predictions
for set_type in ("train", "test"):
num_predictions = len([
row for row in db_engine.execute('''
select * from {}_results.{}_predictions
join model_metadata.models using (model_id)'''.format(
set_type, set_type))
])
assert num_predictions > 0
# 4. evaluations linked to predictions linked to models, for training and testing
for set_type in ("train", "test"):
num_evaluations = len([
row for row in db_engine.execute('''
select * from {}_results.{}_evaluations e
join model_metadata.models using (model_id)
join {}_results.{}_predictions p on (
e.model_id = p.model_id and
e.evaluation_start_time <= p.as_of_date and
e.evaluation_end_time >= p.as_of_date)
'''.format(set_type, set_type, set_type, set_type))
])
assert num_evaluations > 0
# 5. experiment
num_experiments = len([
row for row in db_engine.execute(
'select * from model_metadata.experiments')
])
assert num_experiments == 1
# 6. that models are linked to experiments
num_models_with_experiment = len([
row for row in db_engine.execute('''
select * from model_metadata.experiments
join model_metadata.models using (experiment_hash)
''')
])
assert num_models == num_models_with_experiment
# 7. that models have the train end date and label timespan
results = [(model['train_end_time'], model['training_label_timespan'])
for model in db_engine.execute(
'select * from model_metadata.models')]
assert sorted(set(results)) == [
(datetime(2012, 6, 1), timedelta(180)),
(datetime(2013, 6, 1), timedelta(180)),
]
# 8. that the right number of individual importances are present
individual_importances = [
row for row in db_engine.execute('''
select * from test_results.individual_importances
join model_metadata.models using (model_id)
''')
]
assert len(
individual_importances) == num_predictions * 2 # only 2 features
# 9. Checking the proper matrices created and stored
matrices = [
row for row in db_engine.execute('''
select matrix_type, n_examples from model_metadata.matrices''')
]
types = [i[0] for i in matrices]
counts = [i[1] for i in matrices]
assert types.count('train') == 2
assert types.count('test') == 2
for i in counts:
assert i > 0
assert len(matrices) == 4
