def test_restart_experiment(experiment_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment = experiment_class( config=sample_config(), db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, ) experiment.run() evaluations = num_linked_evaluations(db_engine) assert evaluations > 0 experiment = experiment_class( config=sample_config(), db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, replace=False, ) experiment.make_entity_date_table = mock.Mock() experiment.run() assert not experiment.make_entity_date_table.called
def shared_db_engine_with_source_data(shared_db_engine): """A successfully-run experiment. Its database schemas and project storage can be queried. Returns: (triage.experiments.SingleThreadedExperiment) """ populate_source_data(shared_db_engine) yield shared_db_engine
def test_filepaths_and_queries_give_same_hashes(experiment_class): with testing.postgresql.Postgresql() as postgresql, TemporaryDirectory( ) as temp_dir, mock.patch("triage.util.conf.open", side_effect=open_side_effect) as mock_file: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) query_config = sample_config(query_source="query") file_config = sample_config(query_source="filepath") experiment_with_queries = experiment_class( config=query_config, db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, ) experiment_with_filepaths = experiment_class( config=file_config, db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, ) assert (experiment_with_queries.experiment_hash == experiment_with_filepaths.experiment_hash) assert (experiment_with_queries.cohort_table_name == experiment_with_filepaths.cohort_table_name) assert (experiment_with_queries.labels_table_name == experiment_with_filepaths.labels_table_name)
def test_experiment_validator(): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) populate_source_data(db_engine) with mock.patch("triage.util.conf.open", side_effect=open_side_effect) as mock_file: ExperimentValidator(db_engine).run(sample_config("query")) ExperimentValidator(db_engine).run(sample_config("filepath"))
def test_profiling(db_engine): populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: project_path = os.path.join(temp_dir, "inspections") SingleThreadedExperiment(config=sample_config(), db_engine=db_engine, project_path=project_path, profile=True).run() assert len(os.listdir(os.path.join(project_path, "profiling_stats"))) == 1
def prepare_experiment(config): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment = SingleThreadedExperiment(config=config, db_engine=db_engine, project_path=os.path.join( temp_dir, 'inspections'), cleanup=False) yield experiment
def test_cleanup_timeout(_clean_up_mock, experiment_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment = experiment_class( config=sample_config(), db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, cleanup_timeout=0.02, # Set short timeout ) with pytest.raises(TimeoutError): experiment()
def test_profiling(db_engine): populate_source_data(db_engine) with TemporaryDirectory() as temp_dir, mock.patch( "triage.util.conf.open", side_effect=open_side_effect) as mock_file: project_path = os.path.join(temp_dir, "inspections") SingleThreadedExperiment( config=sample_config(), db_engine=db_engine, project_path=project_path, profile=True, ).run() assert len(os.listdir(os.path.join(project_path, "profiling_stats"))) == 1
def test_validate_default(experiment_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment = experiment_class( config=sample_config(), db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, ) experiment.validate = mock.MagicMock() experiment.run() experiment.validate.assert_called_once()
def finished_experiment(shared_db_engine, shared_project_storage): """A successfully-run experiment. Its database schemas and project storage can be queried. Returns: (triage.experiments.SingleThreadedExperiment) """ populate_source_data(shared_db_engine) base_config = sample_config() experiment = SingleThreadedExperiment( base_config, db_engine=shared_db_engine, project_path=shared_project_storage.project_path ) experiment.run() return experiment
def finished_experiment(shared_db_engine, shared_project_storage): """A successfully-run experiment. Its database schemas and project storage can be queried. Returns: (triage.experiments.SingleThreadedExperiment) """ populate_source_data(shared_db_engine) base_config = sample_config() with mock.patch("triage.util.conf.open", side_effect=open_side_effect) as mock_file: experiment = SingleThreadedExperiment( base_config, db_engine=shared_db_engine, project_path=shared_project_storage.project_path, ) experiment.run() return experiment
def test_validate_default(experiment_class): with testing.postgresql.Postgresql() as postgresql, TemporaryDirectory( ) as temp_dir, mock.patch("triage.util.conf.open", side_effect=open_side_effect) as mock_file: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) experiment = experiment_class( config=sample_config(), db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=True, ) experiment.validate = mock.MagicMock() experiment.run() experiment.validate.assert_called_once()
def prepare_experiment(config): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: with mock.patch( "triage.util.conf.open", side_effect=open_side_effect ) as mock_file: experiment = SingleThreadedExperiment( config=config, db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), cleanup=False, partial_run=True, ) yield experiment
def test_fill_timechop_config_missing(): remove_keys = [ 'model_update_frequency', 'training_as_of_date_frequencies', 'test_as_of_date_frequencies', 'max_training_histories', 'test_durations', 'feature_start_time', 'feature_end_time', 'label_start_time', 'label_end_time', 'training_label_timespans', 'test_label_timespans' ] # ensure redundant keys properly raise errors config = sample_config() config['temporal_config']['label_timespans'] = '1y' with pytest.raises(KeyError): timechop_config = fill_timechop_config_missing(config, None) with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) populate_source_data(db_engine) config = sample_config() for key in remove_keys: config['temporal_config'].pop(key) config['temporal_config']['label_timespans'] = '1y' timechop_config = fill_timechop_config_missing(config, db_engine) assert timechop_config['model_update_frequency'] == '100y' assert timechop_config['training_as_of_date_frequencies'] == '100y' assert timechop_config['test_as_of_date_frequencies'] == '100y' assert timechop_config['max_training_histories'] == '0d' assert timechop_config['test_durations'] == '0d' assert timechop_config['training_label_timespans'] == '1y' assert timechop_config['test_label_timespans'] == '1y' assert 'label_timespans' not in timechop_config.keys() assert timechop_config['feature_start_time'] == '2010-10-01' assert timechop_config['feature_end_time'] == '2013-10-01' assert timechop_config['label_start_time'] == '2010-10-01' assert timechop_config['label_end_time'] == '2013-10-01'
def model_evaluator(shared_db_engine, shared_project_storage): """Returns an instantiated ModelEvaluator available at module scope""" populate_source_data(shared_db_engine) base_config = sample_config() # We need to have an ensemble model to test ModelEvaluator correctly # so we can't use the finished_experiment fixture""" base_config['grid_config'] = { 'sklearn.ensemble.ExtraTreesClassifier': { 'n_estimators': [10], 'criterion': ['gini'], 'max_depth': [1], 'max_features': ['sqrt'], 'min_samples_split': [2], } } SingleThreadedExperiment( base_config, db_engine=shared_db_engine, project_path=shared_project_storage.project_path).run() return ModelEvaluator(1, 1, shared_db_engine)
def model_evaluator(shared_db_engine, shared_project_storage): """Returns an instantiated ModelEvaluator available at module scope""" populate_source_data(shared_db_engine) base_config = sample_config() # We need to have an ensemble model to test ModelEvaluator correctly # so we can't use the finished_experiment fixture""" base_config["grid_config"] = { "sklearn.ensemble.ExtraTreesClassifier": { "n_estimators": [10], "criterion": ["gini"], "max_depth": [1], "max_features": ["sqrt"], "min_samples_split": [2], } } with mock.patch("triage.util.conf.open", side_effect=open_side_effect) as mock_file: SingleThreadedExperiment( base_config, db_engine=shared_db_engine, project_path=shared_project_storage.project_path, ).run() return ModelEvaluator(1, 1, shared_db_engine)
def test_simple_experiment(experiment_class, matrix_storage_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment_class( config=sample_config(), db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), matrix_storage_class=matrix_storage_class, cleanup=True, ).run() # assert # 1. that model groups entries are present num_mgs = len([ row for row in db_engine.execute( "select * from model_metadata.model_groups") ]) assert num_mgs > 0 # 2. that model entries are present, and linked to model groups num_models = len([ row for row in db_engine.execute(""" select * from model_metadata.model_groups join model_metadata.models using (model_group_id) where model_comment = 'test2-final-final' """) ]) assert num_models > 0 # 3. predictions, linked to models for both training and testing predictions for set_type in ("train", "test"): num_predictions = len([ row for row in db_engine.execute(""" select * from {}_results.predictions join model_metadata.models using (model_id)""".format( set_type, set_type)) ]) assert num_predictions > 0 # 4. evaluations linked to predictions linked to models, for training and testing for set_type in ("train", "test"): num_evaluations = len([ row for row in db_engine.execute(""" select * from {}_results.evaluations e join model_metadata.models using (model_id) join {}_results.predictions p on ( e.model_id = p.model_id and e.evaluation_start_time <= p.as_of_date and e.evaluation_end_time >= p.as_of_date) """.format(set_type, set_type, set_type)) ]) assert num_evaluations > 0 # 5. subset evaluations linked to subsets and predictions linked to # models, for training and testing for set_type in ("train", "test"): num_evaluations = len([ row for row in db_engine.execute(""" select e.model_id, e.subset_hash from {}_results.evaluations e join model_metadata.models using (model_id) join model_metadata.subsets using (subset_hash) join {}_results.predictions p on ( e.model_id = p.model_id and e.evaluation_start_time <= p.as_of_date and e.evaluation_end_time >= p.as_of_date) group by e.model_id, e.subset_hash """.format(set_type, set_type)) ]) # 4 model groups trained/tested on 2 splits, with 1 metric + parameter assert num_evaluations == 8 # 6. experiment num_experiments = len([ row for row in db_engine.execute( "select * from model_metadata.experiments") ]) assert num_experiments == 1 # 7. that models are linked to experiments num_models_with_experiment = len([ row for row in db_engine.execute(""" select * from model_metadata.experiments join model_metadata.experiment_models using (experiment_hash) join model_metadata.models using (model_hash) """) ]) assert num_models == num_models_with_experiment # 8. that models have the train end date and label timespan results = [(model["train_end_time"], model["training_label_timespan"]) for model in db_engine.execute( "select * from model_metadata.models")] assert sorted(set(results)) == [ (datetime(2012, 6, 1), timedelta(180)), (datetime(2013, 6, 1), timedelta(180)), ] # 9. that the right number of individual importances are present individual_importances = [ row for row in db_engine.execute(""" select * from test_results.individual_importances join model_metadata.models using (model_id) """) ] assert len( individual_importances) == num_predictions * 2 # only 2 features # 10. Checking the proper matrices created and stored matrices = [ row for row in db_engine.execute(""" select matrix_type, num_observations from model_metadata.matrices""" ) ] types = [i[0] for i in matrices] counts = [i[1] for i in matrices] assert types.count("train") == 2 assert types.count("test") == 2 for i in counts: assert i > 0 assert len(matrices) == 4 # 11. Checking that all matrices are associated with the experiment linked_matrices = list( db_engine.execute("""select * from model_metadata.matrices join model_metadata.experiment_matrices using (matrix_uuid) join model_metadata.experiments using (experiment_hash)""")) assert len(linked_matrices) == len(matrices)
def test_baselines_with_missing_features(experiment_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) populate_source_data(db_engine) # set up the config with the baseline model and feature group mixing config = sample_config() config["grid_config"] = { "triage.component.catwalk.baselines.rankers.PercentileRankOneFeature": { "feature": ["entity_features_entity_id_1year_cat_sightings_count"] } } config["feature_group_definition"] = { "tables": [ "entity_features_aggregation_imputed", "zip_code_features_aggregation_imputed", ] } config["feature_group_strategies"] = ["leave-one-in"] with TemporaryDirectory() as temp_dir: experiment_class( config=config, db_engine=db_engine, project_path=os.path.join(temp_dir, "inspections"), ).run() # assert # 1. that model groups entries are present num_mgs = len([ row for row in db_engine.execute( "select * from model_metadata.model_groups") ]) assert num_mgs > 0 # 2. that model entries are present, and linked to model groups num_models = len([ row for row in db_engine.execute(""" select * from model_metadata.model_groups join model_metadata.models using (model_group_id) where model_comment = 'test2-final-final' """) ]) assert num_models > 0 # 3. predictions, linked to models num_predictions = len([ row for row in db_engine.execute(""" select * from test_results.predictions join model_metadata.models using (model_id)""") ]) assert num_predictions > 0 # 4. evaluations linked to predictions linked to models num_evaluations = len([ row for row in db_engine.execute(""" select * from test_results.evaluations e join model_metadata.models using (model_id) join test_results.predictions p on ( e.model_id = p.model_id and e.evaluation_start_time <= p.as_of_date and e.evaluation_end_time >= p.as_of_date) """) ]) assert num_evaluations > 0 # 5. experiment num_experiments = len([ row for row in db_engine.execute( "select * from model_metadata.experiments") ]) assert num_experiments == 1 # 6. that models are linked to experiments num_models_with_experiment = len([ row for row in db_engine.execute(""" select * from model_metadata.experiments join model_metadata.experiment_models using (experiment_hash) join model_metadata.models using (model_hash) """) ]) assert num_models == num_models_with_experiment # 7. that models have the train end date and label timespan results = [(model["train_end_time"], model["training_label_timespan"]) for model in db_engine.execute( "select * from model_metadata.models")] assert sorted(set(results)) == [ (datetime(2012, 6, 1), timedelta(180)), (datetime(2013, 6, 1), timedelta(180)), ] # 8. that the right number of individual importances are present individual_importances = [ row for row in db_engine.execute(""" select * from test_results.individual_importances join model_metadata.models using (model_id) """) ] assert len( individual_importances) == num_predictions * 2 # only 2 features
def test_simple_experiment(experiment_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment_class(config=sample_config(), db_engine=db_engine, model_storage_class=FSModelStorageEngine, project_path=os.path.join(temp_dir, 'inspections')).run() # assert # 1. that model groups entries are present num_mgs = len([ row for row in db_engine.execute('select * from results.model_groups') ]) assert num_mgs > 0 # 2. that model entries are present, and linked to model groups num_models = len([ row for row in db_engine.execute(''' select * from results.model_groups join results.models using (model_group_id) where model_comment = 'test2-final-final' ''') ]) assert num_models > 0 # 3. predictions, linked to models num_predictions = len([ row for row in db_engine.execute(''' select * from results.predictions join results.models using (model_id)''') ]) assert num_predictions > 0 # 4. evaluations linked to predictions linked to models num_evaluations = len([ row for row in db_engine.execute(''' select * from results.evaluations e join results.models using (model_id) join results.predictions p on ( e.model_id = p.model_id and e.evaluation_start_time <= p.as_of_date and e.evaluation_end_time >= p.as_of_date) ''') ]) assert num_evaluations > 0 # 5. experiment num_experiments = len([ row for row in db_engine.execute('select * from results.experiments') ]) assert num_experiments == 1 # 6. that models are linked to experiments num_models_with_experiment = len([ row for row in db_engine.execute(''' select * from results.experiments join results.models using (experiment_hash) ''') ]) assert num_models == num_models_with_experiment # 7. that models have the train end date and label timespan results = [ (model['train_end_time'], model['training_label_timespan']) for model in db_engine.execute('select * from results.models') ] assert sorted(set(results)) == [ (datetime(2012, 6, 1), timedelta(180)), (datetime(2013, 6, 1), timedelta(180)), ] # 8. that the right number of individual importances are present individual_importances = [ row for row in db_engine.execute(''' select * from results.individual_importances join results.models using (model_id) ''') ] assert len( individual_importances) == num_predictions * 2 # only 2 features
def test_experiment_validator(): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) populate_source_data(db_engine) ExperimentValidator(db_engine).run(sample_config())
def test_simple_experiment(experiment_class): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) populate_source_data(db_engine) with TemporaryDirectory() as temp_dir: experiment_class(config=sample_config(), db_engine=db_engine, model_storage_class=FSModelStorageEngine, project_path=os.path.join(temp_dir, 'inspections')).run() # assert # 1. that model groups entries are present num_mgs = len([ row for row in db_engine.execute( 'select * from model_metadata.model_groups') ]) assert num_mgs > 0 # 2. that model entries are present, and linked to model groups num_models = len([ row for row in db_engine.execute(''' select * from model_metadata.model_groups join model_metadata.models using (model_group_id) where model_comment = 'test2-final-final' ''') ]) assert num_models > 0 # 3. predictions, linked to models for both training and testing predictions for set_type in ("train", "test"): num_predictions = len([ row for row in db_engine.execute(''' select * from {}_results.{}_predictions join model_metadata.models using (model_id)'''.format( set_type, set_type)) ]) assert num_predictions > 0 # 4. evaluations linked to predictions linked to models, for training and testing for set_type in ("train", "test"): num_evaluations = len([ row for row in db_engine.execute(''' select * from {}_results.{}_evaluations e join model_metadata.models using (model_id) join {}_results.{}_predictions p on ( e.model_id = p.model_id and e.evaluation_start_time <= p.as_of_date and e.evaluation_end_time >= p.as_of_date) '''.format(set_type, set_type, set_type, set_type)) ]) assert num_evaluations > 0 # 5. experiment num_experiments = len([ row for row in db_engine.execute( 'select * from model_metadata.experiments') ]) assert num_experiments == 1 # 6. that models are linked to experiments num_models_with_experiment = len([ row for row in db_engine.execute(''' select * from model_metadata.experiments join model_metadata.models using (experiment_hash) ''') ]) assert num_models == num_models_with_experiment # 7. that models have the train end date and label timespan results = [(model['train_end_time'], model['training_label_timespan']) for model in db_engine.execute( 'select * from model_metadata.models')] assert sorted(set(results)) == [ (datetime(2012, 6, 1), timedelta(180)), (datetime(2013, 6, 1), timedelta(180)), ] # 8. that the right number of individual importances are present individual_importances = [ row for row in db_engine.execute(''' select * from test_results.individual_importances join model_metadata.models using (model_id) ''') ] assert len( individual_importances) == num_predictions * 2 # only 2 features # 9. Checking the proper matrices created and stored matrices = [ row for row in db_engine.execute(''' select matrix_type, n_examples from model_metadata.matrices''') ] types = [i[0] for i in matrices] counts = [i[1] for i in matrices] assert types.count('train') == 2 assert types.count('test') == 2 for i in counts: assert i > 0 assert len(matrices) == 4