def test_all_same_labels(db_engine_with_results_schema): num_entities = 5 trained_model, model_id = fake_trained_model( db_engine_with_results_schema, train_end_time=TRAIN_END_TIME, ) for label_value in [0, 1]: labels = [label_value] * num_entities # We should be able to calculate accuracy even if all of the labels # are the same, but ROC_AUC requires some positive and some # negative labels, so we should get one NULL value # for this config training_metric_groups = [{"metrics": ["accuracy", "roc_auc"]}] # Acquire fake data and objects to be used in the tests model_evaluator = ModelEvaluator( {}, training_metric_groups, db_engine_with_results_schema, ) fake_matrix_store = MockMatrixStore( matrix_type="train", matrix_uuid=str(labels), label_count=num_entities, db_engine=db_engine_with_results_schema, init_labels=pd.DataFrame( { "label_value": labels, "entity_id": list(range(num_entities)), "as_of_date": [TRAIN_END_TIME] * num_entities, } ) .set_index(["entity_id", "as_of_date"]) .label_value, init_as_of_dates=[TRAIN_END_TIME], ) model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_matrix_store, model_id ) for metric, best, worst, stochastic in db_engine_with_results_schema.execute( f"""select metric, best_value, worst_value, stochastic_value from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_matrix_store.as_of_dates[0]), ): if metric == "accuracy": assert best is not None assert worst is not None assert stochastic is not None else: assert best is None assert worst is None assert stochastic is None
def test_ModelEvaluator_needs_evaluation_with_bias_audit( db_engine_with_results_schema): # test that if a bias audit config is passed, and there are no matching bias audits # in the database, needs_evaluation returns true # this all assumes that evaluations are populated. those tests are in the 'no_bias_audit' test model_evaluator = ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": { "top_n": [3] }, }, ], training_metric_groups=[], bias_config={'thresholds': { 'top_n': [2] }}, db_engine=db_engine_with_results_schema, ) model_with_evaluations = ModelFactory() eval_time = datetime.datetime(2016, 1, 1) as_of_date_frequency = "3d" for subset_hash in [""]: EvaluationFactory( model_rel=model_with_evaluations, evaluation_start_time=eval_time, evaluation_end_time=eval_time, as_of_date_frequency=as_of_date_frequency, metric="precision@", parameter="3_abs", subset_hash=subset_hash, ) session.commit() # make a test matrix to pass in metadata_overrides = { 'as_of_date_frequency': as_of_date_frequency, 'as_of_times': [eval_time], } test_matrix_store = MockMatrixStore("test", "1234", 5, db_engine_with_results_schema, metadata_overrides=metadata_overrides) assert model_evaluator.needs_evaluations( matrix_store=test_matrix_store, model_id=model_with_evaluations.model_id, subset_hash="", )
def test_subset_labels_and_predictions(db_engine_with_results_schema): num_entities = 5 labels = [0, 1, 0, 1, 0] predictions_proba = np.array([0.6, 0.4, 0.55, 0.70, 0.3]) fake_matrix_store = MockMatrixStore( matrix_type="test", matrix_uuid="abcde", label_count=num_entities, db_engine=db_engine_with_results_schema, init_labels=pd.DataFrame( { "label_value": labels, "entity_id": list(range(num_entities)), "as_of_date": [TRAIN_END_TIME] * num_entities, } ) .set_index(["entity_id", "as_of_date"]) .label_value, init_as_of_dates=[TRAIN_END_TIME], ) for subset in SUBSETS: if subset["name"] == "evens": expected_result = 3 elif subset["name"] == "odds": expected_result = 2 elif subset["name"] == "empty": expected_result = 0 populate_subset_data( db_engine_with_results_schema, subset, list(range(num_entities)) ) ( subset_labels, subset_predictions, subset_protected_df, ) = subset_labels_and_predictions( subset_df=query_subset_table( db_engine_with_results_schema, fake_matrix_store.as_of_dates, get_subset_table_name(subset), ), predictions_proba=predictions_proba, labels=fake_matrix_store.labels, protected_df=pd.DataFrame(), ) assert len(subset_labels) == expected_result assert len(subset_predictions) == expected_result
def test_evaluation_with_protected_df(db_engine_with_results_schema): # Test that if a protected_df is passed (along with bias config, the only real needed one # being threshold info), an Aequitas report is written to the database. model_evaluator = ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": { "top_n": [3] }, }, ], training_metric_groups=[], bias_config={'thresholds': { 'top_n': [2] }}, db_engine=db_engine_with_results_schema, ) testing_labels = np.array([1, 0]) testing_prediction_probas = np.array([0.56, 0.55]) fake_test_matrix_store = MockMatrixStore("test", "1234", 5, db_engine_with_results_schema, testing_labels) trained_model, model_id = fake_trained_model( db_engine_with_results_schema, train_end_time=TRAIN_END_TIME, ) protected_df = pd.DataFrame({ "entity_id": fake_test_matrix_store.design_matrix.index.levels[0].tolist(), "protectedattribute1": "value1" }) model_evaluator.evaluate(testing_prediction_probas, fake_test_matrix_store, model_id, protected_df) for record in db_engine_with_results_schema.execute( """select * from test_results.aequitas where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_test_matrix_store.as_of_dates[0]), ): assert record['model_id'] == model_id assert record['parameter'] == '2_abs' assert record['attribute_name'] == 'protectedattribute1' assert record['attribute_value'] == 'value1'
def test_evaluation_with_sort_ties(db_engine_with_results_schema): model_evaluator = ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": { "top_n": [3] }, }, ], training_metric_groups=[], db_engine=db_engine_with_results_schema, ) testing_labels = np.array([1, 0, 1, 0, 0]) testing_prediction_probas = np.array([0.56, 0.55, 0.5, 0.5, 0.3]) fake_test_matrix_store = MockMatrixStore("test", "1234", 5, db_engine_with_results_schema, testing_labels) trained_model, model_id = fake_trained_model( db_engine_with_results_schema, train_end_time=TRAIN_END_TIME, ) model_evaluator.evaluate(testing_prediction_probas, fake_test_matrix_store, model_id) for record in db_engine_with_results_schema.execute( """select * from test_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_test_matrix_store.as_of_dates[0]), ): assert record["num_labeled_examples"] == 5 assert record["num_positive_labels"] == 2 assert_almost_equal(float(record["worst_value"]), 0.33333, 5) assert_almost_equal(float(record["best_value"]), 0.66666, 5) assert record["num_sort_trials"] == SORT_TRIALS assert record["stochastic_value"] > record["worst_value"] assert record["stochastic_value"] < record["best_value"] assert record["standard_deviation"]
def test_evaluating_early_warning(): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) testing_metric_groups = [ { "metrics": [ "precision@", "recall@", "true positives@", "true negatives@", "false positives@", "false negatives@", ], "thresholds": {"percentiles": [5.0, 10.0], "top_n": [5, 10]}, }, { "metrics": [ "f1", "mediocre", "accuracy", "roc_auc", "average precision score", ] }, {"metrics": ["fbeta@"], "parameters": [{"beta": 0.75}, {"beta": 1.25}]}, ] training_metric_groups = [{"metrics": ["accuracy", "roc_auc"]}] custom_metrics = {"mediocre": always_half} model_evaluator = ModelEvaluator( testing_metric_groups, training_metric_groups, db_engine, custom_metrics=custom_metrics, ) labels = fake_labels(5) fake_train_matrix_store = MockMatrixStore("train", "efgh", 5, db_engine, labels) fake_test_matrix_store = MockMatrixStore("test", "1234", 5, db_engine, labels) trained_model, model_id = fake_trained_model(db_engine) # Evaluate the testing metrics and test for all of them. model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_test_matrix_store, model_id ) records = [ row[0] for row in db_engine.execute( """select distinct(metric || parameter) from test_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_test_matrix_store.as_of_dates[0]), ) ] assert records == [ "accuracy", "average precision score", "f1", "false [email protected]_pct", "false negatives@10_abs", "false [email protected]_pct", "false negatives@5_abs", "false [email protected]_pct", "false positives@10_abs", "false [email protected]_pct", "false positives@5_abs", "[email protected]_beta", "[email protected]_beta", "mediocre", "[email protected]_pct", "precision@10_abs", "[email protected]_pct", "precision@5_abs", "[email protected]_pct", "recall@10_abs", "[email protected]_pct", "recall@5_abs", "roc_auc", "true [email protected]_pct", "true negatives@10_abs", "true [email protected]_pct", "true negatives@5_abs", "true [email protected]_pct", "true positives@10_abs", "true [email protected]_pct", "true positives@5_abs", ] # Evaluate the training metrics and test model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_train_matrix_store, model_id ) records = [ row[0] for row in db_engine.execute( """select distinct(metric || parameter) from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_train_matrix_store.as_of_dates[0]), ) ] assert records == ["accuracy", "roc_auc"]
def test_model_scoring_inspections(): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) testing_metric_groups = [ { "metrics": ["precision@", "recall@", "fpr@"], "thresholds": {"percentiles": [50.0], "top_n": [3]}, }, { # ensure we test a non-thresholded metric as well "metrics": ["accuracy"] }, ] training_metric_groups = [ {"metrics": ["accuracy"], "thresholds": {"percentiles": [50.0]}} ] model_evaluator = ModelEvaluator( testing_metric_groups, training_metric_groups, db_engine ) testing_labels = numpy.array([True, False, numpy.nan, True, False]) testing_prediction_probas = numpy.array([0.56, 0.4, 0.55, 0.5, 0.3]) training_labels = numpy.array( [False, False, True, True, True, False, True, True] ) training_prediction_probas = numpy.array( [0.6, 0.4, 0.55, 0.70, 0.3, 0.2, 0.8, 0.6] ) fake_train_matrix_store = MockMatrixStore( "train", "efgh", 5, db_engine, training_labels ) fake_test_matrix_store = MockMatrixStore( "test", "1234", 5, db_engine, testing_labels ) trained_model, model_id = fake_trained_model(db_engine) # Evaluate testing matrix and test the results model_evaluator.evaluate( testing_prediction_probas, fake_test_matrix_store, model_id ) for record in db_engine.execute( """select * from test_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_test_matrix_store.as_of_dates[0]), ): assert record["num_labeled_examples"] == 4 assert record["num_positive_labels"] == 2 if record["parameter"] == "": assert record["num_labeled_above_threshold"] == 4 elif "pct" in record["parameter"]: assert record["num_labeled_above_threshold"] == 1 else: assert record["num_labeled_above_threshold"] == 2 # Evaluate the training matrix and test the results model_evaluator.evaluate( training_prediction_probas, fake_train_matrix_store, model_id ) for record in db_engine.execute( """select * from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_train_matrix_store.as_of_dates[0]), ): assert record["num_labeled_examples"] == 8 assert record["num_positive_labels"] == 5 assert record["value"] == 0.625
def test_evaluation_sorting_with_protected_df(db_engine_with_results_schema): # Test that if a protected_df is passed (along with bias config, the only real needed one # being threshold info), an Aequitas report is written to the database. model_evaluator = ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": {"top_n": [3]}, }, ], training_metric_groups=[], bias_config={"thresholds": {"top_n": [2]}}, db_engine=db_engine_with_results_schema, ) testing_labels = np.array([1, 1, 1, 0, 1]) testing_prediction_probas = np.array([0.56, 0.55, 0.92, 0.85, 0.24]) fake_test_matrix_store = MockMatrixStore( "test", "1234", 5, db_engine_with_results_schema, metadata_overrides={"as_of_times": [TRAIN_END_TIME]}, matrix=pd.DataFrame.from_dict( { "entity_id": [1, 2, 3, 4, 5], "as_of_date": [pd.Timestamp(2016, 1, 1)] * 5, "feature_one": [3, 4, 3, 4, 3], "feature_two": [5, 6, 5, 6, 5], "label": testing_labels, } ).set_index(MatrixStore.indices), init_labels=pd.DataFrame( { "label_value": testing_labels, "entity_id": [1, 2, 3, 4, 5], "as_of_date": [pd.Timestamp(2016, 1, 1)] * 5, } ) .set_index(["entity_id", "as_of_date"]) .label_value, init_as_of_dates=[TRAIN_END_TIME], ) trained_model, model_id = fake_trained_model( db_engine_with_results_schema, train_end_time=TRAIN_END_TIME, ) protected_df = pd.DataFrame( { # "entity_id": fake_test_matrix_store.design_matrix.index.levels[0].tolist(), # "as_of_date": fake_test_matrix_store.design_matrix.index.levels[1].tolist(), "protectedattribute1": ["low", "low", "low", "high", "high"] }, index=fake_test_matrix_store.design_matrix.index, ) # should be low has 3 records, all 1's; high has 2 records, one 1 expected = { "low": {"group_size": 3, "group_label_neg": 0, "group_label_pos": 3}, "high": {"group_size": 2, "group_label_neg": 1, "group_label_pos": 1}, } model_evaluator.evaluate( testing_prediction_probas, fake_test_matrix_store, model_id, protected_df ) for record in db_engine_with_results_schema.execute( """select * from test_results.aequitas where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_test_matrix_store.as_of_dates[0]), ): assert record["model_id"] == model_id assert record["parameter"] == "2_abs" assert record["attribute_name"] == "protectedattribute1" for col, value in expected[record["attribute_value"]].items(): assert record[col] == value
def test_ModelEvaluator_needs_evaluation_no_bias_audit(db_engine_with_results_schema): # TEST SETUP: # create two models: one that has zero evaluations, # one that has an evaluation for precision@100_abs # both overall and for each subset model_with_evaluations = ModelFactory() model_without_evaluations = ModelFactory() eval_time = datetime.datetime(2016, 1, 1) as_of_date_frequency = "3d" for subset_hash in [""] + [filename_friendly_hash(subset) for subset in SUBSETS]: EvaluationFactory( model_rel=model_with_evaluations, evaluation_start_time=eval_time, evaluation_end_time=eval_time, as_of_date_frequency=as_of_date_frequency, metric="precision@", parameter="100_abs", subset_hash=subset_hash, ) session.commit() # make a test matrix to pass in metadata_overrides = { "as_of_date_frequency": as_of_date_frequency, "as_of_times": [eval_time], } test_matrix_store = MockMatrixStore( "test", "1234", 5, db_engine_with_results_schema, metadata_overrides=metadata_overrides, ) train_matrix_store = MockMatrixStore( "train", "2345", 5, db_engine_with_results_schema, metadata_overrides=metadata_overrides, ) # the evaluated model has test evaluations for precision, but not recall, # so this needs evaluations for subset in SUBSETS: if not subset: subset_hash = "" else: subset_hash = filename_friendly_hash(subset) assert ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@", "recall@"], "thresholds": {"top_n": [100]}, } ], training_metric_groups=[], db_engine=db_engine_with_results_schema, ).needs_evaluations( matrix_store=test_matrix_store, model_id=model_with_evaluations.model_id, subset_hash=subset_hash, ) # the evaluated model has test evaluations for precision, # so this should not need evaluations for subset in SUBSETS: if not subset: subset_hash = "" else: subset_hash = filename_friendly_hash(subset) assert not ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": {"top_n": [100]}, } ], training_metric_groups=[], db_engine=db_engine_with_results_schema, ).needs_evaluations( matrix_store=test_matrix_store, model_id=model_with_evaluations.model_id, subset_hash=subset_hash, ) # the non-evaluated model has no evaluations, # so this should need evaluations for subset in SUBSETS: if not subset: subset_hash = "" else: subset_hash = filename_friendly_hash(subset) assert ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": {"top_n": [100]}, } ], training_metric_groups=[], db_engine=db_engine_with_results_schema, ).needs_evaluations( matrix_store=test_matrix_store, model_id=model_without_evaluations.model_id, subset_hash=subset_hash, ) # the evaluated model has no *train* evaluations, # so the train matrix should need evaluations for subset in SUBSETS: if not subset: subset_hash = "" else: subset_hash = filename_friendly_hash(subset) assert ModelEvaluator( testing_metric_groups=[ { "metrics": ["precision@"], "thresholds": {"top_n": [100]}, } ], training_metric_groups=[ { "metrics": ["precision@"], "thresholds": {"top_n": [100]}, } ], db_engine=db_engine_with_results_schema, ).needs_evaluations( matrix_store=train_matrix_store, model_id=model_with_evaluations.model_id, subset_hash=subset_hash, ) session.close() session.remove()
def test_model_scoring_inspections(db_engine_with_results_schema): testing_metric_groups = [ { "metrics": ["precision@", "recall@", "fpr@"], "thresholds": {"percentiles": [50.0], "top_n": [3]}, }, { # ensure we test a non-thresholded metric as well "metrics": ["accuracy"] }, ] training_metric_groups = [ {"metrics": ["accuracy"], "thresholds": {"percentiles": [50.0]}} ] model_evaluator = ModelEvaluator( testing_metric_groups, training_metric_groups, db_engine_with_results_schema, ) testing_labels = np.array([1, 0, np.nan, 1, 0]) testing_prediction_probas = np.array([0.56, 0.4, 0.55, 0.5, 0.3]) training_labels = np.array([0, 0, 1, 1, 1, 0, 1, 1]) training_prediction_probas = np.array([0.6, 0.4, 0.55, 0.70, 0.3, 0.2, 0.8, 0.6]) fake_train_matrix_store = MockMatrixStore( "train", "efgh", 5, db_engine_with_results_schema, training_labels ) fake_test_matrix_store = MockMatrixStore( "test", "1234", 5, db_engine_with_results_schema, testing_labels ) trained_model, model_id = fake_trained_model( db_engine_with_results_schema, train_end_time=TRAIN_END_TIME, ) # Evaluate testing matrix and test the results model_evaluator.evaluate( testing_prediction_probas, fake_test_matrix_store, model_id ) for record in db_engine_with_results_schema.execute( """select * from test_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_test_matrix_store.as_of_dates[0]), ): assert record["num_labeled_examples"] == 4 assert record["num_positive_labels"] == 2 if record["parameter"] == "": assert record["num_labeled_above_threshold"] == 4 elif "pct" in record["parameter"]: assert record["num_labeled_above_threshold"] == 1 else: assert record["num_labeled_above_threshold"] == 2 # Evaluate the training matrix and test the results model_evaluator.evaluate( training_prediction_probas, fake_train_matrix_store, model_id ) for record in db_engine_with_results_schema.execute( """select * from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_train_matrix_store.as_of_dates[0]), ): assert record["num_labeled_examples"] == 8 assert record["num_positive_labels"] == 5 assert record["worst_value"] == 0.625 assert record["best_value"] == 0.625 assert record["stochastic_value"] == 0.625 # best/worst are same, should shortcut trials assert record["num_sort_trials"] == 0 assert record["standard_deviation"] == 0
def test_evaluating_early_warning(db_engine_with_results_schema): num_entities = 10 labels = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] # Set up testing configuration parameters testing_metric_groups = [ { "metrics": [ "precision@", "recall@", "true positives@", "true negatives@", "false positives@", "false negatives@", ], "thresholds": {"percentiles": [5.0, 10.0], "top_n": [5, 10]}, }, { "metrics": [ "f1", "mediocre", "accuracy", "roc_auc", "average precision score", ] }, {"metrics": ["fbeta@"], "parameters": [{"beta": 0.75}, {"beta": 1.25}]}, ] training_metric_groups = [{"metrics": ["accuracy", "roc_auc"]}] custom_metrics = {"mediocre": always_half} # Acquire fake data and objects to be used in the tests model_evaluator = ModelEvaluator( testing_metric_groups, training_metric_groups, db_engine_with_results_schema, custom_metrics=custom_metrics, ) fake_test_matrix_store = MockMatrixStore( matrix_type="test", matrix_uuid="efgh", label_count=num_entities, db_engine=db_engine_with_results_schema, init_labels=pd.DataFrame( { "label_value": labels, "entity_id": list(range(num_entities)), "as_of_date": [TRAIN_END_TIME] * num_entities, } ) .set_index(["entity_id", "as_of_date"]) .label_value, init_as_of_dates=[TRAIN_END_TIME], ) fake_train_matrix_store = MockMatrixStore( matrix_type="train", matrix_uuid="1234", label_count=num_entities, db_engine=db_engine_with_results_schema, init_labels=pd.DataFrame( { "label_value": labels, "entity_id": list(range(num_entities)), "as_of_date": [TRAIN_END_TIME] * num_entities, } ) .set_index(["entity_id", "as_of_date"]) .label_value, init_as_of_dates=[TRAIN_END_TIME], ) trained_model, model_id = fake_trained_model( db_engine_with_results_schema, train_end_time=TRAIN_END_TIME, ) # ensure that the matrix uuid is present matrix_uuids = [ row[0] for row in db_engine_with_results_schema.execute( "select matrix_uuid from test_results.evaluations" ) ] assert all(matrix_uuid == "efgh" for matrix_uuid in matrix_uuids) # Evaluate the training metrics and test model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_train_matrix_store, model_id ) records = [ row[0] for row in db_engine_with_results_schema.execute( """select distinct(metric || parameter) from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1""", (model_id, fake_train_matrix_store.as_of_dates[0]), ) ] assert records == ["accuracy", "roc_auc"] # Run tests for overall and subset evaluations for subset in SUBSETS: if subset is None: where_hash = "" else: populate_subset_data( db_engine_with_results_schema, subset, list(range(num_entities)) ) SubsetFactory(subset_hash=filename_friendly_hash(subset)) session.commit() where_hash = f"and subset_hash = '{filename_friendly_hash(subset)}'" # Evaluate the testing metrics and test for all of them. model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_test_matrix_store, model_id, subset=subset, ) records = [ row[0] for row in db_engine_with_results_schema.execute( f"""\ select distinct(metric || parameter) from test_results.evaluations where model_id = %s and evaluation_start_time = %s {where_hash} order by 1 """, (model_id, fake_test_matrix_store.as_of_dates[0]), ) ] assert records == [ "accuracy", "average precision score", "f1", "false [email protected]_pct", "false negatives@10_abs", "false [email protected]_pct", "false negatives@5_abs", "false [email protected]_pct", "false positives@10_abs", "false [email protected]_pct", "false positives@5_abs", "[email protected]_beta", "[email protected]_beta", "mediocre", "[email protected]_pct", "precision@10_abs", "[email protected]_pct", "precision@5_abs", "[email protected]_pct", "recall@10_abs", "[email protected]_pct", "recall@5_abs", "roc_auc", "true [email protected]_pct", "true negatives@10_abs", "true [email protected]_pct", "true negatives@5_abs", "true [email protected]_pct", "true positives@10_abs", "true [email protected]_pct", "true positives@5_abs", ] # Evaluate the training metrics and test model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_train_matrix_store, model_id, subset=subset, ) records = [ row[0] for row in db_engine_with_results_schema.execute( f"""select distinct(metric || parameter) from train_results.evaluations where model_id = %s and evaluation_start_time = %s {where_hash} order by 1""", (model_id, fake_train_matrix_store.as_of_dates[0]), ) ] assert records == ["accuracy", "roc_auc"] # ensure that the matrix uuid is present matrix_uuids = [ row[0] for row in db_engine_with_results_schema.execute( "select matrix_uuid from train_results.evaluations" ) ] assert all(matrix_uuid == "1234" for matrix_uuid in matrix_uuids)
def test_evaluating_early_warning(): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) testing_metric_groups = [{ 'metrics': ['precision@', 'recall@', 'true positives@', 'true negatives@', 'false positives@', 'false negatives@'], 'thresholds': { 'percentiles': [5.0, 10.0], 'top_n': [5, 10] } }, { 'metrics': ['f1', 'mediocre', 'accuracy', 'roc_auc', 'average precision score'], }, { 'metrics': ['fbeta@'], 'parameters': [{'beta': 0.75}, {'beta': 1.25}] }] training_metric_groups = [{'metrics': ['accuracy', 'roc_auc']}] custom_metrics = {'mediocre': always_half} model_evaluator = ModelEvaluator(testing_metric_groups, training_metric_groups, db_engine, custom_metrics=custom_metrics ) labels = fake_labels(5) fake_train_matrix_store = MockMatrixStore('train', 'efgh', 5, db_engine, labels) fake_test_matrix_store = MockMatrixStore('test', '1234', 5, db_engine, labels) trained_model, model_id = fake_trained_model( 'myproject', InMemoryModelStorageEngine('myproject'), db_engine ) # Evaluate the testing metrics and test for all of them. model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_test_matrix_store, model_id, ) records = [ row[0] for row in db_engine.execute( '''select distinct(metric || parameter) from test_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1''', (model_id, fake_test_matrix_store.as_of_dates[0]) ) ] assert records == [ 'accuracy', 'average precision score', 'f1', 'false [email protected]_pct', 'false negatives@10_abs', 'false [email protected]_pct', 'false negatives@5_abs', 'false [email protected]_pct', 'false positives@10_abs', 'false [email protected]_pct', 'false positives@5_abs', '[email protected]_beta', '[email protected]_beta', 'mediocre', '[email protected]_pct', 'precision@10_abs', '[email protected]_pct', 'precision@5_abs', '[email protected]_pct', 'recall@10_abs', '[email protected]_pct', 'recall@5_abs', 'roc_auc', 'true [email protected]_pct', 'true negatives@10_abs', 'true [email protected]_pct', 'true negatives@5_abs', 'true [email protected]_pct', 'true positives@10_abs', 'true [email protected]_pct', 'true positives@5_abs' ] # Evaluate the training metrics and test model_evaluator.evaluate( trained_model.predict_proba(labels)[:, 1], fake_train_matrix_store, model_id, ) records = [ row[0] for row in db_engine.execute( '''select distinct(metric || parameter) from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1''', (model_id, fake_train_matrix_store.as_of_dates[0]) ) ] assert records == ['accuracy', 'roc_auc']
def test_model_scoring_inspections(): with testing.postgresql.Postgresql() as postgresql: db_engine = create_engine(postgresql.url()) ensure_db(db_engine) testing_metric_groups = [{ 'metrics': ['precision@', 'recall@', 'fpr@'], 'thresholds': {'percentiles': [50.0], 'top_n': [3]} }, { # ensure we test a non-thresholded metric as well 'metrics': ['accuracy'], }] training_metric_groups = [{'metrics': ['accuracy'], 'thresholds': {'percentiles': [50.0]}}] model_evaluator = ModelEvaluator(testing_metric_groups, training_metric_groups, db_engine) testing_labels = numpy.array([True, False, numpy.nan, True, False]) testing_prediction_probas = numpy.array([0.56, 0.4, 0.55, 0.5, 0.3]) training_labels = numpy.array([False, False, True, True, True, False, True, True]) training_prediction_probas = numpy.array([0.6, 0.4, 0.55, 0.70, 0.3, 0.2, 0.8, 0.6]) fake_train_matrix_store = MockMatrixStore('train', 'efgh', 5, db_engine, training_labels) fake_test_matrix_store = MockMatrixStore('test', '1234', 5, db_engine, testing_labels) trained_model, model_id = fake_trained_model( 'myproject', InMemoryModelStorageEngine('myproject'), db_engine ) # Evaluate testing matrix and test the results model_evaluator.evaluate( testing_prediction_probas, fake_test_matrix_store, model_id, ) for record in db_engine.execute( '''select * from test_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1''', (model_id, fake_test_matrix_store.as_of_dates[0]) ): assert record['num_labeled_examples'] == 4 assert record['num_positive_labels'] == 2 if record['parameter'] == '': assert record['num_labeled_above_threshold'] == 4 elif 'pct' in record['parameter']: assert record['num_labeled_above_threshold'] == 1 else: assert record['num_labeled_above_threshold'] == 2 # Evaluate the training matrix and test the results model_evaluator.evaluate( training_prediction_probas, fake_train_matrix_store, model_id, ) for record in db_engine.execute( '''select * from train_results.evaluations where model_id = %s and evaluation_start_time = %s order by 1''', (model_id, fake_train_matrix_store.as_of_dates[0]) ): assert record['num_labeled_examples'] == 8 assert record['num_positive_labels'] == 5 assert record['value'] == 0.625
def test_ModelEvaluator_needs_evaluation(db_engine): ensure_db(db_engine) init_engine(db_engine) # TEST SETUP: # create two models: one that has zero evaluations, # one that has an evaluation for precision@100_abs model_with_evaluations = ModelFactory() model_without_evaluations = ModelFactory() eval_time = datetime.datetime(2016, 1, 1) as_of_date_frequency = "3d" EvaluationFactory(model_rel=model_with_evaluations, evaluation_start_time=eval_time, evaluation_end_time=eval_time, as_of_date_frequency=as_of_date_frequency, metric="precision@", parameter="100_abs") session.commit() # make a test matrix to pass in metadata_overrides = { 'as_of_date_frequency': as_of_date_frequency, 'end_time': eval_time, } test_matrix_store = MockMatrixStore("test", "1234", 5, db_engine, metadata_overrides=metadata_overrides) train_matrix_store = MockMatrixStore("train", "2345", 5, db_engine, metadata_overrides=metadata_overrides) # the evaluated model has test evaluations for precision, but not recall, # so this needs evaluations assert ModelEvaluator(testing_metric_groups=[{ "metrics": ["precision@", "recall@"], "thresholds": { "top_n": [100] }, }], training_metric_groups=[], db_engine=db_engine).needs_evaluations( matrix_store=test_matrix_store, model_id=model_with_evaluations.model_id, ) # the evaluated model has test evaluations for precision, # so this should not need evaluations assert not ModelEvaluator(testing_metric_groups=[{ "metrics": ["precision@"], "thresholds": { "top_n": [100] }, }], training_metric_groups=[], db_engine=db_engine).needs_evaluations( matrix_store=test_matrix_store, model_id=model_with_evaluations.model_id, ) # the non-evaluated model has no evaluations, # so this should need evaluations assert ModelEvaluator(testing_metric_groups=[{ "metrics": ["precision@"], "thresholds": { "top_n": [100] }, }], training_metric_groups=[], db_engine=db_engine).needs_evaluations( matrix_store=test_matrix_store, model_id=model_without_evaluations.model_id, ) # the evaluated model has no *train* evaluations, # so the train matrix should need evaluations assert ModelEvaluator(testing_metric_groups=[{ "metrics": ["precision@"], "thresholds": { "top_n": [100] }, }], training_metric_groups=[{ "metrics": ["precision@"], "thresholds": { "top_n": [100] }, }], db_engine=db_engine).needs_evaluations( matrix_store=train_matrix_store, model_id=model_with_evaluations.model_id, ) session.close() session.remove()