def test_uniform_distribution(): with rig_engines() as (db_engine, project_storage): model = ModelFactory() feature_importances = [ FeatureImportanceFactory(model_rel=model, feature="feature_{}".format(i)) for i in range(0, 10) ] data_dict = {"entity_id": [1, 1], "as_of_date": ["2016-01-01", "2017-01-01"], "label": [0, 1]} for imp in feature_importances: data_dict[imp.feature] = [0.5, 0.5] metadata = matrix_metadata_creator() test_store = get_matrix_store( project_storage, pandas.DataFrame.from_dict(data_dict), metadata, ) results = uniform_distribution( db_engine, model_id=model.model_id, as_of_date=datetime.date(2016, 1, 1), test_matrix_store=test_store, n_ranks=5, ) assert len(results) == 5 # 5 features x 1 entity for this as_of_date for result in results: assert "entity_id" in result assert "feature_name" in result assert "score" in result assert "feature_value" in result assert result["feature_value"] == 0.5 assert result["score"] >= 0 assert result["score"] <= 1 assert isinstance(result["feature_name"], str) assert result["entity_id"] in [1, 2]
def test_predictor_save_predictions(matrix_type, predict_setup_args): (project_storage, db_engine, model_id) = predict_setup_args # if save_predictions is sent as False, don't save predictor = Predictor(project_storage.model_storage_engine(), db_engine, save_predictions=False) matrix = matrix_creator(index="entity_id") metadata = matrix_metadata_creator(end_time=AS_OF_DATE, matrix_type=matrix_type, indices=["entity_id"]) matrix_store = get_matrix_store(project_storage, matrix, metadata) train_matrix_columns = matrix.columns[0:-1].tolist() predict_proba = predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=train_matrix_columns, ) # assert # 1. that the returned predictions are of the desired length assert len(predict_proba) == 2 # 2. that the predictions table entries are present and # can be linked to the original models assert not table_has_data(f"{matrix_type}_predictions", db_engine)
def test_uniform_distribution_entity_id_index(): with rig_engines() as (db_engine, project_storage): model = ModelFactory() feature_importances = [ FeatureImportanceFactory(model_rel=model, feature='feature_{}'.format(i)) for i in range(0, 10) ] data_dict = {'entity_id': [1, 2]} for imp in feature_importances: data_dict[imp.feature] = [0.5, 0.5] metadata = matrix_metadata_creator(indices='entity_id') test_store = get_matrix_store( project_storage, pandas.DataFrame.from_dict(data_dict).set_index( metadata['indices']), metadata) results = uniform_distribution(db_engine, model_id=model.model_id, as_of_date='2016-01-01', test_matrix_store=test_store, n_ranks=5) assert len(results) == 10 # 5 features x 2 entities for result in results: assert 'entity_id' in result assert 'feature_name' in result assert 'score' in result assert 'feature_value' in result assert result['feature_value'] == 0.5 assert result['score'] >= 0 assert result['score'] <= 1 assert isinstance(result['feature_name'], str) assert result['entity_id'] in [1, 2]
def test_predictor_get_train_columns(): with prepare() as (project_storage, db_engine, model_id): predictor = Predictor(project_storage.model_storage_engine(), db_engine) train_store = get_matrix_store( project_storage=project_storage, matrix=matrix_creator(), metadata=matrix_metadata_creator(matrix_type="train"), ) # flip the order of some feature columns in the test matrix other_order_matrix = matrix_creator() order = other_order_matrix.columns.tolist() order[0], order[1] = order[1], order[0] other_order_matrix = other_order_matrix[order] test_store = get_matrix_store( project_storage=project_storage, matrix=other_order_matrix, metadata=matrix_metadata_creator(matrix_type="test"), ) # Runs the same test for training and testing predictions for store, mat_type in zip((train_store, test_store), ("train", "test")): predict_proba = predictor.predict( model_id, store, misc_db_parameters=dict(), train_matrix_columns=train_store.columns(), ) # assert # 1. that we calculated predictions assert len(predict_proba) > 0 # 2. that the predictions table entries are present and # can be linked to the original models records = [ row for row in db_engine.execute("""select entity_id, as_of_date from {}_results.predictions join model_metadata.models using (model_id)""".format( mat_type, mat_type)) ] assert len(records) > 0
def test_predictor_retrieve(): with prepare() as (project_storage, db_engine, model_id): predictor = Predictor(project_storage.model_storage_engine(), db_engine, replace=False) # create prediction set matrix = matrix_creator() metadata = matrix_metadata_creator() matrix_store = get_matrix_store(project_storage, matrix, metadata) predict_proba = predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=matrix.columns[0:-1].tolist()) # When run again, the predictions retrieved from the database # should match. # # Some trickiness here. Let's explain: # # If we are not careful, retrieving predictions from the database and # presenting them as a numpy array can result in a bad ordering, # since the given matrix may not be 'ordered' by some criteria # that can be easily represented by an ORDER BY clause. # # It will sometimes work, because without ORDER BY you will get # it back in the table's physical order, which unless something has # happened to the table will be the order you inserted it, # which could very well be the order in the matrix. # So it's not a bug that would necessarily immediately show itself, # but when it does go wrong your scores will be garbage. # # So we simulate a table order mutation that can happen over time: # Remove the first row and put it at the end. # If the Predictor doesn't explicitly reorder the results, this will fail # Only running on TestPrediction because TrainPrediction behaves the exact same way reorder_session = sessionmaker(bind=db_engine)() obj = reorder_session.query(TestPrediction).first() reorder_session.delete(obj) reorder_session.commit() make_transient(obj) reorder_session = sessionmaker(bind=db_engine)() reorder_session.add(obj) reorder_session.commit() predictor.load_model = Mock() new_predict_proba = predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=matrix.columns[0:-1].tolist()) assert_array_equal(new_predict_proba, predict_proba) assert not predictor.load_model.called
def test_replace_true_rerun(self): with testing.postgresql.Postgresql() as postgresql: # create an engine and generate a table with fake feature data engine = create_engine(postgresql.url()) ensure_db(engine) create_schemas( engine=engine, features_tables=features_tables, labels=labels, states=states, ) matrix_metadata = matrix_metadata_creator(state="active", test_duration="1month", label_name="booking") dates = [ datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0), datetime.datetime(2016, 3, 1, 0, 0), ] feature_dictionary = { "features0": ["f1", "f2"], "features1": ["f3", "f4"] } uuid = filename_friendly_hash(matrix_metadata) build_args = dict( as_of_times=dates, label_name="booking", label_type="binary", feature_dictionary=feature_dictionary, matrix_metadata=matrix_metadata, matrix_uuid=uuid, matrix_type="test", ) with get_matrix_storage_engine() as matrix_storage_engine: builder = MatrixBuilder( db_config=db_config, matrix_storage_engine=matrix_storage_engine, experiment_hash=experiment_hash, engine=engine, replace=True, ) builder.build_matrix(**build_args) assert len( matrix_storage_engine.get_store(uuid).design_matrix) == 5 assert builder.sessionmaker().query(Matrix).get(uuid) # rerun builder.build_matrix(**build_args) assert len( matrix_storage_engine.get_store(uuid).design_matrix) == 5 assert builder.sessionmaker().query(Matrix).get(uuid)
def setup_model_train_tester(project_storage, replace, additional_bigtrain_classnames=None): matrix_storage_engine = MatrixStorageEngine(project_storage) train_matrix_store = get_matrix_store( project_storage, metadata=matrix_metadata_creator(matrix_type="train"), write_to_db=False) test_matrix_store = get_matrix_store( project_storage, metadata=matrix_metadata_creator(matrix_type="test"), write_to_db=False) sample_train_kwargs = { 'matrix_store': train_matrix_store, 'class_path': None, 'parameters': {}, 'model_hash': None, 'misc_db_parameters': {} } train_test_task = { 'train_kwargs': sample_train_kwargs, 'train_store': train_matrix_store, 'test_store': test_matrix_store } predictor = MagicMock(spec_set=Predictor) trainer = MagicMock(spec_set=ModelTrainer) evaluator = MagicMock(spec_set=ModelEvaluator) individual_importance_calculator = MagicMock( spec_set=IndividualImportanceCalculator) protected_groups_generator = MagicMock(spec_set=ProtectedGroupsGenerator) train_tester = ModelTrainTester( matrix_storage_engine=matrix_storage_engine, model_trainer=trainer, model_evaluator=evaluator, individual_importance_calculator=individual_importance_calculator, predictor=predictor, subsets=[], replace=replace, protected_groups_generator=protected_groups_generator, additional_bigtrain_classnames=additional_bigtrain_classnames) return train_tester, train_test_task
def test_calculate_and_save(): with rig_engines() as (db_engine, project_storage): train_store = get_matrix_store( project_storage, matrix_creator(), matrix_metadata_creator(matrix_type='train'), ) test_store = get_matrix_store( project_storage, matrix_creator(), matrix_metadata_creator(matrix_type='test'), ) calculator = IndividualImportanceCalculator(db_engine, methods=['sample'], replace=False) # given a trained model # and a test matrix _, model_id = \ fake_trained_model( db_engine, train_matrix_uuid=train_store.uuid ) # i expect to be able to call calculate and save calculator.calculate_and_save_all_methods_and_dates( model_id, test_store) # and find individual importances in the results schema afterwards records = [ row for row in db_engine.execute('''select entity_id, as_of_date from test_results.individual_importances join model_metadata.models using (model_id)''') ] assert len(records) > 0 # and that when run again, has the same result calculator.calculate_and_save_all_methods_and_dates( model_id, test_store) new_records = [ row for row in db_engine.execute('''select entity_id, as_of_date from test_results.individual_importances join model_metadata.models using (model_id)''') ] assert len(records) == len(new_records) assert records == new_records
def test_replace_true_rerun(self): with testing.postgresql.Postgresql() as postgresql: # create an engine and generate a table with fake feature data engine = create_engine(postgresql.url()) ensure_db(engine) create_schemas(engine=engine, features_tables=features_tables, labels=labels, states=states) matrix_metadata = matrix_metadata_creator( state='state_one and state_two', test_duration='1month') dates = [ datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0), datetime.datetime(2016, 3, 1, 0, 0) ] feature_dictionary = { 'features0': ['f1', 'f2'], 'features1': ['f3', 'f4'], } uuid = metta.generate_uuid(matrix_metadata) build_args = dict(as_of_times=dates, label_name='booking', label_type='binary', feature_dictionary=feature_dictionary, matrix_metadata=matrix_metadata, matrix_uuid=uuid, matrix_type='test') with get_matrix_storage_engine() as matrix_storage_engine: builder = MatrixBuilder( db_config=db_config, matrix_storage_engine=matrix_storage_engine, engine=engine, replace=True) builder.build_matrix(**build_args) assert len(matrix_storage_engine.get_store(uuid).matrix) == 5 assert builder.sessionmaker().query(Matrix).get(uuid) # rerun builder.build_matrix(**build_args) assert len(matrix_storage_engine.get_store(uuid).matrix) == 5 assert builder.sessionmaker().query(Matrix).get(uuid)
def test_predictor_composite_index(): with prepare() as (project_storage, db_engine, model_id): predictor = Predictor(project_storage.model_storage_engine(), db_engine) dayone = datetime.datetime(2011, 1, 1) daytwo = datetime.datetime(2011, 1, 2) source_dict = { "entity_id": [1, 2, 1, 2], "as_of_date": [dayone, dayone, daytwo, daytwo], "feature_one": [3, 4, 5, 6], "feature_two": [5, 6, 7, 8], "label": [7, 8, 8, 7], } # Runs the same test for training and testing predictions for mat_type in ("train", "test"): matrix = pandas.DataFrame.from_dict(source_dict).set_index( ["entity_id", "as_of_date"]) metadata = matrix_metadata_creator(matrix_type=mat_type) matrix_store = get_matrix_store(project_storage, matrix, metadata) predict_proba = predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=["feature_one", "feature_two"], ) # assert # 1. that the returned predictions are of the desired length assert len(predict_proba) == 4 # 2. that the predictions table entries are present and # can be linked to the original models records = [ row for row in db_engine.execute("""select entity_id, as_of_date from {}_results.predictions join model_metadata.models using (model_id)""".format( mat_type, mat_type)) ] assert len(records) == 4
def test_predictor_needs_predictions(matrix_type, predict_setup_args): """Test that the logic that figures out if predictions are needed for a given model/matrix""" (project_storage, db_engine, model_id) = predict_setup_args # if not all of the predictions for the given model id and matrix are present in the db, # needs_predictions should return true. else, false predictor = Predictor(project_storage.model_storage_engine(), db_engine, 'worst') metadata = matrix_metadata_creator(matrix_type=matrix_type) matrix_store = get_matrix_store(project_storage, metadata=metadata) train_matrix_columns = matrix_store.columns() # we haven't done anything yet, this should definitely need predictions assert predictor.needs_predictions(matrix_store, model_id) predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=train_matrix_columns, ) # now that predictions have been made, this should no longer need predictions assert not predictor.needs_predictions(matrix_store, model_id)
def test_uniform_distribution_entity_id_index(): with rig_engines() as (db_engine, project_storage): model = ModelFactory() feature_importances = [ FeatureImportanceFactory(model_rel=model, feature="feature_{}".format(i)) for i in range(0, 10) ] data_dict = {"entity_id": [1, 2]} for imp in feature_importances: data_dict[imp.feature] = [0.5, 0.5] metadata = matrix_metadata_creator(indices="entity_id") test_store = get_matrix_store( project_storage, pandas.DataFrame.from_dict(data_dict).set_index( metadata["indices"]), metadata, ) results = uniform_distribution( db_engine, model_id=model.model_id, as_of_date="2016-01-01", test_matrix_store=test_store, n_ranks=5, ) assert len(results) == 10 # 5 features x 2 entities for result in results: assert "entity_id" in result assert "feature_name" in result assert "score" in result assert "feature_value" in result assert result["feature_value"] == 0.5 assert result["score"] >= 0 assert result["score"] <= 1 assert isinstance(result["feature_name"], str) assert result["entity_id"] in [1, 2]
def test_predictor_needs_predictions(matrix_type, predict_setup_args): (project_storage, db_engine, model_id) = predict_setup_args # if not all of the predictions for the given model id and matrix are present in the db, # needs_predictions should return true. else, false predictor = Predictor(project_storage.model_storage_engine(), db_engine) matrix = matrix_creator(index="entity_id") metadata = matrix_metadata_creator(end_time=AS_OF_DATE, matrix_type=matrix_type, indices=["entity_id"]) matrix_store = get_matrix_store(project_storage, matrix, metadata) train_matrix_columns = matrix.columns[0:-1].tolist() # we haven't done anything yet, this should definitely need predictions assert predictor.needs_predictions(matrix_store, model_id) predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=train_matrix_columns, ) # now that predictions have been made, this should no longer need predictions assert not predictor.needs_predictions(matrix_store, model_id)
def prediction_results(matrix_type, predictor, predict_setup_args): (project_storage, db_engine, model_id) = predict_setup_args dayone = datetime.datetime(2011, 1, 1) daytwo = datetime.datetime(2011, 1, 2) source_dict = { "entity_id": [1, 2, 3, 1, 2, 3], "as_of_date": [dayone, dayone, dayone, daytwo, daytwo, daytwo], "feature_one": [3] * 6, "feature_two": [5] * 6, "label": [True, False] * 3 } matrix = pd.DataFrame.from_dict(source_dict) metadata = matrix_metadata_creator(matrix_type=matrix_type) matrix_store = get_matrix_store(project_storage, matrix, metadata) predict_proba = predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=["feature_one", "feature_two"], ) return predict_proba
def test_predictor_entity_index(): with prepare() as (project_storage, db_engine, model_id): predictor = Predictor(project_storage.model_storage_engine(), db_engine) # Runs the same test for training and testing predictions for mat_type in ("train", "test"): matrix = matrix_creator(index="entity_id") metadata = matrix_metadata_creator(end_time=AS_OF_DATE, matrix_type=mat_type, indices=["entity_id"]) matrix_store = get_matrix_store(project_storage, matrix, metadata) train_matrix_columns = matrix.columns[0:-1].tolist() predict_proba = predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=train_matrix_columns, ) # assert # 1. that the returned predictions are of the desired length assert len(predict_proba) == 2 # 2. that the predictions table entries are present and # can be linked to the original models records = [ row for row in db_engine.execute("""select entity_id, as_of_date from {}_results.predictions join model_metadata.models using (model_id)""".format( mat_type, mat_type)) ] assert len(records) == 2 # 3. that the contained as_of_dates match what we sent in for record in records: assert record[1].date() == AS_OF_DATE # 4. that the entity ids match the given dataset assert sorted([record[0] for record in records]) == [1, 2] # 5. running with same model_id, different as of date # then with same as of date only replaces the records # with the same date # Runs the same test for training and testing predictions for mat_type in ("train", "test"): new_matrix = matrix_creator(index="entity_id") new_metadata = matrix_metadata_creator( end_time=AS_OF_DATE + datetime.timedelta(days=1), matrix_type=mat_type, indices=["entity_id"], ) new_matrix_store = get_matrix_store(project_storage, new_matrix, new_metadata) predictor.predict( model_id, new_matrix_store, misc_db_parameters=dict(), train_matrix_columns=train_matrix_columns, ) predictor.predict( model_id, matrix_store, misc_db_parameters=dict(), train_matrix_columns=train_matrix_columns, ) records = [ row for row in db_engine.execute("""select entity_id, as_of_date from {}_results.predictions join model_metadata.models using (model_id)""".format( mat_type, mat_type)) ] assert len(records) == 4 # 6. That we can delete the model when done prediction on it predictor.delete_model(model_id) assert predictor.load_model(model_id) is None
def test_integration(): with rig_engines() as (db_engine, project_storage): train_store = get_matrix_store( project_storage, matrix_creator(), matrix_metadata_creator(matrix_type='train')) as_of_dates = [datetime.date(2016, 12, 21), datetime.date(2017, 1, 21)] test_stores = [] for as_of_date in as_of_dates: matrix_store = get_matrix_store( project_storage, pandas.DataFrame.from_dict({ 'entity_id': [3], 'feature_one': [8], 'feature_two': [5], 'label': [0] }).set_index('entity_id'), matrix_metadata_creator(end_time=as_of_date, indices=['entity_id'])) test_stores.append(matrix_store) model_storage_engine = ModelStorageEngine(project_storage) experiment_hash = save_experiment_and_get_hash({}, db_engine) # instantiate pipeline objects trainer = ModelTrainer( experiment_hash=experiment_hash, model_storage_engine=model_storage_engine, db_engine=db_engine, ) predictor = Predictor(model_storage_engine, db_engine) model_evaluator = ModelEvaluator([{ 'metrics': ['precision@'], 'thresholds': { 'top_n': [5] } }], [{}], db_engine) # run the pipeline grid_config = { 'sklearn.linear_model.LogisticRegression': { 'C': [0.00001, 0.0001], 'penalty': ['l1', 'l2'], 'random_state': [2193] } } model_ids = trainer.train_models(grid_config=grid_config, misc_db_parameters=dict(), matrix_store=train_store) for model_id in model_ids: for as_of_date, test_store in zip(as_of_dates, test_stores): predictions_proba = predictor.predict( model_id, test_store, misc_db_parameters=dict(), train_matrix_columns=['feature_one', 'feature_two']) model_evaluator.evaluate( predictions_proba, test_store, model_id, ) # assert # 1. that the predictions table entries are present and # can be linked to the original models records = [ row for row in db_engine.execute( '''select entity_id, model_id, as_of_date from test_results.predictions join model_metadata.models using (model_id) order by 3, 2''') ] assert records == [ (3, 1, datetime.datetime(2016, 12, 21)), (3, 2, datetime.datetime(2016, 12, 21)), (3, 3, datetime.datetime(2016, 12, 21)), (3, 4, datetime.datetime(2016, 12, 21)), (3, 1, datetime.datetime(2017, 1, 21)), (3, 2, datetime.datetime(2017, 1, 21)), (3, 3, datetime.datetime(2017, 1, 21)), (3, 4, datetime.datetime(2017, 1, 21)), ] # that evaluations are there records = [ row for row in db_engine.execute(''' select model_id, evaluation_start_time, metric, parameter from test_results.evaluations order by 2, 1''') ] assert records == [ (1, datetime.datetime(2016, 12, 21), 'precision@', '5_abs'), (2, datetime.datetime(2016, 12, 21), 'precision@', '5_abs'), (3, datetime.datetime(2016, 12, 21), 'precision@', '5_abs'), (4, datetime.datetime(2016, 12, 21), 'precision@', '5_abs'), (1, datetime.datetime(2017, 1, 21), 'precision@', '5_abs'), (2, datetime.datetime(2017, 1, 21), 'precision@', '5_abs'), (3, datetime.datetime(2017, 1, 21), 'precision@', '5_abs'), (4, datetime.datetime(2017, 1, 21), 'precision@', '5_abs'), ]
def test_integration(): with rig_engines() as (db_engine, project_storage): train_store = get_matrix_store( project_storage, matrix_creator(), matrix_metadata_creator(matrix_type="train"), ) as_of_dates = [datetime.date(2016, 12, 21), datetime.date(2017, 1, 21)] test_stores = [] for as_of_date in as_of_dates: matrix_store = get_matrix_store( project_storage, pandas.DataFrame.from_dict({ "entity_id": [3], "feature_one": [8], "feature_two": [5], "label": [0], }).set_index("entity_id"), matrix_metadata_creator(end_time=as_of_date, indices=["entity_id"]), ) test_stores.append(matrix_store) model_storage_engine = ModelStorageEngine(project_storage) experiment_hash = save_experiment_and_get_hash({}, db_engine) # instantiate pipeline objects trainer = ModelTrainer( experiment_hash=experiment_hash, model_storage_engine=model_storage_engine, db_engine=db_engine, ) predictor = Predictor(model_storage_engine, db_engine) model_evaluator = ModelEvaluator([{ "metrics": ["precision@"], "thresholds": { "top_n": [5] } }], [{}], db_engine) # run the pipeline grid_config = { "sklearn.linear_model.LogisticRegression": { "C": [0.00001, 0.0001], "penalty": ["l1", "l2"], "random_state": [2193], } } model_ids = trainer.train_models(grid_config=grid_config, misc_db_parameters=dict(), matrix_store=train_store) for model_id in model_ids: for as_of_date, test_store in zip(as_of_dates, test_stores): predictions_proba = predictor.predict( model_id, test_store, misc_db_parameters=dict(), train_matrix_columns=["feature_one", "feature_two"], ) model_evaluator.evaluate(predictions_proba, test_store, model_id) # assert # 1. that the predictions table entries are present and # can be linked to the original models records = [ row for row in db_engine.execute( """select entity_id, model_id, as_of_date from test_results.predictions join model_metadata.models using (model_id) order by 3, 2""") ] assert records == [ (3, 1, datetime.datetime(2016, 12, 21)), (3, 2, datetime.datetime(2016, 12, 21)), (3, 3, datetime.datetime(2016, 12, 21)), (3, 4, datetime.datetime(2016, 12, 21)), (3, 1, datetime.datetime(2017, 1, 21)), (3, 2, datetime.datetime(2017, 1, 21)), (3, 3, datetime.datetime(2017, 1, 21)), (3, 4, datetime.datetime(2017, 1, 21)), ] # that evaluations are there records = [ row for row in db_engine.execute(""" select model_id, evaluation_start_time, metric, parameter from test_results.evaluations order by 2, 1""") ] assert records == [ (1, datetime.datetime(2016, 12, 21), "precision@", "5_abs"), (2, datetime.datetime(2016, 12, 21), "precision@", "5_abs"), (3, datetime.datetime(2016, 12, 21), "precision@", "5_abs"), (4, datetime.datetime(2016, 12, 21), "precision@", "5_abs"), (1, datetime.datetime(2017, 1, 21), "precision@", "5_abs"), (2, datetime.datetime(2017, 1, 21), "precision@", "5_abs"), (3, datetime.datetime(2017, 1, 21), "precision@", "5_abs"), (4, datetime.datetime(2017, 1, 21), "precision@", "5_abs"), ]