示例#1
0
def test_evaluation_factories():
    with testing.postgresql.Postgresql() as postgresql:
        engine = create_engine(postgresql.url())
        schema.Base.metadata.create_all(engine)
        init_engine(engine)

        # create some basic evaluations, but with the same model group and
        # model to test the factory relationships
        model_group = ModelGroupFactory()
        model = ModelFactory(model_group_rel=model_group)
        for metric, value in [
            ('precision@', 0.4),
            ('recall@', 0.3),
        ]:
            EvaluationFactory(model_rel=model,
                              metric=metric,
                              parameter='100_abs',
                              value=value)
        session.commit()
        results = engine.execute('''
            select
                model_group_id,
                m.model_id,
                e.metric,
                e.value
            from
                results.evaluations e
                join results.models m using (model_id)
        ''')
        for model_group_id, model_id, metric, value in results:
            # if the evaluations are created with the model group and model,
            # as opposed to an autoprovisioned one,
            # the ids in a fresh DB should be 1
            assert model_group_id == 1
            assert model_id == 1
示例#2
0
def test_uniform_distribution_entity_date_index():
    with testing.postgresql.Postgresql() as postgresql:
        db_engine = create_engine(postgresql.url())
        ensure_db(db_engine)
        init_engine(db_engine)
        model = ModelFactory()
        feature_importances = [
            FeatureImportanceFactory(model_rel=model,
                                     feature='feature_{}'.format(i))
            for i in range(0, 10)
        ]
        data_dict = {
            'entity_id': [1, 1],
            'as_of_date': ['2016-01-01', '2017-01-01']
        }
        for imp in feature_importances:
            data_dict[imp.feature] = [0.5, 0.5]
        test_store = InMemoryMatrixStore(
            matrix=pandas.DataFrame.from_dict(data_dict).set_index(
                ['entity_id', 'as_of_date']),
            metadata=sample_metadata())
        session.commit()
        results = uniform_distribution(db_engine,
                                       model_id=model.model_id,
                                       as_of_date='2016-01-01',
                                       test_matrix_store=test_store,
                                       n_ranks=5)

        assert len(results) == 5  # 5 features x 1 entity for this as_of_date
        for result in results:
            assert 'entity_id' in result
            assert 'feature_name' in result
            assert 'score' in result
            assert 'feature_value' in result
            assert result['feature_value'] == 0.5
            assert result['score'] >= 0
            assert result['score'] <= 1
            assert isinstance(result['feature_name'], str)
            assert result['entity_id'] in [1, 2]
示例#3
0
def test_prediction_factories():
    with testing.postgresql.Postgresql() as postgresql:
        engine = create_engine(postgresql.url())
        schema.Base.metadata.create_all(engine)
        init_engine(engine)

        # create some basic predictions, but with the same model group and
        # model to test the factory relationships
        model_group = ModelGroupFactory()
        model = ModelFactory(model_group_rel=model_group)
        for entity_id, as_of_date in [
            (1, '2016-01-01'),
            (1, '2016-04-01'),
            (2, '2016-01-01'),
            (2, '2016-04-01'),
            (3, '2016-01-01'),
            (3, '2016-04-01'),
        ]:
            PredictionFactory(
                model_rel=model,
                entity_id=entity_id,
                as_of_date=as_of_date,
            )
            IndividualImportanceFactory(
                model_rel=model,
                entity_id=entity_id,
                as_of_date=as_of_date,
            )
        session.commit()
        results = engine.execute('''
            select m.*, p.*
            from
                results.predictions p
                join results.models m using (model_id)
                join results.individual_importances i using (model_id, entity_id, as_of_date)
        ''')
        assert len([row for row in results]) == 6
示例#4
0
 def setup_data(self, engine):
     ensure_db(engine)
     init_engine(engine)
     ModelGroupFactory(model_group_id=1, model_type='modelType1')
     ModelGroupFactory(model_group_id=2, model_type='modelType2')
     ModelGroupFactory(model_group_id=3, model_type='modelType3')
     ModelGroupFactory(model_group_id=4, model_type='modelType4')
     ModelGroupFactory(model_group_id=5, model_type='modelType5')
     session.commit()
     distance_table = DistanceFromBestTable(db_engine=engine,
                                            models_table='models',
                                            distance_table='dist_table')
     distance_table._create()
     distance_rows = [
         # 2014: model group 1 should pass both close and min checks
         (1, 1, '2014-01-01', 'precision@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (1, 1, '2014-01-01', 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (1, 1, '2014-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2015: model group 1 should not pass close check
         (1, 2, '2015-01-01', 'precision@', '100_abs', 0.5, 0.88, 0.38, 0.0
          ),
         (1, 2, '2015-01-01', 'recall@', '100_abs', 0.5, 0.88, 0.38, 0.0),
         (1, 2, '2015-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         (1, 3, '2016-01-01', 'precision@', '100_abs', 0.46, 0.46, 0.0,
          0.11),
         (1, 3, '2016-01-01', 'recall@', '100_abs', 0.46, 0.46, 0.0, 0.11),
         (1, 3, '2016-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2014: model group 2 should not pass min check
         (2, 4, '2014-01-01', 'precision@', '100_abs', 0.39, 0.5, 0.11, 0.5
          ),
         (2, 4, '2014-01-01', 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (2, 4, '2014-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2015: model group 2 should pass both checks
         (2, 5, '2015-01-01', 'precision@', '100_abs', 0.69, 0.88, 0.19,
          0.12),
         (2, 5, '2015-01-01', 'recall@', '100_abs', 0.69, 0.88, 0.19, 0.0),
         (2, 5, '2015-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         (2, 6, '2016-01-01', 'precision@', '100_abs', 0.34, 0.46, 0.12,
          0.11),
         (2, 6, '2016-01-01', 'recall@', '100_abs', 0.46, 0.46, 0.0, 0.11),
         (2, 6, '2016-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # model group 3 not included in this round
         (3, 7, '2014-01-01', 'precision@', '100_abs', 0.28, 0.5, 0.22, 0.0
          ),
         (3, 7, '2014-01-01', 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (3, 7, '2014-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         (3, 8, '2015-01-01', 'precision@', '100_abs', 0.88, 0.88, 0.0,
          0.02),
         (3, 8, '2015-01-01', 'recall@', '100_abs', 0.5, 0.88, 0.38, 0.0),
         (3, 8, '2015-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         (3, 9, '2016-01-01', 'precision@', '100_abs', 0.44, 0.46, 0.02,
          0.11),
         (3, 9, '2016-01-01', 'recall@', '100_abs', 0.46, 0.46, 0.0, 0.11),
         (3, 9, '2016-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2014: model group 4 should not pass any checks
         (4, 10, '2014-01-01', 'precision@', '100_abs', 0.29, 0.5, 0.21,
          0.21),
         (4, 10, '2014-01-01', 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (4, 10, '2014-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2015: model group 4 should not pass close check
         (4, 11, '2015-01-01', 'precision@', '100_abs', 0.67, 0.88, 0.21,
          0.21),
         (4, 11, '2015-01-01', 'recall@', '100_abs', 0.5, 0.88, 0.38, 0.0),
         (4, 11, '2015-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         (4, 12, '2016-01-01', 'precision@', '100_abs', 0.25, 0.46, 0.21,
          0.21),
         (4, 12, '2016-01-01', 'recall@', '100_abs', 0.46, 0.46, 0.0, 0.11),
         (4, 12, '2016-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2014: model group 5 should not pass because precision is good but not recall
         (5, 13, '2014-01-01', 'precision@', '100_abs', 0.5, 0.38, 0.0, 0.38
          ),
         (5, 13, '2014-01-01', 'recall@', '100_abs', 0.3, 0.5, 0.2, 0.38),
         (5, 13, '2014-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2015: model group 5 should not pass because precision is good but not recall
         (5, 14, '2015-01-01', 'precision@', '100_abs', 0.5, 0.88, 0.38, 0.0
          ),
         (5, 14, '2015-01-01', 'recall@', '100_abs', 0.3, 0.88, 0.58, 0.0),
         (5, 14, '2015-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         (5, 15, '2016-01-01', 'precision@', '100_abs', 0.46, 0.46, 0.0,
          0.11),
         (5, 15, '2016-01-01', 'recall@', '100_abs', 0.3, 0.46, 0.16, 0.11),
         (5, 15, '2016-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
         # 2014: model group 6 is failed by false positives
         (6, 16, '2014-01-01', 'precision@', '100_abs', 0.5, 0.5, 0.0, 0.38
          ),
         (6, 16, '2014-01-01', 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (6, 16, '2014-01-01', 'false positives@', '100_abs', 60, 30, 30,
          10),
         # 2015: model group 6 is failed by false positives
         (6, 17, '2015-01-01', 'precision@', '100_abs', 0.5, 0.88, 0.38, 0.0
          ),
         (6, 17, '2015-01-01', 'recall@', '100_abs', 0.5, 0.38, 0.0, 0.38),
         (6, 17, '2015-01-01', 'false positives@', '100_abs', 60, 30, 30,
          10),
         (6, 18, '2016-01-01', 'precision@', '100_abs', 0.46, 0.46, 0.0,
          0.11),
         (6, 18, '2016-01-01', 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.38),
         (6, 18, '2016-01-01', 'false positives@', '100_abs', 40, 30, 10,
          10),
     ]
     for dist_row in distance_rows:
         engine.execute(
             'insert into dist_table values (%s, %s, %s, %s, %s, %s, %s, %s, %s)',
             dist_row)
     thresholder = ModelGroupThresholder(
         distance_from_best_table=distance_table,
         train_end_times=['2014-01-01', '2015-01-01'],
         initial_model_group_ids=[1, 2, 4, 5, 6],
         initial_metric_filters=self.metric_filters)
     return thresholder
示例#5
0
def test_DistanceFromBestTable():
    with testing.postgresql.Postgresql() as postgresql:
        engine = create_engine(postgresql.url())
        ensure_db(engine)
        init_engine(engine)
        model_groups = {
            'stable': ModelGroupFactory(model_type='myStableClassifier'),
            'bad': ModelGroupFactory(model_type='myBadClassifier'),
            'spiky': ModelGroupFactory(model_type='mySpikeClassifier'),
        }

        class StableModelFactory(ModelFactory):
            model_group_rel = model_groups['stable']

        class BadModelFactory(ModelFactory):
            model_group_rel = model_groups['bad']

        class SpikyModelFactory(ModelFactory):
            model_group_rel = model_groups['spiky']

        models = {
            'stable_3y_ago': StableModelFactory(train_end_time='2014-01-01'),
            'stable_2y_ago': StableModelFactory(train_end_time='2015-01-01'),
            'stable_1y_ago': StableModelFactory(train_end_time='2016-01-01'),
            'bad_3y_ago': BadModelFactory(train_end_time='2014-01-01'),
            'bad_2y_ago': BadModelFactory(train_end_time='2015-01-01'),
            'bad_1y_ago': BadModelFactory(train_end_time='2016-01-01'),
            'spiky_3y_ago': SpikyModelFactory(train_end_time='2014-01-01'),
            'spiky_2y_ago': SpikyModelFactory(train_end_time='2015-01-01'),
            'spiky_1y_ago': SpikyModelFactory(train_end_time='2016-01-01'),
        }

        class ImmediateEvalFactory(EvaluationFactory):
            evaluation_start_time = factory.LazyAttribute(
                lambda o: o.model_rel.train_end_time)
            evaluation_end_time = factory.LazyAttribute(
                lambda o: _sql_add_days(o.model_rel.train_end_time, 1))

        class MonthOutEvalFactory(EvaluationFactory):
            evaluation_start_time = factory.LazyAttribute(
                lambda o: _sql_add_days(o.model_rel.train_end_time, 31))
            evaluation_end_time = factory.LazyAttribute(
                lambda o: _sql_add_days(o.model_rel.train_end_time, 32))

        class Precision100Factory(ImmediateEvalFactory):
            metric = 'precision@'
            parameter = '100_abs'

        class Precision100FactoryMonthOut(MonthOutEvalFactory):
            metric = 'precision@'
            parameter = '100_abs'

        class Recall100Factory(ImmediateEvalFactory):
            metric = 'recall@'
            parameter = '100_abs'

        class Recall100FactoryMonthOut(MonthOutEvalFactory):
            metric = 'recall@'
            parameter = '100_abs'

        for add_val, PrecFac, RecFac in [
            (0, Precision100Factory, Recall100Factory),
            (-0.15, Precision100FactoryMonthOut, Recall100FactoryMonthOut)
        ]:
            PrecFac(model_rel=models['stable_3y_ago'], value=0.6 + add_val)
            PrecFac(model_rel=models['stable_2y_ago'], value=0.57 + add_val)
            PrecFac(model_rel=models['stable_1y_ago'], value=0.59 + add_val)
            PrecFac(model_rel=models['bad_3y_ago'], value=0.4 + add_val)
            PrecFac(model_rel=models['bad_2y_ago'], value=0.39 + add_val)
            PrecFac(model_rel=models['bad_1y_ago'], value=0.43 + add_val)
            PrecFac(model_rel=models['spiky_3y_ago'], value=0.8 + add_val)
            PrecFac(model_rel=models['spiky_2y_ago'], value=0.4 + add_val)
            PrecFac(model_rel=models['spiky_1y_ago'], value=0.4 + add_val)
            RecFac(model_rel=models['stable_3y_ago'], value=0.55 + add_val)
            RecFac(model_rel=models['stable_2y_ago'], value=0.56 + add_val)
            RecFac(model_rel=models['stable_1y_ago'], value=0.55 + add_val)
            RecFac(model_rel=models['bad_3y_ago'], value=0.35 + add_val)
            RecFac(model_rel=models['bad_2y_ago'], value=0.34 + add_val)
            RecFac(model_rel=models['bad_1y_ago'], value=0.36 + add_val)
            RecFac(model_rel=models['spiky_3y_ago'], value=0.35 + add_val)
            RecFac(model_rel=models['spiky_2y_ago'], value=0.8 + add_val)
            RecFac(model_rel=models['spiky_1y_ago'], value=0.36 + add_val)
        session.commit()
        distance_table = DistanceFromBestTable(db_engine=engine,
                                               models_table='models',
                                               distance_table='dist_table')
        metrics = [{
            'metric': 'precision@',
            'parameter': '100_abs'
        }, {
            'metric': 'recall@',
            'parameter': '100_abs'
        }]
        model_group_ids = [mg.model_group_id for mg in model_groups.values()]
        distance_table.create_and_populate(
            model_group_ids, ['2014-01-01', '2015-01-01', '2016-01-01'],
            metrics)

        # get an ordered list of the models/groups for a particular metric/time
        query = '''
            select model_id, raw_value, dist_from_best_case, dist_from_best_case_next_time
            from dist_table where metric = %s and parameter = %s and train_end_time = %s
            order by dist_from_best_case
        '''

        prec_3y_ago = engine.execute(query,
                                     ('precision@', '100_abs', '2014-01-01'))
        assert [row for row in prec_3y_ago] == [
            (models['spiky_3y_ago'].model_id, 0.8, 0, 0.17),
            (models['stable_3y_ago'].model_id, 0.6, 0.2, 0),
            (models['bad_3y_ago'].model_id, 0.4, 0.4, 0.18),
        ]

        recall_2y_ago = engine.execute(query,
                                       ('recall@', '100_abs', '2015-01-01'))
        assert [row for row in recall_2y_ago] == [
            (models['spiky_2y_ago'].model_id, 0.8, 0, 0.19),
            (models['stable_2y_ago'].model_id, 0.56, 0.24, 0),
            (models['bad_2y_ago'].model_id, 0.34, 0.46, 0.19),
        ]

        assert distance_table.observed_bounds == {
            ('precision@', '100_abs'): (0.39, 0.8),
            ('recall@', '100_abs'): (0.34, 0.8),
        }
示例#6
0
def create_sample_distance_table(engine):
    ensure_db(engine)
    init_engine(engine)
    model_groups = {
        'stable': ModelGroupFactory(model_type='myStableClassifier'),
        'spiky': ModelGroupFactory(model_type='mySpikeClassifier'),
    }

    class StableModelFactory(ModelFactory):
        model_group_rel = model_groups['stable']

    class SpikyModelFactory(ModelFactory):
        model_group_rel = model_groups['spiky']

    models = {
        'stable_3y_ago': StableModelFactory(train_end_time='2014-01-01'),
        'stable_2y_ago': StableModelFactory(train_end_time='2015-01-01'),
        'stable_1y_ago': StableModelFactory(train_end_time='2016-01-01'),
        'spiky_3y_ago': SpikyModelFactory(train_end_time='2014-01-01'),
        'spiky_2y_ago': SpikyModelFactory(train_end_time='2015-01-01'),
        'spiky_1y_ago': SpikyModelFactory(train_end_time='2016-01-01'),
    }
    session.commit()
    distance_table = DistanceFromBestTable(
        db_engine=engine,
        models_table='models',
        distance_table='dist_table'
    )
    distance_table._create()
    stable_grp = model_groups['stable'].model_group_id
    spiky_grp = model_groups['spiky'].model_group_id
    stable_3y_id = models['stable_3y_ago'].model_id
    stable_3y_end = models['stable_3y_ago'].train_end_time
    stable_2y_id = models['stable_2y_ago'].model_id
    stable_2y_end = models['stable_2y_ago'].train_end_time
    stable_1y_id = models['stable_1y_ago'].model_id
    stable_1y_end = models['stable_1y_ago'].train_end_time
    spiky_3y_id = models['spiky_3y_ago'].model_id
    spiky_3y_end = models['spiky_3y_ago'].train_end_time
    spiky_2y_id = models['spiky_2y_ago'].model_id
    spiky_2y_end = models['spiky_2y_ago'].train_end_time
    spiky_1y_id = models['spiky_1y_ago'].model_id
    spiky_1y_end = models['spiky_1y_ago'].train_end_time
    distance_rows = [
        (stable_grp, stable_3y_id, stable_3y_end, 'precision@', '100_abs', 0.5, 0.6, 0.1, 0.5, 0.15),
        (stable_grp, stable_2y_id, stable_2y_end, 'precision@', '100_abs', 0.5, 0.84, 0.34, 0.5, 0.18),
        (stable_grp, stable_1y_id, stable_1y_end, 'precision@', '100_abs', 0.46, 0.67, 0.21, 0.5, 0.11),
        (spiky_grp, spiky_3y_id, spiky_3y_end, 'precision@', '100_abs', 0.45, 0.6, 0.15, 0.5, 0.19),
        (spiky_grp, spiky_2y_id, spiky_2y_end, 'precision@', '100_abs', 0.84, 0.84, 0.0, 0.5, 0.3),
        (spiky_grp, spiky_1y_id, spiky_1y_end, 'precision@', '100_abs', 0.45, 0.67, 0.22, 0.5, 0.12),
        (stable_grp, stable_3y_id, stable_3y_end, 'recall@', '100_abs', 0.4, 0.4, 0.0, 0.4, 0.0),
        (stable_grp, stable_2y_id, stable_2y_end, 'recall@', '100_abs', 0.5, 0.5, 0.0, 0.5, 0.0),
        (stable_grp, stable_1y_id, stable_1y_end, 'recall@', '100_abs', 0.6, 0.6, 0.0, 0.6, 0.0),
        (spiky_grp, spiky_3y_id, spiky_3y_end, 'recall@', '100_abs', 0.65, 0.65, 0.0, 0.65, 0.0),
        (spiky_grp, spiky_2y_id, spiky_2y_end, 'recall@', '100_abs', 0.55, 0.55, 0.0, 0.55, 0.0),
        (spiky_grp, spiky_1y_id, spiky_1y_end, 'recall@', '100_abs', 0.45, 0.45, 0.0, 0.45, 0.0),
    ]
    for dist_row in distance_rows:
        engine.execute(
            'insert into dist_table values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)',
            dist_row
        )
    return distance_table, model_groups
示例#7
0
def test_Auditioner():
    with testing.postgresql.Postgresql() as postgresql:
        db_engine = create_engine(postgresql.url())
        ensure_db(db_engine)
        init_engine(db_engine)
        # set up data, randomly generated by the factories but conforming
        # generally to what we expect results schema data to look like
        num_model_groups = 10
        model_types = [
            'classifier type {}'.format(i) for i in range(0, num_model_groups)
        ]
        model_groups = [
            ModelGroupFactory(model_type=model_type)
            for model_type in model_types
        ]
        train_end_times = [
            datetime(2013, 1, 1),
            datetime(2014, 1, 1),
            datetime(2015, 1, 1),
            datetime(2016, 1, 1),
        ]
        models = [
            ModelFactory(model_group_rel=model_group,
                         train_end_time=train_end_time)
            for model_group in model_groups
            for train_end_time in train_end_times
        ]
        metrics = [
            ('precision@', '100_abs'),
            ('recall@', '100_abs'),
            ('precision@', '50_abs'),
            ('recall@', '50_abs'),
            ('fpr@', '10_pct'),
        ]

        class ImmediateEvalFactory(EvaluationFactory):
            evaluation_start_time = factory.LazyAttribute(
                lambda o: o.model_rel.train_end_time)

        _ = [
            ImmediateEvalFactory(model_rel=model,
                                 metric=metric,
                                 parameter=parameter)
            for metric, parameter in metrics for model in models
        ]
        session.commit()

        # define a very loose filtering that should admit all model groups
        no_filtering = [{
            'metric': 'precision@',
            'parameter': '100_abs',
            'max_from_best': 1.0,
            'threshold_value': 0.0
        }, {
            'metric': 'recall@',
            'parameter': '100_abs',
            'max_from_best': 1.0,
            'threshold_value': 0.0
        }]
        model_group_ids = [mg.model_group_id for mg in model_groups]
        auditioner = Auditioner(
            db_engine,
            model_group_ids,
            train_end_times,
            no_filtering,
        )
        assert len(auditioner.thresholded_model_group_ids) == num_model_groups
        auditioner.plot_model_groups()

        # here, we pick thresholding rules that should definitely remove
        # all model groups from contention because they are too strict.
        remove_all = [{
            'metric': 'precision@',
            'parameter': '100_abs',
            'max_from_best': 0.0,
            'threshold_value': 1.1
        }, {
            'metric': 'recall@',
            'parameter': '100_abs',
            'max_from_best': 0.0,
            'threshold_value': 1.1
        }]

        auditioner.update_metric_filters(remove_all)
        assert len(auditioner.thresholded_model_group_ids) == 0

        # one potential place for bugs would be when we pull back the rules
        # for being too restrictive. we want to make sure that the original list is
        # always used for thresholding, or else such a move would be impossible
        auditioner.update_metric_filters(no_filtering)
        assert len(auditioner.thresholded_model_group_ids) == num_model_groups

        # now, we want to take this partially thresholded list and run it through
        # a grid of selection rules, meant to pick winners by a variety of user-defined
        # criteria
        rule_grid = [{
            'shared_parameters': [
                {
                    'metric': 'precision@',
                    'parameter': '100_abs'
                },
                {
                    'metric': 'recall@',
                    'parameter': '100_abs'
                },
            ],
            'selection_rules': [{
                'name': 'most_frequent_best_dist',
                'dist_from_best_case': [0.1, 0.2, 0.3]
            }, {
                'name': 'best_current_value'
            }]
        }, {
            'shared_parameters': [
                {
                    'metric1': 'precision@',
                    'parameter1': '100_abs'
                },
            ],
            'selection_rules': [
                {
                    'name': 'best_average_two_metrics',
                    'metric2': ['recall@'],
                    'parameter2': ['100_abs'],
                    'metric1_weight': [0.4, 0.5, 0.6]
                },
            ]
        }]
        auditioner.register_selection_rule_grid(rule_grid, plot=False)
        final_model_group_ids = auditioner.selection_rule_model_group_ids

        # we expect the result to be a mapping of selection rule name to model group id
        assert isinstance(final_model_group_ids, dict)

        # we expect that there is one winner for each selection rule
        assert sorted(final_model_group_ids.keys()) == \
            sorted([rule.descriptive_name for rule in auditioner.selection_rules])

        # we expect that the results written to the yaml file are the
        # chosen model groups and their rules
        # however because the source data is randomly generated we could have a
        # different list on consecutive runs
        # and don't want to introduce non-determinism to the test
        with tempfile.NamedTemporaryFile() as tf:
            auditioner.write_tyra_config(tf.name)
            assert sorted(yaml.load(tf)['selection_rule_model_groups'].keys()) == \
                sorted(final_model_group_ids.keys())