Python HyperbandIterationManager示例

示例#1

文件： test_iteration_managers.py 项目： yu-iskw/polyaxon

 def setUp(self):
     super().setUp()
     self.experiment_group = ExperimentGroupFactory(
         content=experiment_group_spec_content_hyperband)
     for _ in range(3):
         ExperimentFactory(experiment_group=self.experiment_group)
     self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)

示例#2

文件： test_iteration_managers.py 项目： ttsvetanov/polyaxon

 def setUp(self):
     super().setUp()
     self.experiment_group = ExperimentGroupFactory(
         content=experiment_group_spec_content_hyperband)
     for _ in range(3):
         ExperimentFactory(experiment_group=self.experiment_group)
     self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)

示例#3

文件： test_iteration_managers.py 项目： yu-iskw/polyaxon

class TestHyperbandIterationManagers(BaseTest):
    def setUp(self):
        super().setUp()
        self.experiment_group = ExperimentGroupFactory(
            content=experiment_group_spec_content_hyperband)
        for _ in range(3):
            ExperimentFactory(experiment_group=self.experiment_group)
        self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)

    @flaky(max_runs=3)
    def test_create_iteration(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        assert isinstance(iteration, ExperimentGroupIteration)
        assert ExperimentGroupIteration.objects.count() == 1
        assert iteration.experiment_group == self.experiment_group
        assert iteration.data == {
            'iteration': 0,
            'num_suggestions': 0,
            'bracket_iteration': 0,
            'experiment_ids': [],
            'experiments_metrics': None,
        }
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert self.experiment_group.iteration_data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'num_suggestions': 0,
            'experiment_ids': experiment_ids,
            'experiments_metrics': None,
        }
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert self.experiment_group.iteration_data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'num_suggestions': 0,
            'experiment_ids': experiment_ids,
            'experiments_metrics': None,
        }
        self.iteration_manager.update_iteration_num_suggestions(num_suggestions=3)
        assert self.experiment_group.iteration_data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'num_suggestions': 3,
            'experiment_ids': experiment_ids,
            'experiments_metrics': None,
        }
        experiment_ids = list(self.experiment_group.experiments.values_list('id', flat=True))
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        self.assertEqual(sorted(self.experiment_group.iteration_data['experiment_ids']),
                         sorted(experiment_ids))

    def test_update_iteration_raises_if_not_iteration_is_created(self):
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 0

    def test_update_iteration(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert iteration.data['experiments_metrics'] is None
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 1

        iteration.refresh_from_db()
        assert iteration.data['experiments_metrics'] == []

        ExperimentMetric.objects.create(
            experiment_id=experiment_ids[0],
            values={self.experiment_group.hptuning_config.hyperband.metric.name: 0.9})
        self.iteration_manager.update_iteration()
        iteration.refresh_from_db()
        assert iteration.data['experiments_metrics'] == [[experiment_ids[0], 0.9]]

    def test_get_reduced_configs(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert iteration.data['experiments_metrics'] is None
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 1

        assert self.iteration_manager.get_reduced_configs() == []

示例#4

文件： test_iteration_managers.py 项目： ttsvetanov/polyaxon

class TestHyperbandIterationManagers(BaseTest):
    DISABLE_RUNNER = True

    def setUp(self):
        super().setUp()
        self.experiment_group = ExperimentGroupFactory(
            content=experiment_group_spec_content_hyperband)
        for _ in range(3):
            ExperimentFactory(experiment_group=self.experiment_group)
        self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)

    def test_create_iteration(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        assert isinstance(iteration, ExperimentGroupIteration)
        assert ExperimentGroupIteration.objects.count() == 1
        assert iteration.experiment_group == self.experiment_group
        assert iteration.data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'experiment_ids': None,
            'experiments_metrics': None,
        }
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        iteration.refresh_from_db()
        assert iteration.data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'experiment_ids': experiment_ids,
            'experiments_metrics': None,
        }

    def test_update_iteration_raises_if_not_iteration_is_created(self):
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 0

    def test_update_iteration(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert iteration.data['experiments_metrics'] is None
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 1

        iteration.refresh_from_db()
        assert iteration.data['experiments_metrics'] == []

        ExperimentMetric.objects.create(
            experiment_id=experiment_ids[0],
            values={self.experiment_group.hptuning_config.hyperband.metric.name: 0.9})
        self.iteration_manager.update_iteration()
        iteration.refresh_from_db()
        assert iteration.data['experiments_metrics'] == [[experiment_ids[0], 0.9]]

    def test_get_reduced_configs(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert iteration.data['experiments_metrics'] is None
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 1

        assert self.iteration_manager.get_reduced_configs() == []

示例#5

文件： test_iteration_managers.py 项目： mashroomxl/polyaxon

class TestHyperbandIterationManagers(BaseTest):
    DISABLE_RUNNER = True

    def setUp(self):
        super().setUp()
        self.experiment_group = ExperimentGroupFactory(
            content=experiment_group_spec_content_hyperband)
        for _ in range(3):
            ExperimentFactory(experiment_group=self.experiment_group)
        self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)

    def test_create_iteration(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        assert isinstance(iteration, ExperimentGroupIteration)
        assert ExperimentGroupIteration.objects.count() == 1
        assert iteration.experiment_group == self.experiment_group
        assert iteration.data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'experiment_ids': None,
            'experiments_metrics': None,
        }
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        iteration.refresh_from_db()
        assert iteration.data == {
            'iteration': 0,
            'bracket_iteration': 0,
            'experiment_ids': experiment_ids,
            'experiments_metrics': None,
        }

    def test_update_iteration_raises_if_not_iteration_is_created(self):
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 0

    def test_update_iteration(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert iteration.data['experiments_metrics'] is None
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 1

        iteration.refresh_from_db()
        assert iteration.data['experiments_metrics'] == []

        ExperimentMetric.objects.create(
            experiment_id=experiment_ids[0],
            values={self.experiment_group.params_config.hyperband.metric.name: 0.9})
        self.iteration_manager.update_iteration()
        iteration.refresh_from_db()
        assert iteration.data['experiments_metrics'] == [[experiment_ids[0], 0.9]]

    def test_get_reduced_configs(self):
        assert ExperimentGroupIteration.objects.count() == 0
        experiment_ids = [self.experiment_group.experiments.first().id]
        iteration = self.iteration_manager.create_iteration()
        self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
        assert iteration.data['experiments_metrics'] is None
        self.iteration_manager.update_iteration()
        assert ExperimentGroupIteration.objects.count() == 1

        assert self.iteration_manager.get_reduced_configs() == []