Пример #1
0
    def test_race_challenger_2(self):
        """
        Makes sure that a racing configuration with better performance,
        that is capped, doesn't substitute the incumbent.
        """
        def target(x):
            time.sleep(1.5)
            return (x["a"] + 1) / 1000.0

        taf = ExecuteTAFuncDict(use_pynisher=False,
                                ta=target,
                                stats=self.stats,
                                run_obj="runtime")
        taf.runhistory = self.rh

        intensifier = Intensifier(
            stats=self.stats,
            traj_logger=TrajLogger(output_dir=None, stats=self.stats),
            rng=np.random.RandomState(12345),
            instances=[1],
        )

        self.rh.add(
            config=self.config1,
            cost=0.001,
            time=0.001,
            status=StatusType.SUCCESS,
            instance_id=1,
            seed=12345,
            additional_info=None,
        )
        intensifier.N = 1
        # config2 should have a timeout (due to adaptive capping)
        # and config1 should still be the incumbent
        inc, instance, seed, cutoff = intensifier._get_next_racer(
            challenger=self.config2,
            incumbent=self.config1,
            run_history=self.rh)
        run_info = RunInfo(
            config=self.config2,
            instance=instance,
            instance_specific="0",
            seed=seed,
            cutoff=cutoff,
            capped=True,
            budget=0.0,
        )

        result = eval_challenger(run_info, taf, self.stats, self.rh)
        inc, perf = intensifier.process_results(
            run_info=run_info,
            incumbent=self.config1,
            run_history=self.rh,
            time_bound=np.inf,
            result=result,
        )

        self.assertEqual(inc, self.config1)
        self.assertEqual(intensifier.num_run, 1)
        self.assertEqual(intensifier.num_chall_run, 1)
Пример #2
0
    def test_race_challenger_1(self):
        """
           Makes sure that a racing configuration with better performance,
           is selected as incumbent
           No adaptive capping
        """
        def target(x):
            return (x['a'] + 1) / 1000.

        taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
        taf.runhistory = self.rh

        intensifier = Intensifier(stats=self.stats,
                                  traj_logger=TrajLogger(output_dir=None,
                                                         stats=self.stats),
                                  rng=np.random.RandomState(12345),
                                  instances=[1],
                                  run_obj_time=False)

        self.rh.add(config=self.config1,
                    cost=1,
                    time=1,
                    status=StatusType.SUCCESS,
                    instance_id=1,
                    seed=None,
                    additional_info=None)

        intensifier.N = 1
        inc, instance, seed, cutoff = intensifier._get_next_racer(
            challenger=self.config2,
            incumbent=self.config1,
            run_history=self.rh)
        run_info = RunInfo(
            config=self.config2,
            instance=instance,
            instance_specific="0",
            cutoff=cutoff,
            seed=seed,
            capped=False,
            budget=0.0,
        )
        result = eval_challenger(run_info, taf, self.stats, self.rh)
        inc, perf = intensifier.process_results(
            run_info=run_info,
            incumbent=self.config1,
            run_history=self.rh,
            time_bound=np.inf,
            result=result,
        )

        self.assertEqual(inc, self.config2)
        self.assertEqual(intensifier.num_run, 1)
        self.assertEqual(intensifier.num_chall_run, 1)
Пример #3
0
    def test_race_challenger_large_blocked_seed(self):
        """
        test _race_challenger whether seeds are blocked for challenger runs
        """
        def target(x):
            return 1

        taf = ExecuteTAFuncDict(use_pynisher=False,
                                ta=target,
                                stats=self.stats)
        taf.runhistory = self.rh

        intensifier = Intensifier(
            stats=self.stats,
            traj_logger=TrajLogger(output_dir=None, stats=self.stats),
            rng=np.random.RandomState(12345),
            instances=list(range(10)),
            run_obj_time=False,
            deterministic=False,
        )

        for i in range(10):
            self.rh.add(
                config=self.config1,
                cost=i + 1,
                time=1,
                status=StatusType.SUCCESS,
                instance_id=i,
                seed=i,
                additional_info=None,
            )

        intensifier.stage = IntensifierStage.RUN_CHALLENGER

        # tie on first instances and then challenger should always win
        # and be returned as inc
        while True:
            if intensifier.continue_challenger:
                config = intensifier.current_challenger
            else:
                config, _ = intensifier.get_next_challenger(
                    challengers=[self.config2, self.config3], chooser=None)
            inc, instance, seed, cutoff = intensifier._get_next_racer(
                challenger=config, incumbent=self.config1, run_history=self.rh)
            run_info = RunInfo(
                config=config,
                instance=instance,
                instance_specific="0",
                seed=seed,
                cutoff=cutoff,
                capped=False,
                budget=0.0,
            )
            result = eval_challenger(run_info, taf, self.stats, self.rh)
            inc, perf = intensifier.process_results(
                run_info=run_info,
                incumbent=self.config1,
                run_history=self.rh,
                time_bound=np.inf,
                result=result,
            )

            # stop when challenger evaluation is over
            if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
                break

        self.assertEqual(inc, self.config2)
        self.assertEqual(self.rh.get_cost(self.config2), 1)

        # get data for config2 to check that the correct run was performed
        runs = self.rh.get_runs_for_config(self.config2,
                                           only_max_observed_budget=True)
        self.assertEqual(len(runs), 10)

        seeds = sorted([r.seed for r in runs])
        self.assertEqual(seeds, list(range(10)), seeds)

        self.assertEqual(intensifier.num_run, 10)
        self.assertEqual(intensifier.num_chall_run, 10)
Пример #4
0
    def test_race_challenger_3(self):
        """
        test _race_challenger with adaptive capping on a previously capped configuration
        """
        def target(config: Configuration, seed: int, instance: str):
            if instance == 1:
                time.sleep(2.1)
            else:
                time.sleep(0.6)
            return (config["a"] + 1) / 1000.0

        taf = ExecuteTAFuncDict(
            use_pynisher=False,
            ta=target,
            stats=self.stats,
            run_obj="runtime",
            par_factor=1,
        )
        taf.runhistory = self.rh

        intensifier = Intensifier(
            stats=self.stats,
            traj_logger=TrajLogger(output_dir=None, stats=self.stats),
            rng=np.random.RandomState(12345),
            cutoff=2,
            instances=[1],
        )

        self.rh.add(
            config=self.config1,
            cost=0.5,
            time=0.5,
            status=StatusType.SUCCESS,
            instance_id=1,
            seed=12345,
            additional_info=None,
        )

        # config2 should have a timeout (due to adaptive capping)
        # and config1 should still be the incumbent
        config, _ = intensifier.get_next_challenger(
            challengers=[self.config2, self.config3], chooser=None)
        inc, instance, seed, cutoff = intensifier._get_next_racer(
            challenger=config, incumbent=self.config1, run_history=self.rh)
        run_info = RunInfo(
            config=config,
            instance=instance,
            instance_specific="0",
            seed=seed,
            cutoff=cutoff,
            capped=True,
            budget=0.0,
        )
        result = eval_challenger(run_info, taf, self.stats, self.rh)
        inc, perf = intensifier.process_results(
            run_info=run_info,
            incumbent=self.config1,
            run_history=self.rh,
            time_bound=np.inf,
            result=result,
        )

        self.assertEqual(inc, self.config1)

        # further run for incumbent
        self.rh.add(
            config=self.config1,
            cost=2,
            time=2,
            status=StatusType.TIMEOUT,
            instance_id=2,
            seed=12345,
            additional_info=None,
        )

        # give config2 a second chance - now it should run on both instances

        # run on instance 1
        config, _ = intensifier.get_next_challenger(
            challengers=[self.config2, self.config3], chooser=None)
        inc, instance, seed, cutoff = intensifier._get_next_racer(
            challenger=config, incumbent=self.config1, run_history=self.rh)
        run_info = RunInfo(
            config=config,
            instance=instance,
            instance_specific="0",
            seed=seed,
            cutoff=cutoff,
            capped=False,
            budget=0.0,
        )
        result = eval_challenger(run_info, taf, self.stats, self.rh)
        inc, perf = intensifier.process_results(
            run_info=run_info,
            incumbent=self.config1,
            run_history=self.rh,
            time_bound=np.inf,
            result=result,
        )

        # run on instance 2
        config, _ = intensifier.get_next_challenger(challengers=[self.config3],
                                                    chooser=None)
        self.assertEqual(config, self.config2)
        self.assertTrue(intensifier.continue_challenger)

        inc, instance, seed, cutoff = intensifier._get_next_racer(
            challenger=config, incumbent=self.config1, run_history=self.rh)
        run_info = RunInfo(
            config=config,
            instance=instance,
            instance_specific="0",
            seed=seed,
            cutoff=cutoff,
            capped=False,
            budget=0.0,
        )

        result = eval_challenger(run_info, taf, self.stats, self.rh)
        inc, perf = intensifier.process_results(
            run_info=run_info,
            incumbent=self.config1,
            run_history=self.rh,
            time_bound=np.inf,
            result=result,
        )

        # the incumbent should still be config1 because
        # config2 should get on inst 1 a full timeout
        # such that c(config1) = 1.25 and c(config2) close to 1.3
        self.assertEqual(inc, self.config1)
        # the capped run should not be counted in runs_perf_config
        self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2)
        self.assertFalse(intensifier.continue_challenger)

        self.assertEqual(intensifier.num_run, 3)
        self.assertEqual(intensifier.num_chall_run, 3)