def testAllocateFreeResourcesWithIncreaseBy(self): scheduler = ResourceChangingScheduler( resources_allocation_function=DistributeResourcesToTopJob( add_bundles=False, increase_by={ "CPU": 2, "GPU": 2 }, metric="metric", mode="max", )) base_pgf = PlacementGroupFactory([{"CPU": 2, "GPU": 2}]) trial1, trial2, trial3, trial4 = self._prepareTrials( scheduler, base_pgf) decision = scheduler.on_trial_result(self.trial_runner, trial2, { "metric": 0.9, "training_iteration": 4 }) assert decision == TrialScheduler.CONTINUE decision = scheduler.on_trial_result(self.trial_runner, trial1, { "metric": 1.0, "training_iteration": 4 }) assert decision == TrialScheduler.CONTINUE trial4.status = Trial.TERMINATED self._allocateAndAssertNewResources( trial1, scheduler, PlacementGroupFactory([{ "CPU": 4, "GPU": 4 }])) decision = scheduler.on_trial_result(self.trial_runner, trial2, { "metric": 1.1, "training_iteration": 4 }) assert decision == TrialScheduler.CONTINUE trial3.status = Trial.TERMINATED self._allocateAndAssertNewResources(trial2, scheduler, PlacementGroupFactory([{ "CPU": 4, "GPU": 4 }]), metric=1.1) trial2.status = Trial.TERMINATED self._allocateAndAssertNewResources(trial1, scheduler, PlacementGroupFactory([{ "CPU": 8, "GPU": 8 }]), metric=1.2)
def testAllocateFreeResourcesWithIncreaseByTimes(self): scheduler = ResourceChangingScheduler( resources_allocation_function=DistributeResources( add_bundles=True, increase_by={"GPU": 2}, increase_by_times=2 ) ) base_pgf = PlacementGroupFactory([{"CPU": 1}, {"GPU": 2}]) trial1, trial2, trial3, trial4 = self._prepareTrials(scheduler, base_pgf) decision = scheduler.on_trial_result( self.trial_runner, trial1, {"metric": 1, "training_iteration": 4} ) assert decision == TrialScheduler.CONTINUE trial4.status = Trial.TERMINATED self._allocateAndAssertNewResources( trial1, scheduler, PlacementGroupFactory([{"CPU": 1}] + [{"GPU": 2}] * 2) ) trial3.status = Trial.TERMINATED self._allocateAndAssertNewResources( trial2, scheduler, PlacementGroupFactory([{"CPU": 1}] + [{"GPU": 2}] * 2) ) trial2.status = Trial.TERMINATED self._allocateAndAssertNewResources( trial1, scheduler, PlacementGroupFactory([{"CPU": 1}] + [{"GPU": 2}] * 3) )