def testAllocateFreeResourcesWithIncreaseBy(self):
scheduler = ResourceChangingScheduler(
resources_allocation_function=DistributeResourcesToTopJob(
add_bundles=False,
increase_by={
"CPU": 2,
"GPU": 2
},
metric="metric",
mode="max",
))
base_pgf = PlacementGroupFactory([{"CPU": 2, "GPU": 2}])
trial1, trial2, trial3, trial4 = self._prepareTrials(
scheduler, base_pgf)
decision = scheduler.on_trial_result(self.trial_runner, trial2, {
"metric": 0.9,
"training_iteration": 4
})
assert decision == TrialScheduler.CONTINUE
decision = scheduler.on_trial_result(self.trial_runner, trial1, {
"metric": 1.0,
"training_iteration": 4
})
assert decision == TrialScheduler.CONTINUE
trial4.status = Trial.TERMINATED
self._allocateAndAssertNewResources(
trial1, scheduler, PlacementGroupFactory([{
"CPU": 4,
"GPU": 4
}]))
decision = scheduler.on_trial_result(self.trial_runner, trial2, {
"metric": 1.1,
"training_iteration": 4
})
assert decision == TrialScheduler.CONTINUE
trial3.status = Trial.TERMINATED
self._allocateAndAssertNewResources(trial2,
scheduler,
PlacementGroupFactory([{
"CPU": 4,
"GPU": 4
}]),
metric=1.1)
trial2.status = Trial.TERMINATED
self._allocateAndAssertNewResources(trial1,
scheduler,
PlacementGroupFactory([{
"CPU": 8,
"GPU": 8
}]),
metric=1.2)
def testAllocateFreeResourcesWithIncreaseByTimes(self):
scheduler = ResourceChangingScheduler(
resources_allocation_function=DistributeResources(
add_bundles=True, increase_by={"GPU": 2}, increase_by_times=2
)
)
base_pgf = PlacementGroupFactory([{"CPU": 1}, {"GPU": 2}])
trial1, trial2, trial3, trial4 = self._prepareTrials(scheduler, base_pgf)
decision = scheduler.on_trial_result(
self.trial_runner, trial1, {"metric": 1, "training_iteration": 4}
)
assert decision == TrialScheduler.CONTINUE
trial4.status = Trial.TERMINATED
self._allocateAndAssertNewResources(
trial1, scheduler, PlacementGroupFactory([{"CPU": 1}] + [{"GPU": 2}] * 2)
)
trial3.status = Trial.TERMINATED
self._allocateAndAssertNewResources(
trial2, scheduler, PlacementGroupFactory([{"CPU": 1}] + [{"GPU": 2}] * 2)
)
trial2.status = Trial.TERMINATED
self._allocateAndAssertNewResources(
trial1, scheduler, PlacementGroupFactory([{"CPU": 1}] + [{"GPU": 2}] * 3)
)