def testAllocateFreeResourcesWithIncreaseBy(self):
scheduler = ResourceChangingScheduler(
resources_allocation_function=DistributeResourcesToTopJob(
add_bundles=False,
increase_by={
"CPU": 2,
"GPU": 2
},
metric="metric",
mode="max",
))
base_pgf = PlacementGroupFactory([{"CPU": 2, "GPU": 2}])
trial1, trial2, trial3, trial4 = self._prepareTrials(
scheduler, base_pgf)
decision = scheduler.on_trial_result(self.trial_runner, trial2, {
"metric": 0.9,
"training_iteration": 4
})
assert decision == TrialScheduler.CONTINUE
decision = scheduler.on_trial_result(self.trial_runner, trial1, {
"metric": 1.0,
"training_iteration": 4
})
assert decision == TrialScheduler.CONTINUE
trial4.status = Trial.TERMINATED
self._allocateAndAssertNewResources(
trial1, scheduler, PlacementGroupFactory([{
"CPU": 4,
"GPU": 4
}]))
decision = scheduler.on_trial_result(self.trial_runner, trial2, {
"metric": 1.1,
"training_iteration": 4
})
assert decision == TrialScheduler.CONTINUE
trial3.status = Trial.TERMINATED
self._allocateAndAssertNewResources(trial2,
scheduler,
PlacementGroupFactory([{
"CPU": 4,
"GPU": 4
}]),
metric=1.1)
trial2.status = Trial.TERMINATED
self._allocateAndAssertNewResources(trial1,
scheduler,
PlacementGroupFactory([{
"CPU": 8,
"GPU": 8
}]),
metric=1.2)
def testDeallocateResources(self):
scheduler = ResourceChangingScheduler(
resources_allocation_function=DistributeResourcesToTopJob(
add_bundles=False, increase_by={"GPU": 2}, metric="metric", mode="max"
)
)
base_pgf = PlacementGroupFactory([{"CPU": 1, "GPU": 2}])
trial1, trial2, trial3, trial4 = self._prepareTrials(scheduler, base_pgf)
trial1.placement_group_factory = PlacementGroupFactory([{"CPU": 1, "GPU": 4}])
trial4.status = Trial.PENDING
self._allocateAndAssertNewResources(
trial1, scheduler, PlacementGroupFactory([{"CPU": 1, "GPU": 2}])
)
