def test_job_request_abort(self): django_job = self._create_job(hosts=[5, 6]) job = scheduler_models.Job(django_job.id) job.request_abort() django_hqes = list(models.HostQueueEntry.objects.filter(job=job.id)) for hqe in django_hqes: self.assertTrue(hqe.aborted)
def test_next_group_name(self): django_job = self._create_job(metahosts=[1]) job = scheduler_models.Job(id=django_job.id) self.assertEqual('group0', job._next_group_name()) for hqe in django_job.hostqueueentry_set.filter(): hqe.execution_subdir = 'my_rack.group0' hqe.save() self.assertEqual('my_rack.group1', job._next_group_name('my/rack'))
def test_atomic_group_scheduling_fail(self): # If synch_count is > the atomic group number of machines, the job # should be aborted immediately. model_job = self._create_job(synchronous=True, atomic_group=1) model_job.synch_count = 4 model_job.save() job = scheduler_models.Job(id=model_job.id) self._run_scheduler() self._check_for_extra_schedulings() queue_entries = job.get_host_queue_entries() self.assertEqual(1, len(queue_entries)) self.assertEqual(queue_entries[0].status, models.HostQueueEntry.Status.ABORTED)
def test_run_if_ready_delays(self): # Also tests Job.run_with_ready_delay() on atomic group jobs. django_job = self._create_job(hosts=[5, 6], atomic_group=1) job = scheduler_models.Job(django_job.id) self.assertEqual(1, job.synch_count) django_hqes = list(models.HostQueueEntry.objects.filter(job=job.id)) self.assertEqual(2, len(django_hqes)) self.assertEqual(2, django_hqes[0].atomic_group.max_number_of_machines) def set_hqe_status(django_hqe, status): django_hqe.status = status django_hqe.save() scheduler_models.HostQueueEntry( django_hqe.id).host.set_status(status) # An initial state, our synch_count is 1 set_hqe_status(django_hqes[0], models.HostQueueEntry.Status.VERIFYING) set_hqe_status(django_hqes[1], models.HostQueueEntry.Status.PENDING) # So that we don't depend on the config file value during the test. self.assert_(scheduler_config.config. secs_to_wait_for_atomic_group_hosts is not None) self.god.stub_with(scheduler_config.config, 'secs_to_wait_for_atomic_group_hosts', 123456) # Get the pending one as a scheduler_models.HostQueueEntry object. hqe = scheduler_models.HostQueueEntry(django_hqes[1].id) self.assert_(not job._delay_ready_task) self.assertTrue(job.is_ready()) # Ready with one pending, one verifying and an atomic group should # result in a DelayCallTask to re-check if we're ready a while later. job.run_if_ready(hqe) self.assertEquals('Waiting', hqe.status) self._dispatcher._schedule_delay_tasks() self.assertEquals('Pending', hqe.status) agent = self._dispatcher._agents[0] self.assert_(job._delay_ready_task) self.assert_(isinstance(agent, monitor_db.Agent)) self.assert_(agent.task) delay_task = agent.task self.assert_(isinstance(delay_task, scheduler_models.DelayedCallTask)) self.assert_(not delay_task.is_done()) self.god.stub_function(delay_task, 'abort') self.god.stub_function(job, 'run') self.god.stub_function(job, '_pending_count') self.god.stub_with(job, 'synch_count', 9) self.god.stub_function(job, 'request_abort') # Test that the DelayedCallTask's callback queued up above does the # correct thing and does not call run if there are not enough hosts # in pending after the delay. job._pending_count.expect_call().and_return(0) job.request_abort.expect_call() delay_task._callback() self.god.check_playback() # Test that the DelayedCallTask's callback queued up above does the # correct thing and returns the Agent returned by job.run() if # there are still enough hosts pending after the delay. job.synch_count = 4 job._pending_count.expect_call().and_return(4) job.run.expect_call(hqe) delay_task._callback() self.god.check_playback() job._pending_count.expect_call().and_return(4) # Adjust the delay deadline so that enough time has passed. job._delay_ready_task.end_time = time.time() - 111111 job.run.expect_call(hqe) # ...the delay_expired condition should cause us to call run() self._dispatcher._handle_agents() self.god.check_playback() delay_task.success = False # Adjust the delay deadline back so that enough time has not passed. job._delay_ready_task.end_time = time.time() + 111111 self._dispatcher._handle_agents() self.god.check_playback() # Now max_number_of_machines HQEs are in pending state. Remaining # delay will now be ignored. other_hqe = scheduler_models.HostQueueEntry(django_hqes[0].id) self.god.unstub(job, 'run') self.god.unstub(job, '_pending_count') self.god.unstub(job, 'synch_count') self.god.unstub(job, 'request_abort') # ...the over_max_threshold test should cause us to call run() delay_task.abort.expect_call() other_hqe.on_pending() self.assertEquals('Starting', other_hqe.status) self.assertEquals('Starting', hqe.status) self.god.stub_function(job, 'run') self.god.unstub(delay_task, 'abort') hqe.set_status('Pending') other_hqe.set_status('Pending') # Now we're not over the max for the atomic group. But all assigned # hosts are in pending state. over_max_threshold should make us run(). hqe.atomic_group.max_number_of_machines += 1 hqe.atomic_group.save() job.run.expect_call(hqe) hqe.on_pending() self.god.check_playback() hqe.atomic_group.max_number_of_machines -= 1 hqe.atomic_group.save() other_hqe = scheduler_models.HostQueueEntry(django_hqes[0].id) self.assertTrue(hqe.job is other_hqe.job) # DBObject classes should reuse instances so these should be the same. self.assertEqual(job, other_hqe.job) self.assertEqual(other_hqe.job, hqe.job) # Be sure our delay was not lost during the other_hqe construction. self.assertEqual(job._delay_ready_task, delay_task) self.assert_(job._delay_ready_task) self.assertFalse(job._delay_ready_task.is_done()) self.assertFalse(job._delay_ready_task.aborted) # We want the real run() to be called below. self.god.unstub(job, 'run') # We pass in the other HQE this time the same way it would happen # for real when one host finishes verifying and enters pending. job.run_if_ready(other_hqe) # The delayed task must be aborted by the actual run() call above. self.assertTrue(job._delay_ready_task.aborted) self.assertFalse(job._delay_ready_task.success) self.assertTrue(job._delay_ready_task.is_done()) # Check that job run() and _finish_run() were called by the above: self._dispatcher._schedule_running_host_queue_entries() agent = self._dispatcher._agents[0] self.assert_(agent.task) task = agent.task self.assert_(isinstance(task, monitor_db.QueueTask)) # Requery these hqes in order to verify the status from the DB. django_hqes = list(models.HostQueueEntry.objects.filter(job=job.id)) for entry in django_hqes: self.assertEqual(models.HostQueueEntry.Status.STARTING, entry.status) # We're already running, but more calls to run_with_ready_delay can # continue to come in due to straggler hosts enter Pending. Make # sure we don't do anything. self.god.stub_function(job, 'run') job.run_with_ready_delay(hqe) self.god.check_playback() self.god.unstub(job, 'run')