def test_save(self): # Dummy Job to avoid creating a one in the HostQueueEntry __init__. class MockJob(object): def __init__(self, id, row): pass def tag(self): return 'MockJob' self.god.stub_with(scheduler_models, 'Job', MockJob) hqe = scheduler_models.HostQueueEntry( new_record=True, row=[0, 1, 2, 'Queued', None, 0, 0, 0, '.', None, False, None, None]) hqe.save() new_id = hqe.id # Force a re-query and verify that the correct data was stored. scheduler_models.DBObject._clear_instance_cache() hqe = scheduler_models.HostQueueEntry(id=new_id) self.assertEqual(hqe.id, new_id) self.assertEqual(hqe.job_id, 1) self.assertEqual(hqe.host_id, 2) self.assertEqual(hqe.status, 'Queued') self.assertEqual(hqe.meta_host, None) self.assertEqual(hqe.active, False) self.assertEqual(hqe.complete, False) self.assertEqual(hqe.deleted, False) self.assertEqual(hqe.execution_subdir, '.') self.assertEqual(hqe.started_on, None) self.assertEqual(hqe.finished_on, None)
def __init__(self, task, extra_command_args): super(SpecialAgentTask, self).__init__() assert self.TASK_TYPE is not None, 'self.TASK_TYPE must be overridden' self.host = rdb_lib.get_hosts([task.host.id])[0] self.host.dbg_str = 'Task: %s' % str(task) self.queue_entry = None if task.queue_entry: self.queue_entry = scheduler_models.HostQueueEntry( id=task.queue_entry.id) self.host.dbg_str += self.queue_entry.get_dbg_str() self.task = task self._extra_command_args = extra_command_args self.host.metadata = self.get_metadata()
def set_hqe_status(django_hqe, status): django_hqe.status = status django_hqe.save() scheduler_models.HostQueueEntry( django_hqe.id).host.set_status(status)
def test_run_if_ready_delays(self): # Also tests Job.run_with_ready_delay() on atomic group jobs. django_job = self._create_job(hosts=[5, 6], atomic_group=1) job = scheduler_models.Job(django_job.id) self.assertEqual(1, job.synch_count) django_hqes = list(models.HostQueueEntry.objects.filter(job=job.id)) self.assertEqual(2, len(django_hqes)) self.assertEqual(2, django_hqes[0].atomic_group.max_number_of_machines) def set_hqe_status(django_hqe, status): django_hqe.status = status django_hqe.save() scheduler_models.HostQueueEntry( django_hqe.id).host.set_status(status) # An initial state, our synch_count is 1 set_hqe_status(django_hqes[0], models.HostQueueEntry.Status.VERIFYING) set_hqe_status(django_hqes[1], models.HostQueueEntry.Status.PENDING) # So that we don't depend on the config file value during the test. self.assert_(scheduler_config.config. secs_to_wait_for_atomic_group_hosts is not None) self.god.stub_with(scheduler_config.config, 'secs_to_wait_for_atomic_group_hosts', 123456) # Get the pending one as a scheduler_models.HostQueueEntry object. hqe = scheduler_models.HostQueueEntry(django_hqes[1].id) self.assert_(not job._delay_ready_task) self.assertTrue(job.is_ready()) # Ready with one pending, one verifying and an atomic group should # result in a DelayCallTask to re-check if we're ready a while later. job.run_if_ready(hqe) self.assertEquals('Waiting', hqe.status) self._dispatcher._schedule_delay_tasks() self.assertEquals('Pending', hqe.status) agent = self._dispatcher._agents[0] self.assert_(job._delay_ready_task) self.assert_(isinstance(agent, monitor_db.Agent)) self.assert_(agent.task) delay_task = agent.task self.assert_(isinstance(delay_task, scheduler_models.DelayedCallTask)) self.assert_(not delay_task.is_done()) self.god.stub_function(delay_task, 'abort') self.god.stub_function(job, 'run') self.god.stub_function(job, '_pending_count') self.god.stub_with(job, 'synch_count', 9) self.god.stub_function(job, 'request_abort') # Test that the DelayedCallTask's callback queued up above does the # correct thing and does not call run if there are not enough hosts # in pending after the delay. job._pending_count.expect_call().and_return(0) job.request_abort.expect_call() delay_task._callback() self.god.check_playback() # Test that the DelayedCallTask's callback queued up above does the # correct thing and returns the Agent returned by job.run() if # there are still enough hosts pending after the delay. job.synch_count = 4 job._pending_count.expect_call().and_return(4) job.run.expect_call(hqe) delay_task._callback() self.god.check_playback() job._pending_count.expect_call().and_return(4) # Adjust the delay deadline so that enough time has passed. job._delay_ready_task.end_time = time.time() - 111111 job.run.expect_call(hqe) # ...the delay_expired condition should cause us to call run() self._dispatcher._handle_agents() self.god.check_playback() delay_task.success = False # Adjust the delay deadline back so that enough time has not passed. job._delay_ready_task.end_time = time.time() + 111111 self._dispatcher._handle_agents() self.god.check_playback() # Now max_number_of_machines HQEs are in pending state. Remaining # delay will now be ignored. other_hqe = scheduler_models.HostQueueEntry(django_hqes[0].id) self.god.unstub(job, 'run') self.god.unstub(job, '_pending_count') self.god.unstub(job, 'synch_count') self.god.unstub(job, 'request_abort') # ...the over_max_threshold test should cause us to call run() delay_task.abort.expect_call() other_hqe.on_pending() self.assertEquals('Starting', other_hqe.status) self.assertEquals('Starting', hqe.status) self.god.stub_function(job, 'run') self.god.unstub(delay_task, 'abort') hqe.set_status('Pending') other_hqe.set_status('Pending') # Now we're not over the max for the atomic group. But all assigned # hosts are in pending state. over_max_threshold should make us run(). hqe.atomic_group.max_number_of_machines += 1 hqe.atomic_group.save() job.run.expect_call(hqe) hqe.on_pending() self.god.check_playback() hqe.atomic_group.max_number_of_machines -= 1 hqe.atomic_group.save() other_hqe = scheduler_models.HostQueueEntry(django_hqes[0].id) self.assertTrue(hqe.job is other_hqe.job) # DBObject classes should reuse instances so these should be the same. self.assertEqual(job, other_hqe.job) self.assertEqual(other_hqe.job, hqe.job) # Be sure our delay was not lost during the other_hqe construction. self.assertEqual(job._delay_ready_task, delay_task) self.assert_(job._delay_ready_task) self.assertFalse(job._delay_ready_task.is_done()) self.assertFalse(job._delay_ready_task.aborted) # We want the real run() to be called below. self.god.unstub(job, 'run') # We pass in the other HQE this time the same way it would happen # for real when one host finishes verifying and enters pending. job.run_if_ready(other_hqe) # The delayed task must be aborted by the actual run() call above. self.assertTrue(job._delay_ready_task.aborted) self.assertFalse(job._delay_ready_task.success) self.assertTrue(job._delay_ready_task.is_done()) # Check that job run() and _finish_run() were called by the above: self._dispatcher._schedule_running_host_queue_entries() agent = self._dispatcher._agents[0] self.assert_(agent.task) task = agent.task self.assert_(isinstance(task, monitor_db.QueueTask)) # Requery these hqes in order to verify the status from the DB. django_hqes = list(models.HostQueueEntry.objects.filter(job=job.id)) for entry in django_hqes: self.assertEqual(models.HostQueueEntry.Status.STARTING, entry.status) # We're already running, but more calls to run_with_ready_delay can # continue to come in due to straggler hosts enter Pending. Make # sure we don't do anything. self.god.stub_function(job, 'run') job.run_with_ready_delay(hqe) self.god.check_playback() self.god.unstub(job, 'run')