def testQueueStatsUpdates(self):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock(side_effect=j._Complete)
# Check that we can find the queued job.
stats = scheduler.QueueStats('mock')
self.assertEquals(stats['queued_jobs'], 1)
self.assertNotIn('running_jobs', stats)
self.assertEquals(len(stats['queue_time_samples']), 0)
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j.Start.called)
job_id, _ = scheduler.PickJobs('mock')[0]
self.assertIsNone(job_id)
# Check that point-in-time stats are zero, and that we have one sample.
stats = scheduler.QueueStats('mock')
self.assertNotIn('queued_jobs', stats)
self.assertNotIn('running_jobs', stats)
self.assertNotEquals(len(stats['queue_time_samples']), 0)
self.assertEquals(len(stats['queue_time_samples'][0]), 2)
def testMultipleQueues(self):
jobs = []
total_jobs = 2
total_queues = 10
for configuration_id in range(total_queues):
for _ in range(total_jobs):
j = job.Job.New((), (),
arguments={
'configuration':
'queue-{}'.format(configuration_id)
},
comparison_mode='performance')
j.Start = mock.MagicMock(side_effect=j._Complete)
scheduler.Schedule(j)
jobs.append(j)
# We ensure that all jobs complete if we poll the fifo-scheduler.
for _ in range(0, total_jobs):
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
# Check for each job that Job.Start() was called.
for index, j in enumerate(jobs):
self.assertTrue(j.Start.Called,
'job at index {} was not run!'.format(index))
def CreateAndSchedule():
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock(side_effect=j._Complete)
return j
def testJobCancellationSucceedsOnRunningJob(self):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock()
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j.Start.called)
# Ensure that the job is still running.
job_id, queue_status = scheduler.PickJobs('mock')[0]
self.assertEqual(job_id, j.job_id)
self.assertEqual(queue_status, 'Running')
# We can cancel a running job.
self.assertTrue(scheduler.Cancel(j))
# Ensure that the job is still running.
job_id, queue_status = scheduler.PickJobs('mock')[0]
self.assertNotEqual(job_id, j.job_id)
self.assertNotEqual(queue_status, 'Running')
def testSingleQueue(self):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock()
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j.Start.called)
# Ensure that the job is still running.
job_id, queue_status = scheduler.PickJobs('mock')[0]
self.assertEqual(job_id, j.job_id)
self.assertEqual(queue_status, 'Running')
# On the next poll, we need to ensure that an ongoing job doesn't get marked
# completed until it really is completed.
j.Start = mock.MagicMock()
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertFalse(j.Start.called)
job_id, queue_status = scheduler.PickJobs('mock')[0]
self.assertEqual(job_id, j.job_id)
self.assertEqual(queue_status, 'Running')
def Post(self):
job = _CreateJob(self.request)
scheduler.Schedule(job)
return {
'jobId': job.job_id,
'jobUrl': job.url,
}
def testCancelForbiddenUser(self):
job = job_module.Job.New((), (), user='******')
scheduler.Schedule(job)
self.addCleanup(scheduler.Cancel, job)
self.Post('/api/job/cancel', {
'job_id': job.job_id,
'reason': 'testing!'
},
status=403)
def testSchedulePriorityOrder(self):
j0 = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j0)
j0.Start = mock.MagicMock( # pylint: disable=invalid-name
side_effect=j0._Complete)
j1 = job.Job.New((), (),
arguments={
'configuration': 'mock',
'priority': '100',
},
comparison_mode='performance',
priority=100)
j1.Start = mock.MagicMock( # pylint: disable=invalid-name
side_effect=j1._Complete)
scheduler.Schedule(j1)
j2 = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j2)
j2.Start = mock.MagicMock( # pylint: disable=invalid-name
side_effect=j2._Complete)
# The first time we call the scheduler, it must mark j0 completed.
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j0.Start.called)
self.assertFalse(j1.Start.called)
self.assertFalse(j2.Start.called)
# Next time, j2 should be completed.
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertFalse(j1.Start.called)
self.assertTrue(j2.Start.called)
# Then we should have j1 completed.
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j1.Start.called)
def testCancelUnknownJob(self):
job = job_module.Job.New((), (), user='******')
scheduler.Schedule(job)
self.addCleanup(scheduler.Cancel, job)
self.Post('/api/job/cancel', {
'job_id': job.job_id + '1',
'reason': 'testing!'
},
status=404)
job = job_module.JobFromId(job.job_id + '1')
self.assertIsNone(job)
def testJobCancellationSucceedsOnQueuedJob(self):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock()
self.assertTrue(scheduler.Cancel(j))
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertFalse(j.Start.called)
def Post(self):
# TODO(dberris): Validate the inputs based on the type of job requested.
job = _CreateJob(self.request)
scheduler.Schedule(job)
job.PostCreationUpdate()
return {
'jobId': job.job_id,
'jobUrl': job.url,
}
def testJobFails(self):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock(side_effect=j.Fail)
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j.Start.called)
job_id, _ = scheduler.PickJobs('mock')[0]
self.assertIsNone(job_id)
def testJobSamplesCapped(self):
for _ in range(51):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance')
scheduler.Schedule(j)
j.Start = mock.MagicMock(side_effect=j._Complete)
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
stats = scheduler.QueueStats('mock')
self.assertLessEqual(len(stats.get('queue_time_samples')), 50)
def Post(self):
# TODO(dberris): Validate the inputs based on the type of job requested.
job = _CreateJob(self.request)
# We apply the cost-based scheduling at job creation time, so that we can
# roll out the feature as jobs come along.
scheduler.Schedule(job, scheduler.Cost(job))
job.PostCreationUpdate()
return {
'jobId': job.job_id,
'jobUrl': job.url,
}
def testCancelKnownJobByAdmin(self):
job = job_module.Job.New((), (), user='******', bug_id=123)
scheduler.Schedule(job)
self.Post(
'/api/job/cancel', {
'job_id': job.job_id,
'reason': 'testing!'
},
status=200)
job = job_module.JobFromId(job.job_id)
self.assertTrue(job.cancelled)
self.assertIn('[email protected]: testing!', job.cancel_reason)
self.ExecuteDeferredTasks('default')
self.assertTrue(self.add_bug_comment.called)
def testJobRunInExecutionEngine(self):
j = job.Job.New((), (),
arguments={'configuration': 'mock'},
comparison_mode='performance',
use_execution_engine=True)
self.PopulateSimpleBisectionGraph(j)
scheduler.Schedule(j)
j.Start = mock.MagicMock(side_effect=j._Complete)
response = self.testapp.get('/cron/fifo-scheduler')
self.assertEqual(response.status_code, 200)
self.ExecuteDeferredTasks('default')
self.assertTrue(j.Start.called)
job_id, _ = scheduler.PickJobs('mock')[0]
self.assertIsNone(job_id)
def testCancelAlreadyRunningJob(self):
job = job_module.Job.New((), (),
arguments={'configuration': 'mock'},
user='******')
scheduler.Schedule(job)
_, status = scheduler.PickJobs(job.configuration)[0]
self.assertEqual(status, 'Queued')
job.task = '123'
job.started = True
job.put()
self.assertTrue(job.running)
self.addCleanup(scheduler.Cancel, job)
self.Post('/api/job/cancel', {
'job_id': job.job_id,
'reason': 'testing!'
},
status=200)
def testCancelCancelledJob(self):
job = job_module.Job.New((), (), user='******')
scheduler.Schedule(job)
self.Post('/api/job/cancel', {
'job_id': job.job_id,
'reason': 'testing!'
},
status=200)
job = job_module.JobFromId(job.job_id)
self.assertTrue(job.cancelled)
self.assertIn('[email protected]: testing!', job.cancel_reason)
self.Post('/api/job/cancel', {
'job_id': job.job_id,
'reason': 'cancelling again!'
},
status=400)
job = job_module.JobFromId(job.job_id)
self.assertTrue(job.cancelled)
self.assertIn('[email protected]: testing!', job.cancel_reason)
