def test_single_pulls(self):
# first pull new task 1
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
# second pull expired task 1
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_EXPIRED)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
# third pull is blank since all recipes have been pulled from
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 0)
# expire all workers except OTHER_WORKER / RECIPE_RUNNING
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING])
# get oldest expired job first ( redo task since it never completes )
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
def test_worker(self):
# no jobs
ignore, job = worker_pull('SAMPLE_WORKER', jobs=0)
self.assertEqual(len(job), 0)
# manual mode ( without force always returns no tasks )
ignore, job = worker_pull('SAMPLE_WORKER', jobs=1)
self.assertEqual(len(job), 0)
# advance time, since current jobs need to expire, artificially ping to keep out of queue
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
# remove time dependency for this test, force all tasks in first recipe
self.recipe.force()
status = self.recipe.get_status()
for task in status['tasks']:
ignore, job = worker_pull('SAMPLE_WORKER', jobs=1)
self.assertEqual(len(job), 1)
job = job[0]
self.assertEqual(job['event'], 'JOB_PENDING')
self.assertEqual(job['instance'], task['instance'])
self.assertEqual(job['hour'], task['hour'])
# job is not run through actual worker, so 'job' key will be missing, simulate it
job['job'] = {
'worker':'SAMPLE_WORKER',
'id':'SAMPLE_JOB_UUID',
'process':None,
'utc':datetime.utcnow(),
}
worker_status(
job['job']['worker'],
job['recipe']['setup']['uuid'],
job['script'],
job['instance'],
job['hour'],
'JOB_END',
"Output is OK.",
""
)
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
# after completing all tasks, check if they whole recipe is done
self.recipe.refresh_from_db()
self.assertTrue(self.recipe.job_done)
status = self.recipe.get_status()
self.assertTrue(
all([
(task['event'] == 'JOB_END')
for task in status['tasks']
])
)
def test_half_done(self):
jobs_all, jobs_new = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs_new, [])
self.assertEqual(jobs_all, [self.RECIPE_DONE])
# first pull marked job as done, now it wont show up
jobs_all, jobs_new = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs_new, [])
self.assertEqual(jobs_all, [])
def test_recipe_stop(self):
resp = self.client.post('/recipe/stop/', {'reference':'THISREFERENCEISINVALID'})
self.assertEqual(resp.status_code, 404)
self.assertEqual(resp.content, b'RECIPE NOT FOUND')
resp = self.client.post('/recipe/stop/', {'reference':self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, b'RECIPE STOPPED')
#self.recipe_new.refresh_from_db()
#self.assertTrue(self.recipe_new.job_done)
assertRecipeDone(self, self.recipe_new)
# recipe is stopped and cannot be pulled
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 0)
# start a recipe run to test interrupt
resp = self.client.post('/recipe/start/', {'reference':self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, b'RECIPE STARTED')
#self.recipe_new.refresh_from_db()
#self.assertFalse(self.recipe_new.job_done)
assertRecipeNotDone(self, self.recipe_new)
# wait for next worker cycle
sleep(WORKER_LOOKBACK_EXPIRE/1000)
# recipe is started and can be pulled
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 1)
resp = self.client.post('/recipe/stop/', {'reference':self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, b'RECIPE INTERRUPTED')
self.recipe_new.refresh_from_db()
#self.assertTrue(self.recipe_new.job_done)
assertRecipeDone(self, self.recipe_new)
# wait for next worker cycle
sleep(WORKER_LOOKBACK_EXPIRE/1000)
# recipe is stopped and cannot be pulled
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 0)
def test_manager(self):
management.call_command('job_worker', worker='TEST_WORKER', jobs=5, timeout=60*60*1, verbose=True, test=True)
jobs = Recipe.objects.filter(worker_uid='TEST_WORKER')
self.assertEqual(len(jobs), 2)
status = jobs[0].get_status()
self.assertEqual(len(status['tasks']), 2)
self.assertEqual(status['tasks'][0]['script'], 'hello')
self.assertEqual(status['tasks'][0]['instance'], 1)
self.assertEqual(status['tasks'][0]['hour'], 0)
self.assertEqual(status['tasks'][0]['event'], 'JOB_END')
self.assertEqual(status['tasks'][1]['script'], 'hello')
self.assertEqual(status['tasks'][1]['instance'], 2)
self.assertEqual(status['tasks'][1]['hour'], 0)
self.assertEqual(status['tasks'][1]['event'], 'JOB_PENDING')
# advance time, since current jobs need to expire, artificially ping to keep out of queue
sleep(WORKER_LOOKBACK_EXPIRE/1000)
worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING])
# second loop through manager
management.call_command('job_worker', worker='TEST_WORKER', jobs=5, timeout=60*60*1, verbose=True, test=True)
jobs = Recipe.objects.filter(worker_uid='TEST_WORKER')
self.assertEqual(len(jobs), 2)
status = jobs[0].get_status()
self.assertEqual(len(status['tasks']), 2)
self.assertEqual(status['tasks'][0]['script'], 'hello')
self.assertEqual(status['tasks'][0]['instance'], 1)
self.assertEqual(status['tasks'][0]['hour'], 0)
self.assertEqual(status['tasks'][0]['event'], 'JOB_END')
self.assertEqual(status['tasks'][1]['script'], 'hello')
self.assertEqual(status['tasks'][1]['instance'], 2)
self.assertEqual(status['tasks'][1]['hour'], 0)
self.assertEqual(status['tasks'][1]['event'], 'JOB_END')
# jobs were also marked as complete
#self.assertTrue(jobs[0].job_done)
#self.assertTrue(jobs[1].job_done)
assertRecipeDone(self, jobs[0])
assertRecipeDone(self, jobs[1])
# advance time, since current jobs need to expire, artificially ping to keep out of queue
sleep(WORKER_LOOKBACK_EXPIRE/1000)
worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING])
# all jobs either run by other workers or done
ignore, jobs = worker_pull('TEST_WORKER', 5)
self.assertEqual(len(jobs), 0)
def test_multiple_pulls(self):
# pull all jobs at once
ignore, jobs = worker_pull('TEST_WORKER', 5)
self.assertEqual(len(jobs), 2)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
self.assertEqual(jobs[1]['recipe']['setup']['uuid'], self.RECIPE_EXPIRED)
self.assertEqual(jobs[1]['script'], 'hello')
self.assertEqual(jobs[1]['instance'], 1)
self.assertEqual(jobs[1]['hour'], 0)
def test_recipe_start(self):
resp = self.client.post('/recipe/start/', {'reference':'THISREFERENCEISINVALID'})
self.assertEqual(resp.status_code, 404)
self.assertEqual(resp.content, b'RECIPE NOT FOUND')
resp = self.client.post('/recipe/start/', {'reference':self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, b'RECIPE STARTED')
# start a recipe run to test interrupt
ignore, jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 1)
resp = self.client.post('/recipe/start/', {'reference':self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, b'RECIPE INTERRUPTED')
def test_worker(self):
# remove time dependency for this test, force all tasks
self.recipe.force()
status = self.recipe.get_status()
for task in status['tasks']:
ignore, job = worker_pull('SAMPLE_WORKER', jobs=1)
self.assertEqual(len(job), 1)
job = job[0]
self.assertEqual(job['event'], 'JOB_PENDING')
self.assertEqual(job['instance'], task['instance'])
self.assertEqual(job['hour'], task['hour'])
# job is not run through actual worker, so 'job' key will be missing, simulate it
job['job'] = {
'worker':'SAMPLE_WORKER',
'id':'SAMPLE_JOB_UUID',
'process':None,
'utc':datetime.utcnow(),
}
worker_status(
job['job']['worker'],
job['recipe']['setup']['uuid'],
job['script'],
job['instance'],
job['hour'],
'JOB_END',
"Output is OK.",
""
)
sleep(WORKER_LOOKBACK_EXPIRE/1000)
# after completing all tasks, check if they whole recipe is done
self.recipe.refresh_from_db()
assertRecipeDone(self, self.recipe)