def test_single_pulls(self):
# first pull new task 1
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
# second pull expired task 1
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'],
self.RECIPE_EXPIRED)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
# third pull is blank since all recipes have been pulled from
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 0)
# expire all workers except OTHER_WORKER / RECIPE_RUNNING
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING])
# get oldest expired job first ( redo task since it never completes )
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
def test_recipe_stop(self):
resp = self.client.post('/recipe/stop/',
{'reference': 'THISREFERENCEISINVALID'})
self.assertEqual(resp.status_code, 404)
self.assertEqual(resp.content, 'RECIPE NOT FOUND')
resp = self.client.post('/recipe/stop/',
{'reference': self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, 'RECIPE STOPPED')
self.recipe_new.refresh_from_db()
self.assertTrue(self.recipe_new.job_done)
# recipe is stopped and cannot be pulled
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 0)
# start a recipe run to test interrupt
resp = self.client.post('/recipe/start/',
{'reference': self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, 'RECIPE STARTED')
self.recipe_new.refresh_from_db()
self.assertFalse(self.recipe_new.job_done)
# wait for next worker cycle
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
# recipe is started and can be pulled
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 1)
resp = self.client.post('/recipe/stop/',
{'reference': self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, 'RECIPE INTERRUPTED')
self.recipe_new.refresh_from_db()
self.assertTrue(self.recipe_new.job_done)
# wait for next worker cycle
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
# recipe is stopped and cannot be pulled
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 0)
def pull(self):
self.lock_thread.acquire()
jobs = worker_pull(self.uid, jobs=self.available())
self.lock_thread.release()
if jobs:
for job in jobs:
self.run(job)
def test_manager(self):
management.call_command('job_worker',
worker='TEST_WORKER',
jobs=5,
timeout=60 * 60 * 1,
verbose=True,
test=True)
jobs = Recipe.objects.filter(worker_uid='TEST_WORKER')
self.assertEqual(len(jobs), 2)
status = jobs[0].get_status()
self.assertEqual(len(status['tasks']), 2)
self.assertEqual(status['tasks'][0]['script'], 'hello')
self.assertEqual(status['tasks'][0]['instance'], 1)
self.assertEqual(status['tasks'][0]['hour'], 0)
self.assertEqual(status['tasks'][0]['event'], 'JOB_END')
self.assertEqual(status['tasks'][1]['script'], 'hello')
self.assertEqual(status['tasks'][1]['instance'], 2)
self.assertEqual(status['tasks'][1]['hour'], 0)
self.assertEqual(status['tasks'][1]['event'], 'JOB_PENDING')
# advance time, since current jobs need to expire, artificially ping to keep out of queue
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING])
# second loop through manager
management.call_command('job_worker',
worker='TEST_WORKER',
jobs=5,
timeout=60 * 60 * 1,
verbose=True,
test=True)
jobs = Recipe.objects.filter(worker_uid='TEST_WORKER')
self.assertEqual(len(jobs), 2)
status = jobs[0].get_status()
self.assertEqual(len(status['tasks']), 2)
self.assertEqual(status['tasks'][0]['script'], 'hello')
self.assertEqual(status['tasks'][0]['instance'], 1)
self.assertEqual(status['tasks'][0]['hour'], 0)
self.assertEqual(status['tasks'][0]['event'], 'JOB_END')
self.assertEqual(status['tasks'][1]['script'], 'hello')
self.assertEqual(status['tasks'][1]['instance'], 2)
self.assertEqual(status['tasks'][1]['hour'], 0)
self.assertEqual(status['tasks'][1]['event'], 'JOB_END')
# jobs were also marked as complete
self.assertTrue(jobs[0].job_done)
self.assertTrue(jobs[1].job_done)
# advance time, since current jobs need to expire, artificially ping to keep out of queue
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING])
# all jobs either run by other workers or done
jobs = worker_pull('TEST_WORKER', 5)
self.assertEqual(len(jobs), 0)
def test_worker(self):
# manual mode ( without force always returns no tasks )
job = worker_pull('SAMPLE_WORKER', jobs=1)
self.assertEqual(len(job), 0)
# advance time, since current jobs need to expire, artificially ping to keep out of queue
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
# remove time dependency for this test, force all tasks
self.recipe.force()
status = self.recipe.get_status()
for task in status['tasks']:
job = worker_pull('SAMPLE_WORKER', jobs=1)
self.assertEqual(len(job), 1)
job = job[0]
self.assertEqual(job['event'], 'JOB_PENDING')
self.assertEqual(job['instance'], task['instance'])
self.assertEqual(job['hour'], task['hour'])
# job is not run through actual worker, so 'job' key will be missing, simulate it
job['job'] = {
'worker': 'SAMPLE_WORKER',
'id': 'SAMPLE_JOB_UUID',
'process': None,
'utc': datetime.utcnow(),
}
worker_status(job['job']['worker'], job['recipe']['setup']['uuid'],
job['script'], job['instance'], job['hour'],
'JOB_END', "Output is OK.", "")
sleep((JOB_LOOKBACK_MS * 2) / 1000.0)
# after completing all tasks, check if they whole recipe is done
self.recipe.refresh_from_db()
self.assertTrue(self.recipe.job_done)
status = self.recipe.get_status()
self.assertTrue(
all([(task['event'] == 'JOB_END') for task in status['tasks']]))
def test_multiple_pulls(self):
# pull all jobs at once
jobs = worker_pull('TEST_WORKER', 5)
self.assertEqual(len(jobs), 2)
self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW)
self.assertEqual(jobs[0]['script'], 'hello')
self.assertEqual(jobs[0]['instance'], 1)
self.assertEqual(jobs[0]['hour'], 0)
self.assertEqual(jobs[1]['recipe']['setup']['uuid'],
self.RECIPE_EXPIRED)
self.assertEqual(jobs[1]['script'], 'hello')
self.assertEqual(jobs[1]['instance'], 1)
self.assertEqual(jobs[1]['hour'], 0)
def test_recipe_start(self):
resp = self.client.post('/recipe/start/',
{'reference': 'THISREFERENCEISINVALID'})
self.assertEqual(resp.status_code, 404)
self.assertEqual(resp.content, 'RECIPE NOT FOUND')
resp = self.client.post('/recipe/start/',
{'reference': self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, 'RECIPE STARTED')
# start a recipe run to test interrupt
jobs = worker_pull('SAMPLE_WORKER', 1)
self.assertEqual(len(jobs), 1)
resp = self.client.post('/recipe/start/',
{'reference': self.recipe_new.reference})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.content, 'RECIPE INTERRUPTED')