def test_single_pulls(self): # first pull new task 1 jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW) self.assertEqual(jobs[0]['script'], 'hello') self.assertEqual(jobs[0]['instance'], 1) self.assertEqual(jobs[0]['hour'], 0) # second pull expired task 1 jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_EXPIRED) self.assertEqual(jobs[0]['script'], 'hello') self.assertEqual(jobs[0]['instance'], 1) self.assertEqual(jobs[0]['hour'], 0) # third pull is blank since all recipes have been pulled from jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(len(jobs), 0) # expire all workers except OTHER_WORKER / RECIPE_RUNNING sleep((JOB_LOOKBACK_MS * 2) / 1000.0) worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING]) # get oldest expired job first ( redo task since it never completes ) jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW) self.assertEqual(jobs[0]['script'], 'hello') self.assertEqual(jobs[0]['instance'], 1) self.assertEqual(jobs[0]['hour'], 0)
def test_recipe_stop(self): resp = self.client.post('/recipe/stop/', {'reference': 'THISREFERENCEISINVALID'}) self.assertEqual(resp.status_code, 404) self.assertEqual(resp.content, 'RECIPE NOT FOUND') resp = self.client.post('/recipe/stop/', {'reference': self.recipe_new.reference}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.content, 'RECIPE STOPPED') self.recipe_new.refresh_from_db() self.assertTrue(self.recipe_new.job_done) # recipe is stopped and cannot be pulled jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(len(jobs), 0) # start a recipe run to test interrupt resp = self.client.post('/recipe/start/', {'reference': self.recipe_new.reference}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.content, 'RECIPE STARTED') self.recipe_new.refresh_from_db() self.assertFalse(self.recipe_new.job_done) # wait for next worker cycle sleep((JOB_LOOKBACK_MS * 2) / 1000.0) # recipe is started and can be pulled jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(len(jobs), 1) resp = self.client.post('/recipe/stop/', {'reference': self.recipe_new.reference}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.content, 'RECIPE INTERRUPTED') self.recipe_new.refresh_from_db() self.assertTrue(self.recipe_new.job_done) # wait for next worker cycle sleep((JOB_LOOKBACK_MS * 2) / 1000.0) # recipe is stopped and cannot be pulled jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(len(jobs), 0)
def pull(self): self.lock_thread.acquire() jobs = worker_pull(self.uid, jobs=self.available()) self.lock_thread.release() if jobs: for job in jobs: self.run(job)
def test_manager(self): management.call_command('job_worker', worker='TEST_WORKER', jobs=5, timeout=60 * 60 * 1, verbose=True, test=True) jobs = Recipe.objects.filter(worker_uid='TEST_WORKER') self.assertEqual(len(jobs), 2) status = jobs[0].get_status() self.assertEqual(len(status['tasks']), 2) self.assertEqual(status['tasks'][0]['script'], 'hello') self.assertEqual(status['tasks'][0]['instance'], 1) self.assertEqual(status['tasks'][0]['hour'], 0) self.assertEqual(status['tasks'][0]['event'], 'JOB_END') self.assertEqual(status['tasks'][1]['script'], 'hello') self.assertEqual(status['tasks'][1]['instance'], 2) self.assertEqual(status['tasks'][1]['hour'], 0) self.assertEqual(status['tasks'][1]['event'], 'JOB_PENDING') # advance time, since current jobs need to expire, artificially ping to keep out of queue sleep((JOB_LOOKBACK_MS * 2) / 1000.0) worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING]) # second loop through manager management.call_command('job_worker', worker='TEST_WORKER', jobs=5, timeout=60 * 60 * 1, verbose=True, test=True) jobs = Recipe.objects.filter(worker_uid='TEST_WORKER') self.assertEqual(len(jobs), 2) status = jobs[0].get_status() self.assertEqual(len(status['tasks']), 2) self.assertEqual(status['tasks'][0]['script'], 'hello') self.assertEqual(status['tasks'][0]['instance'], 1) self.assertEqual(status['tasks'][0]['hour'], 0) self.assertEqual(status['tasks'][0]['event'], 'JOB_END') self.assertEqual(status['tasks'][1]['script'], 'hello') self.assertEqual(status['tasks'][1]['instance'], 2) self.assertEqual(status['tasks'][1]['hour'], 0) self.assertEqual(status['tasks'][1]['event'], 'JOB_END') # jobs were also marked as complete self.assertTrue(jobs[0].job_done) self.assertTrue(jobs[1].job_done) # advance time, since current jobs need to expire, artificially ping to keep out of queue sleep((JOB_LOOKBACK_MS * 2) / 1000.0) worker_ping('OTHER_WORKER', [self.RECIPE_RUNNING]) # all jobs either run by other workers or done jobs = worker_pull('TEST_WORKER', 5) self.assertEqual(len(jobs), 0)
def test_worker(self): # manual mode ( without force always returns no tasks ) job = worker_pull('SAMPLE_WORKER', jobs=1) self.assertEqual(len(job), 0) # advance time, since current jobs need to expire, artificially ping to keep out of queue sleep((JOB_LOOKBACK_MS * 2) / 1000.0) # remove time dependency for this test, force all tasks self.recipe.force() status = self.recipe.get_status() for task in status['tasks']: job = worker_pull('SAMPLE_WORKER', jobs=1) self.assertEqual(len(job), 1) job = job[0] self.assertEqual(job['event'], 'JOB_PENDING') self.assertEqual(job['instance'], task['instance']) self.assertEqual(job['hour'], task['hour']) # job is not run through actual worker, so 'job' key will be missing, simulate it job['job'] = { 'worker': 'SAMPLE_WORKER', 'id': 'SAMPLE_JOB_UUID', 'process': None, 'utc': datetime.utcnow(), } worker_status(job['job']['worker'], job['recipe']['setup']['uuid'], job['script'], job['instance'], job['hour'], 'JOB_END', "Output is OK.", "") sleep((JOB_LOOKBACK_MS * 2) / 1000.0) # after completing all tasks, check if they whole recipe is done self.recipe.refresh_from_db() self.assertTrue(self.recipe.job_done) status = self.recipe.get_status() self.assertTrue( all([(task['event'] == 'JOB_END') for task in status['tasks']]))
def test_multiple_pulls(self): # pull all jobs at once jobs = worker_pull('TEST_WORKER', 5) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0]['recipe']['setup']['uuid'], self.RECIPE_NEW) self.assertEqual(jobs[0]['script'], 'hello') self.assertEqual(jobs[0]['instance'], 1) self.assertEqual(jobs[0]['hour'], 0) self.assertEqual(jobs[1]['recipe']['setup']['uuid'], self.RECIPE_EXPIRED) self.assertEqual(jobs[1]['script'], 'hello') self.assertEqual(jobs[1]['instance'], 1) self.assertEqual(jobs[1]['hour'], 0)
def test_recipe_start(self): resp = self.client.post('/recipe/start/', {'reference': 'THISREFERENCEISINVALID'}) self.assertEqual(resp.status_code, 404) self.assertEqual(resp.content, 'RECIPE NOT FOUND') resp = self.client.post('/recipe/start/', {'reference': self.recipe_new.reference}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.content, 'RECIPE STARTED') # start a recipe run to test interrupt jobs = worker_pull('SAMPLE_WORKER', 1) self.assertEqual(len(jobs), 1) resp = self.client.post('/recipe/start/', {'reference': self.recipe_new.reference}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.content, 'RECIPE INTERRUPTED')