def test_celery_integration(self):
executor = CeleryExecutor()
executor.start()
with start_worker(app=app, logfile=sys.stdout, loglevel='debug'):
success_command = ['true', 'some_parameter']
fail_command = ['false', 'some_parameter']
executor.execute_async(key='success', command=success_command)
# errors are propagated for some reason
try:
executor.execute_async(key='fail', command=fail_command)
except:
pass
executor.running['success'] = True
executor.running['fail'] = True
executor.end(synchronous=True)
self.assertTrue(executor.event_buffer['success'], State.SUCCESS)
self.assertTrue(executor.event_buffer['fail'], State.FAILED)
self.assertNotIn('success', executor.tasks)
self.assertNotIn('fail', executor.tasks)
self.assertNotIn('success', executor.last_state)
self.assertNotIn('fail', executor.last_state)
def test_celery_integration(self):
executor = CeleryExecutor()
executor.start()
with start_worker(app=app, logfile=sys.stdout, loglevel='debug'):
success_command = ['true', 'some_parameter']
fail_command = ['false', 'some_parameter']
executor.execute_async(key='success', command=success_command)
# errors are propagated for some reason
try:
executor.execute_async(key='fail', command=fail_command)
except:
pass
executor.running['success'] = True
executor.running['fail'] = True
executor.end(synchronous=True)
self.assertTrue(executor.event_buffer['success'], State.SUCCESS)
self.assertTrue(executor.event_buffer['fail'], State.FAILED)
self.assertNotIn('success', executor.tasks)
self.assertNotIn('fail', executor.tasks)
self.assertNotIn('success', executor.last_state)
self.assertNotIn('fail', executor.last_state)
def test_celery_integration(self):
executor = CeleryExecutor()
executor.start()
with start_worker(app=app, logfile=sys.stdout, loglevel='debug'):
success_command = ['true', 'some_parameter']
fail_command = ['false', 'some_parameter']
cached_celery_backend = execute_command.backend
task_tuples_to_send = [
('success', 'fake_simple_ti', success_command,
celery_configuration['task_default_queue'], execute_command),
('fail', 'fake_simple_ti', fail_command,
celery_configuration['task_default_queue'], execute_command)
]
chunksize = executor._num_tasks_per_send_process(
len(task_tuples_to_send))
num_processes = min(len(task_tuples_to_send),
executor._sync_parallelism)
send_pool = Pool(processes=num_processes)
key_and_async_results = send_pool.map(send_task_to_executor,
task_tuples_to_send,
chunksize=chunksize)
send_pool.close()
send_pool.join()
for key, command, result in key_and_async_results:
# Only pops when enqueued successfully, otherwise keep it
# and expect scheduler loop to deal with it.
result.backend = cached_celery_backend
executor.running[key] = command
executor.tasks[key] = result
executor.last_state[key] = celery_states.PENDING
executor.running['success'] = True
executor.running['fail'] = True
executor.end(synchronous=True)
self.assertTrue(executor.event_buffer['success'], State.SUCCESS)
self.assertTrue(executor.event_buffer['fail'], State.FAILED)
self.assertNotIn('success', executor.tasks)
self.assertNotIn('fail', executor.tasks)
self.assertNotIn('success', executor.last_state)
self.assertNotIn('fail', executor.last_state)