def test_celery_integration(self): executor = CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='debug'): success_command = ['true', 'some_parameter'] fail_command = ['false', 'some_parameter'] executor.execute_async(key='success', command=success_command) # errors are propagated for some reason try: executor.execute_async(key='fail', command=fail_command) except: pass executor.running['success'] = True executor.running['fail'] = True executor.end(synchronous=True) self.assertTrue(executor.event_buffer['success'], State.SUCCESS) self.assertTrue(executor.event_buffer['fail'], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)
def test_celery_integration(self): executor = CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='debug'): success_command = ['true', 'some_parameter'] fail_command = ['false', 'some_parameter'] cached_celery_backend = execute_command.backend task_tuples_to_send = [ ('success', 'fake_simple_ti', success_command, celery_configuration['task_default_queue'], execute_command), ('fail', 'fake_simple_ti', fail_command, celery_configuration['task_default_queue'], execute_command) ] chunksize = executor._num_tasks_per_send_process( len(task_tuples_to_send)) num_processes = min(len(task_tuples_to_send), executor._sync_parallelism) send_pool = Pool(processes=num_processes) key_and_async_results = send_pool.map(send_task_to_executor, task_tuples_to_send, chunksize=chunksize) send_pool.close() send_pool.join() for key, command, result in key_and_async_results: # Only pops when enqueued successfully, otherwise keep it # and expect scheduler loop to deal with it. result.backend = cached_celery_backend executor.running[key] = command executor.tasks[key] = result executor.last_state[key] = celery_states.PENDING executor.running['success'] = True executor.running['fail'] = True executor.end(synchronous=True) self.assertTrue(executor.event_buffer['success'], State.SUCCESS) self.assertTrue(executor.event_buffer['fail'], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)