def test_celery_integration(self, broker_url): success_command = ['airflow', 'run', 'true', 'some_parameter'] fail_command = ['airflow', 'version'] def fake_execute_command(command): if command != success_command: raise AirflowException("fail") with self._prepare_app(broker_url, execute=fake_execute_command) as app: executor = celery_executor.CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='debug'): cached_celery_backend = celery_executor.execute_command.backend task_tuples_to_send = [ ('success', 'fake_simple_ti', success_command, celery_executor. celery_configuration['task_default_queue'], celery_executor.execute_command), ('fail', 'fake_simple_ti', fail_command, celery_executor. celery_configuration['task_default_queue'], celery_executor.execute_command) ] chunksize = executor._num_tasks_per_send_process( len(task_tuples_to_send)) num_processes = min(len(task_tuples_to_send), executor._sync_parallelism) send_pool = Pool(processes=num_processes) key_and_async_results = send_pool.map( celery_executor.send_task_to_executor, task_tuples_to_send, chunksize=chunksize) send_pool.close() send_pool.join() for key, command, result in key_and_async_results: # Only pops when enqueued successfully, otherwise keep it # and expect scheduler loop to deal with it. result.backend = cached_celery_backend executor.running[key] = command executor.tasks[key] = result executor.last_state[key] = celery_states.PENDING executor.running['success'] = True executor.running['fail'] = True executor.end(synchronous=True) self.assertTrue(executor.event_buffer['success'], State.SUCCESS) self.assertTrue(executor.event_buffer['fail'], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)
def test_celery_integration(self): executor = CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='debug'): success_command = ['true', 'some_parameter'] fail_command = ['false', 'some_parameter'] executor.execute_async(key='success', command=success_command) # errors are propagated for some reason try: executor.execute_async(key='fail', command=fail_command) except: pass executor.running['success'] = True executor.running['fail'] = True executor.end(synchronous=True) self.assertTrue(executor.event_buffer['success'], State.SUCCESS) self.assertTrue(executor.event_buffer['fail'], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)
def celery_session_worker(request, celery_session_app, celery_worker_pool, celery_worker_parameters): """Session Fixture: Start worker that lives throughout test suite.""" with worker.start_worker(celery_session_app, pool=celery_worker_pool, **celery_worker_parameters) as worker_context: yield worker_context
def flask_celery_worker(flask_celery_app: celery.Celery, db): # type: ignore # pylint: disable=W0621 with worker.start_worker( flask_celery_app, pool="solo", perform_ping_check=False, ) as w: yield w db.session.close_all()
def test_celery_integration(self): executor = CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='debug'): success_command = ['true', 'some_parameter'] fail_command = ['false', 'some_parameter'] executor.execute_async(key='success', command=success_command) # errors are propagated for some reason try: executor.execute_async(key='fail', command=fail_command) except: pass executor.running['success'] = True executor.running['fail'] = True executor.end(synchronous=True) self.assertTrue(executor.event_buffer['success'], State.SUCCESS) self.assertTrue(executor.event_buffer['fail'], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)
def setUp(self): Organism.objects.create( display_name=display_name, short_name=short_name, tax_id=tax_id) organism = Organism.objects.get(short_name=short_name) prepare_test_fasta_file() self.files = test_files HmmerDB.objects.create( fasta_file=FileObject('/blast/db/clec_peptide_example_BLASTdb.fa'), organism=organism, is_shown=True, title=title) if not DEBUG: # headless chrome driver options = webdriver.ChromeOptions() options.add_argument('headless') options.add_argument('window-size=1280x800') self.driver = webdriver.Chrome(chrome_options=options) else: # use with header self.driver = webdriver.Chrome() # Or use different webdriver # self.driver = webdriver.PhantomJS() # self.driver = webdriver.Firefox() self.driver.set_window_size(1280, 800) # Start up celery worker for testing self.celery_worker = start_worker(app) self.celery_worker.__enter__()
def test_start_worker_with_exception(self): """Make sure that start_worker does not hang on exception""" with pytest.raises(NotImplementedError): with start_worker(app=self.app, loglevel=0): result = self.error_task.apply_async() result.get(timeout=5)
def setUpClass(cls): super().setUpClass() # Start up celery worker app.loader.import_module('celery.contrib.testing.tasks') cls.celery_worker = start_worker(app) cls.celery_worker.__enter__()
def setUpClass(cls): super().setUpClass() # start celery worker with test app context and in-memory context celery_app = make_celery(app) cls.celery_worker = start_worker(celery_app, perform_ping_check=False) # spawn celery worker cls.celery_worker.__enter__()
def test_celery_integration(self, broker_url): success_command = ['airflow', 'tasks', 'run', 'true', 'some_parameter'] fail_command = ['airflow', 'version'] def fake_execute_command(command): if command != success_command: raise AirflowException("fail") with _prepare_app(broker_url, execute=fake_execute_command) as app: executor = celery_executor.CeleryExecutor() self.assertEqual(executor.tasks, {}) executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='info'): execute_date = datetime.now() task_tuples_to_send = [ (('success', 'fake_simple_ti', execute_date, 0), None, success_command, celery_executor. celery_configuration['task_default_queue'], celery_executor.execute_command), (('fail', 'fake_simple_ti', execute_date, 0), None, fail_command, celery_executor. celery_configuration['task_default_queue'], celery_executor.execute_command) ] # "Enqueue" them. We don't have a real SimpleTaskInstance, so directly edit the dict for (key, simple_ti, command, queue, task) in task_tuples_to_send: # pylint: disable=W0612 executor.queued_tasks[key] = (command, 1, queue, simple_ti) executor._process_tasks(task_tuples_to_send) self.assertEqual( list(executor.tasks.keys()), [('success', 'fake_simple_ti', execute_date, 0), ('fail', 'fake_simple_ti', execute_date, 0)]) self.assertEqual( executor.event_buffer[('success', 'fake_simple_ti', execute_date, 0)][0], State.QUEUED) self.assertEqual( executor.event_buffer[('fail', 'fake_simple_ti', execute_date, 0)][0], State.QUEUED) executor.end(synchronous=True) self.assertEqual( executor.event_buffer[('success', 'fake_simple_ti', execute_date, 0)][0], State.SUCCESS) self.assertEqual( executor.event_buffer[('fail', 'fake_simple_ti', execute_date, 0)][0], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertEqual(executor.queued_tasks, {}) self.assertEqual(timedelta(0, 600), executor.task_adoption_timeout)
def setUpClass(cls): """Start a celery worker""" super().setUpClass() # Special namespace loading of methods needed by start_worker, per the celery docs app.loader.import_module("celery.contrib.testing.tasks") cls.clear_worker() cls.celery_worker = start_worker(app, concurrency=1) cls.celery_worker.__enter__()
def setUpClass(cls) -> None: super().setUpClass() # https://stackoverflow.com/questions/46530784/make-django-test-case-database-visible-to-celery/46564964#46564964 cls.celery_worker = start_worker(app, perform_ping_check=False) cls.celery_worker.__enter__() # show logs during testing logging.getLogger().setLevel(logging.WARNING)
def swh_scheduler_celery_worker( swh_scheduler_celery_app, swh_scheduler_celery_includes, ): """Spawn a worker""" for module in swh_scheduler_celery_includes: swh_scheduler_celery_app.loader.import_task_module(module) with worker.start_worker(swh_scheduler_celery_app, pool="solo") as w: yield w
def application_worker(request, application, celery_includes, celery_worker_pool, celery_worker_parameters): # type: (Any, Celery, Sequence[str], str) -> WorkController """Fixture: Start worker in a thread, stop it when the test returns.""" for module in celery_includes: application.loader.import_task_module(module) with worker.start_worker(application, pool=celery_worker_pool, **celery_worker_parameters) as w: yield w
def setUpClass(cls): logging.disable(logging.WARNING) super().setUpClass() # route tasks to this worker by using the default 'celery' queue # that is exclusively used for the automated tests celery_app.conf.update(task_routes=None) cls.celery_worker = start_worker(celery_app, concurrency=2, perform_ping_check=False) cls.celery_worker.__enter__()
def setUpClass(cls): super().setUpClass() # import the ping task, because the start worker function needs it app.loader.import_module("celery.contrib.testing.tasks") # Start up celery worker cls.celery_worker = start_worker(app) cls.celery_worker.__enter__() cls.setupTestData()
def test_celery_integration(self, broker_url): with self._prepare_app(broker_url) as app: executor = celery_executor.CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='info'): success_command = ['true', 'some_parameter'] fail_command = ['false', 'some_parameter'] execute_date = datetime.datetime.now() cached_celery_backend = celery_executor.execute_command.backend task_tuples_to_send = [ (('success', 'fake_simple_ti', execute_date, 0), None, success_command, celery_executor.celery_configuration['task_default_queue'], celery_executor.execute_command), (('fail', 'fake_simple_ti', execute_date, 0), None, fail_command, celery_executor.celery_configuration['task_default_queue'], celery_executor.execute_command) ] chunksize = executor._num_tasks_per_send_process(len(task_tuples_to_send)) num_processes = min(len(task_tuples_to_send), executor._sync_parallelism) send_pool = Pool(processes=num_processes) key_and_async_results = send_pool.map( celery_executor.send_task_to_executor, task_tuples_to_send, chunksize=chunksize) send_pool.close() send_pool.join() for task_instance_key, _, result in key_and_async_results: # Only pops when enqueued successfully, otherwise keep it # and expect scheduler loop to deal with it. result.backend = cached_celery_backend executor.running.add(task_instance_key) executor.tasks[task_instance_key] = result executor.last_state[task_instance_key] = celery_states.PENDING executor.running.add(('success', 'fake_simple_ti', execute_date, 0)) executor.running.add(('fail', 'fake_simple_ti', execute_date, 0)) executor.end(synchronous=True) self.assertEqual(executor.event_buffer[('success', 'fake_simple_ti', execute_date, 0)], State.SUCCESS) self.assertEqual(executor.event_buffer[('fail', 'fake_simple_ti', execute_date, 0)], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)
def setUpClass(cls): super().setUpClass() # Instantiate the browser object profile = cls._create_firefox_profile() cls.browser = Browser("remote", command_executor=SELENIUM_URL, browser_profile=profile) if cls.requires_celery: app.loader.import_module("celery.contrib.testing.tasks") cls.clear_worker() cls.celery_worker = start_worker(app, concurrency=1) cls.celery_worker.__enter__()
def celery_test_fixture(user_object): """ This Fixture starts a celery worker in the running test container and exits it. With this manual entering and exiting it is possible to use the time set by freezgun and also use the Testdatabase. :param user_object: :param contract_ending_in_february: :return: """ worker = start_worker(app) worker.__enter__() yield worker.__exit__(None, None, None)
def setUp(self): celery_app = TestApp(set_as_current=False, enable_logging=True, config=self['celery_config'], **self['celery_parameters']) self['celery_app_fixture'] = setup_default_app(celery_app) self['celery_app_fixture'].__enter__() for module in self['celery_includes']: celery_app.loader.import_task_module(module) self['celery_worker_fixture'] = start_worker( celery_app, pool='prefork', **self['celery_worker_parameters']) self['celery_worker_fixture'].__enter__()
def test_celery_integration(self, broker_url): with self._prepare_app(broker_url) as app: executor = celery_executor.CeleryExecutor() executor.start() with start_worker(app=app, logfile=sys.stdout, loglevel='debug'): success_command = ['true', 'some_parameter'] fail_command = ['false', 'some_parameter'] cached_celery_backend = celery_executor.execute_command.backend task_tuples_to_send = [('success', 'fake_simple_ti', success_command, celery_executor.celery_configuration['task_default_queue'], celery_executor.execute_command), ('fail', 'fake_simple_ti', fail_command, celery_executor.celery_configuration['task_default_queue'], celery_executor.execute_command)] chunksize = executor._num_tasks_per_send_process(len(task_tuples_to_send)) num_processes = min(len(task_tuples_to_send), executor._sync_parallelism) send_pool = Pool(processes=num_processes) key_and_async_results = send_pool.map( celery_executor.send_task_to_executor, task_tuples_to_send, chunksize=chunksize) send_pool.close() send_pool.join() for key, command, result in key_and_async_results: # Only pops when enqueued successfully, otherwise keep it # and expect scheduler loop to deal with it. result.backend = cached_celery_backend executor.running[key] = command executor.tasks[key] = result executor.last_state[key] = celery_states.PENDING executor.running['success'] = True executor.running['fail'] = True executor.end(synchronous=True) self.assertTrue(executor.event_buffer['success'], State.SUCCESS) self.assertTrue(executor.event_buffer['fail'], State.FAILED) self.assertNotIn('success', executor.tasks) self.assertNotIn('fail', executor.tasks) self.assertNotIn('success', executor.last_state) self.assertNotIn('fail', executor.last_state)
def celery_worker(request, celery_app, celery_includes, celery_worker_pool, celery_worker_parameters): # type: (Any, Celery, Sequence[str], str) -> WorkController """Fixture: Start worker in a thread, stop it when the test returns.""" if not NO_WORKER: for module in celery_includes: celery_app.loader.import_task_module(module) with worker.start_worker(celery_app, pool=celery_worker_pool, **celery_worker_parameters) as w: yield w
def celery_session_worker(request, celery_session_app, celery_includes, celery_worker_pool, celery_worker_parameters): # type: (Any, Celery, Sequence[str], str) -> WorkController """Session Fixture: Start worker that lives throughout test suite.""" if not NO_WORKER: for module in celery_includes: celery_session_app.loader.import_task_module(module) with worker.start_worker(celery_session_app, pool=celery_worker_pool, **celery_worker_parameters) as w: yield w
def test_worker_status(exporter, celery_app): threading.Thread(target=exporter.run, args=(exporter.cfg, ), daemon=True).start() time.sleep(5) with start_worker(celery_app, without_heartbeat=False) as celery_worker: hostname = celery_worker.hostname time.sleep(2) assert (exporter.registry.get_sample_value( "celery_worker_up", labels={"hostname": hostname}) == 1.0) time.sleep(2) assert (exporter.registry.get_sample_value("celery_worker_up", labels={"hostname": hostname}) == 0.0)
def celery_test_fixture_end_of_year_test(december_contract): """ This fixture creates a Contract which starts on 1.1.2019 and ends at 29.2.2020. The expected output is as in every other month: -1200 minutes on the automatically created Report at the beginning of a month. :param user_object: :param contract_ending_in_february: :return: """ worker = start_worker(app) worker.__enter__() yield worker.__exit__(None, None, None)
def setUpClass(cls): super().setUpClass() app.loader.import_module('celery.contrib.testing.tasks') # Start up celery worker cls.celery_worker = start_worker(app) cls.celery_worker.__enter__() # create a dummy job in the in-memory database job = Job( name="test job", status="started", ) job.save() cls.job_id = job.id # call command for populating database call_command('populateHierarchies', verbosity=0)
def setUpClass(cls): super().setUpClass() # Instantiate the browser object. profile = cls._create_firefox_profile() cls.browser = Browser( "remote", command_executor=SELENIUM_URL, browser_profile=profile, # See: https://developer.mozilla.org/en-US/docs/Web/WebDriver/Timeouts # desired_capabilities={"timeouts": {"implicit": 60 * 60 * 1000 }}, # 1 hour timeout ) if cls.requires_celery: app.loader.import_module("celery.contrib.testing.tasks") cls.clear_worker() cls.celery_worker = start_worker(app, concurrency=1) cls.celery_worker.__enter__()
def setUp(self): if not DEBUG: # headless chrome driver options = webdriver.ChromeOptions() options.add_argument('headless') options.add_argument('window-size=1280x800') self.driver = webdriver.Chrome(chrome_options=options) else: # use with header self.driver = webdriver.Chrome() # Or use different webdriver # self.driver = webdriver.PhantomJS() # self.driver = webdriver.Firefox() self.driver.set_window_size(1280, 800) # Start up celery worker for testing self.celery_worker = start_worker(app) self.celery_worker.__enter__()
def celery_test_fixture_correct_minutes(user_object, contract_ending_in_february): """ This fixture modifies the automatically created report for January to have symbolicaly 600 minutes of work documented. It is needed to test whether the automatic Report creation carries over the minutes of the last month. :param user_object: :param contract_ending_in_february: :return: """ january_report = contract_ending_in_february.reports.get( month_year__month=1) january_report.worktime = datetime.timedelta(minutes=600) january_report.save() worker = start_worker(app) worker.__enter__() yield worker.__exit__(None, None, None)
def setUpClass(self): super(QueryTestCase, self).setUpClass() # Start up celery worker for testing self.celery_worker = start_worker(app) self.celery_worker.__enter__() Organism.objects.create(display_name=display_name, short_name=short_name, tax_id=tax_id) organism = Organism.objects.get(short_name=short_name) sequence = SequenceType.objects.create(molecule_type='prot', dataset_type=dataset_type) prepare_test_fasta_file() self.files = test_files BlastDb.objects.create( fasta_file=FileObject('/blast/db/clec_peptide_example_BLASTdb.fa'), organism=organism, type=sequence, is_shown=False, title=title) organism = Organism.objects.get(short_name=short_name) blastdb = BlastDb.objects.get(organism=organism) returncode, error, output = blastdb.makeblastdb() returncode, error, output = blastdb.index_fasta() blastdb.is_shown = True blastdb.save() if not DEBUG: # headless chrome driver options = webdriver.ChromeOptions() options.add_argument('headless') options.add_argument('window-size=1280x800') self.driver = webdriver.Chrome(chrome_options=options) else: # use with header self.driver = webdriver.Chrome() # Or use different webdriver # self.driver = webdriver.PhantomJS() # self.driver = webdriver.Firefox() self.driver.set_window_size(1280, 800)
def setUp(self): User = get_user_model() self.username = '******' self.password = '******' User.objects.create_superuser( username=self.username, password=self.password, email='*****@*****.**') if not DEBUG: # headless chrome driver options = webdriver.ChromeOptions() options.add_argument('headless') options.add_argument('window-size=1280x800') self.driver = webdriver.Chrome(chrome_options=options) else: # use with header self.driver = webdriver.Chrome() # Or use different webdriver # self.driver = webdriver.PhantomJS() # self.driver = webdriver.Firefox() self.driver.set_window_size(1280, 800) # Start up celery worker for testing self.celery_worker = start_worker(app) self.celery_worker.__enter__()
def setUpClass(cls): super().setUpClass() app = Celery('openwisp2') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() cls.celery_worker = start_worker(app)