def test_watchdog_needs_envvar(envname): try: with env.unset_env([envname]): handler_watchdog(None, None) except AssertionError as err: assert 'envvar should be set' in str(err) else: raise RuntimeError('Had not raised an AssertionError')
def test_watchdog_shutdown(monkeypatch): queue_url = 'redis://' _env = dict( CELERY_SERVERLESS_QUEUE_URL=queue_url, CELERY_SERVERLESS_LOCK_URL=queue_url, CELERY_SERVERLESS_INTERCOM_URL=queue_url, ) redis = pytest.importorskip('redis') from redis.exceptions import ConnectionError conn = redis.StrictRedis.from_url(queue_url) try: conn.ping() except ConnectionError as err: pytest.skip('Redis server is not available: %s' % err) monkeypatch.setattr( 'celery_serverless.watchdog.Watchdog.get_workers_count', lambda self: 1, ) _shutdown_requested_responses = [False, False, False, True, 'success'] monkeypatch.setattr( 'celery_serverless.watchdog.Watchdog.is_shutdown_requested', lambda self: _shutdown_requested_responses.pop(0), ) with env.set_env(**_env): response = handler_watchdog(None, None) assert _shutdown_requested_responses == [ 'success' ], 'handler not stopped with `is_shutdown_requested() = True`'
def test_watchdog_handler_minimal_call(): _env = dict( CELERY_SERVERLESS_QUEUE_URL='disabled', CELERY_SERVERLESS_LOCK_URL='dummy_threading://', CELERY_SERVERLESS_INTERCOM_URL='disabled', ) with env.set_env(**_env): response = handler_watchdog(None, None) assert response
def test_watchdog_monitor_redis_queues(monkeypatch): queue_url = 'redis://' queue_name = 'celery' redis = pytest.importorskip('redis') from redis.exceptions import ConnectionError conn = redis.StrictRedis.from_url(queue_url) try: conn.ping() except ConnectionError as err: pytest.skip('Redis server is not available: %s' % err) # Redis is available. # Lets set it up before test the watchdog def _simulate_worker_invocation(*args, **kwargs): """ Simulates a Worker invocation cycle via Redis keys changes """ ## Expects kwargs{} maybe containing: # 'data': { # 'worker_id': '2a3dc6a4-78a7-11e8-86e7-4c32758cbd8b', # 'worker_trigger_time': 1529951999.836363, # 'prefix': 'celery_serverless:watchdog:worker:', # } ## logger.warning('Simulating an Worker invocation: START') watchdog_data = kwargs.get('data', {}) worker_id = watchdog_data.get('worker_id', str(uuid.uuid1())) worker_prefix = watchdog_data.get('prefix', None) # Worker takes some time to init, then notify the Cache time.sleep(2) # celeryd_init hook happens at this point # Worker connecting to the broker time.sleep(2) # worker_ready hook happens at this point # Delay to acquire the task time.sleep(0.5) conn.rpop(queue_name) # Simulate task removal from the queue. # task_prerun hook happens at this point watchdog.inform_worker_busy( conn, worker_id, prefix=worker_prefix) # Moves itself from "started" to "working". # Worker got a job. Started working time.sleep(2) # Worker finished the job. # task_postrun hook happens at this point watchdog.inform_worker_leave( conn, worker_id, prefix=worker_prefix) # Unsubscribe itself from "working" list. logger.warning('Simulating an Worker invocation: END') conn.flushdb() assert watchdog._get_workers_count( conn) == 0, 'The redis is not starting empty' _simulate_worker_invocation() # Just be sure that it works. assert watchdog._get_workers_count( conn) == 0, 'Worker simulation is not informing its finish.' jobs = ['one', 'two', 'three'] with conn.pipeline() as pipe: pipe.delete(queue_name) pipe.lpush(queue_name, *jobs) pipe.llen(queue_name) pipe_result = pipe.execute() assert pipe_result[-1] == 3, 'Are our Redis misbehaving or something?' with ThreadPoolExecutor() as executor: monkeypatch.setattr( 'celery_serverless.watchdog.invoke_worker', lambda data: (True, executor.submit(_simulate_worker_invocation, data=data)), ) _env = dict( CELERY_SERVERLESS_QUEUE_URL=queue_url, CELERY_SERVERLESS_LOCK_URL=queue_url, CELERY_SERVERLESS_INTERCOM_URL=queue_url, ) with env.set_env(**_env): response = handler_watchdog(None, None) assert response assert conn.llen( queue_name) == 0, 'Watchdog finished but the queue is not empty' # Should I really test for this? assert watchdog._get_workers_count( conn) == 0, 'Watchdog finished but not the workers'