def q_application():
""" Initialise a QT application with a Xvfb server
"""
try:
from PyQt4 import QtGui
except ImportError:
pytest.skip('PyQT4 not installed, skipping test')
global TestQtApp
assert hasattr(TestQtApp, 'app'), "Can only initialize QApplication once per process"
if TestQtApp.app is None:
# TODO: investigate if this is still the case, if not just use the regular xvfb_server fixture
if 'PYDEV_CONSOLE_ENCODING' in os.environ:
# PyDev destroys session scoped fixtures after each test, so we can't clean up the XvfbServer
global server
server = XvfbServer()
with set_env(XAUTHORITY=server.authfile, DISPLAY=server.display):
TestQtApp.app = QtGui.QApplication([__name__, '-display', server.display])
yield TestQtApp
else:
with XvfbServer() as server:
with set_env(XAUTHORITY=server.authfile, DISPLAY=server.display):
TestQtApp.app = QtGui.QApplication([__name__, '-display', server.display])
yield TestQtApp
TestQtApp.app.exit()
del TestQtApp.app
gc.collect()
else:
yield TestQtApp
def test_set_env_ok_if_not_exists():
if TEMP_NAME in os.environ:
del os.environ[TEMP_NAME]
with env.set_env(TEMP_NAME, "anything"):
assert os.environ[TEMP_NAME] == "anything"
assert TEMP_NAME not in os.environ
def test_set_env_ok_if_exists():
ev = os.environ[TEMP_NAME] = "junk_name"
try:
with env.set_env(TEMP_NAME, "not_junk"):
assert os.environ[TEMP_NAME] == "not_junk"
assert os.environ[TEMP_NAME] == ev
finally:
del os.environ[TEMP_NAME]
def test_PYTHONPATH_not_present_in_testing_env_if_set():
with env.set_env('PYTHONPATH', 'fred'):
with mock.patch.object(venv.Workspace, 'run') as run:
venv.VirtualEnv()
call = run.mock_calls[0]
assert 'PYTHONPATH' not in call[2]['env']
venv.VirtualEnv({'PYTHONPATH': 'john'})
call = run.mock_calls[1]
assert 'PYTHONPATH' not in call[2]['env']
def test_set_env_with_kwargs_updates():
test_env = {"TEST_ACME_TESTING_A": "a", "TEST_ACME_TESTING_B": "b", "TEST_ACME_TESTING_C": "c"}
os.environ.update(test_env)
with env.set_env("TEST_ACME_TESTING_A", 1, TEST_ACME_TESTING_B="fred", TEST_ACME_TESTING_C=None):
assert os.environ["TEST_ACME_TESTING_A"] == "1"
assert os.environ["TEST_ACME_TESTING_B"] == "fred"
assert "C" not in os.environ
assert os.environ["TEST_ACME_TESTING_A"] == "a"
assert os.environ["TEST_ACME_TESTING_B"] == "b"
assert os.environ["TEST_ACME_TESTING_C"] == "c"
def test_set_env_ok_if_not_exists():
if TEMP_NAME in os.environ:
del os.environ[TEMP_NAME]
with env.set_env(TEMP_NAME, 'anything'):
out = run.run('env', capture_stdout=True)
for o in out.split('\n'):
if o.startswith(TEMP_NAME):
assert o == '%s=anything' % TEMP_NAME
break
else:
assert False, '%s not found in os.environ' % TEMP_NAME
def test_set_env_ok_if_exists():
ev = os.environ[TEMP_NAME] = 'junk_name'
try:
with env.set_env(TEMP_NAME, 'anything'):
out = run.run('env', capture_stdout=True)
for o in out.split('\n'):
if o.startswith(TEMP_NAME):
assert o == '%s=anything' % TEMP_NAME
break
else:
assert False, '%s not found in os.environ' % TEMP_NAME
assert os.environ[TEMP_NAME] == ev
finally:
if TEMP_NAME in os.environ:
del os.environ[TEMP_NAME]
def test_set_env_with_kwargs_updates():
test_env = {
"TEST_ACME_TESTING_A": "a",
"TEST_ACME_TESTING_B": "b",
"TEST_ACME_TESTING_C": "c"
}
os.environ.update(test_env)
with env.set_env("TEST_ACME_TESTING_A",
1,
TEST_ACME_TESTING_B="fred",
TEST_ACME_TESTING_C=None):
assert os.environ["TEST_ACME_TESTING_A"] == "1"
assert os.environ["TEST_ACME_TESTING_B"] == "fred"
assert "C" not in os.environ
assert os.environ["TEST_ACME_TESTING_A"] == "a"
assert os.environ["TEST_ACME_TESTING_B"] == "b"
assert os.environ["TEST_ACME_TESTING_C"] == "c"
def oembed_providers(files_dir):
providers_filepath = str(files_dir / 'test_providers.json')
with env.set_env(oembed.OEMBED_PROVIDERS_ENV_VAR_NAME, providers_filepath):
return oembed.prepare_providers(oembed.load_providers())
def test_watchdog_monitor_redis_queues(monkeypatch):
lock_url = 'redis://'
redis = pytest.importorskip('redis')
from redis.exceptions import ConnectionError
conn = redis.StrictRedis.from_url(lock_url)
try:
conn.ping()
except ConnectionError as err:
pytest.skip('Redis server is not available: %s' % err)
# Redis is available.
# Lets set it up before test the watchdog
times_invoked = 0
def _simulate_watchdog_invocation(*args, **kwargs):
"""
Simulates a Watchdog invocation cycle via Redis locks changes
"""
logger.warning('Simulating an Watchdog invocation: START')
nonlocal times_invoked
times_invoked += 1
# 1) Watchdog fetches its lock
lock, lock_name = watchdog.get_watchdog_lock()
# 2) It runs with the lock or cancels
with_lock = lock.acquire(False)
if not with_lock:
logger.info('Watchdog COULD NOT got the lock')
else:
logger.info('Watchdog GOT the lock')
time.sleep(5)
lock.release()
logger.info('Watchdog RELEASED the lock')
logger.warning('Simulating an Watchdog invocation: END')
conn.flushdb()
with env.set_env(CELERY_SERVERLESS_LOCK_URL=lock_url):
_simulate_watchdog_invocation() # Just be sure that it works.
with ThreadPoolExecutor() as executor:
monkeypatch.setattr(
'celery_serverless.invoker.invoke',
lambda *ar, **kw:
(True, executor.submit(_simulate_watchdog_invocation)),
)
client_futures = []
times_invoked = 0
with env.set_env(CELERY_SERVERLESS_LOCK_URL=lock_url,
CELERY_SERVERLESS_CLIENT_LOCK_URL=lock_url):
for i in range(20):
client_futures.append(
executor.submit(invoker.client_invoke_watchdog))
client_invokes = [
fut.result() for fut in as_completed(client_futures)
]
result_flags, results = zip(*client_invokes)
assert times_invoked == 1, 'More than one client succeeded to invoke_watchdog'
assert len([i for i in result_flags
if i == True]) == 1, 'More than one client got a lock'
def test_oembed_load_exception():
with env.set_env(oembed.OEMBED_PROVIDERS_ENV_VAR_NAME, 'test.json'):
with pytest.raises(oembed.LoadOembedProvidersException):
oembed.load_providers()
def test_worker_handler_minimal_call():
with env.set_env(CELERY_SERVERLESS_INTERCOM_URL='disabled'):
response = handler_worker(None, None)
assert response
def test_watchdog_monitor_redis_queues(monkeypatch):
queue_url = 'redis://'
queue_name = 'celery'
redis = pytest.importorskip('redis')
from redis.exceptions import ConnectionError
conn = redis.StrictRedis.from_url(queue_url)
try:
conn.ping()
except ConnectionError as err:
pytest.skip('Redis server is not available: %s' % err)
# Redis is available.
# Lets set it up before test the watchdog
def _simulate_worker_invocation(*args, **kwargs):
"""
Simulates a Worker invocation cycle via Redis keys changes
"""
## Expects kwargs{} maybe containing:
# 'data': {
# 'worker_id': '2a3dc6a4-78a7-11e8-86e7-4c32758cbd8b',
# 'worker_trigger_time': 1529951999.836363,
# 'prefix': 'celery_serverless:watchdog:worker:',
# }
##
logger.warning('Simulating an Worker invocation: START')
watchdog_data = kwargs.get('data', {})
worker_id = watchdog_data.get('worker_id', str(uuid.uuid1()))
worker_prefix = watchdog_data.get('prefix', None)
# Worker takes some time to init, then notify the Cache
time.sleep(2)
# celeryd_init hook happens at this point
# Worker connecting to the broker
time.sleep(2)
# worker_ready hook happens at this point
# Delay to acquire the task
time.sleep(0.5)
conn.rpop(queue_name) # Simulate task removal from the queue.
# task_prerun hook happens at this point
watchdog.inform_worker_busy(
conn, worker_id,
prefix=worker_prefix) # Moves itself from "started" to "working".
# Worker got a job. Started working
time.sleep(2)
# Worker finished the job.
# task_postrun hook happens at this point
watchdog.inform_worker_leave(
conn, worker_id,
prefix=worker_prefix) # Unsubscribe itself from "working" list.
logger.warning('Simulating an Worker invocation: END')
conn.flushdb()
assert watchdog._get_workers_count(
conn) == 0, 'The redis is not starting empty'
_simulate_worker_invocation() # Just be sure that it works.
assert watchdog._get_workers_count(
conn) == 0, 'Worker simulation is not informing its finish.'
jobs = ['one', 'two', 'three']
with conn.pipeline() as pipe:
pipe.delete(queue_name)
pipe.lpush(queue_name, *jobs)
pipe.llen(queue_name)
pipe_result = pipe.execute()
assert pipe_result[-1] == 3, 'Are our Redis misbehaving or something?'
with ThreadPoolExecutor() as executor:
monkeypatch.setattr(
'celery_serverless.watchdog.invoke_worker',
lambda data:
(True, executor.submit(_simulate_worker_invocation, data=data)),
)
_env = dict(
CELERY_SERVERLESS_QUEUE_URL=queue_url,
CELERY_SERVERLESS_LOCK_URL=queue_url,
CELERY_SERVERLESS_INTERCOM_URL=queue_url,
)
with env.set_env(**_env):
response = handler_watchdog(None, None)
assert response
assert conn.llen(
queue_name) == 0, 'Watchdog finished but the queue is not empty'
# Should I really test for this?
assert watchdog._get_workers_count(
conn) == 0, 'Watchdog finished but not the workers'
