def launch_celery_worker_environment(task_desc: TaskDescription,
redis_params: dict,
workers_per_node: int = None):
redis_port = redis_params['port']
redis_host = pbs.hostname()
redis_password = cr.get_redis_password(generate_if_missing=True)
redis_shutdown = cr.launch_redis(password=redis_password, **redis_params)
if not redis_shutdown:
raise RuntimeError('Failed to launch Redis')
_LOG.info('Launched Redis at %s:%d', redis_host, redis_port)
for i in range(5):
if cr.check_redis(redis_host, redis_port, redis_password) is False:
sleep(0.5)
executor = cr.CeleryExecutor(
redis_host,
redis_port,
password=redis_password,
)
logger_shutdown = multiprocessing.Value('b', False, lock=False)
log_proc = multiprocessing.Process(
target=_run_celery_task_logging,
args=(logger_shutdown, cr.app, task_desc)
)
log_proc.start()
worker_procs = list(_spawn_pbs_workers(redis_host,
redis_port,
workers_per_node))
_LOG.info('%d workers launched.', len(worker_procs))
def start_shutdown():
cr.app.control.shutdown()
def shutdown():
start_shutdown()
_LOG.info('Waiting for workers to quit')
# TODO: time limit followed by kill
for i, proc in enumerate(worker_procs):
_LOG.debug("Waiting on node %s", i)
proc.wait()
# We deliberately don't want to stop the logger until all worker have stopped completely,
# to capture all of their logs.
_LOG.info('Stopping log process')
logger_shutdown.value = True
log_proc.join()
_LOG.info('Shutting down redis-server')
redis_shutdown()
return executor, shutdown
def test_launch_redis_with_config_password():
redis_stop = cr.launch_redis(PORT, password='', loglevel='verbose')
assert redis_stop is not None
sleep(REDIS_WAIT)
is_running = cr.check_redis(port=PORT, password='')
assert is_running is True
redis_stop()
sleep(REDIS_WAIT)
is_running = cr.check_redis(port=PORT, password='')
assert is_running is False
def test_launch_redis_with_config_password():
is_running = cr.check_redis(port=PORT)
assert is_running is False, "Redis should not be running at the start of the test"
redis_stop = cr.launch_redis(PORT, password='', loglevel='verbose')
assert redis_stop is not None
sleep(REDIS_WAIT)
is_running = cr.check_redis(port=PORT, password='')
assert is_running is True
redis_stop()
sleep(REDIS_WAIT)
is_running = cr.check_redis(port=PORT, password='')
assert is_running is False