def launch_celery_worker_environment(task_desc: TaskDescription, redis_params: dict, workers_per_node: int = None): redis_port = redis_params['port'] redis_host = pbs.hostname() redis_password = cr.get_redis_password(generate_if_missing=True) redis_shutdown = cr.launch_redis(password=redis_password, **redis_params) if not redis_shutdown: raise RuntimeError('Failed to launch Redis') _LOG.info('Launched Redis at %s:%d', redis_host, redis_port) for i in range(5): if cr.check_redis(redis_host, redis_port, redis_password) is False: sleep(0.5) executor = cr.CeleryExecutor( redis_host, redis_port, password=redis_password, ) logger_shutdown = multiprocessing.Value('b', False, lock=False) log_proc = multiprocessing.Process( target=_run_celery_task_logging, args=(logger_shutdown, cr.app, task_desc) ) log_proc.start() worker_procs = list(_spawn_pbs_workers(redis_host, redis_port, workers_per_node)) _LOG.info('%d workers launched.', len(worker_procs)) def start_shutdown(): cr.app.control.shutdown() def shutdown(): start_shutdown() _LOG.info('Waiting for workers to quit') # TODO: time limit followed by kill for i, proc in enumerate(worker_procs): _LOG.debug("Waiting on node %s", i) proc.wait() # We deliberately don't want to stop the logger until all worker have stopped completely, # to capture all of their logs. _LOG.info('Stopping log process') logger_shutdown.value = True log_proc.join() _LOG.info('Shutting down redis-server') redis_shutdown() return executor, shutdown
def test_launch_redis_with_config_password(): redis_stop = cr.launch_redis(PORT, password='', loglevel='verbose') assert redis_stop is not None sleep(REDIS_WAIT) is_running = cr.check_redis(port=PORT, password='') assert is_running is True redis_stop() sleep(REDIS_WAIT) is_running = cr.check_redis(port=PORT, password='') assert is_running is False
def test_launch_redis_with_config_password(): is_running = cr.check_redis(port=PORT) assert is_running is False, "Redis should not be running at the start of the test" redis_stop = cr.launch_redis(PORT, password='', loglevel='verbose') assert redis_stop is not None sleep(REDIS_WAIT) is_running = cr.check_redis(port=PORT, password='') assert is_running is True redis_stop() sleep(REDIS_WAIT) is_running = cr.check_redis(port=PORT, password='') assert is_running is False