def main():
logger.info("Running abaco health checks. Now: {}".format(time.time()))
try:
clean_up_ipc_dirs()
except Exception as e:
logger.error("Got exception from clean_up_ipc_dirs: {}".format(e))
try:
ttl = Config.get('workers', 'worker_ttl')
except Exception as e:
logger.error(
"Could not get worker_ttl config. Exception: {}".format(e))
if not container_running(name='spawner*'):
logger.critical("No spawners running! Launching new spawner..")
command = 'python3 -u /actors/spawner.py'
# check logging strategy to determine log file name:
try:
run_container_with_docker(
AE_IMAGE,
command,
name='abaco_spawner_0',
environment={'AE_IMAGE': AE_IMAGE.split(':')[0]},
mounts=[],
log_file=None)
except Exception as e:
logger.critical(
"Could not restart spawner. Exception: {}".format(e))
try:
ttl = int(ttl)
except Exception as e:
logger.error("Invalid ttl config: {}. Setting to -1.".format(e))
ttl = -1
ids = get_actor_ids()
logger.info("Found {} actor(s). Now checking status.".format(len(ids)))
for id in ids:
check_workers(id, ttl)
def main():
print("Running abaco health checks. Now: {}".format(time.time()))
ttl = Config.get('workers', 'worker_ttl')
if not container_running(name='spawner*'):
print("No spawners running! Launching new spawner..")
command = 'python3 -u /actors/spawner.py'
run_container_with_docker(AE_IMAGE, command, name='abaco_spawner_0', environment={'AE_IMAGE': AE_IMAGE})
try:
ttl = int(ttl)
except Exception:
ttl = -1
ids = get_actor_ids()
print("Found {} actor(s). Now checking status.".format(len(ids)))
for id in ids:
check_workers(id, ttl)
def main():
print("Running abaco health checks. Now: {}".format(time.time()))
ttl = Config.get('workers', 'worker_ttl')
if not container_running(name='spawner*'):
print("No spawners running! Launching new spawner..")
command = 'python3 -u /actors/spawner.py'
run_container_with_docker(AE_IMAGE, command, name='abaco_spawner_0', environment={'AE_IMAGE': AE_IMAGE})
try:
ttl = int(ttl)
except Exception:
ttl = -1
ids = get_actor_ids()
print("Found {} actor(s). Now checking status.".format(len(ids)))
for id in ids:
check_workers(id, ttl)
def check_spawner(queue):
"""
Check the health and existence of a spawner on this host for a particular queue.
:param queue: (str) - the queue to check on.
:return:
"""
logger.debug("top of check_spawner for queue: {}".format(queue))
# spawner container names by convention should have the format <project>_<queue>_spawner_<count>; for example
# abaco_default_spawner_2.
# so, we look for container names containing a string with that format:
spawner_name_segment = '{}_spawner'.format(queue)
if not container_running(name=spawner_name_segment):
logger.critical(
"No spawners running for queue {}! Launching new spawner..".format(
queue))
start_spawner(queue)
else:
logger.debug("spawner for queue {} already running.".format(queue))