def process_worker_ch(tenant, worker_ch, actor_id, worker_id, actor_ch, ag_client):
""" Target for a thread to listen on the worker channel for a message to stop processing.
:param worker_ch:
:return:
"""
global keep_running
logger.info("Worker subscribing to worker channel...{}_{}".format(actor_id, worker_id))
while keep_running:
try:
msg, msg_obj = worker_ch.get_one()
except Exception as e:
logger.error(f"worker {worker_id} got exception trying to read the worker channel! "
f"sleeping for 10 seconds and then will try again; e: {e}")
time.sleep(10)
continue
# receiving the message is enough to ack it - resiliency is currently handled in the calling code.
msg_obj.ack()
logger.debug("Received message in worker channel; msg: {}; {}_{}".format(msg, actor_id, worker_id))
logger.debug("Type(msg)={}".format(type(msg)))
if msg == 'status':
# this is a health check, return 'ok' to the reply_to channel.
logger.debug("received health check. updating worker_health_time.")
try:
Worker.update_worker_health_time(actor_id, worker_id)
except Exception as e:
logger.error(f"worker {worker_id} got exception trying to update its health time! "
f"sleeping for 10 seconds and then will try again; e: {e}")
time.sleep(10)
continue
elif msg == 'force_quit':
logger.info("Worker with worker_id: {} (actor_id: {}) received a force_quit message, "
"forcing the execution to halt...".format(worker_id, actor_id))
globals.force_quit = True
globals.keep_running = False
elif msg == 'stop' or msg == 'stop-no-delete':
logger.info("Worker with worker_id: {} (actor_id: {}) received stop message, "
"stopping worker...".format(worker_id, actor_id))
globals.keep_running = False
# set the worker status to SHUTTING_DOWN:
try:
Worker.update_worker_status(actor_id, worker_id, SHUTTING_DOWN)
except Exception as e:
logger.error(
f"worker got exception trying to update status to SHUTTING_DOWN. actor_id: {actor_id};"
f"worker_id: {worker_id}; exception: {e}")
# when an actor's image is updated, old workers are deleted while new workers are
# created. Deleting the actor msg channel in this case leads to race conditions
delete_actor_ch = True
if msg == 'stop-no-delete':
logger.info("Got stop-no-delete; will not delete actor_ch. {}_{}".format(actor_id, worker_id))
delete_actor_ch = False
# if a `stop` was sent, the actor is being deleted, and so we want to immediately shutdown processing.
else:
globals.force_quit = True
# first, delete an associated client
# its possible this worker was not passed a client,
# but if so, we need to delete it before shutting down.
if ag_client:
logger.info("worker {}_{} Requesting client {} be deleted.".format(actor_id, worker_id, ag_client.api_key))
secret = os.environ.get('_abaco_secret')
clients_ch = ClientsChannel()
msg = clients_ch.request_delete_client(tenant=tenant,
actor_id=actor_id,
worker_id=worker_id,
client_id=ag_client.api_key,
secret=secret)
if msg['status'] == 'ok':
logger.info("Client delete request completed successfully for "
"worker_id: {}, client_id: {}.".format(worker_id, ag_client.api_key))
else:
logger.error("Error deleting client for "
"worker_id: {}, client_id: {}. Message: {}".format(worker_id, msg['message'],
ag_client.api_key))
clients_ch.close()
else:
logger.info("Did not receive client. Not issuing delete. Exiting. {}_{}".format(actor_id, worker_id))
try:
Worker.delete_worker(actor_id, worker_id)
except WorkerException as e:
logger.info("Got WorkerException from delete_worker(). "
"worker_id: {}"
"Exception: {}".format(worker_id, e))
# delete associated channels:
# it is possible the actor channel was already deleted, in which case we just keep processing
if delete_actor_ch:
try:
actor_ch.delete()
logger.info("ActorChannel deleted for actor: {} worker_id: {}".format(actor_id, worker_id))
except Exception as e:
logger.info("Got exception deleting ActorChannel for actor: {} "
"worker_id: {}; exception: {}".format(actor_id, worker_id, e))
try:
worker_ch.delete()
logger.info("WorkerChannel deleted for actor: {} worker_id: {}".format(actor_id, worker_id))
except Exception as e:
logger.info("Got exception deleting WorkerChannel for actor: {} "
"worker_id: {}; exception: {}".format(actor_id, worker_id, e))
logger.info("Worker with worker_id: {} is now exiting.".format(worker_id))
_thread.interrupt_main()
logger.info("main thread interrupted, worker {}_{} issuing os._exit()...".format(actor_id, worker_id))
os._exit(0)
def check_workers(actor_id, ttl):
"""Check health of all workers for an actor."""
logger.info("Checking health for actor: {}".format(actor_id))
try:
workers = Worker.get_workers(actor_id)
except Exception as e:
logger.error("Got exception trying to retrieve workers: {}".format(e))
return None
logger.debug("workers: {}".format(workers))
host_id = os.environ.get('SPAWNER_HOST_ID',
Config.get('spawner', 'host_id'))
logger.debug("host_id: {}".format(host_id))
for worker in workers:
# if the worker has only been requested, it will not have a host_id.
if 'host_id' not in worker:
# @todo- we will skip for now, but we need something more robust in case the worker is never claimed.
continue
# ignore workers on different hosts
if not host_id == worker['host_id']:
continue
# first check if worker is responsive; if not, will need to manually kill
logger.info("Checking health for worker: {}".format(worker))
ch = WorkerChannel(worker_id=worker['id'])
worker_id = worker.get('id')
result = None
try:
logger.debug("Issuing status check to channel: {}".format(
worker['ch_name']))
result = ch.put_sync('status', timeout=5)
except channelpy.exceptions.ChannelTimeoutException:
logger.info(
"Worker did not respond, removing container and deleting worker."
)
try:
rm_container(worker['cid'])
except DockerError:
pass
try:
Worker.delete_worker(actor_id, worker_id)
logger.info("worker {} deleted from store".format(worker_id))
except Exception as e:
logger.error(
"Got exception trying to delete worker: {}".format(e))
# if the put_sync timed out and we removed the worker, we also need to delete the channel
# otherwise the un-acked message will remain.
try:
ch.delete()
except Exception as e:
logger.error(
"Got exception: {} while trying to delete worker channel for worker: {}"
.format(e, worker_id))
finally:
try:
ch.close()
except Exception as e:
logger.error(
"Got an error trying to close the worker channel for dead worker. Exception: {}"
.format(e))
if result and not result == 'ok':
logger.error(
"Worker responded unexpectedly: {}, deleting worker.".format(
result))
try:
rm_container(worker['cid'])
Worker.delete_worker(actor_id, worker_id)
except Exception as e:
logger.error(
"Got error removing/deleting worker: {}".format(e))
else:
# worker is healthy so update last health check:
Worker.update_worker_health_time(actor_id, worker_id)
logger.info("Worker ok.")
# now check if the worker has been idle beyond the ttl:
if ttl < 0:
# ttl < 0 means infinite life
logger.info("Infinite ttl configured; leaving worker")
return
# we don't shut down workers that are currently running:
if not worker['status'] == codes.BUSY:
last_execution = worker.get('last_execution_time', 0)
# if worker has made zero executions, use the create_time
if last_execution == 0:
last_execution = worker.get('create_time',
datetime.datetime.min)
logger.debug("using last_execution: {}".format(last_execution))
try:
assert type(last_execution) == datetime.datetime
except:
logger.error(
"Time received for TTL measurements is not of type datetime."
)
last_execution = datetime.datetime.min
if last_execution + datetime.timedelta(
seconds=ttl) < datetime.datetime.utcnow():
# shutdown worker
logger.info("Shutting down worker beyond ttl.")
shutdown_worker(actor_id, worker['id'])
else:
logger.info("Still time left for this worker.")
if worker['status'] == codes.ERROR:
# shutdown worker
logger.info("Shutting down worker in error status.")
shutdown_worker(actor_id, worker['id'])
def check_workers(actor_id, ttl):
"""Check health of all workers for an actor."""
logger.info("Checking health for actor: {}".format(actor_id))
try:
workers = Worker.get_workers(actor_id)
except Exception as e:
logger.error("Got exception trying to retrieve workers: {}".format(e))
return None
logger.debug("workers: {}".format(workers))
for _, worker in workers.items():
# if the worker has only been requested, it will not have a host_id.
if 'host_id' not in worker:
# @todo- we will skip for now, but we need something more robust in case the worker is never claimed.
continue
# ignore workers on different hosts
if not Config.get('spawner', 'host_id') == worker['host_id']:
continue
# first check if worker is responsive; if not, will need to manually kill
logger.info("Checking health for worker: {}".format(worker))
ch = WorkerChannel(worker_id=worker['id'])
worker_id = worker.get('id')
try:
logger.debug("Issuing status check to channel: {}".format(
worker['ch_name']))
result = ch.put_sync('status', timeout=5)
except channelpy.exceptions.ChannelTimeoutException:
logger.info(
"Worker did not respond, removing container and deleting worker."
)
try:
rm_container(worker['cid'])
except DockerError:
pass
try:
Worker.delete_worker(actor_id, worker_id)
except Exception as e:
logger.error(
"Got exception trying to delete worker: {}".format(e))
finally:
try:
ch.close()
except Exception as e:
logger.error(
"Got an error trying to close the worker channel for dead worker. Exception: {}"
.format(e))
if not result == 'ok':
logger.error(
"Worker responded unexpectedly: {}, deleting worker.".format(
result))
try:
rm_container(worker['cid'])
Worker.delete_worker(actor_id, worker_id)
except Exception as e:
logger.error(
"Got error removing/deleting worker: {}".format(e))
else:
# worker is healthy so update last health check:
Worker.update_worker_health_time(actor_id, worker_id)
logger.info("Worker ok.")
# now check if the worker has been idle beyond the ttl:
if ttl < 0:
# ttl < 0 means infinite life
logger.info("Infinite ttl configured; leaving worker")
return
# we don't shut down workers that are currently running:
if not worker['status'] == codes.BUSY:
last_execution = int(float(worker.get('last_execution_time', 0)))
if last_execution + ttl < time.time():
# shutdown worker
logger.info("Shutting down worker beyond ttl.")
shutdown_worker(worker['id'])
else:
logger.info("Still time left for this worker.")
elif worker['status'] == codes.ERROR:
# shutdown worker
logger.info("Shutting down worker in error status.")
shutdown_worker(worker['id'])
else:
logger.debug("Worker not in READY status, will postpone.")
