def main(worker_ch_name, image):
worker_ch = WorkerChannel(name=worker_ch_name)
# first, attempt to pull image from docker hub:
try:
print("Worker pulling image {}...".format(image))
pull_image(image)
except DockerError as e:
# return a message to the spawner that there was an error pulling image and abort
worker_ch.put({'status': 'error', 'msg': str(e)})
raise e
# inform spawner that image pulled successfully
print("Image pulled successfully")
# wait to receive message from spawner that it is time to subscribe to the actor channel
print("Worker waiting on message from spawner...")
result = worker_ch.put_sync({'status': 'ok'})
if result['status'] == 'error':
print("Worker received error message from spawner: {}. Quiting...".format(str(result)))
raise WorkerException(str(result))
actor_id = result.get('actor_id')
tenant = result.get('tenant')
print("Worker received ok from spawner. Message: {}, actor_id:{}".format(result, actor_id))
api_server = None
client_id = None
client_secret = None
access_token = None
refresh_token = None
if result.get('client') == 'yes':
api_server = result.get('api_server')
client_id = result.get('client_id')
client_secret = result.get('client_secret')
access_token = result.get('access_token')
refresh_token = result.get('refresh_token')
else:
print("Did not get client:yes, got client:{}".format(result.get('client')))
Actor.set_status(actor_id, READY)
subscribe(tenant,
actor_id,
api_server,
client_id,
client_secret,
access_token,
refresh_token,
worker_ch)
def check_workers(actor_id, ttl):
"""Check health of all workers for an actor."""
print("Checking health for actors: {}".format(actor_id))
workers = Worker.get_workers(actor_id)
print("workers: {}".format(workers))
for _, worker in workers.items():
# if the worker has only been requested, it will not have a host_id.
if 'host_id' not in worker:
# @todo- we will skip for now, but we need something more robust in case the worker is never claimed.
continue
# ignore workers on different hosts
if not Config.get('spawner', 'host_id') == worker['host_id']:
continue
# first check if worker is responsive; if not, will need to manually kill
print("Checking health for worker: {}".format(worker))
ch = WorkerChannel(name=worker['ch_name'])
try:
print("Issuing status check to channel: {}".format(worker['ch_name']))
result = ch.put_sync('status', timeout=5)
except channelpy.exceptions.ChannelTimeoutException:
print("Worker did not respond, removing container and deleting worker.")
try:
rm_container(worker['cid'])
except DockerError:
pass
Worker.delete_worker(actor_id, worker['ch_name'])
continue
if not result == 'ok':
print("Worker responded unexpectedly: {}, deleting worker.".format(result))
rm_container(worker['cid'])
Worker.delete_worker(actor_id, worker['ch_name'])
else:
print("Worker ok.")
# now check if the worker has been idle beyond the ttl:
if ttl < 0:
# ttl < 0 means infinite life
print("Infinite ttl configured; leaving worker")
return
if worker['status'] == codes.READY and \
worker['last_execution'] + ttl < time.time():
# shutdown worker
print("Shutting down worker beyond ttl.")
shutdown_worker(worker['ch_name'])
else:
print("Worker still has life.")
def main(worker_ch_name, worker_id, image):
worker_ch = WorkerChannel(name=worker_ch_name)
# first, attempt to pull image from docker hub:
try:
print("Worker pulling image {}...".format(image))
pull_image(image)
except DockerError as e:
# return a message to the spawner that there was an error pulling image and abort
worker_ch.put({'status': 'error', 'msg': str(e)})
raise e
# inform spawner that image pulled successfully
print("Image pulled successfully")
# wait to receive message from spawner that it is time to subscribe to the actor channel
print("Worker waiting on message from spawner...")
result = worker_ch.put_sync({'status': 'ok'})
if result['status'] == 'error':
print("Worker received error message from spawner: {}. Quiting...".
format(str(result)))
raise WorkerException(str(result))
actor_id = result.get('actor_id')
tenant = result.get('tenant')
print("Worker received ok from spawner. Message: {}, actor_id:{}".format(
result, actor_id))
api_server = None
client_id = None
client_secret = None
access_token = None
refresh_token = None
if result.get('client') == 'yes':
api_server = result.get('api_server')
client_id = result.get('client_id')
client_secret = result.get('client_secret')
access_token = result.get('access_token')
refresh_token = result.get('refresh_token')
else:
print("Did not get client:yes, got client:{}".format(
result.get('client')))
Actor.set_status(actor_id, READY)
subscribe(tenant, actor_id, worker_id, api_server, client_id,
client_secret, access_token, refresh_token, worker_ch)
def check_workers(actor_id, ttl):
"""Check health of all workers for an actor."""
print("Checking health for actors: {}".format(actor_id))
workers = Worker.get_workers(actor_id)
print("workers: {}".format(workers))
for _, worker in workers.items():
# ignore workers on different hosts
if not Config.get('spawner', 'host_id') == worker['host_id']:
continue
# first check if worker is responsive; if not, will need to manually kill
print("Checking health for worker: {}".format(worker))
ch = WorkerChannel(name=worker['ch_name'])
try:
print("Issuing status check to channel: {}".format(worker['ch_name']))
result = ch.put_sync('status', timeout=5)
except channelpy.exceptions.ChannelTimeoutException:
print("Worker did not respond, removing container and deleting worker.")
try:
rm_container(worker['cid'])
except DockerError:
pass
Worker.delete_worker(actor_id, worker['ch_name'])
continue
if not result == 'ok':
print("Worker responded unexpectedly: {}, deleting worker.".format(result))
rm_container(worker['cid'])
Worker.delete_worker(actor_id, worker['ch_name'])
else:
print("Worker ok.")
# now check if the worker has been idle beyond the ttl:
if ttl < 0:
# ttl < 0 means infinite life
print("Infinite ttl configured; leaving worker")
return
if worker['status'] == codes.READY and \
worker['last_execution'] + ttl < time.time():
# shutdown worker
print("Shutting down worker beyond ttl.")
shutdown_worker(worker['ch_name'])
else:
print("Worker still has life.")
def main(worker_ch_name, image):
worker_ch = WorkerChannel(name=worker_ch_name)
# first, attempt to pull image from docker hub:
try:
print("Worker pulling image {}...".format(image))
pull_image(image)
except DockerError as e:
# return a message to the spawner that there was an error pulling image and abort
worker_ch.put({"status": "error", "msg": str(e)})
raise e
# inform spawner that image pulled successfully
print("Image pulled successfully")
# wait to receive message from spawner that it is time to subscribe to the actor channel
print("Worker waiting on message from spawner...")
result = worker_ch.put_sync({"status": "ok"})
if result["status"] == "error":
print("Worker received error message from spawner: {}. Quiting...".format(str(result)))
raise WorkerException(str(result))
actor_id = result.get("actor_id")
print("Worker received ok from spawner. Message: {}, actor_id:{}".format(result, actor_id))
Actor.set_status(actor_id, READY)
subscribe(actor_id, worker_ch)
def check_workers(actor_id, ttl):
"""Check health of all workers for an actor."""
logger.info("Checking health for actor: {}".format(actor_id))
try:
workers = Worker.get_workers(actor_id)
except Exception as e:
logger.error("Got exception trying to retrieve workers: {}".format(e))
return None
logger.debug("workers: {}".format(workers))
host_id = os.environ.get('SPAWNER_HOST_ID',
Config.get('spawner', 'host_id'))
logger.debug("host_id: {}".format(host_id))
for worker in workers:
# if the worker has only been requested, it will not have a host_id.
if 'host_id' not in worker:
# @todo- we will skip for now, but we need something more robust in case the worker is never claimed.
continue
# ignore workers on different hosts
if not host_id == worker['host_id']:
continue
# first check if worker is responsive; if not, will need to manually kill
logger.info("Checking health for worker: {}".format(worker))
ch = WorkerChannel(worker_id=worker['id'])
worker_id = worker.get('id')
result = None
try:
logger.debug("Issuing status check to channel: {}".format(
worker['ch_name']))
result = ch.put_sync('status', timeout=5)
except channelpy.exceptions.ChannelTimeoutException:
logger.info(
"Worker did not respond, removing container and deleting worker."
)
try:
rm_container(worker['cid'])
except DockerError:
pass
try:
Worker.delete_worker(actor_id, worker_id)
logger.info("worker {} deleted from store".format(worker_id))
except Exception as e:
logger.error(
"Got exception trying to delete worker: {}".format(e))
# if the put_sync timed out and we removed the worker, we also need to delete the channel
# otherwise the un-acked message will remain.
try:
ch.delete()
except Exception as e:
logger.error(
"Got exception: {} while trying to delete worker channel for worker: {}"
.format(e, worker_id))
finally:
try:
ch.close()
except Exception as e:
logger.error(
"Got an error trying to close the worker channel for dead worker. Exception: {}"
.format(e))
if result and not result == 'ok':
logger.error(
"Worker responded unexpectedly: {}, deleting worker.".format(
result))
try:
rm_container(worker['cid'])
Worker.delete_worker(actor_id, worker_id)
except Exception as e:
logger.error(
"Got error removing/deleting worker: {}".format(e))
else:
# worker is healthy so update last health check:
Worker.update_worker_health_time(actor_id, worker_id)
logger.info("Worker ok.")
# now check if the worker has been idle beyond the ttl:
if ttl < 0:
# ttl < 0 means infinite life
logger.info("Infinite ttl configured; leaving worker")
return
# we don't shut down workers that are currently running:
if not worker['status'] == codes.BUSY:
last_execution = worker.get('last_execution_time', 0)
# if worker has made zero executions, use the create_time
if last_execution == 0:
last_execution = worker.get('create_time',
datetime.datetime.min)
logger.debug("using last_execution: {}".format(last_execution))
try:
assert type(last_execution) == datetime.datetime
except:
logger.error(
"Time received for TTL measurements is not of type datetime."
)
last_execution = datetime.datetime.min
if last_execution + datetime.timedelta(
seconds=ttl) < datetime.datetime.utcnow():
# shutdown worker
logger.info("Shutting down worker beyond ttl.")
shutdown_worker(actor_id, worker['id'])
else:
logger.info("Still time left for this worker.")
if worker['status'] == codes.ERROR:
# shutdown worker
logger.info("Shutting down worker in error status.")
shutdown_worker(actor_id, worker['id'])
def main(worker_ch_name, worker_id, image):
"""
Main function for the worker process.
This function
"""
logger.info(
"Entering main() for worker: {}, channel: {}, image: {}".format(
worker_id, worker_ch_name, image))
worker_ch = WorkerChannel(name=worker_ch_name)
# first, attempt to pull image from docker hub:
try:
logger.info("Worker pulling image {}...".format(image))
pull_image(image)
except DockerError as e:
# return a message to the spawner that there was an error pulling image and abort
# this is not necessarily an error state: the user simply could have provided an
# image name that does not exist in the registry. This is the first time we would
# find that out.
logger.info(
"worker got a DockerError trying to pull image. Error: {}.".format(
e))
worker_ch.put({'status': 'error', 'msg': str(e)})
raise e
logger.info("Image {} pulled successfully.".format(image))
# inform spawner that image pulled successfully and, simultaneously,
# wait to receive message from spawner that it is time to subscribe to the actor channel
logger.debug("Worker waiting on message from spawner...")
result = worker_ch.put_sync({'status': 'ok'})
logger.info(
"Worker received reply from spawner. result: {}.".format(result))
if result['status'] == 'error':
# we do not expect to get an error response at this point. this needs investigation
logger.error(
"Worker received error message from spawner: {}. Quiting...".
format(str(result)))
raise WorkerException(str(result))
actor_id = result.get('actor_id')
tenant = result.get('tenant')
logger.info(
"Worker received ok from spawner. Message: {}, actor_id:{}".format(
result, actor_id))
api_server = None
client_id = None
client_secret = None
access_token = None
refresh_token = None
if result.get('client') == 'yes':
logger.info("Got client: yes, result: {}".format(result))
api_server = result.get('api_server')
client_id = result.get('client_id')
client_secret = result.get('client_secret')
access_token = result.get('access_token')
refresh_token = result.get('refresh_token')
else:
logger.info("Did not get client:yes, got result:{}".format(result))
Actor.set_status(actor_id, READY)
logger.info("Actor status set to READY. subscribing to inbox.")
subscribe(tenant, actor_id, worker_id, api_server, client_id,
client_secret, access_token, refresh_token, worker_ch)
def check_workers(actor_id, ttl):
"""Check health of all workers for an actor."""
logger.info("Checking health for actor: {}".format(actor_id))
try:
workers = Worker.get_workers(actor_id)
except Exception as e:
logger.error("Got exception trying to retrieve workers: {}".format(e))
return None
logger.debug("workers: {}".format(workers))
for _, worker in workers.items():
# if the worker has only been requested, it will not have a host_id.
if 'host_id' not in worker:
# @todo- we will skip for now, but we need something more robust in case the worker is never claimed.
continue
# ignore workers on different hosts
if not Config.get('spawner', 'host_id') == worker['host_id']:
continue
# first check if worker is responsive; if not, will need to manually kill
logger.info("Checking health for worker: {}".format(worker))
ch = WorkerChannel(worker_id=worker['id'])
worker_id = worker.get('id')
try:
logger.debug("Issuing status check to channel: {}".format(
worker['ch_name']))
result = ch.put_sync('status', timeout=5)
except channelpy.exceptions.ChannelTimeoutException:
logger.info(
"Worker did not respond, removing container and deleting worker."
)
try:
rm_container(worker['cid'])
except DockerError:
pass
try:
Worker.delete_worker(actor_id, worker_id)
except Exception as e:
logger.error(
"Got exception trying to delete worker: {}".format(e))
finally:
try:
ch.close()
except Exception as e:
logger.error(
"Got an error trying to close the worker channel for dead worker. Exception: {}"
.format(e))
if not result == 'ok':
logger.error(
"Worker responded unexpectedly: {}, deleting worker.".format(
result))
try:
rm_container(worker['cid'])
Worker.delete_worker(actor_id, worker_id)
except Exception as e:
logger.error(
"Got error removing/deleting worker: {}".format(e))
else:
# worker is healthy so update last health check:
Worker.update_worker_health_time(actor_id, worker_id)
logger.info("Worker ok.")
# now check if the worker has been idle beyond the ttl:
if ttl < 0:
# ttl < 0 means infinite life
logger.info("Infinite ttl configured; leaving worker")
return
# we don't shut down workers that are currently running:
if not worker['status'] == codes.BUSY:
last_execution = int(float(worker.get('last_execution_time', 0)))
if last_execution + ttl < time.time():
# shutdown worker
logger.info("Shutting down worker beyond ttl.")
shutdown_worker(worker['id'])
else:
logger.info("Still time left for this worker.")
elif worker['status'] == codes.ERROR:
# shutdown worker
logger.info("Shutting down worker in error status.")
shutdown_worker(worker['id'])
else:
logger.debug("Worker not in READY status, will postpone.")