def stop_workers(self, actor_id, worker_ids):
"""Stop existing workers; used when updating an actor's image."""
logger.debug("Top of stop_workers() for actor: {}.".format(actor_id))
try:
workers_list = workers_store.items({'actor_id': actor_id})
except KeyError:
logger.debug("workers_store had no workers for actor: {}".format(actor_id))
workers_list = []
# if there are existing workers, we need to close the actor message channel and
# gracefully shutdown the existing worker processes.
if len(workers_list) > 0:
logger.info(f"Found {len(workers_list)} workers to stop.")
# first, close the actor msg channel to prevent any new messages from being pulled
# by the old workers.
actor_ch = ActorMsgChannel(actor_id)
actor_ch.close()
logger.info("Actor channel closed for actor: {}".format(actor_id))
# now, send messages to workers for a graceful shutdown:
for worker in workers_list:
# don't stop the new workers:
if worker['id'] not in worker_ids:
ch = WorkerChannel(worker_id=worker['id'])
# since this is an update, there are new workers being started, so
# don't delete the actor msg channel:
ch.put('stop-no-delete')
logger.info(f"Sent 'stop-no-delete' message to worker_id: {worker['id']}")
ch.close()
else:
logger.debug("skipping worker {worker} as it it not in worker_ids.")
else:
logger.info("No workers to stop.")
def get_worker(wid):
"""
Check to see if a string `wid` is the id of a worker in the worker store.
If so, return it; if not, return None.
"""
worker = workers_store.items({'id': wid})
if worker:
return worker
return None
def shutdown_all_workers():
"""
Utility function for properly shutting down all existing workers.
This function is useful when deploying a new version of the worker code.
"""
# iterate over the workers_store directly, not the actors_store, since there could be data integrity issue.
for actor_id, worker in workers_store.items():
# call check_workers with ttl = 0 so that all will be shut down:
check_workers(actor_id, 0)
def get_tot_workers(self):
logger.debug("top of get_tot_workers")
self.tot_workers = 0
logger.debug('spawner host_id: {}'.format(self.host_id))
for k,v in workers_store.items():
for wid, worker in v.items():
if worker.get('host_id') == self.host_id:
self.tot_workers += 1
logger.debug("returning total workers: {}".format(self.tot_workers))
return self.tot_workers
def get_worker(wid):
"""
Check to see if a string `wid` is the id of a worker in the worker store.
If so, return it; if not, return None.
"""
for actor_id, value in workers_store.items():
for worker_id, worker in value.items():
if worker_id == wid:
return worker
return None
def zero_out_workers_db():
"""
Set all workers collections in the db to empty. Run this as part of a maintenance; steps:
1) remove all docker containers
2) run this function
3) run clean_up_clients_store().
:return:
"""
for k, _ in workers_store.items():
workers_store[k] = {}
def zero_out_workers_db():
"""
Set all workers collections in the db to empty. Run this as part of a maintenance; steps:
1) remove all docker containers
2) run this function
3) run clean_up_apim_clients().
4) run zero_out_clients_db()
:return:
"""
for worker in workers_store.items(proj_inp=None):
del workers_store[worker['_id']]
def get_tot_workers(self):
logger.debug("top of get_tot_workers")
self.tot_workers = 0
logger.debug('spawner host_id: {}'.format(self.host_id))
for worker_info in workers_store.items():
try:
if worker_info['host_id'] == self.host_id:
self.tot_workers += 1
except KeyError:
continue
logger.debug("returning total workers: {}".format(self.tot_workers))
return self.tot_workers
def shutdown_all_workers():
"""
Utility function for properly shutting down all existing workers.
This function is useful when deploying a new version of the worker code.
"""
# iterate over the workers_store directly, not the actors_store, since there could be data integrity issue.
logger.debug("Top of shutdown_all_workers.")
actors_with_workers = set()
for worker in workers_store.items():
actors_with_workers.add(worker['actor_id'])
for actor_id in actors_with_workers:
check_workers(actor_id, 0)
def check_workers_store(ttl):
logger.debug("Top of check_workers_store.")
"""Run through all workers in workers_store and ensure there is no data integrity issue."""
for worker in workers_store.items():
aid = worker['actor_id']
check_worker_health(aid, worker, ttl)
