def stop_workers(self, actor_id, worker_ids): """Stop existing workers; used when updating an actor's image.""" logger.debug("Top of stop_workers() for actor: {}.".format(actor_id)) try: workers_list = workers_store.items({'actor_id': actor_id}) except KeyError: logger.debug("workers_store had no workers for actor: {}".format(actor_id)) workers_list = [] # if there are existing workers, we need to close the actor message channel and # gracefully shutdown the existing worker processes. if len(workers_list) > 0: logger.info(f"Found {len(workers_list)} workers to stop.") # first, close the actor msg channel to prevent any new messages from being pulled # by the old workers. actor_ch = ActorMsgChannel(actor_id) actor_ch.close() logger.info("Actor channel closed for actor: {}".format(actor_id)) # now, send messages to workers for a graceful shutdown: for worker in workers_list: # don't stop the new workers: if worker['id'] not in worker_ids: ch = WorkerChannel(worker_id=worker['id']) # since this is an update, there are new workers being started, so # don't delete the actor msg channel: ch.put('stop-no-delete') logger.info(f"Sent 'stop-no-delete' message to worker_id: {worker['id']}") ch.close() else: logger.debug("skipping worker {worker} as it it not in worker_ids.") else: logger.info("No workers to stop.")
def get_worker(wid): """ Check to see if a string `wid` is the id of a worker in the worker store. If so, return it; if not, return None. """ worker = workers_store.items({'id': wid}) if worker: return worker return None
def shutdown_all_workers(): """ Utility function for properly shutting down all existing workers. This function is useful when deploying a new version of the worker code. """ # iterate over the workers_store directly, not the actors_store, since there could be data integrity issue. for actor_id, worker in workers_store.items(): # call check_workers with ttl = 0 so that all will be shut down: check_workers(actor_id, 0)
def get_tot_workers(self): logger.debug("top of get_tot_workers") self.tot_workers = 0 logger.debug('spawner host_id: {}'.format(self.host_id)) for k,v in workers_store.items(): for wid, worker in v.items(): if worker.get('host_id') == self.host_id: self.tot_workers += 1 logger.debug("returning total workers: {}".format(self.tot_workers)) return self.tot_workers
def get_worker(wid): """ Check to see if a string `wid` is the id of a worker in the worker store. If so, return it; if not, return None. """ for actor_id, value in workers_store.items(): for worker_id, worker in value.items(): if worker_id == wid: return worker return None
def zero_out_workers_db(): """ Set all workers collections in the db to empty. Run this as part of a maintenance; steps: 1) remove all docker containers 2) run this function 3) run clean_up_clients_store(). :return: """ for k, _ in workers_store.items(): workers_store[k] = {}
def zero_out_workers_db(): """ Set all workers collections in the db to empty. Run this as part of a maintenance; steps: 1) remove all docker containers 2) run this function 3) run clean_up_apim_clients(). 4) run zero_out_clients_db() :return: """ for worker in workers_store.items(proj_inp=None): del workers_store[worker['_id']]
def get_tot_workers(self): logger.debug("top of get_tot_workers") self.tot_workers = 0 logger.debug('spawner host_id: {}'.format(self.host_id)) for worker_info in workers_store.items(): try: if worker_info['host_id'] == self.host_id: self.tot_workers += 1 except KeyError: continue logger.debug("returning total workers: {}".format(self.tot_workers)) return self.tot_workers
def shutdown_all_workers(): """ Utility function for properly shutting down all existing workers. This function is useful when deploying a new version of the worker code. """ # iterate over the workers_store directly, not the actors_store, since there could be data integrity issue. logger.debug("Top of shutdown_all_workers.") actors_with_workers = set() for worker in workers_store.items(): actors_with_workers.add(worker['actor_id']) for actor_id in actors_with_workers: check_workers(actor_id, 0)
def check_workers_store(ttl): logger.debug("Top of check_workers_store.") """Run through all workers in workers_store and ensure there is no data integrity issue.""" for worker in workers_store.items(): aid = worker['actor_id'] check_worker_health(aid, worker, ttl)