def shutdown(self, reason=None): self._shutting_down = True errmsg = "" if reason is not None: errmsg = "Shutting down sandboxagent due to reason: " + reason + "..." self._logger.info(errmsg) else: self._logger.info("Gracefully shutting down sandboxagent...") if self._frontend_process is not None: self._logger.info("Shutting down the frontend...") self._frontend_process.terminate() else: self._logger.info("No frontend; most probably it was the reason of the shutdown.") # shutting down function workers depends on the queue service if self._queue_service_process is not None: self._logger.info("Shutting down the function worker(s)...") self._deployment.shutdown() # shut down the local queue client, so that we can also shut down the queue service self._local_queue_client.removeTopic(self._instructions_topic) self._local_queue_client.shutdown() self._logger.info("Shutting down the queue service...") process_utils.terminate_and_wait_child(self._queue_service_process, "queue service", 5, self._logger) else: self._logger.info("No queue service; most probably it was the reason of the shutdown.") self._logger.info("Force shutting down the function worker(s)...") self._deployment.force_shutdown() # we can't do this here, because there may be other sandboxes running the same workflow #self._management_data_layer_client.put("workflow_status_" + self._workflowid, "undeployed") self._management_data_layer_client.shutdown() self._logger.info("Shutting down fluent-bit...") time.sleep(2) # flush interval of fluent-bit process_utils.terminate_and_wait_child(self._fluentbit_process, "fluent-bit", 5, self._logger) self._is_running = False if self._frontend_process is not None: try: self._frontend_process.wait(30) except subprocess.TimeoutExpired as exc: self._frontend_process.kill() _, _ = self._frontend_process.communicate() self._logger.info("Shutdown complete.") if reason is not None: self._update_deployment_status(True, errmsg) self._management_data_layer_client.shutdown() os._exit(1) else: self._update_deployment_status(False, errmsg) self._management_data_layer_client.shutdown() os._exit(0)
def force_shutdown(self): # called when the queue service has crashed and we need to shut down the function workers for state in self._functionworker_process_map: p = self._functionworker_process_map[state] process_utils.terminate_and_wait_child(p, "FunctionWorker", 5, self._logger) for jrh_process in self._javarequesthandler_process_list: process_utils.terminate_and_wait_child(jrh_process, "JavaRequestHandler", 5, self._logger) self._local_queue_client.shutdown()
def shutdown(self): shutdown_message = {} shutdown_message["action"] = "stop" lqcm_shutdown = LocalQueueClientMessage(key="0l", value=json.dumps(shutdown_message)) workflow_nodes = self._workflow.getWorkflowNodeMap() for function_topic in workflow_nodes: ack = self._local_queue_client.addMessage(function_topic, lqcm_shutdown, True) while not ack: ack = self._local_queue_client.addMessage(function_topic, lqcm_shutdown, True) self._logger.info("Waiting for function workers to shutdown") self._wait_for_child_processes() for jrh_process in self._javarequesthandler_process_list: process_utils.terminate_and_wait_child(jrh_process, "JavaRequestHandler", 5, self._logger) self._local_queue_client.shutdown()