def shutdown(self, reason=None):
self._shutting_down = True
errmsg = ""
if reason is not None:
errmsg = "Shutting down sandboxagent due to reason: " + reason + "..."
self._logger.info(errmsg)
else:
self._logger.info("Gracefully shutting down sandboxagent...")
if self._frontend_process is not None:
self._logger.info("Shutting down the frontend...")
self._frontend_process.terminate()
else:
self._logger.info("No frontend; most probably it was the reason of the shutdown.")
# shutting down function workers depends on the queue service
if self._queue_service_process is not None:
self._logger.info("Shutting down the function worker(s)...")
self._deployment.shutdown()
# shut down the local queue client, so that we can also shut down the queue service
self._local_queue_client.removeTopic(self._instructions_topic)
self._local_queue_client.shutdown()
self._logger.info("Shutting down the queue service...")
process_utils.terminate_and_wait_child(self._queue_service_process, "queue service", 5, self._logger)
else:
self._logger.info("No queue service; most probably it was the reason of the shutdown.")
self._logger.info("Force shutting down the function worker(s)...")
self._deployment.force_shutdown()
# we can't do this here, because there may be other sandboxes running the same workflow
#self._management_data_layer_client.put("workflow_status_" + self._workflowid, "undeployed")
self._management_data_layer_client.shutdown()
self._logger.info("Shutting down fluent-bit...")
time.sleep(2) # flush interval of fluent-bit
process_utils.terminate_and_wait_child(self._fluentbit_process, "fluent-bit", 5, self._logger)
self._is_running = False
if self._frontend_process is not None:
try:
self._frontend_process.wait(30)
except subprocess.TimeoutExpired as exc:
self._frontend_process.kill()
_, _ = self._frontend_process.communicate()
self._logger.info("Shutdown complete.")
if reason is not None:
self._update_deployment_status(True, errmsg)
self._management_data_layer_client.shutdown()
os._exit(1)
else:
self._update_deployment_status(False, errmsg)
self._management_data_layer_client.shutdown()
os._exit(0)
def force_shutdown(self):
# called when the queue service has crashed and we need to shut down the function workers
for state in self._functionworker_process_map:
p = self._functionworker_process_map[state]
process_utils.terminate_and_wait_child(p, "FunctionWorker", 5, self._logger)
for jrh_process in self._javarequesthandler_process_list:
process_utils.terminate_and_wait_child(jrh_process, "JavaRequestHandler", 5, self._logger)
self._local_queue_client.shutdown()
def shutdown(self):
shutdown_message = {}
shutdown_message["action"] = "stop"
lqcm_shutdown = LocalQueueClientMessage(key="0l", value=json.dumps(shutdown_message))
workflow_nodes = self._workflow.getWorkflowNodeMap()
for function_topic in workflow_nodes:
ack = self._local_queue_client.addMessage(function_topic, lqcm_shutdown, True)
while not ack:
ack = self._local_queue_client.addMessage(function_topic, lqcm_shutdown, True)
self._logger.info("Waiting for function workers to shutdown")
self._wait_for_child_processes()
for jrh_process in self._javarequesthandler_process_list:
process_utils.terminate_and_wait_child(jrh_process, "JavaRequestHandler", 5, self._logger)
self._local_queue_client.shutdown()