def load_configuration(f=None, interactive=sys.stdout.isatty()): error = None defaults = _Struct(docker=Docker(), index=DockerIndex(), containers=Containers(), uris=URIs(), state=State(), log=Log( console=logging.DEBUG if interactive else None, syslog=logging.INFO if not interactive else None)) parsed = None try: f = f if f else path() if f: parsed = parse(f) except Exception as e: error = e finally: confs = defaults.merge(parsed) if parsed else defaults deimos.logger.initialize(**dict(confs.log.items())) if error: log.exception((("Error loading %s: " % f) if f else "") + str(error)) sys.exit(16) if parsed: log.info("Loaded configuration from %s" % f) for _, conf in parsed.items(): log.debug("Found: %r", conf) return confs
def load_configuration(f=None, interactive=sys.stdout.isatty()): error = None defaults = _Struct( docker=Docker(), index=DockerIndex(), containers=Containers(), uris=URIs(), state=State(), hooks=Hooks(), log=Log(console=(logging.DEBUG if interactive else None), syslog=(logging.INFO if not interactive else None)), ) parsed = None try: f = f if f else path() if f: parsed = parse(f) except Exception as e: error = e finally: confs = defaults.merge(parsed) if parsed else defaults deimos.logger.initialize(**dict(confs.log.items())) if error: pre = ("Error loading %s: " % f) if f else "" log.exception(pre + str(error)) sys.exit(16) if parsed: log.info("Loaded configuration from %s" % f) for _, conf in parsed.items(): log.debug("Found: %r", conf) return confs
def construct(path, name=None): "Selects an appropriate CGroup subclass for the given CGroup path." name = name if name else path.split("/")[4] classes = {"memory": Memory, "cpu": CPU, "cpuacct": CPUAcct} constructor = classes.get(name, CGroup) log.debug("Chose %s for: %s", constructor.__name__, path) return constructor(path, name)
def serialize(cls, **properties): """ With a Protobuf class and properties as keyword arguments, sets all the properties on a new instance of the class and serializes the resulting value. """ obj = cls() for k, v in properties.iteritems(): log.debug("%s.%s = %r", cls.__name__, k, v) setattr(obj, k, v) return obj.SerializeToString()
def proto_out(cls, **properties): """ With a Protobuf class and properies as keyword arguments, sets all the properties on a new instance of the class and serializes the resulting value to stdout. """ obj = cls() for k, v in properties.iteritems(): log.debug("%s.%s = %r", cls.__name__, k, v) setattr(obj, k, v) data = obj.SerializeToString() sys.stdout.write(data) sys.stdout.flush()
def stdio(containerizer, *args): """Connect containerizer class to command line args and STDIN Dispatches to an appropriate containerizer method based on the first argument and parses the input using an appropriate Protobuf type. launch < containerizer::Launch update < containerizer::Update usage < containerizer::Usage > mesos::ResourceStatistics wait < containerizer::Wait > containerizer::Termination destroy < containerizer::Destroy containers > containerizer::Containers recover Output serialization must be handled by the containerizer method (it doesn't necessarily happen at the end). Not really part of the containerizer protocol but exposed by Deimos as a subcommand: # Follows a Docker ID, PID, &c and exits with an appropriate, matching # exit code, in a manner specific to the containerizer observe <id> """ try: name = args[0] method, proto = { "launch": (containerizer.launch, Launch), "update": (containerizer.update, Update), "usage": (containerizer.usage, Usage), "wait": (containerizer.wait, Wait), "destroy": (containerizer.destroy, Destroy), "containers": (containerizer.containers, None), "recover": (containerizer.recover, None), "observe": (containerizer.observe, None) }[name] except IndexError: raise Err("Please choose a subcommand") except KeyError: raise Err("Subcommand %s is not valid for containerizers" % name) log.debug("%r", (method, proto)) if proto is not None: method(recordio.read(proto), *args[1:]) else: method(*args[1:])
def stop_docker_and_resume(self, signum): if self.state is not None and self.state.cid() is not None: cid = self.state.cid() log.info("Trying to stop Docker container: %s", cid) container_id = self.state.docker_id log.debug("Unwiring the container %s from MidoNet", container_id) try: log.info("state = %s", dir(self.state)) midonet.unwire_container_from_midonet(container_id) log.debug("Successfully unwired the container %s from MidoNet "\ "bridge", container_id) except Exception as ex: log.error(traceback.format_exc()) try: Run()(deimos.docker.stop(cid)) except subprocess.CalledProcessError: pass return deimos.sig.Resume()
def stdio(containerizer, *args): """Connect containerizer class to command line args and STDIN Dispatches to an appropriate containerizer method based on the first argument and parses the input using an appropriate Protobuf type. launch < containerizer::Launch update < containerizer::Update usage < containerizer::Usage > mesos::ResourceStatistics wait < containerizer::Wait > containerizer::Termination destroy < containerizer::Destroy containers > containerizer::Containers recover Output serialization must be handled by the containerizer method (it doesn't necessarily happen at the end). Not really part of the containerizer protocol but exposed by Deimos as a subcommand: # Follows a Docker ID, PID, &c and exits with an appropriate, matching # exit code, in a manner specific to the containerizer observe <id> """ try: name = args[0] method, proto = {"launch": (containerizer.launch, Launch), "update": (containerizer.update, Update), "usage": (containerizer.usage, Usage), "wait": (containerizer.wait, Wait), "destroy": (containerizer.destroy, Destroy), "containers": (containerizer.containers, None), "recover": (containerizer.recover, None), "observe": (containerizer.observe, None)}[name] except IndexError: raise Err("Please choose a subcommand") except KeyError: raise Err("Subcommand %s is not valid for containerizers" % name) log.debug("%r", (method, proto)) if proto is not None: return method(recordio.read(proto), *args[1:]) else: return method(*args[1:])
def destroy(self, destroy_pb, *args): log.info(" ".join(args)) container_id = destroy_pb.container_id.value state = deimos.state.State(self.state_root, mesos_id=container_id) state.await_launch() lk_d = state.lock("destroy", LOCK_EX) if state.exit() is None: container_id = state.docker_id log.debug("Unwiring the container %s from MidoNet", container_id) try: log.info("state_root = %s", dir(state)) midonet.unwire_container_from_midonet(container_id) log.debug("Successfully unwired the container %s from MidoNet " \ "bridge", container_id) except Exception as ex: log.error(traceback.format_exc()) Run()(deimos.docker.stop(state.cid())) else: log.info("Container is stopped") return 0
def launch(self, container_id, *args): log.info(" ".join([container_id] + list(args))) deimos.sig.install(self.sig_proxy) run_options = [] state = deimos.state.State(self.state_root, mesos_id=container_id) state.push() lk_l = state.lock("launch", LOCK_EX) mesos_directory() task = protos.TaskInfo() task.ParseFromString(sys.stdin.read()) for line in proto_lines(task): log.debug(line) state.executor_id = executor_id(task) state.push() state.ids() url, options = self.container_settings.override(*container(task)) pre, image = url.split("docker:///") if pre != "": raise Err("URL '%s' is not a valid docker:// URL!" % url) if image == "": image = self.default_image(task) log.info("image = %s", image) run_options += [ "--sig-proxy" ] run_options += [ "--rm" ] # This is how we ensure container cleanup run_options += [ "--cidfile", state.resolve("cid") ] place_uris(task, self.shared_dir, self.optimistic_unpack) run_options += [ "-w", self.workdir ] # Docker requires an absolute path to a source filesystem, separated # from the bind path in the container with a colon, but the absolute # path to the Mesos sandbox might have colons in it (TaskIDs with # timestamps can cause this situation). So we create a soft link to it # and mount that. shared_full = os.path.abspath(self.shared_dir) sandbox_symlink = state.sandbox_symlink(shared_full) run_options += [ "-v", "%s:%s" % (sandbox_symlink, self.workdir) ] cpus, mems = cpu_and_mem(task) env = [(_.name, _.value) for _ in task.command.environment.variables] run_options += options # We need to wrap the call to Docker in a call to the Mesos executor # if no executor is passed as part of the task. We need to pass the # MESOS_* environment variables in to the container if we're going to # start an executor. observer_argv = None if needs_executor_wrapper(task): options = ["--mesos-executor", "--observer"] if not(len(args) > 1 and args[0] in options): raise Err("Task %s needs --observer to be set!" % state.eid()) observer_argv = list(args[1:]) + [ deimos.path.me(), "wait", "--docker" ] else: env += mesos_env() + [("MESOS_DIRECTORY", self.workdir)] runner_argv = deimos.docker.run(run_options, image, argv(task), env=env, ports=ports(task), cpus=cpus, mems=mems) log_mesos_env(logging.DEBUG) observer = None with open("stdout", "w") as o: # This awkward multi 'with' is a with open("stderr", "w") as e: # concession to 2.6 compatibility with open(os.devnull) as devnull: log.info(deimos.cmd.present(runner_argv)) self.runner = subprocess.Popen(runner_argv, stdin=devnull, stdout=o, stderr=e) state.pid(self.runner.pid) state.await_cid() state.push() lk_w = state.lock("wait", LOCK_EX) lk_l.unlock() state.ids() proto_out(protos.ExternalStatus, message="launch: ok") sys.stdout.close() # Mark STDOUT as closed for Python code os.close(1) # Use low-level call to close OS side of STDOUT if observer_argv is not None: observer_argv += [state.cid()] log.info(deimos.cmd.present(observer_argv)) call = deimos.cmd.in_sh(observer_argv) # TODO: Collect these leaking file handles. obs_out = open(state.resolve("observer.out"), "w+") obs_err = open(state.resolve("observer.err"), "w+") # If the Mesos executor sees LIBPROCESS_PORT=0 (which # is passed by the slave) there are problems when it # attempts to bind. ("Address already in use"). # Purging both LIBPROCESS_* net variables, to be safe. for v in ["LIBPROCESS_PORT", "LIBPROCESS_IP"]: if v in os.environ: del os.environ[v] observer = subprocess.Popen(call, stdin=devnull, stdout=obs_out, stderr=obs_err, close_fds=True) data = Run(data=True)(deimos.docker.wait(state.cid())) state.exit(data) lk_w.unlock() for p, arr in [(self.runner, runner_argv), (observer, observer_argv)]: if p is None or p.wait() == 0: continue log.warning(deimos.cmd.present(arr, p.wait())) return state.exit()
def __init__(self, **cgroups_path_mapping): properties = {} for k, v in cgroups_path_mapping.items(): properties[k] = construct(v, k) _Struct.__init__(self, **properties) log.debug(" ".join(self.keys()))
def deserialize(cls, data): obj = cls() obj.ParseFromString(data) for line in lines(obj): log.debug(line) return obj
def launch(self, launch_pb, *args): log.info(" ".join(args)) fork = False if "--no-fork" in args else True deimos.sig.install(self.log_signal) run_options = [] launchy = deimos.mesos.Launch(launch_pb) state = deimos.state.State(self.state_root, mesos_id=launchy.container_id) state.push() lk_l = state.lock("launch", LOCK_EX) state.executor_id = launchy.executor_id state.push() state.ids() mesos_directory() # Redundant? if launchy.directory: os.chdir(launchy.directory) # TODO: if launchy.user: # os.seteuid(launchy.user) url, options = launchy.container options, trailing_argv = split_on(options, "//") url, options = self.container_settings.override(url, options) true_argv = launchy.argv if trailing_argv is None else trailing_argv image = self.determine_image(url, launchy) log.info("image = %s", image) run_options += ["--sig-proxy"] run_options += ["--rm"] # This is how we ensure container cleanup run_options += ["--cidfile", state.resolve("cid")] place_uris(launchy, self.shared_dir, self.optimistic_unpack) run_options += ["-w", self.workdir] # Docker requires an absolute path to a source filesystem, separated # from the bind path in the container with a colon, but the absolute # path to the Mesos sandbox might have colons in it (TaskIDs with # timestamps can cause this situation). So we create a soft link to it # and mount that. shared_full = os.path.abspath(self.shared_dir) sandbox_symlink = state.sandbox_symlink(shared_full) run_options += ["-v", "%s:%s" % (sandbox_symlink, self.workdir)] cpus, mems = launchy.cpu_and_mem env = launchy.env run_options += options env_dict = dict(env) if env_dict.get("MIDONET_BRIDGE_ID", None): run_options += ["--net=none"] # We need to wrap the call to Docker in a call to the Mesos executor # if no executor is passed as part of the task. We need to pass the # MESOS_* environment variables in to the container if we're going to # start an executor. observer_argv = None if launchy.needs_observer: # NB: The "@@docker@@" variant is a work around for Mesos's option # parser. There is a fix in the pipeline. observer_argv = [ mesos_executor(), "--override", deimos.path.me(), "observe", state.mesos_id ] state.lock("observe", LOCK_EX | LOCK_NB) # Explanation of Locks # When the observer is running, we would like its call to # observe() to finish before all the wait(); and we'd like the # observer to have a chance to report TASK_FINISHED before the # calls to wait() report their results (which would result in a # TASK_FAILED). # # For this reason, we take the "observe" lock in launch(), before # we call the observer and before releasing the "launch" or "wait" # locks. # # Calls to observe() actually skip locking "observe"; but wait() # calls must take this lock. The "observe" lock is held by # launch() until the observer executor completes, at which point # we can be reasonably sure its status was propagated to the Mesos # slave. else: env += mesos_env() + [("MESOS_DIRECTORY", self.workdir)] self.place_dockercfg() runner_argv = deimos.docker.run(run_options, image, true_argv, env=env, ports=launchy.ports, cpus=cpus, mems=mems) log_mesos_env(logging.DEBUG) observer = None with open("stdout", "w") as o: # This awkward multi 'with' is a with open("stderr", "w") as e: # concession to 2.6 compatibility with open(os.devnull) as devnull: log.info(deimos.cmd.present(runner_argv)) self.runner = subprocess.Popen(runner_argv, stdin=devnull, stdout=o, stderr=e) state.pid(self.runner.pid) state.await_cid() log.debug("Wiring the container to MidoNet") try: bridge_id = env_dict.get( "MIDONET_BRIDGE_ID", "78488c47-d1de-4d16-a27a-4e6419dc4f88") container_id = state.docker_id ip_addr = env_dict.get("MIDONET_IP_ADDRESS", "192.168.100.42") default_gw = env_dict.get("MIDONET_DEFAULT_GATEWAY", None) midonet.wire_container_to_midonet( container_id, bridge_id, ip_addr, default_gw) log.debug("Successfully wired the container %s to MidoNet " \ "bridge %s", container_id, bridge_id) except Exception as ex: log.error(traceback.format_exc()) state.push() lk_w = state.lock("wait", LOCK_EX) lk_l.unlock() if fork: pid = os.fork() if pid is not 0: state.ids() log.info("Forking watcher into child...") return state.ids() if observer_argv is not None: log.info(deimos.cmd.present(observer_argv)) call = deimos.cmd.in_sh(observer_argv, allstderr=False) # TODO: Collect these leaking file handles. obs_out = open(state.resolve("observer.out"), "w+") obs_err = open(state.resolve("observer.err"), "w+") # If the Mesos executor sees LIBPROCESS_PORT=0 (which # is passed by the slave) there are problems when it # attempts to bind. ("Address already in use"). # Purging both LIBPROCESS_* net variables, to be safe. for v in ["LIBPROCESS_PORT", "LIBPROCESS_IP"]: if v in os.environ: del os.environ[v] observer = subprocess.Popen(call, stdin=devnull, stdout=obs_out, stderr=obs_err, close_fds=True) data = Run(data=True)(deimos.docker.wait(state.cid())) state.exit(data) lk_w.unlock() for p, arr in [(self.runner, runner_argv), (observer, observer_argv)]: if p is None: continue thread = threading.Thread(target=p.wait) thread.start() thread.join(10) if thread.is_alive(): log.warning(deimos.cmd.present(arr, "SIGTERM after 10s")) p.terminate() thread.join(1) if thread.is_alive(): log.warning(deimos.cmd.present(arr, "SIGKILL after 1s")) p.kill() msg = deimos.cmd.present(arr, p.wait()) if p.wait() == 0: log.info(msg) else: log.warning(msg) return state.exit()
def launch(self, container_id, *args): log.info(" ".join([container_id] + list(args))) deimos.sig.install(self.sig_proxy) run_options = [] state = deimos.state.State(self.state_root, mesos_id=container_id) state.push() lk_l = state.lock("launch", LOCK_EX) mesos_directory() task = protos.TaskInfo() task.ParseFromString(sys.stdin.read()) for line in proto_lines(task): log.debug(line) state.executor_id = executor_id(task) state.push() state.ids() url, options = self.container_settings.override(*container(task)) pre, image = url.split("docker:///") if pre != "": raise Err("URL '%s' is not a valid docker:// URL!" % url) if image == "": image = self.default_image(task) log.info("image = %s", image) run_options += ["--sig-proxy"] run_options += ["--rm"] # This is how we ensure container cleanup run_options += ["--cidfile", state.resolve("cid")] place_uris(task, self.shared_dir, self.optimistic_unpack) run_options += ["-w", self.workdir] # Docker requires an absolute path to a source filesystem, separated # from the bind path in the container with a colon, but the absolute # path to the Mesos sandbox might have colons in it (TaskIDs with # timestamps can cause this situation). So we create a soft link to it # and mount that. shared_full = os.path.abspath(self.shared_dir) sandbox_symlink = state.sandbox_symlink(shared_full) run_options += ["-v", "%s:%s" % (sandbox_symlink, self.workdir)] cpus, mems = cpu_and_mem(task) env = [(_.name, _.value) for _ in task.command.environment.variables] run_options += options # We need to wrap the call to Docker in a call to the Mesos executor # if no executor is passed as part of the task. We need to pass the # MESOS_* environment variables in to the container if we're going to # start an executor. observer_argv = None if needs_executor_wrapper(task): options = ["--mesos-executor", "--observer"] if not (len(args) > 1 and args[0] in options): raise Err("Task %s needs --observer to be set!" % state.eid()) observer_argv = list( args[1:]) + [deimos.path.me(), "wait", "--docker"] else: env += mesos_env() + [("MESOS_DIRECTORY", self.workdir)] runner_argv = deimos.docker.run(run_options, image, argv(task), env=env, ports=ports(task), cpus=cpus, mems=mems) log_mesos_env(logging.DEBUG) observer = None with open("stdout", "w") as o: # This awkward multi 'with' is a with open("stderr", "w") as e: # concession to 2.6 compatibility with open(os.devnull) as devnull: log.info(deimos.cmd.present(runner_argv)) self.runner = subprocess.Popen(runner_argv, stdin=devnull, stdout=o, stderr=e) state.pid(self.runner.pid) state.await_cid() state.push() lk_w = state.lock("wait", LOCK_EX) lk_l.unlock() state.ids() proto_out(protos.ExternalStatus, message="launch: ok") sys.stdout.close() # Mark STDOUT as closed for Python code os.close( 1) # Use low-level call to close OS side of STDOUT if observer_argv is not None: observer_argv += [state.cid()] log.info(deimos.cmd.present(observer_argv)) call = deimos.cmd.in_sh(observer_argv) # TODO: Collect these leaking file handles. obs_out = open(state.resolve("observer.out"), "w+") obs_err = open(state.resolve("observer.err"), "w+") # If the Mesos executor sees LIBPROCESS_PORT=0 (which # is passed by the slave) there are problems when it # attempts to bind. ("Address already in use"). # Purging both LIBPROCESS_* net variables, to be safe. for v in ["LIBPROCESS_PORT", "LIBPROCESS_IP"]: if v in os.environ: del os.environ[v] observer = subprocess.Popen(call, stdin=devnull, stdout=obs_out, stderr=obs_err, close_fds=True) data = Run(data=True)(deimos.docker.wait(state.cid())) state.exit(data) lk_w.unlock() for p, arr in [(self.runner, runner_argv), (observer, observer_argv)]: if p is None or p.wait() == 0: continue log.warning(deimos.cmd.present(arr, p.wait())) return state.exit()
def launch(self, launch_pb, *args): log.info(" ".join(args)) fork = False if "--no-fork" in args else True deimos.sig.install(self.log_signal) run_options = [] launchy = deimos.mesos.Launch(launch_pb) state = deimos.state.State(self.state_root, mesos_id=launchy.container_id) state.push() lk_l = state.lock("launch", LOCK_EX) state.executor_id = launchy.executor_id state.push() state.ids() mesos_directory() # Redundant? if launchy.directory: os.chdir(launchy.directory) # TODO: if launchy.user: # os.seteuid(launchy.user) url, options = launchy.container options, trailing_argv = split_on(options, "//") url, options = self.container_settings.override(url, options) true_argv = launchy.argv if trailing_argv is None else trailing_argv image = self.determine_image(url, launchy) log.info("image = %s", image) run_options += ["--sig-proxy"] run_options += ["--rm"] # This is how we ensure container cleanup run_options += ["--cidfile", state.resolve("cid")] place_uris(launchy, self.shared_dir, self.optimistic_unpack) run_options += ["-w", self.workdir] # Docker requires an absolute path to a source filesystem, separated # from the bind path in the container with a colon, but the absolute # path to the Mesos sandbox might have colons in it (TaskIDs with # timestamps can cause this situation). So we create a soft link to it # and mount that. shared_full = os.path.abspath(self.shared_dir) sandbox_symlink = state.sandbox_symlink(shared_full) run_options += ["-v", "%s:%s" % (sandbox_symlink, self.workdir)] cpus, mems = launchy.cpu_and_mem env = launchy.env run_options += options env_dict = dict(env) if env_dict.get("MIDONET_BRIDGE_ID", None): run_options += ["--net=none"] # We need to wrap the call to Docker in a call to the Mesos executor # if no executor is passed as part of the task. We need to pass the # MESOS_* environment variables in to the container if we're going to # start an executor. observer_argv = None if launchy.needs_observer: # NB: The "@@docker@@" variant is a work around for Mesos's option # parser. There is a fix in the pipeline. observer_argv = [mesos_executor(), "--override", deimos.path.me(), "observe", state.mesos_id] state.lock("observe", LOCK_EX | LOCK_NB) # Explanation of Locks # When the observer is running, we would like its call to # observe() to finish before all the wait(); and we'd like the # observer to have a chance to report TASK_FINISHED before the # calls to wait() report their results (which would result in a # TASK_FAILED). # # For this reason, we take the "observe" lock in launch(), before # we call the observer and before releasing the "launch" or "wait" # locks. # # Calls to observe() actually skip locking "observe"; but wait() # calls must take this lock. The "observe" lock is held by # launch() until the observer executor completes, at which point # we can be reasonably sure its status was propagated to the Mesos # slave. else: env += mesos_env() + [("MESOS_DIRECTORY", self.workdir)] self.place_dockercfg() runner_argv = deimos.docker.run(run_options, image, true_argv, env=env, ports=launchy.ports, cpus=cpus, mems=mems) log_mesos_env(logging.DEBUG) observer = None with open("stdout", "w") as o: # This awkward multi 'with' is a with open("stderr", "w") as e: # concession to 2.6 compatibility with open(os.devnull) as devnull: log.info(deimos.cmd.present(runner_argv)) self.runner = subprocess.Popen(runner_argv, stdin=devnull, stdout=o, stderr=e) state.pid(self.runner.pid) state.await_cid() log.debug("Wiring the container to MidoNet") try: bridge_id = env_dict.get( "MIDONET_BRIDGE_ID", "78488c47-d1de-4d16-a27a-4e6419dc4f88") container_id = state.docker_id ip_addr = env_dict.get( "MIDONET_IP_ADDRESS", "192.168.100.42") default_gw = env_dict.get("MIDONET_DEFAULT_GATEWAY", None) midonet.wire_container_to_midonet( container_id, bridge_id, ip_addr, default_gw) log.debug("Successfully wired the container %s to MidoNet " \ "bridge %s", container_id, bridge_id) except Exception as ex: log.error(traceback.format_exc()) state.push() lk_w = state.lock("wait", LOCK_EX) lk_l.unlock() if fork: pid = os.fork() if pid is not 0: state.ids() log.info("Forking watcher into child...") return state.ids() if observer_argv is not None: log.info(deimos.cmd.present(observer_argv)) call = deimos.cmd.in_sh(observer_argv, allstderr=False) # TODO: Collect these leaking file handles. obs_out = open(state.resolve("observer.out"), "w+") obs_err = open(state.resolve("observer.err"), "w+") # If the Mesos executor sees LIBPROCESS_PORT=0 (which # is passed by the slave) there are problems when it # attempts to bind. ("Address already in use"). # Purging both LIBPROCESS_* net variables, to be safe. for v in ["LIBPROCESS_PORT", "LIBPROCESS_IP"]: if v in os.environ: del os.environ[v] observer = subprocess.Popen(call, stdin=devnull, stdout=obs_out, stderr=obs_err, close_fds=True) data = Run(data=True)(deimos.docker.wait(state.cid())) state.exit(data) lk_w.unlock() for p, arr in [(self.runner, runner_argv), (observer, observer_argv)]: if p is None: continue thread = threading.Thread(target=p.wait) thread.start() thread.join(10) if thread.is_alive(): log.warning(deimos.cmd.present(arr, "SIGTERM after 10s")) p.terminate() thread.join(1) if thread.is_alive(): log.warning(deimos.cmd.present(arr, "SIGKILL after 1s")) p.kill() msg = deimos.cmd.present(arr, p.wait()) if p.wait() == 0: log.info(msg) else: log.warning(msg) return state.exit()