def example_app():
"""Example app fixture."""
current_dir = os.getcwd()
# go to example directory
project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
exampleappdir = os.path.join(project_dir, 'examples')
os.chdir(exampleappdir)
# setup example
cmd = './app-setup.sh'
exit_status = subprocess.call(cmd, shell=True)
assert exit_status == 0
# setup example
cmd = './app-fixtures.sh'
exit_status = subprocess.call(cmd, shell=True)
assert exit_status == 0
# Starting example web app
cmd = 'FLASK_APP=app.py flask run --debugger -p 5000'
webapp = subprocess.Popen(cmd, stdout=subprocess.PIPE,
preexec_fn=os.setsid, shell=True)
time.sleep(20)
# return webapp
yield webapp
# stop server
os.killpg(webapp.pid, signal.SIGTERM)
# tear down example app
cmd = './app-teardown.sh'
subprocess.call(cmd, shell=True)
# return to the original directory
os.chdir(current_dir)
def run_regular_process(cmd, kwargs, cargs={}):
"""
Executes a subprocess command by using subprocess.Popen
"""
proc = subprocess.Popen(cmd, **kwargs)
if kwargs.get('stdout') or kwargs.get('stderr'):
try:
out, err = proc.communicate(**cargs)
except TimeoutExpired:
if kwargs.get('start_new_session') and hasattr(os, 'killpg'):
os.killpg(proc.pid, signal.SIGKILL)
else:
proc.kill()
out, err = proc.communicate()
exc = TimeoutExpired(proc.args, timeout=cargs['timeout'], output=out)
exc.stderr = err
raise exc
status = proc.returncode
else:
out, err = (None, None)
try:
status = proc.wait(**cargs)
except TimeoutExpired as e:
if kwargs.get('start_new_session') and hasattr(os, 'killpg'):
os.killpg(proc.pid, signal.SIGKILL)
else:
proc.kill()
proc.wait()
raise e
return status, out, err
def _kill_server(server_process):
"""
kill the server subprocess group
"""
if server_process:
print "\nKilling the stale instance of a server..."
os.killpg(server_process.pid, signal.SIGTERM)
def ssh_output(cmd):
""" Runs an SSH command and returns the stdout/stderr.
:param cmd: command to run
:type cmd: str
:rtype: (str, str)
"""
# ssh must run with stdin attached to a tty
master, slave = pty.openpty()
proc = subprocess.Popen(cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
close_fds=True,
shell=True)
os.close(slave)
# wait for the ssh connection
time.sleep(8)
# kill the whole process group
os.killpg(os.getpgid(proc.pid), 15)
os.close(master)
stdout, stderr = proc.communicate()
print('SSH STDOUT: {}'.format(stdout.decode('utf-8')))
print('SSH STDERR: {}'.format(stderr.decode('utf-8')))
return stdout, stderr
def end_test(self, retcode):
if self.jmeter_process:
self.log.info("Terminating jmeter process group with PID %s", self.jmeter_process.pid)
try:
os.killpg(self.jmeter_process.pid, signal.SIGTERM)
except OSError, exc:
self.log.debug("Seems JMeter exited itself: %s", exc)
def reload(self):
"""
Reload and re-apply configuration settings
Existing child processes are sent a SIGHUP signal
and will exit after completing existing requests.
New child processes, which will have the updated
configuration, are spawned. This allows preventing
interruption to the service.
"""
def _has_changed(old, new, param):
old = old.get(param)
new = getattr(new, param)
return (new != old)
old_conf = utils.stash_conf_values()
has_changed = functools.partial(_has_changed, old_conf, CONF)
CONF.reload_config_files()
os.killpg(self.pgid, signal.SIGHUP)
self.stale_children = self.children
self.children = set()
# Ensure any logging config changes are picked up
logging.setup(CONF, 'glance')
config.set_config_defaults()
self.configure(old_conf, has_changed)
self.start_wsgi()
def run_with_timeout(command, timeout):
""" Run given command for timeout seconds. If command does not exit
before timeout is reached, forcibly kill spawned process.
"""
global result
global output
global process
def thread_fun():
global process
global output
global result
print "Blog Thread Running " + command + " started"
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, preexec_fn=os.setsid)
output = process.communicate()[0]
result = process.returncode
thread = threading.Thread(target=thread_fun)
thread.start()
thread.join(timeout)
if thread.isAlive():
print "Killing Example"
try:
output = ""
result = -1
#process.kill()
os.killpg(process.pid, signal.SIGTERM)
except OSError:
pass
print "Going To Sleep"
time.sleep(10)
print "Woken Up"
thread.join()
return
def progressive_kill(pid, kill_process_group=False):
if sys.platform.startswith("win"):
commands_to_try = ['TASKKILL /PID {pid}', 'TASKKILL /F /PID {pid}']
with open(os.devnull) as devnull:
for command in commands_to_try:
kill_proc = subprocess.Popen(command.format(pid=pid), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out = kill_proc.communicate()[0]
if kill_proc.poll() == 0:
LOG.debug("{command} succeeded".format(command=command))
return
else:
LOG.debug("{command} failed".format(command=command))
LOG.debug(out)
else:
signals_to_try = ['SIGINT', 'SIGTERM', 'SIGKILL']
for signal_name in signals_to_try:
signal_num = getattr(signal, signal_name)
try:
if kill_process_group:
os.killpg(pid, signal_num)
else:
os.kill(pid, signal_num)
LOG.debug("{signal_name} succeeded".format(signal_name=signal_name))
return
except Exception as e:
LOG.debug("{signal_name} failed".format(signal_name=signal_name))
LOG.debug(traceback.format_exc(e))
def hup(*args):
# Shuts down the server(s), but allows running requests to complete
self.LOG.error(_LE('SIGHUP received'))
signal.signal(signal.SIGHUP, signal.SIG_IGN)
os.killpg(0, signal.SIGHUP)
signal.signal(signal.SIGHUP, hup)
def on_rfcomm_disconnect(self, port):
node = '/dev/rfcomm%i' % port
for bdaddr, scripts in self.scripts.items():
process = scripts.get(node)
if process:
logging.info("Sending HUP to %s" % process.pid)
os.killpg(process.pid, signal.SIGHUP)
def python(code):
pypy = Popen(['pypy-sandbox'],
stdin=PIPE, stdout=PIPE,
stderr=PIPE,
env={'PATH': PATH},
universal_newlines=True,
preexec_fn=os.setpgrp)
try:
stdout, stderr = pypy.communicate(code, timeout=5)
except TimeoutExpired:
os.killpg(pypy.pid, signal.SIGKILL)
return "Timed out..."
errlines = stderr.split('\n')
if len(errlines) > 3:
for i in range(1, len(errlines)):
line = errlines[-i] # iterate backwards
if line:
return line[:250]
else:
for line in stdout.split('\n'):
if line.startswith('>>>> '):
while line[:5] in ['>>>> ', '.... ']:
line = line[5:]
return line[:250]
def stop_job( self, job ):
# if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
client = self.get_client( job.destination_params, job.job_runner_external_id )
job_ext_output_metadata = job.get_external_output_metadata()
if not PulsarJobRunner.__remote_metadata( client ) and job_ext_output_metadata:
pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them
if pid in [ None, '' ]:
log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.id )
return
pid = int( pid )
if not self.check_pid( pid ):
log.warning( "stop_job(): %s: PID %d was already dead or can't be signaled" % ( job.id, pid ) )
return
for sig in [ 15, 9 ]:
try:
os.killpg( pid, sig )
except OSError as e:
log.warning( "stop_job(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % ( job.id, errno.errorcode[e.errno], sig, pid, e.strerror ) )
return # give up
sleep( 2 )
if not self.check_pid( pid ):
log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" % ( job.id, pid, sig ) )
return
else:
log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job.id, pid ) )
else:
# Remote kill
pulsar_url = job.job_runner_name
job_id = job.job_runner_external_id
log.debug("Attempt remote Pulsar kill of job with url %s and id %s" % (pulsar_url, job_id))
client = self.get_client(job.destination_params, job_id)
client.kill()
def kill_spawned_jobs(verbose=False):
"""
INPUT:
- ``verbose`` -- bool (default: False); if True, display a
message each time a process is sent a kill signal
EXAMPLES::
sage: gp.eval('a=10')
'10'
sage: sage.interfaces.quit.kill_spawned_jobs(verbose=False)
sage: sage.interfaces.quit.expect_quitall()
sage: gp.eval('a=10')
'10'
sage: sage.interfaces.quit.kill_spawned_jobs(verbose=True)
Killing spawned job ...
After doing the above, we do the following to avoid confusion in other doctests::
sage: sage.interfaces.quit.expect_quitall()
"""
from sage.misc.misc import SAGE_TMP
file = os.path.join(SAGE_TMP, 'spawned_processes')
if not os.path.exists(file):
return
for L in open(file).readlines():
i = L.find(' ')
pid = L[:i].strip()
try:
if verbose:
print("Killing spawned job %s" % pid)
os.killpg(int(pid), 9)
except OSError:
pass
def playFile(self):
"""play/stop file"""
if len(self.selectedItems()) == 1:
item_sel = self.selectedItems()[0].text(0).split(" ")
if item_sel[0].lower().endswith(".mp3"):
playfile = int(AudioFileTools.readOptionsXml(xml_options_root, "playfile"))
selected_book = self.selectedItems()[0].parent().text(0)
files = AudioFileTools.readFromXml(xml_cache_root, selected_book, files = True)
# search for the clicked item in xml file and return path
file_path = [each_file[2] for each_file in files if each_file[2] == item_sel[1]][0]
if os.path.exists(file_path):
afplayCMD = ["afplay", file_path]
if not playfile:
# play sound and set pid
process = subprocess.Popen(afplayCMD, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, preexec_fn = os.setsid)
AudioFileTools.saveOptionsXml(xml_options_root, _options_dir, process.pid, "playfile")
else:
try:
os.killpg(playfile, signal.SIGTERM)
AudioFileTools.saveOptionsXml(xml_options_root, _options_dir, "0", "playfile")
except OSError:
process = subprocess.Popen(afplayCMD, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, preexec_fn = os.setsid)
AudioFileTools.saveOptionsXml(xml_options_root, _options_dir, process.pid, "playfile")
def run(self, timeout=10, no_assert=False, pass_status=0, do_not_buffer_output=False):
def target():
"""
subprocess.Popen wrapper that takes care of the dirty business of
child processes spawned within the shell
"""
if self.do_not_buffer_output:
self.process = subprocess.Popen([self.exec_cmd], shell=True, preexec_fn=os.setsid)
self.process.communicate()
else:
self.process = subprocess.Popen([self.exec_cmd], stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True, preexec_fn=os.setsid)
self.stdout, self.stderr = self.process.communicate()
self.cmd_status = self.process.returncode
# this optional flag is useful when no output buffering is desired (e.g. for debugging scripts)
self.do_not_buffer_output = do_not_buffer_output
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive() and self.process:
#print 'Timeout occured Terminating process'
# Timeout occured, kill all processes in the group
os.killpg(self.process.pid, signal.SIGTERM)
time.sleep(2)
if self.process.poll() is None: # Force kill if process is still alive
time.sleep(3)
os.killpg(self.process.pid, signal.SIGKILL)
thread.join()
self.cmd_timeout = True
# If Command failed or timedout
if not no_assert:
if (self.cmd_status != pass_status or self.cmd_timeout):
raise Exception("Command \'%s\' failed with output [%s], error [%s]" % (self.exec_cmd, self.stdout, self.stderr))
def handle(self, *args, **options):
if len(args) != 4:
raise Exception("Arguments needed: %s" % self.args)
# parse arguments
runserver_host = "%s:%s" % (args[0], args[2])
runserver_cmd = "python manage.py runserver %s" % runserver_host
nodejs_cmd = "node %s %s %s %s" % (nodjango_indexjs, args[1], args[2], args[3])
try:
# start nodejs proxy
proc2 = Popen(nodejs_cmd, shell=True, preexec_fn=os.setsid, stdout=sys.stdout, stderr=sys.stderr)
# start django runserver
proc1 = Popen(runserver_cmd, shell=True, preexec_fn=os.setsid, stdout=sys.stdout, stderr=sys.stderr)
time.sleep(2)
# start django private socketio
self.socket = SocketIO('127.0.0.1', int(args[3]), Namespace=DjangoNamespace)
print '* * *\t django socket started'
self.socket.wait()
finally:
print 'killing...'
os.killpg(proc1.pid, signal.SIGTERM)
os.killpg(proc2.pid, signal.SIGTERM)
print 'killed!'
def kill_children(*args):
"""Kills the entire process group."""
self.logger.error(_('SIGTERM or SIGINT received'))
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.running = False
os.killpg(0, signal.SIGTERM)
def monitor_processes(processes, logger):
while processes:
time.sleep(1)
kill_all = False
processes_running = []
for p in processes:
status = p.poll()
if status is not None:
logger.info("Process %s exited with code %d" % (p.pid, status))
if status != 0:
kill_all = True
logger.error("Process %s failed. Will kill the remaining processes." % p.pid)
else:
processes_running.append(p)
if kill_all:
for p in processes:
status = p.poll()
if status is None:
try:
os.killpg(p.pid, signal.SIGTERM)
except OSError, e:
try:
p.terminate()
except OSError, ex:
logger.write("Could not kill the process\n")
sys.exit(1)
def run(self):
start_time = datetime.now()
delta = 0
# poll() returns the returncode attribute, which is either the return
# code of the child process (which could be zero), or None if the
# process has not yet terminated.
while delta < self.timeout and self.proc.poll() is None:
time.sleep(1)
delta = (datetime.now() - start_time).total_seconds()
# if the test is not finished after timeout, first try to terminate it
# and if that fails, send SIGKILL to all processes in the test's
# process group
if self.proc.poll() is None:
self.status = 1
self.proc.terminate()
time.sleep(5)
if self.proc.poll() is None:
self.status = 2
if hasattr(os, 'killpg'):
os.killpg(self.proc.pid, signal.SIGKILL)
self.proc.wait()
def execute (command, timeout = -1):
start_time = time.time()
processPid = [None]
stdoutOutput = [None]
stderrOutput = [None]
def target():
process = Popen(command, stdout=PIPE, stderr=STDOUT, close_fds=True)
processPid[0] = process.pid;
(stdoutOutput[0], stderrOutput[0]) = process.communicate();
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
# Kill Process
try:
os.killpg(processPid[0], signal.SIGKILL)
except:
pass
os.waitpid(-1, os.WNOHANG)
thread.join()
elapsed_time = time.time() - start_time
output = stdoutOutput[0]
return (output.strip(), elapsed_time);
def tearDownClass(cls):
super(UploaderBase, cls).tearDownClass()
# kill server process group
os.killpg(cls._runserver.pid, signal.SIGKILL)
if os.path.exists('integration_settings.py'):
os.unlink('integration_settings.py')
def killProcesses(pid, pgrp):
""" kill a job upon request """
pUtil.tolog("killProcesses() called")
# if there is a known subprocess pgrp, then it should be enough to kill the group in one go
status = False
_sleep = True
if pgrp != 0:
# kill the process gracefully
pUtil.tolog("Killing group process %d" % (pgrp))
try:
os.killpg(pgrp, signal.SIGTERM)
except Exception,e:
pUtil.tolog("WARNING: Exception thrown when killing the child group process with SIGTERM: %s" % (e))
_sleep = False
else:
pUtil.tolog("(SIGTERM sent)")
if _sleep:
_t = 30
pUtil.tolog("Sleeping %d s to allow processes to exit" % (_t))
time.sleep(_t)
try:
os.killpg(pgrp, signal.SIGKILL)
except Exception,e:
pUtil.tolog("WARNING: Exception thrown when killing the child group process with SIGKILL: %s" % (e))
def run(self, timeout):
def target():
self.process = subprocess.Popen(self.cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE, preexec_fn=os.setsid)
self.out, self.err = self.process.communicate()
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
print "killing..."
self.process.terminate()
thread.join(1)
if thread.is_alive():
print "re-killing..."
self.process.kill()
thread.join(1)
if thread.is_alive():
print "re-re-killing..."
os.killpg(self.process.pid, signal.SIGTERM)
thread.join()
return (self.process.returncode, self.out, self.err)
def ws_cli(service_cls, *args, **kwargs):
"""Parse command line, start/stop ad-hoc server.
service_cls - Class to launch web service. Must have the constants
service_cls.NS and service_cls.UTIL. *args and **kwargs are
passed to its constructor.
"""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("non_interactive", "service_root_mode")
opts, args = opt_parser.parse_args()
arg = None
if args:
arg = args[0]
if arg == "start":
port = None
if args[1:]:
port = args[1]
_ws_init(service_cls, port, opts.service_root_mode, *args, **kwargs)
else:
report = Reporter(opts.verbosity - opts.quietness)
status = _get_server_status(service_cls)
level = Reporter.DEFAULT
if arg != "stop":
level = 0
for key, value in sorted(status.items()):
report("%s=%s\n" % (key, value), level=level)
if (arg == "stop" and status.get("pid") and
(opts.non_interactive or
raw_input("Stop server? y/n (default=n)") == "y")):
os.killpg(int(status["pid"]), signal.SIGTERM)
def _stop_pid( self, pid, job_id ):
"""
This method stops the given process id whether it's a task or job.
It is meant to be a private helper method, but it is mostly reusable.
The first argument is the process id to stop, and the second id is the
job's id (which is used for logging messages only right now).
"""
pid = int( pid )
log.debug( "Stopping pid %s" % pid )
if not self._check_pid( pid ):
log.warning( "_stop_pid(): %s: PID %d was already dead or can't be signaled" % ( job_id, pid ) )
return
for sig in [ 15, 9 ]:
try:
os.killpg( pid, sig )
except OSError as e:
# This warning could be bogus; many tasks are stopped with
# SIGTERM (signal 15), but ymmv depending on the platform.
log.warning( "_stop_pid(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % ( job_id, errno.errorcode[e.errno], sig, pid, e.strerror ) )
return
# TODO: If we're stopping lots of tasks, then we will want to put this
# avoid a two-second overhead using some other asynchronous method.
sleep( 2 )
if not self._check_pid( pid ):
log.debug( "_stop_pid(): %s: PID %d successfully killed with signal %d" % ( job_id, pid, sig ) )
return
else:
log.warning( "_stop_pid(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job_id, pid ) )
def child_cleanup(self, local_ids):
'''Let the forker know that we are done with the child process data.
and clean up. Only call this if you have some sort of return code.
Operates on a list of ids
'''
for local_id in local_ids:
if not self.children.has_key(local_id):
continue
#kill child if still running.
pg = self.children[local_id].proc
pid = pg.pid
pg.poll()
if pg.returncode == None:
try:
if pg.poll() == None:
os.killpg(pid, signal.SIGTERM)
sleep(5)
if pg.poll() == None:
os.killpg(pid, signal.SIGKILL)
except OSError:
#apparently we're already dead.
pass
#now that we're dead...
if self.children[local_id].runid != None:
self.active_runids.remove(self.children[local_id].runid)
del self.children[local_id]
def test_serve():
if is_port_8000_in_use():
pytest.skip("port 8000 already in use")
p = subprocess.Popen([os.path.join(wpt.localpaths.repo_root, "wpt"), "serve"],
preexec_fn=os.setsid)
start = time.time()
try:
while True:
if p.poll() is not None:
assert False, "server not running"
if time.time() - start > 60:
assert False, "server did not start responding within 60s"
try:
resp = urllib2.urlopen("http://web-platform.test:8000")
print resp
except urllib2.URLError:
print "URLError"
time.sleep(1)
else:
assert resp.code == 200
break
finally:
os.killpg(p.pid, 15)
def _terminate_processes(pids):
print_err("At least one health check timed out.")
for pid in pids:
try:
os.killpg(pid, signal.SIGTERM)
except OSError:
pass
def _killtasks(self, tasks=None, sig=signal.SIGKILL, kill_pgid=False):
"""
Kill all tasks
@param: tasks list of processids
@param: sig, signal to use to kill
@apram: kill_pgid, send kill to group
"""
if tasks is None:
self.log.error("killtasks no tasks passed")
elif isinstance(tasks, basestring):
try:
tasks = [int(tasks)]
except:
self.log.error("killtasks failed to convert tasks string %s to int" % tasks)
for pid in tasks:
pgid = os.getpgid(pid)
try:
os.kill(int(pid), sig)
if kill_pgid:
os.killpg(pgid, sig)
self.log.debug("Killed %s with signal %s" % (pid, sig))
except OSError, err:
# ERSCH is no such process, so no issue
if not err.errno == errno.ESRCH:
self.log.error("Failed to kill %s: %s" % (pid, err))
except Exception, err:
self.log.error("Failed to kill %s: %s" % (pid, err))
def clean_up_and_exit(in_signal, stack):
print "Killing webserver process..."
if mac:
print "<br />"
sys.stdout.flush()
try:
proc['server'].terminate()
except:
pass
try:
print "Killing jupyter notebook process [pid={0}]...".format(proc['jupyternotebook'].pid)
if mac:
print "<br /></body></html>"
sys.stdout.flush()
pgrp = os.getpgid(proc['jupyternotebook'].pid)
os.killpg(pgrp, signal.SIGINT)
time.sleep(0.1)
os.killpg(pgrp, signal.SIGTERM)
except Exception as e:
print "Failed to kill notebook: {0}".format(e)
if signal == None:
exit(0)
else:
exit(-1)
def tearDown(self):
os.killpg(self.fastproxy.pid, signal.SIGTERM)
def kill_children(*args):
"""Kills the entire process group."""
logger.error('SIGTERM received')
signal.signal(signal.SIGTERM, signal.SIG_IGN)
running[0] = False
os.killpg(0, signal.SIGTERM)
def main(argv=None):
parser = argparse.ArgumentParser(description='Submit cloudsim run')
parser.add_argument("config",
nargs="?",
default=str(pathlib.Path('./run.toml')))
parser.add_argument("-n", action="store_true", help="dry run")
parser.add_argument("-s",
"--show-simulator-window",
action="store_true",
help="Display Gazebo simulator window.")
args = parser.parse_args(argv)
config_file = pathlib.Path(args.config).resolve()
if not config_file.is_file():
sys.exit(f"{config_file} not found")
with open(args.config, 'r') as f:
config = toml.load(f)
client = docker.from_env()
version = client.version()["Version"]
if version < "19.03":
sys.exit("Please install at least version 19.03 of docker")
world = validate_world(config["world"])
circuit = validate_circuit(world)
if "timeout" in config:
config["robots"]["T{}".format(config["timeout"])] = "teambase"
robots = validate_robots(config["robots"])
image = validate_image(client, config["image"])
now = datetime.datetime.now(datetime.timezone.utc)
strnow = f"{now.year}-{now.month:02d}-{now.day:02d}T{now.hour:02d}.{now.minute:02d}.{now.second:02d}"
logdir = config_file.with_name(f"{strnow}-{config_file.stem}")
print(f"circuit: {circuit}")
print(f"world: {world}")
print(f"image: {image}")
print(f"logdir: {logdir}")
print(f"robots: ")
for name, kind in robots.items():
print(f" {name:>10s}: {kind}")
print()
_prune_containers(client, robots)
_xauth()
logdir.mkdir()
symlink = config_file.with_name(config_file.stem)
if symlink.exists():
symlink.unlink()
symlink.symlink_to(f"{strnow}-{config_file.stem}")
should_stop = False
def _sigint(signum, frame):
nonlocal should_stop
print("got signal, stopping")
should_stop = True
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGINT, _sigint)
sim = _run_sim(client, circuit, logdir, world, robots,
args.show_simulator_window)
to_stop = [sim]
stdout = [sim]
to_wait = []
for n, (name, kind) in enumerate(robots.items(), start=1):
if not should_stop:
b = _run_bridge(client, circuit, world, name, kind, n)
to_stop.append(b)
stdout.append(b)
if not should_stop:
r = _run_robot(client, logdir, image, name, n)
to_wait.append(r)
stdout.append(r)
if not should_stop:
print("Waiting for robot containers to finish....")
while not should_stop and len(to_wait) > 0:
time.sleep(1)
for r in to_wait:
r.reload()
for r in to_wait:
if r.status == "exited":
print(f"Container {r.name} exited.")
if r.name.startswith('T'): #teambase
should_stop = True
to_wait = [r for r in to_wait if r.status != "exited"]
for s in to_stop:
s.reload()
if s.status == "exited":
print(f"Container {s.name} unexpectedly exited.")
break
else:
continue
break
if len(to_wait) > 0:
print("Stopping robot containers...")
for r in to_wait:
r.kill(signal.SIGINT) # robot containers should respond to signals
for r in to_wait:
try:
r.wait(timeout=10)
except Exception: # requests.exceptions.ReadTimeout:
print(f"Container {r.name} ignored SIGINT! Stopping anyway.")
r.stop(timeout=0)
for s in to_stop:
try:
s.reload()
if s.status == "exited":
print(f"Container {s.name} exited.")
else:
print(f"Stopping {s.name} container...")
#s.kill(signal.SIGINT) # not working because ign containers don't forward signals
s.stop(
timeout=0
) # there is no point in waiting since nobody is listening for signals
except docker.errors.APIError as e:
print(e)
for s in stdout:
with open(logdir / f"{s.name}-stdout.txt", "wb") as f:
f.write(s.logs())
s.remove()
networks = client.networks.list()
network_names = {"simnet"}
network_names.update(f"relay{n+1}net" for n in range(len(robots)))
for net in networks:
if net.name in network_names:
print(f"Removing {net.name} network...")
net.remove()
if should_stop:
# [How to be a proper program](https://www.cons.org/cracauer/sigint.html)
signal.signal(signal.SIGINT, signal.SIG_DFL)
os.killpg(
os.getpid(), signal.SIGINT
) # signal.raise_signal(signal.SIGINT) available only since python 3.8
def kill_fn(s):
os.killpg(gid, s)
def on_timeout():
if proc.returncode is None:
print_status('TIMED OUT\n')
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
proc.kill()
def _terminate(self, proc):
os.killpg(proc.pid, 15)
sleep(1)
if proc.poll() is None:
os.killpg(proc.pid, 9)
return proc.wait() # reap
def on_press(self):
global alarm_pid
if (alarm_pid != 999999999999):
os.killpg(alarm_pid, signal.SIGTERM)
alarm_pid = 999999999999
for n, test in enumerate(test_to_run):
path = test[0]
prefix = '[%d/%d]' % (n + 1, n_total)
print_status('%s RUNNING %s' % (prefix, path))
signal.signal(signal.SIGALRM, alarm_handler)
if args.jenkins and test[1] == 'boost':
path = path + " --output_format=XML --log_level=all --report_level=no"
proc = subprocess.Popen(path.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env,preexec_fn=os.setsid)
signal.alarm(args.timeout)
out = None
err = None
try:
out, err = proc.communicate()
signal.alarm(0)
except:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
proc.kill()
proc.returncode = -1
finally:
if proc.returncode:
print_status('FAILED: %s\n' % (path))
if proc.returncode == -1:
print_status('TIMED OUT\n')
if out:
print('=== stdout START ===')
print(out.decode())
print('=== stdout END ===')
if err:
print('=== stderr START ===')
print(err.decode())
print('=== stderr END ===')
def run(self):
if SETPROCTITLE:
setproctitle.setproctitle('homecontrol-scripts-' + self.Index)
levelprevious = -1
IDExternal = self.Settings["scriptid" + self.Index] + "/"
self.ComQueue[MQTT].put([
MQTT_PUBLISH_NORETAIN, IDExternal + "interface",
self.Settings[SETTINGS_IPTOPIC]
])
self.ComQueue[MQTT].put([MQTT_SUBSCRIBE, IDExternal + "#"])
InitCommands = []
SliderLevelLast = "0"
ProcessRunning = {}
ProcessThread = {}
CommandsQueue = {} #Queue to control shell scripts or processes
for i in range(0, 25):
try:
self.Settings["scriptstart" + self.Index + "," + str(i)]
except:
levels = i
interval = 1 / i
break
time.sleep(int(self.Index)) #staggered Initialization
Process = [0 for i in range(levels + 1)] # preload array with 0
l = i
#Collect incomming commands on startup, and modify some of them (e.g. sliderlastlevel)
time.sleep(int(self.Settings["waitforstatusupdate"]))
while not self.ComQueue[self.IDInternal].empty():
temp = self.ComQueue[self.IDInternal].get()
InitCommands.append(temp)
for i in range(0, len(InitCommands)):
temp = InitCommands[i]
if temp[0] == "levellast":
SliderLevelLast = temp[1]
elif temp[0] == "levellasttoggle":
continue
else:
self.ComQueue[self.IDInternal].put(temp)
if self.Settings["firstrun"] == "1":
self.ComQueue[self.IDInternal].put(["init"])
while True:
data = self.ComQueue[self.IDInternal].get()
if data[0] == SCRIPTS_TERMINATE_PROCESS:
ProcessThread[data[1]].join()
ProcessRunning[data[1]] = False
elif data[0] == "state": #Toggle
if data[1] == "1":
#Kill previous started process in case still running
if 1 in ProcessRunning:
if ProcessRunning[1]:
CommandsQueue[1].put(SCRIPTS_TERMINATE_PROCESS)
ProcessThread[1].join()
ProcessRunning[1] = False
#Start process
File = self.Settings["scriptstart" + self.Index + ",0"]
if os.path.isfile(File):
CommandsQueue[0] = multiprocessing.Queue()
ProcessThread[0] = Processes(
self.Settings["scriptstart" + self.Index + ",0"],
0, CommandsQueue, self.ComQueue, self.IDInternal)
ProcessThread[0].start()
ProcessRunning[0] = True
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "STATE", "1"])
elif data[1] == "0":
#Kill previous started process in case still running
if 0 in ProcessRunning:
if ProcessRunning[0]:
CommandsQueue[0].put(SCRIPTS_TERMINATE_PROCESS)
ProcessThread[0].join()
ProcessRunning[0] = False
#Start process
File = self.Settings["scriptstop" + self.Index + ",0"]
if os.path.isfile(File):
CommandsQueue[1] = multiprocessing.Queue()
ProcessThread[1] = Processes(
self.Settings["scriptstop" + self.Index + ",0"], 0,
CommandsQueue, self.ComQueue, self.IDInternal)
ProcessThread[1].start()
ProcessRunning[1] = True
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "STATE", "0"])
elif data[1] == "undefined":
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "STATE", "undefined"])
elif data[0] == "level":
if data[1] == "undefined":
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVEL", "undefined"])
else:
temp = float(data[1])
for i in range(0, levels):
if temp >= interval * i and temp <= interval * (i + 1):
if temp > 0:
SliderLevelLast = data[1]
self.ComQueue[MQTT].put([
MQTT_PUBLISH, IDExternal + "LEVELLAST",
data[1]
])
self.ComQueue[MQTT].put([
MQTT_PUBLISH,
IDExternal + "LEVELLASTTOGGLE", "1"
])
else:
self.ComQueue[MQTT].put([
MQTT_PUBLISH,
IDExternal + "LEVELLASTTOGGLE", "0"
])
if levelprevious in ProcessRunning:
if ProcessRunning[levelprevious]:
CommandsQueue[levelprevious].put(
SCRIPTS_TERMINATE_PROCESS)
ProcessThread[levelprevious].join()
ProcessRunning[levelprevious] = False
File = self.Settings["scriptstart" + self.Index +
"," + str(i)]
if os.path.isfile(File):
CommandsQueue[i] = multiprocessing.Queue()
ProcessThread[i] = Processes(
self.Settings["scriptstart" + self.Index +
"," + str(i)], i,
CommandsQueue, self.ComQueue,
self.IDInternal)
ProcessThread[i].start()
ProcessRunning[i] = True
levelprevious = i
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVEL", data[1]])
elif data[0] == "init":
if Process[0] != 0:
os.killpg(os.getpgid(Process[0].pid), signal.SIGTERM)
Process[0] = 0
if Process[1] != 0:
os.killpg(os.getpgid(Process[1].pid), signal.SIGTERM)
Process[1] = 0
if l == 1: # only one level, must be a "state" script
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "STATE", "0"])
else: #more then one level, must be a multilevel script
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVEL", "0"])
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVELLAST", "0"])
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "0"])
continue
elif data[0] == "levellasttoggle":
if data[1] == "0":
self.ComQueue[self.IDInternal].put(["level", "0"])
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "0"])
else:
self.ComQueue[self.IDInternal].put(
["level", SliderLevelLast])
self.ComQueue[MQTT].put(
[MQTT_PUBLISH, IDExternal + "LEVELLASTTOGGLE", "1"])
def run_single_test(self, test):
if test.fname[0].endswith('.jar'):
cmd = ['java', '-jar'] + test.fname
elif not test.is_cross_built and run_with_mono(test.fname[0]):
cmd = ['mono'] + test.fname
else:
if test.is_cross_built:
if test.exe_runner is None:
# Can not run test on cross compiled executable
# because there is no execute wrapper.
cmd = None
else:
cmd = [test.exe_runner] + test.fname
else:
cmd = test.fname
if cmd is None:
res = 'SKIP'
duration = 0.0
stdo = 'Not run because can not execute cross compiled binaries.'
stde = None
returncode = GNU_SKIP_RETURNCODE
else:
test_opts = deepcopy(self.options)
test_env = self.get_test_env(test_opts, test)
wrap = self.get_wrapper(test_opts)
if test_opts.gdb:
test.timeout = None
cmd = wrap + cmd + test.cmd_args + self.options.test_args
starttime = time.time()
if len(test.extra_paths) > 0:
test_env['PATH'] = os.pathsep.join(test.extra_paths +
['']) + test_env['PATH']
# If MALLOC_PERTURB_ is not set, or if it is set to an empty value,
# (i.e., the test or the environment don't explicitly set it), set
# it ourselves. We do this unconditionally for regular tests
# because it is extremely useful to have.
# Setting MALLOC_PERTURB_="0" will completely disable this feature.
if ('MALLOC_PERTURB_' not in test_env
or not test_env['MALLOC_PERTURB_']
) and not self.options.benchmark:
test_env['MALLOC_PERTURB_'] = str(random.randint(1, 255))
stdout = None
stderr = None
if not self.options.verbose:
stdout = subprocess.PIPE
stderr = subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT
# Let gdb handle ^C instead of us
if test_opts.gdb:
previous_sigint_handler = signal.getsignal(signal.SIGINT)
# Make the meson executable ignore SIGINT while gdb is running.
signal.signal(signal.SIGINT, signal.SIG_IGN)
def preexec_fn():
if test_opts.gdb:
# Restore the SIGINT handler for the child process to
# ensure it can handle it.
signal.signal(signal.SIGINT, signal.SIG_DFL)
else:
# We don't want setsid() in gdb because gdb needs the
# terminal in order to handle ^C and not show tcsetpgrp()
# errors avoid not being able to use the terminal.
os.setsid()
p = subprocess.Popen(
cmd,
stdout=stdout,
stderr=stderr,
env=test_env,
cwd=test.workdir,
preexec_fn=preexec_fn if not is_windows() else None)
timed_out = False
kill_test = False
if test.timeout is None:
timeout = None
elif test_opts.timeout_multiplier is not None:
timeout = test.timeout * test_opts.timeout_multiplier
else:
timeout = test.timeout
try:
(stdo, stde) = p.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
if self.options.verbose:
print("%s time out (After %d seconds)" %
(test.name, timeout))
timed_out = True
except KeyboardInterrupt:
mlog.warning("CTRL-C detected while running %s" % (test.name))
kill_test = True
finally:
if test_opts.gdb:
# Let us accept ^C again
signal.signal(signal.SIGINT, previous_sigint_handler)
if kill_test or timed_out:
# Python does not provide multiplatform support for
# killing a process and all its children so we need
# to roll our own.
if is_windows():
subprocess.call(
['taskkill', '/F', '/T', '/PID',
str(p.pid)])
else:
try:
os.killpg(os.getpgid(p.pid), signal.SIGKILL)
except ProcessLookupError:
# Sometimes (e.g. with Wine) this happens.
# There's nothing we can do (maybe the process
# already died) so carry on.
pass
(stdo, stde) = p.communicate()
endtime = time.time()
duration = endtime - starttime
stdo = decode(stdo)
if stde:
stde = decode(stde)
if timed_out:
res = 'TIMEOUT'
self.timeout_count += 1
self.fail_count += 1
elif p.returncode == GNU_SKIP_RETURNCODE:
res = 'SKIP'
self.skip_count += 1
elif test.should_fail == bool(p.returncode):
res = 'OK'
self.success_count += 1
else:
res = 'FAIL'
self.fail_count += 1
returncode = p.returncode
result = TestRun(res, returncode, test.should_fail, duration, stdo,
stde, cmd, test.env)
return result
push = 0
level = 0
while True :
time.sleep(0.1)
#different camera angles
if GPIO.input(RearView_Switch)==0 and run == 0:
print " Started Full Screen"
rpistr = "raspivid -t 0 -vf -h 480 -w 800"
p=subprocess.Popen(rpistr,shell=True, preexec_fn=os.setsid)
run = 1
while GPIO.input(RearView_Switch)==0:
time.sleep(0.1)
if GPIO.input(RearView_Switch)==0 and push == 1:
os.killpg(p.pid, signal.SIGTERM)
print " Started Full Screen Transparent"
rpistr = "raspivid -t 0 -vf -op 128 -h 480 -w 800"
p=subprocess.Popen(rpistr,shell=True, preexec_fn=os.setsid)
run = 2
while GPIO.input(RearView_Switch)==0:
time.sleep(0.1)
if GPIO.input(RearView_Switch)==0 and push == 2:
os.killpg(p.pid, signal.SIGTERM)
print " Started PIP Right side"
rpistr = "raspivid -t 0 -vf -p 350,1,480,320"
p=subprocess.Popen(rpistr,shell=True, preexec_fn=os.setsid)
run = push
while GPIO.input(RearView_Switch)==0:
time.sleep(0.1)
def term(sig, frame):
nonlocal terminating
terminating = True
os.killpg(os.getpgid(sshprocess.pid), signal.SIGTERM)
def stop(self):
print 'Thread::[{}] successfully stopped.'.format(self.objectName())
if self.process is not None:
killpg(getpgid(self.process.pid), signal.SIGTERM)
path_plan_result = wait_for_socket('pathPlan', timeout=30)
else:
path_plan_result = True
carstate_result = wait_for_socket('carState', timeout=30)
print("Check if everything is running")
running = manager.get_running()
controlsd_running = running['controlsd'].is_alive()
radard_running = running['radard'].is_alive()
plannerd_running = running['plannerd'].is_alive()
manager.kill_managed_process("controlsd")
manager.kill_managed_process("radard")
manager.kill_managed_process("plannerd")
os.killpg(os.getpgid(unlogger.pid), signal.SIGTERM)
sockets_ok = all([
controls_state_result, radarstate_result, plan_result,
path_plan_result, carstate_result, controlsd_running,
radard_running, plannerd_running
])
params_ok = True
failures = []
if not controlsd_running:
failures.append('controlsd')
if not radard_running:
failures.append('radard')
if not radarstate_result:
failures.append('radarState')
def _atexit(self):
with contextlib.suppress(ProcessLookupError):
os.killpg(self.pgid, signal.SIGTERM)
def kill(self):
import signal, subprocess
if os.name == "nt":
subprocess.Popen("taskkill /F /T /pid %i" % self.pid, shell=True)
else:
os.killpg(self.pid, signal.SIGKILL)
def panic(msg):
logging.error("Panic: %s", msg, exc_info=True)
os.killpg(0, 9)
sys.exit(-3)
def generic_handler(event, context_dict, custom_handler_env=None):
"""
event is from the invoker, and contains job information
context_dict is generic infromation about the context
that we are running in, provided by the scheduler
custom_handler_env are environment variables we should set
based on the platform we are on.
"""
pid = os.getpid()
response_status = {'exception': None}
try:
if event['storage_config']['storage_backend'] != 's3':
raise NotImplementedError(
("Using {} as storage backend is not supported " +
"yet.").format(event['storage_config']['storage_backend']))
s3_client = boto3.client("s3")
s3_bucket = event['storage_config']['backend_config']['bucket']
logger.info("invocation started")
# download the input
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "Pywren version mismatch",
version.__version__, event['pywren_version'])
start_time = time.time()
response_status['start_time'] = start_time
runtime_s3_bucket = event['runtime']['s3_bucket']
runtime_s3_key = event['runtime']['s3_key']
if event.get('runtime_url'):
# NOTE(shivaram): Right now we only support S3 urls.
runtime_s3_bucket_used, runtime_s3_key_used = wrenutil.split_s3_url(
event['runtime_url'])
else:
runtime_s3_bucket_used = runtime_s3_bucket
runtime_s3_key_used = runtime_s3_key
job_max_runtime = event.get("job_max_runtime",
290) # default for lambda
response_status['func_key'] = func_key
response_status['data_key'] = data_key
response_status['output_key'] = output_key
response_status['status_key'] = status_key
data_key_size = get_key_size(s3_client, s3_bucket, data_key)
#logger.info("bucket=", s3_bucket, "key=", data_key, "status: ", data_key_size, "bytes" )
while data_key_size is None:
logger.warning("WARNING COULD NOT GET FIRST KEY")
data_key_size = get_key_size(s3_client, s3_bucket, data_key)
if not event['use_cached_runtime']:
subprocess.check_output("rm -Rf {}/*".format(RUNTIME_LOC),
shell=True)
free_disk_bytes = free_disk_space("/tmp")
response_status['free_disk_bytes'] = free_disk_bytes
response_status['runtime_s3_key_used'] = runtime_s3_key_used
response_status['runtime_s3_bucket_used'] = runtime_s3_bucket_used
if (custom_handler_env != None):
delete_old_runtimes = custom_handler_env.get(
'delete_old_runtimes', False)
else:
delete_old_runtimes = False
runtime_cached = download_runtime_if_necessary(s3_client,
runtime_s3_bucket_used,
runtime_s3_key_used,
delete_old_runtimes)
logger.info("Runtime ready, cached={}".format(runtime_cached))
response_status['runtime_cached'] = runtime_cached
cwd = os.getcwd()
jobrunner_path = os.path.join(cwd, "jobrunner.py")
extra_env = event.get('extra_env', {})
extra_env['PYTHONPATH'] = "{}".format(os.getcwd())
call_id = event['call_id']
callset_id = event['callset_id']
response_status['call_id'] = call_id
response_status['callset_id'] = callset_id
runtime_meta = s3_client.head_object(Bucket=runtime_s3_bucket_used,
Key=runtime_s3_key_used)
ETag = str(runtime_meta['ETag'])[1:-1]
conda_runtime_dir = CONDA_RUNTIME_DIR.format(ETag)
conda_python_path = conda_runtime_dir + "/bin"
conda_python_runtime = os.path.join(conda_python_path, "python")
# pass a full json blob
jobrunner_config_filename = JOBRUNNER_CONFIG_FILENAME.format(pid)
jobrunner_stats_filename = JOBRUNNER_STATS_FILENAME.format(pid)
python_module_path = PYTHON_MODULE_PATH.format(pid)
jobrunner_config = {
'func_bucket': s3_bucket,
'func_key': func_key,
'data_bucket': s3_bucket,
'data_key': data_key,
'data_byte_range': data_byte_range,
'python_module_path': python_module_path,
'output_bucket': s3_bucket,
'output_key': output_key,
'stats_filename': jobrunner_stats_filename
}
with open(jobrunner_config_filename, 'w') as jobrunner_fid:
json.dump(jobrunner_config, jobrunner_fid)
if os.path.exists(jobrunner_stats_filename):
os.remove(jobrunner_stats_filename)
cmdstr = "{} {} {}".format(conda_python_runtime, jobrunner_path,
jobrunner_config_filename)
setup_time = time.time()
response_status['setup_time'] = setup_time - start_time
local_env = os.environ.copy()
if custom_handler_env is not None:
local_env.update(custom_handler_env)
local_env.update(extra_env)
local_env['PATH'] = "{}:{}".format(conda_python_path,
local_env.get("PATH", ""))
logger.debug("command str=%s", cmdstr)
# This is copied from http://stackoverflow.com/a/17698359/4577954
# reasons for setting process group: http://stackoverflow.com/a/4791612
process = subprocess.Popen(cmdstr,
shell=True,
env=local_env,
bufsize=1,
stdout=subprocess.PIPE,
preexec_fn=os.setsid)
logger.info("launched process")
def consume_stdout(stdout, queue):
with stdout:
for line in iter(stdout.readline, b''):
queue.put(line)
q = Queue()
t = Thread(target=consume_stdout, args=(process.stdout, q))
t.daemon = True
t.start()
stdout = b""
while t.isAlive():
try:
line = q.get_nowait()
stdout += line
logger.info(line)
except Empty:
time.sleep(PROCESS_STDOUT_SLEEP_SECS)
total_runtime = time.time() - start_time
if total_runtime > job_max_runtime:
logger.warning(
"Process exceeded maximum runtime of {} sec".format(
job_max_runtime))
# Send the signal to all the process groups
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
raise Exception(
"OUTATIME", "Process executed for too long and was killed")
logger.info("command execution finished")
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ")
float_value = float(value)
response_status[key] = float_value
end_time = time.time()
response_status['stdout'] = stdout.decode("ascii")
response_status['exec_time'] = time.time() - setup_time
response_status['end_time'] = end_time
response_status['host_submit_time'] = event['host_submit_time']
response_status['server_info'] = get_server_info()
response_status.update(context_dict)
except Exception as e:
# internal runtime exceptions
response_status['exception'] = str(e)
response_status['exception_args'] = e.args
response_status['exception_traceback'] = traceback.format_exc()
finally:
# creating new client in case the client has not been created
boto3.client("s3").put_object(Bucket=s3_bucket,
Key=status_key,
Body=json.dumps(response_status))
def excepthook(type, value, traceback):
sys.__excepthook__(type, value, traceback)
logging.critical('Crash! Killing all miners.')
os.killpg(os.getpgid(0), signal.SIGKILL) # (This also kills us.)
def executeCmd(realCmd,
timeout=60,
shell=True,
output=False,
tout_kill_tree=True,
success_codes=(0, ),
varsDict=None):
"""Executes a command.
Parameters
----------
realCmd : str
The command to execute.
timeout : int
The time out in seconds for the command.
shell : bool
If shell is True (default), the specified command (may be a string) will be
executed through the shell.
output : bool
If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode).
If False, just indication of success is returned
varsDict: dict
variables supplied to the command (or to the shell script)
Returns
-------
bool or (bool, str, str, int)
True if the command succeeded and with stdout, stderr, returncode if output was set to True
Raises
------
OSError
If command fails to be executed.
RuntimeError
If command execution times out.
"""
stdout = stderr = None
retcode = None
popen = env = None
if varsDict:
if shell:
# build map as array of vars and command line array:
realCmd = Utils.buildShellCmd(realCmd, varsDict)
else: # pragma: no cover - currently unused
env = _merge_dicts(os.environ, varsDict)
realCmdId = id(realCmd)
logCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId,
realCmd)
try:
popen = subprocess.Popen(
realCmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell,
env=env,
preexec_fn=os.
setsid # so that killpg does not kill our process
)
# wait with timeout for process has terminated:
retcode = popen.poll()
if retcode is None:
def _popen_wait_end():
retcode = popen.poll()
return (True, retcode) if retcode is not None else None
# popen.poll is fast operation so we can use the shortest sleep interval:
retcode = Utils.wait_for(_popen_wait_end, timeout,
Utils.DEFAULT_SHORTEST_INTERVAL)
if retcode:
retcode = retcode[1]
# if timeout:
if retcode is None:
if logCmd:
logCmd(logging.ERROR)
logCmd = None
logSys.error("%x -- timed out after %s seconds." %
(realCmdId, timeout))
pgid = os.getpgid(popen.pid)
# if not tree - first try to terminate and then kill, otherwise - kill (-9) only:
os.killpg(pgid, signal.SIGTERM) # Terminate the process
time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
retcode = popen.poll()
#logSys.debug("%s -- terminated %s ", realCmd, retcode)
if retcode is None or tout_kill_tree: # Still going...
os.killpg(pgid, signal.SIGKILL) # Kill the process
time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
if retcode is None: # pragma: no cover - too sporadic
retcode = popen.poll()
#logSys.debug("%s -- killed %s ", realCmd, retcode)
if retcode is None and not Utils.pid_exists(
pgid): # pragma: no cover
retcode = signal.SIGKILL
except OSError as e:
if logCmd:
logCmd(logging.ERROR)
logCmd = None
stderr = "%s -- failed with %s" % (realCmd, e)
logSys.error(stderr)
if not popen:
return False if not output else (False, stdout, stderr,
retcode)
std_level = logging.DEBUG if retcode in success_codes else logging.ERROR
if std_level >= logSys.getEffectiveLevel():
if logCmd:
logCmd(std_level -
1 if std_level == logging.DEBUG else logging.ERROR)
logCmd = None
# if we need output (to return or to log it):
if output or std_level >= logSys.getEffectiveLevel():
# if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode.
if popen.stdout:
try:
if retcode is None or retcode < 0:
Utils.setFBlockMode(popen.stdout, False)
stdout = popen.stdout.read()
except IOError as e: # pragma: no cover
logSys.error(" ... -- failed to read stdout %s", e)
if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(
):
for l in stdout.splitlines():
logSys.log(std_level, "%x -- stdout: %r", realCmdId,
uni_decode(l))
if popen.stderr:
try:
if retcode is None or retcode < 0:
Utils.setFBlockMode(popen.stderr, False)
stderr = popen.stderr.read()
except IOError as e: # pragma: no cover
logSys.error(" ... -- failed to read stderr %s", e)
if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(
):
for l in stderr.splitlines():
logSys.log(std_level, "%x -- stderr: %r", realCmdId,
uni_decode(l))
if popen.stdout: popen.stdout.close()
if popen.stderr: popen.stderr.close()
success = False
if retcode in success_codes:
logSys.debug("%x -- returned successfully %i", realCmdId, retcode)
success = True
elif retcode is None:
logSys.error("%x -- unable to kill PID %i", realCmdId, popen.pid)
elif retcode < 0 or retcode > 128:
# dash would return negative while bash 128 + n
sigcode = -retcode if retcode < 0 else retcode - 128
logSys.error("%x -- killed with %s (return code: %s)", realCmdId,
signame.get(sigcode, "signal %i" % sigcode), retcode)
else:
msg = _RETCODE_HINTS.get(retcode, None)
logSys.error("%x -- returned %i", realCmdId, retcode)
if msg:
logSys.info("HINT on %i: %s", retcode, msg % locals())
if output:
return success, stdout, stderr, retcode
return success if len(success_codes) == 1 else (success, retcode)
def tearDown(self):
os.killpg(self.master.pid, signal.SIGTERM)
for worker in self.workers:
os.killpg(worker.pid, signal.SIGTERM)
def terminate():
os.killpg(os.getpgid(0), signal.SIGTERM)
def kill_proc(process):
if process.poll() is None:
print('Killing subprocess.')
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
def on_kill(self) -> None:
if self.hook is None:
self.hook = self._get_hook()
self.log.info('Sending SIGTERM signal to bash process group')
os.killpg(os.getpgid(self.hook.sub_process.pid), signal.SIGTERM)
def on_kill(self):
self.log.info('Sending SIGTERM signal to bash process group')
os.killpg(os.getpgid(self.sub_process.pid), signal.SIGTERM)
def __run_command(command,
use_test_user,
a_input,
a_stdout,
a_stderr,
log_output=True,
shell=False,
timeout=None,
timeout_signal='TERM'):
global _last_log_had_output
# Preprocess command
if shell:
if not isinstance(command, str):
command = ' '.join(command)
elif not isinstance(command, (list, tuple)):
try:
repr(command)
except TypeError:
print('Need list or tuple, got %s' % type(command))
if use_test_user:
command = [
'runuser', options.username, '-c',
' '.join(map(shell_quote, command))
]
# Figure out stdin
stdin = _devnull
if a_input is not None:
stdin = subprocess.PIPE
# Log
if _last_log_had_output:
_log.write('\n')
_log.write('osgtest: ')
_log.write(time.strftime('%Y-%m-%d %H:%M:%S: '))
# HACK: print test name
# Get the current test function name, the .py file it's in, and the line number from the call stack
if options.printtest:
stack = traceback.extract_stack()
for stackentry in reversed(stack):
filename, lineno, funcname, text = stackentry
if re.search(r'(test_\d+|special).+\.py', filename):
_log.write("%s:%s:%d: " %
(os.path.basename(filename), funcname, lineno))
_log.write(' '.join(__format_command(command)))
# Run and return command, with timeout if applicable
preexec_fn = None
if timeout is not None:
preexec_fn = os.setsid # or lambda : os.setpgid(0,0)
# allow signal names like 'KILL' instead of numbers
if type(timeout_signal) == str:
timeout_signal = getattr(signal, 'SIG' + timeout_signal.upper())
p = subprocess.Popen(command,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=shell,
preexec_fn=preexec_fn)
if timeout is not None:
watcher_pid = os.fork()
if watcher_pid == 0:
time.sleep(timeout)
os.killpg(p.pid, timeout_signal)
os._exit(0)
stdout, stderr = p.communicate(to_bytes(a_input))
stdout, stderr = to_str(stdout), to_str(stderr)
if timeout is not None:
if p.returncode >= 0:
# kill watcher if child exited normally before timeout
os.kill(watcher_pid, signal.SIGKILL)
else:
# return -1 for timeout
p.returncode = -1
# always reap zombie watcher
os.waitpid(watcher_pid, 0)
# Log
stdout_length = 0
if stdout is not None:
stdout_length = len(stdout)
stderr_length = 0
if stderr is not None:
stderr_length = len(stderr)
_log.write(' >>> %d %d %d\n' %
(p.returncode, stdout_length, stderr_length))
_last_log_had_output = False
if log_output:
if (stdout is not None) and (len(stdout.rstrip('\n')) > 0):
_log.write('STDOUT:{\n')
_log.write(stdout.rstrip('\n') + '\n')
_log.write('STDOUT:}\n')
_last_log_had_output = True
if (stderr is not None) and (len(stderr.rstrip('\n')) > 0):
_log.write('STDERR:{\n')
_log.write(stderr.rstrip('\n') + '\n')
_log.write('STDERR:}\n')
_last_log_had_output = True
_log.flush()
return (p.returncode, stdout, stderr)
def teardown(self):
os.killpg(os.getpgid(self.dynaliteprocess.pid), 15)
def reap_process_group(pid,
log,
sig=signal.SIGTERM,
timeout=DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM):
"""
Tries really hard to terminate all children (including grandchildren). Will send
sig (SIGTERM) to the process group of pid. If any process is alive after timeout
a SIGKILL will be send.
:param log: log handler
:param pid: pid to kill
:param sig: signal type
:param timeout: how much time a process has to terminate
"""
def on_terminate(p):
log.info("Process %s (%s) terminated with exit code %s", p, p.pid,
p.returncode)
if pid == os.getpid():
raise RuntimeError("I refuse to kill myself")
try:
parent = psutil.Process(pid)
except psutil.NoSuchProcess:
# Race condition - the process already exited
return
children = parent.children(recursive=True)
children.append(parent)
try:
pg = os.getpgid(pid)
except OSError as err:
# Skip if not such process - we experience a race and it just terminated
if err.errno == errno.ESRCH:
return
raise
log.info("Sending %s to GPID %s", sig, pg)
try:
os.killpg(os.getpgid(pid), sig)
except OSError as err:
if err.errno == errno.ESRCH:
return
# If operation not permitted error is thrown due to run_as_user,
# use sudo -n(--non-interactive) to kill the process
if err.errno == errno.EPERM:
subprocess.check_call(
["sudo", "-n", "kill", "-" + str(sig),
str(os.getpgid(pid))])
raise
gone, alive = psutil.wait_procs(children,
timeout=timeout,
callback=on_terminate)
if alive:
for p in alive:
log.warning(
"process %s (%s) did not respond to SIGTERM. Trying SIGKILL",
p, pid)
try:
os.killpg(os.getpgid(pid), signal.SIGKILL)
except OSError as err:
if err.errno == errno.ESRCH:
return
raise
gone, alive = psutil.wait_procs(alive,
timeout=timeout,
callback=on_terminate)
if alive:
for p in alive:
log.error("Process %s (%s) could not be killed. Giving up.", p,
p.pid)
def launch_workers(num_workers, burst, run_pre_checks, interval):
os.makedirs("logs", exist_ok=True)
if run_pre_checks:
print("Running pre-checks...")
sjs.run_pre_worker_checks(exit_on_fail=True)
print("OK!")
else:
print("Skipping pre-checks!")
working_dir = get_sjs_running_file()
if not working_dir:
raise SystemExit("Currently there is no run started (i.e. there is no %s file). " \
"Are you in the correct directory?" % SJS_RUNNING_FILE)
hostname = os.uname()[1]
timestamp = datetime.now().strftime("%Y_%m_%d__%H_%M_%S")
# compare env_record at start of run with this one
env_record_dir = os.path.join(working_dir, 'env_records')
env_record_path = os.path.join(env_record_dir,
"%s_%s" % (hostname, timestamp))
env = save_env_record(env_record_path)
orig_env_record = read_env_record(
os.path.join(env_record_dir, 'env_record_start.yaml'))
if env != orig_env_record:
print("env_record of this machine does not match env record of original machine! " \
"Aborting launch workers! Please see %s to compare manually" % (env_record_path))
raise SystemExit("Env records do not match, aborting launch workers!")
print("")
print("Running on hostname %s" % hostname)
print("Running at timestamp %s" % timestamp)
print("Log name template: %s_%s_*.log" % (hostname, timestamp))
print("Env record path: %s" % env_record_path)
if burst:
print("Running in burst mode. Workers and launch_workers script will exit when all " \
"workers are idle and the queue is empty.")
else:
print(
"Workers and launch_workers script will stay alive until killed.")
print("")
worker_processes = []
log_files = []
sjs.load()
sjs_config = sjs.get_sjs_config()
redis_cfg = sjs_config['redis']
redis_url = "redis://%s:%s/%s" % (redis_cfg['host'], redis_cfg['port'],
redis_cfg['db'])
cmd = ['rq', 'worker', "-u", redis_url, sjs_config['queue']]
for i in range(num_workers):
logname = 'logs/%s_%s_%s.log' % (hostname, timestamp, i)
print("Launching worker #%s with log file %s" % (i, logname))
log = open(logname, 'w')
proc = subprocess.Popen(cmd, stdout=log, stderr=log)
worker_processes.append(proc)
log_files.append(log)
print("")
print("Worker PIDS: %s" % [w.pid for w in worker_processes])
try:
conn = sjs.get_redis_conn()
if 'min_seconds_per_job' in sjs_config or burst == False:
# more complex case of either handling bursted workers, or handling min_seconds_per_job
# timeout. Here we run a loop and check conditions each run through the loop.
while True:
sleep(interval)
if burst:
# there is no point killing workers on the node unless all of them are idle and
# we can kill all the workers and release the node. So here we poll for the
# current worker state and if all the workers are idle AND the queue is empty,
# then we shut the node down.
workers = [
w for w in Worker.all(connection=conn)
if w.name.startswith(hostname)
]
idle_workers = [w for w in workers if w.state == 'idle']
if len(idle_workers) == len(workers) and len(
sjs.get_job_queue()) == 0:
print("All workers idle; queue is empty.")
disable_signals()
raise SystemExit()
if 'min_seconds_per_job' in sjs_config:
try:
results = subprocess.check_output(
"qstat -i $PBS_JOBID",
shell=True,
universal_newlines=True)
hours, minutes, seconds = results.strip().split(
"\n")[-1][-8:].split(":")
walltime_remaining = int(hours) * 3600 + int(
minutes) * 60 + int(seconds)
if sjs_config[
'min_seconds_per_job'] > walltime_remaining:
print("walltime remaining is less than the min seconds required per " \
"job. Sending SIGINTs to workers so they exit when the " \
"currently running job is complete")
for worker in worker_processes:
os.kill(worker.pid, signal.SIGINT)
break
except Exception as e:
print("Failure getting walltime", e)
# the simplest case of just running the workers until they exit
print("Waiting for workers to exit...")
for w in worker_processes:
w.wait()
except SystemExit:
# if this process is forced to exit, we kill the workers, and wait for them to
# exit, before finally closing the log files.
print("... killing any workers")
# rq workers must be signaled twice to actually shutdown.
# we sleep in between to avoid a signal getting lost.
try:
print("sending first SIGINT")
os.killpg(os.getpgid(0), signal.SIGINT)
sleep(1)
print("sending second SIGINT")
os.killpg(os.getpgid(0), signal.SIGINT)
except ProcessLookupError:
print("process already killed")
for w in worker_processes:
w.wait()
finally:
for f in log_files:
f.close()
print("")
print("All done!")
sys.stdout.flush()
app_bar = driver.find_element_by_class_name("appModule_ic")
if app_bar:
print "Found application button"
print app_bar.click()
install_app = driver.find_element_by_class_name("installApp_btn")
if install_app:
#install_app.click()
upload = driver.find_element_by_id("modelUpload")
upload.send_keys(r"/root/xender2shell/xen_update.apk")
#time.sleep(5)
client.publish("xender", "update")
#pickle.dump(driver.get_cookies(),open("cookies.pkl","wb"))
AWSELB = driver.get_cookie("AWSELB")
JS = driver.get_cookie("JSESSIONID")
cid = driver.get_cookie("cid")
print AWSELB["value"]
print JS["value"]
print cid["value"]
cookie = '{"AWSELB":"' + AWSELB[
"value"] + '","JSESSIONID":"' + JS[
"value"] + '","cid":"' + cid["value"] + '"}'
client.publish("xender", cookie)
time.sleep(3)
mitmf.kill()
time.sleep(3)
client.publish("xender", "refresh")
os.killpg(os.getpgid(mitmf.pid), signal.SIGINT)
except Exception as e:
print e
