def do_execute(self, params, callback):
env = self.create_env(params)
if self.output == 'combined':
child = Subprocess(
self.filename,
env=env,
stdout=Subprocess.STREAM,
stderr=subprocess.STDOUT,
io_loop=IOLoop.instance()
)
retcode, stdout = yield [
gen.Task(child.set_exit_callback),
gen.Task(child.stdout.read_until_close)
]
callback((child.returncode, stdout.split()))
else:
child = Subprocess(
self.filename,
env=env,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
io_loop=IOLoop.instance()
)
retcode, stdout, stderr = yield [
gen.Task(child.set_exit_callback),
gen.Task(child.stdout.read_until_close),
gen.Task(child.stderr.read_until_close)
]
callback((child.returncode, stdout.splitlines(), stderr.splitlines()))
def sed(self, script, fn, stdout, stderr, **kw):
cmd = [self.sedexe, '-u', '-e', script]
if fn:
cmd.append(fn)
p = Subprocess(cmd, stdout=stdout, stderr=stderr, **kw)
log.debug('running sed %s, pid: %s', cmd, p.proc.pid)
return p
def awk(self, script, fn, stdout, stderr, **kw):
cmd = [self.awkexe, '--sandbox', script]
if fn:
cmd.append(fn)
p = Subprocess(cmd, stdout=stdout, stderr=stderr, **kw)
log.debug('running awk %s, pid: %s', cmd, p.proc.pid)
return p
def _start_data_stream(self, log_path):
tail_proc = Subprocess(['tail', '-f', log_path, '-n', '0'],
stdout=Subprocess.STREAM,
bufsize=1)
tail_proc.set_exit_callback(self._close)
self._add_proc(self.task_id, tail_proc)
tail_proc.stdout.read_until('\n', self.write_line_to_clients)
def call_subprocess(cmd, stdin_data=None, stdin_async=True):
"""异步进行系统调用，call sub process async
Args:
cmd: str, commands
stdin_data: str, data for standard in
stdin_async: bool, whether use async for stdin
"""
stdin = Subprocess.STREAM if stdin_async else subprocess.PIPE
sub_process = Subprocess(
shlex.split(cmd),
stdin=stdin,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
)
if stdin_data:
if stdin_async:
yield Task(sub_process.stdin.write, stdin_data)
else:
sub_process.stdin.write(stdin_data)
if stdin_async or stdin_data:
sub_process.stdin.close()
result, error = yield [
Task(sub_process.stdout.read_until_close),
Task(sub_process.stderr.read_until_close),
]
raise Return((result, error))
def run_command(cmd, input=None, env=None):
proc = Subprocess(cmd,
shell=True,
env=env,
stdin=Subprocess.STREAM,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM)
inbytes = None
if input:
inbytes = input.encode()
try:
yield proc.stdin.write(inbytes)
except StreamClosedError as exp:
# Apparently harmless
pass
proc.stdin.close()
out = yield proc.stdout.read_until_close()
eout = yield proc.stderr.read_until_close()
proc.stdout.close()
proc.stderr.close()
eout = eout.decode().strip()
try:
err = yield proc.wait_for_exit()
except CalledProcessError:
#self.log.error("Subprocess returned exitcode %s" % proc.returncode)
#self.log.error(eout)
raise RuntimeError(eout)
if err != 0:
return err # exit error?
else:
out = out.decode().strip()
return out
def do(self, action, **kwargs):
"""Instruct the mediator process to take a given action"""
kwargs['action'] = action
cmd = ['sudo', '-u', self.user.name]
cmd.extend(self.sudo_args)
cmd.append(self.sudospawner_path)
if self.debug_mediator:
cmd.append('--logging=debug')
self.log.debug("Spawning %s", cmd)
p = Subprocess(cmd, stdin=Subprocess.STREAM, stdout=Subprocess.STREAM, stderr=Subprocess.STREAM, preexec_fn=self.make_preexec_fn())
stderr_future = self.relog_stderr(p.stderr)
# hand the stderr future to the IOLoop so it isn't orphaned,
# even though we aren't going to wait for it unless there's an error
IOLoop.current().add_callback(lambda : stderr_future)
yield p.stdin.write(json.dumps(kwargs).encode('utf8'))
p.stdin.close()
data = yield p.stdout.read_until_close()
if p.returncode:
yield stderr_future
raise RuntimeError("sudospawner subprocess failed with exit code: %r" % p.returncode)
data_str = data.decode('utf8', 'replace')
try:
data_str = data_str[data_str.index('{'):data_str.rindex('}')+1]
response = json.loads(data_str)
except ValueError:
self.log.error("Failed to get JSON result from mediator: %r" % data_str)
raise
return response
def open(self):
id_ = self.get_argument('id')
self.p = Subprocess(['tail', '-f', PROCESSES[id_][self.stream]],
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM)
self.p.set_exit_callback(self._close)
self.p.stdout.read_until('\n', self.write_line)
def __terminate_pid_tree(self, pids):
tree_pids = pids.strip() + ' ' + str(self.process.pid)
cmd_str = 'kill ' + tree_pids
Subprocess(shlex.split(cmd_str),
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
io_loop=self.io_loop)
def run(self):
self.start_ts = int(time.time())
self.process = Subprocess(self.cmd,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
env=self.env,
io_loop=self.io_loop)
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=self.process.pid,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id'),
job_id=self.params.get('job_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
self.watcher = self.watch(command)
self.command = command
def __children_pids(self, callback):
cmd_str = 'pgrep -P {pid}'.format(pid=self.process.pid)
sub = Subprocess(shlex.split(cmd_str),
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
io_loop=self.io_loop)
sub.stdout.read_until_close(callback=callback)
async def call_subprocess(
cmd: Union[str, list], stdin_data: Optional[str] = None) \
-> Tuple[int, Union[str, bytes], Union[str, bytes]]:
"""Call sub process async."""
if isinstance(cmd, str):
cmd = shlex.split(cmd)
try:
sub_process = Subprocess(cmd,
stdin=Subprocess.STREAM,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM)
except OSError as e:
return e.errno, '', e.strerror
if stdin_data:
await sub_process.stdin.write(stdin_data)
sub_process.stdin.close()
code, result, error = await multi([
sub_process.wait_for_exit(raise_error=False),
sub_process.stdout.read_until_close(),
sub_process.stderr.read_until_close()
])
result = result.strip()
error = error.strip()
return code, result, error
def _start_feed_raspivid(self):
# The streaming part can be tested independantly of the raspivid command :
# (1) make a test sample.h264 with 'raspivid -t 0 -o - -w 1296 -h 972 -fps 12 -pf baseline > sample.h264'
# (2) cmd = ['sh', '-c', 'cat /opt/camera/sample.h264 | pv -L 500000 -B 500000']
# The raspivid command can be tested over the nerwork with :
# (1) raspivid -t 0 -o - -w 1296 -h 972 -fps 12 -pf baseline | netcat -l -p 9999
# (2) netcat 192.168.1.12 9999 | vlc --demux h264 -
@gen.coroutine
def stream_output():
# https://stackoverflow.com/questions/41431882/live-stream-stdout-and-stdin-with-websocket
try:
while True:
buffered = yield self.streamproc.stdout.read_bytes(
1024 * 1024 * 1, partial=True)
self.streamsplitter.process(buffered)
except StreamClosedError as e:
stamp("stream_output() coroutine closed with error : " +
str(e))
exit()
debug("Opening raspivid process...")
cmd = [
"raspivid", "-t", "0", "-o", "-", "-w",
str(self.xres), "-h",
str(self.yres), "-fps",
str(self.fps), "-pf", "baseline"
]
self.streamproc = Subprocess(cmd, stdout=Subprocess.STREAM)
IOLoop.current().spawn_callback(stream_output)
debug("...raspivid process opened")
def start_env(self, executable, on_stdout, on_stderr):
environment_id = uuid.uuid4().hex
args = [
executable,
str(self.server.PORT),
environment_id
]
stream = Subprocess.STREAM
env = {
'PYTHONUNBUFFERED': '0',
'PYTHONIOENCODING': 'utf-8'
}
try:
sub_process = Subprocess(args=args, executable=executable, stdout=stream,
stderr=stream, env=env)
except Exception as e:
logger.error(e)
raise
def decode_data(func):
def _decode(data):
return func(data.decode('utf-8'))
return _decode
_on_stdout = decode_data(partial(on_stdout, environment_id))
_on_stderr = decode_data(partial(on_stderr, environment_id))
sub_process.stdout.read_until_close(lambda a: a, streaming_callback=_on_stdout)
sub_process.stderr.read_until_close(lambda a: a, streaming_callback=_on_stderr)
self._connections[environment_id] = Future()
return self._connections[environment_id]
def _call_ipptool(self, request):
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_file.write(bytes(request, encoding='utf-8'))
from tornado.process import Subprocess
process = Subprocess([self.config['ipptool_path'],
self.authenticated_uri, '-X',
temp_file.name],
stdin=subprocess.PIPE,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
io_loop=self.io_loop)
future = []
self.io_loop.add_timeout(self.io_loop.time() + self.config['timeout'],
functools.partial(self.timeout_handler,
process.proc, future))
try:
stdout, stderr = yield [Task(process.stdout.read_until_close),
Task(process.stderr.read_until_close)]
if future:
raise TimeoutError
finally:
os.unlink(temp_file.name)
result = plistlib.readPlistFromString(stdout)
try:
raise Return(result['Tests'][0])
except (IndexError, KeyError):
logger = logging.getLogger(__name__)
logger.error('ipptool command failed: {} {}'.format(stdout,
stderr))
raise
def open(self):
filename = "/tmp/simple_foobar.log"
self.proc = Subprocess(["tail", "-f", filename, "-n", "0"],
stdout=Subprocess.STREAM,
bufsize=1)
self.proc.set_exit_callback(self._close)
self.proc.stdout.read_until("\n", self.write_line)
def execute_command(self, command, logger=None):
"""
Executes a command.
Parameters
----------
command : str
Command to be executed.
logger : logging.Logger
Logger.
Returns
-------
future : tornado.gen.Future
Future which will have the execution result.
"""
log = logger or self.log
log.debug('Executing command: %s', command)
proc = Subprocess(shlex.split(command),
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM)
try:
while True:
line_bytes = yield proc.stdout.read_until(b'\n')
line = to_unicode(line_bytes).strip('\r\n')
log.info(line)
except StreamClosedError:
pass
def test_tornado_spa_example(tornado_spa_example_dir):
command = 'jaffle start --disable-color -y'
proc = Subprocess(shlex.split(command),
stdin=None,
stderr=Subprocess.STREAM,
preexec_fn=os.setpgrp)
stdout = []
try:
while True:
line_bytes = yield proc.stderr.read_until(b'\n')
line = to_unicode(line_bytes).rstrip()
stdout.append(line)
if re.search(r'Kernel py_kernel \(.*\) is ready', line):
break
except StreamClosedError:
pass
os.killpg(os.getpgid(proc.proc.pid), signal.SIGINT)
joined_stdout = '\n'.join(stdout)
assert 'Jaffle port:' in stdout[0]
assert 'Starting kernel: py_kernel' in stdout[1]
assert 'Kernel started:' in stdout[2]
assert 'Initializing jaffle.app.watchdog.WatchdogApp on py_kernel' in joined_stdout
assert 'Initializing jaffle.app.tornado.TornadoBridgeApp on py_kernel' in joined_stdout
assert 'Initializing jaffle.app.pytest.PyTestRunnerApp on py_kernel' in joined_stdout
def do(self, action, **kwargs):
"""Instruct the mediator process to take a given action"""
kwargs['action'] = action
cmd = ['sudo', '-u', self.user.name]
cmd.extend(self.sudo_args)
cmd.append(self.sudospawner_path)
if self.debug_mediator:
cmd.append('--logging=debug')
p = Subprocess(cmd,
stdin=Subprocess.STREAM,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM)
f = self.relog_stderr(p.stderr)
# hand the stderr future to the IOLoop so it isn't orphaned,
# even though we aren't going to wait for it
IOLoop.current().add_callback(lambda: f)
yield p.stdin.write(json.dumps(kwargs).encode('utf8'))
p.stdin.close()
data = yield p.stdout.read_until_close()
data_str = data.decode('utf8')
# Trim data outside the json block
data_str = data_str[data_str.index('{'):data_str.rindex('}') + 1]
if p.returncode:
raise RuntimeError("Spawner subprocess failed with exit code: %r" %
p.returncode)
return json.loads(data_str)
def _trace_done(self, returncode):
self.nbapp.log.info("reprozip: tracing done, returned %d", returncode)
if returncode == 0:
# Pack
if self._pack_file.exists():
self._pack_file.remove()
proc = Subprocess([
'reprozip', 'pack', '-d', self._tempdir.path,
self._pack_file.path
],
stdin=subprocess.PIPE)
proc.stdin.close()
proc.set_exit_callback(self._packing_done)
self.nbapp.log.info("reprozip: started packing...")
else:
self._tempdir.rmtree()
if returncode == 3:
self.set_header('Content-Type', 'application/json')
self.finish({
'error':
"There was an error running the notebook. "
"Please make sure that it can run from top to "
"bottom without error before packing."
})
else:
self.send_error(500)
async def send_to_ruby(self, request_json):
env = {
"GEM_HOME": self.__gem_home,
"PCSD_DEBUG": "true" if self.__debug else "false"
}
if self.__no_proxy is not None:
env["NO_PROXY"] = self.__no_proxy
if self.__https_proxy is not None:
env["HTTPS_PROXY"] = self.__https_proxy
pcsd_ruby = Subprocess(
[
self.__ruby_executable, "-I",
self.__pcsd_dir,
self.__pcsd_cmdline_entry
],
stdin=Subprocess.STREAM,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
env=env
)
await Task(pcsd_ruby.stdin.write, str.encode(request_json))
pcsd_ruby.stdin.close()
return await multi([
Task(pcsd_ruby.stdout.read_until_close),
Task(pcsd_ruby.stderr.read_until_close),
])
def _run_process(self, cmd, env=None):
"""
Execute the subprocess of `docker-compose up`
"""
proc = Subprocess(cmd,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
env=env)
app_log.info('read outputs from Subprocess: %s', cmd)
outputs = yield [
proc.stdout.read_until_close(),
proc.stderr.read_until_close()
]
try:
yield proc.wait_for_exit()
except CalledProcessError:
app_log.error('Process is exited abnormally. stderr: %s',
outputs[1])
raise
return {
'stdout': to_unicode(outputs[0]),
'stderr': to_unicode(outputs[1])
}
def open(self):
filename = self.application.logfile
self.proc = Subprocess(["tail", "-f", "-n", "0", filename],
stdout=Subprocess.STREAM,
bufsize=1)
self.proc.set_exit_callback(self._close)
self.proc.stdout.read_until(b"\n", self.write_line)
def test_subprocess(self):
if IOLoop.configured_class().__name__.endswith('LayeredTwistedIOLoop'):
# This test fails non-deterministically with LayeredTwistedIOLoop.
# (the read_until('\n') returns '\n' instead of 'hello\n')
# This probably indicates a problem with either TornadoReactor
# or TwistedIOLoop, but I haven't been able to track it down
# and for now this is just causing spurious travis-ci failures.
raise unittest.SkipTest("Subprocess tests not compatible with "
"LayeredTwistedIOLoop")
subproc = Subprocess([sys.executable, '-u', '-i'],
stdin=Subprocess.STREAM,
stdout=Subprocess.STREAM, stderr=subprocess.STDOUT,
io_loop=self.io_loop)
self.addCleanup(lambda: (subproc.proc.terminate(), subproc.proc.wait()))
subproc.stdout.read_until(b'>>> ', self.stop)
self.wait()
subproc.stdin.write(b"print('hello')\n")
subproc.stdout.read_until(b'\n', self.stop)
data = self.wait()
self.assertEqual(data, b"hello\n")
subproc.stdout.read_until(b">>> ", self.stop)
self.wait()
subproc.stdin.write(b"raise SystemExit\n")
subproc.stdout.read_until_close(self.stop)
data = self.wait()
self.assertEqual(data, b"")
def test_wait_for_exit_raise_disabled(self):
skip_if_twisted()
Subprocess.initialize()
self.addCleanup(Subprocess.uninitialize)
subproc = Subprocess([sys.executable, '-c', 'import sys; sys.exit(1)'])
ret = yield subproc.wait_for_exit(raise_error=False)
self.assertEqual(ret, 1)
def test_wait_for_exit_raise(self):
Subprocess.initialize()
self.addCleanup(Subprocess.uninitialize)
subproc = Subprocess([sys.executable, "-c", "import sys; sys.exit(1)"])
with self.assertRaises(subprocess.CalledProcessError) as cm:
yield subproc.wait_for_exit()
self.assertEqual(cm.exception.returncode, 1)
def test_sigchild_signal(self):
Subprocess.initialize()
self.addCleanup(Subprocess.uninitialize)
subproc = Subprocess(
[sys.executable, "-c", "import time; time.sleep(30)"],
stdout=Subprocess.STREAM,
)
self.addCleanup(subproc.stdout.close)
subproc.set_exit_callback(self.stop)
os.kill(subproc.pid, signal.SIGTERM)
try:
ret = self.wait(timeout=1.0)
except AssertionError:
# We failed to get the termination signal. This test is
# occasionally flaky on pypy, so try to get a little more
# information: did the process close its stdout
# (indicating that the problem is in the parent process's
# signal handling) or did the child process somehow fail
# to terminate?
fut = subproc.stdout.read_until_close()
fut.add_done_callback(lambda f: self.stop()) # type: ignore
try:
self.wait(timeout=1.0)
except AssertionError:
raise AssertionError("subprocess failed to terminate")
else:
raise AssertionError("subprocess closed stdout but failed to "
"get termination signal")
self.assertEqual(subproc.returncode, ret)
self.assertEqual(ret, -signal.SIGTERM)
def test_sigchild_future(self):
Subprocess.initialize()
self.addCleanup(Subprocess.uninitialize)
subproc = Subprocess([sys.executable, "-c", "pass"])
ret = yield subproc.wait_for_exit()
self.assertEqual(ret, 0)
self.assertEqual(subproc.returncode, ret)
def countdown_handler(self, interval, count):
command = '{0}/countdown -i {1} {2}'.format(os.getcwd(), interval,
count)
proc = Subprocess(shlex.split(command), stdout=Subprocess.STREAM)
try:
while True:
line_bytes = yield proc.stdout.read_until(b'\n')
line = to_unicode(line_bytes)[:-1]
self.log.info('command read: %s', line)
timestamp = datetime.now().timestamp()
self.zmq_stream.send_multipart([
b'0',
utf8(
json_encode({
'stdout': line,
'finished': False,
'timestamp': timestamp
}))
])
except StreamClosedError:
self.log.info('command closed')
timestamp = datetime.now().timestamp()
self.zmq_stream.send_multipart([
b'0',
utf8(
json_encode({
'stdout': None,
'finished': True,
'timestamp': timestamp
}))
])
async def make_thumbnail(path):
duration = None
ex = 'jpg'
if path.split('.')[-1].lower() == 'png' or path.split(
'.')[-1].lower() == 'gif':
ex = 'png'
tname = "uploads/{0}_thumb.{1}".format(get_basename(path), ex)
if get_extension(path) in image_extensions:
width, height = await get_image_size(path)
scale = min(
float(thumbnail_size[0]) / width,
float(thumbnail_size[1]) / height, 1.0)
twidth = int(scale * width)
theight = int(scale * height)
tsize = '%sx%s!' % (twidth, theight)
ps = Subprocess(
['convert', path + '[0]', '-thumbnail', tsize, '-strip', tname],
stderr=PIPE,
stdout=PIPE)
try:
ret = await ps.wait_for_exit()
except:
return 'static/missing_thumbnail.jpg'
return tname
elif get_extension(path) in video_extensions:
width, height, duration = await get_video_size(path)
scale = min(
float(thumbnail_size[0]) / width,
float(thumbnail_size[1]) / height, 1.0)
twidth = int(scale * width)
theight = int(scale * height)
tsize = '%sx%s' % (twidth, theight)
ps = Subprocess([
'ffmpeg', '-i', path, '-y', '-s', tsize, '-vframes', '1', '-f',
'image2', '-c:v', 'mjpeg', tname
],
stderr=PIPE,
stdout=PIPE)
try:
ret = await ps.wait_for_exit()
except:
return 'static/missing_thumbnail.jpg'
return tname
elif get_extension(path) in audio_extensions:
return None
else:
raise Exception("Format not supported")
