async def shell_out(*args, **kwargs):
stdin = kwargs.get('stdin', None)
if stdin is not None:
p = process.Subprocess(args,
stdin=process.Subprocess.STREAM,
stdout=process.Subprocess.STREAM)
await p.stdin.write(stdin.encode('utf-8'))
p.stdin.close()
else:
p = process.Subprocess(args, stdout=process.Subprocess.STREAM)
out = await p.stdout.read_until_close()
exit_code = await p.wait_for_exit(raise_error=True)
return out.decode('utf-8')
def play_stream(self, info):
self.kill_stream()
download_cmd = ["rtmpdump",
"-r", "rtmpe://%s%s" % (info['streamHost'], info['streamApp']),
"-a", info['streamApp'][1:],
"-y", "mp3:" + info['surl'],
"-o", "-",
]
play_cmd = ["mplayer", "-cache", "2048", "-quiet", "-"]
self.download_p = process.Subprocess(
download_cmd, stdout=subprocess.PIPE, io_loop=ioloop.IOLoop.instance())
self.play_p = process.Subprocess(
play_cmd, stdin=self.download_p.stdout, io_loop=ioloop.IOLoop.instance())
self.play_p.set_exit_callback(self.stream_ended_cb(self.play_p))
def grep(self, regex, fn, stdout, stderr, **kw):
cmd = [self.toolpaths.cmd_grep, '--text', '--line-buffered', '--color=never', '-e', regex]
if fn:
cmd.append(fn)
proc = process.Subprocess(cmd, stdout=stdout, stderr=stderr, **kw)
log.debug('running grep %s, pid: %s', cmd, proc.proc.pid)
return proc
def open(self, *args, **kwargs):
log.info("Spawning ccls subprocess")
# Create an instance of the language server
proc = process.Subprocess([
'ccls',
'--init={"capabilities": {"foldingRangeProvider": false}, "index": {"onChange": true, "trackDependency":2}, "clang": {"resourceDir": "/home/CppLanguageServer/clang+llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04/lib/clang/10.0.0"}}'
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Create a writer that formats json messages with the correct LSP headers
self.writer = streams.JsonRpcStreamWriter(proc.stdin)
# Create a reader for consuming stdout of the language server. We need to
# consume this in another thread
def consume():
# Start a tornado IOLoop for reading/writing to the process in this thread
ioloop.IOLoop()
reader = streams.JsonRpcStreamReader(proc.stdout)
reader.listen(lambda msg: self.write_message(json.dumps(msg)))
thread = threading.Thread(target=consume)
thread.daemon = True
thread.start()
def _Run(callback):
"""Grab the lock and run all commands an subprocesess."""
job_set = options.options.job_set
assert job_set in kJobSets.keys(
), '--job_set must be one of %r' % kJobSets.keys()
jobs = kJobSets[job_set]
client = db_client.DBClient.Instance()
job = Job(client, job_set)
got_lock = yield gen.Task(job.AcquireLock)
if not got_lock:
logging.warning('Failed to acquire job lock: exiting.')
callback()
return
# Wrap entire call inside a try to make sure we always release the lock.
try:
for title, args in jobs:
logging.info('[%s] running %s' % (title, ' '.join(args)))
# Run the task and wait for the termination callback.
proc = process.Subprocess(args, io_loop=ioloop.IOLoop.instance())
code = yield gen.Task(proc.set_exit_callback)
logging.info('[%s] finished with code: %r' % (title, code))
except:
logging.error(traceback.format_exc())
finally:
yield gen.Task(job.ReleaseLock)
callback()
def handle_connection(connection, address):
stream = iostream.IOStream(connection)
callback = functools.partial(_handle_headers, stream)
stream.read_until("\r\n\r\n", callback)
stream.write("HTTP/1.0 200 OK\r\n")
stream.write("Content-Type: image/gif\r\n")
stream.write("\r\n")
stream.write(HEADER_DATA)
stream.write(INITIAL_FRAME)
#exec traceroute against host ip
remote_ip = address[0]
traceroute_proc = process.Subprocess(
['mtr', '-c', '3', '-r', '-o', 'LSD BAW', remote_ip],
stdout=process.Subprocess.STREAM)
traceroute_proc.initialize()
output_buffer = []
_cb = functools.partial(_handleResult, output_buffer)
result = traceroute_proc.stdout.read_until_close(callback=_cb,
streaming_callback=_cb)
if CLOSE_ON_TIMEOUT:
callback = functools.partial(closestream, stream, traceroute_proc)
ioloop.IOLoop.instance().add_timeout(
datetime.timedelta(seconds=MAX_TIME), callback)
streams.append((stream, traceroute_proc, output_buffer, 0))
def tail(self, ip, n, fn, stdout, stderr, **kw):
flag_follow = '-F' if self.follow_names else '-f'
if ip:
user, password = self.get_user_pass(ip)
if '\\' in fn: #win system
cmd = [
self.toolpaths.cmd_sshpass, '-p', password,
self.toolpaths.cmd_ssh, '{}@{}'.format(user,
ip), 'tail', '-n',
str(n), flag_follow, '\"{}\"'.format(fn[1:])
]
else:
cmd = [
self.toolpaths.cmd_sshpass, '-p', password,
self.toolpaths.cmd_ssh, '{}@{}'.format(user, ip),
self.toolpaths.cmd_tail, '-n',
str(n), flag_follow, fn
]
else:
cmd = [self.toolpaths.cmd_tail, '-n', str(n), flag_follow, fn]
proc = process.Subprocess(cmd,
stdout=stdout,
stderr=stderr,
bufsize=1,
**kw)
log.debug('running tail %s, pid: %s', cmd, proc.proc.pid)
return proc
def awk(self, script, fn, stdout, stderr, **kw):
cmd = [self.toolpaths.cmd_awk, '--sandbox', script]
if fn:
cmd.append(fn)
proc = process.Subprocess(cmd, stdout=stdout, stderr=stderr, **kw)
log.debug('running awk %s, pid: %s', cmd, proc.proc.pid)
return proc
def sed(self, script, fn, stdout, stderr, **kw):
cmd = [self.toolpaths.cmd_sed, '-u', '-e', script]
if fn:
cmd.append(fn)
proc = process.Subprocess(cmd, stdout=stdout, stderr=stderr, **kw)
log.debug('running sed %s, pid: %s', cmd, proc.proc.pid)
return proc
def convert_videos():
global conversion_queue
global converting_lock
try:
remaining = [
key for key in conversion_queue.keys()
if not conversion_queue[key]['status']
]
video = sorted(remaining,
key=lambda v: conversion_queue[v]['added'])[0]
conversion_queue[video]['status'] = True
except Exception:
return
with (yield converting_lock.acquire()):
logging.info('Converting: %s', video)
audio_file = './audio/{}.mp3'.format(video)
ffmpeg_process = process.Subprocess([
'ffmpeg', '-loglevel', 'panic', '-y', '-i',
get_youtube_url(video), '-f', 'mp3', audio_file + '.temp'
])
try:
yield ffmpeg_process.wait_for_exit()
os.rename(audio_file + '.temp', audio_file)
except Exception as ex:
logging.error('Error converting file: %s', ex.reason)
os.remove(audio_file + '.temp')
finally:
del conversion_queue[video]
async def async_call(self, args=None, timeout=None):
"""
Calls system command and return parsed output or standard error output
"""
if args is None:
args = []
# Executing command with Tornado subprocess is possible only in main thread
if threading.main_thread().ident != threading.get_ident():
return self.call(args=args, timeout=timeout)
all_args = [
self.CMD if self.CMD is not None else cfg['tools.%s.cmd' %
self.NAME]
]
all_args.extend(self.COMMON_ARGS)
all_args.extend(args)
cmd = ' '.join(all_args),
log.debug('Executing: %s', cmd)
if self._cancelled:
raise Exception('Task was cancelled')
task = process.Subprocess(all_args,
stderr=process.Subprocess.STREAM,
stdout=process.Subprocess.STREAM)
self.proc = task.proc
coroutine = gen.multi([
task.wait_for_exit(raise_error=False),
task.stdout.read_until_close(),
task.stderr.read_until_close()
])
if not timeout:
return_code, stdout, stderr = await coroutine
else:
try:
return_code, stdout, stderr = await gen.with_timeout(
timedelta(seconds=timeout), coroutine)
except gen.TimeoutError as exception:
log.exception(
"Command %s timed out after %s while executing %s",
self.NAME, timeout, cmd)
task.proc.kill()
raise exception
self.proc = None
if return_code != 0:
log.warning("Command '%s' failed wit exit code: %s", cmd,
return_code)
log.debug("Command '%s':\nSTDOUT:\n%s\nSTDERR:\n%s", cmd, stdout,
stderr)
if self.RAISE_ERROR:
raise subprocess.CalledProcessError(return_code, cmd)
return self.parser.parse(stdout.decode('utf-8'),
stderr.decode('utf-8'))
def process(self):
self.setState(self.JOB_PROCESSING)
Log("[JOB:%d] process", self.d['id'])
cmd = './processor/' + self.d['processor']
argv = [cmd, str(self.d['id']), self.d['args']]
sp = process.Subprocess(argv)
sp.set_exit_callback(self.process_callback)
def run(self):
process.Subprocess.initialize()
self.p = process.Subprocess(['marian-server', '-c',
self.service['configuration'],
'--quiet-translation',
'-p', self.service['port']])
self.p.set_exit_callback(self.on_exit)
ret = yield self.p.wait_for_exit()
def zcat(self, fn, stdout, stderr, **kw):
cmd = [self.toolpaths.cmd_zcat, '-f', '-r', fn]
proc = process.Subprocess(cmd,
stdout=stdout,
stderr=stderr,
bufsize=1,
**kw)
log.debug('running zcat %s, pid: %s', cmd, proc.proc.pid)
return proc
async def test_fork(server):
"""
This test should not fail. Some Subprocess'es can make the ioloop hang, this tests fails when that happens.
"""
i = 0
while i < 5:
i += 1
sub_process = process.Subprocess(["true"])
await sub_process.wait_for_exit(raise_error=False)
def start_process(self):
"""
Start the rstudio process
"""
self.state['starting'] = True
try:
cmd = self.cmd + [
'--user-identity=' + getpass.getuser(),
'--www-port=' + str(self.port)
]
server_env = os.environ.copy()
# Seed RStudio's R and RSTUDIO env variables
server_env.update(self.env)
try:
r_vars = detectR()
server_env.update(r_vars)
except:
raise web.HTTPError(reason='could not detect R',
status_code=500)
@gen.coroutine
def exit_callback(code):
"""
Callback when the rsessionproxy dies
"""
self.log.info(
'rsession process died with code {}'.format(code))
del self.state['proc']
if code != 0:
yield self.start_process()
# Runs rsession in background
proc = process.Subprocess(cmd, env=server_env)
self.log.info('Starting rsession process...')
self.state['proc'] = proc
proc.set_exit_callback(exit_callback)
for i in range(5):
if (yield self.is_running()):
self.log.info('rsession startup complete')
break
# Simple exponential backoff
wait_time = max(1.4**i, 5)
self.log.debug(
'Waiting {} before checking if rstudio is up'.format(
wait_time))
yield gen.sleep(wait_time)
else:
raise web.HTTPError('could not start rsession in time',
status_code=500)
finally:
self.state['starting'] = False
async def get(self, field):
args = journalArgs.copy()
args.append('--field={}'.format(field))
journalProc = process.Subprocess(args,
stdout=process.Subprocess.STREAM)
values = await journalProc.stdout.read_until_close()
self.writeJson(values.decode().splitlines())
await journalProc.wait_for_exit(raise_error=False)
def test_fork(server, io_loop):
"""
This test should not fail. Some Subprocess'es can make the ioloop hang, this tests fails when that happens.
"""
i = 0
while i < 5:
i += 1
sub_process = process.Subprocess(["true"])
yield sub_process.wait_for_exit(raise_error=False)
sub_process.uninitialize()
def tail(self, n, fn, stdout, stderr, **kw):
flag_follow = '-F' if self.follow_names else '-f'
cmd = [self.toolpaths.cmd_tail, '-n', str(n), flag_follow, fn]
proc = process.Subprocess(cmd,
stdout=stdout,
stderr=stderr,
bufsize=1,
**kw)
log.debug('running tail %s, pid: %s', cmd, proc.proc.pid)
return proc
async def start_process(self):
"""
Start the process
"""
if 'starting' in self.state:
raise Exception(
"Process {} start already pending, can not start again".format(
self.name))
if 'proc' in self.state:
raise Exception("Process {} already running, can not start".format(
self.name))
self.state['starting'] = True
cmd = self.get_cmd()
server_env = os.environ.copy()
# Set up extra environment variables for process
server_env.update(self.get_env())
def exit_callback(code):
"""
Callback when the process dies
"""
self.log.info('{} died with code {}'.format(self.name, code))
self.state.pop('proc', None)
if code != 0 and not 'starting' in self.state:
ioloop.IOLoop.current().add_callback(self.start_process)
# Runs process in background
self.log.info('Starting process...')
proc = process.Subprocess(cmd, env=server_env, cwd=self.get_cwd())
proc.set_exit_callback(exit_callback)
for i in range(8):
if (await self.is_running(proc)):
self.log.info('{} startup complete'.format(self.name))
break
# Simple exponential backoff
wait_time = 1.4**i
self.log.debug(
'Waiting {} seconds before checking if {} is up'.format(
wait_time, self.name))
await gen.sleep(wait_time)
else:
# clear starting state for failed start
self.state.pop('starting', None)
# terminate process
proc.terminate()
raise web.HTTPError(500,
'could not start {} in time'.format(self.name))
# add proc to state only after we are sure it has started
self.state['proc'] = proc
del self.state['starting']
async def get_network_info():
fd_network = process.Subprocess("netstat -i | tail -n +2",
**SUBPROCESS_OPTS).stdout
network_result = await fd_network.read_until_close()
network_result = network_result.decode('utf-8').strip()
network_result = network_result.split("\n")
header = network_result[0].split()
interfaces = network_result[1:]
result = []
for iface in interfaces:
result.append(
{key: value
for key, value in zip(header, iface.split())})
return result
def open(self, *args, **kwargs):
proc = process.Subprocess(
[
'pyls', '-v'
], # 具体的LSP实现进程，如 'pyls -v'、'ccls --init={"index": {"onChange": true}}'等
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
self.writer = streams.JsonRpcStreamWriter(proc.stdin)
def consume():
ioloop.IOLoop()
reader = streams.JsonRpcStreamReader(proc.stdout)
reader.listen(lambda msg: self.write_message(json.dumps(msg)))
thread = threading.Thread(target=consume)
thread.daemon = True
thread.start()
def _exec(self):
self.clean() # Clean up old plans
with tempfile.TemporaryDirectory() as tmpdir:
cmd = (
'/usr/bin/env', 'python3',
bbctrl.get_resource('plan.py'),
os.path.abspath(self.gcode), json.dumps(self.state),
json.dumps(self.config),
'--max-time=%s' % self.preplanner.max_plan_time,
'--max-loop=%s' % self.preplanner.max_loop_time
)
self.preplanner.log.info('Running: %s', cmd)
proc = process.Subprocess(cmd, stdout = process.Subprocess.STREAM,
stderr = process.Subprocess.STREAM,
cwd = tmpdir)
errs = ''
self.pid = proc.proc.pid
try:
try:
while True:
line = yield proc.stdout.read_until(b'\n')
self.progress = float(line.strip())
if self.cancel: return
except iostream.StreamClosedError: pass
self.progress = 1
ret = yield proc.wait_for_exit(False)
if ret:
errs = yield proc.stderr.read_until_close()
raise Exception('Plan failed: ' + errs.decode('utf8'))
finally:
proc.stderr.close()
proc.stdout.close()
if not self.cancel:
os.rename(tmpdir + '/meta.json', self.files[0])
os.rename(tmpdir + '/positions.gz', self.files[1])
os.rename(tmpdir + '/speeds.gz', self.files[2])
os.sync()
def prefetch(self):
'''Prefetch files.'''
path_set = set([self.code_path])
for root, _, files in os.walk(self.res_path):
for filename in files:
path_set.add(os.path.abspath(os.path.join(root, filename)))
path_list = list(path_set)
proc_list = []
with StackContext(Privilege.fileaccess):
for idx in range(0, len(path_list), 16):
proc_list.append(process.Subprocess(
['./Prefetch.py'] + path_list[idx:idx + 16],
stdout=process.Subprocess.STREAM))
for proc in proc_list:
yield proc.stdout.read_bytes(2)
async def call_subprocess(args: List[str], stdin_data: Optional[bytes] = None) -> Tuple[int, bytes, bytes]:
p = tornado_process.Subprocess(
args,
stdin=tornado_process.Subprocess.STREAM,
stdout=tornado_process.Subprocess.STREAM,
stderr=tornado_process.Subprocess.STREAM
)
if stdin_data:
await p.stdin.write(stdin_data)
p.stdin.close()
exit_future = p.wait_for_exit(raise_error=False)
stdout_future = p.stdout.read_until_close()
stderr_future = p.stderr.read_until_close()
await asyncio.wait({exit_future, stdout_future, stderr_future})
return exit_future.result(), stdout_future.result(), stderr_future.result()
async def readJournal(self, args):
journalProc = process.Subprocess(args,
stdout=process.Subprocess.STREAM)
try:
while not self.task.cancelled():
msg = await journalProc.stdout.read_until(b'\n')
msg = json.loads(msg.decode())
if not '_SOURCE_REALTIME_TIMESTAMP' in msg:
msg['_SOURCE_REALTIME_TIMESTAMP'] = msg[
'__REALTIME_TIMESTAMP']
msg = {k: v for k, v in msg.items() if not k.startswith('__')}
self.write_messageJson(msg)
except (iostream.StreamClosedError, asyncio.CancelledError):
pass
finally:
self.close()
journalProc.proc.terminate()
await journalProc.wait_for_exit(raise_error=False)
def open(self, *args, **kwargs):
log.info("Spawning pyls subprocess")
# Create an instance of the language server
proc = process.Subprocess(['pyls', '-v'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Create a writer that formats json messages with the correct LSP headers
self.writer = streams.JsonRpcStreamWriter(proc.stdin)
# Create a reader for consuming stdout of the language server. We need to
# consume this in another thread
def consume():
# Start a tornado IOLoop for reading/writing to the process in this thread
ioloop.IOLoop()
reader = streams.JsonRpcStreamReader(proc.stdout)
reader.listen(lambda msg: self.write_message(json.dumps(msg)))
thread = threading.Thread(target=consume)
thread.daemon = True
thread.start()
def run(self):
process.Subprocess.initialize()
self.p = process.Subprocess([
'marian-server',
'-c',
self.service['configuration'],
'-p',
self.service['port'],
'--allow-unk',
# enables translation with a mini-batch size of 64, i.e. translating 64 sentences at once, with a beam-size of 6.
'-b',
'6',
'--mini-batch',
'64',
# use a length-normalization weight of 0.6 (this usually increases BLEU a bit).
'--normalize',
'0.6',
'--maxi-batch-sort',
'src',
'--maxi-batch',
'100',
])
self.p.set_exit_callback(self.on_exit)
ret = yield self.p.wait_for_exit()
def export(self, dry_run, agent_map):
"""
Export a version to the embedded server
"""
inmanta_path = [sys.executable, "-m", "inmanta.app"]
cmd = inmanta_path + [
"-vvv", "export", "-e",
str(self._environment_id), "--server_address", "localhost",
"--server_port",
str(self._server_port)
]
sub_process = process.Subprocess(cmd,
stdout=process.Subprocess.STREAM,
stderr=process.Subprocess.STREAM)
log_out, log_err, returncode = yield [
sub_process.stdout.read_until_close(),
sub_process.stderr.read_until_close(),
sub_process.wait_for_exit(raise_error=False)
]
sub_process.uninitialize()
if returncode > 0:
print("An error occurred while compiling the model:")
if len(log_out) > 0:
print(log_out.decode())
if len(log_err) > 0:
print(log_err.decode())
return False
LOGGER.info("Export of model complete")
yield self.deploy(dry_run, agent_map)
return True
def mv(from_path, to_path):
"""移动文件"""
cmd = ['mv', from_path, to_path]
ps = process.Subprocess(cmd)
result = yield ps.wait_for_exit(raise_error=False)
raise gen.Return(result == 0)