def test_feeder(self):
feeder = Feeder()
p = capture_stdout([sys.executable, 'echoer.py'],
input=feeder,
async=True)
try:
lines = ('hello', 'goodbye')
gen = iter(lines)
# p.commands may not be set yet (separate thread)
while not p.commands or p.commands[0].returncode is None:
logger.debug('commands: %s', p.commands)
try:
data = next(gen)
except StopIteration:
break
feeder.feed(data + '\n')
if p.commands:
p.commands[0].poll()
time.sleep(0.05) # wait for child to return echo
finally:
# p.commands may not be set yet (separate thread)
if p.commands:
p.commands[0].terminate()
feeder.close()
self.assertEqual(p.stdout.text.splitlines(),
['hello hello', 'goodbye goodbye'])
def cli_cmd_sync(cmd, log_obj=None, write_dots=False, on_out=None, on_err=None, cwd=None):
"""
Runs command line task synchronously
:return:
"""
feeder = Feeder()
p = run(cmd,
input=feeder, async=True,
stdout=Capture(buffer_size=1),
stderr=Capture(buffer_size=1),
cwd=cwd)
out_acc = []
err_acc = []
ret_code = 1
log = None
close_log = False
# Logging - either filename or logger itself
if log_obj is not None:
if isinstance(log_obj, types.StringTypes):
delete_file_backup(log_obj, chmod=0o600)
log = safe_open(log_obj, mode='w', chmod=0o600)
close_log = True
else:
log = log_obj
try:
while len(p.commands) == 0:
time.sleep(0.15)
while p.commands[0].returncode is None:
out = p.stdout.readline()
err = p.stderr.readline()
# If output - react on input challenges
if out is not None and len(out) > 0:
out_acc.append(out)
if log is not None:
log.write(out)
log.flush()
if write_dots:
sys.stderr.write('.')
if on_out is not None:
on_out(out, feeder, p)
# Collect error
if err is not None and len(err) > 0:
err_acc.append(err)
if log is not None:
log.write(err)
log.flush()
if write_dots:
sys.stderr.write('.')
if on_err is not None:
on_err(err, feeder, p)
p.commands[0].poll()
time.sleep(0.01)
ret_code = p.commands[0].returncode
# Collect output to accumulator
rest_out = p.stdout.readlines()
if rest_out is not None and len(rest_out) > 0:
for out in rest_out:
out_acc.append(out)
if log is not None:
log.write(out)
log.flush()
if on_out is not None:
on_out(out, feeder, p)
# Collect error to accumulator
rest_err = p.stderr.readlines()
if rest_err is not None and len(rest_err) > 0:
for err in rest_err:
err_acc.append(err)
if log is not None:
log.write(err)
log.flush()
if on_err is not None:
on_err(err, feeder, p)
return ret_code, out_acc, err_acc
finally:
feeder.close()
if close_log:
log.close()
def cli_cmd_sync(cmd,
log_obj=None,
write_dots=False,
on_out=None,
on_err=None,
cwd=None,
shell=True,
readlines=True,
env=None,
**kwargs):
"""
Runs command line task synchronously
:return: return code, out_acc, err_acc
"""
from sarge import run, Capture, Feeder
import time
import sys
feeder = Feeder()
p = run(cmd,
input=feeder,
async=True,
stdout=Capture(timeout=7, buffer_size=1),
stderr=Capture(timeout=7, buffer_size=1),
cwd=cwd,
shell=shell,
env=env,
**kwargs)
out_acc = []
err_acc = []
ret_code = 1
log = None
close_log = False
# Logging - either filename or logger itself
if log_obj is not None:
if isinstance(log_obj, basestring):
delete_file_backup(log_obj, chmod=0o600)
log = safe_open(log_obj, mode='w', chmod=0o600)
close_log = True
else:
log = log_obj
try:
while len(p.commands) == 0:
time.sleep(0.15)
while p.commands[0].returncode is None:
out, err = None, None
if readlines:
out = p.stdout.readline()
err = p.stderr.readline()
else:
out = p.stdout.read(1)
err = p.stdout.read(1)
# If output - react on input challenges
if out is not None and len(out) > 0:
out_acc.append(out)
if log is not None:
log.write(out)
log.flush()
if write_dots:
sys.stderr.write('.')
if on_out is not None:
on_out(out, feeder, p)
# Collect error
if err is not None and len(err) > 0:
err_acc.append(err)
if log is not None:
log.write(err)
log.flush()
if write_dots:
sys.stderr.write('.')
if on_err is not None:
on_err(err, feeder, p)
p.commands[0].poll()
time.sleep(0.01)
ret_code = p.commands[0].returncode
logger.debug('Command terminated with code: %s' % ret_code)
# Collect output to accumulator
rest_out = p.stdout.readlines()
if rest_out is not None and len(rest_out) > 0:
for out in rest_out:
out_acc.append(out)
if log is not None:
log.write(out)
log.flush()
if on_out is not None:
on_out(out, feeder, p)
# Collect error to accumulator
rest_err = p.stderr.readlines()
if rest_err is not None and len(rest_err) > 0:
for err in rest_err:
err_acc.append(err)
if log is not None:
log.write(err)
log.flush()
if on_err is not None:
on_err(err, feeder, p)
return ret_code, out_acc, err_acc
finally:
feeder.close()
if close_log:
log.close()
def run_internal(self):
def preexec_function():
os.setpgrp()
cmd = self.cmd
if self.shell:
args_str = (" ".join(self.args) if isinstance(
self.args, (list, tuple)) else self.args)
if isinstance(cmd, (list, tuple)):
cmd = " ".join(cmd)
if args_str and len(args_str) > 0:
cmd += " " + args_str
else:
if self.args and not isinstance(self.args, (list, tuple)):
raise ValueError("!Shell requires array of args")
if self.args:
cmd += self.args
self.using_stdout_cap = self.stdout is None
self.using_stderr_cap = self.stderr is None
self.feeder = Feeder()
logger.debug("Starting command %s in %s" % (cmd, self.cwd))
run_args = {}
if self.preexec_setgrp:
run_args['preexec_fn'] = preexec_function
p = run(cmd,
input=self.feeder,
async_=True,
stdout=self.stdout or Capture(timeout=0.1, buffer_size=1),
stderr=self.stderr or Capture(timeout=0.1, buffer_size=1),
cwd=self.cwd,
env=self.env,
shell=self.shell,
**run_args)
self.time_start = time.time()
self.proc = p
self.ret_code = 1
self.out_acc, self.err_acc = [], []
out_cur, err_cur = [""], [""]
def process_line(line, is_err=False):
dst = self.err_acc if is_err else self.out_acc
dst.append(line)
if self.log_out_during:
if self.no_log_just_write:
dv = sys.stderr if is_err else sys.stdout
dv.write(line + "\n")
dv.flush()
else:
logger.debug("Out: %s" % line.strip())
if self.on_output:
self.on_output(self, line, is_err)
def add_output(buffers, is_err=False, finish=False):
buffers = [
x.decode("utf8") for x in buffers if x is not None and x != ""
]
lines = [""]
if not buffers and not finish:
return
dst_cur = err_cur if is_err else out_cur
for x in buffers:
clines = [v.strip("\r") for v in x.split("\n")]
lines[-1] += clines[0]
lines.extend(clines[1:])
nlines = len(lines)
dst_cur[0] += lines[0]
if nlines > 1:
process_line(dst_cur[0], is_err)
dst_cur[0] = ""
for line in lines[1:-1]:
process_line(line, is_err)
if not finish and nlines > 1:
dst_cur[0] = lines[-1] or ""
if finish:
cline = dst_cur[0] if nlines == 1 else lines[-1]
if cline:
process_line(cline, is_err)
try:
while len(p.commands) == 0:
time.sleep(0.15)
logger.debug("Program started, progs: %s" % len(p.commands))
if p.commands[0] is None:
self.is_running = False
self.was_running = True
logger.error("Program could not be started")
return
self.is_running = True
self.on_change()
out = None
err = None
while p.commands[0] and p.commands[0].returncode is None:
if self.using_stdout_cap:
out = p.stdout.read(-1, False)
add_output([out], is_err=False)
if self.using_stderr_cap:
err = p.stderr.read(-1, False)
add_output([err], is_err=True)
if self.on_tick:
self.on_tick(self)
p.commands[0].poll()
if self.terminating and p.commands[0].returncode is None:
logger.debug("Terminating by sigint %s" % p.commands[0])
sarge_sigint(p.commands[0], signal.SIGTERM)
sarge_sigint(p.commands[0], signal.SIGINT)
logger.debug("Sigint sent")
logger.debug("Process closed")
# If there is data, consume it right away.
if (self.using_stdout_cap and out) or (self.using_stderr_cap
and err):
continue
time.sleep(0.15)
logger.debug("Runner while ended")
p.wait()
self.ret_code = p.commands[0].returncode if p.commands[0] else -1
if self.using_stdout_cap:
try_fnc(lambda: p.stdout.close())
add_output(self.drain_stream(p.stdout, True), finish=True)
if self.using_stderr_cap:
try_fnc(lambda: p.stderr.close())
add_output(self.drain_stream(p.stderr, True),
is_err=True,
finish=True)
self.was_running = True
self.is_running = False
self.on_change()
logger.debug("Program ended with code: %s" % self.ret_code)
logger.debug("Command: %s" % cmd)
if self.log_out_after:
logger.debug("Std out: %s" % "\n".join(self.out_acc))
logger.debug("Error out: %s" % "\n".join(self.err_acc))
except Exception as e:
self.is_running = False
logger.error("Exception in async runner: %s" % (e, ))
finally:
self.was_running = True
self.time_elapsed = time.time() - self.time_start
try_fnc(lambda: self.feeder.close())
try_fnc(lambda: self.proc.close())
if self.on_finished:
self.on_finished(self)
def run_internal(self):
def preexec_function():
os.setpgrp()
def preexec_setsid():
logger.debug("setsid called")
os.setsid()
cmd = self.cmd
if self.shell:
args_str = (" ".join(self.args) if isinstance(
self.args, (list, tuple)) else self.args)
if isinstance(cmd, (list, tuple)):
cmd = " ".join(cmd)
if args_str and len(args_str) > 0:
cmd += " " + args_str
else:
if self.args and not isinstance(self.args, (list, tuple)):
raise ValueError("!Shell requires array of args")
if self.args:
cmd += self.args
self.using_stdout_cap = self.stdout is None
self.using_stderr_cap = self.stderr is None
self.feeder = Feeder()
logger.debug("Starting command %s in %s" % (cmd, self.cwd))
run_args = {}
if self.create_new_group:
if self.is_win:
self.win_create_process_group = True
else:
self.preexec_setsid = True
if self.preexec_setgrp:
run_args['preexec_fn'] = preexec_function
if self.preexec_setsid:
run_args['preexec_fn'] = preexec_setsid
# https://stackoverflow.com/questions/44124338/trying-to-implement-signal-ctrl-c-event-in-python3-6
if self.win_create_process_group:
run_args['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
p = run(cmd,
input=self.feeder,
async_=True,
stdout=self.stdout or Capture(timeout=0.1, buffer_size=1),
stderr=self.stderr or Capture(timeout=0.1, buffer_size=1),
cwd=self.cwd,
env=self.env,
shell=self.shell,
**run_args)
self.p = p
self.time_start = time.time()
self.proc = p
self.ret_code = 1
self.out_acc, self.err_acc = [], []
out_cur, err_cur = [""], [""]
def process_line(line, is_err=False):
dst = self.err_acc if is_err else self.out_acc
dst.append(line)
if self.log_out_during:
if self.no_log_just_write:
dv = sys.stderr if is_err else sys.stdout
dv.write(line + "\n")
dv.flush()
else:
logger.debug("Out: %s" % line.strip())
if self.on_output:
self.on_output(self, line, is_err)
def add_output(buffers, is_err=False, finish=False):
buffers = [
x.decode("utf8") for x in buffers if x is not None and x != ""
]
lines = [""]
if not buffers and not finish:
return
dst_cur = err_cur if is_err else out_cur
for x in buffers:
clines = [v.strip("\r") for v in x.split("\n")]
lines[-1] += clines[0]
lines.extend(clines[1:])
nlines = len(lines)
dst_cur[0] += lines[0]
if nlines > 1:
process_line(dst_cur[0], is_err)
dst_cur[0] = ""
for line in lines[1:-1]:
process_line(line, is_err)
if not finish and nlines > 1:
dst_cur[0] = lines[-1] or ""
if finish:
cline = dst_cur[0] if nlines == 1 else lines[-1]
if cline:
process_line(cline, is_err)
try:
while len(p.commands) == 0:
time.sleep(0.15)
logger.debug("Program started, progs: %s, pid: %s" %
(len(p.commands), self.get_pid()))
if p.commands[0] is None:
self.is_running = False
self.was_running = True
logger.error("Program could not be started")
return
self.is_running = True
self.on_change()
out = None
err = None
while p.commands[0] and p.commands[0].returncode is None:
if self.using_stdout_cap:
out = p.stdout.read(-1, False)
add_output([out], is_err=False)
if self.using_stderr_cap:
err = p.stderr.read(-1, False)
add_output([err], is_err=True)
if self.on_tick:
self.on_tick(self)
p.commands[0].poll()
if self.terminating and p.commands[0].returncode is None:
self.send_term_signals()
logger.debug("Process closed")
# If there is data, consume it right away.
if (self.using_stdout_cap and out) or (self.using_stderr_cap
and err):
continue
time.sleep(0.15)
try_fnc(lambda: p.commands[0].poll())
self.ret_code = p.commands[0].returncode if p.commands[0] else -1
logger.debug("Runner while-loop ended, retcode: %s" %
(p.commands[0].returncode, ))
if self.do_not_block_runner_thread_on_termination:
logger.debug(
"Not blocking runner thread on termination. Finishing, some output may be lost"
)
self.was_running = True
self.is_running = False
return
if self.force_runner_thread_termination:
self.was_running = True
self.is_running = False
return
logger.debug("Waiting for process to complete")
p.wait()
self.ret_code = p.commands[0].returncode if p.commands[0] else -1
if self.do_drain_streams and self.using_stdout_cap:
logger.debug("Draining stdout stream")
try_fnc(lambda: p.stdout.close())
add_output(self.drain_stream(p.stdout, True), finish=True)
if self.do_drain_streams and self.using_stderr_cap:
logger.debug("Draining stderr stream")
try_fnc(lambda: p.stderr.close())
add_output(self.drain_stream(p.stderr, True),
is_err=True,
finish=True)
self.was_running = True
self.is_running = False
self.on_change()
logger.debug("Program ended with code: %s" % self.ret_code)
logger.debug("Command: %s" % cmd)
if self.log_out_after:
logger.debug("Std out: %s" % "\n".join(self.out_acc))
logger.debug("Error out: %s" % "\n".join(self.err_acc))
except Exception as e:
self.is_running = False
logger.error("Exception in async runner: %s" % (e, ))
finally:
self.was_running = True
self.time_elapsed = time.time() - self.time_start
rtt_utils.try_fnc(lambda: self.feeder.close())
if not self.do_not_block_runner_thread_on_termination:
rtt_utils.try_fnc(lambda: self.proc.close())
if self.on_finished:
self.on_finished(self)
import re
from sarge import Capture, Feeder, run
f = Feeder()
c = Capture(buffer_size=1)
p = run('python login_test.py', async_=True, stdout=c, input=f)
c.expect('Username:'******'input username')
f.feed('user\n')
c.expect('Password:'******'input password')
f.feed('pass\n')
VERIFICATION_CODE_REGEX = re.compile(rb'Input verification code \((\d{4})\): ')
match = c.expect(VERIFICATION_CODE_REGEX)
print('input verification code', match.group(1))
f.feed(match.group(1) + b'\n')
c.expect('>>>', timeout=5)
f.feed('print(1 + 1)\n')
f.feed('exit()\n')
p.wait()
print('final output:\n', b''.join(c.readlines()).decode('utf-8'))