def wait(self):
with self._wait_lock:
# See if another thread already waited. If so, return the status we
# got before. If not, immediately release the status lock, and move
# on to call wait ourselves.
with self._status_lock:
if self._status is not None:
return self._status
# No other thread has waited, we're holding the wait lock, and
# we've released the status lock. It's now our job to wait. As
# documented above, if os.waitid is defined, use that function to
# await the child without reaping it. Otherwise we do an ordinary
# Popen.wait and accept the race condition on some platforms.
if HAS_WAITID:
os.waitid(os.P_PID, self._child.pid, os.WEXITED | os.WNOWAIT)
else:
# Python does synchronize this internally, so it won't race
# with other calls to wait() or poll(). Unfortunately it still
# races with kill(), which is what all of this is about.
self._child.wait()
# Finally, while still holding the wait lock, re-acquire the status
# lock to reap the child and write the result. Since we know the
# child has already exited, this won't block. Any other waiting
# threads that were blocked on us will see our result.
with self._status_lock:
# If the child was already reaped above in the !HAS_WAITID
# branch, this will just return the same status again.
self._status = self._child.wait()
return self._status
def run_watchdog(args):
watchdog = multiprocessing.Value('d', time.time() + 1)
pid = os.fork()
if pid == 0:
run(args, watchdog=watchdog)
else:
install_sighandler()
while last_signal is None:
if time.time() - watchdog.value >= args.watchdog:
logger.error("Child is not responding, killing")
os.kill(pid, signal.SIGKILL)
return True
res = os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG)
if res is not None:
if res.si_code !=0:
logger.error("Child exited with code %d, restarting", res.si_code)
return True
logger.info("Killing child with signal %d", last_signal)
os.kill(pid, last_signal)
s = time.time()
while time.time() - s < 1:
res = os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG)
if res is not None:
break
if res is None:
logger.error("Killing child")
os.kill(pid, signal.SIGKILL)
def execute(self, cmd):
"""Execute command in container.
cmd is a string which is executed within a bash shell.
"""
import os
import subprocess
import sys
options.log.info("%s %s" % (self.cid[:6], cmd))
docker_cmd = ["docker", "exec", "--interactive"]
docker_cmd += [self.cid, "/bin/bash"]
docker = subprocess.Popen(docker_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
universal_newlines=True)
docker.stdin.write(cmd + "\n")
docker.stdin.close()
stdout_reader = os.fork()
if stdout_reader == 0:
self._reader(docker, docker.stdout, "stdout")
sys.exit(0)
stderr_reader = os.fork()
if stderr_reader == 0:
self._reader(docker, docker.stderr, "stderr")
sys.exit(0)
os.waitid(os.P_PID, stdout_reader, os.WEXITED)
os.waitid(os.P_PID, stderr_reader, os.WEXITED)
docker.wait()
if docker.returncode != 0:
options.log.error("running in container %s" % (str(self.cid)))
raise Exception("failed command \"%s\"" % (cmd))
def execute(self, cmd):
"""Execute command in container.
cmd is a string which is executed within a bash shell.
"""
import os
import subprocess
import sys
options.log.info("%s %s" % (self.cid[:6], cmd))
docker_cmd = ["docker", "exec", "--interactive"]
docker_cmd += [self.cid, "/bin/bash"]
docker = subprocess.Popen(docker_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
universal_newlines=True)
docker.stdin.write(cmd + "\n")
docker.stdin.close()
stdout_reader = os.fork()
if stdout_reader == 0:
try:
self._reader(docker, docker.stdout, "stdout")
except KeyboardInterrupt:
pass
sys.exit(0)
stderr_reader = os.fork()
if stderr_reader == 0:
try:
self._reader(docker, docker.stderr, "stderr")
except KeyboardInterrupt:
pass
sys.exit(0)
try:
os.waitid(os.P_PID, stdout_reader, os.WEXITED)
os.waitid(os.P_PID, stderr_reader, os.WEXITED)
docker.wait()
except KeyboardInterrupt:
try:
options.log.info("received keyboard interrupt")
docker.terminate()
self.shell()
options.log.info("return from shell, initiating shutdown")
self.cleanup()
sys.exit(0)
except OSError:
pass
docker.wait()
if docker.returncode != 0:
options.log.error("running in container %s" % (str(self.cid)))
raise ExecuteFailure("failed command \"%s\"" % (cmd))
def wait_for_children(self):
# forward signal to children
for signum in (
signal.SIGINT,
signal.SIGTERM,
signal.SIGHUP,
signal.SIGQUIT,
signal.SIGABRT,
signal.SIGWINCH,
signal.SIGUSR1,
signal.SIGUSR2,
):
signal.signal(signum, self._signal_children)
# just wait for them
pgid = os.getpgid(self.pid)
os.waitid(os.P_PGID, pgid, os.WEXITED)
def run_test_case(path):
pid = os.fork()
if not pid:
# child
result = doctest.testfile(path)
if result.failed == 0:
s = 'pass'
st = 0
else:
s = 'fail'
st = 1
print('[%s] %s' % (s, path))
sys.exit(st)
else:
# parent
while True:
try:
r = os.waitid(
os.P_PID, pid, os.WEXITED | os.WSTOPPED | os.WCONTINUED
| os.WNOWAIT | os.WNOHANG)
except ChildProcessError:
pass
if not r:
time.sleep(0.1)
continue
#print('Child PID %d exited with status %d' % (pid, r.si_status))
break
def reap_children(active_children: set) -> None:
for child_pid in active_children.copy():
active_child_pid, _ = os.waitid(child_pid, os.WNOHANG)
if active_child_pid:
active_children.discard(active_child_pid)
pass
def status(self):
"""
Returns one of
"pending" - not yet begun (will begin if resources to do so available)
"running" - still running
"finished" - completed, ready to be removed
"crashed" - completed with non-0 exit code
"wallclock timeout" - killed by time check
"cpu timeout" - killed by rlimit check
"""
import time
if self._status == 'pending' and PIDWrap.count < PIDWrap.limit:
self.begin()
if self._status == 'running':
ans = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if ans is None:
if time.time() >= self.killat:
os.kill(self.pid, 9)
os.waitpid(self.pid, 0)
self._status = 'wallclock timeout'
else:
self._status = 'running'
elif ans.si_status == 24:
self._status = 'cpu timeout'
elif ans.si_status != 0:
self._status = 'crashed'
else:
self._status = 'finished'
if self._status != 'running': PIDWrap.count -= 1
return self._status
def _getlog(self, wait=5, jobid=None):
logf = b''
quit = wait * 2
logn = self._logcnt(False)
code1 = "%put E3969440A681A24088859985"+logn+";\nE3969440A681A24088859985"+logn
while True:
log = self.stderr.read1(4096)
if len(log) > 0:
logf += log
else:
quit -= 1
if quit < 0 or len(logf) > 0:
break
sleep(0.5)
x = logf.decode(self.sascfg.encoding, errors='replace').replace(code1, " ")
self._log += x
if self.pid == None:
return "No SAS process attached. SAS process has terminated unexpectedly."
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return 'SAS process has terminated unexpectedly. Pid State= '+str(rc)
return x
def _getlsttxt(self, wait=5, jobid=None):
f2 = [None]
lstf = b''
quit = wait * 2
eof = 0
self._asubmit("data _null_;file print;put 'Tom was here';run;", "text")
while True:
lst = self.stdout.read1(4096)
if len(lst) > 0:
lstf += lst
lenf = len(lstf)
eof = lstf.find(b"Tom was here", lenf - 25, lenf)
if (eof != -1):
final = lstf.partition(b"Tom was here")
f2 = final[0].decode(self.sascfg.encoding, errors='replace').rpartition(chr(12))
break
lst = f2[0]
if self.pid == None:
return "No SAS process attached. SAS process has terminated unexpectedly."
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return 'SAS process has terminated unexpectedly. Pid State= '+str(rc)
return lst.replace(chr(12), '\n')
def _getlsttxt(self, wait=5, jobid=None):
f2 = [None]
lstf = b''
quit = wait * 2
eof = 0
self._asubmit("data _null_;file print;put 'Tom was here';run;", "text")
while True:
lst = self.stdout.read1(4096)
if len(lst) > 0:
lstf += lst
lenf = len(lstf)
eof = lstf.find(b"Tom was here", lenf - 25, lenf)
if (eof != -1):
final = lstf.partition(b"Tom was here")
f2 = final[0].decode(self.sascfg.encoding).rpartition(chr(12))
break
lst = f2[0]
if self.pid == None:
return "No SAS process attached. SAS process has terminated unexpectedly."
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return 'SAS process has terminated unexpectedly. Pid State= '+str(rc)
return lst.replace(chr(12), '\n')
async def child_waiter(app):
while True:
try:
status = os.waitid(os.P_ALL, 0, os.WNOHANG | os.WEXITED)
logger.debug("Child completed with status %s", str(status))
except ChildProcessError:
await asyncio.sleep(600)
def _getlog(self, wait=5, jobid=None):
logf = b''
quit = wait * 2
logn = self._logcnt(False)
code1 = "%put E3969440A681A24088859985"+logn+";\nE3969440A681A24088859985"+logn
while True:
log = self.stderr.read1(4096)
if len(log) > 0:
logf += log
else:
quit -= 1
if quit < 0 or len(logf) > 0:
break
sleep(0.5)
x = logf.decode(self.sascfg.encoding).replace(code1, " ")
self._log += x
if self.pid == None:
return "No SAS process attached. SAS process has terminated unexpectedly."
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return 'SAS process has terminated unexpectedly. Pid State= '+str(rc)
return x
def try_wait(self):
with self._status_lock:
if self._status is not None:
return self._status
# The child hasn't been waited on yet, so we need to do a
# non-blocking check to see if it's still running. The Popen type
# provides the poll() method for this, but that might reap the
# child and free its PID, which would make this a race with
# concurrent callers of the blocking wait() method above, who might
# be about to call os.waitid on that PID. When os.waitid is
# available, use that again here, with the WNOHANG flag. Otherwise
# just use poll() and rely on Python's internal synchronization.
if HAS_WAITID:
poll_result = os.waitid(os.P_PID, self._child.pid,
os.WEXITED | os.WNOWAIT | os.WNOHANG)
else:
poll_result = self._child.poll()
# If either of the poll approaches above returned non-None, do a full
# wait to reap the child, which will not block. Note that we've
# released the status lock here, because wait() will re-acquire it.
if poll_result is not None:
return self.wait()
else:
return None
def spawn_daemon(function, *args, **kwargs):
# First fork
pid = os.fork()
if pid != 0: # parent returns
os.waitid(os.P_PID, pid, os.WEXITED)
return
# Decouple from parent environment
# os.chdir("/")
os.setsid()
os.umask(0)
# Do second fork
pid = os.fork()
if pid != 0: # parent exits
os._exit(0)
function(*args, **kwargs)
def initialize_vim(working_path, pipe_name):
"""
Initialize the vim process along with creating a named pipe to communicate with it
:param working_path: working path of main.py and other files
:param pipe_name: pathname of pipe to be created
:return: pid of vim process
"""
# File initializations relative to working paths
plugin_file = os.path.join(working_path, "plugin.vim")
# Make our pipe.
utils.remove_file(pipe_name)
os.mkfifo(pipe_name)
# Create the vim command that opens up vim instance in new terminal
# with proper settings, etc.
profile_path = os.path.join(working_path, "profile.log")
startup_command = "call AutoLogInfo() | profile start " + profile_path + " | profile func * | profile file *"
vim_command = "vim -S %r -c %r" % (plugin_file, startup_command)
# Send command to open up new process, wait for command process to die
# before getting the pid of the actual vim process
args = [env_command[desktop_env], "-e", vim_command]
proc = subprocess.Popen(args).pid
os.waitid(os.P_PID, int(proc), os.WEXITED)
while True:
try:
logging.debug(vim_command)
proc = (
subprocess.Popen(["pgrep", "-f", vim_command[:7]], stdout=subprocess.PIPE)
.stdout.read()
.decode("utf-8")
.rstrip()
)
logging.debug(proc)
proc = int(proc)
break
except:
pass
return proc
def _endsas(self):
rc = 0
if self.pid:
code = ";*\';*\";*/;\n;quit;endsas;"
self._getlog(wait=1)
self._asubmit(code,'text')
sleep(1)
try:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
except (subprocess.TimeoutExpired):
print("SAS didn't shutdown w/in 5 seconds; killing it to be sure")
os.kill(self.pid, signal.SIGKILL)
print("SAS Connection terminated. Subprocess id was "+str(self.pid))
self.pid = None
return rc
def _endsas(self):
rc = 0
if self.pid:
code = b";*\';*\";*/;\n;quit;endsas;\n"
self.stderr.read1(4096)
self.stdin.write(code)
self.stdin.flush()
sleep(1)
try:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
except subprocess.TimeoutExpired:
print("SAS didn't shutdown w/in 5 seconds; killing it to be sure")
os.kill(self.pid, signal.SIGKILL)
self.pid = None
return rc
def sync_wait_reapable(pid):
P_PID = 1
WEXITED = 0x00000004
if sys.platform == 'darwin': # pragma: no cover
# waitid() is not exposed on Python on Darwin but does
# work through CFFI; note that we typically won't get
# here since Darwin also defines kqueue
WNOWAIT = 0x00000020
else:
WNOWAIT = 0x01000000
result = waitid_ffi.new("siginfo_t *")
while waitid(P_PID, pid, result, WEXITED | WNOWAIT) < 0:
got_errno = waitid_ffi.errno
if got_errno == errno.EINTR:
continue
raise OSError(got_errno, os.strerror(got_errno))
def _endsas(self):
rc = 0
if self.pid:
code = b";*\';*\";*/;\n;quit;endsas;\n"
self.stderr.read1(4096)
self.stdin.write(code)
self.stdin.flush()
sleep(1)
try:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
except subprocess.TimeoutExpired:
print(
"SAS didn't shutdown w/in 5 seconds; killing it to be sure"
)
os.kill(self.pid, signal.SIGKILL)
self.pid = None
return rc
def _handle_SIGCHLD(self, pid):
_logger.debug("calling waitid() on process %d", pid)
waitid_result = os.waitid(os.P_PID, pid, os.WNOHANG | os.WEXITED)
_logger.debug("waitid() returned %r", waitid_result)
if waitid_result is None:
_logger.warning("waitid() returned nothing?")
return
if waitid_result.si_code == CLD_EXITED:
returncode = waitid_result.si_status
_logger.debug("Saw CLD_EXITED with return code: %r", returncode)
return returncode
elif waitid_result.si_code == CLD_KILLED:
signal_num = waitid_result.si_status
_logger.debug("Saw CLD_KILLED with signal: %r", signal_num)
return -signal_num
else:
_bug_logger.error("Unexpected si_code: %d", waitid_result.si_code)
def status(self):
import time
if self._status == 'pending' and PIDWrap.count < PIDWrap.limit:
self.begin()
if self._status == 'running':
ans = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if ans is None:
if time.time() >= self.killat:
os.kill(self.pid, 9)
os.waitpid(self.pid, 0)
self._status = 'wallclock timeout'
else:
self._status = 'running'
elif ans.si_status == 24:
self._status = 'cpu timeout'
elif ans.si_status != 0:
self._status = 'crashed'
else:
self._status = 'finished'
if self._status != 'running': PIDWrap.count -= 1
return self._status
def _getlst(self, wait=5, jobid=None):
lstf = b''
quit = wait * 2
eof = 0
bof = False
lenf = 0
while True:
lst = self.stdout.read1(4096)
if len(lst) > 0:
lstf += lst
if ((not bof) and lst.count(b"<!DOCTYPE html>", 0, 20) > 0):
bof = True
else:
lenf = len(lstf)
if (lenf > 15):
eof = lstf.count(b"</html>", (lenf - 15), lenf)
if (eof > 0):
break
if not bof:
quit -= 1
if quit < 0:
break
sleep(0.5)
if self.pid == None:
return "No SAS process attached. SAS process has terminated unexpectedly."
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return 'SAS process has terminated unexpectedly. Pid State= '+str(rc)
if eof:
return lstf.decode(errors='replace')
else:
return lstf.decode(self.sascfg.encoding, errors='replace')
def _getlst(self, wait=5, jobid=None):
lstf = b''
quit = wait * 2
eof = 0
bof = False
lenf = 0
while True:
lst = self.stdout.read1(4096)
if len(lst) > 0:
lstf += lst
if ((not bof) and lst.count(b"<!DOCTYPE html>", 0, 20) > 0):
bof = True
else:
lenf = len(lstf)
if (lenf > 15):
eof = lstf.count(b"</html>", (lenf - 15), lenf)
if (eof > 0):
break
if not bof:
quit -= 1
if quit < 0:
break
sleep(0.5)
if self.pid == None:
return "No SAS process attached. SAS process has terminated unexpectedly."
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return 'SAS process has terminated unexpectedly. Pid State= '+str(rc)
return lstf.decode(self.sascfg.encoding)
def wait_process(self):
siginfo = os.waitid(os.P_PID, self.process_dmtcp_wrapped.pid,
os.WEXITED | os.WNOWAIT)
self.process_dmtcp_wrapped.wait()
states.setProcessFinished()
CLD_KILLED = 3
if siginfo.si_code == CLD_KILLED:
utils.stderr_and_log(
"Process recevied signal: %s" %
signals.strsignal(siginfo.si_status), logger)
self.returncode = self.process_dmtcp_wrapped.returncode
if self.returncode == 0:
context.plugin_hook.done(exeinfo=self)
# TODO check rerunable fail
else:
context.plugin_hook.fail(exeinfo=self)
logger.info("Process to be checkpointed finished with ret code :%d." %
self.returncode)
def submit(self, code, results="html", prompt={}):
odsopen = b"ods listing close;ods html5 file=stdout options(bitmap_mode='inline') device=png; ods graphics on / outputfmt=png;\n"
odsclose = b"ods html5 close;ods listing;\n"
ods = True
mj = b";*\';*\";*/;"
lstf = ''
logf = ''
bail = False
eof = 5
bc = False
done = False
logn = self._logcnt()
logcodei = "%put E3969440A681A24088859985" + logn + ";"
logcodeo = "\nE3969440A681A24088859985" + logn
pcodei = ''
pcodeo = ''
if self.pid is None:
print(
"No SAS process attached. SAS process has terminated unexpectedly."
)
return dict(
LOG=
"No SAS process attached. SAS process has terminated unexpectedly.",
LST='')
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(
LOG='SAS process has terminated unexpectedly. Pid State= ' +
str(rc),
LST='')
if results.upper() != "HTML":
ods = False
if len(prompt):
pcodei += 'options nosource nonotes;\n'
pcodeo += 'options nosource nonotes;\n'
for key in prompt:
gotit = False
while not gotit:
var = self.sascfg._prompt(
'Please enter value for macro variable ' + key + ' ',
pw=prompt[key])
if len(var) > 0:
gotit = True
else:
print("Sorry, didn't get a value for that variable.")
pcodei += '%let ' + key + '=' + var + ';\n'
pcodeo += '%symdel ' + key + ';\n'
pcodei += 'options source notes;\n'
pcodeo += 'options source notes;\n'
if ods:
self.stdin.write(odsopen)
out = self.stdin.write(mj + b'\n' + pcodei.encode() + code.encode() +
b'\n' + pcodeo.encode() + b'\n' + mj)
if ods:
self.stdin.write(odsclose)
out = self.stdin.write(b'\n' + logcodei.encode() + b'\n')
self.stdin.flush()
while not done:
try:
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(
LOG=
'SAS process has terminated unexpectedly. Pid State= '
+ str(rc),
LST='')
if bail:
eof -= 1
if eof < 0:
break
lst = self.stdout.read1(4096).decode()
if len(lst) > 0:
lstf += lst
else:
log = self.stderr.read1(4096).decode()
if len(log) > 0:
logf += log
if logf.count(logcodeo) >= 1:
bail = True
if not bail and bc:
self.stdin.write(odsclose + logcodei.encode() +
b'\n')
self.stdin.flush()
bc = False
done = True
except (KeyboardInterrupt, SystemExit):
print('Exception caught!')
ll = self._breakprompt(logcodeo)
if ll.get('ABORT', False):
return ll
logf += ll['LOG']
lstf += ll['LST']
bc = ll['BC']
if not bc:
print('Exception handled :)\n')
else:
print('Exception ignored, continuing to process...\n')
self.stdin.write(odsclose + logcodei.encode() + b'\n')
self.stdin.flush()
trip = lstf.rpartition("/*]]>*/")
if len(trip[1]) > 0 and len(trip[2]) < 100:
lstf = ''
self._log += logf
final = logf.partition(logcodei)
z = final[0].rpartition(chr(10))
prev = '%08d' % (self._log_cnt - 1)
zz = z[0].rpartition("\nE3969440A681A24088859985" + prev + '\n')
logd = zz[2].replace(mj.decode(), '')
lstd = lstf.replace(chr(12),
chr(10)).replace('<body class="c body">',
'<body class="l body">').replace(
"font-size: x-small;",
"font-size: normal;")
return dict(LOG=logd, LST=lstd)
def get_child_status(self):
if self.pid is not None:
try:
return os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
except ChildProcessError:
self.pid = None
def submit(self, code, results="html", prompt={}):
odsopen = b"ods listing close;ods html5 file=stdout options(bitmap_mode='inline') device=png; ods graphics on / outputfmt=png;\n"
odsclose = b"ods html5 close;ods listing;\n"
ods = True
mj = b";*\';*\";*/;"
lstf = ''
logf = ''
bail = False
eof = 5
bc = False
done = False
logn = self._logcnt()
logcodei = "%put E3969440A681A24088859985" + logn + ";"
logcodeo = "\nE3969440A681A24088859985" + logn
pcodei = ''
pcodeiv = ''
pcodeo = ''
if self.pid is None:
print("No SAS process attached. SAS process has terminated unexpectedly.")
return dict(LOG="No SAS process attached. SAS process has terminated unexpectedly.", LST='')
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' + str(rc), LST='')
if results.upper() != "HTML":
ods = False
if len(prompt):
pcodei += 'options nosource nonotes;\n'
pcodeo += 'options nosource nonotes;\n'
for key in prompt:
gotit = False
while not gotit:
var = self.sascfg._prompt('Please enter value for macro variable '+key+' ', pw=prompt[key])
if len(var) > 0:
gotit = True
else:
print("Sorry, didn't get a value for that variable.")
if prompt[key]:
pcodei += '%let '+key+'='+var+';\n'
pcodeo += '%symdel '+key+';\n'
else:
pcodeiv += '%let '+key+'='+var+';\n'
pcodei += 'options source notes;\n'
pcodeo += 'options source notes;\n'
if ods:
self.stdin.write(odsopen)
out = self.stdin.write(mj+b'\n'+pcodei.encode()+pcodeiv.encode()+code.encode()+b'\n'+pcodeo.encode()+b'\n'+mj)
if ods:
self.stdin.write(odsclose)
out = self.stdin.write(b'\n' + logcodei.encode() + b'\n')
self.stdin.flush()
while not done:
try:
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' +
str(rc), LST='')
if bail:
eof -= 1
if eof < 0:
break
lst = self.stdout.read1(4096).decode()
if len(lst) > 0:
lstf += lst
else:
log = self.stderr.read1(4096).decode()
if len(log) > 0:
logf += log
if logf.count(logcodeo) >= 1:
bail = True
if not bail and bc:
self.stdin.write(odsclose+logcodei.encode() + b'\n')
self.stdin.flush()
bc = False
done = True
except (KeyboardInterrupt, SystemExit):
print('Exception caught!')
ll = self._breakprompt(logcodeo)
if ll.get('ABORT', False):
return ll
logf += ll['LOG']
lstf += ll['LST']
bc = ll['BC']
if not bc:
print('Exception handled :)\n')
else:
print('Exception ignored, continuing to process...\n')
self.stdin.write(odsclose+logcodei.encode()+b'\n')
self.stdin.flush()
trip = lstf.rpartition("/*]]>*/")
if len(trip[1]) > 0 and len(trip[2]) < 100:
lstf = ''
self._log += logf
final = logf.partition(logcodei)
z = final[0].rpartition(chr(10))
prev = '%08d' % (self._log_cnt - 1)
zz = z[0].rpartition("\nE3969440A681A24088859985" + prev +'\n')
logd = zz[2].replace(mj.decode(), '')
lstd = lstf.replace(chr(12), chr(10)).replace('<body class="c body">',
'<body class="l body">').replace("font-size: x-small;",
"font-size: normal;")
return dict(LOG=logd, LST=lstd)
def submit(self, code, results="html"):
odsopen = b"ods listing close;ods html5 file=stdout options(bitmap_mode='inline') device=png; ods graphics on / outputfmt=png;\n"
odsclose = b"ods html5 close;ods listing;\n"
ods = True
htm = "html HTML"
mj = b";*\';*\";*/;"
lstf = ''
logf = ''
bail = False
eof = 5
bc = False
done = False
logn = self._logcnt()
logcodei = "%put E3969440A681A24088859985" + logn + ";"
logcodeo = "\nE3969440A681A24088859985" + logn
if self.pid is None:
return dict(LOG="No SAS process attached. SAS process has terminated unexpectedly.", LST='')
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' + str(rc), LST='')
if htm.find(results) < 0:
ods = False
if ods:
self.stdin.write(odsopen)
out = self.stdin.write(mj + b'\n' + code.encode() + b'\n' + mj)
if ods:
self.stdin.write(odsclose)
out = self.stdin.write(b'\n' + logcodei.encode() + b'\n')
self.stdin.flush()
while not done:
try:
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' +
str(rc), LST='')
if bail:
eof -= 1
if eof < 0:
break
lst = self.stdout.read1(4096).decode()
if len(lst) > 0:
lstf += lst
else:
log = self.stderr.read1(4096).decode()
if len(log) > 0:
logf += log
if logf.count(logcodeo) >= 1:
bail = True
if not bail and bc:
self.stdin.write(odsclose+logcodei.encode() + b'\n')
self.stdin.flush()
bc = False
done = True
except (KeyboardInterrupt, SystemExit):
print('Exception caught!')
ll = self._breakprompt(logcodeo)
if ll.get('ABORT', False):
return ll
logf += ll['LOG']
lstf += ll['LST']
bc = ll['BC']
if not bc:
print('Exception handled :)\n')
else:
print('Exception ignored, continuing to process...\n')
self.stdin.write(odsclose+logcodei.encode()+b'\n')
self.stdin.flush()
trip = lstf.rpartition("/*]]>*/")
if len(trip[1]) > 0 and len(trip[2]) < 100:
lstf = ''
self._log += logf
final = logf.partition(logcodei)
z = final[0].rpartition(chr(10))
prev = '%08d' % (self._log_cnt - 1)
zz = z[0].rpartition("\nE3969440A681A24088859985" + prev +'\n')
logd = zz[2].replace(mj.decode(), '')
lstd = lstf.replace(chr(12), chr(10)).replace('<body class="c body">',
'<body class="l body">').replace("font-size: x-small;",
"font-size: normal;")
return dict(LOG=logd, LST=lstd)
def submit(self, code: str, results: str ="html", prompt: dict ={}) -> dict:
'''
This method is used to submit any SAS code. It returns the Log and Listing as a python dictionary.
code - the SAS statements you want to execute
results - format of results, HTML is default, TEXT is the alternative
prompt - dict of names:flags to prompt for; create macro variables (used in submitted code), then keep or delete
The keys are the names of the macro variables and the boolean flag is to either hide what you type and delete
the macros, or show what you type and keep the macros (they will still be available later)
for example (what you type for pw will not be displayed, user and dsname will):
results = sas.submit(
"""
libname tera teradata server=teracop1 user=&user pw=&pw;
proc print data=tera.&dsname (obs=10); run;
""" ,
prompt = {'user': False, 'pw': True, 'dsname': False}
)
Returns - a Dict containing two keys:values, [LOG, LST]. LOG is text and LST is 'results' (HTML or TEXT)
NOTE: to view HTML results in the ipykernel, issue: from IPython.display import HTML and use HTML() instead of print()
i.e,: results = sas.submit("data a; x=1; run; proc print;run')
print(results['LOG'])
HTML(results['LST'])
'''
odsopen = b"ods listing close;ods "+self.sascfg.output.encode()+ \
b" (id=saspy_internal) file=stdout options(bitmap_mode='inline') device=svg style="+self._sb.HTML_Style.encode()+ \
b"; ods graphics on / outputfmt=png;\n"
odsclose = b"ods "+self.sascfg.output.encode()+b" (id=saspy_internal) close;ods listing;\n"
ods = True;
mj = b";*\';*\";*/;"
lstf = ''
logf = ''
bail = False
eof = 5
bc = False
done = False
logn = self._logcnt()
logcodei = "%put E3969440A681A24088859985" + logn + ";"
logcodeo = "\nE3969440A681A24088859985" + logn
pcodei = ''
pcodeiv = ''
pcodeo = ''
if self.pid == None:
print("No SAS process attached. SAS process has terminated unexpectedly.")
return dict(LOG="No SAS process attached. SAS process has terminated unexpectedly.", LST='')
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= '+str(rc), LST='')
# to cover the possibility of an _asubmit w/ lst output not read; no known cases now; used to be __flushlst__()
# removing this and adding comment in _asubmit to use _getlst[txt] so this will never be necessary; delete later
#while(len(self.stdout.read1(4096)) > 0):
# continue
if results.upper() != "HTML":
ods = False
if len(prompt):
pcodei += 'options nosource nonotes;\n'
pcodeo += 'options nosource nonotes;\n'
for key in prompt:
gotit = False
while not gotit:
var = self.sascfg._prompt('Please enter value for macro variable '+key+' ', pw=prompt[key])
if var is None:
raise KeyboardInterrupt
if len(var) > 0:
gotit = True
else:
print("Sorry, didn't get a value for that variable.")
if prompt[key]:
pcodei += '%let '+key+'='+var+';\n'
pcodeo += '%symdel '+key+';\n'
else:
pcodeiv += '%let '+key+'='+var+';\n'
pcodei += 'options source notes;\n'
pcodeo += 'options source notes;\n'
if ods:
self.stdin.write(odsopen)
pgm = mj+b'\n'+pcodei.encode(self.sascfg.encoding)+pcodeiv.encode(self.sascfg.encoding)
pgm += code.encode(self.sascfg.encoding)+b'\n'+pcodeo.encode(self.sascfg.encoding)+b'\n'+mj
out = self.stdin.write(pgm)
if ods:
self.stdin.write(odsclose)
out = self.stdin.write(b'\n'+logcodei.encode(self.sascfg.encoding)+b'\n')
self.stdin.flush()
while not done:
try:
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
log = ''
try:
log = self.stderr.read1(4096).decode(self.sascfg.encoding, errors='replace')
if len(log) > 0:
logf += log
self._log += logf
except:
pass
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' +
str(rc)+'\n'+logf, LST='')
if bail:
eof -= 1
if eof < 0:
break
if ods:
lst = self.stdout.read1(4096).decode(errors='replace')
else:
lst = self.stdout.read1(4096).decode(self.sascfg.encoding, errors='replace')
if len(lst) > 0:
lstf += lst
else:
log = self.stderr.read1(4096).decode(self.sascfg.encoding, errors='replace')
if len(log) > 0:
logf += log
if logf.count(logcodeo) >= 1:
bail = True
if not bail and bc:
self.stdin.write(odsclose+logcodei.encode(self.sascfg.encoding) + b'\n')
self.stdin.flush()
bc = False
done = True
except (ConnectionResetError):
log = ''
try:
log = self.stderr.read1(4096).decode(self.sascfg.encoding, errors='replace')
if len(log) > 0:
logf += log
self._log += logf
except:
pass
rc = 0
rc = os.waitpid(self.pid, 0)
self.pid = None
return dict(LOG=logf.partition(logcodeo)[0]+'\nConnection Reset: SAS process has terminated unexpectedly. '+
'Pid State= '+str(rc)+'\n'+logf, LST='')
except (KeyboardInterrupt, SystemExit):
print('Exception caught!')
ll = self._breakprompt(logcodeo)
if ll.get('ABORT', False):
return ll
logf += ll['LOG']
lstf += ll['LST']
bc = ll['BC']
if not bc:
print('Exception handled :)\n')
else:
print('Exception ignored, continuing to process...\n')
self.stdin.write(odsclose+logcodei.encode(self.sascfg.encoding)+b'\n')
self.stdin.flush()
trip = lstf.rpartition("/*]]>*/")
if len(trip[1]) > 0 and len(trip[2]) < 100:
lstf = ''
self._log += logf
final = logf.partition(logcodei)
z = final[0].rpartition(chr(10))
prev = '%08d' % (self._log_cnt - 1)
zz = z[0].rpartition("\nE3969440A681A24088859985" + prev +'\n')
logd = zz[2].replace(mj.decode(self.sascfg.encoding), '')
lstd = lstf.replace(chr(12), chr(10)).replace('<body class="c body">',
'<body class="l body">').replace("font-size: x-small;",
"font-size: normal;")
return dict(LOG=logd, LST=lstd)
def submit(self, code: str, results: str ="html", prompt: dict ={}) -> dict:
'''
This method is used to submit any SAS code. It returns the Log and Listing as a python dictionary.
code - the SAS statements you want to execute
results - format of results, HTML is default, TEXT is the alternative
prompt - dict of names:flags to prompt for; create marco variables (used in submitted code), then keep or delete
The keys are the names of the macro variables and the boolean flag is to either hide what you type and delete
the macros, or show what you type and keep the macros (they will still be available later)
for example (what you type for pw will not be displayed, user and dsname will):
results = sas.submit(
"""
libname tera teradata server=teracop1 user=&user pw=&pw;
proc print data=tera.&dsname (obs=10); run;
""" ,
prompt = {'user': False, 'pw': True, 'dsname': False}
)
Returns - a Dict containing two keys:values, [LOG, LST]. LOG is text and LST is 'results' (HTML or TEXT)
NOTE: to view HTML results in the ipykernel, issue: from IPython.display import HTML and use HTML() instead of print()
i.e,: results = sas.submit("data a; x=1; run; proc print;run')
print(results['LOG'])
HTML(results['LST'])
'''
odsopen = b"ods listing close;ods html5 (id=saspy_internal) file=stdout options(bitmap_mode='inline') device=svg; ods graphics on / outputfmt=png;\n"
odsclose = b"ods html5 (id=saspy_internal) close;ods listing;\n"
ods = True;
mj = b";*\';*\";*/;"
lstf = ''
logf = ''
bail = False
eof = 5
bc = False
done = False
logn = self._logcnt()
logcodei = "%put E3969440A681A24088859985" + logn + ";"
logcodeo = "\nE3969440A681A24088859985" + logn
pcodei = ''
pcodeiv = ''
pcodeo = ''
if self.pid == None:
print("No SAS process attached. SAS process has terminated unexpectedly.")
return dict(LOG="No SAS process attached. SAS process has terminated unexpectedly.", LST='')
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= '+str(rc), LST='')
# to cover the possibility of an _asubmit w/ lst output not read; no known cases now; used to be __flushlst__()
# removing this and adding comment in _asubmit to use _getlst[txt] so this will never be necessary; delete later
#while(len(self.stdout.read1(4096)) > 0):
# continue
if results.upper() != "HTML":
ods = False
if len(prompt):
pcodei += 'options nosource nonotes;\n'
pcodeo += 'options nosource nonotes;\n'
for key in prompt:
gotit = False
while not gotit:
var = self.sascfg._prompt('Please enter value for macro variable '+key+' ', pw=prompt[key])
if len(var) > 0:
gotit = True
else:
print("Sorry, didn't get a value for that variable.")
if prompt[key]:
pcodei += '%let '+key+'='+var+';\n'
pcodeo += '%symdel '+key+';\n'
else:
pcodeiv += '%let '+key+'='+var+';\n'
pcodei += 'options source notes;\n'
pcodeo += 'options source notes;\n'
if ods:
self.stdin.write(odsopen)
pgm = mj+b'\n'+pcodei.encode(self.sascfg.encoding)+pcodeiv.encode(self.sascfg.encoding)
pgm += code.encode(self.sascfg.encoding)+b'\n'+pcodeo.encode(self.sascfg.encoding)+b'\n'+mj
out = self.stdin.write(pgm)
if ods:
self.stdin.write(odsclose)
out = self.stdin.write(b'\n'+logcodei.encode(self.sascfg.encoding)+b'\n')
self.stdin.flush()
while not done:
try:
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' +
str(rc), LST='')
if bail:
eof -= 1
if eof < 0:
break
lst = self.stdout.read1(4096).decode(self.sascfg.encoding)
if len(lst) > 0:
lstf += lst
else:
log = self.stderr.read1(4096).decode(self.sascfg.encoding)
if len(log) > 0:
logf += log
if logf.count(logcodeo) >= 1:
bail = True
if not bail and bc:
self.stdin.write(odsclose+logcodei.encode(self.sascfg.encoding) + b'\n')
self.stdin.flush()
bc = False
done = True
except (ConnectionResetError):
rc = 0
rc = os.waitpid(self.pid, 0)
self.pid = None
return dict(LOG=logf.partition(logcodeo)[0]+'\nConnection Reset: SAS process has terminated unexpectedly. Pid State= '+str(rc), LST='')
except (KeyboardInterrupt, SystemExit):
print('Exception caught!')
ll = self._breakprompt(logcodeo)
if ll.get('ABORT', False):
return ll
logf += ll['LOG']
lstf += ll['LST']
bc = ll['BC']
if not bc:
print('Exception handled :)\n')
else:
print('Exception ignored, continuing to process...\n')
self.stdin.write(odsclose+logcodei.encode(self.sascfg.encoding)+b'\n')
self.stdin.flush()
trip = lstf.rpartition("/*]]>*/")
if len(trip[1]) > 0 and len(trip[2]) < 100:
lstf = ''
self._log += logf
final = logf.partition(logcodei)
z = final[0].rpartition(chr(10))
prev = '%08d' % (self._log_cnt - 1)
zz = z[0].rpartition("\nE3969440A681A24088859985" + prev +'\n')
logd = zz[2].replace(mj.decode(self.sascfg.encoding), '')
lstd = lstf.replace(chr(12), chr(10)).replace('<body class="c body">',
'<body class="l body">').replace("font-size: x-small;",
"font-size: normal;")
return dict(LOG=logd, LST=lstd)
def _interact_leave(self, signum, sigframe):
while os.waitid(os.P_ALL, 0, os.WEXITED | os.WNOHANG) is not None:
pass
def save_output(args, output_file):
"""Run args[0] with arguments args[1:] and save standard output to output_file."""
with open(output_file, 'w', encoding='utf-8') as f:
p = subprocess.Popen(args, stfout=f)
os.waitid(p.pid, 0)
def _startsas(self):
#import pdb;pdb.set_trace()
if self.pid:
return self.pid
pgm, parms = self._buildcommand(self.sascfg)
s = ''
for i in range(len(parms)):
s += parms[i]+' '
PIPE_READ = 0
PIPE_WRITE = 1
pin = os.pipe()
pout = os.pipe()
perr = os.pipe()
try:
pidpty = os.forkpty()
except:
import pty
pidpty = pty.fork()
if pidpty[0]:
# we are the parent
pid = pidpty[0]
os.close(pin[PIPE_READ])
os.close(pout[PIPE_WRITE])
os.close(perr[PIPE_WRITE])
else:
# we are the child
signal.signal(signal.SIGINT, signal.SIG_DFL)
os.close(0)
os.close(1)
os.close(2)
os.dup2(pin[PIPE_READ], 0)
os.dup2(pout[PIPE_WRITE], 1)
os.dup2(perr[PIPE_WRITE], 2)
os.close(pin[PIPE_READ])
os.close(pin[PIPE_WRITE])
os.close(pout[PIPE_READ])
os.close(pout[PIPE_WRITE])
os.close(perr[PIPE_READ])
os.close(perr[PIPE_WRITE])
try:
#sleep(5)
os.execv(pgm, parms)
except OSError as e:
print("The OS Error was:\n"+e.strerror+'\n')
print("SAS Connection failed. No connection established. Double check you settings in sascfg.py file.\n")
print("Attempted to run program "+pgm+" with the following parameters:"+str(parms)+"\n")
print("If no OS Error above, try running the following command (where saspy is running) manually to see what is wrong:\n"+s+"\n")
os._exit(-6)
except:
print("Subprocess failed to start. Double check you settings in sascfg.py file.\n")
os._exit(-6)
self.pid = pidpty[0]
self.stdin = os.fdopen(pin[PIPE_WRITE], mode='wb')
self.stderr = os.fdopen(perr[PIPE_READ], mode='rb')
self.stdout = os.fdopen(pout[PIPE_READ], mode='rb')
fcntl.fcntl(self.stdout, fcntl.F_SETFL, os.O_NONBLOCK)
fcntl.fcntl(self.stderr, fcntl.F_SETFL, os.O_NONBLOCK)
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc != None:
self.pid = None
lst = self.stdout.read1(4096)
print("stdout from subprocess is:\n"+lst.decode())
if self.pid is None:
print("SAS Connection failed. No connection established. Double check you settings in sascfg.py file.\n")
print("Attempted to run program "+pgm+" with the following parameters:"+str(parms)+"\n")
print("Try running the following command (where saspy is running) manually to see if you can get more information on what went wrong:\n"+s+"\n")
return NULL
else:
self.submit("options svgtitle='svgtitle'; options validvarname=any; ods graphics on;", "text")
if self.pid is None:
print("SAS Connection failed. No connection established. Double check you settings in sascfg.py file.\n")
print("Attempted to run program "+pgm+" with the following parameters:"+str(parms)+"\n")
print("Try running the following command (where saspy is running) manually to see if you can get more information on what went wrong:\n"+s+"\n")
return None
if self.sascfg.verbose:
print("SAS Connection established. Subprocess id is "+str(self.pid)+"\n")
return self.pid
def _get_startup(self):
"""
get startup information based on new vimrc
:return:
"""
vimrc_new_path = os.path.join(self.working_path, 'vimrc')
# get valid plugins
valid_plugins = []
with self.status_lock:
for i, p in enumerate(self.all_plugins):
if self.plugin_statuses[i]:
valid_plugins.append('Plugin ' + p)
# create process to open vim using --startuptime
with self.startup_lock:
with open(self.vimrc_path, 'r') as vimrc_orig:
with open(vimrc_new_path, 'w') as vimrc_new:
for line in vimrc_orig:
if line.startswith('Plugin') and line.rstrip() not in valid_plugins:
continue
else:
vimrc_new.write(line)
startup_file_path = os.path.join(self.working_path, 'startup.log')
vim_command = 'vim -u ' + vimrc_new_path + ' --startuptime ' + startup_file_path
args = ['gnome-terminal', '-e', vim_command]
proc = subprocess.Popen(args).pid
os.waitid(os.P_PID, int(proc), os.WEXITED)
# check if startup file successfully made
while not os.path.exists(startup_file_path):
pass
while True:
with open(startup_file_path, 'r') as startup_file:
for line in startup_file:
pass
if 'VIM STARTED' in line:
total_elapsed_time = line
break
proc = (subprocess.Popen(['pgrep', '-f', vim_command[:7]],
stdout=subprocess.PIPE)
.stdout.read().decode('utf-8').rstrip())
os.kill(int(proc), 9)
swap_file_path = os.path.join(self.working_path, '.swp')
try:
os.remove(swap_file_path)
except OSError:
pass
# gather source info from startuptime file
source_times = []
with open(startup_file_path, 'r') as startup_file:
for line in startup_file:
logging.debug(line.rsplit(' '))
try:
float(line.rsplit(' ')[2])
except (ValueError, IndexError):
continue
source_times.append(line)
# sort and place info into analysis queue
source_times.sort(key=lambda s_line: float(s_line.split(' ')[2]),
reverse=True)
source_times.insert(0, total_elapsed_time)
self.analysis_queue.put(source_times)
self.analysis_event.set()
os.remove(vimrc_new_path)
os.remove(startup_file_path)
def _breakprompt(self, eos):
found = False
logf = ''
lstf = ''
bc = False
if self.pid is None:
return dict(LOG="No SAS process attached. SAS process has terminated unexpectedly.", LST='', ABORT=True)
if self.sascfg.ssh:
response = self.sascfg._prompt(
"SAS attention handling not supported over ssh. Please enter (T) to terminate SAS or (C) to continue.")
while True:
if response.upper() == 'C':
return dict(LOG='', LST='', BC=True)
if response.upper() == 'T':
break
response = self.sascfg._prompt("Please enter (T) to terminate SAS or (C) to continue.")
interrupt = signal.SIGINT
os.kill(self.pid, interrupt)
sleep(.25)
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
outrc = str(rc)
return dict(LOG='SAS process has terminated unexpectedly. Pid State= ' +
outrc, LST='',ABORT=True)
lst = self.stdout.read1(4096).decode(self.sascfg.encoding)
lstf += lst
if len(lst) > 0:
lsts = lst.rpartition('Select:')
if lsts[0] != '' and lsts[1] != '':
found = True
query = lsts[1] + lsts[2].rsplit('\n?')[0] + '\n'
print('Processing interrupt\nAttn handler Query is\n\n' + query)
response = self.sascfg._prompt("Please enter your Response: ")
self.stdin.write(response.encode(self.sascfg.encoding) + b'\n')
self.stdin.flush()
if (response == 'C' or response == 'c') and query.count("C. Cancel") >= 1:
bc = True
break
else:
lsts = lst.rpartition('Press')
if lsts[0] != '' and lsts[1] != '':
query = lsts[1] + lsts[2].rsplit('\n?')[0] + '\n'
print('Secondary Query is:\n\n' + query)
response = self.sascfg._prompt("Please enter your Response: ")
self.stdin.write(response.encode(self.sascfg.encoding) + b'\n')
self.stdin.flush()
if (response == 'N' or response == 'n') and query.count("N to continue") >= 1:
bc = True
break
else:
#print("******************No 'Select' or 'Press' found in lst=")
pass
else:
log = self.stderr.read1(4096).decode(self.sascfg.encoding)
logf += log
self._log += log
if log.count(eos) >= 1:
print("******************Found end of step. No interrupt processed")
found = True
if found:
break
sleep(.25)
lstr = lstf
logr = logf
return dict(LOG=logr, LST=lstr, BC=bc)
def sync_wait_reapable(pid):
waitid(os.P_PID, pid, os.WEXITED | os.WNOWAIT)
def _interact_leave(self, signum, sigframe):
while os.waitid(os.P_ALL, 0, os.WEXITED | os.WNOHANG) is not None:
pass
p_bmc_r, p_bmc_w = os.pipe()
os.set_inheritable(p_bmc_r,True)
id_bmc=os.fork()
if id_bmc==0:
sys.stdin.close()
os.dup2(p_bmc_r, 0)
os.execl(binpath+'vvt-bmc','vvt-bmc','--solver='+binpath+'z3 -smt2 -in','-i','-d','50')
os.close(p_bmc_r)
while True:
inp = sys.stdin.buffer.read(1024)
if len(inp)==0:
os.close(p_ic3_w)
os.close(p_bmc_w)
break
os.write(p_ic3_w,inp)
os.write(p_bmc_w,inp)
id_exit,res = os.wait()
if id_exit==id_ic3:
if res==0:
os.kill(id_bmc,signal.SIGKILL)
else:
os.waitid(os.P_PID,id_bmc,os.WEXITED)
else:
if res==0:
os.kill(id_ic3,signal.SIGKILL)
else:
os.waitid(os.P_PID,id_ic3,os.WEXITED)
def save_output(args, output_file):
"""Run args[0] with arguments args[1:] and save standard output to output_file."""
with open(output_file, 'w', encoding='utf-8') as f:
p = subprocess.Popen(args, stfout=f)
os.waitid(p.pid, 0)
#status = os.system("python preprocess.py -train_src data/src-train.txt -train_tgt data/tgt-train.txt -valid_src data/src-val.txt -valid_tgt data/tgt-val.txt -save_data data/corpus")
#if (status != 0):
#sys.exit(1)
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
status = os.system(
"./train.py -data data/corpus -save_model corpus_model -world_size 1 -gpu_ranks 0 -batch_size "
+ str(args.onmt_batch) +
" -seed 1 -log_file omnt.log -train_steps " +
str(args.onmt_steps) + " -valid_steps " +
str(args.onmt_valid_steps))
if (status != 0):
sys.exit(1)
#sys.exit(0)
else:
#wait for processes to exit
res1 = os.waitid(os.P_PID, pid_summarunner, os.WEXITED)
res2 = os.waitid(os.P_PID, pid_opennmt, os.WEXITED)
print(str(res1.si_status) + " " + str(res2.si_status))
log.write("Training finished, ")
if ((res1.si_status == 0) & (res2.si_status == 0)):
log.write("no erros encountered\n")
log.close()
sys.exit(0)
elif ((res1.si_status != 0) & (res2.si_status == 0)):
log.write("SummaRuNNer finished with error\n")
log.close()
sys.exit(1)
elif ((res1.si_status == 0) & (res2.si_status != 0)):
log.write("OpenNMT finished with error\n")
log.close()
sys.exit(1)
def _breakprompt(self, eos):
found = False
logf = ''
lstf = ''
bc = False
if self.pid is None:
return dict(
LOG=
b"No SAS process attached. SAS process has terminated unexpectedly.",
LST=b'',
ABORT=True)
interrupt = signal.SIGINT
os.kill(self.pid, interrupt)
sleep(.25)
while True:
rc = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
if rc is not None:
self.pid = None
outrc = str(rc)
return dict(
LOG=b'SAS process has terminated unexpectedly. Pid State= '
+ outrc.encode(),
LST=b'',
ABORT=True)
lst = self.stdout.read1(4096).decode()
lstf += lst
if len(lst) > 0:
lsts = lst.rpartition('Select:')
if lsts[0] != '' and lsts[1] != '':
found = True
query = lsts[1] + lsts[2].rsplit('\n?')[0] + '\n'
print('Processing interrupt\nAttn handler Query is\n\n' +
query)
response = self.sascfg._prompt(
"Please enter your Response: ")
self.stdin.write(response.encode() + b'\n')
self.stdin.flush()
if (response == 'C' or response
== 'c') and query.count("C. Cancel") >= 1:
bc = True
break
else:
lsts = lst.rpartition('Press')
if lsts[0] != '' and lsts[1] != '':
query = lsts[1] + lsts[2].rsplit('\n?')[0] + '\n'
print('Secondary Query is:\n\n' + query)
response = self.sascfg._prompt(
"Please enter your Response: ")
self.stdin.write(response.encode() + b'\n')
self.stdin.flush()
if (response == 'N' or response
== 'n') and query.count("N to continue") >= 1:
bc = True
break
else:
#print("******************No 'Select' or 'Press' found in lst=")
pass
else:
log = self.stderr.read1(4096).decode()
logf += log
self._log += log
if log.count(eos) >= 1:
print(
"******************Found end of step. No interrupt processed"
)
found = True
if found:
break
sleep(.25)
lstr = lstf
logr = logf
return dict(LOG=logr, LST=lstr, BC=bc)
def main():
# Block signals so that they aren't handled
# according to their default dispositions
mask = sigset_t()
fdsi = signalfd_siginfo()
sigemptyset(mask)
sigaddset(mask, SIGINT)
sigaddset(mask, SIGQUIT)
sigaddset(mask, SIGCHLD)
sigaddset(mask, SIGPIPE)
print("Blocking signals")
sigprocmask(SIG_BLOCK, mask, None)
# Get a signalfd descriptor
sfd = signalfd(-1, mask, SFD_CLOEXEC | SFD_NONBLOCK)
print("Got signalfd", sfd)
# Get a epoll descriptor
ep = epoll()
print("Got epollfd", ep.fileno())
print("Adding signalfd fd {} to epoll".format(sfd))
ep.register(sfd, EPOLLIN)
# Get two pair of pipes, one for stdout and one for stderr
stdout_pair = (c_int * 2)()
pipe2(byref(stdout_pair), O_CLOEXEC | O_NONBLOCK)
print("Got stdout pipe pair", stdout_pair[0], stdout_pair[1])
print("Adding pipe fd {} to epoll".format(stdout_pair[0]))
ep.register(stdout_pair[0], EPOLLIN)
stderr_pair = (c_int * 2)()
pipe2(byref(stderr_pair), O_CLOEXEC | O_NONBLOCK)
print("Got stderr pipe pair", stderr_pair[0], stderr_pair[1])
print("Adding pipe fd {} to epoll".format(stdout_pair[0]))
ep.register(stderr_pair[0], EPOLLIN)
prog = argv[1:]
if not prog:
prog = ['echo', 'usage: demo.py PROG [ARGS]']
print("Going to start program:", prog)
# Fork :-)
pid = fork()
if pid == 0: # Child.
# NOTE: we are not closing any of the pipe ends. Why? Because they are
# all O_CLOEXEC and will thus not live across the execlp call down
# below.
dup3(stdout_pair[1], 1, 0)
dup3(stderr_pair[1], 2, 0)
execlp(prog[0], *prog)
return -1
else:
close(stdout_pair[1])
close(stderr_pair[1])
with fdopen(sfd, 'rb', 0) as sfd_stream:
waiting_for = set(['stdout', 'stderr', 'proc'])
while waiting_for:
print("Waiting for events...", ' '.join(waiting_for), flush=True)
event_list = ep.poll(maxevents=10)
print("epoll_wait() read {} events".format(len(event_list)))
for fd, events in event_list:
print("[event]")
event_bits = []
if events & EPOLLIN == EPOLLIN:
event_bits.append('EPOLLIN')
if events & EPOLLOUT == EPOLLOUT:
event_bits.append('EPOLLOUT')
if events & EPOLLRDHUP == EPOLLRDHUP:
event_bits.append('EPOLLRDHUP')
if events & EPOLLPRI == EPOLLPRI:
event_bits.append('EPOLLPRI')
if events & EPOLLERR == EPOLLERR:
event_bits.append('EPOLLERR')
if events & EPOLLHUP == EPOLLHUP:
event_bits.append('EPOLLHUP')
if fd == sfd:
fd_name = 'signalfd()'
elif fd == stdout_pair[0]:
fd_name = 'stdout pipe2()'
elif fd == stderr_pair[0]:
fd_name = 'stderr pipe2()'
else:
fd_name = "???"
print(" events: {} ({})".format(
events, ' | '.join(event_bits)))
print(" fd: {} ({})".format(fd, fd_name))
if fd == sfd:
print("signalfd() descriptor ready")
if events & EPOLLIN:
print("Reading data from signalfd()...")
# Read the next delivered signal
sfd_stream.readinto(fdsi)
if fdsi.ssi_signo == SIGINT:
print("Got SIGINT")
elif fdsi.ssi_signo == SIGQUIT:
print("Got SIGQUIT")
raise SystemExit("exiting prematurly")
elif fdsi.ssi_signo == SIGCHLD:
print("Got SIGCHLD")
waitid_result = waitid(
P_PID, pid,
WNOHANG |
WEXITED | WSTOPPED | WCONTINUED |
WUNTRACED)
if waitid_result is None:
print("child not ready")
else:
print("child event")
print("si_pid:", waitid_result.si_pid)
print("si_uid:", waitid_result.si_uid)
print("si_signo:", waitid_result.si_signo)
assert waitid_result.si_signo == SIGCHLD
print("si_status:", waitid_result.si_status)
print("si_code:", waitid_result.si_code)
if waitid_result.si_code == CLD_EXITED:
# assert WIFEXITED(waitid_result.si_status)
print("child exited normally")
print("exit code:",
WEXITSTATUS(waitid_result.si_status))
waiting_for.remove('proc')
if 'stdout' in waiting_for:
ep.unregister(stdout_pair[0])
close(stdout_pair[0])
waiting_for.remove('stdout')
if 'stderr' in waiting_for:
ep.unregister(stderr_pair[0])
close(stderr_pair[0])
waiting_for.remove('stderr')
elif waitid_result.si_code == CLD_KILLED:
assert WIFSIGNALED(waitid_result.si_status)
print("child was killed by signal")
print("death signal:",
waitid_result.si_status)
waiting_for.remove('proc')
elif waitid_result.si_code == CLD_DUMPED:
assert WIFSIGNALED(waitid_result.si_status)
print("core:",
WCOREDUMP(waitid_result.si_status))
elif waitid_result.si_code == CLD_STOPPED:
print("child was stopped")
print("stop signal:",
waitid_result.si_status)
elif waitid_result.si_code == CLD_TRAPPED:
print("child was trapped")
# TODO: we could explore trap stuff here
elif waitid_result.si_code == CLD_CONTINUED:
print("child was continued")
else:
raise SystemExit(
"Unknown CLD_ code: {}".format(
waitid_result.si_code))
elif fdsi.ssi_signo == SIGPIPE:
print("Got SIGPIPE")
else:
print("Read unexpected signal: {}".format(
fdsi.ssi_signo))
elif fd == stdout_pair[0]:
print("pipe() (stdout) descriptor ready")
if events & EPOLLIN:
print("Reading data from stdout...")
data = read(stdout_pair[0], PIPE_BUF)
print("Read {} bytes from stdout".format(len(data)))
print(data)
if events & EPOLLHUP:
print("Removing stdout pipe from epoll")
ep.unregister(stdout_pair[0])
print("Closing stdout pipe")
close(stdout_pair[0])
waiting_for.remove('stdout')
elif fd == stderr_pair[0]:
print("pipe() (stderr) descriptor ready")
if events & EPOLLIN:
print("Reading data from stdout...")
data = read(stderr_pair[0], PIPE_BUF)
print("Read {} bytes from stderr".format(len(data)))
print(data)
if events & EPOLLHUP:
print("Removing stderr pipe from epoll")
ep.unregister(stderr_pair[0])
print("Closing stderr pipe")
close(stderr_pair[0])
waiting_for.remove('stderr')
else:
# FIXME: we are still getting weird activation events on fd
# 0 (stdin) with events == 0 (nothing). I cannot explain
# this yet.
print("Unexpected descriptor ready:", fd)
assert not waiting_for
print("Closing", ep)
ep.close()
print("Unblocking signals")
sigprocmask(SIG_UNBLOCK, mask, None)
print("Exiting normally")
os.set_inheritable(p_bmc_r, True)
id_bmc = os.fork()
if id_bmc == 0:
sys.stdin.close()
os.dup2(p_bmc_r, 0)
os.execl(binpath + 'vvt-bmc', 'vvt-bmc',
'--solver=' + binpath + 'z3 -smt2 -in', '-i', '-d', '50')
os.close(p_bmc_r)
while True:
inp = sys.stdin.buffer.read(1024)
if len(inp) == 0:
os.close(p_ic3_w)
os.close(p_bmc_w)
break
os.write(p_ic3_w, inp)
os.write(p_bmc_w, inp)
id_exit, res = os.wait()
if id_exit == id_ic3:
if res == 0:
os.kill(id_bmc, signal.SIGKILL)
else:
os.waitid(os.P_PID, id_bmc, os.WEXITED)
else:
if res == 0:
os.kill(id_ic3, signal.SIGKILL)
else:
os.waitid(os.P_PID, id_ic3, os.WEXITED)
def main():
with ExitStack() as stack:
# Block signals so that they aren't handled according to their default
# dispositions until after this block is terminated, or, in our case,
# never since we also use signalfd to collect them.
print("Blocking signals...")
stack.enter_context(pthread_sigmask([SIGCHLD]))
print("Setting up epoll...")
es = EpollSelector()
print("Got", es)
print("Setting up signalfd...")
sfd_obj = stack.enter_context(
signalfd([SIGCHLD], SFD_CLOEXEC | SFD_NONBLOCK))
print("Got", sfd_obj)
print("Adding signalfd epoll...")
es.register(sfd_obj, EVENT_READ, 'signalfd(2)')
print("Setting up stdout pipe...")
stdout_pair = pipe2(O_CLOEXEC | O_NONBLOCK)
print("Got", stdout_pair)
print("Adding stdout pipe to epoll...")
es.register(stdout_pair[0], EVENT_READ, 'stdout')
print("Getting stderr pipe...")
stderr_pair = pipe2(O_CLOEXEC | O_NONBLOCK)
print("Got", stderr_pair)
print("Adding stderr pipe to epoll...")
es.register(stderr_pair[0], EVENT_READ, 'stderr')
prog = argv[1:]
if not prog:
prog = ['echo', 'usage: demo.py PROG [ARGS]']
print("Going to start program:", prog)
# Fork :-)
pid = fork()
if pid == 0: # Child.
# NOTE: we are not closing any of the pipe ends. Why? Because they
# are all O_CLOEXEC and will thus not live across the execlp call
# down below.
dup3(stdout_pair[1], 1, 0)
dup3(stderr_pair[1], 2, 0)
execlp(prog[0], *prog)
return -1
else:
close(stdout_pair[1])
close(stderr_pair[1])
waiting_for = set(['stdout', 'stderr', 'proc'])
while waiting_for:
print("Waiting for events...", ' '.join(waiting_for), flush=True)
event_list = es.select()
print("EpollSelector.select() read {} events".format(
len(event_list)))
for key, events in event_list:
print("[event]")
print(" key: {!r}".format(key))
print(" events: {!r}".format(events))
if key.fd == sfd_obj.fileno():
print("signalfd() descriptor ready")
if events & EVENT_READ:
print("Reading data from signalfd()...")
# Read the next delivered signal
try:
fdsi = sfd_obj.read()[0]
except IndexError:
continue
if fdsi.ssi_signo == SIGINT:
print("Got SIGINT")
elif fdsi.ssi_signo == SIGQUIT:
print("Got SIGQUIT")
raise SystemExit("exiting prematurly")
elif fdsi.ssi_signo == SIGCHLD:
print("Got SIGCHLD")
waitid_result = waitid(
P_PID, pid,
WNOHANG |
WEXITED | WSTOPPED | WCONTINUED |
WUNTRACED)
if waitid_result is None:
print("child not ready")
else:
print("child event")
print("si_pid:", waitid_result.si_pid)
print("si_uid:", waitid_result.si_uid)
print("si_signo:", waitid_result.si_signo)
assert waitid_result.si_signo == SIGCHLD
print("si_status:", waitid_result.si_status)
print("si_code:", waitid_result.si_code)
if waitid_result.si_code == CLD_EXITED:
# assert WIFEXITED(waitid_result.si_status)
print("child exited normally")
print("exit code:",
WEXITSTATUS(waitid_result.si_status))
waiting_for.remove('proc')
if 'stdout' in waiting_for:
es.unregister(stdout_pair[0])
close(stdout_pair[0])
waiting_for.remove('stdout')
if 'stderr' in waiting_for:
es.unregister(stderr_pair[0])
close(stderr_pair[0])
waiting_for.remove('stderr')
elif waitid_result.si_code == CLD_KILLED:
assert WIFSIGNALED(waitid_result.si_status)
print("child was killed by signal")
print("death signal:",
waitid_result.si_status)
waiting_for.remove('proc')
elif waitid_result.si_code == CLD_DUMPED:
assert WIFSIGNALED(waitid_result.si_status)
print("core:",
WCOREDUMP(waitid_result.si_status))
elif waitid_result.si_code == CLD_STOPPED:
print("child was stopped")
print("stop signal:",
waitid_result.si_status)
elif waitid_result.si_code == CLD_TRAPPED:
print("child was trapped")
# TODO: we could explore trap stuff here
elif waitid_result.si_code == CLD_CONTINUED:
print("child was continued")
else:
raise SystemExit(
"Unknown CLD_ code: {}".format(
waitid_result.si_code))
elif fdsi.ssi_signo == SIGPIPE:
print("Got SIGPIPE")
else:
print("Read unexpected signal: {}".format(
fdsi.ssi_signo))
elif key.fd in (stdout_pair[0], stderr_pair[0]):
print("{} pipe() descriptor ready".format(key.data))
if events & EVENT_READ:
print("Reading data from {} pipe...".format(key.data))
data = read(key.fd, PIPE_BUF)
print("Read {} bytes from {} pipe".format(
len(data), key.data))
print("Data:", data)
if len(data) == 0:
print("Removing {} pipe from EpollSelector".format(
key.data))
es.unregister(key.fd)
print("Closing pipe")
close(key.fd)
waiting_for.remove(key.data)
else:
# FIXME: we are still getting weird activation events on fd
# 0 (stdin) with events == 0 (nothing). I cannot explain
# this yet.
print("Unexpected descriptor ready:", key.fd)
assert not waiting_for
