def run(self):
# Open our pipes.
in_pipe = os.fdopen(os.dup(0), 'r')
out_pipe = os.fdopen(os.dup(1), 'w')
devnull = open("/dev/null", 'r')
os.dup2(devnull.fileno(), 0)
devnull.close()
os.dup2(2, 1)
# Dump our startup message.
robj = {
"id": None,
"result": None
}
out_pipe.write(pickle.dumps(robj))
out_pipe.flush()
sys.stderr.write("proxy %d: started.\n" % os.getpid())
# Get the call from the other side.
while True:
try:
obj = pickle.load(in_pipe)
sys.stderr.write("proxy %d: <- %s\n" % (os.getpid(), obj))
except:
# We're done!
break
def closure(obj, out_pipe):
def fn():
self._process(obj, out_pipe)
return fn
t = threading.Thread(target=closure(obj, out_pipe))
t.start()
def test_is_same_output(self):
fd1 = sys.stderr.fileno()
fd2 = os.dup(fd1)
try:
self.assertTrue(ConfigureOutputHandler._is_same_output(fd1, fd2))
finally:
os.close(fd2)
fd2, path = tempfile.mkstemp()
try:
self.assertFalse(ConfigureOutputHandler._is_same_output(fd1, fd2))
fd3 = os.dup(fd2)
try:
self.assertTrue(ConfigureOutputHandler._is_same_output(fd2, fd3))
finally:
os.close(fd3)
with open(path, 'a') as fh:
fd3 = fh.fileno()
self.assertTrue(
ConfigureOutputHandler._is_same_output(fd2, fd3))
finally:
os.close(fd2)
os.remove(path)
def printfiles():
print('Writing to files...')
old = dup(1)
close(1)
open('height{}_find{}_binsearch.tex'.format(height, find), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY)
print('\\begin{tikzpicture}')
print('% height={}, finding={}, blocksize={}'.format(height, find, blocksize))
print()
print('% Binary search')
printarray(array, 0)
print('\\end{tikzpicture}')
close(1)
open('height{}_find{}_bfstree.tex'.format(height, find), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY)
print('\\begin{tikzpicture}')
print('% height={}, finding={}, blocksize={}'.format(height, find, blocksize))
print()
print('% BFS tree')
printtree(bfs, 0)
print('\\end{tikzpicture}')
close(1)
open('height{}_find{}_vebtree.tex'.format(height, find), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY)
print('\\begin{tikzpicture}')
print('% height={}, finding={}, blocksize={}'.format(height, find, blocksize))
print()
print('% vEB tree')
printtree(veb, 0)
print('\\end{tikzpicture}')
close(1)
dup(old)
close(old)
print('Done')
def build_extensions(silent=False):
if silent:
devnull = open(os.devnull, 'w')
oldstdout = os.dup(sys.stdout.fileno())
os.dup2(devnull.fileno(), sys.stdout.fileno())
oldstderr = os.dup(sys.stderr.fileno())
os.dup2(devnull.fileno(), sys.stderr.fileno())
try:
build_extensions(silent=False)
finally:
os.dup2(oldstdout, sys.stdout.fileno())
os.dup2(oldstderr, sys.stderr.fileno())
devnull.close()
return
cur_dir = os.getcwd()
try:
os.chdir(os.path.dirname(os.path.abspath(__file__)))
setup(
name='game',
py_modules=['game'],
ext_modules=[
Extension('_game',
['game.i', 'game.cpp'],
depends=['game.h'],
swig_opts=['-c++'],
extra_compile_args = ['--std=c++0x'],
),
],
script_args=['build_ext', '--inplace']
)
finally:
os.chdir(cur_dir)
def worker(sock):
"""
Called by a worker process after the fork().
"""
signal.signal(SIGHUP, SIG_DFL)
signal.signal(SIGCHLD, SIG_DFL)
signal.signal(SIGTERM, SIG_DFL)
# restore the handler for SIGINT,
# it's useful for debugging (show the stacktrace before exit)
signal.signal(SIGINT, signal.default_int_handler)
# Read the socket using fdopen instead of socket.makefile() because the latter
# seems to be very slow; note that we need to dup() the file descriptor because
# otherwise writes also cause a seek that makes us miss data on the read side.
infile = os.fdopen(os.dup(sock.fileno()), "a+", 65536)
outfile = os.fdopen(os.dup(sock.fileno()), "a+", 65536)
exit_code = 0
try:
worker_main(infile, outfile)
except SystemExit as exc:
exit_code = compute_real_exit_code(exc.code)
finally:
try:
outfile.flush()
except Exception:
pass
return exit_code
def open_network_connections(port):
s = socket.socket()
s.connect(('localhost', port))
in_stream = ph_fdopen(os.dup(s.fileno()), 'r', BUF_SIZE)
out_stream = ph_fdopen(os.dup(s.fileno()), 'w', BUF_SIZE)
return NetworkConnections(BinaryDownStreamFilter(in_stream),
BinaryUpStreamFilter(out_stream), s, port)
def do_ipy(self, args):
"""Start an interactive Python interpreter"""
from c1218.data import C1218Packet
from c1219.access.general import C1219GeneralAccess
from c1219.access.security import C1219SecurityAccess
from c1219.access.log import C1219LogAccess
from c1219.access.telephone import C1219TelephoneAccess
vars = {
'__version__': __version__,
'frmwk': self.frmwk,
'C1218Packet': C1218Packet,
'C1219GeneralAccess': C1219GeneralAccess,
'C1219SecurityAccess': C1219SecurityAccess,
'C1219LogAccess': C1219LogAccess,
'C1219TelephoneAccess': C1219TelephoneAccess
}
banner = 'The Framework Instance Is In The Variable \'frmwk\'' + os.linesep
if self.frmwk.is_serial_connected():
vars['conn'] = self.frmwk.serial_connection
banner = banner + 'The Connection Instance Is In The Variable \'conn\'' + os.linesep
pyconsole = code.InteractiveConsole(vars)
savestdin = os.dup(sys.stdin.fileno())
savestdout = os.dup(sys.stdout.fileno())
savestderr = os.dup(sys.stderr.fileno())
try:
pyconsole.interact(banner)
except SystemExit:
sys.stdin = os.fdopen(savestdin, 'r', 0)
sys.stdout = os.fdopen(savestdout, 'w', 0)
sys.stderr = os.fdopen(savestderr, 'w', 0)
def daemonize(self):
"""Makeself a daemon process.
Double fork, close standard pipes, start a new session and
open logs.
"""
pid = os.fork()
if pid == 0: # first child
os.setsid()
pid = os.fork()
if pid == 0: # second child
# Can't chdir to root if we have relative paths to
# conffile and other modules
#os.chdir('/')
os.umask(0)
else:
os._exit(0)
else:
os._exit(0)
# close stdin, stdout and stderr ...
for fd in range(3):
try:
os.close(fd)
except OSError:
pass
# ... and replace them with /dev/null
os.open('/dev/null', os.O_RDWR)
os.dup(0)
os.dup(0)
syslog.openlog('hip-mgmt-iface',
syslog.LOG_PID | syslog.LOG_NDELAY,
syslog.LOG_DAEMON)
syslog.syslog('FirewallController started.')
def execute (self):
# Python stdout, stderr, stdin redirection
sys.stdout, self.stdout = self.stdout, sys.stdout
sys.stderr, self.stderr = self.stderr, sys.stderr
sys.stdin, self.stdin = self.stdin, sys.stdin
# System stdout, stderr redirection
sys_stdout = os.dup(1)
sys_stderr = os.dup(2)
os.dup2 (self.pipewrite, 1)
os.dup2 (self.pipewrite, 2)
self.shell.eval(self)
self.view.scroll_mark_onscreen(self.buffer.get_insert())
while Gtk.events_pending():
Gtk.main_iteration()
# Get system output and remove system redirection
os.dup2 (sys_stdout, 1)
os.dup2 (sys_stderr, 2)
os.close (sys_stdout)
os.close (sys_stderr)
# Remove python redirection
sys.stdout, self.stdout = self.stdout, sys.stdout
sys.stderr, self.stderr = self.stderr, sys.stderr
sys.stdin, self.stdin = self.stdin, sys.stdin
def __init__(self, mixed_out_err = False):
"Start capture of the Unix-level stdout and stderr."
if (sys.platform == 'win32' or # os.tmpfile fails, cpython issue #2232
not hasattr(os, 'tmpfile') or
not hasattr(os, 'dup') or
not hasattr(os, 'dup2') or
not hasattr(os, 'fdopen')):
self.dummy = 1
else:
self.dummy = 0
# make new stdout/stderr files if needed
self.localoutfd = os.dup(1)
self.localerrfd = os.dup(2)
if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1:
self.saved_stdout = sys.stdout
sys.stdout = os.fdopen(self.localoutfd, 'w', 1)
else:
self.saved_stdout = None
if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2:
self.saved_stderr = sys.stderr
sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
else:
self.saved_stderr = None
self.tmpout = os.tmpfile()
if mixed_out_err:
self.tmperr = self.tmpout
else:
self.tmperr = os.tmpfile()
os.dup2(self.tmpout.fileno(), 1)
os.dup2(self.tmperr.fileno(), 2)
def init_popen_io(execmodel):
if not hasattr(os, 'dup'): # jython
io = Popen2IO(sys.stdout, sys.stdin, execmodel)
import tempfile
sys.stdin = tempfile.TemporaryFile('r')
sys.stdout = tempfile.TemporaryFile('w')
else:
try:
devnull = os.devnull
except AttributeError:
if os.name == 'nt':
devnull = 'NUL'
else:
devnull = '/dev/null'
# stdin
stdin = execmodel.fdopen(os.dup(0), 'r', 1)
fd = os.open(devnull, os.O_RDONLY)
os.dup2(fd, 0)
os.close(fd)
# stdout
stdout = execmodel.fdopen(os.dup(1), 'w', 1)
fd = os.open(devnull, os.O_WRONLY)
os.dup2(fd, 1)
# stderr for win32
if os.name == 'nt':
sys.stderr = execmodel.fdopen(os.dup(2), 'w', 1)
os.dup2(fd, 2)
os.close(fd)
io = Popen2IO(stdout, stdin, execmodel)
sys.stdin = execmodel.fdopen(0, 'r', 1)
sys.stdout = execmodel.fdopen(1, 'w', 1)
return io
def __init__(self, mixed_out_err = False):
"Start capture of the Unix-level stdout and stderr."
if (not hasattr(os, 'tmpfile') or
not hasattr(os, 'dup') or
not hasattr(os, 'dup2') or
not hasattr(os, 'fdopen')):
self.dummy = 1
else:
try:
self.tmpout = os.tmpfile()
if mixed_out_err:
self.tmperr = self.tmpout
else:
self.tmperr = os.tmpfile()
except OSError: # bah? on at least one Windows box
self.dummy = 1
return
self.dummy = 0
# make new stdout/stderr files if needed
self.localoutfd = os.dup(1)
self.localerrfd = os.dup(2)
if hasattr(sys.stdout, 'fileno') and sys.stdout.fileno() == 1:
self.saved_stdout = sys.stdout
sys.stdout = os.fdopen(self.localoutfd, 'w', 1)
else:
self.saved_stdout = None
if hasattr(sys.stderr, 'fileno') and sys.stderr.fileno() == 2:
self.saved_stderr = sys.stderr
sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
else:
self.saved_stderr = None
os.dup2(self.tmpout.fileno(), 1)
os.dup2(self.tmperr.fileno(), 2)
def spawn(prog, args):
p2cread, p2cwrite = os.pipe()
c2pread, c2pwrite = os.pipe()
pid = os.fork()
if pid == 0:
# Child
for i in 0, 1, 2:
try:
os.close(i)
except os.error:
pass
if os.dup(p2cread) != 0:
sys.stderr.write('popen2: bad read dup\n')
if os.dup(c2pwrite) != 1:
sys.stderr.write('popen2: bad write dup\n')
if os.dup(c2pwrite) != 2:
sys.stderr.write('popen2: bad write dup\n')
os.closerange(3, MAXFD)
try:
os.execvp(prog, args)
finally:
sys.stderr.write('execvp failed\n')
os._exit(1)
os.close(p2cread)
os.close(c2pwrite)
return pid, c2pread, p2cwrite
def start(self):
"""Start tee-ing all stdout and stderr output to the file."""
# Flush and save old file descriptors.
sys.stdout.flush()
sys.stderr.flush()
self._old_stdout_fd = os.dup(sys.stdout.fileno())
self._old_stderr_fd = os.dup(sys.stderr.fileno())
# Save file objects
self._old_stdout = sys.stdout
self._old_stderr = sys.stderr
# Replace std[out|err] with unbuffered file objects
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
# Create a tee subprocess.
self._tee = _TeeProcess([self._file], True, self._old_stderr_fd,
os.getpid())
self._tee.start()
# Redirect stdout and stderr to the tee subprocess.
writer_pipe = self._tee.writer_pipe
os.dup2(writer_pipe, sys.stdout.fileno())
os.dup2(writer_pipe, sys.stderr.fileno())
os.close(writer_pipe)
def does_stuff():
a, b = os.pipe()
c = os.dup(a)
d = os.dup(b)
assert a != b
assert a != c
assert a != d
assert b != c
assert b != d
assert c != d
os.close(c)
os.dup2(d, c)
e, f = os.pipe()
assert e != a
assert e != b
assert e != c
assert e != d
assert f != a
assert f != b
assert f != c
assert f != d
assert f != e
os.close(a)
os.close(b)
os.close(c)
os.close(d)
os.close(e)
os.close(f)
return 42
def redirect_streams(self):
# redirect stdout and stderr of steam api
devnull = os.open(os.devnull, 777)
self.old_stdout = os.dup(1)
self.old_stderr = os.dup(2)
os.dup2(devnull, 1)
os.dup2(devnull, 2)
def spawn(prog, args):
p2cread, p2cwrite = os.pipe()
c2pread, c2pwrite = os.pipe()
pid = os.fork()
if pid == 0:
# Child
os.close(0)
os.close(1)
os.close(2)
if os.dup(p2cread) <> 0:
sys.stderr.write('popen2: bad read dup\n')
if os.dup(c2pwrite) <> 1:
sys.stderr.write('popen2: bad write dup\n')
if os.dup(c2pwrite) <> 2:
sys.stderr.write('popen2: bad write dup\n')
for i in range(3, MAXFD):
try:
os.close(i)
except:
pass
try:
os.execvp(prog, args)
finally:
sys.stderr.write('execvp failed\n')
os._exit(1)
os.close(p2cread)
os.close(c2pwrite)
return pid, c2pread, p2cwrite
def __enter__(self):
# prepare standard file descriptors for raw manipulation
self.was_blocking = os.get_blocking(0)
os.set_blocking(0, False)
try:
self.terminal_attr_stdin = termios.tcgetattr(0)
self.terminal_attr_stdout = termios.tcgetattr(1)
self.terminal_attr_stderr = termios.tcgetattr(2)
tty.setraw(0)
tty.setraw(1)
tty.setraw(2)
except termios.error: # probably redirected
self.terminal_attr_stdin = None
# redirect standard file descriptors to new PTY
master, slave = pty.openpty()
os.set_blocking(master, False)
self.real_stdin = os.dup(0)
self.real_stdout = os.dup(1)
self.real_stderr = os.dup(2)
os.close(0)
os.close(1)
os.close(2)
os.dup2(slave, 0)
os.dup2(slave, 1)
os.dup2(slave, 2)
os.close(slave)
self.terminal_pipe = master
# start REPL in separate thread
threading.Thread(target=repl, args=(self,), daemon=True).start()
return self
def decorated( *fargs, **fkwargs ):
suppress = args.suppress_output
if suppress:
# Get rid of what is already there ( should be nothing for this script )
sys.stdout.flush()
# Save file descriptors so it can be reactivated later
saved_stdout = os.dup( 1 )
saved_stderr = os.dup( 2 )
# /dev/null is used just to discard what is being printed
devnull = os.open( '/dev/null', os.O_WRONLY )
# Duplicate the file descriptor for /dev/null
# and overwrite the value for stdout (file descriptor 1)
os.dup2( devnull, 1 )
os.dup2( devnull, 2 )
result = f( *fargs, **fkwargs )
if suppress:
# Close devnull after duplication (no longer needed)
os.close( devnull )
# Reenable stdout and stderr
os.dup2( saved_stdout, 1 )
os.dup2( saved_stderr, 2 )
return result
def __exit__( self, exc_type, exc_val, exc_tb ):
if exc_type is None:
_r, _w = os.pipe( )
def copy( ):
with os.fdopen( _w, 'w' ) as w:
w.write( self.stdin.getvalue( ) )
t = Thread( target=copy )
t.start( )
try:
_stdin = sys.stdin.fileno( )
_old_stdin = os.dup( _stdin )
os.close( _stdin )
assert _stdin == os.dup( _r )
# monkey-patch Fabric
_input_loop = fabric.operations.input_loop
fabric.operations.input_loop = input_loop
try:
self.stdin.result = self._run( )
finally:
fabric.operations.input_loop = _input_loop
os.close( _stdin )
os.dup( _old_stdin )
finally:
t.join( )
return False
def posix_redirect_output(filename=None, permanent=True):
"""
Redirect stdout/stderr to a file, using posix dup2.
"""
sys.stdout.flush()
sys.stderr.flush()
stdout_fd = sys.stdout.fileno()
stderr_fd = sys.stderr.fileno()
if not permanent:
stdout_fd_copy = os.dup(stdout_fd)
stderr_fd_copy = os.dup(stderr_fd)
if filename is None:
out_fd, filename = tempfile.mkstemp()
else:
out_fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
try:
# Redirect stdout and stderr to file
os.dup2(out_fd, stdout_fd)
os.dup2(out_fd, stderr_fd)
yield filename
finally:
sys.stdout.flush()
sys.stderr.flush()
os.close(out_fd)
if not permanent:
os.dup2(stdout_fd_copy, stdout_fd)
os.dup2(stderr_fd_copy, stderr_fd)
os.close(stdout_fd_copy)
os.close(stderr_fd_copy)
def grabOutput(self, flag):
if flag:
if not self.rpmout:
# Grab rpm output, but not the python one.
self.stdout = sys.stdout
self.stderr = sys.stderr
writer = codecs.getwriter(ENCODING)
reader = codecs.getreader(ENCODING)
sys.stdout = writer(os.fdopen(os.dup(1), "w"))
sys.stderr = writer(os.fdopen(os.dup(2), "w"))
pipe = os.pipe()
os.dup2(pipe[1], 1)
os.dup2(pipe[1], 2)
os.close(pipe[1])
self.rpmout = reader(os.fdopen(pipe[0], "r"))
setCloseOnExec(self.rpmout.fileno())
flags = fcntl.fcntl(self.rpmout.fileno(), fcntl.F_GETFL, 0)
flags |= os.O_NONBLOCK
fcntl.fcntl(self.rpmout.fileno(), fcntl.F_SETFL, flags)
else:
if self.rpmout:
self._rpmout()
os.dup2(sys.stdout.fileno(), 1)
os.dup2(sys.stderr.fileno(), 2)
sys.stdout = self.stdout
sys.stderr = self.stderr
del self.stdout
del self.stderr
self.rpmout.close()
self.rpmout = None
def spawnProc(tag, cmdline, copy=[]):
initWorkDir(tag)
os.chdir(workDir(tag))
for i in copy:
debug("Copying %s into workdir of %s" % (i, tag))
os.system("cp -r %s %s" % (i, workDir(tag)))
debug("Spawning '%s' as %s" % (" ".join(cmdline), tag))
saved_stdin = os.dup(0)
saved_stdout = os.dup(1)
saved_stderr = os.dup(2)
child_stdin = open("/dev/null", "r")
child_stdout = open("stdout.log", "w")
child_stderr = open("stderr.log", "w")
os.dup2(child_stdin.fileno(), 0)
os.dup2(child_stdout.fileno(), 1)
os.dup2(child_stderr.fileno(), 2)
pid = os.spawnvp(os.P_NOWAIT, cmdline[0], cmdline)
os.dup2(saved_stdin, 0)
os.dup2(saved_stdout, 1)
os.dup2(saved_stderr, 2)
Running[tag] = pid
Running[pid] = tag
def check_runnable_weight_field():
# Define the bpf program for checking purpose
bpf_check_text = """
#include <linux/sched.h>
unsigned long dummy(struct sched_entity *entity)
{
return entity->runnable_weight;
}
"""
# Get a temporary file name
tmp_file = NamedTemporaryFile(delete=False)
tmp_file.close();
# Duplicate and close stderr (fd = 2)
old_stderr = dup(2)
close(2)
# Open a new file, should get fd number 2
# This will avoid printing llvm errors on the screen
fd = open(tmp_file.name, O_WRONLY)
try:
t = BPF(text=bpf_check_text)
success_compile = True
except:
success_compile = False
# Release the fd 2, and next dup should restore old stderr
close(fd)
dup(old_stderr)
close(old_stderr)
# remove the temporary file and return
unlink(tmp_file.name)
return success_compile
def grabOutput(self, flag):
if flag:
if not self.rpmout:
# Grab rpm output, but not the python one.
self.stdout = sys.stdout
self.stderr = sys.stderr
writer = codecs.getwriter(ENCODING)
reader = codecs.getreader(ENCODING)
sys.stdout = writer(os.fdopen(os.dup(1), "w"),
errors="replace")
sys.stderr = writer(os.fdopen(os.dup(2), "w"),
errors="replace")
fd, rpmoutpath = tempfile.mkstemp("-smart-rpm-out.txt")
os.dup2(fd, 1)
os.dup2(fd, 2)
os.close(fd)
self.rpmout = reader(open(rpmoutpath))
os.unlink(rpmoutpath)
else:
if self.rpmout:
self._process_rpmout()
os.dup2(sys.stdout.fileno(), 1)
os.dup2(sys.stderr.fileno(), 2)
sys.stdout = self.stdout
sys.stderr = self.stderr
del self.stdout
del self.stderr
self.rpmout.close()
self.rpmout = None
self.rpmoutbuffer = ""
def _becomeLogSlave(self, slaveFd, loggerPid):
""" hand over control of io to logging process, grab info
from pseudo tty
"""
self.loggerPid = loggerPid
if self.withStdin and sys.stdin.isatty():
self.oldTermios = termios.tcgetattr(sys.stdin.fileno())
else:
self.oldTermios = None
newTermios = termios.tcgetattr(slaveFd)
# Don't wait after receiving a character
newTermios[6][termios.VTIME] = '\x00'
# Read at least these many characters before returning
newTermios[6][termios.VMIN] = '\x01'
termios.tcsetattr(slaveFd, termios.TCSADRAIN, newTermios)
# Raw mode
tty.setraw(slaveFd)
self.oldStderr = os.dup(sys.stderr.fileno())
self.oldStdout = os.dup(sys.stdout.fileno())
if self.withStdin:
self.oldStdin = os.dup(sys.stdin.fileno())
os.dup2(slaveFd, 0)
else:
self.oldStdin = sys.stdin.fileno()
os.dup2(slaveFd, 1)
os.dup2(slaveFd, 2)
os.close(slaveFd)
self.logging = True
def stdouterrin_setnull():
""" redirect file descriptors 0 and 1 (and possibly 2) to /dev/null.
note that this function may run remotely without py lib support.
"""
# complete confusion (this is independent from the sys.stdout
# and sys.stderr redirection that gateway.remote_exec() can do)
# note that we redirect fd 2 on win too, since for some reason that
# blocks there, while it works (sending to stderr if possible else
# ignoring) on *nix
import sys, os
try:
devnull = os.devnull
except AttributeError:
if os.name == 'nt':
devnull = 'NUL'
else:
devnull = '/dev/null'
# stdin
sys.stdin = os.fdopen(os.dup(0), 'rb', 0)
fd = os.open(devnull, os.O_RDONLY)
os.dup2(fd, 0)
os.close(fd)
# stdout
sys.stdout = os.fdopen(os.dup(1), 'wb', 0)
fd = os.open(devnull, os.O_WRONLY)
os.dup2(fd, 1)
# stderr for win32
if os.name == 'nt':
sys.stderr = os.fdopen(os.dup(2), 'wb', 0)
os.dup2(fd, 2)
os.close(fd)
def uriToBrowser(uri=None):
'''
Method that launches the URI in the default browser of the system. This returns no new entity.
:param uri: uri to open.
'''
# Temporally deactivating standard ouptut and error:
# Source: <https://stackoverflow.com/questions/2323080/how-can-i-disable-the-webbrowser-message-in-python>
# Cloning stdout (1) and stderr (2)
savout1 = os.dup(1)
savout2 = os.dup(2)
# Closing them
os.close(1)
os.close(2)
os.open(os.devnull, os.O_RDWR)
try:
# Opening the Tor URI using onion.cab proxy
if ".onion" in uri:
wb.get().open(uri.replace(".onion", ".onion.cab"), new=2)
else:
wb.get().open(uri, new=2)
finally:
# Reopening them...
os.dup2(savout1, 1)
os.dup2(savout2, 2)
def launch_client(cmd, fdin, fdout): cmd = cmd.split() pid = os.fork() if pid == 0: # child os.close(0) os.close(1) if os.dup(fdin) != 0: print "cannot duplicate fdin" if os.dup(fdout) != 1: print "cannot duplicate fdout" for i in range(3, MAXFD): try: os.close(i) except: pass try: os.execv(cmd[0], cmd) finally: print >>sys.stdrr, "failed to execv %s!" % cmd os._exit(1) else: # parent print "started %s %d" % (cmd, pid) return pid
def main():
s_o = os.dup(1)
s_e = os.dup(2)
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s")
logging.info('init')
print os.environ.get('PYTHONUNBUFFERED')
os.environ['PYTHONUNBUFFERED'] = '1'
fd = os.open('test.log', os.O_RDWR | os.O_CREAT)
os.dup2(fd, 1)
os.dup2(fd, 2)
print 'out1'
print >> sys.stderr, 'err1'
logging.info('init-1')
reload(sys)
print >> sys.stderr, os.environ.get('PYTHONUNBUFFERED')
logging.info('initi-2')
print 'out2'
print >> sys.stderr, 'err2'
os.dup2(s_o, 1)
os.dup2(s_e, 2)
print 'finish, err'
print >> sys.stderr, 'finish err'
logging.info('end')
# to the 'Test/' directory. from threading import Thread import time import cmor, numpy import contextlib import unittest import signal import sys, os import tempfile import cdms2 # ------------------------------------------------------ # Copy stdout and stderr file descriptor for cmor output # ------------------------------------------------------ newstdout = os.dup(1) newstderr = os.dup(2) # -------------- # Create tmpfile # -------------- tmpfile = tempfile.mkstemp( ) #tempfile[0] = File number, tempfile[1] = File name. os.dup2(tmpfile[0], 1) os.dup2(tmpfile[0], 2) os.close(tmpfile[0]) global testOK testOK = [] # ==============================
def __enter__(self):
sys.stdout.flush()
self.oldstdout_fno = os.dup(sys.stdout.fileno())
os.dup2(self._new_stdout.fileno(), 1)
def __init__(self, fd):
self.fd = os.dup(fd)
def setUp(self):
self.out = sys.stdout
self.out_fd = os.dup(1)
def __enter__(self):
self.stdout = os.dup(self.stdout_fd)
os.dup2(self.stdout_pipe_write, self.stdout_fd)
return self
filename = url.split('/')[-1]
filesize = int(requests.head(url).headers['Content-Length'])
print('%s filesize:%s' % (filename, filesize))
threadnum = 3
threading.BoundedSemaphore(threadnum) # 允许线程个数
step = filesize // threadnum
mtd_list = []
start = 0
end = -1
tempf = open('./' + filename, 'w')
tempf.close()
mtd_list = []
with open('./' + filename, 'rb+') as f:
# 获得文件句柄
fileno = f.fileno() # 返回一个整型的文件描述符，可用于底层操作系统的 I/O 操作
while end < filesize - 1:
start = end + 1
end = start + step - 1
if end > filesize:
end = filesize
print('Start:%s,end:%s' % (start, end))
dup = os.dup(fileno) # 复制文件句柄
fd = os.fdopen(dup, 'rb+', -1)
t = Mythread(url, start, end, fd)
t.start()
mtd_list.append(t)
for i in mtd_list:
i.join()
f.close()
def _start_execution_in_container(self, args, stdin, stdout, stderr, env,
cwd, temp_dir, cgroups, output_dir,
result_files_patterns, parent_setup_fn,
child_setup_fn, parent_cleanup_fn):
"""Execute the given command and measure its resource usage similarly to super()._start_execution(),
but inside a container implemented using Linux namespaces.
The command has no network access (only loopback),
a fresh directory as /tmp and no write access outside of this,
and it does not see other processes except itself.
"""
assert self._use_namespaces
args = self._build_cmdline(args, env=env)
# We have three processes involved:
# parent: the current Python process in which RunExecutor is executing
# child: child process in new namespace (PID 1 in inner namespace),
# configures inner namespace, serves as dummy init,
# collects result of grandchild and passes it to parent
# grandchild: child of child process (PID 2 in inner namespace), exec()s tool
# We need the following communication steps between these proceses:
# 1a) grandchild tells parent its PID (in outer namespace).
# 1b) grandchild tells parent that it is ready and measurement should begin.
# 2) parent tells grandchild that measurement has begun and tool should
# be exec()ed.
# 3) child tells parent about return value and resource consumption of grandchild.
# 1a and 1b are done together by sending the PID through a pipe.
# 2 is done by sending a null byte through a pipe.
# 3 is done by sending a pickled object through the same pipe as #2.
# We cannot use the same pipe for both directions, because otherwise a sender might
# read the bytes it has sent itself.
# Error codes from child to parent
CHILD_OSERROR = 128
CHILD_UNKNOWN_ERROR = 129
from_parent, to_grandchild = os.pipe(
) # "downstream" pipe parent->grandchild
from_grandchild, to_parent = os.pipe(
) # "upstream" pipe grandchild/child->parent
# If the current directory is within one of the bind mounts we create,
# we need to cd into this directory again, otherwise we would not see the bind mount,
# but the directory behind it. Thus we always set cwd to force a change of directory.
cwd = os.path.abspath(cwd or os.curdir)
def grandchild():
"""Setup everything inside the process that finally exec()s the tool."""
try:
# We know that this process has PID 2 in the inner namespace,
# but we actually need to know its PID in the outer namespace
# such that parent can put us into the correct cgroups.
# According to http://man7.org/linux/man-pages/man7/pid_namespaces.7.html,
# there are two ways to achieve this: sending a message with the PID
# via a socket (but Python < 3.3 lacks a convenient API for sendmsg),
# and reading /proc/self in the outer procfs instance (that's what we do).
my_outer_pid = container.get_my_pid_from_procfs()
container.mount_proc()
container.drop_capabilities()
container.reset_signal_handling()
child_setup_fn() # Do some other setup the caller wants.
# Signal readiness to parent by sending our PID and wait until parent is also ready
os.write(to_parent, str(my_outer_pid).encode())
received = os.read(from_parent, 1)
assert received == b'\0', received
finally:
# close remaining ends of pipe
os.close(from_parent)
os.close(to_parent)
# here Python will exec() the tool for us
def child():
"""Setup everything inside the container, start the tool, and wait for result."""
try:
logging.debug(
"Child: child process of RunExecutor with PID %d started",
container.get_my_pid_from_procfs())
# Put all received signals on hold until we handle them later.
container.block_all_signals()
# We want to avoid leaking file descriptors to the executed child.
# It is also nice if the child has only the minimal necessary file descriptors,
# to avoid keeping other pipes and files open, e.g., those that the parent
# uses to communicate with other containers (if containers are started in parallel).
# Thus we do not use the close_fds feature of subprocess.Popen,
# but do the same here manually.
# We keep the relevant ends of our pipes, and stdin/out/err of child and grandchild.
necessary_fds = {
sys.stdin, sys.stdout, sys.stderr, to_parent, from_parent,
stdin, stdout, stderr
} - {None}
container.close_open_fds(keep_files=necessary_fds)
try:
if not self._allow_network:
container.activate_network_interface("lo")
self._setup_container_filesystem(temp_dir)
except EnvironmentError as e:
logging.critical("Failed to configure container: %s", e)
return CHILD_OSERROR
try:
os.chdir(cwd)
except EnvironmentError as e:
logging.critical(
"Cannot change into working directory inside container: %s",
e)
return CHILD_OSERROR
try:
grandchild_proc = subprocess.Popen(args,
stdin=stdin,
stdout=stdout,
stderr=stderr,
env=env,
close_fds=False,
preexec_fn=grandchild)
except (EnvironmentError, RuntimeError) as e:
logging.critical("Cannot start process: %s", e)
return CHILD_OSERROR
container.drop_capabilities()
# Close other fds that were still necessary above.
container.close_open_fds(
keep_files={sys.stdout, sys.stderr, to_parent})
# Set up signal handlers to forward signals to grandchild
# (because we are PID 1, there is a special signal handling otherwise).
# cf. dumb-init project: https://github.com/Yelp/dumb-init
# Also wait for grandchild and return its result.
if _HAS_SIGWAIT:
grandchild_result = container.wait_for_child_and_forward_all_signals(
grandchild_proc.pid, args[0])
else:
container.forward_all_signals_async(
grandchild_proc.pid, args[0])
grandchild_result = self._wait_for_process(
grandchild_proc.pid, args[0])
logging.debug(
"Child: process %s terminated with exit code %d.", args[0],
grandchild_result[0])
os.write(to_parent, pickle.dumps(grandchild_result))
os.close(to_parent)
return 0
except EnvironmentError as e:
logging.exception("Error in child process of RunExecutor")
return CHILD_OSERROR
except:
# Need to catch everything because this method always needs to return a int
# (we are inside a C callback that requires returning int).
logging.exception("Error in child process of RunExecutor")
return CHILD_UNKNOWN_ERROR
try: # parent
try:
child_pid = container.execute_in_namespace(
child, use_network_ns=not self._allow_network)
except OSError as e:
raise BenchExecException(
"Creating namespace for container mode failed: " +
os.strerror(e.errno))
logging.debug(
"Parent: child process of RunExecutor with PID %d started.",
child_pid)
def check_child_exit_code():
"""Check if the child process terminated cleanly and raise an error otherwise."""
child_exitcode, unused_child_rusage = self._wait_for_process(
child_pid, args[0])
child_exitcode = util.ProcessExitCode.from_raw(child_exitcode)
logging.debug(
"Parent: child process of RunExecutor with PID %d terminated with %s.",
child_pid, child_exitcode)
if child_exitcode:
if child_exitcode.value:
if child_exitcode.value == CHILD_OSERROR:
# This was an OSError in the child, details were already logged
raise BenchExecException(
"execution in container failed, check log for details"
)
elif child_exitcode.value == CHILD_UNKNOWN_ERROR:
raise BenchExecException(
"unexpected error in container")
raise OSError(child_exitcode.value,
os.strerror(child_exitcode.value))
raise OSError(
0, "Child process of RunExecutor terminated with " +
str(child_exitcode))
# Close unnecessary ends of pipes such that read() does not block forever
# if all other processes have terminated.
os.close(from_parent)
os.close(to_parent)
container.setup_user_mapping(child_pid,
uid=self._uid,
gid=self._gid)
try:
grandchild_pid = int(os.read(
from_grandchild, 10)) # 10 bytes is enough for 32bit int
except ValueError:
# probably empty read, i.e., pipe closed, i.e., child or grandchild failed
check_child_exit_code()
assert False, "Child process of RunExecutor terminated cleanly but did not send expected data."
logging.debug(
"Parent: executing %s in grand child with PID %d via child with PID %d.",
args[0], grandchild_pid, child_pid)
# start measurements
cgroups.add_task(grandchild_pid)
parent_setup = parent_setup_fn()
# Signal grandchild that setup is finished
os.write(to_grandchild, b'\0')
# Copy file descriptor, otherwise we could not close from_grandchild in finally block
# and would leak a file descriptor in case of exception.
from_grandchild_copy = os.dup(from_grandchild)
finally:
os.close(from_grandchild)
os.close(to_grandchild)
def wait_for_grandchild():
# 1024 bytes ought to be enough for everyone^Wour pickled result
try:
received = os.read(from_grandchild_copy, 1024)
except OSError as e:
if self.PROCESS_KILLED and e.errno == errno.EINTR:
# Read was interrupted because of Ctrl+C, we just try again
received = os.read(from_grandchild_copy, 1024)
else:
raise e
parent_cleanup = parent_cleanup_fn(parent_setup)
os.close(from_grandchild_copy)
check_child_exit_code()
if result_files_patterns:
self._transfer_output_files(temp_dir, cwd, output_dir,
result_files_patterns)
exitcode, ru_child = pickle.loads(received)
return exitcode, ru_child, parent_cleanup
return grandchild_pid, wait_for_grandchild
def __enter__(self):
self.orig_stream_dup = os.dup(self.orig_stream_fileno)
self.devnull = open(os.devnull, 'w')
os.dup2(self.devnull.fileno(), self.orig_stream_fileno)
model_list = os.listdir(
os.path.join(ShapeNetv1_dir, data_type, class_id))
for model_id in model_list:
start = time.time()
exr_dir = os.path.join(output_dir, 'exr', model_id)
pose_dir = os.path.join(output_dir, 'pose', model_id)
if os.path.exists(os.path.join(exr_dir, '32.exr')):
print("skip " + exr_dir)
continue
os.makedirs(exr_dir, exist_ok=True)
os.makedirs(pose_dir, exist_ok=True)
# Redirect output to log file
old_os_out = os.dup(1)
os.close(1)
os.open('blender.log', os.O_WRONLY)
# Import mesh model
model_path = os.path.join(ShapeNetv1_dir, data_type, class_id,
model_id, 'model.obj')
bpy.ops.import_scene.obj(filepath=model_path)
# Rotate model by 90 degrees around x-axis (z-up => y-up) to match ShapeNet's coordinates
bpy.ops.transform.rotate(value=-np.pi / 2, axis=(1, 0, 0))
# Render
for i in range(viewspace.shape[0]):
scene.frame_set(i)
cam_pose = mathutils.Vector(
def connect(ssh_cmd, rhostport, python, stderr, options):
main_exe = sys.argv[0]
portl = []
rhostIsIPv6 = False
if (rhostport or '').count(':') > 1:
rhostIsIPv6 = True
if rhostport.count(']') or rhostport.count('['):
result = rhostport.split(']')
rhost = result[0].strip('[')
if len(result) > 1:
result[1] = result[1].strip(':')
if result[1] is not '':
portl = ['-p', str(int(result[1]))]
else: # can't disambiguate IPv6 colons and a port number. pass the hostname through.
rhost = rhostport
else: # IPv4
l = (rhostport or '').split(':', 1)
rhost = l[0]
if len(l) > 1:
portl = ['-p', str(int(l[1]))]
if rhost == '-':
rhost = None
ipv6flag = []
if rhostIsIPv6:
ipv6flag = ['-6']
z = zlib.compressobj(1)
content = readfile('assembler.py')
optdata = ''.join("%s=%r\n" % (k, v) for (k, v) in options.items())
content2 = (empackage(z, 'cmdline_options.py', optdata) +
empackage(z, 'helpers.py') +
empackage(z, 'compat/ssubprocess.py') +
empackage(z, 'ssnet.py') + empackage(z, 'hostwatch.py') +
empackage(z, 'server.py') + "\n")
pyscript = r"""
import sys;
skip_imports=1;
verbosity=%d;
exec compile(sys.stdin.read(%d), "assembler.py", "exec")
""" % (helpers.verbose or 0, len(content))
pyscript = re.sub(r'\s+', ' ', pyscript.strip())
if not rhost:
# ignore the --python argument when running locally; we already know
# which python version works.
argv = [sys.argv[1], '-c', pyscript]
else:
if ssh_cmd:
sshl = ssh_cmd.split(' ')
else:
sshl = ['ssh']
if python:
pycmd = "'%s' -c '%s'" % (python, pyscript)
else:
pycmd = ("P=python2; $P -V 2>/dev/null || P=python; "
"\"$P\" -c '%s'") % pyscript
argv = (sshl + portl + ipv6flag + [rhost, '--', pycmd])
(s1, s2) = socket.socketpair()
def setup():
# runs in the child process
s2.close()
s1a, s1b = os.dup(s1.fileno()), os.dup(s1.fileno())
s1.close()
debug2('executing: %r\n' % argv)
p = ssubprocess.Popen(argv,
stdin=s1a,
stdout=s1b,
preexec_fn=setup,
close_fds=True,
stderr=stderr)
os.close(s1a)
os.close(s1b)
s2.sendall(content)
s2.sendall(content2)
return p, s2
def __init__(self):
# Open a pair of null files
self.null_fds = [os.open(os.devnull, os.O_RDWR) for x in range(2)]
# Save the actual stdout (1) and stderr (2) file descriptors.
self.save_fds = [os.dup(1), os.dup(2)]
def fork(pkg, function, dirty, fake):
"""Fork a child process to do part of a spack build.
Args:
pkg (PackageBase): package whose environment we should set up the
forked process for.
function (callable): argless function to run in the child
process.
dirty (bool): If True, do NOT clean the environment before
building.
fake (bool): If True, skip package setup b/c it's not a real build
Usage::
def child_fun():
# do stuff
build_env.fork(pkg, child_fun)
Forked processes are run with the build environment set up by
spack.build_environment. This allows package authors to have full
control over the environment, etc. without affecting other builds
that might be executed in the same spack call.
If something goes wrong, the child process catches the error and
passes it to the parent wrapped in a ChildError. The parent is
expected to handle (or re-raise) the ChildError.
"""
def child_process(child_pipe, input_stream):
# We are in the child process. Python sets sys.stdin to
# open(os.devnull) to prevent our process and its parent from
# simultaneously reading from the original stdin. But, we assume
# that the parent process is not going to read from it till we
# are done with the child, so we undo Python's precaution.
if input_stream is not None:
sys.stdin = input_stream
try:
if not fake:
setup_package(pkg, dirty=dirty)
return_value = function()
child_pipe.send(return_value)
except StopIteration as e:
# StopIteration is used to stop installations
# before the final stage, mainly for debug purposes
tty.msg(e)
child_pipe.send(None)
except BaseException:
# catch ANYTHING that goes wrong in the child process
exc_type, exc, tb = sys.exc_info()
# Need to unwind the traceback in the child because traceback
# objects can't be sent to the parent.
tb_string = traceback.format_exc()
# build up some context from the offending package so we can
# show that, too.
package_context = get_package_context(tb)
build_log = None
if hasattr(pkg, 'log_path'):
build_log = pkg.log_path
# make a pickleable exception to send to parent.
msg = "%s: %s" % (exc_type.__name__, str(exc))
ce = ChildError(msg, exc_type.__module__, exc_type.__name__,
tb_string, build_log, package_context)
child_pipe.send(ce)
finally:
child_pipe.close()
parent_pipe, child_pipe = multiprocessing.Pipe()
input_stream = None
try:
# Forward sys.stdin when appropriate, to allow toggling verbosity
if sys.stdin.isatty() and hasattr(sys.stdin, 'fileno'):
input_stream = os.fdopen(os.dup(sys.stdin.fileno()))
p = multiprocessing.Process(target=child_process,
args=(child_pipe, input_stream))
p.start()
except InstallError as e:
e.pkg = pkg
raise
finally:
# Close the input stream in the parent process
if input_stream is not None:
input_stream.close()
child_result = parent_pipe.recv()
p.join()
# let the caller know which package went wrong.
if isinstance(child_result, InstallError):
child_result.pkg = pkg
# If the child process raised an error, print its output here rather
# than waiting until the call to SpackError.die() in main(). This
# allows exception handling output to be logged from within Spack.
# see spack.main.SpackCommand.
if isinstance(child_result, ChildError):
child_result.print_context()
raise child_result
return child_result
def main():
"""
Check starting conditions and start GUI.
First, check command line arguments and start loggers. Set log levels. Try
all imports and exit verbosely if a library is not found. Disable outputs
to stdout and start the GUI.
"""
# Set ERROR level for PyQt5 logger
qtlogger = logging.getLogger('PyQt5')
qtlogger.setLevel(logging.ERROR)
parser = argparse.ArgumentParser(
description="edclient - Espdrone graphical control client")
parser.add_argument('--debug',
'-d',
nargs=1,
default='info',
type=str,
help="set debug level "
"[minimal, info, debug, debugfile]")
args = parser.parse_args()
debug = args.debug
cflogger = logging.getLogger('')
# Set correct logging fuctionality according to commandline
if ("debugfile" in debug):
logging.basicConfig(level=logging.DEBUG)
# Add extra format options for file logger (thread and time)
formatter = logging.Formatter('%(asctime)s:%(threadName)s:%(name)'
's:%(levelname)s:%(message)s')
filename = "debug-%s.log" % datetime.datetime.now()
filehandler = logging.FileHandler(filename)
filehandler.setLevel(logging.DEBUG)
filehandler.setFormatter(formatter)
cflogger.addHandler(filehandler)
elif ("debug" in debug):
logging.basicConfig(level=logging.DEBUG)
elif ("minimal" in debug):
logging.basicConfig(level=logging.WARNING)
elif ("info" in debug):
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
logger.debug("Using config path {}".format(edclient.config_path))
logger.debug("sys.path={}".format(sys.path))
# Try all the imports used in the project here to control what happens....
try:
import usb # noqa
except ImportError:
logger.critical("No pyusb installation found, exiting!")
sys.exit(1)
if not sys.platform.startswith('linux'):
try:
import sdl2 # noqa
except ImportError:
logger.critical("No pysdl2 installation found, exiting!")
sys.exit(1)
try:
import PyQt5 # noqa
except ImportError:
logger.critical("No PyQT5 installation found, exiting!")
sys.exit(1)
# Disable printouts from STL
if os.name == 'posix':
stdout = os.dup(1)
os.dup2(os.open('/dev/null', os.O_WRONLY), 1)
sys.stdout = os.fdopen(stdout, 'w')
logger.info("Disabling STL printouts")
if os.name == 'nt':
stdout = os.dup(1)
os.dup2(os.open('NUL', os.O_WRONLY), 1)
sys.stdout = os.fdopen(stdout, 'w')
logger.info("Disabling STL printouts")
if sys.platform == 'darwin':
try:
import Foundation
bundle = Foundation.NSBundle.mainBundle()
if bundle:
info = (bundle.localizedInfoDictionary()
or bundle.infoDictionary())
if info:
info['CFBundleName'] = 'Espdrone'
except ImportError:
logger.info("Foundation not found. Menu will show python as "
"application name")
# Start up the main user-interface
from .ui.main import MainUI
from PyQt5.QtWidgets import QApplication
from PyQt5.QtGui import QIcon
app = QApplication(sys.argv)
# Create and set an event loop that combines qt and asyncio
loop = QSelectorEventLoop(app)
asyncio.set_event_loop(loop)
app.setWindowIcon(QIcon(edclient.module_path + "/icon-256.png"))
# Make sure the right icon is set in Windows 7+ taskbar
if os.name == 'nt':
import ctypes
try:
myappid = 'mycompany.myproduct.subproduct.version'
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(
myappid)
except Exception:
pass
main_window = MainUI()
main_window.show()
sys.exit(app.exec_())
def __enter__(self):
self.read_fd, self.write_fd = os.pipe()
self.copy_fd = os.dup(self.capture_fd)
os.dup2(self.write_fd, self.capture_fd)
return self
def __enter__(self):
self.output_fd_backup = os.dup(self.output.fileno())
self.output.flush()
os.close(self.output.fileno())
os.open(self.redirected_path, os.O_WRONLY)
return self
def dup(fd):
# 复制文件描述符fd
os.dup(fd)
def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite):
"""Execute program (POSIX version)"""
if isinstance(args, types.StringTypes):
args = [args]
else:
args = list(args)
if shell:
args = ['/bin/sh', '-c'] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
def _close_in_parent(fd):
os.close(fd)
to_close.remove(fd)
errpipe_read, errpipe_write = self.pipe_cloexec()
try:
try:
gc_was_enabled = gc.isenabled()
gc.disable()
try:
self.pid = os.fork()
except:
if gc_was_enabled:
gc.enable()
raise
self._child_created = True
if self.pid == 0:
try:
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
def _dup2(a, b):
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
return
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
closed = {None}
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
if close_fds:
self._close_fds(but=errpipe_write)
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
exc_lines = traceback.format_exception(exc_type, exc_value, tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
os._exit(255)
if gc_was_enabled:
gc.enable()
finally:
os.close(errpipe_write)
data = _eintr_retry_call(os.read, errpipe_read, 1048576)
finally:
if p2cread is not None and p2cwrite is not None:
_close_in_parent(p2cread)
if c2pwrite is not None and c2pread is not None:
_close_in_parent(c2pwrite)
if errwrite is not None and errread is not None:
_close_in_parent(errwrite)
os.close(errpipe_read)
if data != '':
try:
_eintr_retry_call(os.waitpid, self.pid, 0)
except OSError as e:
if e.errno != errno.ECHILD:
raise
child_exception = pickle.loads(data)
raise child_exception
return
def __exit__(self, exc_type, exc_value, traceback):
os.close(self.output.fileno())
os.dup(self.output_fd_backup)
os.close(self.output_fd_backup)
def _get_next_fds(self, n=1):
# dup uses the lowest-numbered unused descriptor for the new descriptor
fds = [os.dup(0) for i in range(n)]
for fd in fds:
os.close(fd)
return fds
def __init__(self, argv = None,
shell = False,
executable = None,
cwd = None,
env = None,
stdin = PIPE,
stdout = PTY,
stderr = STDOUT,
close_fds = True,
preexec_fn = lambda: None,
raw = True,
aslr = None,
setuid = None,
where = 'local',
display = None,
alarm = None,
*args,
**kwargs
):
super(process, self).__init__(*args,**kwargs)
# Permit using context.binary
if argv is None:
if context.binary:
argv = [context.binary.path]
else:
raise TypeError('Must provide argv or set context.binary')
#: :class:`subprocess.Popen` object that backs this process
self.proc = None
if not shell:
executable, argv, env = self._validate(cwd, executable, argv, env)
# Avoid the need to have to deal with the STDOUT magic value.
if stderr is STDOUT:
stderr = stdout
# Determine which descriptors will be attached to a new PTY
handles = (stdin, stdout, stderr)
#: Which file descriptor is the controlling TTY
self.pty = handles.index(PTY) if PTY in handles else None
#: Whether the controlling TTY is set to raw mode
self.raw = raw
#: Whether ASLR should be left on
self.aslr = aslr if aslr is not None else context.aslr
#: Whether setuid is permitted
self._setuid = setuid if setuid is None else bool(setuid)
# Create the PTY if necessary
stdin, stdout, stderr, master, slave = self._handles(*handles)
#: Arguments passed on argv
self.argv = argv
#: Full path to the executable
self.executable = executable
if self.executable is None:
if shell:
self.executable = '/bin/sh'
else:
self.executable = which(self.argv[0])
#: Environment passed on envp
self.env = os.environ if env is None else env
self._cwd = os.path.realpath(cwd or os.path.curdir)
#: Alarm timeout of the process
self.alarm = alarm
self.preexec_fn = preexec_fn
self.display = display or self.program
self._qemu = False
self._corefile = None
message = "Starting %s process %r" % (where, self.display)
if self.isEnabledFor(logging.DEBUG):
if self.argv != [self.executable]: message += ' argv=%r ' % self.argv
if self.env != os.environ: message += ' env=%r ' % self.env
with self.progress(message) as p:
if not self.aslr:
self.warn_once("ASLR is disabled!")
# In the event the binary is a foreign architecture,
# and binfmt is not installed (e.g. when running on
# Travis CI), re-try with qemu-XXX if we get an
# 'Exec format error'.
prefixes = [([], self.executable)]
exception = None
for prefix, executable in prefixes:
try:
args = argv
if prefix:
args = prefix + args
self.proc = subprocess.Popen(args = args,
shell = shell,
executable = executable,
cwd = cwd,
env = env,
stdin = stdin,
stdout = stdout,
stderr = stderr,
close_fds = close_fds,
preexec_fn = self.__preexec_fn)
break
except OSError as exception:
if exception.errno != errno.ENOEXEC:
raise
prefixes.append(self.__on_enoexec(exception))
p.success('pid %i' % self.pid)
if self.pty is not None:
if stdin is slave:
self.proc.stdin = os.fdopen(os.dup(master), 'r+b', 0)
if stdout is slave:
self.proc.stdout = os.fdopen(os.dup(master), 'r+b', 0)
if stderr is slave:
self.proc.stderr = os.fdopen(os.dup(master), 'r+b', 0)
os.close(master)
os.close(slave)
# Set in non-blocking mode so that a call to call recv(1000) will
# return as soon as a the first byte is available
if self.proc.stdout:
fd = self.proc.stdout.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
# Save off information about whether the binary is setuid / setgid
self.uid = os.getuid()
self.gid = os.getgid()
self.suid = -1
self.sgid = -1
st = os.stat(self.executable)
if self._setuid:
if (st.st_mode & stat.S_ISUID):
self.setuid = st.st_uid
if (st.st_mode & stat.S_ISGID):
self.setgid = st.st_gid
def main():
# Our parent expects to read JSON from our stdout, so if anyone
# uses print, buck will complain with a helpful "but I wanted an
# array!" message and quit. Redirect stdout to stderr so that
# doesn't happen. Actually dup2 the file handle so that writing
# to file descriptor 1, os.system, and so on work as expected too.
to_parent = os.fdopen(os.dup(sys.stdout.fileno()), 'a')
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
parser = optparse.OptionParser()
parser.add_option('--project_root',
action='store',
type='string',
dest='project_root')
parser.add_option('--build_file_name',
action='store',
type='string',
dest="build_file_name")
parser.add_option(
'--allow_empty_globs',
action='store_true',
dest='allow_empty_globs',
help=
'Tells the parser not to raise an error when glob returns no results.')
parser.add_option('--include', action='append', dest='include')
(options, args) = parser.parse_args()
# Even though project_root is absolute path, it may not be concise. For
# example, it might be like "C:\project\.\rule".
#
# Under cygwin, the project root will be invoked from buck as C:\path, but
# the cygwin python uses UNIX-style paths. They can be converted using
# cygpath, which is necessary because abspath will treat C:\path as a
# relative path.
options.project_root = cygwin_adjusted_path(options.project_root)
project_root = os.path.abspath(options.project_root)
buildFileProcessor = BuildFileProcessor(project_root,
options.build_file_name,
options.allow_empty_globs,
implicit_includes=options.include
or [])
buildFileProcessor.install_builtins(__builtin__.__dict__)
for build_file in args:
build_file = cygwin_adjusted_path(build_file)
values = buildFileProcessor.process(build_file)
to_parent.write(json.dumps(values, sort_keys=True))
to_parent.flush()
# "for ... in sys.stdin" in Python 2.x hangs until stdin is closed.
for build_file in iter(sys.stdin.readline, ''):
build_file = cygwin_adjusted_path(build_file)
values = buildFileProcessor.process(build_file.rstrip())
to_parent.write(json.dumps(values, sort_keys=True))
to_parent.flush()
# Python tries to flush/close stdout when it quits, and if there's a dead
# pipe on the other end, it will spit some warnings to stderr. This breaks
# tests sometimes. Prevent that by explicitly catching the error.
try:
to_parent.close()
except IOError:
pass
def _start_execution_in_container(
self,
args,
stdin,
stdout,
stderr,
env,
root_dir,
cwd,
temp_dir,
memlimit,
memory_nodes,
cgroups,
output_dir,
result_files_patterns,
parent_setup_fn,
child_setup_fn,
parent_cleanup_fn,
):
"""Execute the given command and measure its resource usage similarly to
super()._start_execution(), but inside a container implemented using Linux
namespaces. The command has no network access (only loopback),
a fresh directory as /tmp and no write access outside of this,
and it does not see other processes except itself.
"""
assert self._use_namespaces
if root_dir is None:
env.update(self._env_override)
# We have three processes involved:
# parent: the current Python process in which RunExecutor is executing
# child: child process in new namespace (PID 1 in inner namespace),
# configures inner namespace, serves as dummy init,
# collects result of grandchild and passes it to parent
# grandchild: child of child process (PID 2 in inner namespace), exec()s tool
# We need the following communication steps between these proceses:
# 1a) grandchild tells parent its PID (in outer namespace).
# 1b) grandchild tells parent that it is ready and measurement should begin.
# 2) parent tells grandchild that measurement has begun and tool should
# be exec()ed.
# 3) child tells parent about return value and resource consumption of
# grandchild.
# 1a and 1b are done together by sending the PID through a pipe.
# 2 is done by sending a null byte through a pipe.
# 3 is done by sending a pickled object through the same pipe as #2.
# We cannot use the same pipe for both directions, because otherwise a sender
# might read the bytes it has sent itself.
# Error codes from child to parent
CHILD_OSERROR = 128 # noqa: N806 local constant
CHILD_UNKNOWN_ERROR = 129 # noqa: N806 local constant
# "downstream" pipe parent->grandchild
from_parent, to_grandchild = os.pipe()
# "upstream" pipe grandchild/child->parent
from_grandchild, to_parent = os.pipe()
# The protocol for these pipes is that first the parent sends the marker for
# user mappings, then the grand child sends its outer PID back,
# and finally the parent sends its completion marker.
# After the run, the child sends the result of the grand child and then waits
# for the post_run marker, before it terminates.
MARKER_USER_MAPPING_COMPLETED = b"A" # noqa: N806 local constant
MARKER_PARENT_COMPLETED = b"B" # noqa: N806 local constant
MARKER_PARENT_POST_RUN_COMPLETED = b"C" # noqa: N806 local constant
# If the current directory is within one of the bind mounts we create,
# we need to cd into this directory again, otherwise we would not see the
# bind mount, but the directory behind it.
# Thus we always set cwd to force a change of directory.
if root_dir is None:
cwd = os.path.abspath(cwd or os.curdir)
else:
root_dir = os.path.abspath(root_dir)
cwd = os.path.abspath(cwd)
def grandchild():
"""Setup everything inside the process that finally exec()s the tool."""
try:
# We know that this process has PID 2 in the inner namespace,
# but we actually need to know its PID in the outer namespace
# such that parent can put us into the correct cgroups. According to
# http://man7.org/linux/man-pages/man7/pid_namespaces.7.html,
# there are two ways to achieve this: sending a message with the PID
# via a socket (but Python 2 lacks a convenient API for sendmsg),
# and reading /proc/self in the outer procfs instance
# (that's what we do).
my_outer_pid = container.get_my_pid_from_procfs()
container.mount_proc(self._container_system_config)
container.drop_capabilities()
container.reset_signal_handling()
child_setup_fn() # Do some other setup the caller wants.
# Signal readiness to parent by sending our PID
# and wait until parent is also ready
os.write(to_parent, str(my_outer_pid).encode())
received = os.read(from_parent, 1)
assert received == MARKER_PARENT_COMPLETED, received
finally:
# close remaining ends of pipe
os.close(from_parent)
os.close(to_parent)
# here Python will exec() the tool for us
def child():
"""Setup everything inside the container,
start the tool, and wait for result."""
try:
logging.debug(
"Child: child process of RunExecutor with PID %d started",
container.get_my_pid_from_procfs(),
)
# Put all received signals on hold until we handle them later.
container.block_all_signals()
# We want to avoid leaking file descriptors to the executed child.
# It is also nice if the child has only the minimal necessary file
# descriptors, to avoid keeping other pipes and files open, e.g.,
# those that the parent uses to communicate with other containers
# (if containers are started in parallel).
# Thus we do not use the close_fds feature of subprocess.Popen,
# but do the same here manually. We keep the relevant ends of our pipes,
# and stdin/out/err of child and grandchild.
necessary_fds = {
sys.stdin,
sys.stdout,
sys.stderr,
to_parent,
from_parent,
stdin,
stdout,
stderr,
} - {None}
container.close_open_fds(keep_files=necessary_fds)
try:
if self._container_system_config:
# A standard hostname increases reproducibility.
libc.sethostname(container.CONTAINER_HOSTNAME)
if not self._allow_network:
container.activate_network_interface("lo")
# Wait until user mapping is finished,
# this is necessary for filesystem writes
received = os.read(from_parent, len(MARKER_USER_MAPPING_COMPLETED))
assert received == MARKER_USER_MAPPING_COMPLETED, received
if root_dir is not None:
self._setup_root_filesystem(root_dir)
else:
self._setup_container_filesystem(
temp_dir,
output_dir if result_files_patterns else None,
memlimit,
memory_nodes,
)
# Marking this process as "non-dumpable" (no core dumps) also
# forbids several other ways how other processes can access and
# influence it:
# ptrace is forbidden and much of /proc/<child>/ is inaccessible.
# We set this to prevent the benchmarked tool from messing with this
# process or using it to escape from the container. More info:
# http://man7.org/linux/man-pages/man5/proc.5.html
# It needs to be done after MARKER_USER_MAPPING_COMPLETED.
libc.prctl(libc.PR_SET_DUMPABLE, libc.SUID_DUMP_DISABLE, 0, 0, 0)
except EnvironmentError as e:
logging.critical("Failed to configure container: %s", e)
return CHILD_OSERROR
try:
os.chdir(cwd)
except EnvironmentError as e:
logging.critical(
"Cannot change into working directory inside container: %s", e
)
return CHILD_OSERROR
container.setup_seccomp_filter()
try:
grandchild_proc = subprocess.Popen(
args,
stdin=stdin,
stdout=stdout,
stderr=stderr,
env=env,
close_fds=False,
preexec_fn=grandchild,
)
except (EnvironmentError, RuntimeError) as e:
logging.critical("Cannot start process: %s", e)
return CHILD_OSERROR
# keep capability for unmount if necessary later
necessary_capabilities = (
[libc.CAP_SYS_ADMIN] if result_files_patterns else []
)
container.drop_capabilities(keep=necessary_capabilities)
# Close other fds that were still necessary above.
container.close_open_fds(
keep_files={sys.stdout, sys.stderr, to_parent, from_parent}
)
# Set up signal handlers to forward signals to grandchild
# (because we are PID 1, there is a special signal handling otherwise).
# cf. dumb-init project: https://github.com/Yelp/dumb-init
# Also wait for grandchild and return its result.
if _HAS_SIGWAIT:
grandchild_result = container.wait_for_child_and_forward_signals(
grandchild_proc.pid, args[0]
)
else:
container.forward_all_signals_async(grandchild_proc.pid, args[0])
grandchild_result = self._wait_for_process(
grandchild_proc.pid, args[0]
)
logging.debug(
"Child: process %s terminated with exit code %d.",
args[0],
grandchild_result[0],
)
if result_files_patterns:
# Remove the bind mount that _setup_container_filesystem added
# such that the parent can access the result files.
libc.umount(temp_dir.encode())
# Re-allow access to /proc/<child>/...,
# this is used by the parent for accessing output files
libc.prctl(libc.PR_SET_DUMPABLE, libc.SUID_DUMP_USER, 0, 0, 0)
os.write(to_parent, pickle.dumps(grandchild_result))
os.close(to_parent)
# Now the parent copies the output files, we need to wait until this is
# finished. If the child terminates, the container file system and its
# tmpfs go away.
assert os.read(from_parent, 1) == MARKER_PARENT_POST_RUN_COMPLETED
os.close(from_parent)
return 0
except EnvironmentError:
logging.exception("Error in child process of RunExecutor")
return CHILD_OSERROR
except: # noqa: E722
# Need to catch everything because this method always needs to return an
# int (we are inside a C callback that requires returning int).
logging.exception("Error in child process of RunExecutor")
return CHILD_UNKNOWN_ERROR
try: # parent
try:
child_pid = container.execute_in_namespace(
child, use_network_ns=not self._allow_network
)
except OSError as e:
if (
e.errno == errno.EPERM
and util.try_read_file("/proc/sys/kernel/unprivileged_userns_clone")
== "0"
):
raise BenchExecException(
"Unprivileged user namespaces forbidden on this system, please "
"enable them with 'sysctl kernel.unprivileged_userns_clone=1' "
"or disable container mode"
)
else:
raise BenchExecException(
"Creating namespace for container mode failed: "
+ os.strerror(e.errno)
)
logging.debug(
"Parent: child process of RunExecutor with PID %d started.", child_pid
)
def check_child_exit_code():
"""Check if the child process terminated cleanly
and raise an error otherwise."""
child_exitcode, unused_child_rusage = self._wait_for_process(
child_pid, args[0]
)
child_exitcode = util.ProcessExitCode.from_raw(child_exitcode)
logging.debug(
"Parent: child process of RunExecutor with PID %d"
" terminated with %s.",
child_pid,
child_exitcode,
)
if child_exitcode:
if child_exitcode.value:
if child_exitcode.value == CHILD_OSERROR:
# This was an OSError in the child,
# details were already logged
raise BenchExecException(
"execution in container failed, check log for details"
)
elif child_exitcode.value == CHILD_UNKNOWN_ERROR:
raise BenchExecException("unexpected error in container")
raise OSError(
child_exitcode.value, os.strerror(child_exitcode.value)
)
raise OSError(
0,
"Child process of RunExecutor terminated with "
+ str(child_exitcode),
)
# Close unnecessary ends of pipes such that read() does not block forever
# if all other processes have terminated.
os.close(from_parent)
os.close(to_parent)
container.setup_user_mapping(child_pid, uid=self._uid, gid=self._gid)
# signal child to continue
os.write(to_grandchild, MARKER_USER_MAPPING_COMPLETED)
try:
# read at most 10 bytes because this is enough for 32bit int
grandchild_pid = int(os.read(from_grandchild, 10))
except ValueError:
# probably empty read, i.e., pipe closed,
# i.e., child or grandchild failed
check_child_exit_code()
assert False, (
"Child process of RunExecutor terminated cleanly"
" but did not send expected data."
)
logging.debug(
"Parent: executing %s in grand child with PID %d"
" via child with PID %d.",
args[0],
grandchild_pid,
child_pid,
)
# start measurements
cgroups.add_task(grandchild_pid)
parent_setup = parent_setup_fn()
# Signal grandchild that setup is finished
os.write(to_grandchild, MARKER_PARENT_COMPLETED)
# Copy file descriptor, otherwise we could not close from_grandchild in
# finally block and would leak a file descriptor in case of exception.
from_grandchild_copy = os.dup(from_grandchild)
to_grandchild_copy = os.dup(to_grandchild)
finally:
os.close(from_grandchild)
os.close(to_grandchild)
def wait_for_grandchild():
# 1024 bytes ought to be enough for everyone^Wour pickled result
try:
received = os.read(from_grandchild_copy, 1024)
except OSError as e:
if self.PROCESS_KILLED and e.errno == errno.EINTR:
# Read was interrupted because of Ctrl+C, we just try again
received = os.read(from_grandchild_copy, 1024)
else:
raise e
if not received:
# Typically this means the child exited prematurely because an error
# occurred, and check_child_exitcode() will handle this.
# We close the pipe first, otherwise child could hang infinitely.
os.close(from_grandchild_copy)
os.close(to_grandchild_copy)
check_child_exit_code()
assert False, "Child process terminated cleanly without sending result"
exitcode, ru_child = pickle.loads(received)
base_path = "/proc/{}/root".format(child_pid)
parent_cleanup = parent_cleanup_fn(
parent_setup, util.ProcessExitCode.from_raw(exitcode), base_path
)
if result_files_patterns:
# As long as the child process exists
# we can access the container file system here
self._transfer_output_files(
base_path + temp_dir, cwd, output_dir, result_files_patterns
)
os.close(from_grandchild_copy)
os.write(to_grandchild_copy, MARKER_PARENT_POST_RUN_COMPLETED)
os.close(to_grandchild_copy) # signal child that it can terminate
check_child_exit_code()
return exitcode, ru_child, parent_cleanup
return grandchild_pid, wait_for_grandchild
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not d.getVarFlag(task, 'task', False):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T')
if not tempdir:
bb.fatal("T variable not set, unable to build")
# Change nice level if we're asked to
nice = localdata.getVar("BB_TASK_NICE_LEVEL")
if nice:
curnice = os.nice(0)
nice = int(nice) - curnice
newnice = os.nice(nice)
logger.debug(1, "Renice to %s " % newnice)
ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
if ionice:
try:
cls, prio = ionice.split(".", 1)
bb.utils.ioprio_set(os.getpid(), int(cls), int(prio))
except:
bb.warn("Invalid ionice level %s" % ionice)
bb.utils.mkdirhier(tempdir)
# Determine the logfile to generate
logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
logbase = logfmt.format(task=task, pid=os.getpid())
# Document the order of the tasks...
logorder = os.path.join(tempdir, 'log.task_order')
try:
with open(logorder, 'a') as logorderfile:
logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
except OSError:
logger.exception("Opening log file '%s'", logorder)
pass
# Setup the courtesy link to the logfn
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, logbase)
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logbase, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
class ErrorCheckHandler(logging.Handler):
def __init__(self):
self.triggered = False
logging.Handler.__init__(self, logging.ERROR)
def emit(self, record):
if getattr(record, 'forcelog', False):
self.triggered = False
else:
self.triggered = True
# Handle logfiles
si = open('/dev/null', 'r')
try:
bb.utils.mkdirhier(os.path.dirname(logfn))
logfile = open(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure Python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
# Always enable full debug output into task logfiles
handler.setLevel(logging.DEBUG - 2)
bblogger.addHandler(handler)
errchk = ErrorCheckHandler()
bblogger.addHandler(errchk)
localdata.setVar('BB_LOGFILE', logfn)
localdata.setVar('BB_RUNTASK', task)
localdata.setVar('BB_TASK_LOGGER', bblogger)
flags = localdata.getVarFlags(task)
try:
try:
event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
except (bb.BBHandledException, SystemExit):
return 1
except FuncFailed as exc:
logger.error(str(exc))
return 1
try:
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
except FuncFailed as exc:
if quieterr:
event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
else:
errprinted = errchk.triggered
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
return 1
except bb.BBHandledException:
event.fire(TaskFailed(task, logfn, localdata, True), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
si.close()
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, logfn, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
make_stamp(task, localdata)
return 0
sys.path.append('odsreader')
from ODSReader import ODSReader
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='ODS to Arduino pins converter.')
parser.add_argument('-i', action='store', dest="input_file", metavar='input-odf-file', required=True, help='Input ODS file')
parser.add_argument('-o', action='store', dest="output_file", metavar='generated-header-file', type=argparse.FileType('wt'), default='arduino_pins.h', help='Output C++ header file for arduino pins.')
parser.add_argument('-s', action='store', dest="sheet_name", required=True, help='Name of Sheet in ODS to read pins definition from.')
parser.add_argument('-m', action='store', dest="device", required=True, help='Name of microcontroler the header is generated (example: ATmega128).')
ns = parser.parse_args()
old = os.dup(1)
os.close(1)
sys.stdout = ns.output_file
cpa = re.compile(r'ADC([0-9]+)')
cpt = re.compile(r'TIMER([0-9]+[A|B|C])')
doc = ODSReader(ns.input_file)
table = doc.getSheet(ns.sheet_name)
ports = {}
spi = []
analogin = {}
timers = {}
pwm = []
pins = []
for r in table[1:]:
if r[0] == '---':
def __init__(self, fileno, encoding):
self.binarystream = fdopen(dup(fileno), 'wb')
self.encoding = encoding
def __init__(self, fd):
self._redirected_fd = fd
self._saved_fd = os.dup(self._redirected_fd)
self._into_file = None
def __execute_child(self, args, executable, preexec_fn, close_fds, cwd,
env, universal_newlines, startupinfo, creationflags,
shell, p2cread, p2cwrite, c2pread, c2pwrite, errread,
errwrite):
"""
Executes the program using posix_spawn().
This is based on the method from the superclass but the
posix_spawn API forces a number of changes. In particular:
* When using fork() FDs are manipulated in the child process
after the fork, but before the program is exec()ed. With
posix_spawn() this is done by passing a data-structure to
the posix_spawn() call, which describes the FD manipulations
to perform.
* The fork() version waits until after the fork before
unsetting the non-blocking flag on the FDs that the child
has inherited. In the posix_spawn() version, we cannot
do that after the fork so we dup the FDs in advance and
unset the flag on the duped FD, which we then pass to the
child.
"""
if preexec_fn is not None:
raise NotImplementedError("preexec_fn not supported")
if close_fds:
raise NotImplementedError("close_fds not implemented")
if cwd:
raise NotImplementedError("cwd not implemented")
if universal_newlines:
raise NotImplementedError()
assert startupinfo is None and creationflags == 0
log.debug("Pipes: p2c %s, %s; c2p %s, %s; err %s, %s", p2cread,
p2cwrite, c2pread, c2pwrite, errread, errwrite)
if isinstance(args, types.StringTypes):
args = [args]
else:
args = [a.encode("ascii") for a in args]
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# The FileActions object is an ordered list of FD operations for
# posix_spawn to do in the child process before it execs the new
# program.
file_actions = FileActions()
# In the child, close parent's pipe ends.
if p2cwrite is not None:
file_actions.add_close(p2cwrite)
if c2pread is not None:
file_actions.add_close(c2pread)
if errread is not None:
file_actions.add_close(errread)
# When duping fds, if there arises a situation where one of the fds
# is either 0, 1 or 2, it is possible that it is overwritten (#12607).
fds_to_close_in_parent = []
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
fds_to_close_in_parent.append(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
fds_to_close_in_parent.append(errwrite)
# Dup stdin/out/err FDs in child.
def _dup2(dup_from, dup_to):
if dup_from is None:
# Pass through the existing FD.
dup_from = dup_to
# Need to take a dup so we can remove the non-blocking flag
a_dup = os.dup(dup_from)
log.debug("Duped %s as %s", dup_from, a_dup)
fds_to_close_in_parent.append(a_dup)
self._remove_nonblock_flag(a_dup)
file_actions.add_dup2(a_dup, dup_to)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds in the child. Make sure we don't close the same fd
# more than once, or standard fds.
for fd in set([p2cread, c2pwrite, errwrite]):
if fd > 2:
file_actions.add_close(fd)
gc_was_enabled = gc.isenabled()
# FIXME Does this bug apply to posix_spawn version?
try:
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
self.pid = posix_spawnp(
executable,
args,
file_actions=file_actions,
env=env,
)
except:
if gc_was_enabled:
gc.enable()
raise
finally:
for fd in fds_to_close_in_parent:
os.close(fd)
# Capture the SIGCHILD.
self._watcher = self._loop.child(self.pid)
self._watcher.start(self._on_child, self._watcher)
if gc_was_enabled:
gc.enable()
# Close the Child's pipe ends in the parent.
if p2cread is not None and p2cwrite is not None:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None:
os.close(c2pwrite)
if errwrite is not None and errread is not None:
os.close(errwrite)
def runCommand(self, params, wait=1):
"""
This method runs a command and returns a tuple: (returncode, stdout, stderr)
"""
## @todo: Convert this into generic method and reuse it in dhcp4 and dhcp6
print("Running command: %s" % (" ".join(params)))
# redirect stdout to a pipe so we can check that our
# process spawning is doing the right thing with stdout
self.stdout_old = os.dup(sys.stdout.fileno())
self.stdout_pipes = os.pipe()
os.dup2(self.stdout_pipes[1], sys.stdout.fileno())
os.close(self.stdout_pipes[1])
# do the same trick for stderr:
self.stderr_old = os.dup(sys.stderr.fileno())
self.stderr_pipes = os.pipe()
os.dup2(self.stderr_pipes[1], sys.stderr.fileno())
os.close(self.stderr_pipes[1])
# note that we use dup2() to restore the original stdout
# to the main program ASAP in each test... this prevents
# hangs reading from the child process (as the pipe is only
# open in the child), and also insures nice pretty output
pi = ProcessInfo('Test Process', params)
pi.spawn()
time.sleep(wait)
os.dup2(self.stdout_old, sys.stdout.fileno())
os.dup2(self.stderr_old, sys.stderr.fileno())
self.assertNotEqual(pi.process, None)
self.assertTrue(type(pi.pid) is int)
# Set non-blocking read on pipes. Process may not print anything
# on specific output and the we would hang without this.
fd = self.stdout_pipes[0]
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
fd = self.stderr_pipes[0]
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
# As we don't know how long the subprocess will take to start and
# produce output, we'll loop and sleep for 250 ms between each
# iteration. To avoid an infinite loop, we'll loop for a maximum
# of five seconds: that should be enough.
for count in range(20):
# Read something from stderr and stdout (these reads don't block).
output = self.readPipe(self.stdout_pipes[0])
error = self.readPipe(self.stderr_pipes[0])
# If the process has already exited, or if it has output something,
# quit the loop now.
if pi.process.poll(
) is not None or len(error) > 0 or len(output) > 0:
break
# Process still running, try again in 250 ms.
time.sleep(0.25)
# Exited loop, kill the process if it is still running
if pi.process.poll() is None:
try:
pi.process.terminate()
except OSError:
print("Ignoring failed kill attempt. Process is dead already.")
# call this to get returncode, process should be dead by now
rc = pi.process.wait()
# Clean up our stdout/stderr munging.
os.dup2(self.stdout_old, sys.stdout.fileno())
os.close(self.stdout_old)
os.close(self.stdout_pipes[0])
os.dup2(self.stderr_old, sys.stderr.fileno())
os.close(self.stderr_old)
os.close(self.stderr_pipes[0])
# Free up resources (file descriptors) from the ProcessInfo object
# TODO: For some reason, this gives an error if the process has ended,
# although it does cause all descriptors still allocated to the
# object to be freed.
pi = None
print(
"Process finished, return code=%d, stdout=%d bytes, stderr=%d bytes"
% (rc, len(output), len(error)))
return (rc, output, error)
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not data.getVarFlag(task, 'task', d):
event.fire(TaskInvalid(task, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug(1, "Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T', True)
if not tempdir:
bb.fatal("T variable not set, unable to build")
bb.utils.mkdirhier(tempdir)
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logbase = 'log.{0}.{1}'.format(task, os.getpid())
logfn = os.path.join(tempdir, logbase)
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logbase, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
class ErrorCheckHandler(logging.Handler):
def __init__(self):
self.triggered = False
logging.Handler.__init__(self, logging.ERROR)
def emit(self, record):
self.triggered = True
# Handle logfiles
si = file('/dev/null', 'r')
try:
logfile = file(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
# Always enable full debug output into task logfiles
handler.setLevel(logging.DEBUG - 2)
bblogger.addHandler(handler)
errchk = ErrorCheckHandler()
bblogger.addHandler(errchk)
localdata.setVar('BB_LOGFILE', logfn)
event.fire(TaskStarted(task, localdata), localdata)
try:
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
except FuncFailed as exc:
if quieterr:
event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
else:
errprinted = errchk.triggered
logger.error(str(exc))
event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
return 1
finally:
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
si.close()
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug(2, "Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
event.fire(TaskSucceeded(task, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
make_stamp(task, localdata)
return 0
from __future__ import print_function
import sys
sys.path = [p for p in sys.path if p.startswith('/')]
__name__ = '__bubble__'
sys.modules[__name__] = sys.modules.pop('__main__')
def debug(msg):
print(msg, file=sys.stderr)
# Reshuffle fds so that we can't break our transport by printing to stdout
import os
infd = os.dup(0)
outfd = os.dup(1)
inpipe = os.fdopen(infd, 'rb')
outpipe = os.fdopen(outfd, 'wb', 0)
sys.stdin.close()
sys.stdin = open(os.devnull, 'rb')
sys.stdout.close()
sys.stdout = open(os.devnull, 'wb')
PY2 = sys.version_info < (3, )
PY3 = not PY2
import threading
if PY2:
__metaclass__ = type
from Queue import Queue
import cPickle as pickle
def exec_(_code_, _globs_=None, _locs_=None):
