def _socket(addr, port, family):
"""
Create a socket, bind and listen.
:param addr: The address to use.
:type addr: :any:`str`
:param port: The port to use.
:type port: :any:`int`
:param family: The type of socket to use.
:type family: :any:`socket.AF_INET` or :any:`socket.AF_INET6`.
:returns: A bound socket.
:rtype: :any:`socket.socket`
:raises: :any:`blackhole.exceptions.BlackholeRuntimeException`
"""
sock = socket.socket(family, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except (AttributeError, OSError):
pass
if family == socket.AF_INET6:
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
try:
sock.bind((addr, port))
except OSError:
msg = 'Cannot bind to {}:{}.'.format(addr, port)
logger.fatal(msg)
sock.close()
raise BlackholeRuntimeException(msg)
os.set_inheritable(sock.fileno(), True)
sock.listen(1024)
sock.setblocking(False)
return sock
def makeSockets(cls):
"""Make socket(s) to communicate with remote process.
Returns string to send to remote process
"""
if ( hasattr(socket, 'AF_UNIX') and hasattr(socket, 'socketpair') ):
# convenient interface
cls.sockfamily = socket.AF_UNIX
sock, socket2 = socket.socketpair(cls.sockfamily,
socket.SOCK_STREAM)
# socket is closed on popen in Python 3.4+ without this (PEP 446)
try:
os.set_inheritable(socket2.fileno(), True)
except AttributeError:
pass
sendtext = 'unix %i\n' % socket2.fileno()
cls.socket2 = socket2 # prevent socket being destroyed
waitaccept = False
else:
# otherwise mess around with internet sockets
# * This is required for windows, which doesn't have AF_UNIX
# * It is required where socketpair is not supported
cls.sockfamily = socket.AF_INET
sock = socket.socket(cls.sockfamily, socket.SOCK_STREAM)
sock.bind( ('localhost', 0) )
interface, port = sock.getsockname()
sock.listen(1)
sendtext = 'internet %s %i\n' % (interface, port)
waitaccept = True
return (sock, sendtext.encode('ascii'), waitaccept)
def run_diff(fd1, fd2, end_nl_q1, end_nl_q2):
cmd = ['diff', '-aU7', '/dev/fd/%d' % fd1, '/dev/fd/%d' % fd2]
logger.debug('running %s', cmd)
if hasattr(os, 'set_inheritable'): # new in Python 3.4
os.set_inheritable(fd1, True)
os.set_inheritable(fd2, True)
p = subprocess.Popen(cmd, shell=False, bufsize=1,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
pass_fds=(fd1, fd2))
p.stdin.close()
os.close(fd1)
os.close(fd2)
parser = DiffParser(p.stdout, end_nl_q1, end_nl_q2)
t_read = Thread(target=parser.parse)
t_read.daemon = True
t_read.start()
t_read.join()
p.wait()
logger.debug('done with diff, returncode %d, parsed %s', p.returncode, parser.success)
if not parser.success and p.returncode not in (0, 1):
raise subprocess.CalledProcessError(p.returncode, cmd, output=diff)
if p.returncode == 0:
return None
return parser.diff
def _socket(addr, port, family):
"""
Create a socket, bind and listen.
:param str addr: The address to use.
:param int port: The port to use.
:param family: The type of socket to use.
:type family: :py:obj:`socket.AF_INET` or :py:obj:`socket.AF_INET6`.
:returns: Bound socket.
:rtype: :py:func:`socket.socket`
:raises BlackholeRuntimeException: When a socket cannot be bound.
"""
sock = socket.socket(family, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except (AttributeError, OSError):
pass
if family == socket.AF_INET6:
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
try:
sock.bind((addr, port))
except OSError:
msg = "Cannot bind to {0}:{1}.".format(addr, port)
logger.critical(msg)
sock.close()
raise BlackholeRuntimeException(msg)
os.set_inheritable(sock.fileno(), True)
sock.listen(1024)
sock.setblocking(False)
return sock
def adb_server():
"""Context manager for an ADB server.
This creates an ADB server and returns the port it's listening on.
"""
port = 5038
# Kill any existing server on this non-default port.
subprocess.check_output(["adb", "-P", str(port), "kill-server"],
stderr=subprocess.STDOUT)
read_pipe, write_pipe = os.pipe()
if sys.platform == "win32":
import msvcrt
write_handle = msvcrt.get_osfhandle(write_pipe)
os.set_handle_inheritable(write_handle, True)
reply_fd = str(write_handle)
else:
os.set_inheritable(write_pipe, True)
reply_fd = str(write_pipe)
proc = subprocess.Popen(["adb", "-L", "tcp:localhost:{}".format(port),
"fork-server", "server",
"--reply-fd", reply_fd], close_fds=False)
try:
os.close(write_pipe)
greeting = os.read(read_pipe, 1024)
assert greeting == b"OK\n", repr(greeting)
yield port
finally:
proc.terminate()
proc.wait()
def create_haproxy_pipe():
logger.debug("create_haproxy_pipe called")
pipefd = os.pipe()
os.set_inheritable(pipefd[0], True)
os.set_inheritable(pipefd[1], True)
logger.debug("create_haproxy_pipe done")
return pipefd
def _mk_inheritable(fd):
if version >= (3, 3):
if sys.platform == 'win32':
# Change to Windwos file handle
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
os.set_handle_inheritable(fdh, True)
return fdh
else:
os.set_inheritable(fd, True)
return fd
elif sys.platform == 'win32':
# TODO: find a hack??
# Not yet working
import msvcrt
import _subprocess
curproc = _subprocess.GetCurrentProcess()
fdh = msvcrt.get_osfhandle(fd)
fdh = _subprocess.DuplicateHandle(
curproc, fdh, curproc, 0,
True, # set inheritable FLAG
_subprocess.DUPLICATE_SAME_ACCESS)
return fdh
else:
return fd
def start(process, options):
if process.alive():
print('Already running as %s' % process.read_pid())
return
create_app_symlinks(options)
args, env = build_java_execution(options, True)
makedirs(dirname(options.launcher_log))
log = open_append(options.launcher_log)
makedirs(options.data_dir)
os.chdir(options.data_dir)
pid = os.fork()
if pid > 0:
process.write_pid(pid)
print('Started as %s' % pid)
return
if hasattr(os, "set_inheritable"):
# See https://docs.python.org/3/library/os.html#inheritance-of-file-descriptors
# Since Python 3.4
os.set_inheritable(process.pid_file.fileno(), True)
os.setsid()
redirect_stdin_to_devnull()
redirect_output(log)
os.close(log)
os.execvpe(args[0], args, env)
def __init__(self, lock, sockname, configuration, featureset):
self.configuration = configuration
self.featureset = featureset
self.sockname = sockname
self.bitbake_lock = lock
self.readypipe, self.readypipein = os.pipe()
# Create server control socket
if os.path.exists(sockname):
os.unlink(sockname)
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
try:
os.chdir(os.path.dirname(sockname))
self.sock.bind(os.path.basename(sockname))
finally:
os.chdir(cwd)
self.sock.listen(1)
os.set_inheritable(self.sock.fileno(), True)
startdatetime = datetime.datetime.now()
bb.daemonize.createDaemon(self._startServer, logfile)
self.sock.close()
self.bitbake_lock.close()
ready = ConnectionReader(self.readypipe)
r = ready.poll(30)
if r:
r = ready.get()
if not r or r != "ready":
ready.close()
bb.error("Unable to start bitbake server")
if os.path.exists(logfile):
logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
started = False
lines = []
with open(logfile, "r") as f:
for line in f:
if started:
lines.append(line)
else:
res = logstart_re.match(line.rstrip())
if res:
ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
if ldatetime >= startdatetime:
started = True
lines.append(line)
if lines:
if len(lines) > 10:
bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:])))
else:
bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
raise SystemExit(1)
ready.close()
os.close(self.readypipein)
def send_fd_scm(self, fd=None, file_path=None):
# In iotest.py, the qmp should always use unix socket.
assert self._qmp.is_scm_available()
if self._socket_scm_helper is None:
raise QEMUMachineError("No path to socket_scm_helper set")
if not os.path.exists(self._socket_scm_helper):
raise QEMUMachineError("%s does not exist" %
self._socket_scm_helper)
# This did not exist before 3.4, but since then it is
# mandatory for our purpose
if hasattr(os, 'set_inheritable'):
os.set_inheritable(self._qmp.get_sock_fd(), True)
if fd is not None:
os.set_inheritable(fd, True)
fd_param = ["%s" % self._socket_scm_helper,
"%d" % self._qmp.get_sock_fd()]
if file_path is not None:
assert fd is None
fd_param.append(file_path)
else:
assert fd is not None
fd_param.append(str(fd))
devnull = open(os.path.devnull, 'rb')
proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, close_fds=False)
output = proc.communicate()[0]
if output:
LOG.debug(output)
return proc.returncode
def __init__(self):
self.read, self.write = os.pipe()
if PY3:
print('PY3')
# emulate old os.pipe
os.set_inheritable(self.read, True)
os.set_inheritable(self.write, True)
def __init__(self):
self.master, self.slave = os.openpty()
self._old_winch = signal.signal(signal.SIGWINCH, self._size_changed)
self._size_changed()
try:
os.set_inheritable(self.slave, True)
except AttributeError:
pass
def gpgauthenticate(self, file, signature, keyring="/etc/apt/trusted.gpg"):
""" authenticated a file against a given signature, if no keyring
is given use the apt default keyring
"""
status_pipe = os.pipe()
logger_pipe = os.pipe()
if sys.version_info >= (3, 4):
os.set_inheritable(status_pipe[1], 1)
os.set_inheritable(logger_pipe[1], 1)
gpg = [
"gpg",
"--status-fd",
"%d" % status_pipe[1],
"--logger-fd",
"%d" % logger_pipe[1],
"--no-options",
"--homedir",
self.tmpdir,
"--no-default-keyring",
"--ignore-time-conflict",
"--keyring",
keyring,
"--verify",
signature,
file,
]
def gpg_preexec():
os.close(status_pipe[0])
os.close(logger_pipe[0])
proc = subprocess.Popen(
gpg, stderr=subprocess.PIPE, preexec_fn=gpg_preexec, close_fds=False, universal_newlines=True
)
os.close(status_pipe[1])
os.close(logger_pipe[1])
status_handle = os.fdopen(status_pipe[0])
logger_handle = os.fdopen(logger_pipe[0])
try:
gpgres = status_handle.read()
ret = proc.wait()
if ret != 0:
# gnupg returned a problem (non-zero exit)
print("gpg exited %d" % ret)
print("Debug information: ")
print(status_handle.read())
print(proc.stderr.read())
print(logger_handle.read())
return False
if "VALIDSIG" in gpgres:
return True
print("invalid result from gpg:")
print(gpgres)
return False
finally:
status_handle.close()
proc.stderr.close()
logger_handle.close()
def _pipe_open(self):
readend, writeend = os.pipe()
if hasattr(os, "set_inheritable"):
# Python 3 by default will close all fds in subprocesses. This will avoid that.
os.set_inheritable(readend, True)
os.set_inheritable(writeend, True)
readend = os.fdopen(readend, "rb", 0)
writeend = os.fdopen(writeend, "wb", 0)
return readend, writeend
def _pipe_open(self):
readend, writeend = os.pipe()
if hasattr(os, "set_inheritable"):
# https://www.python.org/dev/peps/pep-0446/
os.set_inheritable(readend, True)
os.set_inheritable(writeend, True)
readend = os.fdopen(readend, "rb", 0)
writeend = os.fdopen(writeend, "wb", 0)
return readend, writeend
def _run(self, *, host, port, worker_num=None, reloader_pid=None,
debug=None):
self._debug = debug or self._debug
if self._debug and not self._log_request:
self._log_request = self._debug
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
os.set_inheritable(sock.fileno(), True)
workers = set()
terminating = False
def stop(sig, frame):
nonlocal terminating
if reloader_pid and sig == signal.SIGHUP:
print('Reload request received')
elif not terminating:
terminating = True
print('Termination request received')
for worker in workers:
worker.terminate()
signal.signal(signal.SIGINT, stop)
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGHUP, stop)
for _ in range(worker_num or 1):
worker = multiprocessing.Process(
target=self.serve,
kwargs=dict(sock=sock, host=host, port=port,
reloader_pid=reloader_pid))
worker.daemon = True
worker.start()
workers.add(worker)
# prevent further operations on socket in parent
sock.close()
for worker in workers:
worker.join()
if worker.exitcode > 0:
print('Worker exited with code {}'.format(worker.exitcode))
elif worker.exitcode < 0:
try:
signame = signames[-worker.exitcode]
except KeyError:
print(
'Worker crashed with unknown code {}!'
.format(worker.exitcode))
else:
print('Worker crashed on signal {}!'.format(signame))
def fork_exec(cmd,
exec_env=None,
logfile=None,
pass_fds=None):
"""
Execute a command using fork/exec.
This is needed for programs system executions that need path
searching but cannot have a shell as their parent process, for
example: glance-api. When glance-api starts it sets itself as
the parent process for its own process group. Thus the pid that
a Popen process would have is not the right pid to use for killing
the process group. This patch gives the test env direct access
to the actual pid.
:param cmd: Command to execute as an array of arguments.
:param exec_env: A dictionary representing the environment with
which to run the command.
:param logile: A path to a file which will hold the stdout/err of
the child process.
:param pass_fds: Sequence of file descriptors passed to the child.
"""
env = os.environ.copy()
if exec_env is not None:
for env_name, env_val in exec_env.items():
if callable(env_val):
env[env_name] = env_val(env.get(env_name))
else:
env[env_name] = env_val
pid = os.fork()
if pid == 0:
if logfile:
fds = [1, 2]
with open(logfile, 'r+b') as fptr:
for desc in fds: # close fds
try:
os.dup2(fptr.fileno(), desc)
except OSError:
pass
if pass_fds and hasattr(os, 'set_inheritable'):
# os.set_inheritable() is only available and needed
# since Python 3.4. On Python 3.3 and older, file descriptors are
# inheritable by default.
for fd in pass_fds:
os.set_inheritable(fd, True)
args = shlex.split(cmd)
os.execvpe(args[0], args, env)
else:
return pid
def __init__(self, transaction, begin=50, end=100):
self.transaction = transaction
self.status = ""
self.progress = 0
self.progress_begin = begin
self.progress_end = end
self._child_exit = -1
self.last_activity = 0
self.child_pid = 0
self.status_parent_fd, self.status_child_fd = os.pipe()
if hasattr(os, "set_inheritable"):
os.set_inheritable(self.status_parent_fd, True)
os.set_inheritable(self.status_child_fd, True)
self.output = ""
self._line_buffer = ""
def serve_multiple(server_settings, workers, stop_event=None):
"""
Starts multiple server processes simultaneously. Stops on interrupt
and terminate signals, and drains connections when complete.
:param server_settings: kw arguments to be passed to the serve function
:param workers: number of workers to launch
:param stop_event: if provided, is used as a stop signal
:return:
"""
if server_settings.get('loop', None) is not None:
if server_settings.get('debug', False):
warnings.simplefilter('default')
warnings.warn("Passing a loop will be deprecated in version 0.4.0"
" https://github.com/channelcat/sanic/pull/335"
" has more information.", DeprecationWarning)
server_settings['reuse_port'] = True
sock = socket()
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind((server_settings['host'], server_settings['port']))
set_inheritable(sock.fileno(), True)
server_settings['sock'] = sock
server_settings['host'] = None
server_settings['port'] = None
if stop_event is None:
stop_event = Event()
signal_func(SIGINT, lambda s, f: stop_event.set())
signal_func(SIGTERM, lambda s, f: stop_event.set())
processes = []
for _ in range(workers):
process = Process(target=serve, kwargs=server_settings)
process.daemon = True
process.start()
processes.append(process)
for process in processes:
process.join()
# the above processes will block this until they're stopped
for process in processes:
process.terminate()
sock.close()
asyncio.get_event_loop().stop()
def _do_sendall(loops, send, recv):
for s in send, recv:
os.set_inheritable(s.fileno(), True)
pid = os.fork()
if not pid:
send.close()
recvall(recv, None)
recv.close()
sys.exit()
return 0
else:
try:
return _sendall(loops, send, BIG_DATA)
finally:
send.close()
recv.close()
def temporary_script(lines : list):
# The script close the original file descriptor upon execution. But if execution fails, TemporaryFile should close it.
with tempfile.TemporaryFile() as file:
fd = file.fileno()
name = '/dev/fd/{}'.format(fd)
writer = io.TextIOWrapper(file)
# Close the original file descriptor, which has been duplicated by bash when this line is executed.
for i in ['exec {}<&-'.format(fd)] + lines:
print(i, file = writer)
writer.flush()
file.seek(0)
# Otherwise, the shell would not be able to access this file descriptor.
os.set_inheritable(fd, True)
yield name
def add_fd(self, fd, fdset, opaque, opts=''):
"""
Pass a file descriptor to the VM
"""
options = ['fd=%d' % fd,
'set=%d' % fdset,
'opaque=%s' % opaque]
if opts:
options.append(opts)
# This did not exist before 3.4, but since then it is
# mandatory for our purpose
if hasattr(os, 'set_inheritable'):
os.set_inheritable(fd, True)
self._args.append('-add-fd')
self._args.append(','.join(options))
return self
def ExecingProcess_Pipe():
"""
This is like multiprocessing.Pipe(duplex=True).
It uses our own ExecingProcess_ConnectionWrapper.
"""
import socket
s1, s2 = socket.socketpair()
# We duplicate the fds because the socket objects will close the fds after they go out of scope.
fd1 = os.dup(s1.fileno())
fd2 = os.dup(s2.fileno())
s1.close()
s2.close()
if hasattr(os, "set_inheritable"):
# Python 3 by default will close all fds in subprocesses. This will avoid that.
os.set_inheritable(fd1, True)
os.set_inheritable(fd2, True)
c1 = ExecingProcess_ConnectionWrapper(fd1)
c2 = ExecingProcess_ConnectionWrapper(fd2)
return c1, c2
def set_inheritable(fd, inheritable):
# On py34+ we can use os.set_inheritable but < py34 we must polyfill
# with fcntl and SetHandleInformation
if hasattr(os, 'get_inheritable'):
if os.get_inheritable(fd) != inheritable:
os.set_inheritable(fd, inheritable)
elif WIN:
h = get_handle(fd)
flags = winapi.HANDLE_FLAG_INHERIT if inheritable else 0
winapi.SetHandleInformation(h, winapi.HANDLE_FLAG_INHERIT, flags)
else:
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
if inheritable:
new_flags = flags & ~fcntl.FD_CLOEXEC
else:
new_flags = flags | fcntl.FD_CLOEXEC
if new_flags != flags:
fcntl.fcntl(fd, fcntl.F_SETFD, new_flags)
def _mk_inheritable(fd):
version = sys.version_info[:2]
if version >= (3, 3):
if sys.platform == 'win32':
# Change to Windwos file handle
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
os.set_handle_inheritable(fdh, True)
return fdh
else:
os.set_inheritable(fd, True)
return fd
elif sys.platform == 'win32':
# TODO: find a hack??
# Not yet working
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
return fdh
else:
return fd
def create_socket(host, family=socket.AF_INET, type=socket.SOCK_STREAM,
backlog=2048, blocking=True, inheritable=False):
if family == socket.AF_UNIX and not host.startswith('unix:'):
raise ValueError('Your host needs to have the unix:/path form')
if host.startswith('unix:'):
family = socket.AF_UNIX
if host.startswith('fd://'):
fd = int(host[5:])
sock = socket.fromfd(fd, family, type)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
else:
sock = socket.socket(family, type)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if host.startswith('unix:'):
filename = host[len('unix:'):]
try:
os.remove(filename)
except OSError:
pass
sock.bind(filename)
else:
if ':' in host:
host, port = host.rsplit(':', 1)
port = int(port)
else:
host, port = '0.0.0.0', int(host)
sock.bind((host, port))
sock.listen(backlog)
if blocking:
sock.setblocking(1)
else:
sock.setblocking(0)
# Required since Python 3.4 to be able to share a socket with a child
# process.
if inheritable and hasattr(os, 'set_inheritable'):
os.set_inheritable(sock.fileno(), True)
return sock
def test_connection_not_allowed_from_different_processes(tmpdir, io_method):
r, w = os.pipe()
os.set_inheritable(r, True)
os.set_inheritable(w, True)
with Model.open(tmpdir) as db:
pid = os.fork()
if pid != 0: # Parent
os.close(w)
with os.fdopen(r, 'rb') as pipe:
try:
getattr(db, io_method)() # This should not raise BadUsageError
except BadUsageError as exc:
assert False, "Unexpected exception in parent %r" % exc
except Exception as exc:
# Possible expected behavior because we are not calling
# every io_method with all the neded parameters
pass
finally:
# Receive child result
result, message = pickle.load(pipe)
assert result, message
os.waitpid(pid, 0)
else: # Child
os.close(r)
with os.fdopen(w, 'wb') as pipe:
try:
getattr(db, io_method)() # This should not raise BadUsageError
except BadUsageError:
pickle.dump((True, "Expected behavior"), pipe)
except Exception as exc:
# This is not expected even when we are calling the
# io_method with incorrect parameters because BadUsageError
# MUST raise first given that is in the decorator.
pickle.dump((False, "Unexpected exception in child %r" % exc), pipe)
else:
pickle.dump((False, "Child did not raise"), pipe)
os._exit(0)
def process(self):
proc_obj = None
log_read_fobj = None
try:
logfd_read, logfd_write = os.pipe()
log_read_fobj = os.fdopen(logfd_read, 'r')
os.set_inheritable(logfd_write, True)
cmd = self.cmd_line[:]
cmd.extend(['--log-fd', str(logfd_write)])
proc_obj = subprocess.Popen(cmd, close_fds=False, stdin=subprocess.DEVNULL, shell=False)
os.close(logfd_write)
res = self.process_with_fd(proc_obj, log_read_fobj)
return res
except Exception as e:
print("Backup failed!!!: %s" % e)
self.job.terminate("Backup failed")
raise
finally:
log_read_fobj.close()
if proc_obj:
proc_obj.kill()
def run_cgi(self, toOpen):
#look for queries
env = cgi_env(self, toOpen)
self.wfile.flush()
#prepare pipe
r,w = os.pipe()
os.set_inheritable(w, True)
pid = os.fork()
if pid == 0:
#LOG
print_log(self)
args = [toOpen]
try:
os.dup2( w , 1)
os.dup2( self.rfile.fileno() , 0)
os.execve(toOpen , args, env)
os._exit(0)
except:
os._exit(1)
else:
# Parent
os.waitpid(pid, 0)
os.close(w)
content = ""
buf = os.read(r, BLOCK_SIZE).decode("utf-8")
while buf != "":
content += buf
buf = os.read(r, BLOCK_SIZE).decode("utf-8")
os.close(r)
length = str( get_content_length(content.encode("utf-8")))
#prepare HEADER
cgi_header(self, content)
self.wfile.write( bytes("Content-Length: "+length+"\n", "utf-8") )
self.wfile.write(content.encode("utf-8"))
self.wfile.flush()
return
def main():
if len(sys.argv) < 4:
print('Usage: {} inputfile outputfile cmd ...'.format(sys.argv[0]), file=sys.stderr)
sys.exit(1)
inputfile, outputfile = sys.argv[1], sys.argv[2]
cmd = sys.argv[3:]
# create the input feeder
(stdin, w) = os.pipe()
inpid = os.fork()
if inpid == 0:
os.close(stdin)
inputFeeder(w, inputfile)
sys.exit(0)
os.close(w)
# create the output watcher
(r, stdout) = os.pipe()
outpid = os.fork()
if outpid == 0:
os.close(stdout)
outputWatcher(r, outputfile)
sys.exit(0)
os.close(r)
# create the stderr watcher
(r, stderr) = os.pipe()
errpid = os.fork()
if errpid == 0:
os.close(stderr)
stderrWatcher(r)
sys.exit(0)
os.close(r)
# launch the main command
pid = os.fork()
if pid == 0:
os.dup2(stdin, 0)
os.set_inheritable(0, True)
os.close(stdin)
os.dup2(stdout, 1)
os.set_inheritable(1, True)
os.close(stdout)
os.dup2(stderr, 2)
os.set_inheritable(2, True)
os.close(stderr)
os.execvp(cmd[0], cmd)
# wait for a child to exit, then kill the rest
(child, status) = os.wait()
if child != inpid: os.kill(inpid, signal.SIGTERM)
if child != outpid: os.kill(outpid, signal.SIGTERM)
if child != errpid: os.kill(errpid, signal.SIGTERM)
if child != pid: os.kill(pid, signal.SIGTERM)
def set_inheritable(self, inheritable):
os.set_inheritable(self.fileno(), inheritable)
import os
import signal
import sys
binpath = sys.argv[1]
p_ic3_r, p_ic3_w = os.pipe()
os.set_inheritable(p_ic3_r, True)
id_ic3 = os.fork()
if id_ic3 == 0:
sys.stdin.close()
os.dup2(p_ic3_r, 0)
os.execl(binpath + 'vvt-verify', 'vvt-verify',
'--backend=cons:' + binpath + 'z3 -smt2 -in',
'--backend=lifting:' + binpath + 'z3 -smt2 -in',
'--backend=domain:' + binpath + 'z3 -smt2 -in',
'--backend=init:' + binpath + 'z3 -smt2 -in',
'--backend=cons:' + binpath + 'mathsat')
os.close(p_ic3_r)
p_bmc_r, p_bmc_w = os.pipe()
os.set_inheritable(p_bmc_r, True)
id_bmc = os.fork()
if id_bmc == 0:
sys.stdin.close()
os.dup2(p_bmc_r, 0)
os.execl(binpath + 'vvt-bmc', 'vvt-bmc',
'--solver=' + binpath + 'z3 -smt2 -in', '-i', '-d', '50')
os.close(p_bmc_r)
while True:
def __init__(self, lock, sockname, configuration, featureset):
self.configuration = configuration
self.featureset = featureset
self.sockname = sockname
self.bitbake_lock = lock
self.readypipe, self.readypipein = os.pipe()
# Create server control socket
if os.path.exists(sockname):
os.unlink(sockname)
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
try:
os.chdir(os.path.dirname(sockname))
self.sock.bind(os.path.basename(sockname))
finally:
os.chdir(cwd)
self.sock.listen(1)
os.set_inheritable(self.sock.fileno(), True)
startdatetime = datetime.datetime.now()
bb.daemonize.createDaemon(self._startServer, logfile)
self.sock.close()
self.bitbake_lock.close()
ready = ConnectionReader(self.readypipe)
r = ready.poll(30)
if r:
r = ready.get()
if not r or r != "ready":
ready.close()
bb.error("Unable to start bitbake server")
if os.path.exists(logfile):
logstart_re = re.compile(self.start_log_format %
('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
started = False
lines = []
with open(logfile, "r") as f:
for line in f:
if started:
lines.append(line)
else:
res = logstart_re.match(line.rstrip())
if res:
ldatetime = datetime.datetime.strptime(
res.group(2),
self.start_log_datetime_format)
if ldatetime >= startdatetime:
started = True
lines.append(line)
if lines:
if len(lines) > 10:
bb.error(
"Last 10 lines of server log for this session (%s):\n%s"
% (logfile, "".join(lines[-10:])))
else:
bb.error("Server log for this session (%s):\n%s" %
(logfile, "".join(lines)))
raise SystemExit(1)
ready.close()
os.close(self.readypipein)
def os_pipe():
r, w = os.pipe()
os.set_inheritable(r, True)
os.set_inheritable(w, True)
return r, w
def _ApplyDiffOperation(self, op, op_name, patch_data, old_part_file,
new_part_file):
"""Applies a SOURCE_BSDIFF, BROTLI_BSDIFF or PUFFDIFF operation.
Args:
op: the operation object
op_name: name string for error reporting
patch_data: the binary patch content
old_part_file: the source partition file object
new_part_file: the target partition file object
Raises:
PayloadError if something goes wrong.
"""
if not old_part_file:
raise PayloadError(
'%s: no source partition file provided for operation type (%d)' %
(op_name, op.type))
block_size = self.block_size
# Dump patch data to file.
with tempfile.NamedTemporaryFile(delete=False) as patch_file:
patch_file_name = patch_file.name
patch_file.write(patch_data)
if (hasattr(new_part_file, 'fileno') and
((not old_part_file) or hasattr(old_part_file, 'fileno'))):
# Construct input and output extents argument for bspatch.
in_extents_arg, _, _ = _ExtentsToBspatchArg(
op.src_extents, block_size, '%s.src_extents' % op_name,
data_length=op.src_length if op.src_length else
self._BytesInExtents(op.src_extents, "%s.src_extents"))
out_extents_arg, pad_off, pad_len = _ExtentsToBspatchArg(
op.dst_extents, block_size, '%s.dst_extents' % op_name,
data_length=op.dst_length if op.dst_length else
self._BytesInExtents(op.dst_extents, "%s.dst_extents"))
new_file_name = '/dev/fd/%d' % new_part_file.fileno()
# Diff from source partition.
old_file_name = '/dev/fd/%d' % old_part_file.fileno()
# In python3, file descriptors(fd) are not passed to child processes by
# default. To pass the fds to the child processes, we need to set the flag
# 'inheritable' in the fds and make the subprocess calls with the argument
# close_fds set to False.
if sys.version_info.major >= 3:
os.set_inheritable(new_part_file.fileno(), True)
os.set_inheritable(old_part_file.fileno(), True)
if op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.BROTLI_BSDIFF):
# Invoke bspatch on partition file with extents args.
bspatch_cmd = [self.bspatch_path, old_file_name, new_file_name,
patch_file_name, in_extents_arg, out_extents_arg]
subprocess.check_call(bspatch_cmd, close_fds=False)
elif op.type == common.OpType.PUFFDIFF:
# Invoke puffpatch on partition file with extents args.
puffpatch_cmd = [self.puffpatch_path,
"--operation=puffpatch",
"--src_file=%s" % old_file_name,
"--dst_file=%s" % new_file_name,
"--patch_file=%s" % patch_file_name,
"--src_extents=%s" % in_extents_arg,
"--dst_extents=%s" % out_extents_arg]
subprocess.check_call(puffpatch_cmd, close_fds=False)
else:
raise PayloadError("Unknown operation %s" % op.type)
# Pad with zeros past the total output length.
if pad_len:
new_part_file.seek(pad_off)
new_part_file.write(b'\0' * pad_len)
else:
# Gather input raw data and write to a temp file.
input_part_file = old_part_file if old_part_file else new_part_file
in_data = _ReadExtents(input_part_file, op.src_extents, block_size,
max_length=op.src_length if op.src_length else
self._BytesInExtents(op.src_extents,
"%s.src_extents"))
with tempfile.NamedTemporaryFile(delete=False) as in_file:
in_file_name = in_file.name
in_file.write(in_data)
# Allocate temporary output file.
with tempfile.NamedTemporaryFile(delete=False) as out_file:
out_file_name = out_file.name
if op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.BROTLI_BSDIFF):
# Invoke bspatch.
bspatch_cmd = [self.bspatch_path, in_file_name, out_file_name,
patch_file_name]
subprocess.check_call(bspatch_cmd)
elif op.type == common.OpType.PUFFDIFF:
# Invoke puffpatch.
puffpatch_cmd = [self.puffpatch_path,
"--operation=puffpatch",
"--src_file=%s" % in_file_name,
"--dst_file=%s" % out_file_name,
"--patch_file=%s" % patch_file_name]
subprocess.check_call(puffpatch_cmd)
else:
raise PayloadError("Unknown operation %s" % op.type)
# Read output.
with open(out_file_name, 'rb') as out_file:
out_data = out_file.read()
if len(out_data) != op.dst_length:
raise PayloadError(
'%s: actual patched data length (%d) not as expected (%d)' %
(op_name, len(out_data), op.dst_length))
# Write output back to partition, with padding.
unaligned_out_len = len(out_data) % block_size
if unaligned_out_len:
out_data += b'\0' * (block_size - unaligned_out_len)
_WriteExtents(new_part_file, out_data, op.dst_extents, block_size,
'%s.dst_extents' % op_name)
# Delete input/output files.
os.remove(in_file_name)
os.remove(out_file_name)
# Delete patch file.
os.remove(patch_file_name)
def piping(args):
# '|' for split command
#args = args.split('|')
'''''
left and right arguments. lArg retrieves data of the pipe's left side
rArg retrieves data of the pipe's right side
'''''
lArg = args[0:args.index("|")]
rArg = args[len(lArg)+1:]
pipeRead, pipeWrite = os.pipe()
rc = os.fork()
# Forking fails if we get a zero returned
if rc < 0:
os.write(2, ("Fork failed, returning %d\n" % rc).encode())
exit()
elif rc == 0:
# redirects child stdout (fd0)
os.close(1)
# duplicates the child's fds and assigns them to pipeWrite
os.dup(pipeWrite)
os.set_inheritable(1, True)
for fdno in (pipeRead, pipeWrite):
# closes all fds
os.close(fdno)
# command will get the left argument
command(lArg)
else:
# redirects child stdin (fd0)
os.close(0)
os.dup(pipeRead)
os.set_inheritable(0, True)
for fdno in (pipeWite, pipeRead):
# closes all fds
os.close(fdno)
if "|" in rArg:
pipe(rArg)
# command will get the right side
command(rArg)
if '<' in lArg:
inRedir("in")
if '>' in lArg:
inRedir("out")
# The right argument will be executed
else:
if '<' in rArg:
outRedir("in")
if '>' in rArg:
outRedir("out")
def set_fd_inheritable(fd):
os.set_inheritable(fd, True)
def main():
try:
# os.system should be a subprocess instead
os.system('cat ~/.bashrc | grep PS1 > ./ps1.txt')
with open('./ps1.txt', 'r') as ps1:
line = ps1.readline()
split = re.split('\'', line)
prompt = split[1]
except FileNotFoundError: # Windows, probably
prompt = 'shel$ '
while (True):
cmd = input(prompt)
if cmd == 'quit':
sys.exit(0)
firstArgs, afterArgs, paths = parse(cmd)
#print(firstArgs, afterArgs, paths)
if afterArgs is None:
rc = os.fork() # !!!
if rc == 0: # child
for dir in paths:
fullPath = '%s/%s' % (dir, firstArgs[0])
try:
os.execve(fullPath, firstArgs, os.environ)
except FileNotFoundError:
pass
else: # parent
cpid = os.wait()
else: # more arguments after
redirect = afterArgs.pop(0) # grab and remove redirect
if redirect == '|':
pr, pw = os.pipe()
print('pr=%d, pw=%d' % (pr, pw))
os.set_inheritable(pr, True)
os.set_inheritable(pw, True)
rc = os.fork() # !!!
if rc == 0: # child
print('Child: pr=%d, pw=%d' % (pr, pw))
stdout = sys.stdout # save stdout
os.close(1) # close stdout
os.set_inheritable(os.dup(pw),
True) # first child's stdout
os.close(pr)
os.close(pw)
for dir in paths:
fullPath = '%s/%s' % (dir, firstArgs[0])
try:
os.execve(fullPath, firstArgs, os.environ)
except FileNotFoundError:
pass
else: # parent
cpid = os.wait()
firstArgs = afterArgs
afterArgs = None
rc1 = os.fork() # !!!
if rc1 == 0: # child
stdin = sys.stdin # save stdin
os.close(0) # close stdin
os.set_inheritable(os.dup(pr), True)
os.close(pr)
os.close(pw)
for dir in paths:
fullPath = '%s/%s' % (dir, firstArgs[0])
try:
os.execve(fullPath, firstArgs, os.environ)
except FileNotFoundError:
pass
else:
cpid = os.wait()
elif redirect == '<':
rc = os.fork() # !!!
if rc == 0:
stdin = sys.stdin # save stdin
os.close(0) # close stdin
os.stdin = open(afterArgs[0], 'r')
os.set_inheritable(0, True)
for dir in paths:
fullPath = '%s/%s' % (dir, firstArgs[0])
try:
os.execve(fullPath, firstArgs, os.environ)
except FileNotFoundError:
pass
else: # parent
cpid = os.wait()
else: # redirect == '>'
rc = os.fork() # !!!
if rc == 0:
stdout = sys.stdout # save stdout
os.close(1) # close stdout
os.stdout = open(afterArgs[0], 'w')
os.set_inheritable(1, True)
for dir in paths:
fullPath = '%s/%s' % (dir, firstArgs[0])
try:
os.execve(fullPath, firstArgs, os.environ)
except FileNotFoundError:
pass
else:
cpid = os.wait()
def test_vhost_user_vga_virgl(self):
"""
:avocado: tags=arch:x86_64
:avocado: tags=device:vhost-user-vga
"""
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
"console=ttyS0 rdinit=/bin/bash")
# FIXME: should check presence of vhost-user-gpu, virgl, memfd etc
if not kvm_available(self.arch, self.qemu_bin):
self.cancel(KVM_NOT_AVAILABLE)
vug = pick_default_vug_bin()
if not vug:
self.cancel("Could not find vhost-user-gpu")
kernel_path = self.fetch_asset(self.KERNEL_URL)
initrd_path = self.fetch_asset(self.INITRD_URL)
# Create socketpair to connect proxy and remote processes
qemu_sock, vug_sock = socket.socketpair(socket.AF_UNIX,
socket.SOCK_STREAM)
os.set_inheritable(qemu_sock.fileno(), True)
os.set_inheritable(vug_sock.fileno(), True)
self._vug_log_path = os.path.join(self.vm._test_dir,
"vhost-user-gpu.log")
self._vug_log_file = open(self._vug_log_path, "wb")
print(self._vug_log_path)
vugp = subprocess.Popen(
[vug, "--virgl", "--fd=%d" % vug_sock.fileno()],
stdin=subprocess.DEVNULL,
stdout=self._vug_log_file,
stderr=subprocess.STDOUT,
shell=False,
close_fds=False,
)
self.vm.set_console()
self.vm.add_args("-cpu", "host")
self.vm.add_args("-m", "2G")
self.vm.add_args("-object", "memory-backend-memfd,id=mem,size=2G")
self.vm.add_args("-machine", "pc,memory-backend=mem,accel=kvm")
self.vm.add_args("-chardev",
"socket,id=vug,fd=%d" % qemu_sock.fileno())
self.vm.add_args("-device", "vhost-user-vga,chardev=vug")
self.vm.add_args("-display", "egl-headless")
self.vm.add_args(
"-kernel",
kernel_path,
"-initrd",
initrd_path,
"-append",
kernel_command_line,
)
self.vm.launch()
self.wait_for_console_pattern("as init process")
exec_command_and_wait_for_pattern(self,
"/usr/sbin/modprobe virtio_gpu", "")
self.wait_for_console_pattern("features: +virgl -edid")
self.vm.shutdown()
qemu_sock.close()
vugp.terminate()
vugp.wait()
def run_simple(hostname, port, application, use_reloader=False,
use_debugger=False, use_evalex=True,
extra_files=None, reloader_interval=1,
reloader_type='auto', threaded=False,
processes=1, request_handler=None, static_files=None,
passthrough_errors=False, ssl_context=None):
"""Start a WSGI application. Optional features include a reloader,
multithreading and fork support.
This function has a command-line interface too::
python -m werkzeug.serving --help
.. versionadded:: 0.5
`static_files` was added to simplify serving of static files as well
as `passthrough_errors`.
.. versionadded:: 0.6
support for SSL was added.
.. versionadded:: 0.8
Added support for automatically loading a SSL context from certificate
file and private key.
.. versionadded:: 0.9
Added command-line interface.
.. versionadded:: 0.10
Improved the reloader and added support for changing the backend
through the `reloader_type` parameter. See :ref:`reloader`
for more information.
:param hostname: The host for the application. eg: ``'localhost'``
:param port: The port for the server. eg: ``8080``
:param application: the WSGI application to execute
:param use_reloader: should the server automatically restart the python
process if modules were changed?
:param use_debugger: should the werkzeug debugging system be used?
:param use_evalex: should the exception evaluation feature be enabled?
:param extra_files: a list of files the reloader should watch
additionally to the modules. For example configuration
files.
:param reloader_interval: the interval for the reloader in seconds.
:param reloader_type: the type of reloader to use. The default is
auto detection. Valid values are ``'stat'`` and
``'watchdog'``. See :ref:`reloader` for more
information.
:param threaded: should the process handle each request in a separate
thread?
:param processes: if greater than 1 then handle each request in a new process
up to this maximum number of concurrent processes.
:param request_handler: optional parameter that can be used to replace
the default one. You can use this to replace it
with a different
:class:`~BaseHTTPServer.BaseHTTPRequestHandler`
subclass.
:param static_files: a dict of paths for static files. This works exactly
like :class:`SharedDataMiddleware`, it's actually
just wrapping the application in that middleware before
serving.
:param passthrough_errors: set this to `True` to disable the error catching.
This means that the server will die on errors but
it can be useful to hook debuggers in (pdb etc.)
:param ssl_context: an SSL context for the connection. Either an
:class:`ssl.SSLContext`, a tuple in the form
``(cert_file, pkey_file)``, the string ``'adhoc'`` if
the server should automatically create one, or ``None``
to disable SSL (which is the default).
"""
if use_debugger:
from werkzeug.debug import DebuggedApplication
application = DebuggedApplication(application, use_evalex)
if static_files:
from werkzeug.wsgi import SharedDataMiddleware
application = SharedDataMiddleware(application, static_files)
def inner():
try:
fd = int(os.environ['WERKZEUG_SERVER_FD'])
except (LookupError, ValueError):
fd = None
make_server(hostname, port, application, threaded,
processes, request_handler,
passthrough_errors, ssl_context,
fd=fd).serve_forever()
if use_reloader:
# If we're not running already in the subprocess that is the
# reloader we want to open up a socket early to make sure the
# port is actually available.
if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
if port == 0 and not can_open_by_fd:
raise ValueError('Cannot bind to a random port with enabled '
'reloader if the Python interpreter does '
'not support socket opening by fd.')
# Create and destroy a socket so that any exceptions are
# raised before we spawn a separate Python interpreter and
# lose this ability.
address_family = select_ip_version(hostname, port)
s = socket.socket(address_family, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((hostname, port))
if hasattr(os, 'set_inheritable'):
os.set_inheritable(s.fileno(), True)
# If we can open the socket by file descriptor, then we can just
# reuse this one and our socket will survive the restarts.
if can_open_by_fd:
os.environ['WERKZEUG_SERVER_FD'] = str(s.fileno())
s.listen(LISTEN_QUEUE)
else:
s.close()
from ._reloader import run_with_reloader
run_with_reloader(inner, extra_files, reloader_interval,
reloader_type)
else:
inner()
def process_start(self):
"""Start soapy_power process"""
if not self.process and self.params:
# Create pipe used for communication with soapy_power process
self.pipe_read_fd, self.pipe_write_fd = os.pipe()
self.pipe_read = open(self.pipe_read_fd, 'rb')
os.set_inheritable(self.pipe_write_fd, True)
if sys.platform == 'win32':
self.pipe_write_handle = subprocess.make_inheritable_handle(
self.pipe_write_fd)
# Prepare soapy_power cmdline parameters
settings = QtCore.QSettings()
cmdline = shlex.split(settings.value("executable", "soapy_power"))
cmdline.extend([
"-f",
"{}M:{}M".format(self.params["start_freq"],
self.params["stop_freq"]),
"-B",
"{}k".format(self.params["bin_size"]),
"-T",
"{}".format(self.params["interval"]),
"-d",
"{}".format(self.params["device"]),
"-r",
"{}".format(self.params["sample_rate"]),
"-p",
"{}".format(self.params["ppm"]),
"-F",
"soapy_power_bin",
"--output-fd",
"{}".format(
int(self.pipe_write_handle) if sys.platform ==
'win32' else self.pipe_write_fd),
])
if self.lnb_lo != 0:
cmdline.extend(["--lnb-lo", "{}".format(self.lnb_lo)])
if self.params["bandwidth"] > 0:
cmdline.extend(["-w", "{}".format(self.params["bandwidth"])])
if self.params["gain"] >= 0:
cmdline.extend(["-g", "{}".format(self.params["gain"])])
if self.params["crop"] > 0:
cmdline.extend(["-k", "{}".format(self.params["crop"])])
if not self.params["single_shot"]:
cmdline.append("-c")
additional_params = settings.value("params",
Info.additional_params)
if additional_params:
cmdline.extend(shlex.split(additional_params))
# Start soapy_power process and close write part of pipe
if sys.platform == 'win32':
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags = 0
self.process = subprocess.Popen(cmdline,
close_fds=False,
universal_newlines=False,
creationflags=creationflags,
console=False)
os.close(self.pipe_write_fd)
if sys.platform == 'win32':
self.pipe_write_handle.Close()
def __init__(self, repo: mzbuild.Repository, name: str):
self.name = name
self.repo = repo
self.images: List[mzbuild.Image] = []
self.workflows: Dict[str, Workflow] = {}
default_tag = os.getenv(f"MZBUILD_TAG", None)
if name in self.repo.compositions:
self.path = self.repo.compositions[name]
else:
raise errors.UnknownComposition
with open(self.path) as f:
compose = yaml.safe_load(f)
workflows = compose.pop("mzworkflows", None)
if workflows is not None:
# TODO: move this into the workflow so that it can use env vars that are
# manually defined.
workflows = _substitute_env_vars(workflows)
for workflow_name, raw_w in workflows.items():
built_steps = []
for raw_step in raw_w["steps"]:
step_name = raw_step.pop("step")
step_ty = Steps.named(step_name)
munged = {
k.replace("-", "_"): v
for k, v in raw_step.items()
}
try:
step = step_ty(**munged)
except TypeError as e:
a = " ".join([f"{k}={v}" for k, v in munged.items()])
raise errors.BadSpec(
f"Unable to construct {step_name} with args {a}: {e}"
)
built_steps.append(step)
env = raw_w.get("env")
if not isinstance(env, dict) and env is not None:
raise errors.BadSpec(
f"Workflow {workflow_name} has wrong type for env: "
f"expected mapping, got {type(env).__name__}: {env}", )
# ensure that integers (e.g. ports) are treated as env vars
if isinstance(env, dict):
env = {k: str(v) for k, v in env.items()}
self.workflows[workflow_name] = Workflow(workflow_name,
built_steps,
env=env,
composition=self)
# Resolve all services that reference an `mzbuild` image to a specific
# `image` reference.
for config in compose["services"].values():
if "mzbuild" in config:
image_name = config["mzbuild"]
if image_name not in self.repo.images:
raise errors.BadSpec(
f"mzcompose: unknown image {image_name}")
image = self.repo.images[image_name]
override_tag = os.getenv(f"MZBUILD_{image.env_var_name()}_TAG",
default_tag)
if override_tag is not None:
config["image"] = image.docker_name(override_tag)
print(
f"mzcompose: warning: overriding {image_name} image to tag {override_tag}",
file=sys.stderr,
)
del config["mzbuild"]
else:
self.images.append(image)
if "propagate-uid-gid" in config:
config["user"] = f"{os.getuid()}:{os.getgid()}"
del config["propagate-uid-gid"]
deps = self.repo.resolve_dependencies(self.images)
for config in compose["services"].values():
if "mzbuild" in config:
config["image"] = deps[config["mzbuild"]].spec()
del config["mzbuild"]
# Emit the munged configuration to a temporary file so that we can later
# pass it to Docker Compose.
tempfile = TemporaryFile()
os.set_inheritable(tempfile.fileno(), True)
yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore
tempfile.flush()
self.file = tempfile
def main(argv):
parser = argparse.ArgumentParser(description="Runs Dolphin as a non-privileged user with access to the privileged tap0 network interface. Run this script with sudo; it will run Dolphin as the user you sudo'ed from and provide it access to the tap interface. It will also automatically configure the tap interface when Dolphin first opens it.")
parser.add_argument(
"--dolphin-path", "-d",
action="store",
dest="dolphin_path",
default="./Dolphin.app/Contents/MacOS/Dolphin",
help="Path to Dolphin executable. Defaults to Dolphin.app/Contents/MacOS/Dolphin in current directory.",
)
parser.add_argument(
"--memwatch-path", "-m",
action="store",
dest="memwatch_path",
default="memwatch",
help="Path to memwatch executable. Defaults to memwatch (assumes it's installed somewhere on $PATH).",
)
parser.add_argument(
"--tap-interface", "-I",
action="store",
dest="tap_interface",
default="/dev/tap0",
help="Tap interface to use. Defaults to /dev/tap0.",
)
parser.add_argument(
"--tap-ip", "-i",
action="store",
dest="tap_ip",
default="192.168.0.5/24",
help="IP address and subnet bits to assign to tap interface. Defaults to 192.168.0.5/24. This should match the gateway and DNS server addresses configured in PSO's network settings.",
)
args = parser.parse_args()
try:
username = os.environ['SUDO_USER']
except KeyError:
print('$SUDO_USER not set; use `sudo -E`')
return 1
tap_match = re.match(r'^/dev/tap([0-9]+)$', args.tap_interface)
if tap_match is None:
print('tap interface name must begin with /dev/tap')
return 1
tap_interface_number = int(tap_match.group(1))
tap_name = 'tap%d' % (tap_interface_number,)
# 1. open tap and configure it
print("starting and configuring " + tap_name)
tap_fd = os.open(args.tap_interface, os.O_RDWR)
os.set_inheritable(tap_fd, True)
subprocess.check_call(['ifconfig', tap_name, args.tap_ip], stderr=subprocess.DEVNULL)
subprocess.check_call(['ifconfig', tap_name, 'up'], stderr=subprocess.DEVNULL)
subprocess.check_call(['ifconfig', tap_name, 'mtu', '9000'], stderr=subprocess.DEVNULL)
# 2. fork a Dolphin process, dropping privileges first
print("starting dolphin")
dolphin_proc = subprocess.Popen([args.dolphin_path],
preexec_fn=lambda: change_user(username), pass_fds=(tap_fd,))
time.sleep(1)
# 3. create a temp file for dolphin to open instead of /dev/tapN
print("creating temp file")
tmpfile_fd = os.open("/tmp/dnet", os.O_CREAT | os.O_RDWR, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
os.close(tmpfile_fd)
os.chmod("/tmp/dnet", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
# 4. modify dolphin's memory to make it upen the temp file
print("redirecting /dev/tap0 for dolphin")
addresses_str = subprocess.check_output([args.memwatch_path, str(dolphin_proc.pid), 'find', '\"/dev/tap0\"'], stderr=subprocess.DEVNULL)
for line in addresses_str.splitlines():
tokens = line.split()
if len(tokens) != 3:
continue
print("redirecting /dev/tap0 to /tmp/dnet at %s in dolphin" % (tokens[1],))
subprocess.check_call([args.memwatch_path, str(dolphin_proc.pid), "access", tokens[1], "rwx"], stderr=subprocess.DEVNULL)
subprocess.check_call([args.memwatch_path, str(dolphin_proc.pid), "write", tokens[1], "\"/tmp/dnet\""], stderr=subprocess.DEVNULL)
subprocess.check_call([args.memwatch_path, str(dolphin_proc.pid), "access", tokens[1], "r-x"], stderr=subprocess.DEVNULL)
# step 5: use lsof to find out when dolphin opens /tmp/dnet
print("waiting for temp file to open")
dolphin_tap0_fd = -1
while dolphin_tap0_fd < 0:
time.sleep(1)
result = subprocess.check_output(["lsof", "-p", str(dolphin_proc.pid)], stderr=subprocess.DEVNULL)
for line in result.splitlines():
if b'/tmp/dnet' not in line:
continue
fd_str = line.split()[3]
dolphin_tap0_fd = int(fd_str[0:-len(fd_str.lstrip(b'0123456789'))])
print("found open tap fd %d in dolphin" % (dolphin_tap0_fd,))
# step 6: use memwatch to move the tap fd into place
print("replacing temp fd %d with tap fd %d in dolphin" % (dolphin_tap0_fd,
tap_fd))
assembly_contents = b"""start:
mov rax, 0x000000000200005A # dup2(from_fd, to_fd)
mov rdi, %d
mov rsi, %d
syscall
# close the original fd. note that rdi is preserved during the syscall so we
# don't need to reload it
mov rax, 0x0000000002000006 # close(fd)
syscall
ret
""" % (tap_fd, dolphin_tap0_fd)
subprocess.run([args.memwatch_path, str(dolphin_proc.pid), "--", "run", "-"], input=assembly_contents, stderr=subprocess.DEVNULL)
# ok we're done; dolphin is running as a non-privileged user with the tap open
return 0
if pid != 0:
os.close(pipe_info[1])
os.close(userns_block[0])
select.select([pipe_info[0]], [], [])
data = json.load(os.fdopen(pipe_info[0]))
child_pid = str(data['child-pid'])
subprocess.call(["newuidmap", child_pid, "0", str(os.getuid()), "1"])
subprocess.call(["newgidmap", child_pid, "0", str(os.getgid()), "1"])
os.write(userns_block[1], b'1')
else:
os.close(pipe_info[0])
os.close(userns_block[1])
if sys.version_info >= (3, 4):
os.set_inheritable(pipe_info[1], True)
os.set_inheritable(userns_block[0], True)
args = [
"bwrap", "bwrap", "--unshare-all", "--unshare-user",
"--userns-block-fd",
"%i" % userns_block[0], "--info-fd",
"%i" % pipe_info[1], "--bind", "/", "/", "cat", "/proc/self/uid_map"
]
os.execlp(*args)
#!/usr/bin/env python
import socket
import subprocess
import os
import sys
import threading
args = getattr(__builtins__,
'raw_input',
lambda: open(0, 'rb', buffering=0, closefd=False).readline()[0:-1])()[1:-1]
s1,s2 = socket.socketpair()
if hasattr(os, 'set_inheritable'):
os.set_inheritable(s1.fileno(), True)
os.set_inheritable(s2.fileno(), True)
p1 = subprocess.Popen(['socat', '-', 'FD:' + str(s1.fileno())], close_fds=False)
env = dict(os.environ)
env['NODE_CHANNEL_FD'] = str(s2.fileno())
p2 = subprocess.Popen(['node', '-e', 'process.argv = [process.argv[0], process.argv[1]].concat(JSON.parse(new Buffer(process.argv[2], "hex"))); setImmediate(function () { require(require("path").resolve(process.argv[1])); })', sys.argv[1], args], env=env, stdin=subprocess.PIPE, stdout=2, close_fds=False)
p2.stdin.close()
#s1.close()
#s2.close()
done = False
class WaitThread(threading.Thread):
def __init__(self, lock, sockname, configuration, featureset):
self.configuration = configuration
self.featureset = featureset
self.sockname = sockname
self.bitbake_lock = lock
self.readypipe, self.readypipein = os.pipe()
# Create server control socket
if os.path.exists(sockname):
os.unlink(sockname)
# Place the log in the builddirectory alongside the lock file
logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# AF_UNIX has path length issues so chdir here to workaround
cwd = os.getcwd()
try:
os.chdir(os.path.dirname(sockname))
self.sock.bind(os.path.basename(sockname))
finally:
os.chdir(cwd)
self.sock.listen(1)
os.set_inheritable(self.sock.fileno(), True)
startdatetime = datetime.datetime.now()
bb.daemonize.createDaemon(self._startServer, logfile)
self.sock.close()
self.bitbake_lock.close()
os.close(self.readypipein)
ready = ConnectionReader(self.readypipe)
r = ready.poll(5)
if not r:
bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
r = ready.poll(90)
if r:
try:
r = ready.get()
except EOFError:
# Trap the child exitting/closing the pipe and error out
r = None
if not r or r[0] != "r":
ready.close()
bb.error("Unable to start bitbake server (%s)" % str(r))
if os.path.exists(logfile):
logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
started = False
lines = []
lastlines = []
with open(logfile, "r") as f:
for line in f:
if started:
lines.append(line)
else:
lastlines.append(line)
res = logstart_re.match(line.rstrip())
if res:
ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
if ldatetime >= startdatetime:
started = True
lines.append(line)
if len(lastlines) > 60:
lastlines = lastlines[-60:]
if lines:
if len(lines) > 60:
bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
else:
bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
elif lastlines:
bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
else:
bb.error("%s doesn't exist" % logfile)
raise SystemExit(1)
ready.close()
while (1):
tokens = None
try:
response = input(ps1)
except EOFError:
sys.exit()
pass
response = response.strip()
if "|" in response:
commands = response.split("|") #since we only need to test 1 pipe
commands[0], commands[1] = commands[0].strip(), commands[1].strip()
rc = os.fork()
if rc == 0:
pr, pw = os.pipe()
for f in (pr, pw):
os.set_inheritable(f, True)
piping = os.fork() #forking twice
if piping == 0: #child is sending output to second child
os.close(1) #snippet modified from pipe demo
os.dup2(pw, 1)
for fd in (pr, pw):
os.close(fd)
tokens = commands[0].split(" ", 1) #tokenizing args
try: #in case the whole path is given
os.execve(tokens[0], tokens, os.environ)
except FileNotFoundError:
pass
for dir in re.split(
":",
os.environ["PATH"]): #searching for program in PATH
program = "%s/%s" % (dir, tokens[0])
os.write(1, ("About to fork (pid=%d)\n" % pid).encode())
rc = os.fork()
if rc < 0:
os.write(2, ("fork failed, returning %d\n" % rc).encode())
sys.exit(1)
elif rc == 0: # child
os.write(1, ("Child: My pid==%d. Parent's pid=%d\n" %
(os.getpid(), pid)).encode())
args = ["wc", "p3-exec.py"]
os.close(1) # redirect child's stdout
os.open("p4-output.txt", os.O_CREAT | os.O_WRONLY)
os.set_inheritable(1, True)
for dir in re.split(":", os.environ['PATH']): # try each directory in path
program = "%s/%s" % (dir, args[0])
try:
os.execve(program, args, os.environ) # try to exec program
except FileNotFoundError: # ...expected
pass # ...fail quietly
os.write(2, ("Child: Error: Could not exec %s\n" % args[0]).encode())
sys.exit(1) # terminate with error
else: # parent (forked ok)
os.write(1, ("Parent: My pid=%d. Child's pid=%d\n" % (pid, rc)).encode())
childPidCode = os.wait()
os.write(1, ("Parent: Child %d terminated with exit code %d\n" %
def run_comand(comand):
rc = os.fork() #create child process
args = comand.copy()
if '&' in args:
args.remove('&')
if 'exit' in args: # exit rpogram
sys.exit(0)
if rc < 0: #frok failed
os.write(2, ("fork failed, returning %d\n" % rc).encode())
sys.exit(1)
elif rc == 0: # child
if '>' in args:
os.close(1) # redirect child's stdout
os.open(args[-1], os.O_CREAT | os.O_WRONLY)
os.set_inheritable(1, True)
argg = args[0:args.index(">")]
exe(argg)
if '<' in args:
os.close(0) # redirect child's stdin
os.open(args[-1], os.O_RDONLY)
os.set_inheritable(0, True)
argg = args[0:args.index("<")]
exe(argg)
if '|' in args: #pipe command
args = ' '.join([str(elem) for elem in args])
pipe = args.split("|")
prog1 = pipe[0].split()
prog2 = pipe[1].split()
pr, pw = os.pipe() #file descriptors for reading and writing
for f in (pr, pw):
os.set_inheritable(f, True)
pipeFork = os.fork()
if pipeFork < 0: #fork failed
os.write(2, ("fork failed").encode())
sys.exit(1)
elif pipeFork == 0: #child process which will write to pipe
os.close(1) #close fd 1 and rederict it
os.dup(pw) #attach fd1 to pipe input fd
os.set_inheritable(1, True) #set fd1 inheritable
for fd in (pr, pw):
os.close(fd)
exe(prog1)
else: #parent
os.close(0)
os.dup(pr)
os.set_inheritable(0, True)
for fd in (pw, pr):
os.close(fd)
exe(prog2)
else:
exe(args)
else: # parent (forked ok)
if not '&' in comand:
os.write(1, ("Parent: My pid=%d. Child's pid=%d\n" %
(pid, rc)).encode())
childPidCode = os.wait(
) #wait and get child pid and return code
os.write(1,
("Parent: Child %d terminated with exit code %d\n" %
childPidCode).encode())
def pipe(command_list):
index = command_list.index("|")
command_left = command_list[:index]
command_right = command_list[index + 1:]
print("command_left: " + str(command_left))
print("command_right: " + str(command_right))
pid = os.getpid() # get and remember pid
pr, pw = os.pipe()
for f in (pr, pw):
os.set_inheritable(f, True)
print("pipe fds: pr=%d, pw=%d" % (pr, pw))
print("About to fork (pid=%d)" % pid)
rc = os.fork()
if rc < 0:
print("fork failed, returning %d\n" % rc, file=sys.stderr)
sys.exit(1)
elif rc == 0: # child - will write to pipe
# print("Child: My pid==%d. Parent's pid=%d" % (os.getpid(), pid), file=sys.stderr)
# args = ["cat", "hello"]
#
# os.close(1) # redirect child's stdout
# os.dup(pw)
# os.close(pw)
# # for fd in (pr, pw):
# # os.close(fd)
# execute(command_left)
saved = os.dup(1)
os.close(1)
# os.dup(pw)
os.dup2(pw, sys.stdout.fileno())
os.close(pw)
os.close(pr)
# execute command
if len(command_left) > 0:
execute(command_left)
# restore
os.dup2(saved, 1)
os.close(saved)
else: # parent (forked ok)
# print("Parent: My pid==%d. Child's pid=%d" % (os.getpid(), rc), file=sys.stderr)
# os.close(0)
# os.dup(pr)
# os.close(pr)
# # for fd in (pw, pr):
# # os.close(fd)
# execute(command_right)
saved = os.dup(0)
os.close(0)
# os.dup(pr)
os.dup2(pr, sys.stdin.fileno())
os.close(pr)
os.close(pw)
# execute command
if len(command_right) > 0:
execute(command_right)
# restore
sys.stdout.flush()
sys.stdin.flush()
os.dup2(saved, 0)
os.close(saved)
child_pid, exit_code = os.wait()
return len(command_list)
def _inner_create_streams(self, path, read, write, append, must_create,
must_not_create, synchronized, inheritable,
fileno, handle, permissions):
# Note : opening broken links works if we're in "w" mode, and raises error in "r" mode,
# like for normal unexisting files.
if handle is not None:
assert fileno is None
self._fileno = self._handle = handle
elif fileno is not None:
assert handle is None
self._fileno = self._handle = fileno
else: # we open the file with low level posix IO - the unix "open()" function
if isinstance(path, unicode):
strname = path.encode(
sys.getfilesystemencoding()
) # let's take no risks - and do not use locale.getpreferredencoding(
# ) here
else:
strname = path
flags = 0 # Note that unix.O_LARGEFILE is actually irrelevant
if synchronized:
flags |= unix.O_SYNC
if read and write:
flags |= unix.O_RDWR
elif write:
flags |= unix.O_WRONLY
else:
flags |= unix.O_RDONLY
if append:
flags |= unix.O_APPEND
if must_not_create:
pass # it's the default case for open() function
elif must_create:
flags |= unix.O_CREAT | unix.O_EXCL
else:
flags |= unix.O_CREAT # by default - we create the file iff it doesn't exists
# print("Creating unix stream with context", locals())
self._fileno = self._handle = unix.open(strname, flags,
permissions)
# on unix we must prevent the opening of directories, but not named fifos or other special files
stats = unix.fstat(self._fileno).st_mode
if stat.S_ISDIR(stats):
raise IOError(errno.EISDIR, "RSFile can't open directories",
self.name)
# we don't use O_CLOEXEC flag (available since Linux 2.6.23) for compatibility and safety
if hasattr(os, "set_inheritable"):
os.set_inheritable(
self._fileno,
inheritable) # for safety we call it in any case
else:
# before PEP0446, newly created file descriptors were inheritable by default
if not inheritable:
old_flags = unix.fcntl(self._fileno, unix.F_GETFD, 0)
if not (old_flags & unix.FD_CLOEXEC):
unix.fcntl(self._fileno, unix.F_SETFD,
old_flags | unix.FD_CLOEXEC)
# WHATEVER the origin of the stream, we initialize these fields:
self._lock_registry_inode = self.unique_id(
) # enforces caching of unique_id
self._lock_registry_descriptor = self._fileno
print('pid', os.getpid())
if not pipe: # these are the checks for 'special' commands
args = my_read_line()
if args[0] == 'exit':
os.write(1, 'Goodbye!\n'.encode())
sys.exit(0)
elif args[0] == 'cd':
os.chdir(args[1])
continue
elif args[-1] == '&':
args = args[:-1]
amper = True
elif '|' in args and not pipe:
pr, pw = os.pipe()
for fd in (pr, pw):
os.set_inheritable(fd, True)
rc = os.fork() # fork here to the child process with gathered information
if rc < 0: # if the fork failed notify the user
os.write(2, 'Fork failed!\n'.encode())
sys.exit(0)
elif rc == 0: # this is the child process
if pipe or '|' in args: # check if we are a reading writing or both pipe command
if pipe: # set appropriate fds for reading and writing to a pipe
#print(os.read(pr, 10000))
print(args)
print('in last command')
os.close(0)
dup_r = os.dup(pr)
os.close(pr)
def main(argv):
args = parseArguments(argv[1:])
# with open(args["a"],"rt") as a_fd:
# a_fdno = a_fd.fileno()
# header_a = a_fd.readline().rstrip().split("\t")
# with open(args["b"],"rt") as b_fd:
# b_fdno = b_fd.fileno()
# header_b = b_fd.readline().rstrip().split("\t")
if args["a"] == "-":
llfd_a = sys.stdin.fileno()
else:
llfd_a = os.open(args["a"], os.O_RDONLY)
if args["b"] == "-":
llfd_b = sys.stdin.fileno()
else:
llfd_b = os.open(args["b"], os.O_RDONLY)
if llfd_a == sys.stdin.fileno() and llfd_b == sys.stdin.fileno():
raise ValueError("-a and -b cannot both read from standard in!")
# Use unbuffered IO to wrap file descriptors
a_fh = os.fdopen(llfd_a, 'rb', 0)
b_fh = os.fdopen(llfd_b, 'rb', 0)
def_enc = sys.getdefaultencoding()
header_a = a_fh.readline().decode(def_enc)
header_b = b_fh.readline().decode(def_enc)
header_a = header_a.rstrip().split("\t")
header_b = header_b.rstrip().split("\t")
# acols, bcols: list of 2-tuples. First value of tuple specifies
# column name in input, second value of tupls specifies column
# name in output (first != second => column is renamed)
acols = args["acols"]
bcols = args["bcols"]
# Check if correct column names specified
missing_a = [x[0] for x in acols if x[0] not in header_a]
missing_b = [x[0] for x in bcols if x[0] not in header_b]
if missing_a or missing_b:
raise ValueError('non-existant column names ' +
','.join(missing_a + missing_b) + ' specified!' +
'\n valid A: {}, B: {}'.format(header_a, header_b))
# pdb.set_trace()
# If no columns are specified: add first columns as key (by) colums
# The columns are appended as 2-tuples: the column name of the input
# and the desired column name of the output
if len(acols) == 0: acols.append((header_a[0], header_a[0]))
if len(bcols) == 0: bcols.append((header_b[0], header_b[0]))
# Add all input columns which are not specified with equal old and new
# column name if --all-a-cols or --all-b-cols, respectively, is set
explicit_old_colnames_a = [oldcn for oldcn, newcn in acols]
explicit_old_colnames_b = [oldcn for oldcn, newcn in bcols]
if args["all_a_cols"]:
acols.extend([(col, col) for col in header_a
if col not in explicit_old_colnames_a])
if args["all_b_cols"]:
bcols.extend([(col, col) for col in header_b
if col not in explicit_old_colnames_b])
# Column indices of columns selected for output
a_cols_i = [header_a.index(x[0]) for x in acols]
b_cols_i = [header_b.index(x[0]) for x in bcols]
a_by = acols[0] # Key column name of table 1 (2-tuple)
b_by = bcols[0] # Key column name of table 2 (2-tuple)
a_by_i = a_cols_i[0] # Key column index of table 1 (0-based)
b_by_i = b_cols_i[0] # Key column index of table 2 (0-based)
# Add pre-/suffixes to all column names from table 1,
# except key (a_by) column.
a_cols_print = [cn_new for cn_old, cn_new in acols if cn_old != a_by[0]]
a_cols_print = [
args["prefix_a"] + cn + args["suffix_a"] for cn in a_cols_print
]
# Add key column, but without pre-/suffix
a_cols_print = [a_by[1]] + a_cols_print
# Add pre-/suffixes to cols from table 2, except key column (omitted)
b_cols_print = [cn_new for cn_old, cn_new in bcols[1:]]
b_cols_print = [
args["prefix_b"] + cn + args["suffix_b"] for cn in b_cols_print
]
# Add outer join arguments
joinargs = []
joinargs.extend(["-1", str(a_by_i + 1)])
joinargs.extend(["-2", str(b_by_i + 1)])
if "a" in args["all"]: joinargs.extend(["-a", "1"])
if "b" in args["all"]: joinargs.extend(["-a", "2"])
#if args["all"] : # Any join
# Old versions of join want this, can't take '-o auto'
fieldspec = ",".join(["1.{}".format(i + 1) for i in a_cols_i] +
["2.{}".format(i + 1) for i in b_cols_i[1:]])
joinargs.extend(["-e", args["empty"]])
joinargs.extend(["-o", fieldspec])
# #namedpipe_a = os.path.join(tmpf.gettempdir(),rnd_string(40))
# namedpipe_a = os.path.join(rnd_string(40))
# atexit.register(os.unlink, namedpipe_a)
# #namedpipe_b = os.path.join(tmpf.gettempdir(),rnd_string(40))
# namedpipe_b = os.path.join(rnd_string(40))
# atexit.register(os.unlink, namedpipe_b)
#
# os.mkfifo(namedpipe_a)
# os.mkfifo(namedpipe_b)
# sp.Popen("{awk} <&{fd} | sort > {fifo}".format(
# awk=awk_select_cols(a_cols_i),
# #tab=args['a'],
# fd = llfd_a,
# fifo=namedpipe_a),
# shell=True, close_fds=False)
# sp.Popen("{awk} <&{fd} | sort > {fifo}".format(
# awk=awk_select_cols(b_cols_i),
# #tab=args['b'],
# fd = llfd_b,
# fifo=namedpipe_b),
# shell=True, close_fds=False)
# cmd = ("join {args} -t '\t' {fifo1} {fifo2}".format(
# args = " ".join(joinargs),
# fifo1=namedpipe_a, fifo2=namedpipe_b))
os.set_inheritable(llfd_a, True)
os.set_inheritable(llfd_b, True)
sort_cmd_a = "| LC_ALL=C sort -k{by} -t$'\\t'".format(by=a_by_i+1) \
if not args["a_sorted"] else ""
sort_cmd_b = "| LC_ALL=C sort -k{by} -t$'\\t'".format(by=b_by_i+1) \
if not args["b_sorted"] else ""
cmd_taba = "{awk} <&{fd} {sort}".format(
awk="cat",
#awk = awk_select_cols(a_cols_i),
fd=llfd_a,
sort=sort_cmd_a)
#sort = "")
cmd_tabb = "{awk} <&{fd} {sort}".format(
awk="cat",
#awk = awk_select_cols(b_cols_i),
fd=llfd_b,
sort=sort_cmd_b)
#sort = "")
cmd = ("join {args} -t$'\\t' <({input_a}) <({input_b})").format(
input_a=cmd_taba, input_b=cmd_tabb, args=" ".join(joinargs))
print("\t".join(a_cols_print + b_cols_print))
sys.stdout.flush()
# Handle SIGPIPE properly, e.g. when piped to head
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Execute join
sp.check_call(cmd, close_fds=False, shell=True, executable="/bin/bash")
sys.stdout.flush()
sys.stderr.flush()
return 0
file_out = re.split(' ', args[1])
file_out = file_out[1]
args = re.split(' ', args[0])
args = args[:-1]
os.close(1)
elif l_direct == True: # input redirection
file_in = re.split(' ', args[1])
file_in = file_in[1]
args = re.split(' ', args[0])
args = args[:-1]
args.append(file_in)
else: # no redirection
args = re.split(' ', args[0])
if file_out != '': # set file to be written to as output
sys.stdout = open(file_out, 'w')
fd = sys.stdout.fileno()
os.set_inheritable(fd, True)
for dir in re.split(":",
os.environ['PATH']): # try each directory in the path
program = "%s/%s" % (dir, args[0])
try:
os.execve(program, args, os.environ) # try to exec program
except FileNotFoundError: # ...expected
pass # ...fail quietly
sys.exit(1) # terminate with error
else: # parent (forked ok)
childPidCode = os.wait()
def run(self):
_logger.info('Start running the thread!')
self.wait_event.clear()
# Set up a pipe for the server to report when it has started.
pipe_in, pipe_out = os.pipe()
# TODO(crbug.com/941669): Remove if condition after python3 migration.
if hasattr(os, 'set_inheritable'):
os.set_inheritable(pipe_out, True)
try:
self._GenerateCommandLineArguments(pipe_out)
# TODO(crbug.com/941669): When this script is ported to Python 3, replace
# 'vpython3' below with sys.executable.
command = [
'vpython3',
os.path.join(_DIR_SOURCE_ROOT, 'net', 'tools', 'testserver',
'testserver.py')
] + self.command_line
_logger.info('Running: %s', command)
# Disable PYTHONUNBUFFERED because it has a bad interaction with the
# testserver. Remove once this interaction is fixed.
unbuf = os.environ.pop('PYTHONUNBUFFERED', None)
# Pass _DIR_SOURCE_ROOT as the child's working directory so that relative
# paths in the arguments are resolved correctly. devnull can be replaced
# with subprocess.DEVNULL in Python 3.
with open(os.devnull, 'r+b') as devnull:
self.process = subprocess.Popen(
command,
preexec_fn=lambda: self._CloseUnnecessaryFDsForTestServerProcess(
pipe_out),
stdin=devnull,
# Preserve stdout and stderr from the test server.
stdout=None,
stderr=None,
cwd=_DIR_SOURCE_ROOT,
close_fds=False)
# Close pipe_out early. If self.process crashes, this will be visible
# in _WaitToStartAndGetPortFromTestServer's select loop.
os.close(pipe_out)
pipe_out = -1
if unbuf:
os.environ['PYTHONUNBUFFERED'] = unbuf
self.is_ready = self._WaitToStartAndGetPortFromTestServer(pipe_in)
if self.is_ready:
port_map = [(0, self.host_port)]
if self.host_ocsp_port:
port_map.extend([(0, self.host_ocsp_port)])
self.port_forwarder.Map(port_map)
self.forwarder_device_port = \
self.port_forwarder.GetDevicePortForHostPort(self.host_port)
if self.host_ocsp_port:
self.forwarder_ocsp_device_port = \
self.port_forwarder.GetDevicePortForHostPort(self.host_ocsp_port)
# Check whether the forwarder is ready on the device.
self.is_ready = self.forwarder_device_port and \
self.port_forwarder.WaitDevicePortReady(self.forwarder_device_port)
# Wake up the request handler thread.
self.ready_event.set()
# Keep thread running until Stop() gets called.
self.stop_event.wait()
if self.process.poll() is None:
self.process.kill()
# Wait for process to actually terminate.
# (crbug.com/946475)
self.process.wait()
self.port_forwarder.Unmap(self.forwarder_device_port)
self.process = None
self.is_ready = False
finally:
if pipe_in >= 0:
os.close(pipe_in)
if pipe_out >= 0:
os.close(pipe_out)
_logger.info('Test-server has died.')
self.wait_event.set()
def pipe(args):
# Get pid
pid = os.getpid()
# Check for pipe
pipe = args.index("|")
# Read input / Write output
pr, pw = os.pipe()
for f in (pr, pw):
os.set_inheritable(f, True)
rc = os.fork()
if rc < 0:
print("fork failed, returning %d\n" % rc, file=sys.stderr)
sys.exit(1)
elif rc == 0: # write to pipe from child
args = args[:pipe]
os.close(1)
fd = os.dup(pw) # duplicate file descriptor output
os.set_inheritable(fd, True)
for fd in (pr, pw):
os.close(fd)
if os.path.isfile(args[0]):
try:
os.execve(args[0], args, os.environ) # try to execute program
except FileNotFoundError:
pass
else:
for dir in re.split(
":",
os.environ['PATH']): # Check for environment variables
program = "%s/%s" % (dir, args[0])
try:
os.execve(program, args,
os.environ) # try to execute program
except FileNotFoundError:
pass
os.write(2, ("%s: command not found\n" % args[0]).encode())
sys.exit(1)
else:
args = args[pipe + 1:]
os.close(0)
fd = os.dup(pr) # duplicate file descriptor for output
os.set_inheritable(fd, True)
for fd in (pw, pr):
os.close(fd) # close file descriptor
if os.path.isfile(args[0]):
try:
os.execve(args[0], args, os.environ) # execute program
except FileNotFoundError:
pass
else:
for dir in re.split(
":",
os.environ['PATH']): # Check for environment variables
program = "%s/%s" % (dir, args[0])
try:
os.execve(program, args, os.environ) # execute program
except FileNotFoundError:
pass
os.write(2,
("%s: command not found\n" %
args[0]).encode()) # return error if command was not found
sys.exit(1)
def commit(self):
def finish_dpkg(term, lock):
""" helper that is run when dpkg finishes """
status = term.get_child_exit_status()
self.exitstatus = posix.WEXITSTATUS(status)
#print "dpkg finished %s %s" % (pid,status)
#print "exit status: %s" % self.exitstatus
#print "was signaled %s" % posix.WIFSIGNALED(status)
try:
lock.release()
except:
logging.exception("lock.release failed")
# get a lock
lock = threading.Lock()
lock.acquire()
# ui
if self.install:
self.status.set_markup("<i>"+_("Installing '%s'...") % \
os.path.basename(self.debfile)+"</i>")
else:
self.status.set_markup("<i>"+_("Removing '%s'...") % \
os.path.basename(self.debfile)+"</i>")
self.progress.pulse()
self.progress.set_text("")
# prepare reading the pipe
(readfd, writefd) = os.pipe()
# File descriptors are not inheritable by child processes by
# default in Python 3.4. We need the dpkg child to inherit the
# write file descriptor.
if hasattr(os, 'set_inheritable'):
os.set_inheritable(writefd, True)
fcntl.fcntl(readfd, fcntl.F_SETFL, os.O_NONBLOCK)
#print("fds (%i,%i)" % (readfd,writefd))
# the command
argv = ["/usr/bin/dpkg", "--auto-deconfigure"]
# ubuntu supports VTE_PTY_KEEP_FD, see
# https://bugzilla.gnome.org/320128 for the upstream bug
if UBUNTU:
argv += ["--status-fd", "%s" % writefd]
if self.install:
argv += ["-i", self.debfile]
else:
argv += ["-r", self.debfile]
env = [
"VTE_PTY_KEEP_FD=%s" % writefd, "DEBIAN_FRONTEND=gnome",
"APT_LISTCHANGES_FRONTEND=gtk"
]
#print argv
#print env
#print self.term
# prepare for the fork
self.term.connect("child-exited", finish_dpkg, lock)
(res, pid) = self.term.fork_command_full(
Vte.PtyFlags.DEFAULT,
"/",
argv,
env,
GLib.SpawnFlags.LEAVE_DESCRIPTORS_OPEN,
# FIXME: add setup_func that closes all fds excpet for writefd
None, #setup_func
None, #setup_data
)
#print "fork_command_full: ", res, pid
raw_read = b""
while lock.locked():
while True:
try:
raw_read += os.read(readfd, 1)
except OSError as e:
# resource temporarly unavailable is ignored
from errno import EAGAIN
if e.errno != EAGAIN:
logging.warn(e.errstr)
break
self.time_last_update = time.time()
if raw_read[-1] == ord("\n"):
read = raw_read.decode("utf-8")
statusl = read.split(":")
if len(statusl) < 3:
logging.warn("got garbage from dpkg: '%s'" % read)
raw_read = ""
break
status = statusl[2].strip()
#print status
if status == "error" or status == "conffile-prompt":
self.term_expander.set_expanded(True)
raw_read = b""
self.progress.pulse()
while Gtk.events_pending():
Gtk.main_iteration()
time.sleep(0.2)
# if the terminal has not reacted for some time, do something
if (not self.term_expander.get_expanded()
and (self.time_last_update + GDEBI_TERMINAL_TIMEOUT) <
time.time()):
self.term_expander.set_expanded(True)
self.progress.set_fraction(1.0)
def exec_(args=[], in_fd=None, out_fd=None, parent_wait=True):
'''
Forks a child process and execs args.
args: list of agruments to be passed to os.execve()
in_fd: integer of new input file descriptor
out_fd: integer of new output file descriptor
parent_wait: if True, then parent waits for child process to terminate
'''
os.write(1, ("Command read...About to fork (pid:%d)\n" % pid).encode())
rc = os.fork()
if rc < 0:
os.write(2, ("fork failed, returning %d\n" % rc).encode())
sys.exit(1)
elif rc == 0: # child
child_pid = os.getpid()
os.write(1, ("Child: My pid=%d. Parent's pid=%d\n" %
(child_pid, pid)).encode())
# change stdin/out fds if given
if in_fd:
os.write(2, ("Child: pid=%d given in_fd: %d\n" %
(child_pid, in_fd)).encode())
os.close(0)
fd = os.dup(in_fd)
os.set_inheritable(fd, True)
os.close(in_fd)
os.write(2, ("Child: pid=%d fd 0 -> %d\n" %
(child_pid, in_fd)).encode())
# os.write(2, ("Child: pid=%d using fd=%d for reading\n" % (child_pid, fd)).encode())
if out_fd:
os.write(2, ("Child: pid=%d given out_fd: %d\n" %
(child_pid, out_fd)).encode())
os.close(1)
fd = os.dup(out_fd)
os.set_inheritable(fd, True)
os.close(out_fd)
os.write(2, ("Child: pid=%d fd 1 -> %d\n" %
(child_pid, out_fd)).encode())
# os.write(2, ("Child: pid=%d using fd=%d for writing\n" % (child_pid, fd)).encode())
os.write(
2, ("Child: pid=%d stdin fd: %d stdout fd: %d\n" %
(child_pid, sys.stdin.fileno(), sys.stdout.fileno())).encode())
for dir in re.split(
":", os.environ['PATH']): # try each directory in the path
program = "%s/%s" % (dir, args[0])
# os.write(1, ("Child: pid=%d ...trying to exec %s\n" % (child_pid, program)).encode())
try:
os.execve(program, args, os.environ) # try to exec program
except FileNotFoundError: # ...expected
pass # ...fail quietly
os.write(2, ("Child: pid=%d Could not exec %s\n" %
(child_pid, args[0])).encode())
sys.exit(1) # terminate with error
else: # parent (forked ok)
os.write(1,
("Parent: My pid=%d. Child's pid=%d\n" % (pid, rc)).encode())
if parent_wait:
childPidCode = os.wait()
os.write(1, ("Parent: Child %d terminated with exit code %d\n" %
childPidCode).encode())
class JobServer(object):
# Whether the job server has been initialized
_initialized = False
# Flag designating whether the `make` program supports the GNU Make
# jobserver interface
_gnu_make = None
# Initialize variables
_load_ok = True
_mem_ok = True
_internal_jobs = []
_max_load = 0
_max_jobs = 0
_job_pipe = os.pipe()
# Setting fd inheritance is required in Python > 3.4
# This is set by default in Python 2.7
# For more info see: https://docs.python.org/3.4/library/os.html#fd-inheritance
if hasattr(os, 'set_inheritable'):
for fd in _job_pipe:
os.set_inheritable(fd, True)
if not os.get_inheritable(fd):
log(
clr('@{yf}@!Warning: jobserver file descriptors are not inheritable.@|'
))
@classmethod
def _set_max_jobs(cls, max_jobs):
"""Set the maximum number of jobs to be used with the jobserver.
This will wait for all active jobs to be completed, then re-initialize the job pipe.
"""
# Read all possible tokens from the pipe
try:
os.read(cls._job_pipe[0], cls._max_jobs)
except (BlockingIOError, InterruptedError):
pass
# Update max jobs
cls._max_jobs = max_jobs
# Initialize the pipe with max_jobs tokens
for i in range(cls._max_jobs):
os.write(cls._job_pipe[1], b'+')
@classmethod
def _set_max_mem(cls, max_mem):
"""
Set the maximum memory to keep instantiating jobs.
:param max_mem: String describing the maximum memory that can be used
on the system. It can either describe memory percentage or absolute
amount. Use 'P%' for percentage or 'N' for absolute value in bytes,
'Nk' for kilobytes, 'Nm' for megabytes, and 'Ng' for gigabytes.
:type max_mem: str
"""
if max_mem is None:
cls._max_mem = None
return
elif type(max_mem) is float or type(max_mem) is int:
mem_percent = max_mem
elif type(max_mem) is str:
m_percent = re.search(r'([0-9]+)\%', max_mem)
m_abs = re.search(r'([0-9]+)([kKmMgG]{0,1})', max_mem)
if m_percent is None and m_abs is None:
cls._max_mem = None
return
if m_percent:
mem_percent = m_abs.group(1)
elif m_abs:
val = float(m_abs.group(1))
mag_symbol = m_abs.group(2)
_, total_mem = memory_usage()
if mag_symbol == '':
mag = 1.0
elif mag_symbol.lower() == 'k':
mag = 1024.0
elif mag_symbol.lower() == 'm':
mag = pow(1024.0, 2)
elif mag_symbol.lower() == 'g':
mag = pow(1024.0, 3)
mem_percent = 100.0 * val * mag / total_mem
cls._max_mem = max(0.0, min(100.0, float(mem_percent)))
@classmethod
def _check_load(cls):
if cls._max_load is not None:
try:
load = os.getloadavg()
if load[0] < cls._max_load:
cls._load_ok = True
else:
cls._load_ok = False
except NotImplementedError:
cls._load_ok = True
return cls._load_ok
@classmethod
def _check_mem(cls):
if cls._max_mem is not None:
mem_used, mem_total = memory_usage()
mem_percent_used = 100.0 * float(mem_used) / float(mem_total)
if mem_percent_used > cls._max_mem:
cls._mem_ok = False
else:
cls._mem_ok = True
return cls._mem_ok
@classmethod
def _check_conditions(cls):
return (cls._check_load()
and cls._check_mem()) or cls._running_jobs() == 0
@classmethod
def _acquire(cls):
"""
Obtain a job server token. Be sure to call _release() to avoid
deadlocks.
"""
try:
# read a token from the job pipe
token = os.read(cls._job_pipe[0], 1)
return token
except (BlockingIOError, InterruptedError):
pass
return None
@classmethod
def _release(cls):
"""
Write a token to the job pipe.
"""
os.write(cls._job_pipe[1], b'+')
@classmethod
def _running_jobs(cls):
try:
buf = array.array('i', [0])
if fcntl.ioctl(cls._job_pipe[0], FIONREAD, buf) == 0:
return cls._max_jobs - buf[0]
except (NotImplementedError, OSError):
pass
return cls._max_jobs
uIn2 = uIn.split()
if len(uIn2) < 3:
print("Invalid command")
if len(uIn2) == 3:
rc0 = os.fork()
if rc0 == 0: #input redirection file with name of another file
args = [uIn2[0]] #command to execute
os.close(0)
sys.stdin = open(uIn2[2],
"r") #open file containing name of another file
rFile = input()
args.append(rFile)
fd = sys.stdin.fileno()
os.set_inheritable(fd, True)
os.write(2, ("Child: opened fd=%d for reading\n" % fd).encode())
for dir in re.split(":", os.environ['PATH']):
program = "%s/%s" % (dir, args[0])
try:
os.execve(program, args,
os.environ) #try and execute program
except FileNotFoundError:
pass
os.write(2, ("Error: Could not exec%s\n" % args[0]).encode())
sys.exit(1)
else: # parent (forked ok)
childPidCode = os.wait() #wait for child to execute
