def _socket(addr, port, family): """ Create a socket, bind and listen. :param addr: The address to use. :type addr: :any:`str` :param port: The port to use. :type port: :any:`int` :param family: The type of socket to use. :type family: :any:`socket.AF_INET` or :any:`socket.AF_INET6`. :returns: A bound socket. :rtype: :any:`socket.socket` :raises: :any:`blackhole.exceptions.BlackholeRuntimeException` """ sock = socket.socket(family, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except (AttributeError, OSError): pass if family == socket.AF_INET6: sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) try: sock.bind((addr, port)) except OSError: msg = 'Cannot bind to {}:{}.'.format(addr, port) logger.fatal(msg) sock.close() raise BlackholeRuntimeException(msg) os.set_inheritable(sock.fileno(), True) sock.listen(1024) sock.setblocking(False) return sock
def makeSockets(cls): """Make socket(s) to communicate with remote process. Returns string to send to remote process """ if ( hasattr(socket, 'AF_UNIX') and hasattr(socket, 'socketpair') ): # convenient interface cls.sockfamily = socket.AF_UNIX sock, socket2 = socket.socketpair(cls.sockfamily, socket.SOCK_STREAM) # socket is closed on popen in Python 3.4+ without this (PEP 446) try: os.set_inheritable(socket2.fileno(), True) except AttributeError: pass sendtext = 'unix %i\n' % socket2.fileno() cls.socket2 = socket2 # prevent socket being destroyed waitaccept = False else: # otherwise mess around with internet sockets # * This is required for windows, which doesn't have AF_UNIX # * It is required where socketpair is not supported cls.sockfamily = socket.AF_INET sock = socket.socket(cls.sockfamily, socket.SOCK_STREAM) sock.bind( ('localhost', 0) ) interface, port = sock.getsockname() sock.listen(1) sendtext = 'internet %s %i\n' % (interface, port) waitaccept = True return (sock, sendtext.encode('ascii'), waitaccept)
def run_diff(fd1, fd2, end_nl_q1, end_nl_q2): cmd = ['diff', '-aU7', '/dev/fd/%d' % fd1, '/dev/fd/%d' % fd2] logger.debug('running %s', cmd) if hasattr(os, 'set_inheritable'): # new in Python 3.4 os.set_inheritable(fd1, True) os.set_inheritable(fd2, True) p = subprocess.Popen(cmd, shell=False, bufsize=1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, pass_fds=(fd1, fd2)) p.stdin.close() os.close(fd1) os.close(fd2) parser = DiffParser(p.stdout, end_nl_q1, end_nl_q2) t_read = Thread(target=parser.parse) t_read.daemon = True t_read.start() t_read.join() p.wait() logger.debug('done with diff, returncode %d, parsed %s', p.returncode, parser.success) if not parser.success and p.returncode not in (0, 1): raise subprocess.CalledProcessError(p.returncode, cmd, output=diff) if p.returncode == 0: return None return parser.diff
def _socket(addr, port, family): """ Create a socket, bind and listen. :param str addr: The address to use. :param int port: The port to use. :param family: The type of socket to use. :type family: :py:obj:`socket.AF_INET` or :py:obj:`socket.AF_INET6`. :returns: Bound socket. :rtype: :py:func:`socket.socket` :raises BlackholeRuntimeException: When a socket cannot be bound. """ sock = socket.socket(family, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except (AttributeError, OSError): pass if family == socket.AF_INET6: sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) try: sock.bind((addr, port)) except OSError: msg = "Cannot bind to {0}:{1}.".format(addr, port) logger.critical(msg) sock.close() raise BlackholeRuntimeException(msg) os.set_inheritable(sock.fileno(), True) sock.listen(1024) sock.setblocking(False) return sock
def adb_server(): """Context manager for an ADB server. This creates an ADB server and returns the port it's listening on. """ port = 5038 # Kill any existing server on this non-default port. subprocess.check_output(["adb", "-P", str(port), "kill-server"], stderr=subprocess.STDOUT) read_pipe, write_pipe = os.pipe() if sys.platform == "win32": import msvcrt write_handle = msvcrt.get_osfhandle(write_pipe) os.set_handle_inheritable(write_handle, True) reply_fd = str(write_handle) else: os.set_inheritable(write_pipe, True) reply_fd = str(write_pipe) proc = subprocess.Popen(["adb", "-L", "tcp:localhost:{}".format(port), "fork-server", "server", "--reply-fd", reply_fd], close_fds=False) try: os.close(write_pipe) greeting = os.read(read_pipe, 1024) assert greeting == b"OK\n", repr(greeting) yield port finally: proc.terminate() proc.wait()
def create_haproxy_pipe(): logger.debug("create_haproxy_pipe called") pipefd = os.pipe() os.set_inheritable(pipefd[0], True) os.set_inheritable(pipefd[1], True) logger.debug("create_haproxy_pipe done") return pipefd
def _mk_inheritable(fd): if version >= (3, 3): if sys.platform == 'win32': # Change to Windwos file handle import msvcrt fdh = msvcrt.get_osfhandle(fd) os.set_handle_inheritable(fdh, True) return fdh else: os.set_inheritable(fd, True) return fd elif sys.platform == 'win32': # TODO: find a hack?? # Not yet working import msvcrt import _subprocess curproc = _subprocess.GetCurrentProcess() fdh = msvcrt.get_osfhandle(fd) fdh = _subprocess.DuplicateHandle( curproc, fdh, curproc, 0, True, # set inheritable FLAG _subprocess.DUPLICATE_SAME_ACCESS) return fdh else: return fd
def start(process, options): if process.alive(): print('Already running as %s' % process.read_pid()) return create_app_symlinks(options) args, env = build_java_execution(options, True) makedirs(dirname(options.launcher_log)) log = open_append(options.launcher_log) makedirs(options.data_dir) os.chdir(options.data_dir) pid = os.fork() if pid > 0: process.write_pid(pid) print('Started as %s' % pid) return if hasattr(os, "set_inheritable"): # See https://docs.python.org/3/library/os.html#inheritance-of-file-descriptors # Since Python 3.4 os.set_inheritable(process.pid_file.fileno(), True) os.setsid() redirect_stdin_to_devnull() redirect_output(log) os.close(log) os.execvpe(args[0], args, env)
def __init__(self, lock, sockname, configuration, featureset): self.configuration = configuration self.featureset = featureset self.sockname = sockname self.bitbake_lock = lock self.readypipe, self.readypipein = os.pipe() # Create server control socket if os.path.exists(sockname): os.unlink(sockname) self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # AF_UNIX has path length issues so chdir here to workaround cwd = os.getcwd() logfile = os.path.join(cwd, "bitbake-cookerdaemon.log") try: os.chdir(os.path.dirname(sockname)) self.sock.bind(os.path.basename(sockname)) finally: os.chdir(cwd) self.sock.listen(1) os.set_inheritable(self.sock.fileno(), True) startdatetime = datetime.datetime.now() bb.daemonize.createDaemon(self._startServer, logfile) self.sock.close() self.bitbake_lock.close() ready = ConnectionReader(self.readypipe) r = ready.poll(30) if r: r = ready.get() if not r or r != "ready": ready.close() bb.error("Unable to start bitbake server") if os.path.exists(logfile): logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False lines = [] with open(logfile, "r") as f: for line in f: if started: lines.append(line) else: res = logstart_re.match(line.rstrip()) if res: ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) if ldatetime >= startdatetime: started = True lines.append(line) if lines: if len(lines) > 10: bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:]))) else: bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines))) raise SystemExit(1) ready.close() os.close(self.readypipein)
def send_fd_scm(self, fd=None, file_path=None): # In iotest.py, the qmp should always use unix socket. assert self._qmp.is_scm_available() if self._socket_scm_helper is None: raise QEMUMachineError("No path to socket_scm_helper set") if not os.path.exists(self._socket_scm_helper): raise QEMUMachineError("%s does not exist" % self._socket_scm_helper) # This did not exist before 3.4, but since then it is # mandatory for our purpose if hasattr(os, 'set_inheritable'): os.set_inheritable(self._qmp.get_sock_fd(), True) if fd is not None: os.set_inheritable(fd, True) fd_param = ["%s" % self._socket_scm_helper, "%d" % self._qmp.get_sock_fd()] if file_path is not None: assert fd is None fd_param.append(file_path) else: assert fd is not None fd_param.append(str(fd)) devnull = open(os.path.devnull, 'rb') proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=False) output = proc.communicate()[0] if output: LOG.debug(output) return proc.returncode
def __init__(self): self.read, self.write = os.pipe() if PY3: print('PY3') # emulate old os.pipe os.set_inheritable(self.read, True) os.set_inheritable(self.write, True)
def __init__(self): self.master, self.slave = os.openpty() self._old_winch = signal.signal(signal.SIGWINCH, self._size_changed) self._size_changed() try: os.set_inheritable(self.slave, True) except AttributeError: pass
def gpgauthenticate(self, file, signature, keyring="/etc/apt/trusted.gpg"): """ authenticated a file against a given signature, if no keyring is given use the apt default keyring """ status_pipe = os.pipe() logger_pipe = os.pipe() if sys.version_info >= (3, 4): os.set_inheritable(status_pipe[1], 1) os.set_inheritable(logger_pipe[1], 1) gpg = [ "gpg", "--status-fd", "%d" % status_pipe[1], "--logger-fd", "%d" % logger_pipe[1], "--no-options", "--homedir", self.tmpdir, "--no-default-keyring", "--ignore-time-conflict", "--keyring", keyring, "--verify", signature, file, ] def gpg_preexec(): os.close(status_pipe[0]) os.close(logger_pipe[0]) proc = subprocess.Popen( gpg, stderr=subprocess.PIPE, preexec_fn=gpg_preexec, close_fds=False, universal_newlines=True ) os.close(status_pipe[1]) os.close(logger_pipe[1]) status_handle = os.fdopen(status_pipe[0]) logger_handle = os.fdopen(logger_pipe[0]) try: gpgres = status_handle.read() ret = proc.wait() if ret != 0: # gnupg returned a problem (non-zero exit) print("gpg exited %d" % ret) print("Debug information: ") print(status_handle.read()) print(proc.stderr.read()) print(logger_handle.read()) return False if "VALIDSIG" in gpgres: return True print("invalid result from gpg:") print(gpgres) return False finally: status_handle.close() proc.stderr.close() logger_handle.close()
def _pipe_open(self): readend, writeend = os.pipe() if hasattr(os, "set_inheritable"): # Python 3 by default will close all fds in subprocesses. This will avoid that. os.set_inheritable(readend, True) os.set_inheritable(writeend, True) readend = os.fdopen(readend, "rb", 0) writeend = os.fdopen(writeend, "wb", 0) return readend, writeend
def _pipe_open(self): readend, writeend = os.pipe() if hasattr(os, "set_inheritable"): # https://www.python.org/dev/peps/pep-0446/ os.set_inheritable(readend, True) os.set_inheritable(writeend, True) readend = os.fdopen(readend, "rb", 0) writeend = os.fdopen(writeend, "wb", 0) return readend, writeend
def _run(self, *, host, port, worker_num=None, reloader_pid=None, debug=None): self._debug = debug or self._debug if self._debug and not self._log_request: self._log_request = self._debug sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((host, port)) os.set_inheritable(sock.fileno(), True) workers = set() terminating = False def stop(sig, frame): nonlocal terminating if reloader_pid and sig == signal.SIGHUP: print('Reload request received') elif not terminating: terminating = True print('Termination request received') for worker in workers: worker.terminate() signal.signal(signal.SIGINT, stop) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGHUP, stop) for _ in range(worker_num or 1): worker = multiprocessing.Process( target=self.serve, kwargs=dict(sock=sock, host=host, port=port, reloader_pid=reloader_pid)) worker.daemon = True worker.start() workers.add(worker) # prevent further operations on socket in parent sock.close() for worker in workers: worker.join() if worker.exitcode > 0: print('Worker exited with code {}'.format(worker.exitcode)) elif worker.exitcode < 0: try: signame = signames[-worker.exitcode] except KeyError: print( 'Worker crashed with unknown code {}!' .format(worker.exitcode)) else: print('Worker crashed on signal {}!'.format(signame))
def fork_exec(cmd, exec_env=None, logfile=None, pass_fds=None): """ Execute a command using fork/exec. This is needed for programs system executions that need path searching but cannot have a shell as their parent process, for example: glance-api. When glance-api starts it sets itself as the parent process for its own process group. Thus the pid that a Popen process would have is not the right pid to use for killing the process group. This patch gives the test env direct access to the actual pid. :param cmd: Command to execute as an array of arguments. :param exec_env: A dictionary representing the environment with which to run the command. :param logile: A path to a file which will hold the stdout/err of the child process. :param pass_fds: Sequence of file descriptors passed to the child. """ env = os.environ.copy() if exec_env is not None: for env_name, env_val in exec_env.items(): if callable(env_val): env[env_name] = env_val(env.get(env_name)) else: env[env_name] = env_val pid = os.fork() if pid == 0: if logfile: fds = [1, 2] with open(logfile, 'r+b') as fptr: for desc in fds: # close fds try: os.dup2(fptr.fileno(), desc) except OSError: pass if pass_fds and hasattr(os, 'set_inheritable'): # os.set_inheritable() is only available and needed # since Python 3.4. On Python 3.3 and older, file descriptors are # inheritable by default. for fd in pass_fds: os.set_inheritable(fd, True) args = shlex.split(cmd) os.execvpe(args[0], args, env) else: return pid
def __init__(self, transaction, begin=50, end=100): self.transaction = transaction self.status = "" self.progress = 0 self.progress_begin = begin self.progress_end = end self._child_exit = -1 self.last_activity = 0 self.child_pid = 0 self.status_parent_fd, self.status_child_fd = os.pipe() if hasattr(os, "set_inheritable"): os.set_inheritable(self.status_parent_fd, True) os.set_inheritable(self.status_child_fd, True) self.output = "" self._line_buffer = ""
def serve_multiple(server_settings, workers, stop_event=None): """ Starts multiple server processes simultaneously. Stops on interrupt and terminate signals, and drains connections when complete. :param server_settings: kw arguments to be passed to the serve function :param workers: number of workers to launch :param stop_event: if provided, is used as a stop signal :return: """ if server_settings.get('loop', None) is not None: if server_settings.get('debug', False): warnings.simplefilter('default') warnings.warn("Passing a loop will be deprecated in version 0.4.0" " https://github.com/channelcat/sanic/pull/335" " has more information.", DeprecationWarning) server_settings['reuse_port'] = True sock = socket() sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) sock.bind((server_settings['host'], server_settings['port'])) set_inheritable(sock.fileno(), True) server_settings['sock'] = sock server_settings['host'] = None server_settings['port'] = None if stop_event is None: stop_event = Event() signal_func(SIGINT, lambda s, f: stop_event.set()) signal_func(SIGTERM, lambda s, f: stop_event.set()) processes = [] for _ in range(workers): process = Process(target=serve, kwargs=server_settings) process.daemon = True process.start() processes.append(process) for process in processes: process.join() # the above processes will block this until they're stopped for process in processes: process.terminate() sock.close() asyncio.get_event_loop().stop()
def _do_sendall(loops, send, recv): for s in send, recv: os.set_inheritable(s.fileno(), True) pid = os.fork() if not pid: send.close() recvall(recv, None) recv.close() sys.exit() return 0 else: try: return _sendall(loops, send, BIG_DATA) finally: send.close() recv.close()
def temporary_script(lines : list): # The script close the original file descriptor upon execution. But if execution fails, TemporaryFile should close it. with tempfile.TemporaryFile() as file: fd = file.fileno() name = '/dev/fd/{}'.format(fd) writer = io.TextIOWrapper(file) # Close the original file descriptor, which has been duplicated by bash when this line is executed. for i in ['exec {}<&-'.format(fd)] + lines: print(i, file = writer) writer.flush() file.seek(0) # Otherwise, the shell would not be able to access this file descriptor. os.set_inheritable(fd, True) yield name
def add_fd(self, fd, fdset, opaque, opts=''): """ Pass a file descriptor to the VM """ options = ['fd=%d' % fd, 'set=%d' % fdset, 'opaque=%s' % opaque] if opts: options.append(opts) # This did not exist before 3.4, but since then it is # mandatory for our purpose if hasattr(os, 'set_inheritable'): os.set_inheritable(fd, True) self._args.append('-add-fd') self._args.append(','.join(options)) return self
def ExecingProcess_Pipe(): """ This is like multiprocessing.Pipe(duplex=True). It uses our own ExecingProcess_ConnectionWrapper. """ import socket s1, s2 = socket.socketpair() # We duplicate the fds because the socket objects will close the fds after they go out of scope. fd1 = os.dup(s1.fileno()) fd2 = os.dup(s2.fileno()) s1.close() s2.close() if hasattr(os, "set_inheritable"): # Python 3 by default will close all fds in subprocesses. This will avoid that. os.set_inheritable(fd1, True) os.set_inheritable(fd2, True) c1 = ExecingProcess_ConnectionWrapper(fd1) c2 = ExecingProcess_ConnectionWrapper(fd2) return c1, c2
def set_inheritable(fd, inheritable): # On py34+ we can use os.set_inheritable but < py34 we must polyfill # with fcntl and SetHandleInformation if hasattr(os, 'get_inheritable'): if os.get_inheritable(fd) != inheritable: os.set_inheritable(fd, inheritable) elif WIN: h = get_handle(fd) flags = winapi.HANDLE_FLAG_INHERIT if inheritable else 0 winapi.SetHandleInformation(h, winapi.HANDLE_FLAG_INHERIT, flags) else: flags = fcntl.fcntl(fd, fcntl.F_GETFD) if inheritable: new_flags = flags & ~fcntl.FD_CLOEXEC else: new_flags = flags | fcntl.FD_CLOEXEC if new_flags != flags: fcntl.fcntl(fd, fcntl.F_SETFD, new_flags)
def _mk_inheritable(fd): version = sys.version_info[:2] if version >= (3, 3): if sys.platform == 'win32': # Change to Windwos file handle import msvcrt fdh = msvcrt.get_osfhandle(fd) os.set_handle_inheritable(fdh, True) return fdh else: os.set_inheritable(fd, True) return fd elif sys.platform == 'win32': # TODO: find a hack?? # Not yet working import msvcrt fdh = msvcrt.get_osfhandle(fd) return fdh else: return fd
def create_socket(host, family=socket.AF_INET, type=socket.SOCK_STREAM, backlog=2048, blocking=True, inheritable=False): if family == socket.AF_UNIX and not host.startswith('unix:'): raise ValueError('Your host needs to have the unix:/path form') if host.startswith('unix:'): family = socket.AF_UNIX if host.startswith('fd://'): fd = int(host[5:]) sock = socket.fromfd(fd, family, type) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) else: sock = socket.socket(family, type) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if host.startswith('unix:'): filename = host[len('unix:'):] try: os.remove(filename) except OSError: pass sock.bind(filename) else: if ':' in host: host, port = host.rsplit(':', 1) port = int(port) else: host, port = '0.0.0.0', int(host) sock.bind((host, port)) sock.listen(backlog) if blocking: sock.setblocking(1) else: sock.setblocking(0) # Required since Python 3.4 to be able to share a socket with a child # process. if inheritable and hasattr(os, 'set_inheritable'): os.set_inheritable(sock.fileno(), True) return sock
def test_connection_not_allowed_from_different_processes(tmpdir, io_method): r, w = os.pipe() os.set_inheritable(r, True) os.set_inheritable(w, True) with Model.open(tmpdir) as db: pid = os.fork() if pid != 0: # Parent os.close(w) with os.fdopen(r, 'rb') as pipe: try: getattr(db, io_method)() # This should not raise BadUsageError except BadUsageError as exc: assert False, "Unexpected exception in parent %r" % exc except Exception as exc: # Possible expected behavior because we are not calling # every io_method with all the neded parameters pass finally: # Receive child result result, message = pickle.load(pipe) assert result, message os.waitpid(pid, 0) else: # Child os.close(r) with os.fdopen(w, 'wb') as pipe: try: getattr(db, io_method)() # This should not raise BadUsageError except BadUsageError: pickle.dump((True, "Expected behavior"), pipe) except Exception as exc: # This is not expected even when we are calling the # io_method with incorrect parameters because BadUsageError # MUST raise first given that is in the decorator. pickle.dump((False, "Unexpected exception in child %r" % exc), pipe) else: pickle.dump((False, "Child did not raise"), pipe) os._exit(0)
def process(self): proc_obj = None log_read_fobj = None try: logfd_read, logfd_write = os.pipe() log_read_fobj = os.fdopen(logfd_read, 'r') os.set_inheritable(logfd_write, True) cmd = self.cmd_line[:] cmd.extend(['--log-fd', str(logfd_write)]) proc_obj = subprocess.Popen(cmd, close_fds=False, stdin=subprocess.DEVNULL, shell=False) os.close(logfd_write) res = self.process_with_fd(proc_obj, log_read_fobj) return res except Exception as e: print("Backup failed!!!: %s" % e) self.job.terminate("Backup failed") raise finally: log_read_fobj.close() if proc_obj: proc_obj.kill()
def run_cgi(self, toOpen): #look for queries env = cgi_env(self, toOpen) self.wfile.flush() #prepare pipe r,w = os.pipe() os.set_inheritable(w, True) pid = os.fork() if pid == 0: #LOG print_log(self) args = [toOpen] try: os.dup2( w , 1) os.dup2( self.rfile.fileno() , 0) os.execve(toOpen , args, env) os._exit(0) except: os._exit(1) else: # Parent os.waitpid(pid, 0) os.close(w) content = "" buf = os.read(r, BLOCK_SIZE).decode("utf-8") while buf != "": content += buf buf = os.read(r, BLOCK_SIZE).decode("utf-8") os.close(r) length = str( get_content_length(content.encode("utf-8"))) #prepare HEADER cgi_header(self, content) self.wfile.write( bytes("Content-Length: "+length+"\n", "utf-8") ) self.wfile.write(content.encode("utf-8")) self.wfile.flush() return
def main(): if len(sys.argv) < 4: print('Usage: {} inputfile outputfile cmd ...'.format(sys.argv[0]), file=sys.stderr) sys.exit(1) inputfile, outputfile = sys.argv[1], sys.argv[2] cmd = sys.argv[3:] # create the input feeder (stdin, w) = os.pipe() inpid = os.fork() if inpid == 0: os.close(stdin) inputFeeder(w, inputfile) sys.exit(0) os.close(w) # create the output watcher (r, stdout) = os.pipe() outpid = os.fork() if outpid == 0: os.close(stdout) outputWatcher(r, outputfile) sys.exit(0) os.close(r) # create the stderr watcher (r, stderr) = os.pipe() errpid = os.fork() if errpid == 0: os.close(stderr) stderrWatcher(r) sys.exit(0) os.close(r) # launch the main command pid = os.fork() if pid == 0: os.dup2(stdin, 0) os.set_inheritable(0, True) os.close(stdin) os.dup2(stdout, 1) os.set_inheritable(1, True) os.close(stdout) os.dup2(stderr, 2) os.set_inheritable(2, True) os.close(stderr) os.execvp(cmd[0], cmd) # wait for a child to exit, then kill the rest (child, status) = os.wait() if child != inpid: os.kill(inpid, signal.SIGTERM) if child != outpid: os.kill(outpid, signal.SIGTERM) if child != errpid: os.kill(errpid, signal.SIGTERM) if child != pid: os.kill(pid, signal.SIGTERM)
def set_inheritable(self, inheritable): os.set_inheritable(self.fileno(), inheritable)
import os import signal import sys binpath = sys.argv[1] p_ic3_r, p_ic3_w = os.pipe() os.set_inheritable(p_ic3_r, True) id_ic3 = os.fork() if id_ic3 == 0: sys.stdin.close() os.dup2(p_ic3_r, 0) os.execl(binpath + 'vvt-verify', 'vvt-verify', '--backend=cons:' + binpath + 'z3 -smt2 -in', '--backend=lifting:' + binpath + 'z3 -smt2 -in', '--backend=domain:' + binpath + 'z3 -smt2 -in', '--backend=init:' + binpath + 'z3 -smt2 -in', '--backend=cons:' + binpath + 'mathsat') os.close(p_ic3_r) p_bmc_r, p_bmc_w = os.pipe() os.set_inheritable(p_bmc_r, True) id_bmc = os.fork() if id_bmc == 0: sys.stdin.close() os.dup2(p_bmc_r, 0) os.execl(binpath + 'vvt-bmc', 'vvt-bmc', '--solver=' + binpath + 'z3 -smt2 -in', '-i', '-d', '50') os.close(p_bmc_r) while True:
def __init__(self, lock, sockname, configuration, featureset): self.configuration = configuration self.featureset = featureset self.sockname = sockname self.bitbake_lock = lock self.readypipe, self.readypipein = os.pipe() # Create server control socket if os.path.exists(sockname): os.unlink(sockname) self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # AF_UNIX has path length issues so chdir here to workaround cwd = os.getcwd() logfile = os.path.join(cwd, "bitbake-cookerdaemon.log") try: os.chdir(os.path.dirname(sockname)) self.sock.bind(os.path.basename(sockname)) finally: os.chdir(cwd) self.sock.listen(1) os.set_inheritable(self.sock.fileno(), True) startdatetime = datetime.datetime.now() bb.daemonize.createDaemon(self._startServer, logfile) self.sock.close() self.bitbake_lock.close() ready = ConnectionReader(self.readypipe) r = ready.poll(30) if r: r = ready.get() if not r or r != "ready": ready.close() bb.error("Unable to start bitbake server") if os.path.exists(logfile): logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False lines = [] with open(logfile, "r") as f: for line in f: if started: lines.append(line) else: res = logstart_re.match(line.rstrip()) if res: ldatetime = datetime.datetime.strptime( res.group(2), self.start_log_datetime_format) if ldatetime >= startdatetime: started = True lines.append(line) if lines: if len(lines) > 10: bb.error( "Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:]))) else: bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines))) raise SystemExit(1) ready.close() os.close(self.readypipein)
def os_pipe(): r, w = os.pipe() os.set_inheritable(r, True) os.set_inheritable(w, True) return r, w
def _ApplyDiffOperation(self, op, op_name, patch_data, old_part_file, new_part_file): """Applies a SOURCE_BSDIFF, BROTLI_BSDIFF or PUFFDIFF operation. Args: op: the operation object op_name: name string for error reporting patch_data: the binary patch content old_part_file: the source partition file object new_part_file: the target partition file object Raises: PayloadError if something goes wrong. """ if not old_part_file: raise PayloadError( '%s: no source partition file provided for operation type (%d)' % (op_name, op.type)) block_size = self.block_size # Dump patch data to file. with tempfile.NamedTemporaryFile(delete=False) as patch_file: patch_file_name = patch_file.name patch_file.write(patch_data) if (hasattr(new_part_file, 'fileno') and ((not old_part_file) or hasattr(old_part_file, 'fileno'))): # Construct input and output extents argument for bspatch. in_extents_arg, _, _ = _ExtentsToBspatchArg( op.src_extents, block_size, '%s.src_extents' % op_name, data_length=op.src_length if op.src_length else self._BytesInExtents(op.src_extents, "%s.src_extents")) out_extents_arg, pad_off, pad_len = _ExtentsToBspatchArg( op.dst_extents, block_size, '%s.dst_extents' % op_name, data_length=op.dst_length if op.dst_length else self._BytesInExtents(op.dst_extents, "%s.dst_extents")) new_file_name = '/dev/fd/%d' % new_part_file.fileno() # Diff from source partition. old_file_name = '/dev/fd/%d' % old_part_file.fileno() # In python3, file descriptors(fd) are not passed to child processes by # default. To pass the fds to the child processes, we need to set the flag # 'inheritable' in the fds and make the subprocess calls with the argument # close_fds set to False. if sys.version_info.major >= 3: os.set_inheritable(new_part_file.fileno(), True) os.set_inheritable(old_part_file.fileno(), True) if op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.BROTLI_BSDIFF): # Invoke bspatch on partition file with extents args. bspatch_cmd = [self.bspatch_path, old_file_name, new_file_name, patch_file_name, in_extents_arg, out_extents_arg] subprocess.check_call(bspatch_cmd, close_fds=False) elif op.type == common.OpType.PUFFDIFF: # Invoke puffpatch on partition file with extents args. puffpatch_cmd = [self.puffpatch_path, "--operation=puffpatch", "--src_file=%s" % old_file_name, "--dst_file=%s" % new_file_name, "--patch_file=%s" % patch_file_name, "--src_extents=%s" % in_extents_arg, "--dst_extents=%s" % out_extents_arg] subprocess.check_call(puffpatch_cmd, close_fds=False) else: raise PayloadError("Unknown operation %s" % op.type) # Pad with zeros past the total output length. if pad_len: new_part_file.seek(pad_off) new_part_file.write(b'\0' * pad_len) else: # Gather input raw data and write to a temp file. input_part_file = old_part_file if old_part_file else new_part_file in_data = _ReadExtents(input_part_file, op.src_extents, block_size, max_length=op.src_length if op.src_length else self._BytesInExtents(op.src_extents, "%s.src_extents")) with tempfile.NamedTemporaryFile(delete=False) as in_file: in_file_name = in_file.name in_file.write(in_data) # Allocate temporary output file. with tempfile.NamedTemporaryFile(delete=False) as out_file: out_file_name = out_file.name if op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.BROTLI_BSDIFF): # Invoke bspatch. bspatch_cmd = [self.bspatch_path, in_file_name, out_file_name, patch_file_name] subprocess.check_call(bspatch_cmd) elif op.type == common.OpType.PUFFDIFF: # Invoke puffpatch. puffpatch_cmd = [self.puffpatch_path, "--operation=puffpatch", "--src_file=%s" % in_file_name, "--dst_file=%s" % out_file_name, "--patch_file=%s" % patch_file_name] subprocess.check_call(puffpatch_cmd) else: raise PayloadError("Unknown operation %s" % op.type) # Read output. with open(out_file_name, 'rb') as out_file: out_data = out_file.read() if len(out_data) != op.dst_length: raise PayloadError( '%s: actual patched data length (%d) not as expected (%d)' % (op_name, len(out_data), op.dst_length)) # Write output back to partition, with padding. unaligned_out_len = len(out_data) % block_size if unaligned_out_len: out_data += b'\0' * (block_size - unaligned_out_len) _WriteExtents(new_part_file, out_data, op.dst_extents, block_size, '%s.dst_extents' % op_name) # Delete input/output files. os.remove(in_file_name) os.remove(out_file_name) # Delete patch file. os.remove(patch_file_name)
def piping(args): # '|' for split command #args = args.split('|') ''''' left and right arguments. lArg retrieves data of the pipe's left side rArg retrieves data of the pipe's right side ''''' lArg = args[0:args.index("|")] rArg = args[len(lArg)+1:] pipeRead, pipeWrite = os.pipe() rc = os.fork() # Forking fails if we get a zero returned if rc < 0: os.write(2, ("Fork failed, returning %d\n" % rc).encode()) exit() elif rc == 0: # redirects child stdout (fd0) os.close(1) # duplicates the child's fds and assigns them to pipeWrite os.dup(pipeWrite) os.set_inheritable(1, True) for fdno in (pipeRead, pipeWrite): # closes all fds os.close(fdno) # command will get the left argument command(lArg) else: # redirects child stdin (fd0) os.close(0) os.dup(pipeRead) os.set_inheritable(0, True) for fdno in (pipeWite, pipeRead): # closes all fds os.close(fdno) if "|" in rArg: pipe(rArg) # command will get the right side command(rArg) if '<' in lArg: inRedir("in") if '>' in lArg: inRedir("out") # The right argument will be executed else: if '<' in rArg: outRedir("in") if '>' in rArg: outRedir("out")
def set_fd_inheritable(fd): os.set_inheritable(fd, True)
def main(): try: # os.system should be a subprocess instead os.system('cat ~/.bashrc | grep PS1 > ./ps1.txt') with open('./ps1.txt', 'r') as ps1: line = ps1.readline() split = re.split('\'', line) prompt = split[1] except FileNotFoundError: # Windows, probably prompt = 'shel$ ' while (True): cmd = input(prompt) if cmd == 'quit': sys.exit(0) firstArgs, afterArgs, paths = parse(cmd) #print(firstArgs, afterArgs, paths) if afterArgs is None: rc = os.fork() # !!! if rc == 0: # child for dir in paths: fullPath = '%s/%s' % (dir, firstArgs[0]) try: os.execve(fullPath, firstArgs, os.environ) except FileNotFoundError: pass else: # parent cpid = os.wait() else: # more arguments after redirect = afterArgs.pop(0) # grab and remove redirect if redirect == '|': pr, pw = os.pipe() print('pr=%d, pw=%d' % (pr, pw)) os.set_inheritable(pr, True) os.set_inheritable(pw, True) rc = os.fork() # !!! if rc == 0: # child print('Child: pr=%d, pw=%d' % (pr, pw)) stdout = sys.stdout # save stdout os.close(1) # close stdout os.set_inheritable(os.dup(pw), True) # first child's stdout os.close(pr) os.close(pw) for dir in paths: fullPath = '%s/%s' % (dir, firstArgs[0]) try: os.execve(fullPath, firstArgs, os.environ) except FileNotFoundError: pass else: # parent cpid = os.wait() firstArgs = afterArgs afterArgs = None rc1 = os.fork() # !!! if rc1 == 0: # child stdin = sys.stdin # save stdin os.close(0) # close stdin os.set_inheritable(os.dup(pr), True) os.close(pr) os.close(pw) for dir in paths: fullPath = '%s/%s' % (dir, firstArgs[0]) try: os.execve(fullPath, firstArgs, os.environ) except FileNotFoundError: pass else: cpid = os.wait() elif redirect == '<': rc = os.fork() # !!! if rc == 0: stdin = sys.stdin # save stdin os.close(0) # close stdin os.stdin = open(afterArgs[0], 'r') os.set_inheritable(0, True) for dir in paths: fullPath = '%s/%s' % (dir, firstArgs[0]) try: os.execve(fullPath, firstArgs, os.environ) except FileNotFoundError: pass else: # parent cpid = os.wait() else: # redirect == '>' rc = os.fork() # !!! if rc == 0: stdout = sys.stdout # save stdout os.close(1) # close stdout os.stdout = open(afterArgs[0], 'w') os.set_inheritable(1, True) for dir in paths: fullPath = '%s/%s' % (dir, firstArgs[0]) try: os.execve(fullPath, firstArgs, os.environ) except FileNotFoundError: pass else: cpid = os.wait()
def test_vhost_user_vga_virgl(self): """ :avocado: tags=arch:x86_64 :avocado: tags=device:vhost-user-vga """ kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + "console=ttyS0 rdinit=/bin/bash") # FIXME: should check presence of vhost-user-gpu, virgl, memfd etc if not kvm_available(self.arch, self.qemu_bin): self.cancel(KVM_NOT_AVAILABLE) vug = pick_default_vug_bin() if not vug: self.cancel("Could not find vhost-user-gpu") kernel_path = self.fetch_asset(self.KERNEL_URL) initrd_path = self.fetch_asset(self.INITRD_URL) # Create socketpair to connect proxy and remote processes qemu_sock, vug_sock = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) os.set_inheritable(qemu_sock.fileno(), True) os.set_inheritable(vug_sock.fileno(), True) self._vug_log_path = os.path.join(self.vm._test_dir, "vhost-user-gpu.log") self._vug_log_file = open(self._vug_log_path, "wb") print(self._vug_log_path) vugp = subprocess.Popen( [vug, "--virgl", "--fd=%d" % vug_sock.fileno()], stdin=subprocess.DEVNULL, stdout=self._vug_log_file, stderr=subprocess.STDOUT, shell=False, close_fds=False, ) self.vm.set_console() self.vm.add_args("-cpu", "host") self.vm.add_args("-m", "2G") self.vm.add_args("-object", "memory-backend-memfd,id=mem,size=2G") self.vm.add_args("-machine", "pc,memory-backend=mem,accel=kvm") self.vm.add_args("-chardev", "socket,id=vug,fd=%d" % qemu_sock.fileno()) self.vm.add_args("-device", "vhost-user-vga,chardev=vug") self.vm.add_args("-display", "egl-headless") self.vm.add_args( "-kernel", kernel_path, "-initrd", initrd_path, "-append", kernel_command_line, ) self.vm.launch() self.wait_for_console_pattern("as init process") exec_command_and_wait_for_pattern(self, "/usr/sbin/modprobe virtio_gpu", "") self.wait_for_console_pattern("features: +virgl -edid") self.vm.shutdown() qemu_sock.close() vugp.terminate() vugp.wait()
def run_simple(hostname, port, application, use_reloader=False, use_debugger=False, use_evalex=True, extra_files=None, reloader_interval=1, reloader_type='auto', threaded=False, processes=1, request_handler=None, static_files=None, passthrough_errors=False, ssl_context=None): """Start a WSGI application. Optional features include a reloader, multithreading and fork support. This function has a command-line interface too:: python -m werkzeug.serving --help .. versionadded:: 0.5 `static_files` was added to simplify serving of static files as well as `passthrough_errors`. .. versionadded:: 0.6 support for SSL was added. .. versionadded:: 0.8 Added support for automatically loading a SSL context from certificate file and private key. .. versionadded:: 0.9 Added command-line interface. .. versionadded:: 0.10 Improved the reloader and added support for changing the backend through the `reloader_type` parameter. See :ref:`reloader` for more information. :param hostname: The host for the application. eg: ``'localhost'`` :param port: The port for the server. eg: ``8080`` :param application: the WSGI application to execute :param use_reloader: should the server automatically restart the python process if modules were changed? :param use_debugger: should the werkzeug debugging system be used? :param use_evalex: should the exception evaluation feature be enabled? :param extra_files: a list of files the reloader should watch additionally to the modules. For example configuration files. :param reloader_interval: the interval for the reloader in seconds. :param reloader_type: the type of reloader to use. The default is auto detection. Valid values are ``'stat'`` and ``'watchdog'``. See :ref:`reloader` for more information. :param threaded: should the process handle each request in a separate thread? :param processes: if greater than 1 then handle each request in a new process up to this maximum number of concurrent processes. :param request_handler: optional parameter that can be used to replace the default one. You can use this to replace it with a different :class:`~BaseHTTPServer.BaseHTTPRequestHandler` subclass. :param static_files: a dict of paths for static files. This works exactly like :class:`SharedDataMiddleware`, it's actually just wrapping the application in that middleware before serving. :param passthrough_errors: set this to `True` to disable the error catching. This means that the server will die on errors but it can be useful to hook debuggers in (pdb etc.) :param ssl_context: an SSL context for the connection. Either an :class:`ssl.SSLContext`, a tuple in the form ``(cert_file, pkey_file)``, the string ``'adhoc'`` if the server should automatically create one, or ``None`` to disable SSL (which is the default). """ if use_debugger: from werkzeug.debug import DebuggedApplication application = DebuggedApplication(application, use_evalex) if static_files: from werkzeug.wsgi import SharedDataMiddleware application = SharedDataMiddleware(application, static_files) def inner(): try: fd = int(os.environ['WERKZEUG_SERVER_FD']) except (LookupError, ValueError): fd = None make_server(hostname, port, application, threaded, processes, request_handler, passthrough_errors, ssl_context, fd=fd).serve_forever() if use_reloader: # If we're not running already in the subprocess that is the # reloader we want to open up a socket early to make sure the # port is actually available. if os.environ.get('WERKZEUG_RUN_MAIN') != 'true': if port == 0 and not can_open_by_fd: raise ValueError('Cannot bind to a random port with enabled ' 'reloader if the Python interpreter does ' 'not support socket opening by fd.') # Create and destroy a socket so that any exceptions are # raised before we spawn a separate Python interpreter and # lose this ability. address_family = select_ip_version(hostname, port) s = socket.socket(address_family, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((hostname, port)) if hasattr(os, 'set_inheritable'): os.set_inheritable(s.fileno(), True) # If we can open the socket by file descriptor, then we can just # reuse this one and our socket will survive the restarts. if can_open_by_fd: os.environ['WERKZEUG_SERVER_FD'] = str(s.fileno()) s.listen(LISTEN_QUEUE) else: s.close() from ._reloader import run_with_reloader run_with_reloader(inner, extra_files, reloader_interval, reloader_type) else: inner()
def process_start(self): """Start soapy_power process""" if not self.process and self.params: # Create pipe used for communication with soapy_power process self.pipe_read_fd, self.pipe_write_fd = os.pipe() self.pipe_read = open(self.pipe_read_fd, 'rb') os.set_inheritable(self.pipe_write_fd, True) if sys.platform == 'win32': self.pipe_write_handle = subprocess.make_inheritable_handle( self.pipe_write_fd) # Prepare soapy_power cmdline parameters settings = QtCore.QSettings() cmdline = shlex.split(settings.value("executable", "soapy_power")) cmdline.extend([ "-f", "{}M:{}M".format(self.params["start_freq"], self.params["stop_freq"]), "-B", "{}k".format(self.params["bin_size"]), "-T", "{}".format(self.params["interval"]), "-d", "{}".format(self.params["device"]), "-r", "{}".format(self.params["sample_rate"]), "-p", "{}".format(self.params["ppm"]), "-F", "soapy_power_bin", "--output-fd", "{}".format( int(self.pipe_write_handle) if sys.platform == 'win32' else self.pipe_write_fd), ]) if self.lnb_lo != 0: cmdline.extend(["--lnb-lo", "{}".format(self.lnb_lo)]) if self.params["bandwidth"] > 0: cmdline.extend(["-w", "{}".format(self.params["bandwidth"])]) if self.params["gain"] >= 0: cmdline.extend(["-g", "{}".format(self.params["gain"])]) if self.params["crop"] > 0: cmdline.extend(["-k", "{}".format(self.params["crop"])]) if not self.params["single_shot"]: cmdline.append("-c") additional_params = settings.value("params", Info.additional_params) if additional_params: cmdline.extend(shlex.split(additional_params)) # Start soapy_power process and close write part of pipe if sys.platform == 'win32': creationflags = subprocess.CREATE_NEW_PROCESS_GROUP else: creationflags = 0 self.process = subprocess.Popen(cmdline, close_fds=False, universal_newlines=False, creationflags=creationflags, console=False) os.close(self.pipe_write_fd) if sys.platform == 'win32': self.pipe_write_handle.Close()
def __init__(self, repo: mzbuild.Repository, name: str): self.name = name self.repo = repo self.images: List[mzbuild.Image] = [] self.workflows: Dict[str, Workflow] = {} default_tag = os.getenv(f"MZBUILD_TAG", None) if name in self.repo.compositions: self.path = self.repo.compositions[name] else: raise errors.UnknownComposition with open(self.path) as f: compose = yaml.safe_load(f) workflows = compose.pop("mzworkflows", None) if workflows is not None: # TODO: move this into the workflow so that it can use env vars that are # manually defined. workflows = _substitute_env_vars(workflows) for workflow_name, raw_w in workflows.items(): built_steps = [] for raw_step in raw_w["steps"]: step_name = raw_step.pop("step") step_ty = Steps.named(step_name) munged = { k.replace("-", "_"): v for k, v in raw_step.items() } try: step = step_ty(**munged) except TypeError as e: a = " ".join([f"{k}={v}" for k, v in munged.items()]) raise errors.BadSpec( f"Unable to construct {step_name} with args {a}: {e}" ) built_steps.append(step) env = raw_w.get("env") if not isinstance(env, dict) and env is not None: raise errors.BadSpec( f"Workflow {workflow_name} has wrong type for env: " f"expected mapping, got {type(env).__name__}: {env}", ) # ensure that integers (e.g. ports) are treated as env vars if isinstance(env, dict): env = {k: str(v) for k, v in env.items()} self.workflows[workflow_name] = Workflow(workflow_name, built_steps, env=env, composition=self) # Resolve all services that reference an `mzbuild` image to a specific # `image` reference. for config in compose["services"].values(): if "mzbuild" in config: image_name = config["mzbuild"] if image_name not in self.repo.images: raise errors.BadSpec( f"mzcompose: unknown image {image_name}") image = self.repo.images[image_name] override_tag = os.getenv(f"MZBUILD_{image.env_var_name()}_TAG", default_tag) if override_tag is not None: config["image"] = image.docker_name(override_tag) print( f"mzcompose: warning: overriding {image_name} image to tag {override_tag}", file=sys.stderr, ) del config["mzbuild"] else: self.images.append(image) if "propagate-uid-gid" in config: config["user"] = f"{os.getuid()}:{os.getgid()}" del config["propagate-uid-gid"] deps = self.repo.resolve_dependencies(self.images) for config in compose["services"].values(): if "mzbuild" in config: config["image"] = deps[config["mzbuild"]].spec() del config["mzbuild"] # Emit the munged configuration to a temporary file so that we can later # pass it to Docker Compose. tempfile = TemporaryFile() os.set_inheritable(tempfile.fileno(), True) yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore tempfile.flush() self.file = tempfile
def main(argv): parser = argparse.ArgumentParser(description="Runs Dolphin as a non-privileged user with access to the privileged tap0 network interface. Run this script with sudo; it will run Dolphin as the user you sudo'ed from and provide it access to the tap interface. It will also automatically configure the tap interface when Dolphin first opens it.") parser.add_argument( "--dolphin-path", "-d", action="store", dest="dolphin_path", default="./Dolphin.app/Contents/MacOS/Dolphin", help="Path to Dolphin executable. Defaults to Dolphin.app/Contents/MacOS/Dolphin in current directory.", ) parser.add_argument( "--memwatch-path", "-m", action="store", dest="memwatch_path", default="memwatch", help="Path to memwatch executable. Defaults to memwatch (assumes it's installed somewhere on $PATH).", ) parser.add_argument( "--tap-interface", "-I", action="store", dest="tap_interface", default="/dev/tap0", help="Tap interface to use. Defaults to /dev/tap0.", ) parser.add_argument( "--tap-ip", "-i", action="store", dest="tap_ip", default="192.168.0.5/24", help="IP address and subnet bits to assign to tap interface. Defaults to 192.168.0.5/24. This should match the gateway and DNS server addresses configured in PSO's network settings.", ) args = parser.parse_args() try: username = os.environ['SUDO_USER'] except KeyError: print('$SUDO_USER not set; use `sudo -E`') return 1 tap_match = re.match(r'^/dev/tap([0-9]+)$', args.tap_interface) if tap_match is None: print('tap interface name must begin with /dev/tap') return 1 tap_interface_number = int(tap_match.group(1)) tap_name = 'tap%d' % (tap_interface_number,) # 1. open tap and configure it print("starting and configuring " + tap_name) tap_fd = os.open(args.tap_interface, os.O_RDWR) os.set_inheritable(tap_fd, True) subprocess.check_call(['ifconfig', tap_name, args.tap_ip], stderr=subprocess.DEVNULL) subprocess.check_call(['ifconfig', tap_name, 'up'], stderr=subprocess.DEVNULL) subprocess.check_call(['ifconfig', tap_name, 'mtu', '9000'], stderr=subprocess.DEVNULL) # 2. fork a Dolphin process, dropping privileges first print("starting dolphin") dolphin_proc = subprocess.Popen([args.dolphin_path], preexec_fn=lambda: change_user(username), pass_fds=(tap_fd,)) time.sleep(1) # 3. create a temp file for dolphin to open instead of /dev/tapN print("creating temp file") tmpfile_fd = os.open("/tmp/dnet", os.O_CREAT | os.O_RDWR, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) os.close(tmpfile_fd) os.chmod("/tmp/dnet", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 4. modify dolphin's memory to make it upen the temp file print("redirecting /dev/tap0 for dolphin") addresses_str = subprocess.check_output([args.memwatch_path, str(dolphin_proc.pid), 'find', '\"/dev/tap0\"'], stderr=subprocess.DEVNULL) for line in addresses_str.splitlines(): tokens = line.split() if len(tokens) != 3: continue print("redirecting /dev/tap0 to /tmp/dnet at %s in dolphin" % (tokens[1],)) subprocess.check_call([args.memwatch_path, str(dolphin_proc.pid), "access", tokens[1], "rwx"], stderr=subprocess.DEVNULL) subprocess.check_call([args.memwatch_path, str(dolphin_proc.pid), "write", tokens[1], "\"/tmp/dnet\""], stderr=subprocess.DEVNULL) subprocess.check_call([args.memwatch_path, str(dolphin_proc.pid), "access", tokens[1], "r-x"], stderr=subprocess.DEVNULL) # step 5: use lsof to find out when dolphin opens /tmp/dnet print("waiting for temp file to open") dolphin_tap0_fd = -1 while dolphin_tap0_fd < 0: time.sleep(1) result = subprocess.check_output(["lsof", "-p", str(dolphin_proc.pid)], stderr=subprocess.DEVNULL) for line in result.splitlines(): if b'/tmp/dnet' not in line: continue fd_str = line.split()[3] dolphin_tap0_fd = int(fd_str[0:-len(fd_str.lstrip(b'0123456789'))]) print("found open tap fd %d in dolphin" % (dolphin_tap0_fd,)) # step 6: use memwatch to move the tap fd into place print("replacing temp fd %d with tap fd %d in dolphin" % (dolphin_tap0_fd, tap_fd)) assembly_contents = b"""start: mov rax, 0x000000000200005A # dup2(from_fd, to_fd) mov rdi, %d mov rsi, %d syscall # close the original fd. note that rdi is preserved during the syscall so we # don't need to reload it mov rax, 0x0000000002000006 # close(fd) syscall ret """ % (tap_fd, dolphin_tap0_fd) subprocess.run([args.memwatch_path, str(dolphin_proc.pid), "--", "run", "-"], input=assembly_contents, stderr=subprocess.DEVNULL) # ok we're done; dolphin is running as a non-privileged user with the tap open return 0
if pid != 0: os.close(pipe_info[1]) os.close(userns_block[0]) select.select([pipe_info[0]], [], []) data = json.load(os.fdopen(pipe_info[0])) child_pid = str(data['child-pid']) subprocess.call(["newuidmap", child_pid, "0", str(os.getuid()), "1"]) subprocess.call(["newgidmap", child_pid, "0", str(os.getgid()), "1"]) os.write(userns_block[1], b'1') else: os.close(pipe_info[0]) os.close(userns_block[1]) if sys.version_info >= (3, 4): os.set_inheritable(pipe_info[1], True) os.set_inheritable(userns_block[0], True) args = [ "bwrap", "bwrap", "--unshare-all", "--unshare-user", "--userns-block-fd", "%i" % userns_block[0], "--info-fd", "%i" % pipe_info[1], "--bind", "/", "/", "cat", "/proc/self/uid_map" ] os.execlp(*args)
#!/usr/bin/env python import socket import subprocess import os import sys import threading args = getattr(__builtins__, 'raw_input', lambda: open(0, 'rb', buffering=0, closefd=False).readline()[0:-1])()[1:-1] s1,s2 = socket.socketpair() if hasattr(os, 'set_inheritable'): os.set_inheritable(s1.fileno(), True) os.set_inheritable(s2.fileno(), True) p1 = subprocess.Popen(['socat', '-', 'FD:' + str(s1.fileno())], close_fds=False) env = dict(os.environ) env['NODE_CHANNEL_FD'] = str(s2.fileno()) p2 = subprocess.Popen(['node', '-e', 'process.argv = [process.argv[0], process.argv[1]].concat(JSON.parse(new Buffer(process.argv[2], "hex"))); setImmediate(function () { require(require("path").resolve(process.argv[1])); })', sys.argv[1], args], env=env, stdin=subprocess.PIPE, stdout=2, close_fds=False) p2.stdin.close() #s1.close() #s2.close() done = False class WaitThread(threading.Thread):
def __init__(self, lock, sockname, configuration, featureset): self.configuration = configuration self.featureset = featureset self.sockname = sockname self.bitbake_lock = lock self.readypipe, self.readypipein = os.pipe() # Create server control socket if os.path.exists(sockname): os.unlink(sockname) # Place the log in the builddirectory alongside the lock file logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log") self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # AF_UNIX has path length issues so chdir here to workaround cwd = os.getcwd() try: os.chdir(os.path.dirname(sockname)) self.sock.bind(os.path.basename(sockname)) finally: os.chdir(cwd) self.sock.listen(1) os.set_inheritable(self.sock.fileno(), True) startdatetime = datetime.datetime.now() bb.daemonize.createDaemon(self._startServer, logfile) self.sock.close() self.bitbake_lock.close() os.close(self.readypipein) ready = ConnectionReader(self.readypipe) r = ready.poll(5) if not r: bb.note("Bitbake server didn't start within 5 seconds, waiting for 90") r = ready.poll(90) if r: try: r = ready.get() except EOFError: # Trap the child exitting/closing the pipe and error out r = None if not r or r[0] != "r": ready.close() bb.error("Unable to start bitbake server (%s)" % str(r)) if os.path.exists(logfile): logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)')) started = False lines = [] lastlines = [] with open(logfile, "r") as f: for line in f: if started: lines.append(line) else: lastlines.append(line) res = logstart_re.match(line.rstrip()) if res: ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format) if ldatetime >= startdatetime: started = True lines.append(line) if len(lastlines) > 60: lastlines = lastlines[-60:] if lines: if len(lines) > 60: bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:]))) else: bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines))) elif lastlines: bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines))) else: bb.error("%s doesn't exist" % logfile) raise SystemExit(1) ready.close()
while (1): tokens = None try: response = input(ps1) except EOFError: sys.exit() pass response = response.strip() if "|" in response: commands = response.split("|") #since we only need to test 1 pipe commands[0], commands[1] = commands[0].strip(), commands[1].strip() rc = os.fork() if rc == 0: pr, pw = os.pipe() for f in (pr, pw): os.set_inheritable(f, True) piping = os.fork() #forking twice if piping == 0: #child is sending output to second child os.close(1) #snippet modified from pipe demo os.dup2(pw, 1) for fd in (pr, pw): os.close(fd) tokens = commands[0].split(" ", 1) #tokenizing args try: #in case the whole path is given os.execve(tokens[0], tokens, os.environ) except FileNotFoundError: pass for dir in re.split( ":", os.environ["PATH"]): #searching for program in PATH program = "%s/%s" % (dir, tokens[0])
os.write(1, ("About to fork (pid=%d)\n" % pid).encode()) rc = os.fork() if rc < 0: os.write(2, ("fork failed, returning %d\n" % rc).encode()) sys.exit(1) elif rc == 0: # child os.write(1, ("Child: My pid==%d. Parent's pid=%d\n" % (os.getpid(), pid)).encode()) args = ["wc", "p3-exec.py"] os.close(1) # redirect child's stdout os.open("p4-output.txt", os.O_CREAT | os.O_WRONLY) os.set_inheritable(1, True) for dir in re.split(":", os.environ['PATH']): # try each directory in path program = "%s/%s" % (dir, args[0]) try: os.execve(program, args, os.environ) # try to exec program except FileNotFoundError: # ...expected pass # ...fail quietly os.write(2, ("Child: Error: Could not exec %s\n" % args[0]).encode()) sys.exit(1) # terminate with error else: # parent (forked ok) os.write(1, ("Parent: My pid=%d. Child's pid=%d\n" % (pid, rc)).encode()) childPidCode = os.wait() os.write(1, ("Parent: Child %d terminated with exit code %d\n" %
def run_comand(comand): rc = os.fork() #create child process args = comand.copy() if '&' in args: args.remove('&') if 'exit' in args: # exit rpogram sys.exit(0) if rc < 0: #frok failed os.write(2, ("fork failed, returning %d\n" % rc).encode()) sys.exit(1) elif rc == 0: # child if '>' in args: os.close(1) # redirect child's stdout os.open(args[-1], os.O_CREAT | os.O_WRONLY) os.set_inheritable(1, True) argg = args[0:args.index(">")] exe(argg) if '<' in args: os.close(0) # redirect child's stdin os.open(args[-1], os.O_RDONLY) os.set_inheritable(0, True) argg = args[0:args.index("<")] exe(argg) if '|' in args: #pipe command args = ' '.join([str(elem) for elem in args]) pipe = args.split("|") prog1 = pipe[0].split() prog2 = pipe[1].split() pr, pw = os.pipe() #file descriptors for reading and writing for f in (pr, pw): os.set_inheritable(f, True) pipeFork = os.fork() if pipeFork < 0: #fork failed os.write(2, ("fork failed").encode()) sys.exit(1) elif pipeFork == 0: #child process which will write to pipe os.close(1) #close fd 1 and rederict it os.dup(pw) #attach fd1 to pipe input fd os.set_inheritable(1, True) #set fd1 inheritable for fd in (pr, pw): os.close(fd) exe(prog1) else: #parent os.close(0) os.dup(pr) os.set_inheritable(0, True) for fd in (pw, pr): os.close(fd) exe(prog2) else: exe(args) else: # parent (forked ok) if not '&' in comand: os.write(1, ("Parent: My pid=%d. Child's pid=%d\n" % (pid, rc)).encode()) childPidCode = os.wait( ) #wait and get child pid and return code os.write(1, ("Parent: Child %d terminated with exit code %d\n" % childPidCode).encode())
def pipe(command_list): index = command_list.index("|") command_left = command_list[:index] command_right = command_list[index + 1:] print("command_left: " + str(command_left)) print("command_right: " + str(command_right)) pid = os.getpid() # get and remember pid pr, pw = os.pipe() for f in (pr, pw): os.set_inheritable(f, True) print("pipe fds: pr=%d, pw=%d" % (pr, pw)) print("About to fork (pid=%d)" % pid) rc = os.fork() if rc < 0: print("fork failed, returning %d\n" % rc, file=sys.stderr) sys.exit(1) elif rc == 0: # child - will write to pipe # print("Child: My pid==%d. Parent's pid=%d" % (os.getpid(), pid), file=sys.stderr) # args = ["cat", "hello"] # # os.close(1) # redirect child's stdout # os.dup(pw) # os.close(pw) # # for fd in (pr, pw): # # os.close(fd) # execute(command_left) saved = os.dup(1) os.close(1) # os.dup(pw) os.dup2(pw, sys.stdout.fileno()) os.close(pw) os.close(pr) # execute command if len(command_left) > 0: execute(command_left) # restore os.dup2(saved, 1) os.close(saved) else: # parent (forked ok) # print("Parent: My pid==%d. Child's pid=%d" % (os.getpid(), rc), file=sys.stderr) # os.close(0) # os.dup(pr) # os.close(pr) # # for fd in (pw, pr): # # os.close(fd) # execute(command_right) saved = os.dup(0) os.close(0) # os.dup(pr) os.dup2(pr, sys.stdin.fileno()) os.close(pr) os.close(pw) # execute command if len(command_right) > 0: execute(command_right) # restore sys.stdout.flush() sys.stdin.flush() os.dup2(saved, 0) os.close(saved) child_pid, exit_code = os.wait() return len(command_list)
def _inner_create_streams(self, path, read, write, append, must_create, must_not_create, synchronized, inheritable, fileno, handle, permissions): # Note : opening broken links works if we're in "w" mode, and raises error in "r" mode, # like for normal unexisting files. if handle is not None: assert fileno is None self._fileno = self._handle = handle elif fileno is not None: assert handle is None self._fileno = self._handle = fileno else: # we open the file with low level posix IO - the unix "open()" function if isinstance(path, unicode): strname = path.encode( sys.getfilesystemencoding() ) # let's take no risks - and do not use locale.getpreferredencoding( # ) here else: strname = path flags = 0 # Note that unix.O_LARGEFILE is actually irrelevant if synchronized: flags |= unix.O_SYNC if read and write: flags |= unix.O_RDWR elif write: flags |= unix.O_WRONLY else: flags |= unix.O_RDONLY if append: flags |= unix.O_APPEND if must_not_create: pass # it's the default case for open() function elif must_create: flags |= unix.O_CREAT | unix.O_EXCL else: flags |= unix.O_CREAT # by default - we create the file iff it doesn't exists # print("Creating unix stream with context", locals()) self._fileno = self._handle = unix.open(strname, flags, permissions) # on unix we must prevent the opening of directories, but not named fifos or other special files stats = unix.fstat(self._fileno).st_mode if stat.S_ISDIR(stats): raise IOError(errno.EISDIR, "RSFile can't open directories", self.name) # we don't use O_CLOEXEC flag (available since Linux 2.6.23) for compatibility and safety if hasattr(os, "set_inheritable"): os.set_inheritable( self._fileno, inheritable) # for safety we call it in any case else: # before PEP0446, newly created file descriptors were inheritable by default if not inheritable: old_flags = unix.fcntl(self._fileno, unix.F_GETFD, 0) if not (old_flags & unix.FD_CLOEXEC): unix.fcntl(self._fileno, unix.F_SETFD, old_flags | unix.FD_CLOEXEC) # WHATEVER the origin of the stream, we initialize these fields: self._lock_registry_inode = self.unique_id( ) # enforces caching of unique_id self._lock_registry_descriptor = self._fileno
print('pid', os.getpid()) if not pipe: # these are the checks for 'special' commands args = my_read_line() if args[0] == 'exit': os.write(1, 'Goodbye!\n'.encode()) sys.exit(0) elif args[0] == 'cd': os.chdir(args[1]) continue elif args[-1] == '&': args = args[:-1] amper = True elif '|' in args and not pipe: pr, pw = os.pipe() for fd in (pr, pw): os.set_inheritable(fd, True) rc = os.fork() # fork here to the child process with gathered information if rc < 0: # if the fork failed notify the user os.write(2, 'Fork failed!\n'.encode()) sys.exit(0) elif rc == 0: # this is the child process if pipe or '|' in args: # check if we are a reading writing or both pipe command if pipe: # set appropriate fds for reading and writing to a pipe #print(os.read(pr, 10000)) print(args) print('in last command') os.close(0) dup_r = os.dup(pr) os.close(pr)
def main(argv): args = parseArguments(argv[1:]) # with open(args["a"],"rt") as a_fd: # a_fdno = a_fd.fileno() # header_a = a_fd.readline().rstrip().split("\t") # with open(args["b"],"rt") as b_fd: # b_fdno = b_fd.fileno() # header_b = b_fd.readline().rstrip().split("\t") if args["a"] == "-": llfd_a = sys.stdin.fileno() else: llfd_a = os.open(args["a"], os.O_RDONLY) if args["b"] == "-": llfd_b = sys.stdin.fileno() else: llfd_b = os.open(args["b"], os.O_RDONLY) if llfd_a == sys.stdin.fileno() and llfd_b == sys.stdin.fileno(): raise ValueError("-a and -b cannot both read from standard in!") # Use unbuffered IO to wrap file descriptors a_fh = os.fdopen(llfd_a, 'rb', 0) b_fh = os.fdopen(llfd_b, 'rb', 0) def_enc = sys.getdefaultencoding() header_a = a_fh.readline().decode(def_enc) header_b = b_fh.readline().decode(def_enc) header_a = header_a.rstrip().split("\t") header_b = header_b.rstrip().split("\t") # acols, bcols: list of 2-tuples. First value of tuple specifies # column name in input, second value of tupls specifies column # name in output (first != second => column is renamed) acols = args["acols"] bcols = args["bcols"] # Check if correct column names specified missing_a = [x[0] for x in acols if x[0] not in header_a] missing_b = [x[0] for x in bcols if x[0] not in header_b] if missing_a or missing_b: raise ValueError('non-existant column names ' + ','.join(missing_a + missing_b) + ' specified!' + '\n valid A: {}, B: {}'.format(header_a, header_b)) # pdb.set_trace() # If no columns are specified: add first columns as key (by) colums # The columns are appended as 2-tuples: the column name of the input # and the desired column name of the output if len(acols) == 0: acols.append((header_a[0], header_a[0])) if len(bcols) == 0: bcols.append((header_b[0], header_b[0])) # Add all input columns which are not specified with equal old and new # column name if --all-a-cols or --all-b-cols, respectively, is set explicit_old_colnames_a = [oldcn for oldcn, newcn in acols] explicit_old_colnames_b = [oldcn for oldcn, newcn in bcols] if args["all_a_cols"]: acols.extend([(col, col) for col in header_a if col not in explicit_old_colnames_a]) if args["all_b_cols"]: bcols.extend([(col, col) for col in header_b if col not in explicit_old_colnames_b]) # Column indices of columns selected for output a_cols_i = [header_a.index(x[0]) for x in acols] b_cols_i = [header_b.index(x[0]) for x in bcols] a_by = acols[0] # Key column name of table 1 (2-tuple) b_by = bcols[0] # Key column name of table 2 (2-tuple) a_by_i = a_cols_i[0] # Key column index of table 1 (0-based) b_by_i = b_cols_i[0] # Key column index of table 2 (0-based) # Add pre-/suffixes to all column names from table 1, # except key (a_by) column. a_cols_print = [cn_new for cn_old, cn_new in acols if cn_old != a_by[0]] a_cols_print = [ args["prefix_a"] + cn + args["suffix_a"] for cn in a_cols_print ] # Add key column, but without pre-/suffix a_cols_print = [a_by[1]] + a_cols_print # Add pre-/suffixes to cols from table 2, except key column (omitted) b_cols_print = [cn_new for cn_old, cn_new in bcols[1:]] b_cols_print = [ args["prefix_b"] + cn + args["suffix_b"] for cn in b_cols_print ] # Add outer join arguments joinargs = [] joinargs.extend(["-1", str(a_by_i + 1)]) joinargs.extend(["-2", str(b_by_i + 1)]) if "a" in args["all"]: joinargs.extend(["-a", "1"]) if "b" in args["all"]: joinargs.extend(["-a", "2"]) #if args["all"] : # Any join # Old versions of join want this, can't take '-o auto' fieldspec = ",".join(["1.{}".format(i + 1) for i in a_cols_i] + ["2.{}".format(i + 1) for i in b_cols_i[1:]]) joinargs.extend(["-e", args["empty"]]) joinargs.extend(["-o", fieldspec]) # #namedpipe_a = os.path.join(tmpf.gettempdir(),rnd_string(40)) # namedpipe_a = os.path.join(rnd_string(40)) # atexit.register(os.unlink, namedpipe_a) # #namedpipe_b = os.path.join(tmpf.gettempdir(),rnd_string(40)) # namedpipe_b = os.path.join(rnd_string(40)) # atexit.register(os.unlink, namedpipe_b) # # os.mkfifo(namedpipe_a) # os.mkfifo(namedpipe_b) # sp.Popen("{awk} <&{fd} | sort > {fifo}".format( # awk=awk_select_cols(a_cols_i), # #tab=args['a'], # fd = llfd_a, # fifo=namedpipe_a), # shell=True, close_fds=False) # sp.Popen("{awk} <&{fd} | sort > {fifo}".format( # awk=awk_select_cols(b_cols_i), # #tab=args['b'], # fd = llfd_b, # fifo=namedpipe_b), # shell=True, close_fds=False) # cmd = ("join {args} -t '\t' {fifo1} {fifo2}".format( # args = " ".join(joinargs), # fifo1=namedpipe_a, fifo2=namedpipe_b)) os.set_inheritable(llfd_a, True) os.set_inheritable(llfd_b, True) sort_cmd_a = "| LC_ALL=C sort -k{by} -t$'\\t'".format(by=a_by_i+1) \ if not args["a_sorted"] else "" sort_cmd_b = "| LC_ALL=C sort -k{by} -t$'\\t'".format(by=b_by_i+1) \ if not args["b_sorted"] else "" cmd_taba = "{awk} <&{fd} {sort}".format( awk="cat", #awk = awk_select_cols(a_cols_i), fd=llfd_a, sort=sort_cmd_a) #sort = "") cmd_tabb = "{awk} <&{fd} {sort}".format( awk="cat", #awk = awk_select_cols(b_cols_i), fd=llfd_b, sort=sort_cmd_b) #sort = "") cmd = ("join {args} -t$'\\t' <({input_a}) <({input_b})").format( input_a=cmd_taba, input_b=cmd_tabb, args=" ".join(joinargs)) print("\t".join(a_cols_print + b_cols_print)) sys.stdout.flush() # Handle SIGPIPE properly, e.g. when piped to head signal.signal(signal.SIGPIPE, signal.SIG_DFL) # Execute join sp.check_call(cmd, close_fds=False, shell=True, executable="/bin/bash") sys.stdout.flush() sys.stderr.flush() return 0
file_out = re.split(' ', args[1]) file_out = file_out[1] args = re.split(' ', args[0]) args = args[:-1] os.close(1) elif l_direct == True: # input redirection file_in = re.split(' ', args[1]) file_in = file_in[1] args = re.split(' ', args[0]) args = args[:-1] args.append(file_in) else: # no redirection args = re.split(' ', args[0]) if file_out != '': # set file to be written to as output sys.stdout = open(file_out, 'w') fd = sys.stdout.fileno() os.set_inheritable(fd, True) for dir in re.split(":", os.environ['PATH']): # try each directory in the path program = "%s/%s" % (dir, args[0]) try: os.execve(program, args, os.environ) # try to exec program except FileNotFoundError: # ...expected pass # ...fail quietly sys.exit(1) # terminate with error else: # parent (forked ok) childPidCode = os.wait()
def run(self): _logger.info('Start running the thread!') self.wait_event.clear() # Set up a pipe for the server to report when it has started. pipe_in, pipe_out = os.pipe() # TODO(crbug.com/941669): Remove if condition after python3 migration. if hasattr(os, 'set_inheritable'): os.set_inheritable(pipe_out, True) try: self._GenerateCommandLineArguments(pipe_out) # TODO(crbug.com/941669): When this script is ported to Python 3, replace # 'vpython3' below with sys.executable. command = [ 'vpython3', os.path.join(_DIR_SOURCE_ROOT, 'net', 'tools', 'testserver', 'testserver.py') ] + self.command_line _logger.info('Running: %s', command) # Disable PYTHONUNBUFFERED because it has a bad interaction with the # testserver. Remove once this interaction is fixed. unbuf = os.environ.pop('PYTHONUNBUFFERED', None) # Pass _DIR_SOURCE_ROOT as the child's working directory so that relative # paths in the arguments are resolved correctly. devnull can be replaced # with subprocess.DEVNULL in Python 3. with open(os.devnull, 'r+b') as devnull: self.process = subprocess.Popen( command, preexec_fn=lambda: self._CloseUnnecessaryFDsForTestServerProcess( pipe_out), stdin=devnull, # Preserve stdout and stderr from the test server. stdout=None, stderr=None, cwd=_DIR_SOURCE_ROOT, close_fds=False) # Close pipe_out early. If self.process crashes, this will be visible # in _WaitToStartAndGetPortFromTestServer's select loop. os.close(pipe_out) pipe_out = -1 if unbuf: os.environ['PYTHONUNBUFFERED'] = unbuf self.is_ready = self._WaitToStartAndGetPortFromTestServer(pipe_in) if self.is_ready: port_map = [(0, self.host_port)] if self.host_ocsp_port: port_map.extend([(0, self.host_ocsp_port)]) self.port_forwarder.Map(port_map) self.forwarder_device_port = \ self.port_forwarder.GetDevicePortForHostPort(self.host_port) if self.host_ocsp_port: self.forwarder_ocsp_device_port = \ self.port_forwarder.GetDevicePortForHostPort(self.host_ocsp_port) # Check whether the forwarder is ready on the device. self.is_ready = self.forwarder_device_port and \ self.port_forwarder.WaitDevicePortReady(self.forwarder_device_port) # Wake up the request handler thread. self.ready_event.set() # Keep thread running until Stop() gets called. self.stop_event.wait() if self.process.poll() is None: self.process.kill() # Wait for process to actually terminate. # (crbug.com/946475) self.process.wait() self.port_forwarder.Unmap(self.forwarder_device_port) self.process = None self.is_ready = False finally: if pipe_in >= 0: os.close(pipe_in) if pipe_out >= 0: os.close(pipe_out) _logger.info('Test-server has died.') self.wait_event.set()
def pipe(args): # Get pid pid = os.getpid() # Check for pipe pipe = args.index("|") # Read input / Write output pr, pw = os.pipe() for f in (pr, pw): os.set_inheritable(f, True) rc = os.fork() if rc < 0: print("fork failed, returning %d\n" % rc, file=sys.stderr) sys.exit(1) elif rc == 0: # write to pipe from child args = args[:pipe] os.close(1) fd = os.dup(pw) # duplicate file descriptor output os.set_inheritable(fd, True) for fd in (pr, pw): os.close(fd) if os.path.isfile(args[0]): try: os.execve(args[0], args, os.environ) # try to execute program except FileNotFoundError: pass else: for dir in re.split( ":", os.environ['PATH']): # Check for environment variables program = "%s/%s" % (dir, args[0]) try: os.execve(program, args, os.environ) # try to execute program except FileNotFoundError: pass os.write(2, ("%s: command not found\n" % args[0]).encode()) sys.exit(1) else: args = args[pipe + 1:] os.close(0) fd = os.dup(pr) # duplicate file descriptor for output os.set_inheritable(fd, True) for fd in (pw, pr): os.close(fd) # close file descriptor if os.path.isfile(args[0]): try: os.execve(args[0], args, os.environ) # execute program except FileNotFoundError: pass else: for dir in re.split( ":", os.environ['PATH']): # Check for environment variables program = "%s/%s" % (dir, args[0]) try: os.execve(program, args, os.environ) # execute program except FileNotFoundError: pass os.write(2, ("%s: command not found\n" % args[0]).encode()) # return error if command was not found sys.exit(1)
def commit(self): def finish_dpkg(term, lock): """ helper that is run when dpkg finishes """ status = term.get_child_exit_status() self.exitstatus = posix.WEXITSTATUS(status) #print "dpkg finished %s %s" % (pid,status) #print "exit status: %s" % self.exitstatus #print "was signaled %s" % posix.WIFSIGNALED(status) try: lock.release() except: logging.exception("lock.release failed") # get a lock lock = threading.Lock() lock.acquire() # ui if self.install: self.status.set_markup("<i>"+_("Installing '%s'...") % \ os.path.basename(self.debfile)+"</i>") else: self.status.set_markup("<i>"+_("Removing '%s'...") % \ os.path.basename(self.debfile)+"</i>") self.progress.pulse() self.progress.set_text("") # prepare reading the pipe (readfd, writefd) = os.pipe() # File descriptors are not inheritable by child processes by # default in Python 3.4. We need the dpkg child to inherit the # write file descriptor. if hasattr(os, 'set_inheritable'): os.set_inheritable(writefd, True) fcntl.fcntl(readfd, fcntl.F_SETFL, os.O_NONBLOCK) #print("fds (%i,%i)" % (readfd,writefd)) # the command argv = ["/usr/bin/dpkg", "--auto-deconfigure"] # ubuntu supports VTE_PTY_KEEP_FD, see # https://bugzilla.gnome.org/320128 for the upstream bug if UBUNTU: argv += ["--status-fd", "%s" % writefd] if self.install: argv += ["-i", self.debfile] else: argv += ["-r", self.debfile] env = [ "VTE_PTY_KEEP_FD=%s" % writefd, "DEBIAN_FRONTEND=gnome", "APT_LISTCHANGES_FRONTEND=gtk" ] #print argv #print env #print self.term # prepare for the fork self.term.connect("child-exited", finish_dpkg, lock) (res, pid) = self.term.fork_command_full( Vte.PtyFlags.DEFAULT, "/", argv, env, GLib.SpawnFlags.LEAVE_DESCRIPTORS_OPEN, # FIXME: add setup_func that closes all fds excpet for writefd None, #setup_func None, #setup_data ) #print "fork_command_full: ", res, pid raw_read = b"" while lock.locked(): while True: try: raw_read += os.read(readfd, 1) except OSError as e: # resource temporarly unavailable is ignored from errno import EAGAIN if e.errno != EAGAIN: logging.warn(e.errstr) break self.time_last_update = time.time() if raw_read[-1] == ord("\n"): read = raw_read.decode("utf-8") statusl = read.split(":") if len(statusl) < 3: logging.warn("got garbage from dpkg: '%s'" % read) raw_read = "" break status = statusl[2].strip() #print status if status == "error" or status == "conffile-prompt": self.term_expander.set_expanded(True) raw_read = b"" self.progress.pulse() while Gtk.events_pending(): Gtk.main_iteration() time.sleep(0.2) # if the terminal has not reacted for some time, do something if (not self.term_expander.get_expanded() and (self.time_last_update + GDEBI_TERMINAL_TIMEOUT) < time.time()): self.term_expander.set_expanded(True) self.progress.set_fraction(1.0)
def exec_(args=[], in_fd=None, out_fd=None, parent_wait=True): ''' Forks a child process and execs args. args: list of agruments to be passed to os.execve() in_fd: integer of new input file descriptor out_fd: integer of new output file descriptor parent_wait: if True, then parent waits for child process to terminate ''' os.write(1, ("Command read...About to fork (pid:%d)\n" % pid).encode()) rc = os.fork() if rc < 0: os.write(2, ("fork failed, returning %d\n" % rc).encode()) sys.exit(1) elif rc == 0: # child child_pid = os.getpid() os.write(1, ("Child: My pid=%d. Parent's pid=%d\n" % (child_pid, pid)).encode()) # change stdin/out fds if given if in_fd: os.write(2, ("Child: pid=%d given in_fd: %d\n" % (child_pid, in_fd)).encode()) os.close(0) fd = os.dup(in_fd) os.set_inheritable(fd, True) os.close(in_fd) os.write(2, ("Child: pid=%d fd 0 -> %d\n" % (child_pid, in_fd)).encode()) # os.write(2, ("Child: pid=%d using fd=%d for reading\n" % (child_pid, fd)).encode()) if out_fd: os.write(2, ("Child: pid=%d given out_fd: %d\n" % (child_pid, out_fd)).encode()) os.close(1) fd = os.dup(out_fd) os.set_inheritable(fd, True) os.close(out_fd) os.write(2, ("Child: pid=%d fd 1 -> %d\n" % (child_pid, out_fd)).encode()) # os.write(2, ("Child: pid=%d using fd=%d for writing\n" % (child_pid, fd)).encode()) os.write( 2, ("Child: pid=%d stdin fd: %d stdout fd: %d\n" % (child_pid, sys.stdin.fileno(), sys.stdout.fileno())).encode()) for dir in re.split( ":", os.environ['PATH']): # try each directory in the path program = "%s/%s" % (dir, args[0]) # os.write(1, ("Child: pid=%d ...trying to exec %s\n" % (child_pid, program)).encode()) try: os.execve(program, args, os.environ) # try to exec program except FileNotFoundError: # ...expected pass # ...fail quietly os.write(2, ("Child: pid=%d Could not exec %s\n" % (child_pid, args[0])).encode()) sys.exit(1) # terminate with error else: # parent (forked ok) os.write(1, ("Parent: My pid=%d. Child's pid=%d\n" % (pid, rc)).encode()) if parent_wait: childPidCode = os.wait() os.write(1, ("Parent: Child %d terminated with exit code %d\n" % childPidCode).encode())
class JobServer(object): # Whether the job server has been initialized _initialized = False # Flag designating whether the `make` program supports the GNU Make # jobserver interface _gnu_make = None # Initialize variables _load_ok = True _mem_ok = True _internal_jobs = [] _max_load = 0 _max_jobs = 0 _job_pipe = os.pipe() # Setting fd inheritance is required in Python > 3.4 # This is set by default in Python 2.7 # For more info see: https://docs.python.org/3.4/library/os.html#fd-inheritance if hasattr(os, 'set_inheritable'): for fd in _job_pipe: os.set_inheritable(fd, True) if not os.get_inheritable(fd): log( clr('@{yf}@!Warning: jobserver file descriptors are not inheritable.@|' )) @classmethod def _set_max_jobs(cls, max_jobs): """Set the maximum number of jobs to be used with the jobserver. This will wait for all active jobs to be completed, then re-initialize the job pipe. """ # Read all possible tokens from the pipe try: os.read(cls._job_pipe[0], cls._max_jobs) except (BlockingIOError, InterruptedError): pass # Update max jobs cls._max_jobs = max_jobs # Initialize the pipe with max_jobs tokens for i in range(cls._max_jobs): os.write(cls._job_pipe[1], b'+') @classmethod def _set_max_mem(cls, max_mem): """ Set the maximum memory to keep instantiating jobs. :param max_mem: String describing the maximum memory that can be used on the system. It can either describe memory percentage or absolute amount. Use 'P%' for percentage or 'N' for absolute value in bytes, 'Nk' for kilobytes, 'Nm' for megabytes, and 'Ng' for gigabytes. :type max_mem: str """ if max_mem is None: cls._max_mem = None return elif type(max_mem) is float or type(max_mem) is int: mem_percent = max_mem elif type(max_mem) is str: m_percent = re.search(r'([0-9]+)\%', max_mem) m_abs = re.search(r'([0-9]+)([kKmMgG]{0,1})', max_mem) if m_percent is None and m_abs is None: cls._max_mem = None return if m_percent: mem_percent = m_abs.group(1) elif m_abs: val = float(m_abs.group(1)) mag_symbol = m_abs.group(2) _, total_mem = memory_usage() if mag_symbol == '': mag = 1.0 elif mag_symbol.lower() == 'k': mag = 1024.0 elif mag_symbol.lower() == 'm': mag = pow(1024.0, 2) elif mag_symbol.lower() == 'g': mag = pow(1024.0, 3) mem_percent = 100.0 * val * mag / total_mem cls._max_mem = max(0.0, min(100.0, float(mem_percent))) @classmethod def _check_load(cls): if cls._max_load is not None: try: load = os.getloadavg() if load[0] < cls._max_load: cls._load_ok = True else: cls._load_ok = False except NotImplementedError: cls._load_ok = True return cls._load_ok @classmethod def _check_mem(cls): if cls._max_mem is not None: mem_used, mem_total = memory_usage() mem_percent_used = 100.0 * float(mem_used) / float(mem_total) if mem_percent_used > cls._max_mem: cls._mem_ok = False else: cls._mem_ok = True return cls._mem_ok @classmethod def _check_conditions(cls): return (cls._check_load() and cls._check_mem()) or cls._running_jobs() == 0 @classmethod def _acquire(cls): """ Obtain a job server token. Be sure to call _release() to avoid deadlocks. """ try: # read a token from the job pipe token = os.read(cls._job_pipe[0], 1) return token except (BlockingIOError, InterruptedError): pass return None @classmethod def _release(cls): """ Write a token to the job pipe. """ os.write(cls._job_pipe[1], b'+') @classmethod def _running_jobs(cls): try: buf = array.array('i', [0]) if fcntl.ioctl(cls._job_pipe[0], FIONREAD, buf) == 0: return cls._max_jobs - buf[0] except (NotImplementedError, OSError): pass return cls._max_jobs
uIn2 = uIn.split() if len(uIn2) < 3: print("Invalid command") if len(uIn2) == 3: rc0 = os.fork() if rc0 == 0: #input redirection file with name of another file args = [uIn2[0]] #command to execute os.close(0) sys.stdin = open(uIn2[2], "r") #open file containing name of another file rFile = input() args.append(rFile) fd = sys.stdin.fileno() os.set_inheritable(fd, True) os.write(2, ("Child: opened fd=%d for reading\n" % fd).encode()) for dir in re.split(":", os.environ['PATH']): program = "%s/%s" % (dir, args[0]) try: os.execve(program, args, os.environ) #try and execute program except FileNotFoundError: pass os.write(2, ("Error: Could not exec%s\n" % args[0]).encode()) sys.exit(1) else: # parent (forked ok) childPidCode = os.wait() #wait for child to execute