async def _testPipeLoggerToPipe(self, test_string, loop): """ Test PipeLogger writing to a pipe connected to a PipeReader. This verifies that PipeLogger does not deadlock when writing to a pipe that's drained by a PipeReader running in the same process (requires non-blocking write). """ input_fd, writer_pipe = os.pipe() _set_nonblocking(writer_pipe) writer_pipe = os.fdopen(writer_pipe, 'wb', 0) writer = asyncio.ensure_future( _writer(writer_pipe, test_string.encode('ascii'))) writer.add_done_callback(lambda writer: writer_pipe.close()) pr, pw = os.pipe() consumer = PipeLogger(background=True, input_fd=input_fd, log_file_path=os.fdopen(pw, 'wb', 0), scheduler=loop) consumer.start() # Before starting the reader, wait here for a moment, in order # to exercise PipeLogger's handling of EAGAIN during write. await asyncio.wait([writer], timeout=0.01) reader = _reader(pr) await writer content = await reader await consumer.async_wait() self.assertEqual(consumer.returncode, os.EX_OK) return content.decode('ascii', 'replace')
def _start(self): in_pr, in_pw = os.pipe() out_pr, out_pw = os.pipe() self._files = {} self._files['pipe_in'] = in_pr self._files['pipe_out'] = out_pw fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl(in_pr, fcntl.F_SETFD, fcntl.fcntl(in_pr, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) self._reg_id = self.scheduler.io_add_watch(in_pr, self.scheduler.IO_IN, self._output_handler) self._registered = True self._proc = SpawnProcess( args=[portage._python_interpreter, os.path.join(portage._bin_path, 'lock-helper.py'), self.path], env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path), fd_pipes={0:out_pr, 1:in_pw, 2:sys.__stderr__.fileno()}, scheduler=self.scheduler) self._proc.addExitListener(self._proc_exit) self._proc.start() os.close(out_pr) os.close(in_pw)
def _start(self): in_pr, in_pw = os.pipe() out_pr, out_pw = os.pipe() self._files = {} self._files["pipe_in"] = in_pr self._files["pipe_out"] = out_pw fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) self.scheduler.add_reader(in_pr, self._output_handler) self._registered = True self._proc = SpawnProcess( args=[ portage._python_interpreter, os.path.join(portage._bin_path, "lock-helper.py"), self.path, ], env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path), fd_pipes={ 0: out_pr, 1: in_pw, 2: sys.__stderr__.fileno() }, scheduler=self.scheduler, ) self._proc.addExitListener(self._proc_exit) self._proc.start() os.close(out_pr) os.close(in_pw)
def _create_pty_or_pipe(copy_term_size=None): """ Try to create a pty and if then fails then create a normal pipe instead. @param copy_term_size: If a tty file descriptor is given then the term size will be copied to the pty. @type copy_term_size: int @rtype: tuple @return: A tuple of (is_pty, master_fd, slave_fd) where is_pty is True if a pty was successfully allocated, and False if a normal pipe was allocated. """ got_pty = False global _disable_openpty, _fbsd_test_pty if _fbsd_test_pty and not _disable_openpty: # Test for python openpty breakage after freebsd7 to freebsd8 # upgrade, which results in a 'Function not implemented' error # and the process being killed. pid = os.fork() if pid == 0: pty.openpty() os._exit(os.EX_OK) pid, status = os.waitpid(pid, 0) if (status & 0xff) == 140: _disable_openpty = True _fbsd_test_pty = False if _disable_openpty: master_fd, slave_fd = os.pipe() else: try: master_fd, slave_fd = pty.openpty() got_pty = True except EnvironmentError as e: _disable_openpty = True writemsg("openpty failed: '%s'\n" % str(e), noiselevel=-1) del e master_fd, slave_fd = os.pipe() if got_pty: # Disable post-processing of output since otherwise weird # things like \n -> \r\n transformations may occur. mode = termios.tcgetattr(slave_fd) mode[1] &= ~termios.OPOST termios.tcsetattr(slave_fd, termios.TCSANOW, mode) if got_pty and \ copy_term_size is not None and \ os.isatty(copy_term_size): rows, columns = get_term_size() set_term_size(rows, columns, slave_fd) return (got_pty, master_fd, slave_fd)
def _create_pty_or_pipe(copy_term_size=None): """ Try to create a pty and if then fails then create a normal pipe instead. @param copy_term_size: If a tty file descriptor is given then the term size will be copied to the pty. @type copy_term_size: int @rtype: tuple @returns: A tuple of (is_pty, master_fd, slave_fd) where is_pty is True if a pty was successfully allocated, and False if a normal pipe was allocated. """ got_pty = False global _disable_openpty, _fbsd_test_pty if _fbsd_test_pty and not _disable_openpty: # Test for python openpty breakage after freebsd7 to freebsd8 # upgrade, which results in a 'Function not implemented' error # and the process being killed. pid = os.fork() if pid == 0: pty.openpty() os._exit(os.EX_OK) pid, status = os.waitpid(pid, 0) if (status & 0xff) == 140: _disable_openpty = True _fbsd_test_pty = False if _disable_openpty: master_fd, slave_fd = os.pipe() else: try: master_fd, slave_fd = pty.openpty() got_pty = True except EnvironmentError as e: _disable_openpty = True writemsg("openpty failed: '%s'\n" % str(e), noiselevel=-1) del e master_fd, slave_fd = os.pipe() if got_pty: # Disable post-processing of output since otherwise weird # things like \n -> \r\n transformations may occur. mode = termios.tcgetattr(slave_fd) mode[1] &= ~termios.OPOST termios.tcsetattr(slave_fd, termios.TCSANOW, mode) if got_pty and \ copy_term_size is not None and \ os.isatty(copy_term_size): rows, columns = get_term_size() set_term_size(rows, columns, slave_fd) return (got_pty, master_fd, slave_fd)
def _async_start(self): pipe_logger = None filter_proc = None try: log_input = None if self.log_path is not None: log_filter_file = self.log_filter_file if log_filter_file is not None: split_value = shlex_split(log_filter_file) log_filter_file = split_value if split_value else None if log_filter_file: filter_input, stdin = os.pipe() log_input, filter_output = os.pipe() try: filter_proc = yield asyncio.create_subprocess_exec( *log_filter_file, env=self.env, stdin=filter_input, stdout=filter_output, stderr=filter_output, loop=self.scheduler) except EnvironmentError: # Maybe the command is missing or broken somehow... os.close(filter_input) os.close(stdin) os.close(log_input) os.close(filter_output) else: self._stdin = os.fdopen(stdin, 'wb', 0) os.close(filter_input) os.close(filter_output) if self._stdin is None: # Since log_filter_file is unspecified or refers to a file # that is missing or broken somehow, create a pipe that # logs directly to pipe_logger. log_input, stdin = os.pipe() self._stdin = os.fdopen(stdin, 'wb', 0) # Set background=True so that pipe_logger does not log to stdout. pipe_logger = PipeLogger(background=True, scheduler=self.scheduler, input_fd=log_input, log_file_path=self.log_path) yield pipe_logger.async_start() except asyncio.CancelledError: if pipe_logger is not None and pipe_logger.poll() is None: pipe_logger.cancel() if filter_proc is not None and filter_proc.returncode is None: filter_proc.terminate() raise self._main_task = asyncio.ensure_future(self._main( pipe_logger, filter_proc=filter_proc), loop=self.scheduler) self._main_task.add_done_callback(self._main_exit)
def _start(self): filter_proc = None log_input = None if self.log_path is not None: log_filter_file = self.log_filter_file if log_filter_file is not None: split_value = shlex_split(log_filter_file) log_filter_file = split_value if split_value else None if log_filter_file: filter_input, stdin = os.pipe() log_input, filter_output = os.pipe() try: filter_proc = PopenProcess( proc=subprocess.Popen( log_filter_file, env=self.env, stdin=filter_input, stdout=filter_output, stderr=filter_output, ), scheduler=self.scheduler, ) filter_proc.start() except EnvironmentError: # Maybe the command is missing or broken somehow... os.close(filter_input) os.close(stdin) os.close(log_input) os.close(filter_output) else: self._stdin = os.fdopen(stdin, "wb", 0) os.close(filter_input) os.close(filter_output) if self._stdin is None: # Since log_filter_file is unspecified or refers to a file # that is missing or broken somehow, create a pipe that # logs directly to pipe_logger. log_input, stdin = os.pipe() self._stdin = os.fdopen(stdin, "wb", 0) # Set background=True so that pipe_logger does not log to stdout. pipe_logger = PipeLogger( background=True, scheduler=self.scheduler, input_fd=log_input, log_file_path=self.log_path, ) pipe_logger.start() self._main_task_cancel = functools.partial(self._main_cancel, filter_proc, pipe_logger) self._main_task = asyncio.ensure_future(self._main( filter_proc, pipe_logger), loop=self.scheduler) self._main_task.add_done_callback(self._main_exit)
def _spawn(self, args, fd_pipes, **kwargs): """ Extend the superclass _spawn method to perform some pre-fork and post-fork actions. """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl( elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK, ) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink( self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd, ) fd_pipes[elog_writer_fd] = elog_writer_fd self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() if not self.unmerge: self._counter = self.vartree.dbapi.counter_tick() self._dblink = mylink self._elog_reader_fd = elog_reader_fd pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs) os.close(elog_writer_fd) self._buf = "" self._elog_keys = set() # Discard messages which will be collected by the subprocess, # in order to avoid duplicates (bug #446136). portage.elog.messages.collect_messages(key=mylink.mycpv) # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) return pids
async def run(self): self.expected = getattr(self, "expected", None) or {"returncode": 0} if self.debug: fd_pipes = {} pr = None pw = None else: pr, pw = os.pipe() fd_pipes = {1: pw, 2: pw} pr = open(pr, "rb", 0) proc = AsyncFunction( scheduler=asyncio.get_event_loop(), target=self._subprocess, args=(self.args, self.cwd, self.env, self.expected, self.debug), fd_pipes=fd_pipes, ) proc.start() if pw is not None: os.close(pw) await proc.async_wait() if pr is None: stdio = None else: stdio = await _reader(pr) self.result = { "stdio": stdio, "result": proc.result, }
def _testPipeLoggerToPipe(self, test_string, loop=None): """ Test PipeLogger writing to a pipe connected to a PipeReader. This verifies that PipeLogger does not deadlock when writing to a pipe that's drained by a PipeReader running in the same process (requires non-blocking write). """ producer = PopenProcess(proc=subprocess.Popen( ["bash", "-c", self._echo_cmd % test_string], stdout=subprocess.PIPE, stderr=subprocess.STDOUT), scheduler=loop) pr, pw = os.pipe() consumer = producer.pipe_reader = PipeLogger( background=True, input_fd=producer.proc.stdout, log_file_path=os.fdopen(pw, 'wb', 0)) reader = _reader(pr, loop=loop) yield producer.async_start() content = yield reader yield producer.async_wait() yield consumer.async_wait() self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) coroutine_return(content.decode('ascii', 'replace'))
def _start(self): pr, pw = os.pipe() self.fd_pipes = {} self.fd_pipes[pw] = pw self._async_func_reader_pw = pw self._async_func_reader = PipeReader(input_files={"input": pr}, scheduler=self.scheduler) self._async_func_reader.addExitListener(self._async_func_reader_exit) self._async_func_reader.start() ForkProcess._start(self) os.close(pw)
def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" if self.background or not sys.stdout.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd)
def _start(self): in_pr, in_pw = os.pipe() out_pr, out_pw = os.pipe() self._files = {} self._files['pipe_in'] = in_pr self._files['pipe_out'] = out_pw fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) self._reg_id = self.scheduler.register(in_pr, self.scheduler.IO_IN, self._output_handler) self._registered = True self._proc = SpawnProcess( args=[portage._python_interpreter, os.path.join(portage._bin_path, 'lock-helper.py'), self.path], env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path), fd_pipes={0:out_pr, 1:in_pw, 2:sys.stderr.fileno()}, scheduler=self.scheduler) self._proc.addExitListener(self._proc_exit) self._proc.start() os.close(out_pr) os.close(in_pw)
def _start(self): pr, pw = os.pipe() self.fd_pipes = {} self.fd_pipes[pw] = pw self._digest_pw = pw self._digest_pipe_reader = PipeReader( input_files={"input":pr}, scheduler=self.scheduler) self._digest_pipe_reader.addExitListener(self._digest_pipe_reader_exit) self._digest_pipe_reader.start() ForkProcess._start(self) os.close(pw)
def testLazyImportPortageBaseline(self): """ Check what modules are imported by a baseline module import. """ env = os.environ.copy() pythonpath = env.get("PYTHONPATH") if pythonpath is not None and not pythonpath.strip(): pythonpath = None if pythonpath is None: pythonpath = "" else: pythonpath = ":" + pythonpath pythonpath = PORTAGE_PYM_PATH + pythonpath env["PYTHONPATH"] = pythonpath # If python is patched to insert the path of the # currently installed portage module into sys.path, # then the above PYTHONPATH override doesn't help. env["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH scheduler = global_event_loop() master_fd, slave_fd = os.pipe() master_file = os.fdopen(master_fd, "rb", 0) slave_file = os.fdopen(slave_fd, "wb") producer = SpawnProcess( args=self._baseline_import_cmd, env=env, fd_pipes={1: slave_fd}, scheduler=scheduler, ) producer.start() slave_file.close() consumer = PipeReader(input_files={"producer": master_file}, scheduler=scheduler) consumer.start() consumer.wait() self.assertEqual(producer.wait(), os.EX_OK) self.assertEqual(consumer.wait(), os.EX_OK) output = consumer.getvalue().decode("ascii", "replace").split() unexpected_modules = " ".join( sorted(x for x in output if self._module_re.match(x) is not None and x not in self._baseline_imports)) self.assertEqual("", unexpected_modules)
def _testAsyncFunctionStdin(self, loop=None): test_string = '1\n2\n3\n' pr, pw = os.pipe() fd_pipes = {0: pr} reader = AsyncFunction(scheduler=loop, fd_pipes=fd_pipes, target=self._read_from_stdin, args=(pw, )) reader.start() os.close(pr) _set_nonblocking(pw) with open(pw, mode='wb', buffering=0) as pipe_write: yield _writer(pipe_write, test_string.encode('utf_8'), loop=loop) self.assertEqual((yield reader.async_wait()), os.EX_OK) self.assertEqual(reader.result, test_string)
async def _testAsyncFunctionStdin(self, loop): test_string = "1\n2\n3\n" pr, pw = os.pipe() fd_pipes = {0: pr} reader = AsyncFunction(scheduler=loop, fd_pipes=fd_pipes, target=self._read_from_stdin, args=(pw, )) reader.start() os.close(pr) _set_nonblocking(pw) with open(pw, mode="wb", buffering=0) as pipe_write: await _writer(pipe_write, test_string.encode("utf_8")) self.assertEqual((await reader.async_wait()), os.EX_OK) self.assertEqual(reader.result, test_string)
def testLazyImportPortageBaseline(self): """ Check what modules are imported by a baseline module import. """ env = os.environ.copy() pythonpath = env.get('PYTHONPATH') if pythonpath is not None and not pythonpath.strip(): pythonpath = None if pythonpath is None: pythonpath = '' else: pythonpath = ':' + pythonpath pythonpath = PORTAGE_PYM_PATH + pythonpath env['PYTHONPATH'] = pythonpath # If python is patched to insert the path of the # currently installed portage module into sys.path, # then the above PYTHONPATH override doesn't help. env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH scheduler = PollScheduler().sched_iface master_fd, slave_fd = os.pipe() master_file = os.fdopen(master_fd, 'rb', 0) slave_file = os.fdopen(slave_fd, 'wb') producer = SpawnProcess( args=self._baseline_import_cmd, env=env, fd_pipes={1:slave_fd}, scheduler=scheduler) producer.start() slave_file.close() consumer = PipeReader( input_files={"producer" : master_file}, scheduler=scheduler) consumer.start() consumer.wait() self.assertEqual(producer.wait(), os.EX_OK) self.assertEqual(consumer.wait(), os.EX_OK) output = consumer.getvalue().decode('ascii', 'replace').split() unexpected_modules = " ".join(sorted(x for x in output \ if self._module_re.match(x) is not None and \ x not in self._baseline_imports)) self.assertEqual("", unexpected_modules)
def _start(self): pr, pw = os.pipe() self._files = {} self._files['pipe_read'] = pr self._files['pipe_write'] = pw for f in self._files.values(): fcntl.fcntl(f, fcntl.F_SETFL, fcntl.fcntl(f, fcntl.F_GETFL) | os.O_NONBLOCK) self._reg_id = self.scheduler.register(self._files['pipe_read'], self.scheduler.IO_IN, self._output_handler) self._registered = True threading_mod = threading if self._force_dummy: threading_mod = dummy_threading self._thread = threading_mod.Thread(target=self._run_lock) self._thread.start()
def _start(self): pr, pw = os.pipe() self._files = {} self._files['pipe_read'] = pr self._files['pipe_write'] = pw for f in self._files.values(): fcntl.fcntl(f, fcntl.F_SETFL, fcntl.fcntl(f, fcntl.F_GETFL) | os.O_NONBLOCK) self._reg_id = self.scheduler.io_add_watch(self._files['pipe_read'], self.scheduler.IO_IN, self._output_handler) self._registered = True threading_mod = threading if self._force_dummy: threading_mod = dummy_threading self._thread = threading_mod.Thread(target=self._run_lock) self._thread.start()
def _start(self): pr, pw = os.pipe() self._files = {} self._files['pipe_read'] = os.fdopen(pr, 'rb', 0) self._files['pipe_write'] = os.fdopen(pw, 'wb', 0) for k, f in self._files.items(): fcntl.fcntl(f.fileno(), fcntl.F_SETFL, fcntl.fcntl(f.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) self._reg_id = self.scheduler.register(self._files['pipe_read'].fileno(), PollConstants.POLLIN, self._output_handler) self._registered = True threading_mod = threading if self._force_dummy: threading_mod = dummy_threading self._thread = threading_mod.Thread(target=self._run_lock) self._thread.start()
def _testPipeReader(self, test_string): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ if self._use_pty: got_pty, master_fd, slave_fd = _create_pty_or_pipe() if not got_pty: os.close(slave_fd) os.close(master_fd) skip_reason = "pty not acquired" self.portage_skip = skip_reason self.fail(skip_reason) return else: master_fd, slave_fd = os.pipe() # WARNING: It is very important to use unbuffered mode here, # in order to avoid issue 5380 with python3. master_file = os.fdopen(master_fd, 'rb', 0) slave_file = os.fdopen(slave_fd, 'wb', 0) task_scheduler = TaskScheduler(max_jobs=2) producer = SpawnProcess( args=["bash", "-c", self._echo_cmd % test_string], env=os.environ, fd_pipes={1: slave_fd}, scheduler=task_scheduler.sched_iface) task_scheduler.add(producer) slave_file.close() consumer = PipeReader(input_files={"producer": master_file}, scheduler=task_scheduler.sched_iface, _use_array=self._use_array) task_scheduler.add(consumer) # This will ensure that both tasks have exited, which # is necessary to avoid "ResourceWarning: unclosed file" # warnings since Python 3.2 (and also ensures that we # don't leave any zombie child processes). task_scheduler.run() self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) return consumer.getvalue().decode('ascii', 'replace')
def _testPipeReader(self, test_string): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ if self._use_pty: got_pty, master_fd, slave_fd = _create_pty_or_pipe() if not got_pty: os.close(slave_fd) os.close(master_fd) skip_reason = "pty not acquired" self.portage_skip = skip_reason self.fail(skip_reason) return else: master_fd, slave_fd = os.pipe() # WARNING: It is very important to use unbuffered mode here, # in order to avoid issue 5380 with python3. master_file = os.fdopen(master_fd, 'rb', 0) slave_file = os.fdopen(slave_fd, 'wb', 0) task_scheduler = TaskScheduler(max_jobs=2) producer = SpawnProcess( args=["bash", "-c", self._echo_cmd % test_string], env=os.environ, fd_pipes={1:slave_fd}, scheduler=task_scheduler.sched_iface) task_scheduler.add(producer) slave_file.close() consumer = PipeReader( input_files={"producer" : master_file}, scheduler=task_scheduler.sched_iface, _use_array=self._use_array) task_scheduler.add(consumer) # This will ensure that both tasks have exited, which # is necessary to avoid "ResourceWarning: unclosed file" # warnings since Python 3.2 (and also ensures that we # don't leave any zombie child processes). task_scheduler.run() self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) return consumer.getvalue().decode('ascii', 'replace')
def _testPipeReader(self, test_string): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ if self._use_pty: got_pty, master_fd, slave_fd = _create_pty_or_pipe() if not got_pty: os.close(slave_fd) os.close(master_fd) skip_reason = "pty not acquired" self.portage_skip = skip_reason self.fail(skip_reason) return else: master_fd, slave_fd = os.pipe() # WARNING: It is very important to use unbuffered mode here, # in order to avoid issue 5380 with python3. master_file = os.fdopen(master_fd, 'rb', 0) scheduler = global_event_loop() consumer = PipeReader( input_files={"producer" : master_file}, _use_array=self._use_array, scheduler=scheduler) producer = PopenProcess( pipe_reader=consumer, proc=subprocess.Popen(["bash", "-c", self._echo_cmd % test_string], stdout=slave_fd), scheduler=scheduler) producer.start() os.close(slave_fd) producer.wait() self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) return consumer.getvalue().decode('ascii', 'replace')
def testPipeReader(self): """ Use a poll loop to read data from a pipe and assert that the data written to the pipe is identical to the data read from the pipe. """ test_string = 2 * "blah blah blah\n" scheduler = PollScheduler().sched_iface master_fd, slave_fd = os.pipe() master_file = os.fdopen(master_fd, 'rb', 0) slave_file = os.fdopen(slave_fd, 'wb') producer = SpawnProcess( args=["bash", "-c", "echo -n '%s'" % test_string], env=os.environ, fd_pipes={1:slave_fd}, scheduler=scheduler) producer.start() slave_file.close() consumer = PipeReader( input_files={"producer" : master_file}, scheduler=scheduler) consumer.start() # This will ensure that both tasks have exited, which # is necessary to avoid "ResourceWarning: unclosed file" # warnings since Python 3.2 (and also ensures that we # don't leave any zombie child processes). scheduler.schedule() self.assertEqual(producer.returncode, os.EX_OK) self.assertEqual(consumer.returncode, os.EX_OK) output = consumer.getvalue().decode('ascii', 'replace') self.assertEqual(test_string, output)
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, path_lookup=True, pre_exec=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: A dict of Key=Value pairs for env variables @type env: Dictionary @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:sys.__stdin__.fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand))) traceback.print_exc() sys.stderr.flush() os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) spawned_pids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] # When it's done, we can remove it from the # global pid list as well. spawned_pids.remove(pid) if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) spawned_pids.remove(pid) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _start(self): settings = self.settings settings.setcpv(self.cpv) ebuild_path = self.ebuild_hash.location # the caller can pass in eapi in order to avoid # redundant _parse_eapi_ebuild_head calls eapi = self.eapi if eapi is None and \ 'parse-eapi-ebuild-head' in settings.features: with io.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: eapi = portage._parse_eapi_ebuild_head(f) if eapi is not None: if not portage.eapi_is_supported(eapi): self.metadata = self.metadata_callback(self.cpv, self.repo_path, {'EAPI' : eapi}, self.ebuild_hash) self._set_returncode((self.pid, os.EX_OK << 8)) self.wait() return settings.configdict['pkg']['EAPI'] = eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) fd_pipes.setdefault(1, sys.stdout.fileno()) fd_pipes.setdefault(2, sys.stderr.fileno()) # flush any pending output for fd in fd_pipes.values(): if fd == sys.stdout.fileno(): sys.stdout.flush() if fd == sys.stderr.fileno(): sys.stderr.flush() fd_pipes_orig = fd_pipes.copy() self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fd_pipes[self._metadata_fd] = slave_fd self._raw_metadata = [] files.ebuild = os.fdopen(master_fd, 'rb', 0) self._reg_id = self.scheduler.register(files.ebuild.fileno(), self._registered_events, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) os.close(slave_fd) null_input.close() if isinstance(retval, int): # doebuild failed before spawning self._unregister() self._set_returncode((self.pid, retval << 8)) self.wait() return self.pid = retval[0] portage.process.spawned_pids.remove(self.pid)
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.io_add_watch( elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=(sys.version_info < (3, 4)), unshare_net=False, unshare_ipc=False, unshare_mount=False, unshare_pid=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: If env is not None, it must be a mapping that defines the environment variables for the new process; these are used instead of the default behavior of inheriting the current process's environment. @type env: None or Mapping @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() env = os.environ if env is None else env if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:portage._get_stdin().fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # This caches the libc library lookup and _unshare_validator results # in the current process, so that results are cached for use in # child processes. unshare_flags = 0 if unshare_net or unshare_ipc or unshare_mount or unshare_pid: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 if unshare_net: unshare_flags |= CLONE_NEWNET if unshare_ipc: unshare_flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace unshare_flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc unshare_flags |= CLONE_NEWPID | CLONE_NEWNS _unshare_validate(unshare_flags) # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _create_pipe(self): return os.pipe()
def _start(self): settings = self.settings settings.setcpv(self.cpv) ebuild_path = self.ebuild_path eapi = None if 'parse-eapi-glep-55' in settings.features: pf, eapi = portage._split_ebuild_name_glep55( os.path.basename(ebuild_path)) if eapi is None and \ 'parse-eapi-ebuild-head' in settings.features: eapi = portage._parse_eapi_ebuild_head( codecs.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace')) if eapi is not None: if not portage.eapi_is_supported(eapi): self.metadata_callback(self.cpv, self.ebuild_path, self.repo_path, {'EAPI': eapi}, self.ebuild_mtime) self.returncode = os.EX_OK self.wait() return settings.configdict['pkg']['EAPI'] = eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} fd_pipes.setdefault(0, sys.stdin.fileno()) fd_pipes.setdefault(1, sys.stdout.fileno()) fd_pipes.setdefault(2, sys.stderr.fileno()) # flush any pending output for fd in fd_pipes.values(): if fd == sys.stdout.fileno(): sys.stdout.flush() if fd == sys.stderr.fileno(): sys.stderr.flush() fd_pipes_orig = fd_pipes.copy() self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fd_pipes[self._metadata_fd] = slave_fd self._raw_metadata = [] files.ebuild = os.fdopen(master_fd, 'rb') self._reg_id = self.scheduler.register(files.ebuild.fileno(), self._registered_events, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings["ROOT"], settings, debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) os.close(slave_fd) if isinstance(retval, int): # doebuild failed before spawning self._unregister() self.returncode = retval self.wait() return self.pid = retval[0] portage.process.spawned_pids.remove(self.pid)
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, scheduler=self.scheduler, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.register(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # TODO: Find out why PyPy 1.8 with close_fds=True triggers # "[Errno 9] Bad file descriptor" in subprocesses. It could # be due to garbage collection of file objects that were not # closed before going out of scope, since PyPy's garbage # collector does not support the refcounting semantics that # CPython does. close_fds = platform.python_implementation() != 'PyPy' portage.process._setup_pipes(fd_pipes, close_fds=close_fds) # Use default signal handlers since the ones inherited # from the parent process are irrelevant here. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # In this subprocess we want mylink._display_merge() to use # stdout/stderr directly since they are pipes. This behavior # is triggered when mylink._scheduler is None. mylink._scheduler = None # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def _start(self): ebuild_path = self.ebuild_hash.location with io.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f) parsed_eapi = self._eapi if parsed_eapi is None: parsed_eapi = "0" if not parsed_eapi: # An empty EAPI setting is invalid. self._eapi_invalid(None) self._set_returncode((self.pid, 1 << 8)) self.wait() return self.eapi_supported = portage.eapi_is_supported(parsed_eapi) if not self.eapi_supported: self.metadata = {"EAPI": parsed_eapi} self._set_returncode((self.pid, os.EX_OK << 8)) self.wait() return settings = self.settings settings.setcpv(self.cpv) settings.configdict['pkg']['EAPI'] = parsed_eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: sys.__stdout__.flush() sys.__stderr__.flush() break self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fd_pipes[self._metadata_fd] = slave_fd self._raw_metadata = [] files.ebuild = master_fd self._reg_id = self.scheduler.register(files.ebuild, self._registered_events, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) os.close(slave_fd) null_input.close() if isinstance(retval, int): # doebuild failed before spawning self._unregister() self._set_returncode((self.pid, retval << 8)) self.wait() return self.pid = retval[0] portage.process.spawned_pids.remove(self.pid)
def _start(self): ebuild_path = self.ebuild_hash.location with io.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f) parsed_eapi = self._eapi if parsed_eapi is None: parsed_eapi = "0" if not parsed_eapi: # An empty EAPI setting is invalid. self._eapi_invalid(None) self.returncode = 1 self._async_wait() return self.eapi_supported = portage.eapi_is_supported(parsed_eapi) if not self.eapi_supported: self.metadata = {"EAPI": parsed_eapi} self.returncode = os.EX_OK self._async_wait() return settings = self.settings settings.setcpv(self.cpv) settings.configdict['pkg']['EAPI'] = parsed_eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: sys.__stdout__.flush() sys.__stderr__.flush() break self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl(master_fd, fcntl.F_SETFD, fcntl.fcntl(master_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) fd_pipes[slave_fd] = slave_fd settings["PORTAGE_PIPE_FD"] = str(slave_fd) self._raw_metadata = [] files.ebuild = master_fd self.scheduler.add_reader(files.ebuild, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) settings.pop("PORTAGE_PIPE_FD", None) os.close(slave_fd) null_input.close() if isinstance(retval, int): # doebuild failed before spawning self.returncode = retval self._async_wait() return self.pid = retval[0]
def testDoebuild(self): """ Invoke portage.doebuild() with the fd_pipes parameter, and check that the expected output appears in the pipe. This functionality is not used by portage internally, but it is supported for API consumers (see bug #475812). """ output_fd = 200 ebuild_body = ["S=${WORKDIR}"] for phase_func in ( "pkg_info", "pkg_nofetch", "pkg_pretend", "pkg_setup", "src_unpack", "src_prepare", "src_configure", "src_compile", "src_test", "src_install", ): ebuild_body.append(("%s() { echo ${EBUILD_PHASE}" " 1>&%s; }") % (phase_func, output_fd)) ebuild_body.append("") ebuild_body = "\n".join(ebuild_body) ebuilds = { "app-misct/foo-1": { "EAPI": "5", "MISC_CONTENT": ebuild_body, } } # Override things that may be unavailable, or may have portability # issues when running tests in exotic environments. # prepstrip - bug #447810 (bash read builtin EINTR problem) true_symlinks = ("find", "prepstrip", "sed", "scanelf") true_binary = portage.process.find_binary("true") self.assertEqual(true_binary is None, False, "true command not found") dev_null = open(os.devnull, "wb") playground = ResolverPlayground(ebuilds=ebuilds) try: QueryCommand._db = playground.trees root_config = playground.trees[playground.eroot]["root_config"] portdb = root_config.trees["porttree"].dbapi settings = portage.config(clone=playground.settings) if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: settings["__PORTAGE_TEST_HARDLINK_LOCKS"] = os.environ[ "__PORTAGE_TEST_HARDLINK_LOCKS"] settings.backup_changes("__PORTAGE_TEST_HARDLINK_LOCKS") settings.features.add("noauto") settings.features.add("test") settings["PORTAGE_PYTHON"] = portage._python_interpreter settings["PORTAGE_QUIET"] = "1" settings["PYTHONDONTWRITEBYTECODE"] = os.environ.get( "PYTHONDONTWRITEBYTECODE", "") fake_bin = os.path.join(settings["EPREFIX"], "bin") portage.util.ensure_dirs(fake_bin) for x in true_symlinks: os.symlink(true_binary, os.path.join(fake_bin, x)) settings["__PORTAGE_TEST_PATH_OVERRIDE"] = fake_bin settings.backup_changes("__PORTAGE_TEST_PATH_OVERRIDE") cpv = "app-misct/foo-1" metadata = dict( zip(Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys))) pkg = Package( built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name="ebuild", ) settings.setcpv(pkg) ebuildpath = portdb.findname(cpv) self.assertNotEqual(ebuildpath, None) for phase in ( "info", "nofetch", "pretend", "setup", "unpack", "prepare", "configure", "compile", "test", "install", "qmerge", "clean", "merge", ): pr, pw = os.pipe() producer = DoebuildProcess( doebuild_pargs=(ebuildpath, phase), doebuild_kwargs={ "settings": settings, "mydbapi": portdb, "tree": "porttree", "vartree": root_config.trees["vartree"], "fd_pipes": { 1: dev_null.fileno(), 2: dev_null.fileno(), output_fd: pw, }, "prev_mtimes": {}, }, ) consumer = PipeReader(input_files={"producer": pr}) task_scheduler = TaskScheduler(iter([producer, consumer]), max_jobs=2) try: task_scheduler.start() finally: # PipeReader closes pr os.close(pw) task_scheduler.wait() output = portage._unicode_decode( consumer.getvalue()).rstrip("\n") if task_scheduler.returncode != os.EX_OK: portage.writemsg(output, noiselevel=-1) self.assertEqual(task_scheduler.returncode, os.EX_OK) if phase not in ("clean", "merge", "qmerge"): self.assertEqual(phase, output) finally: dev_null.close() playground.cleanup() QueryCommand._db = None
def make_pipes(): return os.pipe(), None
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). TODO: Share code with ForkProcess. """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() parent_pid = os.getpid() pid = None try: pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError( "fork returned non-integer: %s" % (repr(pid),)) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.locks._close_fds() # We don't exec, so use close_fds=False # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: os._exit(rval) finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1)
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None, path_lookup=True, pre_exec=None, close_fds=(sys.version_info < (3, 4)), unshare_net=False, unshare_ipc=False, unshare_mount=False, unshare_pid=False, cgroup=None): """ Spawns a given command. @param mycommand: the command to execute @type mycommand: String or List (Popen style list) @param env: If env is not None, it must be a mapping that defines the environment variables for the new process; these are used instead of the default behavior of inheriting the current process's environment. @type env: None or Mapping @param opt_name: an optional name for the spawn'd process (defaults to the binary name) @type opt_name: String @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example (default is {0:stdin, 1:stdout, 2:stderr}) @type fd_pipes: Dictionary @param returnpid: Return the Process IDs for a successful spawn. NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them. @type returnpid: Boolean @param uid: User ID to spawn as; useful for dropping privilages @type uid: Integer @param gid: Group ID to spawn as; useful for dropping privilages @type gid: Integer @param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts. @type groups: List @param umask: An integer representing the umask for the process (see man chmod for umask details) @type umask: Integer @param cwd: Current working directory @type cwd: String @param logfile: name of a file to use for logging purposes @type logfile: String @param path_lookup: If the binary is not fully specified then look for it in PATH @type path_lookup: Boolean @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @param close_fds: If True, then close all file descriptors except those referenced by fd_pipes (default is True for python3.3 and earlier, and False for python3.4 and later due to non-inheritable file descriptor behavior from PEP 446). @type close_fds: Boolean @param unshare_net: If True, networking will be unshared from the spawned process @type unshare_net: Boolean @param unshare_ipc: If True, IPC will be unshared from the spawned process @type unshare_ipc: Boolean @param unshare_mount: If True, mount namespace will be unshared and mounts will be private to the namespace @type unshare_mount: Boolean @param unshare_pid: If True, PID ns will be unshared from the spawned process @type unshare_pid: Boolean @param cgroup: CGroup path to bind the process to @type cgroup: String logfile requires stdout and stderr to be assigned to this process (ie not pointed somewhere else.) """ # mycommand is either a str or a list if isinstance(mycommand, basestring): mycommand = mycommand.split() env = os.environ if env is None else env if sys.hexversion < 0x3000000: # Avoid a potential UnicodeEncodeError from os.execve(). env_bytes = {} for k, v in env.items(): env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ _unicode_encode(v, encoding=_encodings['content']) env = env_bytes del env_bytes # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \ (not os.path.isabs(binary) or not os.path.isfile(binary) or not os.access(binary, os.X_OK)): binary = path_lookup and find_binary(binary) or None if not binary: raise CommandNotFound(mycommand[0]) # If we haven't been told what file descriptors to use # default to propagating our stdin, stdout and stderr. if fd_pipes is None: fd_pipes = { 0:portage._get_stdin().fileno(), 1:sys.__stdout__.fileno(), 2:sys.__stderr__.fileno(), } # mypids will hold the pids of all processes created. mypids = [] if logfile: # Using a log file requires that stdout and stderr # are assigned to the process we're running. if 1 not in fd_pipes or 2 not in fd_pipes: raise ValueError(fd_pipes) # Create a pipe (pr, pw) = os.pipe() # Create a tee process, giving it our stdout and stderr # as well as the read end of the pipe. mypids.extend(spawn(('tee', '-i', '-a', logfile), returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) # We don't need the read end of the pipe, so close it. os.close(pr) # Assign the write end of the pipe to our stdout and stderr. fd_pipes[1] = pw fd_pipes[2] = pw # Cache _has_ipv6() result for use in child processes. _has_ipv6() # This caches the libc library lookup and _unshare_validator results # in the current process, so that results are cached for use in # child processes. unshare_flags = 0 if unshare_net or unshare_ipc or unshare_mount or unshare_pid: # from /usr/include/bits/sched.h CLONE_NEWNS = 0x00020000 CLONE_NEWUTS = 0x04000000 CLONE_NEWIPC = 0x08000000 CLONE_NEWPID = 0x20000000 CLONE_NEWNET = 0x40000000 if unshare_net: # UTS namespace to override hostname unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS if unshare_ipc: unshare_flags |= CLONE_NEWIPC if unshare_mount: # NEWNS = mount namespace unshare_flags |= CLONE_NEWNS if unshare_pid: # we also need mount namespace for slave /proc unshare_flags |= CLONE_NEWPID | CLONE_NEWNS _unshare_validate(unshare_flags) # Force instantiation of portage.data.userpriv_groups before the # fork, so that the result is cached in the main process. bool(groups) parent_pid = os.getpid() pid = None try: pid = os.fork() if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid, unshare_flags, cgroup) except SystemExit: raise except Exception as e: # We need to catch _any_ exception so that it doesn't # propagate out of this function and cause exiting # with anything other than os._exit() writemsg("%s:\n %s\n" % (e, " ".join(mycommand)), noiselevel=-1) traceback.print_exc() sys.stderr.flush() finally: if pid == 0 or (pid is None and os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This # finally block has to be setup before the fork # in order to avoid a race condition. os._exit(1) if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) # Add the pid to our local and the global pid lists. mypids.append(pid) # If we started a tee process the write side of the pipe is no # longer needed, so close it. if logfile: os.close(pw) # If the caller wants to handle cleaning up the processes, we tell # it about all processes that were created. if returnpid: return mypids # Otherwise we clean them up. while mypids: # Pull the last reader in the pipe chain. If all processes # in the pipe are well behaved, it will die when the process # it is reading from dies. pid = mypids.pop(0) # and wait for it. retval = os.waitpid(pid, 0)[1] if retval: # If it failed, kill off anything else that # isn't dead yet. for pid in mypids: # With waitpid and WNOHANG, only check the # first element of the tuple since the second # element may vary (bug #337465). if os.waitpid(pid, os.WNOHANG)[0] == 0: os.kill(pid, signal.SIGTERM) os.waitpid(pid, 0) # If it got a signal, return the signal that was sent. if (retval & 0xff): return ((retval & 0xff) << 8) # Otherwise, return its exit code. return (retval >> 8) # Everything succeeded return 0
def _start(self): settings = self.settings settings.setcpv(self.cpv) ebuild_path = self.ebuild_hash.location # the caller can pass in eapi in order to avoid # redundant _parse_eapi_ebuild_head calls eapi = self.eapi if eapi is None and \ 'parse-eapi-ebuild-head' in settings.features: with io.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: eapi = portage._parse_eapi_ebuild_head(f) if eapi is not None: if not portage.eapi_is_supported(eapi): self.metadata = self.metadata_callback(self.cpv, self.repo_path, {'EAPI': eapi}, self.ebuild_hash) self._set_returncode((self.pid, os.EX_OK << 8)) self.wait() return settings.configdict['pkg']['EAPI'] = eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) fd_pipes.setdefault(1, sys.stdout.fileno()) fd_pipes.setdefault(2, sys.stderr.fileno()) # flush any pending output for fd in fd_pipes.values(): if fd == sys.stdout.fileno(): sys.stdout.flush() if fd == sys.stderr.fileno(): sys.stderr.flush() self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fd_pipes[self._metadata_fd] = slave_fd self._raw_metadata = [] files.ebuild = master_fd self._reg_id = self.scheduler.register(files.ebuild, self._registered_events, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) os.close(slave_fd) null_input.close() if isinstance(retval, int): # doebuild failed before spawning self._unregister() self._set_returncode((self.pid, retval << 8)) self.wait() return self.pid = retval[0] portage.process.spawned_pids.remove(self.pid)
def _start(self): ebuild_path = self.ebuild_hash.location with io.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f) parsed_eapi = self._eapi if parsed_eapi is None: parsed_eapi = "0" if not parsed_eapi: # An empty EAPI setting is invalid. self._eapi_invalid(None) self.returncode = 1 self._async_wait() return self.eapi_supported = portage.eapi_is_supported(parsed_eapi) if not self.eapi_supported: self.metadata = {"EAPI": parsed_eapi} self.returncode = os.EX_OK self._async_wait() return settings = self.settings settings.setcpv(self.cpv) settings.configdict['pkg']['EAPI'] = parsed_eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} null_input = open('/dev/null', 'rb') fd_pipes.setdefault(0, null_input.fileno()) fd_pipes.setdefault(1, sys.__stdout__.fileno()) fd_pipes.setdefault(2, sys.__stderr__.fileno()) # flush any pending output stdout_filenos = (sys.__stdout__.fileno(), sys.__stderr__.fileno()) for fd in fd_pipes.values(): if fd in stdout_filenos: sys.__stdout__.flush() sys.__stderr__.flush() break self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl( master_fd, fcntl.F_SETFD, fcntl.fcntl(master_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) fd_pipes[slave_fd] = slave_fd settings["PORTAGE_PIPE_FD"] = str(slave_fd) self._raw_metadata = [] files.ebuild = master_fd self.scheduler.add_reader(files.ebuild, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) settings.pop("PORTAGE_PIPE_FD", None) os.close(slave_fd) null_input.close() if isinstance(retval, int): # doebuild failed before spawning self.returncode = retval self._async_wait() return self.pid = retval[0]
def _start(self): settings = self.settings settings.setcpv(self.cpv) ebuild_path = self.ebuild_path eapi = None if 'parse-eapi-glep-55' in settings.features: pf, eapi = portage._split_ebuild_name_glep55( os.path.basename(ebuild_path)) if eapi is None and \ 'parse-eapi-ebuild-head' in settings.features: eapi = portage._parse_eapi_ebuild_head( codecs.open(_unicode_encode(ebuild_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace')) if eapi is not None: if not portage.eapi_is_supported(eapi): self.metadata_callback(self.cpv, self.ebuild_path, self.repo_path, {'EAPI' : eapi}, self.ebuild_mtime) self.returncode = os.EX_OK self.wait() return settings.configdict['pkg']['EAPI'] = eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None slave_fd = None fd_pipes = None if self.fd_pipes is not None: fd_pipes = self.fd_pipes.copy() else: fd_pipes = {} fd_pipes.setdefault(0, sys.stdin.fileno()) fd_pipes.setdefault(1, sys.stdout.fileno()) fd_pipes.setdefault(2, sys.stderr.fileno()) # flush any pending output for fd in fd_pipes.values(): if fd == sys.stdout.fileno(): sys.stdout.flush() if fd == sys.stderr.fileno(): sys.stderr.flush() fd_pipes_orig = fd_pipes.copy() self._files = self._files_dict() files = self._files master_fd, slave_fd = os.pipe() fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fd_pipes[self._metadata_fd] = slave_fd self._raw_metadata = [] files.ebuild = os.fdopen(master_fd, 'rb') self._reg_id = self.scheduler.register(files.ebuild.fileno(), self._registered_events, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", settings["ROOT"], settings, debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) os.close(slave_fd) if isinstance(retval, int): # doebuild failed before spawning self._unregister() self.returncode = retval self.wait() return self.pid = retval[0] portage.process.spawned_pids.remove(self.pid)
def _pipe(self, fd_pipes): """ @type fd_pipes: dict @param fd_pipes: pipes from which to copy terminal size if desired. """ return os.pipe()
def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call dblink.merge(). """ elog_reader_fd, elog_writer_fd = os.pipe() fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) blockers = None if self.blockers is not None: # Query blockers in the main process, since closing # of file descriptors in the subprocess can prevent # access to open database connections such as that # used by the sqlite metadata cache module. blockers = self.blockers() mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings, treetype=self.treetype, vartree=self.vartree, blockers=blockers, scheduler=self.scheduler, pipe=elog_writer_fd) fd_pipes[elog_writer_fd] = elog_writer_fd self._elog_reg_id = self.scheduler.register(elog_reader_fd, self._registered_events, self._elog_output_handler) # If a concurrent emerge process tries to install a package # in the same SLOT as this one at the same time, there is an # extremely unlikely chance that the COUNTER values will not be # ordered correctly unless we lock the vdb here. # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() counter = None if not self.unmerge: counter = self.vartree.dbapi.counter_tick() pid = os.fork() if pid != 0: if not isinstance(pid, int): raise AssertionError("fork returned non-integer: %s" % (repr(pid), )) os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" self._elog_keys = set() # invalidate relevant vardbapi caches if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None self.vartree.dbapi._pkgs_changed = True self.vartree.dbapi._clear_pkg_cache(mylink) portage.process.spawned_pids.append(pid) return [pid] os.close(elog_reader_fd) # TODO: Find out why PyPy 1.8 with close_fds=True triggers # "[Errno 9] Bad file descriptor" in subprocesses. It could # be due to garbage collection of file objects that were not # closed before going out of scope, since PyPy's garbage # collector does not support the refcounting semantics that # CPython does. close_fds = platform.python_implementation() != 'PyPy' portage.process._setup_pipes(fd_pipes, close_fds=close_fds) # Use default signal handlers since the ones inherited # from the parent process are irrelevant here. signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') # In this subprocess we want mylink._display_merge() to use # stdout/stderr directly since they are pipes. This behavior # is triggered when mylink._scheduler is None. mylink._scheduler = None # Avoid wastful updates of the vdb cache. self.vartree.dbapi._flush_cache_enabled = False # In this subprocess we don't want PORTAGE_BACKGROUND to # suppress stdout/stderr output since they are pipes. We # also don't want to open PORTAGE_LOG_FILE, since it will # already be opened by the parent process, so we set the # "subprocess" value for use in conditional logging code # involving PORTAGE_LOG_FILE. if not self.unmerge: # unmerge phases have separate logs if self.settings.get("PORTAGE_BACKGROUND") == "1": self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1" else: self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0" self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE") self.settings["PORTAGE_BACKGROUND"] = "subprocess" self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 try: if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( ldpath_mtimes=self.prev_mtimes) == os.EX_OK: mylink.lockdb() try: mylink.delete() finally: mylink.unlockdb() rval = os.EX_OK else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) except SystemExit: raise except: traceback.print_exc() finally: # Call os._exit() from finally block, in order to suppress any # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval)
def testDoebuild(self): """ Invoke portage.doebuild() with the fd_pipes parameter, and check that the expected output appears in the pipe. This functionality is not used by portage internally, but it is supported for API consumers (see bug #475812). """ output_fd = 200 ebuild_body = ['S=${WORKDIR}'] for phase_func in ('pkg_info', 'pkg_nofetch', 'pkg_pretend', 'pkg_setup', 'src_unpack', 'src_prepare', 'src_configure', 'src_compile', 'src_test', 'src_install'): ebuild_body.append(('%s() { echo ${EBUILD_PHASE}' ' 1>&%s; }') % (phase_func, output_fd)) ebuild_body.append('') ebuild_body = '\n'.join(ebuild_body) ebuilds = { 'app-misct/foo-1': { 'EAPI' : '5', "MISC_CONTENT": ebuild_body, } } # Override things that may be unavailable, or may have portability # issues when running tests in exotic environments. # prepstrip - bug #447810 (bash read builtin EINTR problem) true_symlinks = ("find", "prepstrip", "sed", "scanelf") true_binary = portage.process.find_binary("true") self.assertEqual(true_binary is None, False, "true command not found") dev_null = open(os.devnull, 'wb') playground = ResolverPlayground(ebuilds=ebuilds) try: QueryCommand._db = playground.trees root_config = playground.trees[playground.eroot]['root_config'] portdb = root_config.trees["porttree"].dbapi settings = portage.config(clone=playground.settings) if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: settings["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] settings.backup_changes("__PORTAGE_TEST_HARDLINK_LOCKS") settings.features.add("noauto") settings.features.add("test") settings['PORTAGE_PYTHON'] = portage._python_interpreter settings['PORTAGE_QUIET'] = "1" settings['PYTHONDONTWRITEBYTECODE'] = os.environ.get("PYTHONDONTWRITEBYTECODE", "") fake_bin = os.path.join(settings["EPREFIX"], "bin") portage.util.ensure_dirs(fake_bin) for x in true_symlinks: os.symlink(true_binary, os.path.join(fake_bin, x)) settings["__PORTAGE_TEST_PATH_OVERRIDE"] = fake_bin settings.backup_changes("__PORTAGE_TEST_PATH_OVERRIDE") cpv = 'app-misct/foo-1' metadata = dict(zip(Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys))) pkg = Package(built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name='ebuild') settings.setcpv(pkg) ebuildpath = portdb.findname(cpv) self.assertNotEqual(ebuildpath, None) for phase in ('info', 'nofetch', 'pretend', 'setup', 'unpack', 'prepare', 'configure', 'compile', 'test', 'install', 'qmerge', 'clean', 'merge'): pr, pw = os.pipe() producer = DoebuildProcess(doebuild_pargs=(ebuildpath, phase), doebuild_kwargs={"settings" : settings, "mydbapi": portdb, "tree": "porttree", "vartree": root_config.trees["vartree"], "fd_pipes": { 1: dev_null.fileno(), 2: dev_null.fileno(), output_fd: pw, }, "prev_mtimes": {}}) consumer = PipeReader( input_files={"producer" : pr}) task_scheduler = TaskScheduler(iter([producer, consumer]), max_jobs=2) try: task_scheduler.start() finally: # PipeReader closes pr os.close(pw) task_scheduler.wait() output = portage._unicode_decode( consumer.getvalue()).rstrip("\n") if task_scheduler.returncode != os.EX_OK: portage.writemsg(output, noiselevel=-1) self.assertEqual(task_scheduler.returncode, os.EX_OK) if phase not in ('clean', 'merge', 'qmerge'): self.assertEqual(phase, output) finally: dev_null.close() playground.cleanup() QueryCommand._db = None