async def _testPipeLoggerToPipe(self, test_string, loop):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
to a pipe that's drained by a PipeReader running in the same
process (requires non-blocking write).
"""
input_fd, writer_pipe = os.pipe()
_set_nonblocking(writer_pipe)
writer_pipe = os.fdopen(writer_pipe, 'wb', 0)
writer = asyncio.ensure_future(
_writer(writer_pipe, test_string.encode('ascii')))
writer.add_done_callback(lambda writer: writer_pipe.close())
pr, pw = os.pipe()
consumer = PipeLogger(background=True,
input_fd=input_fd,
log_file_path=os.fdopen(pw, 'wb', 0),
scheduler=loop)
consumer.start()
# Before starting the reader, wait here for a moment, in order
# to exercise PipeLogger's handling of EAGAIN during write.
await asyncio.wait([writer], timeout=0.01)
reader = _reader(pr)
await writer
content = await reader
await consumer.async_wait()
self.assertEqual(consumer.returncode, os.EX_OK)
return content.decode('ascii', 'replace')
def _async_start(self):
pipe_logger = None
filter_proc = None
try:
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = yield asyncio.create_subprocess_exec(
*log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
loop=self.scheduler)
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, 'wb', 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, 'wb', 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path)
yield pipe_logger.async_start()
except asyncio.CancelledError:
if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
if filter_proc is not None and filter_proc.returncode is None:
filter_proc.terminate()
raise
self._main_task = asyncio.ensure_future(self._main(
pipe_logger, filter_proc=filter_proc),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def _start(self):
filter_proc = None
log_input = None
if self.log_path is not None:
log_filter_file = self.log_filter_file
if log_filter_file is not None:
split_value = shlex_split(log_filter_file)
log_filter_file = split_value if split_value else None
if log_filter_file:
filter_input, stdin = os.pipe()
log_input, filter_output = os.pipe()
try:
filter_proc = PopenProcess(
proc=subprocess.Popen(
log_filter_file,
env=self.env,
stdin=filter_input,
stdout=filter_output,
stderr=filter_output,
),
scheduler=self.scheduler,
)
filter_proc.start()
except EnvironmentError:
# Maybe the command is missing or broken somehow...
os.close(filter_input)
os.close(stdin)
os.close(log_input)
os.close(filter_output)
else:
self._stdin = os.fdopen(stdin, "wb", 0)
os.close(filter_input)
os.close(filter_output)
if self._stdin is None:
# Since log_filter_file is unspecified or refers to a file
# that is missing or broken somehow, create a pipe that
# logs directly to pipe_logger.
log_input, stdin = os.pipe()
self._stdin = os.fdopen(stdin, "wb", 0)
# Set background=True so that pipe_logger does not log to stdout.
pipe_logger = PipeLogger(
background=True,
scheduler=self.scheduler,
input_fd=log_input,
log_file_path=self.log_path,
)
pipe_logger.start()
self._main_task_cancel = functools.partial(self._main_cancel,
filter_proc, pipe_logger)
self._main_task = asyncio.ensure_future(self._main(
filter_proc, pipe_logger),
loop=self.scheduler)
self._main_task.add_done_callback(self._main_exit)
def testLazyImportPortageBaseline(self):
"""
Check what modules are imported by a baseline module import.
"""
env = os.environ.copy()
pythonpath = env.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env["PYTHONPATH"] = pythonpath
# If python is patched to insert the path of the
# currently installed portage module into sys.path,
# then the above PYTHONPATH override doesn't help.
env["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH
scheduler = global_event_loop()
master_fd, slave_fd = os.pipe()
master_file = os.fdopen(master_fd, "rb", 0)
slave_file = os.fdopen(slave_fd, "wb")
producer = SpawnProcess(
args=self._baseline_import_cmd,
env=env,
fd_pipes={1: slave_fd},
scheduler=scheduler,
)
producer.start()
slave_file.close()
consumer = PipeReader(input_files={"producer": master_file},
scheduler=scheduler)
consumer.start()
consumer.wait()
self.assertEqual(producer.wait(), os.EX_OK)
self.assertEqual(consumer.wait(), os.EX_OK)
output = consumer.getvalue().decode("ascii", "replace").split()
unexpected_modules = " ".join(
sorted(x for x in output if self._module_re.match(x) is not None
and x not in self._baseline_imports))
self.assertEqual("", unexpected_modules)
def testLazyImportPortageBaseline(self):
"""
Check what modules are imported by a baseline module import.
"""
env = os.environ.copy()
pythonpath = env.get('PYTHONPATH')
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is None:
pythonpath = ''
else:
pythonpath = ':' + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env['PYTHONPATH'] = pythonpath
# If python is patched to insert the path of the
# currently installed portage module into sys.path,
# then the above PYTHONPATH override doesn't help.
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
scheduler = PollScheduler().sched_iface
master_fd, slave_fd = os.pipe()
master_file = os.fdopen(master_fd, 'rb', 0)
slave_file = os.fdopen(slave_fd, 'wb')
producer = SpawnProcess(
args=self._baseline_import_cmd,
env=env, fd_pipes={1:slave_fd},
scheduler=scheduler)
producer.start()
slave_file.close()
consumer = PipeReader(
input_files={"producer" : master_file},
scheduler=scheduler)
consumer.start()
consumer.wait()
self.assertEqual(producer.wait(), os.EX_OK)
self.assertEqual(consumer.wait(), os.EX_OK)
output = consumer.getvalue().decode('ascii', 'replace').split()
unexpected_modules = " ".join(sorted(x for x in output \
if self._module_re.match(x) is not None and \
x not in self._baseline_imports))
self.assertEqual("", unexpected_modules)
def add_manifest(self, mymanifests, myheaders, myupdates, myremoved,
commitmessage):
myfiles = mymanifests[:]
# If there are no header (SVN/CVS keywords) changes in
# the files, this Manifest commit must include the
# other (yet uncommitted) files.
if not myheaders:
myfiles += myupdates
myfiles += myremoved
myfiles.sort()
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile)
# cleanup the commit message before possibly exiting
try:
os.unlink(commitmessagefile)
except OSError:
pass
if retval != os.EX_OK:
writemsg_level("!!! Exiting on %s (shell) "
"error code: %s\n" %
(self.vcs_settings.vcs, retval),
level=logging.ERROR,
noiselevel=-1)
sys.exit(retval)
def _start(self):
pr, pw = os.pipe()
self._files = {}
self._files['pipe_read'] = os.fdopen(pr, 'rb', 0)
self._files['pipe_write'] = os.fdopen(pw, 'wb', 0)
for k, f in self._files.items():
fcntl.fcntl(f.fileno(), fcntl.F_SETFL,
fcntl.fcntl(f.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
self._reg_id = self.scheduler.register(self._files['pipe_read'].fileno(),
PollConstants.POLLIN, self._output_handler)
self._registered = True
threading_mod = threading
if self._force_dummy:
threading_mod = dummy_threading
self._thread = threading_mod.Thread(target=self._run_lock)
self._thread.start()
def priming_commit(self, myupdates, myremoved, commitmessage):
myfiles = myupdates + myremoved
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
separator = '-' * 78
print()
print(green("Using commit message:"))
print(green(separator))
print(commitmessage)
print(green(separator))
print()
# Having a leading ./ prefix on file paths can trigger a bug in
# the cvs server when committing files to multiple directories,
# so strip the prefix.
myfiles = [f.lstrip("./") for f in myfiles]
retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile)
# cleanup the commit message before possibly exiting
try:
os.unlink(commitmessagefile)
except OSError:
pass
if retval != os.EX_OK:
writemsg_level("!!! Exiting on %s (shell) "
"error code: %s\n" %
(self.vcs_settings.vcs, retval),
level=logging.ERROR,
noiselevel=-1)
sys.exit(retval)
def _testPipeReader(self, master_fd, slave_fd, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
scheduler = global_event_loop()
consumer = PipeReader(input_files={"producer": master_file},
_use_array=self._use_array,
scheduler=scheduler)
consumer.start()
producer = scheduler.run_until_complete(
asyncio.create_subprocess_exec("bash",
"-c",
self._echo_cmd % test_string,
stdout=slave_fd,
loop=scheduler))
os.close(slave_fd)
scheduler.run_until_complete(producer.wait())
scheduler.run_until_complete(consumer.async_wait())
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
return consumer.getvalue().decode('ascii', 'replace')
def add_manifest(self, mymanifests, myheaders, myupdates, myremoved,
commitmessage):
myfiles = mymanifests[:]
# If there are no header (SVN/CVS keywords) changes in
# the files, this Manifest commit must include the
# other (yet uncommitted) files.
if not myheaders:
myfiles += myupdates
myfiles += myremoved
myfiles.sort()
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile)
# cleanup the commit message before possibly exiting
try:
os.unlink(commitmessagefile)
except OSError:
pass
if retval != os.EX_OK:
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" % (self.vcs_settings.vcs, retval),
level=logging.ERROR, noiselevel=-1)
sys.exit(retval)
def _testPipeReader(self, master_fd, slave_fd, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
scheduler = global_event_loop()
consumer = PipeReader(
input_files={"producer" : master_file},
_use_array=self._use_array,
scheduler=scheduler)
producer = PopenProcess(
pipe_reader=consumer,
proc=subprocess.Popen(["bash", "-c", self._echo_cmd % test_string],
stdout=slave_fd),
scheduler=scheduler)
producer.start()
os.close(slave_fd)
producer.wait()
consumer.wait()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
return consumer.getvalue().decode('ascii', 'replace')
def setUp(self):
fd, self.testfile = tempfile.mkstemp(suffix=".testdata",
prefix=self.__class__.__name__,
text=True)
f = os.fdopen(fd, "w")
f.write("\n".join(test_cps))
f.close()
def priming_commit(self, myupdates, myremoved, commitmessage):
myfiles = myupdates + myremoved
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
separator = '-' * 78
print()
print(green("Using commit message:"))
print(green(separator))
print(commitmessage)
print(green(separator))
print()
# Having a leading ./ prefix on file paths can trigger a bug in
# the cvs server when committing files to multiple directories,
# so strip the prefix.
myfiles = [f.lstrip("./") for f in myfiles]
retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile)
# cleanup the commit message before possibly exiting
try:
os.unlink(commitmessagefile)
except OSError:
pass
if retval != os.EX_OK:
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" % (self.vcs_settings.vcs, retval),
level=logging.ERROR, noiselevel=-1)
sys.exit(retval)
def _testPipeLoggerToPipe(self, test_string, loop=None):
"""
Test PipeLogger writing to a pipe connected to a PipeReader.
This verifies that PipeLogger does not deadlock when writing
to a pipe that's drained by a PipeReader running in the same
process (requires non-blocking write).
"""
producer = PopenProcess(proc=subprocess.Popen(
["bash", "-c", self._echo_cmd % test_string],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=loop)
pr, pw = os.pipe()
consumer = producer.pipe_reader = PipeLogger(
background=True,
input_fd=producer.proc.stdout,
log_file_path=os.fdopen(pw, 'wb', 0))
reader = _reader(pr, loop=loop)
yield producer.async_start()
content = yield reader
yield producer.async_wait()
yield consumer.async_wait()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
coroutine_return(content.decode('ascii', 'replace'))
def _testPipeReader(self, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
if self._use_pty:
got_pty, master_fd, slave_fd = _create_pty_or_pipe()
if not got_pty:
os.close(slave_fd)
os.close(master_fd)
skip_reason = "pty not acquired"
self.portage_skip = skip_reason
self.fail(skip_reason)
return
else:
master_fd, slave_fd = os.pipe()
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
slave_file = os.fdopen(slave_fd, 'wb', 0)
task_scheduler = TaskScheduler(max_jobs=2)
producer = SpawnProcess(
args=["bash", "-c", self._echo_cmd % test_string],
env=os.environ,
fd_pipes={1: slave_fd},
scheduler=task_scheduler.sched_iface)
task_scheduler.add(producer)
slave_file.close()
consumer = PipeReader(input_files={"producer": master_file},
scheduler=task_scheduler.sched_iface,
_use_array=self._use_array)
task_scheduler.add(consumer)
# This will ensure that both tasks have exited, which
# is necessary to avoid "ResourceWarning: unclosed file"
# warnings since Python 3.2 (and also ensures that we
# don't leave any zombie child processes).
task_scheduler.run()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
return consumer.getvalue().decode('ascii', 'replace')
def _testPipeReader(self, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
if self._use_pty:
got_pty, master_fd, slave_fd = _create_pty_or_pipe()
if not got_pty:
os.close(slave_fd)
os.close(master_fd)
skip_reason = "pty not acquired"
self.portage_skip = skip_reason
self.fail(skip_reason)
return
else:
master_fd, slave_fd = os.pipe()
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
slave_file = os.fdopen(slave_fd, 'wb', 0)
task_scheduler = TaskScheduler(max_jobs=2)
producer = SpawnProcess(
args=["bash", "-c", self._echo_cmd % test_string],
env=os.environ, fd_pipes={1:slave_fd},
scheduler=task_scheduler.sched_iface)
task_scheduler.add(producer)
slave_file.close()
consumer = PipeReader(
input_files={"producer" : master_file},
scheduler=task_scheduler.sched_iface, _use_array=self._use_array)
task_scheduler.add(consumer)
# This will ensure that both tasks have exited, which
# is necessary to avoid "ResourceWarning: unclosed file"
# warnings since Python 3.2 (and also ensures that we
# don't leave any zombie child processes).
task_scheduler.run()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
return consumer.getvalue().decode('ascii', 'replace')
def _processItem(self, item):
filename = None
fd, filename = mkstemp()
f = os.fdopen(fd, 'w')
f.write(item)
f.close()
try:
return NewsItem(filename, 0)
except TypeError:
self.fail("Error while processing news item %s" % filename)
def _start(self):
self._files = self._files_dict()
input_fd = os.open(self.input_fifo, os.O_RDONLY | os.O_NONBLOCK)
self._files.pipe_in = os.fdopen(input_fd, 'rb')
self._reg_id = self.scheduler.register(self._files.pipe_in.fileno(),
self._registered_events,
self._input_handler)
self._registered = True
def _start(self): self._files = self._files_dict() input_fd = os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) self._files.pipe_in = os.fdopen(input_fd, 'rb') self._reg_id = self.scheduler.register( self._files.pipe_in.fileno(), self._registered_events, self._input_handler) self._registered = True
def setUp(self): fd, self.testfile = tempfile.mkstemp(suffix=".testdata", prefix=self.__class__.__name__, text=True) f = os.fdopen(fd, 'w') for i in range(0, len(test_cps)): atom = test_cps[i] if i % 2 == 0: f.write(atom + ' abc def\n') else: f.write(atom + '\n') f.close()
def _processItem(self, item):
filename = None
fd, filename = mkstemp()
f = os.fdopen(fd, 'w')
f.write(item)
f.close()
try:
return NewsItem(filename, 0)
except TypeError:
self.fail("Error while processing news item %s" % filename)
def add_manifest(self, mymanifests, myheaders, myupdates, myremoved,
commitmessage):
myfiles = mymanifests[:]
# If there are no header (SVN/CVS keywords) changes in
# the files, this Manifest commit must include the
# other (yet uncommitted) files.
if not myheaders:
myfiles += myupdates
myfiles += myremoved
myfiles.sort()
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
commit_cmd = []
if self.options.pretend and self.vcs_settings.vcs is None:
# substitute a bogus value for pretend output
commit_cmd.append("cvs")
else:
commit_cmd.append(self.vcs_settings.vcs)
commit_cmd.extend(self.vcs_settings.vcs_global_opts)
commit_cmd.append("commit")
commit_cmd.extend(self.vcs_settings.vcs_local_opts)
if self.vcs_settings.vcs == "hg":
commit_cmd.extend(["--logfile", commitmessagefile])
commit_cmd.extend(myfiles)
else:
commit_cmd.extend(["-F", commitmessagefile])
commit_cmd.extend(f.lstrip("./") for f in myfiles)
try:
if self.options.pretend:
print("(%s)" % (" ".join(commit_cmd), ))
else:
retval = spawn(commit_cmd, env=self.repo_settings.commit_env)
if retval != os.EX_OK:
if self.repo_settings.repo_config.sign_commit and self.vcs_settings.vcs == 'git' and \
not git_supports_gpg_sign():
# Inform user that newer git is needed (bug #403323).
logging.error(
"Git >=1.7.9 is required for signed commits!")
writemsg_level("!!! Exiting on %s (shell) "
"error code: %s\n" %
(self.vcs_settings.vcs, retval),
level=logging.ERROR,
noiselevel=-1)
sys.exit(retval)
finally:
try:
os.unlink(commitmessagefile)
except OSError:
pass
def add_manifest(self, mymanifests, myheaders, myupdates, myremoved,
commitmessage):
myfiles = mymanifests[:]
# If there are no header (SVN/CVS keywords) changes in
# the files, this Manifest commit must include the
# other (yet uncommitted) files.
if not myheaders:
myfiles += myupdates
myfiles += myremoved
myfiles.sort()
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
commit_cmd = []
if self.options.pretend and self.vcs_settings.vcs is None:
# substitute a bogus value for pretend output
commit_cmd.append("cvs")
else:
commit_cmd.append(self.vcs_settings.vcs)
commit_cmd.extend(self.vcs_settings.vcs_global_opts)
commit_cmd.append("commit")
commit_cmd.extend(self.vcs_settings.vcs_local_opts)
if self.vcs_settings.vcs == "hg":
commit_cmd.extend(["--logfile", commitmessagefile])
commit_cmd.extend(myfiles)
else:
commit_cmd.extend(["-F", commitmessagefile])
commit_cmd.extend(f.lstrip("./") for f in myfiles)
try:
if self.options.pretend:
print("(%s)" % (" ".join(commit_cmd),))
else:
retval = spawn(commit_cmd, env=self.repo_settings.commit_env)
if retval != os.EX_OK:
if self.repo_settings.repo_config.sign_commit and self.vcs_settings.vcs == 'git' and \
not git_supports_gpg_sign():
# Inform user that newer git is needed (bug #403323).
logging.error(
"Git >=1.7.9 is required for signed commits!")
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" % (self.vcs_settings.vcs, retval),
level=logging.ERROR, noiselevel=-1)
sys.exit(retval)
finally:
try:
os.unlink(commitmessagefile)
except OSError:
pass
def setUp(self):
fd, self.testfile = tempfile.mkstemp(
suffix=".testdata", prefix=self.__class__.__name__, text=True
)
f = os.fdopen(fd, "w")
for i in range(0, len(test_cps)):
atom = test_cps[i]
if i % 2 == 0:
f.write(atom + " abc def\n")
else:
f.write(atom + "\n")
f.close()
def _reopen_input(self): """ Re-open the input stream, in order to suppress POLLHUP events (bug #339976). """ self._files.pipe_in.close() input_fd = os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) self._files.pipe_in = os.fdopen(input_fd, 'rb', 0) self.scheduler.unregister(self._reg_id) self._reg_id = self.scheduler.register( self._files.pipe_in.fileno(), self._registered_events, self._input_handler)
def _start(self):
in_pr, in_pw = os.pipe()
out_pr, out_pw = os.pipe()
self._files = {}
self._files['pipe_in'] = os.fdopen(in_pr, 'rb', 0)
self._files['pipe_out'] = os.fdopen(out_pw, 'wb', 0)
fcntl.fcntl(in_pr, fcntl.F_SETFL,
fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK)
self._reg_id = self.scheduler.register(in_pr,
PollConstants.POLLIN, self._output_handler)
self._registered = True
self._proc = SpawnProcess(
args=[portage._python_interpreter,
os.path.join(portage._bin_path, 'lock-helper.py'), self.path],
env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path),
fd_pipes={0:out_pr, 1:in_pw, 2:sys.stderr.fileno()},
scheduler=self.scheduler)
self._proc.addExitListener(self._proc_exit)
self._proc.start()
os.close(out_pr)
os.close(in_pw)
def _start(self): self._files = self._files_dict() input_fd = os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) # File streams are in unbuffered mode since we do atomic # read and write of whole pickles. self._files.pipe_in = os.fdopen(input_fd, 'rb', 0) self._reg_id = self.scheduler.register( self._files.pipe_in.fileno(), self._registered_events, self._input_handler) self._registered = True
def testPipeReader(self):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
test_string = 2 * "blah blah blah\n"
scheduler = PollScheduler().sched_iface
master_fd, slave_fd = os.pipe()
master_file = os.fdopen(master_fd, 'rb', 0)
slave_file = os.fdopen(slave_fd, 'wb')
producer = SpawnProcess(
args=["bash", "-c", "echo -n '%s'" % test_string],
env=os.environ, fd_pipes={1:slave_fd},
scheduler=scheduler)
producer.start()
slave_file.close()
consumer = PipeReader(
input_files={"producer" : master_file},
scheduler=scheduler)
consumer.start()
# This will ensure that both tasks have exited, which
# is necessary to avoid "ResourceWarning: unclosed file"
# warnings since Python 3.2 (and also ensures that we
# don't leave any zombie child processes).
scheduler.schedule()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
output = consumer.getvalue().decode('ascii', 'replace')
self.assertEqual(test_string, output)
def _start(self):
log_file_path = self.log_file_path
if hasattr(log_file_path, 'write'):
self._log_file_nb = True
self._log_file = log_file_path
_set_nonblocking(self._log_file.fileno())
elif log_file_path is not None:
try:
self._log_file = open(
_unicode_encode(log_file_path,
encoding=_encodings["fs"],
errors="strict"),
mode="ab",
)
if log_file_path.endswith(".gz"):
self._log_file_real = self._log_file
self._log_file = gzip.GzipFile(filename="",
mode="ab",
fileobj=self._log_file)
portage.util.apply_secpass_permissions(
log_file_path,
uid=portage.portage_uid,
gid=portage.portage_gid,
mode=0o660,
)
except FileNotFoundError:
if self._was_cancelled():
self._async_wait()
return
raise
if isinstance(self.input_fd, int):
self.input_fd = os.fdopen(self.input_fd, 'rb', 0)
fd = self.input_fd.fileno()
fcntl.fcntl(fd, fcntl.F_SETFL,
fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
self._io_loop_task = asyncio.ensure_future(self._io_loop(
self.input_fd),
loop=self.scheduler)
self._io_loop_task.add_done_callback(self._io_loop_done)
self._registered = True
def priming_commit(self, myupdates, myremoved, commitmessage):
myfiles = myupdates + myremoved
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
separator = '-' * 78
print()
print(green("Using commit message:"))
print(green(separator))
print(commitmessage)
print(green(separator))
print()
# Having a leading ./ prefix on file paths can trigger a bug in
# the cvs server when committing files to multiple directories,
# so strip the prefix.
myfiles = [f.lstrip("./") for f in myfiles]
commit_cmd = [self.vcs_settings.vcs]
commit_cmd.extend(self.vcs_settings.vcs_global_opts)
commit_cmd.append("commit")
commit_cmd.extend(self.vcs_settings.vcs_local_opts)
commit_cmd.extend(["-F", commitmessagefile])
commit_cmd.extend(myfiles)
try:
if self.options.pretend:
print("(%s)" % (" ".join(commit_cmd), ))
else:
retval = spawn(commit_cmd, env=self.repo_settings.commit_env)
if retval != os.EX_OK:
writemsg_level("!!! Exiting on %s (shell) "
"error code: %s\n" %
(self.vcs_settings.vcs, retval),
level=logging.ERROR,
noiselevel=-1)
sys.exit(retval)
finally:
try:
os.unlink(commitmessagefile)
except OSError:
pass
def priming_commit(self, myupdates, myremoved, commitmessage):
myfiles = myupdates + myremoved
fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
mymsg = os.fdopen(fd, "wb")
mymsg.write(_unicode_encode(commitmessage))
mymsg.close()
separator = '-' * 78
print()
print(green("Using commit message:"))
print(green(separator))
print(commitmessage)
print(green(separator))
print()
# Having a leading ./ prefix on file paths can trigger a bug in
# the cvs server when committing files to multiple directories,
# so strip the prefix.
myfiles = [f.lstrip("./") for f in myfiles]
commit_cmd = [self.vcs_settings.vcs]
commit_cmd.extend(self.vcs_settings.vcs_global_opts)
commit_cmd.append("commit")
commit_cmd.extend(self.vcs_settings.vcs_local_opts)
commit_cmd.extend(["-F", commitmessagefile])
commit_cmd.extend(myfiles)
try:
if self.options.pretend:
print("(%s)" % (" ".join(commit_cmd),))
else:
retval = spawn(commit_cmd, env=self.repo_settings.commit_env)
if retval != os.EX_OK:
writemsg_level(
"!!! Exiting on %s (shell) "
"error code: %s\n" % (self.vcs_settings.vcs, retval),
level=logging.ERROR, noiselevel=-1)
sys.exit(retval)
finally:
try:
os.unlink(commitmessagefile)
except OSError:
pass
def _testPipeReader(self, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
if self._use_pty:
got_pty, master_fd, slave_fd = _create_pty_or_pipe()
if not got_pty:
os.close(slave_fd)
os.close(master_fd)
skip_reason = "pty not acquired"
self.portage_skip = skip_reason
self.fail(skip_reason)
return
else:
master_fd, slave_fd = os.pipe()
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
scheduler = global_event_loop()
consumer = PipeReader(
input_files={"producer" : master_file},
_use_array=self._use_array,
scheduler=scheduler)
producer = PopenProcess(
pipe_reader=consumer,
proc=subprocess.Popen(["bash", "-c", self._echo_cmd % test_string],
stdout=slave_fd),
scheduler=scheduler)
producer.start()
os.close(slave_fd)
producer.wait()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
return consumer.getvalue().decode('ascii', 'replace')
def _start(self):
log_file_path = self.log_file_path
if hasattr(log_file_path, 'write'):
self._log_file_nb = True
self._log_file = log_file_path
_set_nonblocking(self._log_file.fileno())
elif log_file_path is not None:
self._log_file = open(_unicode_encode(log_file_path,
encoding=_encodings['fs'], errors='strict'), mode='ab')
if log_file_path.endswith('.gz'):
self._log_file_real = self._log_file
self._log_file = gzip.GzipFile(filename='', mode='ab',
fileobj=self._log_file)
portage.util.apply_secpass_permissions(log_file_path,
uid=portage.portage_uid, gid=portage.portage_gid,
mode=0o660)
if isinstance(self.input_fd, int):
self.input_fd = os.fdopen(self.input_fd, 'rb', 0)
fd = self.input_fd.fileno()
fcntl.fcntl(fd, fcntl.F_SETFL,
fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(fd, fcntl.F_SETFD,
fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
self._io_loop_task = asyncio.ensure_future(self._io_loop(self.input_fd), loop=self.scheduler)
self._io_loop_task.add_done_callback(self._io_loop_done)
self._registered = True
def testPipeReader(self):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
test_string = 2 * "blah blah blah\n"
master_fd, slave_fd = self._create_pipe()
master_file = os.fdopen(master_fd, 'rb')
task_scheduler = TaskScheduler(max_jobs=2)
scheduler = task_scheduler.sched_iface
class Producer(SpawnProcess):
def _spawn(self, args, **kwargs):
rval = SpawnProcess._spawn(self, args, **kwargs)
os.close(kwargs['fd_pipes'][1])
return rval
producer = Producer(
args=["bash", "-c", "echo -n '%s'" % test_string],
fd_pipes={1:slave_fd}, scheduler=scheduler)
consumer = PipeReader(
input_files={"producer" : master_file},
scheduler=scheduler)
task_scheduler.add(producer)
task_scheduler.add(consumer)
task_scheduler.run()
if sys.hexversion >= 0x3000000:
test_string = test_string.encode()
self._assertEqual(test_string, consumer.getvalue())
def _start(self):
if self.cancelled:
return
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
fd_pipes.setdefault(0, sys.stdin.fileno())
fd_pipes.setdefault(1, sys.stdout.fileno())
fd_pipes.setdefault(2, sys.stderr.fileno())
# flush any pending output
for fd in fd_pipes.values():
if fd == sys.stdout.fileno():
sys.stdout.flush()
if fd == sys.stderr.fileno():
sys.stderr.flush()
self._files = self._files_dict()
files = self._files
master_fd, slave_fd = self._pipe(fd_pipes)
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
logfile = None
if self._can_log(slave_fd):
logfile = self.logfile
null_input = None
fd_pipes_orig = fd_pipes.copy()
if self.background:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = open('/dev/null', 'rb')
fd_pipes[0] = null_input.fileno()
else:
fd_pipes[0] = fd_pipes_orig[0]
files.process = os.fdopen(master_fd, 'rb')
if logfile is not None:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
files.log = open(_unicode_encode(logfile,
encoding=_encodings['fs'],
errors='strict'),
mode='ab')
if logfile.endswith('.gz'):
files.log = gzip.GzipFile(filename='',
mode='ab',
fileobj=files.log)
portage.util.apply_secpass_permissions(logfile,
uid=portage.portage_uid,
gid=portage.portage_gid,
mode=0o660)
if not self.background:
files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb')
output_handler = self._output_handler
else:
# Create a dummy pipe so the scheduler can monitor
# the process from inside a poll() loop.
fd_pipes[self._dummy_pipe_fd] = slave_fd
if self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
output_handler = self._dummy_handler
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
self._reg_id = self.scheduler.register(files.process.fileno(),
self._registered_events,
output_handler)
self._registered = True
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
null_input.close()
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, 'w')
f.write("%s %s" % (self.cpv, ' '.join(self.useflags)))
f.close()
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, 'w')
f.write("%s %s" % (self.cpv, ' '.join(self.useflags)))
f.close()
def _send_reply(self, reply): output_fd = os.open(self.output_fifo, os.O_WRONLY|os.O_NONBLOCK) output_file = os.fdopen(output_fd, 'wb') pickle.dump(reply, output_file) output_file.close()
def _start(self):
settings = self.settings
settings.setcpv(self.cpv)
ebuild_path = self.ebuild_path
eapi = None
if 'parse-eapi-glep-55' in settings.features:
pf, eapi = portage._split_ebuild_name_glep55(
os.path.basename(ebuild_path))
if eapi is None and \
'parse-eapi-ebuild-head' in settings.features:
eapi = portage._parse_eapi_ebuild_head(
codecs.open(_unicode_encode(ebuild_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace'))
if eapi is not None:
if not portage.eapi_is_supported(eapi):
self.metadata_callback(self.cpv, self.ebuild_path,
self.repo_path, {'EAPI' : eapi}, self.ebuild_mtime)
self.returncode = os.EX_OK
self.wait()
return
settings.configdict['pkg']['EAPI'] = eapi
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
fd_pipes = None
if self.fd_pipes is not None:
fd_pipes = self.fd_pipes.copy()
else:
fd_pipes = {}
fd_pipes.setdefault(0, sys.stdin.fileno())
fd_pipes.setdefault(1, sys.stdout.fileno())
fd_pipes.setdefault(2, sys.stderr.fileno())
# flush any pending output
for fd in fd_pipes.values():
if fd == sys.stdout.fileno():
sys.stdout.flush()
if fd == sys.stderr.fileno():
sys.stderr.flush()
fd_pipes_orig = fd_pipes.copy()
self._files = self._files_dict()
files = self._files
master_fd, slave_fd = os.pipe()
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fd_pipes[self._metadata_fd] = slave_fd
self._raw_metadata = []
files.ebuild = os.fdopen(master_fd, 'rb')
self._reg_id = self.scheduler.register(files.ebuild.fileno(),
self._registered_events, self._output_handler)
self._registered = True
retval = portage.doebuild(ebuild_path, "depend",
settings["ROOT"], settings, debug,
mydbapi=self.portdb, tree="porttree",
fd_pipes=fd_pipes, returnpid=True)
os.close(slave_fd)
if isinstance(retval, int):
# doebuild failed before spawning
self._unregister()
self.returncode = retval
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, 'w')
for c in self.cpv:
f.write("%s %s\n" % (c, ' '.join(self.keywords)))
f.close()
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, 'w')
for k, v in self.items.items():
f.write('%s=%s\n' % (k,v))
f.close()
def _send_reply(self, reply):
output_fd = os.open(self.output_fifo, os.O_WRONLY | os.O_NONBLOCK)
output_file = os.fdopen(output_fd, 'wb')
pickle.dump(reply, output_file)
output_file.close()
def _start(self):
if self.cancelled:
return
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
fd_pipes.setdefault(0, sys.stdin.fileno())
fd_pipes.setdefault(1, sys.stdout.fileno())
fd_pipes.setdefault(2, sys.stderr.fileno())
# flush any pending output
for fd in fd_pipes.values():
if fd == sys.stdout.fileno():
sys.stdout.flush()
if fd == sys.stderr.fileno():
sys.stderr.flush()
self._files = self._files_dict()
files = self._files
master_fd, slave_fd = self._pipe(fd_pipes)
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
logfile = None
if self._can_log(slave_fd):
logfile = self.logfile
null_input = None
fd_pipes_orig = fd_pipes.copy()
if self.background:
# TODO: Use job control functions like tcsetpgrp() to control
# access to stdin. Until then, use /dev/null so that any
# attempts to read from stdin will immediately return EOF
# instead of blocking indefinitely.
null_input = open('/dev/null', 'rb')
fd_pipes[0] = null_input.fileno()
else:
fd_pipes[0] = fd_pipes_orig[0]
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
files.process = os.fdopen(master_fd, 'rb', 0)
if logfile is not None:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
files.log = open(_unicode_encode(logfile,
encoding=_encodings['fs'], errors='strict'), mode='ab')
if logfile.endswith('.gz'):
self._log_file_real = files.log
files.log = gzip.GzipFile(filename='', mode='ab',
fileobj=files.log)
portage.util.apply_secpass_permissions(logfile,
uid=portage.portage_uid, gid=portage.portage_gid,
mode=0o660)
if not self.background:
files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb')
output_handler = self._output_handler
else:
# Create a dummy pipe so the scheduler can monitor
# the process from inside a poll() loop.
fd_pipes[self._dummy_pipe_fd] = slave_fd
if self.background:
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
output_handler = self._dummy_handler
kwargs = {}
for k in self._spawn_kwarg_names:
v = getattr(self, k)
if v is not None:
kwargs[k] = v
kwargs["fd_pipes"] = fd_pipes
kwargs["returnpid"] = True
kwargs.pop("logfile", None)
self._reg_id = self.scheduler.register(files.process.fileno(),
self._registered_events, output_handler)
self._registered = True
retval = self._spawn(self.args, **kwargs)
os.close(slave_fd)
if null_input is not None:
null_input.close()
if isinstance(retval, int):
# spawn failed
self._unregister()
self._set_returncode((self.pid, retval))
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
def _test_pty_eof(fdopen_buffered=False):
"""
Returns True if this issues is fixed for the currently
running version of python: http://bugs.python.org/issue5380
Raises an EnvironmentError from openpty() if it fails.
NOTE: This issue is only problematic when array.fromfile()
is used, rather than os.read(). However, array.fromfile()
is preferred since it is approximately 10% faster.
New development: It appears that array.fromfile() is usable
with python3 as long as fdopen is called with a bufsize
argument of 0.
"""
use_fork = False
test_string = 2 * "blah blah blah\n"
test_string = _unicode_decode(test_string,
encoding='utf_8', errors='strict')
# may raise EnvironmentError
master_fd, slave_fd = pty.openpty()
# Non-blocking mode is required for Darwin kernel.
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# Disable post-processing of output since otherwise weird
# things like \n -> \r\n transformations may occur.
mode = termios.tcgetattr(slave_fd)
mode[1] &= ~termios.OPOST
termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
# Simulate a subprocess writing some data to the
# slave end of the pipe, and then exiting.
pid = None
if use_fork:
pids = spawn_bash(_unicode_encode("echo -n '%s'" % test_string,
encoding='utf_8', errors='strict'), env=os.environ,
fd_pipes={0:sys.stdin.fileno(), 1:slave_fd, 2:slave_fd},
returnpid=True)
if isinstance(pids, int):
os.close(master_fd)
os.close(slave_fd)
raise EnvironmentError('spawn failed')
pid = pids[0]
else:
os.write(slave_fd, _unicode_encode(test_string,
encoding='utf_8', errors='strict'))
os.close(slave_fd)
# If using a fork, we must wait for the child here,
# in order to avoid a race condition that would
# lead to inconsistent results.
if pid is not None:
os.waitpid(pid, 0)
if fdopen_buffered:
master_file = os.fdopen(master_fd, 'rb')
else:
master_file = os.fdopen(master_fd, 'rb', 0)
eof = False
data = []
iwtd = [master_file]
owtd = []
ewtd = []
while not eof:
events = select.select(iwtd, owtd, ewtd)
if not events[0]:
eof = True
break
buf = array.array('B')
try:
buf.fromfile(master_file, 1024)
except (EOFError, IOError):
eof = True
if not buf:
eof = True
else:
try:
# Python >=3.2
data.append(buf.tobytes())
except AttributeError:
data.append(buf.tostring())
master_file.close()
return test_string == _unicode_decode(b''.join(data), encoding='utf_8', errors='strict')
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, "w")
f.write("\n".join(test_cps))
f.close()
def setUp(self):
fd, self.testfile = tempfile.mkstemp(suffix=".testdata", prefix=self.__class__.__name__, text=True)
f = os.fdopen(fd, 'w')
f.write("\n".join(test_cps))
f.close()
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, 'w')
for k, v in self.items.items():
f.write('%s=%s\n' % (k, v))
f.close()
def _start(self):
settings = self.settings
settings.setcpv(self.cpv)
ebuild_path = self.ebuild_hash.location
# the caller can pass in eapi in order to avoid
# redundant _parse_eapi_ebuild_head calls
eapi = self.eapi
if eapi is None and \
'parse-eapi-ebuild-head' in settings.features:
with io.open(_unicode_encode(ebuild_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace') as f:
eapi = portage._parse_eapi_ebuild_head(f)
if eapi is not None:
if not portage.eapi_is_supported(eapi):
self.metadata = self.metadata_callback(self.cpv,
self.repo_path, {'EAPI' : eapi}, self.ebuild_hash)
self._set_returncode((self.pid, os.EX_OK << 8))
self.wait()
return
settings.configdict['pkg']['EAPI'] = eapi
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
fd_pipes = None
if self.fd_pipes is not None:
fd_pipes = self.fd_pipes.copy()
else:
fd_pipes = {}
null_input = open('/dev/null', 'rb')
fd_pipes.setdefault(0, null_input.fileno())
fd_pipes.setdefault(1, sys.stdout.fileno())
fd_pipes.setdefault(2, sys.stderr.fileno())
# flush any pending output
for fd in fd_pipes.values():
if fd == sys.stdout.fileno():
sys.stdout.flush()
if fd == sys.stderr.fileno():
sys.stderr.flush()
fd_pipes_orig = fd_pipes.copy()
self._files = self._files_dict()
files = self._files
master_fd, slave_fd = os.pipe()
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fd_pipes[self._metadata_fd] = slave_fd
self._raw_metadata = []
files.ebuild = os.fdopen(master_fd, 'rb', 0)
self._reg_id = self.scheduler.register(files.ebuild.fileno(),
self._registered_events, self._output_handler)
self._registered = True
retval = portage.doebuild(ebuild_path, "depend",
settings=settings, debug=debug,
mydbapi=self.portdb, tree="porttree",
fd_pipes=fd_pipes, returnpid=True)
os.close(slave_fd)
null_input.close()
if isinstance(retval, int):
# doebuild failed before spawning
self._unregister()
self._set_returncode((self.pid, retval << 8))
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
def _test_pty_eof():
"""
Returns True if this issues is fixed for the currently
running version of python: http://bugs.python.org/issue5380
Raises an EnvironmentError from openpty() if it fails.
"""
use_fork = False
test_string = 2 * "blah blah blah\n"
test_string = _unicode_decode(test_string,
encoding='utf_8',
errors='strict')
# may raise EnvironmentError
master_fd, slave_fd = pty.openpty()
# Non-blocking mode is required for Darwin kernel.
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
# Disable post-processing of output since otherwise weird
# things like \n -> \r\n transformations may occur.
mode = termios.tcgetattr(slave_fd)
mode[1] &= ~termios.OPOST
termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
# Simulate a subprocess writing some data to the
# slave end of the pipe, and then exiting.
pid = None
if use_fork:
pids = spawn_bash(_unicode_encode("echo -n '%s'" % test_string,
encoding='utf_8',
errors='strict'),
env=os.environ,
fd_pipes={
0: sys.stdin.fileno(),
1: slave_fd,
2: slave_fd
},
returnpid=True)
if isinstance(pids, int):
os.close(master_fd)
os.close(slave_fd)
raise EnvironmentError('spawn failed')
pid = pids[0]
else:
os.write(
slave_fd,
_unicode_encode(test_string, encoding='utf_8', errors='strict'))
os.close(slave_fd)
# If using a fork, we must wait for the child here,
# in order to avoid a race condition that would
# lead to inconsistent results.
if pid is not None:
os.waitpid(pid, 0)
master_file = os.fdopen(master_fd, 'rb')
eof = False
data = []
iwtd = [master_file]
owtd = []
ewtd = []
while not eof:
events = select.select(iwtd, owtd, ewtd)
if not events[0]:
eof = True
break
buf = array.array('B')
try:
buf.fromfile(master_file, 1024)
except (EOFError, IOError):
eof = True
if not buf:
eof = True
else:
data.append(
_unicode_decode(buf.tostring(),
encoding='utf_8',
errors='strict'))
master_file.close()
return test_string == ''.join(data)
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, "w")
f.write("\n".join(test_cps))
f.close()
def _start(self):
settings = self.settings
settings.setcpv(self.cpv)
ebuild_path = self.ebuild_path
eapi = None
if eapi is None and \
'parse-eapi-ebuild-head' in settings.features:
eapi = portage._parse_eapi_ebuild_head(
codecs.open(_unicode_encode(ebuild_path,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace'))
if eapi is not None:
if not portage.eapi_is_supported(eapi):
self.metadata_callback(self.cpv, self.ebuild_path,
self.repo_path, {'EAPI': eapi},
self.ebuild_mtime)
self._set_returncode((self.pid, os.EX_OK))
self.wait()
return
settings.configdict['pkg']['EAPI'] = eapi
debug = settings.get("PORTAGE_DEBUG") == "1"
master_fd = None
slave_fd = None
fd_pipes = None
if self.fd_pipes is not None:
fd_pipes = self.fd_pipes.copy()
else:
fd_pipes = {}
fd_pipes.setdefault(0, sys.stdin.fileno())
fd_pipes.setdefault(1, sys.stdout.fileno())
fd_pipes.setdefault(2, sys.stderr.fileno())
# flush any pending output
for fd in fd_pipes.values():
if fd == sys.stdout.fileno():
sys.stdout.flush()
if fd == sys.stderr.fileno():
sys.stderr.flush()
fd_pipes_orig = fd_pipes.copy()
self._files = self._files_dict()
files = self._files
master_fd, slave_fd = os.pipe()
fcntl.fcntl(master_fd, fcntl.F_SETFL,
fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fd_pipes[self._metadata_fd] = slave_fd
self._raw_metadata = []
files.ebuild = os.fdopen(master_fd, 'rb')
self._reg_id = self.scheduler.register(files.ebuild.fileno(),
self._registered_events,
self._output_handler)
self._registered = True
retval = portage.doebuild(ebuild_path,
"depend",
settings["ROOT"],
settings,
debug,
mydbapi=self.portdb,
tree="porttree",
fd_pipes=fd_pipes,
returnpid=True)
os.close(slave_fd)
if isinstance(retval, int):
# doebuild failed before spawning
self._unregister()
self._set_returncode((self.pid, retval))
self.wait()
return
self.pid = retval[0]
portage.process.spawned_pids.remove(self.pid)
def BuildFile(self):
fd, self.fname = mkstemp()
f = os.fdopen(fd, 'w')
for c in self.cpv:
f.write("%s %s\n" % (c,' '.join(self.keywords)))
f.close()
