def _start(self):
saved_env_path = self._get_saved_env_path()
dest_env_path = self._get_dest_env_path()
shell_cmd = "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -c -- %s > %s" % \
(_shell_quote(saved_env_path),
_shell_quote(dest_env_path))
extractor_proc = SpawnProcess(
args=[BASH_BINARY, "-c", shell_cmd],
background=self.background,
env=self.settings.environ(),
scheduler=self.scheduler,
logfile=self.settings.get('PORTAGE_LOG_FILE'))
self._start_task(extractor_proc, self._extractor_exit)
def _pkg_install_mask_cleanup(self, proc):
if self._default_exit(proc) != os.EX_OK:
self.wait()
else:
self._start_task(
SpawnProcess(
args=["rm", "-rf", self._proot],
background=self.background,
env=self.settings.environ(),
scheduler=self.scheduler,
logfile=self.logfile,
),
self._default_final_exit,
)
def testLazyImportPortageBaseline(self):
"""
Check what modules are imported by a baseline module import.
"""
env = os.environ.copy()
pythonpath = env.get("PYTHONPATH")
if pythonpath is not None and not pythonpath.strip():
pythonpath = None
if pythonpath is None:
pythonpath = ""
else:
pythonpath = ":" + pythonpath
pythonpath = PORTAGE_PYM_PATH + pythonpath
env["PYTHONPATH"] = pythonpath
# If python is patched to insert the path of the
# currently installed portage module into sys.path,
# then the above PYTHONPATH override doesn't help.
env["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH
scheduler = global_event_loop()
master_fd, slave_fd = os.pipe()
master_file = os.fdopen(master_fd, "rb", 0)
slave_file = os.fdopen(slave_fd, "wb")
producer = SpawnProcess(
args=self._baseline_import_cmd,
env=env,
fd_pipes={1: slave_fd},
scheduler=scheduler,
)
producer.start()
slave_file.close()
consumer = PipeReader(input_files={"producer": master_file},
scheduler=scheduler)
consumer.start()
consumer.wait()
self.assertEqual(producer.wait(), os.EX_OK)
self.assertEqual(consumer.wait(), os.EX_OK)
output = consumer.getvalue().decode("ascii", "replace").split()
unexpected_modules = " ".join(
sorted(x for x in output if self._module_re.match(x) is not None
and x not in self._baseline_imports))
self.assertEqual("", unexpected_modules)
async def _check_call(self, cmd):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
"""
p = SpawnProcess(args=cmd,
scheduler=asyncio.get_event_loop(),
**self._spawn_kwargs)
p.start()
if await p.async_wait() != os.EX_OK:
raise RepoStorageException(
"command exited with status {}: {}".format(
p.returncode, " ".join(cmd)))
def testIpcDaemon(self):
tmpdir = tempfile.mkdtemp()
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = tmpdir
input_fifo = os.path.join(tmpdir, '.ipc_in')
output_fifo = os.path.join(tmpdir, '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
task_scheduler = TaskScheduler(max_jobs=2)
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env,
scheduler=task_scheduler.sched_iface)
def exit_command_callback():
proc.cancel()
daemon.cancel()
exit_command.reply_hook = exit_command_callback
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run()
self.assertEqual(exit_command.exitcode, exitcode)
finally:
shutil.rmtree(tmpdir)
def _testPipeReader(self, test_string):
"""
Use a poll loop to read data from a pipe and assert that
the data written to the pipe is identical to the data
read from the pipe.
"""
if self._use_pty:
got_pty, master_fd, slave_fd = _create_pty_or_pipe()
if not got_pty:
os.close(slave_fd)
os.close(master_fd)
skip_reason = "pty not acquired"
self.portage_skip = skip_reason
self.fail(skip_reason)
return
else:
master_fd, slave_fd = os.pipe()
# WARNING: It is very important to use unbuffered mode here,
# in order to avoid issue 5380 with python3.
master_file = os.fdopen(master_fd, 'rb', 0)
slave_file = os.fdopen(slave_fd, 'wb', 0)
task_scheduler = TaskScheduler(max_jobs=2)
producer = SpawnProcess(
args=["bash", "-c", self._echo_cmd % test_string],
env=os.environ,
fd_pipes={1: slave_fd},
scheduler=task_scheduler.sched_iface)
task_scheduler.add(producer)
slave_file.close()
consumer = PipeReader(input_files={"producer": master_file},
scheduler=task_scheduler.sched_iface,
_use_array=self._use_array)
task_scheduler.add(consumer)
# This will ensure that both tasks have exited, which
# is necessary to avoid "ResourceWarning: unclosed file"
# warnings since Python 3.2 (and also ensures that we
# don't leave any zombie child processes).
task_scheduler.run()
self.assertEqual(producer.returncode, os.EX_OK)
self.assertEqual(consumer.returncode, os.EX_OK)
return consumer.getvalue().decode('ascii', 'replace')
def _check_call(self, cmd, privileged=False):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
@param privileged: run with maximum privileges
@type privileged: bool
"""
if privileged:
kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes'))
else:
kwargs = self._spawn_kwargs
p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(), **kwargs)
p.start()
if (yield p.async_wait()) != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def testLogfile(self):
logfile = None
try:
fd, logfile = tempfile.mkstemp()
os.close(fd)
null_fd = os.open('/dev/null', os.O_RDWR)
test_string = 2 * "blah blah blah\n"
task_scheduler = TaskScheduler()
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
"echo -n '%s'" % test_string],
env={},
fd_pipes={
0: sys.stdin.fileno(),
1: null_fd,
2: null_fd
},
scheduler=task_scheduler.sched_iface,
logfile=logfile)
task_scheduler.add(proc)
task_scheduler.run()
os.close(null_fd)
f = codecs.open(_unicode_encode(logfile,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='strict')
log_content = f.read()
f.close()
# When logging passes through a pty, this comparison will fail
# unless the oflag terminal attributes have the termios.OPOST
# bit disabled. Otherwise, tranformations such as \n -> \r\n
# may occur.
self.assertEqual(test_string, log_content)
finally:
if logfile:
try:
os.unlink(logfile)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
def _start(self):
try:
with io.open(
_unicode_encode(
os.path.join(
self.settings["PORTAGE_BUILDDIR"],
"build-info",
"PKG_INSTALL_MASK",
),
encoding=_encodings["fs"],
errors="strict",
),
mode="r",
encoding=_encodings["repo.content"],
errors="replace",
) as f:
self._pkg_install_mask = InstallMask(f.read())
except EnvironmentError:
self._pkg_install_mask = None
if self._pkg_install_mask:
self._proot = os.path.join(self.settings["T"], "packaging")
self._start_task(
SpawnProcess(
args=[
self._shell_binary,
"-e",
"-c",
(
"rm -rf {PROOT}; "
'cp -pPR $(cp --help | grep -q -- "^[[:space:]]*-l," && echo -l)'
' "${{D}}" {PROOT}'
).format(PROOT=portage._shell_quote(self._proot)),
],
background=self.background,
env=self.settings.environ(),
scheduler=self.scheduler,
logfile=self.logfile,
),
self._copy_proot_exit,
)
else:
self._proot = self.settings["D"]
self._start_package_phase()
def _extractor_exit(self, extractor):
if self._default_exit(extractor) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self.wait()
return
try:
with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace') as f:
self._build_prefix = f.read().rstrip('\n')
except IOError:
self._build_prefix = ""
if self._build_prefix == self.settings["EPREFIX"]:
ensure_dirs(self.settings["ED"])
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
env = self.settings.environ()
env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"]
chpathtool = SpawnProcess(
args=[
portage._python_interpreter,
os.path.join(self.settings["PORTAGE_BIN_PATH"],
"chpathtool.py"), self.settings["D"],
self._build_prefix, self.settings["EPREFIX"]
],
background=self.background,
env=env,
scheduler=self.scheduler,
logfile=self.settings.get('PORTAGE_LOG_FILE'))
self._writemsg_level(">>> Adjusting Prefix to %s\n" %
self.settings["EPREFIX"])
self._start_task(chpathtool, self._chpathtool_exit)
def testLogfile(self):
logfile = None
try:
fd, logfile = tempfile.mkstemp()
os.close(fd)
null_fd = os.open('/dev/null', os.O_RDWR)
test_string = 2 * "blah blah blah\n"
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
"echo -n '%s'" % test_string],
env={},
fd_pipes={
0: portage._get_stdin().fileno(),
1: null_fd,
2: null_fd
},
scheduler=global_event_loop(),
logfile=logfile)
global_event_loop().run_until_complete(proc.async_start())
os.close(null_fd)
self.assertEqual(proc.wait(), os.EX_OK)
f = io.open(_unicode_encode(logfile,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='strict')
log_content = f.read()
f.close()
# When logging passes through a pty, this comparison will fail
# unless the oflag terminal attributes have the termios.OPOST
# bit disabled. Otherwise, tranformations such as \n -> \r\n
# may occur.
self.assertEqual(test_string, log_content)
finally:
if logfile:
try:
os.unlink(logfile)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
def _start(self):
in_pr, in_pw = os.pipe()
out_pr, out_pw = os.pipe()
self._files = {}
self._files['pipe_in'] = in_pr
self._files['pipe_out'] = out_pw
fcntl.fcntl(in_pr, fcntl.F_SETFL,
fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK)
self._reg_id = self.scheduler.io_add_watch(in_pr,
self.scheduler.IO_IN, self._output_handler)
self._registered = True
self._proc = SpawnProcess(
args=[portage._python_interpreter,
os.path.join(portage._bin_path, 'lock-helper.py'), self.path],
env=dict(os.environ, PORTAGE_PYM_PATH=portage._pym_path),
fd_pipes={0:out_pr, 1:in_pw, 2:sys.stderr.fileno()},
scheduler=self.scheduler)
self._proc.addExitListener(self._proc_exit)
self._proc.start()
os.close(out_pr)
os.close(in_pw)
def _start(self):
in_pr, in_pw = os.pipe()
out_pr, out_pw = os.pipe()
self._files = {}
self._files['pipe_in'] = in_pr
self._files['pipe_out'] = out_pw
fcntl.fcntl(in_pr, fcntl.F_SETFL,
fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK)
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(
in_pr, fcntl.F_SETFD,
fcntl.fcntl(in_pr, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
self._reg_id = self.scheduler.io_add_watch(in_pr, self.scheduler.IO_IN,
self._output_handler)
self._registered = True
self._proc = SpawnProcess(args=[
portage._python_interpreter,
os.path.join(portage._bin_path, 'lock-helper.py'), self.path
],
env=dict(os.environ,
PORTAGE_PYM_PATH=portage._pym_path),
fd_pipes={
0: out_pr,
1: in_pw,
2: sys.__stderr__.fileno()
},
scheduler=self.scheduler)
self._proc.addExitListener(self._proc_exit)
self._proc.start()
os.close(out_pr)
os.close(in_pw)
def testIpcDaemon(self):
event_loop = global_event_loop()
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
env['PYTHONDONTWRITEBYTECODE'] = os.environ.get(
'PYTHONDONTWRITEBYTECODE', '')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
build_dir = EbuildBuildDir(scheduler=event_loop, settings=env)
event_loop.run_until_complete(build_dir.async_lock())
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2,
event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for EventLoop/AsyncScheduler.
# Use a ridiculously long sleep_time_s in case the user's
# system is heavily loaded (see bug #436334).
sleep_time_s = 600 # seconds
short_timeout_s = 0.010 # seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SleepProcess(seconds=sleep_time_s)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2,
event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, short_timeout_s)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
event_loop.run_until_complete(build_dir.async_unlock())
shutil.rmtree(tmpdir)
def _unpack_contents_exit(self, unpack_contents):
if self._default_exit(unpack_contents) != os.EX_OK:
unpack_contents.future.result()
self._writemsg_level(
"!!! Error Extracting '%s'\n" % self._pkg_path,
noiselevel=-1,
level=logging.ERROR,
)
self._async_unlock_builddir(returncode=self.returncode)
return
# Before anything else, let's do an integrity check.
(provides,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["PROVIDES"])
if check_dyn_libs_inconsistent(self.settings["D"], provides):
self._writemsg_level(
colorize(
"BAD",
"!!! Error! Installing dynamic libraries (.so) with blank PROVIDES!",
),
noiselevel=-1,
level=logging.ERROR,
)
try:
with io.open(
_unicode_encode(
os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings["fs"],
errors="strict",
),
mode="r",
encoding=_encodings["repo.content"],
errors="replace",
) as f:
self._build_prefix = f.read().rstrip("\n")
except IOError:
self._build_prefix = ""
if self._build_prefix == self.settings["EPREFIX"]:
ensure_dirs(self.settings["ED"])
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
env = self.settings.environ()
env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"]
chpathtool = SpawnProcess(
args=[
portage._python_interpreter,
os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"),
self.settings["D"],
self._build_prefix,
self.settings["EPREFIX"],
],
background=self.background,
env=env,
scheduler=self.scheduler,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
)
self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"])
self._start_task(chpathtool, self._chpathtool_exit)
def testIpcDaemon(self):
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
task_scheduler = TaskScheduler(max_jobs=2)
build_dir = EbuildBuildDir(scheduler=task_scheduler.sched_iface,
settings=env)
build_dir.lock()
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env,
scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.clear()
task_scheduler.wait()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT)
task_scheduler.clear()
task_scheduler.wait()
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for QueueScheduler.run()
sleep_time_s = 10 # 10.000 seconds
short_timeout_ms = 10 # 0.010 seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
'exec sleep %d' % sleep_time_s],
env=env,
scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.clear()
task_scheduler.wait()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run(timeout=short_timeout_ms)
task_scheduler.clear()
task_scheduler.wait()
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
build_dir.unlock()
shutil.rmtree(tmpdir)
