def _testAsynchronousLock(self): scheduler = global_event_loop() tempdir = tempfile.mkdtemp() try: path = os.path.join(tempdir, 'lock_me') for force_async, async_unlock in itertools.product( (True, False), repeat=2): for force_dummy in (True, False): async_lock = AsynchronousLock(path=path, scheduler=scheduler, _force_async=force_async, _force_thread=True, _force_dummy=force_dummy) async_lock.start() self.assertEqual(async_lock.wait(), os.EX_OK) self.assertEqual(async_lock.returncode, os.EX_OK) if async_unlock: scheduler.run_until_complete(async_lock.async_unlock()) else: async_lock.unlock() async_lock = AsynchronousLock(path=path, scheduler=scheduler, _force_async=force_async, _force_process=True) async_lock.start() self.assertEqual(async_lock.wait(), os.EX_OK) self.assertEqual(async_lock.returncode, os.EX_OK) if async_unlock: scheduler.run_until_complete(async_lock.async_unlock()) else: async_lock.unlock() finally: shutil.rmtree(tempdir)
def _testAsynchronousLock(self):
scheduler = global_event_loop()
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, 'lock_me')
for force_async, async_unlock in itertools.product((True, False),
repeat=2):
for force_dummy in (True, False):
async_lock = AsynchronousLock(path=path,
scheduler=scheduler,
_force_async=force_async,
_force_thread=True,
_force_dummy=force_dummy)
async_lock.start()
self.assertEqual(async_lock.wait(), os.EX_OK)
self.assertEqual(async_lock.returncode, os.EX_OK)
if async_unlock:
scheduler.run_until_complete(async_lock.async_unlock())
else:
async_lock.unlock()
async_lock = AsynchronousLock(path=path,
scheduler=scheduler,
_force_async=force_async,
_force_process=True)
async_lock.start()
self.assertEqual(async_lock.wait(), os.EX_OK)
self.assertEqual(async_lock.returncode, os.EX_OK)
if async_unlock:
scheduler.run_until_complete(async_lock.async_unlock())
else:
async_lock.unlock()
finally:
shutil.rmtree(tempdir)
def _testAsynchronousLockWait(self):
scheduler = global_event_loop()
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, "lock_me")
lock1 = AsynchronousLock(path=path, scheduler=scheduler)
lock1.start()
self.assertEqual(lock1.wait(), os.EX_OK)
self.assertEqual(lock1.returncode, os.EX_OK)
# lock2 requires _force_async=True since the portage.locks
# module is not designed to work as intended here if the
# same process tries to lock the same file more than
# one time concurrently.
lock2 = AsynchronousLock(path=path,
scheduler=scheduler,
_force_async=True,
_force_process=True)
lock2.start()
# lock2 should be waiting for lock1 to release
self.assertEqual(lock2.poll(), None)
self.assertEqual(lock2.returncode, None)
scheduler.run_until_complete(lock1.async_unlock())
self.assertEqual(lock2.wait(), os.EX_OK)
self.assertEqual(lock2.returncode, os.EX_OK)
scheduler.run_until_complete(lock2.async_unlock())
finally:
shutil.rmtree(tempdir)
def _testAsynchronousLock(self):
scheduler = global_event_loop()
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, "lock_me")
for force_async in (True, False):
for force_dummy in ((False, ) if dummy_threading is None else
(True, False)):
async_lock = AsynchronousLock(
path=path,
scheduler=scheduler,
_force_async=force_async,
_force_thread=True,
_force_dummy=force_dummy,
)
async_lock.start()
self.assertEqual(async_lock.wait(), os.EX_OK)
self.assertEqual(async_lock.returncode, os.EX_OK)
scheduler.run_until_complete(async_lock.async_unlock())
async_lock = AsynchronousLock(
path=path,
scheduler=scheduler,
_force_async=force_async,
_force_process=True,
)
async_lock.start()
self.assertEqual(async_lock.wait(), os.EX_OK)
self.assertEqual(async_lock.returncode, os.EX_OK)
scheduler.run_until_complete(async_lock.async_unlock())
finally:
shutil.rmtree(tempdir)
def async_unlock(self):
"""
Release the lock asynchronously. Release notification is available
via the add_done_callback method of the returned Future instance.
@returns: Future, result is None
"""
if self._lock_obj is not None:
yield self._lock_obj.async_unlock()
self._lock_obj = None
self.locked = False
self.settings.pop('PORTAGE_BUILDDIR_LOCKED', None)
catdir_lock = AsynchronousLock(path=self._catdir,
scheduler=self.scheduler)
try:
yield catdir_lock.async_start()
yield catdir_lock.async_wait()
except asyncio.CancelledError:
if catdir_lock.poll() is None:
catdir_lock.cancel()
raise
if catdir_lock.returncode == os.EX_OK:
try:
os.rmdir(self._catdir)
except OSError:
pass
yield catdir_lock.async_unlock()
def _testAsynchronousLockWaitKill(self):
scheduler = global_event_loop()
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, "lock_me")
lock1 = AsynchronousLock(path=path, scheduler=scheduler)
lock1.start()
self.assertEqual(lock1.wait(), os.EX_OK)
self.assertEqual(lock1.returncode, os.EX_OK)
lock2 = AsynchronousLock(path=path,
scheduler=scheduler,
_force_async=True,
_force_process=True)
lock2.start()
# lock2 should be waiting for lock1 to release
self.assertEqual(lock2.poll(), None)
self.assertEqual(lock2.returncode, None)
# Kill lock2's process and then check wait() and
# returncode results. This is intended to simulate
# a SIGINT sent via the controlling tty.
self.assertEqual(lock2._imp is not None, True)
self.assertEqual(lock2._imp._proc is not None, True)
self.assertEqual(lock2._imp._proc.pid is not None, True)
lock2._imp._kill_test = True
os.kill(lock2._imp._proc.pid, signal.SIGTERM)
self.assertEqual(lock2.wait() == os.EX_OK, False)
self.assertEqual(lock2.returncode == os.EX_OK, False)
self.assertEqual(lock2.returncode is None, False)
scheduler.run_until_complete(lock1.async_unlock())
finally:
shutil.rmtree(tempdir)
def _testAsynchronousLockWaitCancel(self):
scheduler = global_event_loop()
tempdir = tempfile.mkdtemp()
try:
path = os.path.join(tempdir, "lock_me")
lock1 = AsynchronousLock(path=path, scheduler=scheduler)
lock1.start()
self.assertEqual(lock1.wait(), os.EX_OK)
self.assertEqual(lock1.returncode, os.EX_OK)
lock2 = AsynchronousLock(path=path,
scheduler=scheduler,
_force_async=True,
_force_process=True)
lock2.start()
# lock2 should be waiting for lock1 to release
self.assertEqual(lock2.poll(), None)
self.assertEqual(lock2.returncode, None)
# Cancel lock2 and then check wait() and returncode results.
lock2.cancel()
self.assertEqual(lock2.wait() == os.EX_OK, False)
self.assertEqual(lock2.returncode == os.EX_OK, False)
self.assertEqual(lock2.returncode is None, False)
scheduler.run_until_complete(lock1.async_unlock())
finally:
shutil.rmtree(tempdir)
class _BinpkgFetcherProcess(SpawnProcess):
__slots__ = ("pkg", "pretend", "locked", "pkg_path", "_lock_obj")
def _start(self):
pkg = self.pkg
pretend = self.pretend
bintree = pkg.root_config.trees["bintree"]
settings = bintree.settings
pkg_path = self.pkg_path
exists = os.path.exists(pkg_path)
resume = exists and os.path.basename(pkg_path) in bintree.invalids
if not (pretend or resume):
# Remove existing file or broken symlink.
try:
os.unlink(pkg_path)
except OSError:
pass
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
fetchcommand = None
resumecommand = None
if bintree._remote_has_index:
remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(
pkg.cpv)]
rel_uri = remote_metadata.get("PATH")
if not rel_uri:
rel_uri = pkg.cpv + ".tbz2"
remote_base_uri = remote_metadata["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
fetchcommand = remote_metadata.get('FETCHCOMMAND')
resumecommand = remote_metadata.get('RESUMECOMMAND')
else:
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
"/" + pkg.pf + ".tbz2"
if pretend:
portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
self.returncode = os.EX_OK
self._async_wait()
return
fcmd = None
if resume:
fcmd = resumecommand
else:
fcmd = fetchcommand
if fcmd is None:
protocol = urllib_parse_urlparse(uri)[0]
fcmd_prefix = "FETCHCOMMAND"
if resume:
fcmd_prefix = "RESUMECOMMAND"
fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
if not fcmd:
fcmd = settings.get(fcmd_prefix)
fcmd_vars = {
"DISTDIR": os.path.dirname(pkg_path),
"URI": uri,
"FILE": os.path.basename(pkg_path)
}
for k in ("PORTAGE_SSH_OPTS", ):
v = settings.get(k)
if v is not None:
fcmd_vars[k] = v
fetch_env = dict(settings.items())
fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
for x in portage.util.shlex_split(fcmd)]
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stdout__.fileno())
self.args = fetch_args
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
SpawnProcess._start(self)
def _pipe(self, fd_pipes):
"""When appropriate, use a pty so that fetcher progress bars,
like wget has, will work properly."""
if self.background or not sys.__stdout__.isatty():
# When the output only goes to a log file,
# there's no point in creating a pty.
return os.pipe()
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def sync_timestamp(self):
# If possible, update the mtime to match the remote package if
# the fetcher didn't already do it automatically.
bintree = self.pkg.root_config.trees["bintree"]
if bintree._remote_has_index:
remote_mtime = bintree._remotepkgs[bintree.dbapi._instance_key(
self.pkg.cpv)].get("_mtime_")
if remote_mtime is not None:
try:
remote_mtime = int(remote_mtime)
except ValueError:
pass
else:
try:
local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME]
except OSError:
pass
else:
if remote_mtime != local_mtime:
try:
os.utime(self.pkg_path,
(remote_mtime, remote_mtime))
except OSError:
pass
def async_lock(self):
"""
This raises an AlreadyLocked exception if lock() is called
while a lock is already held. In order to avoid this, call
unlock() or check whether the "locked" attribute is True
or False before calling lock().
"""
if self._lock_obj is not None:
raise self.AlreadyLocked((self._lock_obj, ))
result = self.scheduler.create_future()
def acquired_lock(async_lock):
if async_lock.wait() == os.EX_OK:
self.locked = True
result.set_result(None)
else:
result.set_exception(
AssertionError(
"AsynchronousLock failed with returncode %s" %
(async_lock.returncode, )))
self._lock_obj = AsynchronousLock(path=self.pkg_path,
scheduler=self.scheduler)
self._lock_obj.addExitListener(acquired_lock)
self._lock_obj.start()
return result
class AlreadyLocked(portage.exception.PortageException):
pass
def async_unlock(self):
if self._lock_obj is None:
raise AssertionError('already unlocked')
result = self._lock_obj.async_unlock()
self._lock_obj = None
self.locked = False
return result
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") + \
("_ebuild_lock",)
# FEATURES displayed prior to setup phase
_features_display = ("ccache", "compressdebug", "distcc", "fakeroot",
"installsources", "keeptemp", "keepwork",
"network-sandbox", "network-sandbox-proxy", "nostrip",
"preserve-libs", "sandbox", "selinux", "sesandbox",
"splitdebug", "suidctl", "test", "userpriv",
"usersandbox")
# Locked phases
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
def _start(self):
self.scheduler.run_until_complete(self._async_start())
@coroutine
def _async_start(self):
need_builddir = self.phase not in EbuildProcess._phases_without_builddir
if need_builddir:
phase_completed_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
".%sed" % self.phase.rstrip('e'))
if not os.path.exists(phase_completed_file):
# If the phase is really going to run then we want
# to eliminate any stale elog messages that may
# exist from a previous run.
try:
os.unlink(
os.path.join(self.settings['T'], 'logging',
self.phase))
except OSError:
pass
if self.phase in ('nofetch', 'pretend', 'setup'):
use = self.settings.get('PORTAGE_BUILT_USE')
if use is None:
use = self.settings['PORTAGE_USE']
maint_str = ""
upstr_str = ""
metadata_xml_path = os.path.join(
os.path.dirname(self.settings['EBUILD']), "metadata.xml")
if MetaDataXML is not None and os.path.isfile(metadata_xml_path):
herds_path = os.path.join(self.settings['PORTDIR'],
'metadata/herds.xml')
try:
metadata_xml = MetaDataXML(metadata_xml_path, herds_path)
maint_str = metadata_xml.format_maintainer_string()
upstr_str = metadata_xml.format_upstream_string()
except SyntaxError:
maint_str = "<invalid metadata.xml>"
msg = []
msg.append("Package: %s" % self.settings.mycpv)
if self.settings.get('PORTAGE_REPO_NAME'):
msg.append("Repository: %s" %
self.settings['PORTAGE_REPO_NAME'])
if maint_str:
msg.append("Maintainer: %s" % maint_str)
if upstr_str:
msg.append("Upstream: %s" % upstr_str)
msg.append("USE: %s" % use)
relevant_features = []
enabled_features = self.settings.features
for x in self._features_display:
if x in enabled_features:
relevant_features.append(x)
if relevant_features:
msg.append("FEATURES: %s" % " ".join(relevant_features))
# Force background=True for this header since it's intended
# for the log and it doesn't necessarily need to be visible
# elsewhere.
yield self._elog('einfo', msg, background=True)
if self.phase == 'package':
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
self.settings['PORTAGE_BINPKG_TMPFILE'] = \
os.path.join(self.settings['PKGDIR'],
self.settings['CATEGORY'], self.settings['PF']) + '.tbz2'
if self.phase in ("pretend", "prerm"):
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler,
settings=self.settings)
if env_extractor.saved_env_exists():
self._current_task = env_extractor
yield env_extractor.async_start()
yield env_extractor.async_wait()
if self._default_exit(env_extractor) != os.EX_OK:
self._async_wait()
coroutine_return()
# If the environment.bz2 doesn't exist, then ebuild.sh will
# source the ebuild as a fallback.
self._start_lock()
def _start_lock(self):
if (self.phase in self._locked_phases
and "ebuild-locks" in self.settings.features):
eroot = self.settings["EROOT"]
lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild")
if os.access(os.path.dirname(lock_path), os.W_OK):
self._ebuild_lock = AsynchronousLock(path=lock_path,
scheduler=self.scheduler)
self._start_task(self._ebuild_lock, self._lock_exit)
return
self._start_ebuild()
def _lock_exit(self, ebuild_lock):
if self._default_exit(ebuild_lock) != os.EX_OK:
self.wait()
return
self._start_ebuild()
def _get_log_path(self):
# Don't open the log file during the clean phase since the
# open file can result in an nfs lock on $T/build.log which
# prevents the clean phase from removing $T.
logfile = None
if self.phase not in ("clean", "cleanrm") and \
self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
logfile = self.settings.get("PORTAGE_LOG_FILE")
return logfile
def _start_ebuild(self):
if self.phase == "package":
self._start_task(
PackagePhase(actionmap=self.actionmap,
background=self.background,
fd_pipes=self.fd_pipes,
logfile=self._get_log_path(),
scheduler=self.scheduler,
settings=self.settings), self._ebuild_exit)
return
if self.phase == "unpack":
alist = self.settings.configdict["pkg"].get("A", "").split()
_prepare_fake_distdir(self.settings, alist)
_prepare_fake_filesdir(self.settings)
fd_pipes = self.fd_pipes
if fd_pipes is None:
if not self.background and self.phase == 'nofetch':
# All the pkg_nofetch output goes to stderr since
# it's considered to be an error message.
fd_pipes = {1: sys.__stderr__.fileno()}
ebuild_process = EbuildProcess(actionmap=self.actionmap,
background=self.background,
fd_pipes=fd_pipes,
logfile=self._get_log_path(),
phase=self.phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(ebuild_process, self._ebuild_exit)
def _ebuild_exit(self, ebuild_process):
self._assert_current(ebuild_process)
if self._ebuild_lock is None:
self._ebuild_exit_unlocked(ebuild_process)
else:
self._start_task(
AsyncTaskFuture(future=self._ebuild_lock.async_unlock()),
functools.partial(self._ebuild_exit_unlocked, ebuild_process))
def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None):
if unlock_task is not None:
self._assert_current(unlock_task)
if unlock_task.cancelled:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.result()
fail = False
if ebuild_process.returncode != os.EX_OK:
self.returncode = ebuild_process.returncode
if self.phase == "test" and \
"test-fail-continue" in self.settings.features:
# mark test phase as complete (bug #452030)
try:
open(
_unicode_encode(os.path.join(
self.settings["PORTAGE_BUILDDIR"], ".tested"),
encoding=_encodings['fs'],
errors='strict'), 'wb').close()
except OSError:
pass
else:
fail = True
if not fail:
self.returncode = None
logfile = self._get_log_path()
if self.phase == "install":
out = io.StringIO()
_check_build_log(self.settings, out=out)
msg = out.getvalue()
self.scheduler.output(msg, log_path=logfile)
if fail:
self._die_hooks()
return
settings = self.settings
_post_phase_userpriv_perms(settings)
if self.phase == "unpack":
# Bump WORKDIR timestamp, in case tar gave it a timestamp
# that will interfere with distfiles / WORKDIR timestamp
# comparisons as reported in bug #332217. Also, fix
# ownership since tar can change that too.
os.utime(settings["WORKDIR"], None)
_prepare_workdir(settings)
elif self.phase == "install":
out = io.StringIO()
_post_src_install_write_metadata(settings)
_post_src_install_uid_fix(settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(msg, log_path=logfile)
elif self.phase == "preinst":
_preinst_bsdflags(settings)
elif self.phase == "postinst":
_postinst_bsdflags(settings)
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
if logfile is not None and self.phase in ("install", ):
# Log to a temporary file, since the code we are running
# reads PORTAGE_LOG_FILE for QA checks, and we want to
# avoid annoying "gzip: unexpected end of file" messages
# when FEATURES=compress-build-logs is enabled.
fd, logfile = tempfile.mkstemp()
os.close(fd)
post_phase = _PostPhaseCommands(background=self.background,
commands=post_phase_cmds,
elog=self._elog,
fd_pipes=self.fd_pipes,
logfile=logfile,
phase=self.phase,
scheduler=self.scheduler,
settings=settings)
self._start_task(post_phase, self._post_phase_exit)
return
# this point is not reachable if there was a failure and
# we returned for die_hooks above, so returncode must
# indicate success (especially if ebuild_process.returncode
# is unsuccessful and test-fail-continue came into play)
self.returncode = os.EX_OK
self._current_task = None
self.wait()
def _post_phase_exit(self, post_phase):
self._assert_current(post_phase)
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
if post_phase.logfile is not None and \
post_phase.logfile != log_path:
# We were logging to a temp file (see above), so append
# temp file to main log and remove temp file.
self._append_temp_log(post_phase.logfile, log_path)
if self._final_exit(post_phase) != os.EX_OK:
writemsg("!!! post %s failed; exiting.\n" % self.phase,
noiselevel=-1)
self._die_hooks()
return
self._current_task = None
self.wait()
return
def _append_temp_log(self, temp_log, log_path):
temp_file = open(
_unicode_encode(temp_log,
encoding=_encodings['fs'],
errors='strict'), 'rb')
log_file, log_file_real = self._open_log(log_path)
for line in temp_file:
log_file.write(line)
temp_file.close()
log_file.close()
if log_file_real is not log_file:
log_file_real.close()
os.unlink(temp_log)
def _open_log(self, log_path):
f = open(_unicode_encode(log_path,
encoding=_encodings['fs'],
errors='strict'),
mode='ab')
f_real = f
if log_path.endswith('.gz'):
f = gzip.GzipFile(filename='', mode='ab', fileobj=f)
return (f, f_real)
def _die_hooks(self):
self.returncode = None
phase = 'die_hooks'
die_hooks = MiscFunctionsProcess(background=self.background,
commands=[phase],
phase=phase,
logfile=self._get_log_path(),
fd_pipes=self.fd_pipes,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(die_hooks, self._die_hooks_exit)
def _die_hooks_exit(self, die_hooks):
if self.phase != 'clean' and \
'noclean' not in self.settings.features and \
'fail-clean' in self.settings.features:
self._default_exit(die_hooks)
self._fail_clean()
return
self._final_exit(die_hooks)
self.returncode = 1
self.wait()
def _fail_clean(self):
self.returncode = None
portage.elog.elog_process(self.settings.mycpv, self.settings)
phase = "clean"
clean_phase = EbuildPhase(background=self.background,
fd_pipes=self.fd_pipes,
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(clean_phase, self._fail_clean_exit)
return
def _fail_clean_exit(self, clean_phase):
self._final_exit(clean_phase)
self.returncode = 1
self.wait()
@coroutine
def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = \
self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
build_logger = None
try:
log_file = None
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
if log_path:
build_logger = BuildLogger(
env=self.settings.environ(),
log_path=log_path,
log_filter_file=self.settings.get(
'PORTAGE_LOG_FILTER_FILE_CMD'),
scheduler=self.scheduler)
yield build_logger.async_start()
log_file = build_logger.stdin
yield self.scheduler.async_output(msg,
log_file=log_file,
background=background)
if build_logger is not None:
build_logger.stdin.close()
yield build_logger.async_wait()
except asyncio.CancelledError:
if build_logger is not None:
build_logger.cancel()
raise
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") + \
("_ebuild_lock",)
# FEATURES displayed prior to setup phase
_features_display = (
"ccache", "compressdebug", "distcc", "distcc-pump", "fakeroot",
"installsources", "keeptemp", "keepwork", "network-sandbox",
"network-sandbox-proxy", "nostrip", "preserve-libs", "sandbox",
"selinux", "sesandbox", "splitdebug", "suidctl", "test",
"userpriv", "usersandbox"
)
# Locked phases
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
def _start(self):
need_builddir = self.phase not in EbuildProcess._phases_without_builddir
if need_builddir:
phase_completed_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
".%sed" % self.phase.rstrip('e'))
if not os.path.exists(phase_completed_file):
# If the phase is really going to run then we want
# to eliminate any stale elog messages that may
# exist from a previous run.
try:
os.unlink(os.path.join(self.settings['T'],
'logging', self.phase))
except OSError:
pass
if self.phase in ('nofetch', 'pretend', 'setup'):
use = self.settings.get('PORTAGE_BUILT_USE')
if use is None:
use = self.settings['PORTAGE_USE']
maint_str = ""
upstr_str = ""
metadata_xml_path = os.path.join(os.path.dirname(self.settings['EBUILD']), "metadata.xml")
if MetaDataXML is not None and os.path.isfile(metadata_xml_path):
herds_path = os.path.join(self.settings['PORTDIR'],
'metadata/herds.xml')
try:
metadata_xml = MetaDataXML(metadata_xml_path, herds_path)
maint_str = metadata_xml.format_maintainer_string()
upstr_str = metadata_xml.format_upstream_string()
except SyntaxError:
maint_str = "<invalid metadata.xml>"
msg = []
msg.append("Package: %s" % self.settings.mycpv)
if self.settings.get('PORTAGE_REPO_NAME'):
msg.append("Repository: %s" % self.settings['PORTAGE_REPO_NAME'])
if maint_str:
msg.append("Maintainer: %s" % maint_str)
if upstr_str:
msg.append("Upstream: %s" % upstr_str)
msg.append("USE: %s" % use)
relevant_features = []
enabled_features = self.settings.features
for x in self._features_display:
if x in enabled_features:
relevant_features.append(x)
if relevant_features:
msg.append("FEATURES: %s" % " ".join(relevant_features))
# Force background=True for this header since it's intended
# for the log and it doesn't necessarily need to be visible
# elsewhere.
self._elog('einfo', msg, background=True)
if self.phase == 'package':
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
self.settings['PORTAGE_BINPKG_TMPFILE'] = \
os.path.join(self.settings['PKGDIR'],
self.settings['CATEGORY'], self.settings['PF']) + '.tbz2'
if self.phase in ("pretend", "prerm"):
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
if env_extractor.saved_env_exists():
self._start_task(env_extractor, self._env_extractor_exit)
return
# If the environment.bz2 doesn't exist, then ebuild.sh will
# source the ebuild as a fallback.
self._start_lock()
def _env_extractor_exit(self, env_extractor):
if self._default_exit(env_extractor) != os.EX_OK:
self.wait()
return
self._start_lock()
def _start_lock(self):
if (self.phase in self._locked_phases and
"ebuild-locks" in self.settings.features):
eroot = self.settings["EROOT"]
lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild")
if os.access(os.path.dirname(lock_path), os.W_OK):
self._ebuild_lock = AsynchronousLock(path=lock_path,
scheduler=self.scheduler)
self._start_task(self._ebuild_lock, self._lock_exit)
return
self._start_ebuild()
def _lock_exit(self, ebuild_lock):
if self._default_exit(ebuild_lock) != os.EX_OK:
self.wait()
return
self._start_ebuild()
def _get_log_path(self):
# Don't open the log file during the clean phase since the
# open file can result in an nfs lock on $T/build.log which
# prevents the clean phase from removing $T.
logfile = None
if self.phase not in ("clean", "cleanrm") and \
self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
logfile = self.settings.get("PORTAGE_LOG_FILE")
return logfile
def _start_ebuild(self):
if self.phase == "package":
self._start_task(PackagePhase(actionmap=self.actionmap,
background=self.background, fd_pipes=self.fd_pipes,
logfile=self._get_log_path(), scheduler=self.scheduler,
settings=self.settings), self._ebuild_exit)
return
if self.phase == "unpack":
alist = self.settings.configdict["pkg"].get("A", "").split()
_prepare_fake_distdir(self.settings, alist)
_prepare_fake_filesdir(self.settings)
fd_pipes = self.fd_pipes
if fd_pipes is None:
if not self.background and self.phase == 'nofetch':
# All the pkg_nofetch output goes to stderr since
# it's considered to be an error message.
fd_pipes = {1 : sys.__stderr__.fileno()}
ebuild_process = EbuildProcess(actionmap=self.actionmap,
background=self.background, fd_pipes=fd_pipes,
logfile=self._get_log_path(), phase=self.phase,
scheduler=self.scheduler, settings=self.settings)
self._start_task(ebuild_process, self._ebuild_exit)
def _ebuild_exit(self, ebuild_process):
self._assert_current(ebuild_process)
if self._ebuild_lock is None:
self._ebuild_exit_unlocked(ebuild_process)
else:
self._start_task(
AsyncTaskFuture(future=self._ebuild_lock.async_unlock()),
functools.partial(self._ebuild_exit_unlocked, ebuild_process))
def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None):
if unlock_task is not None:
self._assert_current(unlock_task)
if unlock_task.cancelled:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.result()
fail = False
if ebuild_process.returncode != os.EX_OK:
self.returncode = ebuild_process.returncode
if self.phase == "test" and \
"test-fail-continue" in self.settings.features:
# mark test phase as complete (bug #452030)
try:
open(_unicode_encode(os.path.join(
self.settings["PORTAGE_BUILDDIR"], ".tested"),
encoding=_encodings['fs'], errors='strict'),
'wb').close()
except OSError:
pass
else:
fail = True
if not fail:
self.returncode = None
logfile = self._get_log_path()
if self.phase == "install":
out = io.StringIO()
_check_build_log(self.settings, out=out)
msg = out.getvalue()
self.scheduler.output(msg, log_path=logfile)
if fail:
self._die_hooks()
return
settings = self.settings
_post_phase_userpriv_perms(settings)
if self.phase == "unpack":
# Bump WORKDIR timestamp, in case tar gave it a timestamp
# that will interfere with distfiles / WORKDIR timestamp
# comparisons as reported in bug #332217. Also, fix
# ownership since tar can change that too.
os.utime(settings["WORKDIR"], None)
_prepare_workdir(settings)
elif self.phase == "install":
out = io.StringIO()
_post_src_install_write_metadata(settings)
_post_src_install_uid_fix(settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(msg, log_path=logfile)
elif self.phase == "preinst":
_preinst_bsdflags(settings)
elif self.phase == "postinst":
_postinst_bsdflags(settings)
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
if logfile is not None and self.phase in ("install",):
# Log to a temporary file, since the code we are running
# reads PORTAGE_LOG_FILE for QA checks, and we want to
# avoid annoying "gzip: unexpected end of file" messages
# when FEATURES=compress-build-logs is enabled.
fd, logfile = tempfile.mkstemp()
os.close(fd)
post_phase = _PostPhaseCommands(background=self.background,
commands=post_phase_cmds, fd_pipes=self.fd_pipes,
logfile=logfile, phase=self.phase, scheduler=self.scheduler,
settings=settings)
self._start_task(post_phase, self._post_phase_exit)
return
# this point is not reachable if there was a failure and
# we returned for die_hooks above, so returncode must
# indicate success (especially if ebuild_process.returncode
# is unsuccessful and test-fail-continue came into play)
self.returncode = os.EX_OK
self._current_task = None
self.wait()
def _post_phase_exit(self, post_phase):
self._assert_current(post_phase)
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
if post_phase.logfile is not None and \
post_phase.logfile != log_path:
# We were logging to a temp file (see above), so append
# temp file to main log and remove temp file.
self._append_temp_log(post_phase.logfile, log_path)
if self._final_exit(post_phase) != os.EX_OK:
writemsg("!!! post %s failed; exiting.\n" % self.phase,
noiselevel=-1)
self._die_hooks()
return
if self.phase == "install":
out = io.StringIO()
_post_src_install_soname_symlinks(self.settings, out)
msg = out.getvalue()
if msg:
self.scheduler.output(msg, log_path=log_path)
self._current_task = None
self.wait()
return
def _append_temp_log(self, temp_log, log_path):
temp_file = open(_unicode_encode(temp_log,
encoding=_encodings['fs'], errors='strict'), 'rb')
log_file, log_file_real = self._open_log(log_path)
for line in temp_file:
log_file.write(line)
temp_file.close()
log_file.close()
if log_file_real is not log_file:
log_file_real.close()
os.unlink(temp_log)
def _open_log(self, log_path):
f = open(_unicode_encode(log_path,
encoding=_encodings['fs'], errors='strict'),
mode='ab')
f_real = f
if log_path.endswith('.gz'):
f = gzip.GzipFile(filename='', mode='ab', fileobj=f)
return (f, f_real)
def _die_hooks(self):
self.returncode = None
phase = 'die_hooks'
die_hooks = MiscFunctionsProcess(background=self.background,
commands=[phase], phase=phase, logfile=self._get_log_path(),
fd_pipes=self.fd_pipes, scheduler=self.scheduler,
settings=self.settings)
self._start_task(die_hooks, self._die_hooks_exit)
def _die_hooks_exit(self, die_hooks):
if self.phase != 'clean' and \
'noclean' not in self.settings.features and \
'fail-clean' in self.settings.features:
self._default_exit(die_hooks)
self._fail_clean()
return
self._final_exit(die_hooks)
self.returncode = 1
self.wait()
def _fail_clean(self):
self.returncode = None
portage.elog.elog_process(self.settings.mycpv, self.settings)
phase = "clean"
clean_phase = EbuildPhase(background=self.background,
fd_pipes=self.fd_pipes, phase=phase, scheduler=self.scheduler,
settings=self.settings)
self._start_task(clean_phase, self._fail_clean_exit)
return
def _fail_clean_exit(self, clean_phase):
self._final_exit(clean_phase)
self.returncode = 1
self.wait()
def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = \
self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path,
background=background)
def async_lock(self):
"""
Acquire the lock asynchronously. Notification is available
via the add_done_callback method of the returned Future instance.
This raises an AlreadyLocked exception if async_lock() is called
while a lock is already held. In order to avoid this, call
async_unlock() or check whether the "locked" attribute is True
or False before calling async_lock().
@returns: Future, result is None
"""
if self._lock_obj is not None:
raise self.AlreadyLocked((self._lock_obj, ))
dir_path = self.settings.get('PORTAGE_BUILDDIR')
if not dir_path:
raise AssertionError('PORTAGE_BUILDDIR is unset')
catdir = os.path.dirname(dir_path)
self._catdir = catdir
try:
portage.util.ensure_dirs(os.path.dirname(catdir),
gid=portage.portage_gid,
mode=0o70,
mask=0)
except PortageException:
if not os.path.isdir(os.path.dirname(catdir)):
raise
catdir_lock = AsynchronousLock(path=catdir, scheduler=self.scheduler)
builddir_lock = AsynchronousLock(path=dir_path,
scheduler=self.scheduler)
try:
yield catdir_lock.async_start()
yield catdir_lock.async_wait()
self._assert_lock(catdir_lock)
try:
portage.util.ensure_dirs(catdir,
gid=portage.portage_gid,
mode=0o70,
mask=0)
except PortageException:
if not os.path.isdir(catdir):
raise
yield builddir_lock.async_start()
yield builddir_lock.async_wait()
except asyncio.CancelledError:
if catdir_lock.poll() is None:
catdir_lock.cancel()
if builddir_lock.poll() is None:
builddir_lock.cancel()
raise
try:
self._assert_lock(builddir_lock)
except AssertionError:
yield catdir_lock.async_unlock()
raise
self._lock_obj = builddir_lock
self.locked = True
self.settings['PORTAGE_BUILDDIR_LOCKED'] = '1'
yield catdir_lock.async_unlock()
class _BinpkgFetcherProcess(SpawnProcess):
__slots__ = ("pkg", "pretend", "locked", "pkg_path", "_lock_obj")
def _start(self):
pkg = self.pkg
pretend = self.pretend
bintree = pkg.root_config.trees["bintree"]
settings = bintree.settings
pkg_path = self.pkg_path
exists = os.path.exists(pkg_path)
resume = exists and os.path.basename(pkg_path) in bintree.invalids
if not (pretend or resume):
# Remove existing file or broken symlink.
try:
os.unlink(pkg_path)
except OSError:
pass
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
if bintree._remote_has_index:
instance_key = bintree.dbapi._instance_key(pkg.cpv)
rel_uri = bintree._remotepkgs[instance_key].get("PATH")
if not rel_uri:
rel_uri = pkg.cpv + ".tbz2"
remote_base_uri = bintree._remotepkgs[
instance_key]["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
else:
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
"/" + pkg.pf + ".tbz2"
if pretend:
portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
self.returncode = os.EX_OK
self._async_wait()
return
protocol = urllib_parse_urlparse(uri)[0]
fcmd_prefix = "FETCHCOMMAND"
if resume:
fcmd_prefix = "RESUMECOMMAND"
fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
if not fcmd:
fcmd = settings.get(fcmd_prefix)
fcmd_vars = {
"DISTDIR" : os.path.dirname(pkg_path),
"URI" : uri,
"FILE" : os.path.basename(pkg_path)
}
for k in ("PORTAGE_SSH_OPTS",):
v = settings.get(k)
if v is not None:
fcmd_vars[k] = v
fetch_env = dict(settings.items())
fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
for x in portage.util.shlex_split(fcmd)]
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stdout__.fileno())
self.args = fetch_args
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
SpawnProcess._start(self)
def _pipe(self, fd_pipes):
"""When appropriate, use a pty so that fetcher progress bars,
like wget has, will work properly."""
if self.background or not sys.__stdout__.isatty():
# When the output only goes to a log file,
# there's no point in creating a pty.
return os.pipe()
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def sync_timestamp(self):
# If possible, update the mtime to match the remote package if
# the fetcher didn't already do it automatically.
bintree = self.pkg.root_config.trees["bintree"]
if bintree._remote_has_index:
remote_mtime = bintree._remotepkgs[
bintree.dbapi._instance_key(
self.pkg.cpv)].get("_mtime_")
if remote_mtime is not None:
try:
remote_mtime = long(remote_mtime)
except ValueError:
pass
else:
try:
local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME]
except OSError:
pass
else:
if remote_mtime != local_mtime:
try:
os.utime(self.pkg_path,
(remote_mtime, remote_mtime))
except OSError:
pass
def async_lock(self):
"""
This raises an AlreadyLocked exception if lock() is called
while a lock is already held. In order to avoid this, call
unlock() or check whether the "locked" attribute is True
or False before calling lock().
"""
if self._lock_obj is not None:
raise self.AlreadyLocked((self._lock_obj,))
result = self.scheduler.create_future()
def acquired_lock(async_lock):
if async_lock.wait() == os.EX_OK:
self.locked = True
result.set_result(None)
else:
result.set_exception(AssertionError(
"AsynchronousLock failed with returncode %s"
% (async_lock.returncode,)))
self._lock_obj = AsynchronousLock(path=self.pkg_path,
scheduler=self.scheduler)
self._lock_obj.addExitListener(acquired_lock)
self._lock_obj.start()
return result
class AlreadyLocked(portage.exception.PortageException):
pass
def async_unlock(self):
if self._lock_obj is None:
raise AssertionError('already unlocked')
result = self._lock_obj.async_unlock()
self._lock_obj = None
self.locked = False
return result