def _start(self):
pkg = self.pkg
root_config = pkg.root_config
bintree = root_config.trees["bintree"]
binpkg_format = self.settings.get(
"BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
)
if binpkg_format == "xpak":
binpkg_tmpfile = os.path.join(
bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid())
)
elif binpkg_format == "gpkg":
binpkg_tmpfile = os.path.join(
bintree.pkgdir, pkg.cpv + ".gpkg.tar." + str(portage.getpid())
)
else:
raise InvalidBinaryPackageFormat(binpkg_format)
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
self._binpkg_tmpfile = binpkg_tmpfile
self.settings["PORTAGE_BINPKG_TMPFILE"] = self._binpkg_tmpfile
package_phase = EbuildPhase(
background=self.background,
phase="package",
scheduler=self.scheduler,
settings=self.settings,
)
self._start_task(package_phase, self._package_phase_exit)
def _safe_loop():
"""
Return an event loop that's safe to use within the current context.
For portage internal callers or external API consumers calling from
the main thread, this returns a globally shared event loop instance.
For external API consumers calling from a non-main thread, an
asyncio loop must be registered for the current thread, or else the
asyncio.get_event_loop() function will raise an error like this:
RuntimeError: There is no current event loop in thread 'Thread-1'.
In order to avoid this RuntimeError, a loop will be automatically
created like this:
asyncio.set_event_loop(asyncio.new_event_loop())
In order to avoid a ResourceWarning, automatically created loops
are added to a WeakValueDictionary, and closed via an atexit hook
if they still exist during exit for the current pid.
@rtype: asyncio.AbstractEventLoop (or compatible)
@return: event loop instance
"""
loop = _get_running_loop()
if loop is not None:
return loop
thread_key = threading.get_ident()
with _thread_weakrefs.lock:
if _thread_weakrefs.pid != portage.getpid():
_thread_weakrefs.pid = portage.getpid()
_thread_weakrefs.mainloop = None
_thread_weakrefs.loops = weakref.WeakValueDictionary()
try:
loop = _thread_weakrefs.loops[thread_key]
except KeyError:
try:
try:
_loop = _real_asyncio.get_running_loop()
except AttributeError:
_loop = _real_asyncio.get_event_loop()
except RuntimeError:
_loop = _real_asyncio.new_event_loop()
_real_asyncio.set_event_loop(_loop)
loop = _thread_weakrefs.loops[thread_key] = _AsyncioEventLoop(loop=_loop)
if (
_thread_weakrefs.mainloop is None
and threading.current_thread() is threading.main_thread()
):
_thread_weakrefs.mainloop = loop
return loop
def hardlock_name(path):
base, tail = os.path.split(path)
return os.path.join(
base,
".%s.hardlock-%s-%s" %
(tail, portage._decode_argv([os.uname()[1]])[0], portage.getpid()),
)
def recompose_mem(self, xpdata, break_hardlinks=True):
"""
Update the xpak segment.
@param xpdata: A new xpak segment to be written, like that returned
from the xpak_mem() function.
@param break_hardlinks: If hardlinks exist, create a copy in order
to break them. This makes it safe to use hardlinks to create
cheap snapshots of the repository, which is useful for solving
race conditions on binhosts as described here:
https://crbug.com/185031
Default is True.
"""
self.scan() # Don't care about condition... We'll rewrite the data anyway.
if break_hardlinks and self.filestat and self.filestat.st_nlink > 1:
tmp_fname = "%s.%d" % (self.file, portage.getpid())
copyfile(self.file, tmp_fname)
try:
portage.util.apply_stat_permissions(self.file, self.filestat)
except portage.exception.OperationNotPermitted:
pass
os.rename(tmp_fname, self.file)
myfile = open(
_unicode_encode(self.file, encoding=_encodings["fs"], errors="strict"),
"ab+",
)
if not myfile:
raise IOError
myfile.seek(-self.xpaksize, 2) # 0,2 or -0,2 just mean EOF.
myfile.truncate()
myfile.write(xpdata + encodeint(len(xpdata)) + b"STOP")
myfile.flush()
myfile.close()
return 1
def process(mysettings, key, logentries, fulltext):
global _items
time_str = _unicode_decode(
time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())),
encoding=_encodings["content"],
errors="replace",
)
header = _(
">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n"
) % {
"pkg": key,
"pid": portage.getpid(),
"time": time_str
}
config_root = mysettings["PORTAGE_CONFIGROOT"]
# Copy needed variables from the config instance,
# since we don't need to hold a reference for the
# whole thing. This also makes it possible to
# rely on per-package variable settings that may
# have come from /etc/portage/package.env, since
# we'll be isolated from any future mutations of
# mysettings.
config_dict = {}
for k in _config_keys:
v = mysettings.get(k)
if v is not None:
config_dict[k] = v
config_dict, items = _items.setdefault(config_root, (config_dict, {}))
items[key] = header + fulltext
def _db_init_connection(self):
config = self._config
self._dbpath = self.location + ".sqlite"
# if os.path.exists(self._dbpath):
# os.unlink(self._dbpath)
connection_kwargs = {}
connection_kwargs["timeout"] = config["timeout"]
try:
if not self.readonly:
self._ensure_dirs()
connection = self._db_module.connect(database=_unicode_decode(
self._dbpath),
**connection_kwargs)
cursor = connection.cursor()
self._db_connection_info = self._connection_info_entry(
connection, cursor, portage.getpid())
self._db_cursor.execute("PRAGMA encoding = %s" %
self._db_escape_string("UTF-8"))
if not self.readonly and not self._ensure_access(self._dbpath):
raise cache_errors.InitializationError(
self.__class__, "can't ensure perms on %s" % self._dbpath)
self._db_init_cache_size(config["cache_bytes"])
self._db_init_synchronous(config["synchronous"])
self._db_init_structures()
except self._db_error as e:
raise cache_errors.InitializationError(self.__class__, e)
def _run_until_complete(self, future):
"""
An implementation of AbstractEventLoop.run_until_complete that supresses
spurious error messages like the following reported in bug 655656:
Exception ignored when trying to write to the signal wakeup fd:
BlockingIOError: [Errno 11] Resource temporarily unavailable
In order to avoid potential interference with API consumers, this
implementation is only used when portage._internal_caller is True.
"""
if self._wakeup_fd != -1:
signal.set_wakeup_fd(self._wakeup_fd)
self._wakeup_fd = -1
# Account for any signals that may have arrived between
# set_wakeup_fd calls.
os.kill(portage.getpid(), signal.SIGCHLD)
try:
return self._loop.run_until_complete(future)
finally:
try:
self._wakeup_fd = signal.set_wakeup_fd(-1)
except ValueError:
# This is intended to fail when not called in the main thread.
pass
def wrapper(kill_switch):
if portage.getpid() == parent_pid:
# thread in main process
def done_callback(result):
result.cancelled() or result.exception() or result.result()
kill_switch.set()
def start_coroutine(future):
result = asyncio.ensure_future(coroutine_func(),
loop=parent_loop)
pending[id(result)] = result
result.add_done_callback(done_callback)
future.set_result(result)
future = Future()
parent_loop.call_soon_threadsafe(start_coroutine, future)
kill_switch.wait()
if not future.done():
future.cancel()
raise asyncio.CancelledError
elif not future.result().done():
future.result().cancel()
raise asyncio.CancelledError
else:
return future.result().result()
# child process
loop = global_event_loop()
try:
return loop.run_until_complete(coroutine_func())
finally:
loop.close()
def _get_running_loop():
with _thread_weakrefs.lock:
if _thread_weakrefs.pid == portage.getpid():
try:
loop = _thread_weakrefs.loops[threading.get_ident()]
except KeyError:
return None
return loop if loop.is_running() else None
def _fetch_uri(self, uri):
if self.config.options.dry_run:
# Simply report success.
logging.info("dry-run: fetch '%s' from '%s'" %
(self.distfile, uri))
self._success()
self.returncode = os.EX_OK
self._async_wait()
return
if self.config.options.temp_dir:
self._fetch_tmp_dir_info = "temp-dir"
distdir = self.config.options.temp_dir
else:
self._fetch_tmp_dir_info = "distfiles"
distdir = self.config.options.distfiles
tmp_basename = self.distfile + "._emirrordist_fetch_.%s" % portage.getpid(
)
variables = {"DISTDIR": distdir, "URI": uri, "FILE": tmp_basename}
self._fetch_tmp_file = os.path.join(distdir, tmp_basename)
try:
os.unlink(self._fetch_tmp_file)
except OSError:
pass
args = portage.util.shlex_split(default_fetchcommand)
args = [portage.util.varexpand(x, mydict=variables) for x in args]
args = [
_unicode_encode(x, encoding=_encodings["fs"], errors="strict")
for x in args
]
null_fd = os.open(os.devnull, os.O_RDONLY)
fetcher = PopenProcess(
background=self.background,
proc=subprocess.Popen(args,
stdin=null_fd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT),
scheduler=self.scheduler,
)
os.close(null_fd)
fetcher.pipe_reader = PipeLogger(
background=self.background,
input_fd=fetcher.proc.stdout,
log_file_path=self._log_path,
scheduler=self.scheduler,
)
self._start_task(fetcher, self._fetcher_exit)
def _thread_weakrefs_atexit():
with _thread_weakrefs.lock:
if _thread_weakrefs.pid == portage.getpid():
while True:
try:
thread_key, loop = _thread_weakrefs.loops.popitem()
except KeyError:
break
else:
loop.close()
def _start(self):
pkg = self.pkg
root_config = pkg.root_config
bintree = root_config.trees["bintree"]
binpkg_tmpfile = os.path.join(
bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid()))
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
self._binpkg_tmpfile = binpkg_tmpfile
self.settings["PORTAGE_BINPKG_TMPFILE"] = self._binpkg_tmpfile
package_phase = EbuildPhase(background=self.background,
phase='package',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(package_phase, self._package_phase_exit)
def _finalize(mysettings, items):
if len(items) == 0:
return
if len(items) == 1:
count = _("one package")
else:
count = _("multiple packages")
if "PORTAGE_ELOG_MAILURI" in mysettings:
myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0]
else:
myrecipient = "root@localhost"
myfrom = mysettings.get("PORTAGE_ELOG_MAILFROM", "")
myfrom = myfrom.replace("${HOST}", socket.getfqdn())
mysubject = mysettings.get("PORTAGE_ELOG_MAILSUBJECT", "")
mysubject = mysubject.replace("${PACKAGE}", count)
mysubject = mysubject.replace("${HOST}", socket.getfqdn())
mybody = _("elog messages for the following packages generated by "
"process %(pid)d on host %(host)s:\n") % {
"pid": portage.getpid(),
"host": socket.getfqdn()
}
for key in items:
mybody += "- %s\n" % key
mymessage = portage.mail.create_message(myfrom,
myrecipient,
mysubject,
mybody,
attachments=list(items.values()))
# Timeout after one minute in case send_mail() blocks indefinitely.
try:
try:
AlarmSignal.register(60)
portage.mail.send_mail(mysettings, mymessage)
finally:
AlarmSignal.unregister()
except AlarmSignal:
writemsg("Timeout in finalize() for elog system 'mail_summary'\n",
noiselevel=-1)
except PortageException as e:
writemsg("%s\n" % (e, ), noiselevel=-1)
return
def _hardlink_atomic(self, src, dest, dir_info, symlink=False):
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(
head, ".%s._mirrordist_hardlink_.%s" % (tail, portage.getpid()))
try:
try:
if symlink:
os.symlink(src, hardlink_tmp)
else:
os.link(src, hardlink_tmp)
except OSError as e:
if e.errno != errno.EXDEV:
msg = "hardlink %s from %s failed: %s" % (
self.distfile,
dir_info,
e,
)
self.scheduler.output(msg + "\n",
background=True,
log_path=self._log_path)
logging.error(msg)
return False
try:
os.rename(hardlink_tmp, dest)
except OSError as e:
msg = "hardlink rename '%s' from %s failed: %s" % (
self.distfile,
dir_info,
e,
)
self.scheduler.output(msg + "\n",
background=True,
log_path=self._log_path)
logging.error(msg)
return False
finally:
try:
os.unlink(hardlink_tmp)
except OSError:
pass
return True
def _internal_caller_exception_handler(loop, context): """ An exception handler which drops to a pdb shell if std* streams refer to a tty, and otherwise kills the process with SIGTERM. In order to avoid potential interference with API consumers, this implementation is only used when portage._internal_caller is True. """ loop.default_exception_handler(context) if 'exception' in context: # Normally emerge will wait for all coroutines to complete # after SIGTERM has been received. However, an unhandled # exception will prevent the interrupted coroutine from # completing, therefore use the default SIGTERM handler # in order to ensure that emerge exits immediately (though # uncleanly). signal.signal(signal.SIGTERM, signal.SIG_DFL) os.kill(portage.getpid(), signal.SIGTERM)
def global_event_loop():
"""
Get a global EventLoop (or compatible object) instance which
belongs exclusively to the current process.
"""
pid = portage.getpid()
instance = _instances.get(pid)
if instance is not None:
return instance
constructor = AsyncioEventLoop
# If the default constructor doesn't support multiprocessing,
# then multiprocessing constructor is used in subprocesses.
if not constructor.supports_multiprocessing and pid != _MAIN_PID:
constructor = EventLoop
# Use the _asyncio_wrapper attribute, so that unit tests can compare
# the reference to one retured from _wrap_loop(), since they should
# not close the loop if it refers to a global event loop.
instance = constructor()._asyncio_wrapper
_instances[pid] = instance
return instance
def get_open_fds():
return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid())
if fd.isdigit())
def process(mysettings, key, logentries, fulltext):
if mysettings.get("PORTAGE_LOGDIR"):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep),
"var", "log", "portage")
if not os.path.isdir(logdir):
# Only initialize group/mode if the directory doesn't
# exist, so that we don't override permissions if they
# were previously set by the administrator.
# NOTE: These permissions should be compatible with our
# default logrotate config as discussed in bug 374287.
logdir_uid = -1
if portage.data.secpass >= 2:
logdir_uid = portage_uid
ensure_dirs(logdir, uid=logdir_uid, gid=portage_gid, mode=0o2770)
elogdir = os.path.join(logdir, "elog")
_ensure_log_subdirs(logdir, elogdir)
# TODO: Locking
elogfilename = elogdir + "/summary.log"
try:
elogfile = io.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'],
errors='strict'),
mode='a',
encoding=_encodings['content'],
errors='backslashreplace')
except IOError as e:
func_call = "open('%s', 'a')" % elogfilename
if e.errno == errno.EACCES:
raise portage.exception.PermissionDenied(func_call)
elif e.errno == errno.EPERM:
raise portage.exception.OperationNotPermitted(func_call)
elif e.errno == errno.EROFS:
raise portage.exception.ReadOnlyFileSystem(func_call)
else:
raise
# Copy group permission bits from parent directory.
elogdir_st = os.stat(elogdir)
elogdir_gid = elogdir_st.st_gid
elogdir_grp_mode = 0o060 & elogdir_st.st_mode
# Copy the uid from the parent directory if we have privileges
# to do so, for compatibility with our default logrotate
# config (see bug 378451). With the "su portage portage"
# directive and logrotate-3.8.0, logrotate's chown call during
# the compression phase will only succeed if the log file's uid
# is portage_uid.
logfile_uid = -1
if portage.data.secpass >= 2:
logfile_uid = elogdir_st.st_uid
apply_permissions(elogfilename,
uid=logfile_uid,
gid=elogdir_gid,
mode=elogdir_grp_mode,
mask=0)
time_fmt = "%Y-%m-%d %H:%M:%S %Z"
time_str = time.strftime(time_fmt, time.localtime(time.time()))
# Avoid potential UnicodeDecodeError in Python 2, since strftime
# returns bytes in Python 2, and %Z may contain non-ascii chars.
time_str = _unicode_decode(time_str,
encoding=_encodings['content'],
errors='replace')
elogfile.write(
_(">>> Messages generated by process "
"%(pid)d on %(time)s for package %(pkg)s:\n\n") % {
"pid": portage.getpid(),
"time": time_str,
"pkg": key
})
elogfile.write(_unicode_decode(fulltext))
elogfile.write("\n")
elogfile.close()
return elogfilename
def movefile(src,
dest,
newmtime=None,
sstat=None,
mysettings=None,
hardlink_candidates=None,
encoding=_encodings['fs']):
"""moves a file from src to dest, preserving all permissions and attributes; mtime will
be preserved even when moving across filesystems. Returns mtime as integer on success
and None on failure. mtime is expressed in seconds in Python <3.3 and nanoseconds in
Python >=3.3. Move is atomic."""
if mysettings is None:
mysettings = portage.settings
src_bytes = _unicode_encode(src, encoding=encoding, errors='strict')
dest_bytes = _unicode_encode(dest, encoding=encoding, errors='strict')
xattr_enabled = "xattr" in mysettings.features
selinux_enabled = mysettings.selinux_enabled()
if selinux_enabled:
selinux = _unicode_module_wrapper(_selinux, encoding=encoding)
_copyfile = selinux.copyfile
_rename = selinux.rename
else:
_copyfile = copyfile
_rename = _os.rename
lchown = _unicode_func_wrapper(portage.data.lchown, encoding=encoding)
os = _unicode_module_wrapper(_os,
encoding=encoding,
overrides=_os_overrides)
try:
if not sstat:
sstat = os.lstat(src)
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % _("Stating source file failed... movefile()"),
noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
destexists = 1
try:
dstat = os.lstat(dest)
except (OSError, IOError):
dstat = os.lstat(os.path.dirname(dest))
destexists = 0
if bsd_chflags:
if destexists and dstat.st_flags != 0:
bsd_chflags.lchflags(dest, 0)
# Use normal stat/chflags for the parent since we want to
# follow any symlinks to the real parent directory.
pflags = os.stat(os.path.dirname(dest)).st_flags
if pflags != 0:
bsd_chflags.chflags(os.path.dirname(dest), 0)
if destexists:
if stat.S_ISLNK(dstat[stat.ST_MODE]):
try:
os.unlink(dest)
destexists = 0
except SystemExit as e:
raise
except Exception as e:
pass
if stat.S_ISLNK(sstat[stat.ST_MODE]):
try:
target = os.readlink(src)
if mysettings and "D" in mysettings and \
target.startswith(mysettings["D"]):
target = target[len(mysettings["D"]) - 1:]
if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
os.unlink(dest)
try:
if selinux_enabled:
selinux.symlink(target, dest, src)
else:
os.symlink(target, dest)
except OSError as e:
# Some programs will create symlinks automatically, so we have
# to tolerate these links being recreated during the merge
# process. In any case, if the link is pointing at the right
# place, we're in good shape.
if e.errno not in (errno.ENOENT, errno.EEXIST) or \
target != os.readlink(dest):
raise
lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
try:
_os.unlink(src_bytes)
except OSError:
pass
try:
os.utime(dest,
ns=(sstat.st_mtime_ns, sstat.st_mtime_ns),
follow_symlinks=False)
except NotImplementedError:
# utimensat() and lutimes() missing in libc.
return os.stat(dest, follow_symlinks=False).st_mtime_ns
else:
return sstat.st_mtime_ns
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % _("failed to properly create symlink:"),
noiselevel=-1)
writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
hardlinked = False
# Since identical files might be merged to multiple filesystems,
# so os.link() calls might fail for some paths, so try them all.
# For atomic replacement, first create the link as a temp file
# and them use os.rename() to replace the destination.
if hardlink_candidates:
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
(tail, portage.getpid()))
try:
os.unlink(hardlink_tmp)
except OSError as e:
if e.errno != errno.ENOENT:
writemsg(_("!!! Failed to remove hardlink temp file: %s\n") % \
(hardlink_tmp,), noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
del e
for hardlink_src in hardlink_candidates:
try:
os.link(hardlink_src, hardlink_tmp)
except OSError:
continue
else:
try:
os.rename(hardlink_tmp, dest)
except OSError as e:
writemsg(_("!!! Failed to rename %s to %s\n") % \
(hardlink_tmp, dest), noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
hardlinked = True
try:
_os.unlink(src_bytes)
except OSError:
pass
break
renamefailed = 1
if hardlinked:
renamefailed = False
if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev):
try:
if selinux_enabled:
selinux.rename(src, dest)
else:
os.rename(src, dest)
renamefailed = 0
except OSError as e:
if e.errno != errno.EXDEV:
# Some random error.
writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") %
{
"src": src,
"dest": dest
},
noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
# Invalid cross-device-link 'bind' mounted or actually Cross-Device
def copy_file(unlink_first=False):
dest_tmp = dest + "#new"
dest_tmp_bytes = _unicode_encode(dest_tmp,
encoding=encoding,
errors='strict')
try: # For safety copy then move it over.
_copyfile(src_bytes, dest_tmp_bytes)
_apply_stat(sstat, dest_tmp_bytes)
if xattr_enabled:
try:
_copyxattr(src_bytes,
dest_tmp_bytes,
exclude=mysettings.get("PORTAGE_XATTR_EXCLUDE",
""))
except SystemExit:
raise
except:
msg = _("Failed to copy extended attributes. "
"In order to avoid this error, set "
"FEATURES=\"-xattr\" in make.conf.")
msg = textwrap.wrap(msg, 65)
for line in msg:
writemsg("!!! %s\n" % (line, ), noiselevel=-1)
raise
_rename(dest_tmp_bytes, dest_bytes)
_os.unlink(src_bytes)
finally:
if _os.path.exists(dest_tmp_bytes):
_os.unlink(dest_tmp_bytes)
if renamefailed:
if stat.S_ISREG(sstat[stat.ST_MODE]):
try:
copy_file()
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') % {
"src": src,
"dest": dest
},
noiselevel=-1)
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
try:
writemsg("!!! unlink %s then retry.\n" % dest,
noiselevel=-1)
if _os.path.exists(dest_bytes):
_os.unlink(dest_bytes)
copy_file()
except SystemExit as e:
raise
except Exception as e:
writemsg("!!! %s\n" % (e, ), noiselevel=-1)
return None
else:
#we don't yet handle special, so we need to fall back to /bin/mv
a = spawn([MOVE_BINARY, '-f', src, dest], env=os.environ)
if a != os.EX_OK:
writemsg(_("!!! Failed to move special file:\n"),
noiselevel=-1)
writemsg(_("!!! '%(src)s' to '%(dest)s'\n") % \
{"src": _unicode_decode(src, encoding=encoding),
"dest": _unicode_decode(dest, encoding=encoding)}, noiselevel=-1)
writemsg("!!! %s\n" % a, noiselevel=-1)
return None # failure
# In Python <3.3 always use stat_obj[stat.ST_MTIME] for the integral timestamp
# which is returned, since the stat_obj.st_mtime float attribute rounds *up*
# if the nanosecond part of the timestamp is 999999881 ns or greater.
try:
if hardlinked:
newmtime = os.stat(dest).st_mtime_ns
else:
# Note: It is not possible to preserve nanosecond precision
# (supported in POSIX.1-2008 via utimensat) with the IEEE 754
# double precision float which only has a 53 bit significand.
if newmtime is not None:
os.utime(dest, ns=(newmtime, newmtime))
else:
newmtime = sstat.st_mtime_ns
if renamefailed:
# If rename succeeded then timestamps are automatically
# preserved with complete precision because the source
# and destination inodes are the same. Otherwise, manually
# update timestamps with nanosecond precision.
os.utime(dest, ns=(newmtime, newmtime))
except OSError:
# The utime can fail here with EPERM even though the move succeeded.
# Instead of failing, use stat to return the mtime if possible.
try:
newmtime = os.stat(dest).st_mtime_ns
except OSError as e:
writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1)
writemsg("!!! %s\n" % dest, noiselevel=-1)
writemsg("!!! %s\n" % str(e), noiselevel=-1)
return None
if bsd_chflags:
# Restore the flags we saved before moving
if pflags:
bsd_chflags.chflags(os.path.dirname(dest), pflags)
return newmtime
def _wrap_coroutine_func(self, coroutine_func):
parent_loop = global_event_loop()
parent_pid = portage.getpid()
pending = weakref.WeakValueDictionary()
# Since ThreadPoolExecutor does not propagate cancellation of a
# parent_future to the underlying coroutine, use kill_switch to
# propagate task cancellation to wrapper, so that HangForever's
# thread returns when retry eventually cancels parent_future.
def wrapper(kill_switch):
if portage.getpid() == parent_pid:
# thread in main process
def done_callback(result):
result.cancelled() or result.exception() or result.result()
kill_switch.set()
def start_coroutine(future):
result = asyncio.ensure_future(coroutine_func(),
loop=parent_loop)
pending[id(result)] = result
result.add_done_callback(done_callback)
future.set_result(result)
future = Future()
parent_loop.call_soon_threadsafe(start_coroutine, future)
kill_switch.wait()
if not future.done():
future.cancel()
raise asyncio.CancelledError
elif not future.result().done():
future.result().cancel()
raise asyncio.CancelledError
else:
return future.result().result()
# child process
loop = global_event_loop()
try:
return loop.run_until_complete(coroutine_func())
finally:
loop.close()
def execute_wrapper():
kill_switch = threading.Event()
parent_future = asyncio.ensure_future(parent_loop.run_in_executor(
self._executor, wrapper, kill_switch),
loop=parent_loop)
def kill_callback(parent_future):
if not kill_switch.is_set():
kill_switch.set()
parent_future.add_done_callback(kill_callback)
return parent_future
try:
yield execute_wrapper
finally:
while True:
try:
_, future = pending.popitem()
except KeyError:
break
try:
parent_loop.run_until_complete(future)
except (Exception, asyncio.CancelledError):
pass
future.cancelled() or future.exception() or future.result()
def _setitem(self, cpv, values):
if "_eclasses_" in values:
values = ProtectedDict(values)
values["INHERITED"] = ' '.join(sorted(values["_eclasses_"]))
new_content = []
for k in self.auxdbkey_order:
new_content.append(values.get(k, ''))
new_content.append('\n')
for i in range(magic_line_count - len(self.auxdbkey_order)):
new_content.append('\n')
new_content = ''.join(new_content)
new_content = _unicode_encode(new_content,
_encodings['repo.content'],
errors='backslashreplace')
new_fp = os.path.join(self.location, cpv)
try:
f = open(
_unicode_encode(new_fp,
encoding=_encodings['fs'],
errors='strict'), 'rb')
except EnvironmentError:
pass
else:
try:
try:
existing_st = os.fstat(f.fileno())
existing_content = f.read()
finally:
f.close()
except EnvironmentError:
pass
else:
existing_mtime = existing_st[stat.ST_MTIME]
if values['_mtime_'] == existing_mtime and \
existing_content == new_content:
return
if self.raise_stat_collision and \
values['_mtime_'] == existing_mtime and \
len(new_content) == existing_st.st_size:
raise cache_errors.StatCollision(cpv, new_fp,
existing_mtime,
existing_st.st_size)
s = cpv.rfind("/")
fp = os.path.join(self.location, cpv[:s],
".update.%i.%s" % (portage.getpid(), cpv[s + 1:]))
try:
myf = open(
_unicode_encode(fp, encoding=_encodings['fs'],
errors='strict'), 'wb')
except EnvironmentError as e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
myf = open(
_unicode_encode(fp,
encoding=_encodings['fs'],
errors='strict'), 'wb')
except EnvironmentError as e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
myf.write(new_content)
finally:
myf.close()
self._ensure_access(fp, mtime=values["_mtime_"])
try:
os.rename(fp, new_fp)
except EnvironmentError as e:
try:
os.unlink(fp)
except EnvironmentError:
pass
raise cache_errors.CacheCorruption(cpv, e)
# Copyright 2012-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import portage
from .EventLoop import EventLoop
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
_MAIN_PID = portage.getpid()
_instances = {}
def global_event_loop():
"""
Get a global EventLoop (or compatible object) instance which
belongs exclusively to the current process.
"""
pid = portage.getpid()
instance = _instances.get(pid)
if instance is not None:
return instance
constructor = AsyncioEventLoop
# If the default constructor doesn't support multiprocessing,
# then multiprocessing constructor is used in subprocesses.
if not constructor.supports_multiprocessing and pid != _MAIN_PID:
constructor = EventLoop
# Use the _asyncio_wrapper attribute, so that unit tests can compare
# the reference to one retured from _wrap_loop(), since they should
# not close the loop if it refers to a global event loop.
def spawn(mycommand,
env=None,
opt_name=None,
fd_pipes=None,
returnpid=False,
uid=None,
gid=None,
groups=None,
umask=None,
cwd=None,
logfile=None,
path_lookup=True,
pre_exec=None,
close_fds=False,
unshare_net=False,
unshare_ipc=False,
unshare_mount=False,
unshare_pid=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: If env is not None, it must be a mapping that defines the environment
variables for the new process; these are used instead of the default behavior
of inheriting the current process's environment.
@type env: None or Mapping
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, str):
mycommand = mycommand.split()
env = os.environ if env is None else env
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(
spawn(('tee', '-i', '-a', logfile),
returnpid=True,
fd_pipes={
0: pr,
1: fd_pipes[1],
2: fd_pipes[2]
}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# Cache _has_ipv6() result for use in child processes.
_has_ipv6()
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
# child processes.
unshare_flags = 0
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWUTS = 0x04000000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
if unshare_net:
# UTS namespace to override hostname
unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS
if unshare_ipc:
unshare_flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
unshare_flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
unshare_flags |= CLONE_NEWPID | CLONE_NEWNS
_unshare_validate(unshare_flags)
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = portage.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
portage._ForkWatcher.hook(portage._ForkWatcher)
try:
_exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups,
uid, umask, cwd, pre_exec, close_fds, unshare_net,
unshare_ipc, unshare_mount, unshare_pid, unshare_flags,
cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
# Don't used portage.getpid() here, due to a race with the above
# portage._ForkWatcher cache update.
if pid == 0 or (pid is None and _os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid), ))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if retval & 0xff:
return (retval & 0xff) << 8
# Otherwise, return its exit code.
return retval >> 8
# Everything succeeded
return 0
def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
cwd, pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount,
unshare_pid, unshare_flags, cgroup):
"""
Execute a given binary with options
@param binary: Name of program to execute
@type binary: String
@param mycommand: Options for program
@type mycommand: String
@param opt_name: Name of process (defaults to binary)
@type opt_name: String
@param fd_pipes: Mapping pipes to destination; { 0:0, 1:1, 2:2 }
@type fd_pipes: Dictionary
@param env: Key,Value mapping for Environmental Variables
@type env: Dictionary
@param gid: Group ID to run the process under
@type gid: Integer
@param groups: Groups the Process should be in.
@type groups: List
@param uid: User ID to run the process under
@type uid: Integer
@param umask: an int representing a unix umask (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param unshare_flags: Flags for the unshare(2) function
@type unshare_flags: Integer
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@return: Never returns (calls os.execve)
"""
# If the process we're creating hasn't been given a name
# assign it the name of the executable.
if not opt_name:
if binary is portage._python_interpreter:
# NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0]
# does not contain the full path of the binary.
opt_name = binary
else:
opt_name = os.path.basename(binary)
# Set up the command's argument list.
myargs = [opt_name]
myargs.extend(mycommand[1:])
# Avoid a potential UnicodeEncodeError from os.execve().
myargs = [
_unicode_encode(x, encoding=_encodings['fs'], errors='strict')
for x in myargs
]
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# Unregister SIGCHLD handler and wakeup_fd for the parent
# process's event loop (bug 655656).
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
try:
wakeup_fd = signal.set_wakeup_fd(-1)
if wakeup_fd > 0:
os.close(wakeup_fd)
except (ValueError, OSError):
pass
# Quiet killing of subprocesses by SIGPIPE (see bug #309001).
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
# Avoid issues triggered by inheritance of SIGQUIT handler from
# the parent process (see bug #289486).
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
_setup_pipes(fd_pipes, close_fds=close_fds, inheritable=True)
# Add to cgroup
# it's better to do it from the child since we can guarantee
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
f.write('%d\n' % portage.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
try:
# Since a failed unshare call could corrupt process
# state, first validate that the call can succeed.
# The parent process should call _unshare_validate
# before it forks, so that all child processes can
# reuse _unshare_validate results that have been
# cached by the parent process.
errno_value = _unshare_validate(unshare_flags)
if errno_value == 0 and libc.unshare(unshare_flags) != 0:
errno_value = ctypes.get_errno()
if errno_value != 0:
involved_features = []
if unshare_ipc:
involved_features.append('ipc-sandbox')
if unshare_mount:
involved_features.append('mount-sandbox')
if unshare_net:
involved_features.append('network-sandbox')
if unshare_pid:
involved_features.append('pid-sandbox')
writemsg(
"Unable to unshare: %s (for FEATURES=\"%s\")\n" %
(errno.errorcode.get(errno_value, '?'),
' '.join(involved_features)),
noiselevel=-1)
else:
if unshare_pid:
main_child_pid = os.fork()
if main_child_pid == 0:
# The portage.getpid() cache may need to be updated here,
# in case the pre_exec function invokes portage APIs.
portage._ForkWatcher.hook(portage._ForkWatcher)
# pid namespace requires us to become init
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
_unicode_encode(
'' if uid is None else str(uid)),
_unicode_encode(
'' if gid is None else str(gid)),
_unicode_encode(
'' if groups is None else ','.join(
str(group) for group in groups)),
_unicode_encode(
'' if umask is None else str(umask)),
_unicode_encode(','.join(
str(fd) for fd in fd_pipes)), binary
] + myargs
uid = None
gid = None
groups = None
umask = None
else:
# Execute a supervisor process which will forward
# signals to init and forward exit status to the
# parent process. The supervisor process runs in
# the global pid namespace, so skip /proc remount
# and other setup that's intended only for the
# init process.
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
os.path.join(portage._bin_path,
'pid-ns-init'),
str(main_child_pid)
]
os.execve(binary, myargs, env)
if unshare_mount:
# mark the whole filesystem as slave to avoid
# mounts escaping the namespace
s = subprocess.Popen(
['mount', '--make-rslave', '/'])
mount_ret = s.wait()
if mount_ret != 0:
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts slave: %d\n" %
(mount_ret, ),
noiselevel=-1)
if unshare_pid:
# we need at least /proc being slave
s = subprocess.Popen(
['mount', '--make-slave', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
# can't proceed with shared /proc
writemsg("Unable to mark /proc slave: %d\n" %
(mount_ret, ),
noiselevel=-1)
os._exit(1)
# mount new /proc for our namespace
s = subprocess.Popen(
['mount', '-n', '-t', 'proc', 'proc', '/proc'])
mount_ret = s.wait()
if mount_ret != 0:
writemsg("Unable to mount new /proc: %d\n" %
(mount_ret, ),
noiselevel=-1)
os._exit(1)
if unshare_net:
# use 'localhost' to avoid hostname resolution problems
try:
# pypy3 does not implement socket.sethostname()
new_hostname = b'localhost'
if hasattr(socket, 'sethostname'):
socket.sethostname(new_hostname)
else:
if libc.sethostname(
new_hostname,
len(new_hostname)) != 0:
errno_value = ctypes.get_errno()
raise OSError(errno_value,
os.strerror(errno_value))
except Exception as e:
writemsg(
"Unable to set hostname: %s (for FEATURES=\"network-sandbox\")\n"
% (e, ),
noiselevel=-1)
_configure_loopback_interface()
except AttributeError:
# unshare() not supported by libc
pass
# Set requested process permissions.
if gid:
# Cast proxies to int, in case it matters.
os.setgid(int(gid))
if groups:
os.setgroups(groups)
if uid:
# Cast proxies to int, in case it matters.
os.setuid(int(uid))
if umask:
os.umask(umask)
if cwd is not None:
os.chdir(cwd)
if pre_exec:
pre_exec()
# And switch to the new process.
os.execve(binary, myargs, env)
def get_open_fds():
return (int(fd) for fd in os.listdir(_fd_dir) if fd.isdigit())
if platform.python_implementation() == 'PyPy':
# EAGAIN observed with PyPy 1.8.
_get_open_fds = get_open_fds
def get_open_fds():
try:
return _get_open_fds()
except OSError as e:
if e.errno != errno.EAGAIN:
raise
return range(max_fd_limit)
elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
# In order for this function to work in forked subprocesses,
# os.getpid() must be called from inside the function.
def get_open_fds():
return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid())
if fd.isdigit())
else:
def get_open_fds():
return range(max_fd_limit)
sandbox_capable = (os.path.isfile(SANDBOX_BINARY)
and os.access(SANDBOX_BINARY, os.X_OK))
def _db_connection(self):
if (self._db_connection_info is None
or self._db_connection_info.pid != portage.getpid()):
self._db_init_connection()
return self._db_connection_info.connection