def rebuild(self, exclude_pkgs=None, include_file=None,
preserve_paths=None):
"""
Raises CommandNotFound if there are preserved libs
and the scanelf binary is not available.
@param exclude_pkgs: A set of packages that should be excluded from
the LinkageMap, since they are being unmerged and their NEEDED
entries are therefore irrelevant and would only serve to corrupt
the LinkageMap.
@type exclude_pkgs: set
@param include_file: The path of a file containing NEEDED entries for
a package which does not exist in the vardbapi yet because it is
currently being merged.
@type include_file: String
@param preserve_paths: Libraries preserved by a package instance that
is currently being merged. They need to be explicitly passed to the
LinkageMap, since they are not registered in the
PreservedLibsRegistry yet.
@type preserve_paths: set
"""
os = _os_merge
root = self._root
root_len = len(root) - 1
self._clear_cache()
self._defpath.update(getlibpaths(self._dbapi.settings['EROOT'],
env=self._dbapi.settings))
libs = self._libs
obj_properties = self._obj_properties
lines = []
# Data from include_file is processed first so that it
# overrides any data from previously installed files.
if include_file is not None:
for line in grabfile(include_file):
lines.append((None, include_file, line))
aux_keys = [self._needed_aux_key]
can_lock = os.access(os.path.dirname(self._dbapi._dbroot), os.W_OK)
if can_lock:
self._dbapi.lock()
try:
for cpv in self._dbapi.cpv_all():
if exclude_pkgs is not None and cpv in exclude_pkgs:
continue
needed_file = self._dbapi.getpath(cpv,
filename=self._needed_aux_key)
for line in self._dbapi.aux_get(cpv, aux_keys)[0].splitlines():
lines.append((cpv, needed_file, line))
finally:
if can_lock:
self._dbapi.unlock()
# have to call scanelf for preserved libs here as they aren't
# registered in NEEDED.ELF.2 files
plibs = {}
if preserve_paths is not None:
plibs.update((x, None) for x in preserve_paths)
if self._dbapi._plib_registry and \
self._dbapi._plib_registry.hasEntries():
for cpv, items in \
self._dbapi._plib_registry.getPreservedLibs().items():
if exclude_pkgs is not None and cpv in exclude_pkgs:
# These preserved libs will either be unmerged,
# rendering them irrelevant, or they will be
# preserved in the replacement package and are
# already represented via the preserve_paths
# parameter.
continue
plibs.update((x, cpv) for x in items)
if plibs:
args = [os.path.join(EPREFIX or "/", "usr/bin/scanelf"), "-qF", "%a;%F;%S;%r;%n"]
args.extend(os.path.join(root, x.lstrip("." + os.sep)) \
for x in plibs)
try:
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
raise CommandNotFound(args[0])
else:
for l in proc.stdout:
try:
l = _unicode_decode(l,
encoding=_encodings['content'], errors='strict')
except UnicodeDecodeError:
l = _unicode_decode(l,
encoding=_encodings['content'], errors='replace')
writemsg_level(_("\nError decoding characters " \
"returned from scanelf: %s\n\n") % (l,),
level=logging.ERROR, noiselevel=-1)
l = l[3:].rstrip("\n")
if not l:
continue
try:
entry = NeededEntry.parse("scanelf", l)
except InvalidData as e:
writemsg_level("\n%s\n\n" % (e,),
level=logging.ERROR, noiselevel=-1)
continue
try:
with open(_unicode_encode(entry.filename,
encoding=_encodings['fs'],
errors='strict'), 'rb') as f:
elf_header = ELFHeader.read(f)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
# File removed concurrently.
continue
entry.multilib_category = compute_multilib_category(elf_header)
entry.filename = entry.filename[root_len:]
owner = plibs.pop(entry.filename, None)
lines.append((owner, "scanelf", _unicode(entry)))
proc.wait()
proc.stdout.close()
if plibs:
# Preserved libraries that did not appear in the scanelf output.
# This is known to happen with statically linked libraries.
# Generate dummy lines for these, so we can assume that every
# preserved library has an entry in self._obj_properties. This
# is important in order to prevent findConsumers from raising
# an unwanted KeyError.
for x, cpv in plibs.items():
lines.append((cpv, "plibs", ";".join(['', x, '', '', ''])))
# Share identical frozenset instances when available,
# in order to conserve memory.
frozensets = {}
for owner, location, l in lines:
l = l.rstrip("\n")
if not l:
continue
if '\0' in l:
# os.stat() will raise "TypeError: must be encoded string
# without NULL bytes, not str" in this case.
writemsg_level(_("\nLine contains null byte(s) " \
"in %s: %s\n\n") % (location, l),
level=logging.ERROR, noiselevel=-1)
continue
try:
entry = NeededEntry.parse(location, l)
except InvalidData as e:
writemsg_level("\n%s\n\n" % (e,),
level=logging.ERROR, noiselevel=-1)
continue
# If NEEDED.ELF.2 contains the new multilib category field,
# then use that for categorization. Otherwise, if a mapping
# exists, map e_machine (entry.arch) to an approximate
# multilib category. If all else fails, use e_machine, just
# as older versions of portage did.
arch = entry.multilib_category
if arch is None:
arch = _approx_multilib_categories.get(
entry.arch, entry.arch)
obj = entry.filename
soname = entry.soname
expand = {"ORIGIN": os.path.dirname(entry.filename)}
path = frozenset(normalize_path(
varexpand(x, expand, error_leader=lambda: "%s: " % location))
for x in entry.runpaths)
path = frozensets.setdefault(path, path)
needed = frozenset(entry.needed)
needed = frozensets.setdefault(needed, needed)
obj_key = self._obj_key(obj)
indexed = True
myprops = obj_properties.get(obj_key)
if myprops is None:
indexed = False
myprops = self._obj_properties_class(
arch, needed, path, soname, [], owner)
obj_properties[obj_key] = myprops
# All object paths are added into the obj_properties tuple.
myprops.alt_paths.append(obj)
# Don't index the same file more that once since only one
# set of data can be correct and therefore mixing data
# may corrupt the index (include_file overrides previously
# installed).
if indexed:
continue
arch_map = libs.get(arch)
if arch_map is None:
arch_map = {}
libs[arch] = arch_map
if soname:
soname_map = arch_map.get(soname)
if soname_map is None:
soname_map = self._soname_map_class(
providers=[], consumers=[])
arch_map[soname] = soname_map
soname_map.providers.append(obj_key)
for needed_soname in needed:
soname_map = arch_map.get(needed_soname)
if soname_map is None:
soname_map = self._soname_map_class(
providers=[], consumers=[])
arch_map[needed_soname] = soname_map
soname_map.consumers.append(obj_key)
for arch, sonames in libs.items():
for soname_node in sonames.values():
soname_node.providers = tuple(set(soname_node.providers))
soname_node.consumers = tuple(set(soname_node.consumers))
def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None,
path_lookup=True, pre_exec=None,
close_fds=(sys.version_info < (3, 4)), unshare_net=False,
unshare_ipc=False, unshare_mount=False, unshare_pid=False,
cgroup=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: If env is not None, it must be a mapping that defines the environment
variables for the new process; these are used instead of the default behavior
of inheriting the current process's environment.
@type env: None or Mapping
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
(default is {0:stdin, 1:stdout, 2:stderr})
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param cwd: Current working directory
@type cwd: String
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
@param close_fds: If True, then close all file descriptors except those
referenced by fd_pipes (default is True for python3.3 and earlier, and False for
python3.4 and later due to non-inheritable file descriptor behavior from PEP 446).
@type close_fds: Boolean
@param unshare_net: If True, networking will be unshared from the spawned process
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
@param unshare_pid: If True, PID ns will be unshared from the spawned process
@type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
env = os.environ if env is None else env
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:portage._get_stdin().fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
# Cache _has_ipv6() result for use in child processes.
_has_ipv6()
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
# child processes.
unshare_flags = 0
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWUTS = 0x04000000
CLONE_NEWIPC = 0x08000000
CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000
if unshare_net:
# UTS namespace to override hostname
unshare_flags |= CLONE_NEWNET | CLONE_NEWUTS
if unshare_ipc:
unshare_flags |= CLONE_NEWIPC
if unshare_mount:
# NEWNS = mount namespace
unshare_flags |= CLONE_NEWNS
if unshare_pid:
# we also need mount namespace for slave /proc
unshare_flags |= CLONE_NEWPID | CLONE_NEWNS
_unshare_validate(unshare_flags)
# Force instantiation of portage.data.userpriv_groups before the
# fork, so that the result is cached in the main process.
bool(groups)
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
unshare_net, unshare_ipc, unshare_mount, unshare_pid,
unshare_flags, cgroup)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
writemsg("%s:\n %s\n" % (e, " ".join(mycommand)),
noiselevel=-1)
traceback.print_exc()
sys.stderr.flush()
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, logfile=None,
path_lookup=True, pre_exec=None):
"""
Spawns a given command.
@param mycommand: the command to execute
@type mycommand: String or List (Popen style list)
@param env: A dict of Key=Value pairs for env variables
@type env: Dictionary
@param opt_name: an optional name for the spawn'd process (defaults to the binary name)
@type opt_name: String
@param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout } for example
@type fd_pipes: Dictionary
@param returnpid: Return the Process IDs for a successful spawn.
NOTE: This requires the caller clean up all the PIDs, otherwise spawn will clean them.
@type returnpid: Boolean
@param uid: User ID to spawn as; useful for dropping privilages
@type uid: Integer
@param gid: Group ID to spawn as; useful for dropping privilages
@type gid: Integer
@param groups: Group ID's to spawn in: useful for having the process run in multiple group contexts.
@type groups: List
@param umask: An integer representing the umask for the process (see man chmod for umask details)
@type umask: Integer
@param logfile: name of a file to use for logging purposes
@type logfile: String
@param path_lookup: If the binary is not fully specified then look for it in PATH
@type path_lookup: Boolean
@param pre_exec: A function to be called with no arguments just prior to the exec call.
@type pre_exec: callable
logfile requires stdout and stderr to be assigned to this process (ie not pointed
somewhere else.)
"""
# mycommand is either a str or a list
if isinstance(mycommand, basestring):
mycommand = mycommand.split()
if sys.hexversion < 0x3000000:
# Avoid a potential UnicodeEncodeError from os.execve().
env_bytes = {}
for k, v in env.items():
env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \
_unicode_encode(v, encoding=_encodings['content'])
env = env_bytes
del env_bytes
# If an absolute path to an executable file isn't given
# search for it unless we've been told not to.
binary = mycommand[0]
if binary not in (BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY) and \
(not os.path.isabs(binary) or not os.path.isfile(binary)
or not os.access(binary, os.X_OK)):
binary = path_lookup and find_binary(binary) or None
if not binary:
raise CommandNotFound(mycommand[0])
# If we haven't been told what file descriptors to use
# default to propagating our stdin, stdout and stderr.
if fd_pipes is None:
fd_pipes = {
0:sys.__stdin__.fileno(),
1:sys.__stdout__.fileno(),
2:sys.__stderr__.fileno(),
}
# mypids will hold the pids of all processes created.
mypids = []
if logfile:
# Using a log file requires that stdout and stderr
# are assigned to the process we're running.
if 1 not in fd_pipes or 2 not in fd_pipes:
raise ValueError(fd_pipes)
# Create a pipe
(pr, pw) = os.pipe()
# Create a tee process, giving it our stdout and stderr
# as well as the read end of the pipe.
mypids.extend(spawn(('tee', '-i', '-a', logfile),
returnpid=True, fd_pipes={0:pr,
1:fd_pipes[1], 2:fd_pipes[2]}))
# We don't need the read end of the pipe, so close it.
os.close(pr)
# Assign the write end of the pipe to our stdout and stderr.
fd_pipes[1] = pw
fd_pipes[2] = pw
pid = os.fork()
if pid == 0:
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, pre_exec)
except SystemExit:
raise
except Exception as e:
# We need to catch _any_ exception so that it doesn't
# propagate out of this function and cause exiting
# with anything other than os._exit()
sys.stderr.write("%s:\n %s\n" % (e, " ".join(mycommand)))
traceback.print_exc()
sys.stderr.flush()
os._exit(1)
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" % (repr(pid),))
# Add the pid to our local and the global pid lists.
mypids.append(pid)
spawned_pids.append(pid)
# If we started a tee process the write side of the pipe is no
# longer needed, so close it.
if logfile:
os.close(pw)
# If the caller wants to handle cleaning up the processes, we tell
# it about all processes that were created.
if returnpid:
return mypids
# Otherwise we clean them up.
while mypids:
# Pull the last reader in the pipe chain. If all processes
# in the pipe are well behaved, it will die when the process
# it is reading from dies.
pid = mypids.pop(0)
# and wait for it.
retval = os.waitpid(pid, 0)[1]
# When it's done, we can remove it from the
# global pid list as well.
spawned_pids.remove(pid)
if retval:
# If it failed, kill off anything else that
# isn't dead yet.
for pid in mypids:
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if os.waitpid(pid, os.WNOHANG)[0] == 0:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
spawned_pids.remove(pid)
# If it got a signal, return the signal that was sent.
if (retval & 0xff):
return ((retval & 0xff) << 8)
# Otherwise, return its exit code.
return (retval >> 8)
# Everything succeeded
return 0
def rebuild(self, exclude_pkgs=None, include_file=None):
"""
Raises CommandNotFound if there are preserved libs
and the scanelf binary is not available.
"""
os = _os_merge
root = self._eroot
root_len = len(root) - 1
self._clear_cache()
self._defpath.update(getlibpaths(self._eroot))
libs = self._libs
obj_properties = self._obj_properties
lines = []
# Data from include_file is processed first so that it
# overrides any data from previously installed files.
if include_file is not None:
lines += grabfile(include_file)
aux_keys = [self._needed_aux_key]
for cpv in self._dbapi.cpv_all():
if exclude_pkgs is not None and cpv in exclude_pkgs:
continue
lines += self._dbapi.aux_get(cpv, aux_keys)[0].split('\n')
# Cache NEEDED.* files avoid doing excessive IO for every rebuild.
self._dbapi.flush_cache()
# have to call scanelf for preserved libs here as they aren't
# registered in NEEDED.ELF.2 files
plibs = set()
if self._dbapi._plib_registry and self._dbapi._plib_registry.getPreservedLibs():
args = ["/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"]
for items in self._dbapi._plib_registry.getPreservedLibs().values():
plibs.update(items)
args.extend(os.path.join(root, x.lstrip("." + os.sep)) \
for x in items)
try:
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
raise CommandNotFound(args[0])
else:
for l in proc.stdout:
try:
l = _unicode_decode(l,
encoding=_encodings['content'], errors='strict')
except UnicodeDecodeError:
l = _unicode_decode(l,
encoding=_encodings['content'], errors='replace')
writemsg_level(_("\nError decoding characters " \
"returned from scanelf: %s\n\n") % (l,),
level=logging.ERROR, noiselevel=-1)
l = l[3:].rstrip("\n")
if not l:
continue
fields = l.split(";")
if len(fields) < 5:
writemsg_level(_("\nWrong number of fields " \
"returned from scanelf: %s\n\n") % (l,),
level=logging.ERROR, noiselevel=-1)
continue
fields[1] = fields[1][root_len:]
plibs.discard(fields[1])
lines.append(";".join(fields))
proc.wait()
if plibs:
# Preserved libraries that did not appear in the scanelf output.
# This is known to happen with statically linked libraries.
# Generate dummy lines for these, so we can assume that every
# preserved library has an entry in self._obj_properties. This
# is important in order to prevent findConsumers from raising
# an unwanted KeyError.
for x in plibs:
lines.append(";".join(['', x, '', '', '']))
for l in lines:
l = l.rstrip("\n")
if not l:
continue
fields = l.split(";")
if len(fields) < 5:
writemsg_level(_("\nWrong number of fields " \
"in %s: %s\n\n") % (self._needed_aux_key, l),
level=logging.ERROR, noiselevel=-1)
continue
arch = fields[0]
obj = fields[1]
soname = fields[2]
path = set([normalize_path(x) \
for x in filter(None, fields[3].replace(
"${ORIGIN}", os.path.dirname(obj)).replace(
"$ORIGIN", os.path.dirname(obj)).split(":"))])
needed = [x for x in fields[4].split(",") if x]
obj_key = self._obj_key(obj)
indexed = True
myprops = obj_properties.get(obj_key)
if myprops is None:
indexed = False
myprops = (arch, needed, path, soname, set())
obj_properties[obj_key] = myprops
# All object paths are added into the obj_properties tuple.
myprops[4].add(obj)
# Don't index the same file more that once since only one
# set of data can be correct and therefore mixing data
# may corrupt the index (include_file overrides previously
# installed).
if indexed:
continue
arch_map = libs.get(arch)
if arch_map is None:
arch_map = {}
libs[arch] = arch_map
if soname:
soname_map = arch_map.get(soname)
if soname_map is None:
soname_map = self._soname_map_class(
providers=set(), consumers=set())
arch_map[soname] = soname_map
soname_map.providers.add(obj_key)
for needed_soname in needed:
soname_map = arch_map.get(needed_soname)
if soname_map is None:
soname_map = self._soname_map_class(
providers=set(), consumers=set())
arch_map[needed_soname] = soname_map
soname_map.consumers.add(obj_key)
