def _init_ipc_fifos(self): input_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo)
def _init_ipc_fifos(self):
input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".ipc_out")
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(
p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770,
stat_cached=st,
)
return (input_fifo, output_fifo)
def _want_userfetch(settings):
"""
Check if it's desirable to drop privileges for userfetch.
@param settings: portage config
@type settings: portage.package.ebuild.config.config
@return: True if desirable, False otherwise
"""
return ('userfetch' in settings.features and
portage.data.secpass >= 2 and os.getuid() == 0)
def _spawn_fetch(settings, args, **kwargs):
"""
Spawn a process with appropriate settings for fetching, including
userfetch and selinux support.
"""
global _userpriv_spawn_kwargs
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
if "fd_pipes" not in kwargs:
kwargs["fd_pipes"] = {
0 : portage._get_stdin().fileno(),
1 : sys.__stdout__.fileno(),
2 : sys.__stdout__.fileno(),
}
logname = None
if "userfetch" in settings.features and \
os.getuid() == 0 and portage_gid and portage_uid and \
hasattr(os, "setgroups"):
kwargs.update(_userpriv_spawn_kwargs)
logname = portage.data._portage_username
spawn_func = spawn
if settings.selinux_enabled():
spawn_func = selinux.spawn_wrapper(spawn_func,
settings["PORTAGE_FETCH_T"])
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
# Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
# does not filter the calling environment (which may contain needed
# proxy variables, as in bug #315421).
phase_backup = settings.get('EBUILD_PHASE')
settings['EBUILD_PHASE'] = 'fetch'
env = settings.environ()
if logname is not None:
env["LOGNAME"] = logname
try:
rval = spawn_func(args, env=env, **kwargs)
finally:
if phase_backup is None:
settings.pop('EBUILD_PHASE', None)
else:
settings['EBUILD_PHASE'] = phase_backup
return rval
def _spawn_fetch(settings, args, **kwargs):
"""
Spawn a process with appropriate settings for fetching, including
userfetch and selinux support.
"""
global _userpriv_spawn_kwargs
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
if "fd_pipes" not in kwargs:
kwargs["fd_pipes"] = {
0 : portage._get_stdin().fileno(),
1 : sys.__stdout__.fileno(),
2 : sys.__stdout__.fileno(),
}
logname = None
if "userfetch" in settings.features and \
os.getuid() == 0 and portage_gid and portage_uid and \
hasattr(os, "setgroups"):
kwargs.update(_userpriv_spawn_kwargs)
logname = portage.data._portage_username
spawn_func = spawn
if settings.selinux_enabled():
spawn_func = selinux.spawn_wrapper(spawn_func,
settings["PORTAGE_FETCH_T"])
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
# Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
# does not filter the calling environment (which may contain needed
# proxy variables, as in bug #315421).
phase_backup = settings.get('EBUILD_PHASE')
settings['EBUILD_PHASE'] = 'fetch'
env = settings.environ()
if logname is not None:
env["LOGNAME"] = logname
try:
rval = spawn_func(args, env=env, **kwargs)
finally:
if phase_backup is None:
settings.pop('EBUILD_PHASE', None)
else:
settings['EBUILD_PHASE'] = phase_backup
return rval
def get_committer_name(env=None):
"""Generate a committer string like echangelog does."""
if env is None:
env = os.environ
if "GENTOO_COMMITTER_NAME" in env and "GENTOO_COMMITTER_EMAIL" in env:
user = "******" % (env["GENTOO_COMMITTER_NAME"], env["GENTOO_COMMITTER_EMAIL"])
elif "GENTOO_AUTHOR_NAME" in env and "GENTOO_AUTHOR_EMAIL" in env:
user = "******" % (env["GENTOO_AUTHOR_NAME"], env["GENTOO_AUTHOR_EMAIL"])
elif "ECHANGELOG_USER" in env:
user = env["ECHANGELOG_USER"]
else:
pwd_struct = pwd.getpwuid(os.getuid())
gecos = pwd_struct.pw_gecos.split(",")[0] # bug #80011
user = "******" % (gecos, pwd_struct.pw_name)
return user
def get_committer_name(env=None):
"""Generate a committer string like echangelog does."""
if env is None:
env = os.environ
if "GENTOO_COMMITTER_NAME" in env and "GENTOO_COMMITTER_EMAIL" in env:
user = "******" % (env["GENTOO_COMMITTER_NAME"], env["GENTOO_COMMITTER_EMAIL"])
elif "GENTOO_AUTHOR_NAME" in env and "GENTOO_AUTHOR_EMAIL" in env:
user = "******" % (env["GENTOO_AUTHOR_NAME"], env["GENTOO_AUTHOR_EMAIL"])
elif "ECHANGELOG_USER" in env:
user = env["ECHANGELOG_USER"]
else:
pwd_struct = pwd.getpwuid(os.getuid())
gecos = pwd_struct.pw_gecos.split(",")[0] # bug #80011
user = "******" % (gecos, pwd_struct.pw_name)
return user
def get_committer_name(env=None):
"""Generate a committer string like echangelog does."""
if env is None:
env = os.environ
if 'GENTOO_COMMITTER_NAME' in env and 'GENTOO_COMMITTER_EMAIL' in env:
user = '******' % (env['GENTOO_COMMITTER_NAME'],
env['GENTOO_COMMITTER_EMAIL'])
elif 'GENTOO_AUTHOR_NAME' in env and 'GENTOO_AUTHOR_EMAIL' in env:
user = '******' % (env['GENTOO_AUTHOR_NAME'],
env['GENTOO_AUTHOR_EMAIL'])
elif 'ECHANGELOG_USER' in env:
user = env['ECHANGELOG_USER']
else:
pwd_struct = pwd.getpwuid(os.getuid())
gecos = pwd_struct.pw_gecos.split(',')[0] # bug #80011
user = '******' % (gecos, pwd_struct.pw_name)
return user
def binTestsInit(): binTestsCleanup() global basedir, env basedir = tempfile.mkdtemp() env = os.environ.copy() env["EAPI"] = "0" env["D"] = os.path.join(basedir, "image") env["T"] = os.path.join(basedir, "temp") env["S"] = os.path.join(basedir, "workdir") env["PF"] = "portage-tests-0.09-r1" env["PATH"] = bindir + ":" + env["PATH"] env["PORTAGE_BIN_PATH"] = bindir env["PORTAGE_PYM_PATH"] = pymdir env["PORTAGE_INST_UID"] = str(os.getuid()) env["PORTAGE_INST_GID"] = str(os.getgid()) env["DESTTREE"] = "/usr" os.mkdir(env["D"]) os.mkdir(env["T"]) os.mkdir(env["S"])
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = {}
env["EAPI"] = "0"
env["D"] = os.path.join(basedir, "image")
env["T"] = os.path.join(basedir, "temp")
env["S"] = os.path.join(basedir, "workdir")
env["PF"] = "portage-tests-0.09-r1"
env["PATH"] = bindir + ":" + os.environ["PATH"]
env["PORTAGE_BIN_PATH"] = bindir
env["PORTAGE_PYM_PATH"] = pymdir
env["PORTAGE_INST_UID"] = str(os.getuid())
env["PORTAGE_INST_GID"] = str(os.getgid())
env["DESTTREE"] = "/usr"
os.mkdir(env["D"])
os.mkdir(env["T"])
os.mkdir(env["S"])
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = {}
env['EAPI'] = '0'
env['D'] = os.path.join(basedir, 'image')
env['T'] = os.path.join(basedir, 'temp')
env['S'] = os.path.join(basedir, 'workdir')
env['PF'] = 'portage-tests-0.09-r1'
env['PATH'] = bindir + ':' + os.environ['PATH']
env['PORTAGE_BIN_PATH'] = bindir
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_INST_UID'] = str(os.getuid())
env['PORTAGE_INST_GID'] = str(os.getgid())
env['DESTTREE'] = '/usr'
os.mkdir(env['D'])
os.mkdir(env['T'])
os.mkdir(env['S'])
def get_committer_name(env=None):
"""Generate a committer string like echangelog does."""
if env is None:
env = os.environ
if 'GENTOO_COMMITTER_NAME' in env and \
'GENTOO_COMMITTER_EMAIL' in env:
user = '******' % (env['GENTOO_COMMITTER_NAME'],
env['GENTOO_COMMITTER_EMAIL'])
elif 'GENTOO_AUTHOR_NAME' in env and \
'GENTOO_AUTHOR_EMAIL' in env:
user = '******' % (env['GENTOO_AUTHOR_NAME'],
env['GENTOO_AUTHOR_EMAIL'])
elif 'ECHANGELOG_USER' in env:
user = env['ECHANGELOG_USER']
else:
pwd_struct = pwd.getpwuid(os.getuid())
gecos = pwd_struct.pw_gecos.split(',')[0] # bug #80011
user = '******' % (gecos, pwd_struct.pw_name)
return user
def binTestsInit():
binTestsCleanup()
global basedir, env
basedir = tempfile.mkdtemp()
env = {}
env['EAPI'] = '0'
env['D'] = os.path.join(basedir, 'image')
env['T'] = os.path.join(basedir, 'temp')
env['S'] = os.path.join(basedir, 'workdir')
env['PF'] = 'portage-tests-0.09-r1'
env['PATH'] = bindir + ':' + os.environ['PATH']
env['PORTAGE_BIN_PATH'] = bindir
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_INST_UID'] = str(os.getuid())
env['PORTAGE_INST_GID'] = str(os.getgid())
env['DESTTREE'] = '/usr'
os.mkdir(env['D'])
os.mkdir(env['T'])
os.mkdir(env['S'])
def _spawn_fetch(settings, args, **kwargs):
"""
Spawn a process with appropriate settings for fetching, including
userfetch and selinux support.
"""
global _userpriv_spawn_kwargs
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
if "fd_pipes" not in kwargs:
kwargs["fd_pipes"] = {
0 : sys.stdin.fileno(),
1 : sys.stdout.fileno(),
2 : sys.stdout.fileno(),
}
if "userfetch" in settings.features and \
os.getuid() == 0 and portage_gid and portage_uid:
kwargs.update(_userpriv_spawn_kwargs)
spawn_func = spawn
if settings.selinux_enabled():
spawn_func = selinux.spawn_wrapper(spawn_func,
settings["PORTAGE_FETCH_T"])
# bash is an allowed entrypoint, while most binaries are not
if args[0] != BASH_BINARY:
args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
rval = spawn_func(args, env=settings.environ(), **kwargs)
return rval
def _do_rsync(self, syncuri, timestamp, opts):
updatecache_flg = False
is_synced = False
if timestamp != 0 and "--quiet" not in opts:
print(">>> Checking server timestamp ...")
rsynccommand = [self.bin_command
] + self.rsync_opts + self.extra_rsync_opts
if self.proto == 'ssh' and self.ssh_opts:
rsynccommand.append("--rsh=ssh " + self.ssh_opts)
if "--debug" in opts:
print(rsynccommand)
local_state_unchanged = False
exitcode = os.EX_OK
servertimestamp = 0
# Even if there's no timestamp available locally, fetch the
# timestamp anyway as an initial probe to verify that the server is
# responsive. This protects us from hanging indefinitely on a
# connection attempt to an unresponsive server which rsync's
# --timeout option does not prevent.
#if True:
# Temporary file for remote server timestamp comparison.
# NOTE: If FEATURES=usersync is enabled then the tempfile
# needs to be in a directory that's readable by the usersync
# user. We assume that ${PORTAGE_TMPDIR}/portage will satisfy this
# requirement, since that's not necessarily true for the
# default directory used by the tempfile module.
if self.usersync_uid is not None:
tmpdir = os.path.join(self.settings['PORTAGE_TMPDIR'], 'portage')
ensure_dirs_kwargs = {}
if portage.secpass >= 1:
ensure_dirs_kwargs['gid'] = portage.portage_gid
ensure_dirs_kwargs['mode'] = 0o70
ensure_dirs_kwargs['mask'] = 0
portage.util.ensure_dirs(tmpdir, **ensure_dirs_kwargs)
else:
# use default dir from tempfile module
tmpdir = None
fd, tmpservertimestampfile = \
tempfile.mkstemp(dir=tmpdir)
os.close(fd)
if self.usersync_uid is not None:
portage.util.apply_permissions(tmpservertimestampfile,
uid=self.usersync_uid)
command = rsynccommand[:]
command.append('--inplace')
command.append(syncuri.rstrip("/") + \
"/metadata/timestamp.chk")
command.append(tmpservertimestampfile)
content = None
pids = []
try:
# Timeout here in case the server is unresponsive. The
# --timeout rsync option doesn't apply to the initial
# connection attempt.
try:
if self.rsync_initial_timeout:
portage.exception.AlarmSignal.register(
self.rsync_initial_timeout)
pids.extend(
portage.process.spawn(command,
returnpid=True,
**self.spawn_kwargs))
exitcode = os.waitpid(pids[0], 0)[1]
if self.usersync_uid is not None:
portage.util.apply_permissions(tmpservertimestampfile,
uid=os.getuid())
content = portage.grabfile(tmpservertimestampfile)
finally:
if self.rsync_initial_timeout:
portage.exception.AlarmSignal.unregister()
try:
os.unlink(tmpservertimestampfile)
except OSError:
pass
except portage.exception.AlarmSignal:
# timed out
print('timed out')
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
os.kill(pids[0], signal.SIGTERM)
os.waitpid(pids[0], 0)
# This is the same code rsync uses for timeout.
exitcode = 30
else:
if exitcode != os.EX_OK:
if exitcode & 0xff:
exitcode = (exitcode & 0xff) << 8
else:
exitcode = exitcode >> 8
if content:
try:
servertimestamp = time.mktime(
time.strptime(content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del command, pids, content
if exitcode == os.EX_OK:
if (servertimestamp != 0) and (servertimestamp == timestamp):
local_state_unchanged = True
is_synced = True
self.logger(self.xterm_titles,
">>> Cancelling sync -- Already current.")
print()
print(">>>")
print(
">>> Timestamps on the server and in the local repository are the same."
)
print(
">>> Cancelling all further sync action. You are already up to date."
)
print(">>>")
print(">>> In order to force sync, remove '%s'." %
self.servertimestampfile)
print(">>>")
print()
elif (servertimestamp != 0) and (servertimestamp < timestamp):
self.logger(self.xterm_titles,
">>> Server out of date: %s" % syncuri)
print()
print(">>>")
print(">>> SERVER OUT OF DATE: %s" % syncuri)
print(">>>")
print(">>> In order to force sync, remove '%s'." %
self.servertimestampfile)
print(">>>")
print()
exitcode = SERVER_OUT_OF_DATE
elif (servertimestamp == 0) or (servertimestamp > timestamp):
# actual sync
command = rsynccommand[:]
submodule_paths = self._get_submodule_paths()
if submodule_paths:
# The only way to select multiple directories to
# sync, without calling rsync multiple times, is
# to use --relative.
command.append("--relative")
for path in submodule_paths:
# /./ is special syntax supported with the
# rsync --relative option.
command.append(syncuri + "/./" + path)
else:
command.append(syncuri + "/")
command.append(self.download_dir)
exitcode = None
try:
exitcode = portage.process.spawn(command,
**self.spawn_kwargs)
finally:
if exitcode is None:
# interrupted
exitcode = 128 + signal.SIGINT
# 0 Success
# 1 Syntax or usage error
# 2 Protocol incompatibility
# 5 Error starting client-server protocol
# 35 Timeout waiting for daemon connection
if exitcode not in (0, 1, 2, 5, 35):
# If the exit code is not among those listed above,
# then we may have a partial/inconsistent sync
# state, so our previously read timestamp as well
# as the corresponding file can no longer be
# trusted.
timestamp = 0
try:
os.unlink(self.servertimestampfile)
except OSError:
pass
else:
updatecache_flg = True
if exitcode in [0, 1, 3, 4, 11, 14, 20, 21]:
is_synced = True
elif exitcode in [1, 3, 4, 11, 14, 20, 21]:
is_synced = True
else:
# Code 2 indicates protocol incompatibility, which is expected
# for servers with protocol < 29 that don't support
# --prune-empty-directories. Retry for a server that supports
# at least rsync protocol version 29 (>=rsync-2.6.4).
pass
return local_state_unchanged, is_synced, exitcode, updatecache_flg
def pre_sync(self, repo):
self.settings, self.trees, self.mtimedb = self.emerge_config
self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Synchronization of repository '%s' located in '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
try:
st = os.stat(repo.location)
except OSError:
st = None
self.usersync_uid = None
spawn_kwargs = {}
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):
user = None
group = None
home = None
spl = sync_user.split(':', 1)
if spl[0]:
username = spl[0]
try:
try:
pw = pwd.getpwnam(username)
except KeyError:
pw = pwd.getpwuid(int(username))
except (ValueError, KeyError):
writemsg("!!! User '%s' invalid or does not exist\n"
% username, noiselevel=-1)
return (user, group, home)
user = pw.pw_uid
group = pw.pw_gid
home = pw.pw_dir
if len(spl) > 1:
groupname = spl[1]
try:
try:
gp = grp.getgrnam(groupname)
except KeyError:
pw = grp.getgrgid(int(groupname))
except (ValueError, KeyError):
writemsg("!!! Group '%s' invalid or does not exist\n"
% groupname, noiselevel=-1)
return (user, group, home)
group = gp.gr_gid
return (user, group, home)
# user or user:group
(uid, gid, home) = get_sync_user_data(repo.sync_user)
if uid is not None:
spawn_kwargs["uid"] = uid
self.usersync_uid = uid
if gid is not None:
spawn_kwargs["gid"] = gid
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
if st is None:
perms = {'mode': 0o755}
# respect sync-user if set
if 'umask' in spawn_kwargs:
perms['mode'] &= ~spawn_kwargs['umask']
if 'uid' in spawn_kwargs:
perms['uid'] = spawn_kwargs['uid']
if 'gid' in spawn_kwargs:
perms['gid'] = spawn_kwargs['gid']
writemsg_level(">>> '%s' not found, creating it."
% _unicode_decode(repo.location))
portage.util.ensure_dirs(repo.location, **perms)
st = os.stat(repo.location)
if (repo.sync_user is None and
'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):
try:
homedir = pwd.getpwuid(st.st_uid).pw_dir
except KeyError:
pass
else:
# Drop privileges when syncing, in order to match
# existing uid/gid settings.
self.usersync_uid = st.st_uid
spawn_kwargs["uid"] = st.st_uid
spawn_kwargs["gid"] = st.st_gid
spawn_kwargs["groups"] = [st.st_gid]
spawn_kwargs["env"]["HOME"] = homedir
umask = 0o002
if not st.st_mode & 0o020:
umask = umask | 0o020
spawn_kwargs["umask"] = umask
# override the defaults when sync_umask is set
if repo.sync_umask is not None:
spawn_kwargs["umask"] = int(repo.sync_umask, 8)
self.spawn_kwargs = spawn_kwargs
if self.usersync_uid is not None:
# PORTAGE_TMPDIR is used below, so validate it and
# bail out if necessary.
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
return rval
os.umask(0o022)
return os.EX_OK
def __init__(self, _unused_param=DeprecationWarning, mysettings=None):
"""
@param _unused_param: deprecated, use mysettings['PORTDIR'] instead
@type _unused_param: None
@param mysettings: an immutable config instance
@type mysettings: portage.config
"""
from portage import config
if mysettings:
self.settings = mysettings
else:
from portage import settings
self.settings = config(clone=settings)
if _unused_param is not DeprecationWarning:
warnings.warn("The first parameter of the " + \
"portage.dbapi.porttree.portdbapi" + \
" constructor is unused since portage-2.1.8. " + \
"mysettings['PORTDIR'] is used instead.",
DeprecationWarning, stacklevel=2)
self.repositories = self.settings.repositories
self.treemap = self.repositories.treemap
# This is strictly for use in aux_get() doebuild calls when metadata
# is generated by the depend phase. It's safest to use a clone for
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
# Make api consumers exempt from sandbox violations
# when doing metadata cache updates.
sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":")
if self.depcachedir not in sandbox_write:
sandbox_write.append(self.depcachedir)
os.environ["SANDBOX_WRITE"] = \
":".join(filter(None, sandbox_write))
self.porttrees = list(self.settings.repositories.repoLocationList())
# This is used as sanity check for aux_get(). If there is no
# root eclass dir, we assume that PORTDIR is invalid or
# missing. This check allows aux_get() to detect a missing
# portage tree and return early by raising a KeyError.
self._have_root_eclass_dir = os.path.isdir(
os.path.join(self.settings.repositories.mainRepoLocation(), "eclass"))
#if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
self.xcache = {}
self.frozen = 0
#Keep a list of repo names, sorted by priority (highest priority first).
self._ordered_repo_name_list = tuple(reversed(self.repositories.prepos_order))
self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule")
self.auxdb = {}
self._pregen_auxdb = {}
# If the current user doesn't have depcachedir write permission,
# then the depcachedir cache is kept here read-only access.
self._ro_auxdb = {}
self._init_cache_dirs()
try:
depcachedir_st = os.stat(self.depcachedir)
depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
except OSError:
depcachedir_st = None
depcachedir_w_ok = False
cache_kwargs = {}
depcachedir_unshared = False
if portage.data.secpass < 1 and \
depcachedir_w_ok and \
depcachedir_st is not None and \
os.getuid() == depcachedir_st.st_uid and \
os.getgid() == depcachedir_st.st_gid:
# If this user owns depcachedir and is not in the
# portage group, then don't bother to set permissions
# on cache entries. This makes it possible to run
# egencache without any need to be a member of the
# portage group.
depcachedir_unshared = True
else:
cache_kwargs.update(portage._native_kwargs({
'gid' : portage_gid,
'perms' : 0o664
}))
# If secpass < 1, we don't want to write to the cache
# since then we won't be able to apply group permissions
# to the cache entries/directories.
if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok:
for x in self.porttrees:
self.auxdb[x] = volatile.database(
self.depcachedir, x, self._known_keys,
**cache_kwargs)
try:
self._ro_auxdb[x] = self.auxdbmodule(self.depcachedir, x,
self._known_keys, readonly=True, **cache_kwargs)
except CacheError:
pass
else:
for x in self.porttrees:
if x in self.auxdb:
continue
# location, label, auxdbkeys
self.auxdb[x] = self.auxdbmodule(
self.depcachedir, x, self._known_keys, **cache_kwargs)
if "metadata-transfer" not in self.settings.features:
for x in self.porttrees:
if x in self._pregen_auxdb:
continue
cache = self._create_pregen_cache(x)
if cache is not None:
self._pregen_auxdb[x] = cache
# Selectively cache metadata in order to optimize dep matching.
self._aux_cache_keys = set(
["DEPEND", "EAPI", "HDEPEND",
"INHERITED", "IUSE", "KEYWORDS", "LICENSE",
"PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository",
"RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"])
self._aux_cache = {}
self._broken_ebuilds = set()
def __init__(self, _unused_param=DeprecationWarning, mysettings=None):
"""
@param _unused_param: deprecated, use mysettings['PORTDIR'] instead
@type _unused_param: None
@param mysettings: an immutable config instance
@type mysettings: portage.config
"""
from portage import config
if mysettings:
self.settings = mysettings
else:
from portage import settings
self.settings = config(clone=settings)
if _unused_param is not DeprecationWarning:
warnings.warn("The first parameter of the " + \
"portage.dbapi.porttree.portdbapi" + \
" constructor is unused since portage-2.1.8. " + \
"mysettings['PORTDIR'] is used instead.",
DeprecationWarning, stacklevel=2)
self.repositories = self.settings.repositories
self.treemap = self.repositories.treemap
# This is strictly for use in aux_get() doebuild calls when metadata
# is generated by the depend phase. It's safest to use a clone for
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
# Make api consumers exempt from sandbox violations
# when doing metadata cache updates.
sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":")
if self.depcachedir not in sandbox_write:
sandbox_write.append(self.depcachedir)
os.environ["SANDBOX_WRITE"] = \
":".join(filter(None, sandbox_write))
self.porttrees = list(self.settings.repositories.repoLocationList())
# This is used as sanity check for aux_get(). If there is no
# root eclass dir, we assume that PORTDIR is invalid or
# missing. This check allows aux_get() to detect a missing
# repository and return early by raising a KeyError.
self._have_root_eclass_dir = os.path.isdir(
os.path.join(self.settings.repositories.mainRepoLocation(), "eclass"))
#if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
self.xcache = {}
self.frozen = 0
#Keep a list of repo names, sorted by priority (highest priority first).
self._ordered_repo_name_list = tuple(reversed(self.repositories.prepos_order))
self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule")
self.auxdb = {}
self._pregen_auxdb = {}
# If the current user doesn't have depcachedir write permission,
# then the depcachedir cache is kept here read-only access.
self._ro_auxdb = {}
self._init_cache_dirs()
try:
depcachedir_st = os.stat(self.depcachedir)
depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
except OSError:
depcachedir_st = None
depcachedir_w_ok = False
cache_kwargs = {}
depcachedir_unshared = False
if portage.data.secpass < 1 and \
depcachedir_w_ok and \
depcachedir_st is not None and \
os.getuid() == depcachedir_st.st_uid and \
os.getgid() == depcachedir_st.st_gid:
# If this user owns depcachedir and is not in the
# portage group, then don't bother to set permissions
# on cache entries. This makes it possible to run
# egencache without any need to be a member of the
# portage group.
depcachedir_unshared = True
else:
cache_kwargs.update({
'gid' : portage_gid,
'perms' : 0o664
})
# If secpass < 1, we don't want to write to the cache
# since then we won't be able to apply group permissions
# to the cache entries/directories.
if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok:
for x in self.porttrees:
self.auxdb[x] = volatile.database(
self.depcachedir, x, self._known_keys,
**cache_kwargs)
try:
self._ro_auxdb[x] = self.auxdbmodule(self.depcachedir, x,
self._known_keys, readonly=True, **cache_kwargs)
except CacheError:
pass
else:
for x in self.porttrees:
if x in self.auxdb:
continue
# location, label, auxdbkeys
self.auxdb[x] = self.auxdbmodule(
self.depcachedir, x, self._known_keys, **cache_kwargs)
if "metadata-transfer" not in self.settings.features:
for x in self.porttrees:
if x in self._pregen_auxdb:
continue
cache = self._create_pregen_cache(x)
if cache is not None:
self._pregen_auxdb[x] = cache
# Selectively cache metadata in order to optimize dep matching.
self._aux_cache_keys = set(
["BDEPEND", "DEPEND", "EAPI",
"INHERITED", "IUSE", "KEYWORDS", "LICENSE",
"PDEPEND", "PROPERTIES", "RDEPEND", "repository",
"RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"])
self._aux_cache = {}
self._better_cache = None
self._broken_ebuilds = set()
def _do_rsync(self, syncuri, timestamp, opts):
is_synced = False
if timestamp != 0 and "--quiet" not in opts:
print(">>> Checking server timestamp ...")
rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
if self.proto == 'ssh' and self.ssh_opts:
rsynccommand.append("--rsh=ssh " + self.ssh_opts)
if "--debug" in opts:
print(rsynccommand)
exitcode = os.EX_OK
servertimestamp = 0
# Even if there's no timestamp available locally, fetch the
# timestamp anyway as an initial probe to verify that the server is
# responsive. This protects us from hanging indefinitely on a
# connection attempt to an unresponsive server which rsync's
# --timeout option does not prevent.
#if True:
# Temporary file for remote server timestamp comparison.
# NOTE: If FEATURES=usersync is enabled then the tempfile
# needs to be in a directory that's readable by the usersync
# user. We assume that PORTAGE_TMPDIR will satisfy this
# requirement, since that's not necessarily true for the
# default directory used by the tempfile module.
if self.usersync_uid is not None:
tmpdir = self.settings['PORTAGE_TMPDIR']
else:
# use default dir from tempfile module
tmpdir = None
fd, tmpservertimestampfile = \
tempfile.mkstemp(dir=tmpdir)
os.close(fd)
if self.usersync_uid is not None:
portage.util.apply_permissions(tmpservertimestampfile,
uid=self.usersync_uid)
command = rsynccommand[:]
command.append(syncuri.rstrip("/") + \
"/metadata/timestamp.chk")
command.append(tmpservertimestampfile)
content = None
pids = []
try:
# Timeout here in case the server is unresponsive. The
# --timeout rsync option doesn't apply to the initial
# connection attempt.
try:
if self.rsync_initial_timeout:
portage.exception.AlarmSignal.register(
self.rsync_initial_timeout)
pids.extend(portage.process.spawn(
command, returnpid=True,
**portage._native_kwargs(self.spawn_kwargs)))
exitcode = os.waitpid(pids[0], 0)[1]
if self.usersync_uid is not None:
portage.util.apply_permissions(tmpservertimestampfile,
uid=os.getuid())
content = portage.grabfile(tmpservertimestampfile)
finally:
if self.rsync_initial_timeout:
portage.exception.AlarmSignal.unregister()
try:
os.unlink(tmpservertimestampfile)
except OSError:
pass
except portage.exception.AlarmSignal:
# timed out
print('timed out')
# With waitpid and WNOHANG, only check the
# first element of the tuple since the second
# element may vary (bug #337465).
if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
os.kill(pids[0], signal.SIGTERM)
os.waitpid(pids[0], 0)
# This is the same code rsync uses for timeout.
exitcode = 30
else:
if exitcode != os.EX_OK:
if exitcode & 0xff:
exitcode = (exitcode & 0xff) << 8
else:
exitcode = exitcode >> 8
if content:
try:
servertimestamp = time.mktime(time.strptime(
content[0], TIMESTAMP_FORMAT))
except (OverflowError, ValueError):
pass
del command, pids, content
if exitcode == os.EX_OK:
if (servertimestamp != 0) and (servertimestamp == timestamp):
self.logger(self.xterm_titles,
">>> Cancelling sync -- Already current.")
print()
print(">>>")
print(">>> Timestamps on the server and in the local repository are the same.")
print(">>> Cancelling all further sync action. You are already up to date.")
print(">>>")
print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
print(">>>")
print()
return is_synced, exitcode
elif (servertimestamp != 0) and (servertimestamp < timestamp):
self.logger(self.xterm_titles,
">>> Server out of date: %s" % syncuri)
print()
print(">>>")
print(">>> SERVER OUT OF DATE: %s" % syncuri)
print(">>>")
print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
print(">>>")
print()
exitcode = SERVER_OUT_OF_DATE
elif (servertimestamp == 0) or (servertimestamp > timestamp):
# actual sync
command = rsynccommand + [syncuri+"/", self.repo.location]
exitcode = None
try:
exitcode = portage.process.spawn(command,
**portage._native_kwargs(self.spawn_kwargs))
finally:
if exitcode is None:
# interrupted
exitcode = 128 + signal.SIGINT
# 0 Success
# 1 Syntax or usage error
# 2 Protocol incompatibility
# 5 Error starting client-server protocol
# 35 Timeout waiting for daemon connection
if exitcode not in (0, 1, 2, 5, 35):
# If the exit code is not among those listed above,
# then we may have a partial/inconsistent sync
# state, so our previously read timestamp as well
# as the corresponding file can no longer be
# trusted.
timestamp = 0
try:
os.unlink(self.servertimestampfile)
except OSError:
pass
if exitcode in [0,1,3,4,11,14,20,21]:
is_synced = True
elif exitcode in [1,3,4,11,14,20,21]:
is_synced = True
else:
# Code 2 indicates protocol incompatibility, which is expected
# for servers with protocol < 29 that don't support
# --prune-empty-directories. Retry for a server that supports
# at least rsync protocol version 29 (>=rsync-2.6.4).
pass
return is_synced, exitcode
def pre_sync(self, repo):
msg = ">>> Syncing repository '%s' into '%s'..." % (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
try:
st = os.stat(repo.location)
except OSError:
st = None
self.usersync_uid = None
spawn_kwargs = {}
# Redirect command stderr to stdout, in order to prevent
# spurious cron job emails (bug 566132).
spawn_kwargs["fd_pipes"] = {0: sys.__stdin__.fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stdout__.fileno()}
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):
user = None
group = None
home = None
logname = None
spl = sync_user.split(":", 1)
if spl[0]:
username = spl[0]
try:
try:
pw = pwd.getpwnam(username)
except KeyError:
pw = pwd.getpwuid(int(username))
except (ValueError, KeyError):
writemsg("!!! User '%s' invalid or does not exist\n" % username, noiselevel=-1)
return (logname, user, group, home)
user = pw.pw_uid
group = pw.pw_gid
home = pw.pw_dir
logname = pw.pw_name
if len(spl) > 1:
groupname = spl[1]
try:
try:
gp = grp.getgrnam(groupname)
except KeyError:
pw = grp.getgrgid(int(groupname))
except (ValueError, KeyError):
writemsg("!!! Group '%s' invalid or does not exist\n" % groupname, noiselevel=-1)
return (logname, user, group, home)
group = gp.gr_gid
return (logname, user, group, home)
# user or user:group
(logname, uid, gid, home) = get_sync_user_data(repo.sync_user)
if uid is not None:
spawn_kwargs["uid"] = uid
self.usersync_uid = uid
if gid is not None:
spawn_kwargs["gid"] = gid
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
if logname is not None:
spawn_kwargs["env"]["LOGNAME"] = logname
if st is None:
perms = {"mode": 0o755}
# respect sync-user if set
if "umask" in spawn_kwargs:
perms["mode"] &= ~spawn_kwargs["umask"]
if "uid" in spawn_kwargs:
perms["uid"] = spawn_kwargs["uid"]
if "gid" in spawn_kwargs:
perms["gid"] = spawn_kwargs["gid"]
portage.util.ensure_dirs(repo.location, **perms)
st = os.stat(repo.location)
if (
repo.sync_user is None
and "usersync" in self.settings.features
and portage.data.secpass >= 2
and (st.st_uid != os.getuid() and st.st_mode & 0o700 or st.st_gid != os.getgid() and st.st_mode & 0o070)
):
try:
pw = pwd.getpwuid(st.st_uid)
except KeyError:
pass
else:
# Drop privileges when syncing, in order to match
# existing uid/gid settings.
self.usersync_uid = st.st_uid
spawn_kwargs["uid"] = st.st_uid
spawn_kwargs["gid"] = st.st_gid
spawn_kwargs["groups"] = [st.st_gid]
spawn_kwargs["env"]["HOME"] = pw.pw_dir
spawn_kwargs["env"]["LOGNAME"] = pw.pw_name
umask = 0o002
if not st.st_mode & 0o020:
umask = umask | 0o020
spawn_kwargs["umask"] = umask
# override the defaults when sync_umask is set
if repo.sync_umask is not None:
spawn_kwargs["umask"] = int(repo.sync_umask, 8)
self.spawn_kwargs = spawn_kwargs
if self.usersync_uid is not None:
# PORTAGE_TMPDIR is used below, so validate it and
# bail out if necessary.
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
return rval
os.umask(0o022)
return os.EX_OK
def pre_sync(self, repo):
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
try:
st = os.stat(repo.location)
except OSError:
st = None
self.usersync_uid = None
spawn_kwargs = {}
# Redirect command stderr to stdout, in order to prevent
# spurious cron job emails (bug 566132).
spawn_kwargs["fd_pipes"] = {
0: sys.__stdin__.fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno()
}
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):
user = None
group = None
home = None
logname = None
spl = sync_user.split(':', 1)
if spl[0]:
username = spl[0]
try:
try:
pw = pwd.getpwnam(username)
except KeyError:
pw = pwd.getpwuid(int(username))
except (ValueError, KeyError):
writemsg("!!! User '%s' invalid or does not exist\n" %
username,
noiselevel=-1)
return (logname, user, group, home)
user = pw.pw_uid
group = pw.pw_gid
home = pw.pw_dir
logname = pw.pw_name
if len(spl) > 1:
groupname = spl[1]
try:
try:
gp = grp.getgrnam(groupname)
except KeyError:
pw = grp.getgrgid(int(groupname))
except (ValueError, KeyError):
writemsg("!!! Group '%s' invalid or does not exist\n" %
groupname,
noiselevel=-1)
return (logname, user, group, home)
group = gp.gr_gid
return (logname, user, group, home)
# user or user:group
(logname, uid, gid, home) = get_sync_user_data(repo.sync_user)
if uid is not None:
spawn_kwargs["uid"] = uid
self.usersync_uid = uid
if gid is not None:
spawn_kwargs["gid"] = gid
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
if logname is not None:
spawn_kwargs["env"]["LOGNAME"] = logname
if st is None:
perms = {'mode': 0o755}
# respect sync-user if set
if 'umask' in spawn_kwargs:
perms['mode'] &= ~spawn_kwargs['umask']
if 'uid' in spawn_kwargs:
perms['uid'] = spawn_kwargs['uid']
if 'gid' in spawn_kwargs:
perms['gid'] = spawn_kwargs['gid']
portage.util.ensure_dirs(repo.location, **perms)
st = os.stat(repo.location)
if (repo.sync_user is None and 'usersync' in self.settings.features
and portage.data.secpass >= 2
and (st.st_uid != os.getuid() and st.st_mode & 0o700
or st.st_gid != os.getgid() and st.st_mode & 0o070)):
try:
pw = pwd.getpwuid(st.st_uid)
except KeyError:
pass
else:
# Drop privileges when syncing, in order to match
# existing uid/gid settings.
self.usersync_uid = st.st_uid
spawn_kwargs["uid"] = st.st_uid
spawn_kwargs["gid"] = st.st_gid
spawn_kwargs["groups"] = [st.st_gid]
spawn_kwargs["env"]["HOME"] = pw.pw_dir
spawn_kwargs["env"]["LOGNAME"] = pw.pw_name
umask = 0o002
if not st.st_mode & 0o020:
umask = umask | 0o020
spawn_kwargs["umask"] = umask
# override the defaults when sync_umask is set
if repo.sync_umask is not None:
spawn_kwargs["umask"] = int(repo.sync_umask, 8)
self.spawn_kwargs = spawn_kwargs
if self.usersync_uid is not None:
# PORTAGE_TMPDIR is used below, so validate it and
# bail out if necessary.
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
return rval
os.umask(0o022)
return os.EX_OK