def __getattr__(self, attr): if attr == 'mtime': # use stat.ST_MTIME; accessing .st_mtime gets you a float # depending on the python version, and int(float) introduces # some rounding issues that aren't present for people using # the straight c api. # thus use the defacto python compatibility work around; # access via index, which guarantees you get the raw int. try: self.mtime = obj = os.stat(self.location)[stat.ST_MTIME] except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): raise FileNotFound(self.location) elif e.errno == PermissionDenied.errno: raise PermissionDenied(self.location) raise return obj if not attr.islower(): # we don't care to allow .mD5 as an alias for .md5 raise AttributeError(attr) hashname = attr.upper() if hashname not in checksum.get_valid_checksum_keys(): raise AttributeError(attr) val = checksum.perform_checksum(self.location, hashname)[0] setattr(self, attr, val) return val
def addFile(self, ftype, fname, hashdict=None, ignoreMissing=False):
""" Add entry to Manifest optionally using hashdict to avoid recalculation of hashes """
if ftype == "AUX" and not fname.startswith("files/"):
fname = os.path.join("files", fname)
if not os.path.exists(self.pkgdir+fname) and not ignoreMissing:
raise FileNotFound(fname)
if not ftype in MANIFEST2_IDENTIFIERS:
raise InvalidDataType(ftype)
if ftype == "AUX" and fname.startswith("files"):
fname = fname[6:]
self.fhashdict[ftype][fname] = {}
if hashdict != None:
self.fhashdict[ftype][fname].update(hashdict)
if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]:
self.updateFileHashes(ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing)
def hardlink_lockfile(lockfilename, max_wait=14400):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE lockfile, that is just a placeholder on the disk.
Then we HARDLINK the real lockfile to that private file.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
We default to a 4 hour timeout.
"""
start_time = time.time()
myhardlock = hardlock_name(lockfilename)
reported_waiting = False
while (time.time() < (start_time + max_wait)):
# We only need it to exist.
myfd = os.open(myhardlock, os.O_CREAT | os.O_RDWR, 0o660)
os.close(myfd)
if not os.path.exists(myhardlock):
raise FileNotFound(
_("Created lockfile is missing: %(filename)s") % \
{"filename" : myhardlock})
try:
res = os.link(myhardlock, lockfilename)
except OSError:
pass
if hardlink_is_mine(myhardlock, lockfilename):
# We have the lock.
if reported_waiting:
writemsg("\n", noiselevel=-1)
return True
if reported_waiting:
writemsg(".", noiselevel=-1)
else:
reported_waiting = True
from portage.const import PORTAGE_BIN_PATH
msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n"
"%(bin_path)s/clean_locks can fix stuck locks.\n"
"Lockfile: %(lockfilename)s\n") % \
{"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename}
writemsg(msg, noiselevel=-1)
time.sleep(3)
os.unlink(myhardlock)
return False
def _readManifest(self, file_path, myhashdict=None, **kwargs):
"""Parse a manifest. If myhashdict is given then data will be added too it.
Otherwise, a new dict will be created and returned."""
try:
with io.open(_unicode_encode(file_path,
encoding=_encodings['fs'], errors='strict'), mode='r',
encoding=_encodings['repo.content'], errors='replace') as f:
if myhashdict is None:
myhashdict = {}
self._parseDigests(f, myhashdict=myhashdict, **kwargs)
return myhashdict
except (OSError, IOError) as e:
if e.errno == errno.ENOENT:
raise FileNotFound(file_path)
else:
raise
def __init__(self, **kwargs):
CompositeTask.__init__(self, **kwargs)
pkg = self.pkg
bintree = pkg.root_config.trees["bintree"]
binpkg_path = None
if bintree._remote_has_index:
instance_key = bintree.dbapi._instance_key(pkg.cpv)
binpkg_path = bintree._remotepkgs[instance_key].get("PATH")
if binpkg_path:
self.pkg_path = binpkg_path + ".partial"
else:
self.pkg_path = (pkg.root_config.trees["bintree"].getname(
pkg.cpv, allocate_new=True) + ".partial")
else:
raise FileNotFound("Binary packages index not found")
def apply_secpass_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1,
stat_cached=None, follow_links=True):
"""A wrapper around apply_permissions that uses secpass and simple
logic to apply as much of the permissions as possible without
generating an obviously avoidable permission exception. Despite
attempts to avoid an exception, it's possible that one will be raised
anyway, so be prepared.
Returns True if all permissions are applied and False if some are left
unapplied."""
if stat_cached is None:
try:
if follow_links:
stat_cached = os.stat(filename)
else:
stat_cached = os.lstat(filename)
except OSError as oe:
func_call = "stat('%s')" % filename
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
else:
raise
all_applied = True
if portage.data.secpass < 2:
if uid != -1 and \
uid != stat_cached.st_uid:
all_applied = False
uid = -1
if gid != -1 and \
gid != stat_cached.st_gid and \
gid not in os.getgroups():
all_applied = False
gid = -1
apply_permissions(filename, uid=uid, gid=gid, mode=mode, mask=mask,
stat_cached=stat_cached, follow_links=follow_links)
return all_applied
def addFile(self, ftype, fname, hashdict=None, ignoreMissing=False):
"""Add entry to Manifest optionally using hashdict to avoid recalculation of hashes"""
if ftype == "AUX":
if not fname.startswith("files/"):
fname = os.path.join("files", fname)
if fname.startswith("files"):
fname = fname[6:]
if not os.path.exists(f"{self.pkgdir}{fname}") and not ignoreMissing:
raise FileNotFound(fname)
if ftype not in MANIFEST2_IDENTIFIERS:
raise InvalidDataType(ftype)
self.fhashdict[ftype][fname] = {}
if hashdict is not None:
self.fhashdict[ftype][fname].update(hashdict)
if self.required_hashes.difference(set(self.fhashdict[ftype][fname])):
self.updateFileHashes(
ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing
)
def make_herd_base(filename):
herd_to_emails = dict()
all_emails = set()
try:
xml_tree = xml.etree.ElementTree.parse(
_unicode_encode(filename,
encoding=_encodings['fs'],
errors='strict'),
parser=xml.etree.ElementTree.XMLParser(target=_HerdsTreeBuilder()))
except ExpatError as e:
raise ParseError("metadata.xml: " + str(e))
except EnvironmentError as e:
func_call = "open('%s')" % filename
if e.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif e.errno == errno.ENOENT:
raise FileNotFound(filename)
raise
herds = xml_tree.findall('herd')
for h in herds:
_herd_name = h.find('name')
if _herd_name is None:
continue
herd_name = _herd_name.text.strip()
del _herd_name
maintainers = h.findall('maintainer')
herd_emails = set()
for m in maintainers:
_m_email = m.find('email')
if _m_email is None:
continue
m_email = _m_email.text.strip()
herd_emails.add(m_email)
all_emails.add(m_email)
herd_to_emails[herd_name] = herd_emails
return HerdBase(herd_to_emails, all_emails)
def write_atomic(file_path, content, **kwargs):
f = None
try:
f = atomic_ofstream(file_path, **kwargs)
f.write(content)
f.close()
except (IOError, OSError) as e:
if f:
f.abort()
func_call = "write_atomic('%s')" % file_path
if e.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif e.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif e.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
elif e.errno == errno.ENOENT:
raise FileNotFound(file_path)
else:
raise
def compression_probe(f):
"""
Identify the compression type of a file. Returns one of the
following identifier strings:
bzip2
gzip
lz4
lzip
lzop
xz
zstd
@param f: a file path, or file-like object
@type f: str or file
@return: a string identifying the compression type, or None if the
compression type is unrecognized
@rtype str or None
"""
open_file = isinstance(f, str)
if open_file:
try:
f = open(
_unicode_encode(f, encoding=_encodings["fs"], errors="strict"),
mode="rb",
)
except IOError as e:
if e.errno == PermissionDenied.errno:
raise PermissionDenied(f)
elif e.errno in (errno.ENOENT, errno.ESTALE):
raise FileNotFound(f)
else:
raise
try:
return _compression_probe_file(f)
finally:
if open_file:
f.close()
def _parse_color_map(config_root='/', onerror=None):
"""
Parse /etc/portage/color.map and return a dict of error codes.
@param onerror: an optional callback to handle any ParseError that would
otherwise be raised
@type onerror: callable
@rtype: dict
@return: a dictionary mapping color classes to color codes
"""
global codes, _styles
myfile = os.path.join(config_root, COLOR_MAP_FILE)
ansi_code_pattern = re.compile("^[0-9;]*m$")
quotes = '\'"'
def strip_quotes(token):
if token[0] in quotes and token[0] == token[-1]:
token = token[1:-1]
return token
try:
with io.open(_unicode_encode(myfile,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace') as f:
lines = f.readlines()
for lineno, line in enumerate(lines):
commenter_pos = line.find("#")
line = line[:commenter_pos].strip()
if len(line) == 0:
continue
split_line = line.split("=")
if len(split_line) != 2:
e = ParseError(_("'%s', line %s: expected exactly one occurrence of '=' operator") % \
(myfile, lineno))
raise e
if onerror:
onerror(e)
else:
raise e
continue
k = strip_quotes(split_line[0].strip())
v = strip_quotes(split_line[1].strip())
if not k in _styles and not k in codes:
e = ParseError(_("'%s', line %s: Unknown variable: '%s'") % \
(myfile, lineno, k))
if onerror:
onerror(e)
else:
raise e
continue
if ansi_code_pattern.match(v):
if k in _styles:
_styles[k] = (esc_seq + v, )
elif k in codes:
codes[k] = esc_seq + v
else:
code_list = []
for x in v.split():
if x in codes:
if k in _styles:
code_list.append(x)
elif k in codes:
code_list.append(codes[x])
else:
e = ParseError(_("'%s', line %s: Undefined: '%s'") % \
(myfile, lineno, x))
if onerror:
onerror(e)
else:
raise e
if k in _styles:
_styles[k] = tuple(code_list)
elif k in codes:
codes[k] = "".join(code_list)
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
raise FileNotFound(myfile)
elif e.errno == errno.EACCES:
raise PermissionDenied(myfile)
raise
def hardlink_lockfile(lockfilename,
max_wait=DeprecationWarning,
waiting_msg=None,
flags=0):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE hardlink to the real lockfile, that is just a
placeholder on the disk.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
"""
if max_wait is not DeprecationWarning:
warnings.warn(
"The 'max_wait' parameter of "
"portage.locks.hardlink_lockfile() is now unused. Use "
"flags=os.O_NONBLOCK instead.",
DeprecationWarning,
stacklevel=2)
global _quiet
out = None
displayed_waiting_msg = False
preexisting = os.path.exists(lockfilename)
myhardlock = hardlock_name(lockfilename)
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# myhardlock must not exist prior to our link() call, and we can
# safely unlink it since its file name is unique to our PID
try:
os.unlink(myhardlock)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
pass
else:
func_call = "unlink('%s')" % myhardlock
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
else:
raise
while True:
# create lockfilename if it doesn't exist yet
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
else:
myfd_st = None
try:
myfd_st = os.fstat(myfd)
if not preexisting:
# Don't chown the file if it is preexisting, since we
# want to preserve existing permissions in that case.
if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
os.fchown(myfd, -1, portage_gid)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
writemsg("%s: fchown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
else:
# another process has removed the file, so we'll have
# to create it again
continue
finally:
os.close(myfd)
# If fstat shows more than one hardlink, then it's extremely
# unlikely that the following link call will result in a lock,
# so optimize away the wasteful link call and sleep or raise
# TryAgain.
if myfd_st is not None and myfd_st.st_nlink < 2:
try:
os.link(lockfilename, myhardlock)
except OSError as e:
func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno in (errno.ESTALE, errno.ENOENT):
# another process has removed the file, so we'll have
# to create it again
continue
else:
raise
else:
if hardlink_is_mine(myhardlock, lockfilename):
if out is not None:
out.eend(os.EX_OK)
break
try:
os.unlink(myhardlock)
except OSError as e:
# This should not happen, since the file name of
# myhardlock is unique to our host and PID,
# and the above link() call succeeded.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
raise FileNotFound(myhardlock)
if flags & os.O_NONBLOCK:
raise TryAgain(lockfilename)
if out is None and not _quiet:
out = portage.output.EOutput()
if out is not None and not displayed_waiting_msg:
displayed_waiting_msg = True
if waiting_msg is None:
waiting_msg = _("waiting for lock on %s\n") % lockfilename
out.ebegin(waiting_msg)
time.sleep(_HARDLINK_POLL_LATENCY)
return True
def _init_index(self):
cp_map = {}
desc_cache = {}
self._desc_cache = desc_cache
self._cp_map = cp_map
index_missing = []
streams = []
for repo_path in self._portdb.porttrees:
outside_repo = os.path.join(self._portdb.depcachedir,
repo_path.lstrip(os.sep))
filenames = []
for parent_dir in (repo_path, outside_repo):
filenames.append(
os.path.join(parent_dir, "metadata", "pkg_desc_index"))
repo_name = self._portdb.getRepositoryName(repo_path)
try:
f = None
for filename in filenames:
try:
f = io.open(filename,
encoding=_encodings["repo.content"])
except IOError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
else:
break
if f is None:
raise FileNotFound(filename)
streams.append(
iter(
IndexStreamIterator(
f,
functools.partial(pkg_desc_index_line_read,
repo=repo_name))))
except FileNotFound:
index_missing.append(repo_path)
if index_missing:
self._unindexed_cp_map = {}
class _NonIndexedStream(object):
def __iter__(self_):
for cp in self._portdb.cp_all(trees=index_missing):
# Don't call cp_list yet, since it's a waste
# if the package name does not match the current
# search.
self._unindexed_cp_map[cp] = index_missing
yield pkg_desc_index_node(cp, (), None)
streams.append(iter(_NonIndexedStream()))
if streams:
if len(streams) == 1:
cp_group_iter = ([node] for node in streams[0])
else:
cp_group_iter = MultiIterGroupBy(streams,
key=operator.attrgetter("cp"))
for cp_group in cp_group_iter:
new_cp = None
cp_list = cp_map.get(cp_group[0].cp)
if cp_list is None:
new_cp = cp_group[0].cp
cp_list = []
cp_map[cp_group[0].cp] = cp_list
for entry in cp_group:
cp_list.extend(entry.cpv_list)
if entry.desc is not None:
for cpv in entry.cpv_list:
desc_cache[cpv] = entry.desc
if new_cp is not None:
yield cp_group[0].cp
def _start(self):
pkg = self.pkg
pretend = self.pretend
bintree = pkg.root_config.trees["bintree"]
settings = bintree.settings
pkg_path = self.pkg_path
exists = os.path.exists(pkg_path)
resume = exists and os.path.basename(pkg_path) in bintree.invalids
if not (pretend or resume):
# Remove existing file or broken symlink.
try:
os.unlink(pkg_path)
except OSError:
pass
# urljoin doesn't work correctly with
# unrecognized protocols like sftp
fetchcommand = None
resumecommand = None
if bintree._remote_has_index:
remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(
pkg.cpv)]
binpkg_format = remote_metadata.get(
"BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0])
if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS:
raise InvalidBinaryPackageFormat(binpkg_format)
rel_uri = remote_metadata.get("PATH")
if not rel_uri:
if binpkg_format == "xpak":
rel_uri = pkg.cpv + ".tbz2"
elif binpkg_format == "gpkg":
rel_uri = pkg.cpv + ".gpkg.tar"
remote_base_uri = remote_metadata["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
fetchcommand = remote_metadata.get("FETCHCOMMAND")
resumecommand = remote_metadata.get("RESUMECOMMAND")
else:
raise FileNotFound("Binary packages index not found")
if pretend:
portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
self.returncode = os.EX_OK
self._async_wait()
return
fcmd = None
if resume:
fcmd = resumecommand
else:
fcmd = fetchcommand
if fcmd is None:
protocol = urllib_parse_urlparse(uri)[0]
fcmd_prefix = "FETCHCOMMAND"
if resume:
fcmd_prefix = "RESUMECOMMAND"
fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
if not fcmd:
fcmd = settings.get(fcmd_prefix)
fcmd_vars = {
"DISTDIR": os.path.dirname(pkg_path),
"URI": uri,
"FILE": os.path.basename(pkg_path),
}
for k in ("PORTAGE_SSH_OPTS", ):
v = settings.get(k)
if v is not None:
fcmd_vars[k] = v
fetch_env = dict(settings.items())
fetch_args = [
portage.util.varexpand(x, mydict=fcmd_vars)
for x in portage.util.shlex_split(fcmd)
]
if self.fd_pipes is None:
self.fd_pipes = {}
fd_pipes = self.fd_pipes
# Redirect all output to stdout since some fetchers like
# wget pollute stderr (if portage detects a problem then it
# can send it's own message to stderr).
fd_pipes.setdefault(0, portage._get_stdin().fileno())
fd_pipes.setdefault(1, sys.__stdout__.fileno())
fd_pipes.setdefault(2, sys.__stdout__.fileno())
self.args = fetch_args
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
self.log_filter_file = settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
SpawnProcess._start(self)
def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1,
stat_cached=None, follow_links=True):
"""Apply user, group, and mode bits to a file if the existing bits do not
already match. The default behavior is to force an exact match of mode
bits. When mask=0 is specified, mode bits on the target file are allowed
to be a superset of the mode argument (via logical OR). When mask>0, the
mode bits that the target file is allowed to have are restricted via
logical XOR.
Returns True if the permissions were modified and False otherwise."""
modified = False
if stat_cached is None:
try:
if follow_links:
stat_cached = os.stat(filename)
else:
stat_cached = os.lstat(filename)
except OSError as oe:
func_call = "stat('%s')" % filename
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
else:
raise
if (uid != -1 and uid != stat_cached.st_uid) or \
(gid != -1 and gid != stat_cached.st_gid):
try:
if follow_links:
os.chown(filename, uid, gid)
else:
portage.data.lchown(filename, uid, gid)
modified = True
except OSError as oe:
func_call = "chown('%s', %i, %i)" % (filename, uid, gid)
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
else:
raise
new_mode = -1
st_mode = stat_cached.st_mode & 0o7777 # protect from unwanted bits
if mask >= 0:
if mode == -1:
mode = 0 # Don't add any mode bits when mode is unspecified.
else:
mode = mode & 0o7777
if (mode & st_mode != mode) or \
((mask ^ st_mode) & st_mode != st_mode):
new_mode = mode | st_mode
new_mode = (mask ^ new_mode) & new_mode
elif mode != -1:
mode = mode & 0o7777 # protect from unwanted bits
if mode != st_mode:
new_mode = mode
# The chown system call may clear S_ISUID and S_ISGID
# bits, so those bits are restored if necessary.
if modified and new_mode == -1 and \
(st_mode & stat.S_ISUID or st_mode & stat.S_ISGID):
if mode == -1:
new_mode = st_mode
else:
mode = mode & 0o7777
if mask >= 0:
new_mode = mode | st_mode
new_mode = (mask ^ new_mode) & new_mode
else:
new_mode = mode
if not (new_mode & stat.S_ISUID or new_mode & stat.S_ISGID):
new_mode = -1
if not follow_links and stat.S_ISLNK(stat_cached.st_mode):
# Mode doesn't matter for symlinks.
new_mode = -1
if new_mode != -1:
try:
os.chmod(filename, new_mode)
modified = True
except OSError as oe:
func_call = "chmod('%s', %s)" % (filename, oct(new_mode))
if oe.errno == errno.EPERM:
raise OperationNotPermitted(func_call)
elif oe.errno == errno.EACCES:
raise PermissionDenied(func_call)
elif oe.errno == errno.EROFS:
raise ReadOnlyFileSystem(func_call)
elif oe.errno == errno.ENOENT:
raise FileNotFound(filename)
raise
return modified