def updateprotect(self): """Update internal state for isprotected() calls. Nonexistent paths are ignored.""" os = _os_merge self.protect = [] self._dirs = set() for x in self.protect_list: ppath = normalize_path( os.path.join(self.myroot, x.lstrip(os.path.sep))) try: if stat.S_ISDIR(os.stat(ppath).st_mode): self._dirs.add(ppath) self.protect.append(ppath) except OSError: # If it doesn't exist, there's no need to protect it. pass self.protectmask = [] for x in self.mask_list: ppath = normalize_path( os.path.join(self.myroot, x.lstrip(os.path.sep))) try: """Use lstat so that anything, even a broken symlink can be protected.""" if stat.S_ISDIR(os.lstat(ppath).st_mode): self._dirs.add(ppath) self.protectmask.append(ppath) """Now use stat in case this is a symlink to a directory.""" if stat.S_ISDIR(os.stat(ppath).st_mode): self._dirs.add(ppath) except OSError: # If it doesn't exist, there's no need to mask it. pass
def _same_device(path1, path2): try: st1 = os.stat(path1) st2 = os.stat(path2) except OSError: return False else: return st1.st_dev == st2.st_dev
def hardlink_is_mine(link, lock):
try:
lock_st = os.stat(lock)
if lock_st.st_nlink == 2:
link_st = os.stat(link)
return lock_st.st_ino == link_st.st_ino and lock_st.st_dev == link_st.st_dev
except OSError:
pass
return False
def exists_raise_eaccess(path):
try:
os.stat(path)
except OSError as e:
if e.errno == PermissionDenied.errno:
raise PermissionDenied("stat('%s')" % path)
return False
else:
return True
def cacheddir(my_original_path, ignorecvs, ignorelist, EmptyOnError, followSymlinks=True): mypath = normalize_path(my_original_path) try: pathstat = os.stat(mypath) if not stat.S_ISDIR(pathstat.st_mode): raise DirectoryNotFound(mypath) except EnvironmentError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mypath) del e return [], [] except PortageException: return [], [] else: try: fpaths = os.listdir(mypath) except EnvironmentError as e: if e.errno != errno.EACCES: raise del e raise PermissionDenied(mypath) ftype = [] for x in fpaths: try: if followSymlinks: pathstat = os.stat(mypath+"/"+x) else: pathstat = os.lstat(mypath+"/"+x) if stat.S_ISREG(pathstat[stat.ST_MODE]): ftype.append(0) elif stat.S_ISDIR(pathstat[stat.ST_MODE]): ftype.append(1) elif stat.S_ISLNK(pathstat[stat.ST_MODE]): ftype.append(2) else: ftype.append(3) except (IOError, OSError): ftype.append(3) if ignorelist or ignorecvs: ret_list = [] ret_ftype = [] for file_path, file_type in zip(fpaths, ftype): if file_path in ignorelist: pass elif ignorecvs: if file_path[:2] != ".#" and \ not (file_type == 1 and file_path in VCS_DIRS): ret_list.append(file_path) ret_ftype.append(file_type) else: ret_list = fpaths ret_ftype = ftype return ret_list, ret_ftype
def getfetchsizes(self, mypkg, useflags=None, debug=0, myrepo=None):
# returns a filename:size dictionnary of remaining downloads
myebuild, mytree = self.findname2(mypkg, myrepo=myrepo)
if myebuild is None:
raise AssertionError(_("ebuild not found for '%s'") % mypkg)
pkgdir = os.path.dirname(myebuild)
mf = self.repositories.get_repo_for_location(
os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
pkgdir, self.settings["DISTDIR"])
checksums = mf.getDigests()
if not checksums:
if debug:
writemsg(_("[empty/missing/bad digest]: %s\n") % (mypkg,))
return {}
filesdict={}
myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree)
#XXX: maybe this should be improved: take partial downloads
# into account? check checksums?
for myfile in myfiles:
try:
fetch_size = int(checksums[myfile]["size"])
except (KeyError, ValueError):
if debug:
writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
continue
file_path = os.path.join(self.settings["DISTDIR"], myfile)
mystat = None
try:
mystat = os.stat(file_path)
except OSError:
pass
if mystat is None:
existing_size = 0
ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS")
if ro_distdirs is not None:
for x in shlex_split(ro_distdirs):
try:
mystat = os.stat(os.path.join(x, myfile))
except OSError:
pass
else:
if mystat.st_size == fetch_size:
existing_size = fetch_size
break
else:
existing_size = mystat.st_size
remaining_size = fetch_size - existing_size
if remaining_size > 0:
# Assume the download is resumable.
filesdict[myfile] = remaining_size
elif remaining_size < 0:
# The existing file is too large and therefore corrupt.
filesdict[myfile] = int(checksums[myfile]["size"])
return filesdict
def load(self):
atoms = []
nonatoms = []
atoms_changed = False
# load atoms and non-atoms from different files so the worldfile is
# backwards-compatible with older versions and other PMs, even though
# it's supposed to be private state data :/
try:
mtime = os.stat(self._filename).st_mtime
except (OSError, IOError):
mtime = None
if (not self._loaded or self._mtime != mtime):
try:
data, errors = self.loader.load()
for fname in errors:
for e in errors[fname]:
self.errors.append(fname+": "+e)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
data = {}
atoms = list(data)
self._mtime = mtime
atoms_changed = True
else:
atoms.extend(self._atoms)
try:
mtime = os.stat(self._filename2).st_mtime
except (OSError, IOError):
mtime = None
if (not self._loaded or self._mtime2 != mtime):
try:
data, errors = self.loader2.load()
for fname in errors:
for e in errors[fname]:
self.errors.append(fname+": "+e)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
data = {}
nonatoms = list(data)
self._mtime2 = mtime
atoms_changed = True
else:
nonatoms.extend(self._nonatoms)
if atoms_changed:
self._setAtoms(atoms+nonatoms)
def _ensure_log_subdirs(logdir, subdir): """ This assumes that logdir exists, and creates subdirectories down to subdir as necessary. The gid of logdir is copied to all subdirectories, along with 0x2070 mode bits if present. Both logdir and subdir are assumed to be normalized absolute paths. """ st = os.stat(logdir) uid = -1 gid = st.st_gid grp_mode = 0o2070 & st.st_mode # If logdir is writable by the portage group but its uid # is not portage_uid, then set the uid to portage_uid if # we have privileges to do so, for compatibility with our # default logrotate config (see bug 378451). With the # "su portage portage" directive and logrotate-3.8.0, # logrotate's chown call during the compression phase will # only succeed if the log file's uid is portage_uid. if grp_mode and gid == portage_gid and \ portage.data.secpass >= 2: uid = portage_uid if st.st_uid != portage_uid: ensure_dirs(logdir, uid=uid) logdir_split_len = len(logdir.split(os.sep)) subdir_split = subdir.split(os.sep)[logdir_split_len:] subdir_split.reverse() current = logdir while subdir_split: current = os.path.join(current, subdir_split.pop()) ensure_dirs(current, uid=uid, gid=gid, mode=grp_mode, mask=0)
def load(self):
try:
mtime = os.stat(self._filename).st_mtime
except (OSError, IOError):
mtime = None
if not self._loaded or self._mtime != mtime:
try:
data, errors = self.loader.load()
for fname in errors:
for e in errors[fname]:
self.errors.append(fname + ": " + e)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
data = {}
if self.greedy:
atoms = []
for a in data:
matches = self.dbapi.match(a)
for cpv in matches:
atoms.append("%s:%s" % (cpv_getkey(cpv), self.dbapi.aux_get(cpv, ["SLOT"])[0]))
# In addition to any installed slots, also try to pull
# in the latest new slot that may be available.
atoms.append(a)
else:
atoms = iter(data)
self._setAtoms(atoms)
self._mtime = mtime
def have_ebuild_dir(path, maxdepth=3):
"""
Try to figure out if 'path' or a subdirectory contains one or more
ebuild files named appropriately for their parent directory.
"""
stack = [(normalize_path(path), 1)]
while stack:
path, depth = stack.pop()
basename = os.path.basename(path)
try:
listdir = os.listdir(path)
except OSError:
continue
for filename in listdir:
abs_filename = os.path.join(path, filename)
try:
st = os.stat(abs_filename)
except OSError:
continue
if stat.S_ISDIR(st.st_mode):
if depth < maxdepth:
stack.append((abs_filename, depth + 1))
elif stat.S_ISREG(st.st_mode):
if filename.endswith(".ebuild") and \
filename.startswith(basename + "-"):
return os.path.dirname(os.path.dirname(path))
def grab_updates(updpath, prev_mtimes=None):
"""Returns all the updates from the given directory as a sorted list of
tuples, each containing (file_path, statobj, content). If prev_mtimes is
given then only updates with differing mtimes are considered."""
try:
mylist = os.listdir(updpath)
except OSError as oe:
if oe.errno == errno.ENOENT:
raise DirectoryNotFound(updpath)
raise
if prev_mtimes is None:
prev_mtimes = {}
# validate the file name (filter out CVS directory, etc...)
mylist = [myfile for myfile in mylist if len(myfile) == 7 and myfile[1:3] == "Q-"]
if len(mylist) == 0:
return []
# update names are mangled to make them sort properly
mylist = [myfile[3:]+"-"+myfile[:2] for myfile in mylist]
mylist.sort()
mylist = [myfile[5:]+"-"+myfile[:4] for myfile in mylist]
update_data = []
for myfile in mylist:
file_path = os.path.join(updpath, myfile)
mystat = os.stat(file_path)
if file_path not in prev_mtimes or \
long(prev_mtimes[file_path]) != mystat[stat.ST_MTIME]:
content = codecs.open(_unicode_encode(file_path,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'], errors='replace'
).read()
update_data.append((file_path, mystat, content))
return update_data
def _apply_max_mtime(self, existing_st, entries): """ Set the Manifest mtime to the max mtime of all relevant files (the existing Manifest mtime is included in order to account for eclass modifications that change DIST entries). This results in a stable/predictable mtime, which is useful when converting thin manifests to thick manifests for distribution via rsync. For portability, the mtime is set with 1 second resolution. @param existing_st: stat result for existing Manifest @type existing_st: posix.stat_result @param entries: list of current Manifest2Entry instances @type entries: list """ # Use stat_result[stat.ST_MTIME] for 1 second resolution, since # it always rounds down. Note that stat_result.st_mtime will round # up from 0.999999999 to 1.0 when precision is lost during conversion # from nanosecond resolution to float. max_mtime = None if existing_st is None else existing_st[stat.ST_MTIME] for entry in entries: if entry.type == 'DIST': continue abs_path = (os.path.join(self.pkgdir, 'files', entry.name) if entry.type == 'AUX' else os.path.join(self.pkgdir, entry.name)) mtime = os.stat(abs_path)[stat.ST_MTIME] if max_mtime is None or mtime > max_mtime: max_mtime = mtime if max_mtime is not None: os.utime(self.getFullname(), (max_mtime, max_mtime))
def _check_distfile(filename, digests, eout, show_errors=1):
"""
@return a tuple of (match, stat_obj) where match is True if filename
matches all given digests (if any) and stat_obj is a stat result, or
None if the file does not exist.
"""
if digests is None:
digests = {}
size = digests.get("size")
if size is not None and len(digests) == 1:
digests = None
try:
st = os.stat(filename)
except OSError:
return (False, None)
if size is not None and size != st.st_size:
return (False, st)
if not digests:
if size is not None:
eout.ebegin(_("%s size ;-)") % os.path.basename(filename))
eout.eend(0)
elif st.st_size == 0:
# Zero-byte distfiles are always invalid.
return (False, st)
else:
if _check_digests(filename, digests, show_errors=show_errors):
eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
" ".join(sorted(digests))))
eout.eend(0)
else:
return (False, st)
return (True, st)
def sync_timestamp(self):
# If possible, update the mtime to match the remote package if
# the fetcher didn't already do it automatically.
bintree = self.pkg.root_config.trees["bintree"]
if bintree._remote_has_index:
remote_mtime = bintree._remotepkgs[
bintree.dbapi._instance_key(
self.pkg.cpv)].get("_mtime_")
if remote_mtime is not None:
try:
remote_mtime = long(remote_mtime)
except ValueError:
pass
else:
try:
local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME]
except OSError:
pass
else:
if remote_mtime != local_mtime:
try:
os.utime(self.pkg_path,
(remote_mtime, remote_mtime))
except OSError:
pass
def _pkgindex_entry(self, cpv): """ Performs checksums and evaluates USE flag conditionals. Raises InvalidDependString if necessary. @rtype: dict @return: a dict containing entry for the give cpv. """ pkg_path = self.getname(cpv) d = dict(zip(self._pkgindex_aux_keys, self.dbapi.aux_get(cpv, self._pkgindex_aux_keys))) d.update(perform_multiple_checksums( pkg_path, hashes=self._pkgindex_hashes)) d["CPV"] = cpv st = os.stat(pkg_path) d["MTIME"] = str(st[stat.ST_MTIME]) d["SIZE"] = str(st.st_size) rel_path = self._pkg_paths[cpv] # record location if it's non-default if rel_path != cpv + ".tbz2": d["PATH"] = rel_path self._eval_use_flags(cpv, d) return d
def load(self):
atoms_changed = False
try:
mtime = os.stat(self._filename).st_mtime
except (OSError, IOError):
mtime = None
if (not self._loaded or self._mtime != mtime):
try:
data, errors = self.loader.load()
for fname in errors:
for e in errors[fname]:
self.errors.append(fname+": "+e)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
del e
data = {}
nonatoms = list(data)
self._mtime = mtime
atoms_changed = True
else:
nonatoms = list(self._nonatoms)
if atoms_changed:
self._setAtoms(nonatoms)
def close(self): """Closes the temporary file, copies permissions (if possible), and performs the atomic replacement via os.rename(). If the abort() method has been called, then the temp file is closed and removed.""" f = object.__getattribute__(self, '_file') real_name = object.__getattribute__(self, '_real_name') if not f.closed: try: f.close() if not object.__getattribute__(self, '_aborted'): try: apply_stat_permissions(f.name, os.stat(real_name)) except OperationNotPermitted: pass except FileNotFound: pass except OSError as oe: # from the above os.stat call if oe.errno in (errno.ENOENT, errno.EPERM): pass else: raise os.rename(f.name, real_name) finally: # Make sure we cleanup the temp file # even if an exception is raised. try: os.unlink(f.name) except OSError as oe: pass
def RecursiveFileLoader(filename):
"""
If filename is of type file, return a generate that yields filename
else if filename is of type directory, return a generator that fields
files in that directory.
Ignore files beginning with . or ending in ~.
Prune CVS directories.
@param filename: name of a file/directory to traverse
@rtype: list
@returns: List of files to process
"""
try:
st = os.stat(filename)
except OSError:
return
if stat.S_ISDIR(st.st_mode):
for root, dirs, files in os.walk(filename):
for d in list(dirs):
if d[:1] == "." or d == "CVS":
dirs.remove(d)
for f in files:
try:
f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
except UnicodeDecodeError:
continue
if f[:1] == "." or f[-1:] == "~":
continue
yield os.path.join(root, f)
else:
yield filename
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self.returncode == os.EX_OK:
# If possible, update the mtime to match the remote package if
# the fetcher didn't already do it automatically.
bintree = self.pkg.root_config.trees["bintree"]
if bintree._remote_has_index:
remote_mtime = bintree._remotepkgs[self.pkg.cpv].get("MTIME")
if remote_mtime is not None:
try:
remote_mtime = long(remote_mtime)
except ValueError:
pass
else:
try:
local_mtime = os.stat(self.pkg_path)[stat.ST_MTIME]
except OSError:
pass
else:
if remote_mtime != local_mtime:
try:
os.utime(self.pkg_path,
(remote_mtime, remote_mtime))
except OSError:
pass
if self.locked:
self.unlock()
def __getattr__(self, attr): if attr == 'mtime': # use stat.ST_MTIME; accessing .st_mtime gets you a float # depending on the python version, and long(float) introduces # some rounding issues that aren't present for people using # the straight c api. # thus use the defacto python compatibility work around; # access via index, which guarantees you get the raw long. try: self.mtime = obj = os.stat(self.location)[stat.ST_MTIME] except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): raise FileNotFound(self.location) elif e.errno == PermissionDenied.errno: raise PermissionDenied(self.location) raise return obj if not attr.islower(): # we don't care to allow .mD5 as an alias for .md5 raise AttributeError(attr) hashname = attr.upper() if hashname not in checksum.hashfunc_map: raise AttributeError(attr) val = checksum.perform_checksum(self.location, hashname)[0] setattr(self, attr, val) return val
def _fetch_fs(self, mirror_info): file_path = os.path.join(mirror_info.location, self.distfile) st = None size_ok = False try: st = os.stat(file_path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): msg = "%s stat failed in %s: %s" % \ (self.distfile, mirror_info.name, e) self.scheduler.output(msg + '\n', background=True, log_path=self._log_path) logging.error(msg) else: size_ok = st.st_size == self.digests["size"] self._current_stat = st if size_ok: self._current_mirror = mirror_info self._start_task( FileDigester(file_path=file_path, hash_names=(self._select_hash(),), background=self.background, logfile=self._log_path), self._fs_mirror_digester_exit) else: self._try_next_mirror()
def _migrate(self, update_location): """ When repo.user_location is a normal directory, migrate it to storage so that it can be replaced with a symlink. After migration, commit the content as the latest snapshot. """ try: os.rename(self._user_location, update_location) except OSError: portage.util.ensure_dirs(update_location) portage.util.apply_stat_permissions(update_location, os.stat(self._user_location)) # It's probably on a different device, so copy it. yield self._check_call(['rsync', '-a', self._user_location + '/', update_location + '/']) # Remove the old copy so that symlink can be created. Run with # maximum privileges, since removal requires write access to # the parent directory. yield self._check_call(['rm', '-rf', user_location], privileged=True) self._update_location = update_location # Make this copy the latest snapshot yield self.commit_update()
def verify_all(filename, mydict, calc_prelink=0, strict=0):
"""
Verify all checksums against a file.
@param filename: File to run the checksums against
@type filename: String
@param calc_prelink: Whether or not to reverse prelink before running the checksum
@type calc_prelink: Integer
@param strict: Enable/Disable strict checking (which stops exactly at a checksum failure and throws an exception)
@type strict: Integer
@rtype: Tuple
@return: Result of the checks and possible message:
1) If size fails, False, and a tuple containing a message, the given size, and the actual size
2) If there is an os error, False, and a tuple containing the system error followed by 2 nulls
3) If a checksum fails, False and a tuple containing a message, the given hash, and the actual hash
4) If all checks succeed, return True and a fake reason
"""
# Dict relates to single file only.
# returns: (passed,reason)
file_is_ok = True
reason = "Reason unknown"
try:
mysize = os.stat(filename)[stat.ST_SIZE]
if mydict["size"] != mysize:
return False, (_("Filesize does not match recorded size"), mysize, mydict["size"])
except OSError as e:
if e.errno == errno.ENOENT:
raise portage.exception.FileNotFound(filename)
return False, (str(e), None, None)
verifiable_hash_types = set(mydict).intersection(hashfunc_map)
verifiable_hash_types.discard("size")
if not verifiable_hash_types:
expected = set(hashfunc_map)
expected.discard("size")
expected = list(expected)
expected.sort()
expected = " ".join(expected)
got = set(mydict)
got.discard("size")
got = list(got)
got.sort()
got = " ".join(got)
return False, (_("Insufficient data for checksum verification"), got, expected)
for x in sorted(mydict):
if x == "size":
continue
elif x in hashfunc_map:
myhash = perform_checksum(filename, x, calc_prelink=calc_prelink)[0]
if mydict[x] != myhash:
if strict:
raise portage.exception.DigestException(
("Failed to verify '$(file)s' on " + "checksum type '%(type)s'") % {"file": filename, "type": x}
)
else:
file_is_ok = False
reason = (("Failed on %s verification" % x), myhash, mydict[x])
break
return file_is_ok, reason
def process(mysettings, key, logentries, fulltext):
if mysettings.get("PORT_LOGDIR"):
logdir = normalize_path(mysettings["PORT_LOGDIR"])
else:
logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep),
"var", "log", "portage")
if not os.path.isdir(logdir):
# Only initialize group/mode if the directory doesn't
# exist, so that we don't override permissions if they
# were previously set by the administrator.
# NOTE: These permissions should be compatible with our
# default logrotate config as discussed in bug 374287.
logdir_uid = -1
if portage.data.secpass >= 2:
logdir_uid = portage_uid
ensure_dirs(logdir, uid=logdir_uid, gid=portage_gid, mode=0o2770)
elogdir = os.path.join(logdir, "elog")
_ensure_log_subdirs(logdir, elogdir)
# TODO: Locking
elogfilename = elogdir+"/summary.log"
elogfile = io.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='backslashreplace')
# Copy group permission bits from parent directory.
elogdir_st = os.stat(elogdir)
elogdir_gid = elogdir_st.st_gid
elogdir_grp_mode = 0o060 & elogdir_st.st_mode
# Copy the uid from the parent directory if we have privileges
# to do so, for compatibility with our default logrotate
# config (see bug 378451). With the "su portage portage"
# directive and logrotate-3.8.0, logrotate's chown call during
# the compression phase will only succeed if the log file's uid
# is portage_uid.
logfile_uid = -1
if portage.data.secpass >= 2:
logfile_uid = elogdir_st.st_uid
apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid,
mode=elogdir_grp_mode, mask=0)
time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.localtime(time.time()))
# Avoid potential UnicodeDecodeError later.
time_str = _unicode_decode(time_str,
encoding=_encodings['content'], errors='replace')
elogfile.write(_unicode_decode(
_(">>> Messages generated by process " +
"%(pid)d on %(time)s for package %(pkg)s:\n\n") %
{"pid": os.getpid(), "time": time_str, "pkg": key}))
elogfile.write(_unicode_decode(fulltext))
elogfile.write(_unicode_decode("\n"))
elogfile.close()
return elogfilename
def process(mysettings, key, logentries, fulltext):
if mysettings.get("PORT_LOGDIR"):
logdir = normalize_path(mysettings["PORT_LOGDIR"])
else:
logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep),
"var", "log", "portage")
if not os.path.isdir(logdir):
# Only initialize group/mode if the directory doesn't
# exist, so that we don't override permissions if they
# were previously set by the administrator.
# NOTE: These permissions should be compatible with our
# default logrotate config as discussed in bug 374287.
uid = -1
if portage.data.secpass >= 2:
uid = portage_uid
ensure_dirs(logdir, uid=uid, gid=portage_gid, mode=0o2770)
cat = mysettings['CATEGORY']
pf = mysettings['PF']
elogfilename = pf + ":" + _unicode_decode(
time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())),
encoding=_encodings['content'], errors='replace') + ".log"
if "split-elog" in mysettings.features:
log_subdir = os.path.join(logdir, "elog", cat)
elogfilename = os.path.join(log_subdir, elogfilename)
else:
log_subdir = os.path.join(logdir, "elog")
elogfilename = os.path.join(log_subdir, cat + ':' + elogfilename)
_ensure_log_subdirs(logdir, log_subdir)
elogfile = io.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='backslashreplace')
elogfile.write(_unicode_decode(fulltext))
elogfile.close()
# Copy group permission bits from parent directory.
elogdir_st = os.stat(log_subdir)
elogdir_gid = elogdir_st.st_gid
elogdir_grp_mode = 0o060 & elogdir_st.st_mode
# Copy the uid from the parent directory if we have privileges
# to do so, for compatibility with our default logrotate
# config (see bug 378451). With the "su portage portage"
# directive and logrotate-3.8.0, logrotate's chown call during
# the compression phase will only succeed if the log file's uid
# is portage_uid.
logfile_uid = -1
if portage.data.secpass >= 2:
logfile_uid = elogdir_st.st_uid
apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid,
mode=elogdir_grp_mode, mask=0)
return elogfilename
def _filter(self, atom):
cpv = self._db.match(atom)[0]
path = self._db.getpath(cpv, filename="COUNTER")
age = (time.time() - os.stat(path).st_mtime) / (3600 * 24)
if (self._mode == "older" and age <= self._age) or (self._mode == "newer" and age >= self._age):
return False
else:
return True
def isdir_raise_eaccess(path):
try:
st = os.stat(path)
except OSError as e:
if e.errno == PermissionDenied.errno:
raise PermissionDenied("stat('%s')" % path)
return False
else:
return stat.S_ISDIR(st.st_mode)
def _apply_max_mtime(self, preserved_stats, entries): """ Set the Manifest mtime to the max mtime of all relevant files and directories. Directory mtimes account for file renames and removals. The existing Manifest mtime accounts for eclass modifications that change DIST entries. This results in a stable/predictable mtime, which is useful when converting thin manifests to thick manifests for distribution via rsync. For portability, the mtime is set with 1 second resolution. @param preserved_stats: maps paths to preserved stat results that should be used instead of os.stat() calls @type preserved_stats: dict @param entries: list of current Manifest2Entry instances @type entries: list """ # Use stat_result[stat.ST_MTIME] for 1 second resolution, since # it always rounds down. Note that stat_result.st_mtime will round # up from 0.999999999 to 1.0 when precision is lost during conversion # from nanosecond resolution to float. max_mtime = None _update_max = (lambda st: max_mtime if max_mtime is not None and max_mtime > st[stat.ST_MTIME] else st[stat.ST_MTIME]) _stat = (lambda path: preserved_stats[path] if path in preserved_stats else os.stat(path)) for stat_result in preserved_stats.values(): max_mtime = _update_max(stat_result) for entry in entries: if entry.type == 'DIST': continue abs_path = (os.path.join(self.pkgdir, 'files', entry.name) if entry.type == 'AUX' else os.path.join(self.pkgdir, entry.name)) max_mtime = _update_max(_stat(abs_path)) if not self.thin: # Account for changes to all relevant nested directories. # This is not necessary for thin manifests because # self.pkgdir is already included via preserved_stats. for parent_dir, dirs, files in os.walk(self.pkgdir.rstrip(os.sep)): try: parent_dir = _unicode_decode(parent_dir, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: # If an absolute path cannot be decoded, then it is # always excluded from the manifest (repoman will # report such problems). pass else: max_mtime = _update_max(_stat(parent_dir)) if max_mtime is not None: for path in preserved_stats: os.utime(path, (max_mtime, max_mtime))
def _checksum_failure_temp_file(settings, distdir, basename):
"""
First try to find a duplicate temp file with the same checksum and return
that filename if available. Otherwise, use mkstemp to create a new unique
filename._checksum_failure_.$RANDOM, rename the given file, and return the
new filename. In any case, filename will be renamed or removed before this
function returns a temp filename.
"""
filename = os.path.join(distdir, basename)
if basename.endswith(_download_suffix):
normal_basename = basename[:-len(_download_suffix)]
else:
normal_basename = basename
size = os.stat(filename).st_size
checksum = None
tempfile_re = re.compile(re.escape(normal_basename) + r'\._checksum_failure_\..*')
for temp_filename in os.listdir(distdir):
if not tempfile_re.match(temp_filename):
continue
temp_filename = os.path.join(distdir, temp_filename)
try:
if size != os.stat(temp_filename).st_size:
continue
except OSError:
continue
try:
temp_checksum = perform_md5(temp_filename)
except FileNotFound:
# Apparently the temp file disappeared. Let it go.
continue
if checksum is None:
checksum = perform_md5(filename)
if checksum == temp_checksum:
os.unlink(filename)
return temp_filename
fd, temp_filename = \
tempfile.mkstemp("", normal_basename + "._checksum_failure_.", distdir)
os.close(fd)
_movefile(filename, temp_filename, mysettings=settings)
return temp_filename
def find_updated_config_files(target_root, config_protect):
"""
Return a tuple of configuration files that needs to be updated.
The tuple contains lists organized like this:
[ protected_dir, file_list ]
If the protected config isn't a protected_dir but a procted_file, list is:
[ protected_file, None ]
If no configuration files needs to be updated, None is returned
"""
os = _os_merge
if config_protect:
# directories with some protect files in them
for x in config_protect:
files = []
x = os.path.join(target_root, x.lstrip(os.path.sep))
if not os.access(x, os.W_OK):
continue
try:
mymode = os.lstat(x).st_mode
except OSError:
continue
if stat.S_ISLNK(mymode):
# We want to treat it like a directory if it
# is a symlink to an existing directory.
try:
real_mode = os.stat(x).st_mode
if stat.S_ISDIR(real_mode):
mymode = real_mode
except OSError:
pass
if stat.S_ISDIR(mymode):
mycommand = \
"find '%s' -name '.*' -type d -prune -o -name '._cfg????_*'" % x
else:
mycommand = "find '%s' -maxdepth 1 -name '._cfg????_%s'" % \
os.path.split(x.rstrip(os.path.sep))
mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0"
a = subprocess_getstatusoutput(mycommand)
if a[0] == 0:
files = a[1].split('\0')
# split always produces an empty string as the last element
if files and not files[-1]:
del files[-1]
if files:
if stat.S_ISDIR(mymode):
yield (x, files)
else:
yield (x, None)
def pre_sync(self, repo):
self.settings, self.trees, self.mtimedb = self.emerge_config
self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:
st = None
self.usersync_uid = None
spawn_kwargs = {}
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):
user = None
group = None
home = None
logname = None
spl = sync_user.split(':', 1)
if spl[0]:
username = spl[0]
try:
try:
pw = pwd.getpwnam(username)
except KeyError:
pw = pwd.getpwuid(int(username))
except (ValueError, KeyError):
writemsg("!!! User '%s' invalid or does not exist\n" %
username,
noiselevel=-1)
return (logname, user, group, home)
user = pw.pw_uid
group = pw.pw_gid
home = pw.pw_dir
logname = pw.pw_name
if len(spl) > 1:
groupname = spl[1]
try:
try:
gp = grp.getgrnam(groupname)
except KeyError:
pw = grp.getgrgid(int(groupname))
except (ValueError, KeyError):
writemsg("!!! Group '%s' invalid or does not exist\n" %
groupname,
noiselevel=-1)
return (logname, user, group, home)
group = gp.gr_gid
return (logname, user, group, home)
# user or user:group
(logname, uid, gid, home) = get_sync_user_data(repo.sync_user)
if uid is not None:
spawn_kwargs["uid"] = uid
self.usersync_uid = uid
if gid is not None:
spawn_kwargs["gid"] = gid
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
if logname is not None:
spawn_kwargs["env"]["LOGNAME"] = logname
if st is None:
perms = {'mode': 0o755}
# respect sync-user if set
if 'umask' in spawn_kwargs:
perms['mode'] &= ~spawn_kwargs['umask']
if 'uid' in spawn_kwargs:
perms['uid'] = spawn_kwargs['uid']
if 'gid' in spawn_kwargs:
perms['gid'] = spawn_kwargs['gid']
portage.util.ensure_dirs(repo.location, **perms)
st = os.stat(repo.location)
if (repo.sync_user is None and 'usersync' in self.settings.features
and portage.data.secpass >= 2
and (st.st_uid != os.getuid() and st.st_mode & 0o700
or st.st_gid != os.getgid() and st.st_mode & 0o070)):
try:
pw = pwd.getpwuid(st.st_uid)
except KeyError:
pass
else:
# Drop privileges when syncing, in order to match
# existing uid/gid settings.
self.usersync_uid = st.st_uid
spawn_kwargs["uid"] = st.st_uid
spawn_kwargs["gid"] = st.st_gid
spawn_kwargs["groups"] = [st.st_gid]
spawn_kwargs["env"]["HOME"] = pw.pw_dir
spawn_kwargs["env"]["LOGNAME"] = pw.pw_name
umask = 0o002
if not st.st_mode & 0o020:
umask = umask | 0o020
spawn_kwargs["umask"] = umask
# override the defaults when sync_umask is set
if repo.sync_umask is not None:
spawn_kwargs["umask"] = int(repo.sync_umask, 8)
self.spawn_kwargs = spawn_kwargs
if self.usersync_uid is not None:
# PORTAGE_TMPDIR is used below, so validate it and
# bail out if necessary.
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
return rval
os.umask(0o022)
return os.EX_OK
def _lockfile_iteration(mypath,
wantnewlockfile=False,
unlinkfile=False,
waiting_msg=None,
flags=0):
"""
Acquire a lock on mypath, without retry. Return None if the lockfile
was removed by previous lock holder (caller must retry).
@param mypath: lock file path
@type mypath: str
@param wantnewlockfile: use a separate new lock file
@type wantnewlockfile: bool
@param unlinkfile: remove lock file prior to unlock
@type unlinkfile: bool
@param waiting_msg: message to show before blocking
@type waiting_msg: str
@param flags: lock flags (only supports os.O_NONBLOCK)
@type flags: int
@rtype: bool
@return: unlockfile tuple on success, None if retry is needed
"""
if not mypath:
raise InvalidData(_("Empty path given"))
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# Support for file object or integer file descriptor parameters is
# deprecated due to ambiguity in whether or not it's safe to close
# the file descriptor, making it prone to "Bad file descriptor" errors
# or file descriptor leaks.
if isinstance(mypath, str) and mypath[-1] == '/':
mypath = mypath[:-1]
lockfilename_path = mypath
if hasattr(mypath, 'fileno'):
warnings.warn(
"portage.locks.lockfile() support for "
"file object parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename_path = getattr(mypath, 'name', None)
mypath = mypath.fileno()
if isinstance(mypath, int):
warnings.warn(
"portage.locks.lockfile() support for integer file "
"descriptor parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
lockfilename_path = lockfilename
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, str):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
while True:
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
if e.errno in (errno.ENOENT,
errno.ESTALE) and os.path.isdir(
os.path.dirname(lockfilename)):
# Retry required for NFS (see bug 636798).
continue
else:
_raise_exc(e)
else:
break
if not preexisting:
try:
if portage.data.secpass >= 1 and os.stat(
lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
os.close(myfd)
return None
writemsg("%s: chown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = portage._eintr_func_wrapper(_get_lock_fn())
try:
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
raise IOError(errno.ENOSYS, "Function not implemented")
locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if not hasattr(e, "errno"):
raise
if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
os.close(myfd)
raise TryAgain(mypath)
global _quiet
if _quiet:
out = None
else:
out = portage.output.EOutput()
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s") % lockfilename
if out is not None:
out.ebegin(waiting_msg)
# try for the exclusive lock now.
enolock_msg_shown = False
while True:
try:
locking_method(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
if e.errno == errno.ENOLCK:
# This is known to occur on Solaris NFS (see
# bug #462694). Assume that the error is due
# to temporary exhaustion of record locks,
# and loop until one becomes available.
if not enolock_msg_shown:
enolock_msg_shown = True
if isinstance(mypath, int):
context_desc = _("Error while waiting "
"to lock fd %i") % myfd
else:
context_desc = _("Error while waiting "
"to lock '%s'") % lockfilename
writemsg("\n!!! %s: %s\n" % (context_desc, e),
noiselevel=-1)
time.sleep(_HARDLINK_POLL_LATENCY)
continue
if out is not None:
out.eend(1, str(e))
raise
else:
break
if out is not None:
out.eend(os.EX_OK)
elif e.errno in (errno.ENOSYS, ):
# We're not allowed to lock on this FS.
if not isinstance(lockfilename, int):
# If a file object was passed in, it's not safe
# to close the file descriptor because it may
# still be in use.
os.close(myfd)
lockfilename_path = _unicode_decode(lockfilename_path,
encoding=_encodings['fs'],
errors='strict')
if not isinstance(lockfilename_path, str):
raise
link_success = hardlink_lockfile(lockfilename_path,
waiting_msg=waiting_msg,
flags=flags)
if not link_success:
raise
lockfilename = lockfilename_path
locking_method = None
myfd = HARDLINK_FD
else:
raise
fstat_result = None
if isinstance(lockfilename, str) and myfd != HARDLINK_FD and unlinkfile:
try:
(removed, fstat_result) = _lockfile_was_removed(myfd, lockfilename)
except Exception:
# Do not leak the file descriptor here.
os.close(myfd)
raise
else:
if removed:
# Removed by previous lock holder... Caller will retry...
os.close(myfd)
return None
if myfd != HARDLINK_FD:
_lock_manager(myfd,
os.fstat(myfd) if fstat_result is None else fstat_result,
mypath)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: a tuple of (removed, fstat_result), where removed is True if
lock_path does not correspond to lock_fd, and False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return (True, None)
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return (True, None)
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
# Create another hardlink in order to detect whether or not
# hardlink inode numbers are expected to match. For example,
# inode numbers are not expected to match for sshfs.
inode_test = hardlink_path + '-inode-test'
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
os.link(hardlink_path, inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return (True, None)
else:
if not os.path.samefile(hardlink_path, inode_test):
# This implies that inode numbers are not expected
# to match for this file system, so use a simple
# stat call to detect if lock_path has been removed.
return (not os.path.exists(lock_path), fstat_st)
finally:
try:
os.unlink(inode_test)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return (True, None)
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return (False, fstat_st)
def emerge_main(args=None):
"""
@param args: command arguments (default: sys.argv[1:])
@type args: list
"""
if args is None:
args = sys.argv[1:]
args = portage._decode_argv(args)
# Use system locale.
try:
locale.setlocale(locale.LC_ALL, "")
except locale.Error as e:
writemsg_level("setlocale: %s\n" % e, level=logging.WARN)
# Disable color until we're sure that it should be enabled (after
# EMERGE_DEFAULT_OPTS has been parsed).
portage.output.havecolor = 0
# This first pass is just for options that need to be known as early as
# possible, such as --config-root. They will be parsed again later,
# together with EMERGE_DEFAULT_OPTS (which may vary depending on the
# the value of --config-root).
myaction, myopts, myfiles = parse_opts(args, silent=True)
if "--debug" in myopts:
os.environ["PORTAGE_DEBUG"] = "1"
if "--config-root" in myopts:
os.environ["PORTAGE_CONFIGROOT"] = myopts["--config-root"]
if "--sysroot" in myopts:
os.environ["SYSROOT"] = myopts["--sysroot"]
if "--root" in myopts:
os.environ["ROOT"] = myopts["--root"]
if "--prefix" in myopts:
os.environ["EPREFIX"] = myopts["--prefix"]
if "--accept-properties" in myopts:
os.environ["ACCEPT_PROPERTIES"] = myopts["--accept-properties"]
if "--accept-restrict" in myopts:
os.environ["ACCEPT_RESTRICT"] = myopts["--accept-restrict"]
# optimize --help (no need to load config / EMERGE_DEFAULT_OPTS)
if myaction == "help":
emerge_help()
return os.EX_OK
if myaction == "moo":
print(COWSAY_MOO % platform.system())
return os.EX_OK
if myaction == "sync":
# need to set this to True now in order for the repository config
# loading to allow new repos with non-existent directories
portage._sync_mode = True
# Verify that /dev/null exists and is a device file as a cheap early
# filter for obviously broken /dev/s.
try:
if os.stat(os.devnull).st_rdev == 0:
writemsg_level(
"Failed to validate a sane '/dev'.\n"
"'/dev/null' is not a device file.\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1
except OSError:
writemsg_level(
"Failed to validate a sane '/dev'.\n"
"'/dev/null' does not exist.\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1
# Verify that BASH process substitution works as another cheap early
# filter. Process substitution uses '/dev/fd'.
with open(os.devnull, "r+b") as dev_null:
fd_pipes = {
0: dev_null.fileno(),
1: dev_null.fileno(),
2: dev_null.fileno(),
}
if (portage.process.spawn_bash("[[ $(< <(echo foo) ) == foo ]]",
fd_pipes=fd_pipes) != 0):
writemsg_level(
"Failed to validate a sane '/dev'.\n"
"bash process substitution doesn't work; this may be an "
"indication of a broken '/dev/fd'.\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1
# Portage needs to ensure a sane umask for the files it creates.
os.umask(0o22)
emerge_config = load_emerge_config(action=myaction,
args=myfiles,
opts=myopts)
# Make locale variables from configuration files (make.defaults, make.conf) affect locale of emerge process.
for locale_var_name in (
"LANGUAGE",
"LC_ALL",
"LC_ADDRESS",
"LC_COLLATE",
"LC_CTYPE",
"LC_IDENTIFICATION",
"LC_MEASUREMENT",
"LC_MESSAGES",
"LC_MONETARY",
"LC_NAME",
"LC_NUMERIC",
"LC_PAPER",
"LC_TELEPHONE",
"LC_TIME",
"LANG",
):
locale_var_value = emerge_config.running_config.settings.get(
locale_var_name)
if locale_var_value is not None:
os.environ.setdefault(locale_var_name, locale_var_value)
try:
locale.setlocale(locale.LC_ALL, "")
except locale.Error as e:
writemsg_level("setlocale: %s\n" % e, level=logging.WARN)
tmpcmdline = []
if "--ignore-default-opts" not in myopts:
tmpcmdline.extend(
portage.util.shlex_split(
emerge_config.target_config.settings.get(
"EMERGE_DEFAULT_OPTS", "")))
tmpcmdline.extend(args)
emerge_config.action, emerge_config.opts, emerge_config.args = parse_opts(
tmpcmdline)
try:
return run_action(emerge_config)
finally:
# Call destructors for our portdbapi instances.
for x in emerge_config.trees.values():
if "porttree" in x.lazy_items:
continue
x["porttree"].dbapi.close_caches()
def UpdateChangeLog(pkgdir,
user,
msg,
skel_path,
category,
package,
new=(),
removed=(),
changed=(),
pretend=False,
quiet=False):
"""
Write an entry to an existing ChangeLog, or create a new one.
Updates copyright year on changed files, and updates the header of
ChangeLog with the contents of skel.ChangeLog.
"""
if '<root@' in user:
if not quiet:
logging.critical('Please set ECHANGELOG_USER or run as non-root')
return None
# ChangeLog times are in UTC
gmtime = time.gmtime()
year = time.strftime('%Y', gmtime)
date = time.strftime('%d %b %Y', gmtime)
cl_path = os.path.join(pkgdir, 'ChangeLog')
clold_lines = []
clnew_lines = []
old_header_lines = []
header_lines = []
clold_file = None
try:
clold_file = io.open(_unicode_encode(cl_path,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace')
except EnvironmentError:
pass
f, clnew_path = tempfile.mkstemp()
# construct correct header first
try:
if clold_file is not None:
# retain header from old ChangeLog
first_line = True
for line in clold_file:
line_strip = line.strip()
if line_strip and line[:1] != "#":
clold_lines.append(line)
break
# always make sure cat/pkg is up-to-date in case we are
# moving packages around, or copied from another pkg, or ...
if first_line:
if line.startswith('# ChangeLog for'):
line = '# ChangeLog for %s/%s\n' % (category, package)
first_line = False
old_header_lines.append(line)
header_lines.append(update_copyright_year(year, line))
if not line_strip:
break
clskel_file = None
if not header_lines:
# delay opening this until we find we need a header
try:
clskel_file = io.open(_unicode_encode(
skel_path, encoding=_encodings['fs'], errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace')
except EnvironmentError:
pass
if clskel_file is not None:
# read skel.ChangeLog up to first empty line
for line in clskel_file:
line_strip = line.strip()
if not line_strip:
break
line = line.replace('<CATEGORY>', category)
line = line.replace('<PACKAGE_NAME>', package)
line = update_copyright_year(year, line)
header_lines.append(line)
header_lines.append('\n')
clskel_file.close()
# write new ChangeLog entry
clnew_lines.extend(header_lines)
newebuild = False
for fn in new:
if not fn.endswith('.ebuild'):
continue
ebuild = fn.split(os.sep)[-1][0:-7]
clnew_lines.append('*%s (%s)\n' % (ebuild, date))
newebuild = True
if newebuild:
clnew_lines.append('\n')
trivial_files = ('ChangeLog', 'Manifest')
display_new = ['+' + elem for elem in new if elem not in trivial_files]
display_removed = ['-' + elem for elem in removed]
display_changed = [
elem for elem in changed if elem not in trivial_files
]
if not (display_new or display_removed or display_changed):
# If there's nothing else to display, show one of the
# trivial files.
for fn in trivial_files:
if fn in new:
display_new = ['+' + fn]
break
elif fn in changed:
display_changed = [fn]
break
display_new.sort()
display_removed.sort()
display_changed.sort()
mesg = '%s; %s %s:' % (date, user, ', '.join(
chain(display_new, display_removed, display_changed)))
for line in textwrap.wrap(mesg,
80,
initial_indent=' ',
subsequent_indent=' ',
break_on_hyphens=False):
clnew_lines.append('%s\n' % line)
for line in textwrap.wrap(msg,
80,
initial_indent=' ',
subsequent_indent=' '):
clnew_lines.append('%s\n' % line)
# Don't append a trailing newline if the file is new.
if clold_file is not None:
clnew_lines.append('\n')
f = io.open(f,
mode='w',
encoding=_encodings['repo.content'],
errors='backslashreplace')
for line in clnew_lines:
f.write(line)
# append stuff from old ChangeLog
if clold_file is not None:
if clold_lines:
# clold_lines may contain a saved non-header line
# that we want to write first.
# Also, append this line to clnew_lines so that the
# unified_diff call doesn't show it as removed.
for line in clold_lines:
f.write(line)
clnew_lines.append(line)
else:
# ensure that there is no more than one blank
# line after our new entry
for line in clold_file:
if line.strip():
f.write(line)
break
# Now prepend old_header_lines to clold_lines, for use
# in the unified_diff call below.
clold_lines = old_header_lines + clold_lines
# Trim any trailing newlines.
lines = clold_file.readlines()
clold_file.close()
while lines and lines[-1] == '\n':
del lines[-1]
f.writelines(lines)
f.close()
# show diff
if not quiet:
for line in difflib.unified_diff(clold_lines,
clnew_lines,
fromfile=cl_path,
tofile=cl_path,
n=0):
util.writemsg_stdout(line, noiselevel=-1)
util.writemsg_stdout("\n", noiselevel=-1)
if pretend:
# remove what we've done
os.remove(clnew_path)
else:
# rename to ChangeLog, and set permissions
try:
clold_stat = os.stat(cl_path)
except OSError:
clold_stat = None
shutil.move(clnew_path, cl_path)
if clold_stat is None:
util.apply_permissions(cl_path, mode=0o644)
else:
util.apply_stat_permissions(cl_path, clold_stat)
if clold_file is None:
return True
else:
return False
except IOError as e:
err = 'Repoman is unable to create/write to Changelog.new file: %s' % (
e, )
logging.critical(err)
# try to remove if possible
try:
os.remove(clnew_path)
except OSError:
pass
return None
def process(mysettings, key, logentries, fulltext):
if mysettings.get("PORTAGE_LOGDIR"):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep),
"var", "log", "portage")
if not os.path.isdir(logdir):
# Only initialize group/mode if the directory doesn't
# exist, so that we don't override permissions if they
# were previously set by the administrator.
# NOTE: These permissions should be compatible with our
# default logrotate config as discussed in bug 374287.
logdir_uid = -1
if portage.data.secpass >= 2:
logdir_uid = portage_uid
ensure_dirs(logdir, uid=logdir_uid, gid=portage_gid, mode=0o2770)
elogdir = os.path.join(logdir, "elog")
_ensure_log_subdirs(logdir, elogdir)
# TODO: Locking
elogfilename = elogdir + "/summary.log"
try:
elogfile = io.open(_unicode_encode(elogfilename,
encoding=_encodings['fs'],
errors='strict'),
mode='a',
encoding=_encodings['content'],
errors='backslashreplace')
except IOError as e:
func_call = "open('%s', 'a')" % elogfilename
if e.errno == errno.EACCES:
raise portage.exception.PermissionDenied(func_call)
elif e.errno == errno.EPERM:
raise portage.exception.OperationNotPermitted(func_call)
elif e.errno == errno.EROFS:
raise portage.exception.ReadOnlyFileSystem(func_call)
else:
raise
# Copy group permission bits from parent directory.
elogdir_st = os.stat(elogdir)
elogdir_gid = elogdir_st.st_gid
elogdir_grp_mode = 0o060 & elogdir_st.st_mode
# Copy the uid from the parent directory if we have privileges
# to do so, for compatibility with our default logrotate
# config (see bug 378451). With the "su portage portage"
# directive and logrotate-3.8.0, logrotate's chown call during
# the compression phase will only succeed if the log file's uid
# is portage_uid.
logfile_uid = -1
if portage.data.secpass >= 2:
logfile_uid = elogdir_st.st_uid
apply_permissions(elogfilename,
uid=logfile_uid,
gid=elogdir_gid,
mode=elogdir_grp_mode,
mask=0)
time_fmt = "%Y-%m-%d %H:%M:%S %Z"
if sys.hexversion < 0x3000000:
time_fmt = _unicode_encode(time_fmt)
time_str = time.strftime(time_fmt, time.localtime(time.time()))
# Avoid potential UnicodeDecodeError in Python 2, since strftime
# returns bytes in Python 2, and %Z may contain non-ascii chars.
time_str = _unicode_decode(time_str,
encoding=_encodings['content'],
errors='replace')
elogfile.write(
_(">>> Messages generated by process "
"%(pid)d on %(time)s for package %(pkg)s:\n\n") % {
"pid": os.getpid(),
"time": time_str,
"pkg": key
})
elogfile.write(_unicode_decode(fulltext))
elogfile.write("\n")
elogfile.close()
return elogfilename
def lockfile(mypath,
wantnewlockfile=0,
unlinkfile=0,
waiting_msg=None,
flags=0):
"""
If wantnewlockfile is True then this creates a lockfile in the parent
directory as the file: '.' + basename + '.portage_lockfile'.
"""
if not mypath:
raise InvalidData(_("Empty path given"))
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# Support for file object or integer file descriptor parameters is
# deprecated due to ambiguity in whether or not it's safe to close
# the file descriptor, making it prone to "Bad file descriptor" errors
# or file descriptor leaks.
if isinstance(mypath, basestring) and mypath[-1] == '/':
mypath = mypath[:-1]
lockfilename_path = mypath
if hasattr(mypath, 'fileno'):
warnings.warn(
"portage.locks.lockfile() support for "
"file object parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename_path = getattr(mypath, 'name', None)
mypath = mypath.fileno()
if isinstance(mypath, int):
warnings.warn(
"portage.locks.lockfile() support for integer file "
"descriptor parameters is deprecated. Use a file path instead.",
DeprecationWarning,
stacklevel=2)
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
lockfilename_path = lockfilename
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, basestring):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
if not preexisting:
try:
if portage.data.secpass >= 1 and os.stat(
lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
return lockfile(mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile,
waiting_msg=waiting_msg,
flags=flags)
else:
writemsg("%s: chown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = portage._eintr_func_wrapper(_get_lock_fn())
try:
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
raise IOError(errno.ENOSYS, "Function not implemented")
locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if not hasattr(e, "errno"):
raise
if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
os.close(myfd)
raise TryAgain(mypath)
global _quiet
if _quiet:
out = None
else:
out = portage.output.EOutput()
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s") % lockfilename
if out is not None:
out.ebegin(waiting_msg)
# try for the exclusive lock now.
enolock_msg_shown = False
while True:
try:
locking_method(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
if e.errno == errno.ENOLCK:
# This is known to occur on Solaris NFS (see
# bug #462694). Assume that the error is due
# to temporary exhaustion of record locks,
# and loop until one becomes available.
if not enolock_msg_shown:
enolock_msg_shown = True
if isinstance(mypath, int):
context_desc = _("Error while waiting "
"to lock fd %i") % myfd
else:
context_desc = _("Error while waiting "
"to lock '%s'") % lockfilename
writemsg("\n!!! %s: %s\n" % (context_desc, e),
noiselevel=-1)
time.sleep(_HARDLINK_POLL_LATENCY)
continue
if out is not None:
out.eend(1, str(e))
raise
else:
break
if out is not None:
out.eend(os.EX_OK)
elif e.errno in (errno.ENOSYS, ):
# We're not allowed to lock on this FS.
if not isinstance(lockfilename, int):
# If a file object was passed in, it's not safe
# to close the file descriptor because it may
# still be in use.
os.close(myfd)
lockfilename_path = _unicode_decode(lockfilename_path,
encoding=_encodings['fs'],
errors='strict')
if not isinstance(lockfilename_path, basestring):
raise
link_success = hardlink_lockfile(lockfilename_path,
waiting_msg=waiting_msg,
flags=flags)
if not link_success:
raise
lockfilename = lockfilename_path
locking_method = None
myfd = HARDLINK_FD
else:
raise
if isinstance(lockfilename, basestring) and \
myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
# The file was deleted on us... Keep trying to make one...
os.close(myfd)
writemsg(_("lockfile recurse\n"), 1)
lockfilename, myfd, unlinkfile, locking_method = lockfile(
mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile,
waiting_msg=waiting_msg,
flags=flags)
if myfd != HARDLINK_FD:
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(
myfd, fcntl.F_SETFD,
fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
_open_fds.add(myfd)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
def verify_all(filename, mydict, calc_prelink=0, strict=0):
"""
Verify all checksums against a file.
@param filename: File to run the checksums against
@type filename: String
@param calc_prelink: Whether or not to reverse prelink before running the checksum
@type calc_prelink: Integer
@param strict: Enable/Disable strict checking (which stops exactly at a checksum failure and throws an exception)
@type strict: Integer
@rtype: Tuple
@return: Result of the checks and possible message:
1) If size fails, False, and a tuple containing a message, the given size, and the actual size
2) If there is an os error, False, and a tuple containing the system error followed by 2 nulls
3) If a checksum fails, False and a tuple containing a message, the given hash, and the actual hash
4) If all checks succeed, return True and a fake reason
"""
# Dict relates to single file only.
# returns: (passed,reason)
file_is_ok = True
reason = "Reason unknown"
try:
mysize = os.stat(filename)[stat.ST_SIZE]
if mydict.get("size") is not None and mydict["size"] != mysize:
return False, (_("Filesize does not match recorded size"), mysize,
mydict["size"])
except OSError as e:
if e.errno == errno.ENOENT:
raise portage.exception.FileNotFound(filename)
return False, (str(e), None, None)
verifiable_hash_types = set(mydict).intersection(hashfunc_map)
verifiable_hash_types.discard("size")
if not verifiable_hash_types:
expected = set(hashfunc_map)
expected.discard("size")
expected = list(expected)
expected.sort()
expected = " ".join(expected)
got = set(mydict)
got.discard("size")
got = list(got)
got.sort()
got = " ".join(got)
return False, (_("Insufficient data for checksum verification"), got,
expected)
for x in sorted(mydict):
if x == "size":
continue
elif x in hashfunc_map:
myhash = perform_checksum(filename, x,
calc_prelink=calc_prelink)[0]
if mydict[x] != myhash:
if strict:
raise portage.exception.DigestException(
("Failed to verify '$(file)s' on " + \
"checksum type '%(type)s'") % \
{"file" : filename, "type" : x})
else:
file_is_ok = False
reason = (("Failed on %s verification" % x), myhash,
mydict[x])
break
return file_is_ok, reason
def check(self, **kwargs):
'''Checks the ebuild sources and files for errors
@param xpkg: the pacakge being checked
@param checkdir: string, directory path
@param checkdir_relative: repolevel determined path
@returns: boolean
'''
xpkg = kwargs.get('xpkg')
checkdir = kwargs.get('checkdir')
checkdir_relative = kwargs.get('checkdir_relative')
changed = kwargs.get('changed').changed
new = kwargs.get('changed').new
_digests = self.digests(checkdir)
fetchlist_dict = portage.FetchlistDict(checkdir, self.repoman_settings,
self.portdb)
myfiles_all = []
self._src_uri_error = False
for mykey in fetchlist_dict:
try:
myfiles_all.extend(fetchlist_dict[mykey])
except portage.exception.InvalidDependString as e:
self._src_uri_error = True
try:
self.portdb.aux_get(mykey, ["SRC_URI"])
except KeyError:
# This will be reported as an "ebuild.syntax" error.
pass
else:
self.qatracker.add_error(
"SRC_URI.syntax", "%s.ebuild SRC_URI: %s" % (mykey, e))
del fetchlist_dict
if not self._src_uri_error:
# This test can produce false positives if SRC_URI could not
# be parsed for one or more ebuilds. There's no point in
# producing a false error here since the root cause will
# produce a valid error elsewhere, such as "SRC_URI.syntax"
# or "ebuild.sytax".
myfiles_all = set(myfiles_all)
for entry in _digests:
if entry not in myfiles_all:
self.qatracker.add_error("digest.unused",
checkdir + "::" + entry)
for entry in myfiles_all:
if entry not in _digests:
self.qatracker.add_error("digest.missing",
checkdir + "::" + entry)
del myfiles_all
if os.path.exists(checkdir + "/files"):
filesdirlist = os.listdir(checkdir + "/files")
# Recurse through files directory, use filesdirlist as a stack;
# appending directories as needed,
# so people can't hide > 20k files in a subdirectory.
while filesdirlist:
y = filesdirlist.pop(0)
relative_path = os.path.join(xpkg, "files", y)
full_path = os.path.join(self.repo_settings.repodir,
relative_path)
try:
mystat = os.stat(full_path)
except OSError as oe:
if oe.errno == 2:
# don't worry about it. it likely was removed via fix above.
continue
else:
raise oe
if S_ISDIR(mystat.st_mode):
if self.vcs_settings.status.isVcsDir(y):
continue
for z in os.listdir(checkdir + "/files/" + y):
if self.vcs_settings.status.isVcsDir(z):
continue
filesdirlist.append(y + "/" + z)
# Current policy is no files over 20 KiB, these are the checks.
# File size between 20 KiB and 60 KiB causes a warning,
# while file size over 60 KiB causes an error.
elif mystat.st_size > 61440:
self.qatracker.add_error(
"file.size-fatal", "(%d KiB) %s/files/%s" %
(mystat.st_size // 1024, xpkg, y))
elif mystat.st_size > 20480:
self.qatracker.add_error(
"file.size", "(%d KiB) %s/files/%s" %
(mystat.st_size // 1024, xpkg, y))
elif mystat.st_size == 0:
self.qatracker.add_error("file.empty",
"%s/files/%s" % (xpkg, y))
index = self.repo_settings.repo_config.find_invalid_path_char(
y)
if index != -1:
y_relative = os.path.join(checkdir_relative, "files", y)
if self.vcs_settings.vcs is not None \
and not vcs_new_changed(y_relative, changed, new):
# If the file isn't in the VCS new or changed set, then
# assume that it's an irrelevant temporary file (Manifest
# entries are not generated for file names containing
# prohibited characters). See bug #406877.
index = -1
if index != -1:
self.qatracker.add_error(
"file.name",
"%s/files/%s: char '%s'" % (checkdir, y, y[index]))
return False
def getentries(mydir, recursive=0):
"""(basedir,recursive=0)
Scans the given directory and returns a datadict of all the entries in
the directory separated as a dirs dict and a files dict."""
myfn = mydir + "/CVS/Entries"
# entries=[dirs, files]
entries = {"dirs": {}, "files": {}}
if not os.path.exists(mydir):
return entries
try:
myfile = io.open(_unicode_encode(myfn,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='strict')
mylines = myfile.readlines()
myfile.close()
except SystemExit as e:
raise
except:
mylines = []
for line in mylines:
if line and line[-1] == "\n":
line = line[:-1]
if not line:
continue
if line == "D": # End of entries file
break
mysplit = line.split("/")
if len(mysplit) != 6:
print("Confused:", mysplit)
continue
if mysplit[0] == "D":
entries["dirs"][mysplit[1]] = {
"dirs": {},
"files": {},
"status": []
}
entries["dirs"][mysplit[1]]["status"] = ["cvs"]
if os.path.isdir(mydir + "/" + mysplit[1]):
entries["dirs"][mysplit[1]]["status"] += ["exists"]
entries["dirs"][mysplit[1]]["flags"] = mysplit[2:]
if recursive:
rentries = getentries(mydir + "/" + mysplit[1], recursive)
entries["dirs"][mysplit[1]]["dirs"] = rentries["dirs"]
entries["dirs"][mysplit[1]]["files"] = rentries["files"]
else:
# [D]/Name/revision/Date/Flags/Tags
entries["files"][mysplit[1]] = {}
entries["files"][mysplit[1]]["revision"] = mysplit[2]
entries["files"][mysplit[1]]["date"] = mysplit[3]
entries["files"][mysplit[1]]["flags"] = mysplit[4]
entries["files"][mysplit[1]]["tags"] = mysplit[5]
entries["files"][mysplit[1]]["status"] = ["cvs"]
if entries["files"][mysplit[1]]["revision"][0] == "-":
entries["files"][mysplit[1]]["status"] += ["removed"]
for file in os.listdir(mydir):
if file == "CVS":
continue
if os.path.isdir(mydir + "/" + file):
if file not in entries["dirs"]:
if ignore_list.match(file) is not None:
continue
entries["dirs"][file] = {"dirs": {}, "files": {}}
# It's normal for a directory to be unlisted in Entries
# when checked out without -P (see bug #257660).
rentries = getentries(mydir + "/" + file, recursive)
entries["dirs"][file]["dirs"] = rentries["dirs"]
entries["dirs"][file]["files"] = rentries["files"]
if "status" in entries["dirs"][file]:
if "exists" not in entries["dirs"][file]["status"]:
entries["dirs"][file]["status"] += ["exists"]
else:
entries["dirs"][file]["status"] = ["exists"]
elif os.path.isfile(mydir + "/" + file):
if file not in entries["files"]:
if ignore_list.match(file) is not None:
continue
entries["files"][file] = {
"revision": "",
"date": "",
"flags": "",
"tags": ""
}
if "status" in entries["files"][file]:
if "exists" not in entries["files"][file]["status"]:
entries["files"][file]["status"] += ["exists"]
else:
entries["files"][file]["status"] = ["exists"]
try:
mystat = os.stat(mydir + "/" + file)
mytime = time.asctime(time.gmtime(mystat[stat.ST_MTIME]))
if "status" not in entries["files"][file]:
entries["files"][file]["status"] = []
if mytime == entries["files"][file]["date"]:
entries["files"][file]["status"] += ["current"]
except SystemExit as e:
raise
except Exception as e:
print("failed to stat", file)
print(e)
return
elif ignore_list.match(file) is not None:
pass
else:
print()
print("File of unknown type:", mydir + "/" + file)
print()
return entries
def fetch_metadata_xsd(metadata_xsd, repoman_settings):
"""
Fetch metadata.xsd if it doesn't exist or the ctime is older than
metadata_xsd_ctime_interval.
@rtype: bool
@return: True if successful, otherwise False
"""
must_fetch = True
metadata_xsd_st = None
current_time = int(time.time())
try:
metadata_xsd_st = os.stat(metadata_xsd)
except EnvironmentError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
del e
else:
# Trigger fetch if metadata.xsd mtime is old or clock is wrong.
if abs(current_time - metadata_xsd_st.st_ctime) \
< metadata_xsd_ctime_interval:
must_fetch = False
if must_fetch:
print()
print(
"%s the local copy of metadata.xsd "
"needs to be refetched, doing that now" % green("***"))
print()
parsed_url = urlparse(metadata_xsd_uri)
setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
fcmd = repoman_settings.get(setting)
if not fcmd:
fcmd = repoman_settings.get('FETCHCOMMAND')
if not fcmd:
logging.error("FETCHCOMMAND is unset")
return False
destdir = repoman_settings["DISTDIR"]
fd, metadata_xsd_tmp = tempfile.mkstemp(
prefix='metadata.xsd.', dir=destdir)
os.close(fd)
try:
if not portage.getbinpkg.file_get(
metadata_xsd_uri, destdir, fcmd=fcmd,
filename=os.path.basename(metadata_xsd_tmp)):
logging.error(
"failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri)
return False
try:
portage.util.apply_secpass_permissions(
metadata_xsd_tmp,
gid=portage.data.portage_gid, mode=0o664, mask=0o2)
except portage.exception.PortageException:
pass
shutil.move(metadata_xsd_tmp, metadata_xsd)
finally:
try:
os.unlink(metadata_xsd_tmp)
except OSError:
pass
return True
def __init__(self, _unused_param=DeprecationWarning, mysettings=None):
"""
@param _unused_param: deprecated, use mysettings['PORTDIR'] instead
@type _unused_param: None
@param mysettings: an immutable config instance
@type mysettings: portage.config
"""
from portage import config
if mysettings:
self.settings = mysettings
else:
from portage import settings
self.settings = config(clone=settings)
if _unused_param is not DeprecationWarning:
warnings.warn("The first parameter of the " + \
"portage.dbapi.porttree.portdbapi" + \
" constructor is unused since portage-2.1.8. " + \
"mysettings['PORTDIR'] is used instead.",
DeprecationWarning, stacklevel=2)
self.repositories = self.settings.repositories
self.treemap = self.repositories.treemap
# This is strictly for use in aux_get() doebuild calls when metadata
# is generated by the depend phase. It's safest to use a clone for
# this purpose because doebuild makes many changes to the config
# instance that is passed in.
self.doebuild_settings = config(clone=self.settings)
self.depcachedir = os.path.realpath(self.settings.depcachedir)
if os.environ.get("SANDBOX_ON") == "1":
# Make api consumers exempt from sandbox violations
# when doing metadata cache updates.
sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":")
if self.depcachedir not in sandbox_write:
sandbox_write.append(self.depcachedir)
os.environ["SANDBOX_WRITE"] = \
":".join(filter(None, sandbox_write))
self.porttrees = list(self.settings.repositories.repoLocationList())
# This is used as sanity check for aux_get(). If there is no
# root eclass dir, we assume that PORTDIR is invalid or
# missing. This check allows aux_get() to detect a missing
# portage tree and return early by raising a KeyError.
self._have_root_eclass_dir = os.path.isdir(
os.path.join(self.settings.repositories.mainRepoLocation(),
"eclass"))
#if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
self.xcache = {}
self.frozen = 0
#Keep a list of repo names, sorted by priority (highest priority first).
self._ordered_repo_name_list = tuple(
reversed(self.repositories.prepos_order))
self.auxdbmodule = self.settings.load_best_module(
"portdbapi.auxdbmodule")
self.auxdb = {}
self._pregen_auxdb = {}
# If the current user doesn't have depcachedir write permission,
# then the depcachedir cache is kept here read-only access.
self._ro_auxdb = {}
self._init_cache_dirs()
try:
depcachedir_st = os.stat(self.depcachedir)
depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
except OSError:
depcachedir_st = None
depcachedir_w_ok = False
cache_kwargs = {}
depcachedir_unshared = False
if portage.data.secpass < 1 and \
depcachedir_w_ok and \
depcachedir_st is not None and \
os.getuid() == depcachedir_st.st_uid and \
os.getgid() == depcachedir_st.st_gid:
# If this user owns depcachedir and is not in the
# portage group, then don't bother to set permissions
# on cache entries. This makes it possible to run
# egencache without any need to be a member of the
# portage group.
depcachedir_unshared = True
else:
cache_kwargs.update({'gid': portage_gid, 'perms': 0o664})
# If secpass < 1, we don't want to write to the cache
# since then we won't be able to apply group permissions
# to the cache entries/directories.
if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok:
for x in self.porttrees:
self.auxdb[x] = volatile.database(self.depcachedir, x,
self._known_keys,
**cache_kwargs)
try:
self._ro_auxdb[x] = self.auxdbmodule(self.depcachedir,
x,
self._known_keys,
readonly=True,
**cache_kwargs)
except CacheError:
pass
else:
for x in self.porttrees:
if x in self.auxdb:
continue
# location, label, auxdbkeys
self.auxdb[x] = self.auxdbmodule(self.depcachedir, x,
self._known_keys,
**cache_kwargs)
if "metadata-transfer" not in self.settings.features:
for x in self.porttrees:
if x in self._pregen_auxdb:
continue
cache = self._create_pregen_cache(x)
if cache is not None:
self._pregen_auxdb[x] = cache
# Selectively cache metadata in order to optimize dep matching.
self._aux_cache_keys = set([
"DEPEND", "EAPI", "HDEPEND", "INHERITED", "IUSE", "KEYWORDS",
"LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND",
"repository", "RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"
])
self._aux_cache = {}
self._broken_ebuilds = set()
def _update_recycle_db(self):
start_time = self._config.start_time
recycle_dir = self._config.options.recycle_dir
recycle_db = self._config.recycle_db
r_deletion_delay = self._config.options.recycle_deletion_delay
# Use a dict optimize access.
recycle_db_cache = dict(recycle_db.items())
for filename in os.listdir(recycle_dir):
recycle_file = os.path.join(recycle_dir, filename)
try:
st = os.stat(recycle_file)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
logging.error(("stat failed for '%s' in "
"recycle: %s") % (filename, e))
continue
value = recycle_db_cache.pop(filename, None)
if value is None:
logging.debug(("add '%s' to " "recycle db") % filename)
recycle_db[filename] = (st.st_size, start_time)
else:
r_size, r_time = value
if long(r_size) != st.st_size:
recycle_db[filename] = (st.st_size, start_time)
elif r_time + r_deletion_delay < start_time:
if self._config.options.dry_run:
logging.info(("dry-run: delete '%s' from "
"recycle") % filename)
logging.info(("drop '%s' from "
"recycle db") % filename)
else:
try:
os.unlink(recycle_file)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
logging.error(
("delete '%s' from "
"recycle failed: %s") % (filename, e))
else:
logging.debug(("delete '%s' from "
"recycle") % filename)
try:
del recycle_db[filename]
except KeyError:
pass
else:
logging.debug(("drop '%s' from "
"recycle db") % filename)
# Existing files were popped from recycle_db_cache,
# so any remaining entries are for files that no
# longer exist.
for filename in recycle_db_cache:
try:
del recycle_db[filename]
except KeyError:
pass
else:
logging.debug(("drop non-existent '%s' from "
"recycle db") % filename)
def FindPortdir(settings):
""" Try to figure out what repo we are in and whether we are in a regular
tree or an overlay.
Basic logic is:
1. Determine what directory we are in (supports symlinks).
2. Build a list of directories from / to our current location
3. Iterate over PORTDIR_OVERLAY, if we find a match, search for a profiles directory
in the overlay. If it has one, make it portdir, otherwise make it portdir_overlay.
4. If we didn't find an overlay in PORTDIR_OVERLAY, see if we are in PORTDIR; if so, set
portdir_overlay to PORTDIR. If we aren't in PORTDIR, see if PWD has a profiles dir, if
so, set portdir_overlay and portdir to PWD, else make them False.
5. If we haven't found portdir_overlay yet, it means the user is doing something odd, report
an error.
6. If we haven't found a portdir yet, set portdir to PORTDIR.
Args:
settings - portage.config instance, preferably repoman_settings
Returns:
list(portdir, portdir_overlay, location)
"""
portdir = None
portdir_overlay = None
location = os.getcwd()
pwd = os.environ.get('PWD', '')
if pwd and pwd != location and os.path.realpath(pwd) == location:
# getcwd() returns the canonical path but that makes it hard for repoman to
# orient itself if the user has symlinks in their portage tree structure.
# We use os.environ["PWD"], if available, to get the non-canonical path of
# the current working directory (from the shell).
location = pwd
location = normalize_path(location)
path_ids = {}
p = location
s = None
while True:
s = os.stat(p)
path_ids[(s.st_dev, s.st_ino)] = p
if p == "/":
break
p = os.path.dirname(p)
if location[-1] != "/":
location += "/"
for overlay in settings["PORTDIR_OVERLAY"].split():
overlay = os.path.realpath(overlay)
try:
s = os.stat(overlay)
except OSError:
continue
overlay = path_ids.get((s.st_dev, s.st_ino))
if overlay is None:
continue
if overlay[-1] != "/":
overlay += "/"
if True:
portdir_overlay = overlay
subdir = location[len(overlay):]
if subdir and subdir[-1] != "/":
subdir += "/"
if have_profile_dir(location, subdir.count("/")):
portdir = portdir_overlay
break
# Couldn't match location with anything from PORTDIR_OVERLAY,
# so fall back to have_profile_dir() checks alone. Assume that
# an overlay will contain at least a "repo_name" file while a
# master repo (portdir) will contain at least a "profiles.desc"
# file.
if not portdir_overlay:
portdir_overlay = have_profile_dir(location, filename="repo_name")
if portdir_overlay:
subdir = location[len(portdir_overlay):]
if subdir and subdir[-1] != os.sep:
subdir += os.sep
if have_profile_dir(location, subdir.count(os.sep)):
portdir = portdir_overlay
if not portdir_overlay:
if (settings["PORTDIR"] + os.path.sep).startswith(location):
portdir_overlay = settings["PORTDIR"]
else:
portdir_overlay = have_profile_dir(location)
portdir = portdir_overlay
if not portdir_overlay:
msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
' from the current working directory'
logging.critical(msg)
return (None, None, None)
if not portdir:
portdir = settings["PORTDIR"]
if not portdir_overlay.endswith('/'):
portdir_overlay += '/'
if not portdir.endswith('/'):
portdir += '/'
return [normalize_path(x) for x in (portdir, portdir_overlay, location)]
def _start(self):
if self.config.options.fetch_log_dir is not None and \
not self.config.options.dry_run:
self._log_path = os.path.join(self.config.options.fetch_log_dir,
self.distfile + '.log')
self._previously_added = True
if self.config.distfiles_db is not None and \
self.distfile not in self.config.distfiles_db:
self._previously_added = False
self.config.distfiles_db[self.distfile] = self.cpv
if not self._have_needed_digests():
msg = "incomplete digests: %s" % " ".join(self.digests)
self.scheduler.output(msg,
background=self.background,
log_path=self._log_path)
self.config.log_failure("%s\t%s\t%s" %
(self.cpv, self.distfile, msg))
self.config.file_failures[self.distfile] = self.cpv
self.returncode = os.EX_OK
self._async_wait()
return
st = None
for layout in self.config.layouts:
distfile_path = os.path.join(self.config.options.distfiles,
layout.get_path(self.distfile))
try:
st = os.stat(distfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
msg = "%s stat failed in %s: %s" % \
(self.distfile, "distfiles", e)
self.scheduler.output(msg + '\n',
background=True,
log_path=self._log_path)
logging.error(msg)
else:
break
size_ok = st is not None and st.st_size == self.digests["size"]
if not size_ok:
if self.config.options.dry_run:
if st is not None:
logging.info(
("dry-run: delete '%s' with "
"wrong size from distfiles") % (self.distfile, ))
else:
# Do the unlink in order to ensure that the path is clear,
# even if stat raised ENOENT, since a broken symlink can
# trigger ENOENT.
unlink_success = True
for layout in self.config.layouts:
unlink_path = os.path.join(self.config.options.distfiles,
layout.get_path(self.distfile))
if self._unlink_file(unlink_path, "distfiles"):
if st is not None:
logging.debug(("delete '%s' with "
"wrong size from distfiles") %
(self.distfile, ))
else:
self.config.log_failure("%s\t%s\t%s" %
(self.cpv, self.distfile,
"unlink failed in distfiles"))
unlink_success = False
if not unlink_success:
self.returncode = os.EX_OK
self._async_wait()
return
if size_ok:
if self.config.options.verify_existing_digest:
self._start_task(
FileDigester(file_path=distfile_path,
hash_names=(self._select_hash(), ),
background=self.background,
logfile=self._log_path),
self._distfiles_digester_exit)
return
self._success()
self.returncode = os.EX_OK
self._async_wait()
return
self._start_fetch()
def testEbuildFetch(self):
distfiles = {
'bar': b'bar\n',
'foo': b'foo\n',
}
ebuilds = {
'dev-libs/A-1': {
'EAPI': '7',
'RESTRICT': 'primaryuri',
'SRC_URI': '''{scheme}://{host}:{port}/distfiles/bar.txt -> bar
{scheme}://{host}:{port}/distfiles/foo.txt -> foo''',
},
}
loop = SchedulerInterface(global_event_loop())
scheme = 'http'
host = '127.0.0.1'
content = {}
for k, v in distfiles.items():
content['/distfiles/{}.txt'.format(k)] = v
with AsyncHTTPServer(host, content, loop) as server:
ebuilds_subst = {}
for cpv, metadata in ebuilds.items():
metadata = metadata.copy()
metadata['SRC_URI'] = metadata['SRC_URI'].format(
scheme=scheme, host=host, port=server.server_port)
ebuilds_subst[cpv] = metadata
playground = ResolverPlayground(ebuilds=ebuilds_subst, distfiles=distfiles)
ro_distdir = tempfile.mkdtemp()
try:
fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND'])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is None:
self.skipTest('FETCHCOMMAND not found: {}'.format(playground.settings['FETCHCOMMAND']))
resumecommand = portage.util.shlex_split(playground.settings['RESUMECOMMAND'])
resume_bin = portage.process.find_binary(resumecommand[0])
if resume_bin is None:
self.skipTest('RESUMECOMMAND not found: {}'.format(playground.settings['RESUMECOMMAND']))
root_config = playground.trees[playground.eroot]['root_config']
portdb = root_config.trees["porttree"].dbapi
settings = config(clone=playground.settings)
# Tests only work with one ebuild at a time, so the config
# pool only needs a single config instance.
class config_pool:
@staticmethod
def allocate():
return settings
@staticmethod
def deallocate(settings):
pass
def async_fetch(pkg, ebuild_path):
fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path,
fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop)
fetcher.start()
return fetcher.async_wait()
for cpv in ebuilds:
metadata = dict(zip(Package.metadata_keys,
portdb.aux_get(cpv, Package.metadata_keys)))
pkg = Package(built=False, cpv=cpv, installed=False,
metadata=metadata, root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
ebuild_path = portdb.findname(pkg.cpv)
portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb)
# Test good files in DISTDIR
for k in settings['AA'].split():
os.stat(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test digestgen with fetch
os.unlink(os.path.join(os.path.dirname(ebuild_path), 'Manifest'))
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
with ForkExecutor(loop=loop) as executor:
self.assertTrue(bool(loop.run_until_complete(
loop.run_in_executor(executor, functools.partial(
digestgen, mysettings=settings, myportdb=portdb)))))
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test missing files in DISTDIR
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test empty files in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
pass
self.assertEqual(os.stat(file_path).st_size, 0)
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test non-empty files containing null bytes in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
f.write(len(distfiles[k]) * b'\0')
self.assertEqual(os.stat(file_path).st_size, len(distfiles[k]))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test PORTAGE_RO_DISTDIRS
settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir)
orig_fetchcommand = settings['FETCHCOMMAND']
orig_resumecommand = settings['RESUMECOMMAND']
try:
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.rename(file_path, os.path.join(ro_distdir, k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
self.assertTrue(os.path.islink(file_path))
with open(file_path, 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
os.unlink(file_path)
finally:
settings.pop('PORTAGE_RO_DISTDIRS')
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test local filesystem in GENTOO_MIRRORS
orig_mirrors = settings['GENTOO_MIRRORS']
orig_fetchcommand = settings['FETCHCOMMAND']
try:
settings['GENTOO_MIRRORS'] = ro_distdir
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['GENTOO_MIRRORS'] = orig_mirrors
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test readonly DISTDIR
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
try:
os.chmod(settings['DISTDIR'], 0o555)
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
os.chmod(settings['DISTDIR'], orig_distdir_mode)
# Test parallel-fetch mode
settings['PORTAGE_PARALLEL_FETCHONLY'] = '1'
try:
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings.pop('PORTAGE_PARALLEL_FETCHONLY')
# Test RESUMECOMMAND
orig_resume_min_size = settings['PORTAGE_FETCH_RESUME_MIN_SIZE']
try:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2'
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.unlink(file_path)
with open(file_path + _download_suffix, 'wb') as f:
f.write(distfiles[k][:2])
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size
finally:
shutil.rmtree(ro_distdir)
playground.cleanup()
def _check_already_fetched(self, settings, uri_map):
digests = self._get_digests()
distdir = settings["DISTDIR"]
allow_missing = self._get_manifest().allow_missing
for filename in uri_map:
# Use stat rather than lstat since fetch() creates
# symlinks when PORTAGE_RO_DISTDIRS is used.
try:
st = os.stat(os.path.join(distdir, filename))
except OSError:
return False
if st.st_size == 0:
return False
expected_size = digests.get(filename, {}).get('size')
if expected_size is None:
continue
if st.st_size != expected_size:
return False
hash_filter = _hash_filter(settings.get("PORTAGE_CHECKSUM_FILTER", ""))
if hash_filter.transparent:
hash_filter = None
stdout_orig = sys.stdout
stderr_orig = sys.stderr
global_havecolor = portage.output.havecolor
out = io.StringIO()
eout = portage.output.EOutput()
eout.quiet = settings.get("PORTAGE_QUIET") == "1"
success = True
try:
sys.stdout = out
sys.stderr = out
if portage.output.havecolor:
portage.output.havecolor = not self.background
for filename in uri_map:
mydigests = digests.get(filename)
if mydigests is None:
if not allow_missing:
success = False
break
continue
ok, st = _check_distfile(os.path.join(distdir, filename),
mydigests,
eout,
show_errors=False,
hash_filter=hash_filter)
if not ok:
success = False
break
except portage.exception.FileNotFound:
# A file disappeared unexpectedly.
return False
finally:
sys.stdout = stdout_orig
sys.stderr = stderr_orig
portage.output.havecolor = global_havecolor
if success:
# When returning unsuccessfully, no messages are produced, since
# we assume that a fetcher process will later be executed in order
# to produce such messages.
msg = out.getvalue()
if msg:
self.scheduler.output(msg, log_path=self.logfile)
return success
def pre_sync(self, repo):
msg = ">>> Syncing repository '%s' into '%s'..." % (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
try:
st = os.stat(repo.location)
except OSError:
st = None
self.usersync_uid = None
spawn_kwargs = {}
# Redirect command stderr to stdout, in order to prevent
# spurious cron job emails (bug 566132).
spawn_kwargs["fd_pipes"] = {
0: portage._get_stdin().fileno(),
1: sys.__stdout__.fileno(),
2: sys.__stdout__.fileno(),
}
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):
user = None
group = None
home = None
logname = None
spl = sync_user.split(":", 1)
if spl[0]:
username = spl[0]
try:
try:
pw = pwd.getpwnam(username)
except KeyError:
pw = pwd.getpwuid(int(username))
except (ValueError, KeyError):
writemsg(
"!!! User '%s' invalid or does not exist\n" % username,
noiselevel=-1,
)
return (logname, user, group, home)
user = pw.pw_uid
group = pw.pw_gid
home = pw.pw_dir
logname = pw.pw_name
if len(spl) > 1:
groupname = spl[1]
try:
try:
gp = grp.getgrnam(groupname)
except KeyError:
pw = grp.getgrgid(int(groupname))
except (ValueError, KeyError):
writemsg(
"!!! Group '%s' invalid or does not exist\n" % groupname,
noiselevel=-1,
)
return (logname, user, group, home)
group = gp.gr_gid
return (logname, user, group, home)
# user or user:group
(logname, uid, gid, home) = get_sync_user_data(repo.sync_user)
if uid is not None:
spawn_kwargs["uid"] = uid
self.usersync_uid = uid
if gid is not None:
spawn_kwargs["gid"] = gid
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
if logname is not None:
spawn_kwargs["env"]["LOGNAME"] = logname
if st is None:
perms = {"mode": 0o755}
# respect sync-user if set
if "umask" in spawn_kwargs:
perms["mode"] &= ~spawn_kwargs["umask"]
if "uid" in spawn_kwargs:
perms["uid"] = spawn_kwargs["uid"]
if "gid" in spawn_kwargs:
perms["gid"] = spawn_kwargs["gid"]
portage.util.ensure_dirs(repo.location, **perms)
st = os.stat(repo.location)
if (
repo.sync_user is None
and "usersync" in self.settings.features
and portage.data.secpass >= 2
and (
st.st_uid != os.getuid()
and st.st_mode & 0o700
or st.st_gid != os.getgid()
and st.st_mode & 0o070
)
):
try:
pw = pwd.getpwuid(st.st_uid)
except KeyError:
pass
else:
# Drop privileges when syncing, in order to match
# existing uid/gid settings.
self.usersync_uid = st.st_uid
spawn_kwargs["uid"] = st.st_uid
spawn_kwargs["gid"] = st.st_gid
spawn_kwargs["groups"] = [st.st_gid]
spawn_kwargs["env"]["HOME"] = pw.pw_dir
spawn_kwargs["env"]["LOGNAME"] = pw.pw_name
umask = 0o002
if not st.st_mode & 0o020:
umask = umask | 0o020
spawn_kwargs["umask"] = umask
# override the defaults when sync_umask is set
if repo.sync_umask is not None:
spawn_kwargs["umask"] = int(repo.sync_umask, 8)
spawn_kwargs.setdefault("umask", 0o022)
self.spawn_kwargs = spawn_kwargs
if self.usersync_uid is not None:
# PORTAGE_TMPDIR is used below, so validate it and
# bail out if necessary.
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
return rval
os.umask(0o022)
return os.EX_OK
def check_isebuild(self, **kwargs):
'''Test the file for qualifications that is is an ebuild
@param checkdirlist: list of files in the current package directory
@param checkdir: current package directory path
@param xpkg: current package directory being checked
@param validity_future: Future instance
@returns: dictionary, including {pkgs, can_force}
'''
checkdirlist = kwargs.get('checkdirlist').get()
checkdir = kwargs.get('checkdir')
xpkg = kwargs.get('xpkg')
fuse = kwargs.get('validity_future')
can_force = kwargs.get('can_force')
self.continue_ = False
ebuildlist = []
pkgs = {}
for y in checkdirlist:
file_is_ebuild = y.endswith(".ebuild")
file_should_be_non_executable = (
y in self.repo_settings.qadata.no_exec or file_is_ebuild)
if file_should_be_non_executable:
file_is_executable = stat.S_IMODE(
os.stat(os.path.join(checkdir, y)).st_mode) & 0o111
if file_is_executable:
self.qatracker.add_error("file.executable",
os.path.join(checkdir, y))
if file_is_ebuild:
pf = y[:-7]
ebuildlist.append(pf)
catdir = xpkg.split("/")[0]
cpv = "%s/%s" % (catdir, pf)
allvars = self.repo_settings.qadata.allvars
try:
myaux = dict(
zip(allvars, self.portdb.aux_get(cpv, allvars)))
except KeyError:
fuse.set(False, ignore_InvalidState=True)
self.qatracker.add_error("ebuild.syntax",
os.path.join(xpkg, y))
continue
except IOError:
fuse.set(False, ignore_InvalidState=True)
self.qatracker.add_error("ebuild.output",
os.path.join(xpkg, y))
continue
except InvalidPackageName:
fuse.set(False, ignore_InvalidState=True)
self.qatracker.add_error("ebuild.invalidname",
os.path.join(xpkg, y))
continue
if not portage.eapi_is_supported(myaux["EAPI"]):
fuse.set(False, ignore_InvalidState=True)
self.qatracker.add_error("EAPI.unsupported",
os.path.join(xpkg, y))
continue
pkgs[pf] = Package(cpv=cpv,
metadata=myaux,
root_config=self.root_config,
type_name="ebuild")
if len(pkgs) != len(ebuildlist):
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
# Do not try to do any more QA checks on this package since missing
# metadata leads to false positives for several checks, and false
# positives confuse users.
self.continue_ = True
can_force.set(False, ignore_InvalidState=True)
self.pkgs = pkgs
# set our updated data
dyn_pkgs = kwargs.get('pkgs')
dyn_pkgs.set(pkgs)
return self.continue_
def getsize(filename):
size = os.stat(filename).st_size
return (size, size)
def _testEbuildFetch(
self,
loop,
scheme,
host,
orig_distfiles,
ebuilds,
content,
server,
playground,
ro_distdir,
):
mirror_layouts = (
(
"[structure]",
"0=filename-hash BLAKE2B 8",
"1=flat",
),
(
"[structure]",
"1=filename-hash BLAKE2B 8",
"0=flat",
),
(
"[structure]",
"0=content-hash SHA512 8:8:8",
"1=flat",
),
)
fetchcommand = portage.util.shlex_split(
playground.settings["FETCHCOMMAND"])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is None:
self.skipTest("FETCHCOMMAND not found: {}".format(
playground.settings["FETCHCOMMAND"]))
eubin = os.path.join(playground.eprefix, "usr", "bin")
os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin)))
resumecommand = portage.util.shlex_split(
playground.settings["RESUMECOMMAND"])
resume_bin = portage.process.find_binary(resumecommand[0])
if resume_bin is None:
self.skipTest("RESUMECOMMAND not found: {}".format(
playground.settings["RESUMECOMMAND"]))
if resume_bin != fetch_bin:
os.symlink(resume_bin,
os.path.join(eubin, os.path.basename(resume_bin)))
root_config = playground.trees[playground.eroot]["root_config"]
portdb = root_config.trees["porttree"].dbapi
def run_async(func, *args, **kwargs):
with ForkExecutor(loop=loop) as executor:
return loop.run_until_complete(
loop.run_in_executor(
executor, functools.partial(func, *args, **kwargs)))
for layout_lines in mirror_layouts:
settings = config(clone=playground.settings)
layout_data = "".join("{}\n".format(line) for line in layout_lines)
mirror_conf = MirrorLayoutConfig()
mirror_conf.read_from_file(io.StringIO(layout_data))
layouts = mirror_conf.get_all_layouts()
content["/distfiles/layout.conf"] = layout_data.encode("utf8")
distfiles = {}
for k, v in orig_distfiles.items():
filename = DistfileName(
k,
digests=dict((algo, checksum_str(v, hashname=algo))
for algo in MANIFEST2_HASH_DEFAULTS),
)
distfiles[filename] = v
# mirror path
for layout in layouts:
content["/distfiles/" + layout.get_path(filename)] = v
# upstream path
content["/distfiles/{}.txt".format(k)] = v
shutil.rmtree(settings["DISTDIR"])
os.makedirs(settings["DISTDIR"])
with open(os.path.join(settings['DISTDIR'], 'layout.conf'),
'wt') as f:
f.write(layout_data)
if any(
isinstance(layout, ContentHashLayout)
for layout in layouts):
content_db = os.path.join(playground.eprefix,
'var/db/emirrordist/content.db')
os.makedirs(os.path.dirname(content_db), exist_ok=True)
try:
os.unlink(content_db)
except OSError:
pass
else:
content_db = None
# Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given.
foo_uri = {
'foo': ('{scheme}://{host}:{port}/distfiles/foo'.format(
scheme=scheme, host=host, port=server.server_port), )
}
foo_path = os.path.join(settings['DISTDIR'], 'foo')
foo_stale_content = b'stale content\n'
with open(foo_path, 'wb') as f:
f.write(b'stale content\n')
self.assertTrue(
bool(run_async(fetch, foo_uri, settings, try_mirrors=False)))
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), foo_stale_content)
with open(foo_path, 'rb') as f:
self.assertNotEqual(f.read(), distfiles['foo'])
# Use force=True to update the stale file.
self.assertTrue(
bool(
run_async(fetch,
foo_uri,
settings,
try_mirrors=False,
force=True)))
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), distfiles['foo'])
# Test force=True with FEATURES=skiprocheck, using read-only DISTDIR.
# FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that
# FETCHCOMMAND must perform atomic rename itself due to read-only
# DISTDIR.
with open(foo_path, 'wb') as f:
f.write(b'stale content\n')
orig_fetchcommand = settings['FETCHCOMMAND']
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
temp_fetchcommand = os.path.join(eubin, 'fetchcommand')
with open(temp_fetchcommand, 'w') as f:
f.write("""
set -e
URI=$1
DISTDIR=$2
FILE=$3
trap 'chmod a-w "${DISTDIR}"' EXIT
chmod ug+w "${DISTDIR}"
%s
mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}"
""" % orig_fetchcommand.replace('${FILE}', '${FILE}.__download__'))
settings[
'FETCHCOMMAND'] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % (
BASH_BINARY, temp_fetchcommand)
settings.features.add('skiprocheck')
settings.features.remove('distlocks')
os.chmod(settings['DISTDIR'], 0o555)
try:
self.assertTrue(
bool(
run_async(fetch,
foo_uri,
settings,
try_mirrors=False,
force=True)))
finally:
settings['FETCHCOMMAND'] = orig_fetchcommand
os.chmod(settings['DISTDIR'], orig_distdir_mode)
settings.features.remove('skiprocheck')
settings.features.add('distlocks')
os.unlink(temp_fetchcommand)
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), distfiles['foo'])
# Test emirrordist invocation.
emirrordist_cmd = (portage._python_interpreter, '-b', '-Wd',
os.path.join(self.bindir, 'emirrordist'),
'--distfiles', settings['DISTDIR'],
'--config-root', settings['EPREFIX'],
'--delete', '--repositories-configuration',
settings.repositories.config_string(), '--repo',
'test_repo', '--mirror')
if content_db is not None:
emirrordist_cmd = emirrordist_cmd + (
'--content-db',
content_db,
)
env = settings.environ()
env['PYTHONPATH'] = ':'.join(
filter(None, [PORTAGE_PYM_PATH] +
os.environ.get('PYTHONPATH', '').split(':')))
for k in distfiles:
try:
os.unlink(os.path.join(settings['DISTDIR'], k))
except OSError:
pass
proc = loop.run_until_complete(
asyncio.create_subprocess_exec(*emirrordist_cmd, env=env))
self.assertEqual(loop.run_until_complete(proc.wait()), 0)
for k in distfiles:
with open(
os.path.join(settings['DISTDIR'],
layouts[0].get_path(k)), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
if content_db is not None:
loop.run_until_complete(
self._test_content_db(
emirrordist_cmd,
env,
layouts,
content_db,
distfiles,
settings,
portdb,
))
# Tests only work with one ebuild at a time, so the config
# pool only needs a single config instance.
class config_pool:
@staticmethod
def allocate():
return settings
@staticmethod
def deallocate(settings):
pass
def async_fetch(pkg, ebuild_path):
fetcher = EbuildFetcher(config_pool=config_pool,
ebuild_path=ebuild_path,
fetchonly=False,
fetchall=True,
pkg=pkg,
scheduler=loop)
fetcher.start()
return fetcher.async_wait()
for cpv in ebuilds:
metadata = dict(
zip(Package.metadata_keys,
portdb.aux_get(cpv, Package.metadata_keys)))
pkg = Package(built=False,
cpv=cpv,
installed=False,
metadata=metadata,
root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
ebuild_path = portdb.findname(pkg.cpv)
portage.doebuild_environment(ebuild_path,
'fetch',
settings=settings,
db=portdb)
# Test good files in DISTDIR
for k in settings['AA'].split():
os.stat(os.path.join(settings['DISTDIR'], k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test digestgen with fetch
os.unlink(
os.path.join(os.path.dirname(ebuild_path), 'Manifest'))
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
with ForkExecutor(loop=loop) as executor:
self.assertTrue(
bool(
loop.run_until_complete(
loop.run_in_executor(
executor,
functools.partial(digestgen,
mysettings=settings,
myportdb=portdb)))))
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test missing files in DISTDIR
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test empty files in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
pass
self.assertEqual(os.stat(file_path).st_size, 0)
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test non-empty files containing null bytes in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
f.write(len(distfiles[k]) * b'\0')
self.assertEqual(
os.stat(file_path).st_size, len(distfiles[k]))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test PORTAGE_RO_DISTDIRS
settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir)
orig_fetchcommand = settings['FETCHCOMMAND']
orig_resumecommand = settings['RESUMECOMMAND']
try:
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.rename(file_path, os.path.join(ro_distdir, k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
self.assertTrue(os.path.islink(file_path))
with open(file_path, 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
os.unlink(file_path)
finally:
settings.pop('PORTAGE_RO_DISTDIRS')
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test local filesystem in GENTOO_MIRRORS
orig_mirrors = settings['GENTOO_MIRRORS']
orig_fetchcommand = settings['FETCHCOMMAND']
try:
settings['GENTOO_MIRRORS'] = ro_distdir
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k),
'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['GENTOO_MIRRORS'] = orig_mirrors
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test readonly DISTDIR
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
try:
os.chmod(settings['DISTDIR'], 0o555)
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k),
'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
os.chmod(settings['DISTDIR'], orig_distdir_mode)
# Test parallel-fetch mode
settings['PORTAGE_PARALLEL_FETCHONLY'] = '1'
try:
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k),
'rb') as f:
self.assertEqual(f.read(), distfiles[k])
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k),
'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings.pop('PORTAGE_PARALLEL_FETCHONLY')
# Test RESUMECOMMAND
orig_resume_min_size = settings[
'PORTAGE_FETCH_RESUME_MIN_SIZE']
try:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2'
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.unlink(file_path)
with open(file_path + _download_suffix, 'wb') as f:
f.write(distfiles[k][:2])
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k),
'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings[
'PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size
# Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR
orig_fetchcommand = settings['FETCHCOMMAND']
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
try:
os.chmod(settings['DISTDIR'], 0o555)
settings[
'FETCHCOMMAND'] = '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"' % (
BASH_BINARY, orig_fetchcommand.replace('"', '\\"'))
settings.features.add('skiprocheck')
settings.features.remove('distlocks')
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
finally:
settings['FETCHCOMMAND'] = orig_fetchcommand
os.chmod(settings['DISTDIR'], orig_distdir_mode)
settings.features.remove('skiprocheck')
settings.features.add('distlocks')
def process(mysettings, key, logentries, fulltext):
if mysettings.get("PORTAGE_LOGDIR"):
logdir = normalize_path(mysettings["PORTAGE_LOGDIR"])
else:
logdir = os.path.join(
os.sep, mysettings["EPREFIX"].lstrip(os.sep), "var", "log", "portage"
)
if not os.path.isdir(logdir):
# Only initialize group/mode if the directory doesn't
# exist, so that we don't override permissions if they
# were previously set by the administrator.
# NOTE: These permissions should be compatible with our
# default logrotate config as discussed in bug 374287.
uid = -1
if portage.data.secpass >= 2:
uid = portage_uid
ensure_dirs(logdir, uid=uid, gid=portage_gid, mode=0o2770)
cat, pf = portage.catsplit(key)
elogfilename = (
pf
+ ":"
+ _unicode_decode(
time.strftime("%Y%m%d-%H%M%S", time.gmtime(time.time())),
encoding=_encodings["content"],
errors="replace",
)
+ ".log"
)
if "split-elog" in mysettings.features:
log_subdir = os.path.join(logdir, "elog", cat)
elogfilename = os.path.join(log_subdir, elogfilename)
else:
log_subdir = os.path.join(logdir, "elog")
elogfilename = os.path.join(log_subdir, cat + ":" + elogfilename)
_ensure_log_subdirs(logdir, log_subdir)
try:
with io.open(
_unicode_encode(elogfilename, encoding=_encodings["fs"], errors="strict"),
mode="w",
encoding=_encodings["content"],
errors="backslashreplace",
) as elogfile:
elogfile.write(_unicode_decode(fulltext))
except IOError as e:
func_call = "open('%s', 'w')" % elogfilename
if e.errno == errno.EACCES:
raise portage.exception.PermissionDenied(func_call)
elif e.errno == errno.EPERM:
raise portage.exception.OperationNotPermitted(func_call)
elif e.errno == errno.EROFS:
raise portage.exception.ReadOnlyFileSystem(func_call)
else:
raise
# Copy group permission bits from parent directory.
elogdir_st = os.stat(log_subdir)
elogdir_gid = elogdir_st.st_gid
elogdir_grp_mode = 0o060 & elogdir_st.st_mode
# Copy the uid from the parent directory if we have privileges
# to do so, for compatibility with our default logrotate
# config (see bug 378451). With the "su portage portage"
# directive and logrotate-3.8.0, logrotate's chown call during
# the compression phase will only succeed if the log file's uid
# is portage_uid.
logfile_uid = -1
if portage.data.secpass >= 2:
logfile_uid = elogdir_st.st_uid
apply_permissions(
elogfilename, uid=logfile_uid, gid=elogdir_gid, mode=elogdir_grp_mode, mask=0
)
return elogfilename
def collect_libraries_from_dir(dirs, mask, logger):
''' Collects all libraries from specified list of directories.
mask is list of pathes, that are ommited in scanning, can be eighter single file or entire directory
Returns tuple composed of: list of libraries, list of symlinks, and toupe with pair
(symlink_id, library_id) for resolving dependencies
'''
# contains list of directories found
# allows us to reduce number of fnc calls
found_directories = set()
found_files = set()
found_symlinks = set()
found_la_files = set() # la libraries
for _dir in dirs:
if _dir in mask:
continue
try:
for _listing in os.listdir(_dir):
listing = os.path.join(_dir, _listing)
if listing in mask or _listing in mask:
continue
if os.path.isdir(listing):
if os.path.islink(listing):
#we do not want scan symlink-directories
pass
else:
found_directories.add(listing)
elif os.path.isfile(listing):
if (listing.endswith('.so') or
listing.endswith('.a') or
'.so.' in listing
):
if os.path.islink(listing):
found_symlinks.add(listing)
else:
found_files.add(listing)
continue
elif listing.endswith('.la'):
if listing in found_la_files:
continue
found_la_files.add(listing)
else:
# sometimes there are binaries in libs' subdir,
# for example in nagios
if not os.path.islink(listing):
#if listing in found_files or listing in found_symlinks:
#continue
prv = os.stat(listing)[stat.ST_MODE]
if prv & stat.S_IXUSR == stat.S_IXUSR or \
prv & stat.S_IXGRP == stat.S_IXGRP or \
prv & stat.S_IXOTH == stat.S_IXOTH:
found_files.add(listing)
except Exception as ex:
logger.debug('\t' +
yellow('Exception collecting libraries: ' +
blue('%s') %str(ex)))
if found_directories:
_file, la_file, link = \
collect_libraries_from_dir(found_directories, mask, logger)
found_files.update(_file)
found_la_files.update(la_file)
found_symlinks.update(link)
return (found_files, found_la_files, found_symlinks)
def digestgen(myarchives=None, mysettings=None, myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@return: 1 on success and 0 on failure
"""
if mysettings is None or myportdb is None:
raise TypeError("portage.digestgen(): 'mysettings' and 'myportdb' parameter are required.")
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
try:
mf = mysettings.repositories.get_repo_for_location(mytree)
except KeyError:
# backward compatibility
mytree = os.path.realpath(mytree)
mf = mysettings.repositories.get_repo_for_location(mytree)
mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict)
if not mf.allow_create:
writemsg_stdout(_(">>> Skipping creating Manifest for %s; "
"repository is configured to not use them\n") % mysettings["O"])
return 1
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.add(MANIFEST2_REQUIRED_HASH)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
if missing_files:
for myfile in missing_files:
uris = set()
all_restrict = set()
for cpv in distfiles_map[myfile]:
uris.update(myportdb.getFetchMap(
cpv, mytree=mytree)[myfile])
restrict = myportdb.aux_get(cpv, ['RESTRICT'],
mytree=mytree)[0]
# Here we ignore conditional parts of RESTRICT since
# they don't apply unconditionally. Assume such
# conditionals only apply on the client side where
# digestgen() does not need to be called.
all_restrict.update(use_reduce(restrict,
flat=True, matchnone=True))
# fetch() uses CATEGORY and PF to display a message
# when fetch restriction is triggered.
cat, pf = catsplit(cpv)
mysettings["CATEGORY"] = cat
mysettings["PF"] = pf
# fetch() uses PORTAGE_RESTRICT to control fetch
# restriction, which is only applied to files that
# are not fetchable via a mirror:// URI.
mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict)
try:
st = os.stat(os.path.join(
mysettings["DISTDIR"],myfile))
except OSError:
st = None
if not fetch({myfile : uris}, mysettings):
myebuild = os.path.join(mysettings["O"],
catsplit(cpv)[1] + ".ebuild")
spawn_nofetch(myportdb, myebuild)
writemsg(_("!!! Fetch failed for %s, can't update "
"Manifest\n") % myfile, noiselevel=-1)
if myfile in dist_hashes and \
st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
writemsg(_("!!! If you would like to "
"forcefully replace the existing "
"Manifest entry\n!!! for %s, use "
"the following command:\n") % myfile + \
"!!! " + colorize("INFORM",
"ebuild --force %s manifest" % \
os.path.basename(myebuild)) + "\n",
noiselevel=-1)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True,
assumeDistHashesAlways=(
"assume-digests" in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update "
"Manifest\n") % e, noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e,), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(
os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" + colorize("WARN",
str(len(auto_assumed)).rjust(18)) + "\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(
" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1
def hardlink_is_mine(link, lock):
try:
return os.stat(link).st_nlink == 2
except OSError:
return False
def lockfile(mypath,
wantnewlockfile=0,
unlinkfile=0,
waiting_msg=None,
flags=0):
"""
If wantnewlockfile is True then this creates a lockfile in the parent
directory as the file: '.' + basename + '.portage_lockfile'.
"""
import fcntl
if not mypath:
raise InvalidData(_("Empty path given"))
if isinstance(mypath, basestring) and mypath[-1] == '/':
mypath = mypath[:-1]
if hasattr(mypath, 'fileno'):
mypath = mypath.fileno()
if isinstance(mypath, int):
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
del base, tail
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, basestring):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
try:
myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
else:
raise
if not preexisting:
try:
if os.stat(lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
return lockfile(mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile,
waiting_msg=waiting_msg,
flags=flags)
else:
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = fcntl.lockf
try:
fcntl.lockf(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if "errno" not in dir(e):
raise
if e.errno in (errno.EACCES, errno.EAGAIN):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
raise TryAgain(mypath)
global _quiet
out = EOutput()
out.quiet = _quiet
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s\n") % lockfilename
out.ebegin(waiting_msg)
# try for the exclusive lock now.
try:
fcntl.lockf(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
out.eend(1, str(e))
raise
out.eend(os.EX_OK)
elif e.errno == errno.ENOLCK:
# We're not allowed to lock on this FS.
os.close(myfd)
link_success = False
if lockfilename == str(lockfilename):
if wantnewlockfile:
try:
if os.stat(lockfilename)[stat.ST_NLINK] == 1:
os.unlink(lockfilename)
except OSError:
pass
link_success = hardlink_lockfile(lockfilename)
if not link_success:
raise
locking_method = None
myfd = HARDLINK_FD
else:
raise
if isinstance(lockfilename, basestring) and \
myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0:
# The file was deleted on us... Keep trying to make one...
os.close(myfd)
writemsg(_("lockfile recurse\n"), 1)
lockfilename, myfd, unlinkfile, locking_method = lockfile(
mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile,
waiting_msg=waiting_msg,
flags=flags)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
def _env_update(makelinks, target_root, prev_mtimes, contents, env,
writemsg_level):
if writemsg_level is None:
writemsg_level = portage.util.writemsg_level
if target_root is None:
target_root = portage.settings["ROOT"]
if prev_mtimes is None:
prev_mtimes = portage.mtimedb["ldpath"]
if env is None:
settings = portage.settings
else:
settings = env
eprefix = settings.get("EPREFIX", "")
eprefix_lstrip = eprefix.lstrip(os.sep)
eroot = normalize_path(os.path.join(target_root, eprefix_lstrip)).rstrip(
os.sep) + os.sep
envd_dir = os.path.join(eroot, "etc", "env.d")
ensure_dirs(envd_dir, mode=0o755)
fns = listdir(envd_dir, EmptyOnError=1)
fns.sort()
templist = []
for x in fns:
if len(x) < 3:
continue
if not x[0].isdigit() or not x[1].isdigit():
continue
if x.startswith(".") or x.endswith("~") or x.endswith(".bak"):
continue
templist.append(x)
fns = templist
del templist
space_separated = set(["CONFIG_PROTECT", "CONFIG_PROTECT_MASK"])
colon_separated = set([
"ADA_INCLUDE_PATH", "ADA_OBJECTS_PATH", "CLASSPATH", "INFODIR",
"INFOPATH", "KDEDIRS", "LDPATH", "MANPATH", "PATH", "PKG_CONFIG_PATH",
"PRELINK_PATH", "PRELINK_PATH_MASK", "PYTHONPATH", "ROOTPATH"
])
config_list = []
for x in fns:
file_path = os.path.join(envd_dir, x)
try:
myconfig = getconfig(file_path, expand=False)
except ParseError as e:
writemsg("!!! '%s'\n" % str(e), noiselevel=-1)
del e
continue
if myconfig is None:
# broken symlink or file removed by a concurrent process
writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1)
continue
config_list.append(myconfig)
if "SPACE_SEPARATED" in myconfig:
space_separated.update(myconfig["SPACE_SEPARATED"].split())
del myconfig["SPACE_SEPARATED"]
if "COLON_SEPARATED" in myconfig:
colon_separated.update(myconfig["COLON_SEPARATED"].split())
del myconfig["COLON_SEPARATED"]
env = {}
specials = {}
for var in space_separated:
mylist = []
for myconfig in config_list:
if var in myconfig:
for item in myconfig[var].split():
if item and not item in mylist:
mylist.append(item)
del myconfig[var] # prepare for env.update(myconfig)
if mylist:
env[var] = " ".join(mylist)
specials[var] = mylist
for var in colon_separated:
mylist = []
for myconfig in config_list:
if var in myconfig:
for item in myconfig[var].split(":"):
if item and not item in mylist:
mylist.append(item)
del myconfig[var] # prepare for env.update(myconfig)
if mylist:
env[var] = ":".join(mylist)
specials[var] = mylist
for myconfig in config_list:
"""Cumulative variables have already been deleted from myconfig so that
they won't be overwritten by this dict.update call."""
env.update(myconfig)
ldsoconf_path = os.path.join(eroot, "etc", "ld.so.conf")
try:
myld = io.open(_unicode_encode(ldsoconf_path,
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['content'],
errors='replace')
myldlines = myld.readlines()
myld.close()
oldld = []
for x in myldlines:
#each line has at least one char (a newline)
if x[:1] == "#":
continue
oldld.append(x[:-1])
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
oldld = None
newld = specials["LDPATH"]
if oldld != newld:
#ld.so.conf needs updating and ldconfig needs to be run
myfd = atomic_ofstream(ldsoconf_path)
myfd.write(
"# ld.so.conf autogenerated by env-update; make all changes to\n")
myfd.write("# contents of /etc/env.d directory\n")
for x in specials["LDPATH"]:
myfd.write(x + "\n")
myfd.close()
potential_lib_dirs = set()
for lib_dir_glob in ('usr/lib*', 'lib*'):
x = os.path.join(eroot, lib_dir_glob)
for y in glob.glob(
_unicode_encode(x, encoding=_encodings['fs'],
errors='strict')):
try:
y = _unicode_decode(y,
encoding=_encodings['fs'],
errors='strict')
except UnicodeDecodeError:
continue
if os.path.basename(y) != 'libexec':
potential_lib_dirs.add(y[len(eroot):])
# Update prelink.conf if we are prelink-enabled
if prelink_capable:
prelink_d = os.path.join(eroot, 'etc', 'prelink.conf.d')
ensure_dirs(prelink_d)
newprelink = atomic_ofstream(os.path.join(prelink_d, 'portage.conf'))
newprelink.write(
"# prelink.conf autogenerated by env-update; make all changes to\n"
)
newprelink.write("# contents of /etc/env.d directory\n")
for x in sorted(potential_lib_dirs) + ['bin', 'sbin']:
newprelink.write('-l /%s\n' % (x, ))
prelink_paths = set()
prelink_paths |= set(specials.get('LDPATH', []))
prelink_paths |= set(specials.get('PATH', []))
prelink_paths |= set(specials.get('PRELINK_PATH', []))
prelink_path_mask = specials.get('PRELINK_PATH_MASK', [])
for x in prelink_paths:
if not x:
continue
if x[-1:] != '/':
x += "/"
plmasked = 0
for y in prelink_path_mask:
if not y:
continue
if y[-1] != '/':
y += "/"
if y == x[0:len(y)]:
plmasked = 1
break
if not plmasked:
newprelink.write("-h %s\n" % (x, ))
for x in prelink_path_mask:
newprelink.write("-b %s\n" % (x, ))
newprelink.close()
# Migration code path. If /etc/prelink.conf was generated by us, then
# point it to the new stuff until the prelink package re-installs.
prelink_conf = os.path.join(eroot, 'etc', 'prelink.conf')
try:
with open(
_unicode_encode(prelink_conf,
encoding=_encodings['fs'],
errors='strict'), 'rb') as f:
if f.readline(
) == b'# prelink.conf autogenerated by env-update; make all changes to\n':
f = atomic_ofstream(prelink_conf)
f.write('-c /etc/prelink.conf.d/*.conf\n')
f.close()
except IOError as e:
if e.errno != errno.ENOENT:
raise
current_time = int(time.time())
mtime_changed = False
lib_dirs = set()
for lib_dir in set(specials['LDPATH']) | potential_lib_dirs:
x = os.path.join(eroot, lib_dir.lstrip(os.sep))
try:
newldpathtime = os.stat(x)[stat.ST_MTIME]
lib_dirs.add(normalize_path(x))
except OSError as oe:
if oe.errno == errno.ENOENT:
try:
del prev_mtimes[x]
except KeyError:
pass
# ignore this path because it doesn't exist
continue
raise
if newldpathtime == current_time:
# Reset mtime to avoid the potential ambiguity of times that
# differ by less than 1 second.
newldpathtime -= 1
os.utime(x, (newldpathtime, newldpathtime))
prev_mtimes[x] = newldpathtime
mtime_changed = True
elif x in prev_mtimes:
if prev_mtimes[x] == newldpathtime:
pass
else:
prev_mtimes[x] = newldpathtime
mtime_changed = True
else:
prev_mtimes[x] = newldpathtime
mtime_changed = True
if makelinks and \
not mtime_changed and \
contents is not None:
libdir_contents_changed = False
for mypath, mydata in contents.items():
if mydata[0] not in ("obj", "sym"):
continue
head, tail = os.path.split(mypath)
if head in lib_dirs:
libdir_contents_changed = True
break
if not libdir_contents_changed:
makelinks = False
if "CHOST" in settings and "CBUILD" in settings and \
settings["CHOST"] != settings["CBUILD"]:
ldconfig = find_binary("%s-ldconfig" % settings["CHOST"])
else:
ldconfig = os.path.join(eroot, "sbin", "ldconfig")
if ldconfig is None:
pass
elif not (os.access(ldconfig, os.X_OK) and os.path.isfile(ldconfig)):
ldconfig = None
# Only run ldconfig as needed
if makelinks and ldconfig:
# ldconfig has very different behaviour between FreeBSD and Linux
if ostype == "Linux" or ostype.lower().endswith("gnu"):
# We can't update links if we haven't cleaned other versions first, as
# an older package installed ON TOP of a newer version will cause ldconfig
# to overwrite the symlinks we just made. -X means no links. After 'clean'
# we can safely create links.
writemsg_level(_(">>> Regenerating %setc/ld.so.cache...\n") % \
(target_root,))
os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root))
elif ostype in ("FreeBSD", "DragonFly"):
writemsg_level(_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") % \
target_root)
os.system(("cd / ; %s -elf -i " + \
"-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'") % \
(ldconfig, target_root, target_root))
del specials["LDPATH"]
notice = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n"
notice += "# DO NOT EDIT THIS FILE."
penvnotice = notice + " CHANGES TO STARTUP PROFILES\n"
cenvnotice = penvnotice[:]
penvnotice += "# GO INTO /etc/profile NOT /etc/profile.env\n\n"
cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n"
#create /etc/profile.env for bash support
profile_env_path = os.path.join(eroot, "etc", "profile.env")
outfile = atomic_ofstream(profile_env_path)
outfile.write(penvnotice)
env_keys = [x for x in env if x != "LDPATH"]
env_keys.sort()
for k in env_keys:
v = env[k]
if v.startswith('$') and not v.startswith('${'):
outfile.write("export %s=$'%s'\n" % (k, v[1:]))
else:
outfile.write("export %s='%s'\n" % (k, v))
outfile.close()
# Create the systemd user environment configuration file
# /etc/environment.d/10-gentoo-env.conf with the
# environment configuration from /etc/env.d.
systemd_environment_dir = os.path.join(eroot, "etc", "environment.d")
os.makedirs(systemd_environment_dir, exist_ok=True)
systemd_gentoo_env_path = os.path.join(systemd_environment_dir,
"10-gentoo-env.conf")
systemd_gentoo_env = atomic_ofstream(systemd_gentoo_env_path)
try:
senvnotice = notice + "\n\n"
systemd_gentoo_env.write(senvnotice)
for env_key in env_keys:
env_key_value = env[env_key]
# Skip variables with the empty string
# as value. Those sometimes appear in
# profile.env (e.g. "export GCC_SPECS=''"),
# but are invalid in systemd's syntax.
if not env_key_value:
continue
# Transform into systemd environment.d
# conf syntax, basically shell variable
# assignment (without "export ").
line = f"{env_key}={env_key_value}\n"
systemd_gentoo_env.write(line)
except:
systemd_gentoo_env.abort()
raise
systemd_gentoo_env.close()
#create /etc/csh.env for (t)csh support
outfile = atomic_ofstream(os.path.join(eroot, "etc", "csh.env"))
outfile.write(cenvnotice)
for x in env_keys:
outfile.write("setenv %s '%s'\n" % (x, env[x]))
outfile.close()
def cacheddir(my_original_path,
ignorecvs,
ignorelist,
EmptyOnError,
followSymlinks=True):
global cacheHit, cacheMiss, cacheStale
mypath = normalize_path(my_original_path)
if mypath in dircache:
cacheHit += 1
cached_mtime, list, ftype = dircache[mypath]
else:
cacheMiss += 1
cached_mtime, list, ftype = -1, [], []
try:
pathstat = os.stat(mypath)
if stat.S_ISDIR(pathstat[stat.ST_MODE]):
mtime = pathstat.st_mtime
else:
raise DirectoryNotFound(mypath)
except EnvironmentError as e:
if e.errno == PermissionDenied.errno:
raise PermissionDenied(mypath)
del e
return [], []
except PortageException:
return [], []
# Python retuns mtime in seconds, so if it was changed in the last few seconds, it could be invalid
if mtime != cached_mtime or time.time() - mtime < 4:
if mypath in dircache:
cacheStale += 1
try:
list = os.listdir(mypath)
except EnvironmentError as e:
if e.errno != errno.EACCES:
raise
del e
raise PermissionDenied(mypath)
ftype = []
for x in list:
try:
if followSymlinks:
pathstat = os.stat(mypath + "/" + x)
else:
pathstat = os.lstat(mypath + "/" + x)
if stat.S_ISREG(pathstat[stat.ST_MODE]):
ftype.append(0)
elif stat.S_ISDIR(pathstat[stat.ST_MODE]):
ftype.append(1)
elif stat.S_ISLNK(pathstat[stat.ST_MODE]):
ftype.append(2)
else:
ftype.append(3)
except (IOError, OSError):
ftype.append(3)
dircache[mypath] = mtime, list, ftype
ret_list = []
ret_ftype = []
for x in range(0, len(list)):
if list[x] in ignorelist:
pass
elif ignorecvs:
if list[x][:2] != ".#" and \
not (ftype[x] == 1 and list[x] in _ignorecvs_dirs):
ret_list.append(list[x])
ret_ftype.append(ftype[x])
else:
ret_list.append(list[x])
ret_ftype.append(ftype[x])
writemsg(
"cacheddirStats: H:%d/M:%d/S:%d\n" % (cacheHit, cacheMiss, cacheStale),
10)
return ret_list, ret_ftype
def singleBuilder(cls, options, settings, trees):
vardbapi = trees["vartree"].dbapi
mode = options.get("mode", "older")
if str(mode).lower() not in ["newer", "older"]:
raise SetConfigError(
_("invalid 'mode' value %s (use either 'newer' or 'older')") %
mode)
formats = []
if options.get("package") is not None:
formats.append("package")
if options.get("filestamp") is not None:
formats.append("filestamp")
if options.get("seconds") is not None:
formats.append("seconds")
if options.get("date") is not None:
formats.append("date")
if not formats:
raise SetConfigError(
_("none of these options specified: 'package', 'filestamp', 'seconds', 'date'"
))
elif len(formats) > 1:
raise SetConfigError(
_("no more than one of these options is allowed: 'package', 'filestamp', 'seconds', 'date'"
))
setformat = formats[0]
if (setformat == "package"):
package = options.get("package")
try:
cpv = vardbapi.match(package)[0]
date, = vardbapi.aux_get(cpv, ('BUILD_TIME', ))
date = int(date)
except (KeyError, ValueError):
raise SetConfigError(
_("cannot determine installation date of package %s") %
package)
elif (setformat == "filestamp"):
filestamp = options.get("filestamp")
try:
date = int(os.stat(filestamp).st_mtime)
except (OSError, ValueError):
raise SetConfigError(
_("cannot determine 'filestamp' of '%s'") % filestamp)
elif (setformat == "seconds"):
try:
date = int(options.get("seconds"))
except ValueError:
raise SetConfigError(_("option 'seconds' must be an integer"))
else:
dateopt = options.get("date")
try:
dateformat = options.get("dateformat", "%x %X")
date = int(time.mktime(time.strptime(dateopt, dateformat)))
except ValueError:
raise SetConfigError(
_("'date=%s' does not match 'dateformat=%s'") %
(dateopt, dateformat))
return DateSet(vardb=vardbapi, date=date, mode=mode)
def update_copyright(fn_path, year, pretend=False):
"""
Check file for a Copyright statement, and update its year. The
patterns used for replacing copyrights are taken from echangelog.
Only the first lines of each file that start with a hash ('#') are
considered, until a line is found that doesn't start with a hash.
Files are read and written in binary mode, so that this function
will work correctly with files encoded in any character set, as
long as the copyright statements consist of plain ASCII.
@param fn_path: file path
@type str
@param year: current year
@type str
@param pretend: pretend mode
@type bool
@rtype: bool
@return: True if copyright update was needed, False otherwise
"""
try:
fn_hdl = io.open(_unicode_encode(fn_path,
encoding=_encodings['fs'],
errors='strict'),
mode='rb')
except EnvironmentError:
return
orig_header = []
new_header = []
for line in fn_hdl:
line_strip = line.strip()
orig_header.append(line)
if not line_strip or line_strip[:1] != b'#':
new_header.append(line)
break
line = update_copyright_year(year, line)
new_header.append(line)
difflines = 0
for diffline in difflib.unified_diff(
[_unicode_decode(diffline) for diffline in orig_header],
[_unicode_decode(diffline) for diffline in new_header],
fromfile=fn_path,
tofile=fn_path,
n=0):
util.writemsg_stdout(diffline, noiselevel=-1)
difflines += 1
util.writemsg_stdout("\n", noiselevel=-1)
# unified diff has three lines to start with
if difflines > 3 and not pretend:
# write new file with changed header
f, fnnew_path = mkstemp()
f = io.open(f, mode='wb')
for line in new_header:
f.write(line)
for line in fn_hdl:
f.write(line)
f.close()
try:
fn_stat = os.stat(fn_path)
except OSError:
fn_stat = None
shutil.move(fnnew_path, fn_path)
if fn_stat is None:
util.apply_permissions(fn_path, mode=0o644)
else:
util.apply_stat_permissions(fn_path, fn_stat)
fn_hdl.close()
return difflines > 3
