def read_config(mandatory_opts): loader = KeyValuePairFileLoader( '/etc/dispatch-conf.conf', None) opts, errors = loader.load() if not opts: print(_('dispatch-conf: Error reading /etc/dispatch-conf.conf; fatal'), file=sys.stderr) sys.exit(1) # Handle quote removal here, since KeyValuePairFileLoader doesn't do that. quotes = "\"'" for k, v in opts.items(): if v[:1] in quotes and v[:1] == v[-1:]: opts[k] = v[1:-1] for key in mandatory_opts: if key not in opts: if key == "merge": opts["merge"] = "sdiff --suppress-common-lines --output='%s' '%s' '%s'" else: print(_('dispatch-conf: Missing option "%s" in /etc/dispatch-conf.conf; fatal') % (key,), file=sys.stderr) if not os.path.exists(opts['archive-dir']): os.mkdir(opts['archive-dir']) # Use restrictive permissions by default, in order to protect # against vulnerabilities (like bug #315603 involving rcs). os.chmod(opts['archive-dir'], 0o700) elif not os.path.isdir(opts['archive-dir']): print(_('dispatch-conf: Config archive dir [%s] must exist; fatal') % (opts['archive-dir'],), file=sys.stderr) sys.exit(1) return opts
def read_config(mandatory_opts): loader = portage.env.loaders.KeyValuePairFileLoader( '/etc/dispatch-conf.conf', None) opts, errors = loader.load() if not opts: print(_('dispatch-conf: Error reading /etc/dispatch-conf.conf; fatal'), file=sys.stderr) sys.exit(1) # Handle quote removal here, since KeyValuePairFileLoader doesn't do that. quotes = "\"'" for k, v in opts.items(): if v[:1] in quotes and v[:1] == v[-1:]: opts[k] = v[1:-1] for key in mandatory_opts: if key not in opts: if key == "merge": opts["merge"] = "sdiff --suppress-common-lines --output='%s' '%s' '%s'" else: print(_('dispatch-conf: Missing option "%s" in /etc/dispatch-conf.conf; fatal') % (key,), file=sys.stderr) if not os.path.exists(opts['archive-dir']): os.mkdir(opts['archive-dir']) elif not os.path.isdir(opts['archive-dir']): print(_('dispatch-conf: Config archive dir [%s] must exist; fatal') % (opts['archive-dir'],), file=sys.stderr) sys.exit(1) return opts
def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False): if all: useflags = None elif useflags is None: if mysettings: useflags = mysettings["USE"].split() myfiles = self.getFetchMap(mypkg, useflags=useflags) myebuild = self.findname(mypkg) if myebuild is None: raise AssertionError("ebuild not found for '%s'" % mypkg) pkgdir = os.path.dirname(myebuild) mf = Manifest(pkgdir, self.settings["DISTDIR"]) mysums = mf.getDigests() failures = {} for x in myfiles: if not mysums or x not in mysums: ok = False reason = _("digest missing") else: try: ok, reason = portage.checksum.verify_all( os.path.join(self.settings["DISTDIR"], x), mysums[x]) except FileNotFound as e: ok = False reason = _("File Not Found: '%s'") % (e,) if not ok: failures[x] = reason if failures: return False return True
def load_unpack_dependencies_configuration(repositories): repo_dict = {} for repo in repositories.repos_with_profiles(): for eapi in _supported_eapis: if eapi_has_automatic_unpack_dependencies(eapi): file_name = os.path.join(repo.location, "profiles", "unpack_dependencies", eapi) lines = grabfile(file_name, recursive=True) for line in lines: elements = line.split() suffix = elements[0].lower() if len(elements) == 1: writemsg(_("--- Missing unpack dependencies for '%s' suffix in '%s'\n") % (suffix, file_name)) depend = " ".join(elements[1:]) try: use_reduce(depend, eapi=eapi) except InvalidDependString as e: writemsg(_("--- Invalid unpack dependencies for '%s' suffix in '%s': '%s'\n" % (suffix, file_name, e))) else: repo_dict.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend ret = {} for repo in repositories.repos_with_profiles(): for repo_name in [x.name for x in repo.masters] + [repo.name]: for eapi in repo_dict.get(repo_name, {}): for suffix, depend in repo_dict.get(repo_name, {}).get(eapi, {}).items(): ret.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend return ret
def multiBuilder(self, options, settings, trees): rValue = {} directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "sets")) name_pattern = options.get("name_pattern", "${name}") if not "$name" in name_pattern and not "${name}" in name_pattern: raise SetConfigError(_("name_pattern doesn't include ${name} placeholder")) greedy = get_boolean(options, "greedy", False) # look for repository path variables match = self._repopath_match.match(directory) if match: try: directory = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], directory) except KeyError: raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"]) try: directory = _unicode_decode(directory, encoding=_encodings['fs'], errors='strict') # Now verify that we can also encode it. _unicode_encode(directory, encoding=_encodings['fs'], errors='strict') except UnicodeError: directory = _unicode_decode(directory, encoding=_encodings['fs'], errors='replace') raise SetConfigError( _("Directory path contains invalid character(s) for encoding '%s': '%s'") \ % (_encodings['fs'], directory)) if os.path.isdir(directory): directory = normalize_path(directory) for parent, dirs, files in os.walk(directory): try: parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue for d in dirs[:]: if d[:1] == '.': dirs.remove(d) for filename in files: try: filename = _unicode_decode(filename, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if filename[:1] == '.': continue if filename.endswith(".metadata"): continue filename = os.path.join(parent, filename)[1 + len(directory):] myname = name_pattern.replace("$name", filename) myname = myname.replace("${name}", filename) rValue[myname] = StaticFileSet( os.path.join(directory, filename), greedy=greedy, dbapi=trees["vartree"].dbapi) return rValue
def hardlock_cleanup(path, remove_all_locks=False): myhost = os.uname()[1] mydl = os.listdir(path) results = [] mycount = 0 mylist = {} for x in mydl: if os.path.isfile(path + "/" + x): parts = x.split(".hardlock-") if len(parts) == 2: filename = parts[0][1:] hostpid = parts[1].split("-") host = "-".join(hostpid[:-1]) pid = hostpid[-1] if filename not in mylist: mylist[filename] = {} if host not in mylist[filename]: mylist[filename][host] = [] mylist[filename][host].append(pid) mycount += 1 results.append(_("Found %(count)s locks") % {"count": mycount}) for x in mylist: if myhost in mylist[x] or remove_all_locks: mylockname = hardlock_name(path + "/" + x) if hardlink_is_mine(mylockname, path + "/" + x) or \ not os.path.exists(path + "/" + x) or \ remove_all_locks: for y in mylist[x]: for z in mylist[x][y]: filename = path + "/." + x + ".hardlock-" + y + "-" + z if filename == mylockname: continue try: # We're sweeping through, unlinking everyone's locks. os.unlink(filename) results.append(_("Unlinked: ") + filename) except OSError: pass try: os.unlink(path + "/" + x) results.append(_("Unlinked: ") + path + "/" + x) os.unlink(mylockname) results.append(_("Unlinked: ") + mylockname) except OSError: pass else: try: os.unlink(mylockname) results.append(_("Unlinked: ") + mylockname) except OSError: pass return results
def process(mysettings, key, logentries, fulltext): if "PORTAGE_ELOG_MAILURI" in mysettings: myrecipient = mysettings["PORTAGE_ELOG_MAILURI"].split()[0] else: myrecipient = "root@localhost" myfrom = mysettings["PORTAGE_ELOG_MAILFROM"] myfrom = myfrom.replace("${HOST}", socket.getfqdn()) mysubject = mysettings["PORTAGE_ELOG_MAILSUBJECT"] mysubject = mysubject.replace("${PACKAGE}", key) mysubject = mysubject.replace("${HOST}", socket.getfqdn()) # look at the phases listed in our logentries to figure out what action was performed action = _("merged") for phase in logentries: # if we found a *rm phase assume that the package was unmerged if phase in ["postrm", "prerm"]: action = _("unmerged") # if we think that the package was unmerged, make sure there was no unexpected # phase recorded to avoid misinformation if action == _("unmerged"): for phase in logentries: if phase not in ["postrm", "prerm", "other"]: action = _("unknown") mysubject = mysubject.replace("${ACTION}", action) mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, fulltext) try: portage.mail.send_mail(mysettings, mymessage) except PortageException as e: writemsg("%s\n" % str(e), noiselevel=-1) return
def _parse_repository_usealiases(self, repositories): ret = {} for repo in repositories.repos_with_profiles(): file_name = os.path.join(repo.location, "profiles", "use.aliases") eapi = read_corresponding_eapi_file( file_name, default=repo.eapi) useflag_re = _get_useflag_re(eapi) raw_file_dict = grabdict(file_name, recursive=True) file_dict = {} for real_flag, aliases in raw_file_dict.items(): if useflag_re.match(real_flag) is None: writemsg(_("--- Invalid real USE flag in '%s': '%s'\n") % (file_name, real_flag), noiselevel=-1) else: for alias in aliases: if useflag_re.match(alias) is None: writemsg(_("--- Invalid USE flag alias for '%s' real USE flag in '%s': '%s'\n") % (real_flag, file_name, alias), noiselevel=-1) else: if any(alias in v for k, v in file_dict.items() if k != real_flag): writemsg(_("--- Duplicated USE flag alias in '%s': '%s'\n") % (file_name, alias), noiselevel=-1) else: file_dict.setdefault(real_flag, []).append(alias) ret[repo.name] = file_dict return ret
def lineParser(self, line, line_num, data, errors): line = line.strip() if line.startswith('#'): # Skip commented lines return if not len(line): # skip empty lines return split = line.split() if len(split) < 1: errors.setdefault(self.fname, []).append( _("Malformed data at line: %s, data: %s") % (line_num + 1, line)) return key = split[0] value = split[1:] if not self._validate(key): errors.setdefault(self.fname, []).append( _("Key validation failed at line: %s, data %s") % (line_num + 1, key)) return if not self._valueValidate(value): errors.setdefault(self.fname, []).append( _("Value validation failed at line: %s, data %s") % (line_num + 1, value)) return if key in data: data[key].append(value) else: data[key] = value
def _run(self): mf = self.repo_config.load_manifest( os.path.join(self.repo_config.location, self.cp), self.distdir, fetchlist_dict=self.fetchlist_dict) try: mf.create(assumeDistHashesAlways=True) except FileNotFound as e: portage.writemsg(_("!!! File %s doesn't exist, can't update " "Manifest\n") % e, noiselevel=-1) return 1 except PortagePackageException as e: portage.writemsg(("!!! %s\n") % (e,), noiselevel=-1) return 1 try: modified = mf.write(sign=False) except PermissionDenied as e: portage.writemsg("!!! %s: %s\n" % (_("Permission Denied"), e,), noiselevel=-1) return 1 else: if modified: return self.MODIFIED else: return os.EX_OK
def _addProfile(self, currentPath): parentsFile = os.path.join(currentPath, "parent") eapi_file = os.path.join(currentPath, "eapi") try: eapi = codecs.open(_unicode_encode(eapi_file, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace' ).readline().strip() except IOError: pass else: if not eapi_is_supported(eapi): raise ParseError(_( "Profile contains unsupported " "EAPI '%s': '%s'") % \ (eapi, os.path.realpath(eapi_file),)) if os.path.exists(parentsFile): parents = grabfile(parentsFile) if not parents: raise ParseError( _("Empty parent file: '%s'") % parentsFile) for parentPath in parents: parentPath = normalize_path(os.path.join( currentPath, parentPath)) if os.path.exists(parentPath): self._addProfile(parentPath) else: raise ParseError( _("Parent '%s' not found: '%s'") % \ (parentPath, parentsFile)) self.profiles.append(currentPath)
def _expand_parent_colon(self, parentsFile, parentPath, repo_loc, repositories): colon = parentPath.find(":") if colon == -1: return parentPath if colon == 0: if repo_loc is None: raise ParseError( _("Parent '%s' not found: '%s'") % \ (parentPath, parentsFile)) else: parentPath = normalize_path(os.path.join( repo_loc, 'profiles', parentPath[colon+1:])) else: p_repo_name = parentPath[:colon] try: p_repo_loc = repositories.get_location_for_name(p_repo_name) except KeyError: raise ParseError( _("Parent '%s' not found: '%s'") % \ (parentPath, parentsFile)) else: parentPath = normalize_path(os.path.join( p_repo_loc, 'profiles', parentPath[colon+1:])) return parentPath
def finalize(mysettings=None): """The mysettings parameter is just for backward compatibility since an older version of portage will import the module from a newer version when it upgrades itself.""" global _items printer = EOutput() for root, key, logentries in _items: print() if root == "/": printer.einfo(_("Messages for package %s:") % colorize("INFORM", key)) else: printer.einfo(_("Messages for package %(pkg)s merged to %(root)s:") % {"pkg": colorize("INFORM", key), "root": root}) print() for phase in EBUILD_PHASES: if phase not in logentries: continue for msgtype, msgcontent in logentries[phase]: fmap = {"INFO": printer.einfo, "WARN": printer.ewarn, "ERROR": printer.eerror, "LOG": printer.einfo, "QA": printer.ewarn} if isinstance(msgcontent, basestring): msgcontent = [msgcontent] for line in msgcontent: fmap[msgtype](line.strip("\n")) _items = [] return
def lineParser(self, line, line_num, data, errors): line = line.strip() if line.startswith("#"): # skip commented lines return if not len(line): # skip empty lines return split = line.split("=", 1) if len(split) < 2: errors.setdefault(self.fname, []).append(_("Malformed data at line: %s, data %s") % (line_num + 1, line)) return key = split[0].strip() value = split[1].strip() if not key: errors.setdefault(self.fname, []).append(_("Malformed key at line: %s, key %s") % (line_num + 1, key)) return if not self._validate(key): errors.setdefault(self.fname, []).append( _("Key validation failed at line: %s, data %s") % (line_num + 1, key) ) return if not self._valueValidate(value): errors.setdefault(self.fname, []).append( _("Value validation failed at line: %s, data %s") % (line_num + 1, value) ) return data[key] = value
def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True, eapi_filter=None): ret = {} location_dict = {} file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True) eapi = read_corresponding_eapi_file(file_name) if eapi_filter is not None and not eapi_filter(eapi): if file_dict: writemsg(_("--- EAPI '%s' does not support '%s': '%s'\n") % (eapi, os.path.basename(file_name), file_name), noiselevel=-1) return ret useflag_re = _get_useflag_re(eapi) for k, v in file_dict.items(): useflags = [] for prefixed_useflag in v: if prefixed_useflag[:1] == "-": useflag = prefixed_useflag[1:] else: useflag = prefixed_useflag if useflag_re.match(useflag) is None: writemsg(_("--- Invalid USE flag for '%s' in '%s': '%s'\n") % (k, file_name, prefixed_useflag), noiselevel=-1) else: useflags.append(prefixed_useflag) location_dict.setdefault(k, []).extend(useflags) for k, v in location_dict.items(): if juststrings: v = " ".join(v) else: v = tuple(v) ret.setdefault(k.cp, {})[k] = v return ret
def _finalize(): global _items printer = EOutput() for root, key, logentries, logfile in _items: print() if root == "/": printer.einfo(_("Messages for package %s:") % colorize("INFORM", key)) else: printer.einfo(_("Messages for package %(pkg)s merged to %(root)s:") % {"pkg": colorize("INFORM", key), "root": root}) if logfile is not None: printer.einfo(_("Log file: %s") % colorize("INFORM", logfile)) print() for phase in EBUILD_PHASES: if phase not in logentries: continue for msgtype, msgcontent in logentries[phase]: fmap = {"INFO": printer.einfo, "WARN": printer.ewarn, "ERROR": printer.eerror, "LOG": printer.einfo, "QA": printer.ewarn} if isinstance(msgcontent, basestring): msgcontent = [msgcontent] for line in msgcontent: fmap[msgtype](line.strip("\n")) _items = [] return
def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. @param filename: File to run the checksums against @type filename: String @param calc_prelink: Whether or not to reverse prelink before running the checksum @type calc_prelink: Integer @param strict: Enable/Disable strict checking (which stops exactly at a checksum failure and throws an exception) @type strict: Integer @rtype: Tuple @return: Result of the checks and possible message: 1) If size fails, False, and a tuple containing a message, the given size, and the actual size 2) If there is an os error, False, and a tuple containing the system error followed by 2 nulls 3) If a checksum fails, False and a tuple containing a message, the given hash, and the actual hash 4) If all checks succeed, return True and a fake reason """ # Dict relates to single file only. # returns: (passed,reason) file_is_ok = True reason = "Reason unknown" try: mysize = os.stat(filename)[stat.ST_SIZE] if mydict["size"] != mysize: return False, (_("Filesize does not match recorded size"), mysize, mydict["size"]) except OSError as e: if e.errno == errno.ENOENT: raise portage.exception.FileNotFound(filename) return False, (str(e), None, None) verifiable_hash_types = set(mydict).intersection(hashfunc_map) verifiable_hash_types.discard("size") if not verifiable_hash_types: expected = set(hashfunc_map) expected.discard("size") expected = list(expected) expected.sort() expected = " ".join(expected) got = set(mydict) got.discard("size") got = list(got) got.sort() got = " ".join(got) return False, (_("Insufficient data for checksum verification"), got, expected) for x in sorted(mydict): if x == "size": continue elif x in hashfunc_map: myhash = perform_checksum(filename, x, calc_prelink=calc_prelink)[0] if mydict[x] != myhash: if strict: raise portage.exception.DigestException( ("Failed to verify '$(file)s' on " + "checksum type '%(type)s'") % {"file": filename, "type": x} ) else: file_is_ok = False reason = (("Failed on %s verification" % x), myhash, mydict[x]) break return file_is_ok, reason
def deprecated_profile_check(settings=None): config_root = "/" if settings is not None: config_root = settings["PORTAGE_CONFIGROOT"] deprecated_profile_file = os.path.join(config_root, DEPRECATED_PROFILE_FILE) if not os.access(deprecated_profile_file, os.R_OK): return False dcontent = codecs.open(_unicode_encode(deprecated_profile_file, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace').readlines() writemsg(colorize("BAD", _("\n!!! Your current profile is " "deprecated and not supported anymore.")) + "\n", noiselevel=-1) writemsg(colorize("BAD", _("!!! Use eselect profile to update your " "profile.")) + "\n", noiselevel=-1) if not dcontent: writemsg(colorize("BAD", _("!!! Please refer to the " "Gentoo Upgrading Guide.")) + "\n", noiselevel=-1) return True newprofile = dcontent[0] writemsg(colorize("BAD", _("!!! Please upgrade to the " "following profile if possible:")) + "\n", noiselevel=-1) writemsg(8*" " + colorize("GOOD", newprofile) + "\n", noiselevel=-1) if len(dcontent) > 1: writemsg(_("To upgrade do the following steps:\n"), noiselevel=-1) for myline in dcontent[1:]: writemsg(myline, noiselevel=-1) writemsg("\n\n", noiselevel=-1) return True
def collect_ebuild_messages(path): """ Collect elog messages generated by the bash logging function stored at 'path'. """ mylogfiles = None try: mylogfiles = os.listdir(path) except OSError: pass # shortcut for packages without any messages if not mylogfiles: return {} # exploit listdir() file order so we process log entries in chronological order mylogfiles.reverse() logentries = {} for msgfunction in mylogfiles: filename = os.path.join(path, msgfunction) if msgfunction not in EBUILD_PHASES: writemsg(_("!!! can't process invalid log file: %s\n") % filename, noiselevel=-1) continue if not msgfunction in logentries: logentries[msgfunction] = [] lastmsgtype = None msgcontent = [] f = io.open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') for l in f: l = l.rstrip('\n') if not l: continue try: msgtype, msg = l.split(" ", 1) except ValueError: writemsg(_("!!! malformed entry in " "log file: '%s'\n") % filename, noiselevel=-1) continue if lastmsgtype is None: lastmsgtype = msgtype if msgtype == lastmsgtype: msgcontent.append(msg) else: if msgcontent: logentries[msgfunction].append((lastmsgtype, msgcontent)) msgcontent = [msg] lastmsgtype = msgtype f.close() if msgcontent: logentries[msgfunction].append((lastmsgtype, msgcontent)) # clean logfiles to avoid repetitions for f in mylogfiles: try: os.unlink(os.path.join(path, f)) except OSError: pass return logentries
def onerror(e): if isinstance(e, OperationNotPermitted): writemsg(_("Operation Not Permitted: %s\n") % str(e), noiselevel=-1) elif isinstance(e, FileNotFound): writemsg(_("File Not Found: '%s'\n") % str(e), noiselevel=-1) else: raise
def display(self): self.out.write( "\r" + colorize("WARN", _("cache miss: '") + str(self.misses) + "'") + " --- " + colorize("GOOD", _("cache hit: '") + str(self.hits) + "'") ) self.out.flush()
def __init__(self, s): if isinstance(s, Atom): # This is an efficiency assertion, to ensure that the Atom # constructor is not called redundantly. raise TypeError(_("Expected %s, got %s") % \ (_atom_base, type(s))) _atom_base.__init__(s) if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) if blocker.overlap.forbid: s = s[2:] else: s = s[1:] else: blocker = False self.__dict__['blocker'] = blocker m = _atom_re.match(s) if m is None: raise InvalidAtom(self) if m.group('op') is not None: base = _atom_re.groupindex['op'] op = m.group(base + 1) cpv = m.group(base + 2) cp = m.group(base + 3) if m.group(base + 4) is not None: raise InvalidAtom(self) elif m.group('star') is not None: base = _atom_re.groupindex['star'] op = '=*' cpv = m.group(base + 1) cp = m.group(base + 2) if m.group(base + 3) is not None: raise InvalidAtom(self) elif m.group('simple') is not None: op = None cpv = cp = m.group(_atom_re.groupindex['simple'] + 1) if m.group(_atom_re.groupindex['simple'] + 2) is not None: raise InvalidAtom(self) else: raise AssertionError(_("required group not found in atom: '%s'") % self) self.__dict__['cp'] = cp self.__dict__['cpv'] = cpv self.__dict__['slot'] = m.group(_atom_re.groups - 1) self.__dict__['operator'] = op use_str = m.group(_atom_re.groups) if use_str is not None: use = _use_dep(dep_getusedeps(s)) without_use = Atom(m.group('without_use')) else: use = None without_use = self self.__dict__['use'] = use self.__dict__['without_use'] = without_use
def check_auto_sync(self): '''Check the auto_sync setting''' if self.repo.auto_sync is None: writemsg_level("!!! %s\n" % _("Repository '%s' is missing auto_sync attribute") % self.repo.name, level=self.logger.ERROR, noiselevel=-1) elif self.repo.auto_sync.lower() not in ["yes", "true", "no", "false"]: writemsg_level("!!! %s\n" % _("Repository '%s' auto_sync attribute must be one of: %s") % (self.repo.name, '{yes, true, no, false}'), level=self.logger.ERROR, noiselevel=-1)
def singleBuilder(cls, options, settings, trees): mode = options.get("mode", "older") if str(mode).lower() not in ["newer", "older"]: raise SetConfigError(_("invalid 'mode' value %s (use either 'newer' or 'older')") % mode) try: age = int(options.get("age", "7")) except ValueError as e: raise SetConfigError(_("value of option 'age' is not an integer")) return AgeSet(vardb=trees["vartree"].dbapi, mode=mode, age=age)
def parse_updates(mycontent): """Valid updates are returned as a list of split update commands.""" myupd = [] errors = [] mylines = mycontent.splitlines() for myline in mylines: mysplit = myline.split() if len(mysplit) == 0: continue if mysplit[0] not in ("move", "slotmove"): errors.append(_("ERROR: Update type not recognized '%s'") % myline) continue if mysplit[0] == "move": if len(mysplit) != 3: errors.append(_("ERROR: Update command invalid '%s'") % myline) continue valid = True for i in (1, 2): try: atom = Atom(mysplit[i]) except InvalidAtom: atom = None else: if atom.blocker or atom != atom.cp: atom = None if atom is not None: mysplit[i] = atom else: errors.append( _("ERROR: Malformed update entry '%s'") % myline) valid = False break if not valid: continue if mysplit[0] == "slotmove": if len(mysplit)!=4: errors.append(_("ERROR: Update command invalid '%s'") % myline) continue pkg, origslot, newslot = mysplit[1], mysplit[2], mysplit[3] try: atom = Atom(pkg) except InvalidAtom: atom = None else: if atom.blocker: atom = None if atom is not None: mysplit[1] = atom else: errors.append(_("ERROR: Malformed update entry '%s'") % myline) continue # The list of valid updates is filtered by continue statements above. myupd.append(mysplit) return myupd, errors
def _read_dirVirtuals(self, profiles): """ Read the 'virtuals' file in all profiles. """ virtuals_list = [] for x in profiles: virtuals_file = os.path.join(x, "virtuals") virtuals_dict = grabdict(virtuals_file) atoms_dict = {} for k, v in virtuals_dict.items(): try: virt_atom = Atom(k) except InvalidAtom: virt_atom = None else: if virt_atom.blocker or \ str(virt_atom) != str(virt_atom.cp): virt_atom = None if virt_atom is None: writemsg(_("--- Invalid virtuals atom in %s: %s\n") % \ (virtuals_file, k), noiselevel=-1) continue providers = [] for atom in v: atom_orig = atom if atom[:1] == '-': # allow incrementals atom = atom[1:] try: atom = Atom(atom) except InvalidAtom: atom = None else: if atom.blocker: atom = None if atom is None: writemsg(_("--- Invalid atom in %s: %s\n") % \ (virtuals_file, atom_orig), noiselevel=-1) else: if atom_orig == str(atom): # normal atom, so return as Atom instance providers.append(atom) else: # atom has special prefix, so return as string providers.append(atom_orig) if providers: atoms_dict[virt_atom] = providers if atoms_dict: virtuals_list.append(atoms_dict) self._dirVirtuals = stack_dictlist(virtuals_list, incremental=True) for virt in self._dirVirtuals: # Preference for virtuals decreases from left to right. self._dirVirtuals[virt].reverse()
def file_archive(archive, curconf, newconf, mrgconf): """Archive existing config to the archive-dir, bumping old versions out of the way into .# versions (log-rotate style). Then, if mrgconf was specified and there is a .dist version, merge the user's changes and the distributed changes and put the result into mrgconf. Lastly, if newconf was specified, archive it as a .dist.new version (which gets moved to the .dist version at the end of the processing).""" try: os.makedirs(os.path.dirname(archive)) except OSError: pass # Archive the current config file if it isn't already saved if os.path.exists(archive) and len(diffstatusoutput("diff -aq '%s' '%s'", curconf, archive)[1]) != 0: suf = 1 while suf < 9 and os.path.exists(archive + "." + str(suf)): suf += 1 while suf > 1: os.rename(archive + "." + str(suf - 1), archive + "." + str(suf)) suf -= 1 os.rename(archive, archive + ".1") if os.path.isfile(curconf): try: shutil.copy2(curconf, archive) except (IOError, os.error) as why: print( _("dispatch-conf: Error copying %(curconf)s to %(archive)s: %(reason)s; fatal") % {"curconf": curconf, "archive": archive, "reason": str(why)}, file=sys.stderr, ) if newconf != "": # Save off new config file in the archive dir with .dist.new suffix try: shutil.copy2(newconf, archive + ".dist.new") except (IOError, os.error) as why: print( _("dispatch-conf: Error copying %(newconf)s to %(archive)s: %(reason)s; fatal") % {"newconf": newconf, "archive": archive + ".dist.new", "reason": str(why)}, file=sys.stderr, ) ret = 0 if mrgconf != "" and os.path.exists(archive + ".dist"): # This puts the results of the merge into mrgconf. ret = os.system(DIFF3_MERGE % (curconf, archive + ".dist", newconf, mrgconf)) mystat = os.lstat(newconf) os.chmod(mrgconf, mystat.st_mode) os.chown(mrgconf, mystat.st_uid, mystat.st_gid) return ret
def _check_locations(self): """Check if repositories location are correct and show a warning message if not""" for (name, r) in self.prepos.items(): if name != 'DEFAULT': if r.location is None: writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1) else: if not isdir_raise_eaccess(r.location) and not portage._sync_mode: self.prepos_order.remove(name) writemsg(_("!!! Invalid Repository Location" " (not a dir): '%s'\n") % r.location, noiselevel=-1)
def setexec(ctx="\n"): ctx = _unicode_encode(ctx, encoding=_encodings['content'], errors='strict') if selinux.setexeccon(ctx) < 0: ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') if selinux.security_getenforce() == 1: raise OSError(_("Failed setting exec() context \"%s\".") % ctx) else: portage.writemsg("!!! " + \ _("Failed setting exec() context \"%s\".") % ctx, \ noiselevel=-1)
def getfetchsizes(self, mypkg, useflags=None, debug=0, myrepo=None): # returns a filename:size dictionnary of remaining downloads myebuild, mytree = self.findname2(mypkg, myrepo=myrepo) if myebuild is None: raise AssertionError(_("ebuild not found for '%s'") % mypkg) pkgdir = os.path.dirname(myebuild) mf = self.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(pkgdir))).load_manifest( pkgdir, self.settings["DISTDIR"]) checksums = mf.getDigests() if not checksums: if debug: writemsg(_("[empty/missing/bad digest]: %s\n") % (mypkg,)) return {} filesdict={} myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree) #XXX: maybe this should be improved: take partial downloads # into account? check checksums? for myfile in myfiles: try: fetch_size = int(checksums[myfile]["size"]) except (KeyError, ValueError): if debug: writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg}) continue file_path = os.path.join(self.settings["DISTDIR"], myfile) mystat = None try: mystat = os.stat(file_path) except OSError: pass if mystat is None: existing_size = 0 ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS") if ro_distdirs is not None: for x in shlex_split(ro_distdirs): try: mystat = os.stat(os.path.join(x, myfile)) except OSError: pass else: if mystat.st_size == fetch_size: existing_size = fetch_size break else: existing_size = mystat.st_size remaining_size = fetch_size - existing_size if remaining_size > 0: # Assume the download is resumable. filesdict[myfile] = remaining_size elif remaining_size < 0: # The existing file is too large and therefore corrupt. filesdict[myfile] = int(checksums[myfile]["size"]) return filesdict
def unlockfile(mytuple): #XXX: Compatability hack. if len(mytuple) == 3: lockfilename, myfd, unlinkfile = mytuple locking_method = fcntl.flock elif len(mytuple) == 4: lockfilename, myfd, unlinkfile, locking_method = mytuple else: raise InvalidData if (myfd == HARDLINK_FD): unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile) return True # myfd may be None here due to myfd = mypath in lockfile() if isinstance(lockfilename, basestring) and \ not os.path.exists(lockfilename): writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1) if myfd is not None: os.close(myfd) _open_fds.remove(myfd) return False try: if myfd is None: myfd = os.open(lockfilename, os.O_WRONLY, 0o660) unlinkfile = 1 locking_method(myfd, fcntl.LOCK_UN) except OSError: if isinstance(lockfilename, basestring): os.close(myfd) _open_fds.remove(myfd) raise IOError(_("Failed to unlock file '%s'\n") % lockfilename) try: # This sleep call was added to allow other processes that are # waiting for a lock to be able to grab it before it is deleted. # lockfile() already accounts for this situation, however, and # the sleep here adds more time than is saved overall, so am # commenting until it is proved necessary. #time.sleep(0.0001) if unlinkfile: locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) # We won the lock, so there isn't competition for it. # We can safely delete the file. writemsg(_("Got the lockfile...\n"), 1) if _fstat_nlink(myfd) == 1: os.unlink(lockfilename) writemsg(_("Unlinked lockfile...\n"), 1) locking_method(myfd, fcntl.LOCK_UN) else: writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1) os.close(myfd) _open_fds.remove(myfd) return False except SystemExit: raise except Exception as e: writemsg(_("Failed to get lock... someone took it.\n"), 1) writemsg(str(e) + "\n", 1) # why test lockfilename? because we may have been handed an # fd originally, and the caller might not like having their # open fd closed automatically on them. if isinstance(lockfilename, basestring): os.close(myfd) _open_fds.remove(myfd) return True
def rebuild(self, exclude_pkgs=None, include_file=None, preserve_paths=None): """ Raises CommandNotFound if there are preserved libs and the scanelf binary is not available. @param exclude_pkgs: A set of packages that should be excluded from the LinkageMap, since they are being unmerged and their NEEDED entries are therefore irrelevant and would only serve to corrupt the LinkageMap. @type exclude_pkgs: set @param include_file: The path of a file containing NEEDED entries for a package which does not exist in the vardbapi yet because it is currently being merged. @type include_file: String @param preserve_paths: Libraries preserved by a package instance that is currently being merged. They need to be explicitly passed to the LinkageMap, since they are not registered in the PreservedLibsRegistry yet. @type preserve_paths: set """ os = _os_merge root = self._root root_len = len(root) - 1 self._clear_cache() self._defpath.update( getlibpaths(self._dbapi.settings["EROOT"], env=self._dbapi.settings)) libs = self._libs obj_properties = self._obj_properties lines = [] # Data from include_file is processed first so that it # overrides any data from previously installed files. if include_file is not None: for line in grabfile(include_file): lines.append((None, include_file, line)) aux_keys = [self._needed_aux_key] can_lock = os.access(os.path.dirname(self._dbapi._dbroot), os.W_OK) if can_lock: self._dbapi.lock() try: for cpv in self._dbapi.cpv_all(): if exclude_pkgs is not None and cpv in exclude_pkgs: continue needed_file = self._dbapi.getpath( cpv, filename=self._needed_aux_key) for line in self._dbapi.aux_get(cpv, aux_keys)[0].splitlines(): lines.append((cpv, needed_file, line)) finally: if can_lock: self._dbapi.unlock() # have to call scanelf for preserved libs here as they aren't # registered in NEEDED.ELF.2 files plibs = {} if preserve_paths is not None: plibs.update((x, None) for x in preserve_paths) if self._dbapi._plib_registry and self._dbapi._plib_registry.hasEntries( ): for cpv, items in self._dbapi._plib_registry.getPreservedLibs( ).items(): if exclude_pkgs is not None and cpv in exclude_pkgs: # These preserved libs will either be unmerged, # rendering them irrelevant, or they will be # preserved in the replacement package and are # already represented via the preserve_paths # parameter. continue plibs.update((x, cpv) for x in items) if plibs: # We don't use scanelf -q, since that would omit libraries like # musl's /usr/lib/libc.so which do not have any DT_NEEDED or # DT_SONAME settings. args = [ os.path.join(EPREFIX or "/", "usr/bin/scanelf"), "-BF", "%a;%F;%S;%r;%n", ] args.extend( os.path.join(root, x.lstrip("." + os.sep)) for x in plibs) try: proc = subprocess.Popen(args, stdout=subprocess.PIPE) except EnvironmentError as e: if e.errno != errno.ENOENT: raise raise CommandNotFound(args[0]) else: for l in proc.stdout: try: l = _unicode_decode(l, encoding=_encodings["content"], errors="strict") except UnicodeDecodeError: l = _unicode_decode(l, encoding=_encodings["content"], errors="replace") writemsg_level( _("\nError decoding characters " "returned from scanelf: %s\n\n") % (l, ), level=logging.ERROR, noiselevel=-1, ) l = l[3:].rstrip("\n") if not l: continue try: entry = NeededEntry.parse("scanelf", l) except InvalidData as e: writemsg_level("\n%s\n\n" % (e, ), level=logging.ERROR, noiselevel=-1) continue try: with open( _unicode_encode( entry.filename, encoding=_encodings["fs"], errors="strict", ), "rb", ) as f: elf_header = ELFHeader.read(f) except EnvironmentError as e: if e.errno != errno.ENOENT: raise # File removed concurrently. continue # Infer implicit soname from basename (bug 715162). if not entry.soname: try: proc = subprocess.Popen( [ b"file", _unicode_encode( entry.filename, encoding=_encodings["fs"], errors="strict", ), ], stdout=subprocess.PIPE, ) out, err = proc.communicate() proc.wait() except EnvironmentError: pass else: if b"SB shared object" in out: entry.soname = os.path.basename(entry.filename) entry.multilib_category = compute_multilib_category( elf_header) entry.filename = entry.filename[root_len:] owner = plibs.pop(entry.filename, None) lines.append((owner, "scanelf", str(entry))) proc.wait() proc.stdout.close() if plibs: # Preserved libraries that did not appear in the scanelf output. # This is known to happen with statically linked libraries. # Generate dummy lines for these, so we can assume that every # preserved library has an entry in self._obj_properties. This # is important in order to prevent findConsumers from raising # an unwanted KeyError. for x, cpv in plibs.items(): lines.append((cpv, "plibs", ";".join(["", x, "", "", ""]))) # Share identical frozenset instances when available, # in order to conserve memory. frozensets = {} owner_entries = collections.defaultdict(list) while True: try: owner, location, l = lines.pop() except IndexError: break l = l.rstrip("\n") if not l: continue if "\0" in l: # os.stat() will raise "TypeError: must be encoded string # without NULL bytes, not str" in this case. writemsg_level( _("\nLine contains null byte(s) " "in %s: %s\n\n") % (location, l), level=logging.ERROR, noiselevel=-1, ) continue try: entry = NeededEntry.parse(location, l) except InvalidData as e: writemsg_level("\n%s\n\n" % (e, ), level=logging.ERROR, noiselevel=-1) continue # If NEEDED.ELF.2 contains the new multilib category field, # then use that for categorization. Otherwise, if a mapping # exists, map e_machine (entry.arch) to an approximate # multilib category. If all else fails, use e_machine, just # as older versions of portage did. if entry.multilib_category is None: entry.multilib_category = _approx_multilib_categories.get( entry.arch, entry.arch) entry.filename = normalize_path(entry.filename) expand = {"ORIGIN": os.path.dirname(entry.filename)} entry.runpaths = frozenset( normalize_path( varexpand( x, expand, error_leader=lambda: "%s: " % location)) for x in entry.runpaths) entry.runpaths = frozensets.setdefault(entry.runpaths, entry.runpaths) owner_entries[owner].append(entry) # In order to account for internal library resolution which a package # may implement (useful at least for handling of bundled libraries), # generate implicit runpath entries for any needed sonames which are # provided by the same owner package. for owner, entries in owner_entries.items(): if owner is None: continue providers = {} for entry in entries: if entry.soname: providers[SonameAtom(entry.multilib_category, entry.soname)] = entry for entry in entries: implicit_runpaths = [] for soname in entry.needed: soname_atom = SonameAtom(entry.multilib_category, soname) provider = providers.get(soname_atom) if provider is None: continue provider_dir = os.path.dirname(provider.filename) if provider_dir not in entry.runpaths: implicit_runpaths.append(provider_dir) if implicit_runpaths: entry.runpaths = frozenset( itertools.chain(entry.runpaths, implicit_runpaths)) entry.runpaths = frozensets.setdefault( entry.runpaths, entry.runpaths) for owner, entry in ((owner, entry) for (owner, entries) in owner_entries.items() for entry in entries): arch = entry.multilib_category obj = entry.filename soname = entry.soname path = entry.runpaths needed = frozenset(entry.needed) needed = frozensets.setdefault(needed, needed) obj_key = self._obj_key(obj) indexed = True myprops = obj_properties.get(obj_key) if myprops is None: indexed = False myprops = self._obj_properties_class(arch, needed, path, soname, [], owner) obj_properties[obj_key] = myprops # All object paths are added into the obj_properties tuple. myprops.alt_paths.append(obj) # Don't index the same file more that once since only one # set of data can be correct and therefore mixing data # may corrupt the index (include_file overrides previously # installed). if indexed: continue arch_map = libs.get(arch) if arch_map is None: arch_map = {} libs[arch] = arch_map if soname: soname_map = arch_map.get(soname) if soname_map is None: soname_map = self._soname_map_class(providers=[], consumers=[]) arch_map[soname] = soname_map soname_map.providers.append(obj_key) for needed_soname in needed: soname_map = arch_map.get(needed_soname) if soname_map is None: soname_map = self._soname_map_class(providers=[], consumers=[]) arch_map[needed_soname] = soname_map soname_map.consumers.append(obj_key) for arch, sonames in libs.items(): for soname_node in sonames.values(): soname_node.providers = tuple(set(soname_node.providers)) soname_node.consumers = tuple(set(soname_node.consumers))
def parse_layout_conf(repo_location, repo_name=None): eapi = read_corresponding_eapi_file( os.path.join(repo_location, REPO_NAME_LOC)) layout_filename = os.path.join(repo_location, "metadata", "layout.conf") layout_file = KeyValuePairFileLoader(layout_filename, None, None) layout_data, layout_errors = layout_file.load() data = {} # None indicates abscence of a masters setting, which later code uses # to trigger a backward compatibility fallback that sets an implicit # master. In order to avoid this fallback behavior, layout.conf can # explicitly set masters to an empty value, which will result in an # empty tuple here instead of None. masters = layout_data.get('masters') if masters is not None: masters = tuple(masters.split()) data['masters'] = masters data['aliases'] = tuple(layout_data.get('aliases', '').split()) data['allow-provide-virtual'] = \ layout_data.get('allow-provide-virtuals', 'false').lower() == 'true' data['eapis-banned'] = tuple(layout_data.get('eapis-banned', '').split()) data['eapis-deprecated'] = tuple( layout_data.get('eapis-deprecated', '').split()) data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \ == 'true' data['sign-manifest'] = layout_data.get('sign-manifests', 'true').lower() \ == 'true' data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \ == 'true' data['repo-name'] = _gen_valid_repo(layout_data.get('repo-name', '')) manifest_policy = layout_data.get('use-manifests', 'strict').lower() data['allow-missing-manifest'] = manifest_policy != 'strict' data['create-manifest'] = manifest_policy != 'false' data['disable-manifest'] = manifest_policy == 'false' # for compatibility w/ PMS, fallback to pms; but also check if the # cache exists or not. cache_formats = layout_data.get('cache-formats', '').lower().split() if not cache_formats: # Auto-detect cache formats, and prefer md5-cache if available. # This behavior was deployed in portage-2.1.11.14, so that the # default egencache format could eventually be changed to md5-dict # in portage-2.1.11.32. WARNING: Versions prior to portage-2.1.11.14 # will NOT recognize md5-dict format unless it is explicitly # listed in layout.conf. cache_formats = [] if os.path.isdir(os.path.join(repo_location, 'metadata', 'md5-cache')): cache_formats.append('md5-dict') if os.path.isdir(os.path.join(repo_location, 'metadata', 'cache')): cache_formats.append('pms') data['cache-formats'] = tuple(cache_formats) manifest_hashes = layout_data.get('manifest-hashes') if manifest_hashes is not None: manifest_hashes = frozenset(manifest_hashes.upper().split()) if MANIFEST2_REQUIRED_HASH not in manifest_hashes: repo_name = _get_repo_name(repo_location, cached=repo_name) warnings.warn( (_("Repository named '%(repo_name)s' has a " "'manifest-hashes' setting that does not contain " "the '%(hash)s' hash which is required by this " "portage version. You will have to upgrade portage " "if you want to generate valid manifests for this " "repository: %(layout_filename)s") % { "repo_name": repo_name or 'unspecified', "hash": MANIFEST2_REQUIRED_HASH, "layout_filename": layout_filename }), DeprecationWarning) unsupported_hashes = manifest_hashes.difference( MANIFEST2_HASH_FUNCTIONS) if unsupported_hashes: repo_name = _get_repo_name(repo_location, cached=repo_name) warnings.warn(( _("Repository named '%(repo_name)s' has a " "'manifest-hashes' setting that contains one " "or more hash types '%(hashes)s' which are not supported by " "this portage version. You will have to upgrade " "portage if you want to generate valid manifests for " "this repository: %(layout_filename)s") % { "repo_name": repo_name or 'unspecified', "hashes": " ".join(sorted(unsupported_hashes)), "layout_filename": layout_filename }), DeprecationWarning) data['manifest-hashes'] = manifest_hashes data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \ == 'true' raw_formats = layout_data.get('profile-formats') if raw_formats is None: if eapi_allows_directories_on_profile_level_and_repository_level(eapi): raw_formats = ('portage-1', ) else: raw_formats = ('portage-1-compat', ) else: raw_formats = set(raw_formats.split()) unknown = raw_formats.difference(_valid_profile_formats) if unknown: repo_name = _get_repo_name(repo_location, cached=repo_name) warnings.warn((_( "Repository named '%(repo_name)s' has unsupported " "profiles in use ('profile-formats = %(unknown_fmts)s' setting in " "'%(layout_filename)s; please upgrade portage.") % dict(repo_name=repo_name or 'unspecified', layout_filename=layout_filename, unknown_fmts=" ".join(unknown))), DeprecationWarning) raw_formats = tuple(raw_formats.intersection(_valid_profile_formats)) data['profile-formats'] = raw_formats return data, layout_errors
def process(mysettings, key, logentries, fulltext): if mysettings.get("PORTAGE_LOGDIR"): logdir = normalize_path(mysettings["PORTAGE_LOGDIR"]) else: logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep), "var", "log", "portage") if not os.path.isdir(logdir): # Only initialize group/mode if the directory doesn't # exist, so that we don't override permissions if they # were previously set by the administrator. # NOTE: These permissions should be compatible with our # default logrotate config as discussed in bug 374287. logdir_uid = -1 if portage.data.secpass >= 2: logdir_uid = portage_uid ensure_dirs(logdir, uid=logdir_uid, gid=portage_gid, mode=0o2770) elogdir = os.path.join(logdir, "elog") _ensure_log_subdirs(logdir, elogdir) # TODO: Locking elogfilename = elogdir + "/summary.log" try: elogfile = io.open(_unicode_encode(elogfilename, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='backslashreplace') except IOError as e: func_call = "open('%s', 'a')" % elogfilename if e.errno == errno.EACCES: raise portage.exception.PermissionDenied(func_call) elif e.errno == errno.EPERM: raise portage.exception.OperationNotPermitted(func_call) elif e.errno == errno.EROFS: raise portage.exception.ReadOnlyFileSystem(func_call) else: raise # Copy group permission bits from parent directory. elogdir_st = os.stat(elogdir) elogdir_gid = elogdir_st.st_gid elogdir_grp_mode = 0o060 & elogdir_st.st_mode # Copy the uid from the parent directory if we have privileges # to do so, for compatibility with our default logrotate # config (see bug 378451). With the "su portage portage" # directive and logrotate-3.8.0, logrotate's chown call during # the compression phase will only succeed if the log file's uid # is portage_uid. logfile_uid = -1 if portage.data.secpass >= 2: logfile_uid = elogdir_st.st_uid apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid, mode=elogdir_grp_mode, mask=0) time_fmt = "%Y-%m-%d %H:%M:%S %Z" if sys.hexversion < 0x3000000: time_fmt = _unicode_encode(time_fmt) time_str = time.strftime(time_fmt, time.localtime(time.time())) # Avoid potential UnicodeDecodeError in Python 2, since strftime # returns bytes in Python 2, and %Z may contain non-ascii chars. time_str = _unicode_decode(time_str, encoding=_encodings['content'], errors='replace') elogfile.write( _(">>> Messages generated by process " "%(pid)d on %(time)s for package %(pkg)s:\n\n") % { "pid": os.getpid(), "time": time_str, "pkg": key }) elogfile.write(_unicode_decode(fulltext)) elogfile.write("\n") elogfile.close() return elogfilename
def _parse(paths, prepos, ignored_map, ignored_location_map, local_config, portdir): """Parse files in paths to load config""" parser = SafeConfigParser() # use read_file/readfp in order to control decoding of unicode try: # Python >=3.2 read_file = parser.read_file source_kwarg = 'source' except AttributeError: read_file = parser.readfp source_kwarg = 'filename' recursive_paths = [] for p in paths: if isinstance(p, basestring): recursive_paths.extend(_recursive_file_list(p)) else: recursive_paths.append(p) for p in recursive_paths: if isinstance(p, basestring): f = None try: f = io.open(_unicode_encode(p, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError: pass else: # The 'source' keyword argument is needed since otherwise # ConfigParser in Python <3.3.3 may throw a TypeError # because it assumes that f.name is a native string rather # than binary when constructing error messages. kwargs = {source_kwarg: p} read_file(f, **portage._native_kwargs(kwargs)) finally: if f is not None: f.close() elif isinstance(p, io.StringIO): kwargs = {source_kwarg: "<io.StringIO>"} read_file(p, **portage._native_kwargs(kwargs)) else: raise TypeError( "Unsupported type %r of element %r of 'paths' argument" % (type(p), p)) prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults(), local_config=local_config) for sname in parser.sections(): optdict = {} for oname in parser.options(sname): optdict[oname] = parser.get(sname, oname) repo = RepoConfig(sname, optdict, local_config=local_config) if repo.sync_type is not None and repo.sync_uri is None: writemsg_level("!!! %s\n" % _( "Repository '%s' has sync-type attribute, but is missing sync-uri attribute" ) % sname, level=logging.ERROR, noiselevel=-1) continue if repo.sync_uri is not None and repo.sync_type is None: writemsg_level("!!! %s\n" % _( "Repository '%s' has sync-uri attribute, but is missing sync-type attribute" ) % sname, level=logging.ERROR, noiselevel=-1) continue if repo.sync_type not in (None, "cvs", "git", "rsync"): writemsg_level("!!! %s\n" % _( "Repository '%s' has sync-type attribute set to unsupported value: '%s'" ) % (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1) continue if repo.sync_type == "cvs" and repo.sync_cvs_repo is None: writemsg_level("!!! %s\n" % _( "Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute" ) % sname, level=logging.ERROR, noiselevel=-1) continue # For backward compatibility with locations set via PORTDIR and # PORTDIR_OVERLAY, delay validation of the location and repo.name # until after PORTDIR and PORTDIR_OVERLAY have been processed. prepos[sname] = repo
def lchown(*pargs, **kwargs): writemsg(colorize("BAD", "!!!") + _(" It seems that os.lchown does not" " exist. Please rebuild python.\n"), noiselevel=-1)
def unmerge(root_config, myopts, unmerge_action, unmerge_files, ldpath_mtimes, autoclean=0, clean_world=1, clean_delay=1, ordered=0, raise_on_error=0, scheduler=None, writemsg_level=portage.util.writemsg_level): """ Returns os.EX_OK if no errors occur, 1 if an error occurs, and 130 if interrupted due to a 'no' answer for --ask. """ if clean_world: clean_world = myopts.get('--deselect') != 'n' rval, pkgmap = _unmerge_display(root_config, myopts, unmerge_action, unmerge_files, clean_delay=clean_delay, ordered=ordered, writemsg_level=writemsg_level) if rval != os.EX_OK: return rval enter_invalid = '--ask-enter-invalid' in myopts vartree = root_config.trees["vartree"] sets = root_config.sets settings = root_config.settings mysettings = portage.config(clone=settings) xterm_titles = "notitles" not in settings.features if "--pretend" in myopts: #we're done... return return os.EX_OK if "--ask" in myopts: uq = UserQuery(myopts) if uq.query("Would you like to unmerge these packages?", enter_invalid) == "No": # enter pretend mode for correct formatting of results myopts["--pretend"] = True print() print("Quitting.") print() return 128 + signal.SIGINT if not vartree.dbapi.writable: writemsg_level("!!! %s\n" % _("Read-only file system: %s") % vartree.dbapi._dbroot, level=logging.ERROR, noiselevel=-1) return 1 #the real unmerging begins, after a short delay unless we're raging.... if not unmerge_action == "rage-clean" and clean_delay and not autoclean: countdown(int(settings["CLEAN_DELAY"]), ">>> Unmerging") all_selected = set() all_selected.update(*[x["selected"] for x in pkgmap]) # Set counter variables curval = 1 maxval = len(all_selected) for x in range(len(pkgmap)): for y in pkgmap[x]["selected"]: emergelog(xterm_titles, "=== Unmerging... ("+y+")") message = ">>> Unmerging ({0} of {1}) {2}...\n".format( colorize("MERGE_LIST_PROGRESS", str(curval)), colorize("MERGE_LIST_PROGRESS", str(maxval)), y) writemsg_level(message, noiselevel=-1) curval += 1 mysplit = y.split("/") #unmerge... retval = portage.unmerge(mysplit[0], mysplit[1], settings=mysettings, vartree=vartree, ldpath_mtimes=ldpath_mtimes, scheduler=scheduler) if retval != os.EX_OK: emergelog(xterm_titles, " !!! unmerge FAILURE: "+y) if raise_on_error: raise UninstallFailure(retval) sys.exit(retval) else: if clean_world and hasattr(sets["selected"], "cleanPackage")\ and hasattr(sets["selected"], "lock"): sets["selected"].lock() if hasattr(sets["selected"], "load"): sets["selected"].load() sets["selected"].cleanPackage(vartree.dbapi, y) sets["selected"].unlock() emergelog(xterm_titles, " >>> unmerge success: "+y) if clean_world and hasattr(sets["selected"], "remove")\ and hasattr(sets["selected"], "lock"): sets["selected"].lock() # load is called inside remove() for s in root_config.setconfig.active: sets["selected"].remove(SETPREFIX + s) sets["selected"].unlock() return os.EX_OK
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ 'mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ 'mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage ]) with open(os.path.join(cgroup_portage, 'release_agent'), 'w') as f: f.write( os.path.join(self.settings['PORTAGE_BIN_PATH'], 'cgroup-release-agent')) with open( os.path.join(cgroup_portage, 'notify_on_release'), 'w') as f: f.write('1') else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join(cgroup_portage, 'release_agent') try: with open(release_agent) as f: release_agent_path = f.readline().rstrip('\n') except EnvironmentError: release_agent_path = None if (release_agent_path is None or not os.path.exists(release_agent_path)): with open(release_agent, 'w') as f: f.write( os.path.join(self.settings['PORTAGE_BIN_PATH'], 'cgroup-release-agent')) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial(self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon)) return else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. @param filename: File to run the checksums against @type filename: String @param calc_prelink: Whether or not to reverse prelink before running the checksum @type calc_prelink: Integer @param strict: Enable/Disable strict checking (which stops exactly at a checksum failure and throws an exception) @type strict: Integer @rtype: Tuple @return: Result of the checks and possible message: 1) If size fails, False, and a tuple containing a message, the given size, and the actual size 2) If there is an os error, False, and a tuple containing the system error followed by 2 nulls 3) If a checksum fails, False and a tuple containing a message, the given hash, and the actual hash 4) If all checks succeed, return True and a fake reason """ # Dict relates to single file only. # returns: (passed,reason) file_is_ok = True reason = "Reason unknown" try: mysize = os.stat(filename)[stat.ST_SIZE] if mydict.get("size") is not None and mydict["size"] != mysize: return False, (_("Filesize does not match recorded size"), mysize, mydict["size"]) except OSError as e: if e.errno == errno.ENOENT: raise portage.exception.FileNotFound(filename) return False, (str(e), None, None) verifiable_hash_types = set(mydict).intersection(hashfunc_map) verifiable_hash_types.discard("size") if not verifiable_hash_types: expected = set(hashfunc_map) expected.discard("size") expected = list(expected) expected.sort() expected = " ".join(expected) got = set(mydict) got.discard("size") got = list(got) got.sort() got = " ".join(got) return False, (_("Insufficient data for checksum verification"), got, expected) for x in sorted(mydict): if x == "size": continue elif x in hashfunc_map: myhash = perform_checksum(filename, x, calc_prelink=calc_prelink)[0] if mydict[x] != myhash: if strict: raise portage.exception.DigestException( ("Failed to verify '$(file)s' on " + \ "checksum type '%(type)s'") % \ {"file" : filename, "type" : x}) else: file_is_ok = False reason = (("Failed on %s verification" % x), myhash, mydict[x]) break return file_is_ok, reason
def _read_dirVirtuals(self, profiles): """ Read the 'virtuals' file in all profiles. """ virtuals_list = [] for x in profiles: virtuals_file = os.path.join(x, "virtuals") virtuals_dict = grabdict(virtuals_file) atoms_dict = {} for k, v in virtuals_dict.items(): try: virt_atom = Atom(k) except InvalidAtom: virt_atom = None else: if virt_atom.blocker or str(virt_atom) != str( virt_atom.cp): virt_atom = None if virt_atom is None: writemsg( _("--- Invalid virtuals atom in %s: %s\n") % (virtuals_file, k), noiselevel=-1, ) continue providers = [] for atom in v: atom_orig = atom if atom[:1] == "-": # allow incrementals atom = atom[1:] try: atom = Atom(atom) except InvalidAtom: atom = None else: if atom.blocker: atom = None if atom is None: writemsg( _("--- Invalid atom in %s: %s\n") % (virtuals_file, atom_orig), noiselevel=-1, ) else: if atom_orig == str(atom): # normal atom, so return as Atom instance providers.append(atom) else: # atom has special prefix, so return as string providers.append(atom_orig) if providers: atoms_dict[virt_atom] = providers if atoms_dict: virtuals_list.append(atoms_dict) self._dirVirtuals = stack_dictlist(virtuals_list, incremental=True) for virt in self._dirVirtuals: # Preference for virtuals decreases from left to right. self._dirVirtuals[virt].reverse()
def parse_updates(mycontent): """Valid updates are returned as a list of split update commands.""" eapi_attrs = _get_eapi_attrs(None) slot_re = _get_slot_re(eapi_attrs) myupd = [] errors = [] mylines = mycontent.splitlines() for myline in mylines: mysplit = myline.split() if len(mysplit) == 0: continue if mysplit[0] not in ("move", "slotmove"): errors.append(_("ERROR: Update type not recognized '%s'") % myline) continue if mysplit[0] == "move": if len(mysplit) != 3: errors.append(_("ERROR: Update command invalid '%s'") % myline) continue valid = True for i in (1, 2): try: atom = Atom(mysplit[i]) except InvalidAtom: atom = None else: if atom.blocker or atom != atom.cp: atom = None if atom is not None: mysplit[i] = atom else: errors.append( _("ERROR: Malformed update entry '%s'") % myline) valid = False break if not valid: continue if mysplit[0] == "slotmove": if len(mysplit) != 4: errors.append(_("ERROR: Update command invalid '%s'") % myline) continue pkg, origslot, newslot = mysplit[1], mysplit[2], mysplit[3] try: atom = Atom(pkg) except InvalidAtom: atom = None else: if atom.blocker: atom = None if atom is not None: mysplit[1] = atom else: errors.append(_("ERROR: Malformed update entry '%s'") % myline) continue invalid_slot = False for slot in (origslot, newslot): m = slot_re.match(slot) if m is None: invalid_slot = True break if "/" in slot: # EAPI 4-slot-abi style SLOT is currently not supported. invalid_slot = True break if invalid_slot: errors.append(_("ERROR: Malformed update entry '%s'") % myline) continue # The list of valid updates is filtered by continue statements above. myupd.append(mysplit) return myupd, errors
def update_config_files(config_root, protect, protect_mask, update_iter, match_callback=None, case_insensitive=False): """Perform global updates on /etc/portage/package.*, /etc/portage/profile/package.*, /etc/portage/profile/packages and /etc/portage/sets. config_root - location of files to update protect - list of paths from CONFIG_PROTECT protect_mask - list of paths from CONFIG_PROTECT_MASK update_iter - list of update commands as returned from parse_updates(), or dict of {repo_name: list} match_callback - a callback which will be called with three arguments: match_callback(repo_name, old_atom, new_atom) and should return boolean value determining whether to perform the update""" repo_dict = None if isinstance(update_iter, dict): repo_dict = update_iter if match_callback is None: def match_callback(repo_name, atoma, atomb): return True config_root = normalize_path(config_root) update_files = {} file_contents = {} myxfiles = [ "package.accept_keywords", "package.env", "package.keywords", "package.license", "package.mask", "package.properties", "package.unmask", "package.use", "sets" ] myxfiles += [ os.path.join("profile", x) for x in ("packages", "package.accept_keywords", "package.keywords", "package.mask", "package.unmask", "package.use", "package.use.force", "package.use.mask", "package.use.stable.force", "package.use.stable.mask") ] abs_user_config = os.path.join(config_root, USER_CONFIG_PATH) recursivefiles = [] for x in myxfiles: config_file = os.path.join(abs_user_config, x) if os.path.isdir(config_file): for parent, dirs, files in os.walk(config_file): try: parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue for y_enc in list(dirs): try: y = _unicode_decode(y_enc, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: dirs.remove(y_enc) continue if y.startswith(".") or y in VCS_DIRS: dirs.remove(y_enc) for y in files: try: y = _unicode_decode(y, encoding=_encodings['fs'], errors='strict') except UnicodeDecodeError: continue if y.startswith("."): continue recursivefiles.append( os.path.join(parent, y)[len(abs_user_config) + 1:]) else: recursivefiles.append(x) myxfiles = recursivefiles for x in myxfiles: f = None try: f = io.open(_unicode_encode(os.path.join(abs_user_config, x), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') file_contents[x] = f.readlines() except IOError: continue finally: if f is not None: f.close() ignore_line_re = re.compile(r'^#|^\s*$') if repo_dict is None: update_items = [(None, update_iter)] else: update_items = [x for x in repo_dict.items() if x[0] != 'DEFAULT'] for repo_name, update_iter in update_items: for update_cmd in update_iter: for x, contents in file_contents.items(): skip_next = False for pos, line in enumerate(contents): if skip_next: skip_next = False continue if ignore_line_re.match(line): continue atom = line.split()[0] if atom[:1] == "-": # package.mask supports incrementals atom = atom[1:] if atom[:1] == "*": # packages file supports "*"-prefixed atoms as indication of system packages. atom = atom[1:] if not isvalidatom(atom): continue new_atom = update_dbentry(update_cmd, atom) if atom != new_atom: if match_callback(repo_name, atom, new_atom): # add a comment with the update command, so # the user can clearly see what happened contents[pos] = "# %s\n" % \ " ".join("%s" % (x,) for x in update_cmd) contents.insert( pos + 1, line.replace("%s" % (atom, ), "%s" % (new_atom, ), 1)) # we've inserted an additional line, so we need to # skip it when it's reached in the next iteration skip_next = True update_files[x] = 1 sys.stdout.write("p") sys.stdout.flush() protect_obj = ConfigProtect(config_root, protect, protect_mask, case_insensitive=case_insensitive) for x in update_files: updating_file = os.path.join(abs_user_config, x) if protect_obj.isprotected(updating_file): updating_file = new_protect_filename(updating_file) try: write_atomic(updating_file, "".join(file_contents[x])) except PortageException as e: writemsg("\n!!! %s\n" % str(e), noiselevel=-1) writemsg(_("!!! An error occurred while updating a config file:") + \ " '%s'\n" % updating_file, noiselevel=-1) continue
def create_conn(baseurl, conn=None): """(baseurl,conn) --- Takes a protocol://site:port/address url, and an optional connection. If connection is already active, it is passed on. baseurl is reduced to address and is returned in tuple (conn,address)""" parts = baseurl.split("://", 1) if len(parts) != 2: raise ValueError( _("Provided URI does not " "contain protocol identifier. '%s'") % baseurl) protocol, url_parts = parts del parts url_parts = url_parts.split("/") host = url_parts[0] if len(url_parts) < 2: address = "/" else: address = "/" + "/".join(url_parts[1:]) del url_parts userpass_host = host.split("@", 1) if len(userpass_host) == 1: host = userpass_host[0] userpass = ["anonymous"] else: host = userpass_host[1] userpass = userpass_host[0].split(":") del userpass_host if len(userpass) > 2: raise ValueError(_("Unable to interpret username/password provided.")) elif len(userpass) == 2: username = userpass[0] password = userpass[1] elif len(userpass) == 1: username = userpass[0] password = None del userpass http_headers = {} http_params = {} if username and password: try: encodebytes = base64.encodebytes except AttributeError: # Python 2 encodebytes = base64.encodestring http_headers = { b"Authorization": "Basic %s" % \ encodebytes(_unicode_encode("%s:%s" % (username, password))).replace( b"\012", b"" ), } if not conn: if protocol == "https": # Use local import since https typically isn't needed, and # this way we can usually avoid triggering the global scope # http.client ImportError handler (like during stage1 -> stage2 # builds where USE=ssl is disabled for python). try: try: from http.client import HTTPSConnection as http_client_HTTPSConnection except ImportError: from httplib import HTTPSConnection as http_client_HTTPSConnection except ImportError: raise NotImplementedError( _("python must have ssl enabled for https support")) conn = http_client_HTTPSConnection(host) elif protocol == "http": conn = http_client_HTTPConnection(host) elif protocol == "ftp": passive = 1 if (host[-1] == "*"): passive = 0 host = host[:-1] conn = ftplib.FTP(host) if password: conn.login(username, password) else: sys.stderr.write(colorize("WARN", _(" * No password provided for username"))+" '%s'" % \ (username,) + "\n\n") conn.login(username) conn.set_pasv(passive) conn.set_debuglevel(0) elif protocol == "sftp": try: import paramiko except ImportError: raise NotImplementedError( _("paramiko must be installed for sftp support")) t = paramiko.Transport(host) t.connect(username=username, password=password) conn = paramiko.SFTPClient.from_transport(t) else: raise NotImplementedError( _("%s is not a supported protocol.") % protocol) return (conn, protocol, address, http_params, http_headers)
def __init__(self, repositories, profiles, abs_user_config, user_config=True, strict_umatched_removal=False): self._punmaskdict = ExtendedAtomDict(list) self._pmaskdict = ExtendedAtomDict(list) # Preserves atoms that are eliminated by negative # incrementals in user_pkgmasklines. self._pmaskdict_raw = ExtendedAtomDict(list) #Read profile/package.mask from every repo. #Repositories inherit masks from their parent profiles and #are able to remove mask from them with -atoms. #Such a removal affects only the current repo, but not the parent. #Add ::repo specs to every atom to make sure atoms only affect #packages from the current repo. # Cache the repository-wide package.mask files as a particular # repo may be often referenced by others as the master. pmask_cache = {} def grab_pmask(loc, repo_config): if loc not in pmask_cache: path = os.path.join(loc, 'profiles', 'package.mask') pmask_cache[loc] = grabfile_package( path, recursive=repo_config.portage1_profiles, remember_source_file=True, verify_eapi=True, eapi_default=repo_config.eapi, allow_build_id=("build-id" in repo_config.profile_formats)) if repo_config.portage1_profiles_compat and os.path.isdir( path): warnings.warn( _("Repository '%(repo_name)s' is implicitly using " "'portage-1' profile format in its profiles/package.mask, but " "the repository profiles are not marked as that format. This will break " "in the future. Please either convert the following paths " "to files, or add\nprofile-formats = portage-1\nto the " "repository's layout.conf.\n") % dict(repo_name=repo_config.name)) return pmask_cache[loc] repo_pkgmasklines = [] for repo in repositories.repos_with_profiles(): lines = [] repo_lines = grab_pmask(repo.location, repo) removals = frozenset(line[0][1:] for line in repo_lines if line[0][:1] == "-") matched_removals = set() for master in repo.masters: master_lines = grab_pmask(master.location, master) for line in master_lines: if line[0] in removals: matched_removals.add(line[0]) # Since we don't stack masters recursively, there aren't any # atoms earlier in the stack to be matched by negative atoms in # master_lines. Also, repo_lines may contain negative atoms # that are intended to negate atoms from a different master # than the one with which we are currently stacking. Therefore, # we disable warn_for_unmatched_removal here (see bug #386569). lines.append( stack_lists([master_lines, repo_lines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=False)) # It's safe to warn for unmatched removal if masters have not # been overridden by the user, which is guaranteed when # user_config is false (when called by repoman). if repo.masters: unmatched_removals = removals.difference(matched_removals) if unmatched_removals and not user_config: source_file = os.path.join(repo.location, "profiles", "package.mask") unmatched_removals = list(unmatched_removals) if len(unmatched_removals) > 3: writemsg(_( "--- Unmatched removal atoms in %s: %s and %s more\n" ) % (source_file, ", ".join( "-" + x for x in unmatched_removals[:3]), len(unmatched_removals) - 3), noiselevel=-1) else: writemsg( _("--- Unmatched removal atom(s) in %s: %s\n") % (source_file, ", ".join( "-" + x for x in unmatched_removals)), noiselevel=-1) else: lines.append( stack_lists([repo_lines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=not user_config, strict_warn_for_unmatched_removal= strict_umatched_removal)) repo_pkgmasklines.extend( append_repo(stack_lists(lines), repo.name, remember_source_file=True)) repo_pkgunmasklines = [] for repo in repositories.repos_with_profiles(): if not repo.portage1_profiles: continue repo_lines = grabfile_package(os.path.join(repo.location, "profiles", "package.unmask"), \ recursive=1, remember_source_file=True, verify_eapi=True, eapi_default=repo.eapi, allow_build_id=("build-id" in repo.profile_formats)) lines = stack_lists([repo_lines], incremental=1, \ remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal) repo_pkgunmasklines.extend( append_repo(lines, repo.name, remember_source_file=True)) #Read package.mask from the user's profile. Stack them in the end #to allow profiles to override masks from their parent profiles. profile_pkgmasklines = [] profile_pkgunmasklines = [] for x in profiles: profile_pkgmasklines.append( grabfile_package(os.path.join(x.location, "package.mask"), recursive=x.portage1_directories, remember_source_file=True, verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_build_id=x.allow_build_id)) if x.portage1_directories: profile_pkgunmasklines.append( grabfile_package(os.path.join(x.location, "package.unmask"), recursive=x.portage1_directories, remember_source_file=True, verify_eapi=True, eapi=x.eapi, eapi_default=None, allow_build_id=x.allow_build_id)) profile_pkgmasklines = stack_lists(profile_pkgmasklines, incremental=1, \ remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal) profile_pkgunmasklines = stack_lists(profile_pkgunmasklines, incremental=1, \ remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal) #Read /etc/portage/package.mask. Don't stack it to allow the user to #remove mask atoms from everywhere with -atoms. user_pkgmasklines = [] user_pkgunmasklines = [] if user_config: user_pkgmasklines = grabfile_package( os.path.join(abs_user_config, "package.mask"), recursive=1, \ allow_wildcard=True, allow_repo=True, remember_source_file=True, verify_eapi=False, allow_build_id=True) user_pkgunmasklines = grabfile_package( os.path.join(abs_user_config, "package.unmask"), recursive=1, \ allow_wildcard=True, allow_repo=True, remember_source_file=True, verify_eapi=False, allow_build_id=True) #Stack everything together. At this point, only user_pkgmasklines may contain -atoms. #Don't warn for unmatched -atoms here, since we don't do it for any other user config file. raw_pkgmasklines = stack_lists([repo_pkgmasklines, profile_pkgmasklines], \ incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, ignore_repo=True) pkgmasklines = stack_lists([repo_pkgmasklines, profile_pkgmasklines, user_pkgmasklines], \ incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, ignore_repo=True) pkgunmasklines = stack_lists([repo_pkgunmasklines, profile_pkgunmasklines, user_pkgunmasklines], \ incremental=1, remember_source_file=True, warn_for_unmatched_removal=False, ignore_repo=True) for x, source_file in raw_pkgmasklines: self._pmaskdict_raw.setdefault(x.cp, []).append(x) for x, source_file in pkgmasklines: self._pmaskdict.setdefault(x.cp, []).append(x) for x, source_file in pkgunmasklines: self._punmaskdict.setdefault(x.cp, []).append(x) for d in (self._pmaskdict_raw, self._pmaskdict, self._punmaskdict): for k, v in d.items(): d[k] = tuple(v)
def _get_global(k): if k in _initialized_globals: return globals()[k] if k == 'secpass': unprivileged = False if hasattr(portage, 'settings'): unprivileged = "unprivileged" in portage.settings.features else: # The config class has equivalent code, but we also need to # do it here if _disable_legacy_globals() has been called. eroot_or_parent = first_existing( os.path.join(_target_root(), _target_eprefix().lstrip(os.sep))) try: eroot_st = os.stat(eroot_or_parent) except OSError: pass else: unprivileged = _unprivileged_mode(eroot_or_parent, eroot_st) v = 0 if uid == 0: v = 2 elif unprivileged: v = 2 elif portage_gid in os.getgroups(): v = 1 elif k in ('portage_gid', 'portage_uid'): #Discover the uid and gid of the portage user/group keyerror = False try: portage_uid = pwd.getpwnam(_get_global('_portage_username')).pw_uid except KeyError: keyerror = True portage_uid = 0 try: portage_gid = grp.getgrnam(_get_global('_portage_grpname')).gr_gid except KeyError: keyerror = True portage_gid = 0 # Suppress this error message if both PORTAGE_GRPNAME and # PORTAGE_USERNAME are set to "root", for things like # Android (see bug #454060). if keyerror and not (_get_global('_portage_username') == "root" and _get_global('_portage_grpname') == "root"): writemsg(colorize( "BAD", _("portage: 'portage' user or group missing.")) + "\n", noiselevel=-1) writemsg(_(" For the defaults, line 1 goes into passwd, " "and 2 into group.\n"), noiselevel=-1) writemsg(colorize("GOOD", " portage:x:250:250:portage:/var/tmp/portage:/bin/false") \ + "\n", noiselevel=-1) writemsg(colorize("GOOD", " portage::250:portage") + "\n", noiselevel=-1) portage_group_warning() globals()['portage_gid'] = portage_gid _initialized_globals.add('portage_gid') globals()['portage_uid'] = portage_uid _initialized_globals.add('portage_uid') if k == 'portage_gid': return portage_gid elif k == 'portage_uid': return portage_uid else: raise AssertionError('unknown name: %s' % k) elif k == 'userpriv_groups': v = [_get_global('portage_gid')] if secpass >= 2: # Get a list of group IDs for the portage user. Do not use # grp.getgrall() since it is known to trigger spurious # SIGPIPE problems with nss_ldap. cmd = ["id", "-G", _portage_username] if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000: # Python 3.1 _execvp throws TypeError for non-absolute executable # path passed as bytes (see http://bugs.python.org/issue8513). fullname = portage.process.find_binary(cmd[0]) if fullname is None: globals()[k] = v _initialized_globals.add(k) return v cmd[0] = fullname encoding = portage._encodings['content'] cmd = [ portage._unicode_encode(x, encoding=encoding, errors='strict') for x in cmd ] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) myoutput = proc.communicate()[0] status = proc.wait() if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: for x in portage._unicode_decode(myoutput, encoding=encoding, errors='strict').split(): try: v.append(int(x)) except ValueError: pass v = sorted(set(v)) # Avoid instantiating portage.settings when the desired # variable is set in os.environ. elif k in ('_portage_grpname', '_portage_username'): v = None if k == '_portage_grpname': env_key = 'PORTAGE_GRPNAME' else: env_key = 'PORTAGE_USERNAME' if env_key in os.environ: v = os.environ[env_key] elif hasattr(portage, 'settings'): v = portage.settings.get(env_key) else: # The config class has equivalent code, but we also need to # do it here if _disable_legacy_globals() has been called. eroot_or_parent = first_existing( os.path.join(_target_root(), _target_eprefix().lstrip(os.sep))) try: eroot_st = os.stat(eroot_or_parent) except OSError: pass else: if _unprivileged_mode(eroot_or_parent, eroot_st): if k == '_portage_grpname': try: grp_struct = grp.getgrgid(eroot_st.st_gid) except KeyError: pass else: v = grp_struct.gr_name else: try: pwd_struct = pwd.getpwuid(eroot_st.st_uid) except KeyError: pass else: v = pwd_struct.pw_name if v is None: v = 'portage' else: raise AssertionError('unknown name: %s' % k) globals()[k] = v _initialized_globals.add(k) return v
def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, hardlink_candidates=None, encoding=_encodings['fs']): """moves a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns mtime as integer on success and None on failure. mtime is expressed in seconds in Python <3.3 and nanoseconds in Python >=3.3. Move is atomic.""" if mysettings is None: mysettings = portage.settings src_bytes = _unicode_encode(src, encoding=encoding, errors='strict') dest_bytes = _unicode_encode(dest, encoding=encoding, errors='strict') xattr_enabled = "xattr" in mysettings.features selinux_enabled = mysettings.selinux_enabled() if selinux_enabled: selinux = _unicode_module_wrapper(_selinux, encoding=encoding) _copyfile = selinux.copyfile _rename = selinux.rename else: _copyfile = copyfile _rename = _os.rename lchown = _unicode_func_wrapper(portage.data.lchown, encoding=encoding) os = _unicode_module_wrapper(_os, encoding=encoding, overrides=_os_overrides) try: if not sstat: sstat = os.lstat(src) except SystemExit as e: raise except Exception as e: writemsg("!!! %s\n" % _("Stating source file failed... movefile()"), noiselevel=-1) writemsg("!!! %s\n" % (e, ), noiselevel=-1) return None destexists = 1 try: dstat = os.lstat(dest) except (OSError, IOError): dstat = os.lstat(os.path.dirname(dest)) destexists = 0 if bsd_chflags: if destexists and dstat.st_flags != 0: bsd_chflags.lchflags(dest, 0) # Use normal stat/chflags for the parent since we want to # follow any symlinks to the real parent directory. pflags = os.stat(os.path.dirname(dest)).st_flags if pflags != 0: bsd_chflags.chflags(os.path.dirname(dest), 0) if destexists: if stat.S_ISLNK(dstat[stat.ST_MODE]): try: os.unlink(dest) destexists = 0 except SystemExit as e: raise except Exception as e: pass if stat.S_ISLNK(sstat[stat.ST_MODE]): try: target = os.readlink(src) if mysettings and "D" in mysettings and \ target.startswith(mysettings["D"]): target = target[len(mysettings["D"]) - 1:] if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): os.unlink(dest) try: if selinux_enabled: selinux.symlink(target, dest, src) else: os.symlink(target, dest) except OSError as e: # Some programs will create symlinks automatically, so we have # to tolerate these links being recreated during the merge # process. In any case, if the link is pointing at the right # place, we're in good shape. if e.errno not in (errno.ENOENT, errno.EEXIST) or \ target != os.readlink(dest): raise lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID]) try: _os.unlink(src_bytes) except OSError: pass try: os.utime(dest, ns=(sstat.st_mtime_ns, sstat.st_mtime_ns), follow_symlinks=False) except NotImplementedError: # utimensat() and lutimes() missing in libc. return os.stat(dest, follow_symlinks=False).st_mtime_ns else: return sstat.st_mtime_ns except SystemExit as e: raise except Exception as e: writemsg("!!! %s\n" % _("failed to properly create symlink:"), noiselevel=-1) writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1) writemsg("!!! %s\n" % (e, ), noiselevel=-1) return None hardlinked = False # Since identical files might be merged to multiple filesystems, # so os.link() calls might fail for some paths, so try them all. # For atomic replacement, first create the link as a temp file # and them use os.rename() to replace the destination. if hardlink_candidates: head, tail = os.path.split(dest) hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \ (tail, portage.getpid())) try: os.unlink(hardlink_tmp) except OSError as e: if e.errno != errno.ENOENT: writemsg(_("!!! Failed to remove hardlink temp file: %s\n") % \ (hardlink_tmp,), noiselevel=-1) writemsg("!!! %s\n" % (e, ), noiselevel=-1) return None del e for hardlink_src in hardlink_candidates: try: os.link(hardlink_src, hardlink_tmp) except OSError: continue else: try: os.rename(hardlink_tmp, dest) except OSError as e: writemsg(_("!!! Failed to rename %s to %s\n") % \ (hardlink_tmp, dest), noiselevel=-1) writemsg("!!! %s\n" % (e, ), noiselevel=-1) return None hardlinked = True try: _os.unlink(src_bytes) except OSError: pass break renamefailed = 1 if hardlinked: renamefailed = False if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev): try: if selinux_enabled: selinux.rename(src, dest) else: os.rename(src, dest) renamefailed = 0 except OSError as e: if e.errno != errno.EXDEV: # Some random error. writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") % { "src": src, "dest": dest }, noiselevel=-1) writemsg("!!! %s\n" % (e, ), noiselevel=-1) return None # Invalid cross-device-link 'bind' mounted or actually Cross-Device if renamefailed: if stat.S_ISREG(sstat[stat.ST_MODE]): dest_tmp = dest + "#new" dest_tmp_bytes = _unicode_encode(dest_tmp, encoding=encoding, errors='strict') success = False try: # For safety copy then move it over. _copyfile(src_bytes, dest_tmp_bytes) _apply_stat(sstat, dest_tmp_bytes) if xattr_enabled: try: _copyxattr(src_bytes, dest_tmp_bytes, exclude=mysettings.get( "PORTAGE_XATTR_EXCLUDE", "")) except SystemExit: raise except: msg = _("Failed to copy extended attributes. " "In order to avoid this error, set " "FEATURES=\"-xattr\" in make.conf.") msg = textwrap.wrap(msg, 65) for line in msg: writemsg("!!! %s\n" % (line, ), noiselevel=-1) raise _rename(dest_tmp_bytes, dest_bytes) _os.unlink(src_bytes) success = True except Exception as e: writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') % { "src": src, "dest": dest }, noiselevel=-1) writemsg("!!! %s\n" % (e, ), noiselevel=-1) return None finally: if not success: try: _os.unlink(dest_tmp_bytes) except OSError: pass else: #we don't yet handle special, so we need to fall back to /bin/mv a = spawn([MOVE_BINARY, '-f', src, dest], env=os.environ) if a != os.EX_OK: writemsg(_("!!! Failed to move special file:\n"), noiselevel=-1) writemsg(_("!!! '%(src)s' to '%(dest)s'\n") % \ {"src": _unicode_decode(src, encoding=encoding), "dest": _unicode_decode(dest, encoding=encoding)}, noiselevel=-1) writemsg("!!! %s\n" % a, noiselevel=-1) return None # failure # In Python <3.3 always use stat_obj[stat.ST_MTIME] for the integral timestamp # which is returned, since the stat_obj.st_mtime float attribute rounds *up* # if the nanosecond part of the timestamp is 999999881 ns or greater. try: if hardlinked: newmtime = os.stat(dest).st_mtime_ns else: # Note: It is not possible to preserve nanosecond precision # (supported in POSIX.1-2008 via utimensat) with the IEEE 754 # double precision float which only has a 53 bit significand. if newmtime is not None: os.utime(dest, ns=(newmtime, newmtime)) else: newmtime = sstat.st_mtime_ns if renamefailed: # If rename succeeded then timestamps are automatically # preserved with complete precision because the source # and destination inodes are the same. Otherwise, manually # update timestamps with nanosecond precision. os.utime(dest, ns=(newmtime, newmtime)) except OSError: # The utime can fail here with EPERM even though the move succeeded. # Instead of failing, use stat to return the mtime if possible. try: newmtime = os.stat(dest).st_mtime_ns except OSError as e: writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1) writemsg("!!! %s\n" % dest, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) return None if bsd_chflags: # Restore the flags we saved before moving if pflags: bsd_chflags.chflags(os.path.dirname(dest), pflags) return newmtime
def __init__(self, paths, settings): """Load config from files in paths""" prepos = {} location_map = {} treemap = {} ignored_map = {} ignored_location_map = {} if "PORTAGE_REPOSITORIES" in settings: portdir = "" portdir_overlay = "" portdir_sync = "" else: portdir = settings.get("PORTDIR", "") portdir_overlay = settings.get("PORTDIR_OVERLAY", "") portdir_sync = settings.get("SYNC", "") try: self._parse(paths, prepos, ignored_map, ignored_location_map, settings.local_config, portdir) except ConfigParserError as e: writemsg(_("!!! Error while reading repo config file: %s\n") % e, noiselevel=-1) # The configparser state is unreliable (prone to quirky # exceptions) after it has thrown an error, so use empty # config and try to fall back to PORTDIR{,_OVERLAY}. prepos.clear() prepos['DEFAULT'] = RepoConfig('DEFAULT', {}, local_config=settings.local_config) location_map.clear() treemap.clear() ignored_map.clear() ignored_location_map.clear() default_portdir = os.path.join(os.sep, settings['EPREFIX'].lstrip(os.sep), 'usr', 'portage') # If PORTDIR_OVERLAY contains a repo with the same repo_name as # PORTDIR, then PORTDIR is overridden. portdir = self._add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map, settings.local_config, default_portdir) if portdir and portdir.strip(): portdir = os.path.realpath(portdir) ignored_repos = tuple((repo_name, tuple(paths)) \ for repo_name, paths in ignored_map.items()) self.missing_repo_names = frozenset( repo.location for repo in prepos.values() if repo.location is not None and repo.missing_repo_name) # Do this before expanding aliases, so that location_map and # treemap consistently map unaliased names whenever available. for repo_name, repo in list(prepos.items()): if repo.location is None: if repo_name != 'DEFAULT': # Skip this warning for repoman (bug #474578). if settings.local_config and paths: writemsg_level("!!! %s\n" % _( "Section '%s' in repos.conf is missing location attribute" ) % repo.name, level=logging.ERROR, noiselevel=-1) del prepos[repo_name] continue else: if not portage._sync_mode: if not isdir_raise_eaccess(repo.location): writemsg_level("!!! %s\n" % _( "Section '%s' in repos.conf has location attribute set " "to nonexistent directory: '%s'") % (repo_name, repo.location), level=logging.ERROR, noiselevel=-1) # Ignore missing directory for 'gentoo' so that # first sync with emerge-webrsync is possible. if repo.name != 'gentoo': del prepos[repo_name] continue # After removing support for PORTDIR_OVERLAY, the following check can be: # if repo.missing_repo_name: if repo.missing_repo_name and repo.name != repo_name: writemsg_level("!!! %s\n" % _( "Section '%s' in repos.conf refers to repository " "without repository name set in '%s'") % (repo_name, os.path.join(repo.location, REPO_NAME_LOC)), level=logging.ERROR, noiselevel=-1) del prepos[repo_name] continue if repo.name != repo_name: writemsg_level("!!! %s\n" % _( "Section '%s' in repos.conf has name different " "from repository name '%s' set inside repository") % (repo_name, repo.name), level=logging.ERROR, noiselevel=-1) del prepos[repo_name] continue location_map[repo.location] = repo_name treemap[repo_name] = repo.location # Add alias mappings, but never replace unaliased mappings. for repo_name, repo in list(prepos.items()): names = set() names.add(repo_name) if repo.aliases: aliases = stack_lists([repo.aliases], incremental=True) names.update(aliases) for name in names: if name in prepos and prepos[name].location is not None: if name == repo_name: # unaliased names already handled earlier continue writemsg_level(_("!!! Repository name or alias '%s', " + \ "defined for repository '%s', overrides " + \ "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1) # Never replace an unaliased mapping with # an aliased mapping. continue prepos[name] = repo if repo.location is not None: if repo.location not in location_map: # Never replace an unaliased mapping with # an aliased mapping. location_map[repo.location] = name treemap[name] = repo.location main_repo = prepos['DEFAULT'].main_repo if main_repo is None or main_repo not in prepos: #setting main_repo if it was not set in repos.conf main_repo = location_map.get(portdir) if main_repo is not None: prepos['DEFAULT'].main_repo = main_repo else: prepos['DEFAULT'].main_repo = None if portdir and not portage._sync_mode: writemsg(_( "!!! main-repo not set in DEFAULT and PORTDIR is empty.\n" ), noiselevel=-1) if main_repo is not None and prepos[main_repo].priority is None: # This happens if main-repo has been set in repos.conf. prepos[main_repo].priority = -1000 # Backward compatible SYNC support for mirrorselect. if portdir_sync and main_repo is not None: if portdir_sync.startswith("rsync://"): prepos[main_repo].sync_uri = portdir_sync prepos[main_repo].sync_type = "rsync" # Include repo.name in sort key, for predictable sorting # even when priorities are equal. prepos_order = sorted(prepos.items(), key=lambda r: (r[1].priority or 0, r[1].name)) # filter duplicates from aliases, by only including # items where repo.name == key prepos_order = [ repo.name for (key, repo) in prepos_order if repo.name == key and key != 'DEFAULT' and repo.location is not None ] self.prepos = prepos self.prepos_order = prepos_order self.ignored_repos = ignored_repos self.location_map = location_map self.treemap = treemap self._prepos_changed = True self._repo_location_list = [] #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig. for repo_name, repo in prepos.items(): if repo_name == "DEFAULT": continue if repo.masters is None: if self.mainRepo() and repo_name != self.mainRepo().name: repo.masters = self.mainRepo(), else: repo.masters = () else: if repo.masters and isinstance(repo.masters[0], RepoConfig): # This one has already been processed # because it has an alias. continue master_repos = [] for master_name in repo.masters: if master_name not in prepos: layout_filename = os.path.join(repo.user_location, "metadata", "layout.conf") writemsg_level(_("Unavailable repository '%s' " \ "referenced by masters entry in '%s'\n") % \ (master_name, layout_filename), level=logging.ERROR, noiselevel=-1) else: master_repos.append(prepos[master_name]) repo.masters = tuple(master_repos) #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths. for repo_name, repo in prepos.items(): if repo_name == "DEFAULT": continue eclass_locations = [] eclass_locations.extend(master_repo.location for master_repo in repo.masters) # Only append the current repo to eclass_locations if it's not # there already. This allows masters to have more control over # eclass override order, which may be useful for scenarios in # which there is a plan to migrate eclasses to a master repo. if repo.location not in eclass_locations: eclass_locations.append(repo.location) if repo.eclass_overrides: for other_repo_name in repo.eclass_overrides: if other_repo_name in self.treemap: eclass_locations.append( self.get_location_for_name(other_repo_name)) else: writemsg_level(_("Unavailable repository '%s' " \ "referenced by eclass-overrides entry for " \ "'%s'\n") % (other_repo_name, repo_name), \ level=logging.ERROR, noiselevel=-1) repo.eclass_locations = tuple(eclass_locations) eclass_dbs = {} for repo_name, repo in prepos.items(): if repo_name == "DEFAULT": continue eclass_db = None for eclass_location in repo.eclass_locations: tree_db = eclass_dbs.get(eclass_location) if tree_db is None: tree_db = eclass_cache.cache(eclass_location) eclass_dbs[eclass_location] = tree_db if eclass_db is None: eclass_db = tree_db.copy() else: eclass_db.append(tree_db) repo.eclass_db = eclass_db for repo_name, repo in prepos.items(): if repo_name == "DEFAULT": continue if repo._masters_orig is None and self.mainRepo() and \ repo.name != self.mainRepo().name and not portage._sync_mode: # TODO: Delete masters code in pym/portage/tests/resolver/ResolverPlayground.py when deleting this warning. writemsg_level( "!!! %s\n" % _("Repository '%s' is missing masters attribute in '%s'") % (repo.name, os.path.join(repo.location, "metadata", "layout.conf")) + "!!! %s\n" % _("Set 'masters = %s' in this file for future compatibility" ) % self.mainRepo().name, level=logging.WARNING, noiselevel=-1) self._prepos_changed = True self._repo_location_list = [] self._check_locations()
def _builderGetRepository(cls, options, repositories): repository = options.get("repository", "porttree") if not repository in repositories: raise SetConfigError(_("invalid repository class '%s'") % repository) return repository
def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map, local_config, default_portdir): """Add overlays in PORTDIR_OVERLAY as repositories""" overlays = [] portdir_orig = None if portdir: portdir = normalize_path(portdir) portdir_orig = portdir overlays.append(portdir) try: port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] except ValueError as e: #File "/usr/lib/python3.2/shlex.py", line 168, in read_token # raise ValueError("No closing quotation") writemsg(_("!!! Invalid PORTDIR_OVERLAY:" " %s: %s\n") % (e, portdir_overlay), noiselevel=-1) port_ov = [] overlays.extend(port_ov) default_repo_opts = {} if prepos['DEFAULT'].aliases is not None: default_repo_opts['aliases'] = \ ' '.join(prepos['DEFAULT'].aliases) if prepos['DEFAULT'].eclass_overrides is not None: default_repo_opts['eclass-overrides'] = \ ' '.join(prepos['DEFAULT'].eclass_overrides) if prepos['DEFAULT'].masters is not None: default_repo_opts['masters'] = \ ' '.join(prepos['DEFAULT'].masters) if overlays: # We need a copy of the original repos.conf data, since we're # going to modify the prepos dict and some of the RepoConfig # objects that we put in prepos may have to be discarded if # they get overridden by a repository with the same name but # a different location. This is common with repoman, for example, # when temporarily overriding an rsync repo with another copy # of the same repo from CVS. repos_conf = prepos.copy() #overlay priority is negative because we want them to be looked before any other repo base_priority = 0 for ov in overlays: # Ignore missing directory for 'gentoo' so that # first sync with emerge-webrsync is possible. if isdir_raise_eaccess(ov) or \ (base_priority == 0 and ov is portdir): repo_opts = default_repo_opts.copy() repo_opts['location'] = ov repo = RepoConfig(None, repo_opts, local_config=local_config) # repos_conf_opts contains options from repos.conf repos_conf_opts = repos_conf.get(repo.name) if repos_conf_opts is not None: # Selectively copy only the attributes which # repos.conf is allowed to override. for k in ('aliases', 'eclass_overrides', 'force', 'masters', 'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'): v = getattr(repos_conf_opts, k, None) if v is not None: setattr(repo, k, v) if repo.name in prepos: # Silently ignore when PORTDIR overrides the location # setting from the default repos.conf (bug #478544). old_location = prepos[repo.name].location if old_location is not None and \ old_location != repo.location and \ not (base_priority == 0 and old_location == default_portdir): ignored_map.setdefault(repo.name, []).append(old_location) ignored_location_map[old_location] = repo.name if old_location == portdir: portdir = repo.user_location if repo.priority is None: if base_priority == 0 and ov == portdir_orig: # If it's the original PORTDIR setting and it's not # in PORTDIR_OVERLAY, then it will be assigned a # special priority setting later. pass else: repo.priority = base_priority base_priority += 1 prepos[repo.name] = repo else: if not portage._sync_mode: writemsg( _("!!! Invalid PORTDIR_OVERLAY (not a dir): '%s'\n" ) % ov, noiselevel=-1) return portdir
def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, waiting_msg=None, flags=0): """ Acquire a lock on mypath, without retry. Return None if the lockfile was removed by previous lock holder (caller must retry). @param mypath: lock file path @type mypath: str @param wantnewlockfile: use a separate new lock file @type wantnewlockfile: bool @param unlinkfile: remove lock file prior to unlock @type unlinkfile: bool @param waiting_msg: message to show before blocking @type waiting_msg: str @param flags: lock flags (only supports os.O_NONBLOCK) @type flags: int @rtype: bool @return: unlockfile tuple on success, None if retry is needed """ if not mypath: raise InvalidData(_("Empty path given")) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, str) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn( "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn( "portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, str): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: while True: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir( os.path.dirname(lockfilename)): # Retry required for NFS (see bug 636798). continue else: _raise_exc(e) else: break if not preexisting: try: if portage.data.secpass >= 1 and os.stat( lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): os.close(myfd) return None writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = portage._eintr_func_wrapper(_get_lock_fn()) try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. enolock_msg_shown = False while True: try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if e.errno == errno.ENOLCK: # This is known to occur on Solaris NFS (see # bug #462694). Assume that the error is due # to temporary exhaustion of record locks, # and loop until one becomes available. if not enolock_msg_shown: enolock_msg_shown = True if isinstance(mypath, int): context_desc = _("Error while waiting " "to lock fd %i") % myfd else: context_desc = _("Error while waiting " "to lock '%s'") % lockfilename writemsg("\n!!! %s: %s\n" % (context_desc, e), noiselevel=-1) time.sleep(_HARDLINK_POLL_LATENCY) continue if out is not None: out.eend(1, str(e)) raise else: break if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS, ): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, str): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise fstat_result = None if isinstance(lockfilename, str) and myfd != HARDLINK_FD and unlinkfile: try: (removed, fstat_result) = _lockfile_was_removed(myfd, lockfilename) except Exception: # Do not leak the file descriptor here. os.close(myfd) raise else: if removed: # Removed by previous lock holder... Caller will retry... os.close(myfd) return None if myfd != HARDLINK_FD: _lock_manager(myfd, os.fstat(myfd) if fstat_result is None else fstat_result, mypath) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def _start(self): tar_options = "" if "xattr" in self.features: process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = process.communicate()[0] if b"--xattrs" in output: tar_options = ["--xattrs", "--xattrs-include='*'"] for x in portage.util.shlex_split(self.env.get("PORTAGE_XATTR_EXCLUDE", "")): tar_options.append(portage._shell_quote("--xattrs-exclude=%s" % x)) tar_options = " ".join(tar_options) decomp = _compressors.get(compression_probe(self.pkg_path)) if decomp is not None: decomp_cmd = decomp.get("decompress") else: decomp_cmd = None if decomp_cmd is None: self.scheduler.output("!!! %s\n" % _("File compression header unrecognized: %s") % self.pkg_path, log_path=self.logfile, background=self.background, level=logging.ERROR) self.returncode = 1 self._async_wait() return try: decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0] except IndexError: decompression_binary = "" if find_binary(decompression_binary) is None: # Try alternative command if it exists if _compressors.get(compression_probe(self.pkg_path)).get("decompress_alt"): decomp_cmd = _compressors.get( compression_probe(self.pkg_path)).get("decompress_alt") try: decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0] except IndexError: decompression_binary = "" if find_binary(decompression_binary) is None: missing_package = _compressors.get(compression_probe(self.pkg_path)).get("package") self.scheduler.output("!!! %s\n" % _("File compression unsupported %s.\n Command was: %s.\n Maybe missing package: %s") % (self.pkg_path, varexpand(decomp_cmd, mydict=self.env), missing_package), log_path=self.logfile, background=self.background, level=logging.ERROR) self.returncode = 1 self._async_wait() return # Add -q to decomp_cmd opts, in order to avoid "trailing garbage # after EOF ignored" warning messages due to xpak trailer. # SIGPIPE handling (128 + SIGPIPE) should be compatible with # assert_sigpipe_ok() that's used by the ebuild unpack() helper. self.args = [self._shell_binary, "-c", ("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \ "p=(${PIPESTATUS[@]}) ; " + \ "if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \ "echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \ "if [ ${p[1]} != 0 ] ; then " + \ "echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \ "exit 0 ;") % \ (decomp_cmd, portage._shell_quote(self.pkg_path), tar_options, portage._shell_quote(self.image_dir))] SpawnProcess._start(self)
def _parse_color_map(config_root='/', onerror=None): """ Parse /etc/portage/color.map and return a dict of error codes. @param onerror: an optional callback to handle any ParseError that would otherwise be raised @type onerror: callable @rtype: dict @return: a dictionary mapping color classes to color codes """ global codes, _styles myfile = os.path.join(config_root, COLOR_MAP_FILE) ansi_code_pattern = re.compile("^[0-9;]*m$") quotes = '\'"' def strip_quotes(token): if token[0] in quotes and token[0] == token[-1]: token = token[1:-1] return token f = None try: f = io.open(_unicode_encode(myfile, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') lineno = 0 for line in f: lineno += 1 commenter_pos = line.find("#") line = line[:commenter_pos].strip() if len(line) == 0: continue split_line = line.split("=") if len(split_line) != 2: e = ParseError(_("'%s', line %s: expected exactly one occurrence of '=' operator") % \ (myfile, lineno)) raise e if onerror: onerror(e) else: raise e continue k = strip_quotes(split_line[0].strip()) v = strip_quotes(split_line[1].strip()) if not k in _styles and not k in codes: e = ParseError(_("'%s', line %s: Unknown variable: '%s'") % \ (myfile, lineno, k)) if onerror: onerror(e) else: raise e continue if ansi_code_pattern.match(v): if k in _styles: _styles[k] = (esc_seq + v, ) elif k in codes: codes[k] = esc_seq + v else: code_list = [] for x in v.split(): if x in codes: if k in _styles: code_list.append(x) elif k in codes: code_list.append(codes[x]) else: e = ParseError(_("'%s', line %s: Undefined: '%s'") % \ (myfile, lineno, x)) if onerror: onerror(e) else: raise e if k in _styles: _styles[k] = tuple(code_list) elif k in codes: codes[k] = "".join(code_list) except (IOError, OSError) as e: if e.errno == errno.ENOENT: raise FileNotFound(myfile) elif e.errno == errno.EACCES: raise PermissionDenied(myfile) raise finally: if f is not None: f.close()
def _start(self): tar_options = "" if "xattr" in self.features: process = subprocess.Popen(["tar", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = process.communicate()[0] if b"--xattrs" in output: tar_options = ["--xattrs", "--xattrs-include='*'"] for x in portage.util.shlex_split(self.env.get("PORTAGE_XATTR_EXCLUDE", "")): tar_options.append(portage._shell_quote("--xattrs-exclude=%s" % x)) tar_options = " ".join(tar_options) decomp = _compressors.get(compression_probe(self.pkg_path)) if decomp is not None: decomp_cmd = decomp.get("decompress") elif tarfile.is_tarfile(portage._unicode_encode(self.pkg_path, encoding=portage._encodings['fs'], errors='strict')): decomp_cmd = 'cat' decomp = { 'compress': 'cat', 'package': 'sys-apps/coreutils', } else: decomp_cmd = None if decomp_cmd is None: self.scheduler.output("!!! %s\n" % _("File compression header unrecognized: %s") % self.pkg_path, log_path=self.logfile, background=self.background, level=logging.ERROR) self.returncode = 1 self._async_wait() return try: decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0] except IndexError: decompression_binary = "" if find_binary(decompression_binary) is None: # Try alternative command if it exists if decomp.get("decompress_alt"): decomp_cmd = decomp.get("decompress_alt") try: decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0] except IndexError: decompression_binary = "" if find_binary(decompression_binary) is None: missing_package = decomp.get("package") self.scheduler.output("!!! %s\n" % _("File compression unsupported %s.\n Command was: %s.\n Maybe missing package: %s") % (self.pkg_path, varexpand(decomp_cmd, mydict=self.env), missing_package), log_path=self.logfile, background=self.background, level=logging.ERROR) self.returncode = 1 self._async_wait() return pkg_xpak = portage.xpak.tbz2(self.pkg_path) pkg_xpak.scan() # SIGPIPE handling (128 + SIGPIPE) should be compatible with # assert_sigpipe_ok() that's used by the ebuild unpack() helper. self.args = [self._shell_binary, "-c", ("cmd0=(head -c %d -- %s) cmd1=(%s) cmd2=(tar -xp %s -C %s -f -); " + \ '"${cmd0[@]}" | "${cmd1[@]}" | "${cmd2[@]}"; ' + \ "p=(${PIPESTATUS[@]}) ; for i in {0..2}; do " + \ "if [[ ${p[$i]} != 0 && ${p[$i]} != %d ]] ; then " + \ "echo command $(eval \"echo \\\"'\\${cmd$i[*]}'\\\"\") " + \ "failed with status ${p[$i]} ; exit ${p[$i]} ; fi ; done; " + \ "if [ ${p[$i]} != 0 ] ; then " + \ "echo command $(eval \"echo \\\"'\\${cmd$i[*]}'\\\"\") " + \ "failed with status ${p[$i]} ; exit ${p[$i]} ; fi ; " + \ "exit 0 ;") % \ (pkg_xpak.filestat.st_size - pkg_xpak.xpaksize, portage._shell_quote(self.pkg_path), decomp_cmd, tar_options, portage._shell_quote(self.image_dir), 128 + signal.SIGPIPE)] SpawnProcess._start(self)
def display(self): self.out.write("\r"+colorize("WARN", _("cache miss: '")+str(self.misses)+"'") + \ " --- "+colorize("GOOD", _("cache hit: '")+str(self.hits)+"'")) self.out.flush()
def listBrokenBinaries(self, debug=False): """ Find binaries and their needed sonames, which have no providers. @param debug: Boolean to enable debug output @type debug: Boolean @rtype: dict (example: {'/usr/bin/foo': set(['libbar.so'])}) @return: The return value is an object -> set-of-sonames mapping, where object is a broken binary and the set consists of sonames needed by object that have no corresponding libraries to fulfill the dependency. """ os = _os_merge class _LibraryCache: """ Caches properties associated with paths. The purpose of this class is to prevent multiple instances of _ObjectKey for the same paths. """ def __init__(cache_self): cache_self.cache = {} def get(cache_self, obj): """ Caches and returns properties associated with an object. @param obj: absolute path (can be symlink) @type obj: string (example: '/usr/lib/libfoo.so') @rtype: 4-tuple with types (string or None, string or None, 2-tuple, Boolean) @return: 4-tuple with the following components: 1. arch as a string or None if it does not exist, 2. soname as a string or None if it does not exist, 3. obj_key as 2-tuple, 4. Boolean representing whether the object exists. (example: ('libfoo.so.1', (123L, 456L), True)) """ if obj in cache_self.cache: return cache_self.cache[obj] obj_key = self._obj_key(obj) # Check that the library exists on the filesystem. if obj_key.file_exists(): # Get the arch and soname from LinkageMap._obj_properties if # it exists. Otherwise, None. obj_props = self._obj_properties.get(obj_key) if obj_props is None: arch = None soname = None else: arch = obj_props.arch soname = obj_props.soname return cache_self.cache.setdefault( obj, (arch, soname, obj_key, True)) return cache_self.cache.setdefault( obj, (None, None, obj_key, False)) rValue = {} cache = _LibraryCache() providers = self.listProviders() # Iterate over all obj_keys and their providers. for obj_key, sonames in providers.items(): obj_props = self._obj_properties[obj_key] arch = obj_props.arch path = obj_props.runpaths objs = obj_props.alt_paths path = path.union(self._defpath) # Iterate over each needed soname and the set of library paths that # fulfill the soname to determine if the dependency is broken. for soname, libraries in sonames.items(): # validLibraries is used to store libraries, which satisfy soname, # so if no valid libraries are found, the soname is not satisfied # for obj_key. If unsatisfied, objects associated with obj_key # must be emerged. validLibraries = set() # It could be the case that the library to satisfy the soname is # not in the obj's runpath, but a symlink to the library is (eg # libnvidia-tls.so.1 in nvidia-drivers). Also, since LinkageMap # does not catalog symlinks, broken or missing symlinks may go # unnoticed. As a result of these cases, check that a file with # the same name as the soname exists in obj's runpath. # XXX If we catalog symlinks in LinkageMap, this could be improved. for directory in path: cachedArch, cachedSoname, cachedKey, cachedExists = cache.get( os.path.join(directory, soname)) # Check that this library provides the needed soname. Doing # this, however, will cause consumers of libraries missing # sonames to be unnecessarily emerged. (eg libmix.so) if cachedSoname == soname and cachedArch == arch: validLibraries.add(cachedKey) if debug and cachedKey not in set( map(self._obj_key_cache.get, libraries)): # XXX This is most often due to soname symlinks not in # a library's directory. We could catalog symlinks in # LinkageMap to avoid checking for this edge case here. writemsg_level( _("Found provider outside of findProviders:") + (" %s -> %s %s\n" % ( os.path.join(directory, soname), self._obj_properties[cachedKey].alt_paths, libraries, )), level=logging.DEBUG, noiselevel=-1, ) # A valid library has been found, so there is no need to # continue. break if (debug and cachedArch == arch and cachedKey in self._obj_properties): writemsg_level( (_("Broken symlink or missing/bad soname: " + "%(dir_soname)s -> %(cachedKey)s " + "with soname %(cachedSoname)s but expecting %(soname)s" ) % { "dir_soname": os.path.join(directory, soname), "cachedKey": self._obj_properties[cachedKey], "cachedSoname": cachedSoname, "soname": soname, }) + "\n", level=logging.DEBUG, noiselevel=-1, ) # This conditional checks if there are no libraries to satisfy the # soname (empty set). if not validLibraries: for obj in objs: rValue.setdefault(obj, set()).add(soname) # If no valid libraries have been found by this point, then # there are no files named with the soname within obj's runpath, # but if there are libraries (from the providers mapping), it is # likely that soname symlinks or the actual libraries are # missing or broken. Thus those libraries are added to rValue # in order to emerge corrupt library packages. for lib in libraries: rValue.setdefault(lib, set()).add(soname) if debug: if not os.path.isfile(lib): writemsg_level( _("Missing library:") + " %s\n" % (lib, ), level=logging.DEBUG, noiselevel=-1, ) else: writemsg_level( _("Possibly missing symlink:") + "%s\n" % (os.path.join( os.path.dirname(lib), soname)), level=logging.DEBUG, noiselevel=-1, ) return rValue
def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None): """(baseurl,conn,chunk_size,verbose) -- """ if not conn: keepconnection = 0 else: keepconnection = 1 cache_path = "/var/cache/edb" metadatafilename = os.path.join(cache_path, 'remote_metadata.pickle') if makepickle is None: makepickle = "/var/cache/edb/metadata.idx.most_recent" try: conn, protocol, address, params, headers = create_conn(baseurl, conn) except _all_errors as e: # ftplib.FTP(host) can raise errors like this: # socket.error: (111, 'Connection refused') sys.stderr.write("!!! %s\n" % (e, )) return {} out = sys.stdout try: metadatafile = open( _unicode_encode(metadatafilename, encoding=_encodings['fs'], errors='strict'), 'rb') mypickle = pickle.Unpickler(metadatafile) try: mypickle.find_global = None except AttributeError: # TODO: If py3k, override Unpickler.find_class(). pass metadata = mypickle.load() out.write(_("Loaded metadata pickle.\n")) out.flush() metadatafile.close() except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError): metadata = {} if baseurl not in metadata: metadata[baseurl] = {} if "indexname" not in metadata[baseurl]: metadata[baseurl]["indexname"] = "" if "timestamp" not in metadata[baseurl]: metadata[baseurl]["timestamp"] = 0 if "unmodified" not in metadata[baseurl]: metadata[baseurl]["unmodified"] = 0 if "data" not in metadata[baseurl]: metadata[baseurl]["data"] = {} if not os.access(cache_path, os.W_OK): sys.stderr.write(_("!!! Unable to write binary metadata to disk!\n")) sys.stderr.write(_("!!! Permission denied: '%s'\n") % cache_path) return metadata[baseurl]["data"] import portage.exception try: filelist = dir_get_list(baseurl, conn) except portage.exception.PortageException as e: sys.stderr.write( _("!!! Error connecting to '%s'.\n") % _hide_url_passwd(baseurl)) sys.stderr.write("!!! %s\n" % str(e)) del e return metadata[baseurl]["data"] tbz2list = match_in_array(filelist, suffix=".tbz2") metalist = match_in_array(filelist, prefix="metadata.idx") del filelist # Determine if our metadata file is current. metalist.sort() metalist.reverse() # makes the order new-to-old. for mfile in metalist: if usingcache and \ ((metadata[baseurl]["indexname"] != mfile) or \ (metadata[baseurl]["timestamp"] < int(time.time()-(60*60*24)))): # Try to download new cache until we succeed on one. data = "" for trynum in [1, 2, 3]: mytempfile = tempfile.TemporaryFile() try: file_get(baseurl + "/" + mfile, mytempfile, conn) if mytempfile.tell() > len(data): mytempfile.seek(0) data = mytempfile.read() except ValueError as e: sys.stderr.write("--- " + str(e) + "\n") if trynum < 3: sys.stderr.write(_("Retrying...\n")) sys.stderr.flush() mytempfile.close() continue if match_in_array([mfile], suffix=".gz"): out.write("gzip'd\n") out.flush() try: import gzip mytempfile.seek(0) gzindex = gzip.GzipFile(mfile[:-3], 'rb', 9, mytempfile) data = gzindex.read() except SystemExit as e: raise except Exception as e: mytempfile.close() sys.stderr.write( _("!!! Failed to use gzip: ") + str(e) + "\n") sys.stderr.flush() mytempfile.close() try: metadata[baseurl]["data"] = pickle.loads(data) del data metadata[baseurl]["indexname"] = mfile metadata[baseurl]["timestamp"] = int(time.time()) metadata[baseurl][ "modified"] = 0 # It's not, right after download. out.write(_("Pickle loaded.\n")) out.flush() break except SystemExit as e: raise except Exception as e: sys.stderr.write( _("!!! Failed to read data from index: ") + str(mfile) + "\n") sys.stderr.write("!!! " + str(e) + "\n") sys.stderr.flush() try: metadatafile = open( _unicode_encode(metadatafilename, encoding=_encodings['fs'], errors='strict'), 'wb') pickle.dump(metadata, metadatafile, protocol=2) metadatafile.close() except SystemExit as e: raise except Exception as e: sys.stderr.write( _("!!! Failed to write binary metadata to disk!\n")) sys.stderr.write("!!! " + str(e) + "\n") sys.stderr.flush() break # We may have metadata... now we run through the tbz2 list and check. class CacheStats(object): from time import time def __init__(self, out): self.misses = 0 self.hits = 0 self.last_update = 0 self.out = out self.min_display_latency = 0.2 def update(self): cur_time = self.time() if cur_time - self.last_update >= self.min_display_latency: self.last_update = cur_time self.display() def display(self): self.out.write("\r"+colorize("WARN", _("cache miss: '")+str(self.misses)+"'") + \ " --- "+colorize("GOOD", _("cache hit: '")+str(self.hits)+"'")) self.out.flush() cache_stats = CacheStats(out) have_tty = os.environ.get('TERM') != 'dumb' and out.isatty() if have_tty: cache_stats.display() binpkg_filenames = set() for x in tbz2list: x = os.path.basename(x) binpkg_filenames.add(x) if x not in metadata[baseurl]["data"]: cache_stats.misses += 1 if have_tty: cache_stats.update() metadata[baseurl]["modified"] = 1 myid = None for retry in range(3): try: myid = file_get_metadata( "/".join((baseurl.rstrip("/"), x.lstrip("/"))), conn, chunk_size) break except http_client_BadStatusLine: # Sometimes this error is thrown from conn.getresponse() in # make_http_request(). The docstring for this error in # httplib.py says "Presumably, the server closed the # connection before sending a valid response". conn, protocol, address, params, headers = create_conn( baseurl) except http_client_ResponseNotReady: # With some http servers this error is known to be thrown # from conn.getresponse() in make_http_request() when the # remote file does not have appropriate read permissions. # Maybe it's possible to recover from this exception in # cases though, so retry. conn, protocol, address, params, headers = create_conn( baseurl) if myid and myid[0]: metadata[baseurl]["data"][x] = make_metadata_dict(myid) elif verbose: sys.stderr.write( colorize("BAD", _("!!! Failed to retrieve metadata on: ")) + str(x) + "\n") sys.stderr.flush() else: cache_stats.hits += 1 if have_tty: cache_stats.update() cache_stats.display() # Cleanse stale cache for files that don't exist on the server anymore. stale_cache = set(metadata[baseurl]["data"]).difference(binpkg_filenames) if stale_cache: for x in stale_cache: del metadata[baseurl]["data"][x] metadata[baseurl]["modified"] = 1 del stale_cache del binpkg_filenames out.write("\n") out.flush() try: if "modified" in metadata[baseurl] and metadata[baseurl]["modified"]: metadata[baseurl]["timestamp"] = int(time.time()) metadatafile = open( _unicode_encode(metadatafilename, encoding=_encodings['fs'], errors='strict'), 'wb') pickle.dump(metadata, metadatafile, protocol=2) metadatafile.close() if makepickle: metadatafile = open( _unicode_encode(makepickle, encoding=_encodings['fs'], errors='strict'), 'wb') pickle.dump(metadata[baseurl]["data"], metadatafile, protocol=2) metadatafile.close() except SystemExit as e: raise except Exception as e: sys.stderr.write(_("!!! Failed to write binary metadata to disk!\n")) sys.stderr.write("!!! " + str(e) + "\n") sys.stderr.flush() if not keepconnection: conn.close() return metadata[baseurl]["data"]
def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): """Given a string (packagename or virtual) expand it into a valid cat/package string. Virtuals use the mydb to determine which provided virtual is a valid choice and defaults to the first element when there are no installed/available candidates.""" myslash = mycpv.split("/") mysplit = _pkgsplit(myslash[-1]) if settings is None: settings = globals()["settings"] virts = settings.getvirtuals() virts_p = settings.get_virts_p() if len(myslash) > 2: # this is illegal case. mysplit = [] mykey = mycpv elif len(myslash) == 2: if mysplit: mykey = myslash[0] + "/" + mysplit[0] else: mykey = mycpv if mydb and virts and mykey in virts: writemsg("mydb.__class__: %s\n" % (mydb.__class__), 1) if hasattr(mydb, "cp_list"): if not mydb.cp_list(mykey, use_cache=use_cache): writemsg("virts[%s]: %s\n" % (str(mykey), virts[mykey]), 1) mykey_orig = mykey[:] for vkey in virts[mykey]: # The virtuals file can contain a versioned atom, so # it may be necessary to remove the operator and # version from the atom before it is passed into # dbapi.cp_list(). if mydb.cp_list(vkey.cp): mykey = str(vkey) writemsg(_("virts chosen: %s\n") % (mykey), 1) break if mykey == mykey_orig: mykey = str(virts[mykey][0]) writemsg(_("virts defaulted: %s\n") % (mykey), 1) #we only perform virtual expansion if we are passed a dbapi else: #specific cpv, no category, ie. "foo-1.0" if mysplit: myp = mysplit[0] else: # "foo" ? myp = mycpv mykey = None matches = [] if mydb and hasattr(mydb, "categories"): for x in mydb.categories: if mydb.cp_list(x + "/" + myp, use_cache=use_cache): matches.append(x + "/" + myp) if len(matches) > 1: virtual_name_collision = False if len(matches) == 2: for x in matches: if not x.startswith("virtual/"): # Assume that the non-virtual is desired. This helps # avoid the ValueError for invalid deps that come from # installed packages (during reverse blocker detection, # for example). mykey = x else: virtual_name_collision = True if not virtual_name_collision: # AmbiguousPackageName inherits from ValueError, # for backward compatibility with calling code # that already handles ValueError. raise AmbiguousPackageName(matches) elif matches: mykey = matches[0] if not mykey and not isinstance(mydb, list): if myp in virts_p: mykey = virts_p[myp][0] #again, we only perform virtual expansion if we have a dbapi (not a list) if not mykey: mykey = "null/" + myp if mysplit: if mysplit[2] == "r0": return mykey + "-" + mysplit[1] else: return mykey + "-" + mysplit[1] + "-" + mysplit[2] else: return mykey
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, waiting_msg=None, flags=0): """ If wantnewlockfile is True then this creates a lockfile in the parent directory as the file: '.' + basename + '.portage_lockfile'. """ if not mypath: raise InvalidData(_("Empty path given")) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # Support for file object or integer file descriptor parameters is # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath if hasattr(mypath, 'fileno'): warnings.warn( "portage.locks.lockfile() support for " "file object parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): warnings.warn( "portage.locks.lockfile() support for integer file " "descriptor parameters is deprecated. Use a file path instead.", DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath if isinstance(mypath, basestring): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) old_mask = os.umask(000) try: try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno == ReadOnlyFileSystem.errno: raise ReadOnlyFileSystem(func_call) else: raise if not preexisting: try: if portage.data.secpass >= 1 and os.stat( lockfilename).st_gid != portage_gid: os.chown(lockfilename, -1, portage_gid) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return lockfile(mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) else: writemsg("%s: chown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) finally: os.umask(old_mask) elif isinstance(mypath, int): myfd = mypath else: raise ValueError(_("Unknown type passed in '%s': '%s'") % \ (type(mypath), mypath)) # try for a non-blocking lock, if it's held, throw a message # we're waiting on lockfile and use a blocking attempt. locking_method = portage._eintr_func_wrapper(_get_lock_fn()) try: if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): raise if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK): # resource temp unavailable; eg, someone beat us to the lock. if flags & os.O_NONBLOCK: os.close(myfd) raise TryAgain(mypath) global _quiet if _quiet: out = None else: out = portage.output.EOutput() if waiting_msg is None: if isinstance(mypath, int): waiting_msg = _("waiting for lock on fd %i") % myfd else: waiting_msg = _("waiting for lock on %s") % lockfilename if out is not None: out.ebegin(waiting_msg) # try for the exclusive lock now. enolock_msg_shown = False while True: try: locking_method(myfd, fcntl.LOCK_EX) except EnvironmentError as e: if e.errno == errno.ENOLCK: # This is known to occur on Solaris NFS (see # bug #462694). Assume that the error is due # to temporary exhaustion of record locks, # and loop until one becomes available. if not enolock_msg_shown: enolock_msg_shown = True if isinstance(mypath, int): context_desc = _("Error while waiting " "to lock fd %i") % myfd else: context_desc = _("Error while waiting " "to lock '%s'") % lockfilename writemsg("\n!!! %s: %s\n" % (context_desc, e), noiselevel=-1) time.sleep(_HARDLINK_POLL_LATENCY) continue if out is not None: out.eend(1, str(e)) raise else: break if out is not None: out.eend(os.EX_OK) elif e.errno in (errno.ENOSYS, ): # We're not allowed to lock on this FS. if not isinstance(lockfilename, int): # If a file object was passed in, it's not safe # to close the file descriptor because it may # still be in use. os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') if not isinstance(lockfilename_path, basestring): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) if not link_success: raise lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: raise if isinstance(lockfilename, basestring) and \ myfd != HARDLINK_FD and _fstat_nlink(myfd) == 0: # The file was deleted on us... Keep trying to make one... os.close(myfd) writemsg(_("lockfile recurse\n"), 1) lockfilename, myfd, unlinkfile, locking_method = lockfile( mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) if myfd != HARDLINK_FD: # FD_CLOEXEC is enabled by default in Python >=3.4. if sys.hexversion < 0x3040000: try: fcntl.FD_CLOEXEC except AttributeError: pass else: fcntl.fcntl( myfd, fcntl.F_SETFD, fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) _open_fds.add(myfd) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) return (lockfilename, myfd, unlinkfile, locking_method)
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, waiting_msg=None, flags=0): """Does the NFS, hardlink shuffle to ensure locking on the disk. We create a PRIVATE hardlink to the real lockfile, that is just a placeholder on the disk. If our file can 2 references, then we have the lock. :) Otherwise we lather, rise, and repeat. """ if max_wait is not DeprecationWarning: warnings.warn( "The 'max_wait' parameter of " "portage.locks.hardlink_lockfile() is now unused. Use " "flags=os.O_NONBLOCK instead.", DeprecationWarning, stacklevel=2) global _quiet out = None displayed_waiting_msg = False preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) # Since Python 3.4, chown requires int type (no proxies). portage_gid = int(portage.data.portage_gid) # myhardlock must not exist prior to our link() call, and we can # safely unlink it since its file name is unique to our PID try: os.unlink(myhardlock) except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): pass else: func_call = "unlink('%s')" % myhardlock if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) else: raise while True: # create lockfilename if it doesn't exist yet try: myfd = os.open(lockfilename, os.O_CREAT | os.O_RDWR, 0o660) except OSError as e: func_call = "open('%s')" % lockfilename if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno == ReadOnlyFileSystem.errno: raise ReadOnlyFileSystem(func_call) else: raise else: myfd_st = None try: myfd_st = os.fstat(myfd) if not preexisting: # Don't chown the file if it is preexisting, since we # want to preserve existing permissions in that case. if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid: os.fchown(myfd, -1, portage_gid) except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): writemsg("%s: fchown('%s', -1, %d)\n" % \ (e, lockfilename, portage_gid), noiselevel=-1) writemsg(_("Cannot chown a lockfile: '%s'\n") % \ lockfilename, noiselevel=-1) writemsg(_("Group IDs of current user: %s\n") % \ " ".join(str(n) for n in os.getgroups()), noiselevel=-1) else: # another process has removed the file, so we'll have # to create it again continue finally: os.close(myfd) # If fstat shows more than one hardlink, then it's extremely # unlikely that the following link call will result in a lock, # so optimize away the wasteful link call and sleep or raise # TryAgain. if myfd_st is not None and myfd_st.st_nlink < 2: try: os.link(lockfilename, myhardlock) except OSError as e: func_call = "link('%s', '%s')" % (lockfilename, myhardlock) if e.errno == OperationNotPermitted.errno: raise OperationNotPermitted(func_call) elif e.errno == PermissionDenied.errno: raise PermissionDenied(func_call) elif e.errno in (errno.ESTALE, errno.ENOENT): # another process has removed the file, so we'll have # to create it again continue else: raise else: if hardlink_is_mine(myhardlock, lockfilename): if out is not None: out.eend(os.EX_OK) break try: os.unlink(myhardlock) except OSError as e: # This should not happen, since the file name of # myhardlock is unique to our host and PID, # and the above link() call succeeded. if e.errno not in (errno.ENOENT, errno.ESTALE): raise raise FileNotFound(myhardlock) if flags & os.O_NONBLOCK: raise TryAgain(lockfilename) if out is None and not _quiet: out = portage.output.EOutput() if out is not None and not displayed_waiting_msg: displayed_waiting_msg = True if waiting_msg is None: waiting_msg = _("waiting for lock on %s\n") % lockfilename out.ebegin(waiting_msg) time.sleep(_HARDLINK_POLL_LATENCY) return True
def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (self.phase, signum) self._eerror(textwrap.wrap(msg, 72))