def get_log_text_dict(filename): """Return the log contents as a dict""" logfile_dict = {} index = 1 for text_line in grablines(filename): logfile_dict[index] = text_line index = index + 1 return logfile_dict, index - 1
def __init__(self, options, portdb, event_loop): self.options = options self.portdb = portdb self.event_loop = event_loop self.added_byte_count = 0 self.added_file_count = 0 self.scheduled_deletion_count = 0 self.delete_count = 0 self.file_owners = {} self.file_failures = {} self.start_time = time.time() self._open_files = [] self.log_success = self._open_log('success', options.success_log, 'a') self.log_failure = self._open_log('failure', options.failure_log, 'a') self.distfiles = None if options.distfiles is not None: self.distfiles = options.distfiles self.mirrors = copy.copy(portdb.settings.thirdpartymirrors()) if options.mirror_overrides is not None: self.mirrors.update(grabdict(options.mirror_overrides)) if options.mirror_skip is not None: for x in options.mirror_skip.split(","): self.mirrors[x] = [] self.whitelist = None if options.whitelist_from is not None: self.whitelist = set() for filename in options.whitelist_from: for line in grablines(filename): line = line.strip() if line and not line.startswith("#"): self.whitelist.add(line) self.restrict_mirror_exemptions = None if options.restrict_mirror_exemptions is not None: self.restrict_mirror_exemptions = frozenset( options.restrict_mirror_exemptions.split(",")) self.recycle_db = None if options.recycle_db is not None: self.recycle_db = self._open_shelve( options.recycle_db, 'recycle') self.distfiles_db = None if options.distfiles_db is not None: self.distfiles_db = self._open_shelve( options.distfiles_db, 'distfiles') self.deletion_db = None if options.deletion_db is not None: self.deletion_db = self._open_shelve( options.deletion_db, 'deletion')
def __init__(self, options, portdb, event_loop): self.options = options self.portdb = portdb self.event_loop = event_loop self.added_byte_count = 0 self.added_file_count = 0 self.scheduled_deletion_count = 0 self.delete_count = 0 self.file_owners = {} self.file_failures = {} self.start_time = time.time() self._open_files = [] self.log_success = self._open_log('success', options.success_log, 'a') self.log_failure = self._open_log('failure', options.failure_log, 'a') self.distfiles = None if options.distfiles is not None: self.distfiles = options.distfiles self.mirrors = copy.copy(portdb.settings.thirdpartymirrors()) if options.mirror_overrides is not None: self.mirrors.update(grabdict(options.mirror_overrides)) if options.mirror_skip is not None: for x in options.mirror_skip.split(","): self.mirrors[x] = [] self.whitelist = None if options.whitelist_from is not None: self.whitelist = set() for filename in options.whitelist_from: for line in grablines(filename): line = line.strip() if line and not line.startswith("#"): self.whitelist.add(line) self.restrict_mirror_exemptions = None if options.restrict_mirror_exemptions is not None: self.restrict_mirror_exemptions = frozenset( options.restrict_mirror_exemptions.split(",")) self.recycle_db = None if options.recycle_db is not None: self.recycle_db = self._open_shelve(options.recycle_db, 'recycle') self.distfiles_db = None if options.distfiles_db is not None: self.distfiles_db = self._open_shelve(options.distfiles_db, 'distfiles') self.deletion_db = None if options.deletion_db is not None: self.deletion_db = self._open_shelve(options.deletion_db, 'deletion')
def _strip_sig(manifest_path): """ Strip an existing signature from a Manifest file. """ line_re = ManifestTask._manifest_line_re lines = grablines(manifest_path) f = None try: f = atomic_ofstream(manifest_path) for line in lines: if line_re.match(line) is not None: f.write(line) f.close() f = None finally: if f is not None: f.abort()
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None): """ If specified, the myrepo argument is assumed to be valid. This should be a safe assumption since portdbapi methods always return valid repo names and valid "repository" metadata from aux_get. """ if settings is None: settings = portage.settings if portdb is None: portdb = portage.portdb mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: metadata = dict( zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo))) except KeyError: if not portdb.cpv_exists(mycpv): raise else: if myrepo is None: myrepo = _gen_valid_repo(metadata["repository"]) elif myrepo is None: myrepo = metadata.get("repository") if myrepo is not None: myrepo = _gen_valid_repo(metadata["repository"]) if metadata is not None and not portage.eapi_is_supported( metadata["EAPI"]): # Return early since otherwise we might produce invalid # results given that the EAPI is not supported. Also, # metadata is mostly useless in this case since it doesn't # contain essential things like SLOT. if return_location: return (None, None) return None # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv try: pkg.slot except AttributeError: pkg = _pkg_str(mycpv, metadata=metadata, repo=myrepo) cpv_slot_list = [pkg] mycp = pkg.cp locations = [] if pkg.repo in settings.repositories: for repo in settings.repositories[pkg.repo].masters + ( settings.repositories[pkg.repo], ): locations.append(os.path.join(repo.location, "profiles")) locations.extend(settings.profiles) locations.append( os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() pmasklists = [] for profile in locations: pmask_filename = os.path.join(profile, "package.mask") node = None for l, recursive_filename in grablines(pmask_filename, recursive=1, remember_source_file=True): if node is None or node[0] != recursive_filename: node = (recursive_filename, []) pmasklists.append(node) node[1].append(l) pmaskdict = settings._mask_manager._pmaskdict if mycp in pmaskdict: for x in pmaskdict[mycp]: if match_from_list(x, cpv_slot_list): x = x.without_repo for pmask in pmasklists: comment = "" comment_valid = -1 pmask_filename = pmask[0] for i in range(len(pmask[1])): l = pmask[1][i].strip() try: l_atom = Atom(l, allow_repo=True, allow_wildcard=True).without_repo except InvalidAtom: l_atom = None if l == "": comment = "" comment_valid = -1 elif l[0] == "#": comment += l + "\n" comment_valid = i + 1 elif l_atom == x: if comment_valid != i: comment = "" if return_location: return (comment, pmask_filename) return comment elif comment_valid != -1: # Apparently this comment applies to multiple masks, so # it remains valid until a blank line is encountered. comment_valid += 1 if return_location: return (None, None) return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None): """ If specified, the myrepo argument is assumed to be valid. This should be a safe assumption since portdbapi methods always return valid repo names and valid "repository" metadata from aux_get. """ if settings is None: settings = portage.settings if portdb is None: portdb = portage.portdb mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo))) except KeyError: if not portdb.cpv_exists(mycpv): raise else: if myrepo is None: myrepo = _gen_valid_repo(metadata["repository"]) elif myrepo is None: myrepo = metadata.get("repository") if myrepo is not None: myrepo = _gen_valid_repo(metadata["repository"]) if metadata is not None and not portage.eapi_is_supported(metadata["EAPI"]): # Return early since otherwise we might produce invalid # results given that the EAPI is not supported. Also, # metadata is mostly useless in this case since it doesn't # contain essential things like SLOT. if return_location: return (None, None) else: return None # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv if metadata is not None: pkg = "".join((mycpv, _slot_separator, metadata["SLOT"])) # At this point myrepo should be None, a valid name, or # Package.UNKNOWN_REPO which we ignore. if myrepo is not None and myrepo != Package.UNKNOWN_REPO: pkg = "".join((pkg, _repo_separator, myrepo)) cpv_slot_list = [pkg] mycp = mysplit[0] + "/" + mysplit[1] # XXX- This is a temporary duplicate of code from the config constructor. locations = [os.path.join(settings["PORTDIR"], "profiles")] locations.extend(settings.profiles) for ov in settings["PORTDIR_OVERLAY"].split(): profdir = os.path.join(normalize_path(ov), "profiles") if os.path.isdir(profdir): locations.append(profdir) locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() pmasklists = [] for profile in locations: pmask_filename = os.path.join(profile, "package.mask") node = None for l, recursive_filename in grablines(pmask_filename, recursive=1, remember_source_file=True): if node is None or node[0] != recursive_filename: node = (recursive_filename, []) pmasklists.append(node) node[1].append(l) pmaskdict = settings._mask_manager._pmaskdict if mycp in pmaskdict: for x in pmaskdict[mycp]: if match_from_list(x, cpv_slot_list): x = x.without_repo for pmask in pmasklists: comment = "" comment_valid = -1 pmask_filename = pmask[0] for i in range(len(pmask[1])): l = pmask[1][i].strip() try: l_atom = Atom(l, allow_repo=True, allow_wildcard=True).without_repo except InvalidAtom: l_atom = None if l == "": comment = "" comment_valid = -1 elif l[0] == "#": comment += l + "\n" comment_valid = i + 1 elif l_atom == x: if comment_valid != i: comment = "" if return_location: return (comment, pmask_filename) else: return comment elif comment_valid != -1: # Apparently this comment applies to multiple masks, so # it remains valid until a blank line is encountered. comment_valid += 1 if return_location: return (None, None) else: return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None): """ If specified, the myrepo argument is assumed to be valid. This should be a safe assumption since portdbapi methods always return valid repo names and valid "repository" metadata from aux_get. """ if settings is None: settings = portage.settings if portdb is None: portdb = portage.portdb mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: metadata = dict( zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo))) except KeyError: if not portdb.cpv_exists(mycpv): raise else: if myrepo is None: myrepo = _gen_valid_repo(metadata["repository"]) elif myrepo is None: myrepo = metadata.get("repository") if myrepo is not None: myrepo = _gen_valid_repo(metadata["repository"]) if metadata is not None and \ not portage.eapi_is_supported(metadata["EAPI"]): # Return early since otherwise we might produce invalid # results given that the EAPI is not supported. Also, # metadata is mostly useless in this case since it doesn't # contain essential things like SLOT. if return_location: return (None, None) else: return None # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv if metadata is not None: pkg = "".join((mycpv, _slot_separator, metadata["SLOT"])) # At this point myrepo should be None, a valid name, or # Package.UNKNOWN_REPO which we ignore. if myrepo is not None and myrepo != Package.UNKNOWN_REPO: pkg = "".join((pkg, _repo_separator, myrepo)) cpv_slot_list = [pkg] mycp = mysplit[0] + "/" + mysplit[1] # XXX- This is a temporary duplicate of code from the config constructor. locations = [os.path.join(settings["PORTDIR"], "profiles")] locations.extend(settings.profiles) for ov in settings["PORTDIR_OVERLAY"].split(): profdir = os.path.join(normalize_path(ov), "profiles") if os.path.isdir(profdir): locations.append(profdir) locations.append( os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() pmasklists = [] for profile in locations: pmask_filename = os.path.join(profile, "package.mask") pmasklists.append( (pmask_filename, grablines(pmask_filename, recursive=1))) pmaskdict = settings._mask_manager._pmaskdict if mycp in pmaskdict: for x in pmaskdict[mycp]: if match_from_list(x, cpv_slot_list): x = x.without_repo for pmask in pmasklists: comment = "" comment_valid = -1 pmask_filename = pmask[0] for i in range(len(pmask[1])): l = pmask[1][i].strip() try: l_atom = Atom(l, allow_repo=True, allow_wildcard=True).without_repo except InvalidAtom: l_atom = None if l == "": comment = "" comment_valid = -1 elif l[0] == "#": comment += (l + "\n") comment_valid = i + 1 elif l_atom == x: if comment_valid != i: comment = "" if return_location: return (comment, pmask_filename) else: return comment elif comment_valid != -1: # Apparently this comment applies to multiple masks, so # it remains valid until a blank line is encountered. comment_valid += 1 if return_location: return (None, None) else: return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False): if settings is None: settings = portage.settings if portdb is None: portdb = portage.portdb mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys))) except KeyError: if not portdb.cpv_exists(mycpv): raise if metadata is None: # Can't access SLOT due to corruption. cpv_slot_list = [mycpv] else: cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])] mycp=mysplit[0]+"/"+mysplit[1] # XXX- This is a temporary duplicate of code from the config constructor. locations = [os.path.join(settings["PORTDIR"], "profiles")] locations.extend(settings.profiles) for ov in settings["PORTDIR_OVERLAY"].split(): profdir = os.path.join(normalize_path(ov), "profiles") if os.path.isdir(profdir): locations.append(profdir) locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations] if mycp in settings.pmaskdict: for x in settings.pmaskdict[mycp]: if match_from_list(x, cpv_slot_list): for pmask in pmasklists: comment = "" comment_valid = -1 pmask_filename = os.path.join(pmask[0], "package.mask") for i in range(len(pmask[1])): l = pmask[1][i].strip() if l == "": comment = "" comment_valid = -1 elif l[0] == "#": comment += (l+"\n") comment_valid = i + 1 elif l == x: if comment_valid != i: comment = "" if return_location: return (comment, pmask_filename) else: return comment elif comment_valid != -1: # Apparently this comment applies to muliple masks, so # it remains valid until a blank line is encountered. comment_valid += 1 if return_location: return (None, None) else: return None
def __init__(self, options, portdb, event_loop): self.options = options self.portdb = portdb self.event_loop = event_loop self.added_byte_count = 0 self.added_file_count = 0 self.scheduled_deletion_count = 0 self.delete_count = 0 self.file_owners = {} self.file_failures = {} self.start_time = time.time() self._open_files = [] self.log_success = self._open_log( "success", getattr(options, "success_log", None), "a" ) self.log_failure = self._open_log( "failure", getattr(options, "failure_log", None), "a" ) self.distfiles = None if getattr(options, "distfiles", None) is not None: self.distfiles = options.distfiles self.mirrors = copy.copy(portdb.settings.thirdpartymirrors()) if getattr(options, "mirror_overrides", None) is not None: self.mirrors.update(grabdict(options.mirror_overrides)) if getattr(options, "mirror_skip", None) is not None: for x in options.mirror_skip.split(","): self.mirrors[x] = [] self.whitelist = None if getattr(options, "whitelist_from", None) is not None: self.whitelist = set() for filename in options.whitelist_from: for line in grablines(filename): line = line.strip() if line and not line.startswith("#"): self.whitelist.add(line) self.restrict_mirror_exemptions = None if getattr(options, "restrict_mirror_exemptions", None) is not None: self.restrict_mirror_exemptions = frozenset( options.restrict_mirror_exemptions.split(",") ) self.recycle_db = None if getattr(options, "recycle_db", None) is not None: self.recycle_db = self._open_shelve(options.recycle_db, "recycle") self.distfiles_db = None if getattr(options, "distfiles_db", None) is not None: self.distfiles_db = self._open_shelve(options.distfiles_db, "distfiles") self.content_db = None if getattr(options, "content_db", None) is not None: self.content_db = ContentDB( self._open_shelve(options.content_db, "content") ) self.deletion_db = None if getattr(options, "deletion_db", None) is not None: self.deletion_db = self._open_shelve(options.deletion_db, "deletion") self.layout_conf = MirrorLayoutConfig() if getattr(options, "layout_conf", None) is None: options.layout_conf = os.path.join(self.distfiles, "layout.conf") self.layout_conf.read_from_file(options.layout_conf) self.layouts = self.layout_conf.get_all_layouts()
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False): if settings is None: settings = portage.settings if portdb is None: portdb = portage.portdb mysplit = catpkgsplit(mycpv) if not mysplit: raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys))) except KeyError: if not portdb.cpv_exists(mycpv): raise if metadata is None: # Can't access SLOT due to corruption. cpv_slot_list = [mycpv] else: cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])] mycp=mysplit[0]+"/"+mysplit[1] # XXX- This is a temporary duplicate of code from the config constructor. locations = [os.path.join(settings["PORTDIR"], "profiles")] locations.extend(settings.profiles) for ov in settings["PORTDIR_OVERLAY"].split(): profdir = os.path.join(normalize_path(ov), "profiles") if os.path.isdir(profdir): locations.append(profdir) locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) locations.reverse() pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations] pmaskdict = settings._mask_manager._pmaskdict if mycp in pmaskdict: for x in pmaskdict[mycp]: if match_from_list(x, cpv_slot_list): for pmask in pmasklists: comment = "" comment_valid = -1 pmask_filename = os.path.join(pmask[0], "package.mask") for i in range(len(pmask[1])): l = pmask[1][i].strip() if l == "": comment = "" comment_valid = -1 elif l[0] == "#": comment += (l+"\n") comment_valid = i + 1 elif l == x: if comment_valid != i: comment = "" if return_location: return (comment, pmask_filename) else: return comment elif comment_valid != -1: # Apparently this comment applies to muliple masks, so # it remains valid until a blank line is encountered. comment_valid += 1 if return_location: return (None, None) else: return None