Exemple #1
0
def  get_log_text_dict(filename):
	"""Return the log contents as a dict"""
	logfile_dict = {}
	index = 1
	for text_line in grablines(filename):
		logfile_dict[index] = text_line
		index = index + 1
	return logfile_dict, index - 1
Exemple #2
0
	def __init__(self, options, portdb, event_loop):
		self.options = options
		self.portdb = portdb
		self.event_loop = event_loop
		self.added_byte_count = 0
		self.added_file_count = 0
		self.scheduled_deletion_count = 0
		self.delete_count = 0
		self.file_owners = {}
		self.file_failures = {}
		self.start_time = time.time()
		self._open_files = []

		self.log_success = self._open_log('success', options.success_log, 'a')
		self.log_failure = self._open_log('failure', options.failure_log, 'a')

		self.distfiles = None
		if options.distfiles is not None:
			self.distfiles = options.distfiles

		self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())

		if options.mirror_overrides is not None:
			self.mirrors.update(grabdict(options.mirror_overrides))

		if options.mirror_skip is not None:
			for x in options.mirror_skip.split(","):
				self.mirrors[x] = []

		self.whitelist = None
		if options.whitelist_from is not None:
			self.whitelist = set()
			for filename in options.whitelist_from:
				for line in grablines(filename):
					line = line.strip()
					if line and not line.startswith("#"):
						self.whitelist.add(line)

		self.restrict_mirror_exemptions = None
		if options.restrict_mirror_exemptions is not None:
			self.restrict_mirror_exemptions = frozenset(
				options.restrict_mirror_exemptions.split(","))

		self.recycle_db = None
		if options.recycle_db is not None:
			self.recycle_db = self._open_shelve(
				options.recycle_db, 'recycle')

		self.distfiles_db = None
		if options.distfiles_db is not None:
			self.distfiles_db = self._open_shelve(
				options.distfiles_db, 'distfiles')

		self.deletion_db = None
		if options.deletion_db is not None:
			self.deletion_db = self._open_shelve(
				options.deletion_db, 'deletion')
Exemple #3
0
    def __init__(self, options, portdb, event_loop):
        self.options = options
        self.portdb = portdb
        self.event_loop = event_loop
        self.added_byte_count = 0
        self.added_file_count = 0
        self.scheduled_deletion_count = 0
        self.delete_count = 0
        self.file_owners = {}
        self.file_failures = {}
        self.start_time = time.time()
        self._open_files = []

        self.log_success = self._open_log('success', options.success_log, 'a')
        self.log_failure = self._open_log('failure', options.failure_log, 'a')

        self.distfiles = None
        if options.distfiles is not None:
            self.distfiles = options.distfiles

        self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())

        if options.mirror_overrides is not None:
            self.mirrors.update(grabdict(options.mirror_overrides))

        if options.mirror_skip is not None:
            for x in options.mirror_skip.split(","):
                self.mirrors[x] = []

        self.whitelist = None
        if options.whitelist_from is not None:
            self.whitelist = set()
            for filename in options.whitelist_from:
                for line in grablines(filename):
                    line = line.strip()
                    if line and not line.startswith("#"):
                        self.whitelist.add(line)

        self.restrict_mirror_exemptions = None
        if options.restrict_mirror_exemptions is not None:
            self.restrict_mirror_exemptions = frozenset(
                options.restrict_mirror_exemptions.split(","))

        self.recycle_db = None
        if options.recycle_db is not None:
            self.recycle_db = self._open_shelve(options.recycle_db, 'recycle')

        self.distfiles_db = None
        if options.distfiles_db is not None:
            self.distfiles_db = self._open_shelve(options.distfiles_db,
                                                  'distfiles')

        self.deletion_db = None
        if options.deletion_db is not None:
            self.deletion_db = self._open_shelve(options.deletion_db,
                                                 'deletion')
Exemple #4
0
    def _strip_sig(manifest_path):
        """
		Strip an existing signature from a Manifest file.
		"""
        line_re = ManifestTask._manifest_line_re
        lines = grablines(manifest_path)
        f = None
        try:
            f = atomic_ofstream(manifest_path)
            for line in lines:
                if line_re.match(line) is not None:
                    f.write(line)
            f.close()
            f = None
        finally:
            if f is not None:
                f.abort()
	def _strip_sig(manifest_path):
		"""
		Strip an existing signature from a Manifest file.
		"""
		line_re = ManifestTask._manifest_line_re
		lines = grablines(manifest_path)
		f = None
		try:
			f = atomic_ofstream(manifest_path)
			for line in lines:
				if line_re.match(line) is not None:
					f.write(line)
			f.close()
			f = None
		finally:
			if f is not None:
				f.abort()
Exemple #6
0
def getmaskingreason(mycpv,
                     metadata=None,
                     settings=None,
                     portdb=None,
                     return_location=False,
                     myrepo=None):
    """
    If specified, the myrepo argument is assumed to be valid. This
    should be a safe assumption since portdbapi methods always
    return valid repo names and valid "repository" metadata from
    aux_get.
    """
    if settings is None:
        settings = portage.settings
    if portdb is None:
        portdb = portage.portdb
    mysplit = catpkgsplit(mycpv)
    if not mysplit:
        raise ValueError(_("invalid CPV: %s") % mycpv)

    if metadata is None:
        db_keys = list(portdb._aux_cache_keys)
        try:
            metadata = dict(
                zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo)))
        except KeyError:
            if not portdb.cpv_exists(mycpv):
                raise
        else:
            if myrepo is None:
                myrepo = _gen_valid_repo(metadata["repository"])

    elif myrepo is None:
        myrepo = metadata.get("repository")
        if myrepo is not None:
            myrepo = _gen_valid_repo(metadata["repository"])

    if metadata is not None and not portage.eapi_is_supported(
            metadata["EAPI"]):
        # Return early since otherwise we might produce invalid
        # results given that the EAPI is not supported. Also,
        # metadata is mostly useless in this case since it doesn't
        # contain essential things like SLOT.
        if return_location:
            return (None, None)
        return None

    # Sometimes we can't access SLOT or repository due to corruption.
    pkg = mycpv
    try:
        pkg.slot
    except AttributeError:
        pkg = _pkg_str(mycpv, metadata=metadata, repo=myrepo)

    cpv_slot_list = [pkg]

    mycp = pkg.cp

    locations = []
    if pkg.repo in settings.repositories:
        for repo in settings.repositories[pkg.repo].masters + (
                settings.repositories[pkg.repo], ):
            locations.append(os.path.join(repo.location, "profiles"))
    locations.extend(settings.profiles)
    locations.append(
        os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH))
    locations.reverse()
    pmasklists = []
    for profile in locations:
        pmask_filename = os.path.join(profile, "package.mask")
        node = None
        for l, recursive_filename in grablines(pmask_filename,
                                               recursive=1,
                                               remember_source_file=True):
            if node is None or node[0] != recursive_filename:
                node = (recursive_filename, [])
                pmasklists.append(node)
            node[1].append(l)

    pmaskdict = settings._mask_manager._pmaskdict
    if mycp in pmaskdict:
        for x in pmaskdict[mycp]:
            if match_from_list(x, cpv_slot_list):
                x = x.without_repo
                for pmask in pmasklists:
                    comment = ""
                    comment_valid = -1
                    pmask_filename = pmask[0]
                    for i in range(len(pmask[1])):
                        l = pmask[1][i].strip()
                        try:
                            l_atom = Atom(l,
                                          allow_repo=True,
                                          allow_wildcard=True).without_repo
                        except InvalidAtom:
                            l_atom = None
                        if l == "":
                            comment = ""
                            comment_valid = -1
                        elif l[0] == "#":
                            comment += l + "\n"
                            comment_valid = i + 1
                        elif l_atom == x:
                            if comment_valid != i:
                                comment = ""
                            if return_location:
                                return (comment, pmask_filename)
                            return comment
                        elif comment_valid != -1:
                            # Apparently this comment applies to multiple masks, so
                            # it remains valid until a blank line is encountered.
                            comment_valid += 1
    if return_location:
        return (None, None)
    return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False, myrepo=None):
    """
	If specified, the myrepo argument is assumed to be valid. This
	should be a safe assumption since portdbapi methods always
	return valid repo names and valid "repository" metadata from
	aux_get.
	"""
    if settings is None:
        settings = portage.settings
    if portdb is None:
        portdb = portage.portdb
    mysplit = catpkgsplit(mycpv)
    if not mysplit:
        raise ValueError(_("invalid CPV: %s") % mycpv)

    if metadata is None:
        db_keys = list(portdb._aux_cache_keys)
        try:
            metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo)))
        except KeyError:
            if not portdb.cpv_exists(mycpv):
                raise
        else:
            if myrepo is None:
                myrepo = _gen_valid_repo(metadata["repository"])

    elif myrepo is None:
        myrepo = metadata.get("repository")
        if myrepo is not None:
            myrepo = _gen_valid_repo(metadata["repository"])

    if metadata is not None and not portage.eapi_is_supported(metadata["EAPI"]):
        # Return early since otherwise we might produce invalid
        # results given that the EAPI is not supported. Also,
        # metadata is mostly useless in this case since it doesn't
        # contain essential things like SLOT.
        if return_location:
            return (None, None)
        else:
            return None

            # Sometimes we can't access SLOT or repository due to corruption.
    pkg = mycpv
    if metadata is not None:
        pkg = "".join((mycpv, _slot_separator, metadata["SLOT"]))
        # At this point myrepo should be None, a valid name, or
        # Package.UNKNOWN_REPO which we ignore.
    if myrepo is not None and myrepo != Package.UNKNOWN_REPO:
        pkg = "".join((pkg, _repo_separator, myrepo))
    cpv_slot_list = [pkg]

    mycp = mysplit[0] + "/" + mysplit[1]

    # XXX- This is a temporary duplicate of code from the config constructor.
    locations = [os.path.join(settings["PORTDIR"], "profiles")]
    locations.extend(settings.profiles)
    for ov in settings["PORTDIR_OVERLAY"].split():
        profdir = os.path.join(normalize_path(ov), "profiles")
        if os.path.isdir(profdir):
            locations.append(profdir)
    locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH))
    locations.reverse()
    pmasklists = []
    for profile in locations:
        pmask_filename = os.path.join(profile, "package.mask")
        node = None
        for l, recursive_filename in grablines(pmask_filename, recursive=1, remember_source_file=True):
            if node is None or node[0] != recursive_filename:
                node = (recursive_filename, [])
                pmasklists.append(node)
            node[1].append(l)

    pmaskdict = settings._mask_manager._pmaskdict
    if mycp in pmaskdict:
        for x in pmaskdict[mycp]:
            if match_from_list(x, cpv_slot_list):
                x = x.without_repo
                for pmask in pmasklists:
                    comment = ""
                    comment_valid = -1
                    pmask_filename = pmask[0]
                    for i in range(len(pmask[1])):
                        l = pmask[1][i].strip()
                        try:
                            l_atom = Atom(l, allow_repo=True, allow_wildcard=True).without_repo
                        except InvalidAtom:
                            l_atom = None
                        if l == "":
                            comment = ""
                            comment_valid = -1
                        elif l[0] == "#":
                            comment += l + "\n"
                            comment_valid = i + 1
                        elif l_atom == x:
                            if comment_valid != i:
                                comment = ""
                            if return_location:
                                return (comment, pmask_filename)
                            else:
                                return comment
                        elif comment_valid != -1:
                            # Apparently this comment applies to multiple masks, so
                            # it remains valid until a blank line is encountered.
                            comment_valid += 1
    if return_location:
        return (None, None)
    else:
        return None
Exemple #8
0
def getmaskingreason(mycpv,
                     metadata=None,
                     settings=None,
                     portdb=None,
                     return_location=False,
                     myrepo=None):
    """
	If specified, the myrepo argument is assumed to be valid. This
	should be a safe assumption since portdbapi methods always
	return valid repo names and valid "repository" metadata from
	aux_get.
	"""
    if settings is None:
        settings = portage.settings
    if portdb is None:
        portdb = portage.portdb
    mysplit = catpkgsplit(mycpv)
    if not mysplit:
        raise ValueError(_("invalid CPV: %s") % mycpv)

    if metadata is None:
        db_keys = list(portdb._aux_cache_keys)
        try:
            metadata = dict(
                zip(db_keys, portdb.aux_get(mycpv, db_keys, myrepo=myrepo)))
        except KeyError:
            if not portdb.cpv_exists(mycpv):
                raise
        else:
            if myrepo is None:
                myrepo = _gen_valid_repo(metadata["repository"])

    elif myrepo is None:
        myrepo = metadata.get("repository")
        if myrepo is not None:
            myrepo = _gen_valid_repo(metadata["repository"])

    if metadata is not None and \
     not portage.eapi_is_supported(metadata["EAPI"]):
        # Return early since otherwise we might produce invalid
        # results given that the EAPI is not supported. Also,
        # metadata is mostly useless in this case since it doesn't
        # contain essential things like SLOT.
        if return_location:
            return (None, None)
        else:
            return None

    # Sometimes we can't access SLOT or repository due to corruption.
    pkg = mycpv
    if metadata is not None:
        pkg = "".join((mycpv, _slot_separator, metadata["SLOT"]))
    # At this point myrepo should be None, a valid name, or
    # Package.UNKNOWN_REPO which we ignore.
    if myrepo is not None and myrepo != Package.UNKNOWN_REPO:
        pkg = "".join((pkg, _repo_separator, myrepo))
    cpv_slot_list = [pkg]

    mycp = mysplit[0] + "/" + mysplit[1]

    # XXX- This is a temporary duplicate of code from the config constructor.
    locations = [os.path.join(settings["PORTDIR"], "profiles")]
    locations.extend(settings.profiles)
    for ov in settings["PORTDIR_OVERLAY"].split():
        profdir = os.path.join(normalize_path(ov), "profiles")
        if os.path.isdir(profdir):
            locations.append(profdir)
    locations.append(
        os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH))
    locations.reverse()
    pmasklists = []
    for profile in locations:
        pmask_filename = os.path.join(profile, "package.mask")
        pmasklists.append(
            (pmask_filename, grablines(pmask_filename, recursive=1)))

    pmaskdict = settings._mask_manager._pmaskdict
    if mycp in pmaskdict:
        for x in pmaskdict[mycp]:
            if match_from_list(x, cpv_slot_list):
                x = x.without_repo
                for pmask in pmasklists:
                    comment = ""
                    comment_valid = -1
                    pmask_filename = pmask[0]
                    for i in range(len(pmask[1])):
                        l = pmask[1][i].strip()
                        try:
                            l_atom = Atom(l,
                                          allow_repo=True,
                                          allow_wildcard=True).without_repo
                        except InvalidAtom:
                            l_atom = None
                        if l == "":
                            comment = ""
                            comment_valid = -1
                        elif l[0] == "#":
                            comment += (l + "\n")
                            comment_valid = i + 1
                        elif l_atom == x:
                            if comment_valid != i:
                                comment = ""
                            if return_location:
                                return (comment, pmask_filename)
                            else:
                                return comment
                        elif comment_valid != -1:
                            # Apparently this comment applies to multiple masks, so
                            # it remains valid until a blank line is encountered.
                            comment_valid += 1
    if return_location:
        return (None, None)
    else:
        return None
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False):
	if settings is None:
		settings = portage.settings
	if portdb is None:
		portdb = portage.portdb
	mysplit = catpkgsplit(mycpv)
	if not mysplit:
		raise ValueError(_("invalid CPV: %s") % mycpv)
	if metadata is None:
		db_keys = list(portdb._aux_cache_keys)
		try:
			metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys)))
		except KeyError:
			if not portdb.cpv_exists(mycpv):
				raise
	if metadata is None:
		# Can't access SLOT due to corruption.
		cpv_slot_list = [mycpv]
	else:
		cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])]
	mycp=mysplit[0]+"/"+mysplit[1]

	# XXX- This is a temporary duplicate of code from the config constructor.
	locations = [os.path.join(settings["PORTDIR"], "profiles")]
	locations.extend(settings.profiles)
	for ov in settings["PORTDIR_OVERLAY"].split():
		profdir = os.path.join(normalize_path(ov), "profiles")
		if os.path.isdir(profdir):
			locations.append(profdir)
	locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
		USER_CONFIG_PATH))
	locations.reverse()
	pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations]

	if mycp in settings.pmaskdict:
		for x in settings.pmaskdict[mycp]:
			if match_from_list(x, cpv_slot_list):
				for pmask in pmasklists:
					comment = ""
					comment_valid = -1
					pmask_filename = os.path.join(pmask[0], "package.mask")
					for i in range(len(pmask[1])):
						l = pmask[1][i].strip()
						if l == "":
							comment = ""
							comment_valid = -1
						elif l[0] == "#":
							comment += (l+"\n")
							comment_valid = i + 1
						elif l == x:
							if comment_valid != i:
								comment = ""
							if return_location:
								return (comment, pmask_filename)
							else:
								return comment
						elif comment_valid != -1:
							# Apparently this comment applies to muliple masks, so
							# it remains valid until a blank line is encountered.
							comment_valid += 1
	if return_location:
		return (None, None)
	else:
		return None
Exemple #10
0
    def __init__(self, options, portdb, event_loop):
        self.options = options
        self.portdb = portdb
        self.event_loop = event_loop
        self.added_byte_count = 0
        self.added_file_count = 0
        self.scheduled_deletion_count = 0
        self.delete_count = 0
        self.file_owners = {}
        self.file_failures = {}
        self.start_time = time.time()
        self._open_files = []

        self.log_success = self._open_log(
            "success", getattr(options, "success_log", None), "a"
        )
        self.log_failure = self._open_log(
            "failure", getattr(options, "failure_log", None), "a"
        )

        self.distfiles = None
        if getattr(options, "distfiles", None) is not None:
            self.distfiles = options.distfiles

        self.mirrors = copy.copy(portdb.settings.thirdpartymirrors())

        if getattr(options, "mirror_overrides", None) is not None:
            self.mirrors.update(grabdict(options.mirror_overrides))

        if getattr(options, "mirror_skip", None) is not None:
            for x in options.mirror_skip.split(","):
                self.mirrors[x] = []

        self.whitelist = None
        if getattr(options, "whitelist_from", None) is not None:
            self.whitelist = set()
            for filename in options.whitelist_from:
                for line in grablines(filename):
                    line = line.strip()
                    if line and not line.startswith("#"):
                        self.whitelist.add(line)

        self.restrict_mirror_exemptions = None
        if getattr(options, "restrict_mirror_exemptions", None) is not None:
            self.restrict_mirror_exemptions = frozenset(
                options.restrict_mirror_exemptions.split(",")
            )

        self.recycle_db = None
        if getattr(options, "recycle_db", None) is not None:
            self.recycle_db = self._open_shelve(options.recycle_db, "recycle")

        self.distfiles_db = None
        if getattr(options, "distfiles_db", None) is not None:
            self.distfiles_db = self._open_shelve(options.distfiles_db, "distfiles")

        self.content_db = None
        if getattr(options, "content_db", None) is not None:
            self.content_db = ContentDB(
                self._open_shelve(options.content_db, "content")
            )

        self.deletion_db = None
        if getattr(options, "deletion_db", None) is not None:
            self.deletion_db = self._open_shelve(options.deletion_db, "deletion")

        self.layout_conf = MirrorLayoutConfig()
        if getattr(options, "layout_conf", None) is None:
            options.layout_conf = os.path.join(self.distfiles, "layout.conf")
        self.layout_conf.read_from_file(options.layout_conf)
        self.layouts = self.layout_conf.get_all_layouts()
Exemple #11
0
def getmaskingreason(mycpv, metadata=None, settings=None, portdb=None, return_location=False):
	if settings is None:
		settings = portage.settings
	if portdb is None:
		portdb = portage.portdb
	mysplit = catpkgsplit(mycpv)
	if not mysplit:
		raise ValueError(_("invalid CPV: %s") % mycpv)
	if metadata is None:
		db_keys = list(portdb._aux_cache_keys)
		try:
			metadata = dict(zip(db_keys, portdb.aux_get(mycpv, db_keys)))
		except KeyError:
			if not portdb.cpv_exists(mycpv):
				raise
	if metadata is None:
		# Can't access SLOT due to corruption.
		cpv_slot_list = [mycpv]
	else:
		cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])]
	mycp=mysplit[0]+"/"+mysplit[1]

	# XXX- This is a temporary duplicate of code from the config constructor.
	locations = [os.path.join(settings["PORTDIR"], "profiles")]
	locations.extend(settings.profiles)
	for ov in settings["PORTDIR_OVERLAY"].split():
		profdir = os.path.join(normalize_path(ov), "profiles")
		if os.path.isdir(profdir):
			locations.append(profdir)
	locations.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
		USER_CONFIG_PATH))
	locations.reverse()
	pmasklists = [(x, grablines(os.path.join(x, "package.mask"), recursive=1)) for x in locations]

	pmaskdict = settings._mask_manager._pmaskdict
	if mycp in pmaskdict:
		for x in pmaskdict[mycp]:
			if match_from_list(x, cpv_slot_list):
				for pmask in pmasklists:
					comment = ""
					comment_valid = -1
					pmask_filename = os.path.join(pmask[0], "package.mask")
					for i in range(len(pmask[1])):
						l = pmask[1][i].strip()
						if l == "":
							comment = ""
							comment_valid = -1
						elif l[0] == "#":
							comment += (l+"\n")
							comment_valid = i + 1
						elif l == x:
							if comment_valid != i:
								comment = ""
							if return_location:
								return (comment, pmask_filename)
							else:
								return comment
						elif comment_valid != -1:
							# Apparently this comment applies to muliple masks, so
							# it remains valid until a blank line is encountered.
							comment_valid += 1
	if return_location:
		return (None, None)
	else:
		return None