예제 #1
0
파일: scanner.py 프로젝트: gmt/portage
	def __init__(self, repo_settings, myreporoot, config_root, options,
				vcs_settings, mydir, env):
		'''Class __init__'''
		self.repo_settings = repo_settings
		self.config_root = config_root
		self.options = options
		self.vcs_settings = vcs_settings
		self.env = env

		# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
		# behave incrementally.
		self.repoman_incrementals = tuple(
			x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')

		self.categories = []
		for path in self.repo_settings.repo_config.eclass_db.porttrees:
			self.categories.extend(portage.util.grabfile(
				os.path.join(path, 'profiles', 'categories')))
		self.repo_settings.repoman_settings.categories = frozenset(
			portage.util.stack_lists([self.categories], incremental=1))
		self.categories = self.repo_settings.repoman_settings.categories

		self.portdb = repo_settings.portdb
		self.portdb.settings = self.repo_settings.repoman_settings
		# We really only need to cache the metadata that's necessary for visibility
		# filtering. Anything else can be discarded to reduce memory consumption.
		self.portdb._aux_cache_keys.clear()
		self.portdb._aux_cache_keys.update(
			["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])

		self.reposplit = myreporoot.split(os.path.sep)
		self.repolevel = len(self.reposplit)

		if self.options.mode == 'commit':
			repochecks.commit_check(self.repolevel, self.reposplit)
			repochecks.conflict_check(self.vcs_settings, self.options)

		# Make startdir relative to the canonical repodir, so that we can pass
		# it to digestgen and it won't have to be canonicalized again.
		if self.repolevel == 1:
			startdir = self.repo_settings.repodir
		else:
			startdir = normalize_path(mydir)
			startdir = os.path.join(
				self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:])

		# get lists of valid keywords, licenses, and use
		new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings)
		kwlist, liclist, uselist, profile_list, \
			global_pmaskdict, liclist_deprecated = new_data
		self.repo_metadata = {
			'kwlist': kwlist,
			'liclist': liclist,
			'uselist': uselist,
			'profile_list': profile_list,
			'pmaskdict': global_pmaskdict,
			'lic_deprecated': liclist_deprecated,
		}

		self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
		self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')

		self.profiles = setup_profile(profile_list)

		check_profiles(self.profiles, self.repo_settings.repoman_settings.archlist())

		scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings)

		self.dev_keywords = dev_profile_keywords(self.profiles)

		self.qatracker = QATracker()

		if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
			self.options.echangelog = 'y'

		if self.vcs_settings.vcs is None:
			self.options.echangelog = 'n'

		self.check = {}
		# The --echangelog option causes automatic ChangeLog generation,
		# which invalidates changelog.ebuildadded and changelog.missing
		# checks.
		# Note: Some don't use ChangeLogs in distributed SCMs.
		# It will be generated on server side from scm log,
		# before package moves to the rsync server.
		# This is needed because they try to avoid merge collisions.
		# Gentoo's Council decided to always use the ChangeLog file.
		# TODO: shouldn't this just be switched on the repo, iso the VCS?
		is_echangelog_enabled = self.options.echangelog in ('y', 'force')
		self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn')
		self.check['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn

		if self.options.mode == "manifest":
			pass
		elif self.options.pretend:
			print(green("\nRepoMan does a once-over of the neighborhood..."))
		else:
			print(green("\nRepoMan scours the neighborhood..."))

		self.changed = Changes(self.options)
		self.changed.scan(self.vcs_settings)

		self.have = {
			'pmasked': False,
			'dev_keywords': False,
		}

		# NOTE: match-all caches are not shared due to potential
		# differences between profiles in _get_implicit_iuse.
		self.caches = {
			'arch': {},
			'arch_xmatch': {},
			'shared_xmatch': {"cp-list": {}},
		}

		self.include_arches = None
		if self.options.include_arches:
			self.include_arches = set()
			self.include_arches.update(*[x.split() for x in self.options.include_arches])

		# Disable the "ebuild.notadded" check when not in commit mode and
		# running `svn status` in every package dir will be too expensive.
		self.check['ebuild_notadded'] = not \
			(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")

		self.effective_scanlist = scanlist
		if self.options.if_modified == "y":
			self.effective_scanlist = sorted(vcs_files_to_cps(
				chain(self.changed.changed, self.changed.new, self.changed.removed),
				self.repolevel, self.reposplit, self.categories))

		self.live_eclasses = portage.const.LIVE_ECLASSES

		# initialize our checks classes here before the big xpkg loop
		self.manifester = Manifests(self.options, self.qatracker, self.repo_settings.repoman_settings)
		self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.qatracker)
		self.filescheck = FileChecks(
			self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings)
		self.status_check = VCSStatus(self.vcs_settings, self.qatracker)
		self.fetchcheck = FetchChecks(
			self.qatracker, self.repo_settings, self.portdb, self.vcs_settings)
		self.pkgmeta = PkgMetadata(self.options, self.qatracker, self.repo_settings.repoman_settings)
		self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker)
		self.use_flag_checks = USEFlagChecks(self.qatracker, uselist)
		self.keywordcheck = KeywordChecks(self.qatracker, self.options)
		self.liveeclasscheck = LiveEclassChecks(self.qatracker)
		self.rubyeclasscheck = RubyEclassChecks(self.qatracker)
		self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings)
		self.descriptioncheck = DescriptionChecks(self.qatracker)
		self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated)
		self.restrictcheck = RestrictChecks(self.qatracker)
예제 #2
0
파일: scanner.py 프로젝트: emlove/portage
	def __init__(self, repo_settings, myreporoot, config_root, options,
				vcs_settings, mydir, env):
		'''Class __init__'''
		self.repo_settings = repo_settings
		self.config_root = config_root
		self.options = options
		self.vcs_settings = vcs_settings
		self.env = env

		# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
		# behave incrementally.
		self.repoman_incrementals = tuple(
			x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')

		self.categories = []
		for path in self.repo_settings.repo_config.eclass_db.porttrees:
			self.categories.extend(portage.util.grabfile(
				os.path.join(path, 'profiles', 'categories')))
		self.repo_settings.repoman_settings.categories = frozenset(
			portage.util.stack_lists([self.categories], incremental=1))
		self.categories = self.repo_settings.repoman_settings.categories

		metadata_dtd = None
		for path in reversed(self.repo_settings.repo_config.eclass_db.porttrees):
			path = os.path.join(path, 'metadata/dtd/metadata.dtd')
			if os.path.exists(path):
				metadata_dtd = path
				break

		self.portdb = repo_settings.portdb
		self.portdb.settings = self.repo_settings.repoman_settings
		# We really only need to cache the metadata that's necessary for visibility
		# filtering. Anything else can be discarded to reduce memory consumption.
		if self.options.mode != "manifest" and self.options.digest != "y":
			# Don't do this when generating manifests, since that uses
			# additional keys if spawn_nofetch is called (RESTRICT and
			# DEFINED_PHASES).
			self.portdb._aux_cache_keys.clear()
			self.portdb._aux_cache_keys.update(
				["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])

		self.reposplit = myreporoot.split(os.path.sep)
		self.repolevel = len(self.reposplit)

		if self.options.mode == 'commit':
			repochecks.commit_check(self.repolevel, self.reposplit)
			repochecks.conflict_check(self.vcs_settings, self.options)

		# Make startdir relative to the canonical repodir, so that we can pass
		# it to digestgen and it won't have to be canonicalized again.
		if self.repolevel == 1:
			startdir = self.repo_settings.repodir
		else:
			startdir = normalize_path(mydir)
			startdir = os.path.join(
				self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:])

		# get lists of valid keywords, licenses, and use
		new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings)
		kwlist, liclist, uselist, profile_list, \
			global_pmaskdict, liclist_deprecated = new_data
		self.repo_metadata = {
			'kwlist': kwlist,
			'liclist': liclist,
			'uselist': uselist,
			'profile_list': profile_list,
			'pmaskdict': global_pmaskdict,
			'lic_deprecated': liclist_deprecated,
		}

		self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
		self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')

		self.profiles = setup_profile(profile_list)

		check_profiles(self.profiles, self.repo_settings.repoman_settings.archlist())

		scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings)

		self.dev_keywords = dev_profile_keywords(self.profiles)

		self.qatracker = QATracker()

		if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
			self.options.echangelog = 'y'

		if self.vcs_settings.vcs is None:
			self.options.echangelog = 'n'

		self.check = {}
		# The --echangelog option causes automatic ChangeLog generation,
		# which invalidates changelog.ebuildadded and changelog.missing
		# checks.
		# Note: Some don't use ChangeLogs in distributed SCMs.
		# It will be generated on server side from scm log,
		# before package moves to the rsync server.
		# This is needed because they try to avoid merge collisions.
		# Gentoo's Council decided to always use the ChangeLog file.
		# TODO: shouldn't this just be switched on the repo, iso the VCS?
		is_echangelog_enabled = self.options.echangelog in ('y', 'force')
		self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn')
		self.check['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn

		if self.options.mode == "manifest" or self.options.quiet:
			pass
		elif self.options.pretend:
			print(green("\nRepoMan does a once-over of the neighborhood..."))
		else:
			print(green("\nRepoMan scours the neighborhood..."))

		self.changed = Changes(self.options)
		# bypass unneeded VCS operations if not needed
		if (self.options.if_modified == "y" or
			self.options.mode not in ("manifest", "manifest-check")):
			self.changed.scan(self.vcs_settings)

		self.have = {
			'pmasked': False,
			'dev_keywords': False,
		}

		# NOTE: match-all caches are not shared due to potential
		# differences between profiles in _get_implicit_iuse.
		self.caches = {
			'arch': {},
			'arch_xmatch': {},
			'shared_xmatch': {"cp-list": {}},
		}

		self.include_arches = None
		if self.options.include_arches:
			self.include_arches = set()
			self.include_arches.update(*[x.split() for x in self.options.include_arches])

		# Disable the "ebuild.notadded" check when not in commit mode and
		# running `svn status` in every package dir will be too expensive.
		self.check['ebuild_notadded'] = not \
			(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")

		self.effective_scanlist = scanlist
		if self.options.if_modified == "y":
			self.effective_scanlist = sorted(vcs_files_to_cps(
				chain(self.changed.changed, self.changed.new, self.changed.removed),
				self.repolevel, self.reposplit, self.categories))

		self.live_eclasses = portage.const.LIVE_ECLASSES

		# initialize our checks classes here before the big xpkg loop
		self.manifester = Manifests(self.options, self.qatracker, self.repo_settings.repoman_settings)
		self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.qatracker)
		self.filescheck = FileChecks(
			self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings)
		self.status_check = VCSStatus(self.vcs_settings, self.qatracker)
		self.fetchcheck = FetchChecks(
			self.qatracker, self.repo_settings, self.portdb, self.vcs_settings)
		self.pkgmeta = PkgMetadata(self.options, self.qatracker,
			self.repo_settings.repoman_settings, metadata_dtd=metadata_dtd)
		self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker)
		self.use_flag_checks = USEFlagChecks(self.qatracker, uselist)
		self.keywordcheck = KeywordChecks(self.qatracker, self.options)
		self.liveeclasscheck = LiveEclassChecks(self.qatracker)
		self.rubyeclasscheck = RubyEclassChecks(self.qatracker)
		self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings)
		self.descriptioncheck = DescriptionChecks(self.qatracker)
		self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated)
		self.restrictcheck = RestrictChecks(self.qatracker)
예제 #3
0
파일: scanner.py 프로젝트: gmt/portage
class Scanner(object):
	'''Primary scan class.  Operates all the small Q/A tests and checks'''

	def __init__(self, repo_settings, myreporoot, config_root, options,
				vcs_settings, mydir, env):
		'''Class __init__'''
		self.repo_settings = repo_settings
		self.config_root = config_root
		self.options = options
		self.vcs_settings = vcs_settings
		self.env = env

		# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
		# behave incrementally.
		self.repoman_incrementals = tuple(
			x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')

		self.categories = []
		for path in self.repo_settings.repo_config.eclass_db.porttrees:
			self.categories.extend(portage.util.grabfile(
				os.path.join(path, 'profiles', 'categories')))
		self.repo_settings.repoman_settings.categories = frozenset(
			portage.util.stack_lists([self.categories], incremental=1))
		self.categories = self.repo_settings.repoman_settings.categories

		self.portdb = repo_settings.portdb
		self.portdb.settings = self.repo_settings.repoman_settings
		# We really only need to cache the metadata that's necessary for visibility
		# filtering. Anything else can be discarded to reduce memory consumption.
		self.portdb._aux_cache_keys.clear()
		self.portdb._aux_cache_keys.update(
			["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])

		self.reposplit = myreporoot.split(os.path.sep)
		self.repolevel = len(self.reposplit)

		if self.options.mode == 'commit':
			repochecks.commit_check(self.repolevel, self.reposplit)
			repochecks.conflict_check(self.vcs_settings, self.options)

		# Make startdir relative to the canonical repodir, so that we can pass
		# it to digestgen and it won't have to be canonicalized again.
		if self.repolevel == 1:
			startdir = self.repo_settings.repodir
		else:
			startdir = normalize_path(mydir)
			startdir = os.path.join(
				self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:])

		# get lists of valid keywords, licenses, and use
		new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings)
		kwlist, liclist, uselist, profile_list, \
			global_pmaskdict, liclist_deprecated = new_data
		self.repo_metadata = {
			'kwlist': kwlist,
			'liclist': liclist,
			'uselist': uselist,
			'profile_list': profile_list,
			'pmaskdict': global_pmaskdict,
			'lic_deprecated': liclist_deprecated,
		}

		self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
		self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')

		self.profiles = setup_profile(profile_list)

		check_profiles(self.profiles, self.repo_settings.repoman_settings.archlist())

		scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings)

		self.dev_keywords = dev_profile_keywords(self.profiles)

		self.qatracker = QATracker()

		if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
			self.options.echangelog = 'y'

		if self.vcs_settings.vcs is None:
			self.options.echangelog = 'n'

		self.check = {}
		# The --echangelog option causes automatic ChangeLog generation,
		# which invalidates changelog.ebuildadded and changelog.missing
		# checks.
		# Note: Some don't use ChangeLogs in distributed SCMs.
		# It will be generated on server side from scm log,
		# before package moves to the rsync server.
		# This is needed because they try to avoid merge collisions.
		# Gentoo's Council decided to always use the ChangeLog file.
		# TODO: shouldn't this just be switched on the repo, iso the VCS?
		is_echangelog_enabled = self.options.echangelog in ('y', 'force')
		self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn')
		self.check['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn

		if self.options.mode == "manifest":
			pass
		elif self.options.pretend:
			print(green("\nRepoMan does a once-over of the neighborhood..."))
		else:
			print(green("\nRepoMan scours the neighborhood..."))

		self.changed = Changes(self.options)
		self.changed.scan(self.vcs_settings)

		self.have = {
			'pmasked': False,
			'dev_keywords': False,
		}

		# NOTE: match-all caches are not shared due to potential
		# differences between profiles in _get_implicit_iuse.
		self.caches = {
			'arch': {},
			'arch_xmatch': {},
			'shared_xmatch': {"cp-list": {}},
		}

		self.include_arches = None
		if self.options.include_arches:
			self.include_arches = set()
			self.include_arches.update(*[x.split() for x in self.options.include_arches])

		# Disable the "ebuild.notadded" check when not in commit mode and
		# running `svn status` in every package dir will be too expensive.
		self.check['ebuild_notadded'] = not \
			(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")

		self.effective_scanlist = scanlist
		if self.options.if_modified == "y":
			self.effective_scanlist = sorted(vcs_files_to_cps(
				chain(self.changed.changed, self.changed.new, self.changed.removed),
				self.repolevel, self.reposplit, self.categories))

		self.live_eclasses = portage.const.LIVE_ECLASSES

		# initialize our checks classes here before the big xpkg loop
		self.manifester = Manifests(self.options, self.qatracker, self.repo_settings.repoman_settings)
		self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.qatracker)
		self.filescheck = FileChecks(
			self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings)
		self.status_check = VCSStatus(self.vcs_settings, self.qatracker)
		self.fetchcheck = FetchChecks(
			self.qatracker, self.repo_settings, self.portdb, self.vcs_settings)
		self.pkgmeta = PkgMetadata(self.options, self.qatracker, self.repo_settings.repoman_settings)
		self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker)
		self.use_flag_checks = USEFlagChecks(self.qatracker, uselist)
		self.keywordcheck = KeywordChecks(self.qatracker, self.options)
		self.liveeclasscheck = LiveEclassChecks(self.qatracker)
		self.rubyeclasscheck = RubyEclassChecks(self.qatracker)
		self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings)
		self.descriptioncheck = DescriptionChecks(self.qatracker)
		self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated)
		self.restrictcheck = RestrictChecks(self.qatracker)


	def scan_pkgs(self, can_force):
		for xpkg in self.effective_scanlist:
			# ebuilds and digests added to cvs respectively.
			logging.info("checking package %s" % xpkg)
			# save memory by discarding xmatch caches from previous package(s)
			self.caches['arch_xmatch'].clear()
			self.eadded = []
			catdir, pkgdir = xpkg.split("/")
			checkdir = self.repo_settings.repodir + "/" + xpkg
			checkdir_relative = ""
			if self.repolevel < 3:
				checkdir_relative = os.path.join(pkgdir, checkdir_relative)
			if self.repolevel < 2:
				checkdir_relative = os.path.join(catdir, checkdir_relative)
			checkdir_relative = os.path.join(".", checkdir_relative)

			if self.manifester.run(checkdir, self.portdb):
				continue
			if not self.manifester.generated_manifest:
				self.manifester.digest_check(xpkg, checkdir)
			if self.options.mode == 'manifest-check':
				continue

			checkdirlist = os.listdir(checkdir)

			self.pkgs, self.allvalid = self.is_ebuild.check(checkdirlist, checkdir, xpkg)
			if self.is_ebuild.continue_:
				# If we can't access all the metadata then it's totally unsafe to
				# commit since there's no way to generate a correct Manifest.
				# Do not try to do any more QA checks on this package since missing
				# metadata leads to false positives for several checks, and false
				# positives confuse users.
				can_force = False
				continue

			self.keywordcheck.prepare()

			# Sort ebuilds in ascending order for the KEYWORDS.dropped check.
			ebuildlist = sorted(self.pkgs.values())
			ebuildlist = [pkg.pf for pkg in ebuildlist]

			self.filescheck.check(
				checkdir, checkdirlist, checkdir_relative, self.changed.changed, self.changed.new)

			self.status_check.check(self.check['ebuild_notadded'], checkdir, checkdir_relative, xpkg)
			self.eadded.extend(self.status_check.eadded)

			self.fetchcheck.check(
				xpkg, checkdir, checkdir_relative, self.changed.changed, self.changed.new)

			if self.check['changelog'] and "ChangeLog" not in checkdirlist:
				self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog")

			self.pkgmeta.check(xpkg, checkdir, checkdirlist, self.repolevel)
			self.muselist = frozenset(self.pkgmeta.musedict)

			changelog_path = os.path.join(checkdir_relative, "ChangeLog")
			self.changelog_modified = changelog_path in self.changed.changelogs

			self._scan_ebuilds(ebuildlist, xpkg, catdir, pkgdir)
		return self.qatracker, can_force


	def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
		# detect unused local USE-descriptions
		used_useflags = set()

		for y_ebuild in ebuildlist:

			ebuild = Ebuild(
				self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings,
				xpkg, y_ebuild)

			if self.check['changelog'] and not self.changelog_modified \
				and ebuild.ebuild_path in self.changed.new_ebuilds:
				self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)

			if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded):
				# ebuild not added to vcs
				self.qatracker.add_error(
					"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")

			if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker):
				continue

			pkg = self.pkgs[y_ebuild]
			if pkg_invalid(pkg, self.qatracker, ebuild):
				self.allvalid = False
				continue

			myaux = pkg._metadata
			eapi = myaux["EAPI"]
			inherited = pkg.inherited
			live_ebuild = self.live_eclasses.intersection(inherited)

			self.eapicheck.check(pkg, ebuild)

			for k, v in myaux.items():
				if not isinstance(v, basestring):
					continue
				m = NON_ASCII_RE.search(v)
				if m is not None:
					self.qatracker.add_error(
						"variable.invalidchar",
						"%s: %s variable contains non-ASCII "
						"character at position %s" %
						(ebuild.relative_path, k, m.start() + 1))

			if not self.fetchcheck.src_uri_error:
				self.thirdparty.check(myaux, ebuild.relative_path)

			if myaux.get("PROVIDE"):
				self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)

			for pos, missing_var in enumerate(missingvars):
				if not myaux.get(missing_var):
					if catdir == "virtual" and \
						missing_var in ("HOMEPAGE", "LICENSE"):
						continue
					if live_ebuild and missing_var == "KEYWORDS":
						continue
					myqakey = missingvars[pos] + ".missing"
					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")

			if catdir == "virtual":
				for var in ("HOMEPAGE", "LICENSE"):
					if myaux.get(var):
						myqakey = var + ".virtual"
						self.qatracker.add_error(myqakey, ebuild.relative_path)

			self.descriptioncheck.check(pkg, ebuild)

			keywords = myaux["KEYWORDS"].split()

			ebuild_archs = set(
				kw.lstrip("~") for kw in keywords if not kw.startswith("-"))

			self.keywordcheck.check(
				pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed,
				live_ebuild, self.repo_metadata['kwlist'], self.profiles)

			if live_ebuild and self.repo_settings.repo_config.name == "gentoo":
				self.liveeclasscheck.check(
					pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict'])

			if self.options.ignore_arches:
				arches = [[
					self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"],
					self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
			else:
				arches = set()
				for keyword in keywords:
					if keyword[0] == "-":
						continue
					elif keyword[0] == "~":
						arch = keyword[1:]
						if arch == "*":
							for expanded_arch in self.profiles:
								if expanded_arch == "**":
									continue
								arches.add(
									(keyword, expanded_arch, (
										expanded_arch, "~" + expanded_arch)))
						else:
							arches.add((keyword, arch, (arch, keyword)))
					else:
						# For ebuilds with stable keywords, check if the
						# dependencies are satisfiable for unstable
						# configurations, since use.stable.mask is not
						# applied for unstable configurations (see bug
						# 563546).
						if keyword == "*":
							for expanded_arch in self.profiles:
								if expanded_arch == "**":
									continue
								arches.add(
									(keyword, expanded_arch, (expanded_arch,)))
								arches.add(
									(keyword, expanded_arch,
										(expanded_arch, "~" + expanded_arch)))
						else:
							arches.add((keyword, keyword, (keyword,)))
							arches.add((keyword, keyword,
								(keyword, "~" + keyword)))
				if not arches:
					# Use an empty profile for checking dependencies of
					# packages that have empty KEYWORDS.
					arches.add(('**', '**', ('**',)))

			unknown_pkgs = set()
			baddepsyntax = False
			badlicsyntax = False
			badprovsyntax = False
			# catpkg = catdir + "/" + y_ebuild

			inherited_java_eclass = "java-pkg-2" in inherited or \
				"java-pkg-opt-2" in inherited
			inherited_wxwidgets_eclass = "wxwidgets" in inherited
			# operator_tokens = set(["||", "(", ")"])
			type_list, badsyntax = [], []
			for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
				mydepstr = myaux[mytype]

				buildtime = mytype in Package._buildtime_keys
				runtime = mytype in Package._runtime_keys
				token_class = None
				if mytype.endswith("DEPEND"):
					token_class = portage.dep.Atom

				try:
					atoms = portage.dep.use_reduce(
						mydepstr, matchall=1, flat=True,
						is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
				except portage.exception.InvalidDependString as e:
					atoms = None
					badsyntax.append(str(e))

				if atoms and mytype.endswith("DEPEND"):
					if runtime and \
						"test?" in mydepstr.split():
						self.qatracker.add_error(
							mytype + '.suspect',
							"%s: 'test?' USE conditional in %s" %
							(ebuild.relative_path, mytype))

					for atom in atoms:
						if atom == "||":
							continue

						is_blocker = atom.blocker

						# Skip dependency.unknown for blockers, so that we
						# don't encourage people to remove necessary blockers,
						# as discussed in bug 382407. We use atom.without_use
						# due to bug 525376.
						if not is_blocker and \
							not self.portdb.xmatch("match-all", atom.without_use) and \
							not atom.cp.startswith("virtual/"):
							unknown_pkgs.add((mytype, atom.unevaluated_atom))

						if catdir != "virtual":
							if not is_blocker and \
								atom.cp in suspect_virtual:
								self.qatracker.add_error(
									'virtual.suspect', ebuild.relative_path +
									": %s: consider using '%s' instead of '%s'" %
									(mytype, suspect_virtual[atom.cp], atom))
							if not is_blocker and \
								atom.cp.startswith("perl-core/"):
								self.qatracker.add_error('dependency.perlcore',
									ebuild.relative_path +
									": %s: please use '%s' instead of '%s'" %
									(mytype,
									atom.replace("perl-core/","virtual/perl-"),
									atom))

						if buildtime and \
							not is_blocker and \
							not inherited_java_eclass and \
							atom.cp == "virtual/jdk":
							self.qatracker.add_error(
								'java.eclassesnotused', ebuild.relative_path)
						elif buildtime and \
							not is_blocker and \
							not inherited_wxwidgets_eclass and \
							atom.cp == "x11-libs/wxGTK":
							self.qatracker.add_error(
								'wxwidgets.eclassnotused',
								"%s: %ss on x11-libs/wxGTK without inheriting"
								" wxwidgets.eclass" % (ebuild.relative_path, mytype))
						elif runtime:
							if not is_blocker and \
								atom.cp in suspect_rdepend:
								self.qatracker.add_error(
									mytype + '.suspect',
									ebuild.relative_path + ": '%s'" % atom)

						if atom.operator == "~" and \
							portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
							qacat = 'dependency.badtilde'
							self.qatracker.add_error(
								qacat, "%s: %s uses the ~ operator"
								" with a non-zero revision: '%s'" %
								(ebuild.relative_path, mytype, atom))

						check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker,
							ebuild.relative_path, myaux)

				type_list.extend([mytype] * (len(badsyntax) - len(type_list)))

			for m, b in zip(type_list, badsyntax):
				if m.endswith("DEPEND"):
					qacat = "dependency.syntax"
				else:
					qacat = m + ".syntax"
				self.qatracker.add_error(
					qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))

			badlicsyntax = len([z for z in type_list if z == "LICENSE"])
			badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
			baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
			badlicsyntax = badlicsyntax > 0
			badprovsyntax = badprovsyntax > 0

			self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist)

			ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags()
			used_useflags = used_useflags.union(ebuild_used_useflags)

			self.rubyeclasscheck.check(pkg, ebuild)

			# license checks
			if not badlicsyntax:
				self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild)

			self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)

			# Syntax Checks
			if not self.vcs_settings.vcs_preserves_mtime:
				if ebuild.ebuild_path not in self.changed.new_ebuilds and \
					ebuild.ebuild_path not in self.changed.ebuilds:
					pkg.mtime = None
			try:
				# All ebuilds should have utf_8 encoding.
				f = io.open(
					_unicode_encode(
						ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
					mode='r', encoding=_encodings['repo.content'])
				try:
					for check_name, e in run_checks(f, pkg):
						self.qatracker.add_error(
							check_name, ebuild.relative_path + ': %s' % e)
				finally:
					f.close()
			except UnicodeDecodeError:
				# A file.UTF8 failure will have already been recorded above.
				pass

			if self.options.force:
				# The dep_check() calls are the most expensive QA test. If --force
				# is enabled, there's no point in wasting time on these since the
				# user is intent on forcing the commit anyway.
				continue

			relevant_profiles = []
			for keyword, arch, groups in arches:
				if arch not in self.profiles:
					# A missing profile will create an error further down
					# during the KEYWORDS verification.
					continue

				if self.include_arches is not None:
					if arch not in self.include_arches:
						continue

				relevant_profiles.extend(
					(keyword, groups, prof) for prof in self.profiles[arch])

			relevant_profiles.sort(key=sort_key)

			for keyword, groups, prof in relevant_profiles:

				is_stable_profile = prof.status == "stable"
				is_dev_profile = prof.status == "dev" and \
					self.options.include_dev
				is_exp_profile = prof.status == "exp" and \
					self.options.include_exp_profiles == 'y'
				if not (is_stable_profile or is_dev_profile or is_exp_profile):
					continue

				dep_settings = self.caches['arch'].get(prof.sub_path)
				if dep_settings is None:
					dep_settings = portage.config(
						config_profile_path=prof.abs_path,
						config_incrementals=self.repoman_incrementals,
						config_root=self.config_root,
						local_config=False,
						_unmatched_removal=self.options.unmatched_removal,
						env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
					dep_settings.categories = self.repo_settings.repoman_settings.categories
					if self.options.without_mask:
						dep_settings._mask_manager_obj = \
							copy.deepcopy(dep_settings._mask_manager)
						dep_settings._mask_manager._pmaskdict.clear()
					self.caches['arch'][prof.sub_path] = dep_settings

				xmatch_cache_key = (prof.sub_path, tuple(groups))
				xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
				if xcache is None:
					self.portdb.melt()
					self.portdb.freeze()
					xcache = self.portdb.xcache
					xcache.update(self.caches['shared_xmatch'])
					self.caches['arch_xmatch'][xmatch_cache_key] = xcache

				self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
				self.portdb.settings = dep_settings
				self.portdb.xcache = xcache

				dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
				# just in case, prevent config.reset() from nuking these.
				dep_settings.backup_changes("ACCEPT_KEYWORDS")

				# This attribute is used in dbapi._match_use() to apply
				# use.stable.{mask,force} settings based on the stable
				# status of the parent package. This is required in order
				# for USE deps of unstable packages to be resolved correctly,
				# since otherwise use.stable.{mask,force} settings of
				# dependencies may conflict (see bug #456342).
				dep_settings._parent_stable = dep_settings._isStable(pkg)

				# Handle package.use*.{force,mask) calculation, for use
				# in dep_check.
				dep_settings.useforce = dep_settings._use_manager.getUseForce(
					pkg, stable=dep_settings._parent_stable)
				dep_settings.usemask = dep_settings._use_manager.getUseMask(
					pkg, stable=dep_settings._parent_stable)

				if not baddepsyntax:
					ismasked = not ebuild_archs or \
						pkg.cpv not in self.portdb.xmatch("match-visible",
						Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)))
					if ismasked:
						if not self.have['pmasked']:
							self.have['pmasked'] = bool(dep_settings._getMaskAtom(
								pkg.cpv, pkg._metadata))
						if self.options.ignore_masked:
							continue
						# we are testing deps for a masked package; give it some lee-way
						suffix = "masked"
						matchmode = "minimum-all-ignore-profile"
					else:
						suffix = ""
						matchmode = "minimum-visible"

					if not self.have['dev_keywords']:
						self.have['dev_keywords'] = \
							bool(self.dev_keywords.intersection(keywords))

					if prof.status == "dev":
						suffix = suffix + "indev"

					for mytype in Package._dep_keys:

						mykey = "dependency.bad" + suffix
						myvalue = myaux[mytype]
						if not myvalue:
							continue

						success, atoms = portage.dep_check(
							myvalue, self.portdb, dep_settings,
							use="all", mode=matchmode, trees=self.repo_settings.trees)

						if success:
							if atoms:

								# Don't bother with dependency.unknown for
								# cases in which *DEPEND.bad is triggered.
								for atom in atoms:
									# dep_check returns all blockers and they
									# aren't counted for *DEPEND.bad, so we
									# ignore them here.
									if not atom.blocker:
										unknown_pkgs.discard(
											(mytype, atom.unevaluated_atom))

								if not prof.sub_path:
									# old-style virtuals currently aren't
									# resolvable with empty profile, since
									# 'virtuals' mappings are unavailable
									# (it would be expensive to search
									# for PROVIDE in all ebuilds)
									atoms = [
										atom for atom in atoms if not (
											atom.cp.startswith('virtual/')
											and not self.portdb.cp_list(atom.cp))]

								# we have some unsolvable deps
								# remove ! deps, which always show up as unsatisfiable
								atoms = [
									str(atom.unevaluated_atom)
									for atom in atoms if not atom.blocker]

								# if we emptied out our list, continue:
								if not atoms:
									continue
								if self.options.output_style in ['column']:
									self.qatracker.add_error(mykey,
										"%s: %s: %s(%s) %s"
										% (ebuild.relative_path, mytype, keyword,
											prof, repr(atoms)))
								else:
									self.qatracker.add_error(mykey,
										"%s: %s: %s(%s)\n%s"
										% (ebuild.relative_path, mytype, keyword,
											prof, pformat(atoms, indent=6)))
						else:
							if self.options.output_style in ['column']:
								self.qatracker.add_error(mykey,
									"%s: %s: %s(%s) %s"
									% (ebuild.relative_path, mytype, keyword,
										prof, repr(atoms)))
							else:
								self.qatracker.add_error(mykey,
									"%s: %s: %s(%s)\n%s"
									% (ebuild.relative_path, mytype, keyword,
										prof, pformat(atoms, indent=6)))

			if not baddepsyntax and unknown_pkgs:
				type_map = {}
				for mytype, atom in unknown_pkgs:
					type_map.setdefault(mytype, set()).add(atom)
				for mytype, atoms in type_map.items():
					self.qatracker.add_error(
						"dependency.unknown", "%s: %s: %s"
						% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))

		# check if there are unused local USE-descriptions in metadata.xml
		# (unless there are any invalids, to avoid noise)
		if self.allvalid:
			for myflag in self.muselist.difference(used_useflags):
				self.qatracker.add_error(
					"metadata.warning",
					"%s/metadata.xml: unused local USE-description: '%s'"
					% (xpkg, myflag))
예제 #4
0
파일: scanner.py 프로젝트: emlove/portage
class Scanner(object):
	'''Primary scan class.  Operates all the small Q/A tests and checks'''

	def __init__(self, repo_settings, myreporoot, config_root, options,
				vcs_settings, mydir, env):
		'''Class __init__'''
		self.repo_settings = repo_settings
		self.config_root = config_root
		self.options = options
		self.vcs_settings = vcs_settings
		self.env = env

		# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
		# behave incrementally.
		self.repoman_incrementals = tuple(
			x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')

		self.categories = []
		for path in self.repo_settings.repo_config.eclass_db.porttrees:
			self.categories.extend(portage.util.grabfile(
				os.path.join(path, 'profiles', 'categories')))
		self.repo_settings.repoman_settings.categories = frozenset(
			portage.util.stack_lists([self.categories], incremental=1))
		self.categories = self.repo_settings.repoman_settings.categories

		metadata_dtd = None
		for path in reversed(self.repo_settings.repo_config.eclass_db.porttrees):
			path = os.path.join(path, 'metadata/dtd/metadata.dtd')
			if os.path.exists(path):
				metadata_dtd = path
				break

		self.portdb = repo_settings.portdb
		self.portdb.settings = self.repo_settings.repoman_settings
		# We really only need to cache the metadata that's necessary for visibility
		# filtering. Anything else can be discarded to reduce memory consumption.
		if self.options.mode != "manifest" and self.options.digest != "y":
			# Don't do this when generating manifests, since that uses
			# additional keys if spawn_nofetch is called (RESTRICT and
			# DEFINED_PHASES).
			self.portdb._aux_cache_keys.clear()
			self.portdb._aux_cache_keys.update(
				["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])

		self.reposplit = myreporoot.split(os.path.sep)
		self.repolevel = len(self.reposplit)

		if self.options.mode == 'commit':
			repochecks.commit_check(self.repolevel, self.reposplit)
			repochecks.conflict_check(self.vcs_settings, self.options)

		# Make startdir relative to the canonical repodir, so that we can pass
		# it to digestgen and it won't have to be canonicalized again.
		if self.repolevel == 1:
			startdir = self.repo_settings.repodir
		else:
			startdir = normalize_path(mydir)
			startdir = os.path.join(
				self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:])

		# get lists of valid keywords, licenses, and use
		new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings)
		kwlist, liclist, uselist, profile_list, \
			global_pmaskdict, liclist_deprecated = new_data
		self.repo_metadata = {
			'kwlist': kwlist,
			'liclist': liclist,
			'uselist': uselist,
			'profile_list': profile_list,
			'pmaskdict': global_pmaskdict,
			'lic_deprecated': liclist_deprecated,
		}

		self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
		self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')

		self.profiles = setup_profile(profile_list)

		check_profiles(self.profiles, self.repo_settings.repoman_settings.archlist())

		scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings)

		self.dev_keywords = dev_profile_keywords(self.profiles)

		self.qatracker = QATracker()

		if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
			self.options.echangelog = 'y'

		if self.vcs_settings.vcs is None:
			self.options.echangelog = 'n'

		self.check = {}
		# The --echangelog option causes automatic ChangeLog generation,
		# which invalidates changelog.ebuildadded and changelog.missing
		# checks.
		# Note: Some don't use ChangeLogs in distributed SCMs.
		# It will be generated on server side from scm log,
		# before package moves to the rsync server.
		# This is needed because they try to avoid merge collisions.
		# Gentoo's Council decided to always use the ChangeLog file.
		# TODO: shouldn't this just be switched on the repo, iso the VCS?
		is_echangelog_enabled = self.options.echangelog in ('y', 'force')
		self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn')
		self.check['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn

		if self.options.mode == "manifest" or self.options.quiet:
			pass
		elif self.options.pretend:
			print(green("\nRepoMan does a once-over of the neighborhood..."))
		else:
			print(green("\nRepoMan scours the neighborhood..."))

		self.changed = Changes(self.options)
		# bypass unneeded VCS operations if not needed
		if (self.options.if_modified == "y" or
			self.options.mode not in ("manifest", "manifest-check")):
			self.changed.scan(self.vcs_settings)

		self.have = {
			'pmasked': False,
			'dev_keywords': False,
		}

		# NOTE: match-all caches are not shared due to potential
		# differences between profiles in _get_implicit_iuse.
		self.caches = {
			'arch': {},
			'arch_xmatch': {},
			'shared_xmatch': {"cp-list": {}},
		}

		self.include_arches = None
		if self.options.include_arches:
			self.include_arches = set()
			self.include_arches.update(*[x.split() for x in self.options.include_arches])

		# Disable the "ebuild.notadded" check when not in commit mode and
		# running `svn status` in every package dir will be too expensive.
		self.check['ebuild_notadded'] = not \
			(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")

		self.effective_scanlist = scanlist
		if self.options.if_modified == "y":
			self.effective_scanlist = sorted(vcs_files_to_cps(
				chain(self.changed.changed, self.changed.new, self.changed.removed),
				self.repolevel, self.reposplit, self.categories))

		self.live_eclasses = portage.const.LIVE_ECLASSES

		# initialize our checks classes here before the big xpkg loop
		self.manifester = Manifests(self.options, self.qatracker, self.repo_settings.repoman_settings)
		self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.qatracker)
		self.filescheck = FileChecks(
			self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings)
		self.status_check = VCSStatus(self.vcs_settings, self.qatracker)
		self.fetchcheck = FetchChecks(
			self.qatracker, self.repo_settings, self.portdb, self.vcs_settings)
		self.pkgmeta = PkgMetadata(self.options, self.qatracker,
			self.repo_settings.repoman_settings, metadata_dtd=metadata_dtd)
		self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker)
		self.use_flag_checks = USEFlagChecks(self.qatracker, uselist)
		self.keywordcheck = KeywordChecks(self.qatracker, self.options)
		self.liveeclasscheck = LiveEclassChecks(self.qatracker)
		self.rubyeclasscheck = RubyEclassChecks(self.qatracker)
		self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings)
		self.descriptioncheck = DescriptionChecks(self.qatracker)
		self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated)
		self.restrictcheck = RestrictChecks(self.qatracker)


	def scan_pkgs(self, can_force):
		for xpkg in self.effective_scanlist:
			# ebuilds and digests added to cvs respectively.
			logging.info("checking package %s" % xpkg)
			# save memory by discarding xmatch caches from previous package(s)
			self.caches['arch_xmatch'].clear()
			self.eadded = []
			catdir, pkgdir = xpkg.split("/")
			checkdir = self.repo_settings.repodir + "/" + xpkg
			checkdir_relative = ""
			if self.repolevel < 3:
				checkdir_relative = os.path.join(pkgdir, checkdir_relative)
			if self.repolevel < 2:
				checkdir_relative = os.path.join(catdir, checkdir_relative)
			checkdir_relative = os.path.join(".", checkdir_relative)

			if self.manifester.run(checkdir, self.portdb):
				continue
			if not self.manifester.generated_manifest:
				self.manifester.digest_check(xpkg, checkdir)
			if self.options.mode == 'manifest-check':
				continue

			checkdirlist = os.listdir(checkdir)

			self.pkgs, self.allvalid = self.is_ebuild.check(checkdirlist, checkdir, xpkg)
			if self.is_ebuild.continue_:
				# If we can't access all the metadata then it's totally unsafe to
				# commit since there's no way to generate a correct Manifest.
				# Do not try to do any more QA checks on this package since missing
				# metadata leads to false positives for several checks, and false
				# positives confuse users.
				can_force = False
				continue

			self.keywordcheck.prepare()

			# Sort ebuilds in ascending order for the KEYWORDS.dropped check.
			ebuildlist = sorted(self.pkgs.values())
			ebuildlist = [pkg.pf for pkg in ebuildlist]

			self.filescheck.check(
				checkdir, checkdirlist, checkdir_relative, self.changed.changed, self.changed.new)

			self.status_check.check(self.check['ebuild_notadded'], checkdir, checkdir_relative, xpkg)
			self.eadded.extend(self.status_check.eadded)

			self.fetchcheck.check(
				xpkg, checkdir, checkdir_relative, self.changed.changed, self.changed.new)

			if self.check['changelog'] and "ChangeLog" not in checkdirlist:
				self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog")

			self.pkgmeta.check(xpkg, checkdir, checkdirlist, self.repolevel)
			self.muselist = frozenset(self.pkgmeta.musedict)

			changelog_path = os.path.join(checkdir_relative, "ChangeLog")
			self.changelog_modified = changelog_path in self.changed.changelogs

			self._scan_ebuilds(ebuildlist, xpkg, catdir, pkgdir)
		return self.qatracker, can_force


	def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
		# detect unused local USE-descriptions
		used_useflags = set()

		for y_ebuild in ebuildlist:

			ebuild = Ebuild(
				self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings,
				xpkg, y_ebuild)

			if self.check['changelog'] and not self.changelog_modified \
				and ebuild.ebuild_path in self.changed.new_ebuilds:
				self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)

			if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded):
				# ebuild not added to vcs
				self.qatracker.add_error(
					"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")

			if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker):
				continue

			pkg = self.pkgs[y_ebuild]
			if pkg_invalid(pkg, self.qatracker, ebuild):
				self.allvalid = False
				continue

			myaux = pkg._metadata
			eapi = myaux["EAPI"]
			inherited = pkg.inherited
			live_ebuild = self.live_eclasses.intersection(inherited)

			self.eapicheck.check(pkg, ebuild)

			for k, v in myaux.items():
				if not isinstance(v, basestring):
					continue
				m = NON_ASCII_RE.search(v)
				if m is not None:
					self.qatracker.add_error(
						"variable.invalidchar",
						"%s: %s variable contains non-ASCII "
						"character at position %s" %
						(ebuild.relative_path, k, m.start() + 1))

			if not self.fetchcheck.src_uri_error:
				self.thirdparty.check(myaux, ebuild.relative_path)

			if myaux.get("PROVIDE"):
				self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)

			for pos, missing_var in enumerate(missingvars):
				if not myaux.get(missing_var):
					if catdir == "virtual" and \
						missing_var in ("HOMEPAGE", "LICENSE"):
						continue
					if live_ebuild and missing_var == "KEYWORDS":
						continue
					myqakey = missingvars[pos] + ".missing"
					self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")

			if catdir == "virtual":
				for var in ("HOMEPAGE", "LICENSE"):
					if myaux.get(var):
						myqakey = var + ".virtual"
						self.qatracker.add_error(myqakey, ebuild.relative_path)

			self.descriptioncheck.check(pkg, ebuild)

			keywords = myaux["KEYWORDS"].split()

			ebuild_archs = set(
				kw.lstrip("~") for kw in keywords if not kw.startswith("-"))

			self.keywordcheck.check(
				pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed,
				live_ebuild, self.repo_metadata['kwlist'], self.profiles)

			if live_ebuild and self.repo_settings.repo_config.name == "gentoo":
				self.liveeclasscheck.check(
					pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict'])

			if self.options.ignore_arches:
				arches = [[
					self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"],
					self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
			else:
				arches = set()
				for keyword in keywords:
					if keyword[0] == "-":
						continue
					elif keyword[0] == "~":
						arch = keyword[1:]
						if arch == "*":
							for expanded_arch in self.profiles:
								if expanded_arch == "**":
									continue
								arches.add(
									(keyword, expanded_arch, (
										expanded_arch, "~" + expanded_arch)))
						else:
							arches.add((keyword, arch, (arch, keyword)))
					else:
						# For ebuilds with stable keywords, check if the
						# dependencies are satisfiable for unstable
						# configurations, since use.stable.mask is not
						# applied for unstable configurations (see bug
						# 563546).
						if keyword == "*":
							for expanded_arch in self.profiles:
								if expanded_arch == "**":
									continue
								arches.add(
									(keyword, expanded_arch, (expanded_arch,)))
								arches.add(
									(keyword, expanded_arch,
										(expanded_arch, "~" + expanded_arch)))
						else:
							arches.add((keyword, keyword, (keyword,)))
							arches.add((keyword, keyword,
								(keyword, "~" + keyword)))
				if not arches:
					# Use an empty profile for checking dependencies of
					# packages that have empty KEYWORDS.
					arches.add(('**', '**', ('**',)))

			unknown_pkgs = set()
			baddepsyntax = False
			badlicsyntax = False
			badprovsyntax = False
			# catpkg = catdir + "/" + y_ebuild

			inherited_java_eclass = "java-pkg-2" in inherited or \
				"java-pkg-opt-2" in inherited
			inherited_wxwidgets_eclass = "wxwidgets" in inherited
			# operator_tokens = set(["||", "(", ")"])
			type_list, badsyntax = [], []
			for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
				mydepstr = myaux[mytype]

				buildtime = mytype in Package._buildtime_keys
				runtime = mytype in Package._runtime_keys
				token_class = None
				if mytype.endswith("DEPEND"):
					token_class = portage.dep.Atom

				try:
					atoms = portage.dep.use_reduce(
						mydepstr, matchall=1, flat=True,
						is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
				except portage.exception.InvalidDependString as e:
					atoms = None
					badsyntax.append(str(e))

				if atoms and mytype.endswith("DEPEND"):
					if runtime and \
						"test?" in mydepstr.split():
						self.qatracker.add_error(
							mytype + '.suspect',
							"%s: 'test?' USE conditional in %s" %
							(ebuild.relative_path, mytype))

					for atom in atoms:
						if atom == "||":
							continue

						is_blocker = atom.blocker

						# Skip dependency.unknown for blockers, so that we
						# don't encourage people to remove necessary blockers,
						# as discussed in bug 382407. We use atom.without_use
						# due to bug 525376.
						if not is_blocker and \
							not self.portdb.xmatch("match-all", atom.without_use) and \
							not atom.cp.startswith("virtual/"):
							unknown_pkgs.add((mytype, atom.unevaluated_atom))

						if catdir != "virtual":
							if not is_blocker and \
								atom.cp in suspect_virtual:
								self.qatracker.add_error(
									'virtual.suspect', ebuild.relative_path +
									": %s: consider using '%s' instead of '%s'" %
									(mytype, suspect_virtual[atom.cp], atom))
							if not is_blocker and \
								atom.cp.startswith("perl-core/"):
								self.qatracker.add_error('dependency.perlcore',
									ebuild.relative_path +
									": %s: please use '%s' instead of '%s'" %
									(mytype,
									atom.replace("perl-core/","virtual/perl-"),
									atom))

						if buildtime and \
							not is_blocker and \
							not inherited_java_eclass and \
							atom.cp == "virtual/jdk":
							self.qatracker.add_error(
								'java.eclassesnotused', ebuild.relative_path)
						elif buildtime and \
							not is_blocker and \
							not inherited_wxwidgets_eclass and \
							atom.cp == "x11-libs/wxGTK":
							self.qatracker.add_error(
								'wxwidgets.eclassnotused',
								"%s: %ss on x11-libs/wxGTK without inheriting"
								" wxwidgets.eclass" % (ebuild.relative_path, mytype))
						elif runtime:
							if not is_blocker and \
								atom.cp in suspect_rdepend:
								self.qatracker.add_error(
									mytype + '.suspect',
									ebuild.relative_path + ": '%s'" % atom)

						if atom.operator == "~" and \
							portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
							qacat = 'dependency.badtilde'
							self.qatracker.add_error(
								qacat, "%s: %s uses the ~ operator"
								" with a non-zero revision: '%s'" %
								(ebuild.relative_path, mytype, atom))

						check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker,
							ebuild.relative_path, myaux)

				type_list.extend([mytype] * (len(badsyntax) - len(type_list)))

			for m, b in zip(type_list, badsyntax):
				if m.endswith("DEPEND"):
					qacat = "dependency.syntax"
				else:
					qacat = m + ".syntax"
				self.qatracker.add_error(
					qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))

			badlicsyntax = len([z for z in type_list if z == "LICENSE"])
			badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
			baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
			badlicsyntax = badlicsyntax > 0
			badprovsyntax = badprovsyntax > 0

			self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist)

			ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags()
			used_useflags = used_useflags.union(ebuild_used_useflags)

			self.rubyeclasscheck.check(pkg, ebuild)

			# license checks
			if not badlicsyntax:
				self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild)

			self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)

			# Syntax Checks
			if not self.vcs_settings.vcs_preserves_mtime:
				if ebuild.ebuild_path not in self.changed.new_ebuilds and \
					ebuild.ebuild_path not in self.changed.ebuilds:
					pkg.mtime = None
			try:
				# All ebuilds should have utf_8 encoding.
				f = io.open(
					_unicode_encode(
						ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
					mode='r', encoding=_encodings['repo.content'])
				try:
					for check_name, e in run_checks(f, pkg):
						self.qatracker.add_error(
							check_name, ebuild.relative_path + ': %s' % e)
				finally:
					f.close()
			except UnicodeDecodeError:
				# A file.UTF8 failure will have already been recorded above.
				pass

			if self.options.force:
				# The dep_check() calls are the most expensive QA test. If --force
				# is enabled, there's no point in wasting time on these since the
				# user is intent on forcing the commit anyway.
				continue

			relevant_profiles = []
			for keyword, arch, groups in arches:
				if arch not in self.profiles:
					# A missing profile will create an error further down
					# during the KEYWORDS verification.
					continue

				if self.include_arches is not None:
					if arch not in self.include_arches:
						continue

				relevant_profiles.extend(
					(keyword, groups, prof) for prof in self.profiles[arch])

			relevant_profiles.sort(key=sort_key)

			for keyword, groups, prof in relevant_profiles:

				is_stable_profile = prof.status == "stable"
				is_dev_profile = prof.status == "dev" and \
					self.options.include_dev
				is_exp_profile = prof.status == "exp" and \
					self.options.include_exp_profiles == 'y'
				if not (is_stable_profile or is_dev_profile or is_exp_profile):
					continue

				dep_settings = self.caches['arch'].get(prof.sub_path)
				if dep_settings is None:
					dep_settings = portage.config(
						config_profile_path=prof.abs_path,
						config_incrementals=self.repoman_incrementals,
						config_root=self.config_root,
						local_config=False,
						_unmatched_removal=self.options.unmatched_removal,
						env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
					dep_settings.categories = self.repo_settings.repoman_settings.categories
					if self.options.without_mask:
						dep_settings._mask_manager_obj = \
							copy.deepcopy(dep_settings._mask_manager)
						dep_settings._mask_manager._pmaskdict.clear()
					self.caches['arch'][prof.sub_path] = dep_settings

				xmatch_cache_key = (prof.sub_path, tuple(groups))
				xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
				if xcache is None:
					self.portdb.melt()
					self.portdb.freeze()
					xcache = self.portdb.xcache
					xcache.update(self.caches['shared_xmatch'])
					self.caches['arch_xmatch'][xmatch_cache_key] = xcache

				self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
				self.portdb.settings = dep_settings
				self.portdb.xcache = xcache

				dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
				# just in case, prevent config.reset() from nuking these.
				dep_settings.backup_changes("ACCEPT_KEYWORDS")

				# This attribute is used in dbapi._match_use() to apply
				# use.stable.{mask,force} settings based on the stable
				# status of the parent package. This is required in order
				# for USE deps of unstable packages to be resolved correctly,
				# since otherwise use.stable.{mask,force} settings of
				# dependencies may conflict (see bug #456342).
				dep_settings._parent_stable = dep_settings._isStable(pkg)

				# Handle package.use*.{force,mask) calculation, for use
				# in dep_check.
				dep_settings.useforce = dep_settings._use_manager.getUseForce(
					pkg, stable=dep_settings._parent_stable)
				dep_settings.usemask = dep_settings._use_manager.getUseMask(
					pkg, stable=dep_settings._parent_stable)

				if not baddepsyntax:
					ismasked = not ebuild_archs or \
						pkg.cpv not in self.portdb.xmatch("match-visible",
						Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)))
					if ismasked:
						if not self.have['pmasked']:
							self.have['pmasked'] = bool(dep_settings._getMaskAtom(
								pkg.cpv, pkg._metadata))
						if self.options.ignore_masked:
							continue
						# we are testing deps for a masked package; give it some lee-way
						suffix = "masked"
						matchmode = "minimum-all-ignore-profile"
					else:
						suffix = ""
						matchmode = "minimum-visible"

					if not self.have['dev_keywords']:
						self.have['dev_keywords'] = \
							bool(self.dev_keywords.intersection(keywords))

					if prof.status == "dev":
						suffix = suffix + "indev"

					for mytype in Package._dep_keys:

						mykey = "dependency.bad" + suffix
						myvalue = myaux[mytype]
						if not myvalue:
							continue

						success, atoms = portage.dep_check(
							myvalue, self.portdb, dep_settings,
							use="all", mode=matchmode, trees=self.repo_settings.trees)

						if success:
							if atoms:

								# Don't bother with dependency.unknown for
								# cases in which *DEPEND.bad is triggered.
								for atom in atoms:
									# dep_check returns all blockers and they
									# aren't counted for *DEPEND.bad, so we
									# ignore them here.
									if not atom.blocker:
										unknown_pkgs.discard(
											(mytype, atom.unevaluated_atom))

								if not prof.sub_path:
									# old-style virtuals currently aren't
									# resolvable with empty profile, since
									# 'virtuals' mappings are unavailable
									# (it would be expensive to search
									# for PROVIDE in all ebuilds)
									atoms = [
										atom for atom in atoms if not (
											atom.cp.startswith('virtual/')
											and not self.portdb.cp_list(atom.cp))]

								# we have some unsolvable deps
								# remove ! deps, which always show up as unsatisfiable
								atoms = [
									str(atom.unevaluated_atom)
									for atom in atoms if not atom.blocker]

								# if we emptied out our list, continue:
								if not atoms:
									continue
								if self.options.output_style in ['column']:
									self.qatracker.add_error(mykey,
										"%s: %s: %s(%s) %s"
										% (ebuild.relative_path, mytype, keyword,
											prof, repr(atoms)))
								else:
									self.qatracker.add_error(mykey,
										"%s: %s: %s(%s)\n%s"
										% (ebuild.relative_path, mytype, keyword,
											prof, pformat(atoms, indent=6)))
						else:
							if self.options.output_style in ['column']:
								self.qatracker.add_error(mykey,
									"%s: %s: %s(%s) %s"
									% (ebuild.relative_path, mytype, keyword,
										prof, repr(atoms)))
							else:
								self.qatracker.add_error(mykey,
									"%s: %s: %s(%s)\n%s"
									% (ebuild.relative_path, mytype, keyword,
										prof, pformat(atoms, indent=6)))

			if not baddepsyntax and unknown_pkgs:
				type_map = {}
				for mytype, atom in unknown_pkgs:
					type_map.setdefault(mytype, set()).add(atom)
				for mytype, atoms in type_map.items():
					self.qatracker.add_error(
						"dependency.unknown", "%s: %s: %s"
						% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))

		# check if there are unused local USE-descriptions in metadata.xml
		# (unless there are any invalids, to avoid noise)
		if self.allvalid:
			for myflag in self.muselist.difference(used_useflags):
				self.qatracker.add_error(
					"metadata.warning",
					"%s/metadata.xml: unused local USE-description: '%s'"
					% (xpkg, myflag))