def digest_regen(
self, updates, removed, manifests, scanner, broken_changelog_manifests
):
"""Regenerate manifests
@param updates: updated files
@param removed: removed files
@param manifests: Manifest files
@param scanner: The repoman.scanner.Scanner instance
@param broken_changelog_manifests: broken changelog manifests
"""
if updates or removed:
for x in sorted(
vcs_files_to_cps(
chain(updates, removed, manifests),
self.repo_settings.repodir,
scanner.repolevel,
scanner.reposplit,
scanner.categories,
)
):
self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x)
digestgen(
mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb
)
def _vcs_files_to_cps(self, vcs_file_iter):
"""
Iterate over the given modified file paths returned from the vcs,
and return a frozenset containing category/pn strings for each
modified package.
@param vcs_file_iter: file paths from vcs module
@type iter
@rtype: frozenset
@return: category/pn strings for each package.
"""
return vcs_files_to_cps(vcs_file_iter, self.repo_settings.repodir,
self.scanner.repolevel, self.scanner.reposplit,
self.scanner.categories)
def sign_manifest(self, myupdates, myremoved, mymanifests):
try:
for x in sorted(vcs_files_to_cps(
chain(myupdates, myremoved, mymanifests),
self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)):
self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x)
manifest_path = os.path.join(self.repoman_settings["O"], "Manifest")
if not need_signature(manifest_path):
continue
gpgsign(manifest_path, self.repoman_settings, self.options)
except portage.exception.PortageException as e:
portage.writemsg("!!! %s\n" % str(e))
portage.writemsg("!!! Disabled FEATURES='sign'\n")
self.repo_settings.sign_manifests = False
def sign_manifest(self, myupdates, myremoved, mymanifests):
try:
for x in sorted(vcs_files_to_cps(
chain(myupdates, myremoved, mymanifests),
self.scanner.repo_settings.repodir,
self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)):
self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x)
manifest_path = os.path.join(self.repoman_settings["O"], "Manifest")
if not need_signature(manifest_path):
continue
gpgsign(manifest_path, self.repoman_settings, self.options)
except portage.exception.PortageException as e:
portage.writemsg("!!! %s\n" % str(e))
portage.writemsg("!!! Disabled FEATURES='sign'\n")
self.repo_settings.sign_manifests = False
def digest_regen(self, updates, removed, manifests, scanner, broken_changelog_manifests): '''Regenerate manifests @param updates: updated files @param removed: removed files @param manifests: Manifest files @param scanner: The repoman.scanner.Scanner instance @param broken_changelog_manifests: broken changelog manifests ''' if updates or removed: for x in sorted(vcs_files_to_cps( chain(updates, removed, manifests), scanner.repolevel, scanner.reposplit, scanner.categories)): self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb)
def __init__(self, repo_settings, myreporoot, config_root, options,
vcs_settings, mydir, env):
'''Class __init__'''
self.repo_settings = repo_settings
self.config_root = config_root
self.options = options
self.vcs_settings = vcs_settings
self.env = env
# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
# behave incrementally.
self.repoman_incrementals = tuple(x for x in portage.const.INCREMENTALS
if x != 'ACCEPT_KEYWORDS')
self.categories = []
for path in self.repo_settings.repo_config.eclass_db.porttrees:
self.categories.extend(
portage.util.grabfile(
os.path.join(path, 'profiles', 'categories')))
self.repo_settings.repoman_settings.categories = frozenset(
portage.util.stack_lists([self.categories], incremental=1))
self.categories = self.repo_settings.repoman_settings.categories
self.portdb = repo_settings.portdb
self.portdb.settings = self.repo_settings.repoman_settings
digest_only = self.options.mode != 'manifest-check' \
and self.options.digest == 'y'
self.generate_manifest = digest_only or self.options.mode in \
("manifest", 'commit', 'fix')
# We really only need to cache the metadata that's necessary for visibility
# filtering. Anything else can be discarded to reduce memory consumption.
if not self.generate_manifest:
# Don't do this when generating manifests, since that uses
# additional keys if spawn_nofetch is called (RESTRICT and
# DEFINED_PHASES).
self.portdb._aux_cache_keys.clear()
self.portdb._aux_cache_keys.update(
["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
self.reposplit = myreporoot.split(os.path.sep)
self.repolevel = len(self.reposplit)
if self.options.mode == 'commit':
repochecks.commit_check(self.repolevel, self.reposplit)
repochecks.conflict_check(self.vcs_settings, self.options)
# Make startdir relative to the canonical repodir, so that we can pass
# it to digestgen and it won't have to be canonicalized again.
if self.repolevel == 1:
startdir = self.repo_settings.repodir
else:
startdir = normalize_path(mydir)
startdir = os.path.join(
self.repo_settings.repodir,
*startdir.split(os.sep)[-2 - self.repolevel + 3:])
# get lists of valid keywords, licenses, and use
new_data = repo_metadata(self.portdb,
self.repo_settings.repoman_settings)
kwlist, liclist, uselist, profile_list, \
global_pmaskdict, liclist_deprecated = new_data
self.repo_metadata = {
'kwlist':
kwlist,
'liclist':
liclist,
'uselist':
uselist,
'profile_list':
profile_list,
'pmaskdict':
global_pmaskdict,
'lic_deprecated':
liclist_deprecated,
'package.deprecated':
InternalPackageSet(
initial_atoms=portage.util.stack_lists([
portage.util.grabfile_package(os.path.join(
path, 'profiles', 'package.deprecated'),
recursive=True)
for path in self.portdb.porttrees
],
incremental=True))
}
self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(
sorted(kwlist))
self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')
profiles = setup_profile(profile_list)
check_profiles(profiles,
self.repo_settings.repoman_settings.archlist())
scanlist = scan(self.repolevel, self.reposplit, startdir,
self.categories, self.repo_settings)
self.dev_keywords = dev_profile_keywords(profiles)
self.qatracker = self.vcs_settings.qatracker
if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
self.options.echangelog = 'y'
if self.vcs_settings.vcs is None:
self.options.echangelog = 'n'
# Initialize the ModuleConfig class here
# TODO Add layout.conf masters repository.yml config to the list to load/stack
self.moduleconfig = ModuleConfig(
self.repo_settings.masters_list,
self.repo_settings.repoman_settings.valid_versions,
repository_modules=self.options.experimental_repository_modules ==
'y')
checks = {}
# The --echangelog option causes automatic ChangeLog generation,
# which invalidates changelog.ebuildadded and changelog.missing
# checks.
# Note: Some don't use ChangeLogs in distributed SCMs.
# It will be generated on server side from scm log,
# before package moves to the rsync server.
# This is needed because they try to avoid merge collisions.
# Gentoo's Council decided to always use the ChangeLog file.
# TODO: shouldn't this just be switched on the repo, iso the VCS?
is_echangelog_enabled = self.options.echangelog in ('y', 'force')
self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs',
'svn')
checks[
'changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn
if self.options.mode == "manifest" or self.options.quiet:
pass
elif self.options.pretend:
print(green("\nRepoMan does a once-over of the neighborhood..."))
else:
print(green("\nRepoMan scours the neighborhood..."))
self.changed = self.vcs_settings.changes
# bypass unneeded VCS operations if not needed
if (self.options.if_modified == "y"
or self.options.mode not in ("manifest", "manifest-check")):
self.changed.scan()
self.have = {
'pmasked': False,
'dev_keywords': False,
}
# NOTE: match-all caches are not shared due to potential
# differences between profiles in _get_implicit_iuse.
self.caches = {
'arch': {},
'arch_xmatch': {},
'shared_xmatch': {
"cp-list": {}
},
}
self.include_arches = None
if self.options.include_arches:
self.include_arches = set()
self.include_arches.update(
*[x.split() for x in self.options.include_arches])
self.include_profiles = None
if self.options.include_profiles:
self.include_profiles = set()
self.include_profiles.update(
*[x.split() for x in self.options.include_profiles])
# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
# running `svn status` in every package dir will be too expensive.
checks['ebuild_notadded'] = not \
(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")
self.effective_scanlist = scanlist
if self.options.if_modified == "y":
self.effective_scanlist = sorted(
vcs_files_to_cps(
chain(self.changed.changed, self.changed.new,
self.changed.removed), self.repo_settings.repodir,
self.repolevel, self.reposplit, self.categories))
# Create our kwargs dict here to initialize the plugins with
self.kwargs = {
"repo_settings": self.repo_settings,
"portdb": self.portdb,
"qatracker": self.qatracker,
"vcs_settings": self.vcs_settings,
"options": self.options,
"metadata_xsd": get_metadata_xsd(self.repo_settings),
"uselist": uselist,
"checks": checks,
"repo_metadata": self.repo_metadata,
"profiles": profiles,
"include_arches": self.include_arches,
"include_profiles": self.include_profiles,
"caches": self.caches,
"repoman_incrementals": self.repoman_incrementals,
"env": self.env,
"have": self.have,
"dev_keywords": self.dev_keywords,
"linechecks": self.moduleconfig.linechecks,
}
# initialize the plugin checks here
self.modules = {}
self._ext_futures = {}
self.pkg_level_futures = None
def __init__(self, repo_settings, myreporoot, config_root, options,
vcs_settings, mydir, env):
'''Class __init__'''
self.repo_settings = repo_settings
self.config_root = config_root
self.options = options
self.vcs_settings = vcs_settings
self.env = env
# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
# behave incrementally.
self.repoman_incrementals = tuple(
x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
self.categories = []
for path in self.repo_settings.repo_config.eclass_db.porttrees:
self.categories.extend(portage.util.grabfile(
os.path.join(path, 'profiles', 'categories')))
self.repo_settings.repoman_settings.categories = frozenset(
portage.util.stack_lists([self.categories], incremental=1))
self.categories = self.repo_settings.repoman_settings.categories
self.portdb = repo_settings.portdb
self.portdb.settings = self.repo_settings.repoman_settings
digest_only = self.options.mode != 'manifest-check' \
and self.options.digest == 'y'
self.generate_manifest = digest_only or self.options.mode in \
("manifest", 'commit', 'fix')
# We really only need to cache the metadata that's necessary for visibility
# filtering. Anything else can be discarded to reduce memory consumption.
if not self.generate_manifest:
# Don't do this when generating manifests, since that uses
# additional keys if spawn_nofetch is called (RESTRICT and
# DEFINED_PHASES).
self.portdb._aux_cache_keys.clear()
self.portdb._aux_cache_keys.update(
["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
self.reposplit = myreporoot.split(os.path.sep)
self.repolevel = len(self.reposplit)
if self.options.mode == 'commit':
repochecks.commit_check(self.repolevel, self.reposplit)
repochecks.conflict_check(self.vcs_settings, self.options)
# Make startdir relative to the canonical repodir, so that we can pass
# it to digestgen and it won't have to be canonicalized again.
if self.repolevel == 1:
startdir = self.repo_settings.repodir
else:
startdir = normalize_path(mydir)
startdir = os.path.join(
self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:])
# get lists of valid keywords, licenses, and use
new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings)
kwlist, liclist, uselist, profile_list, \
global_pmaskdict, liclist_deprecated = new_data
self.repo_metadata = {
'kwlist': kwlist,
'liclist': liclist,
'uselist': uselist,
'profile_list': profile_list,
'pmaskdict': global_pmaskdict,
'lic_deprecated': liclist_deprecated,
}
self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST')
profiles = setup_profile(profile_list)
check_profiles(profiles, self.repo_settings.repoman_settings.archlist())
scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings)
self.dev_keywords = dev_profile_keywords(profiles)
self.qatracker = self.vcs_settings.qatracker
if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog:
self.options.echangelog = 'y'
if self.vcs_settings.vcs is None:
self.options.echangelog = 'n'
# Initialize the ModuleConfig class here
# TODO Add layout.conf masters repository.yml config to the list to load/stack
self.moduleconfig = ModuleConfig(self.repo_settings.masters_list,
self.repo_settings.repoman_settings.valid_versions,
repository_modules=self.options.experimental_repository_modules == 'y')
checks = {}
# The --echangelog option causes automatic ChangeLog generation,
# which invalidates changelog.ebuildadded and changelog.missing
# checks.
# Note: Some don't use ChangeLogs in distributed SCMs.
# It will be generated on server side from scm log,
# before package moves to the rsync server.
# This is needed because they try to avoid merge collisions.
# Gentoo's Council decided to always use the ChangeLog file.
# TODO: shouldn't this just be switched on the repo, iso the VCS?
is_echangelog_enabled = self.options.echangelog in ('y', 'force')
self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn')
checks['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn
if self.options.mode == "manifest" or self.options.quiet:
pass
elif self.options.pretend:
print(green("\nRepoMan does a once-over of the neighborhood..."))
else:
print(green("\nRepoMan scours the neighborhood..."))
self.changed = self.vcs_settings.changes
# bypass unneeded VCS operations if not needed
if (self.options.if_modified == "y" or
self.options.mode not in ("manifest", "manifest-check")):
self.changed.scan()
self.have = {
'pmasked': False,
'dev_keywords': False,
}
# NOTE: match-all caches are not shared due to potential
# differences between profiles in _get_implicit_iuse.
self.caches = {
'arch': {},
'arch_xmatch': {},
'shared_xmatch': {"cp-list": {}},
}
self.include_arches = None
if self.options.include_arches:
self.include_arches = set()
self.include_arches.update(*[x.split() for x in self.options.include_arches])
# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
# running `svn status` in every package dir will be too expensive.
checks['ebuild_notadded'] = not \
(self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit")
self.effective_scanlist = scanlist
if self.options.if_modified == "y":
self.effective_scanlist = sorted(vcs_files_to_cps(
chain(self.changed.changed, self.changed.new, self.changed.removed),
self.repo_settings.repodir,
self.repolevel, self.reposplit, self.categories))
# Create our kwargs dict here to initialize the plugins with
self.kwargs = {
"repo_settings": self.repo_settings,
"portdb": self.portdb,
"qatracker": self.qatracker,
"vcs_settings": self.vcs_settings,
"options": self.options,
"metadata_xsd": get_metadata_xsd(self.repo_settings),
"uselist": uselist,
"checks": checks,
"repo_metadata": self.repo_metadata,
"profiles": profiles,
"include_arches": self.include_arches,
"caches": self.caches,
"repoman_incrementals": self.repoman_incrementals,
"env": self.env,
"have": self.have,
"dev_keywords": self.dev_keywords,
"linechecks": self.moduleconfig.linechecks,
}
# initialize the plugin checks here
self.modules = {}
self._ext_futures = {}
self.pkg_level_futures = None
def changelogs(self, myupdates, mymanifests, myremoved, mychanged, myautoadd,
mynew, changelog_msg):
broken_changelog_manifests = []
if self.options.echangelog in ('y', 'force'):
logging.info("checking for unmodified ChangeLog files")
committer_name = utilities.get_committer_name(env=self.repoman_settings)
for x in sorted(vcs_files_to_cps(
chain(myupdates, mymanifests, myremoved),
self.repo_settings.repodir,
self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)):
catdir, pkgdir = x.split("/")
checkdir = self.repo_settings.repodir + "/" + x
checkdir_relative = ""
if self.scanner.repolevel < 3:
checkdir_relative = os.path.join(pkgdir, checkdir_relative)
if self.scanner.repolevel < 2:
checkdir_relative = os.path.join(catdir, checkdir_relative)
checkdir_relative = os.path.join(".", checkdir_relative)
changelog_path = os.path.join(checkdir_relative, "ChangeLog")
changelog_modified = changelog_path in self.scanner.changed.changelogs
if changelog_modified and self.options.echangelog != 'force':
continue
# get changes for this package
cdrlen = len(checkdir_relative)
check_relative = lambda e: e.startswith(checkdir_relative)
split_relative = lambda e: e[cdrlen:]
clnew = list(map(split_relative, filter(check_relative, mynew)))
clremoved = list(map(split_relative, filter(check_relative, myremoved)))
clchanged = list(map(split_relative, filter(check_relative, mychanged)))
# Skip ChangeLog generation if only the Manifest was modified,
# as discussed in bug #398009.
nontrivial_cl_files = set()
nontrivial_cl_files.update(clnew, clremoved, clchanged)
nontrivial_cl_files.difference_update(['Manifest'])
if not nontrivial_cl_files and self.options.echangelog != 'force':
continue
new_changelog = utilities.UpdateChangeLog(
checkdir_relative, committer_name, changelog_msg,
os.path.join(self.repo_settings.repodir, 'skel.ChangeLog'),
catdir, pkgdir,
new=clnew, removed=clremoved, changed=clchanged,
pretend=self.options.pretend)
if new_changelog is None:
writemsg_level(
"!!! Updating the ChangeLog failed\n",
level=logging.ERROR, noiselevel=-1)
sys.exit(1)
# if the ChangeLog was just created, add it to vcs
if new_changelog:
myautoadd.append(changelog_path)
# myautoadd is appended to myupdates below
else:
myupdates.append(changelog_path)
if self.options.ask and not self.options.pretend:
# regenerate Manifest for modified ChangeLog (bug #420735)
self.repoman_settings["O"] = checkdir
digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb)
else:
broken_changelog_manifests.append(x)
if myautoadd:
print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
self.vcs_settings.changes.add_items(myautoadd)
myupdates += myautoadd
return myupdates, broken_changelog_manifests
def changelogs(self, myupdates, mymanifests, myremoved, mychanged,
myautoadd, mynew, changelog_msg):
broken_changelog_manifests = []
if self.options.echangelog in ('y', 'force'):
logging.info("checking for unmodified ChangeLog files")
committer_name = utilities.get_committer_name(
env=self.repoman_settings)
for x in sorted(
vcs_files_to_cps(chain(myupdates, mymanifests, myremoved),
self.repo_settings.repodir,
self.scanner.repolevel,
self.scanner.reposplit,
self.scanner.categories)):
catdir, pkgdir = x.split("/")
checkdir = self.repo_settings.repodir + "/" + x
checkdir_relative = ""
if self.scanner.repolevel < 3:
checkdir_relative = os.path.join(pkgdir, checkdir_relative)
if self.scanner.repolevel < 2:
checkdir_relative = os.path.join(catdir, checkdir_relative)
checkdir_relative = os.path.join(".", checkdir_relative)
changelog_path = os.path.join(checkdir_relative, "ChangeLog")
changelog_modified = changelog_path in self.scanner.changed.changelogs
if changelog_modified and self.options.echangelog != 'force':
continue
# get changes for this package
cdrlen = len(checkdir_relative)
check_relative = lambda e: e.startswith(checkdir_relative)
split_relative = lambda e: e[cdrlen:]
clnew = list(map(split_relative, filter(check_relative,
mynew)))
clremoved = list(
map(split_relative, filter(check_relative, myremoved)))
clchanged = list(
map(split_relative, filter(check_relative, mychanged)))
# Skip ChangeLog generation if only the Manifest was modified,
# as discussed in bug #398009.
nontrivial_cl_files = set()
nontrivial_cl_files.update(clnew, clremoved, clchanged)
nontrivial_cl_files.difference_update(['Manifest'])
if not nontrivial_cl_files and self.options.echangelog != 'force':
continue
new_changelog = utilities.UpdateChangeLog(
checkdir_relative,
committer_name,
changelog_msg,
os.path.join(self.repo_settings.repodir, 'skel.ChangeLog'),
catdir,
pkgdir,
new=clnew,
removed=clremoved,
changed=clchanged,
pretend=self.options.pretend)
if new_changelog is None:
writemsg_level("!!! Updating the ChangeLog failed\n",
level=logging.ERROR,
noiselevel=-1)
sys.exit(1)
# if the ChangeLog was just created, add it to vcs
if new_changelog:
myautoadd.append(changelog_path)
# myautoadd is appended to myupdates below
else:
myupdates.append(changelog_path)
if self.options.ask and not self.options.pretend:
# regenerate Manifest for modified ChangeLog (bug #420735)
self.repoman_settings["O"] = checkdir
digestgen(mysettings=self.repoman_settings,
myportdb=self.repo_settings.portdb)
else:
broken_changelog_manifests.append(x)
if myautoadd:
print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...")
self.vcs_settings.changes.add_items(myautoadd)
myupdates += myautoadd
return myupdates, broken_changelog_manifests
