def save(self):
# ensure essential directories exist
hints_dir = os.path.join(self._base_dir, 'input', 'hints')
os.makedirs(os.path.join(self._base_dir, 'output', 'target'),
exist_ok=True)
os.makedirs(os.path.join(self._base_dir, 'state'), exist_ok=True)
os.makedirs(hints_dir, exist_ok=True)
# ensure essential settings are set
# NOTE: All of this seriously needs some refactoring, the direct translation from
# D code is pretty bad at the moment.
assert self._paths_set
assert self._components_set
assert self._archs_set
if not self._broken_archs_set:
self.set_broken_architectures([])
if not self._new_archs_set:
self.set_new_architectures([])
if not self._delays_set:
self.set_delays([])
# save configuration
conf_fname = os.path.join(self._base_dir, 'britney.conf')
log.debug('Saving Britney config to "{}"'.format(conf_fname))
with open(conf_fname, 'wt') as f:
for line in self._contents:
f.write(line + '\n')
if self._hint_contents:
hints_fname = os.path.join(self._base_dir, 'input', 'hints',
'laniakea')
with open(hints_fname, 'wt') as f:
for line in self._hint_contents:
f.write(line + '\n')
def command_repo(options):
''' Import repository data '''
suite_names = []
if options.suite:
suite_names.append(options.suite)
else:
log.debug('Importing data from all mutable suites.')
with session_scope() as session:
suite_names = [
r[0] for r in session.query(ArchiveSuite.name).filter(
ArchiveSuite.frozen == False).all()
] # noqa
# TODO: Optimize this so we can run it in parallel, as well as skip
# imports if they are not needed
for suite_name in suite_names:
log.debug('Importing data for suite "{}".'.format(suite_name))
import_suite_packages(suite_name)
def package_is_removable(self, package_name: str, suite_name: str) -> bool:
''' Check if a package can be removed without breaking reverse dependencies. '''
log.debug('Testing package \'{}\' removal from \'{}\''.format(
package_name, suite_name))
# simulate package removal
args = [
'rm', '-R', '-m', 'RID: Removed from Debian', '-C', 'janitor@dak',
'-n', '-s', suite_name, package_name
]
ret, out = self._run_dak(args, check=False)
if ret != 0:
raise Exception(
'Unable to check if package \'{}\' is removable from \'{}\': {}'
.format(package_name, suite_name, out))
return 'No dependency problem found.' in out
def _collect_urgencies(self, mi_wspace: str):
urgencies = ''
for subdir, dirs, files in os.walk(
self._lconf.archive_urgencies_export_dir):
for fbasename in files:
fname = os.path.join(subdir, fbasename)
if not os.path.isfile(fname):
continue
if not fbasename.startswith('install-urgencies'):
continue
log.debug('Reading urgencies from {}'.format(fname))
with open(fname, 'r') as f:
urgencies = urgencies + f.read()
log.info('Writing urgency policy file.')
urgency_policy_file = os.path.join(mi_wspace, 'state',
'age-policy-urgencies')
with open(urgency_policy_file, 'w') as f:
f.write(urgencies)
def autosync(self, session, sync_conf, remove_cruft: bool = True):
''' Synchronize all packages that are newer '''
self._synced_source_pkgs = []
active_src_pkgs = [] # source packages which should have their binary packages updated
res_issues = []
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
sync_conf = session.query(SynchrotronConfig) \
.join(SynchrotronConfig.destination_suite) \
.join(SynchrotronConfig.source) \
.filter(ArchiveSuite.name == self._target_suite_name,
SynchrotronSource.suite_name == self._source_suite_name).one_or_none()
for component in target_suite.components:
dest_pkg_map = self._get_target_source_packages(component.name)
# The source package lists contains many different versions, some source package
# versions are explicitly kept for GPL-compatibility.
# Sometimes a binary package migrates into another suite, dragging a newer source-package
# that it was built against with itslf into the target suite.
# These packages then have a source with a high version number, but might not have any
# binaries due to them migrating later.
# We need to care for that case when doing binary syncs (TODO: and maybe safeguard against it
# when doing source-only syncs too?), That's why we don't filter out the newest packages in
# binary-sync-mode.
if sync_conf.sync_binaries:
src_pkg_range = self._source_repo.source_packages(ArchiveSuite(self._source_suite_name), component)
else:
src_pkg_range = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component).values()
for spkg in src_pkg_range:
# ignore blacklisted packages in automatic sync
if spkg.name in self._sync_blacklist:
continue
dpkg = dest_pkg_map.get(spkg.name)
if dpkg:
if version_compare(dpkg.version, spkg.version) >= 0:
log.debug('Skipped sync of {}: Target version \'{}\' is equal/newer than source version \'{}\'.'
.format(spkg.name, dpkg.version, spkg.version))
continue
# check if we have a modified target package,
# indicated via its Debian revision, e.g. "1.0-0tanglu1"
if self._distro_tag in version_revision(dpkg.version):
log.info('Not syncing {}/{}: Destination has modifications (found {}).'
.format(spkg.name, spkg.version, dpkg.version))
# add information that this package needs to be merged to the issue list
issue = SynchrotronIssue()
issue.package_name = spkg.name
issue.source_version = spkg.version
issue.target_version = dpkg.version
issue.kind = SynchrotronIssueKind.MERGE_REQUIRED
res_issues.append(issue)
continue
# sync source package
# the source package must always be known to dak first
ret = self._import_source_package(spkg, component.name)
if not ret:
return False, []
# a new source package is always active and needs it's binary packages synced, in
# case we do binary syncs.
active_src_pkgs.append(spkg)
# all packages in the target distribution are considered active, as long as they don't
# have modifications.
for spkg in dest_pkg_map.values():
if self._distro_tag in version_revision(spkg.version):
active_src_pkgs.append(spkg)
# import binaries as well. We test for binary updates for all available active source packages,
# as binNMUs might have happened in the source distribution.
# (an active package in this context is any source package which doesn't have modifications in the
# target distribution)
ret = self._import_binaries_for_source(sync_conf, target_suite, component.name, active_src_pkgs)
if not ret:
return False, []
# test for cruft packages
target_pkg_index = {}
for component in target_suite.components:
dest_pkg_map = self._get_repo_source_package_map(self._target_repo,
target_suite.name,
component.name)
for pkgname, pkg in dest_pkg_map.items():
target_pkg_index[pkgname] = pkg
# check which packages are present in the target, but not in the source suite
for component in target_suite.components:
src_pkg_map = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component.name)
for pkgname in src_pkg_map.keys():
target_pkg_index.pop(pkgname, None)
# remove cruft packages
if remove_cruft:
for pkgname, dpkg in target_pkg_index.items():
dpkg_ver_revision = version_revision(dpkg.version, False)
# native packages are never removed
if not dpkg_ver_revision:
continue
# check if the package is intoduced as new in the distro, in which case we won't remove it
if dpkg_ver_revision.startswith('0' + self._distro_tag):
continue
# if this package was modified in the target distro, we will also not remove it, but flag it
# as "potential cruft" for someone to look at.
if self._distro_tag in dpkg_ver_revision:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.MAYBE_CRUFT
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
res_issues.append(issue)
continue
# check if we can remove this package without breaking stuff
if self._dak.package_is_removable(dpkg.name, target_suite.name):
# try to remove the package
try:
self._dak.remove_package(dpkg.name, target_suite.name)
except Exception as e:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = str(e)
res_issues.append(issue)
else:
# looks like we can not remove this
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = 'This package can not be removed without breaking other packages. It needs manual removal.'
res_issues.append(issue)
self._publish_synced_spkg_events(sync_conf.source.os_name,
sync_conf.source.suite_name,
sync_conf.destination_suite.name,
False)
return True, res_issues
def _import_binaries_for_source(self, sync_conf, target_suite, component: str, spkgs: List[SourcePackage],
ignore_target_changes: bool = False) -> bool:
''' Import binary packages for the given set of source packages into the archive. '''
if not sync_conf.sync_binaries:
log.debug('Skipping binary syncs.')
return True
# list of valid architectrures supported by the target
target_archs = [a.name for a in target_suite.architectures]
# cache of binary-package mappings for the source
src_bpkg_arch_map = {}
for aname in target_archs:
src_bpkg_arch_map[aname] = self._get_repo_binary_package_map(self._source_repo, self._source_suite_name, component, aname)
# cache of binary-package mappings from the target repository
dest_bpkg_arch_map = {}
for aname in target_archs:
dest_bpkg_arch_map[aname] = self._get_repo_binary_package_map(self._target_repo, self._target_suite_name, component, aname)
for spkg in spkgs:
bin_files_synced = False
existing_packages = False
for arch_name in target_archs:
if arch_name not in src_bpkg_arch_map:
continue
src_bpkg_map = src_bpkg_arch_map[arch_name]
dest_bpkg_map = dest_bpkg_arch_map[arch_name]
bin_files = []
for bin_i in spkg.binaries:
if bin_i.name not in src_bpkg_map:
if bin_i.name in dest_bpkg_map:
existing_packages = True # package only exists in target
continue
if arch_name != 'all' and bin_i.architectures == ['all']:
# we handle arch:all explicitly
continue
bpkg = src_bpkg_map[bin_i.name]
if bin_i.version != bpkg.source_version:
log.debug('Not syncing binary package \'{}\': Version number \'{}\' does not match source package version \'{}\'.'
.format(bpkg.name, bin_i.version, bpkg.source_version))
continue
ebpkg = dest_bpkg_map.get(bpkg.name)
if ebpkg:
if version_compare(ebpkg.version, bpkg.version) >= 0:
log.debug('Not syncing binary package \'{}/{}\': Existing binary package with bigger/equal version \'{}\' found.'
.format(bpkg.name, bpkg.version, ebpkg.version))
existing_packages = True
continue
# Filter out manual rebuild uploads matching the pattern XbY.
# sometimes rebuild uploads of not-modified packages happen, and if the source
# distro did a binNMU, we don't want to sync that, even if it's bigger
# This rebuild-upload check must only happen if we haven't just updated the source package
# (in that case the source package version will be bigger than the existing binary package version)
if version_compare(spkg.version, ebpkg.version) >= 0:
if re.match(r'(.*)b([0-9]+)', ebpkg.version):
log.debug('Not syncing binary package \'{}/{}\': Existing binary package with rebuild upload \'{}\' found.'
.format(bpkg.name, bpkg.version, ebpkg.version))
existing_packages = True
continue
if not ignore_target_changes and self._distro_tag in version_revision(ebpkg.version):
# safety measure, we should never get here as packages with modifications were
# filtered out previously.
log.debug('Can not sync binary package {}/{}: Target has modifications.'.format(bin_i.name, bin_i.version))
continue
fname = self._source_repo.get_file(bpkg.bin_file)
bin_files.append(fname)
# now import the binary packages, if there is anything to import
if bin_files:
bin_files_synced = True
ret = self._import_package_files(self._target_suite_name, component, bin_files)
if not ret:
return False
if not bin_files_synced and not existing_packages:
log.warning('No binary packages synced for source {}/{}'.format(spkg.name, spkg.version))
return True
def update_appstream_data(session, local_repo, repo, suite, component, arch):
'''
Import AppStream metadata about software components and associate it with the
binary packages the data belongs to.
'''
if arch.name == 'all':
# arch:all has no AppStream components, those are always associated with an architecture
# and are included in arch-specific files (even if the package they belong to is arch:all)
return
arch_all = session.query(ArchiveArchitecture) \
.filter(ArchiveArchitecture.name == 'all').one()
yaml_fname = local_repo.index_file(
suite,
os.path.join(component.name, 'dep11',
'Components-{}.yml.xz'.format(arch.name)))
if not yaml_fname:
return
cidmap_fname = local_repo.index_file(
suite,
os.path.join(component.name, 'dep11',
'CID-Index-{}.json.gz'.format(arch.name)),
check=False)
if not cidmap_fname:
return
with gzip.open(cidmap_fname, 'rb') as f:
cid_map = json.loads(f.read())
with lzma.open(yaml_fname, 'r') as f:
yaml_data = str(f.read(), 'utf-8')
mdata = AppStream.Metadata()
mdata.set_locale('ALL')
mdata.set_format_style(AppStream.FormatStyle.COLLECTION)
mdata.set_parse_flags(AppStream.ParseFlags.IGNORE_MEDIABASEURL)
mdata.parse(yaml_data, AppStream.FormatKind.YAML)
cpts = mdata.get_components()
if len(cpts) == 0:
return
log.debug('Found {} software components in {}/{}'.format(
len(cpts), suite.name, component.name))
# create context for direct serialization to collection XML chunk
context = AppStream.Context()
context.set_locale('ALL')
context.set_style(AppStream.FormatStyle.COLLECTION)
for cpt in cpts:
cpt.set_active_locale('C')
pkgname = cpt.get_pkgname()
if not pkgname:
# we skip these for now, web-apps have no package assigned - we might need a better way to map
# those to their packages, likely with an improved appstream-generator integration
log.debug(
'Found DEP-11 component without package name in {}/{}: {}'.
format(suite.name, component.name, cpt.get_id()))
continue
# fetch package this component belongs to
bin_pkg = session.query(BinaryPackage) \
.filter(BinaryPackage.name == pkgname) \
.filter(BinaryPackage.repo_id == repo.id) \
.filter(BinaryPackage.architecture_id.in_((arch.id, arch_all.id))) \
.filter(BinaryPackage.component_id == component.id) \
.filter(BinaryPackage.suites.any(ArchiveSuite.id == suite.id)) \
.order_by(BinaryPackage.version.desc()).first()
if not bin_pkg:
log.info('Found orphaned DEP-11 component in {}/{}: {}'.format(
suite.name, component.name, cpt.get_id()))
continue
dcpt = SoftwareComponent()
dcpt.kind = int(cpt.get_kind())
dcpt.cid = cpt.get_id()
dcpt.xml = cpt.to_xml_data(context)
dcpt.gcid = cid_map.get(dcpt.cid)
if not dcpt.gcid:
log.info('Found DEP-11 component without GCID in {}/{}: {}'.format(
suite.name, component.name, cpt.get_id()))
# create UUID for this component (based on GCID or XML data)
dcpt.update_uuid()
existing_dcpt = session.query(SoftwareComponent) \
.filter(SoftwareComponent.uuid == dcpt.uuid).one_or_none()
if existing_dcpt:
if bin_pkg in existing_dcpt.bin_packages:
continue # the binary package is already registered with this component
existing_dcpt.bin_packages.append(bin_pkg)
continue # we already have this component, no need to add it again
# add new software component to database
dcpt.name = cpt.get_name()
dcpt.summary = cpt.get_summary()
dcpt.description = cpt.get_description()
for icon in cpt.get_icons():
if icon.get_kind() == AppStream.IconKind.CACHED:
dcpt.icon_name = icon.get_name()
break
dcpt.project_license = cpt.get_project_license()
dcpt.developer_name = cpt.get_developer_name()
# test for free software
dcpt.is_free = False
if not dcpt.project_license:
# We have no license set.
# If we are in the 'main' component, we
# assume we have free software
if bin_pkg.component.name == 'main':
dcpt.is_free = True
else:
# have AppStream test the SPDX license expression for free software
dcpt.is_free = AppStream.license_is_free_license(
dcpt.project_license)
dcpt.categories = []
for cat in cpt.get_categories():
dcpt.categories.append(cat)
dcpt.bin_packages = [bin_pkg]
session.add(dcpt)
log.debug('Added new software component \'{}\' to database'.format(
dcpt.cid))
session.commit()
def set_trusted(self, trusted):
self._trusted = trusted
if self._trusted:
log.debug('Explicitly marked repository "{}" as trusted.'.format(self.location))
def _create_faux_packages(self, session, mi_wspace: str,
suites_source: List[ArchiveSuite],
suite_target: ArchiveSuite):
'''
If we have a partial source and target suite, we need to let Britney know about the
parent packages somehow.
At the moment, we simply abuse the FauxPackages system for that.
'''
# we don't support more than one source suite for this feature at the moment
if len(suites_source) > 1:
log.info(
'Not auto-generating faux packages: Multiple suites set as sources.'
)
return
suite_source = suites_source[0]
if suite_source.parent and suite_target.parent:
log.info(
'Creating faux-packages to aid resolving of partial suites.')
else:
log.info(
'No auto-generating faux packages: No source and target suite parents, generation is unnecessary.'
)
return
existing_pkg_arch_set = set()
log.debug(
'Creating index of valid packages that do not need a faux package.'
)
# we need repository information to only generate faux packages if a package doesn't exist
# in our source suite(s) already
repo = self._get_local_repo(session)
for suite in suites_source:
esuite = session.query(ArchiveSuite) \
.options(joinedload(ArchiveSuite.components)) \
.options(joinedload(ArchiveSuite.architectures)) \
.filter(ArchiveSuite.id == suite.id).one()
session.expunge(
esuite
) # we don't want packages accidentally added to the database here
for component in esuite.components:
for arch in esuite.architectures:
aname = arch.name
for bpkg in repo.binary_packages(esuite, component, arch):
existing_pkg_arch_set.add(aname + ':' + bpkg.name)
for spkg in repo.source_packages(esuite, component):
existing_pkg_arch_set.add(aname + ':' + spkg.name)
archive_root_dir = self._lconf.archive_root_dir
fauxpkg_fname = os.path.join(mi_wspace, 'input', 'faux-packages')
log.debug('Generating faux packages list')
fauxpkg_data = {}
for component in suite_target.parent.components:
for installer_dir in ['', 'debian-installer']:
for arch in suite_target.parent.architectures:
pfile = os.path.join(archive_root_dir, 'dists',
suite_target.parent.name,
component.name, installer_dir,
'binary-{}'.format(arch.name),
'Packages.xz')
if not os.path.isfile(pfile):
continue
log.debug('Reading data for faux packages list: {}'.format(
pfile))
with TagFile(pfile) as tf:
for e in tf:
pkgname = e['Package']
pkgversion = e['Version']
pkgarch = e['Architecture']
pkid = '{}-{}-{}'.format(pkgname, pkgversion,
pkgarch)
if pkid in fauxpkg_data:
continue
pkgname_arch = pkgarch + ':' + pkgname
if pkgname_arch in existing_pkg_arch_set:
continue
provides = e.get('Provides', '')
data = 'Package: {}\nVersion: {}'.format(
pkgname, pkgversion)
if pkgarch and pkgarch != 'all':
data = data + '\nArchitecture: {}'.format(
pkgarch)
if provides:
data = data + '\nProvides: {}'.format(provides)
if component.name != 'main':
data = data + '\nComponent: {}'.format(
component.name)
fauxpkg_data[pkid] = data
# FIXME: We shouldn't have to special-case this :any case,
# rather Britney should do the right thing and recognize this
# notation for faux-packages. But until that is fixed
# properly and since a dependency on python3:any is so common, we
# will work around this issue
if pkgname == 'python3':
pkid = '{}-{}-{}'.format(
'python3:any', pkgversion, pkgarch)
if pkid in fauxpkg_data:
continue
fauxpkg_data[pkid] = data.replace(
'Package: python3\n',
'Package: python3:any\n')
with open(fauxpkg_fname, 'w') as f:
for segment in fauxpkg_data.values():
f.write(segment + '\n\n')
def _prepare_source_data(self, session, mi_wspace: str,
suites_source: List[ArchiveSuite],
suite_target: ArchiveSuite):
'''
If there is more than one source suite, we need to give Britney an amalgamation
of the data of the two source suites.
This function prepares this data.
'''
# only one suite means we can use the suite's data directly
if len(suites_source) <= 1:
return
archive_root_dir = self._lconf.archive_root_dir
fake_dists_dir = self._get_source_suite_dists_dir(
mi_wspace, suites_source)
for component in suite_target.components:
for arch in suite_target.architectures:
if arch.name == 'all':
continue
packages_files = []
for installer_dir in ['', 'debian-installer']:
for suite_source in suites_source:
pfile = os.path.join(archive_root_dir, 'dists',
suite_source.name, component.name,
installer_dir,
'binary-{}'.format(arch.name),
'Packages.xz')
if os.path.isfile(pfile):
log.debug(
'Looking for packages in: {}'.format(pfile))
packages_files.append(pfile)
if not installer_dir and not installer_dir:
raise Exception(
'No packages found on {}/{} in sources for migration "{}": Can not continue.'
.format(
component.name, arch.name,
self._get_migration_id(suites_source,
suite_target)))
# create new merged Packages file
target_packages_file = os.path.join(
fake_dists_dir, component.name, installer_dir,
'binary-{}'.format(arch.name), 'Packages.xz')
log.debug('Generating combined new fake packages file: {}'.
format(target_packages_file))
os.makedirs(os.path.dirname(target_packages_file),
exist_ok=True)
data = b''
for fname in packages_files:
with open_compressed(fname) as f:
data = data + f.read()
with lzma.open(target_packages_file, 'w') as f:
f.write(data)
sources_files = []
for suite_source in suites_source:
sfile = os.path.join(archive_root_dir, 'dists',
suite_source.name, component.name,
'source', 'Sources.xz')
if os.path.isfile(sfile):
log.debug(
'Looking for source packages in: {}'.format(sfile))
sources_files.append(sfile)
if not sources_files:
raise Exception(
'No source packages found in "{}" sources for migration "{}": Can not continue.'
.format(
component.name,
self._get_migrarion_id(suites_source, suite_target)))
# Create new merged Sources file
target_sources_file = os.path.join(fake_dists_dir, component.name,
'source', 'Sources.xz')
log.debug('Generating combined new fake sources file: {}'.format(
target_sources_file))
os.makedirs(os.path.dirname(target_sources_file), exist_ok=True)
data = b''
for fname in sources_files:
with open_compressed(fname) as f:
data = data + f.read()
with lzma.open(target_sources_file, 'w') as f:
f.write(data)
# Britney needs a Release file to determine the source suites components and architectures.
# To keep things simple, we just copy one of the source Release files.
# TODO: Synthesize a dedicated file instead and be less lazy
release_file = os.path.join(archive_root_dir, 'dists',
suites_source[0].name, 'Release')
target_release_file = os.path.join(fake_dists_dir, 'Release')
log.debug('Using Release file for fake suite: {}'.format(
target_release_file))
if os.path.join(target_release_file):
os.remove(target_release_file)
shutil.copyfile(release_file, target_release_file)
