def __init__(self, repo_entity):
lconf = LocalConfig()
self._repo = Repository(lconf.archive_root_dir,
'master',
entity=repo_entity)
self._repo_entity = repo_entity
self._repo.set_trusted(True)
def _get_local_repo(self, session, repo_name='master'):
repo = session.query(ArchiveRepository) \
.filter(ArchiveRepository.name == repo_name).one()
local_repo = Repository(self._lconf.archive_root_dir,
repo.name,
trusted_keyrings=[])
# we unconditionally trust the local repository - for now
local_repo.set_trusted(True)
return local_repo
def test_repo_local(samplesdir, localconfig):
keyrings = localconfig.trusted_gpg_keyrings
repo_location = os.path.join(samplesdir, 'samplerepo', 'dummy')
suite = ArchiveSuite('testing')
component = ArchiveComponent('main')
arch = ArchiveArchitecture('amd64')
arch_all = ArchiveArchitecture('all')
repo = Repository(repo_location, 'Dummy', trusted_keyrings=[])
# we have no keyrings set, so this should fail
with pytest.raises(GpgException):
src_pkgs = repo.source_packages(suite, component)
# try again!
repo = Repository(repo_location, 'Dummy', trusted_keyrings=keyrings)
src_pkgs = repo.source_packages(suite, component)
bin_pkgs = repo.binary_packages(suite, component, arch)
assert len(bin_pkgs) == 4
bin_pkgs.extend(repo.binary_packages(suite, component, arch_all))
# check packages
assert len(src_pkgs) == 8
assert len(bin_pkgs) == 7
validate_src_packages(src_pkgs)
validate_bin_packages(bin_pkgs)
def __init__(self, target_suite_name: str, source_suite_name: str):
self._lconf = LocalConfig()
self._dak = DakBridge()
# FIXME: Don't hardcode this!
repo_name = 'master'
# the repository of the distribution we import stuff into
self._target_repo = Repository(self._lconf.archive_root_dir, repo_name)
self._target_repo.set_trusted(True)
self._target_suite_name = target_suite_name
self._source_suite_name = source_suite_name
self._distro_tag = config_get_distro_tag()
self._synced_source_pkgs: list[SourcePackage] = []
with session_scope() as session:
sync_source = session.query(SynchrotronSource) \
.filter(SynchrotronSource.suite_name == self._source_suite_name).one()
# FIXME: Synchrotron needs adjustments to work
# better with the new "multiple autosync tasks" model.
# This code will need to be revised for that
# (currently it is just a 1:1 translation from D code)
# the repository of the distribution we use to sync stuff from
self._source_repo = Repository(
sync_source.repo_url, sync_source.os_name,
self._lconf.synchrotron_sourcekeyrings)
# we trust everything by default
self._imports_trusted = True
with session_scope() as session:
self._sync_blacklist = set([
value for value, in session.query(SyncBlacklistEntry.pkgname)
])
class SyncEngine:
'''
Execute package synchronization in Synchrotron
'''
def __init__(self, target_suite_name: str, source_suite_name: str):
self._lconf = LocalConfig()
self._dak = DakBridge()
# FIXME: Don't hardcode this!
repo_name = 'master'
# the repository of the distribution we import stuff into
self._target_repo = Repository(self._lconf.archive_root_dir,
repo_name)
self._target_repo.set_trusted(True)
self._target_suite_name = target_suite_name
self._source_suite_name = source_suite_name
self._distro_tag = config_get_distro_tag()
self._synced_source_pkgs = []
with session_scope() as session:
sync_source = session.query(SynchrotronSource) \
.filter(SynchrotronSource.suite_name == self._source_suite_name).one()
# FIXME: Synchrotron needs adjustments to work
# better with the new "multiple autosync tasks" model.
# This code will need to be revised for that
# (currently it is just a 1:1 translation from D code)
# the repository of the distribution we use to sync stuff from
self._source_repo = Repository(sync_source.repo_url,
sync_source.os_name,
self._lconf.synchrotron_sourcekeyrings)
# we trust everything by default
self._imports_trusted = True
with session_scope() as session:
self._sync_blacklist = set([value for value, in session.query(SyncBlacklistEntry.pkgname)])
def _publish_synced_spkg_events(self, src_os, src_suite, dest_suite, forced=False, emitter=None):
''' Submit events for the synced source packages to the message stream '''
if not emitter:
emitter = EventEmitter(LkModule.SYNCHROTRON)
for spkg in self._synced_source_pkgs:
data = {'name': spkg.name,
'version': spkg.version,
'src_os': src_os,
'suite_src': src_suite,
'suite_dest': dest_suite,
'forced': forced}
emitter.submit_event('src-package-imported', data)
def _get_repo_source_package_map(self, repo, suite_name: str, component_name: str):
''' Get an associative array of the newest source packages present in a repository. '''
suite = ArchiveSuite(suite_name)
component = ArchiveComponent(component_name)
spkgs = repo.source_packages(suite, component)
return make_newest_packages_dict(spkgs)
def _get_repo_binary_package_map(self, repo, suite_name: str, component_name: str,
arch_name: str = None, with_installer: bool = True):
''' Get an associative array of the newest binary packages present in a repository. '''
suite = ArchiveSuite(suite_name)
component = ArchiveComponent(component_name)
arch = ArchiveArchitecture(arch_name)
arch_all = ArchiveArchitecture('all')
bpkgs = repo.binary_packages(suite, component, arch)
bpkgs.extend(repo.binary_packages(suite, component, arch_all)) # always append arch:all packages
if with_installer:
# add d-i packages to the mix
bpkgs.extend(repo.installer_packages(suite, component, arch))
bpkgs.extend(repo.installer_packages(suite, component, arch_all)) # always append arch:all packages
return make_newest_packages_dict(bpkgs)
def _get_target_source_packages(self, component: str):
''' Get mapping of all sources packages in a suite and its parent suite. '''
with session_scope() as session:
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
suite_pkgmap = self._get_repo_source_package_map(self._target_repo,
target_suite.name,
component)
if target_suite.parent:
# we have a parent suite
parent_map = self._get_repo_source_package_map(self._target_repo,
target_suite.parent.name,
component)
# merge the two arrays, keeping only the latest versions
suite_pkgmap = make_newest_packages_dict(list(parent_map.values()) + list(suite_pkgmap.values()))
return suite_pkgmap
def _import_package_files(self, suite: str, component: str, fnames: List[str]):
''' Import an arbitrary amount of packages via the archive management software. '''
return self._dak.import_package_files(suite, component, fnames, self._imports_trusted, True)
def _import_source_package(self, spkg: SourcePackage, component: str) -> bool:
'''
Import a source package from the source repository into the
target repo.
'''
dscfile = None
for f in spkg.files:
# the source repository might be on a remote location, so we need to
# request each file to be there.
# (dak will fetch the files referenced in the .dsc file from the same directory)
if f.fname.endswith('.dsc'):
dscfile = self._source_repo.get_file(f)
self._source_repo.get_file(f)
if not dscfile:
log.error('Critical consistency error: Source package {} in repository {} has no .dsc file.'
.format(spkg.name, self._source_repo.base_dir))
return False
if self._import_package_files(self._target_suite_name, component, [dscfile]):
self._synced_source_pkgs.append(spkg)
return True
return False
def _import_binaries_for_source(self, sync_conf, target_suite, component: str, spkgs: List[SourcePackage],
ignore_target_changes: bool = False) -> bool:
''' Import binary packages for the given set of source packages into the archive. '''
if not sync_conf.sync_binaries:
log.debug('Skipping binary syncs.')
return True
# list of valid architectrures supported by the target
target_archs = [a.name for a in target_suite.architectures]
# cache of binary-package mappings for the source
src_bpkg_arch_map = {}
for aname in target_archs:
src_bpkg_arch_map[aname] = self._get_repo_binary_package_map(self._source_repo, self._source_suite_name, component, aname)
# cache of binary-package mappings from the target repository
dest_bpkg_arch_map = {}
for aname in target_archs:
dest_bpkg_arch_map[aname] = self._get_repo_binary_package_map(self._target_repo, self._target_suite_name, component, aname)
for spkg in spkgs:
bin_files_synced = False
existing_packages = False
for arch_name in target_archs:
if arch_name not in src_bpkg_arch_map:
continue
src_bpkg_map = src_bpkg_arch_map[arch_name]
dest_bpkg_map = dest_bpkg_arch_map[arch_name]
bin_files = []
for bin_i in spkg.binaries:
if bin_i.name not in src_bpkg_map:
if bin_i.name in dest_bpkg_map:
existing_packages = True # package only exists in target
continue
if arch_name != 'all' and bin_i.architectures == ['all']:
# we handle arch:all explicitly
continue
bpkg = src_bpkg_map[bin_i.name]
if bin_i.version != bpkg.source_version:
log.debug('Not syncing binary package \'{}\': Version number \'{}\' does not match source package version \'{}\'.'
.format(bpkg.name, bin_i.version, bpkg.source_version))
continue
ebpkg = dest_bpkg_map.get(bpkg.name)
if ebpkg:
if version_compare(ebpkg.version, bpkg.version) >= 0:
log.debug('Not syncing binary package \'{}/{}\': Existing binary package with bigger/equal version \'{}\' found.'
.format(bpkg.name, bpkg.version, ebpkg.version))
existing_packages = True
continue
# Filter out manual rebuild uploads matching the pattern XbY.
# sometimes rebuild uploads of not-modified packages happen, and if the source
# distro did a binNMU, we don't want to sync that, even if it's bigger
# This rebuild-upload check must only happen if we haven't just updated the source package
# (in that case the source package version will be bigger than the existing binary package version)
if version_compare(spkg.version, ebpkg.version) >= 0:
if re.match(r'(.*)b([0-9]+)', ebpkg.version):
log.debug('Not syncing binary package \'{}/{}\': Existing binary package with rebuild upload \'{}\' found.'
.format(bpkg.name, bpkg.version, ebpkg.version))
existing_packages = True
continue
if not ignore_target_changes and self._distro_tag in version_revision(ebpkg.version):
# safety measure, we should never get here as packages with modifications were
# filtered out previously.
log.debug('Can not sync binary package {}/{}: Target has modifications.'.format(bin_i.name, bin_i.version))
continue
fname = self._source_repo.get_file(bpkg.bin_file)
bin_files.append(fname)
# now import the binary packages, if there is anything to import
if bin_files:
bin_files_synced = True
ret = self._import_package_files(self._target_suite_name, component, bin_files)
if not ret:
return False
if not bin_files_synced and not existing_packages:
log.warning('No binary packages synced for source {}/{}'.format(spkg.name, spkg.version))
return True
def sync_packages(self, component: str, pkgnames: List[str], force: bool = False):
self._synced_source_pkgs = []
with session_scope() as session:
sync_conf = session.query(SynchrotronConfig) \
.join(SynchrotronConfig.destination_suite) \
.join(SynchrotronConfig.source) \
.filter(ArchiveSuite.name == self._target_suite_name,
SynchrotronSource.suite_name == self._source_suite_name).one_or_none()
if not sync_conf:
log.error('Unable to find a sync config for this source/destination combination.')
return False
if not sync_conf.sync_enabled:
log.error('Can not synchronize package: Synchronization is disabled for this configuration.')
return False
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
dest_pkg_map = self._get_target_source_packages(component)
src_pkg_map = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component)
for pkgname in pkgnames:
spkg = src_pkg_map.get(pkgname)
dpkg = dest_pkg_map.get(pkgname)
if not spkg:
log.info('Can not sync {}: Does not exist in source.'.format(pkgname))
continue
if pkgname in self._sync_blacklist:
log.info('Can not sync {}: The package is blacklisted.'.format(pkgname))
continue
if dpkg:
if version_compare(dpkg.version, spkg.version) >= 0:
if force:
log.warning('{}: Target version \'{}\' is newer/equal than source version \'{}\'.'
.format(pkgname, dpkg.version, spkg.version))
else:
log.info('Can not sync {}: Target version \'{}\' is newer/equal than source version \'{}\'.'
.format(pkgname, dpkg.version, spkg.version))
continue
if not force:
if self._distro_tag in version_revision(dpkg.version):
log.error('Not syncing {}/{}: Destination has modifications (found {}).'
.format(spkg.name, spkg.version, dpkg.version))
continue
# sync source package
# the source package must always be known to dak first
ret = self._import_source_package(spkg, component)
if not ret:
return False
ret = self._import_binaries_for_source(sync_conf, target_suite, component, self._synced_source_pkgs, force)
# TODO: Analyze the input, fetch the packages from the source distribution and
# import them into the target in their correct order.
# Then apply the correct, synced override from the source distro.
self._publish_synced_spkg_events(sync_conf.source.os_name,
sync_conf.source.suite_name,
sync_conf.destination_suite.name,
force)
return ret
def autosync(self, session, sync_conf, remove_cruft: bool = True):
''' Synchronize all packages that are newer '''
self._synced_source_pkgs = []
active_src_pkgs = [] # source packages which should have their binary packages updated
res_issues = []
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
sync_conf = session.query(SynchrotronConfig) \
.join(SynchrotronConfig.destination_suite) \
.join(SynchrotronConfig.source) \
.filter(ArchiveSuite.name == self._target_suite_name,
SynchrotronSource.suite_name == self._source_suite_name).one_or_none()
for component in target_suite.components:
dest_pkg_map = self._get_target_source_packages(component.name)
# The source package lists contains many different versions, some source package
# versions are explicitly kept for GPL-compatibility.
# Sometimes a binary package migrates into another suite, dragging a newer source-package
# that it was built against with itslf into the target suite.
# These packages then have a source with a high version number, but might not have any
# binaries due to them migrating later.
# We need to care for that case when doing binary syncs (TODO: and maybe safeguard against it
# when doing source-only syncs too?), That's why we don't filter out the newest packages in
# binary-sync-mode.
if sync_conf.sync_binaries:
src_pkg_range = self._source_repo.source_packages(ArchiveSuite(self._source_suite_name), component)
else:
src_pkg_range = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component).values()
for spkg in src_pkg_range:
# ignore blacklisted packages in automatic sync
if spkg.name in self._sync_blacklist:
continue
dpkg = dest_pkg_map.get(spkg.name)
if dpkg:
if version_compare(dpkg.version, spkg.version) >= 0:
log.debug('Skipped sync of {}: Target version \'{}\' is equal/newer than source version \'{}\'.'
.format(spkg.name, dpkg.version, spkg.version))
continue
# check if we have a modified target package,
# indicated via its Debian revision, e.g. "1.0-0tanglu1"
if self._distro_tag in version_revision(dpkg.version):
log.info('Not syncing {}/{}: Destination has modifications (found {}).'
.format(spkg.name, spkg.version, dpkg.version))
# add information that this package needs to be merged to the issue list
issue = SynchrotronIssue()
issue.package_name = spkg.name
issue.source_version = spkg.version
issue.target_version = dpkg.version
issue.kind = SynchrotronIssueKind.MERGE_REQUIRED
res_issues.append(issue)
continue
# sync source package
# the source package must always be known to dak first
ret = self._import_source_package(spkg, component.name)
if not ret:
return False, []
# a new source package is always active and needs it's binary packages synced, in
# case we do binary syncs.
active_src_pkgs.append(spkg)
# all packages in the target distribution are considered active, as long as they don't
# have modifications.
for spkg in dest_pkg_map.values():
if self._distro_tag in version_revision(spkg.version):
active_src_pkgs.append(spkg)
# import binaries as well. We test for binary updates for all available active source packages,
# as binNMUs might have happened in the source distribution.
# (an active package in this context is any source package which doesn't have modifications in the
# target distribution)
ret = self._import_binaries_for_source(sync_conf, target_suite, component.name, active_src_pkgs)
if not ret:
return False, []
# test for cruft packages
target_pkg_index = {}
for component in target_suite.components:
dest_pkg_map = self._get_repo_source_package_map(self._target_repo,
target_suite.name,
component.name)
for pkgname, pkg in dest_pkg_map.items():
target_pkg_index[pkgname] = pkg
# check which packages are present in the target, but not in the source suite
for component in target_suite.components:
src_pkg_map = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component.name)
for pkgname in src_pkg_map.keys():
target_pkg_index.pop(pkgname, None)
# remove cruft packages
if remove_cruft:
for pkgname, dpkg in target_pkg_index.items():
dpkg_ver_revision = version_revision(dpkg.version, False)
# native packages are never removed
if not dpkg_ver_revision:
continue
# check if the package is intoduced as new in the distro, in which case we won't remove it
if dpkg_ver_revision.startswith('0' + self._distro_tag):
continue
# if this package was modified in the target distro, we will also not remove it, but flag it
# as "potential cruft" for someone to look at.
if self._distro_tag in dpkg_ver_revision:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.MAYBE_CRUFT
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
res_issues.append(issue)
continue
# check if we can remove this package without breaking stuff
if self._dak.package_is_removable(dpkg.name, target_suite.name):
# try to remove the package
try:
self._dak.remove_package(dpkg.name, target_suite.name)
except Exception as e:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = str(e)
res_issues.append(issue)
else:
# looks like we can not remove this
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = 'This package can not be removed without breaking other packages. It needs manual removal.'
res_issues.append(issue)
self._publish_synced_spkg_events(sync_conf.source.os_name,
sync_conf.source.suite_name,
sync_conf.destination_suite.name,
False)
return True, res_issues
def import_suite_packages(suite_name):
# FIXME: Don't hardcode the "master" repository here, fully implement
# the "multiple repositories" feature
repo_name = 'master'
session = session_factory()
suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == suite_name).one()
repo = session.query(ArchiveRepository) \
.filter(ArchiveRepository.name == repo_name).one()
lconf = LocalConfig()
local_repo = Repository(lconf.archive_root_dir,
repo.name,
trusted_keyrings=[],
entity=repo)
# we unconditionally trust the local repository - for now
local_repo.set_trusted(True)
# event emitted for message passing
emitter = EventEmitter(LkModule.ARCHIVE)
for component in suite.components:
# fetch all source packages for the given repository
# FIXME: Urgh... Can this be more efficient?
existing_spkgs = dict()
all_existing_src_packages = session.query(SourcePackage) \
.options(joinedload(SourcePackage.suites)) \
.filter(SourcePackage.repo_id == repo.id) \
.filter(SourcePackage.component_id == component.id).all()
for e_spkg in all_existing_src_packages:
existing_spkgs[e_spkg.uuid] = e_spkg
for spkg in local_repo.source_packages(suite, component):
db_spkg = existing_spkgs.pop(spkg.uuid, None)
if db_spkg:
session.expunge(spkg)
if suite in db_spkg.suites:
continue # the source package is already registered with this suite
db_spkg.suites.append(suite)
_emit_package_event(emitter,
'source-package-published-in-suite', spkg,
{'suite_new': suite.name})
continue
session.add(spkg)
_emit_package_event(emitter, 'source-package-published', spkg)
for old_spkg in existing_spkgs.values():
if suite in old_spkg.suites:
old_spkg.suites.remove(suite)
_emit_package_event(emitter, 'source-package-suite-removed',
old_spkg, {'suite_old': suite.name})
if len(old_spkg.suites) <= 0:
for f in old_spkg.files:
session.delete(f)
session.delete(old_spkg)
_emit_package_event(emitter, 'removed-source-package',
old_spkg)
# commit the source package changes already
session.commit()
for arch in suite.architectures:
# Get all binary packages UUID/suite-id combinations for the given architecture and suite
# FIXME: Urgh... Can this be more efficient?
bpkg_b = Bundle('bin_package', BinaryPackage.uuid)
suite_b = Bundle('archive_suite', ArchiveSuite.id)
existing_bpkgs = dict()
for e_bpkg, suite_i in session.query(bpkg_b, suite_b) \
.filter(BinaryPackage.repo_id == repo.id) \
.filter(BinaryPackage.component_id == component.id) \
.filter(BinaryPackage.architecture_id == arch.id).join(BinaryPackage.suites):
sl = existing_bpkgs.get(e_bpkg.uuid)
if not sl:
existing_bpkgs[e_bpkg.uuid] = [
suite_i.id
] # if there is just one suite, we may get a scalar here
else:
sl.append(suite_i.id)
# add information about regular binary packages
existing_bpkgs = _register_binary_packages(
session, repo, suite, component, arch, existing_bpkgs,
local_repo.binary_packages(suite, component, arch), emitter)
# add information about debian-installer packages
existing_bpkgs = _register_binary_packages(
session, repo, suite, component, arch, existing_bpkgs,
local_repo.installer_packages(suite, component, arch), emitter)
session.commit()
for old_bpkg_uuid, suites in existing_bpkgs.items():
suites_count = len(suites)
if suite.id in suites:
rc = session.query(binpkg_suite_assoc_table) \
.filter(binpkg_suite_assoc_table.c.suite_id == suite.id) \
.filter(binpkg_suite_assoc_table.c.bin_package_uuid == old_bpkg_uuid) \
.delete(synchronize_session=False)
if rc > 0:
suites_count -= 1
if suites_count <= 0:
# delete the old package, we don't need it anymore if it is in no suites
session.query(ArchiveFile) \
.filter(ArchiveFile.binpkg_id == old_bpkg_uuid).delete()
session.query(BinaryPackage) \
.filter(BinaryPackage.uuid == old_bpkg_uuid).delete()
# NOTE: We do not emit messages for removed binary packages, as they are usually
# deleted with their source package (unless we have an arch-specific removal) and we
# don't want to spam messages which may be uninteresting to current Laniakea modules.
session.commit()
# import new AppStream component metadata / delete old components
update_appstream_data(session, local_repo, repo, suite, component,
arch)
# delete orphaned AppStream metadata
for cpt in session.query(SoftwareComponent).filter(
~SoftwareComponent.bin_packages.any()).all():
session.delete(cpt)
session.commit()
class Debcheck:
'''
Analyze the archive's dependency chain.
'''
def __init__(self, repo_entity):
lconf = LocalConfig()
self._repo = Repository(lconf.archive_root_dir, 'master', entity=repo_entity)
self._repo_entity = repo_entity
self._repo.set_trusted(True)
def _execute_dose(self, dose_exe, args, files=[]):
yaml_data = ''
cmd = [dose_exe]
cmd.extend(args)
cmd.extend(files)
pipe = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
while True:
line = pipe.stdout.readline()
if line == '' and pipe.poll() is not None:
break
if line:
yaml_data += line
if not yaml_data.startswith('output-version'):
# the output is weird, assume an error
return False, yaml_data + '\n' + pipe.stderr.read()
return True, yaml_data
def _get_full_index_info(self, suite, arch, sources=False):
'''
Get a list of index files for the specific suite and architecture,
for all components, as well as all the suites it depends on.
The actual indices belonging to the suite are added as "foreground" (fg), the
ones of the dependencies are added as "background" (bg).
'''
res = {'fg': [], 'bg': []}
bin_arch = suite.primary_architecture
for component in suite.components:
if sources:
fname = self._repo.index_file(suite, os.path.join(component.name, 'source', 'Sources.xz'))
if fname:
res['fg'].append(fname)
fname = self._repo.index_file(suite, os.path.join(component.name, 'binary-{}'.format(arch.name), 'Packages.xz'))
if fname:
res['bg'].append(fname)
else:
fname = self._repo.index_file(suite, os.path.join(component.name, 'binary-{}'.format(arch.name), 'Packages.xz'))
if fname:
res['fg'].append(fname)
if arch.name == 'all':
fname = self._repo.index_file(suite, os.path.join(component.name, 'binary-{}'.format(bin_arch.name), 'Packages.xz'))
if fname:
res['bg'].append(fname)
# add base suite packages to the background
if suite.parent:
parent_indices = self._get_full_index_info(suite.parent, arch, False)
res['bg'].extend(parent_indices['bg'])
res['bg'].extend(parent_indices['fg'])
return res
def _generate_build_depcheck_yaml(self, suite):
'''
Get Dose YAML data for build dependency issues in the selected suite.
'''
arch_issue_map = {}
for arch in suite.architectures:
# fetch source-package-centric index list
indices = self._get_full_index_info(suite, arch, True)
if not indices['fg']:
if arch.name == 'all':
continue
raise Exception('Unable to get any indices for {}/{} to check for dependency issues.'.format(suite.name, arch.name))
dose_args = ['--quiet',
'--latest=1',
'-e',
'-f',
'--summary',
'--deb-emulate-sbuild',
'--deb-native-arch={}'.format(suite.primary_architecture.name if arch.name == 'all' else arch.name)]
# run builddepcheck
success, data = self._execute_dose('dose-builddebcheck', dose_args, indices['bg'] + indices['fg'])
if not success:
raise Exception('Unable to run Dose for {}/{}: {}'.format(suite.name, arch.name, data))
arch_issue_map[arch.name] = data
return arch_issue_map
def _generate_depcheck_yaml(self, suite):
'''
Get Dose YAML data for build installability issues in the selected suite.
'''
arch_issue_map = {}
for arch in suite.architectures:
# fetch binary-package index list
indices = self._get_full_index_info(suite, arch, False)
if not indices['fg']:
if arch.name == 'all':
continue
raise Exception('Unable to get any indices for {}/{} to check for dependency issues.'.format(suite.name, arch.name))
dose_args = ['--quiet',
'--latest=1',
'-e',
'-f',
'--summary',
'--deb-native-arch={}'.format(suite.primary_architecture.name if arch.name == 'all' else arch.name)]
# run depcheck
indices_args = []
for f in indices['bg']:
indices_args.append('--bg={}'.format(f))
for f in indices['fg']:
indices_args.append('--fg={}'.format(f))
success, data = self._execute_dose('dose-debcheck', dose_args, indices_args)
if not success:
log.error('Dose debcheck command failed: ' + ' '.join(dose_args) + ' ' + ' '.join(indices_args) + '\n' + data)
raise Exception('Unable to run Dose for {}/{}: {}'.format(suite.name, arch.name, data))
arch_issue_map[arch.name] = data
return arch_issue_map
def _dose_yaml_to_issues(self, yaml_data, suite, arch_name):
def set_basic_package_info(v, entry):
if 'type' in entry and entry['type'] == "src":
v.package_type = PackageType.SOURCE
else:
v.package_type = PackageType.BINARY
v.package_name = str(entry['package'])
v.package_version = str(entry['version'])
v.architectures = str(entry['architecture']).split(',')
res = []
yroot = yaml.safe_load(yaml_data)
report = yroot['report']
arch_is_all = arch_name == 'all'
# if the report is empty, we have no issues to generate and can quit
if not report:
return res
for entry in report:
if not arch_is_all:
# we ignore entries from "all" unless we are explicitly reading information
# for that fake architecture.
if entry['architecture'] == 'all':
continue
issue = DebcheckIssue()
issue.repo = self._repo_entity
issue.time = datetime.utcnow()
issue.suite = suite
missing = []
conflicts = []
set_basic_package_info(issue, entry)
reasons = entry['reasons']
for reason in reasons:
if 'missing' in reason:
# we have a missing package issue
ymissing = reason['missing']['pkg']
pkgissue = PackageIssue()
set_basic_package_info(pkgissue, ymissing)
pkgissue.unsat_dependency = ymissing['unsat-dependency']
missing.append(pkgissue)
elif 'conflict' in reason:
# we have a conflict in the dependency chain
yconflict = reason['conflict']
conflict = PackageConflict()
conflict.pkg1 = PackageIssue()
conflict.pkg2 = PackageIssue()
conflict.depchain1 = []
conflict.depchain2 = []
set_basic_package_info(conflict.pkg1, yconflict['pkg1'])
if 'unsat-conflict' in yconflict['pkg1']:
conflict.pkg1.unsat_conflict = yconflict['pkg1']['unsat-conflict']
set_basic_package_info(conflict.pkg2, yconflict['pkg2'])
if 'unsat-conflict' in yconflict['pkg2']:
conflict.pkg2.unsat_conflict = yconflict['pkg2']['unsat-conflict']
# parse the depchain
if 'depchain1' in yconflict:
for ypkg in yconflict['depchain1'][0]['depchain']:
pkgissue = PackageIssue()
set_basic_package_info(pkgissue, ypkg)
pkgissue.depends = ypkg.get('depends')
conflict.depchain1.append(pkgissue)
if 'depchain2' in yconflict:
for ypkg in yconflict['depchain2'][0]['depchain']:
pkgissue = PackageIssue()
set_basic_package_info(pkgissue, ypkg)
pkgissue.depends = ypkg.get('depends')
conflict.depchain2.append(pkgissue)
conflicts.append(conflict)
else:
raise Exception('Found unknown dependency issue: ' + str(reason))
issue.missing = missing
issue.conflicts = conflicts
res.append(issue)
return res
def build_depcheck_issues(self, suite):
''' Get a list of build-dependency issues affecting the suite '''
issues = []
issues_yaml = self._generate_build_depcheck_yaml(suite)
for arch_name, yaml_data in issues_yaml.items():
issues.extend(self._dose_yaml_to_issues(yaml_data, suite, arch_name))
return issues
def depcheck_issues(self, suite):
''' Get a list of dependency issues affecting the suite '''
issues = []
issues_yaml = self._generate_depcheck_yaml(suite)
for arch_name, yaml_data in issues_yaml.items():
issues.extend(self._dose_yaml_to_issues(yaml_data, suite, arch_name))
return issues