def test_repo_local(samplesdir, localconfig):
keyrings = localconfig.trusted_gpg_keyrings
repo_location = os.path.join(samplesdir, 'samplerepo', 'dummy')
suite = ArchiveSuite('testing')
component = ArchiveComponent('main')
arch = ArchiveArchitecture('amd64')
arch_all = ArchiveArchitecture('all')
repo = Repository(repo_location, 'Dummy', trusted_keyrings=[])
# we have no keyrings set, so this should fail
with pytest.raises(GpgException):
src_pkgs = repo.source_packages(suite, component)
# try again!
repo = Repository(repo_location, 'Dummy', trusted_keyrings=keyrings)
src_pkgs = repo.source_packages(suite, component)
bin_pkgs = repo.binary_packages(suite, component, arch)
assert len(bin_pkgs) == 4
bin_pkgs.extend(repo.binary_packages(suite, component, arch_all))
# check packages
assert len(src_pkgs) == 8
assert len(bin_pkgs) == 7
validate_src_packages(src_pkgs)
validate_bin_packages(bin_pkgs)
def _get_repo_source_package_map(self, repo, suite_name: str, component_name: str):
''' Get an associative array of the newest source packages present in a repository. '''
suite = ArchiveSuite(suite_name)
component = ArchiveComponent(component_name)
spkgs = repo.source_packages(suite, component)
return make_newest_packages_dict(spkgs)
def _get_repo_binary_package_map(self, repo, suite_name: str, component_name: str,
arch_name: str = None, with_installer: bool = True):
''' Get an associative array of the newest binary packages present in a repository. '''
suite = ArchiveSuite(suite_name)
component = ArchiveComponent(component_name)
arch = ArchiveArchitecture(arch_name)
arch_all = ArchiveArchitecture('all')
bpkgs = repo.binary_packages(suite, component, arch)
bpkgs.extend(repo.binary_packages(suite, component, arch_all)) # always append arch:all packages
if with_installer:
# add d-i packages to the mix
bpkgs.extend(repo.installer_packages(suite, component, arch))
bpkgs.extend(repo.installer_packages(suite, component, arch_all)) # always append arch:all packages
return make_newest_packages_dict(bpkgs)
def autosync(self, session, sync_conf, remove_cruft: bool = True):
''' Synchronize all packages that are newer '''
self._synced_source_pkgs = []
active_src_pkgs = [] # source packages which should have their binary packages updated
res_issues = []
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
sync_conf = session.query(SynchrotronConfig) \
.join(SynchrotronConfig.destination_suite) \
.join(SynchrotronConfig.source) \
.filter(ArchiveSuite.name == self._target_suite_name,
SynchrotronSource.suite_name == self._source_suite_name).one_or_none()
for component in target_suite.components:
dest_pkg_map = self._get_target_source_packages(component.name)
# The source package lists contains many different versions, some source package
# versions are explicitly kept for GPL-compatibility.
# Sometimes a binary package migrates into another suite, dragging a newer source-package
# that it was built against with itslf into the target suite.
# These packages then have a source with a high version number, but might not have any
# binaries due to them migrating later.
# We need to care for that case when doing binary syncs (TODO: and maybe safeguard against it
# when doing source-only syncs too?), That's why we don't filter out the newest packages in
# binary-sync-mode.
if sync_conf.sync_binaries:
src_pkg_range = self._source_repo.source_packages(ArchiveSuite(self._source_suite_name), component)
else:
src_pkg_range = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component).values()
for spkg in src_pkg_range:
# ignore blacklisted packages in automatic sync
if spkg.name in self._sync_blacklist:
continue
dpkg = dest_pkg_map.get(spkg.name)
if dpkg:
if version_compare(dpkg.version, spkg.version) >= 0:
log.debug('Skipped sync of {}: Target version \'{}\' is equal/newer than source version \'{}\'.'
.format(spkg.name, dpkg.version, spkg.version))
continue
# check if we have a modified target package,
# indicated via its Debian revision, e.g. "1.0-0tanglu1"
if self._distro_tag in version_revision(dpkg.version):
log.info('Not syncing {}/{}: Destination has modifications (found {}).'
.format(spkg.name, spkg.version, dpkg.version))
# add information that this package needs to be merged to the issue list
issue = SynchrotronIssue()
issue.package_name = spkg.name
issue.source_version = spkg.version
issue.target_version = dpkg.version
issue.kind = SynchrotronIssueKind.MERGE_REQUIRED
res_issues.append(issue)
continue
# sync source package
# the source package must always be known to dak first
ret = self._import_source_package(spkg, component.name)
if not ret:
return False, []
# a new source package is always active and needs it's binary packages synced, in
# case we do binary syncs.
active_src_pkgs.append(spkg)
# all packages in the target distribution are considered active, as long as they don't
# have modifications.
for spkg in dest_pkg_map.values():
if self._distro_tag in version_revision(spkg.version):
active_src_pkgs.append(spkg)
# import binaries as well. We test for binary updates for all available active source packages,
# as binNMUs might have happened in the source distribution.
# (an active package in this context is any source package which doesn't have modifications in the
# target distribution)
ret = self._import_binaries_for_source(sync_conf, target_suite, component.name, active_src_pkgs)
if not ret:
return False, []
# test for cruft packages
target_pkg_index = {}
for component in target_suite.components:
dest_pkg_map = self._get_repo_source_package_map(self._target_repo,
target_suite.name,
component.name)
for pkgname, pkg in dest_pkg_map.items():
target_pkg_index[pkgname] = pkg
# check which packages are present in the target, but not in the source suite
for component in target_suite.components:
src_pkg_map = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component.name)
for pkgname in src_pkg_map.keys():
target_pkg_index.pop(pkgname, None)
# remove cruft packages
if remove_cruft:
for pkgname, dpkg in target_pkg_index.items():
dpkg_ver_revision = version_revision(dpkg.version, False)
# native packages are never removed
if not dpkg_ver_revision:
continue
# check if the package is intoduced as new in the distro, in which case we won't remove it
if dpkg_ver_revision.startswith('0' + self._distro_tag):
continue
# if this package was modified in the target distro, we will also not remove it, but flag it
# as "potential cruft" for someone to look at.
if self._distro_tag in dpkg_ver_revision:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.MAYBE_CRUFT
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
res_issues.append(issue)
continue
# check if we can remove this package without breaking stuff
if self._dak.package_is_removable(dpkg.name, target_suite.name):
# try to remove the package
try:
self._dak.remove_package(dpkg.name, target_suite.name)
except Exception as e:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = str(e)
res_issues.append(issue)
else:
# looks like we can not remove this
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = 'This package can not be removed without breaking other packages. It needs manual removal.'
res_issues.append(issue)
self._publish_synced_spkg_events(sync_conf.source.os_name,
sync_conf.source.suite_name,
sync_conf.destination_suite.name,
False)
return True, res_issues
def database(localconfig, podman_ip, podman_services):
'''
Retrieve a pristine, empty Laniakea database connection.
This will wipe the global database, so tests using this can
never run in parallel.
'''
import json
from laniakea.db import Database, session_scope, ArchiveRepository, ArchiveSuite, \
ArchiveComponent, ArchiveArchitecture
from laniakea.db.core import config_set_project_name, config_set_distro_tag
# get IP of our database container
db_port = podman_services.port_for('postgres', 5432)
# update database URL to use scratch database in our container
pgdb_url = 'postgresql://*****:*****@{}:{}/laniakea_unittest'.format(
podman_ip, db_port)
LocalConfig.instance._database_url = pgdb_url
assert localconfig.database_url == pgdb_url
# update the on-disk configuration, we may pass that on to independent modules
with open(localconfig.fname, 'r') as f:
config_json = json.load(f)
config_json['Database']['host'] = podman_ip
config_json['Database']['port'] = db_port
with open(localconfig.fname, 'w') as f:
json.dump(config_json, f)
# create database factory singleton, if it didn't exist yet
db = Database(localconfig)
# wait for the database to become available
podman_services.wait_until_responsive(
timeout=60.0,
pause=0.5,
check=lambda: pgsql_test_available(session_scope))
# clear database tables so test function has a pristine database to work with
with session_scope() as session:
session.execute('DROP owned BY lkdbuser_test;')
db.downgrade('base')
db.create_tables()
# add core configuration data to the database
config_set_project_name('Test Project')
config_set_distro_tag('test')
with session_scope() as session:
# master repository, currently the only one we support
repo = ArchiveRepository('master')
session.add(repo)
# components
acpt_main = ArchiveComponent('main')
acpt_contrib = ArchiveComponent('contrib')
acpt_nonfree = ArchiveComponent('non-free')
acpt_contrib.parent_component = acpt_main
acpt_nonfree.parent_component = acpt_main
all_components = [acpt_main, acpt_contrib, acpt_nonfree]
session.add_all(all_components)
# architectures
arch_all = ArchiveArchitecture('all')
arch_amd64 = ArchiveArchitecture('amd64')
arch_arm64 = ArchiveArchitecture('arm64')
all_architectures = [arch_all, arch_amd64, arch_arm64]
session.add_all(all_architectures)
# add 'unstable' suite
suite_us = ArchiveSuite('unstable')
suite_us.repos = [repo]
suite_us.components = all_components
suite_us.architectures = all_architectures
suite_us.accept_uploads = True
session.add(suite_us)
# add 'testing' suite
suite_te = ArchiveSuite('testing')
suite_te.repos = [repo]
suite_te.components = all_components
suite_te.architectures = all_architectures
suite_te.devel_target = True
session.add(suite_te)
# add 'experimental' suite
suite_ex = ArchiveSuite('experimental')
suite_ex.repos = [repo]
suite_ex.components = all_components
suite_ex.architectures = [arch_all, arch_amd64]
suite_ex.accept_uploads = True
suite_ex.parent = suite_us
session.add(suite_ex)
return db
def _add_new_suite(session):
'''
Interactively register a new suite.
'''
from laniakea.db import ArchiveRepository, ArchiveSuite, ArchiveComponent, ArchiveArchitecture
repo = session.query(ArchiveRepository) \
.filter(ArchiveRepository.name == 'master').one()
suite_name = input_str('Adding a new suite. Please set a name')
suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == suite_name).one_or_none()
if suite:
print_note('Removing existing suite with the same name ("{}")'.format(
suite_name))
session.delete(suite)
session.commit()
suite = ArchiveSuite(suite_name)
suite.repos = [repo]
component_names = input_list('List of components for suite "{}"'.format(
suite.name))
add_main_dep = 'main' in component_names
if add_main_dep:
component_names.remove('main')
component_names.insert(0, 'main')
suite.components = []
main_component = None
for cname in component_names:
component = session.query(ArchiveComponent) \
.filter(ArchiveComponent.name == cname).one_or_none()
if not component:
component = ArchiveComponent(cname)
if add_main_dep and main_component and cname != 'main':
component.parent_component = main_component
session.add(component)
if cname == 'main':
main_component = component
suite.components.append(component)
arch_names = input_list('List of architectures for suite "{}"'.format(
suite.name))
# every suite has the "all" architecture, so add it straight away
if 'all' not in arch_names:
arch_names.insert(0, 'all')
suite.architectures = []
for aname in arch_names:
arch = session.query(ArchiveArchitecture) \
.filter(ArchiveArchitecture.name == aname).one_or_none()
if not arch:
arch = ArchiveArchitecture(aname)
session.add(arch)
suite.architectures.append(arch)
parent_suite = None
while not parent_suite:
parent_suite_name = input_str(
'Set a name of the suite this suite is an overlay to. Leave empty for primary suite. (The overlay suite must have been added first!)',
allow_empty=True)
if not parent_suite_name:
break
parent_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == parent_suite_name).one_or_none()
if not parent_suite:
print_note(
'Parent suite "{}" was not found.'.format(parent_suite_name))
suite.parent = parent_suite
session.add(suite)
session.commit()
def _add_new_suite(session):
'''
Interactively register a new suite.
'''
from laniakea.db import ArchiveRepository, ArchiveSuite, ArchiveComponent, ArchiveArchitecture
repo = session.query(ArchiveRepository) \
.filter(ArchiveRepository.name == 'master').one()
suite_name = input_str('Adding a new suite. Please set a name')
suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == suite_name).one_or_none()
if suite:
print_note('Removing existing suite with the same name ("{}")'.format(suite_name))
session.delete(suite)
session.commit()
suite = ArchiveSuite()
suite.name = suite_name
suite.repos = [repo]
component_names = input_list('List of components for suite "{}"'.format(suite.name))
add_main_dep = 'main' in component_names
if add_main_dep:
component_names.remove('main')
component_names.insert(0, 'main')
suite.components = []
main_component = None
for cname in component_names:
component = session.query(ArchiveComponent) \
.filter(ArchiveComponent.name == cname).one_or_none()
if not component:
component = ArchiveComponent(cname)
if add_main_dep and main_component and cname != 'main':
component.parent_component = main_component
session.add(component)
if cname == 'main':
main_component = component
suite.components.append(component)
arch_names = input_list('List of architectures for suite "{}"'.format(suite.name))
# every suite has the "all" architecture, so add it straight away
if 'all' not in arch_names:
arch_names.insert(0, 'all')
suite.architectures = []
for aname in arch_names:
arch = session.query(ArchiveArchitecture) \
.filter(ArchiveArchitecture.name == aname).one_or_none()
if not arch:
arch = ArchiveArchitecture(aname)
session.add(arch)
suite.architectures.append(arch)
parent_suite = None
while not parent_suite:
parent_suite_name = input_str('Set a name of the suite this suite is an overlay to. Leave empty for primary suite. (The overlay suite must have been added first!)',
allow_empty=True)
if not parent_suite_name:
break
parent_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == parent_suite_name).one_or_none()
if not parent_suite:
print_note('Parent suite "{}" was not found.'.format(parent_suite_name))
suite.parent_suite = parent_suite
session.add(suite)
session.commit()
def database(localconfig):
'''
Retrieve a pristine, empty Laniakea database connection.
This will wipe the global database, so tests using this can
never run in parallel.
'''
from laniakea.db import Database, session_scope, ArchiveRepository, ArchiveSuite, \
ArchiveComponent, ArchiveArchitecture
from laniakea.db.core import config_set_project_name, config_set_distro_tag
db = Database(localconfig) # create singleton, if it didn't exist yet
# clear database tables so test function has a pristine database to work with
with session_scope() as session:
session.execute('DROP owned BY lkdbuser_test;')
db.downgrade('base')
db.create_tables()
# add core configuration data to the database
config_set_project_name('Test Project')
config_set_distro_tag('test')
with session_scope() as session:
# master repository, currently the only one we support
repo = ArchiveRepository('master')
session.add(repo)
# components
acpt_main = ArchiveComponent('main')
acpt_contrib = ArchiveComponent('contrib')
acpt_nonfree = ArchiveComponent('non-free')
acpt_contrib.parent_component = acpt_main
acpt_nonfree.parent_component = acpt_main
all_components = [acpt_main, acpt_contrib, acpt_nonfree]
session.add_all(all_components)
# architectures
arch_all = ArchiveArchitecture('all')
arch_amd64 = ArchiveArchitecture('amd64')
arch_arm64 = ArchiveArchitecture('arm64')
all_architectures = [arch_all, arch_amd64, arch_arm64]
session.add_all(all_architectures)
# add 'unstable' suite
suite_us = ArchiveSuite()
suite_us.name = 'unstable'
suite_us.repos = [repo]
suite_us.components = all_components
suite_us.architectures = all_architectures
suite_us.accept_uploads = True
session.add(suite_us)
# add 'testing' suite
suite_te = ArchiveSuite()
suite_te.name = 'testing'
suite_te.repos = [repo]
suite_te.components = all_components
suite_te.architectures = all_architectures
suite_te.devel_target = True
session.add(suite_te)
# add 'experimental' suite
suite_ex = ArchiveSuite()
suite_ex.name = 'experimental'
suite_ex.repos = [repo]
suite_ex.components = all_components
suite_ex.architectures = [arch_all, arch_amd64]
suite_ex.accept_uploads = True
suite_ex.parent = suite_us
session.add(suite_ex)
return db