def import_package_files(self,
suite: str,
component: str,
fnames: List[str],
ignore_signature: bool = False,
add_overrides: bool = True) -> bool:
# run dak import command.
args = ['import']
if ignore_signature:
args.append('-s')
if add_overrides:
args.append('-a')
args.append(suite)
args.append(component)
args.extend(fnames)
ret, out = self._run_dak(args, check=False)
if ret != 0:
raise Exception('Unable to import package files \'{}\': {}'.format(
' '.join(fnames), out))
log.info('Imported \'{}\' into \'{}/{}\'.'.format(
' '.join([os.path.basename(f) for f in fnames]), suite, component))
return True
def _run_migration_internal(self, session, suites_from: List[ArchiveSuite],
suite_to: ArchiveSuite):
mi_wspace = self._get_migrate_workspace(suites_from, suite_to)
britney_conf = os.path.join(mi_wspace, 'britney.conf')
if not os.path.isfile(britney_conf):
log.warning(
'No Britney config for migration run "{}" - maybe the configuration was not yet updated?'
.format(self._get_migration_name(suites_from, suite_to)))
return None
log.info('Migration run for "{}"'.format(
self._get_migration_name(suites_from, suite_to)))
# ensure prerequisites are met and Britney is fed with all the data it needs
self._prepare_source_data(session, mi_wspace, suites_from, suite_to)
self._create_faux_packages(session, mi_wspace, suites_from, suite_to)
self._collect_urgencies(mi_wspace)
self._setup_dates(mi_wspace)
self._setup_various(mi_wspace, suites_from, suite_to)
# execute the migration tester
self._britney.run(mi_wspace, britney_conf)
# tell dak to import the new data (overriding the target suite)
dak = DakBridge()
heidi_result = self._postprocess_heidi_file(mi_wspace)
ret = dak.set_suite_to_britney_result(suite_to.name, heidi_result)
if not ret:
return None
res = self._retrieve_excuses(session, mi_wspace, suites_from, suite_to)
return res
def _setup_dates(self, mi_wspace: str):
dates_policy_file = os.path.join(mi_wspace, 'state',
'age-policy-dates')
if os.path.isfile(dates_policy_file):
return
log.info('Writing dates policy file.')
# just make an empty file for now
with open(dates_policy_file, 'w') as f:
f.write('\n')
def remove_package(self, package_name: str, suite_name: str) -> bool:
''' Remove a package from a specified suite. '''
log.info('Removing \'{}\' from \'{}\''.format(package_name,
suite_name))
# actually remove a package
args = [
'rm', '-m', 'RID: Removed from Debian', '-C', 'janitor@dak', '-s',
suite_name, package_name
]
ret, out = self._run_dak(args, 'y\n', check=False)
if ret != 0:
raise Exception(
'Unable to remove package \'{}\' from \'{}\': {}'.format(
package_name, suite_name, out))
return True
def _collect_urgencies(self, mi_wspace: str):
urgencies = ''
for subdir, dirs, files in os.walk(
self._lconf.archive_urgencies_export_dir):
for fbasename in files:
fname = os.path.join(subdir, fbasename)
if not os.path.isfile(fname):
continue
if not fbasename.startswith('install-urgencies'):
continue
log.debug('Reading urgencies from {}'.format(fname))
with open(fname, 'r') as f:
urgencies = urgencies + f.read()
log.info('Writing urgency policy file.')
urgency_policy_file = os.path.join(mi_wspace, 'state',
'age-policy-urgencies')
with open(urgency_policy_file, 'w') as f:
f.write(urgencies)
def set_suite_to_britney_result(self, suite_name: str,
heidi_file: str) -> bool:
'''
Import a Britney result (HeidiResult file) into the dak database.
This will *override* all existing package information in the target suite.
Use this command with great care!
'''
# do some sanity checks
if not os.path.isfile(heidi_file):
log.warning(
'Britney result not imported: File "{}" does not exist.'.
format(heidi_file))
return False
# an empty file might cause us to delete the whole repository contents.
# this is a safeguard against that.
heidi_data = None
with open(heidi_file, 'r') as f:
heidi_data = f.read().strip()
if not heidi_data:
log.warning(
'Stopped Britney result import: File "{}" is empty.'.format(
heidi_file))
return True
log.info('Importing britney result from {}'.format(heidi_file))
# run dak control-suite command.
args = ['control-suite']
args.extend(['--set', suite_name, '--britney'])
ret, out = self._run_dak(args, input_data=heidi_data, check=False)
if ret != 0:
raise Exception('Unable apply Britney result to "{}": {}'.format(
suite_name, out))
log.info('Updated packages in "{}" based on Britney result.'.format(
suite_name))
return True
def _setup_various(self, mi_wspace: str, suites_source: List[ArchiveSuite],
suite_target: ArchiveSuite):
# set up some random files which we do not use at all currently
for suite in suites_source:
rcbugs_policy_file_u = os.path.join(
mi_wspace, 'state', 'rc-bugs-{}'.format(suite.name))
if not os.path.isfile(rcbugs_policy_file_u):
log.info('Writing RC bugs policy file (source).')
# just make an empty file for now
with open(rcbugs_policy_file_u, 'w') as f:
f.write('')
rcbugs_policy_file_t = os.path.join(
mi_wspace, 'state', 'rc-bugs-{}'.format(suite_target.name))
if not os.path.isfile(rcbugs_policy_file_t):
log.info('Writing RC bugs policy file (target).')
# just make an empty file for now
with open(rcbugs_policy_file_t, 'w') as f:
f.write('')
# there is no support for Piuparts yet, but Britney crashes without these files
piupats_dummy_json = (
'{"_id": "Piuparts Package Test Results Summary", '
'"_version": "1.0", '
'"packages": {}}\n')
for suite in suites_source:
piuparts_file_u = os.path.join(
mi_wspace, 'state',
'piuparts-summary-{}.json'.format(suite.name))
if not os.path.isfile(piuparts_file_u):
log.info('Writing Piuparts summary file (source).')
# just make an empty file for now
with open(piuparts_file_u, 'w') as f:
f.write(piupats_dummy_json)
piuparts_file_t = os.path.join(
mi_wspace, 'state',
'piuparts-summary-{}.json'.format(suite_target.name))
if not os.path.isfile(piuparts_file_t):
log.info('Writing Piuparts summary file (target).')
# just make an empty file for now
with open(piuparts_file_t, 'w') as f:
f.write(piupats_dummy_json)
def update_config(self):
'''
Update configuration and distribution
copy of Britney
'''
log.info('Updating configuration')
with session_scope() as session:
for mentry in session.query(SpearsMigrationEntry).all():
si_res = self._suites_from_migration_entry(session, mentry)
if si_res['error']:
continue
suites_from = si_res['from']
suite_to = si_res['to']
assert len(suites_from) >= 1
log.info('Refreshing Britney config for "{}"'.format(
self._get_migration_name(suites_from, suite_to)))
mi_wspace = self._get_migrate_workspace(suites_from, suite_to)
bc = BritneyConfig(mi_wspace)
bc.set_archive_paths(
self._get_source_suite_dists_dir(mi_wspace, suites_from),
os.path.join(self._lconf.archive_root_dir, 'dists',
suite_to.name))
bc.set_components([c.name for c in suite_to.components])
bc.set_architectures([a.name for a in suite_to.architectures])
bc.set_delays(mentry.delays)
hints = session.query(SpearsHint).filter(
SpearsHint.migration_id ==
mentry.make_migration_id()).all()
bc.set_hints(hints)
bc.save()
log.info('Updating Britney')
self._britney.update_dist()
return True
def install_trusted_keyfile(options):
''' Install a public key to trust a client node. '''
from shutil import copyfile
if not options.name:
print('No name for this public key / client given!')
sys.exit(1)
source_keyfile = options.keyfile
if not source_keyfile:
print('No public key file given!')
sys.exit(1)
if not os.path.isfile(source_keyfile):
print('Public key file "{}" was not found.'.format(source_keyfile))
sys.exit(1)
pub_key = None
sec_key = None
try:
pub_key, sec_key = zmq.auth.load_certificate(source_keyfile)
except ValueError:
pass
if not pub_key:
log.info('The given keyfile does not contain a public ZCurve key!')
if sec_key:
print('')
print(
'/!\\ The current file contains a secret ZCurve key. This file should never leave the client machine it is installed on.'
)
print('')
_, verify_key = keyfile_read_verify_key(source_keyfile)
if not verify_key:
log.info('The given keyfile does not contain a verification key!')
if not verify_key and not pub_key:
log.error(
'The keyfile does not contain either a public encryption, nor a verification key. Can not continue.'
)
sys.exit(4)
_, sign_key = keyfile_read_signing_key(source_keyfile)
if sign_key:
print('')
print(
'/!\\ The current file contains a secret signing key. This file should never leave the client machine it is installed on.'
)
print('')
lconf = LocalConfig()
target_keyfile = os.path.join(lconf.trusted_curve_keys_dir,
'{}.pub.key'.format(options.name))
if os.path.isfile(target_keyfile) and not options.force:
print(
'We already trust a key for "{}" on this machine. You can override the existing one by specifying "--force".'
.format(options.name))
sys.exit(2)
try:
copyfile(source_keyfile, target_keyfile)
except Exception as e:
print('Failed to install new public key as {}: {}'.format(
target_keyfile, str(e)))
sys.exit(3)
print('Installed as {}'.format(target_keyfile))
def autosync(self, session, sync_conf, remove_cruft: bool = True):
''' Synchronize all packages that are newer '''
self._synced_source_pkgs = []
active_src_pkgs = [] # source packages which should have their binary packages updated
res_issues = []
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
sync_conf = session.query(SynchrotronConfig) \
.join(SynchrotronConfig.destination_suite) \
.join(SynchrotronConfig.source) \
.filter(ArchiveSuite.name == self._target_suite_name,
SynchrotronSource.suite_name == self._source_suite_name).one_or_none()
for component in target_suite.components:
dest_pkg_map = self._get_target_source_packages(component.name)
# The source package lists contains many different versions, some source package
# versions are explicitly kept for GPL-compatibility.
# Sometimes a binary package migrates into another suite, dragging a newer source-package
# that it was built against with itslf into the target suite.
# These packages then have a source with a high version number, but might not have any
# binaries due to them migrating later.
# We need to care for that case when doing binary syncs (TODO: and maybe safeguard against it
# when doing source-only syncs too?), That's why we don't filter out the newest packages in
# binary-sync-mode.
if sync_conf.sync_binaries:
src_pkg_range = self._source_repo.source_packages(ArchiveSuite(self._source_suite_name), component)
else:
src_pkg_range = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component).values()
for spkg in src_pkg_range:
# ignore blacklisted packages in automatic sync
if spkg.name in self._sync_blacklist:
continue
dpkg = dest_pkg_map.get(spkg.name)
if dpkg:
if version_compare(dpkg.version, spkg.version) >= 0:
log.debug('Skipped sync of {}: Target version \'{}\' is equal/newer than source version \'{}\'.'
.format(spkg.name, dpkg.version, spkg.version))
continue
# check if we have a modified target package,
# indicated via its Debian revision, e.g. "1.0-0tanglu1"
if self._distro_tag in version_revision(dpkg.version):
log.info('Not syncing {}/{}: Destination has modifications (found {}).'
.format(spkg.name, spkg.version, dpkg.version))
# add information that this package needs to be merged to the issue list
issue = SynchrotronIssue()
issue.package_name = spkg.name
issue.source_version = spkg.version
issue.target_version = dpkg.version
issue.kind = SynchrotronIssueKind.MERGE_REQUIRED
res_issues.append(issue)
continue
# sync source package
# the source package must always be known to dak first
ret = self._import_source_package(spkg, component.name)
if not ret:
return False, []
# a new source package is always active and needs it's binary packages synced, in
# case we do binary syncs.
active_src_pkgs.append(spkg)
# all packages in the target distribution are considered active, as long as they don't
# have modifications.
for spkg in dest_pkg_map.values():
if self._distro_tag in version_revision(spkg.version):
active_src_pkgs.append(spkg)
# import binaries as well. We test for binary updates for all available active source packages,
# as binNMUs might have happened in the source distribution.
# (an active package in this context is any source package which doesn't have modifications in the
# target distribution)
ret = self._import_binaries_for_source(sync_conf, target_suite, component.name, active_src_pkgs)
if not ret:
return False, []
# test for cruft packages
target_pkg_index = {}
for component in target_suite.components:
dest_pkg_map = self._get_repo_source_package_map(self._target_repo,
target_suite.name,
component.name)
for pkgname, pkg in dest_pkg_map.items():
target_pkg_index[pkgname] = pkg
# check which packages are present in the target, but not in the source suite
for component in target_suite.components:
src_pkg_map = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component.name)
for pkgname in src_pkg_map.keys():
target_pkg_index.pop(pkgname, None)
# remove cruft packages
if remove_cruft:
for pkgname, dpkg in target_pkg_index.items():
dpkg_ver_revision = version_revision(dpkg.version, False)
# native packages are never removed
if not dpkg_ver_revision:
continue
# check if the package is intoduced as new in the distro, in which case we won't remove it
if dpkg_ver_revision.startswith('0' + self._distro_tag):
continue
# if this package was modified in the target distro, we will also not remove it, but flag it
# as "potential cruft" for someone to look at.
if self._distro_tag in dpkg_ver_revision:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.MAYBE_CRUFT
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
res_issues.append(issue)
continue
# check if we can remove this package without breaking stuff
if self._dak.package_is_removable(dpkg.name, target_suite.name):
# try to remove the package
try:
self._dak.remove_package(dpkg.name, target_suite.name)
except Exception as e:
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = str(e)
res_issues.append(issue)
else:
# looks like we can not remove this
issue = SynchrotronIssue()
issue.kind = SynchrotronIssueKind.REMOVAL_FAILED
issue.source_suite = self._source_suite_name
issue.target_suite = self._target_suite_name
issue.package_name = dpkg.name
issue.source_version = None
issue.target_version = dpkg.version
issue.details = 'This package can not be removed without breaking other packages. It needs manual removal.'
res_issues.append(issue)
self._publish_synced_spkg_events(sync_conf.source.os_name,
sync_conf.source.suite_name,
sync_conf.destination_suite.name,
False)
return True, res_issues
def sync_packages(self, component: str, pkgnames: List[str], force: bool = False):
self._synced_source_pkgs = []
with session_scope() as session:
sync_conf = session.query(SynchrotronConfig) \
.join(SynchrotronConfig.destination_suite) \
.join(SynchrotronConfig.source) \
.filter(ArchiveSuite.name == self._target_suite_name,
SynchrotronSource.suite_name == self._source_suite_name).one_or_none()
if not sync_conf:
log.error('Unable to find a sync config for this source/destination combination.')
return False
if not sync_conf.sync_enabled:
log.error('Can not synchronize package: Synchronization is disabled for this configuration.')
return False
target_suite = session.query(ArchiveSuite) \
.filter(ArchiveSuite.name == self._target_suite_name).one()
dest_pkg_map = self._get_target_source_packages(component)
src_pkg_map = self._get_repo_source_package_map(self._source_repo,
self._source_suite_name,
component)
for pkgname in pkgnames:
spkg = src_pkg_map.get(pkgname)
dpkg = dest_pkg_map.get(pkgname)
if not spkg:
log.info('Can not sync {}: Does not exist in source.'.format(pkgname))
continue
if pkgname in self._sync_blacklist:
log.info('Can not sync {}: The package is blacklisted.'.format(pkgname))
continue
if dpkg:
if version_compare(dpkg.version, spkg.version) >= 0:
if force:
log.warning('{}: Target version \'{}\' is newer/equal than source version \'{}\'.'
.format(pkgname, dpkg.version, spkg.version))
else:
log.info('Can not sync {}: Target version \'{}\' is newer/equal than source version \'{}\'.'
.format(pkgname, dpkg.version, spkg.version))
continue
if not force:
if self._distro_tag in version_revision(dpkg.version):
log.error('Not syncing {}/{}: Destination has modifications (found {}).'
.format(spkg.name, spkg.version, dpkg.version))
continue
# sync source package
# the source package must always be known to dak first
ret = self._import_source_package(spkg, component)
if not ret:
return False
ret = self._import_binaries_for_source(sync_conf, target_suite, component, self._synced_source_pkgs, force)
# TODO: Analyze the input, fetch the packages from the source distribution and
# import them into the target in their correct order.
# Then apply the correct, synced override from the source distro.
self._publish_synced_spkg_events(sync_conf.source.os_name,
sync_conf.source.suite_name,
sync_conf.destination_suite.name,
force)
return ret
def update_appstream_data(session, local_repo, repo, suite, component, arch):
'''
Import AppStream metadata about software components and associate it with the
binary packages the data belongs to.
'''
if arch.name == 'all':
# arch:all has no AppStream components, those are always associated with an architecture
# and are included in arch-specific files (even if the package they belong to is arch:all)
return
arch_all = session.query(ArchiveArchitecture) \
.filter(ArchiveArchitecture.name == 'all').one()
yaml_fname = local_repo.index_file(
suite,
os.path.join(component.name, 'dep11',
'Components-{}.yml.xz'.format(arch.name)))
if not yaml_fname:
return
cidmap_fname = local_repo.index_file(
suite,
os.path.join(component.name, 'dep11',
'CID-Index-{}.json.gz'.format(arch.name)),
check=False)
if not cidmap_fname:
return
with gzip.open(cidmap_fname, 'rb') as f:
cid_map = json.loads(f.read())
with lzma.open(yaml_fname, 'r') as f:
yaml_data = str(f.read(), 'utf-8')
mdata = AppStream.Metadata()
mdata.set_locale('ALL')
mdata.set_format_style(AppStream.FormatStyle.COLLECTION)
mdata.set_parse_flags(AppStream.ParseFlags.IGNORE_MEDIABASEURL)
mdata.parse(yaml_data, AppStream.FormatKind.YAML)
cpts = mdata.get_components()
if len(cpts) == 0:
return
log.debug('Found {} software components in {}/{}'.format(
len(cpts), suite.name, component.name))
# create context for direct serialization to collection XML chunk
context = AppStream.Context()
context.set_locale('ALL')
context.set_style(AppStream.FormatStyle.COLLECTION)
for cpt in cpts:
cpt.set_active_locale('C')
pkgname = cpt.get_pkgname()
if not pkgname:
# we skip these for now, web-apps have no package assigned - we might need a better way to map
# those to their packages, likely with an improved appstream-generator integration
log.debug(
'Found DEP-11 component without package name in {}/{}: {}'.
format(suite.name, component.name, cpt.get_id()))
continue
# fetch package this component belongs to
bin_pkg = session.query(BinaryPackage) \
.filter(BinaryPackage.name == pkgname) \
.filter(BinaryPackage.repo_id == repo.id) \
.filter(BinaryPackage.architecture_id.in_((arch.id, arch_all.id))) \
.filter(BinaryPackage.component_id == component.id) \
.filter(BinaryPackage.suites.any(ArchiveSuite.id == suite.id)) \
.order_by(BinaryPackage.version.desc()).first()
if not bin_pkg:
log.info('Found orphaned DEP-11 component in {}/{}: {}'.format(
suite.name, component.name, cpt.get_id()))
continue
dcpt = SoftwareComponent()
dcpt.kind = int(cpt.get_kind())
dcpt.cid = cpt.get_id()
dcpt.xml = cpt.to_xml_data(context)
dcpt.gcid = cid_map.get(dcpt.cid)
if not dcpt.gcid:
log.info('Found DEP-11 component without GCID in {}/{}: {}'.format(
suite.name, component.name, cpt.get_id()))
# create UUID for this component (based on GCID or XML data)
dcpt.update_uuid()
existing_dcpt = session.query(SoftwareComponent) \
.filter(SoftwareComponent.uuid == dcpt.uuid).one_or_none()
if existing_dcpt:
if bin_pkg in existing_dcpt.bin_packages:
continue # the binary package is already registered with this component
existing_dcpt.bin_packages.append(bin_pkg)
continue # we already have this component, no need to add it again
# add new software component to database
dcpt.name = cpt.get_name()
dcpt.summary = cpt.get_summary()
dcpt.description = cpt.get_description()
for icon in cpt.get_icons():
if icon.get_kind() == AppStream.IconKind.CACHED:
dcpt.icon_name = icon.get_name()
break
dcpt.project_license = cpt.get_project_license()
dcpt.developer_name = cpt.get_developer_name()
# test for free software
dcpt.is_free = False
if not dcpt.project_license:
# We have no license set.
# If we are in the 'main' component, we
# assume we have free software
if bin_pkg.component.name == 'main':
dcpt.is_free = True
else:
# have AppStream test the SPDX license expression for free software
dcpt.is_free = AppStream.license_is_free_license(
dcpt.project_license)
dcpt.categories = []
for cat in cpt.get_categories():
dcpt.categories.append(cat)
dcpt.bin_packages = [bin_pkg]
session.add(dcpt)
log.debug('Added new software component \'{}\' to database'.format(
dcpt.cid))
session.commit()
def _create_faux_packages(self, session, mi_wspace: str,
suites_source: List[ArchiveSuite],
suite_target: ArchiveSuite):
'''
If we have a partial source and target suite, we need to let Britney know about the
parent packages somehow.
At the moment, we simply abuse the FauxPackages system for that.
'''
# we don't support more than one source suite for this feature at the moment
if len(suites_source) > 1:
log.info(
'Not auto-generating faux packages: Multiple suites set as sources.'
)
return
suite_source = suites_source[0]
if suite_source.parent and suite_target.parent:
log.info(
'Creating faux-packages to aid resolving of partial suites.')
else:
log.info(
'No auto-generating faux packages: No source and target suite parents, generation is unnecessary.'
)
return
existing_pkg_arch_set = set()
log.debug(
'Creating index of valid packages that do not need a faux package.'
)
# we need repository information to only generate faux packages if a package doesn't exist
# in our source suite(s) already
repo = self._get_local_repo(session)
for suite in suites_source:
esuite = session.query(ArchiveSuite) \
.options(joinedload(ArchiveSuite.components)) \
.options(joinedload(ArchiveSuite.architectures)) \
.filter(ArchiveSuite.id == suite.id).one()
session.expunge(
esuite
) # we don't want packages accidentally added to the database here
for component in esuite.components:
for arch in esuite.architectures:
aname = arch.name
for bpkg in repo.binary_packages(esuite, component, arch):
existing_pkg_arch_set.add(aname + ':' + bpkg.name)
for spkg in repo.source_packages(esuite, component):
existing_pkg_arch_set.add(aname + ':' + spkg.name)
archive_root_dir = self._lconf.archive_root_dir
fauxpkg_fname = os.path.join(mi_wspace, 'input', 'faux-packages')
log.debug('Generating faux packages list')
fauxpkg_data = {}
for component in suite_target.parent.components:
for installer_dir in ['', 'debian-installer']:
for arch in suite_target.parent.architectures:
pfile = os.path.join(archive_root_dir, 'dists',
suite_target.parent.name,
component.name, installer_dir,
'binary-{}'.format(arch.name),
'Packages.xz')
if not os.path.isfile(pfile):
continue
log.debug('Reading data for faux packages list: {}'.format(
pfile))
with TagFile(pfile) as tf:
for e in tf:
pkgname = e['Package']
pkgversion = e['Version']
pkgarch = e['Architecture']
pkid = '{}-{}-{}'.format(pkgname, pkgversion,
pkgarch)
if pkid in fauxpkg_data:
continue
pkgname_arch = pkgarch + ':' + pkgname
if pkgname_arch in existing_pkg_arch_set:
continue
provides = e.get('Provides', '')
data = 'Package: {}\nVersion: {}'.format(
pkgname, pkgversion)
if pkgarch and pkgarch != 'all':
data = data + '\nArchitecture: {}'.format(
pkgarch)
if provides:
data = data + '\nProvides: {}'.format(provides)
if component.name != 'main':
data = data + '\nComponent: {}'.format(
component.name)
fauxpkg_data[pkid] = data
# FIXME: We shouldn't have to special-case this :any case,
# rather Britney should do the right thing and recognize this
# notation for faux-packages. But until that is fixed
# properly and since a dependency on python3:any is so common, we
# will work around this issue
if pkgname == 'python3':
pkid = '{}-{}-{}'.format(
'python3:any', pkgversion, pkgarch)
if pkid in fauxpkg_data:
continue
fauxpkg_data[pkid] = data.replace(
'Package: python3\n',
'Package: python3:any\n')
with open(fauxpkg_fname, 'w') as f:
for segment in fauxpkg_data.values():
f.write(segment + '\n\n')
def command_autosync(options):
''' Automatically synchronize packages '''
with session_scope() as session:
sync_sources = session.query(SynchrotronSource).all()
autosyncs = session.query(SynchrotronConfig) \
.filter(SynchrotronConfig.sync_enabled == True) \
.filter(SynchrotronConfig.sync_auto_enabled == True).all() # noqa: E712
for autosync in autosyncs:
log.info('Synchronizing packages from {}/{} with {}'.format(
autosync.source.os_name, autosync.source.suite_name,
autosync.destination_suite.name))
emitter = EventEmitter(LkModule.SYNCHROTRON)
engine = SyncEngine(autosync.destination_suite.name,
autosync.source.suite_name)
ret, issue_data = engine.autosync(session, autosync,
autosync.auto_cruft_remove)
if not ret:
sys.exit(2)
return
existing_sync_issues = {}
for ssource in sync_sources:
all_issues = session.query(SynchrotronIssue) \
.filter(SynchrotronIssue.source_suite == ssource.suite_name,
SynchrotronIssue.target_suite == autosync.destination_suite.name,
SynchrotronIssue.config_id == autosync.id) \
.all()
for eissue in all_issues:
eid = '{}-{}-{}:{}'.format(eissue.package_name,
eissue.source_version,
eissue.target_version,
str(eissue.kind))
existing_sync_issues[eid] = eissue
for info in issue_data:
eid = '{}-{}-{}:{}'.format(info.package_name,
info.source_version,
info.target_version, str(info.kind))
issue = existing_sync_issues.pop(eid, None)
if issue:
# the issue already exists, so we just update it
new_issue = False
else:
new_issue = True
issue = info
issue.config = autosync
if new_issue:
session.add(issue)
data = {
'name': issue.package_name,
'src_os': autosync.source.os_name,
'suite_src': issue.source_suite,
'suite_dest': issue.target_suite,
'version_src': issue.source_version,
'version_dest': issue.target_version,
'kind': str(issue.kind)
}
emitter.submit_event('new-autosync-issue', data)
for eissue in existing_sync_issues.values():
session.delete(eissue)
data = {
'name': eissue.package_name,
'src_os': autosync.source.os_name,
'suite_src': eissue.source_suite,
'suite_dest': eissue.target_suite,
'version_src': eissue.source_version,
'version_dest': eissue.target_version,
'kind': str(eissue.kind)
}
emitter.submit_event('resolved-autosync-issue', data)
def command_autosync(options):
''' Automatically synchronize packages '''
with session_scope() as session:
autosyncs = session.query(SynchrotronConfig).filter(SynchrotronConfig.sync_enabled == True) \
.filter(SynchrotronConfig.sync_auto_enabled == True).all() # noqa: E712
bconf, sconf = get_sync_config()
blacklist_pkgnames = get_package_blacklist() # the blacklist is global for now
for autosync in autosyncs:
incoming_suite = get_suiteinfo_for_suite(autosync.destination_suite)
sconf.syncBinaries = autosync.sync_binaries
sconf.source.defaultSuite = autosync.source.suite_name
sconf.source.repoUrl = autosync.source.repo_url
log.info('Synchronizing packages from {}/{} with {}'.format(autosync.source.os_name, autosync.source.suite_name,
autosync.destination_suite.name))
emitter = EventEmitter(LkModule.SYNCHROTRON)
engine = SyncEngine(bconf, sconf, incoming_suite)
engine.setBlacklist(blacklist_pkgnames)
ret, issue_data = engine.autosync()
publish_synced_spkg_events(engine,
autosync.source.os_name,
autosync.source.suite_name,
autosync.destination_suite.name,
emitter=emitter)
if not ret:
sys.exit(2)
return
for ssuite in sconf.source.suites:
session.query(SynchrotronIssue) \
.filter(SynchrotronIssue.source_suite == ssuite.name,
SynchrotronIssue.target_suite == incoming_suite.name,
SynchrotronIssue.config_id == autosync.id) \
.delete()
for info in issue_data:
issue = SynchrotronIssue()
issue.config = autosync
issue.kind = SynchrotronIssueKind(info.kind)
issue.package_name = info.packageName
issue.source_suite = info.sourceSuite
issue.target_suite = info.targetSuite
issue.source_version = info.sourceVersion
issue.target_version = info.targetVersion
issue.details = info.details
session.add(issue)
data = {'name': issue.package_name,
'src_os': autosync.source.os_name,
'src_suite': issue.source_suite,
'dest_suite': issue.target_suite,
'src_version': issue.source_version,
'dest_version': issue.target_version,
'kind': str(issue.kind)}
emitter.submit_event('autosync-issue', data)
def command_autosync(options):
''' Automatically synchronize packages '''
with session_scope() as session:
autosyncs = session.query(SynchrotronConfig).filter(SynchrotronConfig.sync_enabled == True) \
.filter(SynchrotronConfig.sync_auto_enabled == True).all() # noqa: E712
bconf, sconf = get_sync_config()
blacklist_pkgnames = get_package_blacklist(
) # the blacklist is global for now
for autosync in autosyncs:
incoming_suite = get_suiteinfo_for_suite(
autosync.destination_suite)
sconf.syncBinaries = autosync.sync_binaries
sconf.source.defaultSuite = autosync.source.suite_name
sconf.source.repoUrl = autosync.source.repo_url
log.info('Synchronizing packages from {}/{} with {}'.format(
autosync.source.os_name, autosync.source.suite_name,
autosync.destination_suite.name))
emitter = EventEmitter(LkModule.SYNCHROTRON)
engine = SyncEngine(bconf, sconf, incoming_suite)
engine.setBlacklist(blacklist_pkgnames)
ret, issue_data = engine.autosync()
publish_synced_spkg_events(engine,
autosync.source.os_name,
autosync.source.suite_name,
autosync.destination_suite.name,
emitter=emitter)
if not ret:
sys.exit(2)
return
existing_sync_issues = {}
for ssuite in sconf.source.suites:
all_issues = session.query(SynchrotronIssue) \
.filter(SynchrotronIssue.source_suite == ssuite.name,
SynchrotronIssue.target_suite == incoming_suite.name,
SynchrotronIssue.config_id == autosync.id) \
.all()
for eissue in all_issues:
eid = '{}-{}-{}:{}'.format(eissue.package_name,
eissue.source_version,
eissue.target_version,
str(eissue.kind))
existing_sync_issues[eid] = eissue
for info in issue_data:
issue_kind = SynchrotronIssueKind(info.kind)
eid = '{}-{}-{}:{}'.format(info.packageName,
info.sourceVersion,
info.targetVersion, str(issue_kind))
issue = existing_sync_issues.pop(eid, None)
if issue:
# the issue already exists, so we just update it
new_issue = False
else:
new_issue = True
issue = SynchrotronIssue()
issue.config = autosync
issue.package_name = info.packageName
issue.source_version = info.sourceVersion
issue.target_version = info.targetVersion
issue.kind = issue_kind
issue.source_suite = info.sourceSuite
issue.target_suite = info.targetSuite
issue.details = info.details
if new_issue:
session.add(issue)
data = {
'name': issue.package_name,
'src_os': autosync.source.os_name,
'suite_src': issue.source_suite,
'suite_dest': issue.target_suite,
'version_src': issue.source_version,
'version_dest': issue.target_version,
'kind': str(issue.kind)
}
emitter.submit_event('new-autosync-issue', data)
for eissue in existing_sync_issues.values():
session.delete(eissue)
data = {
'name': eissue.package_name,
'src_os': autosync.source.os_name,
'suite_src': eissue.source_suite,
'suite_dest': eissue.target_suite,
'version_src': eissue.source_version,
'version_dest': eissue.target_version,
'kind': str(eissue.kind)
}
emitter.submit_event('resolved-autosync-issue', data)
