def eject_from_mash(self, update, reason):
update.locked = False
text = '%s ejected from the push because %r' % (update.title, reason)
log.warn(text)
update.comment(self.db, text, author=u'bodhi')
# Remove the pending tag as well
if update.request is UpdateRequest.stable:
update.remove_tag(update.release.pending_stable_tag,
koji=buildsys.get_session())
elif update.request is UpdateRequest.testing:
update.remove_tag(update.release.pending_testing_tag,
koji=buildsys.get_session())
update.request = None
if update.title in self.state['updates']:
self.state['updates'].remove(update.title)
if update in self.updates:
self.updates.remove(update)
notifications.publish(
topic="update.eject",
msg=dict(
repo=self.id,
update=update,
reason=reason,
request=self.request,
release=self.release,
agent=self.agent,
),
force=True,
)
def get_rpm_header(nvr, tries=0):
"""
Get the rpm header for a given build.
Args:
nvr (basestring): The name-version-release string of the build you want headers for.
tries (int): The number of attempts that have been made to retrieve the nvr so far. Defaults
to 0.
Returns:
dict: A dictionary mapping RPM header names to their values, as returned by the Koji client.
"""
tries += 1
headers = [
'name', 'summary', 'version', 'release', 'url', 'description',
'changelogtime', 'changelogname', 'changelogtext',
]
rpmID = nvr + '.src'
koji_session = buildsys.get_session()
try:
result = koji_session.getRPMHeaders(rpmID=rpmID, headers=headers)
except Exception as e:
msg = "Failed %i times to get rpm header data from koji for %s: %s"
log.warning(msg % (tries, nvr, str(e)))
if tries < 3:
# Try again...
return get_rpm_header(nvr, tries=tries)
else:
# Give up for good and re-raise the failure...
raise
if result:
return result
raise ValueError("No rpm headers found in koji for %r" % nvr)
def main():
"""Check build tags and sign those we missed."""
db_factory = transactional_session_maker()
older_than = datetime.utcnow() - timedelta(
days=config.get('check_signed_builds_delay'))
with db_factory() as session:
updates = models.Update.query.filter(
models.Update.status == models.UpdateStatus.pending).filter(
models.Update.release_id == models.Release.id).filter(
models.Release.state.in_([
models.ReleaseState.current,
models.ReleaseState.pending,
models.ReleaseState.frozen,
])).all()
if len(updates) == 0:
log.debug('No stuck Updates found')
return
kc = buildsys.get_session()
stuck_builds = []
for update in updates:
# Let Bodhi have its times
if update.date_submitted >= older_than:
continue
builds = update.builds
# Clean Updates with no builds
if len(builds) == 0:
log.debug(f'Obsoleting empty update {update.alias}')
update.obsolete(session)
session.flush()
continue
pending_signing_tag = update.release.pending_signing_tag
pending_testing_tag = update.release.pending_testing_tag
for build in builds:
if build.signed:
log.debug(f'{build.nvr} already marked as signed')
continue
build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]
if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:
# Our composer missed the message that the build got signed
log.debug(f'Changing signed status of {build.nvr}')
build.signed = True
if pending_signing_tag in build_tags and pending_testing_tag not in build_tags:
# autosign missed the message that the build is waiting to be signed
log.debug(
f'{build.nvr} is stuck waiting to be signed, let\'s try again'
)
stuck_builds.append(build.nvr)
session.flush()
if stuck_builds:
kc.multicall = True
for b in stuck_builds:
kc.untagBuild(pending_signing_tag, b, force=True)
kc.multiCall()
for b in stuck_builds:
kc.tagBuild(pending_signing_tag, b, force=True)
kc.multiCall()
def _perform_tag_actions(self):
koji = buildsys.get_session()
for i, batches in enumerate([(self.add_tags_sync, self.move_tags_sync),
(self.add_tags_async, self.move_tags_async)]):
add, move = batches
if i == 0:
koji.multicall = False
else:
koji.multicall = True
for action in add:
tag, build = action
self.log.info("Adding tag %s to %s" % (tag, build))
koji.tagBuild(tag, build, force=True)
for action in move:
from_tag, to_tag, build = action
self.log.info('Moving %s from %s to %s' % (
build, from_tag, to_tag))
koji.moveBuild(from_tag, to_tag, build, force=True)
if i != 0:
results = koji.multiCall()
failed_tasks = buildsys.wait_for_tasks([task[0] for task in results],
koji, sleep=15)
if failed_tasks:
raise Exception("Failed to move builds: %s" % failed_tasks)
def __call__(self, message: fedora_messaging.api.Message) -> None:
"""Comment on related update.
Args:
message: The message we are processing.
"""
body = message.body
missing = []
for mandatory in ('contact', 'run', 'artifact', 'pipeline', 'test',
'generated_at', 'version'):
if mandatory not in body:
missing.append(mandatory)
if missing:
log.debug(f"Received incomplete CI message. Missing: {', '.join(missing)}")
return
nvr = body['artifact'].get('nvr', None)
build_id = body['pipeline'].get('id', None)
run_url = body['run'].get('url', None)
koji = buildsys.get_session()
if not nvr and build_id:
kbuildinfo = koji.getBuild(build_id)
log.debug(kbuildinfo)
if not kbuildinfo:
log.debug(f"Can't find Koji build with id '{build_id}'.")
return
elif 'nvr' not in kbuildinfo:
log.debug(f"Koji build info with id '{build_id}' doesn't contain 'nvr'.")
return
else:
nvr = kbuildinfo['nvr']
elif not nvr and not build_id:
log.debug("Received incomplete CI message. Missing: 'artifact.nvr', 'pipeline.id'.")
return
with self.db_factory() as dbsession:
build = dbsession.query(Build).filter_by(nvr=nvr).first()
if not build:
log.debug(f"Can't get build for '{nvr}'.")
return
if build.update:
if build.update.from_tag:
log.debug("Update is created from tag. Skipping comment.")
return
comment = "CI testing started."
if run_url:
comment = f"CI testing started: '{run_url}'."
build.update.comment(
dbsession, text=comment, author='bodhi', email_notification=False)
else:
log.debug(f"No update in Bodhi for '{nvr}'. Nothing to comment on.")
return
log.debug("Committing changes to the database.")
dbsession.commit()
def main(argv=sys.argv):
"""
Remove the pending and testing tags from branched updates.
Args:
argv (list): The arguments passed to the script. Defaults to sys.argv.
"""
if len(argv) != 2:
usage(argv)
config_uri = argv[1]
setup_logging(config_uri)
log = logging.getLogger(__name__)
settings = get_appsettings(config_uri)
initialize_db(settings)
db = Session()
koji = buildsys.get_session()
one_day = timedelta(days=1)
now = datetime.utcnow()
try:
for release in db.query(Release).filter_by(
state=ReleaseState.pending).all():
log.info(release.name)
for update in db.query(Update).filter_by(
release=release, status=UpdateStatus.stable).all():
if now - update.date_stable > one_day:
for build in update.builds:
tags = build.get_tags()
stable_tag = release.dist_tag
testing_tag = release.testing_tag
pending_signing_tag = release.pending_signing_tag
pending_testing_tag = release.pending_testing_tag
if stable_tag not in tags:
log.error('%s not tagged as stable %s' %
(build.nvr, tags))
continue
if testing_tag in tags:
log.info('Removing %s from %s' %
(testing_tag, build.nvr))
koji.untagBuild(testing_tag, build.nvr)
if pending_signing_tag in tags:
log.info('Removing %s from %s' %
(pending_signing_tag, build.nvr))
koji.untagBuild(pending_signing_tag, build.nvr)
if pending_testing_tag in tags:
log.info('Removing %s from %s' %
(pending_testing_tag, build.nvr))
koji.untagBuild(pending_testing_tag, build.nvr)
db.commit()
except Exception as e:
log.error(e)
db.rollback()
Session.remove()
sys.exit(1)
def test_build_tag_when_cache_tags_fails(self, mock_cache_tags):
"""Assert that the validator fails if getting tags from koji fails"""
self.request.validated = {'builds': ['foo-1-1.f17']}
self.request.buildinfo = {'foo-1-1.f17': {
'nvr': ('foo', '1-1', 'f17'),
}}
self.request.koji = buildsys.get_session()
mock_cache_tags.return_value = None
result = validators.validate_build_tags(self.request)
assert result is None
def get_koji(request):
"""
Return a Koji client, or a duck-type of a Koji client, depending on config.
Args:
request (pyramid.request.Request): The current web request. Unused.
Returns:
koji.ClientSession or DevBuildSys: A Koji client, or a dev Koji mock.
"""
return buildsys.get_session()
def get_all_packages():
"""
Return a list of all packages in Koji.
Returns:
list: The list of package_names from the koji.listPackages() call.
"""
log.debug('Fetching list of all packages...')
koji = buildsys.get_session()
return [pkg['package_name'] for pkg in koji.listPackages()]
def test_tag_pending_signing_side_tag(self):
update = self.db.query(models.Update).first()
update.from_tag = "f17-build-side-1234"
side_tag_pending_signing = "f17-build-side-1234-signing-pending"
self.db.commit()
tag_update_builds_main(side_tag_pending_signing, update.builds)
koji = buildsys.get_session()
assert (side_tag_pending_signing, update.builds[0]) in koji.__added__
def test_side_tag_composed_by_bodhi(self):
u = self.db.query(models.Update).first()
from_tag = "f17-build-side-1234"
builds = [b.nvr for b in u.builds]
handle_srtags_main(builds, u.release.pending_signing_tag, from_tag,
None, u.release.candidate_tag)
koji = buildsys.get_session()
assert ('f17-updates-signing-pending',
'bodhi-2.0-1.fc17') in koji.__added__
assert ('f17-updates-candidate', 'bodhi-2.0-1.fc17') in koji.__added__
def test_side_tag_not_composed_by_bodhi(self):
u = self.db.query(models.Update).first()
from_tag = "f32-build-side-1234"
side_tag_signing_pending = u.release.get_pending_signing_side_tag(from_tag)
side_tag_testing_pending = u.release.get_testing_side_tag(from_tag)
builds = [b.nvr for b in u.builds]
handle_srtags_main(builds, side_tag_signing_pending, from_tag,
side_tag_testing_pending, None)
koji = buildsys.get_session()
assert ('f32-build-side-1234-signing-pending', 'bodhi-2.0-1.fc17') in koji.__added__
assert "f32-build-side-1234-signing-pending" in koji.__tags__[0][0]
assert "f32-build-side-1234-testing-pending" in koji.__tags__[1][0]
def test_build_tag_when_cache_tags_fails_cache_release(self, mock_cache_tags):
"""Assert that the cache_release returns None if getting tags
with cache_tags from koji fails"""
self.request.validated = {'builds': ['foo-1-1.f17']}
self.request.buildinfo = {'foo-1-1.f17': {
'nvr': ('foo', '1-1', 'f17'), 'tags': ['tag'],
}}
self.request.from_tag_inherited = []
self.request.koji = buildsys.get_session()
mock_cache_tags.side_effect = [['tag'], None]
result = validators.validate_build_tags(self.request)
assert result is None
def test_unable_to_infer_content_type_not_implemented(self):
"""Test error handler when Bodhi can't determine the content type due to NotImplemented."""
request = self.get_mock_request()
request.koji = buildsys.get_session()
request.validated = {'builds': [b.nvr for b in models.Build.query.all()]}
with mock.patch('bodhi.server.validators.ContentType.infer_content_class',
side_effect=NotImplementedError('oh no')):
validators.validate_acls(request)
assert request.errors == [
{'location': 'body', 'name': 'builds',
'description': "Unable to infer content_type. 'oh no'"}
]
assert request.errors.status == 501
def remove_pending_tags(self):
""" Remove all pending tags from these updates """
self.log.debug("Removing pending tags from builds")
koji = buildsys.get_session()
koji.multicall = True
for update in self.updates:
if update.request is UpdateRequest.stable:
update.remove_tag(update.release.pending_stable_tag,
koji=koji)
elif update.request is UpdateRequest.testing:
update.remove_tag(update.release.pending_testing_tag,
koji=koji)
result = koji.multiCall()
self.log.debug('remove_pending_tags koji.multiCall result = %r',
result)
def main(tag: str, builds: typing.List[str]):
"""Handle tagging builds for an update in Koji.
Args:
tag: a koji tag to apply on the builds.
builds: list of new build added to the update.
"""
try:
kc = buildsys.get_session()
kc.multicall = True
for build in builds:
kc.tagBuild(tag, build)
log.info(f"Tagging build {build} in {tag}")
kc.multiCall()
except Exception:
log.exception("There was an error handling tagging builds in koji.")
def _fetch_updates(self):
"""Based on our given koji tag, populate a list of Update objects."""
log.debug("Fetching builds tagged with '%s'" % self.tag)
kojiBuilds = get_session().listTagged(self.tag, latest=True)
nonexistent = []
log.debug("%d builds found" % len(kojiBuilds))
for build in kojiBuilds:
self.builds[build['nvr']] = build
build_obj = self.db.query(Build).filter_by(nvr=six.text_type(build['nvr'])).first()
if build_obj:
if build_obj.update:
self.updates.add(build_obj.update)
else:
log.warn('%s does not have a corresponding update' % build['nvr'])
else:
nonexistent.append(build['nvr'])
if nonexistent:
log.warning("Couldn't find the following koji builds tagged as "
"%s in bodhi: %s" % (self.tag, nonexistent))
def main(builds: typing.List[str], pending_signing_tag: str, from_tag: str,
pending_testing_tag: typing.Optional[str],
candidate_tag: typing.Optional[str]):
"""Handle side-tags and related tags for updates in Koji.
Args:
builds: a list of builds to be tagged.
pending_signing_tag: the pending signing tag to apply on the builds.
from_tag: the tag into which the builds were built.
pending_testing_tag: the pending_testing_tag to create if not None.
candidate_tag: the candidate tag needed for update that are composed by bodhi.
"""
try:
koji = buildsys.get_session()
tags = [pending_signing_tag]
if pending_testing_tag is not None:
# Validate that <koji_tag>-pending-signing and <koji-tag>-testing-signing exists
# if not create them.
if not koji.getTag(pending_signing_tag):
log.info(f"Create {pending_signing_tag} in koji")
koji.createTag(pending_signing_tag, parent=from_tag)
if not koji.getTag(pending_testing_tag):
log.info(f"Create {pending_testing_tag} in koji")
koji.createTag(pending_testing_tag, parent=from_tag)
koji.editTag2(pending_testing_tag, perm="autosign")
elif candidate_tag is not None:
# If we don't provide a pending_testing_tag, then we have merged the
# side tag into the release pending_signing and candidate tag.
# We can remove the side tag.
tags.append(candidate_tag)
koji.removeSideTag(from_tag)
koji.multicall = True
for b in builds:
for t in tags:
log.info(f"Tagging build {b} in {t}")
koji.tagBuild(t, b)
koji.multiCall()
except Exception:
log.exception("There was an error handling side-tags updates")
def test_unable_to_infer_content_type(self):
"""Test the error handler for when Bodhi cannot determine the content type of a build."""
request = self.get_mock_request()
request.koji = buildsys.get_session()
request.validated = {
'builds': [b.nvr for b in models.Build.query.all()]
}
with mock.patch(
'bodhi.server.validators.ContentType.infer_content_class',
side_effect=IOError('oh no')):
validators.validate_acls(request)
self.assertEqual(
request.errors,
[{
'location': 'body',
'name': 'builds',
'description': "Unable to infer content_type. 'oh no'"
}])
self.assertEqual(request.errors.status, 400)
def _get_build_repository(build):
"""
Return the registry repository name for the given Build from the container's pull string.
Examples -
'candidate-registry.fedoraproject.org/f29/cockpit:176-5.fc28' => 'f29/cockpit'.
'candidate-registry.fedoraproject.org/myrepo@sha256:<hash>' => 'myrepo'.
Args:
build (bodhi.server.models.Build): A Build representing a container or flatpak.
Returns:
str: The registry repository name for the build.
"""
koji = buildsys.get_session()
koji_build = koji.getBuild(build.nvr)
pull_specs = koji_build['extra']['image']['index']['pull']
# All the pull specs should have the same repository, so which one we use is arbitrary
base, tag = re.compile(r'[:@]').split(pull_specs[0], 1)
server, repository = base.split('/', 1)
return repository
def add_update(self, update):
"""Generate the extended metadata for a given update"""
rec = cr.UpdateRecord()
rec.version = __version__
rec.fromstr = config.get('bodhi_email')
rec.status = update.status.value
rec.type = update.type.value
rec.id = to_bytes(update.alias)
rec.title = to_bytes(update.title)
rec.summary = to_bytes('%s %s update' %
(update.get_title(), update.type.value))
rec.description = to_bytes(update.notes)
rec.release = to_bytes(update.release.long_name)
rec.rights = config.get('updateinfo_rights')
if update.date_pushed:
rec.issued_date = update.date_pushed
if update.date_modified:
rec.updated_date = update.date_modified
col = cr.UpdateCollection()
col.name = to_bytes(update.release.long_name)
col.shortname = to_bytes(update.release.name)
koji = get_session()
for build in update.builds:
rpms = self.get_rpms(koji, build.nvr)
for rpm in rpms:
pkg = cr.UpdateCollectionPackage()
pkg.name = rpm['name']
pkg.version = rpm['version']
pkg.release = rpm['release']
if rpm['epoch'] is not None:
pkg.epoch = str(rpm['epoch'])
else:
pkg.epoch = '0'
pkg.arch = rpm['arch']
# TODO: how do we handle UpdateSuggestion.logout, etc?
pkg.reboot_suggested = update.suggest is UpdateSuggestion.reboot
filename = '%s.%s.rpm' % (rpm['nvr'], rpm['arch'])
pkg.filename = filename
# Build the URL
if rpm['arch'] == 'src':
arch = 'SRPMS'
elif rpm['arch'] in ('noarch', 'i686'):
arch = 'i386'
else:
arch = rpm['arch']
pkg.src = os.path.join(
config.get('file_url'),
update.status is UpdateStatus.testing and 'testing' or '',
str(update.release.version), arch, filename[0], filename)
col.append(pkg)
rec.append_collection(col)
# Create references for each bug
for bug in update.bugs:
ref = cr.UpdateReference()
ref.type = 'bugzilla'
ref.id = to_bytes(bug.bug_id)
ref.href = to_bytes(bug.url)
ref.title = to_bytes(bug.title)
rec.append_reference(ref)
# Create references for each CVE
for cve in update.cves:
ref = cr.UpdateReference()
ref.type = 'cve'
ref.id = to_bytes(cve.cve_id)
ref.href = to_bytes(cve.url)
rec.append_reference(ref)
self.uinfo.append(rec)
def test_tag_pending_signing_builds(self):
update = self.db.query(models.Update).first()
pending_signing_tag = update.release.pending_signing_tag
tag_update_builds_main(pending_signing_tag, update.builds)
koji = buildsys.get_session()
assert (pending_signing_tag, update.builds[0]) in koji.__added__
def approve_update(update: Update, db: Session):
"""Add a comment to an update if it is ready for stable.
Check that the update is eligible to be pushed to stable but hasn't had comments from Bodhi to
this effect. Add a comment stating that the update may now be pushed to stable.
Args:
update: an update in testing that may be ready for stable.
"""
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
log.info(
f"{update.release.name} doesn't have mandatory days in testing")
return
# If this update was already commented, skip it
if update.has_stable_comment:
return
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if not update.meets_testing_requirements:
return
log.info(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi)
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
log.info(f"Automatically marking {update.alias} as stable")
# For now only rawhide update can be created using side tag
# Do not add the release.pending_stable_tag if the update
# was created from a side tag.
if update.release.composed_by_bodhi:
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
else:
# Single and Multi build update
conflicting_builds = update.find_conflicting_builds()
if conflicting_builds:
builds_str = str.join(", ", conflicting_builds)
update.comment(
db, "This update cannot be pushed to stable. "
f"These builds {builds_str} have a more recent "
f"build in koji's {update.release.stable_tag} tag.",
author="bodhi")
update.request = None
if update.from_tag is not None:
update.status = UpdateStatus.pending
update.remove_tag(
update.release.get_testing_side_tag(update.from_tag))
else:
update.status = UpdateStatus.obsolete
update.remove_tag(update.release.pending_testing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
return
update.add_tag(update.release.stable_tag)
update.status = UpdateStatus.stable
update.request = None
update.pushed = True
update.date_stable = update.date_pushed = func.current_timestamp()
update.comment(
db,
"This update has been submitted for stable by bodhi",
author=u'bodhi')
update.modify_bugs()
db.commit()
# Multi build update
if update.from_tag:
# Merging the side tag should happen here
pending_signing_tag = update.release.get_pending_signing_side_tag(
update.from_tag)
testing_tag = update.release.get_testing_side_tag(
update.from_tag)
update.remove_tag(pending_signing_tag)
update.remove_tag(testing_tag)
update.remove_tag(update.from_tag)
koji = buildsys.get_session()
koji.deleteTag(pending_signing_tag)
koji.deleteTag(testing_tag)
else:
# Single build update
update.remove_tag(update.release.pending_testing_tag)
update.remove_tag(update.release.pending_stable_tag)
update.remove_tag(update.release.pending_signing_tag)
update.remove_tag(update.release.candidate_tag)
def main(argv=sys.argv):
"""
Comment on updates that are eligible to be pushed to stable.
Queries for updates in the testing state that have a NULL request, looping over them looking for
updates that are eligible to be pushed to stable but haven't had comments from Bodhi to this
effect. For each such update it finds it will add a comment stating that the update may now be
pushed to stable.
This function is the entry point for the bodhi-approve-testing console script.
Args:
argv (list): A list of command line arguments. Defaults to sys.argv.
"""
logging.basicConfig(level=logging.ERROR)
if len(argv) != 2:
usage(argv)
settings = get_appsettings(argv[1])
initialize_db(settings)
db = Session()
buildsys.setup_buildsystem(config)
try:
testing = db.query(Update).filter_by(status=UpdateStatus.testing,
request=None)
for update in testing:
if not update.release.mandatory_days_in_testing and not update.autotime:
# If this release does not have any testing requirements and is not autotime,
# skip it
print(
f"{update.release.name} doesn't have mandatory days in testing"
)
continue
# If this update was already commented, skip it
if update.has_stable_comment:
continue
# If updates have reached the testing threshold, say something! Keep in mind
# that we don't care about karma here, because autokarma updates get their request set
# to stable by the Update.comment() workflow when they hit the required threshold. Thus,
# this function only needs to consider the time requirements because these updates have
# not reached the karma threshold.
if update.meets_testing_requirements:
print(f'{update.alias} now meets testing requirements')
# Only send email notification about the update reaching
# testing approval on releases composed by bodhi
update.comment(
db,
str(config.get('testing_approval_msg')),
author='bodhi',
email_notification=update.release.composed_by_bodhi)
notifications.publish(
update_schemas.UpdateRequirementsMetStableV1.from_dict(
dict(update=update)))
if update.autotime and update.days_in_testing >= update.stable_days:
print(f"Automatically marking {update.alias} as stable")
# For now only rawhide update can be created using side tag
# Do not add the release.pending_stable_tag if the update
# was created from a side tag.
if update.release.composed_by_bodhi:
update.set_request(db=db,
action=UpdateRequest.stable,
username="******")
# For updates that are not included in composes run by bodhi itself,
# mark them as stable
else:
# Single and Multi build update
conflicting_builds = update.find_conflicting_builds()
if conflicting_builds:
builds_str = str.join(", ", conflicting_builds)
update.comment(
db, "This update cannot be pushed to stable. "
f"These builds {builds_str} have a more recent "
f"build in koji's {update.release.stable_tag} tag.",
author="bodhi")
update.request = None
if update.from_tag is not None:
update.status = UpdateStatus.pending
update.remove_tag(
update.release.get_testing_side_tag(
update.from_tag))
else:
update.status = UpdateStatus.obsolete
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
continue
update.add_tag(update.release.stable_tag)
update.status = UpdateStatus.stable
update.request = None
update.pushed = True
update.date_stable = update.date_pushed = func.current_timestamp(
)
update.comment(
db,
"This update has been submitted for stable by bodhi",
author=u'bodhi')
# Multi build update
if update.from_tag:
# Merging the side tag should happen here
pending_signing_tag = update.release.get_pending_signing_side_tag(
update.from_tag)
testing_tag = update.release.get_testing_side_tag(
update.from_tag)
update.remove_tag(pending_signing_tag)
update.remove_tag(testing_tag)
update.remove_tag(update.from_tag)
koji = buildsys.get_session()
koji.deleteTag(pending_signing_tag)
koji.deleteTag(testing_tag)
# Removes the tag and the build target from koji.
koji.removeSideTag(update.from_tag)
else:
# Single build update
update.remove_tag(
update.release.pending_testing_tag)
update.remove_tag(
update.release.pending_stable_tag)
update.remove_tag(
update.release.pending_signing_tag)
update.remove_tag(update.release.candidate_tag)
db.commit()
except Exception as e:
print(str(e))
db.rollback()
Session.remove()
sys.exit(1)
def test_buildsys_lock(self, mock_lock, mock_buildsystem):
"""Assert the buildsystem lock is aquired and released in get_session"""
buildsys.get_session()
mock_lock.__enter__.assert_called_once()
mock_lock.__exit__.assert_called_once()
mock_buildsystem.assert_called_once_with()
def new_update(request):
"""
Save an update.
This entails either creating a new update, or editing an existing one. To
edit an existing update, the update's alias must be specified in
the ``edited`` parameter.
If the ``from_tag`` parameter is specified and ``builds`` is missing or
empty, the list of builds will be filled with the latest builds in this
Koji tag. This is done by validate_from_tag() because the list of builds
needs to be available in validate_acls().
Ensure that related tags ``from_tag``-pending-signing and ``from_tag``-testing
exists and if not create them in Koji.
Args:
request (pyramid.request): The current request.
"""
data = request.validated
log.debug('validated = %s' % data)
# This has already been validated at this point, but we need to ditch
# it since the models don't care about a csrf argument.
data.pop('csrf_token')
# Same here, but it can be missing.
data.pop('builds_from_tag', None)
build_nvrs = data.get('builds', [])
from_tag = data.get('from_tag')
caveats = []
try:
releases = set()
builds = []
# Create the Package and Build entities
for nvr in build_nvrs:
name, version, release = request.buildinfo[nvr]['nvr']
package = Package.get_or_create(request.buildinfo[nvr])
# Also figure out the build type and create the build if absent.
build_class = ContentType.infer_content_class(
base=Build, build=request.buildinfo[nvr]['info'])
build = build_class.get(nvr)
if build is None:
log.debug("Adding nvr %s, type %r", nvr, build_class)
build = build_class(nvr=nvr, package=package)
request.db.add(build)
request.db.flush()
build.package = package
build.release = request.buildinfo[build.nvr]['release']
builds.append(build)
releases.add(request.buildinfo[build.nvr]['release'])
# We want to go ahead and commit the transaction now so that the Builds are in the database.
# Otherwise, there will be a race condition between robosignatory signing the Builds and the
# signed handler attempting to mark the builds as signed. When we lose that race, the signed
# handler doesn't see the Builds in the database and gives up. After that, nothing will mark
# the builds as signed.
request.db.commit()
# After we commit the transaction, we need to get the builds and releases again,
# since they were tied to the previous session that has now been terminated.
builds = []
releases = set()
for nvr in build_nvrs:
# At this moment, we are sure the builds are in the database (that is what the commit
# was for actually).
build = Build.get(nvr)
builds.append(build)
releases.add(build.release)
if data.get('edited'):
log.info('Editing update: %s' % data['edited'])
data['release'] = list(releases)[0]
data['builds'] = [b.nvr for b in builds]
data['from_tag'] = from_tag
result, _caveats = Update.edit(request, data)
caveats.extend(_caveats)
else:
if len(releases) > 1:
caveats.append({
'name': 'releases',
'description': 'Your update is being split '
'into %i, one for each release.' % len(releases)
})
updates = []
for release in releases:
_data = copy.copy(data) # Copy it because .new(..) mutates it
_data['builds'] = [b for b in builds if b.release == release]
_data['release'] = release
_data['from_tag'] = from_tag
log.info('Creating new update: %r' % _data['builds'])
result, _caveats = Update.new(request, _data)
log.debug('%s update created', result.alias)
updates.append(result)
caveats.extend(_caveats)
if len(releases) > 1:
result = dict(updates=updates)
if from_tag:
koji = buildsys.get_session()
for update in updates:
# Validate that <koji_tag>-pending-signing and <koji-tag>-testing exists,
# if not create them
side_tag_pending_signing = update.release.get_pending_signing_side_tag(
from_tag)
side_tag_testing = update.release.get_testing_side_tag(from_tag)
if not koji.getTag(side_tag_pending_signing):
koji.createTag(side_tag_pending_signing, parent=from_tag)
if not koji.getTag(side_tag_testing):
koji.createTag(side_tag_testing, parent=from_tag)
to_tag = side_tag_pending_signing
# Move every new build to <from_tag>-pending-signing tag
update.add_tag(to_tag)
except LockedUpdateException as e:
log.warning(str(e))
request.errors.add('body', 'builds', "%s" % str(e))
return
except Exception as e:
log.exception('Failed to create update')
request.errors.add(
'body', 'builds', 'Unable to create update. %s' % str(e))
return
# Obsolete older updates for three different cases...
# editing an update, submitting a new single update, submitting multiple.
if isinstance(result, dict):
updates = result['updates']
else:
updates = [result]
for update in updates:
try:
caveats.extend(update.obsolete_older_updates(request.db))
except Exception as e:
caveats.append({
'name': 'update',
'description': 'Problem obsoleting older updates: %s' % str(e),
})
if not isinstance(result, dict):
result = result.__json__()
result['caveats'] = caveats
return result
def add_update(self, update):
"""
Generate the extended metadata for a given update, adding it to self.uinfo.
Args:
update (bodhi.server.models.Update): The Update to be added to self.uinfo.
"""
rec = cr.UpdateRecord()
rec.version = __version__
rec.fromstr = config.get('bodhi_email')
rec.status = update.status.value
rec.type = update.type.value
rec.id = to_bytes(update.alias)
rec.title = to_bytes(update.title)
rec.severity = util.severity_updateinfo_str(update.severity.value)
rec.summary = to_bytes('%s %s update' %
(update.get_title(), update.type.value))
rec.description = to_bytes(update.notes)
rec.release = to_bytes(update.release.long_name)
rec.rights = config.get('updateinfo_rights')
if update.date_pushed:
rec.issued_date = update.date_pushed
else:
# Sometimes we only set the date_pushed after it's pushed out, however,
# it seems that Satellite does not like update entries without issued_date.
# Since we know that we are pushing it now, and the next push will get the data
# correctly, let's just insert utcnow().
rec.issued_date = datetime.utcnow()
if update.date_modified:
rec.updated_date = update.date_modified
else:
rec.updated_date = datetime.utcnow()
col = cr.UpdateCollection()
col.name = to_bytes(update.release.long_name)
col.shortname = to_bytes(update.release.name)
koji = get_session()
for build in update.builds:
rpms = self.get_rpms(koji, build.nvr)
for rpm in rpms:
pkg = cr.UpdateCollectionPackage()
pkg.name = rpm['name']
pkg.version = rpm['version']
pkg.release = rpm['release']
if rpm['epoch'] is not None:
pkg.epoch = str(rpm['epoch'])
else:
pkg.epoch = '0'
pkg.arch = rpm['arch']
# TODO: how do we handle UpdateSuggestion.logout, etc?
pkg.reboot_suggested = update.suggest is UpdateSuggestion.reboot
filename = '%s.%s.rpm' % (rpm['nvr'], rpm['arch'])
pkg.filename = filename
# Build the URL
if rpm['arch'] == 'src':
arch = 'SRPMS'
elif rpm['arch'] in ('noarch', 'i686'):
arch = 'i386'
else:
arch = rpm['arch']
pkg.src = os.path.join(
config.get('file_url'),
update.status is UpdateStatus.testing and 'testing' or '',
str(update.release.version), arch, filename[0], filename)
col.append(pkg)
rec.append_collection(col)
# Create references for each bug
for bug in update.bugs:
ref = cr.UpdateReference()
ref.type = 'bugzilla'
ref.id = to_bytes(bug.bug_id)
ref.href = to_bytes(bug.url)
ref.title = to_bytes(bug.title)
rec.append_reference(ref)
# Create references for each CVE
for cve in update.cves:
ref = cr.UpdateReference()
ref.type = 'cve'
ref.id = to_bytes(cve.cve_id)
ref.href = to_bytes(cve.url)
rec.append_reference(ref)
self.uinfo.append(rec)
def __call__(self, message: fedora_messaging.api.Message) -> None:
"""Create updates from appropriately tagged builds.
Args:
message: The message we are processing.
"""
body = message.body
missing = []
for mandatory in ('tag', 'build_id', 'name', 'version', 'release'):
if mandatory not in body:
missing.append(mandatory)
if missing:
log.debug(
f"Received incomplete tag message. Missing: {', '.join(missing)}"
)
return
btag = body['tag']
bnvr = '{name}-{version}-{release}'.format(**body)
koji = buildsys.get_session()
kbuildinfo = koji.getBuild(bnvr)
if not kbuildinfo:
log.debug(f"Can't find Koji build for {bnvr}.")
return
if 'nvr' not in kbuildinfo:
log.debug(f"Koji build info for {bnvr} doesn't contain 'nvr'.")
return
if 'owner_name' not in kbuildinfo:
log.debug(
f"Koji build info for {bnvr} doesn't contain 'owner_name'.")
return
if kbuildinfo['owner_name'] in config.get(
'automatic_updates_blacklist'):
log.debug(
f"{bnvr} owned by {kbuildinfo['owner_name']} who is listed in "
"automatic_updates_blacklist, skipping.")
return
# some APIs want the Koji build info, some others want the same
# wrapped in a larger (request?) structure
rbuildinfo = {
'info': kbuildinfo,
'nvr': kbuildinfo['nvr'].rsplit('-', 2),
}
with self.db_factory() as dbsession:
rel = dbsession.query(Release).filter_by(
create_automatic_updates=True, candidate_tag=btag).first()
if not rel:
log.debug(
f"Ignoring build being tagged into {btag!r}, no release configured for "
"automatic updates for it found.")
return
bcls = ContentType.infer_content_class(Build, kbuildinfo)
build = bcls.get(bnvr)
if build and build.update:
log.info(
f"Build, active update for {bnvr} exists already, skipping."
)
return
if not build:
log.debug(f"Build for {bnvr} doesn't exist yet, creating.")
# Package.get_or_create() infers content type already
log.debug("Getting/creating related package object.")
pkg = Package.get_or_create(dbsession, rbuildinfo)
log.debug("Creating build object, adding it to the DB.")
build = bcls(nvr=bnvr, package=pkg, release=rel)
dbsession.add(build)
owner_name = kbuildinfo['owner_name']
user = User.get(owner_name)
if not user:
log.debug(f"Creating bodhi user for '{owner_name}'.")
# Leave email, groups blank, these will be filled
# in or updated when they log into Bodhi next time, see
# bodhi.server.security:remember_me().
user = User(name=owner_name)
dbsession.add(user)
log.debug(f"Creating new update for {bnvr}.")
try:
changelog = build.get_changelog(lastupdate=True)
except ValueError:
# Often due to bot-generated builds
# https://github.com/fedora-infra/bodhi/issues/4146
changelog = None
except Exception:
# Re-raise exception, so that the message can be re-queued
raise
closing_bugs = []
if changelog:
log.debug("Adding changelog to update notes.")
notes = f"""Automatic update for {bnvr}.
##### **Changelog**
```
{changelog}
```"""
if rel.name not in config.get('bz_exclude_rels'):
for b in re.finditer(config.get('bz_regex'), changelog,
re.IGNORECASE):
idx = int(b.group(1))
log.debug(f'Adding bug #{idx} to the update.')
bug = Bug.get(idx)
if bug is None:
bug = Bug(bug_id=idx)
dbsession.add(bug)
dbsession.flush()
if bug not in closing_bugs:
closing_bugs.append(bug)
else:
notes = f"Automatic update for {bnvr}."
update = Update(
release=rel,
builds=[build],
bugs=closing_bugs,
notes=notes,
type=UpdateType.unspecified,
stable_karma=3,
unstable_karma=-3,
autokarma=False,
user=user,
status=UpdateStatus.pending,
critpath=Update.contains_critpath_component([build],
rel.branch),
)
# Comment on the update that it was automatically created.
update.comment(
dbsession,
str("This update was automatically created"),
author="bodhi",
)
update.add_tag(update.release.pending_signing_tag)
log.debug("Adding new update to the database.")
dbsession.add(update)
log.debug("Flushing changes to the database.")
dbsession.flush()
# Obsolete older updates which may be stuck in testing due to failed gating
try:
update.obsolete_older_updates(dbsession)
except Exception as e:
log.error(f'Problem obsoleting older updates: {e}')
alias = update.alias
buglist = [b.bug_id for b in update.bugs]
# This must be run after dbsession is closed so changes are committed to db
work_on_bugs_task.delay(alias, buglist)
def __call__(self, message: fedora_messaging.api.Message) -> None:
"""Create updates from appropriately tagged builds.
Args:
message: The message we are processing.
"""
body = message.body
missing = []
for mandatory in ('tag', 'build_id', 'name', 'version', 'release'):
if mandatory not in body:
missing.append(mandatory)
if missing:
log.debug(
f"Received incomplete tag message. Missing: {', '.join(missing)}"
)
return
btag = body['tag']
bnvr = '{name}-{version}-{release}'.format(**body)
koji = buildsys.get_session()
kbuildinfo = koji.getBuild(bnvr)
if not kbuildinfo:
log.debug(f"Can't find Koji build for {bnvr}.")
return
if 'nvr' not in kbuildinfo:
log.debug(f"Koji build info for {bnvr} doesn't contain 'nvr'.")
return
if 'owner_name' not in kbuildinfo:
log.debug(
f"Koji build info for {bnvr} doesn't contain 'owner_name'.")
return
if kbuildinfo['owner_name'] in config.get(
'automatic_updates_blacklist'):
log.debug(
f"{bnvr} owned by {kbuildinfo['owner_name']} who is listed in "
"automatic_updates_blacklist, skipping.")
return
# some APIs want the Koji build info, some others want the same
# wrapped in a larger (request?) structure
rbuildinfo = {
'info': kbuildinfo,
'nvr': kbuildinfo['nvr'].rsplit('-', 2),
}
with self.db_factory() as dbsession:
rel = dbsession.query(Release).filter_by(
create_automatic_updates=True, candidate_tag=btag).first()
if not rel:
log.debug(
f"Ignoring build being tagged into {btag!r}, no release configured for "
"automatic updates for it found.")
return
bcls = ContentType.infer_content_class(Build, kbuildinfo)
build = bcls.get(bnvr)
if build and build.update:
log.info(
f"Build, active update for {bnvr} exists already, skipping."
)
return
if not build:
log.debug(f"Build for {bnvr} doesn't exist yet, creating.")
# Package.get_or_create() infers content type already
log.debug("Getting/creating related package object.")
pkg = Package.get_or_create(dbsession, rbuildinfo)
log.debug("Creating build object, adding it to the DB.")
build = bcls(nvr=bnvr, package=pkg, release=rel)
dbsession.add(build)
owner_name = kbuildinfo['owner_name']
user = User.get(owner_name)
if not user:
log.debug(f"Creating bodhi user for '{owner_name}'.")
# Leave email, groups blank, these will be filled
# in or updated when they log into Bodhi next time, see
# bodhi.server.security:remember_me().
user = User(name=owner_name)
dbsession.add(user)
log.debug(f"Creating new update for {bnvr}.")
changelog = build.get_changelog(lastupdate=True)
if changelog:
notes = f"""Automatic update for {bnvr}.
##### **Changelog**
```
{changelog}
```"""
else:
notes = f"Automatic update for {bnvr}."
update = Update(
release=rel,
builds=[build],
notes=notes,
type=UpdateType.unspecified,
stable_karma=3,
unstable_karma=-3,
autokarma=False,
user=user,
status=UpdateStatus.pending,
)
# Comment on the update that it was automatically created.
update.comment(
dbsession,
str("This update was automatically created"),
author="bodhi",
)
update.add_tag(update.release.pending_signing_tag)
log.debug("Adding new update to the database.")
dbsession.add(update)
log.debug("Committing changes to the database.")
dbsession.commit()