def __call__(self, message: fedora_messaging.api.Message):
"""Handle messages arriving with the configured topic."""
msg = message.body
if not msg:
log.debug("Ignoring message without body.")
return
subject_identifier = msg.get("subject_identifier")
if subject_identifier is None:
log.debug("Couldn't find subject_identifier in Greenwave message")
return
subject_type = msg.get("subject_type")
if subject_type == "compose":
log.debug("Not requesting a decision for a compose")
return
with self.db_factory():
build = Build.get(subject_identifier)
if build is None:
log.debug(f"Couldn't find build {subject_identifier} in DB")
return
log.info(
f"Updating the test_gating_status for: {build.update.alias}")
build.update.update_test_gating_status()
def get_override(request):
"""
Return a dictionary with key "override" indexing the override that matches the given nvr.
Args:
request (pyramid.request): The current request, which should have a query parameter "nvr",
providing the nvr of the requested override.
Returns:
dict: A dictionary with key "override" that indexes the override matching the given nvr.
"""
nvr = request.matchdict.get('nvr')
build = Build.get(nvr)
if not build:
request.errors.add('url', 'nvr', 'No such build')
request.errors.status = HTTPNotFound.code
return
if not build.override:
request.errors.add('url', 'nvr',
'No buildroot override for this build')
request.errors.status = HTTPNotFound.code
return
return dict(override=build.override)
def __call__(self, message: fedora_messaging.api.Message):
"""
Handle messages arriving with the configured topic.
This marks a build as signed if it is assigned to the pending testing release tag.
Example message format::
{
'body': {
'i': 628,
'timestamp': 1484692585,
'msg_id': '2017-821031da-be3a-4f4b-91df-0baa834ca8a4',
'crypto': 'x509',
'topic': 'org.fedoraproject.prod.buildsys.tag',
'signature': '100% real please trust me',
'msg': {
'build_id': 442562,
'name': 'colord',
'tag_id': 214,
'instance': 's390',
'tag': 'f26-updates-testing-pending',
'user': '******',
'version': '1.3.4',
'owner': 'sharkcz',
'release': '1.fc26'
},
},
}
The message can contain additional keys.
Args:
message: The incoming message in the format described above.
"""
message = message.body['msg']
build_nvr = '%(name)s-%(version)s-%(release)s' % message
tag = message['tag']
log.info("%s tagged into %s" % (build_nvr, tag))
with self.db_factory():
build = Build.get(build_nvr)
if not build:
log.info("Build was not submitted, skipping")
return
if not build.release:
log.info('Build is not assigned to release, skipping')
return
if build.release.pending_testing_tag != tag:
log.info("Tag is not pending_testing tag, skipping")
return
# This build was moved into the pending_testing tag for the applicable release, which
# is done by RoboSignatory to indicate that the build has been correctly signed and
# written out. Mark it as such.
log.info("Build has been signed, marking")
build.signed = True
log.info("Build %s has been marked as signed" % build_nvr)
def __call__(self, message: fedora_messaging.api.Message):
"""
Handle messages arriving with the configured topic.
This marks a build as signed if it is assigned to the pending testing release tag.
Example message format::
{
'body': {
'build_id': 442562,
'name': 'colord',
'tag_id': 214,
'instance': 's390',
'tag': 'f26-updates-testing-pending',
'user': '******',
'version': '1.3.4',
'owner': 'sharkcz',
'release': '1.fc26'
},
}
The message can contain additional keys.
Duplicate messages: this method is idempotent.
Args:
message: The incoming message in the format described above.
"""
message = message.body
build_nvr = '%(name)s-%(version)s-%(release)s' % message
tag = message['tag']
log.info("%s tagged into %s" % (build_nvr, tag))
with self.db_factory():
build = Build.get(build_nvr)
if not build:
log.info("Build was not submitted, skipping")
return
if not build.release:
log.info('Build is not assigned to release, skipping')
return
if build.release.pending_testing_tag != tag:
log.info("Tag is not pending_testing tag, skipping")
return
if build.signed:
log.info("Build was already marked as signed (maybe a duplicate message)")
return
# This build was moved into the pending_testing tag for the applicable release, which
# is done by RoboSignatory to indicate that the build has been correctly signed and
# written out. Mark it as such.
log.info("Build has been signed, marking")
build.signed = True
log.info("Build %s has been marked as signed" % build_nvr)
def update_from_db_message(msgid: str, itemdict: dict):
"""
Find and return update for waiverdb or resultsdb message.
Used by the resultsdb and waiverdb consumers.
Args:
msgid: the message ID (for logging purposes)
itemdict: the relevant dict from the message. 'subject' dict
for a waiverdb message, 'item' dict for resultsdb.
Returns:
bodhi.server.models.Update or None: the relevant update, if
found.
"""
itemtype = itemdict.get("type")
if not itemtype:
log.error(f"Couldn't find item type in message {msgid}")
return None
if isinstance(itemtype, list):
# In resultsdb.result.new messages, the values are all lists
# for some reason
itemtype = itemtype[0]
if itemtype not in ("koji_build", "bodhi_update"):
log.debug(f"Irrelevant item type {itemtype}")
return None
# find the update
if itemtype == "bodhi_update":
updateid = itemdict.get("item")
if isinstance(updateid, list):
updateid = updateid[0]
if not updateid:
log.error(f"Couldn't find update ID in message {msgid}")
return None
update = Update.get(updateid)
if not update:
log.error(f"Couldn't find update {updateid} in DB")
return None
else:
nvr = itemdict.get("nvr", itemdict.get("item"))
if isinstance(nvr, list):
nvr = nvr[0]
if not nvr:
log.error(f"Couldn't find nvr in message {msgid}")
return None
build = Build.get(nvr)
if not build:
log.error(f"Couldn't find build {nvr} in DB")
return None
update = build.update
return update
def get_build(request):
"""
Retrieve a Build by name-version-release, specified via an "nvr" query string parameter.
Args:
request (pyramid.request): The current web request.
Returns:
bodhi.server.models.Build or None: The Build matching the search, or None if there is no
Build with the given NVR.
"""
nvr = request.matchdict.get('nvr')
build = Build.get(nvr)
if not build:
request.errors.add('body', 'nvr', 'No such build')
request.errors.status = HTTPNotFound.code
return
return build
def setup_method(self, method):
"""Set up environment for each test."""
super().setup_method(method)
release = self.db.query(Release).filter_by(name='F17').first()
package = self.db.query(Package).first()
user = self.db.query(User).first()
self.update = Update(
release=release,
stable_karma=0,
unstable_karma=0,
notes='',
type=UpdateType.bugfix,
user=user,
)
self.build = Build(nvr='colord-1.3.4-1.fc26',
update=self.update,
type=package.type,
package=package)
self.db.add(self.update)
self.db.add(self.build)
self.db.commit()
body = {
'contact': '*****@*****.**',
'run': {
'url': 'https://example.url',
},
'artifact': {
'nvr': 'colord-1.3.4-1.fc26'
},
'pipeline': {
'id': 442562,
},
'test': 'test',
'generated_at': 'somedate',
'version': '1.3.4',
}
self.sample_message = Message(topic='', body=body)
self.db_factory = base.TransactionalSessionMaker(self.Session)
self.handler = CIHandler(self.db_factory)
def new_update(request):
"""
Save an update.
This entails either creating a new update, or editing an existing one. To
edit an existing update, the update's alias must be specified in
the ``edited`` parameter.
If the ``from_tag`` parameter is specified and ``builds`` is missing or
empty, the list of builds will be filled with the latest builds in this
Koji tag. This is done by validate_from_tag() because the list of builds
needs to be available in validate_acls().
If the release is composed by Bodhi (i.e. a branched or stable release
after the Bodhi activation point), ensure that related tags
``from_tag``-pending-signing and ``from_tag``-testing exists and if not
create them in Koji. If the state of the release is not `pending`, add its
pending-signing tag and remove it if it's a side tag.
Args:
request (pyramid.request): The current request.
"""
data = request.validated
log.debug('validated = %s' % data)
# This has already been validated at this point, but we need to ditch
# it since the models don't care about a csrf argument.
data.pop('csrf_token')
# Same here, but it can be missing.
data.pop('builds_from_tag', None)
data.pop('sidetag_owner', None)
build_nvrs = data.get('builds', [])
from_tag = data.get('from_tag')
caveats = []
try:
releases = set()
builds = []
# Create the Package and Build entities
for nvr in build_nvrs:
name, version, release = request.buildinfo[nvr]['nvr']
package = Package.get_or_create(request.db, request.buildinfo[nvr])
# Also figure out the build type and create the build if absent.
build_class = ContentType.infer_content_class(
base=Build, build=request.buildinfo[nvr]['info'])
build = build_class.get(nvr)
if build is None:
log.debug("Adding nvr %s, type %r", nvr, build_class)
build = build_class(nvr=nvr, package=package)
request.db.add(build)
request.db.flush()
build.package = package
build.release = request.buildinfo[build.nvr]['release']
builds.append(build)
releases.add(request.buildinfo[build.nvr]['release'])
# Disable manual updates for releases not composed by Bodhi
# see #4058
if not from_tag:
for release in releases:
if not release.composed_by_bodhi:
request.errors.add(
'body', 'builds',
"Cannot manually create updates for a Release which is not "
"composed by Bodhi.\nRead the 'Automatic updates' page in "
"Bodhi docs about this error.")
request.db.rollback()
return
# We want to go ahead and commit the transaction now so that the Builds are in the database.
# Otherwise, there will be a race condition between robosignatory signing the Builds and the
# signed handler attempting to mark the builds as signed. When we lose that race, the signed
# handler doesn't see the Builds in the database and gives up. After that, nothing will mark
# the builds as signed.
request.db.commit()
# After we commit the transaction, we need to get the builds and releases again,
# since they were tied to the previous session that has now been terminated.
builds = []
releases = set()
for nvr in build_nvrs:
# At this moment, we are sure the builds are in the database (that is what the commit
# was for actually).
build = Build.get(nvr)
builds.append(build)
releases.add(build.release)
if data.get('edited'):
log.info('Editing update: %s' % data['edited'])
data['release'] = list(releases)[0]
data['builds'] = [b.nvr for b in builds]
data['from_tag'] = from_tag
result, _caveats = Update.edit(request, data)
caveats.extend(_caveats)
else:
if len(releases) > 1:
caveats.append({
'name':
'releases',
'description':
'Your update is being split '
'into %i, one for each release.' % len(releases)
})
updates = []
for release in releases:
_data = copy.copy(data) # Copy it because .new(..) mutates it
_data['builds'] = [b for b in builds if b.release == release]
_data['release'] = release
_data['from_tag'] = from_tag
log.info('Creating new update: %r' % _data['builds'])
result, _caveats = Update.new(request, _data)
log.debug('%s update created', result.alias)
updates.append(result)
caveats.extend(_caveats)
if len(releases) > 1:
result = dict(updates=updates)
if from_tag:
for u in updates:
builds = [b.nvr for b in u.builds]
if not u.release.composed_by_bodhi:
# Before the Bodhi activation point of a release, keep builds tagged
# with the side-tag and its associate tags.
side_tag_signing_pending = u.release.get_pending_signing_side_tag(
from_tag)
side_tag_testing_pending = u.release.get_pending_testing_side_tag(
from_tag)
handle_side_and_related_tags_task.delay(
builds=builds,
pending_signing_tag=side_tag_signing_pending,
from_tag=from_tag,
pending_testing_tag=side_tag_testing_pending)
else:
# After the Bodhi activation point of a release, add the pending-signing tag
# of the release to funnel the builds back into a normal workflow for a
# stable release.
pending_signing_tag = u.release.pending_signing_tag
candidate_tag = u.release.candidate_tag
handle_side_and_related_tags_task.delay(
builds=builds,
pending_signing_tag=pending_signing_tag,
from_tag=from_tag,
candidate_tag=candidate_tag)
except LockedUpdateException as e:
log.warning(str(e))
request.errors.add('body', 'builds', "%s" % str(e))
return
except Exception as e:
log.exception('Failed to create update')
request.errors.add('body', 'builds',
'Unable to create update. %s' % str(e))
return
# Obsolete older updates for three different cases...
# editing an update, submitting a new single update, submitting multiple.
if isinstance(result, dict):
updates = result['updates']
else:
updates = [result]
for update in updates:
try:
caveats.extend(update.obsolete_older_updates(request.db))
except Exception as e:
caveats.append({
'name':
'update',
'description':
'Problem obsoleting older updates: %s' % str(e),
})
if not isinstance(result, dict):
result = result.__json__()
result['caveats'] = caveats
return result
def save_override(request):
"""
Create or edit a buildroot override.
This entails either creating a new buildroot override, or editing an
existing one. To edit an existing buildroot override, the buildroot
override's original id needs to be specified in the ``edited`` parameter.
Args:
request (pyramid.request): The current web request.
Returns:
dict: The new or edited override.
"""
data = request.validated
edited = data.pop("edited")
caveats = []
try:
submitter = User.get(request.user.name)
if edited is None:
builds = data['builds']
overrides = []
if len(builds) > 1:
caveats.append({
'name':
'nvrs',
'description':
'Your override submission was '
'split into %i.' % len(builds)
})
for build in builds:
log.info("Creating a new buildroot override: %s" % build.nvr)
existing_override = BuildrootOverride.get(build.id)
if existing_override:
if not existing_override.expired_date:
data['expiration_date'] = max(
existing_override.expiration_date,
data['expiration_date'])
new_notes = f"""{data['notes']}
_____________
_@{existing_override.submitter.name} ({existing_override.submission_date.strftime('%b %d, %Y')})_
{existing_override.notes}"""
# Truncate notes at 2000 chars
if len(new_notes) > 2000:
new_notes = new_notes[:1972] + '(...)\n___Notes truncated___'
overrides.append(
BuildrootOverride.edit(
request,
edited=build,
submitter=submitter,
submission_date=datetime.now(),
notes=new_notes,
expiration_date=data['expiration_date'],
expired=None,
))
else:
overrides.append(
BuildrootOverride.new(
request,
build=build,
submitter=submitter,
notes=data['notes'],
expiration_date=data['expiration_date'],
))
if len(builds) > 1:
result = dict(overrides=overrides)
else:
result = overrides[0]
else:
log.info("Editing buildroot override: %s" % edited)
edited = Build.get(edited)
if edited is None:
request.errors.add('body', 'edited', 'No such build')
return
result = BuildrootOverride.edit(
request,
edited=edited,
submitter=submitter,
notes=data["notes"],
expired=data["expired"],
expiration_date=data["expiration_date"])
if not result:
# Some error inside .edit(...)
return
except Exception as e:
log.exception(e)
request.errors.add('body', 'override',
'Unable to save buildroot override: %s' % e)
return
if not isinstance(result, dict):
result = result.__json__()
result['caveats'] = caveats
return result
def new_update(request):
"""
Save an update.
This entails either creating a new update, or editing an existing one. To
edit an existing update, the update's alias must be specified in
the ``edited`` parameter.
Args:
request (pyramid.request): The current request.
"""
data = request.validated
log.debug('validated = %s' % data)
# This has already been validated at this point, but we need to ditch
# it since the models don't care about a csrf argument.
data.pop('csrf_token')
caveats = []
try:
releases = set()
builds = []
# Create the Package and Build entities
for nvr in data['builds']:
name, version, release = request.buildinfo[nvr]['nvr']
package = Package.get_or_create(request.buildinfo[nvr])
# Also figure out the build type and create the build if absent.
build_class = ContentType.infer_content_class(
base=Build, build=request.buildinfo[nvr]['info'])
build = build_class.get(nvr)
if build is None:
log.debug("Adding nvr %s, type %r", nvr, build_class)
build = build_class(nvr=nvr, package=package)
request.db.add(build)
request.db.flush()
build.package = package
build.release = request.buildinfo[build.nvr]['release']
builds.append(build)
releases.add(request.buildinfo[build.nvr]['release'])
# We want to go ahead and commit the transaction now so that the Builds are in the database.
# Otherwise, there will be a race condition between robosignatory signing the Builds and the
# signed handler attempting to mark the builds as signed. When we lose that race, the signed
# handler doesn't see the Builds in the database and gives up. After that, nothing will mark
# the builds as signed.
request.db.commit()
# After we commit the transaction, we need to get the builds and releases again, since they
# were tied to the previous session that has now been terminated.
builds = []
releases = set()
for nvr in data['builds']:
# At this moment, we are sure the builds are in the database (that is what the commit
# was for actually).
build = Build.get(nvr)
builds.append(build)
releases.add(build.release)
if data.get('edited'):
log.info('Editing update: %s' % data['edited'])
data['release'] = list(releases)[0]
data['builds'] = [b.nvr for b in builds]
result, _caveats = Update.edit(request, data)
caveats.extend(_caveats)
else:
if len(releases) > 1:
caveats.append({
'name':
'releases',
'description':
'Your update is being split '
'into %i, one for each release.' % len(releases)
})
updates = []
for release in releases:
_data = copy.copy(data) # Copy it because .new(..) mutates it
_data['builds'] = [b for b in builds if b.release == release]
_data['release'] = release
log.info('Creating new update: %r' % _data['builds'])
result, _caveats = Update.new(request, _data)
log.debug('%s update created', result.alias)
updates.append(result)
caveats.extend(_caveats)
if len(releases) > 1:
result = dict(updates=updates)
except LockedUpdateException as e:
log.warning(str(e))
request.errors.add('body', 'builds', "%s" % str(e))
return
except Exception as e:
log.exception('Failed to create update')
request.errors.add('body', 'builds',
'Unable to create update. %s' % str(e))
return
# Obsolete older updates for three different cases...
# editing an update, submitting a new single update, submitting multiple.
if isinstance(result, dict):
updates = result['updates']
else:
updates = [result]
for update in updates:
try:
caveats.extend(update.obsolete_older_updates(request.db))
except Exception as e:
caveats.append({
'name':
'update',
'description':
'Problem obsoleting older updates: %s' % str(e),
})
if not isinstance(result, dict):
result = result.__json__()
result['caveats'] = caveats
return result
def __call__(self, message: fedora_messaging.api.Message):
"""
Handle messages arriving with the configured topic.
This marks a build as signed if it is assigned to the pending testing release tag.
Example message format::
{
'body': {
'build_id': 442562,
'name': 'colord',
'tag_id': 214,
'instance': 's390',
'tag': 'f26-updates-testing-pending',
'user': '******',
'version': '1.3.4',
'owner': 'sharkcz',
'release': '1.fc26'
},
}
The message can contain additional keys.
Duplicate messages: this method is idempotent.
Args:
message: The incoming message in the format described above.
"""
message = message.body
build_nvr = '%(name)s-%(version)s-%(release)s' % message
tag = message['tag']
log.info("%s tagged into %s" % (build_nvr, tag))
with self.db_factory() as dbsession:
build = Build.get(build_nvr)
if not build:
log.info("Build was not submitted, skipping")
return
if not build.release:
log.info('Build is not assigned to release, skipping')
return
if build.update and build.update.from_tag:
koji_testing_tag = build.release.get_testing_side_tag(
build.update.from_tag)
if tag != koji_testing_tag:
log.info("Tag is not testing side tag, skipping")
return
else:
if build.release.pending_testing_tag != tag:
log.info("Tag is not pending_testing tag, skipping")
return
if build.signed:
log.info(
"Build was already marked as signed (maybe a duplicate message)"
)
return
# This build was moved into the pending_testing tag for the applicable release, which
# is done by RoboSignatory to indicate that the build has been correctly signed and
# written out. Mark it as such.
log.info("Build has been signed, marking")
build.signed = True
dbsession.flush()
log.info("Build %s has been marked as signed" % build_nvr)
# If every build in update is signed change status to testing
if build.update \
and not build.update.release.composed_by_bodhi \
and build.update.signed:
log.info(
"Every build in update is signed, set status to testing")
build.update.status = UpdateStatus.testing
build.update.date_testing = func.current_timestamp()
build.update.request = None
build.update.pushed = True
if config.get("test_gating.required"):
log.debug(
'Test gating is required, marking the update as waiting on test '
'gating and updating it from Greenwave to get the real status.'
)
build.update.test_gating_status = TestGatingStatus.waiting
build.update.update_test_gating_status()
log.info(
f"Update {build.update.display_name} status has been set to testing"
)
def save_override(request):
"""
Create or edit a buildroot override.
This entails either creating a new buildroot override, or editing an
existing one. To edit an existing buildroot override, the buildroot
override's original id needs to be specified in the ``edited`` parameter.
Args:
request (pyramid.request): The current web request.
Returns:
dict: The new or edited override.
"""
data = request.validated
edited = data.pop("edited")
caveats = []
try:
submitter = User.get(request.user.name)
if edited is None:
builds = data['builds']
overrides = []
if len(builds) > 1:
caveats.append({
'name': 'nvrs',
'description': 'Your override submission was '
'split into %i.' % len(builds)
})
for build in builds:
log.info("Creating a new buildroot override: %s" % build.nvr)
if BuildrootOverride.get(build.id):
request.errors.add('body', 'builds',
'Buildroot override for %s already exists' % build.nvr)
return
else:
overrides.append(BuildrootOverride.new(
request,
build=build,
submitter=submitter,
notes=data['notes'],
expiration_date=data['expiration_date'],
))
if len(builds) > 1:
result = dict(overrides=overrides)
else:
result = overrides[0]
else:
log.info("Editing buildroot override: %s" % edited)
edited = Build.get(edited)
if edited is None:
request.errors.add('body', 'edited', 'No such build')
return
result = BuildrootOverride.edit(
request, edited=edited, submitter=submitter,
notes=data["notes"], expired=data["expired"],
expiration_date=data["expiration_date"])
if not result:
# Some error inside .edit(...)
return
except Exception as e:
log.exception(e)
request.errors.add('body', 'override',
'Unable to save buildroot override: %s' % e)
return
if not isinstance(result, dict):
result = result.__json__()
result['caveats'] = caveats
return result
