def create_repo_notification(repo, event_name, method_name, method_config, event_config, title=None):
event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name)
method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == method_name)
return RepositoryNotification.create(repository=repo, event=event, method=method,
config_json=json.dumps(method_config), title=title,
event_config_json=json.dumps(event_config))
def find_manifests_for_sec_notification(manifest_digest):
"""
Finds all manifests matching the given digest that live in a repository with a registered
notification event for security scan results.
"""
return (Manifest.select(
Manifest,
Repository).join(Repository).join(RepositoryNotification).where(
Manifest.digest == manifest_digest,
RepositoryNotification.event == ExternalNotificationEvent.get(
name="vulnerability_found"),
))
def lookup_secscan_notification_severities(repository_id):
"""
Returns the configured security scanner notification severities for the repository
or None if none.
"""
try:
repo = Repository.get(id=repository_id)
except Repository.DoesNotExist:
return None
event_kind = ExternalNotificationEvent.get(name="vulnerability_found")
for event in RepositoryNotification.select().where(
RepositoryNotification.repository == repository_id,
RepositoryNotification.event == event_kind,
):
severity = json.loads(event.event_config_json).get(
"vulnerability", {}).get("priority")
if severity:
yield severity
def lookup_notifiable_tags_for_legacy_image(docker_image_id, storage_uuid,
event_name):
""" Yields any alive Tags found in repositories with an event with the given name registered
and whose legacy Image has the given docker image ID and storage UUID.
"""
event = ExternalNotificationEvent.get(name=event_name)
images = (Image.select().join(ImageStorage).where(
Image.docker_image_id == docker_image_id,
ImageStorage.uuid == storage_uuid))
for image in list(images):
# Ensure the image is under a repository that supports the event.
try:
RepositoryNotification.get(repository=image.repository_id,
event=event)
except RepositoryNotification.DoesNotExist:
continue
# If found in a repository with the valid event, yield the tag(s) that contains the image.
for tag in tags_containing_legacy_image(image):
yield tag
def initialize_database():
db_encrypter.initialize(FieldEncrypter("anothercrazykey!"))
db.create_tables(all_models)
Role.create(name="admin")
Role.create(name="write")
Role.create(name="read")
TeamRole.create(name="admin")
TeamRole.create(name="creator")
TeamRole.create(name="member")
Visibility.create(name="public")
Visibility.create(name="private")
LoginService.create(name="google")
LoginService.create(name="github")
LoginService.create(name="quayrobot")
LoginService.create(name="ldap")
LoginService.create(name="jwtauthn")
LoginService.create(name="keystone")
LoginService.create(name="dex")
LoginService.create(name="oidc")
BuildTriggerService.create(name="github")
BuildTriggerService.create(name="custom-git")
BuildTriggerService.create(name="bitbucket")
BuildTriggerService.create(name="gitlab")
AccessTokenKind.create(name="build-worker")
AccessTokenKind.create(name="pushpull-token")
LogEntryKind.create(name="account_change_plan")
LogEntryKind.create(name="account_change_cc")
LogEntryKind.create(name="account_change_password")
LogEntryKind.create(name="account_convert")
LogEntryKind.create(name="create_robot")
LogEntryKind.create(name="delete_robot")
LogEntryKind.create(name="create_repo")
LogEntryKind.create(name="push_repo")
LogEntryKind.create(name="pull_repo")
LogEntryKind.create(name="delete_repo")
LogEntryKind.create(name="create_tag")
LogEntryKind.create(name="move_tag")
LogEntryKind.create(name="delete_tag")
LogEntryKind.create(name="revert_tag")
LogEntryKind.create(name="add_repo_permission")
LogEntryKind.create(name="change_repo_permission")
LogEntryKind.create(name="delete_repo_permission")
LogEntryKind.create(name="change_repo_visibility")
LogEntryKind.create(name="change_repo_trust")
LogEntryKind.create(name="add_repo_accesstoken")
LogEntryKind.create(name="delete_repo_accesstoken")
LogEntryKind.create(name="set_repo_description")
LogEntryKind.create(name="change_repo_state")
LogEntryKind.create(name="build_dockerfile")
LogEntryKind.create(name="org_create_team")
LogEntryKind.create(name="org_delete_team")
LogEntryKind.create(name="org_invite_team_member")
LogEntryKind.create(name="org_delete_team_member_invite")
LogEntryKind.create(name="org_add_team_member")
LogEntryKind.create(name="org_team_member_invite_accepted")
LogEntryKind.create(name="org_team_member_invite_declined")
LogEntryKind.create(name="org_remove_team_member")
LogEntryKind.create(name="org_set_team_description")
LogEntryKind.create(name="org_set_team_role")
LogEntryKind.create(name="create_prototype_permission")
LogEntryKind.create(name="modify_prototype_permission")
LogEntryKind.create(name="delete_prototype_permission")
LogEntryKind.create(name="setup_repo_trigger")
LogEntryKind.create(name="delete_repo_trigger")
LogEntryKind.create(name="create_application")
LogEntryKind.create(name="update_application")
LogEntryKind.create(name="delete_application")
LogEntryKind.create(name="reset_application_client_secret")
# Note: These next two are deprecated.
LogEntryKind.create(name="add_repo_webhook")
LogEntryKind.create(name="delete_repo_webhook")
LogEntryKind.create(name="add_repo_notification")
LogEntryKind.create(name="delete_repo_notification")
LogEntryKind.create(name="reset_repo_notification")
LogEntryKind.create(name="regenerate_robot_token")
LogEntryKind.create(name="repo_verb")
LogEntryKind.create(name="repo_mirror_enabled")
LogEntryKind.create(name="repo_mirror_disabled")
LogEntryKind.create(name="repo_mirror_config_changed")
LogEntryKind.create(name="repo_mirror_sync_started")
LogEntryKind.create(name="repo_mirror_sync_failed")
LogEntryKind.create(name="repo_mirror_sync_success")
LogEntryKind.create(name="repo_mirror_sync_now_requested")
LogEntryKind.create(name="repo_mirror_sync_tag_success")
LogEntryKind.create(name="repo_mirror_sync_tag_failed")
LogEntryKind.create(name="repo_mirror_sync_test_success")
LogEntryKind.create(name="repo_mirror_sync_test_failed")
LogEntryKind.create(name="repo_mirror_sync_test_started")
LogEntryKind.create(name="service_key_create")
LogEntryKind.create(name="service_key_approve")
LogEntryKind.create(name="service_key_delete")
LogEntryKind.create(name="service_key_modify")
LogEntryKind.create(name="service_key_extend")
LogEntryKind.create(name="service_key_rotate")
LogEntryKind.create(name="take_ownership")
LogEntryKind.create(name="manifest_label_add")
LogEntryKind.create(name="manifest_label_delete")
LogEntryKind.create(name="change_tag_expiration")
LogEntryKind.create(name="toggle_repo_trigger")
LogEntryKind.create(name="create_app_specific_token")
LogEntryKind.create(name="revoke_app_specific_token")
ImageStorageLocation.create(name="local_eu")
ImageStorageLocation.create(name="local_us")
ApprBlobPlacementLocation.create(name="local_eu")
ApprBlobPlacementLocation.create(name="local_us")
ImageStorageTransformation.create(name="squash")
ImageStorageTransformation.create(name="aci")
ImageStorageSignatureKind.create(name="gpg2")
# NOTE: These MUST be copied over to NotificationKind, since every external
# notification can also generate a Quay.io notification.
ExternalNotificationEvent.create(name="repo_push")
ExternalNotificationEvent.create(name="build_queued")
ExternalNotificationEvent.create(name="build_start")
ExternalNotificationEvent.create(name="build_success")
ExternalNotificationEvent.create(name="build_cancelled")
ExternalNotificationEvent.create(name="build_failure")
ExternalNotificationEvent.create(name="vulnerability_found")
ExternalNotificationEvent.create(name="repo_mirror_sync_started")
ExternalNotificationEvent.create(name="repo_mirror_sync_success")
ExternalNotificationEvent.create(name="repo_mirror_sync_failed")
ExternalNotificationMethod.create(name="quay_notification")
ExternalNotificationMethod.create(name="email")
ExternalNotificationMethod.create(name="webhook")
ExternalNotificationMethod.create(name="flowdock")
ExternalNotificationMethod.create(name="hipchat")
ExternalNotificationMethod.create(name="slack")
NotificationKind.create(name="repo_push")
NotificationKind.create(name="build_queued")
NotificationKind.create(name="build_start")
NotificationKind.create(name="build_success")
NotificationKind.create(name="build_cancelled")
NotificationKind.create(name="build_failure")
NotificationKind.create(name="vulnerability_found")
NotificationKind.create(name="service_key_submitted")
NotificationKind.create(name="password_required")
NotificationKind.create(name="over_private_usage")
NotificationKind.create(name="expiring_license")
NotificationKind.create(name="maintenance")
NotificationKind.create(name="org_team_invite")
NotificationKind.create(name="repo_mirror_sync_started")
NotificationKind.create(name="repo_mirror_sync_success")
NotificationKind.create(name="repo_mirror_sync_failed")
NotificationKind.create(name="test_notification")
QuayRegion.create(name="us")
QuayService.create(name="quay")
MediaType.create(name="text/plain")
MediaType.create(name="application/json")
MediaType.create(name="text/markdown")
MediaType.create(name="application/vnd.cnr.blob.v0.tar+gzip")
MediaType.create(name="application/vnd.cnr.package-manifest.helm.v0.json")
MediaType.create(name="application/vnd.cnr.package-manifest.kpm.v0.json")
MediaType.create(
name="application/vnd.cnr.package-manifest.docker-compose.v0.json")
MediaType.create(name="application/vnd.cnr.package.kpm.v0.tar+gzip")
MediaType.create(name="application/vnd.cnr.package.helm.v0.tar+gzip")
MediaType.create(
name="application/vnd.cnr.package.docker-compose.v0.tar+gzip")
MediaType.create(name="application/vnd.cnr.manifests.v0.json")
MediaType.create(name="application/vnd.cnr.manifest.list.v0.json")
for media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
MediaType.create(name=media_type)
for media_type in DOCKER_SCHEMA2_CONTENT_TYPES:
MediaType.create(name=media_type)
for media_type in OCI_CONTENT_TYPES:
MediaType.create(name=media_type)
LabelSourceType.create(name="manifest")
LabelSourceType.create(name="api", mutable=True)
LabelSourceType.create(name="internal")
UserPromptKind.create(name="confirm_username")
UserPromptKind.create(name="enter_name")
UserPromptKind.create(name="enter_company")
RepositoryKind.create(name="image")
RepositoryKind.create(name="application")
ApprTagKind.create(name="tag")
ApprTagKind.create(name="release")
ApprTagKind.create(name="channel")
DisableReason.create(name="user_toggled")
DisableReason.create(name="successive_build_failures")
DisableReason.create(name="successive_build_internal_errors")
TagKind.create(name="tag")
def _analyze(self, layer, force_parents=False):
""" Analyzes a single layer.
Return a tuple of two bools:
- The first one tells us if we should evaluate its children.
- The second one is set to False when another worker pre-empted the candidate's analysis
for us.
"""
# If the parent couldn't be analyzed with the target version or higher, we can't analyze
# this image. Mark it as failed with the current target version.
if not force_parents and (
layer.parent_id and not layer.parent.security_indexed and
layer.parent.security_indexed_engine >= self._target_version):
if not set_secscan_status(layer, False, self._target_version):
raise PreemptedException
# Nothing more to do.
return
# Make sure the image's storage is not marked as uploading. If so, nothing more to do.
if layer.storage.uploading:
if not set_secscan_status(layer, False, self._target_version):
raise PreemptedException
# Nothing more to do.
return
# Analyze the image.
previously_security_indexed_successfully = layer.security_indexed
previous_security_indexed_engine = layer.security_indexed_engine
logger.info('Analyzing layer %s', layer.docker_image_id)
analyzed_version = self._api.analyze_layer(layer)
logger.info('Analyzed layer %s successfully with version %s',
layer.docker_image_id, analyzed_version)
# Mark the image as analyzed.
if not set_secscan_status(layer, True, analyzed_version):
# If the image was previously successfully marked as resolved, then set_secscan_status
# might return False because we're not changing it (since this is a fixup).
if not previously_security_indexed_successfully:
raise PreemptedException
# If we are the one who've done the job successfully first, then we need to decide if we should
# send notifications. Notifications are sent if:
# 1) This is a new layer
# 2) This is an existing layer that previously did not index properly
# We don't always send notifications as if we are re-indexing a successful layer for a newer
# feature set in the security scanner, notifications will be spammy.
is_new_image = previous_security_indexed_engine == IMAGE_NOT_SCANNED_ENGINE_VERSION
is_existing_image_unindexed = not is_new_image and not previously_security_indexed_successfully
if (features.SECURITY_NOTIFICATIONS
and (is_new_image or is_existing_image_unindexed)):
# Get the tags of the layer we analyzed.
repository_map = defaultdict(list)
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = list(
filter_tags_have_repository_event(get_tags_for_image(layer.id),
event))
for tag in matching:
repository_map[tag.repository_id].append(tag)
# If there is at least one tag,
# Lookup the vulnerabilities for the image, now that it is analyzed.
if len(repository_map) > 0:
logger.debug('Loading data for layer %s', layer.id)
try:
layer_data = self._api.get_layer_data(
layer, include_vulnerabilities=True)
except APIRequestFailure:
raise
if layer_data is not None:
# Dispatch events for any detected vulnerabilities
logger.debug('Got data for layer %s: %s', layer.id,
layer_data)
found_features = layer_data['Layer'].get('Features', [])
for repository_id in repository_map:
tags = repository_map[repository_id]
vulnerabilities = dict()
# Collect all the vulnerabilities found for the layer under each repository and send
# as a batch notification.
for feature in found_features:
if 'Vulnerabilities' not in feature:
continue
for vulnerability in feature.get(
'Vulnerabilities', []):
vuln_data = {
'id':
vulnerability['Name'],
'description':
vulnerability.get('Description', None),
'link':
vulnerability.get('Link', None),
'has_fix':
'FixedBy' in vulnerability,
# TODO: Change this key name if/when we change the event format.
'priority':
vulnerability.get('Severity', 'Unknown'),
}
vulnerabilities[
vulnerability['Name']] = vuln_data
# TODO: remove when more endpoints have been converted to using
# interfaces
repository = AttrDict({
'namespace_name':
tags[0].repository.namespace_user.username,
'name':
tags[0].repository.name,
})
repo_vulnerabilities = list(vulnerabilities.values())
if not repo_vulnerabilities:
continue
priority_key = lambda v: PRIORITY_LEVELS.get(
v['priority'], {}).get('index', 100)
repo_vulnerabilities.sort(key=priority_key)
event_data = {
'tags': [tag.name for tag in tags],
'vulnerabilities': repo_vulnerabilities,
'vulnerability': repo_vulnerabilities[
0], # For back-compat with existing events.
}
spawn_notification(repository, 'vulnerability_found',
event_data)
def initialize_database():
db.create_tables(all_models)
Role.create(name='admin')
Role.create(name='write')
Role.create(name='read')
TeamRole.create(name='admin')
TeamRole.create(name='creator')
TeamRole.create(name='member')
Visibility.create(name='public')
Visibility.create(name='private')
LoginService.create(name='google')
LoginService.create(name='github')
LoginService.create(name='quayrobot')
LoginService.create(name='ldap')
LoginService.create(name='jwtauthn')
LoginService.create(name='keystone')
LoginService.create(name='dex')
LoginService.create(name='oidc')
BuildTriggerService.create(name='github')
BuildTriggerService.create(name='custom-git')
BuildTriggerService.create(name='bitbucket')
BuildTriggerService.create(name='gitlab')
AccessTokenKind.create(name='build-worker')
AccessTokenKind.create(name='pushpull-token')
LogEntryKind.create(name='account_change_plan')
LogEntryKind.create(name='account_change_cc')
LogEntryKind.create(name='account_change_password')
LogEntryKind.create(name='account_convert')
LogEntryKind.create(name='create_robot')
LogEntryKind.create(name='delete_robot')
LogEntryKind.create(name='create_repo')
LogEntryKind.create(name='push_repo')
LogEntryKind.create(name='pull_repo')
LogEntryKind.create(name='delete_repo')
LogEntryKind.create(name='create_tag')
LogEntryKind.create(name='move_tag')
LogEntryKind.create(name='delete_tag')
LogEntryKind.create(name='revert_tag')
LogEntryKind.create(name='add_repo_permission')
LogEntryKind.create(name='change_repo_permission')
LogEntryKind.create(name='delete_repo_permission')
LogEntryKind.create(name='change_repo_visibility')
LogEntryKind.create(name='change_repo_trust')
LogEntryKind.create(name='add_repo_accesstoken')
LogEntryKind.create(name='delete_repo_accesstoken')
LogEntryKind.create(name='set_repo_description')
LogEntryKind.create(name='change_repo_state')
LogEntryKind.create(name='build_dockerfile')
LogEntryKind.create(name='org_create_team')
LogEntryKind.create(name='org_delete_team')
LogEntryKind.create(name='org_invite_team_member')
LogEntryKind.create(name='org_delete_team_member_invite')
LogEntryKind.create(name='org_add_team_member')
LogEntryKind.create(name='org_team_member_invite_accepted')
LogEntryKind.create(name='org_team_member_invite_declined')
LogEntryKind.create(name='org_remove_team_member')
LogEntryKind.create(name='org_set_team_description')
LogEntryKind.create(name='org_set_team_role')
LogEntryKind.create(name='create_prototype_permission')
LogEntryKind.create(name='modify_prototype_permission')
LogEntryKind.create(name='delete_prototype_permission')
LogEntryKind.create(name='setup_repo_trigger')
LogEntryKind.create(name='delete_repo_trigger')
LogEntryKind.create(name='create_application')
LogEntryKind.create(name='update_application')
LogEntryKind.create(name='delete_application')
LogEntryKind.create(name='reset_application_client_secret')
# Note: These next two are deprecated.
LogEntryKind.create(name='add_repo_webhook')
LogEntryKind.create(name='delete_repo_webhook')
LogEntryKind.create(name='add_repo_notification')
LogEntryKind.create(name='delete_repo_notification')
LogEntryKind.create(name='reset_repo_notification')
LogEntryKind.create(name='regenerate_robot_token')
LogEntryKind.create(name='repo_verb')
LogEntryKind.create(name='repo_mirror_enabled')
LogEntryKind.create(name='repo_mirror_disabled')
LogEntryKind.create(name='repo_mirror_config_changed')
LogEntryKind.create(name='repo_mirror_sync_started')
LogEntryKind.create(name='repo_mirror_sync_failed')
LogEntryKind.create(name='repo_mirror_sync_success')
LogEntryKind.create(name='repo_mirror_sync_now_requested')
LogEntryKind.create(name='repo_mirror_sync_tag_success')
LogEntryKind.create(name='repo_mirror_sync_tag_failed')
LogEntryKind.create(name='repo_mirror_sync_test_success')
LogEntryKind.create(name='repo_mirror_sync_test_failed')
LogEntryKind.create(name='repo_mirror_sync_test_started')
LogEntryKind.create(name='service_key_create')
LogEntryKind.create(name='service_key_approve')
LogEntryKind.create(name='service_key_delete')
LogEntryKind.create(name='service_key_modify')
LogEntryKind.create(name='service_key_extend')
LogEntryKind.create(name='service_key_rotate')
LogEntryKind.create(name='take_ownership')
LogEntryKind.create(name='manifest_label_add')
LogEntryKind.create(name='manifest_label_delete')
LogEntryKind.create(name='change_tag_expiration')
LogEntryKind.create(name='toggle_repo_trigger')
LogEntryKind.create(name='create_app_specific_token')
LogEntryKind.create(name='revoke_app_specific_token')
ImageStorageLocation.create(name='local_eu')
ImageStorageLocation.create(name='local_us')
ApprBlobPlacementLocation.create(name='local_eu')
ApprBlobPlacementLocation.create(name='local_us')
ImageStorageTransformation.create(name='squash')
ImageStorageTransformation.create(name='aci')
ImageStorageSignatureKind.create(name='gpg2')
# NOTE: These MUST be copied over to NotificationKind, since every external
# notification can also generate a Quay.io notification.
ExternalNotificationEvent.create(name='repo_push')
ExternalNotificationEvent.create(name='build_queued')
ExternalNotificationEvent.create(name='build_start')
ExternalNotificationEvent.create(name='build_success')
ExternalNotificationEvent.create(name='build_cancelled')
ExternalNotificationEvent.create(name='build_failure')
ExternalNotificationEvent.create(name='vulnerability_found')
ExternalNotificationEvent.create(name='repo_mirror_sync_started')
ExternalNotificationEvent.create(name='repo_mirror_sync_success')
ExternalNotificationEvent.create(name='repo_mirror_sync_failed')
ExternalNotificationMethod.create(name='quay_notification')
ExternalNotificationMethod.create(name='email')
ExternalNotificationMethod.create(name='webhook')
ExternalNotificationMethod.create(name='flowdock')
ExternalNotificationMethod.create(name='hipchat')
ExternalNotificationMethod.create(name='slack')
NotificationKind.create(name='repo_push')
NotificationKind.create(name='build_queued')
NotificationKind.create(name='build_start')
NotificationKind.create(name='build_success')
NotificationKind.create(name='build_cancelled')
NotificationKind.create(name='build_failure')
NotificationKind.create(name='vulnerability_found')
NotificationKind.create(name='service_key_submitted')
NotificationKind.create(name='password_required')
NotificationKind.create(name='over_private_usage')
NotificationKind.create(name='expiring_license')
NotificationKind.create(name='maintenance')
NotificationKind.create(name='org_team_invite')
NotificationKind.create(name='repo_mirror_sync_started')
NotificationKind.create(name='repo_mirror_sync_success')
NotificationKind.create(name='repo_mirror_sync_failed')
NotificationKind.create(name='test_notification')
QuayRegion.create(name='us')
QuayService.create(name='quay')
MediaType.create(name='text/plain')
MediaType.create(name='application/json')
MediaType.create(name='text/markdown')
MediaType.create(name='application/vnd.cnr.blob.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package-manifest.helm.v0.json')
MediaType.create(name='application/vnd.cnr.package-manifest.kpm.v0.json')
MediaType.create(
name='application/vnd.cnr.package-manifest.docker-compose.v0.json')
MediaType.create(name='application/vnd.cnr.package.kpm.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package.helm.v0.tar+gzip')
MediaType.create(
name='application/vnd.cnr.package.docker-compose.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.manifests.v0.json')
MediaType.create(name='application/vnd.cnr.manifest.list.v0.json')
for media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
MediaType.create(name=media_type)
for media_type in DOCKER_SCHEMA2_CONTENT_TYPES:
MediaType.create(name=media_type)
LabelSourceType.create(name='manifest')
LabelSourceType.create(name='api', mutable=True)
LabelSourceType.create(name='internal')
UserPromptKind.create(name='confirm_username')
UserPromptKind.create(name='enter_name')
UserPromptKind.create(name='enter_company')
RepositoryKind.create(name='image')
RepositoryKind.create(name='application')
ApprTagKind.create(name='tag')
ApprTagKind.create(name='release')
ApprTagKind.create(name='channel')
DisableReason.create(name='user_toggled')
DisableReason.create(name='successive_build_failures')
DisableReason.create(name='successive_build_internal_errors')
TagKind.create(name='tag')
