def update_images(namespace_name, repo_name): permission = ModifyRepositoryPermission(namespace_name, repo_name) if permission.can(): logger.debug("Looking up repository") repository_ref = registry_model.lookup_repository(namespace_name, repo_name, kind_filter="image") if repository_ref is None: # Make sure the repo actually exists. image_pushes.labels("v1", 404, "").inc() abort(404, message="Unknown repository", issue="unknown-repo") builder = lookup_manifest_builder(repository_ref, session.get("manifest_builder"), storage, docker_v2_signing_key) if builder is None: image_pushes.labels("v1", 400, "").inc() abort(400) # Generate a job for each notification that has been added to this repo logger.debug("Adding notifications for repository") event_data = { "updated_tags": [tag.name for tag in builder.committed_tags], } builder.done() track_and_log("push_repo", repository_ref) spawn_notification(repository_ref, "repo_push", event_data) image_pushes.labels("v1", 204, "").inc() return make_response("Updated", 204) image_pushes.labels("v1", 403, "").inc() abort(403)
def emit_log(mirror, log_kind, verb, message, tag=None, tags=None, stdout=None, stderr=None): logs_model.log_action( log_kind, namespace_name=mirror.repository.namespace_user.username, repository_name=mirror.repository.name, metadata={ "verb": verb, "namespace": mirror.repository.namespace_user.username, "repo": mirror.repository.name, "message": message, "tag": tag, "tags": tags, "stdout": stdout, "stderr": stderr, }, ) if log_kind in ( "repo_mirror_sync_started", "repo_mirror_sync_failed", "repo_mirror_sync_success", ): spawn_notification(wrap_repository(mirror.repository), log_kind, {"message": message})
def _write_manifest_and_log(namespace_name, repo_name, tag_name, manifest_impl): repository_ref, manifest, tag = _write_manifest( namespace_name, repo_name, tag_name, manifest_impl ) # Queue all blob manifests for replication. if features.STORAGE_REPLICATION: blobs = registry_model.get_manifest_local_blobs(manifest) if blobs is None: logger.error("Could not lookup blobs for manifest `%s`", manifest.digest) else: with queue_replication_batch(namespace_name) as queue_storage_replication: for blob_digest in blobs: queue_storage_replication(blob_digest) track_and_log("push_repo", repository_ref, tag=tag_name) spawn_notification(repository_ref, "repo_push", {"updated_tags": [tag_name]}) metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, "v2", True]) return Response( "OK", status=202, headers={ "Docker-Content-Digest": manifest.digest, "Location": url_for( "v2.fetch_manifest_by_digest", repository="%s/%s" % (namespace_name, repo_name), manifest_ref=manifest.digest, ), }, )
def _write_manifest_and_log(namespace_name, repo_name, tag_name, manifest_impl): repository_ref, manifest, tag = _write_manifest(namespace_name, repo_name, tag_name, manifest_impl) # Queue all blob manifests for replication. if features.STORAGE_REPLICATION: blobs = registry_model.get_manifest_local_blobs(manifest) if blobs is None: logger.error('Could not lookup blobs for manifest `%s`', manifest.digest) else: with queue_replication_batch( namespace_name) as queue_storage_replication: for blob_digest in blobs: queue_storage_replication(blob_digest) track_and_log('push_repo', repository_ref, tag=tag_name) spawn_notification(repository_ref, 'repo_push', {'updated_tags': [tag_name]}) metric_queue.repository_push.Inc( labelvalues=[namespace_name, repo_name, 'v2', True]) return Response( 'OK', status=202, headers={ 'Docker-Content-Digest': manifest.digest, 'Location': url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name), manifest_ref=manifest.digest), }, )
def update_images(namespace_name, repo_name): permission = ModifyRepositoryPermission(namespace_name, repo_name) if permission.can(): logger.debug('Looking up repository') repository_ref = registry_model.lookup_repository(namespace_name, repo_name, kind_filter='image') if repository_ref is None: # Make sure the repo actually exists. abort(404, message='Unknown repository', issue='unknown-repo') builder = lookup_manifest_builder(repository_ref, session.get('manifest_builder'), storage, docker_v2_signing_key) if builder is None: abort(400) # Generate a job for each notification that has been added to this repo logger.debug('Adding notifications for repository') event_data = { 'updated_tags': [tag.name for tag in builder.committed_tags], } builder.done() track_and_log('push_repo', repository_ref) spawn_notification(repository_ref, 'repo_push', event_data) metric_queue.repository_push.Inc( labelvalues=[namespace_name, repo_name, 'v1', True]) return make_response('Updated', 204) abort(403)
def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None): with UseThenDisconnect(app.config): tags = self.build_config.get("docker_tags", ["latest"]) trigger = self.repo_build.trigger if trigger is not None and trigger.id is not None: trigger_kind = trigger.service.name else: trigger_kind = None event_data = { "build_id": self.repo_build.uuid, "build_name": self.repo_build.display_name, "docker_tags": tags, "trigger_id": trigger.uuid if trigger is not None else None, "trigger_kind": trigger_kind, "trigger_metadata": self.build_config.get("trigger_metadata", {}), } if image_id is not None: event_data["image_id"] = image_id if manifest_digests: event_data["manifest_digests"] = manifest_digests if error_message is not None: event_data["error_message"] = error_message # TODO: remove when more endpoints have been converted to using # interfaces repo = AttrDict({ "namespace_name": self.repo_build.repository.namespace_user.username, "name": self.repo_build.repository.name, }) spawn_notification( repo, kind, event_data, subpage="build/%s" % self.repo_build.uuid, pathargs=["build", self.repo_build.uuid], )
def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None): with UseThenDisconnect(app.config): tags = self.build_config.get('docker_tags', ['latest']) trigger = self.repo_build.trigger if trigger is not None and trigger.id is not None: trigger_kind = trigger.service.name else: trigger_kind = None event_data = { 'build_id': self.repo_build.uuid, 'build_name': self.repo_build.display_name, 'docker_tags': tags, 'trigger_id': trigger.uuid if trigger is not None else None, 'trigger_kind': trigger_kind, 'trigger_metadata': self.build_config.get('trigger_metadata', {}) } if image_id is not None: event_data['image_id'] = image_id if manifest_digests: event_data['manifest_digests'] = manifest_digests if error_message is not None: event_data['error_message'] = error_message # TODO: remove when more endpoints have been converted to using # interfaces repo = AttrDict({ 'namespace_name': self.repo_build.repository.namespace_user.username, 'name': self.repo_build.repository.name, }) spawn_notification(repo, kind, event_data, subpage='build/%s' % self.repo_build.uuid, pathargs=['build', self.repo_build.uuid])
def _analyze(self, layer, force_parents=False): """ Analyzes a single layer. Return a tuple of two bools: - The first one tells us if we should evaluate its children. - The second one is set to False when another worker pre-empted the candidate's analysis for us. """ # If the parent couldn't be analyzed with the target version or higher, we can't analyze # this image. Mark it as failed with the current target version. if not force_parents and ( layer.parent_id and not layer.parent.security_indexed and layer.parent.security_indexed_engine >= self._target_version): if not set_secscan_status(layer, False, self._target_version): raise PreemptedException # Nothing more to do. return # Make sure the image's storage is not marked as uploading. If so, nothing more to do. if layer.storage.uploading: if not set_secscan_status(layer, False, self._target_version): raise PreemptedException # Nothing more to do. return # Analyze the image. previously_security_indexed_successfully = layer.security_indexed previous_security_indexed_engine = layer.security_indexed_engine logger.info('Analyzing layer %s', layer.docker_image_id) analyzed_version = self._api.analyze_layer(layer) logger.info('Analyzed layer %s successfully with version %s', layer.docker_image_id, analyzed_version) # Mark the image as analyzed. if not set_secscan_status(layer, True, analyzed_version): # If the image was previously successfully marked as resolved, then set_secscan_status # might return False because we're not changing it (since this is a fixup). if not previously_security_indexed_successfully: raise PreemptedException # If we are the one who've done the job successfully first, then we need to decide if we should # send notifications. Notifications are sent if: # 1) This is a new layer # 2) This is an existing layer that previously did not index properly # We don't always send notifications as if we are re-indexing a successful layer for a newer # feature set in the security scanner, notifications will be spammy. is_new_image = previous_security_indexed_engine == IMAGE_NOT_SCANNED_ENGINE_VERSION is_existing_image_unindexed = not is_new_image and not previously_security_indexed_successfully if (features.SECURITY_NOTIFICATIONS and (is_new_image or is_existing_image_unindexed)): # Get the tags of the layer we analyzed. repository_map = defaultdict(list) event = ExternalNotificationEvent.get(name='vulnerability_found') matching = list( filter_tags_have_repository_event(get_tags_for_image(layer.id), event)) for tag in matching: repository_map[tag.repository_id].append(tag) # If there is at least one tag, # Lookup the vulnerabilities for the image, now that it is analyzed. if len(repository_map) > 0: logger.debug('Loading data for layer %s', layer.id) try: layer_data = self._api.get_layer_data( layer, include_vulnerabilities=True) except APIRequestFailure: raise if layer_data is not None: # Dispatch events for any detected vulnerabilities logger.debug('Got data for layer %s: %s', layer.id, layer_data) found_features = layer_data['Layer'].get('Features', []) for repository_id in repository_map: tags = repository_map[repository_id] vulnerabilities = dict() # Collect all the vulnerabilities found for the layer under each repository and send # as a batch notification. for feature in found_features: if 'Vulnerabilities' not in feature: continue for vulnerability in feature.get( 'Vulnerabilities', []): vuln_data = { 'id': vulnerability['Name'], 'description': vulnerability.get('Description', None), 'link': vulnerability.get('Link', None), 'has_fix': 'FixedBy' in vulnerability, # TODO: Change this key name if/when we change the event format. 'priority': vulnerability.get('Severity', 'Unknown'), } vulnerabilities[ vulnerability['Name']] = vuln_data # TODO: remove when more endpoints have been converted to using # interfaces repository = AttrDict({ 'namespace_name': tags[0].repository.namespace_user.username, 'name': tags[0].repository.name, }) repo_vulnerabilities = list(vulnerabilities.values()) if not repo_vulnerabilities: continue priority_key = lambda v: PRIORITY_LEVELS.get( v['priority'], {}).get('index', 100) repo_vulnerabilities.sort(key=priority_key) event_data = { 'tags': [tag.name for tag in tags], 'vulnerabilities': repo_vulnerabilities, 'vulnerability': repo_vulnerabilities[ 0], # For back-compat with existing events. } spawn_notification(repository, 'vulnerability_found', event_data)
def start_build(repository, prepared_build, pull_robot_name=None): # Ensure that builds are only run in image repositories. if repository.kind.name != "image": raise Exception( "Attempt to start a build for application repository %s" % repository.id) # Ensure the repository isn't in mirror or read-only mode. if repository.state != RepositoryState.NORMAL: raise Exception( ("Attempt to start a build for a non-normal repository: %s %s" % (repository.id, repository.state))) # Ensure that disabled triggers are not run. if prepared_build.trigger is not None and not prepared_build.trigger.enabled: raise BuildTriggerDisabledException if repository.namespace_user.maximum_queued_builds_count is not None: queue_item_canonical_name = [repository.namespace_user.username] alive_builds = dockerfile_build_queue.num_alive_jobs( queue_item_canonical_name) if alive_builds >= repository.namespace_user.maximum_queued_builds_count: logger.debug( "Prevented queueing of build under namespace %s due to reaching max: %s", repository.namespace_user.username, repository.namespace_user.maximum_queued_builds_count, ) raise MaximumBuildsQueuedException() host = app.config["SERVER_HOSTNAME"] repo_path = "%s/%s/%s" % (host, repository.namespace_user.username, repository.name) new_token = model.token.create_access_token( repository, "write", kind="build-worker", friendly_name="Repository Build Token") logger.debug( "Creating build %s with repo %s tags %s", prepared_build.build_name, repo_path, prepared_build.tags, ) job_config = { "docker_tags": prepared_build.tags, "registry": host, "build_subdir": prepared_build.subdirectory, "context": prepared_build.context, "trigger_metadata": prepared_build.metadata or {}, "is_manual": prepared_build.is_manual, "manual_user": get_authenticated_user().username if get_authenticated_user() else None, "archive_url": prepared_build.archive_url, } with app.config["DB_TRANSACTION_FACTORY"](db): build_request = model.build.create_repository_build( repository, new_token, job_config, prepared_build.dockerfile_id, prepared_build.build_name, prepared_build.trigger, pull_robot_name=pull_robot_name, ) pull_creds = model.user.get_pull_credentials( pull_robot_name) if pull_robot_name else None json_data = json.dumps({ "build_uuid": build_request.uuid, "pull_credentials": pull_creds }) queue_id = dockerfile_build_queue.put( [repository.namespace_user.username, repository.name], json_data, retries_remaining=3) build_request.queue_id = queue_id build_request.save() # Add the build to the repo's log and spawn the build_queued notification. event_log_metadata = { "build_id": build_request.uuid, "docker_tags": prepared_build.tags, "repo": repository.name, "namespace": repository.namespace_user.username, "is_manual": prepared_build.is_manual, "manual_user": get_authenticated_user().username if get_authenticated_user() else None, } if prepared_build.trigger: event_log_metadata["trigger_id"] = prepared_build.trigger.uuid event_log_metadata[ "trigger_kind"] = prepared_build.trigger.service.name event_log_metadata["trigger_metadata"] = prepared_build.metadata or {} logs_model.log_action( "build_dockerfile", repository.namespace_user.username, ip=get_request_ip(), metadata=event_log_metadata, repository=repository, ) # TODO: remove when more endpoints have been converted to using interfaces repo = AttrDict({ "namespace_name": repository.namespace_user.username, "name": repository.name, }) spawn_notification( repo, "build_queued", event_log_metadata, subpage="build/%s" % build_request.uuid, pathargs=["build", build_request.uuid], ) return build_request
def start_build(repository, prepared_build, pull_robot_name=None): # Ensure that builds are only run in image repositories. if repository.kind.name != 'image': raise Exception( 'Attempt to start a build for application repository %s' % repository.id) # Ensure the repository isn't in mirror or read-only mode. if repository.state != RepositoryState.NORMAL: raise Exception( ('Attempt to start a build for a non-normal repository: %s %s' % (repository.id, repository.state))) # Ensure that disabled triggers are not run. if prepared_build.trigger is not None and not prepared_build.trigger.enabled: raise BuildTriggerDisabledException if repository.namespace_user.maximum_queued_builds_count is not None: queue_item_canonical_name = [repository.namespace_user.username] alive_builds = dockerfile_build_queue.num_alive_jobs( queue_item_canonical_name) if alive_builds >= repository.namespace_user.maximum_queued_builds_count: logger.debug( 'Prevented queueing of build under namespace %s due to reaching max: %s', repository.namespace_user.username, repository.namespace_user.maximum_queued_builds_count) raise MaximumBuildsQueuedException() host = app.config['SERVER_HOSTNAME'] repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) new_token = model.token.create_access_token( repository, 'write', kind='build-worker', friendly_name='Repository Build Token') logger.debug('Creating build %s with repo %s tags %s', prepared_build.build_name, repo_path, prepared_build.tags) job_config = { 'docker_tags': prepared_build.tags, 'registry': host, 'build_subdir': prepared_build.subdirectory, 'context': prepared_build.context, 'trigger_metadata': prepared_build.metadata or {}, 'is_manual': prepared_build.is_manual, 'manual_user': get_authenticated_user().username if get_authenticated_user() else None, 'archive_url': prepared_build.archive_url } with app.config['DB_TRANSACTION_FACTORY'](db): build_request = model.build.create_repository_build( repository, new_token, job_config, prepared_build.dockerfile_id, prepared_build.build_name, prepared_build.trigger, pull_robot_name=pull_robot_name) pull_creds = model.user.get_pull_credentials( pull_robot_name) if pull_robot_name else None json_data = json.dumps({ 'build_uuid': build_request.uuid, 'pull_credentials': pull_creds }) queue_id = dockerfile_build_queue.put( [repository.namespace_user.username, repository.name], json_data, retries_remaining=3) build_request.queue_id = queue_id build_request.save() # Add the queueing of the build to the metrics queue. metric_queue.repository_build_queued.Inc( labelvalues=[repository.namespace_user.username, repository.name]) # Add the build to the repo's log and spawn the build_queued notification. event_log_metadata = { 'build_id': build_request.uuid, 'docker_tags': prepared_build.tags, 'repo': repository.name, 'namespace': repository.namespace_user.username, 'is_manual': prepared_build.is_manual, 'manual_user': get_authenticated_user().username if get_authenticated_user() else None } if prepared_build.trigger: event_log_metadata['trigger_id'] = prepared_build.trigger.uuid event_log_metadata[ 'trigger_kind'] = prepared_build.trigger.service.name event_log_metadata['trigger_metadata'] = prepared_build.metadata or {} logs_model.log_action('build_dockerfile', repository.namespace_user.username, ip=get_request_ip(), metadata=event_log_metadata, repository=repository) # TODO: remove when more endpoints have been converted to using interfaces repo = AttrDict({ 'namespace_name': repository.namespace_user.username, 'name': repository.name, }) spawn_notification(repo, 'build_queued', event_log_metadata, subpage='build/%s' % build_request.uuid, pathargs=['build', build_request.uuid]) return build_request