Ejemplo n.º 1
0
def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
    """
    Delete the manifest specified by the digest.

    Note: there is no equivalent method for deleting by tag name because it is
    forbidden by the spec.
    """
    with db_disallow_replica_use():
        repository_ref = registry_model.lookup_repository(
            namespace_name, repo_name)
        if repository_ref is None:
            raise NameUnknown()

        manifest = registry_model.lookup_manifest_by_digest(
            repository_ref, manifest_ref)
        if manifest is None:
            raise ManifestUnknown()

        tags = registry_model.delete_tags_for_manifest(manifest)
        if not tags:
            raise ManifestUnknown()

        for tag in tags:
            track_and_log("delete_tag",
                          repository_ref,
                          tag=tag.name,
                          digest=manifest_ref)

        return Response(status=202)
Ejemplo n.º 2
0
  def post(self, namespace_name, repository_name):
    """ Create a RepoMirrorConfig for a given Repository. """
    # TODO: Tidy up this function
    # TODO: Specify only the data we want to pass on when creating the RepoMirrorConfig. Avoid
    #       the possibility of data injection.

    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    if model.repo_mirror.get_mirror(repo):
      return {'detail': 'Mirror configuration already exits for repository %s/%s' % (
        namespace_name, repository_name)}, 409

    data = request.get_json()

    data['sync_start_date'] = self._string_to_dt(data['sync_start_date'])

    rule = model.repo_mirror.create_rule(repo, data['root_rule']['rule_value'])
    del data['root_rule']

    # Verify the robot is part of the Repository's namespace
    robot = self._setup_robot_for_mirroring(namespace_name, repository_name, data['robot_username'])
    del data['robot_username']

    mirror = model.repo_mirror.enable_mirroring_for_repository(repo, root_rule=rule,
                                                               internal_robot=robot, **data)
    if mirror:
      track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed='external_reference', to=data['external_reference'])
      return '', 201
    else:
      # TODO: Determine appropriate Response
      return {'detail': 'RepoMirrorConfig already exists for this repository.'}, 409
Ejemplo n.º 3
0
    def log_action(
        self,
        event_name,
        namespace_name,
        repo_name=None,
        analytics_name=None,
        analytics_sample=1,
        metadata=None,
    ):
        metadata = {} if metadata is None else metadata

        repo = None
        if repo_name is not None:
            db_repo = data.model.repository.get_repository(
                namespace_name, repo_name, kind_filter="application")
            repo = AttrDict({
                "id":
                db_repo.id,
                "name":
                db_repo.name,
                "namespace_name":
                db_repo.namespace_user.username,
                "is_free_namespace":
                db_repo.namespace_user.stripe_id is None,
            })
        track_and_log(event_name,
                      repo,
                      analytics_name=analytics_name,
                      analytics_sample=analytics_sample,
                      **metadata)
Ejemplo n.º 4
0
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref, registry_model):
    try:
        repository_ref = registry_model.lookup_repository(
            namespace_name, repo_name, raise_on_error=True, manifest_ref=manifest_ref
        )
    except RepositoryDoesNotExist as e:
        image_pulls.labels("v2", "tag", 404).inc()
        raise NameUnknown("repository not found")

    try:
        tag = registry_model.get_repo_tag(repository_ref, manifest_ref, raise_on_error=True)
    except TagDoesNotExist as e:
        if registry_model.has_expired_tag(repository_ref, manifest_ref):
            logger.debug(
                "Found expired tag %s for repository %s/%s", manifest_ref, namespace_name, repo_name
            )
            msg = (
                "Tag %s was deleted or has expired. To pull, revive via time machine" % manifest_ref
            )
            image_pulls.labels("v2", "tag", 404).inc()
            raise TagExpired(msg)

        image_pulls.labels("v2", "tag", 404).inc()
        raise ManifestUnknown(str(e))

    manifest = registry_model.get_manifest_for_tag(tag)
    if manifest is None:
        # Something went wrong.
        image_pulls.labels("v2", "tag", 400).inc()
        raise ManifestInvalid()

    try:
        manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary(
            namespace_name, repo_name, manifest_ref, manifest, registry_model
        )
    except (ManifestException, ManifestDoesNotExist) as e:
        image_pulls.labels("v2", "tag", 404).inc()
        raise ManifestUnknown(str(e))

    if manifest_bytes is None:
        image_pulls.labels("v2", "tag", 404).inc()
        raise ManifestUnknown()

    track_and_log(
        "pull_repo",
        repository_ref,
        analytics_name="pull_repo_100x",
        analytics_sample=0.01,
        tag=manifest_ref,
    )
    image_pulls.labels("v2", "tag", 200).inc()

    return Response(
        manifest_bytes.as_unicode(),
        status=200,
        headers={
            "Content-Type": manifest_media_type,
            "Docker-Content-Digest": manifest_digest,
        },
    )
Ejemplo n.º 5
0
def update_images(namespace_name, repo_name):
    permission = ModifyRepositoryPermission(namespace_name, repo_name)
    if permission.can():
        logger.debug("Looking up repository")
        repository_ref = registry_model.lookup_repository(namespace_name,
                                                          repo_name,
                                                          kind_filter="image")
        if repository_ref is None:
            # Make sure the repo actually exists.
            image_pushes.labels("v1", 404, "").inc()
            abort(404, message="Unknown repository", issue="unknown-repo")

        builder = lookup_manifest_builder(repository_ref,
                                          session.get("manifest_builder"),
                                          storage, docker_v2_signing_key)
        if builder is None:
            image_pushes.labels("v1", 400, "").inc()
            abort(400)

        # Generate a job for each notification that has been added to this repo
        logger.debug("Adding notifications for repository")
        event_data = {
            "updated_tags": [tag.name for tag in builder.committed_tags],
        }

        builder.done()

        track_and_log("push_repo", repository_ref)
        spawn_notification(repository_ref, "repo_push", event_data)
        image_pushes.labels("v1", 204, "").inc()
        return make_response("Updated", 204)

    image_pushes.labels("v1", 403, "").inc()
    abort(403)
Ejemplo n.º 6
0
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref, registry_model):
    try:
        repository_ref = registry_model.lookup_repository(
            namespace_name, repo_name, raise_on_error=True, manifest_ref=manifest_ref
        )
    except RepositoryDoesNotExist as e:
        image_pulls.labels("v2", "manifest", 404).inc()
        raise NameUnknown("repository not found")

    try:
        manifest = registry_model.lookup_manifest_by_digest(
            repository_ref, manifest_ref, raise_on_error=True
        )
    except ManifestDoesNotExist as e:
        image_pulls.labels("v2", "manifest", 404).inc()
        raise ManifestUnknown(str(e))

    track_and_log("pull_repo", repository_ref, manifest_digest=manifest_ref)
    image_pulls.labels("v2", "manifest", 200).inc()

    return Response(
        manifest.internal_manifest_bytes.as_unicode(),
        status=200,
        headers={
            "Content-Type": manifest.media_type,
            "Docker-Content-Digest": manifest.digest,
        },
    )
Ejemplo n.º 7
0
def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
    """
    Delete the manifest specified by the digest.

    Note: there is no equivalent method for deleting by tag name because it is
    forbidden by the spec.
    """
    with db_disallow_replica_use():
        repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
        if repository_ref is None:
            raise NameUnknown("repository not found")

        manifest = registry_model.lookup_manifest_by_digest(repository_ref, manifest_ref)
        if manifest is None:
            raise ManifestUnknown()

        tags = registry_model.delete_tags_for_manifest(manifest)
        if not tags:
            raise ManifestUnknown()

        for tag in tags:
            track_and_log("delete_tag", repository_ref, tag=tag.name, digest=manifest_ref)

        if app.config.get("FEATURE_QUOTA_MANAGEMENT", False):
            repository.force_cache_repo_size(repository_ref.id)

        return Response(status=202)
Ejemplo n.º 8
0
def _write_manifest_and_log(namespace_name, repo_name, tag_name,
                            manifest_impl):
    repository_ref, manifest, tag = _write_manifest(namespace_name, repo_name,
                                                    tag_name, manifest_impl)

    # Queue all blob manifests for replication.
    if features.STORAGE_REPLICATION:
        blobs = registry_model.get_manifest_local_blobs(manifest)
        if blobs is None:
            logger.error('Could not lookup blobs for manifest `%s`',
                         manifest.digest)
        else:
            with queue_replication_batch(
                    namespace_name) as queue_storage_replication:
                for blob_digest in blobs:
                    queue_storage_replication(blob_digest)

    track_and_log('push_repo', repository_ref, tag=tag_name)
    spawn_notification(repository_ref, 'repo_push',
                       {'updated_tags': [tag_name]})
    metric_queue.repository_push.Inc(
        labelvalues=[namespace_name, repo_name, 'v2', True])

    return Response(
        'OK',
        status=202,
        headers={
            'Docker-Content-Digest':
            manifest.digest,
            'Location':
            url_for('v2.fetch_manifest_by_digest',
                    repository='%s/%s' % (namespace_name, repo_name),
                    manifest_ref=manifest.digest),
        },
    )
Ejemplo n.º 9
0
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
    repository_ref = registry_model.lookup_repository(namespace_name,
                                                      repo_name)
    if repository_ref is None:
        raise NameUnknown()

    manifest = registry_model.lookup_manifest_by_digest(
        repository_ref, manifest_ref)
    if manifest is None:
        raise ManifestUnknown()

    manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary(
        namespace_name, repo_name, '$digest', manifest)
    if manifest_digest is None:
        raise ManifestUnknown()

    track_and_log('pull_repo', repository_ref, manifest_digest=manifest_ref)
    metric_queue.repository_pull.Inc(
        labelvalues=[namespace_name, repo_name, 'v2', True])

    return Response(manifest_bytes.as_unicode(),
                    status=200,
                    headers={
                        'Content-Type': manifest_media_type,
                        'Docker-Content-Digest': manifest_digest,
                    })
Ejemplo n.º 10
0
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
    repository_ref = registry_model.lookup_repository(namespace_name,
                                                      repo_name)
    if repository_ref is None:
        image_pulls.labels("v2_1", "manifest", 404).inc()
        raise NameUnknown()

    manifest = registry_model.lookup_manifest_by_digest(
        repository_ref, manifest_ref)
    if manifest is None:
        image_pulls.labels("v2_1", "manifest", 404).inc()
        raise ManifestUnknown()

    manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary(
        namespace_name, repo_name, "$digest", manifest)
    if manifest_digest is None:
        image_pulls.labels("v2_1", "manifest", 404).inc()
        raise ManifestUnknown()

    track_and_log("pull_repo", repository_ref, manifest_digest=manifest_ref)
    image_pulls.labels("v2_1", "manifest", 200).inc()

    return Response(
        manifest_bytes.as_unicode(),
        status=200,
        headers={
            "Content-Type": manifest_media_type,
            "Docker-Content-Digest": manifest_digest,
        },
    )
Ejemplo n.º 11
0
def get_repository_images(namespace_name, repo_name):
    repository_ref = registry_model.lookup_repository(namespace_name,
                                                      repo_name,
                                                      kind_filter="image")

    permission = ReadRepositoryPermission(namespace_name, repo_name)
    if permission.can() or (repository_ref and repository_ref.is_public):
        # We can't rely on permissions to tell us if a repo exists anymore
        if repository_ref is None:
            image_pulls.labels("v1", "tag", 404).inc()
            abort(404, message="Unknown repository", issue="unknown-repo")

        logger.debug("Building repository image response")
        resp = make_response(json.dumps([]), 200)
        resp.mimetype = "application/json"

        track_and_log("pull_repo",
                      repository_ref,
                      analytics_name="pull_repo_100x",
                      analytics_sample=0.01)
        image_pulls.labels("v1", "tag", 200).inc()
        return resp

    image_pulls.labels("v1", "tag", 403).inc()
    abort(403)
Ejemplo n.º 12
0
def update_images(namespace_name, repo_name):
    permission = ModifyRepositoryPermission(namespace_name, repo_name)

    if permission.can():
        logger.debug('Looking up repository')
        repository_ref = registry_model.lookup_repository(namespace_name,
                                                          repo_name,
                                                          kind_filter='image')
        if repository_ref is None:
            # Make sure the repo actually exists.
            abort(404, message='Unknown repository', issue='unknown-repo')

        builder = lookup_manifest_builder(repository_ref,
                                          session.get('manifest_builder'),
                                          storage, docker_v2_signing_key)
        if builder is None:
            abort(400)

        # Generate a job for each notification that has been added to this repo
        logger.debug('Adding notifications for repository')
        event_data = {
            'updated_tags': [tag.name for tag in builder.committed_tags],
        }

        builder.done()

        track_and_log('push_repo', repository_ref)
        spawn_notification(repository_ref, 'repo_push', event_data)
        metric_queue.repository_push.Inc(
            labelvalues=[namespace_name, repo_name, 'v1', True])
        return make_response('Updated', 204)

    abort(403)
Ejemplo n.º 13
0
def _write_manifest_and_log(namespace_name, repo_name, tag_name, manifest_impl):
    repository_ref, manifest, tag = _write_manifest(
        namespace_name, repo_name, tag_name, manifest_impl
    )

    # Queue all blob manifests for replication.
    if features.STORAGE_REPLICATION:
        blobs = registry_model.get_manifest_local_blobs(manifest)
        if blobs is None:
            logger.error("Could not lookup blobs for manifest `%s`", manifest.digest)
        else:
            with queue_replication_batch(namespace_name) as queue_storage_replication:
                for blob_digest in blobs:
                    queue_storage_replication(blob_digest)

    track_and_log("push_repo", repository_ref, tag=tag_name)
    spawn_notification(repository_ref, "repo_push", {"updated_tags": [tag_name]})
    metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, "v2", True])

    return Response(
        "OK",
        status=202,
        headers={
            "Docker-Content-Digest": manifest.digest,
            "Location": url_for(
                "v2.fetch_manifest_by_digest",
                repository="%s/%s" % (namespace_name, repo_name),
                manifest_ref=manifest.digest,
            ),
        },
    )
Ejemplo n.º 14
0
def _torrent_repo_verb(repository, tag, manifest, verb, **kwargs):
    """
    Handles returning a torrent for the given verb on the given image and tag.
    """
    if not features.BITTORRENT:
        # Torrent feature is not enabled.
        abort(406)

    # Lookup an *existing* derived storage for the verb. If the verb's image storage doesn't exist,
    # we cannot create it here, so we 406.
    derived_image = registry_model.lookup_derived_image(
        manifest,
        verb,
        storage,
        varying_metadata={"tag": tag.name},
        include_placements=True)
    if derived_image is None:
        abort(406)

    # Return the torrent.
    torrent = _torrent_for_blob(
        derived_image.blob, model.repository.is_repository_public(repository))

    # Log the action.
    track_and_log("repo_verb",
                  wrap_repository(repository),
                  tag=tag.name,
                  verb=verb,
                  torrent=True,
                  **kwargs)
    return torrent
Ejemplo n.º 15
0
def delete_tag(namespace_name, repo_name, tag):
    permission = ModifyRepositoryPermission(namespace_name, repo_name)
    repository_ref = registry_model.lookup_repository(namespace_name,
                                                      repo_name,
                                                      kind_filter="image")

    if permission.can() and repository_ref is not None:
        if not registry_model.delete_tag(repository_ref, tag):
            abort(404)

        track_and_log("delete_tag", repository_ref, tag=tag)
        return make_response("Deleted", 200)

    abort(403)
Ejemplo n.º 16
0
  def post(self, namespace_name, repository_name):
    """ Update the sync_status for a given Repository's mirroring configuration. """
    repo = model.repository.get_repository(namespace_name, repository_name)
    if not repo:
      raise NotFound()

    mirror = model.repo_mirror.get_mirror(repository=repo)
    if not mirror:
      raise NotFound()

    if mirror and model.repo_mirror.update_sync_status_to_cancel(mirror):
      track_and_log('repo_mirror_config_changed', wrap_repository(repo), changed="sync_status", to="SYNC_CANCEL")
      return '', 204

    raise NotFound()
Ejemplo n.º 17
0
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
    repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
    if repository_ref is None:
        raise NameUnknown()

    tag = registry_model.get_repo_tag(repository_ref, manifest_ref)
    if tag is None:
        if registry_model.has_expired_tag(repository_ref, manifest_ref):
            logger.debug(
                "Found expired tag %s for repository %s/%s", manifest_ref, namespace_name, repo_name
            )
            msg = (
                "Tag %s was deleted or has expired. To pull, revive via time machine" % manifest_ref
            )
            raise TagExpired(msg)

        raise ManifestUnknown()

    manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
    if manifest is None:
        # Something went wrong.
        raise ManifestInvalid()

    manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary(
        namespace_name, repo_name, manifest_ref, manifest
    )
    if manifest_bytes is None:
        raise ManifestUnknown()

    track_and_log(
        "pull_repo",
        repository_ref,
        analytics_name="pull_repo_100x",
        analytics_sample=0.01,
        tag=manifest_ref,
    )
    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, "v2", True])

    return Response(
        manifest_bytes.as_unicode(),
        status=200,
        headers={"Content-Type": manifest_media_type, "Docker-Content-Digest": manifest_digest,},
    )
Ejemplo n.º 18
0
    def put(self, namespace_name, repository_name):
        """
    Update an existing RepoMirrorRule
    """
        repo = model.repository.get_repository(namespace_name, repository_name)
        if not repo:
            raise NotFound()

        rule = model.repo_mirror.get_root_rule(repo)
        if not rule:
            return {'detail': 'The rule appears to be missing.'}, 400

        data = request.get_json()
        if model.repo_mirror.change_rule_value(
                rule, data['root_rule']['rule_value']):
            track_and_log('repo_mirror_config_changed',
                          wrap_repository(repo),
                          changed="mirror_rule",
                          to=data['root_rule']['rule_value'])
            return 200
        else:
            return {'detail': 'Unable to update rule.'}, 400
Ejemplo n.º 19
0
def get_repository_images(namespace_name, repo_name):
    repository_ref = registry_model.lookup_repository(namespace_name,
                                                      repo_name,
                                                      kind_filter='image')

    permission = ReadRepositoryPermission(namespace_name, repo_name)
    if permission.can() or (repository_ref and repository_ref.is_public):
        # We can't rely on permissions to tell us if a repo exists anymore
        if repository_ref is None:
            abort(404, message='Unknown repository', issue='unknown-repo')

        logger.debug('Building repository image response')
        resp = make_response(json.dumps([]), 200)
        resp.mimetype = 'application/json'

        track_and_log('pull_repo',
                      repository_ref,
                      analytics_name='pull_repo_100x',
                      analytics_sample=0.01)
        metric_queue.repository_pull.Inc(
            labelvalues=[namespace_name, repo_name, 'v1', True])
        return resp

    abort(403)
Ejemplo n.º 20
0
    def put(self, namespace_name, repository_name):
        """
        Allow users to modifying the repository's mirroring configuration.
        """
        values = request.get_json()

        repo = model.repository.get_repository(namespace_name, repository_name)
        if not repo:
            raise NotFound()

        mirror = model.repo_mirror.get_mirror(repo)
        if not mirror:
            raise NotFound()

        if "is_enabled" in values:
            if values["is_enabled"] == True:
                if model.repo_mirror.enable_mirror(repo):
                    track_and_log(
                        "repo_mirror_config_changed",
                        wrap_repository(repo),
                        changed="is_enabled",
                        to=True,
                    )
            if values["is_enabled"] == False:
                if model.repo_mirror.disable_mirror(repo):
                    track_and_log(
                        "repo_mirror_config_changed",
                        wrap_repository(repo),
                        changed="is_enabled",
                        to=False,
                    )

        if "external_reference" in values:
            if values["external_reference"] == "":
                return {
                    "detail": "Empty string is an invalid repository location."
                }, 400
            if model.repo_mirror.change_remote(repo,
                                               values["external_reference"]):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="external_reference",
                    to=values["external_reference"],
                )

        if "robot_username" in values:
            robot_username = values["robot_username"]
            robot = self._setup_robot_for_mirroring(namespace_name,
                                                    repository_name,
                                                    robot_username)
            if model.repo_mirror.set_mirroring_robot(repo, robot):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="robot_username",
                    to=robot_username,
                )

        if "sync_start_date" in values:
            try:
                sync_start_date = self._string_to_dt(values["sync_start_date"])
            except ValueError as e:
                return {
                    "detail": "Incorrect DateTime format for sync_start_date."
                }, 400
            if model.repo_mirror.change_sync_start_date(repo, sync_start_date):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="sync_start_date",
                    to=sync_start_date,
                )

        if "sync_interval" in values:
            if model.repo_mirror.change_sync_interval(repo,
                                                      values["sync_interval"]):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="sync_interval",
                    to=values["sync_interval"],
                )

        if "external_registry_username" in values and "external_registry_password" in values:
            username = values["external_registry_username"]
            password = values["external_registry_password"]
            if username is None and password is not None:
                return {
                    "detail":
                    "Unable to delete username while setting a password."
                }, 400
            if model.repo_mirror.change_credentials(repo, username, password):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="external_registry_username",
                    to=username,
                )
                if password is None:
                    track_and_log(
                        "repo_mirror_config_changed",
                        wrap_repository(repo),
                        changed="external_registry_password",
                        to=None,
                    )
                else:
                    track_and_log(
                        "repo_mirror_config_changed",
                        wrap_repository(repo),
                        changed="external_registry_password",
                        to="********",
                    )

        elif "external_registry_username" in values:
            username = values["external_registry_username"]
            if model.repo_mirror.change_username(repo, username):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="external_registry_username",
                    to=username,
                )

        # Do not allow specifying a password without setting a username
        if "external_registry_password" in values and "external_registry_username" not in values:
            return (
                {
                    "detail":
                    "Unable to set a new password without also specifying a username."
                },
                400,
            )

        if "external_registry_config" in values:
            external_registry_config = values.get("external_registry_config",
                                                  {})

            if "verify_tls" in external_registry_config:
                updates = {
                    "verify_tls": external_registry_config["verify_tls"]
                }
                if model.repo_mirror.change_external_registry_config(
                        repo, updates):
                    track_and_log(
                        "repo_mirror_config_changed",
                        wrap_repository(repo),
                        changed="verify_tls",
                        to=external_registry_config["verify_tls"],
                    )

            if "unsigned_images" in external_registry_config:
                updates = {
                    "unsigned_images":
                    external_registry_config["unsigned_images"]
                }
                if model.repo_mirror.change_external_registry_config(
                        repo, updates):
                    track_and_log(
                        "repo_mirror_config_changed",
                        wrap_repository(repo),
                        changed="unsigned_images",
                        to=external_registry_config["unsigned_images"],
                    )

            if "proxy" in external_registry_config:
                proxy_values = external_registry_config.get("proxy", {})

                if "http_proxy" in proxy_values:
                    updates = {
                        "proxy": {
                            "http_proxy": proxy_values["http_proxy"]
                        }
                    }
                    if model.repo_mirror.change_external_registry_config(
                            repo, updates):
                        track_and_log(
                            "repo_mirror_config_changed",
                            wrap_repository(repo),
                            changed="http_proxy",
                            to=proxy_values["http_proxy"],
                        )

                if "https_proxy" in proxy_values:
                    updates = {
                        "proxy": {
                            "https_proxy": proxy_values["https_proxy"]
                        }
                    }
                    if model.repo_mirror.change_external_registry_config(
                            repo, updates):
                        track_and_log(
                            "repo_mirror_config_changed",
                            wrap_repository(repo),
                            changed="https_proxy",
                            to=proxy_values["https_proxy"],
                        )

                if "no_proxy" in proxy_values:
                    updates = {"proxy": {"no_proxy": proxy_values["no_proxy"]}}
                    if model.repo_mirror.change_external_registry_config(
                            repo, updates):
                        track_and_log(
                            "repo_mirror_config_changed",
                            wrap_repository(repo),
                            changed="no_proxy",
                            to=proxy_values["no_proxy"],
                        )

        if "root_rule" in values:

            if values["root_rule"]["rule_kind"] != "tag_glob_csv":
                raise ValidationError(
                    'validation failed: rule_kind must be "tag_glob_csv"')

            if model.repo_mirror.change_rule(
                    repo, RepoMirrorRuleType.TAG_GLOB_CSV,
                    values["root_rule"]["rule_value"]):
                track_and_log(
                    "repo_mirror_config_changed",
                    wrap_repository(repo),
                    changed="mirror_rule",
                    to=values["root_rule"]["rule_value"],
                )

        return "", 201
Ejemplo n.º 21
0
def _repo_verb(namespace,
               repository,
               tag_name,
               verb,
               formatter,
               sign=False,
               checker=None,
               **kwargs):
    # Verify that the image exists and that we have access to it.
    logger.debug(
        'Verifying repo verb %s for repository %s/%s with user %s with mimetype %s',
        verb, namespace, repository, get_authenticated_user(),
        request.accept_mimetypes.best)
    tag, manifest, schema1_manifest = _verify_repo_verb(
        storage, namespace, repository, tag_name, verb, checker)

    # Load the repository for later.
    repo = model.repository.get_repository(namespace, repository)
    if repo is None:
        abort(404)

    # Check for torrent. If found, we return a torrent for the repo verb image (if the derived
    # image already exists).
    if request.accept_mimetypes.best == 'application/x-bittorrent':
        metric_queue.repository_pull.Inc(
            labelvalues=[namespace, repository, verb + '+torrent', True])
        return _torrent_repo_verb(repo, tag, manifest, verb, **kwargs)

    # Log the action.
    track_and_log('repo_verb',
                  wrap_repository(repo),
                  tag=tag.name,
                  verb=verb,
                  **kwargs)
    metric_queue.repository_pull.Inc(
        labelvalues=[namespace, repository, verb, True])

    is_readonly = app.config.get('REGISTRY_STATE', 'normal') == 'readonly'

    # Lookup/create the derived image for the verb and repo image.
    if is_readonly:
        derived_image = registry_model.lookup_derived_image(
            manifest,
            verb,
            storage,
            varying_metadata={'tag': tag.name},
            include_placements=True)
    else:
        derived_image = registry_model.lookup_or_create_derived_image(
            manifest,
            verb,
            storage.preferred_locations[0],
            storage,
            varying_metadata={'tag': tag.name},
            include_placements=True)
        if derived_image is None:
            logger.error(
                'Could not create or lookup a derived image for manifest %s',
                manifest)
            abort(400)

    if derived_image is not None and not derived_image.blob.uploading:
        logger.debug('Derived %s image %s exists in storage', verb,
                     derived_image)
        is_head_request = request.method == 'HEAD'

        metric_queue.pull_byte_count.Inc(derived_image.blob.compressed_size,
                                         labelvalues=[verb])

        download_url = storage.get_direct_download_url(
            derived_image.blob.placements,
            derived_image.blob.storage_path,
            head=is_head_request)
        if download_url:
            logger.debug('Redirecting to download URL for derived %s image %s',
                         verb, derived_image)
            return redirect(download_url)

        # Close the database handle here for this process before we send the long download.
        database.close_db_filter(None)

        logger.debug('Sending cached derived %s image %s', verb, derived_image)
        return send_file(storage.stream_read_file(
            derived_image.blob.placements, derived_image.blob.storage_path),
                         mimetype=LAYER_MIMETYPE)

    logger.debug('Building and returning derived %s image', verb)

    # Close the database connection before any process forking occurs. This is important because
    # the Postgres driver does not react kindly to forking, so we need to make sure it is closed
    # so that each process will get its own unique connection.
    database.close_db_filter(None)

    def _cleanup():
        # Close any existing DB connection once the process has exited.
        database.close_db_filter(None)

    hasher = PieceHasher(app.config['BITTORRENT_PIECE_SIZE'])

    def _store_metadata_and_cleanup():
        if is_readonly:
            return

        with database.UseThenDisconnect(app.config):
            registry_model.set_torrent_info(
                derived_image.blob, app.config['BITTORRENT_PIECE_SIZE'],
                hasher.final_piece_hashes())
            registry_model.set_derived_image_size(derived_image,
                                                  hasher.hashed_bytes)

    # Create a queue process to generate the data. The queue files will read from the process
    # and send the results to the client and storage.
    unique_id = (derived_image.unique_id if derived_image is not None else
                 hashlib.sha256('%s:%s' % (verb, uuid.uuid4())).hexdigest())
    handlers = [hasher.update]
    reporter = VerbReporter(verb)
    args = (formatter, tag, schema1_manifest, unique_id, handlers, reporter)
    queue_process = QueueProcess(
        _open_stream,
        8 * 1024,
        10 * 1024 * 1024,  # 8K/10M chunk/max
        args,
        finished=_store_metadata_and_cleanup)

    client_queue_file = QueueFile(queue_process.create_queue(), 'client')

    if not is_readonly:
        storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')

        # If signing is required, add a QueueFile for signing the image as we stream it out.
        signing_queue_file = None
        if sign and signer.name:
            signing_queue_file = QueueFile(queue_process.create_queue(),
                                           'signing')

    # Start building.
    queue_process.run()

    # Start the storage saving.
    if not is_readonly:
        storage_args = (verb, derived_image, storage_queue_file)
        QueueProcess.run_process(_write_derived_image_to_storage,
                                 storage_args,
                                 finished=_cleanup)

        if sign and signer.name:
            signing_args = (verb, derived_image, signing_queue_file)
            QueueProcess.run_process(_sign_derived_image,
                                     signing_args,
                                     finished=_cleanup)

    # Close the database handle here for this process before we send the long download.
    database.close_db_filter(None)

    # Return the client's data.
    return send_file(client_queue_file, mimetype=LAYER_MIMETYPE)
Ejemplo n.º 22
0
    def put(self, namespace_name, repository_name):
        """ Allow users to modifying the repository's mirroring configuration. """
        values = request.get_json()

        repo = model.repository.get_repository(namespace_name, repository_name)
        if not repo:
            raise NotFound()

        mirror = model.repo_mirror.get_mirror(repo)
        if not mirror:
            raise NotFound()

        if 'is_enabled' in values:
            if values['is_enabled'] == True:
                if model.repo_mirror.enable_mirror(repo):
                    track_and_log('repo_mirror_config_changed',
                                  wrap_repository(repo),
                                  changed='is_enabled',
                                  to=True)
            if values['is_enabled'] == False:
                if model.repo_mirror.disable_mirror(repo):
                    track_and_log('repo_mirror_config_changed',
                                  wrap_repository(repo),
                                  changed='is_enabled',
                                  to=False)

        if 'external_reference' in values:
            if values['external_reference'] == '':
                return {
                    'detail': 'Empty string is an invalid repository location.'
                }, 400
            if model.repo_mirror.change_remote(repo,
                                               values['external_reference']):
                track_and_log('repo_mirror_config_changed',
                              wrap_repository(repo),
                              changed='external_reference',
                              to=values['external_reference'])

        if 'robot_username' in values:
            robot_username = values['robot_username']
            robot = self._setup_robot_for_mirroring(namespace_name,
                                                    repository_name,
                                                    robot_username)
            if model.repo_mirror.set_mirroring_robot(repo, robot):
                track_and_log('repo_mirror_config_changed',
                              wrap_repository(repo),
                              changed='robot_username',
                              to=robot_username)

        if 'sync_start_date' in values:
            try:
                sync_start_date = self._string_to_dt(values['sync_start_date'])
            except ValueError as e:
                return {
                    'detail': 'Incorrect DateTime format for sync_start_date.'
                }, 400
            if model.repo_mirror.change_sync_start_date(repo, sync_start_date):
                track_and_log('repo_mirror_config_changed',
                              wrap_repository(repo),
                              changed='sync_start_date',
                              to=sync_start_date)

        if 'sync_interval' in values:
            if model.repo_mirror.change_sync_interval(repo,
                                                      values['sync_interval']):
                track_and_log('repo_mirror_config_changed',
                              wrap_repository(repo),
                              changed='sync_interval',
                              to=values['sync_interval'])

        if 'external_registry_username' in values and 'external_registry_password' in values:
            username = values['external_registry_username']
            password = values['external_registry_password']
            if username is None and password is not None:
                return {
                    'detail':
                    'Unable to delete username while setting a password.'
                }, 400
            if model.repo_mirror.change_credentials(repo, username, password):
                track_and_log('repo_mirror_config_changed',
                              wrap_repository(repo),
                              changed='external_registry_username',
                              to=username)
                if password is None:
                    track_and_log('repo_mirror_config_changed',
                                  wrap_repository(repo),
                                  changed='external_registry_password',
                                  to=None)
                else:
                    track_and_log('repo_mirror_config_changed',
                                  wrap_repository(repo),
                                  changed='external_registry_password',
                                  to="********")

        elif 'external_registry_username' in values:
            username = values['external_registry_username']
            if model.repo_mirror.change_username(repo, username):
                track_and_log('repo_mirror_config_changed',
                              wrap_repository(repo),
                              changed='external_registry_username',
                              to=username)

        # Do not allow specifying a password without setting a username
        if 'external_registry_password' in values and 'external_registry_username' not in values:
            return {
                'detail':
                'Unable to set a new password without also specifying a username.'
            }, 400

        if 'external_registry_config' in values:
            external_registry_config = values.get('external_registry_config',
                                                  {})

            if 'verify_tls' in external_registry_config:
                updates = {
                    'verify_tls': external_registry_config['verify_tls']
                }
                if model.repo_mirror.change_external_registry_config(
                        repo, updates):
                    track_and_log('repo_mirror_config_changed',
                                  wrap_repository(repo),
                                  changed='verify_tls',
                                  to=external_registry_config['verify_tls'])

            if 'proxy' in external_registry_config:
                proxy_values = external_registry_config.get('proxy', {})

                if 'http_proxy' in proxy_values:
                    updates = {
                        'proxy': {
                            'http_proxy': proxy_values['http_proxy']
                        }
                    }
                    if model.repo_mirror.change_external_registry_config(
                            repo, updates):
                        track_and_log('repo_mirror_config_changed',
                                      wrap_repository(repo),
                                      changed='http_proxy',
                                      to=proxy_values['http_proxy'])

                if 'https_proxy' in proxy_values:
                    updates = {
                        'proxy': {
                            'https_proxy': proxy_values['https_proxy']
                        }
                    }
                    if model.repo_mirror.change_external_registry_config(
                            repo, updates):
                        track_and_log('repo_mirror_config_changed',
                                      wrap_repository(repo),
                                      changed='https_proxy',
                                      to=proxy_values['https_proxy'])

                if 'no_proxy' in proxy_values:
                    updates = {'proxy': {'no_proxy': proxy_values['no_proxy']}}
                    if model.repo_mirror.change_external_registry_config(
                            repo, updates):
                        track_and_log('repo_mirror_config_changed',
                                      wrap_repository(repo),
                                      changed='no_proxy',
                                      to=proxy_values['no_proxy'])

        return '', 201
Ejemplo n.º 23
0
def _repo_verb(
    namespace, repository, tag_name, verb, formatter, sign=False, checker=None, **kwargs
):
    # Verify that the image exists and that we have access to it.
    logger.debug(
        "Verifying repo verb %s for repository %s/%s with user %s with mimetype %s",
        verb,
        namespace,
        repository,
        get_authenticated_user(),
        request.accept_mimetypes.best,
    )
    tag, manifest, schema1_manifest = _verify_repo_verb(
        storage, namespace, repository, tag_name, verb, checker
    )

    # Load the repository for later.
    repo = model.repository.get_repository(namespace, repository)
    if repo is None:
        abort(404)

    # Check for torrent, which is no longer supported.
    if request.accept_mimetypes.best == "application/x-bittorrent":
        abort(406)

    # Log the action.
    track_and_log("repo_verb", wrap_repository(repo), tag=tag.name, verb=verb, **kwargs)

    is_readonly = app.config.get("REGISTRY_STATE", "normal") == "readonly"

    # Lookup/create the derived image for the verb and repo image.
    if is_readonly:
        derived_image = registry_model.lookup_derived_image(
            manifest, verb, storage, varying_metadata={"tag": tag.name}, include_placements=True
        )
    else:
        derived_image = registry_model.lookup_or_create_derived_image(
            manifest,
            verb,
            storage.preferred_locations[0],
            storage,
            varying_metadata={"tag": tag.name},
            include_placements=True,
        )
        if derived_image is None:
            logger.error("Could not create or lookup a derived image for manifest %s", manifest)
            abort(400)

    if derived_image is not None and not derived_image.blob.uploading:
        logger.debug("Derived %s image %s exists in storage", verb, derived_image)
        is_head_request = request.method == "HEAD"

        if derived_image.blob.compressed_size:
            image_pulled_bytes.labels("verbs").inc(derived_image.blob.compressed_size)

        download_url = storage.get_direct_download_url(
            derived_image.blob.placements, derived_image.blob.storage_path, head=is_head_request
        )
        if download_url:
            logger.debug("Redirecting to download URL for derived %s image %s", verb, derived_image)
            return redirect(download_url)

        # Close the database handle here for this process before we send the long download.
        database.close_db_filter(None)

        logger.debug("Sending cached derived %s image %s", verb, derived_image)
        return send_file(
            storage.stream_read_file(
                derived_image.blob.placements, derived_image.blob.storage_path
            ),
            mimetype=LAYER_MIMETYPE,
        )

    logger.debug("Building and returning derived %s image", verb)
    hasher = SimpleHasher()

    # Close the database connection before any process forking occurs. This is important because
    # the Postgres driver does not react kindly to forking, so we need to make sure it is closed
    # so that each process will get its own unique connection.
    database.close_db_filter(None)

    def _cleanup():
        # Close any existing DB connection once the process has exited.
        database.close_db_filter(None)

    def _store_metadata_and_cleanup():
        if is_readonly:
            return

        with database.UseThenDisconnect(app.config):
            registry_model.set_derived_image_size(derived_image, hasher.hashed_bytes)

    # Create a queue process to generate the data. The queue files will read from the process
    # and send the results to the client and storage.
    unique_id = (
        derived_image.unique_id
        if derived_image is not None
        else hashlib.sha256(("%s:%s" % (verb, uuid.uuid4())).encode("utf-8")).hexdigest()
    )
    handlers = [hasher.update]
    reporter = VerbReporter(verb)
    args = (formatter, tag, schema1_manifest, unique_id, handlers, reporter)
    queue_process = QueueProcess(
        _open_stream,
        8 * 1024,
        10 * 1024 * 1024,  # 8K/10M chunk/max
        args,
        finished=_store_metadata_and_cleanup,
    )

    client_queue_file = QueueFile(
        queue_process.create_queue(), "client", timeout=QUEUE_FILE_TIMEOUT
    )

    if not is_readonly:
        storage_queue_file = QueueFile(
            queue_process.create_queue(), "storage", timeout=QUEUE_FILE_TIMEOUT
        )

        # If signing is required, add a QueueFile for signing the image as we stream it out.
        signing_queue_file = None
        if sign and signer.name:
            signing_queue_file = QueueFile(
                queue_process.create_queue(), "signing", timeout=QUEUE_FILE_TIMEOUT
            )

    # Start building.
    queue_process.run()

    # Start the storage saving.
    if not is_readonly:
        storage_args = (verb, derived_image, storage_queue_file, namespace, repository, tag_name)
        QueueProcess.run_process(_write_derived_image_to_storage, storage_args, finished=_cleanup)

        if sign and signer.name:
            signing_args = (verb, derived_image, signing_queue_file)
            QueueProcess.run_process(_sign_derived_image, signing_args, finished=_cleanup)

    # Close the database handle here for this process before we send the long download.
    database.close_db_filter(None)

    # Return the client's data.
    return send_file(client_queue_file, mimetype=LAYER_MIMETYPE)