Exemple #1
0
def find_repository_with_garbage(limit_to_gc_policy_s):
    """ Returns a repository that has garbage (defined as an expired Tag that is past
        the repo's namespace's expiration window) or None if none.
    """
    expiration_timestamp = get_epoch_timestamp_ms() - (limit_to_gc_policy_s *
                                                       1000)

    try:
        candidates = (Tag.select(Tag.repository).join(Repository).join(
            Namespace, on=(Repository.namespace_user == Namespace.id)).where(
                ~(Tag.lifetime_end_ms >> None),
                (Tag.lifetime_end_ms <= expiration_timestamp),
                (Namespace.removed_tag_expiration_s == limit_to_gc_policy_s),
                (Namespace.enabled == True),
                (Repository.state != RepositoryState.MARKED_FOR_DELETION),
            ).limit(GC_CANDIDATE_COUNT).distinct().alias("candidates"))

        found = (Tag.select(
            candidates.c.repository_id).from_(candidates).order_by(
                db_random_func()).get())

        if found is None:
            return

        return Repository.get(Repository.id == found.repository_id)
    except Tag.DoesNotExist:
        return None
    except Repository.DoesNotExist:
        return None
Exemple #2
0
def set_tag_end_ts(tag, end_ts):
    """ Sets the end timestamp for a tag. Should only be called by change_tag_expiration
      or tests.
  """
    end_ms = end_ts * 1000 if end_ts is not None else None

    with db_transaction():
        # Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
        # not changed while we were updating it expiration.
        result = (RepositoryTag.update(lifetime_end_ts=end_ts).where(
            RepositoryTag.id == tag.id,
            RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts).execute())

        # Check for a mapping to an OCI tag.
        try:
            oci_tag = (Tag.select().join(TagToRepositoryTag).where(
                TagToRepositoryTag.repository_tag == tag).get())

            (Tag.update(lifetime_end_ms=end_ms).where(
                Tag.id == oci_tag.id,
                Tag.lifetime_end_ms == oci_tag.lifetime_end_ms).execute())
        except Tag.DoesNotExist:
            pass

        return (tag.lifetime_end_ts, result > 0)
    def test_does_not_bump_tag_expiration_when_manifest_is_cached_and_upstream_errors(
            self, create_repo, proxy_manifest_response):
        repo_ref = create_repo(self.orgname, self.upstream_repository,
                               self.user)
        proxy_mock = proxy_manifest_response(
            UBI8_8_4_DIGEST, UBI8_8_4_MANIFEST_SCHEMA2,
            DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
        with patch("data.registry_model.registry_proxy_model.Proxy",
                   MagicMock(return_value=proxy_mock)):
            proxy_model = ProxyModel(
                self.orgname,
                self.upstream_repository,
                self.user,
            )
            manifest = proxy_model.lookup_manifest_by_digest(
                repo_ref, UBI8_8_4_DIGEST)
        assert manifest is not None
        first_tag = Tag.get(manifest_id=manifest.id)

        proxy_mock = proxy_manifest_response("not-existing-ref", "", "")
        with patch("data.registry_model.registry_proxy_model.Proxy",
                   MagicMock(return_value=proxy_mock)):
            proxy_model = ProxyModel(
                self.orgname,
                self.upstream_repository,
                self.user,
            )
            manifest = proxy_model.lookup_manifest_by_digest(
                repo_ref, UBI8_8_4_DIGEST)
        assert manifest is not None
        tag = Tag.get(manifest_id=manifest.id)
        assert tag.id == first_tag.id
        assert tag.lifetime_end_ms == first_tag.lifetime_end_ms
    def test_returns_None_when_manifest_no_longer_exists_upstream_and_local_cache_is_expired(
            self, create_repo, proxy_manifest_response):
        repo_ref = create_repo(self.orgname, self.upstream_repository,
                               self.user)
        proxy_mock = proxy_manifest_response(
            self.tag, UBI8_8_5_MANIFEST_SCHEMA2,
            DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
        with patch("data.registry_model.registry_proxy_model.Proxy",
                   MagicMock(return_value=proxy_mock)):
            proxy_model = ProxyModel(
                self.orgname,
                self.upstream_repository,
                self.user,
            )
            tag = proxy_model.get_repo_tag(repo_ref, self.tag)
        assert tag is not None

        # expire the tag by setting start and end time to the past
        before_ms = get_epoch_timestamp_ms() - timedelta(
            hours=24).total_seconds() * 1000
        Tag.update(
            lifetime_start_ms=before_ms,
            lifetime_end_ms=before_ms + 5,
        ).where(Tag.id == tag.id).execute()

        proxy_mock = proxy_manifest_response("not-existing-ref", "", "")
        with patch("data.registry_model.registry_proxy_model.Proxy",
                   MagicMock(return_value=proxy_mock)):
            proxy_model = ProxyModel(
                self.orgname,
                self.upstream_repository,
                self.user,
            )
            tag = proxy_model.get_repo_tag(repo_ref, self.tag)
        assert tag is None
Exemple #5
0
def test_get_current_tag_with_multiple_expired_tags(initialized_db):
    repo = model.repository.create_repository("devtable", "newrepo", None)
    manifest, _ = create_manifest_for_testing(repo, "1")
    nowms = get_epoch_timestamp_ms()
    count = (Tag.update(
        lifetime_start_ms=nowms - timedelta(hours=24).total_seconds() * 1000,
        lifetime_end_ms=nowms - timedelta(hours=12).total_seconds() * 1000,
    ).where(Tag.manifest == manifest.id).execute())
    expired_tag = create_temporary_tag_if_necessary(manifest, 3600)
    expired_tag = Tag.create(
        name="v6.6.6",
        repository=repo.id,
        lifetime_start_ms=nowms - timedelta(hours=10).total_seconds() * 1000,
        lifetime_end_ms=nowms - timedelta(hours=8).total_seconds() * 1000,
        reversion=False,
        hidden=False,
        manifest=manifest,
        tag_kind=Tag.tag_kind.get_id("tag"),
    )
    tag = Tag.create(
        name="v6.6.6",
        repository=repo.id,
        lifetime_start_ms=nowms - timedelta(hours=5).total_seconds() * 1000,
        lifetime_end_ms=nowms + timedelta(hours=5).total_seconds() * 1000,
        reversion=False,
        hidden=False,
        manifest=manifest,
        tag_kind=Tag.tag_kind.get_id("tag"),
    )
    current_tag = get_current_tag(repo.id, tag.name)
    assert current_tag.id == tag.id
Exemple #6
0
def create_temporary_tag_if_necessary(manifest, expiration_sec):
    """
    Creates a temporary tag pointing to the given manifest, with the given expiration in seconds,
    unless there is an existing tag that will keep the manifest around.
    """
    tag_name = "$temp-%s" % str(uuid.uuid4())
    now_ms = get_epoch_timestamp_ms()
    end_ms = now_ms + (expiration_sec * 1000)

    # Check if there is an existing tag on the manifest that won't expire within the
    # timeframe. If so, no need for a temporary tag.
    with db_transaction():
        try:
            (Tag.select().where(
                Tag.manifest == manifest,
                (Tag.lifetime_end_ms >> None) |
                (Tag.lifetime_end_ms >= end_ms),
            ).get())
            return None
        except Tag.DoesNotExist:
            pass

        return Tag.create(
            name=tag_name,
            repository=manifest.repository_id,
            lifetime_start_ms=now_ms,
            lifetime_end_ms=end_ms,
            reversion=False,
            hidden=True,
            manifest=manifest,
            tag_kind=Tag.tag_kind.get_id("tag"),
        )
Exemple #7
0
def test_set_tag_expiration_for_manifest(initialized_db):
    tag = Tag.get()
    manifest = tag.manifest
    assert manifest is not None

    set_tag_expiration_for_manifest(manifest, datetime.utcnow() + timedelta(weeks=1))

    updated_tag = Tag.get(id=tag.id)
    assert updated_tag.lifetime_end_ms is not None
Exemple #8
0
def clear_rows(initialized_db):
    # Remove all new-style rows so we can backfill.
    TagToRepositoryTag.delete().execute()
    Tag.delete().execute()
    TagManifestLabelMap.delete().execute()
    ManifestLabel.delete().execute()
    ManifestBlob.delete().execute()
    ManifestLegacyImage.delete().execute()
    TagManifestToManifest.delete().execute()
    Manifest.delete().execute()
Exemple #9
0
def find_matching_tag(repository_id, tag_names, tag_kinds=None):
    """
    Finds an alive tag in the specified repository with one of the specified tag names and returns
    it or None if none.

    Tag's returned are joined with their manifest.
    """
    assert repository_id
    assert tag_names

    query = (
        Tag.select(Tag, Manifest)
        .join(Manifest)
        .where(Tag.repository == repository_id)
        .where(Tag.name << tag_names)
    )

    if tag_kinds:
        query = query.where(Tag.tag_kind << tag_kinds)

    try:
        found = filter_to_alive_tags(query).get()
        assert not found.hidden
        return found
    except Tag.DoesNotExist:
        return None
 def test_create_temp_tags_for_newly_created_sub_manifests_on_manifest_list(
         self, create_repo):
     repo_ref = create_repo(self.orgname, self.upstream_repository,
                            self.user)
     input_manifest = parse_manifest_from_bytes(
         Bytes.for_string_or_unicode(UBI8_LATEST_MANIFEST_LIST),
         DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
         sparse_manifest_support=True,
     )
     proxy_model = ProxyModel(
         self.orgname,
         self.upstream_repository,
         self.user,
     )
     manifest, _ = proxy_model._create_manifest_and_retarget_tag(
         repo_ref, input_manifest, self.tag)
     mchildren = ManifestChild.select(
         ManifestChild.child_manifest_id).where(
             ManifestChild.manifest == manifest.id)
     tags = Tag.select().join(
         ManifestChild,
         on=(Tag.manifest_id == ManifestChild.child_manifest_id))
     assert mchildren.count() == tags.count()
     assert all([t.hidden
                 for t in tags]), "all sub manifest tags must be hidden"
     assert all(
         [t.name != self.tag for t in tags]
     ), "sub manifest tags must have temp tag name, not parent manifest name"
Exemple #11
0
def cache_namespace_repository_sizes(namespace_name):
    namespace = user.get_user_or_org(namespace_name)
    now_ms = get_epoch_timestamp_ms()

    subquery = (Tag.select(Tag.repository_id).where(
        Tag.hidden == False).where((Tag.lifetime_end_ms >> None)
                                   | (Tag.lifetime_end_ms > now_ms)).group_by(
                                       Tag.repository_id).having(
                                           fn.Count(Tag.name) > 0))

    namespace_repo_sizes = (Manifest.select(
        (Repository.id).alias("repository_id"),
        (Repository.name).alias("repository_name"),
        fn.sum(Manifest.layers_compressed_size).alias("repository_size"),
    ).join(Repository).join(
        subquery, on=(subquery.c.repository_id == Repository.id)).where(
            Repository.namespace_user == namespace.id).group_by(Repository.id))

    insert_query = (namespace_repo_sizes.select(
        Repository.id, fn.sum(Manifest.layers_compressed_size)).join_from(
            Repository, RepositorySize,
            JOIN.LEFT_OUTER).where(RepositorySize.repository_id.is_null()))

    RepositorySize.insert_from(
        insert_query,
        fields=[RepositorySize.repository_id, RepositorySize.size_bytes],
    ).execute()
Exemple #12
0
def list_repository_tag_history(repository_id,
                                page,
                                page_size,
                                specific_tag_name=None,
                                active_tags_only=False,
                                since_time_ms=None):
    """ Returns a tuple of the full set of tags found in the specified repository, including those
      that are no longer alive (unless active_tags_only is True), and whether additional tags exist.
      If specific_tag_name is given, the tags are further filtered by name. If since is given, tags
      are further filtered to newer than that date.

      Note that the returned Manifest will not contain the manifest contents.
  """
    query = (Tag.select(Tag, Manifest.id, Manifest.digest,
                        Manifest.media_type).join(Manifest).where(
                            Tag.repository == repository_id).order_by(
                                Tag.lifetime_start_ms.desc(),
                                Tag.name).limit(page_size + 1).offset(
                                    page_size * (page - 1)))

    if specific_tag_name is not None:
        query = query.where(Tag.name == specific_tag_name)

    if since_time_ms is not None:
        query = query.where((Tag.lifetime_start_ms > since_time_ms)
                            | (Tag.lifetime_end_ms > since_time_ms))

    if active_tags_only:
        query = filter_to_alive_tags(query)

    query = filter_to_visible_tags(query)
    results = list(query)

    return results[0:page_size], len(results) > page_size
Exemple #13
0
def _delete_tag(tag, now_ms):
    """
    Deletes the given tag by marking it as expired.
    """
    now_ts = int(now_ms // 1000)

    with db_transaction():
        updated = (Tag.update(lifetime_end_ms=now_ms).where(
            Tag.id == tag.id,
            Tag.lifetime_end_ms == tag.lifetime_end_ms).execute())
        if updated != 1:
            return None

        # TODO: Remove the linkage code once RepositoryTag is gone.
        try:
            old_style_tag = (TagToRepositoryTag.select(
                TagToRepositoryTag, RepositoryTag).join(RepositoryTag).where(
                    TagToRepositoryTag.tag == tag).get()).repository_tag

            old_style_tag.lifetime_end_ts = now_ts
            old_style_tag.save()
        except TagToRepositoryTag.DoesNotExist:
            pass

        return tag
Exemple #14
0
def get_tag_by_id(tag_id):
    """ Returns the tag with the given ID, joined with its manifest or None if none. """
    try:
        return Tag.select(
            Tag, Manifest).join(Manifest).where(Tag.id == tag_id).get()
    except Tag.DoesNotExist:
        return None
Exemple #15
0
def test_create_temporary_tag_if_necessary(initialized_db):
    tag = Tag.get()
    manifest = tag.manifest
    assert manifest is not None

    # Ensure no tag is created, since an existing one is present.
    created = create_temporary_tag_if_necessary(manifest, 60)
    assert created is None

    # Mark the tag as deleted.
    tag.lifetime_end_ms = 1
    tag.save()

    # Now create a temp tag.
    created = create_temporary_tag_if_necessary(manifest, 60)
    assert created is not None
    assert created.hidden
    assert created.name.startswith("$temp-")
    assert created.manifest == manifest
    assert created.lifetime_end_ms is not None
    assert created.lifetime_end_ms == (created.lifetime_start_ms + 60000)

    # Try again and ensure it is not created.
    created = create_temporary_tag_if_necessary(manifest, 30)
    assert created is None
Exemple #16
0
def _purge_oci_tag(tag, context, allow_non_expired=False):
  assert tag.repository_id == context.repository.id

  if not allow_non_expired:
    assert tag.lifetime_end_ms is not None
    assert tag.lifetime_end_ms <= oci_tag.get_epoch_timestamp_ms()

  # Add the manifest to be GCed.
  context.add_manifest_id(tag.manifest_id)

  with db_transaction():
    # Reload the tag and verify its lifetime_end_ms has not changed.
    try:
      reloaded_tag = db_for_update(Tag.select().where(Tag.id == tag.id)).get()
    except Tag.DoesNotExist:
      return False

    assert reloaded_tag.id == tag.id
    assert reloaded_tag.repository_id == context.repository.id
    if reloaded_tag.lifetime_end_ms != tag.lifetime_end_ms:
      return False

    # Delete mapping rows.
    TagToRepositoryTag.delete().where(TagToRepositoryTag.tag == tag).execute()

    # Delete the tag.
    tag.delete_instance()
Exemple #17
0
def set_tag_end_ms(tag, end_ms):
    """
    Sets the end timestamp for a tag.

    Should only be called by change_tag_expiration or tests.
    """

    with db_transaction():
        updated = (Tag.update(lifetime_end_ms=end_ms).where(
            Tag.id == tag).where(
                Tag.lifetime_end_ms == tag.lifetime_end_ms).execute())
        if updated != 1:
            return (None, False)

        # TODO: Remove the linkage code once RepositoryTag is gone.
        try:
            old_style_tag = (TagToRepositoryTag.select(
                TagToRepositoryTag, RepositoryTag).join(RepositoryTag).where(
                    TagToRepositoryTag.tag == tag).get()).repository_tag

            old_style_tag.lifetime_end_ts = end_ms // 1000 if end_ms is not None else None
            old_style_tag.save()
        except TagToRepositoryTag.DoesNotExist:
            pass

        return (tag.lifetime_end_ms, True)
Exemple #18
0
def test_list_repository_tag_history_all_tags(initialized_db):
    for tag in Tag.select():
        repo = tag.repository
        with assert_query_count(1):
            results, _ = list_repository_tag_history(repo, 1, 1000)

        assert (tag in results) == (not tag.hidden)
Exemple #19
0
def test_lookup_alive_tags_shallow(initialized_db):
    found = False
    for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
        tags = lookup_alive_tags_shallow(tag.repository)
        found = True
        assert tag in tags

    assert found

    # Ensure hidden tags cannot be listed.
    tag = Tag.get()
    tag.hidden = True
    tag.save()

    tags = lookup_alive_tags_shallow(tag.repository)
    assert tag not in tags
Exemple #20
0
def test_lookup_manifest_child_tag(initialized_db):
    repository = create_repository("devtable", "newrepo", None)
    manifest, manifest_impl = create_manifest_for_testing(repository)

    # Mark the hidden tag as dead.
    hidden_tag = Tag.get(manifest=manifest, hidden=True)
    hidden_tag.lifetime_end_ms = hidden_tag.lifetime_start_ms
    hidden_tag.save()

    # Ensure the manifest cannot currently be looked up, as it is not pointed to by an alive tag.
    assert lookup_manifest(repository, manifest.digest) is None
    assert lookup_manifest(repository, manifest.digest,
                           allow_dead=True) is not None

    # Populate a manifest list.
    list_builder = DockerSchema2ManifestListBuilder()
    list_builder.add_manifest(manifest_impl, "amd64", "linux")
    manifest_list = list_builder.build()

    # Write the manifest list, which should also write the manifests themselves.
    created_tuple = get_or_create_manifest(repository, manifest_list, storage)
    assert created_tuple is not None

    # Since the manifests are not yet referenced by a tag, they cannot be found.
    assert lookup_manifest(repository, manifest.digest) is None
    assert lookup_manifest(repository, manifest_list.digest) is None

    # Unless we ask for "dead" manifests.
    assert lookup_manifest(repository, manifest.digest,
                           allow_dead=True) is not None
    assert lookup_manifest(repository, manifest_list.digest,
                           allow_dead=True) is not None
Exemple #21
0
def change_tag_expiration(tag_id, expiration_datetime):
    """
    Changes the expiration of the specified tag to the given expiration datetime.

    If the expiration datetime is None, then the tag is marked as not expiring. Returns a tuple of
    the previous expiration timestamp in seconds (if any), and whether the operation succeeded.
    """
    try:
        tag = Tag.get(id=tag_id)
    except Tag.DoesNotExist:
        return (None, False)

    new_end_ms = None
    min_expire_sec = convert_to_timedelta(
        config.app_config.get("LABELED_EXPIRATION_MINIMUM", "1h"))
    max_expire_sec = convert_to_timedelta(
        config.app_config.get("LABELED_EXPIRATION_MAXIMUM", "104w"))

    if expiration_datetime is not None:
        lifetime_start_ts = int(tag.lifetime_start_ms // 1000)

        offset = timegm(expiration_datetime.utctimetuple()) - lifetime_start_ts
        offset = min(max(offset, min_expire_sec.total_seconds()),
                     max_expire_sec.total_seconds())
        new_end_ms = tag.lifetime_start_ms + (offset * 1000)

    if new_end_ms == tag.lifetime_end_ms:
        return (None, True)

    return set_tag_end_ms(tag, new_end_ms)
Exemple #22
0
    def test_creates_manifest_on_first_pull(self, test_name,
                                            proxy_manifest_response):
        test_params = storage_test_cases[test_name]
        repo = f"{self.orgname}/{test_params['image_name']}"
        params = {
            "repository": repo,
            "manifest_ref": test_params["manifest_ref"],
        }
        proxy_mock = proxy_manifest_response(
            test_params["manifest_ref"],
            test_params["manifest_json"],
            test_params["manifest_type"],
        )
        with patch("data.registry_model.registry_proxy_model.Proxy",
                   MagicMock(return_value=proxy_mock)):
            headers = _get_auth_headers(self.sub, self.ctx, repo)
            headers["Accept"] = ", ".join(
                DOCKER_SCHEMA2_CONTENT_TYPES.union(OCI_CONTENT_TYPES).union(
                    DOCKER_SCHEMA1_CONTENT_TYPES))
            conduct_call(
                self.client,
                test_params["view_name"],
                url_for,
                "GET",
                params,
                expected_code=200,
                headers=headers,
            )

        repository_ref = registry_model.lookup_repository(
            self.orgname, test_params["image_name"])
        assert repository_ref is not None
        tag = registry_model.get_repo_tag(repository_ref,
                                          test_params["manifest_ref"])

        # when testing the fetch_manifest_by_digest view the tag created is temporary,
        # and it does not refer to the manifest digest (manifest_ref), so we need to
        # fetch it by its link to the repository instead.
        if test_params["ref_type"] == "digest":
            tag = Tag.filter(Tag.repository_id == repository_ref.id).get()
            # get_manifest_for_tag returns a tag of datatypes.Tag, so we convert
            # the one we have to that type.
            tag = datatypes.Tag.for_tag(tag, SyntheticIDHandler())

        assert tag is not None
        manifest = registry_model.get_manifest_for_tag(tag)
        assert manifest is not None

        output_manifest = manifest.get_parsed_manifest()
        input_manifest = parse_manifest_from_bytes(
            Bytes.for_string_or_unicode(test_params["manifest_json"]),
            test_params["manifest_type"],
            sparse_manifest_support=True,
        )
        assert output_manifest.schema_version == input_manifest.schema_version
        assert output_manifest.media_type == input_manifest.media_type
        assert output_manifest.is_manifest_list == input_manifest.is_manifest_list
        assert output_manifest.digest == input_manifest.digest
        assert output_manifest.manifest_dict == input_manifest.manifest_dict
Exemple #23
0
def test_lookup_manifest(initialized_db):
    found = False
    for tag in filter_to_alive_tags(Tag.select()):
        found = True
        repo = tag.repository
        digest = tag.manifest.digest
        with assert_query_count(1):
            assert lookup_manifest(repo, digest) == tag.manifest

    assert found

    for tag in Tag.select():
        repo = tag.repository
        digest = tag.manifest.digest
        with assert_query_count(1):
            assert lookup_manifest(repo, digest,
                                   allow_dead=True) == tag.manifest
Exemple #24
0
def list_alive_tags(repository_id):
    """ Returns a list of all the tags alive in the specified repository.
      Tag's returned are joined with their manifest.
  """
    query = (Tag.select(
        Tag, Manifest).join(Manifest).where(Tag.repository == repository_id))

    return filter_to_alive_tags(query)
Exemple #25
0
def _purge_repository_contents(repo):
  """ Purges all the contents of a repository, removing all of its tags,
      manifests and images.
  """
  logger.debug('Purging repository %s', repo)

  # Purge via all the tags.
  while True:
    found = False
    for tags in _chunk_iterate_for_deletion(Tag.select().where(Tag.repository == repo)):
      logger.debug('Found %s tags to GC under repository %s', len(tags), repo)
      found = True
      context = _GarbageCollectorContext(repo)
      for tag in tags:
        logger.debug('Deleting tag %s under repository %s', tag, repo)
        assert tag.repository_id == repo.id
        _purge_oci_tag(tag, context, allow_non_expired=True)

      _run_garbage_collection(context)

    if not found:
      break

  # TODO: remove this once we're fully on the OCI data model.
  while True:
    found = False
    repo_tag_query = RepositoryTag.select().where(RepositoryTag.repository == repo)
    for tags in _chunk_iterate_for_deletion(repo_tag_query):
      logger.debug('Found %s tags to GC under repository %s', len(tags), repo)
      found = True
      context = _GarbageCollectorContext(repo)

      for tag in tags:
        logger.debug('Deleting tag %s under repository %s', tag, repo)
        assert tag.repository_id == repo.id
        _purge_pre_oci_tag(tag, context, allow_non_expired=True)

      _run_garbage_collection(context)

    if not found:
      break

  # Add all remaining images to a new context. We do this here to minimize the number of images
  # we need to load.
  while True:
    found_image = False
    image_context = _GarbageCollectorContext(repo)
    for image in Image.select().where(Image.repository == repo):
      found_image = True
      logger.debug('Deleting image %s under repository %s', image, repo)
      assert image.repository_id == repo.id
      image_context.add_legacy_image_id(image.id)

    _run_garbage_collection(image_context)

    if not found_image:
      break
Exemple #26
0
def test_delete_tags_for_manifest(initialized_db):
    for tag in list(filter_to_visible_tags(filter_to_alive_tags(Tag.select()))):
        repo = tag.repository
        assert get_tag(repo, tag.name) == tag

        with assert_query_count(4):
            assert delete_tags_for_manifest(tag.manifest) == [tag]

        assert get_tag(repo, tag.name) is None
Exemple #27
0
def get_tags_for_legacy_image(image_id):
    """ Returns the Tag's that have the associated legacy image. 
    
        NOTE: This is for legacy support in the old security notification worker and should
        be removed once that code is no longer necessary.
    """
    return filter_to_alive_tags(
        Tag.select().distinct().join(Manifest).join(ManifestLegacyImage).where(
            ManifestLegacyImage.image == image_id))
Exemple #28
0
def purge_repository(repo, force=False):
    """
    Completely delete all traces of the repository.

    Will return True upon complete success, and False upon partial or total failure. Garbage
    collection is incremental and repeatable, so this return value does not need to be checked or
    responded to.
    """
    assert repo.state == RepositoryState.MARKED_FOR_DELETION or force

    # Delete the repository of all Appr-referenced entries.
    # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
    # and MySQL is... particular... about such relationships when deleting.
    if repo.kind.name == "application":
        ApprTag.delete().where(ApprTag.repository == repo,
                               ~(ApprTag.linked_tag >> None)).execute()
        ApprTag.delete().where(ApprTag.repository == repo).execute()
    else:
        # GC to remove the images and storage.
        _purge_repository_contents(repo)

    # Ensure there are no additional tags, manifests, images or blobs in the repository.
    assert ApprTag.select().where(ApprTag.repository == repo).count() == 0
    assert Tag.select().where(Tag.repository == repo).count() == 0
    assert RepositoryTag.select().where(
        RepositoryTag.repository == repo).count() == 0
    assert Manifest.select().where(Manifest.repository == repo).count() == 0
    assert ManifestBlob.select().where(
        ManifestBlob.repository == repo).count() == 0
    assert Image.select().where(Image.repository == repo).count() == 0

    # Delete any repository build triggers, builds, and any other large-ish reference tables for
    # the repository.
    _chunk_delete_all(repo, RepositoryPermission, force=force)
    _chunk_delete_all(repo, RepositoryBuild, force=force)
    _chunk_delete_all(repo, RepositoryBuildTrigger, force=force)
    _chunk_delete_all(repo, RepositoryActionCount, force=force)
    _chunk_delete_all(repo, Star, force=force)
    _chunk_delete_all(repo, AccessToken, force=force)
    _chunk_delete_all(repo, RepositoryNotification, force=force)
    _chunk_delete_all(repo, BlobUpload, force=force)
    _chunk_delete_all(repo, RepoMirrorConfig, force=force)
    _chunk_delete_all(repo, RepositoryAuthorizedEmail, force=force)

    # Delete any marker rows for the repository.
    DeletedRepository.delete().where(
        DeletedRepository.repository == repo).execute()

    # Delete the rest of the repository metadata.
    try:
        # Make sure the repository still exists.
        fetched = Repository.get(id=repo.id)
    except Repository.DoesNotExist:
        return False

    fetched.delete_instance(recursive=True, delete_nullable=False, force=force)
    return True
Exemple #29
0
def _check_manifest_used(manifest_id):
  assert manifest_id is not None

  with db_transaction():
    # Check if the manifest is referenced by any other tag.
    try:
      Tag.select().where(Tag.manifest == manifest_id).get()
      return True
    except Tag.DoesNotExist:
      pass

    # Check if the manifest is referenced as a child of another manifest.
    try:
      ManifestChild.select().where(ManifestChild.child_manifest == manifest_id).get()
      return True
    except ManifestChild.DoesNotExist:
      pass

  return False
Exemple #30
0
def test_get_tag(initialized_db):
    found = False
    for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
        repo = tag.repository

        with assert_query_count(1):
            assert get_tag(repo, tag.name) == tag
        found = True

    assert found