def test_get_current_tag_with_multiple_expired_tags(initialized_db): repo = model.repository.create_repository("devtable", "newrepo", None) manifest, _ = create_manifest_for_testing(repo, "1") nowms = get_epoch_timestamp_ms() count = (Tag.update( lifetime_start_ms=nowms - timedelta(hours=24).total_seconds() * 1000, lifetime_end_ms=nowms - timedelta(hours=12).total_seconds() * 1000, ).where(Tag.manifest == manifest.id).execute()) expired_tag = create_temporary_tag_if_necessary(manifest, 3600) expired_tag = Tag.create( name="v6.6.6", repository=repo.id, lifetime_start_ms=nowms - timedelta(hours=10).total_seconds() * 1000, lifetime_end_ms=nowms - timedelta(hours=8).total_seconds() * 1000, reversion=False, hidden=False, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) tag = Tag.create( name="v6.6.6", repository=repo.id, lifetime_start_ms=nowms - timedelta(hours=5).total_seconds() * 1000, lifetime_end_ms=nowms + timedelta(hours=5).total_seconds() * 1000, reversion=False, hidden=False, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) current_tag = get_current_tag(repo.id, tag.name) assert current_tag.id == tag.id
def create_temporary_tag_if_necessary(manifest, expiration_sec): """ Creates a temporary tag pointing to the given manifest, with the given expiration in seconds, unless there is an existing tag that will keep the manifest around. """ tag_name = "$temp-%s" % str(uuid.uuid4()) now_ms = get_epoch_timestamp_ms() end_ms = now_ms + (expiration_sec * 1000) # Check if there is an existing tag on the manifest that won't expire within the # timeframe. If so, no need for a temporary tag. with db_transaction(): try: (Tag.select().where( Tag.manifest == manifest, (Tag.lifetime_end_ms >> None) | (Tag.lifetime_end_ms >= end_ms), ).get()) return None except Tag.DoesNotExist: pass return Tag.create( name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms, lifetime_end_ms=end_ms, reversion=False, hidden=True, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), )
def backfill_tag(repositorytag): logger.info("Backfilling tag %s", repositorytag.id) # Ensure that a mapping row doesn't already exist. If it does, nothing more to do. if lookup_map_row(repositorytag): return False # Grab the manifest for the RepositoryTag, backfilling as necessary. manifest_id = _get_manifest_id(repositorytag) if manifest_id is None: return True lifetime_start_ms = ( repositorytag.lifetime_start_ts * 1000 if repositorytag.lifetime_start_ts is not None else None ) lifetime_end_ms = ( repositorytag.lifetime_end_ts * 1000 if repositorytag.lifetime_end_ts is not None else None ) # Create the new Tag. with db_transaction(): if lookup_map_row(repositorytag): return False try: created = Tag.create( name=repositorytag.name, repository=repositorytag.repository, lifetime_start_ms=lifetime_start_ms, lifetime_end_ms=lifetime_end_ms, reversion=repositorytag.reversion, manifest=manifest_id, tag_kind=Tag.tag_kind.get_id("tag"), ) TagToRepositoryTag.create( tag=created, repository_tag=repositorytag, repository=repositorytag.repository ) except IntegrityError: logger.exception("Could not create tag for repo tag `%s`", repositorytag.id) return False logger.info("Backfilled tag %s", repositorytag.id) return True
def associate_generated_tag_manifest_with_tag(tag, manifest, storage_id_map): oci_manifest = _populate_manifest_and_blobs(tag.repository, manifest, storage_id_map) with db_transaction(): try: (Tag.select().join(TagToRepositoryTag).where( TagToRepositoryTag.repository_tag == tag)).get() except Tag.DoesNotExist: oci_tag = Tag.create( repository=tag.repository, manifest=oci_manifest, name=tag.name, reversion=tag.reversion, lifetime_start_ms=tag.lifetime_start_ts * 1000, lifetime_end_ms=(tag.lifetime_end_ts * 1000 if tag.lifetime_end_ts else None), tag_kind=Tag.tag_kind.get_id("tag"), ) TagToRepositoryTag.create(tag=oci_tag, repository_tag=tag, repository=tag.repository) return _associate_manifest(tag, oci_manifest)
def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False, oci_manifest=None, now_ms=None): now_ms = now_ms or get_epoch_timestamp_ms() now_ts = int(now_ms / 1000) with db_transaction(): try: tag = db_for_update( _tag_alive( RepositoryTag.select().where( RepositoryTag.repository == repository_id, RepositoryTag.name == tag_name), now_ts, )).get() tag.lifetime_end_ts = now_ts tag.save() # Check for an OCI tag. try: oci_tag = db_for_update( Tag.select().join(TagToRepositoryTag).where( TagToRepositoryTag.repository_tag == tag)).get() oci_tag.lifetime_end_ms = now_ms oci_tag.save() except Tag.DoesNotExist: pass except RepositoryTag.DoesNotExist: pass except IntegrityError: msg = "Tag with name %s was stale when we tried to update it; Please retry the push" raise StaleTagException(msg % tag_name) try: image_obj = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repository_id) except Image.DoesNotExist: raise DataModelException("Invalid image with id: %s" % tag_docker_image_id) try: created = RepositoryTag.create( repository=repository_id, image=image_obj, name=tag_name, lifetime_start_ts=now_ts, reversion=reversion, ) if oci_manifest: # Create the OCI tag as well. oci_tag = Tag.create( repository=repository_id, manifest=oci_manifest, name=tag_name, lifetime_start_ms=now_ms, reversion=reversion, tag_kind=Tag.tag_kind.get_id("tag"), ) TagToRepositoryTag.create(tag=oci_tag, repository_tag=created, repository=repository_id) return created except IntegrityError: msg = "Tag with name %s and lifetime start %s already exists" raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
def retarget_tag( tag_name, manifest_id, is_reversion=False, now_ms=None, raise_on_error=False, ): """ Creates or updates a tag with the specified name to point to the given manifest under its repository. If this action is a reversion to a previous manifest, is_reversion should be set to True. Returns the newly created tag row or None on error. """ try: manifest = (Manifest.select( Manifest, MediaType).join(MediaType).where(Manifest.id == manifest_id).get()) except Manifest.DoesNotExist: if raise_on_error: raise RetargetTagException("Manifest requested no longer exists") return None # CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different # name. if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES: try: parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode( manifest.manifest_bytes), validate=False) if parsed.tag != tag_name: logger.error( "Tried to re-target schema1 manifest with tag `%s` to tag `%s", parsed.tag, tag_name, ) return None except MalformedSchema1Manifest as msme: logger.exception("Could not parse schema1 manifest") if raise_on_error: raise RetargetTagException(msme) return None legacy_image = get_legacy_image_for_manifest(manifest) now_ms = now_ms or get_epoch_timestamp_ms() now_ts = int(now_ms // 1000) with db_transaction(): # Lookup an existing tag in the repository with the same name and, if present, mark it # as expired. existing_tag = get_tag(manifest.repository_id, tag_name) if existing_tag is not None: _, okay = set_tag_end_ms(existing_tag, now_ms) # TODO: should we retry here and/or use a for-update? if not okay: return None # Create a new tag pointing to the manifest with a lifetime start of now. created = Tag.create( name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms, reversion=is_reversion, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) return created
def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None, adjust_old_model=True): """ Creates or updates a tag with the specified name to point to the given manifest under its repository. If this action is a reversion to a previous manifest, is_reversion should be set to True. Returns the newly created tag row or None on error. """ try: manifest = (Manifest.select( Manifest, MediaType).join(MediaType).where(Manifest.id == manifest_id).get()) except Manifest.DoesNotExist: return None # CHECK: Make sure that we are not mistargeting a schema 1 manifest to a tag with a different # name. if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES: try: parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode( manifest.manifest_bytes), validate=False) if parsed.tag != tag_name: logger.error( "Tried to re-target schema1 manifest with tag `%s` to tag `%s", parsed.tag, tag_name, ) return None except MalformedSchema1Manifest: logger.exception("Could not parse schema1 manifest") return None legacy_image = get_legacy_image_for_manifest(manifest) now_ms = now_ms or get_epoch_timestamp_ms() now_ts = int(now_ms / 1000) with db_transaction(): # Lookup an existing tag in the repository with the same name and, if present, mark it # as expired. existing_tag = get_tag(manifest.repository_id, tag_name) if existing_tag is not None: _, okay = set_tag_end_ms(existing_tag, now_ms) # TODO: should we retry here and/or use a for-update? if not okay: return None # Create a new tag pointing to the manifest with a lifetime start of now. created = Tag.create( name=tag_name, repository=manifest.repository_id, lifetime_start_ms=now_ms, reversion=is_reversion, manifest=manifest, tag_kind=Tag.tag_kind.get_id("tag"), ) # TODO: Remove the linkage code once RepositoryTag is gone. # If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only # be pullable via the new OCI model. if adjust_old_model: if (manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None): old_style_tag = RepositoryTag.create( repository=manifest.repository_id, image=legacy_image, name=tag_name, lifetime_start_ts=now_ts, reversion=is_reversion, ) TagToRepositoryTag.create(tag=created, repository_tag=old_style_tag, repository=manifest.repository_id) tag_manifest = TagManifest.create( tag=old_style_tag, digest=manifest.digest, json_data=manifest.manifest_bytes) TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest, repository=manifest.repository_id) return created