def backfill_replication():
encountered = set()
query = (
Image.select(Image, ImageStorage, Repository, User)
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(User)
)
for image in query:
if image.storage.uuid in encountered:
continue
namespace = image.repository.namespace_user
locations = model.user.get_region_locations(namespace)
locations_required = locations | set(storage.default_locations)
query = (
ImageStoragePlacement.select(ImageStoragePlacement, ImageStorageLocation)
.where(ImageStoragePlacement.storage == image.storage)
.join(ImageStorageLocation)
)
existing_locations = set([p.location.name for p in query])
locations_missing = locations_required - existing_locations
if locations_missing:
print("Enqueueing image storage %s to be replicated" % (image.storage.uuid))
encountered.add(image.storage.uuid)
if not image_replication_queue.alive([image.storage.uuid]):
queue_storage_replication(image.repository.namespace_user.username, image.storage)
def put_image_layer(namespace, repository, image_id):
logger.debug("Checking repo permissions")
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
repository_ref = registry_model.lookup_repository(namespace,
repository,
kind_filter="image")
if repository_ref is None:
abort(403)
logger.debug("Checking for image in manifest builder")
builder = lookup_manifest_builder(repository_ref,
session.get("manifest_builder"), store,
docker_v2_signing_key)
if builder is None:
abort(400)
layer = builder.lookup_layer(image_id)
if layer is None:
abort(404)
logger.debug("Storing layer data")
input_stream = request.stream
if request.headers.get("transfer-encoding") == "chunked":
# Careful, might work only with WSGI servers supporting chunked
# encoding (Gunicorn)
input_stream = request.environ["wsgi.input"]
expiration_sec = app.config["PUSH_TEMP_TAG_EXPIRATION_SEC"]
settings = BlobUploadSettings(
maximum_blob_size=app.config["MAXIMUM_LAYER_SIZE"],
committed_blob_expiration=expiration_sec,
)
extra_handlers = []
# Add a handler that copies the data into a temp file. This is used to calculate the tarsum,
# which is only needed for older versions of Docker.
requires_tarsum = bool(builder.get_layer_checksums(layer))
if requires_tarsum:
tmp, tmp_hndlr = store.temp_store_handler()
extra_handlers.append(tmp_hndlr)
# Add a handler which computes the simple Docker V1 checksum.
h, sum_hndlr = checksums.simple_checksum_handler(layer.v1_metadata_string)
extra_handlers.append(sum_hndlr)
uploaded_blob = None
try:
with upload_blob(repository_ref,
store,
settings,
extra_blob_stream_handlers=extra_handlers) as manager:
manager.upload_chunk(app.config, input_stream)
uploaded_blob = manager.commit_to_blob(app.config)
except BlobUploadException:
logger.exception("Exception when writing image data")
abort(520,
"Image %(image_id)s could not be written. Please try again.",
image_id=image_id)
# Compute the final checksum
csums = []
csums.append("sha256:{0}".format(h.hexdigest()))
try:
if requires_tarsum:
tmp.seek(0)
csums.append(
checksums.compute_tarsum(tmp, layer.v1_metadata_string))
tmp.close()
except (IOError, checksums.TarError) as exc:
logger.debug("put_image_layer: Error when computing tarsum %s", exc)
# If there was already a precomputed checksum, validate against it now.
if builder.get_layer_checksums(layer):
checksum = builder.get_layer_checksums(layer)[0]
if not builder.validate_layer_checksum(layer, checksum):
logger.debug(
"put_image_checksum: Wrong checksum. Given: %s and expected: %s",
checksum,
builder.get_layer_checksums(layer),
)
abort(
400,
"Checksum mismatch for image: %(image_id)s",
issue="checksum-mismatch",
image_id=image_id,
)
# Assign the blob to the layer in the manifest.
if not builder.assign_layer_blob(layer, uploaded_blob, csums):
abort(500, "Something went wrong")
# Send a job to the work queue to replicate the image layer.
# TODO: move this into a better place.
queue_storage_replication(namespace, uploaded_blob)
return make_response("true", 200)