def update_images(namespace_name, repo_name):
permission = ModifyRepositoryPermission(namespace_name, repo_name)
if permission.can():
logger.debug("Looking up repository")
repository_ref = registry_model.lookup_repository(namespace_name,
repo_name,
kind_filter="image")
if repository_ref is None:
# Make sure the repo actually exists.
image_pushes.labels("v1", 404, "").inc()
abort(404, message="Unknown repository", issue="unknown-repo")
builder = lookup_manifest_builder(repository_ref,
session.get("manifest_builder"),
storage, docker_v2_signing_key)
if builder is None:
image_pushes.labels("v1", 400, "").inc()
abort(400)
# Generate a job for each notification that has been added to this repo
logger.debug("Adding notifications for repository")
event_data = {
"updated_tags": [tag.name for tag in builder.committed_tags],
}
builder.done()
track_and_log("push_repo", repository_ref)
spawn_notification(repository_ref, "repo_push", event_data)
image_pushes.labels("v1", 204, "").inc()
return make_response("Updated", 204)
image_pushes.labels("v1", 403, "").inc()
abort(403)
def update_images(namespace_name, repo_name):
permission = ModifyRepositoryPermission(namespace_name, repo_name)
if permission.can():
logger.debug('Looking up repository')
repository_ref = registry_model.lookup_repository(namespace_name,
repo_name,
kind_filter='image')
if repository_ref is None:
# Make sure the repo actually exists.
abort(404, message='Unknown repository', issue='unknown-repo')
builder = lookup_manifest_builder(repository_ref,
session.get('manifest_builder'),
storage, docker_v2_signing_key)
if builder is None:
abort(400)
# Generate a job for each notification that has been added to this repo
logger.debug('Adding notifications for repository')
event_data = {
'updated_tags': [tag.name for tag in builder.committed_tags],
}
builder.done()
track_and_log('push_repo', repository_ref)
spawn_notification(repository_ref, 'repo_push', event_data)
metric_queue.repository_push.Inc(
labelvalues=[namespace_name, repo_name, 'v1', True])
return make_response('Updated', 204)
abort(403)
def put_image_json(namespace, repository, image_id):
logger.debug('Checking repo permissions')
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
repository_ref = registry_model.lookup_repository(namespace,
repository,
kind_filter='image')
if repository_ref is None:
abort(403)
builder = lookup_manifest_builder(repository_ref,
session.get('manifest_builder'), store,
docker_v2_signing_key)
if builder is None:
abort(400)
logger.debug('Parsing image JSON')
try:
uploaded_metadata = request.data
data = json.loads(uploaded_metadata.decode('utf8'))
except ValueError:
pass
if not data or not isinstance(data, dict):
abort(400,
'Invalid JSON for image: %(image_id)s\nJSON: %(json)s',
issue='invalid-request',
image_id=image_id,
json=request.data)
if 'id' not in data:
abort(400,
'Missing key `id` in JSON for image: %(image_id)s',
issue='invalid-request',
image_id=image_id)
if image_id != data['id']:
abort(400,
'JSON data contains invalid id for image: %(image_id)s',
issue='invalid-request',
image_id=image_id)
logger.debug('Looking up repo image')
location_pref = store.preferred_locations[0]
username = get_authenticated_user() and get_authenticated_user().username
layer = builder.start_layer(image_id, uploaded_metadata, location_pref,
username,
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
if layer is None:
abort(400,
'Image %(image_id)s has invalid metadata',
issue='invalid-request',
image_id=image_id)
return make_response('true', 200)
def put_image_checksum(namespace, repository, image_id):
logger.debug("Checking repo permissions")
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
repository_ref = registry_model.lookup_repository(namespace,
repository,
kind_filter="image")
if repository_ref is None:
abort(403)
# Docker Version < 0.10 (tarsum+sha):
old_checksum = request.headers.get("X-Docker-Checksum")
# Docker Version >= 0.10 (sha):
new_checksum = request.headers.get("X-Docker-Checksum-Payload")
checksum = new_checksum or old_checksum
if not checksum:
abort(
400,
"Missing checksum for image %(image_id)s",
issue="missing-checksum",
image_id=image_id,
)
logger.debug("Checking for image in manifest builder")
builder = lookup_manifest_builder(repository_ref,
session.get("manifest_builder"), store,
docker_v2_signing_key)
if builder is None:
abort(400)
layer = builder.lookup_layer(image_id)
if layer is None:
abort(404)
if old_checksum:
builder.save_precomputed_checksum(layer, checksum)
return make_response("true", 200)
if not builder.validate_layer_checksum(layer, checksum):
logger.debug(
"put_image_checksum: Wrong checksum. Given: %s and expected: %s",
checksum,
builder.get_layer_checksums(layer),
)
abort(
400,
"Checksum mismatch for image: %(image_id)s",
issue="checksum-mismatch",
image_id=image_id,
)
return make_response("true", 200)
def put_tag(namespace_name, repo_name, tag):
permission = ModifyRepositoryPermission(namespace_name, repo_name)
repository_ref = registry_model.lookup_repository(namespace_name,
repo_name,
kind_filter="image")
if permission.can() and repository_ref is not None:
if not TAG_REGEX.match(tag):
abort(400, TAG_ERROR)
image_id = json.loads(request.data)
# Check for the image ID first in a builder (for an in-progress push).
builder = lookup_manifest_builder(repository_ref,
session.get("manifest_builder"),
storage, docker_v2_signing_key)
if builder is not None:
layer = builder.lookup_layer(image_id)
if layer is not None:
commited_tag = builder.commit_tag_and_manifest(tag, layer)
if commited_tag is None:
abort(400)
return make_response("Created", 200)
# Check if there is an existing image we should use (for PUT calls outside of a normal push
# operation).
legacy_image = registry_model.get_legacy_image(repository_ref,
image_id)
if legacy_image is None:
abort(400)
if (registry_model.retarget_tag(repository_ref, tag, legacy_image,
storage, docker_v2_signing_key) is
None):
abort(400)
return make_response("Created", 200)
abort(403)
def test_build_manifest(layers, fake_session, registry_model):
repository_ref = registry_model.lookup_repository("devtable", "complex")
storage = DistributedStorage({"local_us": FakeStorage(None)}, ["local_us"])
settings = BlobUploadSettings("2M", 3600)
app_config = {"TESTING": True}
builder = create_manifest_builder(repository_ref, storage, docker_v2_signing_key)
assert (
lookup_manifest_builder(repository_ref, "anotherid", storage, docker_v2_signing_key) is None
)
assert (
lookup_manifest_builder(repository_ref, builder.builder_id, storage, docker_v2_signing_key)
is not None
)
blobs_by_layer = {}
for layer_id, parent_id, layer_bytes in layers:
# Start a new layer.
assert builder.start_layer(
layer_id, json.dumps({"id": layer_id, "parent": parent_id}), "local_us", None, 60
)
checksum = hashlib.sha1(layer_bytes).hexdigest()
# Assign it a blob.
with upload_blob(repository_ref, storage, settings) as uploader:
uploader.upload_chunk(app_config, BytesIO(layer_bytes))
blob = uploader.commit_to_blob(app_config)
blobs_by_layer[layer_id] = blob
builder.assign_layer_blob(builder.lookup_layer(layer_id), blob, [checksum])
# Validate the checksum.
assert builder.validate_layer_checksum(builder.lookup_layer(layer_id), checksum)
# Commit the manifest to a tag.
tag = builder.commit_tag_and_manifest("somenewtag", builder.lookup_layer(layers[-1][0]))
assert tag
assert tag in builder.committed_tags
# Mark the builder as done.
builder.done()
# Verify the legacy image for the tag.
found = registry_model.get_repo_tag(repository_ref, "somenewtag", include_legacy_image=True)
assert found
assert found.name == "somenewtag"
assert found.legacy_image.docker_image_id == layers[-1][0]
# Verify the blob and manifest.
manifest = registry_model.get_manifest_for_tag(found)
assert manifest
parsed = manifest.get_parsed_manifest()
assert len(list(parsed.layers)) == len(layers)
for index, (layer_id, parent_id, layer_bytes) in enumerate(layers):
assert list(parsed.blob_digests)[index] == blobs_by_layer[layer_id].digest
assert list(parsed.layers)[index].v1_metadata.image_id == layer_id
assert list(parsed.layers)[index].v1_metadata.parent_image_id == parent_id
assert parsed.leaf_layer_v1_image_id == layers[-1][0]
def put_image_json(namespace, repository, image_id):
logger.debug("Checking repo permissions")
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
repository_ref = registry_model.lookup_repository(namespace,
repository,
kind_filter="image")
if repository_ref is None:
abort(403)
builder = lookup_manifest_builder(repository_ref,
session.get("manifest_builder"), store,
docker_v2_signing_key)
if builder is None:
abort(400)
logger.debug("Parsing image JSON")
try:
uploaded_metadata = request.data
uploaded_metadata_string = uploaded_metadata.decode("utf-8")
data = json.loads(uploaded_metadata_string)
except ValueError:
pass
if not data or not isinstance(data, dict):
abort(
400,
"Invalid JSON for image: %(image_id)s\nJSON: %(json)s",
issue="invalid-request",
image_id=image_id,
json=request.data,
)
if "id" not in data:
abort(
400,
"Missing key `id` in JSON for image: %(image_id)s",
issue="invalid-request",
image_id=image_id,
)
if image_id != data["id"]:
abort(
400,
"JSON data contains invalid id for image: %(image_id)s",
issue="invalid-request",
image_id=image_id,
)
logger.debug("Looking up repo image")
location_pref = store.preferred_locations[0]
username = get_authenticated_user() and get_authenticated_user().username
layer = builder.start_layer(
image_id,
uploaded_metadata_string,
location_pref,
username,
app.config["PUSH_TEMP_TAG_EXPIRATION_SEC"],
)
if layer is None:
abort(
400,
"Image %(image_id)s has invalid metadata",
issue="invalid-request",
image_id=image_id,
)
return make_response("true", 200)
def put_image_layer(namespace, repository, image_id):
logger.debug("Checking repo permissions")
permission = ModifyRepositoryPermission(namespace, repository)
if not permission.can():
abort(403)
repository_ref = registry_model.lookup_repository(namespace,
repository,
kind_filter="image")
if repository_ref is None:
abort(403)
logger.debug("Checking for image in manifest builder")
builder = lookup_manifest_builder(repository_ref,
session.get("manifest_builder"), store,
docker_v2_signing_key)
if builder is None:
abort(400)
layer = builder.lookup_layer(image_id)
if layer is None:
abort(404)
logger.debug("Storing layer data")
input_stream = request.stream
if request.headers.get("transfer-encoding") == "chunked":
# Careful, might work only with WSGI servers supporting chunked
# encoding (Gunicorn)
input_stream = request.environ["wsgi.input"]
expiration_sec = app.config["PUSH_TEMP_TAG_EXPIRATION_SEC"]
settings = BlobUploadSettings(
maximum_blob_size=app.config["MAXIMUM_LAYER_SIZE"],
committed_blob_expiration=expiration_sec,
)
extra_handlers = []
# Add a handler that copies the data into a temp file. This is used to calculate the tarsum,
# which is only needed for older versions of Docker.
requires_tarsum = bool(builder.get_layer_checksums(layer))
if requires_tarsum:
tmp, tmp_hndlr = store.temp_store_handler()
extra_handlers.append(tmp_hndlr)
# Add a handler which computes the simple Docker V1 checksum.
h, sum_hndlr = checksums.simple_checksum_handler(layer.v1_metadata_string)
extra_handlers.append(sum_hndlr)
uploaded_blob = None
try:
with upload_blob(repository_ref,
store,
settings,
extra_blob_stream_handlers=extra_handlers) as manager:
manager.upload_chunk(app.config, input_stream)
uploaded_blob = manager.commit_to_blob(app.config)
except BlobUploadException:
logger.exception("Exception when writing image data")
abort(520,
"Image %(image_id)s could not be written. Please try again.",
image_id=image_id)
# Compute the final checksum
csums = []
csums.append("sha256:{0}".format(h.hexdigest()))
try:
if requires_tarsum:
tmp.seek(0)
csums.append(
checksums.compute_tarsum(tmp, layer.v1_metadata_string))
tmp.close()
except (IOError, checksums.TarError) as exc:
logger.debug("put_image_layer: Error when computing tarsum %s", exc)
# If there was already a precomputed checksum, validate against it now.
if builder.get_layer_checksums(layer):
checksum = builder.get_layer_checksums(layer)[0]
if not builder.validate_layer_checksum(layer, checksum):
logger.debug(
"put_image_checksum: Wrong checksum. Given: %s and expected: %s",
checksum,
builder.get_layer_checksums(layer),
)
abort(
400,
"Checksum mismatch for image: %(image_id)s",
issue="checksum-mismatch",
image_id=image_id,
)
# Assign the blob to the layer in the manifest.
if not builder.assign_layer_blob(layer, uploaded_blob, csums):
abort(500, "Something went wrong")
# Send a job to the work queue to replicate the image layer.
# TODO: move this into a better place.
queue_storage_replication(namespace, uploaded_blob)
return make_response("true", 200)
