def update_images(namespace_name, repo_name): permission = ModifyRepositoryPermission(namespace_name, repo_name) if permission.can(): logger.debug("Looking up repository") repository_ref = registry_model.lookup_repository(namespace_name, repo_name, kind_filter="image") if repository_ref is None: # Make sure the repo actually exists. image_pushes.labels("v1", 404, "").inc() abort(404, message="Unknown repository", issue="unknown-repo") builder = lookup_manifest_builder(repository_ref, session.get("manifest_builder"), storage, docker_v2_signing_key) if builder is None: image_pushes.labels("v1", 400, "").inc() abort(400) # Generate a job for each notification that has been added to this repo logger.debug("Adding notifications for repository") event_data = { "updated_tags": [tag.name for tag in builder.committed_tags], } builder.done() track_and_log("push_repo", repository_ref) spawn_notification(repository_ref, "repo_push", event_data) image_pushes.labels("v1", 204, "").inc() return make_response("Updated", 204) image_pushes.labels("v1", 403, "").inc() abort(403)
def update_images(namespace_name, repo_name): permission = ModifyRepositoryPermission(namespace_name, repo_name) if permission.can(): logger.debug('Looking up repository') repository_ref = registry_model.lookup_repository(namespace_name, repo_name, kind_filter='image') if repository_ref is None: # Make sure the repo actually exists. abort(404, message='Unknown repository', issue='unknown-repo') builder = lookup_manifest_builder(repository_ref, session.get('manifest_builder'), storage, docker_v2_signing_key) if builder is None: abort(400) # Generate a job for each notification that has been added to this repo logger.debug('Adding notifications for repository') event_data = { 'updated_tags': [tag.name for tag in builder.committed_tags], } builder.done() track_and_log('push_repo', repository_ref) spawn_notification(repository_ref, 'repo_push', event_data) metric_queue.repository_push.Inc( labelvalues=[namespace_name, repo_name, 'v1', True]) return make_response('Updated', 204) abort(403)
def put_image_json(namespace, repository, image_id): logger.debug('Checking repo permissions') permission = ModifyRepositoryPermission(namespace, repository) if not permission.can(): abort(403) repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image') if repository_ref is None: abort(403) builder = lookup_manifest_builder(repository_ref, session.get('manifest_builder'), store, docker_v2_signing_key) if builder is None: abort(400) logger.debug('Parsing image JSON') try: uploaded_metadata = request.data data = json.loads(uploaded_metadata.decode('utf8')) except ValueError: pass if not data or not isinstance(data, dict): abort(400, 'Invalid JSON for image: %(image_id)s\nJSON: %(json)s', issue='invalid-request', image_id=image_id, json=request.data) if 'id' not in data: abort(400, 'Missing key `id` in JSON for image: %(image_id)s', issue='invalid-request', image_id=image_id) if image_id != data['id']: abort(400, 'JSON data contains invalid id for image: %(image_id)s', issue='invalid-request', image_id=image_id) logger.debug('Looking up repo image') location_pref = store.preferred_locations[0] username = get_authenticated_user() and get_authenticated_user().username layer = builder.start_layer(image_id, uploaded_metadata, location_pref, username, app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) if layer is None: abort(400, 'Image %(image_id)s has invalid metadata', issue='invalid-request', image_id=image_id) return make_response('true', 200)
def put_image_checksum(namespace, repository, image_id): logger.debug("Checking repo permissions") permission = ModifyRepositoryPermission(namespace, repository) if not permission.can(): abort(403) repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter="image") if repository_ref is None: abort(403) # Docker Version < 0.10 (tarsum+sha): old_checksum = request.headers.get("X-Docker-Checksum") # Docker Version >= 0.10 (sha): new_checksum = request.headers.get("X-Docker-Checksum-Payload") checksum = new_checksum or old_checksum if not checksum: abort( 400, "Missing checksum for image %(image_id)s", issue="missing-checksum", image_id=image_id, ) logger.debug("Checking for image in manifest builder") builder = lookup_manifest_builder(repository_ref, session.get("manifest_builder"), store, docker_v2_signing_key) if builder is None: abort(400) layer = builder.lookup_layer(image_id) if layer is None: abort(404) if old_checksum: builder.save_precomputed_checksum(layer, checksum) return make_response("true", 200) if not builder.validate_layer_checksum(layer, checksum): logger.debug( "put_image_checksum: Wrong checksum. Given: %s and expected: %s", checksum, builder.get_layer_checksums(layer), ) abort( 400, "Checksum mismatch for image: %(image_id)s", issue="checksum-mismatch", image_id=image_id, ) return make_response("true", 200)
def put_tag(namespace_name, repo_name, tag): permission = ModifyRepositoryPermission(namespace_name, repo_name) repository_ref = registry_model.lookup_repository(namespace_name, repo_name, kind_filter="image") if permission.can() and repository_ref is not None: if not TAG_REGEX.match(tag): abort(400, TAG_ERROR) image_id = json.loads(request.data) # Check for the image ID first in a builder (for an in-progress push). builder = lookup_manifest_builder(repository_ref, session.get("manifest_builder"), storage, docker_v2_signing_key) if builder is not None: layer = builder.lookup_layer(image_id) if layer is not None: commited_tag = builder.commit_tag_and_manifest(tag, layer) if commited_tag is None: abort(400) return make_response("Created", 200) # Check if there is an existing image we should use (for PUT calls outside of a normal push # operation). legacy_image = registry_model.get_legacy_image(repository_ref, image_id) if legacy_image is None: abort(400) if (registry_model.retarget_tag(repository_ref, tag, legacy_image, storage, docker_v2_signing_key) is None): abort(400) return make_response("Created", 200) abort(403)
def test_build_manifest(layers, fake_session, registry_model): repository_ref = registry_model.lookup_repository("devtable", "complex") storage = DistributedStorage({"local_us": FakeStorage(None)}, ["local_us"]) settings = BlobUploadSettings("2M", 3600) app_config = {"TESTING": True} builder = create_manifest_builder(repository_ref, storage, docker_v2_signing_key) assert ( lookup_manifest_builder(repository_ref, "anotherid", storage, docker_v2_signing_key) is None ) assert ( lookup_manifest_builder(repository_ref, builder.builder_id, storage, docker_v2_signing_key) is not None ) blobs_by_layer = {} for layer_id, parent_id, layer_bytes in layers: # Start a new layer. assert builder.start_layer( layer_id, json.dumps({"id": layer_id, "parent": parent_id}), "local_us", None, 60 ) checksum = hashlib.sha1(layer_bytes).hexdigest() # Assign it a blob. with upload_blob(repository_ref, storage, settings) as uploader: uploader.upload_chunk(app_config, BytesIO(layer_bytes)) blob = uploader.commit_to_blob(app_config) blobs_by_layer[layer_id] = blob builder.assign_layer_blob(builder.lookup_layer(layer_id), blob, [checksum]) # Validate the checksum. assert builder.validate_layer_checksum(builder.lookup_layer(layer_id), checksum) # Commit the manifest to a tag. tag = builder.commit_tag_and_manifest("somenewtag", builder.lookup_layer(layers[-1][0])) assert tag assert tag in builder.committed_tags # Mark the builder as done. builder.done() # Verify the legacy image for the tag. found = registry_model.get_repo_tag(repository_ref, "somenewtag", include_legacy_image=True) assert found assert found.name == "somenewtag" assert found.legacy_image.docker_image_id == layers[-1][0] # Verify the blob and manifest. manifest = registry_model.get_manifest_for_tag(found) assert manifest parsed = manifest.get_parsed_manifest() assert len(list(parsed.layers)) == len(layers) for index, (layer_id, parent_id, layer_bytes) in enumerate(layers): assert list(parsed.blob_digests)[index] == blobs_by_layer[layer_id].digest assert list(parsed.layers)[index].v1_metadata.image_id == layer_id assert list(parsed.layers)[index].v1_metadata.parent_image_id == parent_id assert parsed.leaf_layer_v1_image_id == layers[-1][0]
def put_image_json(namespace, repository, image_id): logger.debug("Checking repo permissions") permission = ModifyRepositoryPermission(namespace, repository) if not permission.can(): abort(403) repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter="image") if repository_ref is None: abort(403) builder = lookup_manifest_builder(repository_ref, session.get("manifest_builder"), store, docker_v2_signing_key) if builder is None: abort(400) logger.debug("Parsing image JSON") try: uploaded_metadata = request.data uploaded_metadata_string = uploaded_metadata.decode("utf-8") data = json.loads(uploaded_metadata_string) except ValueError: pass if not data or not isinstance(data, dict): abort( 400, "Invalid JSON for image: %(image_id)s\nJSON: %(json)s", issue="invalid-request", image_id=image_id, json=request.data, ) if "id" not in data: abort( 400, "Missing key `id` in JSON for image: %(image_id)s", issue="invalid-request", image_id=image_id, ) if image_id != data["id"]: abort( 400, "JSON data contains invalid id for image: %(image_id)s", issue="invalid-request", image_id=image_id, ) logger.debug("Looking up repo image") location_pref = store.preferred_locations[0] username = get_authenticated_user() and get_authenticated_user().username layer = builder.start_layer( image_id, uploaded_metadata_string, location_pref, username, app.config["PUSH_TEMP_TAG_EXPIRATION_SEC"], ) if layer is None: abort( 400, "Image %(image_id)s has invalid metadata", issue="invalid-request", image_id=image_id, ) return make_response("true", 200)
def put_image_layer(namespace, repository, image_id): logger.debug("Checking repo permissions") permission = ModifyRepositoryPermission(namespace, repository) if not permission.can(): abort(403) repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter="image") if repository_ref is None: abort(403) logger.debug("Checking for image in manifest builder") builder = lookup_manifest_builder(repository_ref, session.get("manifest_builder"), store, docker_v2_signing_key) if builder is None: abort(400) layer = builder.lookup_layer(image_id) if layer is None: abort(404) logger.debug("Storing layer data") input_stream = request.stream if request.headers.get("transfer-encoding") == "chunked": # Careful, might work only with WSGI servers supporting chunked # encoding (Gunicorn) input_stream = request.environ["wsgi.input"] expiration_sec = app.config["PUSH_TEMP_TAG_EXPIRATION_SEC"] settings = BlobUploadSettings( maximum_blob_size=app.config["MAXIMUM_LAYER_SIZE"], committed_blob_expiration=expiration_sec, ) extra_handlers = [] # Add a handler that copies the data into a temp file. This is used to calculate the tarsum, # which is only needed for older versions of Docker. requires_tarsum = bool(builder.get_layer_checksums(layer)) if requires_tarsum: tmp, tmp_hndlr = store.temp_store_handler() extra_handlers.append(tmp_hndlr) # Add a handler which computes the simple Docker V1 checksum. h, sum_hndlr = checksums.simple_checksum_handler(layer.v1_metadata_string) extra_handlers.append(sum_hndlr) uploaded_blob = None try: with upload_blob(repository_ref, store, settings, extra_blob_stream_handlers=extra_handlers) as manager: manager.upload_chunk(app.config, input_stream) uploaded_blob = manager.commit_to_blob(app.config) except BlobUploadException: logger.exception("Exception when writing image data") abort(520, "Image %(image_id)s could not be written. Please try again.", image_id=image_id) # Compute the final checksum csums = [] csums.append("sha256:{0}".format(h.hexdigest())) try: if requires_tarsum: tmp.seek(0) csums.append( checksums.compute_tarsum(tmp, layer.v1_metadata_string)) tmp.close() except (IOError, checksums.TarError) as exc: logger.debug("put_image_layer: Error when computing tarsum %s", exc) # If there was already a precomputed checksum, validate against it now. if builder.get_layer_checksums(layer): checksum = builder.get_layer_checksums(layer)[0] if not builder.validate_layer_checksum(layer, checksum): logger.debug( "put_image_checksum: Wrong checksum. Given: %s and expected: %s", checksum, builder.get_layer_checksums(layer), ) abort( 400, "Checksum mismatch for image: %(image_id)s", issue="checksum-mismatch", image_id=image_id, ) # Assign the blob to the layer in the manifest. if not builder.assign_layer_blob(layer, uploaded_blob, csums): abort(500, "Something went wrong") # Send a job to the work queue to replicate the image layer. # TODO: move this into a better place. queue_storage_replication(namespace, uploaded_blob) return make_response("true", 200)