def validate_docker_image_async(*, pk: uuid.UUID, app_label: str, model_name: str): model = apps.get_model(app_label=app_label, model_name=model_name) instance = model.objects.get(pk=pk) if not instance.image: # Create the image from the staged file uploaded_image = StagedAjaxFile(instance.staged_image_uuid) with uploaded_image.open() as f: instance.image.save(uploaded_image.name, File(f)) try: with instance.image.open(mode="rb") as im, tarfile.open(fileobj=im, mode="r") as t: member = dict(zip(t.getnames(), t.getmembers()))["manifest.json"] manifest = t.extractfile(member).read() except (KeyError, tarfile.ReadError): model.objects.filter(pk=pk).update(status=( "manifest.json not found at the root of the container image file. " "Was this created with docker save?")) raise ValidationError("Invalid Dockerfile") manifest = json.loads(manifest) if len(manifest) != 1: model.objects.filter(pk=pk).update( status=(f"The container image file should only have 1 image. " f"This file contains {len(manifest)}.")) raise ValidationError("Invalid Dockerfile") model.objects.filter(pk=pk).update( image_sha256=f"sha256:{manifest[0]['Config'][:64]}", ready=True)
def test_missing_file(): file_content = b"HelloWorld" * 5 uploaded_file_uuid = create_uploaded_file(file_content, [len(file_content)]) tested_file = StagedAjaxFile(uploaded_file_uuid) assert tested_file.exists assert tested_file.is_complete chunks = StagedFile.objects.filter(file_id=tested_file.uuid).all() chunks.delete() assert not tested_file.exists assert not tested_file.is_complete with pytest.raises(NotFoundError): tested_file.name with pytest.raises(NotFoundError): tested_file.size with pytest.raises(NotFoundError): tested_file.delete() with pytest.raises(IOError): tested_file.open()
def test_staged_file_to_django_file(): file_content = b"HelloWorld" * 5 uploaded_file_uuid = create_uploaded_file(file_content, client_filename="bla") tested_file = StagedAjaxFile(uploaded_file_uuid) assert tested_file.name == "bla" with tested_file.open() as f: djangofile = files.File(f) assert djangofile.read() == file_content assert djangofile.read(1) == b""
def test_staged_file_to_django_file(): file_content = b"HelloWorld" * 5 uploaded_file_uuid = create_uploaded_file( file_content, client_filename="bla" ) tested_file = StagedAjaxFile(uploaded_file_uuid) assert tested_file.name == "bla" with tested_file.open() as f: djangofile = files.File(f) assert djangofile.read() == file_content assert djangofile.read(1) == b""
def test_missing_file(): file_content = b"HelloWorld" * 5 uploaded_file_uuid = create_uploaded_file( file_content, [len(file_content)] ) tested_file = StagedAjaxFile(uploaded_file_uuid) assert tested_file.exists assert tested_file.is_complete chunks = StagedFile.objects.filter(file_id=tested_file.uuid).all() chunks.delete() assert not tested_file.exists assert not tested_file.is_complete with pytest.raises(NotFoundError): tested_file.name with pytest.raises(NotFoundError): tested_file.size with pytest.raises(NotFoundError): tested_file.delete() with pytest.raises(IOError): tested_file.open()
def test_rfc7233_implementation_api(client): content = load_test_data() upload_id = generate_new_upload_id( test_rfc7233_implementation_api, content ) url = reverse("api:staged-file-list") _, token = AuthToken.objects.create(user=UserFactory()) part_1_response = create_partial_upload_file_request( client, upload_id, content, 0, 10, url=url, extra_headers={"HTTP_AUTHORIZATION": f"Bearer {token}"}, ) assert part_1_response.status_code == 201 part_2_response = create_partial_upload_file_request( client, upload_id, content, 10, len(content) // 2, url=url, extra_headers={"HTTP_AUTHORIZATION": f"Bearer {token}"}, ) assert part_2_response.status_code == 201 part_3_response = create_partial_upload_file_request( client, upload_id, content, len(content) // 2, len(content), url=url, extra_headers={"HTTP_AUTHORIZATION": f"Bearer {token}"}, ) assert part_3_response.status_code == 201 parsed_json = part_3_response.json() staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert len(staged_content) == len(content) assert hash(staged_content) == hash(content) assert staged_content == content
def copy_to_tmpdir(image_file: RawImageFile): staged_file = StagedAjaxFile(image_file.staged_file_id) if not staged_file.exists: raise ValueError( f"staged file {image_file.staged_file_id} does not exist") with open(provisioning_dir / staged_file.name, "wb") as dest_file: with staged_file.open() as src_file: BUFFER_SIZE = 0x10000 first = True while first or (len(buffer) >= BUFFER_SIZE): first = False buffer = src_file.read(BUFFER_SIZE) dest_file.write(buffer)
def copy_to_tmpdir(image_file: RawImageFile): staged_file = StagedAjaxFile(image_file.staged_file_id) if not staged_file.exists: raise ValueError( f"staged file {image_file.staged_file_id} does not exist" ) with open(provisioning_dir / staged_file.name, "wb") as dest_file: with staged_file.open() as src_file: BUFFER_SIZE = 0x10000 first = True while first or (len(buffer) >= BUFFER_SIZE): first = False buffer = src_file.read(BUFFER_SIZE) dest_file.write(buffer)
def test_single_chunk(rf: RequestFactory): widget = AjaxUploadWidget(ajax_target_path="/ajax") widget.timeout = timedelta(seconds=1) filename = 'test.bin' post_request = create_upload_file_request(rf, filename=filename) response = widget.handle_ajax(post_request) assert isinstance(response, JsonResponse) parsed_json = json.loads(response.content) assert len(parsed_json) == 1 assert parsed_json[0]["filename"] == filename assert "uuid" in parsed_json[0] assert "extra_attrs" in parsed_json[0] staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert staged_content == load_test_data()
def test_single_chunk(rf: RequestFactory): widget = AjaxUploadWidget(ajax_target_path="/ajax") widget.timeout = timedelta(seconds=1) filename = "test.bin" post_request = create_upload_file_request(rf, filename=filename) response = widget.handle_ajax(post_request) assert isinstance(response, JsonResponse) parsed_json = json.loads(response.content) assert len(parsed_json) == 1 assert parsed_json[0]["filename"] == filename assert "uuid" in parsed_json[0] assert "extra_attrs" in parsed_json[0] staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert staged_content == load_test_data()
def test_rfc7233_implementation_client_api(client): content = load_test_data() upload_id = generate_new_upload_id(test_rfc7233_implementation_client_api, content) token = Token.objects.create(user=UserFactory()) filename = "whatever.bin" start_byte = 0 content_io = BytesIO(content) max_chunk_length = 2**15 assert len(content) > 3 * max_chunk_length while True: chunk = content_io.read(max_chunk_length) if not chunk: break end_byte = start_byte + len(chunk) response = client.post( path=reverse("api:staged-file-list"), data={ filename: BytesIO(chunk), "X-Upload-ID": upload_id }, format="multipart", HTTP_CONTENT_RANGE= f"bytes {start_byte}-{end_byte - 1}/{len(content)}", HTTP_AUTHORIZATION=f"Token {token}", ) assert response.status_code == 201 start_byte += len(chunk) parsed_json = response.json() staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert len(staged_content) == len(content) assert hash(staged_content) == hash(content) assert staged_content == content
def validate_docker_image(*, pk: uuid.UUID, app_label: str, model_name: str): model = apps.get_model(app_label=app_label, model_name=model_name) instance = model.objects.get(pk=pk) if not instance.image: # Create the image from the staged file uploaded_image = StagedAjaxFile(instance.staged_image_uuid) with uploaded_image.open() as f: instance.image.save(uploaded_image.name, File(f)) try: image_sha256 = _validate_docker_image_manifest(model=model, instance=instance) except ValidationError: send_invalid_dockerfile_email(container_image=instance) raise model.objects.filter(pk=instance.pk).update( image_sha256=f"sha256:{image_sha256}", ready=True)
def test_single_chunk_api(client): filename = "test.bin" token = Token.objects.create(user=UserFactory()) response = create_upload_file_request( rf=client, filename=filename, url=reverse("api:staged-file-list"), extra_headers={"HTTP_AUTHORIZATION": f"Token {token}"}, ) assert response.status_code == 201 parsed_json = response.json() assert parsed_json[0]["filename"] == filename assert "uuid" in parsed_json[0] assert "extra_attrs" in parsed_json[0] staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert staged_content == load_test_data()
def test_rfc7233_implementation(rf: RequestFactory): content = load_test_data() upload_id = generate_new_upload_id(test_rfc7233_implementation, content) part_1 = create_partial_upload_file_request(rf, upload_id, content, 0, 10) part_2 = create_partial_upload_file_request(rf, upload_id, content, 10, len(content) // 2) part_3 = create_partial_upload_file_request(rf, upload_id, content, len(content) // 2, len(content)) widget = AjaxUploadWidget(ajax_target_path="/ajax") widget.timeout = timedelta(seconds=1) response = widget.handle_ajax(part_1) assert isinstance(response, JsonResponse) response = widget.handle_ajax(part_2) assert isinstance(response, JsonResponse) response = widget.handle_ajax(part_3) assert isinstance(response, JsonResponse) parsed_json = json.loads(response.content) staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert staged_content == content
def test_rfc7233_implementation(rf: RequestFactory): content = load_test_data() upload_id = generate_new_upload_id(test_rfc7233_implementation, content) part_1 = create_partial_upload_file_request(rf, upload_id, content, 0, 10) part_2 = create_partial_upload_file_request( rf, upload_id, content, 10, len(content) // 2 ) part_3 = create_partial_upload_file_request( rf, upload_id, content, len(content) // 2, len(content) ) widget = AjaxUploadWidget(ajax_target_path="/ajax") widget.timeout = timedelta(seconds=1) response = widget.handle_ajax(part_1) assert isinstance(response, JsonResponse) response = widget.handle_ajax(part_2) assert isinstance(response, JsonResponse) response = widget.handle_ajax(part_3) assert isinstance(response, JsonResponse) parsed_json = json.loads(response.content) staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert staged_content == content
def validate_docker_image_async( *, pk: uuid.UUID, app_label: str, model_name: str ): model = apps.get_model(app_label=app_label, model_name=model_name) instance = model.objects.get(pk=pk) if not instance.image: # Create the image from the staged file uploaded_image = StagedAjaxFile(instance.staged_image_uuid) with uploaded_image.open() as f: instance.image.save(uploaded_image.name, File(f)) try: image_sha256 = _validate_docker_image_manifest( model=model, instance=instance ) except ValidationError: send_invalid_dockerfile_email(container_image=instance) raise model.objects.filter(pk=instance.pk).update( image_sha256=f"sha256:{image_sha256}", ready=True )
def test_single_chunk_client_api(client): filename = "test.bin" content = load_test_data() token = Token.objects.create(user=UserFactory()) response = client.post( path=reverse("api:staged-file-list"), data={filename: BytesIO(content)}, format="multipart", HTTP_AUTHORIZATION=f"Token {token}", ) assert response.status_code == 201 parsed_json = response.json() assert len(parsed_json) == 1 assert parsed_json[0]["filename"] == filename assert "uuid" in parsed_json[0] assert "extra_attrs" in parsed_json[0] staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"])) with staged_file.open() as f: staged_content = f.read() assert staged_content == load_test_data()