def validate_docker_image_async(*, pk: uuid.UUID, app_label: str,
model_name: str):
model = apps.get_model(app_label=app_label, model_name=model_name)
instance = model.objects.get(pk=pk)
if not instance.image:
# Create the image from the staged file
uploaded_image = StagedAjaxFile(instance.staged_image_uuid)
with uploaded_image.open() as f:
instance.image.save(uploaded_image.name, File(f))
try:
with instance.image.open(mode="rb") as im, tarfile.open(fileobj=im,
mode="r") as t:
member = dict(zip(t.getnames(), t.getmembers()))["manifest.json"]
manifest = t.extractfile(member).read()
except (KeyError, tarfile.ReadError):
model.objects.filter(pk=pk).update(status=(
"manifest.json not found at the root of the container image file. "
"Was this created with docker save?"))
raise ValidationError("Invalid Dockerfile")
manifest = json.loads(manifest)
if len(manifest) != 1:
model.objects.filter(pk=pk).update(
status=(f"The container image file should only have 1 image. "
f"This file contains {len(manifest)}."))
raise ValidationError("Invalid Dockerfile")
model.objects.filter(pk=pk).update(
image_sha256=f"sha256:{manifest[0]['Config'][:64]}", ready=True)
def test_missing_file():
file_content = b"HelloWorld" * 5
uploaded_file_uuid = create_uploaded_file(file_content,
[len(file_content)])
tested_file = StagedAjaxFile(uploaded_file_uuid)
assert tested_file.exists
assert tested_file.is_complete
chunks = StagedFile.objects.filter(file_id=tested_file.uuid).all()
chunks.delete()
assert not tested_file.exists
assert not tested_file.is_complete
with pytest.raises(NotFoundError):
tested_file.name
with pytest.raises(NotFoundError):
tested_file.size
with pytest.raises(NotFoundError):
tested_file.delete()
with pytest.raises(IOError):
tested_file.open()
def test_staged_file_to_django_file():
file_content = b"HelloWorld" * 5
uploaded_file_uuid = create_uploaded_file(file_content,
client_filename="bla")
tested_file = StagedAjaxFile(uploaded_file_uuid)
assert tested_file.name == "bla"
with tested_file.open() as f:
djangofile = files.File(f)
assert djangofile.read() == file_content
assert djangofile.read(1) == b""
def test_staged_file_to_django_file():
file_content = b"HelloWorld" * 5
uploaded_file_uuid = create_uploaded_file(
file_content, client_filename="bla"
)
tested_file = StagedAjaxFile(uploaded_file_uuid)
assert tested_file.name == "bla"
with tested_file.open() as f:
djangofile = files.File(f)
assert djangofile.read() == file_content
assert djangofile.read(1) == b""
def test_missing_file():
file_content = b"HelloWorld" * 5
uploaded_file_uuid = create_uploaded_file(
file_content, [len(file_content)]
)
tested_file = StagedAjaxFile(uploaded_file_uuid)
assert tested_file.exists
assert tested_file.is_complete
chunks = StagedFile.objects.filter(file_id=tested_file.uuid).all()
chunks.delete()
assert not tested_file.exists
assert not tested_file.is_complete
with pytest.raises(NotFoundError):
tested_file.name
with pytest.raises(NotFoundError):
tested_file.size
with pytest.raises(NotFoundError):
tested_file.delete()
with pytest.raises(IOError):
tested_file.open()
def test_rfc7233_implementation_api(client):
content = load_test_data()
upload_id = generate_new_upload_id(
test_rfc7233_implementation_api, content
)
url = reverse("api:staged-file-list")
_, token = AuthToken.objects.create(user=UserFactory())
part_1_response = create_partial_upload_file_request(
client,
upload_id,
content,
0,
10,
url=url,
extra_headers={"HTTP_AUTHORIZATION": f"Bearer {token}"},
)
assert part_1_response.status_code == 201
part_2_response = create_partial_upload_file_request(
client,
upload_id,
content,
10,
len(content) // 2,
url=url,
extra_headers={"HTTP_AUTHORIZATION": f"Bearer {token}"},
)
assert part_2_response.status_code == 201
part_3_response = create_partial_upload_file_request(
client,
upload_id,
content,
len(content) // 2,
len(content),
url=url,
extra_headers={"HTTP_AUTHORIZATION": f"Bearer {token}"},
)
assert part_3_response.status_code == 201
parsed_json = part_3_response.json()
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert len(staged_content) == len(content)
assert hash(staged_content) == hash(content)
assert staged_content == content
def copy_to_tmpdir(image_file: RawImageFile):
staged_file = StagedAjaxFile(image_file.staged_file_id)
if not staged_file.exists:
raise ValueError(
f"staged file {image_file.staged_file_id} does not exist")
with open(provisioning_dir / staged_file.name, "wb") as dest_file:
with staged_file.open() as src_file:
BUFFER_SIZE = 0x10000
first = True
while first or (len(buffer) >= BUFFER_SIZE):
first = False
buffer = src_file.read(BUFFER_SIZE)
dest_file.write(buffer)
def copy_to_tmpdir(image_file: RawImageFile):
staged_file = StagedAjaxFile(image_file.staged_file_id)
if not staged_file.exists:
raise ValueError(
f"staged file {image_file.staged_file_id} does not exist"
)
with open(provisioning_dir / staged_file.name, "wb") as dest_file:
with staged_file.open() as src_file:
BUFFER_SIZE = 0x10000
first = True
while first or (len(buffer) >= BUFFER_SIZE):
first = False
buffer = src_file.read(BUFFER_SIZE)
dest_file.write(buffer)
def test_single_chunk(rf: RequestFactory):
widget = AjaxUploadWidget(ajax_target_path="/ajax")
widget.timeout = timedelta(seconds=1)
filename = 'test.bin'
post_request = create_upload_file_request(rf, filename=filename)
response = widget.handle_ajax(post_request)
assert isinstance(response, JsonResponse)
parsed_json = json.loads(response.content)
assert len(parsed_json) == 1
assert parsed_json[0]["filename"] == filename
assert "uuid" in parsed_json[0]
assert "extra_attrs" in parsed_json[0]
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert staged_content == load_test_data()
def test_single_chunk(rf: RequestFactory):
widget = AjaxUploadWidget(ajax_target_path="/ajax")
widget.timeout = timedelta(seconds=1)
filename = "test.bin"
post_request = create_upload_file_request(rf, filename=filename)
response = widget.handle_ajax(post_request)
assert isinstance(response, JsonResponse)
parsed_json = json.loads(response.content)
assert len(parsed_json) == 1
assert parsed_json[0]["filename"] == filename
assert "uuid" in parsed_json[0]
assert "extra_attrs" in parsed_json[0]
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert staged_content == load_test_data()
def test_rfc7233_implementation_client_api(client):
content = load_test_data()
upload_id = generate_new_upload_id(test_rfc7233_implementation_client_api,
content)
token = Token.objects.create(user=UserFactory())
filename = "whatever.bin"
start_byte = 0
content_io = BytesIO(content)
max_chunk_length = 2**15
assert len(content) > 3 * max_chunk_length
while True:
chunk = content_io.read(max_chunk_length)
if not chunk:
break
end_byte = start_byte + len(chunk)
response = client.post(
path=reverse("api:staged-file-list"),
data={
filename: BytesIO(chunk),
"X-Upload-ID": upload_id
},
format="multipart",
HTTP_CONTENT_RANGE=
f"bytes {start_byte}-{end_byte - 1}/{len(content)}",
HTTP_AUTHORIZATION=f"Token {token}",
)
assert response.status_code == 201
start_byte += len(chunk)
parsed_json = response.json()
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert len(staged_content) == len(content)
assert hash(staged_content) == hash(content)
assert staged_content == content
def validate_docker_image(*, pk: uuid.UUID, app_label: str, model_name: str):
model = apps.get_model(app_label=app_label, model_name=model_name)
instance = model.objects.get(pk=pk)
if not instance.image:
# Create the image from the staged file
uploaded_image = StagedAjaxFile(instance.staged_image_uuid)
with uploaded_image.open() as f:
instance.image.save(uploaded_image.name, File(f))
try:
image_sha256 = _validate_docker_image_manifest(model=model,
instance=instance)
except ValidationError:
send_invalid_dockerfile_email(container_image=instance)
raise
model.objects.filter(pk=instance.pk).update(
image_sha256=f"sha256:{image_sha256}", ready=True)
def test_single_chunk_api(client):
filename = "test.bin"
token = Token.objects.create(user=UserFactory())
response = create_upload_file_request(
rf=client,
filename=filename,
url=reverse("api:staged-file-list"),
extra_headers={"HTTP_AUTHORIZATION": f"Token {token}"},
)
assert response.status_code == 201
parsed_json = response.json()
assert parsed_json[0]["filename"] == filename
assert "uuid" in parsed_json[0]
assert "extra_attrs" in parsed_json[0]
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert staged_content == load_test_data()
def test_rfc7233_implementation(rf: RequestFactory):
content = load_test_data()
upload_id = generate_new_upload_id(test_rfc7233_implementation, content)
part_1 = create_partial_upload_file_request(rf, upload_id, content, 0, 10)
part_2 = create_partial_upload_file_request(rf, upload_id, content, 10,
len(content) // 2)
part_3 = create_partial_upload_file_request(rf, upload_id, content,
len(content) // 2,
len(content))
widget = AjaxUploadWidget(ajax_target_path="/ajax")
widget.timeout = timedelta(seconds=1)
response = widget.handle_ajax(part_1)
assert isinstance(response, JsonResponse)
response = widget.handle_ajax(part_2)
assert isinstance(response, JsonResponse)
response = widget.handle_ajax(part_3)
assert isinstance(response, JsonResponse)
parsed_json = json.loads(response.content)
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert staged_content == content
def test_rfc7233_implementation(rf: RequestFactory):
content = load_test_data()
upload_id = generate_new_upload_id(test_rfc7233_implementation, content)
part_1 = create_partial_upload_file_request(rf, upload_id, content, 0, 10)
part_2 = create_partial_upload_file_request(
rf, upload_id, content, 10, len(content) // 2
)
part_3 = create_partial_upload_file_request(
rf, upload_id, content, len(content) // 2, len(content)
)
widget = AjaxUploadWidget(ajax_target_path="/ajax")
widget.timeout = timedelta(seconds=1)
response = widget.handle_ajax(part_1)
assert isinstance(response, JsonResponse)
response = widget.handle_ajax(part_2)
assert isinstance(response, JsonResponse)
response = widget.handle_ajax(part_3)
assert isinstance(response, JsonResponse)
parsed_json = json.loads(response.content)
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert staged_content == content
def validate_docker_image_async(
*, pk: uuid.UUID, app_label: str, model_name: str
):
model = apps.get_model(app_label=app_label, model_name=model_name)
instance = model.objects.get(pk=pk)
if not instance.image:
# Create the image from the staged file
uploaded_image = StagedAjaxFile(instance.staged_image_uuid)
with uploaded_image.open() as f:
instance.image.save(uploaded_image.name, File(f))
try:
image_sha256 = _validate_docker_image_manifest(
model=model, instance=instance
)
except ValidationError:
send_invalid_dockerfile_email(container_image=instance)
raise
model.objects.filter(pk=instance.pk).update(
image_sha256=f"sha256:{image_sha256}", ready=True
)
def test_single_chunk_client_api(client):
filename = "test.bin"
content = load_test_data()
token = Token.objects.create(user=UserFactory())
response = client.post(
path=reverse("api:staged-file-list"),
data={filename: BytesIO(content)},
format="multipart",
HTTP_AUTHORIZATION=f"Token {token}",
)
assert response.status_code == 201
parsed_json = response.json()
assert len(parsed_json) == 1
assert parsed_json[0]["filename"] == filename
assert "uuid" in parsed_json[0]
assert "extra_attrs" in parsed_json[0]
staged_file = StagedAjaxFile(uuid.UUID(parsed_json[0]["uuid"]))
with staged_file.open() as f:
staged_content = f.read()
assert staged_content == load_test_data()
