def test_error_dereferencing(api, users, location, es,
httpserver: pytest_httpserver.HTTPServer):
with api.test_request_context():
record = SWORDDeposit.create({})
object_version = ObjectVersion.create(bucket=record.bucket,
key="some-file.txt")
TagManager(object_version).update({
ObjectTagKey.ByReferenceURL:
httpserver.url_for("some-file.txt"),
# This one should get removed after dereferencing
ObjectTagKey.ByReferenceNotDeleted:
"true",
ObjectTagKey.Packaging:
PackagingFormat.SimpleZip,
})
httpserver.expect_request("/some-file.txt").respond_with_data(
b"", status=HTTPStatus.GONE)
db.session.refresh(object_version)
with pytest.raises(urllib.error.HTTPError):
tasks.dereference_object(record.id, object_version.version_id)
db.session.refresh(object_version)
tags = TagManager(object_version)
assert tags.get(ObjectTagKey.FileState) == FileState.Error
def test_dereference_missing_upload(api, users, location, task_delay):
with api.test_request_context():
# Assemble a segmented upload from parts, and complete it
segmented_upload_record: SegmentedUploadRecord = SegmentedUploadRecord.create(
{}
)
MultipartObject.create(
bucket=segmented_upload_record.bucket,
key="some-key",
size=15,
chunk_size=10,
)
record: SWORDDeposit = SWORDDeposit.create({})
record.set_by_reference_files(
[
ByReferenceFileDefinition(
temporary_id=segmented_upload_record.id,
content_disposition="attachment; filename=something.txt",
content_type="text/plain",
packaging=PackagingFormat.Binary,
dereference=True,
),
],
lambda *args: True,
"http://localhost/",
)
object_version = ObjectVersion.query.one()
with pytest.raises(ValueError):
tasks.dereference_object(record.id, object_version.version_id)
assert TagManager(object_version)[ObjectTagKey.FileState] == FileState.Error
def test_post_by_reference_segmented(api, users, location, task_delay):
with api.test_request_context(), api.test_client() as client:
# Assemble a segmented upload from parts, and complete it
segmented_upload_record: SegmentedUploadRecord = SegmentedUploadRecord.create(
{}
)
multipart_object = MultipartObject.create(
bucket=segmented_upload_record.bucket,
key="some-key",
size=15,
chunk_size=10,
)
Part.create(multipart_object, 0, stream=io.BytesIO(b"abcdefghij"))
Part.create(multipart_object, 1, stream=io.BytesIO(b"klmno"))
multipart_object.complete()
login(client)
ttl = (
datetime.datetime.now(tz=datetime.timezone.utc)
+ datetime.timedelta(0, 3600)
).isoformat()
response = client.post(
"/sword/service-document",
data=json.dumps(
{
"@context": JSON_LD_CONTEXT,
"@type": "ByReference",
"byReferenceFiles": [
{
"@id": f"http://localhost/sword/staging/{segmented_upload_record.id}",
"contentDisposition": "attachment; filename=some-resource.json",
"contentType": "application/json",
"dereference": True,
"ttl": ttl,
}
],
}
),
headers={
"Content-Disposition": "attachment; by-reference=true",
"Content-Type": "application/ld+json",
},
)
assert response.status_code == HTTPStatus.CREATED
object_version = ObjectVersion.query.one()
tags = TagManager(object_version)
assert tags == {
ObjectTagKey.Packaging: "http://purl.org/net/sword/3.0/package/Binary",
ObjectTagKey.ByReferenceTemporaryID: str(segmented_upload_record.id),
ObjectTagKey.FileState: FileState.Pending,
ObjectTagKey.ByReferenceDereference: "true",
ObjectTagKey.ByReferenceNotDeleted: "true",
ObjectTagKey.OriginalDeposit: "true",
ObjectTagKey.ByReferenceTTL: ttl,
}
def test_error_unpacking(api, users, location, es):
with api.test_request_context():
record = SWORDDeposit.create({})
object_version = ObjectVersion.create(bucket=record.bucket,
key="some-file.txt",
mimetype="text/plain")
TagManager(object_version).update({
ObjectTagKey.Packaging:
PackagingFormat.SimpleZip,
})
db.session.refresh(object_version)
with pytest.raises(ContentTypeNotAcceptable):
tasks.unpack_object(record.id, object_version.version_id)
db.session.refresh(object_version)
tags = TagManager(object_version)
assert tags.get(ObjectTagKey.FileState) == FileState.Error
def test_dereference_task(api, users, location, es,
httpserver: pytest_httpserver.HTTPServer):
file_contents = "File contents.\n"
with api.test_request_context():
record = SWORDDeposit.create({})
object_version = ObjectVersion.create(bucket=record.bucket,
key="some-file.txt")
TagManager(object_version).update({
ObjectTagKey.ByReferenceURL:
httpserver.url_for("some-file.txt"),
# This one should get removed after dereferencing
ObjectTagKey.ByReferenceNotDeleted:
"true",
ObjectTagKey.Packaging:
PackagingFormat.SimpleZip,
})
httpserver.expect_request("/some-file.txt").respond_with_data(
file_contents)
db.session.refresh(object_version)
tasks.dereference_object(record.id, object_version.version_id)
# Check requests
assert len(httpserver.log) == 1
assert httpserver.log[0][0].path == "/some-file.txt"
db.session.refresh(object_version)
assert object_version.file is not None
assert object_version.file.storage().open().read(
) == file_contents.encode("utf-8")
assert TagManager(object_version) == {
ObjectTagKey.ByReferenceURL: httpserver.url_for("some-file.txt"),
ObjectTagKey.Packaging: PackagingFormat.SimpleZip,
ObjectTagKey.FileState: FileState.Pending,
}
def test_tag_manager_update(api, users, location, es, update_style):
with api.test_request_context():
bucket = Bucket.create()
object_version = ObjectVersion.create(bucket=bucket, key="hello")
ObjectVersionTag.create(
object_version=object_version,
key=ObjectTagKey.Packaging.value,
value="old-packaging",
)
ObjectVersionTag.create(
object_version=object_version,
key=ObjectTagKey.MetadataFormat.value,
value="old-metadata",
)
tags = TagManager(object_version)
assert (
ObjectVersionTag.query.filter_by(object_version=object_version).count() == 2
)
assert tags == {
ObjectTagKey.Packaging: "old-packaging",
ObjectTagKey.MetadataFormat: "old-metadata",
}
if update_style == "dict":
tags.update(
{
ObjectTagKey.MetadataFormat: "new-metadata",
ObjectTagKey.DerivedFrom: "new-derived-from",
}
)
elif update_style == "kwargs":
tags.update(
**{
ObjectTagKey.MetadataFormat.value: "new-metadata",
ObjectTagKey.DerivedFrom.value: "new-derived-from",
}
)
assert tags == {
ObjectTagKey.Packaging: "old-packaging",
ObjectTagKey.MetadataFormat: "new-metadata",
ObjectTagKey.DerivedFrom: "new-derived-from",
}
assert (
ObjectVersionTag.query.filter_by(object_version=object_version).count() == 3
)
db.session.refresh(object_version)
assert object_version.get_tags() == {
ObjectTagKey.Packaging.value: "old-packaging",
ObjectTagKey.MetadataFormat.value: "new-metadata",
ObjectTagKey.DerivedFrom.value: "new-derived-from",
}
def test_by_reference_sets_tag(api, users, location, task_delay):
with api.test_request_context():
# Assemble a segmented upload from parts, and complete it
segmented_upload_record: SegmentedUploadRecord = SegmentedUploadRecord.create(
{}
)
multipart_object = MultipartObject.create(
bucket=segmented_upload_record.bucket,
key="some-key",
size=15,
chunk_size=10,
)
Part.create(multipart_object, 0, stream=io.BytesIO(b"abcdefghij"))
Part.create(multipart_object, 1, stream=io.BytesIO(b"klmno"))
multipart_object.complete()
record: SWORDDeposit = SWORDDeposit.create({})
record.set_by_reference_files(
[
ByReferenceFileDefinition(
temporary_id=segmented_upload_record.id,
content_disposition="attachment; filename=something.txt",
content_type="text/plain",
packaging=PackagingFormat.Binary,
dereference=True,
),
],
lambda *args: True,
"http://localhost/",
)
object_version = ObjectVersion.query.one()
tags = TagManager(object_version)
assert tags == {
ObjectTagKey.OriginalDeposit: "true",
ObjectTagKey.ByReferenceTemporaryID: str(segmented_upload_record.id),
ObjectTagKey.Packaging: "http://purl.org/net/sword/3.0/package/Binary",
ObjectTagKey.FileState: FileState.Pending,
ObjectTagKey.ByReferenceDereference: "true",
ObjectTagKey.ByReferenceNotDeleted: "true",
}
tasks.dereference_object(record.id, object_version.version_id)
assert object_version.file.storage().open().read() == b"abcdefghijklmno"
def test_tag_manager_setitem(api, users, location, es):
value = "http://example.org/"
with api.test_request_context():
bucket = Bucket.create()
object_version = ObjectVersion.create(bucket=bucket, key="hello")
tags = TagManager(object_version)
assert (
ObjectVersionTag.query.filter_by(object_version=object_version).count() == 0
)
tags[ObjectTagKey.Packaging] = value
# It's updated the TagManager
assert tags[ObjectTagKey.Packaging] == value
# We've created a tag
tag = ObjectVersionTag.query.filter_by(object_version=object_version).one()
assert tag.key == ObjectTagKey.Packaging.value
assert tag.value == value
def test_dereference_already_dereferenced(
api, location, es, httpserver: pytest_httpserver.HTTPServer):
with api.test_request_context():
record = SWORDDeposit.create({})
object_version = ObjectVersion.create(bucket=record.bucket,
key="some-file.txt",
stream=io.BytesIO(b"data"))
TagManager(object_version).update({
ObjectTagKey.ByReferenceURL:
httpserver.url_for("some-file.txt"),
ObjectTagKey.Packaging:
PackagingFormat.SimpleZip,
})
httpserver.expect_request("/some-file.txt").respond_with_data(b"data")
db.session.refresh(object_version)
result = tasks.dereference_object(record.id, object_version.version_id)
assert result == ["some-file.txt"]
assert httpserver.log == []
def test_tag_manager_delitem(api, users, location, es):
value = "http://example.org/"
with api.test_request_context():
bucket = Bucket.create()
object_version = ObjectVersion.create(bucket=bucket, key="hello")
ObjectVersionTag.create(
object_version=object_version, key=ObjectTagKey.Packaging.value, value=value
)
tags = TagManager(object_version)
assert tags == {ObjectTagKey.Packaging: value}
assert tags[ObjectTagKey.Packaging] == value
del tags[ObjectTagKey.Packaging]
assert tags == {}
with pytest.raises(KeyError):
_ = tags[ObjectTagKey.Packaging]
# We've deleted the database object
assert (
ObjectVersionTag.query.filter_by(object_version=object_version).count() == 0
)