def send_all_files_to_giles(sender, **kwargs): # Hey, that rhymes!
"""
Create a :class:`.GilesUpload` instance to indicate that an upload should
be performed.
"""
instance = kwargs.get('instance', None)
logger.debug('received post_save for ContentRelation %i' % instance.id)
if not instance.content_resource.file:
return
logger.debug('create giles upload for %i' % instance.id)
giles.create_giles_upload(instance.for_resource.id, instance.id,
instance.for_resource.created_by.username,
delete_on_complete=settings.DELETE_LOCAL_FILES)
def test_check_upload_status(self, mock_get, mock_post):
mock_get.return_value = MockDataResponse(
202, {
"msg": "Upload in progress. Please check back later.",
"msgCode": "010"
})
upload_id = "PROGQ3Fm2J"
mock_post.return_value = MockDataResponse(
200, {
"id": upload_id,
"checkUrl":
"http://giles/giles/rest/files/upload/check/PROGQ3Fm2J"
})
pk = giles.create_giles_upload(self.resource.id,
self.content_relation.id,
self.user.username)
upload = GilesUpload.objects.get(pk=pk)
giles.send_giles_upload(pk, self.user.username)
giles.check_upload_status(self.user.username, upload_id)
upload.refresh_from_db()
self.assertEqual(upload.state, GilesUpload.SENT)
self.assertEqual(upload.upload_id, upload_id)
self.assertEqual(mock_get.call_count, 1)
args, kwargs = mock_get.call_args
self.assertTrue(args[0].endswith(upload_id))
self.assertIn('headers', kwargs)
self.assertIn('Authorization', kwargs['headers'])
def test_send_to_giles(self, mock_post):
"""
Sends a file indicated in a :class:`.GilesUpload` to Giles.
"""
upload_id = "PROGQ3Fm2J"
mock_post.return_value = MockDataResponse(
200, {
"id": upload_id,
"checkUrl":
"http://giles/giles/rest/files/upload/check/PROGQ3Fm2J"
})
pk = giles.create_giles_upload(self.resource.id,
self.content_relation.id,
self.user.username)
upload = GilesUpload.objects.get(pk=pk)
giles.send_giles_upload(pk, self.user.username)
upload.refresh_from_db()
self.assertEqual(upload.state, GilesUpload.SENT)
self.assertEqual(upload.upload_id, upload_id)
self.assertEqual(mock_post.call_count, 1)
args, kwargs = mock_post.call_args
self.assertIn('files', kwargs)
self.assertIn('data', kwargs)
self.assertIn('headers', kwargs)
def trigger_giles_submission(request, resource_id, relation_id):
"""
Manually start the Giles upload process.
"""
resource = _get_resource_by_id(request, resource_id)
instance = resource.content.get(pk=relation_id)
import mimetypes
content_type = instance.content_resource.content_type or mimetypes.guess_type(
instance.content_resource.file.name)[0]
if instance.content_resource.is_local and instance.content_resource.file.name is not None:
# All files should be sent.
upload_pk = giles.create_giles_upload(resource.id, instance.id,
request.user.username,
settings.DELETE_LOCAL_FILES)
def test_send_to_giles_503(self, mock_post):
"""
If Giles response with a 503 server error, the GilesUpdate state should
be set to SEND_ERROR.
"""
message = 'blargh'
mock_post.return_value = MockDataResponse(503,
message,
content=message)
pk = giles.create_giles_upload(self.resource.id,
self.content_relation.id,
self.user.username)
upload = GilesUpload.objects.get(pk=pk)
giles.send_giles_upload(pk, self.user.username)
self.assertEqual(mock_post.call_count, 1)
upload.refresh_from_db()
self.assertEqual(upload.state, GilesUpload.SEND_ERROR)
self.assertIn(message, upload.message)
def test_giles_can_upload(request):
"""
Send a test file to Giles.
"""
from django.core.files import File
import os
user = request.user
resource = Resource.objects.create(name='test resource', created_by=user)
container = ResourceContainer.objects.create(primary=resource,
created_by=user)
resource.container = container
resource.save()
file_path = os.path.join(settings.MEDIA_ROOT, 'test.ack')
with open(file_path, 'w') as f:
test_file = File(f)
test_file.write('this is a test file')
with open(file_path, 'r') as f:
test_file = File(f)
content_resource = Resource.objects.create(content_resource=True,
file=test_file,
created_by=user,
container=container)
content_relation = ContentRelation.objects.create(
for_resource=resource,
content_resource=content_resource,
created_by=user,
container=container)
upload_pk = giles.create_giles_upload(resource.id,
content_relation.id,
user.username,
delete_on_complete=True)
giles.send_giles_upload(upload_pk, user.username)
upload = GilesUpload.objects.get(pk=upload_pk)
context = {
'status': upload.state,
'upload_id': upload.upload_id,
'container_id': container.id,
}
return JsonResponse(context)
def test_create_giles_upload(self):
"""
Creates a new :class:`.GilesUpload`\.
"""
import jsonpickle
pk = giles.create_giles_upload(self.resource.id,
self.content_relation.id,
self.user.username)
self.assertIsInstance(pk, int)
try:
upload = GilesUpload.objects.get(pk=pk)
except GilesUpload.DoesNotExist:
self.fail('Did not create a GilesUpload instance')
self.assertEqual(upload.state, GilesUpload.PENDING)
self.assertEqual(upload.file_path, self.content_resource.file.name)
self.assertEqual(upload.created_by, self.user)
self.assertEqual(len(jsonpickle.decode(upload.on_complete)), 2)
def test_send_to_giles_401(self, mock_post):
"""
When we attempt to send a GilesUpload, if a 401 unauthorized is returned
we should get a new auth token and try again.
"""
upload_id = "PROGQ3Fm2J"
mock_post.return_value = MockDataResponse(
200, {
"id": upload_id,
"checkUrl":
"http://giles/giles/rest/files/upload/check/PROGQ3Fm2J"
})
pk = giles.create_giles_upload(self.resource.id,
self.content_relation.id,
self.user.username)
upload = GilesUpload.objects.get(pk=pk)
giles.send_giles_upload(pk, self.user.username)
self.assertEqual(mock_post.call_count, 3)
upload.refresh_from_db()
self.assertEqual(upload.state, GilesUpload.SENT)
def test_process_upload_known(self, mock_get, mock_post):
__text__ = Type.objects.create(uri='http://purl.org/dc/dcmitype/Text')
__image__ = Type.objects.create(
uri='http://purl.org/dc/dcmitype/Image')
__document__ = Type.objects.create(
uri='http://xmlns.com/foaf/0.1/Document')
__part__ = Field.objects.create(
uri='http://purl.org/dc/terms/isPartOf')
__creator__ = Field.objects.create(
uri='http://purl.org/dc/elements/1.1/creator')
__dataset__ = Type.objects.create(
uri='http://purl.org/dc/dcmitype/Dataset')
mock_get.return_value = MockDataResponse(200, [{
"documentId":
"DOC123edf",
"uploadId":
"UPxx456",
"uploadedDate":
"2016-09-20T14:03:00.152Z",
"access":
"PRIVATE",
"uploadedFile": {
"filename": "uploadedFile.pdf",
"id": "FILE466tgh",
"url":
"http://your-giles-host.net/giles/rest/files/FILE466tgh/content",
"path": "username/UPxx456/DOC123edf/uploadedFile.pdf",
"content-type": "application/pdf",
"size": 3852180
},
"extractedText": {
"filename": "uploadedFile.pdf.txt",
"id": "FILE123cvb",
"url":
"http://your-giles-host.net/giles/rest/files/FILE123cvb/content",
"path": "username/UPxx456/DOC123edf/uploadedFile.pdf.txt",
"content-type": "text/plain",
"size": 39773
},
"pages": [{
"nr": 0,
"image": {
"filename": "uploadedFile.pdf.0.tiff",
"id": "FILEYUI678",
"url":
"http://your-giles-host.net/giles/rest/digilib?fn=username%FILEYUI678%2FDOC123edf0%2FuploadedFile.pdf.0.tiff",
"path":
"username/UPxx456/DOC123edf/uploadedFile.pdf.0.tiff",
"content-type": "image/tiff",
"size": 2032405
},
"text": {
"filename": "uploadedFile.pdf.0.txt",
"id": "FILE789UIO",
"url":
"http://your-giles-host.net/giles/rest/files/FILE789UIO/content",
"path":
"username/UPxx456/DOC123edf/uploadedFile.pdf.0.txt",
"content-type": "text/plain",
"size": 4658
},
"ocr": {
"filename": "uploadedFile.pdf.0.tiff.txt",
"id": "FILE789U12",
"url":
"http://your-giles-host.net/giles/rest/files/FILE789U12/content",
"path":
"username/UPxx456/DOC123edf/uploadedFile.pdf.0.tiff.txt",
"content-type": "text/plain",
"size": 4658
}
}, {
"nr": 1,
"image": {
"filename": "uploadedFile.pdf.1.tiff",
"id": "FILE045tyhG",
"url":
"http://your-giles-host.net/giles/rest/digilib?fn=username%2FFILE045tyhG%2FDOC123edf0%2FuploadedFile.pdf.1.tiff",
"path": "username/UPxx456/DOC123edf/uploadedFile.1.tiff",
"content-type": "image/tiff",
"size": 2512354
},
"text": {
"filename": "uploadedFile.pdf.1.txt",
"id": "FILEMDSPfeVm",
"url":
"http://your-giles-host.net/giles/rest/files/FILEMDSPfeVm/content",
"path":
"username/UPxx456/DOC123edf/uploadedFile.pdf.1.txt",
"content-type": "text/plain",
"size": 5799
},
"ocr": {
"filename": "uploadedFile.pdf.1.tiff.txt",
"id": "FILEMDSPfe12",
"url":
"http://your-giles-host.net/giles/rest/files/FILEMDSPfe12/content",
"path":
"username/UPxx456/DOC123edf/uploadedFile.pdf.1.tiff.txt",
"content-type": "text/plain",
"size": 5799
}
}]
}])
upload_id = "UPxx456"
mock_post.return_value = MockDataResponse(
200, {
"id": upload_id,
"checkUrl":
"http://giles/giles/rest/files/upload/check/PROGQ3Fm2J"
})
pk = giles.create_giles_upload(self.resource.id,
self.content_relation.id,
self.user.username)
giles.send_giles_upload(pk, self.user.username)
upload = GilesUpload.objects.get(pk=pk)
fpath = upload.file_path
self.assertTrue(
os.path.exists(os.path.join(settings.MEDIA_ROOT, fpath)))
giles.process_upload(upload_id, self.user.username)
upload = GilesUpload.objects.get(upload_id=upload_id)
self.assertEqual(upload.state, GilesUpload.DONE)
self.assertEqual(upload.resource, self.resource)
self.assertEqual(upload.resource.relations_to.count(), 2)
self.assertEqual(upload.resource.content.count(), 3)
self.assertEqual(
upload.resource.content.filter(is_deleted=False).count(), 2)
deleted_resource = upload.resource.content.filter(
is_deleted=True).first().content_resource
self.assertFalse(
os.path.exists(os.path.join(settings.MEDIA_ROOT, fpath)))
for cr in upload.resource.content.all():
self.assertFalse(cr.content_resource.public)