def post_store_hook(job, sip_uuid):
"""
Hook for doing any work after an AIP is stored successfully.
"""
update_es = "transfers" in mcpclient_settings.SEARCH_ENABLED
if update_es:
elasticSearchFunctions.setup_reading_from_conf(mcpclient_settings)
client = elasticSearchFunctions.get_client()
else:
logger.info(
"Skipping indexing: Transfers indexing is currently disabled.")
# SIP ARRANGEMENT
# Mark files in this SIP as in an AIP (aip_created)
file_uuids = models.File.objects.filter(sip=sip_uuid).values_list(
"uuid", flat=True)
models.SIPArrange.objects.filter(file_uuid__in=file_uuids).update(
aip_created=True)
# Check if any of component transfers are completely stored
# TODO Storage service should index AIPs, knows when to update ES
transfer_uuids = set(
models.SIPArrange.objects.filter(file_uuid__in=file_uuids).values_list(
"transfer_uuid", flat=True))
for transfer_uuid in transfer_uuids:
job.pyprint("Checking if transfer", transfer_uuid,
"is fully stored...")
arranged_uuids = set(
models.SIPArrange.objects.filter(
transfer_uuid=transfer_uuid).filter(
aip_created=True).values_list("file_uuid", flat=True))
backlog_uuids = set(
models.File.objects.filter(transfer=transfer_uuid).values_list(
"uuid", flat=True))
# If all backlog UUIDs have been arranged
if arranged_uuids == backlog_uuids:
job.pyprint(
"Transfer",
transfer_uuid,
"fully stored, sending delete request to storage service, deleting from transfer backlog",
)
# Submit delete req to SS (not actually delete), remove from ES
storage_service.request_file_deletion(
uuid=transfer_uuid,
user_id=0,
user_email="archivematica system",
reason_for_deletion="All files in Transfer are now in AIPs.",
)
if update_es:
elasticSearchFunctions.remove_sip_transfer_files(
client, transfer_uuid)
# DSPACE HANDLE TO ARCHIVESSPACE
dspace_handle_to_archivesspace(job, sip_uuid)
# POST-STORE CALLBACK
storage_service.post_store_aip_callback(sip_uuid)
def aip_delete(request, uuid):
try:
reason_for_deletion = request.POST.get('reason_for_deletion', '')
response = storage_service.request_file_deletion(
uuid,
request.user.id,
request.user.email,
reason_for_deletion
)
messages.info(request, response['message'])
elasticSearchFunctions.connect_and_mark_deletion_requested(uuid)
except requests.exceptions.ConnectionError:
error_message = 'Unable to connect to storage server. Please contact your administrator.'
messages.warning(request, error_message)
except slumber.exceptions.HttpClientError:
raise Http404
# It would be more elegant to redirect to the AIP storage overview page, but because
# ElasticSearch processes updates asynchronously this would often result in the user
# having to refresh the page to get an up-to-date result
return render(request, 'archival_storage/delete_request_results.html', locals())
def delete(request, uuid):
"""
Request deletion of a package from a backlog transfer
:param request: The Django request object
:param uuid: The UUID of the package requested for deletion.
:return: Redirects the user back to the backlog page
"""
try:
reason_for_deletion = request.POST.get("reason_for_deletion", "")
response = storage_service.request_file_deletion(
uuid, request.user.id, request.user.email, reason_for_deletion
)
messages.info(request, response["message"])
es_client = es.get_client()
es.mark_backlog_deletion_requested(es_client, uuid)
except requests.exceptions.ConnectionError:
messages.warning(
request,
_(
"Unable to connect to storage server. Please contact your administrator."
),
)
except requests.exceptions.RequestException:
raise Http404
return redirect("backlog:backlog_index")
def post_store_hook(sip_uuid):
"""
Hook for doing any work after an AIP is stored successfully.
"""
# SIP ARRANGEMENT
# Mark files in this SIP as in an AIP (aip_created)
file_uuids = models.File.objects.filter(sip=sip_uuid).values_list(
'uuid', flat=True)
models.SIPArrange.objects.filter(file_uuid__in=file_uuids).update(
aip_created=True)
# Check if any of component transfers are completely stored
# TODO Storage service should index AIPs, knows when to update ES
transfer_uuids = set(
models.SIPArrange.objects.filter(file_uuid__in=file_uuids).values_list(
'transfer_uuid', flat=True))
for transfer_uuid in transfer_uuids:
print 'Checking if transfer', transfer_uuid, 'is fully stored...'
arranged_uuids = set(
models.SIPArrange.objects.filter(
transfer_uuid=transfer_uuid).filter(
aip_created=True).values_list('file_uuid', flat=True))
backlog_uuids = set(
models.File.objects.filter(transfer=transfer_uuid).values_list(
'uuid', flat=True))
# If all backlog UUIDs have been arranged
if arranged_uuids == backlog_uuids:
print 'Transfer', transfer_uuid, 'fully stored, sending delete request to storage service, deleting from transfer backlog'
# Submit delete req to SS (not actually delete), remove from ES
storage_service.request_file_deletion(
uuid=transfer_uuid,
user_id=0,
user_email='archivematica system',
reason_for_deletion='All files in Transfer are now in AIPs.')
elasticSearchFunctions.connect_and_remove_transfer_files(
transfer_uuid)
# POST-STORE CALLBACK
storage_service.post_store_aip_callback(sip_uuid)
def delete(request, uuid):
"""
Request deletion of a package from a backlog transfer
:param request: The Django request object
:param uuid: The UUID of the package requested for deletion.
:return: Redirects the user back to the backlog page
"""
try:
reason_for_deletion = request.POST.get('reason_for_deletion', '')
response = storage_service.request_file_deletion(
uuid, request.user.id, request.user.email, reason_for_deletion)
messages.info(request, response['message'])
es_client = elasticSearchFunctions.get_client()
elasticSearchFunctions.mark_backlog_deletion_requested(es_client, uuid)
except requests.exceptions.ConnectionError:
error_message = 'Unable to connect to storage server. Please contact your administrator.'
messages.warning(request, error_message)
except slumber.exceptions.HttpClientError:
raise Http404
return redirect('backlog_index')
def view_aip(request, uuid):
es_client = es.get_client()
try:
es_aip_doc = es.get_aip_data(
es_client, uuid, fields="name,size,created,status,filePath,encrypted"
)
except IndexError:
raise Http404
source = es_aip_doc["_source"]
name = source.get("name")
active_tab = None
form_upload = forms.UploadMetadataOnlyAtomForm(prefix="upload")
form_reingest = forms.ReingestAIPForm(prefix="reingest")
form_delete = forms.DeleteAIPForm(prefix="delete", uuid=uuid)
# Process metadata-only DIP upload form
if request.POST and "submit-upload-form" in request.POST:
form_upload = forms.UploadMetadataOnlyAtomForm(request.POST, prefix="upload")
active_tab = "upload"
if form_upload.is_valid():
try:
file_slug = upload_dip_metadata_to_atom(
name, uuid, form_upload.cleaned_data["slug"]
)
except AtomMetadataUploadError:
messages.error(
request,
_(
"Metadata-only DIP upload failed, check the logs for more details"
),
)
logger.error(
"Unexepected error during metadata-only DIP upload (UUID: %s)",
uuid,
exc_info=True,
)
else:
messages.success(
request,
_(
"Metadata-only DIP upload has been completed successfully. New resource has slug: %(slug)s"
)
% {"slug": file_slug},
)
form_upload = forms.UploadMetadataOnlyAtomForm(
prefix="upload"
) # Reset form
# Process reingest form
if request.POST and "submit-reingest-form" in request.POST:
form_reingest = forms.ReingestAIPForm(request.POST, prefix="reingest")
active_tab = "reingest"
if form_reingest.is_valid():
response = storage_service.request_reingest(
uuid,
form_reingest.cleaned_data["reingest_type"],
form_reingest.cleaned_data["processing_config"],
)
error = response.get("error", True)
message = response.get("message", "An unknown error occurred.")
if error:
messages.error(
request,
_("Error re-ingesting package: %(message)s") % {"message": message},
)
else:
messages.success(request, message)
return redirect("archival_storage:archival_storage_index")
# Process delete form
if request.POST and "submit-delete-form" in request.POST:
form_delete = forms.DeleteAIPForm(request.POST, prefix="delete", uuid=uuid)
active_tab = "delete"
if form_delete.is_valid():
response = storage_service.request_file_deletion(
uuid,
request.user.id,
request.user.email,
form_delete.cleaned_data["reason"],
)
messages.info(request, response["message"])
es_client = es.get_client()
es.mark_aip_deletion_requested(es_client, uuid)
return redirect("archival_storage:archival_storage_index")
context = {
"uuid": uuid,
"name": name,
"created": source.get("created"),
"status": AIP_STATUS_DESCRIPTIONS[source.get("status", es.STATUS_UPLOADED)],
"encrypted": source.get("encrypted", False),
"size": "{0:.2f} MB".format(source.get("size", 0)),
"location_basename": os.path.basename(source.get("filePath")),
"active_tab": active_tab,
"forms": {
"upload": form_upload,
"reingest": form_reingest,
"delete": form_delete,
},
}
return render(request, "archival_storage/view.html", context)
def view_aip(request, uuid):
es_client = elasticSearchFunctions.get_client()
try:
es_aip_doc = elasticSearchFunctions.get_aip_data(es_client, uuid, fields='name,size,created,status,filePath,encrypted')
except IndexError:
raise Http404
name = _get_es_field(es_aip_doc['fields'], 'name')
active_tab = None
form_upload = forms.UploadMetadataOnlyAtomForm(prefix='upload')
form_reingest = forms.ReingestAIPForm(prefix='reingest')
form_delete = forms.DeleteAIPForm(prefix='delete', uuid=uuid)
# Process metadata-only DIP upload form
if request.POST and 'submit-upload-form' in request.POST:
form_upload = forms.UploadMetadataOnlyAtomForm(request.POST, prefix='upload')
active_tab = 'upload'
if form_upload.is_valid():
try:
file_slug = upload_dip_metadata_to_atom(name, uuid, form_upload.cleaned_data['slug'])
except AtomMetadataUploadError:
messages.error(request, _('Metadata-only DIP upload failed, check the logs for more details'))
logger.error('Unexepected error during metadata-only DIP upload (UUID: %s)', uuid, exc_info=True)
else:
messages.success(request, _('Metadata-only DIP upload has been completed successfully. New resource has slug: %(slug)s') % {'slug': file_slug})
form_upload = forms.UploadMetadataOnlyAtomForm(prefix='upload') # Reset form
# Process reingest form
if request.POST and 'submit-reingest-form' in request.POST:
form_reingest = forms.ReingestAIPForm(request.POST, prefix='reingest')
active_tab = 'reingest'
if form_reingest.is_valid():
response = storage_service.request_reingest(uuid, form_reingest.cleaned_data['reingest_type'], form_reingest.cleaned_data['processing_config'])
error = response.get('error', True)
message = response.get('message', 'An unknown error occurred.')
if error:
messages.error(request, _('Error re-ingesting package: %(message)s') % {'message': message})
else:
messages.success(request, message)
return redirect('archival_storage_index')
# Process delete form
if request.POST and 'submit-delete-form' in request.POST:
form_delete = forms.DeleteAIPForm(request.POST, prefix='delete', uuid=uuid)
active_tab = 'delete'
if form_delete.is_valid():
response = storage_service.request_file_deletion(uuid, request.user.id, request.user.email, form_delete.cleaned_data['reason'])
messages.info(request, response['message'])
es_client = elasticSearchFunctions.get_client()
elasticSearchFunctions.mark_aip_deletion_requested(es_client, uuid)
return redirect('archival_storage_index')
context = {
'uuid': uuid,
'name': name,
'created': _get_es_field(es_aip_doc['fields'], 'created'),
'status': AIP_STATUS_DESCRIPTIONS[_get_es_field(es_aip_doc['fields'], 'status', 'UPLOADED')],
'encrypted': _get_es_field(es_aip_doc['fields'], 'encrypted', False),
'size': '{0:.2f} MB'.format(_get_es_field(es_aip_doc['fields'], 'size', 0)),
'location_basename': os.path.basename(_get_es_field(es_aip_doc['fields'], 'filePath')),
'active_tab': active_tab,
'forms': {
'upload': form_upload,
'reingest': form_reingest,
'delete': form_delete,
},
}
return render(request, 'archival_storage/view.html', context)