def _package_contents(self, request, bundle, **kwargs):
"""
Returns metadata about every file within a specified Storage Service
package, specified via Storage Service UUID.
The file properties provided are the properties of the ~:class:`~locations.models.event.File` class; see the class definition for more information.
:returns: a JSON object in the following format:
{
"success": True,
"package": "UUID (as string)",
"files": [
# array containing zero or more objects containing
# all of the file's properties, in the format:
{
"source_id": "",
# ...
}
]
}
"""
response = {"success": True, "package": bundle.obj.uuid, "files": []}
for f in bundle.obj.file_set.all():
response["files"].append({
attr: getattr(f, attr)
for attr in ('source_id', 'name', 'source_package', 'checksum',
'accessionid', 'origin')
})
return http.HttpResponse(status=200,
content=json.dumps(response),
content_type='application/json')
def delete_aip_request(self, request, bundle, **kwargs):
request_info = bundle.data
package = bundle.obj
if package.package_type not in Package.PACKAGE_TYPE_CAN_DELETE:
# Can only request deletion on AIPs
response = {"message": "Deletes not allowed on this package type."}
response_json = json.dumps(response)
return http.HttpMethodNotAllowed(response_json,
content_type='application/json')
(status_code, response) = self._attempt_package_request_event(
package, request_info, Event.DELETE, Package.DEL_REQ)
if status_code == 202:
# This isn't configured by default
site_url = getattr(settings, "SITE_BASE_URL", None)
signals.deletion_request.send(sender=self,
url=site_url,
uuid=package.uuid,
location=package.full_path)
else:
response = {
'message': 'A deletion request already exists for this AIP.'
}
self.log_throttled_access(request)
response_json = json.dumps(response)
return http.HttpResponse(status=status_code,
content=response_json,
mimetype='application/json')
def file_data(self, request, **kwargs):
"""
Returns file metadata as a JSON array of objects.
This maps properties of the File class to the names of the
Elasticsearch indices' Transferfile index, allowing this to directly
substitute for Elasticsearch when looking up metadata on specific files.
Acceptable parameters are:
* relative_path (searches the `name` field)
* fileuuid (searches the `source_id` field)
* accessionid (searches the `accessionid` field)
* sipuuid (searches the `source_package` field)
:returns: an array of one or more objects. See the transferfile
index for information on the return format.
If no results are found for the specified query, returns 404.
If no acceptable query parameters are found, returns 400.
"""
property_map = {
"relative_path": "name",
"fileuuid": "source_id",
"accessionid": "accessionid",
"sipuuid": "source_package"
}
query = {}
for source, dest in property_map.iteritems():
try:
query[dest] = request.GET[source]
except KeyError:
pass
if not query:
response = {
"success": False,
"error": "No supported query properties found!"
}
return http.HttpBadRequest(content=json.dumps(response),
content_type="application/json")
files = File.objects.filter(**query)
if not files.exists():
return http.HttpNotFound()
response = []
for f in files:
response.append({
"accessionid": f.accessionid,
"file_extension": os.path.splitext(f.name)[1],
"filename": os.path.basename(f.name),
"relative_path": f.name,
"fileuuid": f.source_id,
"origin": f.origin,
"sipuuid": f.source_package
})
return http.HttpResponse(content=json.dumps(response),
content_type="application/json")
def async_result(self, request, task_id, **kwargs):
"""
Task results.
If request is ready, return Http 200 response with serialized data.
If request is not ready (or doesn't exist - we can't tell this),
return Http 404 Not Found.
"""
# hack to allow local testing
if not getattr(settings, 'CELERY_ALWAYS_EAGER'):
task = AsyncResult(task_id)
else:
task = EAGER_RESULTS[task_id]
if task.ready():
try:
result = self.process_result(task.get())
except Exception, error:
result = {'error': unicode(error)}
if isinstance(result, http.HttpResponse):
return result
elif isinstance(result, basestring):
return http.HttpResponse(result)
elif isinstance(result, list):
objects = result
sorted_objects = self.apply_sorting(objects,
options=request.GET)
paginator = self._meta.paginator_class(
request.GET,
sorted_objects,
resource_uri=self.get_resource_uri(),
limit=self._meta.limit,
max_limit=self._meta.max_limit,
collection_name=self._meta.collection_name)
to_be_serialized = paginator.page()
# Dehydrate the bundles in preparation for serialization.
bundles = []
for obj in to_be_serialized[self._meta.collection_name]:
bundle = self.build_bundle(obj=obj, request=request)
bundles.append(self.full_dehydrate(bundle, for_list=True))
to_be_serialized[self._meta.collection_name] = bundles
to_be_serialized = self.alter_list_data_to_serialize(
request, to_be_serialized)
return self.create_response(request, to_be_serialized)
elif isinstance(result, dict):
bundle = self.build_bundle(obj=result, request=request)
self.full_dehydrate(bundle)
return self.create_response(request, bundle)
else:
bundle = self.build_bundle(obj=result, request=request)
bundle = self.full_dehydrate(bundle)
bundle = self.alter_detail_data_to_serialize(request, bundle)
return self.create_response(request, bundle)
def extract_file_request(self, request, bundle, **kwargs):
"""
Returns a single file from the Package, extracting if necessary.
"""
relative_path_to_file = request.GET.get('relative_path_to_file')
relative_path_to_file = urllib.unquote(relative_path_to_file)
temp_dir = extracted_file_path = ''
# Get Package details
package = bundle.obj
# If local file exists - return that
if not package.is_compressed:
full_path = package.fetch_local_path()
# The basename of the AIP may be included with the request, because
# all AIPs contain a base directory. That directory may already be
# inside the full path though, so remove the basename only if the
# relative path begins with it.
basename = os.path.join(os.path.basename(full_path), '')
if relative_path_to_file.startswith(basename):
relative_path_to_file = relative_path_to_file.replace(
basename, '', 1)
extracted_file_path = os.path.join(full_path,
relative_path_to_file)
if not os.path.exists(extracted_file_path):
return http.HttpResponse(
status=404,
content="Requested file, {}, not found in AIP".format(
relative_path_to_file))
elif package.package_type in Package.PACKAGE_TYPE_CAN_EXTRACT:
# If file doesn't exist, try to extract it
(extracted_file_path,
temp_dir) = package.extract_file(relative_path_to_file)
else:
# If the package is compressed and we can't extract it,
return http.HttpResponse(
status=501,
content="Unable to extract package of type: {}".format(
package.package_type))
response = utils.download_file_stream(extracted_file_path, temp_dir)
return response
def process_request(self, request):
if 'HTTP_ACCESS_CONTROL_REQUEST_METHOD' in request.META:
response = http.HttpResponse()
response[
'Access-Control-Allow-Origin'] = XS_SHARING_ALLOWED_ORIGINS
response['Access-Control-Allow-Methods'] = ",".join(
XS_SHARING_ALLOWED_METHODS)
return response
return None
def obj_get(self, request, **kwargs):
# This is kinda weird and we don't really need to go through the entire
# object dehydrate cycle, but just fake out our response.
try:
obj = Delayable.objects.get(**kwargs)
except Delayable.DoesNotExist:
raise ImmediateHttpResponse(response=http.HttpNotFound())
response = http.HttpResponse(obj.content,
content_type='application/json')
response['Solitude-Async'] = 'yes' if obj.run else 'no'
response.status_code = obj.status_code
raise ImmediateHttpResponse(response=response)
def recover_aip_request(self, request, bundle, **kwargs):
request_info = bundle.data
package = bundle.obj
if package.package_type not in Package.PACKAGE_TYPE_CAN_RECOVER:
# Can only request recovery of AIPs
response = {
"message": "Recovery not allowed on this package type."
}
response_json = json.dumps(response)
return http.HttpMethodNotAllowed(response_json,
content_type='application/json')
(status_code, response) = self._attempt_package_request_event(
package, request_info, Event.RECOVER, Package.RECOVER_REQ)
self.log_throttled_access(request)
response_json = json.dumps(response)
return http.HttpResponse(status=status_code,
content=response_json,
mimetype='application/json')
def dispatch(self, request_type, request, **kw):
method = request.META['REQUEST_METHOD']
delay = request.META.get('HTTP_SOLITUDE_ASYNC', False)
if delay:
# Move the import here to remove warnings in management commands.
from lib.delayable.tasks import delayable
# Only do async on these requests.
if method not in ['PATCH', 'POST', 'PUT']:
raise ImmediateHttpResponse(
response=http.HttpMethodNotAllowed())
# Create a delayed dispatch.
uid = str(uuid.uuid4())
# We only need a subset of meta.
whitelist = ['PATH_INFO', 'REQUEST_METHOD', 'QUERY_STRING']
meta = dict([k, request.META[k]] for k in whitelist)
# Celery could magically serialise some of this, but I don't
# trust it that much.
delayable.delay(self.__class__.__module__, self.__class__.__name__,
request_type, meta, request.body, kw, uid)
content = json.dumps({
'replay': '/delay/replay/%s/' % uid,
'result': '/delay/result/%s/' % uid
})
return http.HttpResponse(content,
status=202,
content_type='application/json')
# Log the call with CEF and logging.
if settings.DUMP_REQUESTS:
print colorize('brace', method), request.get_full_path()
else:
log.info('%s %s' %
(colorize('brace', method), request.get_full_path()))
msg = '%s:%s' % (kw.get('api_name',
'unknown'), kw.get('resource_name', 'unknown'))
log_cef(msg, request, severity=2)
return super(TastypieBaseResource,
self).dispatch(request_type, request, **kw)
def check_fixity_request(self, request, bundle, **kwargs):
success, failures, message = bundle.obj.check_fixity()
response = {
"success": success,
"message": message,
"failures": {
"files": {
"missing": [],
"changed": [],
"untracked": [],
}
}
}
for failure in failures:
if isinstance(failure, bagit.FileMissing):
info = {"path": failure.path, "message": str(failure)}
response["failures"]["files"]["missing"].append(info)
if isinstance(failure, bagit.ChecksumMismatch):
info = {
"path": failure.path,
"expected": failure.expected,
"actual": failure.found,
"hash_type": failure.algorithm,
"message": str(failure),
}
response["failures"]["files"]["changed"].append(info)
if isinstance(failure, bagit.UnexpectedFile):
info = {"path": failure.path, "message": str(failure)}
response["failures"]["files"]["untracked"].append(info)
report = json.dumps(response)
if not success:
signals.failed_fixity_check.send(sender=self,
uuid=bundle.obj.uuid,
location=bundle.obj.full_path,
report=report)
return http.HttpResponse(report, mimetype="application/json")
def register_in_raffle(self, request, **kwargs):
from server.bom.raffle import Participant
self.method_check(request, allowed=['post'])
self.throttle_check(request)
draw_id = kwargs['pk']
try:
data = json.loads(request.body)
except TypeError:
data = json.loads(request.body.decode('utf-8'))
try:
participant_id = data['participant_id']
participant_name = data['participant_name']
except KeyError:
raise exceptions.ImmediateHttpResponse(
response=http.HttpBadRequest(
"Missing participant_id or participant_name"))
participant = Participant(id=participant_id, name=participant_name)
try:
bom_draw = self._client.retrieve_draw(draw_id)
except mongodb.MongoDriver.NotFoundError:
raise exceptions.ImmediateHttpResponse(
response=http.HttpNotFound("the draw does not exists"))
if not isinstance(bom_draw, bom.RaffleDraw):
raise exceptions.ImmediateHttpResponse(
response=http.HttpBadRequest(
"Registration is only available in Raffles"))
try:
bom_draw.register_participant(participant)
except bom.RaffleDraw.RegistrationError as e:
raise exceptions.ImmediateHttpResponse(
response=http.HttpBadRequest(e))
except bom.RaffleDraw.AlreadyRegisteredError:
raise exceptions.ImmediateHttpResponse(
response=http.HttpNotModified())
self._client.save_draw(bom_draw)
self.log_throttled_access(request)
return http.HttpResponse()
def in_bulk(request):
"""
Parses an uploaded HDF5 'Delta' file with new/changed objects tree and
creates/updates the database using appropriate API Resources.
Tests for this function are available only at the client side.
:param request: multipart/form-data request with 'raw_file' delta file
that contains objects to be saved
:return "top"-object as normal JSON response
"""
# always save file to disk by removing MemoryFileUploadHandler
for handler in request.upload_handlers:
if isinstance(handler, MemoryFileUploadHandler):
request.upload_handlers.remove(handler)
if not (request.method == 'POST' and len(request.FILES) > 0):
return http.HttpBadRequest("Supporting only POST multipart/form-data"
" requests with files")
if not 'raw_file' in request.FILES:
return http.HttpBadRequest("The name of the field should be "
"'raw_file'")
if not request.user.is_authenticated():
return http.HttpUnauthorized("Must authorize before uploading")
path = request.FILES['raw_file'].temporary_file_path()
try:
f = h5py.File(path, 'r')
except IOError:
return http.HttpBadRequest("Uploaded file is not an HDF5 file")
incoming_locations = f.keys()
todo = [] # array of ids to process as an ordered sequence
ids_map = {} # map of the temporary IDs to the new IDs of created objects
saved = [] # collector of processed objects
# this loop sorts object tree as "breadth-first" sequence based on their
# parent <- children relations
while incoming_locations:
location = incoming_locations[0]
json_obj = json.loads(f[location]['json'].value)
model_name = location.split('-')[4] # FIXME make more robust
fk_names = get_fk_field_names(model_name)
m2m_names = get_m2m_field_names(model_name)
parents = [v for k, v in json_obj.items() if k in fk_names]
m2ms = [v for k, v in json_obj.items() if k in m2m_names]
m2m_flat = [v for m2m in m2ms for v in m2m]
match = lambda x: len(
[k for k in incoming_locations if k.split('-')[5] == x]) > 0
if len([x for x in parents + m2m_flat if match(x)]) == 0:
todo.append(location)
incoming_locations.remove(location)
else:
incoming_locations.append(incoming_locations.pop(0))
# this loop saves actual objects
while todo:
location = todo[0]
group = f[location]
json_obj = json.loads(group['json'].value)
_, _, _, _, model_name, obj_id, _ = location.split(
'-') # FIXME robust?
fk_names = get_fk_field_names(model_name)
m2m_names = get_m2m_field_names(model_name)
# update parent IDs to the IDs of created objects
to_update = [k for k in json_obj.keys() if k in fk_names]
for name in to_update:
value = json_obj[name]
if value is not None and value.startswith('TEMP'):
json_obj[name] = ids_map[value]
# update m2m IDs to the IDs of created objects
to_update = [k for k in json_obj.keys() if k in m2m_names]
for name in to_update:
m2m_list = json_obj[name]
if m2m_list is not None:
json_obj[name] = [
ids_map[x] if x.startswith('TEMP') else x for x in m2m_list
]
res = RESOURCES[model_name]
if obj_id.startswith('TEMP'): # create new object
bundle = res.build_bundle(request=request, data=json_obj)
res_bundle = res.obj_create(bundle)
ids_map[obj_id] = res_bundle.obj.local_id
else: # update object
request_bundle = res.build_bundle(request=request)
obj = res.obj_get(request_bundle, pk=obj_id)
bundle = res.build_bundle(obj=obj, data=json_obj, request=request)
res_bundle = res.obj_update(bundle)
# update data fields. no need to check permissions as they must be
# already validated with the object update
data_fields = [k for k in group.keys() if not k == 'json']
temp_paths = [] # collector of temp data files
for name in data_fields:
filename = uuid.uuid1().hex + ".h5"
path = os.path.join(tmp.gettempdir(), filename)
with h5py.File(path, 'w') as temp_f:
temp_f.create_dataset(name=res_bundle.obj.local_id,
data=group[name].value)
setattr(res_bundle.obj, name, File(open(path), name=filename))
temp_paths.append(path)
if len(data_fields) > 0: # clean temp files
res_bundle.obj.save()
for path in temp_paths:
os.remove(path)
saved.append((model_name, res_bundle.obj.local_id))
todo.remove(location)
model_name, obj_id = saved[0] # return top object
res = RESOURCES[model_name]
bundle = res.build_bundle(request=request)
obj = res.obj_get(bundle, pk=obj_id)
res_bundle = res.build_bundle(obj=obj, request=request)
response = http.HttpResponse(
res.serialize(None, res.full_dehydrate(res_bundle),
'application/json'))
return response
def _add_files_to_package(self, request, bundle, **kwargs):
"""
Adds a set of files to a package.
The PUT body must be a list of zero or more JavaScript objects in the following format:
{
"relative_path": "string",
"fileuuid": "string",
"accessionid", "string",
"sipuuid": "string",
"origin": "string"
}
"""
try:
files_list = json.load(request)
except ValueError:
response = {
"success": False,
"error": "No JSON object could be decoded from POST body."
}
return http.HttpBadRequest(json.dumps(response),
content_type="application/json")
if not isinstance(files_list, list):
response = {
"success": False,
"error": "JSON request must contain a list of objects."
}
return http.HttpBadRequest(json.dumps(response),
content_type="application/json")
property_map = {
"relative_path": "name",
"fileuuid": "source_id",
"accessionid": "accessionid",
"sipuuid": "source_package",
"origin": "origin",
}
if len(files_list) == 0:
return http.HttpResponse()
created_files = []
for f in files_list:
kwargs = {"package": bundle.obj}
for source, dest in property_map.iteritems():
try:
kwargs[dest] = f[source]
except KeyError:
response = {
"success": False,
"error": "File object was missing key: " + source
}
return http.HttpBadRequest(json.dumps(response),
content_type="application_json")
created_files.append(File(**kwargs))
for f in created_files:
f.save()
response = {
"success":
True,
"message":
"{} files created in package {}".format(len(created_files),
bundle.obj.uuid)
}
return http.HttpCreated(json.dumps(response),
content_type="application_json")
