def list_display(request):
if 'aips' not in settings.SEARCH_ENABLED:
return render(request, 'archival_storage/list.html')
current_page_number = int(request.GET.get('page', 1))
logger.debug('Current page: %s', current_page_number)
# get count of AIP files
es_client = elasticSearchFunctions.get_client()
aip_indexed_file_count = aip_file_count(es_client)
# get AIPs
order_by = request.GET.get('order_by', 'name_unanalyzed')
sort_by = request.GET.get('sort_by', 'up')
if sort_by == 'down':
sort_direction = 'desc'
else:
sort_direction = 'asc'
sort_specification = order_by + ':' + sort_direction
sort_params = 'order_by=' + order_by + '&sort_by=' + sort_by
# get list of UUIDs of AIPs that are deleted or pending deletion
aips_deleted_or_pending_deletion = []
should_haves = [
{'match': {'status': 'DEL_REQ'}},
{'match': {'status': 'DELETED'}},
]
query = {
"query": {
"bool": {
"should": should_haves
}
}
}
deleted_aip_results = es_client.search(
body=query,
index='aips',
doc_type='aip',
fields='uuid,status'
)
for deleted_aip in deleted_aip_results['hits']['hits']:
aips_deleted_or_pending_deletion.append(deleted_aip['fields']['uuid'][0])
# Fetch results and paginate
def es_pager(page, page_size):
"""
Fetch one page of normalized entries from Elasticsearch.
:param page: 1-indexed page to fetch
:param page_size: Number of entries on a page
:return: List of dicts for each entry, where keys and values have been cleaned up
"""
start = (page - 1) * page_size
results = es_client.search(
index='aips',
doc_type='aip',
body=elasticSearchFunctions.MATCH_ALL_QUERY,
fields='origin,uuid,filePath,created,name,size,encrypted',
sort=sort_specification,
size=page_size,
from_=start,
)
# normalize results - each of the fields contains a single value,
# but is returned from the ES API as a single-length array
# e.g. {"fields": {"uuid": ["abcd"], "name": ["aip"] ...}}
return [elasticSearchFunctions.normalize_results_dict(d) for d in results['hits']['hits']]
items_per_page = 10
count = es_client.count(index='aips', doc_type='aip', body=elasticSearchFunctions.MATCH_ALL_QUERY)['count']
results = LazyPagedSequence(es_pager, page_size=items_per_page, length=count)
# Paginate
page = helpers.pager(
results,
items_per_page,
current_page_number
)
# process deletion, etc., and format results
aips = []
for aip in page.object_list:
# If an AIP was deleted or is pending deletion, react if status changed
if aip['uuid'] in aips_deleted_or_pending_deletion:
# check with storage server to see current status
api_results = storage_service.get_file_info(uuid=aip['uuid'])
try:
aip_status = api_results[0]['status']
except IndexError:
# Storage service does not know about this AIP
# TODO what should happen here?
logger.info("AIP not found in storage service: {}".format(aip))
continue
# delete AIP metadata in ElasticSearch if AIP has been deleted from the
# storage server
# TODO: handle this asynchronously
if aip_status == 'DELETED':
elasticSearchFunctions.delete_aip(es_client, aip['uuid'])
elasticSearchFunctions.delete_aip_files(es_client, aip['uuid'])
elif aip_status != 'DEL_REQ':
# update the status in ElasticSearch for this AIP
elasticSearchFunctions.mark_aip_stored(es_client, aip['uuid'])
else:
aip_status = 'UPLOADED'
# Tweak AIP presentation and add to display array
if aip_status != 'DELETED':
aip['status'] = AIP_STATUS_DESCRIPTIONS[aip_status]
try:
size = '{0:.2f} MB'.format(float(aip['size']))
except (TypeError, ValueError):
size = 'Removed'
aip['size'] = size
aip['href'] = aip['filePath'].replace(AIPSTOREPATH + '/', "AIPsStore/")
aip['date'] = aip['created']
aips.append(aip)
total_size = total_size_of_aips(es_client)
# Find out which AIPs are encrypted
return render(request, 'archival_storage/list.html',
{
'total_size': total_size,
'aip_indexed_file_count': aip_indexed_file_count,
'aips': aips,
'page': page,
'search_params': sort_params,
}
)
def list_display(request):
if "aips" not in settings.SEARCH_ENABLED:
return render(request, "archival_storage/list.html")
current_page_number = int(request.GET.get("page", 1))
logger.debug("Current page: %s", current_page_number)
# get count of AIP files
es_client = elasticSearchFunctions.get_client()
aip_indexed_file_count = aip_file_count(es_client)
# get AIPs
order_by = request.GET.get("order_by", "name")
sort_by = request.GET.get("sort_by", "up")
sort_params = "order_by=" + order_by + "&sort_by=" + sort_by
# use raw subfield to sort by name
if order_by == "name":
order_by = order_by + ".raw"
# change sort_by param to ES sort directions
if sort_by == "down":
sort_by = "desc"
else:
sort_by = "asc"
sort_specification = order_by + ":" + sort_by
# get list of UUIDs of AIPs that are deleted or pending deletion
aips_deleted_or_pending_deletion = []
should_haves = [{
"match": {
"status": "DEL_REQ"
}
}, {
"match": {
"status": "DELETED"
}
}]
query = {"query": {"bool": {"should": should_haves}}}
deleted_aip_results = es_client.search(body=query,
index="aips",
_source="uuid,status")
for deleted_aip in deleted_aip_results["hits"]["hits"]:
aips_deleted_or_pending_deletion.append(deleted_aip["_source"]["uuid"])
# Fetch results and paginate
def es_pager(page, page_size):
"""
Fetch one page of normalized entries from Elasticsearch.
:param page: 1-indexed page to fetch
:param page_size: Number of entries on a page
:return: List of dicts for each entry, where keys and values have been cleaned up
"""
start = (page - 1) * page_size
results = es_client.search(
index="aips",
body={"query": {
"match_all": {}
}},
_source="origin,uuid,filePath,created,name,size,encrypted",
sort=sort_specification,
size=page_size,
from_=start,
)
return [d["_source"] for d in results["hits"]["hits"]]
items_per_page = 10
count = es_client.count(index="aips", body={"query": {
"match_all": {}
}})["count"]
results = LazyPagedSequence(es_pager,
page_size=items_per_page,
length=count)
# Paginate
page = helpers.pager(results, items_per_page, current_page_number)
# process deletion, etc., and format results
aips = []
for aip in page.object_list:
# If an AIP was deleted or is pending deletion, react if status changed
if aip["uuid"] in aips_deleted_or_pending_deletion:
# check with storage server to see current status
api_results = storage_service.get_file_info(uuid=aip["uuid"])
try:
aip_status = api_results[0]["status"]
except IndexError:
# Storage service does not know about this AIP
# TODO what should happen here?
logger.info("AIP not found in storage service: {}".format(aip))
continue
# delete AIP metadata in ElasticSearch if AIP has been deleted from the
# storage server
# TODO: handle this asynchronously
if aip_status == "DELETED":
elasticSearchFunctions.delete_aip(es_client, aip["uuid"])
elasticSearchFunctions.delete_aip_files(es_client, aip["uuid"])
elif aip_status != "DEL_REQ":
# update the status in ElasticSearch for this AIP
elasticSearchFunctions.mark_aip_stored(es_client, aip["uuid"])
else:
aip_status = "UPLOADED"
# Tweak AIP presentation and add to display array
if aip_status != "DELETED":
aip["status"] = AIP_STATUS_DESCRIPTIONS[aip_status]
try:
size = "{0:.2f} MB".format(float(aip["size"]))
except (TypeError, ValueError):
size = "Removed"
aip["size"] = size
aip["href"] = aip["filePath"].replace(AIPSTOREPATH + "/",
"AIPsStore/")
aip["date"] = aip["created"]
aips.append(aip)
total_size = total_size_of_aips(es_client)
# Find out which AIPs are encrypted
return render(
request,
"archival_storage/list.html",
{
"total_size": total_size,
"aip_indexed_file_count": aip_indexed_file_count,
"aips": aips,
"page": page,
"search_params": sort_params,
},
)