def list_display(request): if 'aips' not in settings.SEARCH_ENABLED: return render(request, 'archival_storage/list.html') current_page_number = int(request.GET.get('page', 1)) logger.debug('Current page: %s', current_page_number) # get count of AIP files es_client = elasticSearchFunctions.get_client() aip_indexed_file_count = aip_file_count(es_client) # get AIPs order_by = request.GET.get('order_by', 'name_unanalyzed') sort_by = request.GET.get('sort_by', 'up') if sort_by == 'down': sort_direction = 'desc' else: sort_direction = 'asc' sort_specification = order_by + ':' + sort_direction sort_params = 'order_by=' + order_by + '&sort_by=' + sort_by # get list of UUIDs of AIPs that are deleted or pending deletion aips_deleted_or_pending_deletion = [] should_haves = [ {'match': {'status': 'DEL_REQ'}}, {'match': {'status': 'DELETED'}}, ] query = { "query": { "bool": { "should": should_haves } } } deleted_aip_results = es_client.search( body=query, index='aips', doc_type='aip', fields='uuid,status' ) for deleted_aip in deleted_aip_results['hits']['hits']: aips_deleted_or_pending_deletion.append(deleted_aip['fields']['uuid'][0]) # Fetch results and paginate def es_pager(page, page_size): """ Fetch one page of normalized entries from Elasticsearch. :param page: 1-indexed page to fetch :param page_size: Number of entries on a page :return: List of dicts for each entry, where keys and values have been cleaned up """ start = (page - 1) * page_size results = es_client.search( index='aips', doc_type='aip', body=elasticSearchFunctions.MATCH_ALL_QUERY, fields='origin,uuid,filePath,created,name,size,encrypted', sort=sort_specification, size=page_size, from_=start, ) # normalize results - each of the fields contains a single value, # but is returned from the ES API as a single-length array # e.g. {"fields": {"uuid": ["abcd"], "name": ["aip"] ...}} return [elasticSearchFunctions.normalize_results_dict(d) for d in results['hits']['hits']] items_per_page = 10 count = es_client.count(index='aips', doc_type='aip', body=elasticSearchFunctions.MATCH_ALL_QUERY)['count'] results = LazyPagedSequence(es_pager, page_size=items_per_page, length=count) # Paginate page = helpers.pager( results, items_per_page, current_page_number ) # process deletion, etc., and format results aips = [] for aip in page.object_list: # If an AIP was deleted or is pending deletion, react if status changed if aip['uuid'] in aips_deleted_or_pending_deletion: # check with storage server to see current status api_results = storage_service.get_file_info(uuid=aip['uuid']) try: aip_status = api_results[0]['status'] except IndexError: # Storage service does not know about this AIP # TODO what should happen here? logger.info("AIP not found in storage service: {}".format(aip)) continue # delete AIP metadata in ElasticSearch if AIP has been deleted from the # storage server # TODO: handle this asynchronously if aip_status == 'DELETED': elasticSearchFunctions.delete_aip(es_client, aip['uuid']) elasticSearchFunctions.delete_aip_files(es_client, aip['uuid']) elif aip_status != 'DEL_REQ': # update the status in ElasticSearch for this AIP elasticSearchFunctions.mark_aip_stored(es_client, aip['uuid']) else: aip_status = 'UPLOADED' # Tweak AIP presentation and add to display array if aip_status != 'DELETED': aip['status'] = AIP_STATUS_DESCRIPTIONS[aip_status] try: size = '{0:.2f} MB'.format(float(aip['size'])) except (TypeError, ValueError): size = 'Removed' aip['size'] = size aip['href'] = aip['filePath'].replace(AIPSTOREPATH + '/', "AIPsStore/") aip['date'] = aip['created'] aips.append(aip) total_size = total_size_of_aips(es_client) # Find out which AIPs are encrypted return render(request, 'archival_storage/list.html', { 'total_size': total_size, 'aip_indexed_file_count': aip_indexed_file_count, 'aips': aips, 'page': page, 'search_params': sort_params, } )
def list_display(request): if "aips" not in settings.SEARCH_ENABLED: return render(request, "archival_storage/list.html") current_page_number = int(request.GET.get("page", 1)) logger.debug("Current page: %s", current_page_number) # get count of AIP files es_client = elasticSearchFunctions.get_client() aip_indexed_file_count = aip_file_count(es_client) # get AIPs order_by = request.GET.get("order_by", "name") sort_by = request.GET.get("sort_by", "up") sort_params = "order_by=" + order_by + "&sort_by=" + sort_by # use raw subfield to sort by name if order_by == "name": order_by = order_by + ".raw" # change sort_by param to ES sort directions if sort_by == "down": sort_by = "desc" else: sort_by = "asc" sort_specification = order_by + ":" + sort_by # get list of UUIDs of AIPs that are deleted or pending deletion aips_deleted_or_pending_deletion = [] should_haves = [{ "match": { "status": "DEL_REQ" } }, { "match": { "status": "DELETED" } }] query = {"query": {"bool": {"should": should_haves}}} deleted_aip_results = es_client.search(body=query, index="aips", _source="uuid,status") for deleted_aip in deleted_aip_results["hits"]["hits"]: aips_deleted_or_pending_deletion.append(deleted_aip["_source"]["uuid"]) # Fetch results and paginate def es_pager(page, page_size): """ Fetch one page of normalized entries from Elasticsearch. :param page: 1-indexed page to fetch :param page_size: Number of entries on a page :return: List of dicts for each entry, where keys and values have been cleaned up """ start = (page - 1) * page_size results = es_client.search( index="aips", body={"query": { "match_all": {} }}, _source="origin,uuid,filePath,created,name,size,encrypted", sort=sort_specification, size=page_size, from_=start, ) return [d["_source"] for d in results["hits"]["hits"]] items_per_page = 10 count = es_client.count(index="aips", body={"query": { "match_all": {} }})["count"] results = LazyPagedSequence(es_pager, page_size=items_per_page, length=count) # Paginate page = helpers.pager(results, items_per_page, current_page_number) # process deletion, etc., and format results aips = [] for aip in page.object_list: # If an AIP was deleted or is pending deletion, react if status changed if aip["uuid"] in aips_deleted_or_pending_deletion: # check with storage server to see current status api_results = storage_service.get_file_info(uuid=aip["uuid"]) try: aip_status = api_results[0]["status"] except IndexError: # Storage service does not know about this AIP # TODO what should happen here? logger.info("AIP not found in storage service: {}".format(aip)) continue # delete AIP metadata in ElasticSearch if AIP has been deleted from the # storage server # TODO: handle this asynchronously if aip_status == "DELETED": elasticSearchFunctions.delete_aip(es_client, aip["uuid"]) elasticSearchFunctions.delete_aip_files(es_client, aip["uuid"]) elif aip_status != "DEL_REQ": # update the status in ElasticSearch for this AIP elasticSearchFunctions.mark_aip_stored(es_client, aip["uuid"]) else: aip_status = "UPLOADED" # Tweak AIP presentation and add to display array if aip_status != "DELETED": aip["status"] = AIP_STATUS_DESCRIPTIONS[aip_status] try: size = "{0:.2f} MB".format(float(aip["size"])) except (TypeError, ValueError): size = "Removed" aip["size"] = size aip["href"] = aip["filePath"].replace(AIPSTOREPATH + "/", "AIPsStore/") aip["date"] = aip["created"] aips.append(aip) total_size = total_size_of_aips(es_client) # Find out which AIPs are encrypted return render( request, "archival_storage/list.html", { "total_size": total_size, "aip_indexed_file_count": aip_indexed_file_count, "aips": aips, "page": page, "search_params": sort_params, }, )