def tasks(request, uuid): job = models.Job.objects.get(jobuuid=uuid) objects = job.task_set.all().order_by('-exitcode', '-endtime', '-starttime', '-createdtime') if (len(objects) == 0): return tasks_subjobs(request, uuid) # Filenames can be any encoding - we want to be able to display # unicode, while just displaying unicode replacement characters # for any other encoding present. for item in objects: item.filename = escape(item.filename) item.arguments = escape(item.arguments) item.stdout = escape(item.stdout) item.stderror = escape(item.stderror) page = helpers.pager(objects, django_settings.TASKS_PER_PAGE, request.GET.get('page', None)) objects = page.object_list # figure out duration in seconds for object in objects: object.duration = helpers.task_duration_in_seconds(object) return render(request, 'main/tasks.html', locals())
def ingest_upload_atk_resource_component(request, uuid, resource_component_id): db = ingest_upload_atk_db_connection() try: query = request.GET.get('query', '').strip() resource_component_data = atk.get_resource_component_and_children( db, resource_component_id, 'description', recurse_max_level=2, search_pattern=query) if resource_component_data['children']: page = helpers.pager(resource_component_data['children'], 10, request.GET.get('page', 1)) except MySQLdb.ProgrammingError: return HttpResponseServerError( 'Database error. Please contact an administrator.') resource_id = ingest_upload_atk_determine_resource_component_resource_id( resource_component_id) if not resource_component_data['children'] and query == '': return HttpResponseRedirect( reverse( 'components.ingest.views_atk.ingest_upload_atk_match_dip_objects_to_resource_component_levels', args=[uuid, resource_component_id])) else: search_params = advanced_search.extract_url_search_params_from_request( request) return render(request, 'ingest/atk/resource_component.html', locals())
def render_resource( client, request, resource_id, query, page, sort_by, search_params, match_redirect_target, resource_detail_template, reset_url, uuid, ): resource_data = client.get_resource_component_and_children( resource_id, "collection", recurse_max_level=2, search_pattern=query, sort_by=sort_by, ) sort_direction = _determine_reverse_sort_direction(sort_by) if resource_data["children"]: page = helpers.pager(resource_data["children"], PAGE_SIZE, page) if not resource_data["children"] and query == "": return HttpResponseRedirect( reverse(match_redirect_target, args=[uuid, resource_id])) else: return render(request, resource_detail_template, locals())
def list_records( client, request, query, identifier, page_number, sort_by, search_params, list_redirect_target, reset_url, uuid, ): resources = LazyPagedSequence( lambda page, page_size: client.find_collections( search_pattern=query, identifier=identifier, page=page, page_size=page_size, sort_by=sort_by, ), PAGE_SIZE, client.count_collections(query, identifier), ) page = helpers.pager(resources, PAGE_SIZE, page_number) sort_direction = _determine_reverse_sort_direction(sort_by) return render(request, list_redirect_target, locals())
def render_resource_component(client, request, resource_component_id, query, page, sort_by, search_params, match_redirect_target, resource_detail_template, reset_url, uuid): resource_component_data = client.get_resource_component_and_children( resource_component_id, 'description', recurse_max_level=2, search_pattern=query, sort_by=sort_by) sort_direction = _determine_reverse_sort_direction(sort_by) if resource_component_data['children']: page = helpers.pager(resource_component_data['children'], PAGE_SIZE, page) if not resource_component_data['children'] and query == '': return HttpResponseRedirect( reverse(match_redirect_target, args=[uuid, resource_component_id])) else: return render( request, resource_detail_template, { 'match_redirect_target': match_redirect_target, 'page': page, 'query': query, 'reset_url': reset_url, 'resource_component_data': resource_component_data, 'resource_component_id': resource_component_id, 'search_params': search_params, 'sort_by': sort_by, 'sort_direction': sort_direction, 'uuid': uuid, })
def archival_storage_list_display(request, current_page_number=None): form = forms.StorageSearchForm() total_size = 0 # get ElasticSearch stats aip_indexed_file_count = advanced_search.indexed_count('aips') # get AIPs conn = elasticSearchFunctions.connect_and_create_index('aips') aipResults = conn.search(pyes.StringQuery('*'), doc_types=['aip']) aips = [] #if aipResults._total != None: if len(aipResults) > 0: for aip in aipResults: aips.append(aip) # handle pagination page = helpers.pager(aips, 10, current_page_number) sips = [] for aip in page['objects']: sip = {} sip['href'] = aip.filePath.replace(AIPSTOREPATH + '/', "AIPsStore/") sip['name'] = aip.name sip['uuid'] = aip.uuid #sip['date'] = str(aip.date)[0:19].replace('T', ' ') sip['date'] = aip.created try: size = float(aip.size) total_size = total_size + size sip['size'] = '{0:.2f} MB'.format(size) except: sip['size'] = 'Removed' sips.append(sip) order_by = request.GET.get('order_by', 'name') sort_by = request.GET.get('sort_by', 'up') def sort_aips(sip): value = 0 if 'name' == order_by: value = sip['name'].lower() else: value = sip[order_by] return value sips = sorted(sips, key=sort_aips) if sort_by == 'down': sips.reverse() total_size = '{0:.2f}'.format(total_size) return render(request, 'archival_storage/archival_storage.html', locals())
def archival_storage_list_display(request, current_page_number=None): form = forms.StorageSearchForm() total_size = 0 # get ElasticSearch stats aip_indexed_file_count = advanced_search.indexed_count('aips') # get AIPs conn = elasticSearchFunctions.connect_and_create_index('aips') aipResults = conn.search(pyes.StringQuery('*'), doc_types=['aip']) aips = [] #if aipResults._total != None: if len(aipResults) > 0: for aip in aipResults: aips.append(aip) # handle pagination page = helpers.pager(aips, 10, current_page_number) sips = [] for aip in page['objects']: sip = {} sip['href'] = aip.filePath.replace(AIPSTOREPATH + '/', "AIPsStore/") sip['name'] = aip.name sip['uuid'] = aip.uuid #sip['date'] = str(aip.date)[0:19].replace('T', ' ') sip['date'] = aip.created try: size = float(aip.size) total_size = total_size + size sip['size'] = '{0:.2f} MB'.format(size) except: sip['size'] = 'Removed' sips.append(sip) order_by = request.GET.get('order_by', 'name'); sort_by = request.GET.get('sort_by', 'up'); def sort_aips(sip): value = 0 if 'name' == order_by: value = sip['name'].lower() else: value = sip[order_by] return value sips = sorted(sips, key = sort_aips) if sort_by == 'down': sips.reverse() total_size = '{0:.2f}'.format(total_size) return render(request, 'archival_storage/archival_storage.html', locals())
def failure_report(request, report_id=None): if report_id is not None: report = models.Report.objects.get(pk=report_id) return render(request, 'administration/reports/failure_detail.html', locals()) else: current_page_number = request.GET.get('page', '1') items_per_page = 10 reports = models.Report.objects.all().order_by('-created') page = helpers.pager(reports, items_per_page, current_page_number) return render(request, 'administration/reports/failures.html', locals())
def tasks(request, uuid): job = models.Job.objects.get(jobuuid=uuid) objects = job.task_set.all().order_by('-exitcode', '-endtime', '-starttime', '-createdtime') page = helpers.pager(objects, django_settings.TASKS_PER_PAGE, request.GET.get('page', None)) objects = page['objects'] # figure out duration in seconds for object in objects: object.duration = helpers.task_duration_in_seconds(object) return render(request, 'main/tasks.html', locals())
def preservation_planning_fpr_search(request, current_page_number = None): if current_page_number == None: current_page_number = 1 query = request.GET.get('query', '') if query == '': # No query in the URL parameters list, try to see if we've got an existing query going from a previous page... query = request.session['fpr_query'] # No query from a previous page either if query == '': query = '*' return HttpResponse('No query.') request.session['fpr_query'] = query # Save this for pagination... conn = pyes.ES(elasticSearchFunctions.getElasticsearchServerHostAndPort()) indexes = conn.get_indices() if 'fpr_file' not in indexes: # Grab relevant FPR data from the DB results = get_fpr_table() request.session['fpr_results'] = results # Setup indexing for some Elastic Search action. for row in results: conn.index(row, 'fpr_file', 'fpr_files') else: results = request.session['fpr_results'] # do fulltext search q = pyes.StringQuery(query) s = pyes.Search(q) try: results = conn.search_raw(s, size=len(results), indices='fpr_file') except: return HttpResponse('Error accessing index.') form = FPRSearchForm() search_hits = [] for row in results.hits.hits: search_hits.append(row['_source'].copy()) page = helpers.pager(search_hits, results_per_page, current_page_number) hit_count = len(search_hits) return render(request, 'main/preservation_planning_fpr.html', locals())
def preservation_planning_fpr_search(request, current_page_number=None): if current_page_number == None: current_page_number = 1 query = request.GET.get("query", "") if query == "": # No query in the URL parameters list, try to see if we've got an existing query going from a previous page... query = request.session["fpr_query"] # No query from a previous page either if query == "": query = "*" return HttpResponse("No query.") request.session["fpr_query"] = query # Save this for pagination... conn = pyes.ES(elasticSearchFunctions.getElasticsearchServerHostAndPort()) indexes = conn.get_indices() if "fpr_file" not in indexes: # Grab relevant FPR data from the DB results = get_fpr_table() request.session["fpr_results"] = results # Setup indexing for some Elastic Search action. for row in results: conn.index(row, "fpr_file", "fpr_files") else: results = request.session["fpr_results"] # do fulltext search q = pyes.StringQuery(query) s = pyes.Search(q) try: results = conn.search_raw(s, size=len(results), indices="fpr_file") except: return HttpResponse("Error accessing index.") form = FPRSearchForm() search_hits = [] for row in results.hits.hits: search_hits.append(row["_source"].copy()) page = helpers.pager(search_hits, results_per_page, current_page_number) hit_count = len(search_hits) return render(request, "main/preservation_planning_fpr.html", locals())
def archival_storage_sip_display(request, current_page_number=None): form = forms.StorageSearchForm() total_size = 0 # get ElasticSearch stats aip_indexed_file_count = archival_storage_indexed_count('aips') # get AIPs from DB aips = models.AIP.objects.all() # handle pagination page = helpers.pager(aips, 10, current_page_number) sips = [] for aip in page['objects']: sip = {} sip['href'] = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/") sip['name'] = aip.sipname sip['uuid'] = aip.sipuuid sip['date'] = aip.sipdate try: size = os.path.getsize(aip.filepath) / float(1024) / float(1024) total_size = total_size + size sip['size'] = '{0:.2f} MB'.format(size) except: sip['size'] = 'Removed' sips.append(sip) order_by = request.GET.get('order_by', 'name'); sort_by = request.GET.get('sort_by', 'up'); def sort_aips(sip): value = 0 if 'name' == order_by: value = sip['name'].lower() else: value = sip[order_by] return value sips = sorted(sips, key = sort_aips) if sort_by == 'down': sips.reverse() total_size = '{0:.2f}'.format(total_size) return render(request, 'archival_storage/archival_storage.html', locals())
def ingest_normalization_report(request, uuid, current_page=None): jobs = models.Job.objects.filter(sipuuid=uuid, subjobof='') job = jobs[0] sipname = utils.get_directory_name(job) objects = getNormalizationReportQuery(sipUUID=uuid) results_per_page = 10 if current_page == None: current_page = 1 page = helpers.pager(objects, results_per_page, current_page) hit_count = len(objects) return render(request, 'ingest/normalization_report.html', locals())
def preservation_planning_fpr_data(request, current_page_number=None): results = get_fpr_table() request.session["fpr_results"] = results if current_page_number == None: current_page_number = 1 form = FPRSearchForm() page = helpers.pager(results, results_per_page, current_page_number) request.session["fpr_query"] = "" item_count = len(results) return render(request, "main/preservation_planning_fpr.html", locals())
def preservation_planning_fpr_data(request, current_page_number = None): results = get_fpr_table() request.session['fpr_results'] = results if current_page_number == None: current_page_number = 1 form = FPRSearchForm() page = helpers.pager(results, results_per_page, current_page_number) request.session['fpr_query'] = '' item_count = len(results) return render(request, 'main/preservation_planning_fpr.html', locals())
def render_resource_component( client, request, resource_component_id, query, page, sort_by, search_params, match_redirect_target, resource_detail_template, reset_url, uuid, ): resource_component_data = client.get_resource_component_and_children( resource_component_id, "description", recurse_max_level=2, search_pattern=query, sort_by=sort_by, ) sort_direction = _determine_reverse_sort_direction(sort_by) if resource_component_data["children"]: page = helpers.pager(resource_component_data["children"], PAGE_SIZE, page) if not resource_component_data["children"] and query == "": return HttpResponseRedirect( reverse(match_redirect_target, args=[uuid, resource_component_id])) else: return render( request, resource_detail_template, { "match_redirect_target": match_redirect_target, "page": page, "query": query, "reset_url": reset_url, "resource_component_data": resource_component_data, "resource_component_id": resource_component_id, "search_params": search_params, "sort_by": sort_by, "sort_direction": sort_direction, "uuid": uuid, }, )
def ingest_normalization_report(request, uuid, current_page=None): jobs = models.Job.objects.filter(sipuuid=uuid, subjobof='') sipname = utils.get_directory_name_from_job(jobs) objects = getNormalizationReportQuery(sipUUID=uuid) for o in objects: o['location'] = archivematicaFunctions.escape(o['location']) results_per_page = 10 if current_page == None: current_page = 1 page = helpers.pager(objects, results_per_page, current_page) hit_count = len(objects) return render(request, 'ingest/normalization_report.html', locals())
def tasks(request, uuid): job = models.Job.objects.get(jobuuid=uuid) objects = job.task_set.all().order_by('-exitcode', '-endtime', '-starttime', '-createdtime') if (len(objects) == 0): return tasks_subjobs(request, uuid) page = helpers.pager(objects, django_settings.TASKS_PER_PAGE, request.GET.get('page', None)) objects = page['objects'] # figure out duration in seconds for object in objects: object.duration = helpers.task_duration_in_seconds(object) return render(request, 'main/tasks.html', locals())
def ingest_normalization_report(request, uuid, current_page=None): jobs = models.Job.objects.filter(sipuuid=uuid, subjobof='') sipname = jobs.get_directory_name() objects = getNormalizationReportQuery(sipUUID=uuid) for o in objects: o['location'] = archivematicaFunctions.escape(o['location']) (o['preservation_derivative_validation_attempted'], o['preservation_derivative_validation_failed'], o['access_derivative_validation_attempted'], o['access_derivative_validation_failed'] ) = derivative_validation_report(o) results_per_page = 10 if current_page is None: current_page = 1 page = helpers.pager(objects, results_per_page, current_page) hit_count = len(objects) return render(request, 'ingest/normalization_report.html', locals())
def ingest_normalization_report(request, uuid, current_page=None): jobs = models.Job.objects.filter(sipuuid=uuid) sipname = jobs.get_directory_name() objects = getNormalizationReportQuery(sipUUID=uuid) for o in objects: o["location"] = escape(o["location"]) ( o["preservation_derivative_validation_attempted"], o["preservation_derivative_validation_failed"], o["access_derivative_validation_attempted"], o["access_derivative_validation_failed"], ) = derivative_validation_report(o) results_per_page = 10 if current_page is None: current_page = 1 page = helpers.pager(objects, results_per_page, current_page) hit_count = len(objects) return render(request, "ingest/normalization_report.html", locals())
def ingest_upload_atk(request, uuid): try: query = request.GET.get('query', '').strip() db = ingest_upload_atk_db_connection() try: resources = ingest_upload_atk_get_collection_ids(db, query) except MySQLdb.OperationalError: return HttpResponseServerError( 'Database connection error. Please contact an administration.') page = helpers.pager(resources, 10, request.GET.get('page', 1)) page.objects = augment_resource_data(db, page.object_list) except (MySQLdb.ProgrammingError, MySQLdb.OperationalError) as e: return HttpResponseServerError( 'Database error {0}. Please contact an administrator.'.format( str(e))) search_params = advanced_search.extract_url_search_params_from_request( request) return render(request, 'ingest/atk/resource_list.html', locals())
def search(request): # deal with transfer mode file_mode = False checked_if_in_file_mode = '' if request.GET.get('mode', '') != '': file_mode = True checked_if_in_file_mode = 'checked' # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep( request) # redirect if no search params have been set if not 'query' in request.GET: return helpers.redirect_with_get_params( 'components.archival_storage.views.search', query='', field='', type='') # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request( request) # set paging variables if not file_mode: items_per_page = 2 else: items_per_page = 20 page = advanced_search.extract_page_number_from_url(request) start = page * items_per_page + 1 # perform search conn = pyes.ES(elasticSearchFunctions.getElasticsearchServerHostAndPort()) try: query = advanced_search.assemble_query(queries, ops, fields, types) # use all results to pull transfer facets if not in file mode # pulling only one field (we don't need field data as we augment # the results using separate queries) if not file_mode: results = conn.search_raw(query=query, indices='aips', type='aipfile', fields='uuid') else: results = conn.search_raw(query=query, indices='aips', type='aipfile', start=start - 1, size=items_per_page, fields='AIPUUID,filePath,FILEUUID') except: return HttpResponse('Error accessing index.') # take note of facet data aip_uuids = results['facets']['AIPUUID']['terms'] if not file_mode: number_of_results = len(aip_uuids) page_data = helpers.pager(aip_uuids, items_per_page, page + 1) aip_uuids = page_data['objects'] search_augment_aip_results(conn, aip_uuids) else: number_of_results = results.hits.total results = search_augment_file_results(results) # set remaining paging variables end, previous_page, next_page = advanced_search.paging_related_values_for_template_use( items_per_page, page, start, number_of_results) # make sure results is set try: if results: pass except: results = False form = forms.StorageSearchForm(initial={'query': queries[0]}) return render(request, 'archival_storage/archival_storage_search.html', locals())
def search(request): # deal with transfer mode file_mode = False checked_if_in_file_mode = '' if request.GET.get('mode', '') != '': file_mode = True checked_if_in_file_mode = 'checked' # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep(request) # redirect if no search params have been set if not 'query' in request.GET: return helpers.redirect_with_get_params( 'components.archival_storage.views.search', query='', field='', type='' ) # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request(request) # set paging variables if not file_mode: items_per_page = 2 else: items_per_page = 20 page = advanced_search.extract_page_number_from_url(request) start = page * items_per_page + 1 # perform search conn = pyes.ES(elasticSearchFunctions.getElasticsearchServerHostAndPort()) try: query=advanced_search.assemble_query(queries, ops, fields, types) # use all results to pull transfer facets if not in file mode # pulling only one field (we don't need field data as we augment # the results using separate queries) if not file_mode: results = conn.search_raw( query=query, indices='aips', type='aipfile', fields='uuid' ) else: results = conn.search_raw( query=query, indices='aips', type='aipfile', start=start - 1, size=items_per_page, fields='AIPUUID,filePath,FILEUUID' ) except: return HttpResponse('Error accessing index.') # take note of facet data aip_uuids = results['facets']['AIPUUID']['terms'] if not file_mode: number_of_results = len(aip_uuids) page_data = helpers.pager(aip_uuids, items_per_page, page + 1) aip_uuids = page_data['objects'] search_augment_aip_results(conn, aip_uuids) else: number_of_results = results.hits.total results = search_augment_file_results(results) # set remaining paging variables end, previous_page, next_page = advanced_search.paging_related_values_for_template_use( items_per_page, page, start, number_of_results ) # make sure results is set try: if results: pass except: results = False form = forms.StorageSearchForm(initial={'query': queries[0]}) return render(request, 'archival_storage/archival_storage_search.html', locals())
def list_display(request): if "aips" not in settings.SEARCH_ENABLED: return render(request, "archival_storage/list.html") current_page_number = int(request.GET.get("page", 1)) logger.debug("Current page: %s", current_page_number) # get count of AIP files es_client = elasticSearchFunctions.get_client() aip_indexed_file_count = aip_file_count(es_client) # get AIPs order_by = request.GET.get("order_by", "name") sort_by = request.GET.get("sort_by", "up") sort_params = "order_by=" + order_by + "&sort_by=" + sort_by # use raw subfield to sort by name if order_by == "name": order_by = order_by + ".raw" # change sort_by param to ES sort directions if sort_by == "down": sort_by = "desc" else: sort_by = "asc" sort_specification = order_by + ":" + sort_by # get list of UUIDs of AIPs that are deleted or pending deletion aips_deleted_or_pending_deletion = [] should_haves = [{ "match": { "status": "DEL_REQ" } }, { "match": { "status": "DELETED" } }] query = {"query": {"bool": {"should": should_haves}}} deleted_aip_results = es_client.search(body=query, index="aips", _source="uuid,status") for deleted_aip in deleted_aip_results["hits"]["hits"]: aips_deleted_or_pending_deletion.append(deleted_aip["_source"]["uuid"]) # Fetch results and paginate def es_pager(page, page_size): """ Fetch one page of normalized entries from Elasticsearch. :param page: 1-indexed page to fetch :param page_size: Number of entries on a page :return: List of dicts for each entry, where keys and values have been cleaned up """ start = (page - 1) * page_size results = es_client.search( index="aips", body={"query": { "match_all": {} }}, _source="origin,uuid,filePath,created,name,size,encrypted", sort=sort_specification, size=page_size, from_=start, ) return [d["_source"] for d in results["hits"]["hits"]] items_per_page = 10 count = es_client.count(index="aips", body={"query": { "match_all": {} }})["count"] results = LazyPagedSequence(es_pager, page_size=items_per_page, length=count) # Paginate page = helpers.pager(results, items_per_page, current_page_number) # process deletion, etc., and format results aips = [] for aip in page.object_list: # If an AIP was deleted or is pending deletion, react if status changed if aip["uuid"] in aips_deleted_or_pending_deletion: # check with storage server to see current status api_results = storage_service.get_file_info(uuid=aip["uuid"]) try: aip_status = api_results[0]["status"] except IndexError: # Storage service does not know about this AIP # TODO what should happen here? logger.info("AIP not found in storage service: {}".format(aip)) continue # delete AIP metadata in ElasticSearch if AIP has been deleted from the # storage server # TODO: handle this asynchronously if aip_status == "DELETED": elasticSearchFunctions.delete_aip(es_client, aip["uuid"]) elasticSearchFunctions.delete_aip_files(es_client, aip["uuid"]) elif aip_status != "DEL_REQ": # update the status in ElasticSearch for this AIP elasticSearchFunctions.mark_aip_stored(es_client, aip["uuid"]) else: aip_status = "UPLOADED" # Tweak AIP presentation and add to display array if aip_status != "DELETED": aip["status"] = AIP_STATUS_DESCRIPTIONS[aip_status] try: size = "{0:.2f} MB".format(float(aip["size"])) except (TypeError, ValueError): size = "Removed" aip["size"] = size aip["href"] = aip["filePath"].replace(AIPSTOREPATH + "/", "AIPsStore/") aip["date"] = aip["created"] aips.append(aip) total_size = total_size_of_aips(es_client) # Find out which AIPs are encrypted return render( request, "archival_storage/list.html", { "total_size": total_size, "aip_indexed_file_count": aip_indexed_file_count, "aips": aips, "page": page, "search_params": sort_params, }, )
def list_display(request): current_page_number = request.GET.get('page', 1) form = forms.StorageSearchForm() # get ElasticSearch stats aip_indexed_file_count = advanced_search.indexed_count('aips') # get AIPs order_by = request.GET.get('order_by', 'name') sort_by = request.GET.get('sort_by', 'up') if sort_by == 'down': sort_direction = 'desc' else: sort_direction = 'asc' sort_specification = order_by + ':' + sort_direction conn = elasticSearchFunctions.connect_and_create_index('aips') items_per_page = 10 start = (int(current_page_number) - 1) * items_per_page aipResults = conn.search(pyes.Search(pyes.MatchAllQuery(), start=start, size=items_per_page), doc_types=['aip'], fields='origin,uuid,filePath,created,name,size', sort=sort_specification) try: len(aipResults) except pyes.exceptions.ElasticSearchException: # there will be an error if no mapping exists for AIPs due to no AIPs # having been created return render(request, 'archival_storage/archival_storage.html', locals()) # handle pagination page = helpers.pager(aipResults, items_per_page, current_page_number) if not page: raise Http404 # augment data sips = [] for aip in page['objects']: sip = {} sip['href'] = aip.filePath.replace(AIPSTOREPATH + '/', "AIPsStore/") sip['name'] = aip.name sip['uuid'] = aip.uuid sip['date'] = aip.created try: size = float(aip.size) sip['size'] = '{0:.2f} MB'.format(size) except: sip['size'] = 'Removed' sips.append(sip) # get total size of all AIPS from ElasticSearch q = pyes.MatchAllQuery().search() q.facet.add(pyes.facets.StatisticalFacet('total', field='size')) aipResults = conn.search(q, doc_types=['aip']) total_size = aipResults.facets.total.total total_size = '{0:.2f}'.format(total_size) return render(request, 'archival_storage/archival_storage.html', locals())
def taxonomy(request): taxonomies = models.Taxonomy.objects.all().order_by('name') page = helpers.pager(taxonomies, 20, request.GET.get('page', 1)) return render(request, 'administration/taxonomy.html', locals())
def search(request): # FIXME there has to be a better way of handling checkboxes than parsing # them by hand here, and displaying 'checked' in # _archival_storage_search_form.html # Parse checkbox for file mode yes_options = ("checked", "yes", "true", "on") if request.GET.get("filemode", "") in yes_options: file_mode = True checked_if_in_file_mode = "checked" items_per_page = 20 else: # AIP list file_mode = False checked_if_in_file_mode = "" items_per_page = 10 # Parse checkbox for show AICs show_aics = "" if request.GET.get("show_aics", "") in yes_options: show_aics = "checked" # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep( request) logger.debug("Queries: %s, Ops: %s, Fields: %s, Types: %s", queries, ops, fields, types) # redirect if no search params have been set if "query" not in request.GET: return helpers.redirect_with_get_params( "components.archival_storage.views.search", query="", field="", type="") # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request( request) current_page_number = int(request.GET.get("page", 1)) # perform search es_client = elasticSearchFunctions.get_client() results = None query = advanced_search.assemble_query(queries, ops, fields, types) try: # Use all results to pull transfer facets if not in file mode # pulling only one field (we don't need field data as we augment # the results using separate queries). if not file_mode: # Fetch all unique AIP UUIDs in the returned set of files # ES query will limit to 10 aggregation results by default, # add size parameter in terms to override. # TODO: Use composite aggregation when it gets out of beta. query["aggs"] = { "aip_uuids": { "terms": { "field": "AIPUUID", "size": "10000" } } } # Don't return results, just the aggregation query["size"] = 0 # Searching for AIPs still actually searches type 'aipfile', and # returns the UUID of the AIP the files are a part of. To search # for an attribute of an AIP, the aipfile must index that # information about their AIP. results = es_client.search(body=query, index="aipfiles") # Given these AIP UUIDs, now fetch the actual information we want from aips/aip buckets = results["aggregations"]["aip_uuids"]["buckets"] uuids = [bucket["key"] for bucket in buckets] uuid_file_counts = { bucket["key"]: bucket["doc_count"] for bucket in buckets } query = {"query": {"terms": {"uuid": uuids}}} index = "aips" fields = ( "name,uuid,size,created,status,AICID,isPartOf,countAIPsinAIC,encrypted" ) sort = "name.raw:desc" else: index = "aipfiles" fields = "AIPUUID,filePath,FILEUUID,encrypted" sort = "sipName.raw:desc" # To reduce amount of data fetched from ES, use LazyPagedSequence def es_pager(page, page_size): """ Fetch one page of normalized aipfile entries from Elasticsearch. :param page: 1-indexed page to fetch :param page_size: Number of entries on a page :return: List of dicts for each entry with additional information """ start = (page - 1) * page_size results = es_client.search( body=query, from_=start, size=page_size, index=index, _source=fields, sort=sort, ) if file_mode: return search_augment_file_results(es_client, results) else: return search_augment_aip_results(results, uuid_file_counts) count = es_client.count(index=index, body={"query": query["query"]})["count"] results = LazyPagedSequence(es_pager, items_per_page, count) except ElasticsearchException: logger.exception("Error accessing index.") return HttpResponse("Error accessing index.") if not file_mode: aic_creation_form = forms.CreateAICForm(initial={"results": uuids}) else: # if file_mode aic_creation_form = None page_data = helpers.pager(results, items_per_page, current_page_number) return render( request, "archival_storage/search.html", { "file_mode": file_mode, "show_aics": show_aics, "checked_if_in_file_mode": checked_if_in_file_mode, "aic_creation_form": aic_creation_form, "results": page_data.object_list, "search_params": search_params, "page": page_data, }, )
def terms(request, taxonomy_uuid): taxonomy = models.Taxonomy.objects.get(pk=taxonomy_uuid) terms = taxonomy.taxonomyterm_set.order_by('term') page = helpers.pager(terms, 20, request.GET.get('page', 1)) return render(request, 'administration/terms.html', locals())
def transfer_backlog(request): # deal with transfer mode file_mode = False checked_if_in_file_mode = '' if request.GET.get('mode', '') != '': file_mode = True checked_if_in_file_mode = 'checked' # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep( request) # redirect if no search params have been set if not 'query' in request.GET: return helpers.redirect_with_get_params( 'components.ingest.views.transfer_backlog', query='', field='', type='') # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request( request) # set paging variables if not file_mode: items_per_page = 10 else: items_per_page = 20 page = advanced_search.extract_page_number_from_url(request) start = page * items_per_page + 1 # perform search conn = elasticSearchFunctions.connect_and_create_index('transfers') try: query = advanced_search.assemble_query( queries, ops, fields, types, must_haves=[pyes.TermQuery('status', 'backlog')]) # use all results to pull transfer facets if not in file mode if not file_mode: results = conn.search_raw( query, indices='transfers', type='transferfile', ) else: # otherwise use pages results results = conn.search_raw(query, indices='transfers', type='transferfile', start=start - 1, size=items_per_page) except: return HttpResponse('Error accessing index.') # take note of facet data file_extension_usage = results['facets']['fileExtension']['terms'] transfer_uuids = results['facets']['sipuuid']['terms'] if not file_mode: # run through transfers to see if they've been created yet awaiting_creation = {} for transfer_instance in transfer_uuids: try: awaiting_creation[transfer_instance. term] = transfer_awaiting_sip_creation_v2( transfer_instance.term) transfer = models.Transfer.objects.get( uuid=transfer_instance.term) transfer_basename = os.path.basename( transfer.currentlocation[:-1]) transfer_instance.name = transfer_basename[:-37] transfer_instance.type = transfer.type if transfer.accessionid != None: transfer_instance.accession = transfer.accessionid else: transfer_instance.accession = '' except: awaiting_creation[transfer_instance.term] = False # page data number_of_results = len(transfer_uuids) page_data = helpers.pager(transfer_uuids, items_per_page, page + 1) transfer_uuids = page_data['objects'] else: # page data number_of_results = results.hits.total results = transfer_backlog_augment_search_results(results) # set remaining paging variables end, previous_page, next_page = advanced_search.paging_related_values_for_template_use( items_per_page, page, start, number_of_results) # make sure results is set try: if results: pass except: results = False form = StorageSearchForm(initial={'query': queries[0]}) return render(request, 'ingest/backlog/search.html', locals())
def transfer_backlog(request): # deal with transfer mode file_mode = False checked_if_in_file_mode = '' if request.GET.get('mode', '') != '': file_mode = True checked_if_in_file_mode = 'checked' # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep(request) # redirect if no search params have been set if not 'query' in request.GET: return helpers.redirect_with_get_params( 'components.ingest.views.transfer_backlog', query='', field='', type='' ) # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request(request) # set paging variables if not file_mode: items_per_page = 10 else: items_per_page = 20 page = advanced_search.extract_page_number_from_url(request) start = page * items_per_page + 1 # perform search conn = elasticSearchFunctions.connect_and_create_index('transfers') try: query = advanced_search.assemble_query( queries, ops, fields, types, must_haves=[pyes.TermQuery('status', 'backlog')] ) # use all results to pull transfer facets if not in file mode if not file_mode: results = conn.search_raw( query, indices='transfers', type='transferfile', ) else: # otherwise use pages results results = conn.search_raw( query, indices='transfers', type='transferfile', start=start - 1, size=items_per_page ) except: return HttpResponse('Error accessing index.') # take note of facet data file_extension_usage = results['facets']['fileExtension']['terms'] transfer_uuids = results['facets']['sipuuid']['terms'] if not file_mode: # run through transfers to see if they've been created yet awaiting_creation = {} for transfer_instance in transfer_uuids: try: awaiting_creation[transfer_instance.term] = transfer_awaiting_sip_creation_v2(transfer_instance.term) transfer = models.Transfer.objects.get(uuid=transfer_instance.term) transfer_basename = os.path.basename(transfer.currentlocation[:-1]) transfer_instance.name = transfer_basename[:-37] transfer_instance.type = transfer.type if transfer.accessionid != None: transfer_instance.accession = transfer.accessionid else: transfer_instance.accession = '' except: awaiting_creation[transfer_instance.term] = False # page data number_of_results = len(transfer_uuids) page_data = helpers.pager(transfer_uuids, items_per_page, page + 1) transfer_uuids = page_data['objects'] else: # page data number_of_results = results.hits.total results = transfer_backlog_augment_search_results(results) # set remaining paging variables end, previous_page, next_page = advanced_search.paging_related_values_for_template_use( items_per_page, page, start, number_of_results ) # make sure results is set try: if results: pass except: results = False form = StorageSearchForm(initial={'query': queries[0]}) return render(request, 'ingest/backlog/search.html', locals())
def list_display(request): if 'aips' not in settings.SEARCH_ENABLED: return render(request, 'archival_storage/list.html') current_page_number = int(request.GET.get('page', 1)) logger.debug('Current page: %s', current_page_number) # get count of AIP files es_client = elasticSearchFunctions.get_client() aip_indexed_file_count = aip_file_count(es_client) # get AIPs order_by = request.GET.get('order_by', 'name_unanalyzed') sort_by = request.GET.get('sort_by', 'up') if sort_by == 'down': sort_direction = 'desc' else: sort_direction = 'asc' sort_specification = order_by + ':' + sort_direction sort_params = 'order_by=' + order_by + '&sort_by=' + sort_by # get list of UUIDs of AIPs that are deleted or pending deletion aips_deleted_or_pending_deletion = [] should_haves = [ {'match': {'status': 'DEL_REQ'}}, {'match': {'status': 'DELETED'}}, ] query = { "query": { "bool": { "should": should_haves } } } deleted_aip_results = es_client.search( body=query, index='aips', doc_type='aip', fields='uuid,status' ) for deleted_aip in deleted_aip_results['hits']['hits']: aips_deleted_or_pending_deletion.append(deleted_aip['fields']['uuid'][0]) # Fetch results and paginate def es_pager(page, page_size): """ Fetch one page of normalized entries from Elasticsearch. :param page: 1-indexed page to fetch :param page_size: Number of entries on a page :return: List of dicts for each entry, where keys and values have been cleaned up """ start = (page - 1) * page_size results = es_client.search( index='aips', doc_type='aip', body=elasticSearchFunctions.MATCH_ALL_QUERY, fields='origin,uuid,filePath,created,name,size,encrypted', sort=sort_specification, size=page_size, from_=start, ) # normalize results - each of the fields contains a single value, # but is returned from the ES API as a single-length array # e.g. {"fields": {"uuid": ["abcd"], "name": ["aip"] ...}} return [elasticSearchFunctions.normalize_results_dict(d) for d in results['hits']['hits']] items_per_page = 10 count = es_client.count(index='aips', doc_type='aip', body=elasticSearchFunctions.MATCH_ALL_QUERY)['count'] results = LazyPagedSequence(es_pager, page_size=items_per_page, length=count) # Paginate page = helpers.pager( results, items_per_page, current_page_number ) # process deletion, etc., and format results aips = [] for aip in page.object_list: # If an AIP was deleted or is pending deletion, react if status changed if aip['uuid'] in aips_deleted_or_pending_deletion: # check with storage server to see current status api_results = storage_service.get_file_info(uuid=aip['uuid']) try: aip_status = api_results[0]['status'] except IndexError: # Storage service does not know about this AIP # TODO what should happen here? logger.info("AIP not found in storage service: {}".format(aip)) continue # delete AIP metadata in ElasticSearch if AIP has been deleted from the # storage server # TODO: handle this asynchronously if aip_status == 'DELETED': elasticSearchFunctions.delete_aip(es_client, aip['uuid']) elasticSearchFunctions.delete_aip_files(es_client, aip['uuid']) elif aip_status != 'DEL_REQ': # update the status in ElasticSearch for this AIP elasticSearchFunctions.mark_aip_stored(es_client, aip['uuid']) else: aip_status = 'UPLOADED' # Tweak AIP presentation and add to display array if aip_status != 'DELETED': aip['status'] = AIP_STATUS_DESCRIPTIONS[aip_status] try: size = '{0:.2f} MB'.format(float(aip['size'])) except (TypeError, ValueError): size = 'Removed' aip['size'] = size aip['href'] = aip['filePath'].replace(AIPSTOREPATH + '/', "AIPsStore/") aip['date'] = aip['created'] aips.append(aip) total_size = total_size_of_aips(es_client) # Find out which AIPs are encrypted return render(request, 'archival_storage/list.html', { 'total_size': total_size, 'aip_indexed_file_count': aip_indexed_file_count, 'aips': aips, 'page': page, 'search_params': sort_params, } )
def search(request): # FIXME there has to be a better way of handling checkboxes than parsing # them by hand here, and displaying 'checked' in # _archival_storage_search_form.html # Parse checkbox for file mode yes_options = ('checked', 'yes', 'true', 'on') if request.GET.get('filemode', '') in yes_options: file_mode = True checked_if_in_file_mode = 'checked' items_per_page = 20 else: # AIP list file_mode = False checked_if_in_file_mode = '' items_per_page = 10 # Parse checkbox for show AICs show_aics = '' if request.GET.get('show_aics', '') in yes_options: show_aics = 'checked' # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep(request) logger.debug('Queries: %s, Ops: %s, Fields: %s, Types: %s', queries, ops, fields, types) # redirect if no search params have been set if 'query' not in request.GET: return helpers.redirect_with_get_params( 'components.archival_storage.views.search', query='', field='', type='' ) # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request(request) current_page_number = int(request.GET.get('page', 1)) # perform search es_client = elasticSearchFunctions.get_client() results = None query = advanced_search.assemble_query(es_client, queries, ops, fields, types, search_index='aips', doc_type='aipfile') try: # use all results to pull transfer facets if not in file mode # pulling only one field (we don't need field data as we augment # the results using separate queries) if not file_mode: # Fetch all unique AIP UUIDs in the returned set of files query['aggs'] = {'aip_uuids': {'terms': {'field': 'AIPUUID', 'size': 0}}} # Don't return results, just the aggregation query['size'] = 0 # Searching for AIPs still actually searches type 'aipfile', and # returns the UUID of the AIP the files are a part of. To search # for an attribute of an AIP, the aipfile must index that # information about their AIP in # elasticSearchFunctions.index_mets_file_metadata results = es_client.search( body=query, index='aips', doc_type='aipfile', sort='sipName:desc', ) # Given these AIP UUIDs, now fetch the actual information we want from aips/aip buckets = results['aggregations']['aip_uuids']['buckets'] uuids = [bucket['key'] for bucket in buckets] uuid_file_counts = {bucket['key']: bucket['doc_count'] for bucket in buckets} query = { 'query': { 'terms': { 'uuid': uuids, }, }, } index = 'aips' doc_type = 'aip' fields = 'name,uuid,size,created,status,AICID,isPartOf,countAIPsinAIC,encrypted' sort = 'name:desc' else: index = 'aips' doc_type = 'aipfile' fields = 'AIPUUID,filePath,FILEUUID,encrypted' sort = 'sipName:desc' # To reduce amount of data fetched from ES, use LazyPagedSequence def es_pager(page, page_size): """ Fetch one page of normalized aipfile entries from Elasticsearch. :param page: 1-indexed page to fetch :param page_size: Number of entries on a page :return: List of dicts for each entry with additional information """ start = (page - 1) * page_size results = es_client.search( body=query, from_=start, size=page_size, index=index, doc_type=doc_type, fields=fields, sort=sort, ) if file_mode: return search_augment_file_results(es_client, results) else: return search_augment_aip_results(results, uuid_file_counts) count = es_client.count(index=index, doc_type=doc_type, body={'query': query['query']})['count'] results = LazyPagedSequence(es_pager, items_per_page, count) except ElasticsearchException: logger.exception('Error accessing index.') return HttpResponse('Error accessing index.') if not file_mode: aic_creation_form = forms.CreateAICForm(initial={'results': uuids}) else: # if file_mode aic_creation_form = None page_data = helpers.pager(results, items_per_page, current_page_number) return render(request, 'archival_storage/search.html', { 'file_mode': file_mode, 'show_aics': show_aics, 'checked_if_in_file_mode': checked_if_in_file_mode, 'aic_creation_form': aic_creation_form, 'results': page_data.object_list, 'search_params': search_params, 'page': page_data, } )
def list_display(request): current_page_number = request.GET.get('page', 1) form = forms.StorageSearchForm() # get ElasticSearch stats aip_indexed_file_count = advanced_search.indexed_count('aips') # get AIPs order_by = request.GET.get('order_by', 'name') sort_by = request.GET.get('sort_by', 'up') if sort_by == 'down': sort_direction = 'desc' else: sort_direction = 'asc' sort_specification = order_by + ':' + sort_direction conn = elasticSearchFunctions.connect_and_create_index('aips') items_per_page = 10 start = (int(current_page_number) - 1) * items_per_page aipResults = conn.search( pyes.Search(pyes.MatchAllQuery(), start=start, size=items_per_page), doc_types=['aip'], fields='origin,uuid,filePath,created,name,size', sort=sort_specification ) try: len(aipResults) except pyes.exceptions.ElasticSearchException: # there will be an error if no mapping exists for AIPs due to no AIPs # having been created return render(request, 'archival_storage/archival_storage.html', locals()) # handle pagination page = helpers.pager( aipResults, items_per_page, current_page_number ) if not page: raise Http404 # augment data sips = [] for aip in page['objects']: sip = {} sip['href'] = aip.filePath.replace(AIPSTOREPATH + '/', "AIPsStore/") sip['name'] = aip.name sip['uuid'] = aip.uuid sip['date'] = aip.created try: size = float(aip.size) sip['size'] = '{0:.2f} MB'.format(size) except: sip['size'] = 'Removed' sips.append(sip) # get total size of all AIPS from ElasticSearch q = pyes.MatchAllQuery().search() q.facet.add(pyes.facets.StatisticalFacet('total', field='size')) aipResults = conn.search(q, doc_types=['aip']) total_size = aipResults.facets.total.total total_size = '{0:.2f}'.format(total_size) return render(request, 'archival_storage/archival_storage.html', locals())
def taxonomy(request): taxonomies = models.Taxonomy.objects.all().order_by("name") page = helpers.pager(taxonomies, 20, request.GET.get("page", 1)) return render(request, "administration/taxonomy.html", locals())