def test_pages_to_show(self):
paginator = Paginator(range(300), 10)
# range of pages at the beginning
pages = pages_to_show(paginator, 1)
self.assertEqual(7, len(pages),
"show pages returns 7 items for first page")
self.assert_(1 in pages, "show pages includes 1 for first page")
self.assert_(6 in pages, "show pages includes 6 for first page")
# default labels
self.assertEqual('1', pages[1])
self.assertEqual('6', pages[6])
# custom labels
pages = pages_to_show(paginator, 1, {1: 'one', 2: 'two'})
self.assertEqual('one', pages[1])
self.assertEqual('two', pages[2])
self.assertEqual('6', pages[6]) # default because not specified
pages = pages_to_show(paginator, 2)
self.assert_(1 in pages, "show pages for page 2 includes 1")
self.assert_(2 in pages, "show pages for page 2 includes 2")
self.assert_(3 in pages, "show pages for page 2 includes 3")
# range of pages in the middle
pages = pages_to_show(paginator, 15)
self.assertEqual(
7, len(pages),
"show pages returns 7 items for middle of page result")
self.assert_(
15 in pages,
"show pages includes current page for middle of page result")
self.assert_(
12 in pages,
"show pages includes third page before current page for middle of page result"
)
self.assert_(
18 in pages,
"show pages includes third page after current page for middle of page result"
)
# range of pages at the end
pages = pages_to_show(paginator, 30)
self.assertEqual(7, len(pages),
"show pages returns 7 items for last page")
self.assert_(30 in pages,
"show pages includes last page for last page of results")
self.assert_(
24 in pages,
"show pages includes 6 pages before last page for last page of results"
)
def test_pages_to_show(self):
paginator = Paginator(range(300), 10)
# range of pages at the beginning
pages = pages_to_show(paginator, 1)
self.assertEqual(7, len(pages), "show pages returns 7 items for first page")
self.assert_(1 in pages, "show pages includes 1 for first page")
self.assert_(6 in pages, "show pages includes 6 for first page")
# default labels
self.assertEqual('1', pages[1])
self.assertEqual('6', pages[6])
# custom labels
pages = pages_to_show(paginator, 1, {1: 'one', 2: 'two'})
self.assertEqual('one', pages[1])
self.assertEqual('two', pages[2])
self.assertEqual('6', pages[6]) # default because not specified
pages = pages_to_show(paginator, 2)
self.assert_(1 in pages, "show pages for page 2 includes 1")
self.assert_(2 in pages, "show pages for page 2 includes 2")
self.assert_(3 in pages, "show pages for page 2 includes 3")
# range of pages in the middle
pages = pages_to_show(paginator, 15)
self.assertEqual(7, len(pages),
"show pages returns 7 items for middle of page result")
self.assert_(15 in pages,
"show pages includes current page for middle of page result")
self.assert_(12 in pages,
"show pages includes third page before current page for middle of page result")
self.assert_(18 in pages,
"show pages includes third page after current page for middle of page result")
# range of pages at the end
pages = pages_to_show(paginator, 30)
self.assertEqual(7, len(pages),
"show pages returns 7 items for last page")
self.assert_(30 in pages,
"show pages includes last page for last page of results")
self.assert_(24 in pages,
"show pages includes 6 pages before last page for last page of results")
def view_item(request, pid):
'''
Display information about a single object. Currently
only supports :class:`eulcm.models.boda.EmailMessage`
and :class:`eulcm.models.boda.Mailbox` objects.
:param pid: The pid of the object to be displayed.
'''
repo = TypeInferringRepository(request=request)
obj = repo.get_object(pid)
context = {'obj': obj}
if isinstance(obj, boda.EmailMessage):
template_name = 'arrangement/email_view.html'
elif isinstance(obj, boda.Mailbox):
template_name = 'arrangement/mailbox_view.html'
# use Solr to find paginated messages in this mailbox
solr = solr_interface()
q = solr.query(isPartOf=obj.uri)
paginator = Paginator(q, 30)
try:
page = int(request.GET.get('page', '1'))
except ValueError:
page = 1
try:
results = paginator.page(page)
except (EmptyPage, InvalidPage):
results = paginator.page(paginator.num_pages)
# calculate page links to show
show_pages = pages_to_show(paginator, page)
# add paginated messages to context
context.update({
'page': results,
'show_pages': show_pages,
'search_opts': request.GET.urlencode()
})
else:
raise Http404
return TemplateResponse(request, template_name, context)
def view_item(request, pid):
'''
Display information about a single object. Currently
only supports :class:`eulcm.models.boda.EmailMessage`
and :class:`eulcm.models.boda.Mailbox` objects.
:param pid: The pid of the object to be displayed.
'''
repo = TypeInferringRepository(request=request)
obj = repo.get_object(pid)
context = {'obj': obj}
if isinstance(obj, boda.EmailMessage):
template_name = 'arrangement/email_view.html'
elif isinstance(obj, boda.Mailbox):
template_name = 'arrangement/mailbox_view.html'
# use Solr to find paginated messages in this mailbox
solr = solr_interface()
q = solr.query(isPartOf=obj.uri)
paginator = Paginator(q, 30)
try:
page = int(request.GET.get('page', '1'))
except ValueError:
page = 1
try:
results = paginator.page(page)
except (EmptyPage, InvalidPage):
results = paginator.page(paginator.num_pages)
# calculate page links to show
show_pages = pages_to_show(paginator, page)
# add paginated messages to context
context.update({
'page': results,
'show_pages': show_pages,
'search_opts': request.GET.urlencode()
})
else:
raise Http404
return TemplateResponse(request, template_name, context)
def paginate(request, query):
'''Common pagination logic, straight out of django docs. Takes a
:class:`~django.http.HttpRequest` and a result set that can be
paginated; returns a tuple of the current
:class:`django.core.paginator.Page` (based on the request) and the
page numbers that should be displayed (generated by
:meth:`eulcommon.searchutil.pages_to_show`).
'''
paginator = Paginator(query, 10)
# get current page number
try:
page = int(request.GET.get('page', '1'))
except ValueError:
page = 1
# return last page if an invalid page is requested
try:
results = paginator.page(page)
except (EmptyPage, InvalidPage):
results = paginator.page(paginator.num_pages)
# calculate page links to be shown
show_pages = pages_to_show(paginator, page)
return results, show_pages
def paginate(request, query):
'''Common pagination logic, straight out of django docs. Takes a
:class:`~django.http.HttpRequest` and a result set that can be
paginated; returns a tuple of the current
:class:`django.core.paginator.Page` (based on the request) and the
page numbers that should be displayed (generated by
:meth:`eulcommon.searchutil.pages_to_show`).
'''
paginator = Paginator(query, 10)
# get current page number
try:
page = int(request.GET.get('page', '1'))
except ValueError:
page = 1
# return last page if an invalid page is requested
try:
results = paginator.page(page)
except (EmptyPage, InvalidPage):
results = paginator.page(paginator.num_pages)
# calculate page links to be shown
show_pages = pages_to_show(paginator, page)
return results, show_pages
def search(request):
'''Search for :class:`~keep.audio.models.AudioObject` or
:class:`~keep.arrangement.models.ArrangementObject`by pid, title,
description, collection, date, rights, etc.'''
# if NO search terms are specified, return an advanced search page
if not request.GET:
return TemplateResponse(request, 'common/advanced-search.html',
{'searchform': commonforms.ItemSearch(prefix='audio')})
form = commonforms.ItemSearch(request.GET, prefix='audio')
ctx_dict = {'searchform': form}
if form.is_valid():
solr = solr_interface()
# solr search options from posted data
search_opts = form.search_options()
# search term/value display info for user based on posted data
ctx_dict['search_info'] = form.search_info()
# solr query to restrict this search to appropriate content models
cm_query = solr.Q(solr.Q(content_model=ArrangementObject.ARRANGEMENT_CONTENT_MODEL) \
| solr.Q(content_model=AudioObject.AUDIO_CONTENT_MODEL)\
| solr.Q(content_model=Video.VIDEO_CONTENT_MODEL))
# for now, sort by most recently created
solrquery = solr.query(**search_opts).filter(cm_query).sort_by('-created')
# if user requested specific display fields, handle output display and formatting
if form.cleaned_data['display_fields']:
fields = form.cleaned_data['display_fields']
# pid and content model are always needed to construct html search results
solr_fields = fields + ['pid', 'content_model']
solrquery = solrquery.field_limit(solr_fields)
class FieldList(list):
# extended list object with pid and content model attributes
def __init__(self, pid=None, content_model=None, values=[]):
super(FieldList, self).__init__(values)
if pid:
self.pid = pid
if content_model:
self.content_model = content_model
else:
self.content_model = []
def field_list(**kwargs):
# method to construct a custom solr result based on the requested field list
l = FieldList(pid=kwargs.get('pid', None),
content_model=kwargs.get('content_model', None))
for f in fields:
val = kwargs.get(f, '')
if solr.schema.fields[f].multi_valued:
val = '; '.join(val)
l.append(val)
return l
solrquery = solrquery.results_as(field_list)
ctx_dict.update({
'display_fields': fields,
'display_labels': [commonforms.ItemSearch.display_field_opts[f] for f in fields]
})
# if CSV is requested with display_fields, return as csv before paginating
if form.cleaned_data['output'] == 'csv':
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=Keep-report_%s.csv' \
% date.today()
writer = unicodecsv.writer(response)
# write out list of field labels
writer.writerow(ctx_dict['display_labels'])
# then append all matching values
# FIXME: csv output for very large results is VERY slow
# TODO: append rows in chunks of 50-100, to handle
# large result sets better - maybe use paginator?
writer.writerows(solrquery)
return response
paginator = Paginator(solrquery, 30)
try:
page = int(request.GET.get('page', '1'))
except ValueError:
page = 1
try:
results = paginator.page(page)
except (EmptyPage, InvalidPage):
results = paginator.page(paginator.num_pages)
# calculate page links to show
show_pages = pages_to_show(paginator, page)
ctx_dict.update({
'results': results.object_list,
'page': results,
'show_pages': show_pages,
# pass search term query opts to view for pagination links
'search_opts': request.GET.urlencode(),
})
return TemplateResponse(request, 'common/search.html', ctx_dict)
def keyword_search(request):
'''Combined keyword search across all :mod:`keep` repository
items.
'''
searchform = KeywordSearch(request.GET)
missing_label = '[null]'
ctx = {'form': searchform}
if searchform.is_valid():
search_terms = searchform.cleaned_data['keyword']
solr = solr_interface()
# start with a default query to add filters & search terms
# *first* filter to restrict to content models user has permission to view
# q = filter_by_perms(solr.query(), request.user)
q = solr.query()
# optional date filter for fixity check
fixity_check_mindate = searchform.cleaned_data.get('fixity_check_mindate', None)
if fixity_check_mindate:
today = date.today()
q = q.query(last_fixity_check__range=(fixity_check_mindate, today))
# use solr grouping queries to cluster original and migrated objects
# if they appear in the same search result set
q = q.group_by('original_pid', limit=5, sort='created desc', format='simple')
# separate out normal and fielded search terms in keyword search string
# TODO: should this logic be shifted to form validation/cleaning?
search_info = MultiValueDict()
terms = []
# add field-based search terms to query and search info for display
for t in search_terms:
field, val = t
# add non-field terms to list of terms
# - no field name
if field is None:
terms.append(val)
# - unrecognized field name or incomplete term
elif val is None or field not in searchform.allowed_fields:
# just search on the text we were given
if val is None:
term = '%s:' % field
else:
if ' ' in val: # assume exact phrase if quoted
val = "%s" % val
term = '%s:%s' % (field, val)
terms.append(term)
# field/value pair
else:
solr_field = searchform.allowed_fields[field]
search_val = val
# special case for searching for collection source id
if field == 'coll' and search_val and search_val.isdigit():
solr_field = 'collection_source_id'
# add wildcard to end of search dates
# (indexed by YYYY-MM-DD; allow match on YYYY or YYYY-MM)
if field == 'created':
search_val += '*'
# add field/value search to the solr query
q = q.query(**{solr_field: search_val})
# add to search info for display to user
field = 'collection' if field == 'coll' else field
search_info.update({field: val})
# search on all collected search terms
q = q.query(*terms)
# FIXME: there should be a way to exclude these by type
# Exclude archival collection (Top-level library)
for p in settings.PID_ALIASES.values():
q = q.exclude(pid=p)
# get a copy of current url options for pagination
# and to generate links to remove active filters
urlopts = request.GET.copy()
# handle facets
display_filters = []
# - list of tuples: display name, link to remove the filter
active_filters = dict((field, []) for field in
searchform.facet_field_names.iterkeys())
# - dictionary of filters in use, for exclusion from displayed
# facets
# filter the solr search based on any facets in the request
for filter_val, facet_field in searchform.facet_field_names.iteritems():
# For multi-valued fields (author, subject), we could have multiple
# filters on the same field; treat all facet fields as lists.
for val in request.GET.getlist(filter_val):
# ignore any facet if the value is not set
if not val:
continue
# special case: search for items without a field
if val == missing_label:
q = q.exclude(**{'%s__any' % facet_field: True})
else:
# filter the current solr query
q = q.filter(**{facet_field: val})
# add to list of active filters
active_filters[filter_val].append(val)
# add to list for user display & removal
# - copy the urlopts and remove only the current value
unfacet_urlopts = urlopts.copy()
val_list = unfacet_urlopts.getlist(filter_val)
val_list.remove(val)
unfacet_urlopts.setlist(filter_val, val_list)
# tuple of filter display value, url to remove it
# - add details to label when the value doesn't make it obvious
if filter_val in ['added by', 'modified by']:
label = '%s %s' % (filter_val, val)
elif filter_val == 'fixity_check':
label = 'fixity check: %s' % 'valid' if val == 'pass' else 'invalid'
elif val == missing_label:
label = '%s: null' % filter_val
elif filter_val == 'access status':
# use access status abbreviation instead of numeric code
label = rights_access_terms_dict[val].abbreviation
else:
label = val
display_filters.append((label,
unfacet_urlopts.urlencode()))
# Update solr query to return values & counts for the
# configured facet fields
q = q.facet_by(searchform.facet_field_names.values(),
mincount=1, limit=15, sort='count',
missing=True)
# NOTE: missing true displays count for items without any value
# for the facet field (e.g., no access code set)
# if there are any *keyword* terms, sort by relevance and display score
# (for fielded search terms, items will either match or not, so relevance
# is not as useful)
if terms:
# NOTE: possibly a change in sunburnt?
# including score now requires specifying *all* fields that
# should be returned
q = q.sort_by('-score').field_limit([
# common item information
"object_type", "content_model", "pid", "label", "title",
"creator", "created", "last_modified", "added_by",
# collection
"archive_short_name", "hasMember",
# item
"collection_id",
# audio
"part", "collection_label", "duration", "has_access_copy",
"access_copy_mimetype", "access_copy_size", "source_id",
# arrangement/disk image
"simpleCollection_label", "rights", "state",
# migrated / original
"original_pid", "isDerivationOf", "hasDerivation",
# format and size, used for disk images display (at least)
"content_size", "content_format"
],
score=True)
ctx['show_relevance'] = True
# then sort by most recently created
# (primary sort when no search terms, secondary otherwise)
q = q.sort_by('-created')
# list of currently known types for display in results
# FIXME: are these used anywhere?
known_object_types = ['audio', 'collection', 'born-digital']
# paginate the solr result set
paginator = Paginator(q, 30)
try:
page = int(request.GET.get('page', '1'))
except ValueError:
page = 1
try:
results = paginator.page(page)
except (EmptyPage, InvalidPage):
results = paginator.page(paginator.num_pages)
# calculate page links to show
show_pages = pages_to_show(paginator, page)
# convert the facets from the solr result for display to user
facets = SortedDict()
facet_fields = results.object_list.facet_counts.facet_fields
for display_name, field in searchform.facet_field_names.iteritems():
#do not display coll facet because it is redundant with the collection facet
if display_name in ['coll', 'fixity_check']:
continue
if field in facet_fields and facet_fields[field]:
show_facets = []
# skip any display facet values that are already in effect
for val in facet_fields[field]:
try:
if val[0] not in active_filters[display_name]:
show_facets.append(val)
except TypeError:
# when solr missing=True is turned on,
# last result is a count of items with no value
# for this field
if val is not 0 and field in searchform.show_missing_facets \
and missing_label not in active_filters[display_name]:
show_facets.append((missing_label, val))
if show_facets:
facets[display_name] = show_facets
ctx.update({
'page': results,
'show_pages': show_pages,
# 'known_types': known_object_types,
'search_opts': request.GET.urlencode(),
'search_terms': terms,
'search_info': search_info,
'url_params': urlopts.urlencode(),
'facets': facets,
'active_filters': display_filters,
})
return TemplateResponse(request, 'repoadmin/results.html', ctx)
