def filter_queryset(self, request, queryset, view):
query = BASE_QUERY.copy()
filters = []
params = request.QUERY_PARAMS
if hasattr(request, 'project') or 'project' in params:
project = params.get('project', request.project.name)
filters.append({ 'term': { 'serialized.project.name': project }})
if 'updater' in params:
filters.append({ 'term': { 'serialized.updater.username':
params.get('updater') }})
if 'q' in params:
q = params.get('q')
query['query']['filtered']['query'] = {
'match': {
'display_title': {
'query': q,
'operator': 'and',
'fuzziness': '0.3'
}
}
}
query['highlight'] = {
'fields': {'_all': {}},
'pre_tags': ['<strong>'],
'post_tags': ['</strong>']
}
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
return en_index.search_model(view.model, query)
def get_item_types():
url = ZOTERO_BASE_URL + '/itemTypes'
if not cache.get('item_types'):
resp = urlopen(url)
data = resp.read()
resp.close()
cache.set('item_types', data, 60 * 24 * 7)
data = json.loads(cache.get('item_types'))
from editorsnotes.main.models import Document
from editorsnotes.search import en_index
facet_search = en_index.search_model(Document, {
'query': {'filtered': {'query': {'match_all': {}}}},
'facets': {
'itemTypes': {
'terms': {
'field': 'serialized.zotero_data.itemType',
'size': 10
}
}
}
})
used_item_types = facet_search['facets']['itemTypes']['terms']
return {
'itemTypes': data,
'common': [item_type['term'] for item_type in used_item_types[:10]]
}
def topic(request, project_slug, topic_node_id):
o = {}
topic_qs = Topic.objects.select_related('topic_node', 'creator',
'last_updater', 'project')\
.prefetch_related('related_topics__topic')
o['topic'] = topic = get_object_or_404(topic_qs,
topic_node_id=topic_node_id,
project__slug=project_slug)
o['project'] = topic.project
o['breadcrumb'] = (
(topic.project.name, topic.project.get_absolute_url()),
('Topics', reverse('all_topics_view',
kwargs={'project_slug': topic.project.slug})),
(topic.preferred_name, None)
)
topic_query = {'query': {'term': {'serialized.related_topics.id': topic.id }}}
topic_query['size'] = 1000
model_searches = ( en_index.search_model(model, topic_query) for model in
(Document, Note, Topic) )
documents, notes, topics = (
[ result['_source']['serialized'] for result in search['hits']['hits'] ]
for search in model_searches)
o['documents'] = Document.objects.filter(id__in=[d['id'] for d in documents])
o['related_topics'] = Topic.objects.filter(id__in=[t['id'] for t in topics])
note_objects = Note.objects.in_bulk([n['id'] for n in notes])
for note in notes:
related_topics = list((topic for topic in note['related_topics']
if topic['url'] != request.path))
note_objects[note['id']] = (note_objects[note['id']], related_topics,)
o['notes'] = note_objects.values()
return render_to_response(
'topic.html', o, context_instance=RequestContext(request))
def all_notes(request, project_slug=None):
o = {}
template = 'all-notes.html'
if project_slug is not None:
project = get_object_or_404(Project, slug=project_slug)
o['project'] = project
o['breadcrumb'] = (
(project.name, project.get_absolute_url()),
('Notes', None),
)
else:
o['breadcrumb'] = (
('Browse', reverse('browse_view')),
('Notes', None),
)
o['filtered'] = False
if request.GET.get('filter'):
template = 'filtered-notes.html'
o['filtered'] = True
# Base query & facets
query = {
'query': { 'filtered': { 'query': { 'match_all': {} } } },
'facets': {
'topic_facet': {
'terms': { 'field': 'serialized.related_topics.name', 'size': 16 }
},
'status_facet': {
'terms': { 'field': 'serialized.status' }
}
},
'size': 1000
}
if project_slug is None:
query['facets']['project_facet'] = {
'terms': { 'field': 'serialized.project.name', 'size': 8 }
}
# Filter query based on facets
filters = []
for topic in request.GET.getlist('topic'):
filters.append({ 'term': { 'serialized.related_topics.name': topic } })
project_filter = project.name if project_slug is not None \
else request.GET.get('project')
if project_filter:
filters.append({ 'term': { 'serialized.project.name': project_filter } })
status = request.GET.get('note_status')
if status:
filters.append({ 'term': { 'serialized.status': status } })
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
# Execute built query
executed_query = en_index.search_model(Note, query)
# This is gibberish that can be improved, but it lets us use the old xapian
# code that I implemented a long time ago.
topic_facets = executed_query['facets']['topic_facet']['terms']
o['topic_facets_1'] = topic_facets[:8]
o['topic_facets_2'] = topic_facets[8:] if len(topic_facets) > 8 else []
if project_slug is None:
project_facets = executed_query['facets']['project_facet']['terms']
o['project_facets'] = project_facets
status_facets = executed_query['facets']['status_facet']['terms']
o['status_facets'] = status_facets
o['notes'] = [n['_source'] for n in executed_query['hits']['hits']]
return render_to_response(
template, o, context_instance=RequestContext(request))
def filter_queryset(self, request, queryset, view):
query = BASE_QUERY.copy()
filters = []
params = request.QUERY_PARAMS
if hasattr(request, 'project') or 'project' in params:
project = params.get('project', request.project.slug)
filters.append({'term': {'serialized.project.url':
'/projects/{}/'.format(project)}})
term = clean_query_string(params.get('autocomplete').lower()).split()
query['query']['filtered']['query'] = {
'bool': { 'must': [], 'should': [] }
}
must = query['query']['filtered']['query']['bool']['must']
should = query['query']['filtered']['query']['bool']['should']
if len(term) > 1:
must.append({
'match': {
'display_title': {
'query': ' '.join(term[:-1]),
'operator': 'and',
'fuzziness': '0.2'
}
}
})
must.append({
'prefix': {
'display_title': {
'value': term[-1]
}
}
})
should.append({
'match_phrase': {
'display_title': params.get('autocomplete').lower()
}
})
should.append({
'match': {
'display_title': {
'query': ' '.join(term),
'operator': 'and'
}
}
})
query['fields'] = ['display_title', 'display_url']
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
if params.get('highlight'):
query['highlight'] = {
'fields': {'display_title': {'number_of_fragments': 0}},
'pre_tags': ['<strong>'],
'post_tags': ['</strong>']
}
if hasattr(view, 'model') and view.model is not None:
return en_index.search_model(view.model, query)
else:
# Should boost notes most of all
return en_index.search(query)
def all_documents(request, project_slug=None):
o = {}
template = 'all-documents.html'
o['filtered'] = False
if project_slug is not None:
project = get_object_or_404(Project, slug=project_slug)
o['project'] = project
o['breadcrumb'] = (
(project.name, project.get_absolute_url()),
('Documents', None),
)
else:
o['breadcrumb'] = (
('Browse', reverse('browse_view')),
('Documents', None),
)
if request.GET.get('filter'):
template = 'filtered-documents.html'
o['filtered'] = True
facet_fields = [
('topic', 'serialized.related_topics.name'),
('archive', 'serialized.zotero_data.archive'),
('publicationTitle', 'serialized.zotero_data.publicationTitle'),
('itemType', 'serialized.zotero_data.itemType'),
('creators', 'serialized.zotero_data.creators'),
('representations', 'serialized.representations'),
]
if project_slug is None:
facet_fields.insert(0, ('project', 'serialized.project.name'))
facet_dict = dict(facet_fields)
query = {
'query': { 'filtered': { 'query': { 'match_all': {} } } },
'facets': {},
'size': 1000
}
for label, field in facet_fields:
query['facets'][label] = { 'terms': { 'field': field, 'size': 30 } }
filters = []
params = [ p for p in request.GET.keys() if p.endswith('[]') ]
for field in params:
facet_field = facet_dict.get(field[:-2], None)
if facet_field is None:
continue
for val in request.GET.getlist(field):
filters.append({ 'term': { facet_field: val } })
if project_slug is not None:
filters.append({ 'term': {'serialized.project.name': project.name }})
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
executed_query = en_index.search_model(Document, query)
o['facets'] = {}
for facet_label, facet_vals in executed_query['facets'].items():
o['facets'][facet_label] = [ (f['term'], f['term'], f['count'])
for f in facet_vals['terms'] ]
o['documents'] = [d['_source'] for d in executed_query['hits']['hits']]
return render_to_response(
template, o, context_instance=RequestContext(request))
