def delete_es_indexes():
en_index = get_index('main')
activity_index = get_index('activity')
if en_index.exists():
en_index.delete()
if activity_index.exists():
activity_index.delete()
def flush_es_indexes():
en_index = get_index('main')
activity_index = get_index('activity')
if en_index.exists():
en_index.delete()
en_index.create()
if activity_index.exists():
activity_index.delete()
activity_index.create()
def user(request, username=None):
o = {}
if not username:
user = request.user
if not user or not user.is_authenticated():
return HttpResponseRedirect(reverse('user_login_view'))
o['own_profile'] = True
else:
user = get_object_or_404(User, username=username)
o['own_profile'] = user == request.user
o['user'] = user
activity_index = get_index('activity')
o['activities'] = activity_index.get_activity_for(user)
# FIX
# o['profile'] = get_for(user)
# o['affiliation'] = o['profile'].affiliation
# o['project_role'] = (o['profile'].get_project_role(o['affiliation'])
# if o['affiliation'] else None)
if ['own_profile']:
o['zotero_status'] = user.zotero_key and user.zotero_uid
return render_to_response(
'user.html', o, context_instance=RequestContext(request))
def get(self, request, format=None):
query = {'query': {}}
params = request.QUERY_PARAMS
if 'autocomplete' in params:
self.filter_backend = ElasticSearchAutocompleteFilterBackend
result = self.filter_queryset(None)
data = {
'count':
result['hits']['total'],
'results': [
OrderedDict(
(('type', doc['_type']),
('title', doc['highlight' if 'highlight' in
doc else 'fields']['display_title']
[0 if 'highlight' in doc else 0:None]),
('url', doc['fields']['display_url'])))
for doc in result['hits']['hits']
]
}
return Response(data)
if 'q' in params:
query['query']['query_string'] = {'query': params.get('q')}
if 'sort' in params:
pass
if 'order' in params:
pass
if 'page' in params:
pass
if not query['query']:
query = {'query': {'match_all': {}}}
en_index = get_index('main')
es_query = en_index.es.search(query, index=en_index.name)
hits = []
for hit in es_query['hits']['hits']:
item_type = hit['_type']
item_data = hit['_source']['serialized']
filter_method = getattr(self, 'filter_{}_hits'.format(item_type),
None)
if filter_method is not None:
item_data = filter_method(item_data)
hits.append({'data': item_data, 'type': item_type})
return Response(
OrderedDict((
('results', es_query['hits']['total']),
('data', hits),
)))
def retrieve(self, request, *args, **kwargs):
if not settings.ELASTICSEARCH_ENABLED:
return super(ElasticSearchRetrieveMixin, self).retrieve(request, *args, **kwargs)
# Still need to get the object to check for perms
self.object = self.get_object()
en_index = get_index('main')
data = en_index.data_for_object(self.object)
return Response(data['_source']['serialized'])
def retrieve(self, request, *args, **kwargs):
if not settings.ELASTICSEARCH_ENABLED:
return super(ElasticSearchRetrieveMixin,
self).retrieve(request, *args, **kwargs)
# Still need to get the object to check for perms
self.object = self.get_object()
en_index = get_index('main')
data = en_index.data_for_object(self.object)
return Response(data['_source']['serialized'])
def search(request):
q = request.GET.get('q', {'query': {'match_all': {}}})
en_index = get_index('main')
results = en_index.search(q, highlight=True, size=50)
o = {
'results': results['hits']['hits'],
'query': q if isinstance(q, basestring) else None
}
return render_to_response(
'search.html', o, context_instance=RequestContext(request))
def setup_test_environment(self, **kwargs):
super(CustomTestSuiteRunner, self).setup_test_environment(**kwargs)
test_index_prefix = settings.ELASTICSEARCH_PREFIX + '-test'
settings.ELASTICSEARCH_PREFIX = test_index_prefix
en_index = get_index('main')
activity_index = get_index('activity')
en_index.name = en_index.get_name()
activity_index.name = activity_index.get_name()
for doctype in en_index.document_types.values():
doctype.index_name = en_index.name
try:
en_index.create()
activity_index.create()
except IndexAlreadyExistsError:
en_index.delete()
en_index.create()
activity_index.delete()
activity_index.create()
def search(request):
q = request.GET.get('q', {'query': {'match_all': {}}})
en_index = get_index('main')
results = en_index.search(q, highlight=True, size=50)
o = {
'results': results['hits']['hits'],
'query': q if isinstance(q, basestring) else None
}
return render_to_response('search.html',
o,
context_instance=RequestContext(request))
def setup_test_environment(self, **kwargs):
super(CustomTestSuiteRunner, self).setup_test_environment(**kwargs)
test_index_prefix = settings.ELASTICSEARCH_PREFIX + '-test'
settings.ELASTICSEARCH_PREFIX = test_index_prefix
en_index = get_index('main')
activity_index = get_index('activity')
en_index.name = en_index.get_name()
activity_index.name = activity_index.get_name()
for doctype in en_index.document_types.values():
doctype.index_name = en_index.name
try:
en_index.create()
activity_index.create()
except IndexAlreadyExistsError:
en_index.delete()
en_index.create()
activity_index.delete()
activity_index.create()
def topic(request, project_slug, topic_node_id):
o = {}
topic_qs = Topic.objects.select_related('topic_node', 'creator',
'last_updater', 'project')\
.prefetch_related('related_topics__topic')
o['topic'] = topic = get_object_or_404(topic_qs,
topic_node_id=topic_node_id,
project__slug=project_slug)
o['project'] = topic.project
o['breadcrumb'] = ((topic.project.name, topic.project.get_absolute_url()),
('Topics',
reverse('all_topics_view',
kwargs={'project_slug': topic.project.slug
})), (topic.preferred_name, None))
topic_query = {
'query': {
'term': {
'serialized.related_topics.id': topic.id
}
}
}
topic_query['size'] = 1000
en_index = get_index('main')
model_searches = (en_index.search_model(model, topic_query)
for model in (Document, Note, Topic))
documents, notes, topics = ([
result['_source']['serialized'] for result in search['hits']['hits']
] for search in model_searches)
o['documents'] = Document.objects.filter(
id__in=[d['id'] for d in documents])
o['related_topics'] = Topic.objects.filter(
id__in=[t['id'] for t in topics])
note_objects = Note.objects.in_bulk([n['id'] for n in notes])
for note in notes:
related_topics = list((topic for topic in note['related_topics']
if topic['url'] != request.path))
note_objects[note['id']] = (
note_objects[note['id']],
related_topics,
)
o['notes'] = note_objects.values()
return render_to_response('topic.html',
o,
context_instance=RequestContext(request))
def filter_queryset(self, request, queryset, view):
query = BASE_QUERY.copy()
filters = []
params = request.QUERY_PARAMS
if hasattr(request, 'project') or 'project' in params:
project = params.get('project', request.project.name)
filters.append({'term': {'serialized.project.name': project}})
if 'updater' in params:
filters.append({
'term': {
'serialized.updater.username': params.get('updater')
}
})
if 'q' in params:
q = params.get('q')
query['query']['filtered']['query'] = {
'match': {
'display_title': {
'query': q,
'operator': 'and',
'fuzziness': '0.3'
}
}
}
query['highlight'] = {
'fields': {
'_all': {}
},
'pre_tags': ['<strong>'],
'post_tags': ['</strong>']
}
if filters:
query['query']['filtered']['filter'] = {'and': filters}
en_index = get_index('main')
return en_index.search_model(view.model, query)
def project(request, project_slug):
o = {}
o['project'] = get_object_or_404(Project, slug=project_slug)
activity_index = get_index('activity')
o['activities'] = activity_index.get_activity_for(o['project'])
if request.user.is_authenticated():
o['project_role'] = o['project'].get_role_for(request.user)
o['can_view_roster'] = request.user.has_project_perm(
o['project'], 'main.view_project_roster')
o['can_change_roster'] = request.user.has_project_perm(
o['project'], 'main.change_project_roster')
o['can_change_featured_items'] = request.user.has_project_perm(
o['project'], 'main.change_featureditem')
o['can_change_project'] = request.user.has_project_perm(
o['project'], 'main.change_project')
o['show_edit_row'] = any((
o['can_view_roster'], o['can_change_roster'],
o['can_change_featured_items'], o['can_change_project']))
return render_to_response(
'project.html', o, context_instance=RequestContext(request))
def topic(request, project_slug, topic_node_id):
o = {}
topic_qs = Topic.objects.select_related('topic_node', 'creator',
'last_updater', 'project')\
.prefetch_related('related_topics__topic')
o['topic'] = topic = get_object_or_404(topic_qs,
topic_node_id=topic_node_id,
project__slug=project_slug)
o['project'] = topic.project
o['breadcrumb'] = (
(topic.project.name, topic.project.get_absolute_url()),
('Topics', reverse('all_topics_view',
kwargs={'project_slug': topic.project.slug})),
(topic.preferred_name, None)
)
topic_query = {'query': {'term': {'serialized.related_topics.id': topic.id }}}
topic_query['size'] = 1000
en_index = get_index('main')
model_searches = ( en_index.search_model(model, topic_query) for model in
(Document, Note, Topic) )
documents, notes, topics = (
[ result['_source']['serialized'] for result in search['hits']['hits'] ]
for search in model_searches)
o['documents'] = Document.objects.filter(id__in=[d['id'] for d in documents])
o['related_topics'] = Topic.objects.filter(id__in=[t['id'] for t in topics])
note_objects = Note.objects.in_bulk([n['id'] for n in notes])
for note in notes:
related_topics = list((topic for topic in note['related_topics']
if topic['url'] != request.path))
note_objects[note['id']] = (note_objects[note['id']], related_topics,)
o['notes'] = note_objects.values()
return render_to_response(
'topic.html', o, context_instance=RequestContext(request))
def get_item_types():
url = ZOTERO_BASE_URL + '/itemTypes'
if not cache.get('item_types'):
resp = urlopen(url)
data = resp.read()
resp.close()
cache.set('item_types', data, 60 * 24 * 7)
data = json.loads(cache.get('item_types'))
from editorsnotes.main.models import Document
from editorsnotes.search import get_index
en_index = get_index('main')
facet_search = en_index.search_model(
Document, {
'query': {
'filtered': {
'query': {
'match_all': {}
}
}
},
'facets': {
'itemTypes': {
'terms': {
'field': 'serialized.zotero_data.itemType',
'size': 10
}
}
}
})
used_item_types = facet_search['facets']['itemTypes']['terms']
return {
'itemTypes': data,
'common': [item_type['term'] for item_type in used_item_types[:10]]
}
def filter_queryset(self, request, queryset, view):
query = BASE_QUERY.copy()
filters = []
params = request.QUERY_PARAMS
if hasattr(request, 'project') or 'project' in params:
project = params.get('project', request.project.name)
filters.append({ 'term': { 'serialized.project.name': project }})
if 'updater' in params:
filters.append({ 'term': { 'serialized.updater.username':
params.get('updater') }})
if 'q' in params:
q = params.get('q')
query['query']['filtered']['query'] = {
'match': {
'display_title': {
'query': q,
'operator': 'and',
'fuzziness': '0.3'
}
}
}
query['highlight'] = {
'fields': {'_all': {}},
'pre_tags': ['<strong>'],
'post_tags': ['</strong>']
}
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
en_index = get_index('main')
return en_index.search_model(view.model, query)
def all_documents(request, project_slug=None):
o = {}
template = 'all-documents.html'
o['filtered'] = False
if project_slug is not None:
project = get_object_or_404(Project, slug=project_slug)
o['project'] = project
o['breadcrumb'] = (
(project.name, project.get_absolute_url()),
('Documents', None),
)
else:
o['breadcrumb'] = (
('Browse', reverse('browse_view')),
('Documents', None),
)
if request.GET.get('filter'):
template = 'filtered-documents.html'
o['filtered'] = True
facet_fields = [
('topic', 'serialized.related_topics.name'),
('archive', 'serialized.zotero_data.archive'),
('publicationTitle', 'serialized.zotero_data.publicationTitle'),
('itemType', 'serialized.zotero_data.itemType'),
('creators', 'serialized.zotero_data.creators'),
('representations', 'serialized.representations'),
]
if project_slug is None:
facet_fields.insert(0, ('project', 'serialized.project.name'))
facet_dict = dict(facet_fields)
query = {
'query': { 'filtered': { 'query': { 'match_all': {} } } },
'facets': {},
'size': 1000
}
for label, field in facet_fields:
query['facets'][label] = { 'terms': { 'field': field, 'size': 30 } }
filters = []
params = [ p for p in request.GET.keys() if p.endswith('[]') ]
for field in params:
facet_field = facet_dict.get(field[:-2], None)
if facet_field is None:
continue
for val in request.GET.getlist(field):
filters.append({ 'term': { facet_field: val } })
if project_slug is not None:
filters.append({ 'term': {'serialized.project.name': project.name }})
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
en_index = get_index('main')
executed_query = en_index.search_model(Document, query)
o['facets'] = {}
for facet_label, facet_vals in executed_query['facets'].items():
o['facets'][facet_label] = [ (f['term'], f['term'], f['count'])
for f in facet_vals['terms'] ]
o['documents'] = [d['_source'] for d in executed_query['hits']['hits']]
return render_to_response(
template, o, context_instance=RequestContext(request))
def get(self, request, format=None, **kwargs):
obj = self.get_object(**kwargs)
data = get_index('activity').get_activity_for(obj)
return Response({'activity': data})
def all_notes(request, project_slug=None):
o = {}
template = 'all-notes.html'
if project_slug is not None:
project = get_object_or_404(Project, slug=project_slug)
o['project'] = project
o['breadcrumb'] = (
(project.name, project.get_absolute_url()),
('Notes', None),
)
else:
o['breadcrumb'] = (
('Browse', reverse('browse_view')),
('Notes', None),
)
o['filtered'] = False
if request.GET.get('filter'):
template = 'filtered-notes.html'
o['filtered'] = True
# Base query & facets
query = {
'query': {
'filtered': {
'query': {
'match_all': {}
}
}
},
'facets': {
'topic_facet': {
'terms': {
'field': 'serialized.related_topics.name',
'size': 16
}
},
'status_facet': {
'terms': {
'field': 'serialized.status'
}
}
},
'size': 1000
}
if project_slug is None:
query['facets']['project_facet'] = {
'terms': {
'field': 'serialized.project.name',
'size': 8
}
}
# Filter query based on facets
filters = []
for topic in request.GET.getlist('topic'):
filters.append({'term': {'serialized.related_topics.name': topic}})
project_filter = project.name if project_slug is not None \
else request.GET.get('project')
if project_filter:
filters.append({'term': {'serialized.project.name': project_filter}})
status = request.GET.get('note_status')
if status:
filters.append({'term': {'serialized.status': status}})
if filters:
query['query']['filtered']['filter'] = {'and': filters}
# Execute built query
en_index = get_index('main')
executed_query = en_index.search_model(Note, query)
# This is gibberish that can be improved, but it lets us use the old xapian
# code that I implemented a long time ago.
topic_facets = executed_query['facets']['topic_facet']['terms']
o['topic_facets_1'] = topic_facets[:8]
o['topic_facets_2'] = topic_facets[8:] if len(topic_facets) > 8 else []
if project_slug is None:
project_facets = executed_query['facets']['project_facet']['terms']
o['project_facets'] = project_facets
status_facets = executed_query['facets']['status_facet']['terms']
o['status_facets'] = status_facets
o['notes'] = [n['_source'] for n in executed_query['hits']['hits']]
return render_to_response(template,
o,
context_instance=RequestContext(request))
def all_notes(request, project_slug=None):
o = {}
template = 'all-notes.html'
if project_slug is not None:
project = get_object_or_404(Project, slug=project_slug)
o['project'] = project
o['breadcrumb'] = (
(project.name, project.get_absolute_url()),
('Notes', None),
)
else:
o['breadcrumb'] = (
('Browse', reverse('browse_view')),
('Notes', None),
)
o['filtered'] = False
if request.GET.get('filter'):
template = 'filtered-notes.html'
o['filtered'] = True
# Base query & facets
query = {
'query': { 'filtered': { 'query': { 'match_all': {} } } },
'facets': {
'topic_facet': {
'terms': { 'field': 'serialized.related_topics.name', 'size': 16 }
},
'status_facet': {
'terms': { 'field': 'serialized.status' }
}
},
'size': 1000
}
if project_slug is None:
query['facets']['project_facet'] = {
'terms': { 'field': 'serialized.project.name', 'size': 8 }
}
# Filter query based on facets
filters = []
for topic in request.GET.getlist('topic'):
filters.append({ 'term': { 'serialized.related_topics.name': topic } })
project_filter = project.name if project_slug is not None \
else request.GET.get('project')
if project_filter:
filters.append({ 'term': { 'serialized.project.name': project_filter } })
status = request.GET.get('note_status')
if status:
filters.append({ 'term': { 'serialized.status': status } })
if filters:
query['query']['filtered']['filter'] = { 'and': filters }
# Execute built query
en_index = get_index('main')
executed_query = en_index.search_model(Note, query)
# This is gibberish that can be improved, but it lets us use the old xapian
# code that I implemented a long time ago.
topic_facets = executed_query['facets']['topic_facet']['terms']
o['topic_facets_1'] = topic_facets[:8]
o['topic_facets_2'] = topic_facets[8:] if len(topic_facets) > 8 else []
if project_slug is None:
project_facets = executed_query['facets']['project_facet']['terms']
o['project_facets'] = project_facets
status_facets = executed_query['facets']['status_facet']['terms']
o['status_facets'] = status_facets
o['notes'] = [n['_source'] for n in executed_query['hits']['hits']]
return render_to_response(
template, o, context_instance=RequestContext(request))
def teardown_test_environment(self, **kwargs):
super(CustomTestSuiteRunner, self).teardown_test_environment(**kwargs)
get_index('main').delete()
get_index('activity').delete()
def get(self, request, format=None, **kwargs):
obj = self.get_object(**kwargs)
data = get_index('activity').get_activity_for(obj)
return Response({ 'activity': data })
def teardown_test_environment(self, **kwargs):
super(CustomTestSuiteRunner, self).teardown_test_environment(**kwargs)
get_index('main').delete()
get_index('activity').delete()