def testUnpublishedModelsNotFound(self):
# Make sure that there are four to find!
self.assertEqual(watson.search("tItle Content Description").count(), 4)
# Unpublish two objects.
self.test11.is_published = False
self.test11.save()
self.test21.is_published = False
self.test21.save()
# This should return 4, but two of them are unpublished.
self.assertEqual(watson.search("tItle Content Description").count(), 2)
def testUnpublishedModelsNotFound(self):
# Make sure that there are four to find!
self.assertEqual(watson.search("tItle Content Description").count(), 4)
# Unpublish two objects.
self.test11.is_published = False
self.test11.save()
self.test21.is_published = False
self.test21.save()
# This should return 4, but two of them are unpublished.
self.assertEqual(watson.search("tItle Content Description").count(), 2)
def testNestedUpdateInSkipContext(self):
with watson.skip_index_update():
self.test21.title = "baar"
self.test21.save()
with watson.update_index():
self.test11.title = "fooo"
self.test11.save()
# We should get "fooo", but not "baar"
self.assertEqual(watson.search("fooo").count(), 1)
self.assertEqual(watson.search("baar").count(), 0)
def test_pagesearchadapter_get_live_queryset(self):
self.assertEqual(len(search.search("Homepage", models=(Page,))), 1)
with publication_manager.select_published(True):
self.assertEqual(len(search.search("Homepage", models=(Page,))), 1)
self.homepage.is_online = False
self.homepage.save()
self.assertEqual(len(search.search("Homepage", models=(Page,))), 0)
def testNestedUpdateInSkipContext(self):
with watson.skip_index_update():
self.test21.title = "baar"
self.test21.save()
with watson.update_index():
self.test11.title = "fooo"
self.test11.save()
# We should get "fooo", but not "baar"
self.assertEqual(watson.search("fooo").count(), 1)
self.assertEqual(watson.search("baar").count(), 0)
def testUpdateSearchIndex(self):
# Update a model and make sure that the search results match.
self.test11.title = "fooo"
self.test11.save()
# Test a search that should get one model.
exact_search = watson.search("fooo")
self.assertEqual(len(exact_search), 1)
self.assertEqual(exact_search[0].title, "fooo")
# Delete a model and make sure that the search results match.
self.test11.delete()
self.assertEqual(watson.search("fooo").count(), 0)
def testUpdateSearchIndex(self):
# Update a model and make sure that the search results match.
self.test11.title = "fooo"
self.test11.save()
# Test a search that should get one model.
exact_search = watson.search("fooo")
self.assertEqual(len(exact_search), 1)
self.assertEqual(exact_search[0].title, "fooo")
# Delete a model and make sure that the search results match.
self.test11.delete()
self.assertEqual(watson.search("fooo").count(), 0)
def testBuildWatsonCommand(self):
# Hack a change into the model using a bulk update, which doesn't send signals.
WatsonTestModel1.objects.filter(id=self.test11.id).update(title="fooo1")
WatsonTestModel2.objects.filter(id=self.test21.id).update(title="fooo2")
# Test that no update has happened.
self.assertEqual(watson.search("fooo1").count(), 0)
self.assertEqual(watson.search("fooo2").count(), 0)
# Run the rebuild command.
call_command("buildwatson", verbosity=0)
# Test that the update is now applied.
self.assertEqual(watson.search("fooo1").count(), 1)
self.assertEqual(watson.search("fooo2").count(), 1)
def testBuildWatsonCommand(self):
# Hack a change into the model using a bulk update, which doesn't send signals.
WatsonTestModel1.objects.filter(id=self.test11.id).update(title="fooo1")
WatsonTestModel2.objects.filter(id=self.test21.id).update(title="fooo2")
# Test that no update has happened.
self.assertEqual(watson.search("fooo1").count(), 0)
self.assertEqual(watson.search("fooo2").count(), 0)
# Run the rebuild command.
call_command("buildwatson", verbosity=0)
# Test that the update is now applied.
self.assertEqual(watson.search("fooo1").count(), 1)
self.assertEqual(watson.search("fooo2").count(), 1)
def test_pagesearchadapter_get_live_queryset(self):
self.assertEqual(len(search.search('Homepage', models=(Page, ))), 1)
with publication_manager.select_published(True):
self.assertEqual(len(search.search('Homepage', models=(Page, ))),
1)
self.homepage.is_online = False
self.homepage.save()
self.assertEqual(len(search.search('Homepage', models=(Page, ))),
0)
def testSearchWithApostrophe(self):
WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description model1 instance13",
)
self.assertEqual(watson.search("d'Argent").count(), 1)
def get_queryset(self):
"""Returns list of querysets containing content a user is allowed to search.
This is determined by a user's organization.
"""
# TODO Future Revise for both org users and contractors
user_org = self.request.org
user = self.request.user
# retrieve all content a user is allowed to search
searchable_org_objects = user_org.get_org_searchable_content()
# unpack the querysets from the list of querysets returned
projects, stories, facets, imageassets, documentassets, audioassets, tasks, events, notes, comments = searchable_org_objects
# XXX If this ever include user notes as well
# searchable_user_objects = user.get_user_searchable_content()
# usernotes = searchable_user_objects
# pass all querysets to search method
return watson.search(self.query,
models=[
projects,
stories,
facets,
tasks,
events,
notes,
comments,
imageassets,
documentassets,
audioassets,
])
def list(self, request):
"""
Respond to list() requests by running a Watson search, then using
api.urls.API_VIEW_MODULES to map each matched model back to its
equivalent serializer.
"""
query = request.GET.get(self.param_name)
if not query:
msg = _('You must specify %r parameter.' % (self.param_name,))
raise rest_framework.exceptions.ParseError(msg)
ids_by_type = {}
results = (watson.search(
query, exclude=self.build_excludes()).prefetch_related(
'object'))[:400]
for result in results:
model = result.object
if model:
ids_by_type.setdefault(type(model), []).append(model.id)
prefetched_results = []
for model_class, ids in ids_by_type.iteritems():
qs = model_class.objects.filter(pk__in=ids)
qs = model_class.prefetch_queryset(qs)
prefetched_results.extend(qs)
json = map(self.convert_result, prefetched_results)
return rest_framework.response.Response(json)
def testSearchWithAccent(self):
WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 café",
description="description model1 instance13",
)
self.assertEqual(watson.search("café").count(), 1)
def _search_indexed(request, query, filters):
results = []
kwargs = {}
order_by = 'event_start'
if filters['future'] == 'true':
kwargs['event_start__gte'] = timezone.now()
else:
# Reverse order when showing all events
order_by = '-' + order_by
if filters['myevents'] == 'true':
kwargs['attendance_event__attendees__user'] = request.user
events = Event.objects.filter(**kwargs).order_by(order_by).prefetch_related(
'attendance_event', 'attendance_event__attendees', 'attendance_event__reserved_seats',
'attendance_event__reserved_seats__reservees')
# Filters events that are restricted
display_events = set()
for event in events:
if event.can_display(request.user):
display_events.add(event.pk)
events = events.filter(pk__in=display_events)
if query:
for result in watson.search(query, models=(events,)):
results.append(result.object)
return results[:10]
return events
def testSearchWithAccent(self):
WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 café",
description="description model1 instance13",
)
self.assertEqual(watson.search("café").count(), 1)
def testSearchWithApostrophe(self):
WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description model1 instance13",
)
self.assertEqual(watson.search("d'Argent").count(), 1)
def search(user_id, search_term):
"""Offload the expensive part of search to avoid blocking the web interface"""
email_subquery = models.Email.objects.viewable(user_id)
inbox_subquery = models.Inbox.objects.viewable(user_id)
results = {
"emails": list(watson_search.search(search_term, models=(email_subquery,)).values_list("id", flat=True)),
"inboxes": list(watson_search.search(search_term, models=(inbox_subquery,)).values_list("id", flat=True)),
}
key = u"{0}-{1}".format(user_id, search_term)
key = urllib.quote(key.encode("utf-8"))
cache.set(key, results, SEARCH_TIMEOUT)
return results
def search(request):
""" Takes in http request and a user-entered search string, returns the search_results html
with the objects found by the search available to that template.
"""
search_str = request.GET["user_search"]
search_results = watson.search(search_str)
# Show 18 movies per page
paginator = Paginator(search_results, 24)
page = request.GET.get('page')
try:
results = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
results = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
results = paginator.page(paginator.num_pages)
template = loader.get_template('movies/search_results.html')
context = {
'results': results,
'search_str': search_str,
}
return HttpResponse(template.render(context, request))
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
query = self.request.GET.get('q')
resultados = {}
if query:
resultados = watson.search(query)
context.update({'resultados': resultados, 'query': query})
return context
def testExcludedModelQuerySet(self):
# Test a search that should get all models.
self.assertEqual(watson.search("TITLE", exclude=(WatsonTestModel1.objects.filter(title__icontains="FOOO"), WatsonTestModel2.objects.filter(title__icontains="FOOO"),)).count(), 4)
# Test a search that should get two models.
self.assertEqual(watson.search("MODEL1", exclude=(WatsonTestModel1.objects.filter(
title__icontains = "INSTANCE21",
description__icontains = "INSTANCE22",
),)).count(), 2)
self.assertEqual(watson.search("MODEL2", exclude=(WatsonTestModel2.objects.filter(
title__icontains = "INSTANCE11",
description__icontains = "INSTANCE12",
),)).count(), 2)
# Test a search that should get one model.
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter(
title__icontains = "MODEL2",
),)).count(), 1)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL1",
),)).count(), 1)
# Test a search that should get no models.
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter(
title__icontains = "MODEL1",
),)).count(), 0)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL2",
),)).count(), 0)
def testExcludedModelQuerySet(self):
# Test a search that should get all models.
self.assertEqual(
watson.search(
"TITLE",
exclude=(
WatsonTestModel1.objects.filter(title__icontains="FOOO"),
WatsonTestModel2.objects.filter(title__icontains="FOOO"),)
).count(), 4)
# Test a search that should get two models.
self.assertEqual(watson.search("MODEL1", exclude=(WatsonTestModel1.objects.filter(
title__icontains="INSTANCE21",
description__icontains="INSTANCE22",
),)).count(), 2)
self.assertEqual(watson.search("MODEL2", exclude=(WatsonTestModel2.objects.filter(
title__icontains="INSTANCE11",
description__icontains="INSTANCE12",
),)).count(), 2)
# Test a search that should get one model.
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter(
title__icontains="MODEL2",
),)).count(), 1)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter(
title__icontains="MODEL1",
),)).count(), 1)
# Test a search that should get no models.
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter(
title__icontains="MODEL1",
),)).count(), 0)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter(
title__icontains="MODEL2",
),)).count(), 0)
def get(self, request, tag=None, page=1, format=None):
q = request.GET.get('q', '') if tag == None else tag
if q != '':
results = watson.search(q)
results = paginate(results, page, 15)
serializer = SearchEntrySerializer(results, many=True)
return Response(serializer.data)
else:
return Response(status=status.HTTP_400_BAD_REQUEST)
def testSearchWithLeadingApostrophe(self):
WatsonTestModel1.objects.create(
title="title model1 instance12",
content="'content model1 instance13",
description="description model1 instance13",
)
self.assertTrue(
watson.search("'content").exists()
) # Some database engines ignore leading apostrophes, some count them.
def testCanOverridePublication(self):
# Unpublish two objects.
self.test11.is_published = False
self.test11.save()
# This should still return 4, since we're overriding the publication.
self.assertEqual(watson.search(
"tItle Content Description",
models=(WatsonTestModel2, WatsonTestModel1._base_manager.all(),)
).count(), 4)
def testSearchWithLeadingApostrophe(self):
WatsonTestModel1.objects.create(
title="title model1 instance12",
content="'content model1 instance13",
description="description model1 instance13",
)
self.assertTrue(
watson.search("'content").exists()
) # Some database engines ignore leading apostrophes, some count them.
def testCanOverridePublication(self):
# Unpublish two objects.
self.test11.is_published = False
self.test11.save()
# This should still return 4, since we're overriding the publication.
self.assertEqual(watson.search(
"tItle Content Description",
models=(WatsonTestModel2, WatsonTestModel1._base_manager.all(),)
).count(), 4)
def testSearchIndexUpdateAbandonedOnError(self):
try:
with watson.update_index():
self.test11.title = "fooo"
self.test11.save()
raise Exception("Foo")
except:
pass
# Test a search that should get not model.
self.assertEqual(watson.search("fooo").count(), 0)
def testSearchIndexUpdateAbandonedOnError(self):
try:
with watson.update_index():
self.test11.title = "fooo"
self.test11.save()
raise Exception("Foo")
except:
pass
# Test a search that should get not model.
self.assertEqual(watson.search("fooo").count(), 0)
def build_search(request):
results = []
q = request.GET.get("q")
if q:
results = watson.search(q)
context = {"query": q, "search_entry_list": results}
return render(request, "build/search.html", context=context)
def search(request, tag=None):
data = {}
template = pages_dir + 'search.html'
q = request.GET.get('q', '') if tag == None else tag
if q != '':
results = watson.search(q)
data['q'] = q
data['results_count'] = results.count()
data['results'] = results[:10]
data['is_search'] = True if tag == None else False
return render(request, template, data)
def search_for_plain_users(query, limit=10):
if not query:
return []
results = []
for result in watson.search(query, models=(User.objects.filter(is_active=True),)):
uobj = result.object
results.append({"id": uobj.id, "value": uobj.get_full_name()})
return results[:limit]
def handle_search(self, query, model):
search_results = watson.search(query, models=(model, ))
search_results = map(lambda x: x.object, search_results)
# remove invalid results
search_results = filter(lambda x: x is not None, search_results)
search_results = [x for x in search_results if x != None]
search_results_ids = [seminar.id for seminar in search_results]
return search_results_ids
def load_search_results(request):
q = request.POST.get('q')
page = request.POST.get('page')
results = watson.search(q)
results = paginate_list(results, page)
results_html = loader.render_to_string(
'social/partials/search-results.html',
{'results': results},
)
data['has_next'] = results.has_next()
data['results_html'] = results_html
return JsonResponse(data)
def search_for_users(query, limit=10):
if not query:
return []
results = []
for result in watson.search(
query, models=(User.objects.filter(privacy__visible_for_other_users=True),)
):
results.append(result.object)
return results[:limit]
def search(request):
context = {}
q = ""
try:
if request.POST:
q = request.POST['q']
else:
q = request.GET['q']
except MultiValueDictKeyError:
pass
context['query'] = q
context['search_entry_list'] = watson.search(q)
return render(request, 'search.html', context)
def testKitchenSink(self):
"""For sanity, let's just test everything together in one giant search of doom!"""
self.assertEqual(watson.search(
"INSTANCE11",
models = (
WatsonTestModel1.objects.filter(title__icontains="INSTANCE11"),
WatsonTestModel2.objects.filter(title__icontains="TITLE"),
),
exclude = (
WatsonTestModel1.objects.filter(title__icontains="MODEL2"),
WatsonTestModel2.objects.filter(title__icontains="MODEL1"),
)
).get().title, "title model1 instance11")
def testKitchenSink(self):
"""For sanity, let's just test everything together in one giant search of doom!"""
self.assertEqual(watson.search(
"INSTANCE11",
models=(
WatsonTestModel1.objects.filter(title__icontains="INSTANCE11"),
WatsonTestModel2.objects.filter(title__icontains="TITLE"),
),
exclude=(
WatsonTestModel1.objects.filter(title__icontains="MODEL2"),
WatsonTestModel2.objects.filter(title__icontains="MODEL1"),
)
).get().title, "title model1 instance11")
def build_search(request):
results = []
q = request.GET.get('q')
if q:
results = watson.search(q)
context = {
'query': q,
'search_entry_list': results,
}
return render(request, 'build/search.html', context=context)
def search(request):
context = {}
q = ""
try:
if request.POST:
q = request.POST['q']
else:
q = request.GET['q']
except MultiValueDictKeyError:
pass
context['query'] = q
context['search_entry_list'] = watson.search(q)
return render(request, 'search.html', context)
def search(user_id, search_term):
"""Offload the expensive part of search to avoid blocking the web interface"""
email_subquery = models.Email.objects.viewable(user_id)
inbox_subquery = models.Inbox.objects.viewable(user_id)
results = {
"emails":
list(
watson_search.search(search_term,
models=(email_subquery, )).values_list(
"id", flat=True)),
"inboxes":
list(
watson_search.search(search_term,
models=(inbox_subquery, )).values_list(
"id", flat=True)),
}
key = u"{0}-{1}".format(user_id, search_term)
key = urllib.parse.quote(key.encode("utf-8"))
cache.set(key, results, SEARCH_TIMEOUT)
return results
def get(self, request):
if request.GET.get('search'):
# watson package를 사용한 full text search
search_results = watson.search(request.GET.get('search'),
models=(HFI, ))
# 검색된 object들의 pk list가 output
index_list = [index.object_id for index in search_results]
# HFI에서 해당 index에 해당하는 데이터 검색 후 Serializer로 복수 처리
queryset = [HFI.objects.get(pk=f'{pk}') for pk in index_list]
serializer = HFISerializer(queryset, many=True)
return Response(serializer.data, status=status.HTTP_200_OK)
else:
return Response({"message": "검색 결과가 없습니다."},
status=status.HTTP_200_OK)
def post(self, request):
"""Returns a Json response of a search query
Args:
request (object): HTTPRequest
Returns:
object: JsonResponse
"""
# get query out of request
query = request.POST['query']
# search for the query in the database
search_res = watson.search(query)
# no search results
if search_res.count() < 1:
response = JsonResponse({'search_res': []})
# search results
else:
# list of the search results
search_res_list = []
# go through all search results and add them to the list
for sres in search_res:
# set the sres to the real result
sres = sres.object
# result dict
res = {}
# set the values of the res to the sres
res['name'] = sres.name
res['overview'] = sres.overview
res['year'] = (sres.first_air_date.year if
sres.first_air_date else None)
res['genres'] = sres.get_genre_list()
# try to get the poster url
try:
res['poster'] = sres.poster_large.url
# no poster is present
except ValueError:
res['poster'] = False
# url of the series
res['url'] = sres.get_absolute_url()
# add the result dict to the search result list
search_res_list.append(res)
response = JsonResponse({'search_res': search_res_list})
return response
def testBuildWatsonForModelCommand(self):
# Hack a change into the model using a bulk update, which doesn't send signals.
WatsonTestModel1.objects.filter(id=self.test11.id).update(title="fooo1_selective")
WatsonTestModel2.objects.filter(id=self.test21.id).update(title="fooo2_selective")
WatsonTestModel3.objects.filter(id=self.test31.id).update(title="fooo3_selective")
# Test that no update has happened.
self.assertEqual(watson.search("fooo1_selective").count(), 0)
self.assertEqual(watson.search("fooo2_selective").count(), 0)
self.assertEqual(watson.search("fooo3_selective").count(), 0)
# Run the rebuild command.
call_command("buildwatson", "test_watson.WatsonTestModel1", verbosity=0)
# Test that the update is now applied to selected model.
self.assertEqual(watson.search("fooo1_selective").count(), 1)
self.assertEqual(watson.search("fooo2_selective").count(), 0)
self.assertEqual(watson.search("fooo3_selective").count(), 0)
call_command(
"buildwatson",
"test_watson.WatsonTestModel1", "test_watson.WatsonTestModel2", "test_watson.WatsonTestModel3",
verbosity=0,
)
# Test that the update is now applied to multiple selected models.
self.assertEqual(watson.search("fooo1_selective").count(), 1)
self.assertEqual(watson.search("fooo2_selective").count(), 1)
self.assertEqual(watson.search("fooo3_selective").count(), 1)
def testSearchWithSpecialChars(self):
WatsonTestModel1.objects.all().delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description model1 instance13",
)
self.assertEqual(watson.search("d'Argent").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="'content model1 instance13",
description="description model1 instance13",
)
# Some database engines ignore leading apostrophes, some count them.
self.assertTrue(watson.search("'content").exists())
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd&efgh",
)
self.assertEqual(watson.search("abcd&efgh").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd.efgh",
)
self.assertEqual(watson.search("abcd.efgh").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd,efgh",
)
self.assertEqual(watson.search("abcd,efgh").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd@efgh",
)
self.assertEqual(watson.search("abcd@efgh").count(), 1)
x.delete()
def testSearchWithSpecialChars(self):
WatsonTestModel1.objects.all().delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description model1 instance13",
)
self.assertEqual(watson.search("d'Argent").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="'content model1 instance13",
description="description model1 instance13",
)
# Some database engines ignore leading apostrophes, some count them.
self.assertTrue(watson.search("'content").exists())
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd&efgh",
)
self.assertEqual(watson.search("abcd&efgh").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd.efgh",
)
self.assertEqual(watson.search("abcd.efgh").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd,efgh",
)
self.assertEqual(watson.search("abcd,efgh").count(), 1)
x.delete()
x = WatsonTestModel1.objects.create(
title="title model1 instance12",
content="content model1 instance13 d'Argent",
description="description abcd@efgh",
)
self.assertEqual(watson.search("abcd@efgh").count(), 1)
x.delete()
def testBuildWatsonForModelCommand(self):
# Hack a change into the model using a bulk update, which doesn't send signals.
WatsonTestModel1.objects.filter(id=self.test11.id).update(title="fooo1_selective")
WatsonTestModel2.objects.filter(id=self.test21.id).update(title="fooo2_selective")
WatsonTestModel3.objects.filter(id=self.test31.id).update(title="fooo3_selective")
# Test that no update has happened.
self.assertEqual(watson.search("fooo1_selective").count(), 0)
self.assertEqual(watson.search("fooo2_selective").count(), 0)
self.assertEqual(watson.search("fooo3_selective").count(), 0)
# Run the rebuild command.
call_command("buildwatson", "test_watson.WatsonTestModel1", verbosity=0)
# Test that the update is now applied to selected model.
self.assertEqual(watson.search("fooo1_selective").count(), 1)
self.assertEqual(watson.search("fooo2_selective").count(), 0)
self.assertEqual(watson.search("fooo3_selective").count(), 0)
call_command(
"buildwatson",
"test_watson.WatsonTestModel1", "test_watson.WatsonTestModel2", "test_watson.WatsonTestModel3",
verbosity=0,
)
# Test that the update is now applied to multiple selected models.
self.assertEqual(watson.search("fooo1_selective").count(), 1)
self.assertEqual(watson.search("fooo2_selective").count(), 1)
self.assertEqual(watson.search("fooo3_selective").count(), 1)
def search(request):
out = {}
search_text = request.GET.get('search', '')
search_results = watson.search(search_text)
search_results_count = search_results.count()
page_number = request.GET.get('page', 1)
start = (int(page_number) - 1) * 10
out.update({'current_page_number': int(page_number)})
all_page_count = search_results.count() / 10 + 1
if search_results.count() % 10:
all_page_count += 1
search_results = search_results[start:start+10]
out.update({'all_page_number': range(1, all_page_count)})
out.update({'menu_active_item': 'search'})
out.update({'search_results': search_results})
out.update({'search_results_count': search_results_count})
out.update({'search_text': search_text})
out.update({'title': u'Поиск'})
return render(request, 'search.html', out)
def testReferencingWatsonRankInAnnotations(self):
"""We should be able to reference watson_rank from annotate expressions"""
entries = watson.search("model1").annotate(
relevant=Case(
When(watson_rank__gt=1.0, then=Value(1)),
default_value=Value(0),
output_field=IntegerField()
)
)
# watson_rank does not return the same value across backends, so we
# can't hard code what those will be. In some cases (e.g. the regex
# backend) all ranking is hard coded to 1.0. That doesn't matter - we
# just want to make sure that Django is able to construct a valid query
for entry in entries:
if entry.watson_rank > 1.0:
self.assertTrue(entry.relevant)
else:
self.assertFalse(entry.relevant)
def testReferencingWatsonRankInAnnotations(self):
"""We should be able to reference watson_rank from annotate expressions"""
entries = watson.search("model1").annotate(
relevant=Case(
When(watson_rank__gt=1.0, then=Value(1)),
default_value=Value(0),
output_field=IntegerField()
)
)
# watson_rank does not return the same value across backends, so we
# can't hard code what those will be. In some cases (e.g. the regex
# backend) all ranking is hard coded to 1.0. That doesn't matter - we
# just want to make sure that Django is able to construct a valid query
for entry in entries:
if entry.watson_rank > 1.0:
self.assertTrue(entry.relevant)
else:
self.assertFalse(entry.relevant)
def search(user_id, search_term, before=None, after=None):
"""Offload the expensive part of search to avoid blocking the web interface"""
if not search_term:
return {
"results": [],
"has_next": False
}
if before and after:
raise ValueError("You can't do this.")
email_subquery = models.Email.objects.viewable(user_id)
inbox_subquery = models.Inbox.objects.viewable(user_id)
search_qs = watson_search.search(search_term, models=(email_subquery, inbox_subquery))
page_kwargs = {
"after": after,
"before": before,
}
if before:
page_kwargs["last"] = SEARCH_PAGE_SIZE
else:
page_kwargs["first"] = SEARCH_PAGE_SIZE
paginator = CursorPaginator(search_qs, ordering=('-watson_rank', '-id'))
page = paginator.page(**page_kwargs)
results = {
"results": [p.id for p in page],
"has_next": page.has_next,
"has_previous": page.has_previous,
}
if len(results["results"]) > 0:
results["last"] = paginator.cursor(page[-1])
results["first"] = paginator.cursor(page[0])
key = create_search_cache_key(user_id, search_term, before, after)
cache.set(key, results, SEARCH_TIMEOUT)
return results
def get_queryset(self):
"""Returns list of querysets containing content a user is allowed to search.
This is determined by a user's organization.
"""
user_org = self.request.user.organization
user = self.request.user
# retrieve all content a user is allowed to search
searchable_org_objects = Organization.get_org_searchable_content(user_org)
searchable_user_objects = User.get_user_searchable_content(user)
# unpack the querysets from the list of querysets returned
series, stories, webfacets, printfacets, audiofacets, videofacets, imageassets, networknotes, orgnotes, seriesnotes, storynotes = (
searchable_org_objects
)
usernotes = searchable_user_objects
# pass all querysets to search method
return watson.search(
self.query,
models=[
series,
stories,
webfacets,
printfacets,
audiofacets,
videofacets,
imageassets,
networknotes,
orgnotes,
seriesnotes,
storynotes,
usernotes,
],
)
def full_text_search(request):
if request.method == "POST":
json_data = json.loads(request.body.decode('utf-8'))
request_type = json_data['request_type']
if request_type == "search":
# print ("json search text is ", json_data["search_text"])
search_results = watson.search(json_data["search_text"])
# print "object is ",search_results
json_response = {
"post_ids":[],
}
if search_results.count() == 0:
# print "no result found"
pass
else:
for a_result in search_results:
json_response["post_ids"].append(a_result.meta["id"])
# print "meta is ",a_result.meta
return JsonResponse(json_response)
else:
return PermissionDenied
else:
return PermissionDenied
def testRankingParamPresentOnSearch(self):
self.assertGreater(watson.search("TITLE")[0].watson_rank, 0)
def testRankingWithSearch(self):
self.assertEqual(
[entry.title for entry in watson.search("FOOO")],
["title model1 instance11 fooo baar fooo", "title model1 instance12"],
)
def testExcludedModelList(self):
# Test a search that should get all models.
self.assertEqual(watson.search("TITLE", exclude=()).count(), 4)
# Test a search that should get two models.
self.assertEqual(watson.search("MODEL1", exclude=()).count(), 2)
self.assertEqual(watson.search("MODEL1", exclude=(WatsonTestModel2,)).count(), 2)
self.assertEqual(watson.search("MODEL2", exclude=()).count(), 2)
self.assertEqual(watson.search("MODEL2", exclude=(WatsonTestModel1,)).count(), 2)
# Test a search that should get one model.
self.assertEqual(watson.search("INSTANCE11", exclude=()).count(), 1)
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel2,)).count(), 1)
self.assertEqual(watson.search("INSTANCE21", exclude=()).count(), 1)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel1,)).count(), 1)
# Test a search that should get zero models.
self.assertEqual(watson.search("MODEL1", exclude=(WatsonTestModel1,)).count(), 0)
self.assertEqual(watson.search("MODEL2", exclude=(WatsonTestModel2,)).count(), 0)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2,)).count(), 0)
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1,)).count(), 0)
def get_queryset(self):
"""Returns the initial queryset."""
return watson.search(self.query, models=self.get_models(), exclude=self.get_exclude())
def simple_search(request):
ip_addresses = []
dashes = []
query = []
tests = None
findings = None
finding_templates = None
products = None
tagged_tests = None
tagged_findings = None
tagged_products = None
tagged_endpoints = None
tagged_engagements = None
tagged_finding_templates = None
engagements = None
endpoints = None
languages = None
app_analysis = None
clean_query = ''
cookie = False
terms = ''
form = SimpleSearchForm()
if request.method == 'GET' and "query" in request.GET:
form = SimpleSearchForm(request.GET)
if form.is_valid():
cookie = True
clean_query = form.cleaned_data['query']
search_operator = ""
# Check for search operator like finding:, endpoint:, test: product:
if ":" in clean_query:
operator = clean_query.split(":")
search_operator = operator[0]
clean_query = operator[1].lstrip()
tags = clean_query
if request.user.is_staff:
if "finding" in search_operator or search_operator is "":
findings = watson.search(clean_query, models=(Finding,))
if "template" in search_operator or search_operator is "":
finding_templates = watson.search(clean_query, models=(Finding_Template,))
if "test" in search_operator or search_operator is "":
tests = watson.search(clean_query, models=(Test,))
if "product" in search_operator or search_operator is "":
products = watson.search(clean_query, models=(Product,))
if "tag" in search_operator or search_operator is "":
tagged_findings = TaggedItem.objects.get_by_model(Finding,
tags)
tagged_finding_templates = TaggedItem.objects.get_by_model(Finding_Template,
tags)
tagged_tests = TaggedItem.objects.get_by_model(Test, tags)
tagged_products = TaggedItem.objects.get_by_model(Product,
tags)
tagged_endpoints = TaggedItem.objects.get_by_model(Endpoint,
tags)
tagged_engagements = TaggedItem.objects.get_by_model(
Engagement, tags)
# endpoints = watson.search(clean_query, models=(Endpoint,))
if "endpoint" in search_operator or search_operator is "":
endpoints = Endpoint.objects.filter(Q(host__icontains=clean_query) | Q(path__icontains=clean_query) | Q(fqdn__icontains=clean_query) | Q(protocol__icontains=clean_query))
if "engagement" in search_operator or search_operator is "":
engagements = watson.search(clean_query, models=(Engagement,))
if "language" in search_operator or search_operator is "":
languages = Languages.objects.filter(language__language__icontains=clean_query)
if "technology" in search_operator or search_operator is "":
app_analysis = App_Analysis.objects.filter(name__icontains=clean_query)
else:
if "finding" in search_operator or search_operator is "":
findings = watson.search(clean_query, models=(
Finding.objects.filter(
test__engagement__product__authorized_users__in=[
request.user]),))
if "template" in search_operator or search_operator is "":
finding_templates = watson.search(clean_query, models=(
Finding_Template.objects.filter(
authorized_users__in=[
request.user]),))
if "test" in search_operator or search_operator is "":
tests = watson.search(
clean_query,
models=(Test.objects.filter(
engagement__product__authorized_users__in=[
request.user]),))
if "product" in search_operator or search_operator is "":
products = watson.search(clean_query, models=(
Product.objects.filter(authorized_users__in=[
request.user]),))
if "tag" in search_operator or search_operator is "":
tagged_findings = TaggedItem.objects.get_by_model(
Finding.objects.filter(
test__engagement__product__authorized_users__in=[
request.user]), tags)
tagged_finding_templates = TaggedItem.objects.get_by_model(
Finding_Template.objects.filter(
authorized_users__in=[
request.user]), tags)
tagged_tests = TaggedItem.objects.get_by_model(
Test.objects.filter(
engagement__product__authorized_users__in=[
request.user]), tags)
tagged_products = TaggedItem.objects.get_by_model(
Product.objects.filter(
authorized_users__in=[request.user]), tags)
tagged_endpoints = TaggedItem.objects.get_by_model(
Endpoint.objects.filter(
product__authorized_users__in=[request.user]), tags)
tagged_engagements = TaggedItem.objects.get_by_model(
Engagement.objects.filter(
product__authorized_users__in=[request.user]), tags)
else:
form = SimpleSearchForm()
add_breadcrumb(title="Simple Search", top_level=True, request=request)
response = render(request, 'dojo/simple_search.html', {
'clean_query': clean_query,
'languages': languages,
'app_analysis': app_analysis,
'tests': tests,
'findings': findings,
'finding_templates': finding_templates,
'products': products,
'tagged_tests': tagged_tests,
'tagged_findings': tagged_findings,
'tagged_finding_templates': tagged_finding_templates,
'tagged_products': tagged_products,
'tagged_endpoints': tagged_endpoints,
'tagged_engagements': tagged_engagements,
'engagements': engagements,
'endpoints': endpoints,
'name': 'Simple Search',
'metric': False,
'user': request.user,
'form': form})
if cookie:
response.set_cookie("highlight", value=clean_query,
max_age=None, expires=None,
path='/', secure=True, httponly=False)
else:
response.delete_cookie("highlight", path='/')
return response
def get_queryset(self):
"""Returns the initial queryset."""
return watson.search(self.get_query(self.request))
def testRankingParamAbsentOnSearch(self):
self.assertRaises(AttributeError, lambda: watson.search("TITLE", ranking=False)[0].watson_rank)
