def query_dict_to_q(query_dict, dataset):
"""
Given a dict validated by query_api.schema.QuerySchema, return a Q instance
representing a single clause of the search to be composed.
"""
filter_type, type_tag, value, exclude = [
query_dict[k] for k in ['filter_type', 'type_tag', 'value', 'exclude']
]
modifiers = [modifier['name'] for modifier in query_dict['modifiers']]
(filter_type, value) = apply_transformations(
filter_type,
value,
modifiers,
TRANSFORMATIONS_DICT.get(dataset, DEFAULT_TRANSFORMATIONS),
)
query_data = {}
if filter_type == 'text_search':
query_data['longsearchablestring__type_tag'] = type_tag
query_data['longsearchablestring__searchable_value'] = SearchQuery(
value,
config='spanish',
)
else:
query_data['searchablestring__type_tag'] = type_tag
query_data[
f'searchablestring__value{FILTERS_DICT[filter_type]}'] = value
query = Q(**query_data)
if exclude:
return ~query
return query
def filter_by_text_search(
cls, search_text: str, queryset: Optional[QuerySet[T]] = None
) -> QuerySet[T]:
if queryset is None:
queryset = QuerySet(model=cls)
search_text = search_text.strip()
if not search_text:
raise ValueError(
"search_text cannot be empty string or only contain spaces."
)
pk_set = set(
SearchFeature.objects.filter(
content_type=ContentType.objects.get_for_model(cast(Model, cls))
)
.filter(
m.Q(search_vector=SearchQuery(search_text))
| m.Q(text_feature__icontains=search_text)
)
.values_list("object_id", flat=True)
)
return queryset.filter(pk__in=pk_set)
def post_search(request):
form = SearchForm()
query = SearchForm()
query = None
results = []
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
query = form.cleaned_data['query']
search_vector = SearchVector('title', weight='A') + SearchVector(
'body', weight='B')
search_query = SearchQuery(query)
#results=Post.published.annotate( search=SearchVector('title','body'),).filter(search=query)
results = Post.published.annotate(
search=search_vector,
rank=SearchRank(search_vector, search_query),
).filter(rank__gte=0.3).order_by('-rank')
print(results)
return render(request, 'post/search.html', {
'form': form,
'query': query,
'results': results
})
def get(self, request):
rank_min = 0.0
query = request.GET.get('q', None)
half_space = ''
query = query.replace(half_space, ' ')
vector = SearchVector('name') + SearchVector('english_name')
# vector = SearchVector('name','english_name')
vector2 = SearchVector('name')
search_query = SearchQuery(query, search_type='phrase')
# categories = Category.objects.annotate(rank=SearchRank(vector, search_query)).filter(
# rank__gt=rank_min).order_by('-rank')
categories = Category.objects.annotate(similarity=TrigramSimilarity('name', query)).filter(
similarity__gt=rank_min).order_by('-similarity')
companies = Company.objects.annotate(rank=SearchRank(vector, search_query)).filter(rank__gt=rank_min).order_by(
'-rank')
labels = ProductLabel.objects.annotate(rank=SearchRank(vector2, search_query)).filter(
rank__gt=rank_min).order_by('-rank')
return CustomJSONRenderer().render({
'categories': CategoryMenuSerializer(categories, pop=['all_chatrbazi', 'open_chatrbazi', 'company'],
many=True, context={'request': request}).data,
'companies': CompanyDetailSerializer(companies, many=True, pop=['description', 'product_company'],
context={'request': request}).data,
'tags': ProductLabelSerializer(labels, many=True, context={'request': request}).data,
})
def search(request):
section = request.GET.get('s')
phrase = request.GET.get('q')
query = SearchQuery(phrase)
items = []
context = {
'title': 'Search Results',
'query': phrase,
'section': section
}
set_context(request, context, True)
context['atevents'] = True
event_vector = SearchVector('title') + SearchVector('content')
items = Event.objects.annotate(
search=event_vector).filter(search=query, status=1).order_by('-created_on')
paginator = Paginator(items, 20) # Show 20 per page.
page_number = request.GET.get('page')
page_obj = paginator.get_page(page_number)
context['page_obj'] = page_obj
return render(request, 'home/search.html', context)
def post_search(request):
form = SearchForm()
query = None
results = []
print(request.GET)
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
query = form.cleaned_data['query']
search_vector = SearchVector('title', weight='A') + SearchVector(
'body', weight='B')
search_query = SearchQuery(query)
# results = Post.objects.annotate(
# search=search_vector,
# rank=SearchRank(search_vector, search_query)
# ).filter(rank__gte=0.3).order_by('-rank')
results = Post.objects.annotate(similarity=TrigramSimilarity(
'title',
query), ).filter(similarity__gt=0.3).order_by('-similarity')
return render(request, 'blog/post/search.html', {
'form': form,
'query': query,
'results': results
})
def post_search(request):
form = SearchForm()
query = None
results = []
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
query = form.cleaned_data['query']
search_vector = SearchVector('title', weight='A') +\
SearchVector('body', weight='B')
search_query = SearchQuery(query)
results = Post.published.annotate(similarity=TrigramSimilarity(
'title',
query), ).filter(similarity__gt=0.1).order_by('-similarity')
# results = Post.published.annotate(
# search=SearchVector('title','body'),
# ).filter(search=query)
return render(request, 'blog/post/search.html', {
'form': form,
'query': query,
'results': results
})
def search(self, queryset):
# TODO: it should take into account selected language. Check only romanian for now.
query = self.request.GET.get("q")
if not query:
return queryset
if hasattr(self, "search_cache") and query in self.search_cache:
return self.search_cache[query]
search_query = SearchQuery(query, config="romanian_unaccent")
vector = SearchVector("name", weight="A", config="romanian_unaccent")
result = (queryset.annotate(
rank=SearchRank(vector, search_query),
similarity=TrigramSimilarity("name", query),
).filter(Q(rank__gte=0.3)
| Q(similarity__gt=0.3)).order_by("name").distinct("name"))
if not hasattr(self, "search_cache"):
self.search_cache = {}
self.search_cache[query] = result
return result
def post_search(request):
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
query = form.cleaned_data['query']
# results = Post.published.annotate(search = SearchVector('title','body')).filter(search=query)
search_vector = SearchVector('title', 'body')
search_query = SearchQuery(query)
results = Post.published.annotate(
search=search_vector,
rank=SearchRank(search_vector, search_query)).filter(
search=search_query).order_by('-rank')
else:
form = SearchForm()
query = None
results = []
return render(request, 'blog/post/search.html', {
'form': form,
'query': query,
'results': results
})
def search_tickets(request):
""" Display results matching'q' search term
Arguments:
request = HttpRequest object
"""
q = request.GET.get('q')
if q:
query = SearchQuery(q)
tickets = Ticket.objects.annotate(search=SearchVector(
'title', 'description'), ).filter(search=query)
reported_tickets = tickets.filter(
Q(state='REPORTED') | Q(state='REQUESTED'))
working_tickets = tickets.filter(state='IN PROGRESS')
completed_tickets = tickets.filter(state='COMPLETED')
context = {'ticket_view': 'search_results'}
return render(
request, "ticket_overview.html", {
"tickets": tickets,
"reported_tickets": reported_tickets,
"working_tickets": working_tickets,
"completed_tickets": completed_tickets,
"search_query": q
}, context)
def filter_search(self, queryset, name, value):
vector = SearchVector(
"name",
"short_description",
"owner__email",
"slug",
"related_work",
"addon_experiment_id",
"pref_key",
"public_name",
"public_description",
"objectives",
"analysis",
"analysis_owner",
"engineering_owner",
"bugzilla_id",
"normandy_slug",
)
query = SearchQuery(value)
return (queryset.annotate(
rank=SearchRank(vector, query),
search=vector).filter(search=value).order_by("-rank"))
def search_threads(request, query, visible_threads):
search_query = SearchQuery(
filter_search(query),
config=settings.MISAGO_SEARCH_CONFIG,
)
search_vector = SearchVector(
'search_document',
config=settings.MISAGO_SEARCH_CONFIG,
)
queryset = Post.objects.filter(
is_event=False,
is_hidden=False,
is_unapproved=False,
thread_id__in=visible_threads.values('id'),
search_vector=search_query,
)
if queryset[:HITS_CEILING + 1].count() > HITS_CEILING:
queryset = queryset.order_by('-id')[:HITS_CEILING]
return Post.objects.filter(id__in=queryset.values('id'), ).annotate(
rank=SearchRank(search_vector,
search_query), ).order_by('-rank', '-id')
def filter_visualisations(query, access, source, user=None):
search = SearchVector('name', weight='A', config='english') + SearchVector(
'short_description', weight='B', config='english')
search_query = SearchQuery(query, config='english')
if user and user.has_perm(
dataset_type_to_manage_unpublished_permission_codename(
DataSetType.VISUALISATION.value)):
published_filter = Q()
else:
published_filter = Q(published=True)
visualisations = VisualisationCatalogueItem.objects.filter(
published_filter).annotate(search=search,
search_rank=SearchRank(
search, search_query))
if query:
visualisations = visualisations.filter(search=search_query)
if user and access:
access_filter = (Q(user_access_type='REQUIRES_AUTHENTICATION')
&
(Q(visualisationuserpermission__user=user)
| Q(visualisationuserpermission__isnull=True))) | Q(
user_access_type='REQUIRES_AUTHORIZATION',
visualisationuserpermission__user=user,
)
visualisations = visualisations.filter(access_filter)
if source:
visualisations = visualisations.filter(
visualisation_template__datasetapplicationtemplatepermission__dataset__source_tags__in
=source)
return visualisations
def post_search(request):
form = SearchForm()
query = None
results = []
if "query" in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
query = form.cleaned_data["query"]
search_vector = SearchVector("title", weight="A") + SearchVector(
"body", weight="B"
)
search_query = SearchQuery(query)
results = (
Post.objects.annotate(
search=search_vector, rank=SearchRank(search_vector, search_query),
)
.filter(rank__gte=0.3)
.order_by("-rank")
)
return render(
request,
"blog/post/search.html",
{"form": form, "query": query, "results": results},
)
def category_list(request, category, tag_slug=None):
news_list = News.objects.filter(
category__name=category).order_by('-created')
category = Category.objects.get(name=category)
tag = None
if tag_slug:
tag = get_object_or_404(Tag, slug=tag_slug)
news_list.filter(tags__in=[tag])
query = None
results = []
form = SearchForm()
if 'query' in request.GET:
form = SearchForm(request.GET)
if form.is_valid():
query = form.cleaned_data['query']
search_vector = SearchVector('title', 'body')
search_query = SearchQuery(query)
results = News.objects.annotate(search=search_vector, rank=SearchRank(search_vector, search_query)) \
.filter(search=search_query).order_by('-rank')
paginator = Paginator(news_list, 5)
page_number = request.GET.get('page')
page_obj = paginator.get_page(page_number)
context = {
'news_list': news_list,
'page_obj': page_obj,
'tag': tag,
'category': category,
'results': results,
'query': query,
'form': form,
}
return render(request, 'news/category_list.html', context=context)
def news_admin_search(request):
if not request.method == 'POST':
if 'search-post' in request.session:
request.POST = request.session['search-post']
request.method = 'POST'
if request.method == 'POST':
request.session['search-post'] = request.POST
keywords = request.POST.get('search_input', '')
vector = SearchVector('text', 'title')
query = SearchQuery(keywords)
news_list = NewsData.objects.annotate(rank=SearchRank(vector, query)).filter(rank__gte=0.04).order_by('-rank').order_by("-pub_date")
#news_list = NewsData.objects.annotate(similarity=TrigramSimilarity('text', keywords),).filter(similarity__gt=0.3).order_by('-similarity')
paginator = Paginator(news_list, 10) # Show 25 contacts per page
page = request.GET.get('page')
try:
news = paginator.page(page)
except PageNotAnInteger:
news = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
news = paginator.page(paginator.num_pages)
return render(request, 'news.html', {'news': news })
def post_search(request):
se_form = SearchForm() #initialize the form
query = None # set the seacrh query to none
results = [] # set results to an empty list
if 'find_posts' in request.GET: #check for query string in the request.GET dictionary
se_form = SearchForm(
request.GET
) #When submitted,instantiate the form with submitted GET data and check for validity
if se_form.is_valid():
query = se_form.cleaned_data['find_posts']
"""
In the search we could use a simple filter,SearchVector or a more complex TrigramSimilarity,which uses weights
"""
search_vector = SearchVector('title', weight='A') + SearchVector(
'body', weight='B')
search_query = SearchQuery(query)
results = Post.objects.annotate(
similarity=TrigramSimilarity('title', query)).filter(
similarity__gt=0.3).order_by('-similarity')
return render(request, 'blog/base.html', {
'se_form': se_form,
'query': query,
'results': results
})
def search(self, query_text, release):
"""Use full-text search to return documents matching query_text."""
query_text = query_text.strip()
if query_text:
search_query = SearchQuery(query_text, config=models.F('config'))
search_rank = SearchRank(models.F('search'), search_query)
similarity = TrigramSimilarity('title', query_text)
return self.get_queryset().prefetch_related(
Prefetch('release',
queryset=DocumentRelease.objects.only(
'lang', 'release')),
Prefetch('release__release',
queryset=Release.objects.only('version')),
).filter(
release_id=release.id,
search=search_query,
).annotate(rank=search_rank + similarity).order_by('-rank').only(
'title',
'path',
'metadata',
'release',
)
else:
return self.get_queryset().none()
def get(self, request, *args, **kwargs):
query = request.query_params
field_names = [field.name for field in Article._meta.fields]
query_key = list(query.keys())[0]
if not query_key in field_names:
raise exceptions.ParseError(
f"Please check the query parameter. Searching on field {query_key} is not possible."
)
if request.user.is_superuser:
queryset = Article.objects.all()
else:
queryset = Article.objects.filter(user=request.user)
if query_key == 'categories':
articles = queryset.filter(categories__icontains=query[query_key])
else:
articles = queryset.annotate(rank=SearchRank(
SearchVector(query_key), SearchQuery(
query[query_key]))).order_by('-rank')
return Response([{
"title":
article.title,
"body":
article.body,
"summary":
article.summary,
"categories":
article.categories,
"file_download":
reverse("article:article-download", kwargs={'id': article.id}),
"url":
reverse("article:article-detail", kwargs={'id': article.id}),
} for article in articles])
def get_queryset(self):
search_type = self.request.GET.get("type", "mentor")
if search_type == "mentee":
all_qualified = self.queryset.filter(
experience__exp_type=Experience.Type.WANT_HELP,
looking_for_mentors=True).exclude(user=self.request.user)
if search_type == "mentor":
all_qualified = self.queryset.filter(
experience__exp_type=Experience.Type.CAN_HELP,
looking_for_mentees=True).exclude(user=self.request.user)
query_text = self.request.GET.get("q", "")
if query_text is not "":
search_vector = SearchVector(
"user__first_name",
"user__last_name",
"bio",
"experience__skill__skill",
)
search_query = SearchQuery(query_text.replace(" ", " | "),
search_type="raw")
search_results = (all_qualified.annotate(
search=search_vector,
rank=SearchRank(search_vector, search_query),
).filter(search=search_query, id=OuterRef("id")).distinct("id"))
ranked = (Profile.objects.annotate(
rank=Subquery(search_results.values("rank"))).filter(
id__in=Subquery(search_results.values("id"))).order_by(
"-rank"))
search_results = ranked
else:
search_results = all_qualified.distinct("id")
return search_results
def search_podcasts(query):
""" Search for podcasts according to 'query' """
logger.debug('Searching for "{query}" podcasts"', query=query)
query = SearchQuery(query)
results = Podcast.objects\
.annotate(
rank=SearchRank(F('search_vector'), query)
)\
.annotate(
order=ExpressionWrapper(
F('rank') * F('subscribers'),
output_field=FloatField())
)\
.filter(rank__gte=SEARCH_CUTOFF)\
.order_by('-order')
logger.debug('Found {count} podcasts for "{query}"',
count=len(results),
query=query)
return results
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
if 'page' in kwargs:
canvas = Canvas.objects.filter(pid=kwargs['page']).first()
else:
canvas = Canvas.objects.filter(position='1').first()
context['page'] = canvas
manifest = Manifest.objects.filter(pid=kwargs['volume']).first()
context['volume'] = manifest
context['collectionlink'] = Page.objects.type(CollectionsPage).first()
context['user_annotation_page_count'] = UserAnnotation.objects.filter(
owner_id=self.request.user.id).filter(
canvas__id=canvas.id).count()
context['user_annotation_count'] = UserAnnotation.objects.filter(
owner_id=self.request.user.id).filter(
canvas__manifest__id=manifest.id).count()
context['mirador_url'] = settings.MIRADOR_URL
qs = Annotation.objects.all()
qs2 = UserAnnotation.objects.all()
try:
search_string = self.request.GET['q']
search_type = self.request.GET['type']
print(search_type)
search_strings = self.request.GET['q'].split()
if search_strings:
if search_type == 'partial':
qq = Q()
query = SearchQuery('')
for search_string in search_strings:
query = query | SearchQuery(search_string)
qq |= Q(content__icontains=search_string)
print(query)
vector = SearchVector('content')
qs = qs.filter(qq).filter(
canvas__manifest__label=manifest.label)
# qs = qs.annotate(search=vector).filter(search=query).filter(canvas__manifest__label=manifest.label)
# qs = qs.annotate(rank=SearchRank(vector, query)).order_by('-rank')
qs = qs.values('canvas__position',
'canvas__manifest__label',
'canvas__pid').annotate(
Count('canvas__position')).order_by(
'canvas__position')
qs1 = qs.exclude(resource_type='dctypes:Text').distinct()
qs2 = qs2.annotate(search=vector).filter(
search=query).filter(
canvas__manifest__label=manifest.label)
qs2 = qs2.annotate(
rank=SearchRank(vector, query)).order_by('-rank')
qs2 = qs2.filter(owner_id=self.request.user.id).distinct()
elif search_type == 'exact':
qq = Q()
query = SearchQuery('')
for search_string in search_strings:
query = query | SearchQuery(search_string)
qq |= Q(content__contains=search_string)
print(query)
vector = SearchVector('content')
# qs = qs.filter(qq).filter(canvas__manifest__label=manifest.label)
qs = qs.annotate(search=vector).filter(
search=query).filter(
canvas__manifest__label=manifest.label)
# qs = qs.annotate(rank=SearchRank(vector, query)).order_by('-rank')
qs = qs.values('canvas__position',
'canvas__manifest__label',
'canvas__pid').annotate(
Count('canvas__position')).order_by(
'canvas__position')
qs1 = qs.exclude(resource_type='dctypes:Text').distinct()
qs2 = qs2.annotate(search=vector).filter(
search=query).filter(
canvas__manifest__label=manifest.label)
qs2 = qs2.annotate(
rank=SearchRank(vector, query)).order_by('-rank')
qs2 = qs2.filter(owner_id=self.request.user.id).distinct()
else:
qs1 = ''
qs2 = ''
context['qs1'] = qs1
context['qs2'] = qs2
except MultiValueDictKeyError:
q = ''
return context
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
collection = self.request.GET.get('collection', None)
COLSET = Collection.objects.values_list('pid', flat=True)
COL_OPTIONS = list(COLSET)
COL_LIST = Collection.objects.values(
'pid', 'label').order_by('label').distinct('label')
collection_url_params = self.request.GET.copy()
qs = self.get_queryset()
try:
search_string = self.request.GET['q']
search_type = self.request.GET['type']
search_strings = self.request.GET['q'].split()
if search_strings:
if search_type == 'partial':
qq = Q()
query = SearchQuery('')
for search_string in search_strings:
query = query | SearchQuery(search_string)
qq |= Q(
canvas__annotation__content__icontains=search_string
)
print(query)
# vector = SearchVector('canvas__annotation__content')
# qs1 = qs.annotate(search=vector).filter(search=query)
# qs1 = qs1.annotate(rank=SearchRank(vector, query)).values('pid', 'label', 'author', 'published_date', 'created_at').annotate(pidcount = Count('pid')).order_by('-pidcount')
qs1 = qs.filter(qq)
qs1 = qs1.values(
'pid', 'label', 'author', 'published_date',
'created_at').annotate(
pidcount=Count('pid')).order_by('-pidcount')
vector2 = SearchVector('label', weight='A') + SearchVector(
'author', weight='B') + SearchVector('summary',
weight='C')
qs3 = qs.annotate(search=vector2).filter(search=query)
qs3 = qs3.annotate(rank=SearchRank(vector2, query)).values(
'pid', 'label', 'author', 'published_date',
'created_at').order_by('-rank')
qs2 = qs.values(
'canvas__pid', 'pid',
'canvas__IIIF_IMAGE_SERVER_BASE__IIIF_IMAGE_SERVER_BASE'
).order_by('pid').distinct('pid')
if collection not in COL_OPTIONS:
collection = None
if collection is not None:
qs1 = qs1.filter(collections__pid=collection)
qs3 = qs3.filter(collections__pid=collection)
if 'collection' in collection_url_params:
del collection_url_params['collection']
# qs1 = qs.exclude(resource_type='dctypes:Text').distinct()
# qs2 = qs2.annotate(search=vector).filter(search=query).filter(canvas__manifest__label=manifest.label)
# qs2 = qs2.annotate(rank=SearchRank(vector, query)).order_by('-rank')
# qs2 = qs2.filter(owner_id=self.request.user.id).distinct()
elif search_type == 'exact':
qq = Q()
query = SearchQuery('')
for search_string in search_strings:
query = query | SearchQuery(search_string)
qq |= Q(
canvas__annotation__content__exact=search_string)
print(query)
vector = SearchVector('canvas__annotation__content')
qs1 = qs.annotate(search=vector).filter(search=query)
qs1 = qs1.annotate(rank=SearchRank(vector, query)).values(
'pid', 'label', 'author', 'published_date',
'created_at').annotate(
pidcount=Count('pid')).order_by('-pidcount')
# qs1 = qs.filter(qq)
# qs1 = qs1.values('pid', 'label', 'author', 'published_date', 'created_at').annotate(pidcount = Count('pid')).order_by('-pidcount')
vector2 = SearchVector('label', weight='A') + SearchVector(
'author', weight='B') + SearchVector('summary',
weight='C')
qs3 = qs.annotate(search=vector2).filter(search=query)
qs3 = qs3.annotate(rank=SearchRank(vector2, query)).values(
'pid', 'label', 'author', 'published_date',
'created_at').order_by('-rank')
qs2 = qs.values(
'canvas__pid', 'pid',
'canvas__IIIF_IMAGE_SERVER_BASE__IIIF_IMAGE_SERVER_BASE'
).order_by('pid').distinct('pid')
if collection not in COL_OPTIONS:
collection = None
if collection is not None:
qs1 = qs1.filter(collections__pid=collection)
qs3 = qs3.filter(collections__pid=collection)
if 'collection' in collection_url_params:
del collection_url_params['collection']
# qs1 = qs.exclude(resource_type='dctypes:Text').distinct()
# qs2 = qs2.annotate(search=vector).filter(search=query).filter(canvas__manifest__label=manifest.label)
# qs2 = qs2.annotate(rank=SearchRank(vector, query)).order_by('-rank')
# qs2 = qs2.filter(owner_id=self.request.user.id).distinct()
else:
search_string = ''
search_strings = ''
qs1 = ''
qs2 = ''
qs3 = ''
context['qs1'] = qs1
context['qs2'] = qs2
context['qs3'] = qs3
# qs1 = ''
# qs2 = ''
# context['qs1'] = qs1
# context['qs2'] = qs2
except MultiValueDictKeyError:
q = ''
search_string = ''
search_strings = ''
context['volumes'] = qs.all
# context['user_annotation'] = UserAnnotation.objects.filter(owner_id=self.request.user.id)
# annocount_list = []
# canvaslist = []
# for volume in qs:
# user_annotation_count = UserAnnotation.objects.filter(owner_id=self.request.user.id).filter(canvas__manifest__id=volume.id).count()
# annocount_list.append({volume.pid: user_annotation_count})
# context['user_annotation_count'] = annocount_list
# print(volume.pid)
# print(user_annotation_count)
# canvasquery = Canvas.objects.filter(is_starting_page=1).filter(manifest__id=volume.id)
# canvasquery2 = list(canvasquery)
# canvaslist.append({volume.pid: canvasquery2})
# context['firstthumbnail'] = canvaslist
# value = 0
# context['value'] = value
annocount_list = []
canvaslist = []
for volume in qs:
user_annotation_count = UserAnnotation.objects.filter(
owner_id=self.request.user.id).filter(
canvas__manifest__id=volume.id).count()
annocount_list.append({volume.pid: user_annotation_count})
context['user_annotation_count'] = annocount_list
canvasquery = Canvas.objects.filter(is_starting_page=1).filter(
manifest__id=volume.id)
canvasquery2 = list(canvasquery)
canvaslist.append({volume.pid: canvasquery2})
context['firstthumbnail'] = canvaslist
context.update({
'collection_url_params':
urlencode(collection_url_params),
'collection':
collection,
'COL_OPTIONS':
COL_OPTIONS,
'COL_LIST':
COL_LIST,
'search_string':
search_string,
'search_strings':
search_strings
})
return context
def test_str(self):
tests = (
(~SearchQuery('a'), '~SearchQuery(Value(a))'),
(
(SearchQuery('a') | SearchQuery('b')) &
(SearchQuery('c') | SearchQuery('d')),
'((SearchQuery(Value(a)) || SearchQuery(Value(b))) && '
'(SearchQuery(Value(c)) || SearchQuery(Value(d))))',
),
(
SearchQuery('a') & (SearchQuery('b') | SearchQuery('c')),
'(SearchQuery(Value(a)) && (SearchQuery(Value(b)) || '
'SearchQuery(Value(c))))',
),
((SearchQuery('a') | SearchQuery('b')) & SearchQuery('c'),
'((SearchQuery(Value(a)) || SearchQuery(Value(b))) && '
'SearchQuery(Value(c)))'),
(
SearchQuery('a') & (SearchQuery('b') &
(SearchQuery('c') | SearchQuery('d'))),
'(SearchQuery(Value(a)) && (SearchQuery(Value(b)) && '
'(SearchQuery(Value(c)) || SearchQuery(Value(d)))))',
),
)
for query, expected_str in tests:
with self.subTest(query=query):
self.assertEqual(str(query), expected_str)
def search_recipes(request, queryset, params):
if request.user.is_authenticated:
search_prefs = request.user.searchpreference
else:
search_prefs = SearchPreference()
search_string = params.get('query', '').strip()
search_rating = int(params.get('rating', 0))
search_keywords = params.getlist('keywords', [])
search_foods = params.getlist('foods', [])
search_books = params.getlist('books', [])
search_steps = params.getlist('steps', [])
search_units = params.get('units', None)
# TODO I think default behavior should be 'AND' which is how most sites operate with facet/filters based on results
search_keywords_or = str2bool(params.get('keywords_or', True))
search_foods_or = str2bool(params.get('foods_or', True))
search_books_or = str2bool(params.get('books_or', True))
search_internal = str2bool(params.get('internal', False))
search_random = str2bool(params.get('random', False))
search_new = str2bool(params.get('new', False))
search_last_viewed = int(params.get('last_viewed', 0))
orderby = []
# only sort by recent not otherwise filtering/sorting
if search_last_viewed > 0:
last_viewed_recipes = ViewLog.objects.filter(
created_by=request.user, space=request.space,
created_at__gte=timezone.now() - timedelta(days=14) # TODO make recent days a setting
).order_by('-pk').values_list('recipe__pk', flat=True)
last_viewed_recipes = list(dict.fromkeys(last_viewed_recipes))[:search_last_viewed] # removes duplicates from list prior to slicing
# return queryset.annotate(last_view=Max('viewlog__pk')).annotate(new=Case(When(pk__in=last_viewed_recipes, then=('last_view')), default=Value(0))).filter(new__gt=0).order_by('-new')
# queryset that only annotates most recent view (higher pk = lastest view)
queryset = queryset.annotate(recent=Coalesce(Max(Case(When(viewlog__created_by=request.user, then='viewlog__pk'))), Value(0)))
orderby += ['-recent']
# TODO create setting for default ordering - most cooked, rating,
# TODO create options for live sorting
# TODO make days of new recipe a setting
if search_new:
queryset = (
queryset.annotate(new_recipe=Case(
When(created_at__gte=(timezone.now() - timedelta(days=7)), then=('pk')), default=Value(0), ))
)
# only sort by new recipes if not otherwise filtering/sorting
orderby += ['-new_recipe']
search_type = search_prefs.search or 'plain'
if len(search_string) > 0:
unaccent_include = search_prefs.unaccent.values_list('field', flat=True)
icontains_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.icontains.values_list('field', flat=True)]
istartswith_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.istartswith.values_list('field', flat=True)]
trigram_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.trigram.values_list('field', flat=True)]
fulltext_include = search_prefs.fulltext.values_list('field', flat=True) # fulltext doesn't use field name directly
# if no filters are configured use name__icontains as default
if len(icontains_include) + len(istartswith_include) + len(trigram_include) + len(fulltext_include) == 0:
filters = [Q(**{"name__icontains": search_string})]
else:
filters = []
# dynamically build array of filters that will be applied
for f in icontains_include:
filters += [Q(**{"%s__icontains" % f: search_string})]
for f in istartswith_include:
filters += [Q(**{"%s__istartswith" % f: search_string})]
if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
language = DICTIONARY.get(translation.get_language(), 'simple')
# django full text search https://docs.djangoproject.com/en/3.2/ref/contrib/postgres/search/#searchquery
# TODO can options install this extension to further enhance search query language https://github.com/caub/pg-tsquery
# trigram breaks full text search 'websearch' and 'raw' capabilities and will be ignored if those methods are chosen
if search_type in ['websearch', 'raw']:
search_trigram = False
else:
search_trigram = True
search_query = SearchQuery(
search_string,
search_type=search_type,
config=language,
)
# iterate through fields to use in trigrams generating a single trigram
if search_trigram and len(trigram_include) > 0:
trigram = None
for f in trigram_include:
if trigram:
trigram += TrigramSimilarity(f, search_string)
else:
trigram = TrigramSimilarity(f, search_string)
queryset = queryset.annotate(similarity=trigram)
filters += [Q(similarity__gt=search_prefs.trigram_threshold)]
if 'name' in fulltext_include:
filters += [Q(name_search_vector=search_query)]
if 'description' in fulltext_include:
filters += [Q(desc_search_vector=search_query)]
if 'instructions' in fulltext_include:
filters += [Q(steps__search_vector=search_query)]
if 'keywords' in fulltext_include:
filters += [Q(keywords__in=Subquery(Keyword.objects.filter(name__search=search_query).values_list('id', flat=True)))]
if 'foods' in fulltext_include:
filters += [Q(steps__ingredients__food__in=Subquery(Food.objects.filter(name__search=search_query).values_list('id', flat=True)))]
query_filter = None
for f in filters:
if query_filter:
query_filter |= f
else:
query_filter = f
# TODO add order by user settings - only do search rank and annotation if rank order is configured
search_rank = (
SearchRank('name_search_vector', search_query, cover_density=True)
+ SearchRank('desc_search_vector', search_query, cover_density=True)
+ SearchRank('steps__search_vector', search_query, cover_density=True)
)
queryset = queryset.filter(query_filter).annotate(rank=search_rank)
orderby += ['-rank']
else:
queryset = queryset.filter(name__icontains=search_string)
if len(search_keywords) > 0:
if search_keywords_or:
# TODO creating setting to include descendants of keywords a setting
# for kw in Keyword.objects.filter(pk__in=search_keywords):
# search_keywords += list(kw.get_descendants().values_list('pk', flat=True))
queryset = queryset.filter(keywords__id__in=search_keywords)
else:
# when performing an 'and' search returned recipes should include a parent OR any of its descedants
# AND other keywords selected so filters are appended using keyword__id__in the list of keywords and descendants
for kw in Keyword.objects.filter(pk__in=search_keywords):
queryset = queryset.filter(keywords__id__in=list(kw.get_descendants_and_self().values_list('pk', flat=True)))
if len(search_foods) > 0:
if search_foods_or:
# TODO creating setting to include descendants of food a setting
queryset = queryset.filter(steps__ingredients__food__id__in=search_foods)
else:
# when performing an 'and' search returned recipes should include a parent OR any of its descedants
# AND other foods selected so filters are appended using steps__ingredients__food__id__in the list of foods and descendants
for fd in Food.objects.filter(pk__in=search_foods):
queryset = queryset.filter(steps__ingredients__food__id__in=list(fd.get_descendants_and_self().values_list('pk', flat=True)))
if len(search_books) > 0:
if search_books_or:
queryset = queryset.filter(recipebookentry__book__id__in=search_books)
else:
for k in search_books:
queryset = queryset.filter(recipebookentry__book__id=k)
if search_rating:
queryset = queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
if search_rating == -1:
queryset = queryset.filter(rating=0)
else:
queryset = queryset.filter(rating__gte=search_rating)
# probably only useful in Unit list view, so keeping it simple
if search_units:
queryset = queryset.filter(steps__ingredients__unit__id=search_units)
# probably only useful in Unit list view, so keeping it simple
if search_steps:
queryset = queryset.filter(steps__id__in=search_steps)
if search_internal:
queryset = queryset.filter(internal=True)
queryset = queryset.distinct()
if search_random:
queryset = queryset.order_by("?")
else:
queryset = queryset.order_by(*orderby)
return queryset
def test_ranking_chaining(self):
searched = Line.objects.filter(character=self.minstrel).annotate(
rank=SearchRank(
SearchVector('dialogue'),
SearchQuery('brave sir robin')), ).filter(rank__gt=0.3)
self.assertSequenceEqual(searched, [self.verse0])
def test_combine_raw_phrase(self):
searched = Line.objects.filter(dialogue__search=(
SearchQuery('burn:*', search_type='raw', config='simple')
| SearchQuery('rode forth from Camelot', search_type='phrase')))
self.assertCountEqual(searched,
[self.verse0, self.verse1, self.verse2])
def test_combined_configs(self):
searched = Line.objects.filter(
dialogue__search=(SearchQuery('nostrils', config='simple')
& SearchQuery('bowels', config='simple')), )
self.assertSequenceEqual(searched, [self.verse2])
def test_combine_different_configs(self):
searched = Line.objects.filter(
dialogue__search=(SearchQuery('cadeau', config='french')
| SearchQuery('nostrils', config='english')))
self.assertCountEqual(searched, [self.french, self.verse2])
def test_query_invert(self):
searched = Line.objects.filter(
character=self.minstrel, dialogue__search=~SearchQuery('kneecaps'))
self.assertCountEqual(searched, [self.verse0, self.verse2])
