def test_search_vector_index(self):
"""SearchVector generates IMMUTABLE SQL in order to be indexable."""
# This test should be moved to test_indexes and use a functional
# index instead once support lands (see #26167).
query = Line.objects.all().query
resolved = SearchVector('id', 'dialogue', config='english').resolve_expression(query)
compiler = query.get_compiler(connection.alias)
sql, params = resolved.as_sql(compiler, connection)
# Indexed function must be IMMUTABLE.
with connection.cursor() as cursor:
cursor.execute(
'CREATE INDEX search_vector_index ON %s USING GIN (%s)' % (Line._meta.db_table, sql),
params,
)
def get(self, request, format=None):
id = self.request.query_params.get('id', None)
category = list(
filter(None,
self.request.query_params.get('category', '').split(",")))
tag = list(
map(
int,
filter(None,
self.request.query_params.get('tag', '').split(","))))
search = self.request.query_params.get('search', None)
similar = self.request.query_params.get('similar', None)
next = self.request.query_params.get('next', None)
prev = self.request.query_params.get('prev', None)
page = self.request.query_params.get('page', 1)
news = News.objects.filter(visibility=True).order_by('-id')
if search:
news = news.annotate(search=SearchVector(
'headline', 'body', 'newsAgency', 'category__name',
'tags__name'), ).filter(search=search)
if similar: news = news.get(id=similar).etags.similar_objects()
if category:
if "independent" in category:
news = news.filter(independent=True)
category.remove("independent")
else:
news = news.filter(independent=False)
if "preference" in category:
if request.user.is_authenticated:
mycategory = MyCategory.objects.filter(user=request.user)
if mycategory:
cats = MyCategorySerializer(
mycategory[0]).data["categorys"]
news = news.filter(category__in=cats)
else:
category.append("trending")
else:
return response.Response(
{"message": "User is not Authenticated"})
category.remove("preference")
if "trending" in category:
time_threshold = datetime.now() - timedelta(days=5)
iso, hr = datetime.now(
timezone.utc).isocalendar(), datetime.now().hour
news = news.filter(created_at__gt=time_threshold).annotate(
popularity=(F('pos') - F('neg') + 0.01) /
((iso[0] - Extract(F('created_at'), 'iso_year')) * 364 *
24 +
(iso[1] - Extract(F('created_at'), 'week')) * 7 * 24 +
(iso[2] - Extract(F('created_at'), 'week_day') % 7 - 1) *
24 + hr - Extract(F('created_at'), 'hour') +
1.01)).order_by('-popularity')
category.remove("trending")
elif category:
news = news.filter(category__name__in=category)
else:
news = news.filter(independent=False)
data = {}
news = news.distinct()
if id:
if next and next != "0":
news1 = news.filter(id__gt=int(id)).order_by('id')[0:int(next)]
serializer = NewsSerializer(news1,
many=True,
context={'request': request})
data['next'] = serializer.data
if prev and prev != "0":
news2 = news.filter(
id__lt=int(id)).order_by('-id')[0:int(prev)]
serializer = NewsSerializer(news2,
many=True,
context={'request': request})
data['prev'] = serializer.data
if not next or next == "0": data['next'] = data['prev']
else: data['prev'] = data['next']
else:
serializer = NewsSerializer(Paginator(news, 20).page(page),
many=True,
context={'request': request})
data['next'] = data['prev'] = serializer.data
return response.Response(data)
def search_page(request):
"""Page with search results."""
def _articles_filter(name):
"""Delete articles in product names
Arguments:
name {str} -- Name or generic name of product
"""
exclude = ("de", "des", "au", "aux", 'en', "le", "la", "les", "et",
"un", "une", "du", "à")
produit = name.lower()
produit = produit.split(" ")
for e in exclude:
for x in produit:
if x in e:
try:
produit.remove(e)
except:
return x
keyword = (" ".join(produit[0:5]))
keyword = keyword.replace(",", "")
return keyword
query = request.GET.get('query')
URL = 'https://fr.openfoodfacts.org/cgi/search.pl?'
PARAMETERS = {
'action': 'process',
'search_terms': query,
'sort_by': 'unique_scans_n',
'axis_x': 'energy',
'axis_y': 'products_n',
'page_size': '1',
'page': '1',
'json': '1'
}
r = requests.get(URL, params=PARAMETERS)
requested_product = r.json()
try:
substitution_products = Product.objects.annotate(
search=SearchVector('product_name'), ).filter(
search=_articles_filter(query))
substitution_products = substitution_products.order_by(
'nutrition_grades')
if not substitution_products:
substitution_products = Product.objects.annotate(
search=SearchVector('product_name'), ).filter(
search=_articles_filter(requested_product['products'][0]
['product_name']))
substitution_products = substitution_products.order_by(
'nutrition_grades')
if not substitution_products:
substitution_products = Product.objects.annotate(
search=SearchVector('product_name'), ).filter(
search=_articles_filter(requested_product['products'][0]
['generic_name_fr']))
substitution_products = substitution_products.order_by(
'nutrition_grades')
except:
raise Http404
bookmark_id = list()
if request.user.is_authenticated:
qs_bookmark = Bookmark.objects.filter(user=request.user)
for i in qs_bookmark:
bookmark_id.append(i.bookmark.id)
if request.method == 'POST':
request_post = request.POST.get('bookmark_product_code')
Bookmark.objects.create(user=request.user, bookmark_id=request_post)
return HttpResponseRedirect(request.get_full_path())
# PAGINATION
# https://docs.djangoproject.com/fr/2.2/topics/pagination/
paginator = Paginator(substitution_products, 9) # 9 products by page
page = request.GET.get('page')
try:
substitution_products = paginator.page(page)
except PageNotAnInteger:
substitution_products = paginator.page(1)
except EmptyPage:
substitution_products = paginator.page(paginator.num_pages)
# /PAGINATION
template_name = 'app/search.html'
context = {
'query': query,
'img': IMG,
'requested_title': f"1er résultat Open Food Facts pour : {query}",
'requested_product_name':
requested_product['products'][0]['product_name'],
'requested_product_brands': requested_product['products'][0]['brands'],
'requested_product_quantity':
requested_product['products'][0]['quantity'],
'requested_product_image':
requested_product['products'][0]['image_url'],
'products': substitution_products,
'paginate': True,
'bookmarks': bookmark_id,
}
return render(request, template_name, context)
def update_search_index(self):
self.update(search_data=SearchVector(*self.model.extra['search']))
return news_query
def orm_get_news_item(news_id: int) -> NewsItem:
news_query = NewsItem.objects.filter(id=news_id)
return news_query[0] if len(news_query) > 0 else None
def orm_get_knowledge_base_items():
news_query = NewsItem.objects.filter(knowledge_base_article=True)
return news_query
_news_title_search_vector = SearchVector('title')
def orm_news_title_fts(search: str):
"""full text search on title field of news items"""
news_query = NewsItem.objects.annotate(
search=_news_title_search_vector).filter(search=SearchQuery(search))
return news_query
_news_text_search_vector = SearchVector('text')
def orm_news_text_fts(search: str):
"""full text search on text field of news items"""
news_query = NewsItem.objects.annotate(
def get_search_vector(self):
return (
SearchVector('name', weight='A', config=SEARCH_LANG) +
SearchVector('active_ingredient', weight='B', config=SEARCH_LANG) +
SearchVector('medical_indication', weight='C', config=SEARCH_LANG)
)
def create_q_from_filter(self, filt):
if "combine_method" in filt:
return self.create_q_from_filter_list(filt["filters"],
filt["combine_method"])
else:
q_kwargs = {}
field = filt["field"]
negate = False
if "not_" == filt["operation"][:4]:
negate = True
operation = FilterGenerator.operators[filt["operation"][4:]]
else:
operation = FilterGenerator.operators[filt["operation"]]
value = filt["value"]
value_format = None
if "value_format" in filt:
value_format = filt["value_format"]
# Special multi-field case for full-text search
if isinstance(field, list) and operation == "__search":
# We create the search vector and attach it to this object
sv = SearchVector(*field)
self.search_vectors.append(sv)
# Our Q object is simpler now
q_kwargs["search"] = value
# Return our Q and skip the rest
if negate:
return ~Q(**q_kwargs)
return Q(**q_kwargs)
# Handle special operations
if operation == "fy":
fy = FiscalYear(value)
if negate:
return ~fy.get_filter_object(field)
return fy.get_filter_object(field)
if operation == "range_intersect":
# If we have a value_format and it is fy, convert it to the
# date range for that fiscal year
if value_format and value_format == "fy":
fy = FiscalYear(value)
value = [fy.fy_start_date, fy.fy_end_date]
if negate:
return ~self.range_intersect(field, value)
return self.range_intersect(field, value)
if operation == "in":
# make in operation case insensitive for string fields
if self.is_string_field(field):
q_obj = Q()
for item in value:
new_q = {}
new_q[field + "__iexact"] = item
new_q = Q(**new_q)
q_obj = q_obj | new_q
if negate:
q_obj = ~q_obj
return q_obj
else:
# Otherwise, use built in django in
operation = "__in"
if operation == "__icontains" and isinstance(value, list):
# In cases where we have a list of contains (e.g. ArrayField searches)
# we need to not do this case insensitive, as ArrayField's don't have
# icontains implemented like contains
operation = "__contains"
if operation == "" and self.is_string_field(field):
# If we're doing a simple comparison, we need to use iexact for
# string fields
operation = "__iexact"
# We don't have a special operation, so handle the remaining cases
# It's unlikely anyone would specify and ignored parameter via post
if field in self.ignored_parameters:
return Q()
if field in self.filter_map:
field = self.filter_map[field]
q_kwargs[field + operation] = value
if negate:
return ~Q(**q_kwargs)
return Q(**q_kwargs)
def doSearch(self):
venue_vector = SearchVector('name', weight='A') + SearchVector(
'location', 'upcomingConcerts', weight='B')
self.venues = Venue.objects.annotate(
rank=SearchRank(venue_vector, self.query)).filter(
rank__gte=0.1).order_by('-rank')
def test_config_query_explicit(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue', config='french'), ).filter(
search=SearchQuery('cadeaux', config='french'))
self.assertSequenceEqual(searched, [self.french])
def test_search_with_non_text(self):
searched = Line.objects.annotate(
search=SearchVector('id'), ).filter(search=str(self.crowd.id))
self.assertSequenceEqual(searched, [self.crowd])
def test_search_with_null(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue'), ).filter(search='bedemir')
self.assertCountEqual(
searched,
[self.bedemir0, self.bedemir1, self.crowd, self.witch, self.duck])
def test_search_two_terms(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue'), ).filter(search='heart forest')
self.assertSequenceEqual(searched, [self.verse2])
def test_non_exact_match(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue'), ).filter(search='heart')
self.assertSequenceEqual(searched, [self.verse2])
def test_simple_on_scene(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue'), ).filter(search='Forest')
self.assertCountEqual(searched, self.verses)
def test_simple_on_dialogue(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue'), ).filter(search='elbows')
self.assertSequenceEqual(searched, [self.verse1])
def ListParts(request, type_id):
filters = {}
partType = Type.objects.get(id=type_id)
parts = Part.objects.filter(partType=partType)
fields = Field.objects.filter(typePart_id=type_id)
name = ''
for field in fields:
if field.fields == "char1":
name = field.name
typeName = partType.name
searchField = None
models = {}
for field in fields:
models[field.fields] = field.name
if request.method == 'POST':
form = FilterForm(request.POST, models=models, typeName=typeName)
manufacturer = request.POST.getlist('manufacturer')
location = request.POST.getlist('location')
char1 = request.POST.getlist('char1')
char2 = request.POST.getlist('char2')
char3 = request.POST.getlist('char3')
char4 = request.POST.getlist('char4')
char5 = request.POST.getlist('char5')
char6 = request.POST.getlist('char6')
char7 = request.POST.getlist('char7')
char8 = request.POST.getlist('char8')
char9 = request.POST.getlist('char9')
char10 = request.POST.getlist('char10')
char11 = request.POST.getlist('char11')
char12 = request.POST.getlist('char12')
char13 = request.POST.getlist('char13')
char14 = request.POST.getlist('char14')
char15 = request.POST.getlist('char15')
char16 = request.POST.getlist('char16')
char17 = request.POST.getlist('char17')
char18 = request.POST.getlist('char18')
char19 = request.POST.getlist('char19')
char20 = request.POST.getlist('char20')
integer1 = request.POST.getlist('integer1')
integer2 = request.POST.getlist('integer2')
searchField = request.POST.get('search')
if len(manufacturer) > 0:
filters['manufacturer__in'] = manufacturer
if len(location) > 0:
filters['location__in'] = location
if len(char1) > 0:
filters['char1__in'] = char1
if len(char2) > 0:
filters['char2__in'] = char2
if len(char3) > 0:
filters['char3__in'] = char3
if len(char4) > 0:
filters['char4__in'] = char4
if len(char5) > 0:
filters['char5__in'] = char5
if len(char6) > 0:
filters['char6__in'] = char6
if len(char7) > 0:
filters['char7__in'] = char7
if len(char8) > 0:
filters['char8__in'] = char8
if len(char9) > 0:
filters['char9__in'] = char9
if len(char10) > 0:
filters['char10__in'] = char10
if len(char11) > 0:
filters['char11__in'] = char11
if len(char12) > 0:
filters['char12__in'] = char12
if len(char13) > 0:
filters['char13__in'] = char13
if len(char14) > 0:
filters['char14__in'] = char14
if len(char15) > 0:
filters['char15__in'] = char15
if len(char16) > 0:
filters['char16__in'] = char15
if len(char17) > 0:
filters['char17__in'] = char15
if len(char18) > 0:
filters['char18__in'] = char15
if len(char19) > 0:
filters['char19__in'] = char15
if len(char20) > 0:
filters['char20__in'] = char15
if len(integer1) > 0:
filters['integer1__in'] = integer1
if len(integer2) > 0:
filters['integer2__in'] = integer2
form = FilterForm(models=models, typeName=typeName)
else:
form = FilterForm(models=models, typeName=typeName)
parts = parts.filter(**filters)
if searchField == "" or searchField is None:
parts = parts.distinct('id')
else:
parts = parts.annotate(
search=SearchVector('manufacturer__name', 'location__name',
'char1', 'char2', Cast('integer1', CharField(
)), Cast('integer2', CharField()))).filter(
search=searchField)
parts = parts.distinct('id')
current_filters = ''
if filters.items():
for k, v in filters.items():
current_filters += "{}, ".format(v)
return render(
request, 'part_list.html', {
'type': partType,
'parts': parts,
'fields': fields,
'form': form,
'name': name
})
def test_config_from_field_implicit(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue',
config=F('dialogue_config')), ).filter(search='cadeaux')
self.assertSequenceEqual(searched, [self.french])
def get(self, request, page):
offset = int(page)
result_dict = dict()
dominantEssence = request.GET.get("p", "-")
settings = SettingsSearch.objects.first()
# =====================================================================================================
# ======================================== Parse input string =========================================
# =====================================================================================================
query = request.GET.get("q","-")
query = query.translate(str.maketrans("", "", "!@#$%^&*_+|+\/:;[]{}<>,."))
query = re.sub( '\s+', ' ', query).strip()
if query == "направления":
i = 100
for entry in Trends.objects.all():
result_dict['/opportunity/' + str(entry.id)] = {
'id': entry.id,
'rank': i,
'title': entry.title,
'titlePreview': None,
'type': 'opportunity'
}
i -= 1
elif query == "территории":
i = 100
for entry in District.objects.all():
result_dict['/territory/' + str(entry.id)] = {
'id': entry.id,
'rank': i,
'title': entry.title,
'titlePreview': None,
'type': 'territory'
}
i -= 1
elif query == "флагманы":
i = 0
for entry in Flagmans.objects.all():
result_dict['/flagship/' + str(entry.id)] = {
'id': entry.id,
'rank': i,
'title': entry.title,
'titlePreview': None,
'type': 'flagship'
}
i -= 1
elif query == "новости":
i = 0
for entry in News.objects.all():
result_dict['/news/' + str(entry.id)] = {
'id': entry.id,
'rank': i,
'title': entry.title,
'titlePreview': None,
'type': 'news'
}
i -= 1
elif query == "возможности":
i = 0
for entry in Opportunities.objects.all():
result_dict['/opportunity/' + str(entry.id)] = {
'id': entry.id,
'rank': i,
'title': entry.title,
'titlePreview': None,
'type': 'opportunity'
}
i -= 1
else:
query_dict = query.split(" ")
len_dict = len(query_dict)
i = 1
while i < len_dict:
j = 0
while j < len_dict-i:
word = ""
l = 0
while l <= i:
word += query_dict[l+j] + " "
l+=1
query_dict.append(word[:-1])
j+=1
i+=1
# =====================================================================================================
# =================================== Search query inside News ========================================
# =====================================================================================================
vectorTitle = SearchVector('title')
vectorPreview = SearchVector('titlePreview')
vector = SearchVector('title') + SearchVector('text')
for word in query_dict:
que = SearchQuery(word)
news_temp = News.objects.annotate(qTitle=SearchRank(vectorTitle, que), qPreview=SearchRank(vectorPreview, que))
for entry in news_temp:
entry.rank = round(entry.qTitle * settings.factorTitle + entry.qPreview * settings.factorPreview, 4)
news_blocks_sum = NewsBlock.objects.filter(id_fk=entry, include=True).annotate(similarity=SearchRank(vector, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity'))
if news_blocks_sum['similarity__sum'] is None:
news_blocks_sum['similarity__sum'] = 0
entry.rank += round(news_blocks_sum['similarity__sum'] * settings.factorText, 4)
if entry.rank >= settings.filterGTE:
if dominantEssence == "news":
entry.rank += settings.factorPage
result_dict['/news/' + str(entry.id)] = {
'id': entry.id,
'rank': entry.rank,
'title': entry.title,
'titlePreview': entry.titlePreview,
'type': 'news'
}
# =====================================================================================================
# =============================== Search query inside Opportunities ===================================
# =====================================================================================================
vectorTitle = SearchVector('title')
vector = SearchVector('title') + SearchVector('text')
for word in query_dict:
que = SearchQuery(word)
opp_temp = Opportunities.objects.annotate(qTitle=SearchRank(vectorTitle, que))
for entry in opp_temp:
entry.rank = round(entry.qTitle * (settings.factorTitle + settings.factorPreview), 4)
opp_blocks_sum = OpportunitiesBlock.objects.filter(id_fk=entry, include=True).annotate(
similarity=SearchRank(
vector, que)).exclude(similarity__lte=0).aggregate(Sum('similarity'))
if opp_blocks_sum['similarity__sum'] is None:
opp_blocks_sum['similarity__sum'] = 0
entry.rank += round(opp_blocks_sum['similarity__sum'] * settings.factorText, 4)
if entry.rank >= settings.filterGTE:
if dominantEssence == "opportunity":
entry.rank += settings.factorPage
result_dict['/opportunity/' + str(entry.id)] = {
'id': entry.id,
'rank': entry.rank,
'title': entry.title,
'titlePreview': None,
'type': 'opportunity'
}
# =====================================================================================================
# =================================== Search query inside Flagmans ====================================
# =====================================================================================================
vectorTitle = SearchVector('title')
vectorAddress = SearchVector('address')
vectorDate = SearchVector('date')
for word in query_dict:
que = SearchQuery(word)
flagship_temp = Flagmans.objects.annotate(qTitle=SearchRank(vectorTitle, que), qText=SearchRank(vectorAddress, que)+SearchRank(vectorDate, que))
for entry in flagship_temp:
entry.rank = round(entry.qTitle * settings.factorTitle + entry.qText * settings.factorPreview, 4)
if entry.rank >= settings.filterGTE:
if dominantEssence == "flagship":
entry.rank += settings.factorPage
result_dict['/flagship/' + str(entry.id)] = {
'id': entry.id,
'rank': entry.rank,
'title': entry.title,
'titlePreview': None,
'type': 'flagship'
}
# =====================================================================================================
# =================================== Search query inside Districts ===================================
# =====================================================================================================
vectorContacts = SearchVector('name') + SearchVector('position')
vectorAgency = SearchVector('title') + SearchVector('text')
vectorDepartment = SearchVector('title')
vectorTitle = SearchVector('title')
for word in query_dict:
que = SearchQuery(word)
district_temp = District.objects.annotate(qTitle=SearchRank(vectorTitle, que))
for entry in district_temp:
entry.rank = round(entry.qTitle * settings.factorText, 4)
dContacts_sum = DistrictContacts.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorContacts, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity'))
dAgency_sum = Agency.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorAgency, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity'))
dDep_sum = Department.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorDepartment, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity'))
if dContacts_sum['similarity__sum'] is None:
dContacts_sum['similarity__sum'] = 0
if dAgency_sum['similarity__sum'] is None:
dAgency_sum['similarity__sum'] = 0
if dDep_sum['similarity__sum'] is None:
dDep_sum['similarity__sum'] = 0
entry.rank += round((dContacts_sum.get('similarity__sum') + dAgency_sum.get('similarity__sum') + dDep_sum.get('similarity__sum')) * settings.factorText, 4)
if entry.rank >= settings.filterGTE:
if dominantEssence == "territory":
entry.rank += settings.factorPage
result_dict['/territory/' + str(entry.id)] = {
'id': entry.id,
'rank': entry.rank,
'title': entry.title,
'titlePreview': None,
'type': 'territory'
}
# =====================================================================================================
# =================================== Search query inside Trends ======================================
# =====================================================================================================
vectorContacts = SearchVector('name') + SearchVector('position')
vectorTitle = SearchVector('title')
vectorText = SearchVector('text')
for word in query_dict:
que = SearchQuery(word)
trends_temp = Trends.objects.annotate(qTitle=SearchRank(vectorTitle, que), qText = SearchRank(vectorText, que))
for entry in trends_temp:
entry.rank = round(entry.qTitle * settings.factorTitle + entry.qText * settings.factorText, 4)
tContacts_sum = TrendsContacts.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorContacts, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity'))
if tContacts_sum['similarity__sum'] is None:
tContacts_sum['similarity__sum'] = 0
entry.rank += round(tContacts_sum.get('similarity__sum') * settings.factorText, 4)
if entry.rank >= settings.filterGTE:
if dominantEssence == "direction":
entry.rank += settings.factorPage
result_dict['/direction/' + str(entry.id)] = {
'id': entry.id,
'rank': entry.rank,
'title': entry.title,
'titlePreview': None,
'type': 'direction'
}
founded_content = ResultDictBuilder(result_dict)
count_content = len(founded_content)
return Response({
"data": founded_content[offset:offset + 12],
"count": count_content,
})
def test_vector_combined_mismatch(self):
msg = ('SearchVector can only be combined with other SearchVector '
'instances, got NoneType.')
with self.assertRaisesMessage(TypeError, msg):
Line.objects.filter(dialogue__search=None +
SearchVector('character__name'))
def update_search_index(self):
search_vector = (
SearchVector('name', weight='A', config=SEARCH_LANG) +
SearchVector('sub_names', weight='B', config=SEARCH_LANG)
)
PharmaCompany.objects.update(search_vector=search_vector)
def test_query_and(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting',
'dialogue'), ).filter(search=SearchQuery('bedemir')
& SearchQuery('scales'))
self.assertSequenceEqual(searched, [self.bedemir0])
def searchRecords(searchString=None):
searchContactIdList = []
searchWordList = searchString.split(" ")
first = True
for searchWord in searchWordList:
resultContactIdList = []
contactIdList = Contact.objects.values_list('contact_id', flat=True).annotate(search = SearchVector('fname', 'mname', 'lname'))\
.filter(search=SearchQuery(searchWord)).order_by('contact_id')
addressContactIdList = Address.objects.values_list('contact_id', flat=True).annotate(search=SearchVector('address_type', 'address', 'city','state', 'zip')) \
.filter(search=SearchQuery(searchWord))
phoneContactIdList = PhoneNumber.objects.values_list('contact_id', flat=True).annotate(search=SearchVector('phone_type', 'area_code', 'number')) \
.filter(search=SearchQuery(searchWord))
dateContactIdList = Date.objects.values_list('contact_id', flat=True).annotate(search=SearchVector('date_type', 'date')) \
.filter(search=SearchQuery(searchWord))
for contactId in contactIdList:
resultContactIdList.append(contactId)
for contactId in addressContactIdList:
resultContactIdList.append(contactId)
for contactId in phoneContactIdList:
resultContactIdList.append(contactId)
for contactId in dateContactIdList:
resultContactIdList.append(contactId)
if first:
searchContactIdList = resultContactIdList
first = False
else:
searchContactIdList = list(set(searchContactIdList) & set(resultContactIdList))
resultContactIdList = list(set(searchContactIdList))
contactList = Contact.objects.filter(contact_id__in=resultContactIdList).distinct()
return contactList
def test_ranking_chaining(self):
searched = Line.objects.filter(character=self.minstrel).annotate(
rank=SearchRank(
SearchVector('dialogue'),
SearchQuery('brave sir robin')), ).filter(rank__gt=0.3)
self.assertSequenceEqual(searched, [self.verse0])
def post(self, request, book_id=None):
""" edit a book cool """
# returns None if no match is found
book = models.Edition.objects.filter(id=book_id).first()
form = forms.EditionForm(request.POST, request.FILES, instance=book)
data = {"book": book, "form": form}
if not form.is_valid():
return TemplateResponse(request, "book/edit_book.html", data)
add_author = request.POST.get("add_author")
# we're adding an author through a free text field
if add_author:
data["add_author"] = add_author
data["author_matches"] = []
for author in add_author.split(","):
if not author:
continue
# check for existing authors
vector = SearchVector("name", weight="A") + SearchVector(
"aliases", weight="B"
)
data["author_matches"].append(
{
"name": author.strip(),
"matches": (
models.Author.objects.annotate(search=vector)
.annotate(rank=SearchRank(vector, author))
.filter(rank__gt=0.4)
.order_by("-rank")[:5]
),
}
)
print(data["author_matches"])
# we're creating a new book
if not book:
# check if this is an edition of an existing work
author_text = book.author_text if book else add_author
data["book_matches"] = connector_manager.local_search(
"%s %s" % (form.cleaned_data.get("title"), author_text),
min_confidence=0.5,
raw=True,
)[:5]
# either of the above cases requires additional confirmation
if add_author or not book:
# creting a book or adding an author to a book needs another step
data["confirm_mode"] = True
# this isn't preserved because it isn't part of the form obj
data["remove_authors"] = request.POST.getlist("remove_authors")
# make sure the dates are passed in as datetime, they're currently a string
# QueryDicts are immutable, we need to copy
formcopy = data["form"].data.copy()
try:
formcopy["first_published_date"] = dateparse(
formcopy["first_published_date"]
)
except (MultiValueDictKeyError, ValueError):
pass
try:
formcopy["published_date"] = dateparse(formcopy["published_date"])
except (MultiValueDictKeyError, ValueError):
pass
data["form"].data = formcopy
return TemplateResponse(request, "book/edit_book.html", data)
remove_authors = request.POST.getlist("remove_authors")
for author_id in remove_authors:
book.authors.remove(author_id)
book = form.save(commit=False)
url = request.POST.get("cover-url")
if url:
image = set_cover_from_url(url)
if image:
book.cover.save(*image, save=False)
book.save()
return redirect("/book/%s" % book.id)
from django.contrib.postgres.fields.jsonb import KeyTextTransform
from django.contrib.postgres.search import SearchVector
from django.db.models import Value
from django.db.models.functions import Coalesce
DOCUMENT_SEARCH_VECTOR = (
SearchVector('title', weight='A') +
SearchVector(Coalesce(KeyTextTransform('slug', 'metadata'), Value('')),
weight='A') +
SearchVector(Coalesce(KeyTextTransform('toc', 'metadata'), Value('')),
weight='B') +
SearchVector(Coalesce(KeyTextTransform('body', 'metadata'), Value('')),
weight='C') +
SearchVector(Coalesce(KeyTextTransform('parents', 'metadata'), Value('')),
weight='D'))
def update_search_vector(sender, instance, **kwargs):
Tweet.objects.filter(pk=instance.pk).update(search=SearchVector('tip'))
def replace(self, term):
synonym = Synonym.objects.annotate(search=SearchVector(
"name", "words", config="french"), ).filter(search=term)
return synonym.first().name if synonym.first() else term
def with_documents(self):
vector = SearchVector('name', weight='A', config='english')
vector += SearchVector('aka', weight='B', config='english')
return self.get_queryset().annotate(document=vector)
def get_document_content(self):
file_name = ''
for event_doc_obj in sorted(
Document.objects.filter(document_content_scanned=False),
key=lambda x: x.id,
reverse=True):
try:
file_name = self.downloadFile(event_doc_obj)
document_content_text = self.getFileContent(
file_name, event_doc_obj.id)
document_content = DocumentContent.objects.create(
content=document_content_text)
event_doc_obj.document_content_scanned = True
event_doc_obj.file_path = file_name
event_doc_obj.doc_content = document_content
event_doc_obj.save()
self.remove_folder_contents()
if CombinedItem.objects.filter(item_id=event_doc_obj.id,
item_type='document').exists():
ci = CombinedItem.objects.get(item_id=event_doc_obj.id,
item_type='document')
ci.doc_content = document_content
ci.save()
self.add_content_text(
ReceivedDocument.objects.filter(
document__url=event_doc_obj.url), 'received_document',
document_content_text)
self.add_content_text(
CouncilAddress.objects.filter(
question_document__url=event_doc_obj.url),
'council_address', document_content_text)
self.add_content_text(
WrittenQuestion.objects.filter(
question_document__url=event_doc_obj.url),
'written_question', document_content_text)
self.add_content_text(
PublicDocument.objects.filter(
document__url=event_doc_obj.url), 'format',
document_content_text)
self.add_content_text(
PolicyDocument.objects.filter(
document__url=event_doc_obj.url), 'policy_document',
document_content_text)
self.add_content_text(
ManagementDocument.objects.filter(
document__url=event_doc_obj.url),
'management_document', document_content_text)
self.add_content_text(
Motion.objects.filter(document__url=event_doc_obj.url),
'motion', document_content_text)
self.add_content_text(
Commitment.objects.filter(
new_document__url=event_doc_obj.url), 'commitment',
document_content_text)
except Exception:
self.remove_folder_contents()
vector = SearchVector('content', config='dutch')
DocumentContent.objects.update(vector=vector)
from wagtail.search.query import And, Boost, MatchAll, Not, Or, Phrase, PlainText
from wagtail.search.utils import ADD, MUL, OR
from wagtail.utils.deprecation import RemovedInWagtail217Warning
from .models import IndexEntry
from .query import Lexeme
from .utils import (get_content_type_pk, get_descendants_content_types_pks,
get_postgresql_connections, get_sql_weights, get_weight)
warnings.warn(
"The wagtail.contrib.postgres_search backend is deprecated and has been replaced by "
"wagtail.search.backends.database. "
"See https://docs.wagtail.org/en/stable/releases/2.15.html#database-search-backends-replaced",
category=RemovedInWagtail217Warning)
EMPTY_VECTOR = SearchVector(Value('', output_field=TextField()))
class ObjectIndexer:
"""
Responsible for extracting data from an object to be inserted into the index.
"""
def __init__(self, obj, backend):
self.obj = obj
self.search_fields = obj.get_search_fields()
self.config = backend.config
self.autocomplete_config = backend.autocomplete_config
def prepare_value(self, value):
if isinstance(value, str):
return value
def rank_with_terms_and_filters(
self,
rank_on_field,
rank_by_values,
rank_threshold,
result_fields,
relations=None,
limit=None,
food_group_filters=None,
query_set=None,
):
"""search for foods in db
Parameters
----------
rank_on_field: str
field than has rank applied to it based on rank_by_values
rank_by_valus: str
search terms
rank_threshold: float
value measures relevance of record to search terms.
result_fields: list
contains fields which are to be returned
relations: list
fields related by foreign key and thereby used for prefetch_related
limit: int
number of records to return
food_group_filters: list
contains food group names with which to filter queryset
queryset: QuerySet object
any existing QuerySet upon which the search can be executed upon
Returns
----------
QuerySet
QuerySet containing dicts corresponding to result_fields arg
"""
if query_set is None:
query_set = self.get_queryset()
if food_group_filters:
query_set = query_set.filter(
food_group__name__in=food_group_filters)
rank_on_field = SearchVector(rank_on_field)
if rank_by_values != ["_all_"]:
rank_by_values = self.make_query(rank_by_values)
query_set = (query_set.annotate(
rank=SearchRank(rank_on_field, rank_by_values)).filter(
rank__gte=rank_threshold).order_by("-rank"))
else:
query_set = query_set.order_by("-date")
if relations:
for relation in relations:
query_set = query_set.prefetch_related(relation)
if limit:
query_set = query_set[:limit]
return query_set.values(*result_fields)
