def test_search_vector_index(self): """SearchVector generates IMMUTABLE SQL in order to be indexable.""" # This test should be moved to test_indexes and use a functional # index instead once support lands (see #26167). query = Line.objects.all().query resolved = SearchVector('id', 'dialogue', config='english').resolve_expression(query) compiler = query.get_compiler(connection.alias) sql, params = resolved.as_sql(compiler, connection) # Indexed function must be IMMUTABLE. with connection.cursor() as cursor: cursor.execute( 'CREATE INDEX search_vector_index ON %s USING GIN (%s)' % (Line._meta.db_table, sql), params, )
def get(self, request, format=None): id = self.request.query_params.get('id', None) category = list( filter(None, self.request.query_params.get('category', '').split(","))) tag = list( map( int, filter(None, self.request.query_params.get('tag', '').split(",")))) search = self.request.query_params.get('search', None) similar = self.request.query_params.get('similar', None) next = self.request.query_params.get('next', None) prev = self.request.query_params.get('prev', None) page = self.request.query_params.get('page', 1) news = News.objects.filter(visibility=True).order_by('-id') if search: news = news.annotate(search=SearchVector( 'headline', 'body', 'newsAgency', 'category__name', 'tags__name'), ).filter(search=search) if similar: news = news.get(id=similar).etags.similar_objects() if category: if "independent" in category: news = news.filter(independent=True) category.remove("independent") else: news = news.filter(independent=False) if "preference" in category: if request.user.is_authenticated: mycategory = MyCategory.objects.filter(user=request.user) if mycategory: cats = MyCategorySerializer( mycategory[0]).data["categorys"] news = news.filter(category__in=cats) else: category.append("trending") else: return response.Response( {"message": "User is not Authenticated"}) category.remove("preference") if "trending" in category: time_threshold = datetime.now() - timedelta(days=5) iso, hr = datetime.now( timezone.utc).isocalendar(), datetime.now().hour news = news.filter(created_at__gt=time_threshold).annotate( popularity=(F('pos') - F('neg') + 0.01) / ((iso[0] - Extract(F('created_at'), 'iso_year')) * 364 * 24 + (iso[1] - Extract(F('created_at'), 'week')) * 7 * 24 + (iso[2] - Extract(F('created_at'), 'week_day') % 7 - 1) * 24 + hr - Extract(F('created_at'), 'hour') + 1.01)).order_by('-popularity') category.remove("trending") elif category: news = news.filter(category__name__in=category) else: news = news.filter(independent=False) data = {} news = news.distinct() if id: if next and next != "0": news1 = news.filter(id__gt=int(id)).order_by('id')[0:int(next)] serializer = NewsSerializer(news1, many=True, context={'request': request}) data['next'] = serializer.data if prev and prev != "0": news2 = news.filter( id__lt=int(id)).order_by('-id')[0:int(prev)] serializer = NewsSerializer(news2, many=True, context={'request': request}) data['prev'] = serializer.data if not next or next == "0": data['next'] = data['prev'] else: data['prev'] = data['next'] else: serializer = NewsSerializer(Paginator(news, 20).page(page), many=True, context={'request': request}) data['next'] = data['prev'] = serializer.data return response.Response(data)
def search_page(request): """Page with search results.""" def _articles_filter(name): """Delete articles in product names Arguments: name {str} -- Name or generic name of product """ exclude = ("de", "des", "au", "aux", 'en', "le", "la", "les", "et", "un", "une", "du", "à") produit = name.lower() produit = produit.split(" ") for e in exclude: for x in produit: if x in e: try: produit.remove(e) except: return x keyword = (" ".join(produit[0:5])) keyword = keyword.replace(",", "") return keyword query = request.GET.get('query') URL = 'https://fr.openfoodfacts.org/cgi/search.pl?' PARAMETERS = { 'action': 'process', 'search_terms': query, 'sort_by': 'unique_scans_n', 'axis_x': 'energy', 'axis_y': 'products_n', 'page_size': '1', 'page': '1', 'json': '1' } r = requests.get(URL, params=PARAMETERS) requested_product = r.json() try: substitution_products = Product.objects.annotate( search=SearchVector('product_name'), ).filter( search=_articles_filter(query)) substitution_products = substitution_products.order_by( 'nutrition_grades') if not substitution_products: substitution_products = Product.objects.annotate( search=SearchVector('product_name'), ).filter( search=_articles_filter(requested_product['products'][0] ['product_name'])) substitution_products = substitution_products.order_by( 'nutrition_grades') if not substitution_products: substitution_products = Product.objects.annotate( search=SearchVector('product_name'), ).filter( search=_articles_filter(requested_product['products'][0] ['generic_name_fr'])) substitution_products = substitution_products.order_by( 'nutrition_grades') except: raise Http404 bookmark_id = list() if request.user.is_authenticated: qs_bookmark = Bookmark.objects.filter(user=request.user) for i in qs_bookmark: bookmark_id.append(i.bookmark.id) if request.method == 'POST': request_post = request.POST.get('bookmark_product_code') Bookmark.objects.create(user=request.user, bookmark_id=request_post) return HttpResponseRedirect(request.get_full_path()) # PAGINATION # https://docs.djangoproject.com/fr/2.2/topics/pagination/ paginator = Paginator(substitution_products, 9) # 9 products by page page = request.GET.get('page') try: substitution_products = paginator.page(page) except PageNotAnInteger: substitution_products = paginator.page(1) except EmptyPage: substitution_products = paginator.page(paginator.num_pages) # /PAGINATION template_name = 'app/search.html' context = { 'query': query, 'img': IMG, 'requested_title': f"1er résultat Open Food Facts pour : {query}", 'requested_product_name': requested_product['products'][0]['product_name'], 'requested_product_brands': requested_product['products'][0]['brands'], 'requested_product_quantity': requested_product['products'][0]['quantity'], 'requested_product_image': requested_product['products'][0]['image_url'], 'products': substitution_products, 'paginate': True, 'bookmarks': bookmark_id, } return render(request, template_name, context)
def update_search_index(self): self.update(search_data=SearchVector(*self.model.extra['search']))
return news_query def orm_get_news_item(news_id: int) -> NewsItem: news_query = NewsItem.objects.filter(id=news_id) return news_query[0] if len(news_query) > 0 else None def orm_get_knowledge_base_items(): news_query = NewsItem.objects.filter(knowledge_base_article=True) return news_query _news_title_search_vector = SearchVector('title') def orm_news_title_fts(search: str): """full text search on title field of news items""" news_query = NewsItem.objects.annotate( search=_news_title_search_vector).filter(search=SearchQuery(search)) return news_query _news_text_search_vector = SearchVector('text') def orm_news_text_fts(search: str): """full text search on text field of news items""" news_query = NewsItem.objects.annotate(
def get_search_vector(self): return ( SearchVector('name', weight='A', config=SEARCH_LANG) + SearchVector('active_ingredient', weight='B', config=SEARCH_LANG) + SearchVector('medical_indication', weight='C', config=SEARCH_LANG) )
def create_q_from_filter(self, filt): if "combine_method" in filt: return self.create_q_from_filter_list(filt["filters"], filt["combine_method"]) else: q_kwargs = {} field = filt["field"] negate = False if "not_" == filt["operation"][:4]: negate = True operation = FilterGenerator.operators[filt["operation"][4:]] else: operation = FilterGenerator.operators[filt["operation"]] value = filt["value"] value_format = None if "value_format" in filt: value_format = filt["value_format"] # Special multi-field case for full-text search if isinstance(field, list) and operation == "__search": # We create the search vector and attach it to this object sv = SearchVector(*field) self.search_vectors.append(sv) # Our Q object is simpler now q_kwargs["search"] = value # Return our Q and skip the rest if negate: return ~Q(**q_kwargs) return Q(**q_kwargs) # Handle special operations if operation == "fy": fy = FiscalYear(value) if negate: return ~fy.get_filter_object(field) return fy.get_filter_object(field) if operation == "range_intersect": # If we have a value_format and it is fy, convert it to the # date range for that fiscal year if value_format and value_format == "fy": fy = FiscalYear(value) value = [fy.fy_start_date, fy.fy_end_date] if negate: return ~self.range_intersect(field, value) return self.range_intersect(field, value) if operation == "in": # make in operation case insensitive for string fields if self.is_string_field(field): q_obj = Q() for item in value: new_q = {} new_q[field + "__iexact"] = item new_q = Q(**new_q) q_obj = q_obj | new_q if negate: q_obj = ~q_obj return q_obj else: # Otherwise, use built in django in operation = "__in" if operation == "__icontains" and isinstance(value, list): # In cases where we have a list of contains (e.g. ArrayField searches) # we need to not do this case insensitive, as ArrayField's don't have # icontains implemented like contains operation = "__contains" if operation == "" and self.is_string_field(field): # If we're doing a simple comparison, we need to use iexact for # string fields operation = "__iexact" # We don't have a special operation, so handle the remaining cases # It's unlikely anyone would specify and ignored parameter via post if field in self.ignored_parameters: return Q() if field in self.filter_map: field = self.filter_map[field] q_kwargs[field + operation] = value if negate: return ~Q(**q_kwargs) return Q(**q_kwargs)
def doSearch(self): venue_vector = SearchVector('name', weight='A') + SearchVector( 'location', 'upcomingConcerts', weight='B') self.venues = Venue.objects.annotate( rank=SearchRank(venue_vector, self.query)).filter( rank__gte=0.1).order_by('-rank')
def test_config_query_explicit(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue', config='french'), ).filter( search=SearchQuery('cadeaux', config='french')) self.assertSequenceEqual(searched, [self.french])
def test_search_with_non_text(self): searched = Line.objects.annotate( search=SearchVector('id'), ).filter(search=str(self.crowd.id)) self.assertSequenceEqual(searched, [self.crowd])
def test_search_with_null(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue'), ).filter(search='bedemir') self.assertCountEqual( searched, [self.bedemir0, self.bedemir1, self.crowd, self.witch, self.duck])
def test_search_two_terms(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue'), ).filter(search='heart forest') self.assertSequenceEqual(searched, [self.verse2])
def test_non_exact_match(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue'), ).filter(search='heart') self.assertSequenceEqual(searched, [self.verse2])
def test_simple_on_scene(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue'), ).filter(search='Forest') self.assertCountEqual(searched, self.verses)
def test_simple_on_dialogue(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue'), ).filter(search='elbows') self.assertSequenceEqual(searched, [self.verse1])
def ListParts(request, type_id): filters = {} partType = Type.objects.get(id=type_id) parts = Part.objects.filter(partType=partType) fields = Field.objects.filter(typePart_id=type_id) name = '' for field in fields: if field.fields == "char1": name = field.name typeName = partType.name searchField = None models = {} for field in fields: models[field.fields] = field.name if request.method == 'POST': form = FilterForm(request.POST, models=models, typeName=typeName) manufacturer = request.POST.getlist('manufacturer') location = request.POST.getlist('location') char1 = request.POST.getlist('char1') char2 = request.POST.getlist('char2') char3 = request.POST.getlist('char3') char4 = request.POST.getlist('char4') char5 = request.POST.getlist('char5') char6 = request.POST.getlist('char6') char7 = request.POST.getlist('char7') char8 = request.POST.getlist('char8') char9 = request.POST.getlist('char9') char10 = request.POST.getlist('char10') char11 = request.POST.getlist('char11') char12 = request.POST.getlist('char12') char13 = request.POST.getlist('char13') char14 = request.POST.getlist('char14') char15 = request.POST.getlist('char15') char16 = request.POST.getlist('char16') char17 = request.POST.getlist('char17') char18 = request.POST.getlist('char18') char19 = request.POST.getlist('char19') char20 = request.POST.getlist('char20') integer1 = request.POST.getlist('integer1') integer2 = request.POST.getlist('integer2') searchField = request.POST.get('search') if len(manufacturer) > 0: filters['manufacturer__in'] = manufacturer if len(location) > 0: filters['location__in'] = location if len(char1) > 0: filters['char1__in'] = char1 if len(char2) > 0: filters['char2__in'] = char2 if len(char3) > 0: filters['char3__in'] = char3 if len(char4) > 0: filters['char4__in'] = char4 if len(char5) > 0: filters['char5__in'] = char5 if len(char6) > 0: filters['char6__in'] = char6 if len(char7) > 0: filters['char7__in'] = char7 if len(char8) > 0: filters['char8__in'] = char8 if len(char9) > 0: filters['char9__in'] = char9 if len(char10) > 0: filters['char10__in'] = char10 if len(char11) > 0: filters['char11__in'] = char11 if len(char12) > 0: filters['char12__in'] = char12 if len(char13) > 0: filters['char13__in'] = char13 if len(char14) > 0: filters['char14__in'] = char14 if len(char15) > 0: filters['char15__in'] = char15 if len(char16) > 0: filters['char16__in'] = char15 if len(char17) > 0: filters['char17__in'] = char15 if len(char18) > 0: filters['char18__in'] = char15 if len(char19) > 0: filters['char19__in'] = char15 if len(char20) > 0: filters['char20__in'] = char15 if len(integer1) > 0: filters['integer1__in'] = integer1 if len(integer2) > 0: filters['integer2__in'] = integer2 form = FilterForm(models=models, typeName=typeName) else: form = FilterForm(models=models, typeName=typeName) parts = parts.filter(**filters) if searchField == "" or searchField is None: parts = parts.distinct('id') else: parts = parts.annotate( search=SearchVector('manufacturer__name', 'location__name', 'char1', 'char2', Cast('integer1', CharField( )), Cast('integer2', CharField()))).filter( search=searchField) parts = parts.distinct('id') current_filters = '' if filters.items(): for k, v in filters.items(): current_filters += "{}, ".format(v) return render( request, 'part_list.html', { 'type': partType, 'parts': parts, 'fields': fields, 'form': form, 'name': name })
def test_config_from_field_implicit(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue', config=F('dialogue_config')), ).filter(search='cadeaux') self.assertSequenceEqual(searched, [self.french])
def get(self, request, page): offset = int(page) result_dict = dict() dominantEssence = request.GET.get("p", "-") settings = SettingsSearch.objects.first() # ===================================================================================================== # ======================================== Parse input string ========================================= # ===================================================================================================== query = request.GET.get("q","-") query = query.translate(str.maketrans("", "", "!@#$%^&*_+|+\/:;[]{}<>,.")) query = re.sub( '\s+', ' ', query).strip() if query == "направления": i = 100 for entry in Trends.objects.all(): result_dict['/opportunity/' + str(entry.id)] = { 'id': entry.id, 'rank': i, 'title': entry.title, 'titlePreview': None, 'type': 'opportunity' } i -= 1 elif query == "территории": i = 100 for entry in District.objects.all(): result_dict['/territory/' + str(entry.id)] = { 'id': entry.id, 'rank': i, 'title': entry.title, 'titlePreview': None, 'type': 'territory' } i -= 1 elif query == "флагманы": i = 0 for entry in Flagmans.objects.all(): result_dict['/flagship/' + str(entry.id)] = { 'id': entry.id, 'rank': i, 'title': entry.title, 'titlePreview': None, 'type': 'flagship' } i -= 1 elif query == "новости": i = 0 for entry in News.objects.all(): result_dict['/news/' + str(entry.id)] = { 'id': entry.id, 'rank': i, 'title': entry.title, 'titlePreview': None, 'type': 'news' } i -= 1 elif query == "возможности": i = 0 for entry in Opportunities.objects.all(): result_dict['/opportunity/' + str(entry.id)] = { 'id': entry.id, 'rank': i, 'title': entry.title, 'titlePreview': None, 'type': 'opportunity' } i -= 1 else: query_dict = query.split(" ") len_dict = len(query_dict) i = 1 while i < len_dict: j = 0 while j < len_dict-i: word = "" l = 0 while l <= i: word += query_dict[l+j] + " " l+=1 query_dict.append(word[:-1]) j+=1 i+=1 # ===================================================================================================== # =================================== Search query inside News ======================================== # ===================================================================================================== vectorTitle = SearchVector('title') vectorPreview = SearchVector('titlePreview') vector = SearchVector('title') + SearchVector('text') for word in query_dict: que = SearchQuery(word) news_temp = News.objects.annotate(qTitle=SearchRank(vectorTitle, que), qPreview=SearchRank(vectorPreview, que)) for entry in news_temp: entry.rank = round(entry.qTitle * settings.factorTitle + entry.qPreview * settings.factorPreview, 4) news_blocks_sum = NewsBlock.objects.filter(id_fk=entry, include=True).annotate(similarity=SearchRank(vector, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity')) if news_blocks_sum['similarity__sum'] is None: news_blocks_sum['similarity__sum'] = 0 entry.rank += round(news_blocks_sum['similarity__sum'] * settings.factorText, 4) if entry.rank >= settings.filterGTE: if dominantEssence == "news": entry.rank += settings.factorPage result_dict['/news/' + str(entry.id)] = { 'id': entry.id, 'rank': entry.rank, 'title': entry.title, 'titlePreview': entry.titlePreview, 'type': 'news' } # ===================================================================================================== # =============================== Search query inside Opportunities =================================== # ===================================================================================================== vectorTitle = SearchVector('title') vector = SearchVector('title') + SearchVector('text') for word in query_dict: que = SearchQuery(word) opp_temp = Opportunities.objects.annotate(qTitle=SearchRank(vectorTitle, que)) for entry in opp_temp: entry.rank = round(entry.qTitle * (settings.factorTitle + settings.factorPreview), 4) opp_blocks_sum = OpportunitiesBlock.objects.filter(id_fk=entry, include=True).annotate( similarity=SearchRank( vector, que)).exclude(similarity__lte=0).aggregate(Sum('similarity')) if opp_blocks_sum['similarity__sum'] is None: opp_blocks_sum['similarity__sum'] = 0 entry.rank += round(opp_blocks_sum['similarity__sum'] * settings.factorText, 4) if entry.rank >= settings.filterGTE: if dominantEssence == "opportunity": entry.rank += settings.factorPage result_dict['/opportunity/' + str(entry.id)] = { 'id': entry.id, 'rank': entry.rank, 'title': entry.title, 'titlePreview': None, 'type': 'opportunity' } # ===================================================================================================== # =================================== Search query inside Flagmans ==================================== # ===================================================================================================== vectorTitle = SearchVector('title') vectorAddress = SearchVector('address') vectorDate = SearchVector('date') for word in query_dict: que = SearchQuery(word) flagship_temp = Flagmans.objects.annotate(qTitle=SearchRank(vectorTitle, que), qText=SearchRank(vectorAddress, que)+SearchRank(vectorDate, que)) for entry in flagship_temp: entry.rank = round(entry.qTitle * settings.factorTitle + entry.qText * settings.factorPreview, 4) if entry.rank >= settings.filterGTE: if dominantEssence == "flagship": entry.rank += settings.factorPage result_dict['/flagship/' + str(entry.id)] = { 'id': entry.id, 'rank': entry.rank, 'title': entry.title, 'titlePreview': None, 'type': 'flagship' } # ===================================================================================================== # =================================== Search query inside Districts =================================== # ===================================================================================================== vectorContacts = SearchVector('name') + SearchVector('position') vectorAgency = SearchVector('title') + SearchVector('text') vectorDepartment = SearchVector('title') vectorTitle = SearchVector('title') for word in query_dict: que = SearchQuery(word) district_temp = District.objects.annotate(qTitle=SearchRank(vectorTitle, que)) for entry in district_temp: entry.rank = round(entry.qTitle * settings.factorText, 4) dContacts_sum = DistrictContacts.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorContacts, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity')) dAgency_sum = Agency.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorAgency, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity')) dDep_sum = Department.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorDepartment, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity')) if dContacts_sum['similarity__sum'] is None: dContacts_sum['similarity__sum'] = 0 if dAgency_sum['similarity__sum'] is None: dAgency_sum['similarity__sum'] = 0 if dDep_sum['similarity__sum'] is None: dDep_sum['similarity__sum'] = 0 entry.rank += round((dContacts_sum.get('similarity__sum') + dAgency_sum.get('similarity__sum') + dDep_sum.get('similarity__sum')) * settings.factorText, 4) if entry.rank >= settings.filterGTE: if dominantEssence == "territory": entry.rank += settings.factorPage result_dict['/territory/' + str(entry.id)] = { 'id': entry.id, 'rank': entry.rank, 'title': entry.title, 'titlePreview': None, 'type': 'territory' } # ===================================================================================================== # =================================== Search query inside Trends ====================================== # ===================================================================================================== vectorContacts = SearchVector('name') + SearchVector('position') vectorTitle = SearchVector('title') vectorText = SearchVector('text') for word in query_dict: que = SearchQuery(word) trends_temp = Trends.objects.annotate(qTitle=SearchRank(vectorTitle, que), qText = SearchRank(vectorText, que)) for entry in trends_temp: entry.rank = round(entry.qTitle * settings.factorTitle + entry.qText * settings.factorText, 4) tContacts_sum = TrendsContacts.objects.filter(id_fk=entry).annotate(similarity=SearchRank(vectorContacts, que)).exclude(similarity__lte = 0).aggregate(Sum('similarity')) if tContacts_sum['similarity__sum'] is None: tContacts_sum['similarity__sum'] = 0 entry.rank += round(tContacts_sum.get('similarity__sum') * settings.factorText, 4) if entry.rank >= settings.filterGTE: if dominantEssence == "direction": entry.rank += settings.factorPage result_dict['/direction/' + str(entry.id)] = { 'id': entry.id, 'rank': entry.rank, 'title': entry.title, 'titlePreview': None, 'type': 'direction' } founded_content = ResultDictBuilder(result_dict) count_content = len(founded_content) return Response({ "data": founded_content[offset:offset + 12], "count": count_content, })
def test_vector_combined_mismatch(self): msg = ('SearchVector can only be combined with other SearchVector ' 'instances, got NoneType.') with self.assertRaisesMessage(TypeError, msg): Line.objects.filter(dialogue__search=None + SearchVector('character__name'))
def update_search_index(self): search_vector = ( SearchVector('name', weight='A', config=SEARCH_LANG) + SearchVector('sub_names', weight='B', config=SEARCH_LANG) ) PharmaCompany.objects.update(search_vector=search_vector)
def test_query_and(self): searched = Line.objects.annotate(search=SearchVector( 'scene__setting', 'dialogue'), ).filter(search=SearchQuery('bedemir') & SearchQuery('scales')) self.assertSequenceEqual(searched, [self.bedemir0])
def searchRecords(searchString=None): searchContactIdList = [] searchWordList = searchString.split(" ") first = True for searchWord in searchWordList: resultContactIdList = [] contactIdList = Contact.objects.values_list('contact_id', flat=True).annotate(search = SearchVector('fname', 'mname', 'lname'))\ .filter(search=SearchQuery(searchWord)).order_by('contact_id') addressContactIdList = Address.objects.values_list('contact_id', flat=True).annotate(search=SearchVector('address_type', 'address', 'city','state', 'zip')) \ .filter(search=SearchQuery(searchWord)) phoneContactIdList = PhoneNumber.objects.values_list('contact_id', flat=True).annotate(search=SearchVector('phone_type', 'area_code', 'number')) \ .filter(search=SearchQuery(searchWord)) dateContactIdList = Date.objects.values_list('contact_id', flat=True).annotate(search=SearchVector('date_type', 'date')) \ .filter(search=SearchQuery(searchWord)) for contactId in contactIdList: resultContactIdList.append(contactId) for contactId in addressContactIdList: resultContactIdList.append(contactId) for contactId in phoneContactIdList: resultContactIdList.append(contactId) for contactId in dateContactIdList: resultContactIdList.append(contactId) if first: searchContactIdList = resultContactIdList first = False else: searchContactIdList = list(set(searchContactIdList) & set(resultContactIdList)) resultContactIdList = list(set(searchContactIdList)) contactList = Contact.objects.filter(contact_id__in=resultContactIdList).distinct() return contactList
def test_ranking_chaining(self): searched = Line.objects.filter(character=self.minstrel).annotate( rank=SearchRank( SearchVector('dialogue'), SearchQuery('brave sir robin')), ).filter(rank__gt=0.3) self.assertSequenceEqual(searched, [self.verse0])
def post(self, request, book_id=None): """ edit a book cool """ # returns None if no match is found book = models.Edition.objects.filter(id=book_id).first() form = forms.EditionForm(request.POST, request.FILES, instance=book) data = {"book": book, "form": form} if not form.is_valid(): return TemplateResponse(request, "book/edit_book.html", data) add_author = request.POST.get("add_author") # we're adding an author through a free text field if add_author: data["add_author"] = add_author data["author_matches"] = [] for author in add_author.split(","): if not author: continue # check for existing authors vector = SearchVector("name", weight="A") + SearchVector( "aliases", weight="B" ) data["author_matches"].append( { "name": author.strip(), "matches": ( models.Author.objects.annotate(search=vector) .annotate(rank=SearchRank(vector, author)) .filter(rank__gt=0.4) .order_by("-rank")[:5] ), } ) print(data["author_matches"]) # we're creating a new book if not book: # check if this is an edition of an existing work author_text = book.author_text if book else add_author data["book_matches"] = connector_manager.local_search( "%s %s" % (form.cleaned_data.get("title"), author_text), min_confidence=0.5, raw=True, )[:5] # either of the above cases requires additional confirmation if add_author or not book: # creting a book or adding an author to a book needs another step data["confirm_mode"] = True # this isn't preserved because it isn't part of the form obj data["remove_authors"] = request.POST.getlist("remove_authors") # make sure the dates are passed in as datetime, they're currently a string # QueryDicts are immutable, we need to copy formcopy = data["form"].data.copy() try: formcopy["first_published_date"] = dateparse( formcopy["first_published_date"] ) except (MultiValueDictKeyError, ValueError): pass try: formcopy["published_date"] = dateparse(formcopy["published_date"]) except (MultiValueDictKeyError, ValueError): pass data["form"].data = formcopy return TemplateResponse(request, "book/edit_book.html", data) remove_authors = request.POST.getlist("remove_authors") for author_id in remove_authors: book.authors.remove(author_id) book = form.save(commit=False) url = request.POST.get("cover-url") if url: image = set_cover_from_url(url) if image: book.cover.save(*image, save=False) book.save() return redirect("/book/%s" % book.id)
from django.contrib.postgres.fields.jsonb import KeyTextTransform from django.contrib.postgres.search import SearchVector from django.db.models import Value from django.db.models.functions import Coalesce DOCUMENT_SEARCH_VECTOR = ( SearchVector('title', weight='A') + SearchVector(Coalesce(KeyTextTransform('slug', 'metadata'), Value('')), weight='A') + SearchVector(Coalesce(KeyTextTransform('toc', 'metadata'), Value('')), weight='B') + SearchVector(Coalesce(KeyTextTransform('body', 'metadata'), Value('')), weight='C') + SearchVector(Coalesce(KeyTextTransform('parents', 'metadata'), Value('')), weight='D'))
def update_search_vector(sender, instance, **kwargs): Tweet.objects.filter(pk=instance.pk).update(search=SearchVector('tip'))
def replace(self, term): synonym = Synonym.objects.annotate(search=SearchVector( "name", "words", config="french"), ).filter(search=term) return synonym.first().name if synonym.first() else term
def with_documents(self): vector = SearchVector('name', weight='A', config='english') vector += SearchVector('aka', weight='B', config='english') return self.get_queryset().annotate(document=vector)
def get_document_content(self): file_name = '' for event_doc_obj in sorted( Document.objects.filter(document_content_scanned=False), key=lambda x: x.id, reverse=True): try: file_name = self.downloadFile(event_doc_obj) document_content_text = self.getFileContent( file_name, event_doc_obj.id) document_content = DocumentContent.objects.create( content=document_content_text) event_doc_obj.document_content_scanned = True event_doc_obj.file_path = file_name event_doc_obj.doc_content = document_content event_doc_obj.save() self.remove_folder_contents() if CombinedItem.objects.filter(item_id=event_doc_obj.id, item_type='document').exists(): ci = CombinedItem.objects.get(item_id=event_doc_obj.id, item_type='document') ci.doc_content = document_content ci.save() self.add_content_text( ReceivedDocument.objects.filter( document__url=event_doc_obj.url), 'received_document', document_content_text) self.add_content_text( CouncilAddress.objects.filter( question_document__url=event_doc_obj.url), 'council_address', document_content_text) self.add_content_text( WrittenQuestion.objects.filter( question_document__url=event_doc_obj.url), 'written_question', document_content_text) self.add_content_text( PublicDocument.objects.filter( document__url=event_doc_obj.url), 'format', document_content_text) self.add_content_text( PolicyDocument.objects.filter( document__url=event_doc_obj.url), 'policy_document', document_content_text) self.add_content_text( ManagementDocument.objects.filter( document__url=event_doc_obj.url), 'management_document', document_content_text) self.add_content_text( Motion.objects.filter(document__url=event_doc_obj.url), 'motion', document_content_text) self.add_content_text( Commitment.objects.filter( new_document__url=event_doc_obj.url), 'commitment', document_content_text) except Exception: self.remove_folder_contents() vector = SearchVector('content', config='dutch') DocumentContent.objects.update(vector=vector)
from wagtail.search.query import And, Boost, MatchAll, Not, Or, Phrase, PlainText from wagtail.search.utils import ADD, MUL, OR from wagtail.utils.deprecation import RemovedInWagtail217Warning from .models import IndexEntry from .query import Lexeme from .utils import (get_content_type_pk, get_descendants_content_types_pks, get_postgresql_connections, get_sql_weights, get_weight) warnings.warn( "The wagtail.contrib.postgres_search backend is deprecated and has been replaced by " "wagtail.search.backends.database. " "See https://docs.wagtail.org/en/stable/releases/2.15.html#database-search-backends-replaced", category=RemovedInWagtail217Warning) EMPTY_VECTOR = SearchVector(Value('', output_field=TextField())) class ObjectIndexer: """ Responsible for extracting data from an object to be inserted into the index. """ def __init__(self, obj, backend): self.obj = obj self.search_fields = obj.get_search_fields() self.config = backend.config self.autocomplete_config = backend.autocomplete_config def prepare_value(self, value): if isinstance(value, str): return value
def rank_with_terms_and_filters( self, rank_on_field, rank_by_values, rank_threshold, result_fields, relations=None, limit=None, food_group_filters=None, query_set=None, ): """search for foods in db Parameters ---------- rank_on_field: str field than has rank applied to it based on rank_by_values rank_by_valus: str search terms rank_threshold: float value measures relevance of record to search terms. result_fields: list contains fields which are to be returned relations: list fields related by foreign key and thereby used for prefetch_related limit: int number of records to return food_group_filters: list contains food group names with which to filter queryset queryset: QuerySet object any existing QuerySet upon which the search can be executed upon Returns ---------- QuerySet QuerySet containing dicts corresponding to result_fields arg """ if query_set is None: query_set = self.get_queryset() if food_group_filters: query_set = query_set.filter( food_group__name__in=food_group_filters) rank_on_field = SearchVector(rank_on_field) if rank_by_values != ["_all_"]: rank_by_values = self.make_query(rank_by_values) query_set = (query_set.annotate( rank=SearchRank(rank_on_field, rank_by_values)).filter( rank__gte=rank_threshold).order_by("-rank")) else: query_set = query_set.order_by("-date") if relations: for relation in relations: query_set = query_set.prefetch_related(relation) if limit: query_set = query_set[:limit] return query_set.values(*result_fields)