def test_attr_int_facet_filter__empty_params(int_qf, compiler): sq = int_qf.apply(SearchQuery(), {}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000) }).to_dict(compiler=compiler)) qf_res = int_qf.process_result( SearchResult( { 'aggregations': { 'qf.attr_int': { 'buckets': [{ 'key': 0x12_00000001, 'doc_count': 123, }, { 'key': 0x144_0000dead, 'doc_count': 99 }, { 'key': 0x12_f0000000, 'doc_count': 1 }] } } }, aggregations=sq.get_context().aggregations))
def test_exists(self): self.client.search_exists.return_value = {"exists" : True} self.assertEqual( SearchQuery(index=self.index, doc_cls=self.index.car).exists(refresh=True), True ) self.client.search_exists.assert_called_with( index='test', doc_type='car', body=None, refresh=True ) self.client.search_exists.return_value = {"exists" : False} self.assertEqual( SearchQuery(index=self.index) .filter(self.index.car.status == 1) .function_score({'boost_factor': 3}) .exists(), False ) self.client.search_exists.assert_called_with( index='test', doc_type='car', body={ "query": { "filtered": { "filter": { "term": {"status": 1} } } } } )
def test_attr_range_facet_filter__single_selected_filter(range_qf, compiler): sq = range_qf.apply(SearchQuery(), {'a8__gte': 2.71}) assert_search_query( sq, SearchQuery().aggs({ 'qf.attr_range.filter': agg.Filter(Range('attr.float', gte=0x8_402d70a4, lte=0x8_7f800000), aggs={ 'qf.attr_range': agg.Terms(script=Script( 'doc[params.field].value >>> 32', lang='painless', params={ 'field': 'attr.float', }), size=100) }), 'qf.attr_range:8': agg.Filter(Range('attr.float', gte=0x8_00000000, lte=0x8_ffffffff), ) }).post_filter(Range('attr.float', gte=0x8_402d70a4, lte=0x8_7f800000)), compiler) qf_res = range_qf.process_results( SearchResult( { 'aggregations': { 'qf.attr_range.filter': { 'doc_count': 32, 'qf.attr_range': { 'buckets': [{ 'key': 8, 'doc_count': 32 }, { 'key': 439, 'doc_count': 18 }] } }, 'qf.attr_range:8': { 'doc_count': 84 } } }, aggregations=sq.get_context().aggregations)) assert qf_res.attr_range.name == 'attr_range' assert qf_res.attr_range.alias == 'a' f = qf_res.attr_range.get_facet(8) assert f.attr_id == 8 assert f.count == 84 assert f.selected is True f = qf_res.attr_range.get_facet(439) assert f.attr_id == 439 assert f.count == 18 assert f.selected is False
def test_attr_int_facet_filter__include_values(int_qf_with_values, compiler): sq = int_qf_with_values.apply(SearchQuery(), {'a18': '58084'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int.filter': agg.Filter( Term('attr.int', 0x12_0000e2e4), aggs={'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000)}), 'qf.attr_int:18': agg.Terms(Field('attr.int'), size=100, include=[0x12_0000e2e4, 0x12_0000e7e5]) }).post_filter(Term('attr.int', 0x12_0000e2e4)).to_dict(compiler=compiler))
def test_attr_int_simple_filter_no_alias(compiler): qf = QueryFilter() qf.add_filter(AttrIntSimpleFilter('attr', Field('attr.int'))) sq = qf.apply(SearchQuery(), {}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a18': '224'}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'attr18': '1234'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Term('attr.int', 0x12_000004d2)).to_dict(compiler=compiler))
def test_count(self): self.client.count.return_value = { "count" : 1024, "_shards" : { "total" : 5, "successful" : 5, "failed" : 0 } } self.assertEqual( SearchQuery(index=self.index, doc_cls=self.index.car) .count(), 1024 ) self.client.count.assert_called_with( index='test', doc_type='car', body=None, ) self.client.count.return_value = { "count" : 2, "_shards" : { "total" : 5, "successful" : 5, "failed" : 0 } } self.assertEqual( SearchQuery(index=self.index) .filter(self.index.car.status == 1) .function_score({'boost_factor': 3}) .count(), 2 ) self.client.count.assert_called_with( index='test', doc_type='car', body={ "query": { "filtered": { "filter": { "term": {"status": 1} } } } } )
def test_attr_int_facet_filter__single_selected_value(int_qf, compiler): params = {'a18': '58084'} sq = int_qf.apply(SearchQuery(), params) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int.filter': agg.Filter(Term('attr.int', 0x12_0000e2e4), aggs={ 'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000), }), 'qf.attr_int:18': agg.Terms(Field('attr.int'), size=100), }).post_filter(Term('attr.int', 0x12_0000e2e4)).to_dict(compiler=compiler)) qf_res = int_qf.process_result( SearchResult( { 'aggregations': { 'qf.attr_int.filter': { 'doc_count': 201, 'qf.attr_int': { 'buckets': [{ 'key': 0x144_0000dead, 'doc_count': 123, }, { 'key': 0x12_0000e2e4, 'doc_count': 119 }, { 'key': 0x144_0000beef, 'doc_count': 1 }] } }, 'qf.attr_int:18': { 'buckets': [ { 'key': 0x12_0000e2e4, 'doc_count': 99 }, { 'key': 0x12_0000e7e5, 'doc_count': 88 }, ] } } }, aggregations=sq.get_context().aggregations))
def test_multi_search_with_error(self): self.client.msearch = MagicMock( return_value={ u'responses': [{ u'_shards': { u'failed': 0, u'successful': 64, u'total': 64 }, u'hits': { u'hits': [], u'max_score': 0.0, u'total': 27802974 }, u'timed_out': False, u'took': 59 }, { u'error': u'SearchPhaseExecutionException[Failed to execute phase [query], all shards failed;' }] }) ProductDoc = self.index.product sq1 = SearchQuery(doc_cls=ProductDoc, search_type='count', routing=123) sq2 = (SearchQuery( index=self.cluster['us'], doc_cls=ProductDoc).filter(ProductDoc.status == 0).limit(1)) self.assertRaises(MultiSearchError, lambda: self.cluster.multi_search([sq1, sq2])) results = self.cluster.multi_search([sq1, sq2], raise_on_error=False) self.assertIs(results[0], sq1.result) self.assertIs(results[1], sq2.result) self.assertEqual(results[0].total, 27802974) self.assertEqual(results[0].took, 59) self.assertEqual(results[0].timed_out, False) self.assertEqual(results[0].max_score, 0.0) self.assertEqual(len(results[0].hits), 0) self.assertRaisesRegexp(DelayedElasticsearchException, r'^SearchPhaseExecutionException', lambda: results[1].total) self.assertRaisesRegexp(DelayedElasticsearchException, r'^SearchPhaseExecutionException', lambda: results[1].hits) self.assertRaises(AttributeError, lambda: results[1].unknown_attr)
def test_multi_search_with_error(self): self.client.msearch = MagicMock( return_value={ u'responses': [ { u'_shards': { u'failed': 0, u'successful': 64, u'total': 64 }, u'hits': { u'hits': [], u'max_score': 0.0, u'total': 27802974 }, u'timed_out': False, u'took': 59 }, { u'error': u'SearchPhaseExecutionException[Failed to execute phase [query], all shards failed;' } ] } ) ProductDoc = self.index.product sq1 = SearchQuery(doc_cls=ProductDoc, search_type='count', routing=123) sq2 = ( SearchQuery(index=self.cluster['us'], doc_cls=ProductDoc) .filter(ProductDoc.status == 0) .limit(1) ) self.assertRaises(MultiSearchError, lambda: self.cluster.multi_search([sq1, sq2])) results = self.cluster.multi_search([sq1, sq2], raise_on_error=False) self.assertIs(results[0], sq1.get_result()) self.assertIs(results[1], sq2.get_result()) self.assertEqual(results[0].total, 27802974) self.assertEqual(results[0].took, 59) self.assertEqual(results[0].timed_out, False) self.assertEqual(results[0].max_score, 0.0) self.assertEqual(len(results[0].hits), 0) self.assertRaisesRegexp(DelayedElasticsearchException, r'^SearchPhaseExecutionException', lambda: results[1].total) self.assertRaisesRegexp(DelayedElasticsearchException, r'^SearchPhaseExecutionException', lambda: results[1].hits) self.assertRaises(AttributeError, lambda: results[1].unknown_attr)
def test_attr_int_facet_filter__existing_post_filter(int_qf, compiler): sq = int_qf.apply(SearchQuery().post_filter(Range('price', lt=100)), {}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int.filter': agg.Filter( Range('price', lt=100), aggs={'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000)}) }).post_filter(Range('price', lt=100)).to_dict(compiler=compiler)) sq = int_qf.apply( SearchQuery().post_filter(Range('price', lt=100), meta={'price': True}), {}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int.filter': agg.Filter( Range('price', lt=100), aggs={'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000)}) }).post_filter(Range('price', lt=100)).to_dict(compiler=compiler))
def test_attr_range_facet_filter__existing_post_filter(range_qf, compiler): sq = range_qf.apply(SearchQuery().post_filter(Field('status').term(0)), {}) assert_search_query( sq, SearchQuery().aggs({ 'qf.attr_range.filter': agg.Filter(Term('status', 0), aggs={ 'qf.attr_range': agg.Terms(script=Script( 'doc[params.field].value >>> 32', lang='painless', params={ 'field': 'attr.float', }), size=100), }), }).post_filter(Term('status', 0)), compiler)
def test_suggest(self): sq = SearchQuery() sq = sq.suggest(text="Complete", in_title={'term': {'size': 3, 'field': 'title'}}) self.assert_expression( sq, { 'suggest': { 'text': 'Complete', 'in_title': { 'term': { 'size': 3, 'field': 'title', } } } } ) sq = sq.suggest(in_body={'completion': {'field': 'body'}}) self.assert_expression( sq, { 'suggest': { 'text': 'Complete', 'in_title': { 'term': { 'size': 3, 'field': 'title', } }, 'in_body': { 'completion': { 'field': 'body', } }, } } ) sq = sq.suggest(None) self.assert_expression(sq, {})
def test_attr_bool_facet_filter__empty_params(bool_qf, compiler): sq = bool_qf.apply(SearchQuery(), {}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_bool': agg.Terms(Field('attr.bool'), size=100) }).to_dict(compiler=compiler)) qf_res = bool_qf.process_result( SearchResult( { 'aggregations': { 'qf.attr_bool': { 'buckets': [{ 'key': 0b11, 'doc_count': 123, }, { 'key': 0b10, 'doc_count': 99 }, { 'key': 0b101, 'doc_count': 1 }] } } }, aggregations=sq.get_context().aggregations)) assert len(qf_res.attr_bool.facets) == 2 facet = qf_res.attr_bool.get_facet(1) assert len(facet.all_values) == 2 assert facet.all_values[0].value is True assert facet.all_values[0].count == 123 assert facet.all_values[0].count_text == '123' assert facet.all_values[0].selected is False assert facet.all_values[1].value is False assert facet.all_values[1].count == 99 assert facet.all_values[1].count_text == '99' assert facet.all_values[1].selected is False facet = qf_res.attr_bool.get_facet(2) assert len(facet.all_values) == 1 assert facet.all_values[0].value is True assert facet.all_values[0].count == 1 assert facet.all_values[0].count_text == '1' assert facet.all_values[0].selected is False
def run(options): """Run benchmark.""" prof = cProfile.Profile() cov = coverage.Coverage() times = OrderedDict.fromkeys(['data_load', 'json_loads', 'searchResult']) start = time.monotonic() * 1000 raw_data = options.input.read() times['data_load'] = time.monotonic() * 1000 - start start = time.monotonic() * 1000 raw_results = json.loads(raw_data) times['json_loads'] = time.monotonic() * 1000 - start query = SearchQuery(MatchAll(), doc_cls=SimpleDocument) if 'aggregations' in raw_results: query = query.aggs(terms=Terms(SimpleDocument.integer_0)) gc.disable() if options.profile: cov.start() prof.enable() SearchResult( raw_results, query._aggregations, doc_cls=query._get_doc_cls(), instance_mapper=query._instance_mapper) times['searchResult'] = time.monotonic() * 1000 - start if options.profile: prof.disable() cov.stop() gc.enable() for key, duration in times.items(): print("Took {} {:10.3f}ms".format(key, duration)) if options.profile: prof.print_stats('cumulative') cov.report() cov.html_report()
def test_attr_range_facet_filter__empty(range_qf, compiler): sq = range_qf.apply(SearchQuery(), {}) assert_search_query( sq, SearchQuery().aggs({ 'qf.attr_range': agg.Terms(script=Script('doc[params.field].value >>> 32', lang='painless', params={ 'field': 'attr.float', }), size=100), }), compiler) qf_res = range_qf.process_results( SearchResult( { 'aggregations': { 'qf.attr_range': { 'buckets': [{ 'key': '8', 'doc_count': 84 }, { 'key': '439', 'doc_count': 28 }] } } }, aggregations=sq.get_context().aggregations)) assert qf_res.attr_range.name == 'attr_range' assert qf_res.attr_range.alias == 'a' f = qf_res.attr_range.get_facet(8) assert f.attr_id == 8 assert f.count == 84 assert f.selected is False f = qf_res.attr_range.get_facet(439) assert f.attr_id == 439 assert f.count == 28 assert f.selected is False
def test_exists(self): self.client.search.return_value = { "hits": {"total": 1, "max_score": 1.0, "hits": []} } self.assertEqual( SearchQuery(index=self.index, doc_cls=self.index['car']).exists(), True ) self.client.search.assert_called_with( index='test', doc_type='car', body={'size': 0}, terminate_after=1, ) self.client.search.return_value = { "hits": {"total": 0, "max_score": 0.0, "hits": []} } self.assertEqual( SearchQuery(index=self.index) .filter(self.index['car'].status == 1) .function_score({'boost_factor': 3}) .exists(), False ) self.client.search.assert_called_with( index='test', doc_type='car', body={ "size": 0, "query": { "bool": { "filter": { "term": {"status": 1} } } } }, terminate_after=1, )
def run(options): """Run benchmark.""" prof = cProfile.Profile() cov = coverage.Coverage() times = OrderedDict.fromkeys(['data_load', 'json_loads', 'searchResult']) start = time.monotonic() * 1000 raw_data = options.input.read() times['data_load'] = time.monotonic() * 1000 - start start = time.monotonic() * 1000 raw_results = json.loads(raw_data) times['json_loads'] = time.monotonic() * 1000 - start query = SearchQuery(MatchAll(), doc_cls=SimpleDocument) if 'aggregations' in raw_results: query = query.aggs(terms=Terms(SimpleDocument.integer_0)) gc.disable() if options.profile: cov.start() prof.enable() SearchResult(raw_results, query._aggregations, doc_cls=query._get_doc_cls(), instance_mapper=query._instance_mapper) times['searchResult'] = time.monotonic() * 1000 - start if options.profile: prof.disable() cov.stop() gc.enable() for key, duration in times.items(): print("Took {} {:10.3f}ms".format(key, duration)) if options.profile: prof.print_stats('cumulative') cov.report() cov.html_report()
def test_search_params(self): sq = SearchQuery() self.assertEqual( sq._search_params, {} ) sq = SearchQuery(search_type='count') self.assertEqual( sq._search_params, { 'search_type': 'count' } ) sq = sq.with_search_type(None) self.assertEqual( sq._search_params, {} ) sq = sq.with_search_params({'search_type': 'count', 'query_cache': True}, unknown_param='none') self.assertEqual( sq._search_params, { 'search_type': 'count', 'query_cache': True, 'unknown_param': 'none', } ) sq = sq.with_routing(1234) self.assertEqual( sq._search_params, { 'routing': 1234, 'search_type': 'count', 'query_cache': True, 'unknown_param': 'none', } )
def test_scroll(es_index, cars): with pytest.warns(UserWarning, match='Cannot determine document class'): search_res = es_index.search( SearchQuery(), scroll='1m', ) assert search_res.total == 2 assert len(search_res.hits) == 2 assert search_res.scroll_id is not None scroll_res = es_index.scroll(search_res.scroll_id, scroll='1m') assert scroll_res.total == 2 assert len(scroll_res.hits) == 0 clear_scroll_res = es_index.clear_scroll(scroll_res.scroll_id) assert clear_scroll_res.succeeded is True
def test_aggregations(self): f = DynamicDocument.fields sq = SearchQuery().aggregations(min_price=agg.Min(f.price)) self.assert_expression( sq, { "aggregations": { "min_price": { "min": {"field": "price"} } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = SearchQuery().aggregations(genders=agg.Terms(f.gender)) self.assert_expression( sq, { "aggregations": { "genders": { "terms": {"field": "gender"} } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .aggregations( type=agg.Terms(f.type, aggs={'min_price': agg.Min(f.price)}) ) ) self.assert_expression( sq, { "aggregations": { "type": { "terms": {"field": "type"}, "aggregations": { "min_price": { "min": {"field": "price"} } } } } }, ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .aggregations( top_tags=( agg.Terms( f.tags, size=3, aggs={ 'top_tag_hits': agg.TopHits( sort=f.last_activity_date.desc(), size=1, _source=Params(include=[f.title])) } ) ) ) ) self.assert_expression( sq, { "aggregations": { "top_tags": { "terms": { "field": "tags", "size": 3 }, "aggregations": { "top_tag_hits": { "top_hits": { "sort": { "last_activity_date": "desc" }, "_source": { "include": ["title"] }, "size" : 1 } } } } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .aggregations({ 'top_sites': agg.Terms( f.domain, order=Sort('top_hit', 'desc'), aggs={ 'top_tags_hits': agg.TopHits(), 'top_hit': agg.Max(script='_doc.score'), } ) }) ) self.assert_expression( sq, { "aggregations": { "top_sites": { "terms": { "field": "domain", "order": { "top_hit": "desc" } }, "aggregations": { "top_tags_hits": { "top_hits": {} }, "top_hit" : { "max": { "script": "_doc.score" } } } } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument})
def test_search_query_compile(self): f = DynamicDocument.fields sq = SearchQuery() self.assert_expression(sq, {}) self.assertEqual(collect_doc_classes(sq), set()) sq = SearchQuery(Term(f.user, 'kimchy')).limit(10).offset(0) self.assert_expression( sq, { "from": 0, "size": 10, "query": { "term": {"user": "******"} } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = SearchQuery(Term(f.user, 'kimchy')).filter(f.age >= 16) self.assert_expression( sq, { "query": { "filtered": { "query": { "term": {"user": "******"} }, "filter": { "range": { "age": {"gte": 16} } } } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = SearchQuery(Term(f.user, 'kimchy'), _compiler=QueryCompiled20).filter(f.age >= 16) self.assert_expression( sq, { "query": { "bool": { "must": { "term": {"user": "******"} }, "filter": { "range": { "age": {"gte": 16} } } } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery(Term(f.user, 'kimchy')) .query(f.user != 'kimchy') ) self.assert_expression( sq, { "query": { "bool": { "must_not": [ { "term": {"user": "******"} } ] } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery(Term(f.user, 'kimchy')) .query(None) ) self.assert_expression(sq, {}) self.assertEqual(collect_doc_classes(sq), set()) sq = ( SearchQuery(Term(f.user, 'kimchy')) .filter(f.age >= 16) .filter(f.lang == 'English') ) self.assert_expression( sq, { "query": { "filtered": { "query": { "term": {"user": "******"} }, "filter": { "bool": { "must": [ { "range": { "age": {"gte": 16} } }, { "term": { "lang": "English" } } ] } } } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .order_by( f.opinion_rating.desc(missing='_last'), f.opinion_count.desc(), f.id ) ) self.assert_expression( sq, { "sort": [ { "opinion_rating": { "order": "desc", "missing": "_last" } }, { "opinion_count": "desc" }, "id" ] } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .order_by( f.opinion_rating.desc(missing='_last'), f.opinion_count.desc(), f.id ) .order_by(None) .order_by(None) ) self.assert_expression(sq, {}) self.assertEqual(collect_doc_classes(sq), set()) sq = SearchQuery().source(f.name, f.company) self.assert_expression( sq, { "_source": ["name", "company"] } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = SearchQuery().source(exclude=[f.name, f.company]) self.assert_expression( sq, { "_source": { "exclude": ["name", "company"] } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .source( include=[f.obj1.wildcard('*'), f.obj2.wildcard('*')], # FIXME: f.wildcard('*') exclude=DynamicDocument.wildcard('*').description ) ) self.assert_expression( sq, { "_source": { "include": ["obj1.*", "obj2.*"], "exclude": "*.description" } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .source(None) .source(f.name, f.company) .source(None) ) self.assert_expression(sq, {}) self.assertEqual(collect_doc_classes(sq), set()) sq = ( SearchQuery() .source(f.name, f.company) .source(False) ) self.assert_expression( sq, { "_source": False } ) self.assertEqual(collect_doc_classes(sq), set()) sq = ( SearchQuery() .source(True) ) self.assert_expression( sq, { "_source": True } ) self.assertEqual(collect_doc_classes(sq), set()) sq = SearchQuery().fields(f.name, f.company) self.assert_expression( sq, { "fields": ["name", "company"] } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .fields(True) ) self.assert_expression( sq, { "fields": '*' } ) self.assertEqual(collect_doc_classes(sq), set()) sq = ( SearchQuery() .fields(None) .fields(f.name, f.company) .fields(None) ) self.assert_expression(sq, {}) self.assertEqual(collect_doc_classes(sq), set()) sq = ( SearchQuery() .fields(f.name, f.company) .fields(False) ) self.assert_expression( sq, { "fields": [] } ) self.assertEqual(collect_doc_classes(sq), set()) self.assert_expression( SearchQuery() .function_score({'random_score': {"seed": 1234}}), { "query": { "function_score": { "functions": [ { "random_score": {"seed": 1234} } ], } } } ) sq = ( SearchQuery(MultiMatch('Iphone 6', fields=[f.name, f.description])) .filter(f.status == 0) .function_score(None) .function_score({'_score': {"seed": 1234}}) .function_score(None) .function_score({'field_value_factor': {'field': f.popularity, 'factor': 1.2, 'modifier': 'sqrt'}}, boost_mode='sum') .function_score({'boost_factor': 3, 'filter': f.region == 12}) ) self.assert_expression( sq, { "query": { "filtered": { "query": { "function_score": { "query": { "multi_match": { "query": "Iphone 6", "fields": ["name", "description"] } }, "functions": [ { "field_value_factor": { "field": "popularity", "factor": 1.2, "modifier": "sqrt" } }, { "filter": { "term": {"region": 12} }, "boost_factor": 3 } ], "boost_mode": "sum" } }, "filter": { "term": {"status": 0} } } } } ) self.assertEqual(collect_doc_classes(sq), {DynamicDocument}) sq = ( SearchQuery() .filter(f.status == 0) .boost_score( {'filter': f.discount_percent >= 10, 'weight': 1000}, {'filter': f.discount_percent >= 50, 'weight': 2000}, {'filter': f.presence == 'available', 'weight': 10000}, ) ) self.assert_expression( sq, { "query": { "filtered": { "query": { "function_score": { "functions": [ { "filter": {"range": {"discount_percent": {"gte": 10}}}, "weight": 1000 }, { "filter": {"range": {"discount_percent": {"gte": 50}}}, "weight": 2000 }, { "filter": {"term": {"presence": "available"}}, "weight": 10000 }, ], "score_mode": "sum", "boost_mode": "sum" } }, "filter": { "term": {"status": 0} } } } } ) sq = ( SearchQuery(f.name.match('test')) .filter(f.status == 0) .function_score( {'field_value_factor': {'field': f.popularity}}, ) .boost_score( {'filter': f.discount_percent >= 10, 'weight': 100}, ) .boost_score(None) .boost_score( {'filter': f.discount_percent >= 10, 'weight': 1000}, {'filter': f.discount_percent >= 50, 'weight': 2000}, score_mode='max', ) ) self.assert_expression( sq, { "query": { "filtered": { "query": { "function_score": { "query": { "function_score": { "query": { "match": { "name": "test" } }, "functions": [ { "field_value_factor": { "field": "popularity" } } ] } }, "functions": [ { "filter": {"range": {"discount_percent": {"gte": 10}}}, "weight": 1000 }, { "filter": {"range": {"discount_percent": {"gte": 50}}}, "weight": 2000 }, ], "score_mode": "max", "boost_mode": "sum" } }, "filter": { "term": {"status": 0} } } } } ) sq = ( SearchQuery() .rescore( QueryRescorer( self.index.t.field1.match('the quick brown', type='phrase', slop=2) ) ) .rescore(None) .rescore( QueryRescorer( self.index.t.field1.match('the quick brown fox', type='phrase', slop=2), query_weight=0.7, rescore_query_weight=1.2 ), window_size=100, ) .rescore( QueryRescorer( FunctionScore(script_score={'script': "log10(doc['numeric'].value + 2)"}), score_mode='multiply' ), window_size=10, ) ) self.assert_expression( sq, { "rescore": [ { "window_size": 100, "query": { "rescore_query": { "match": { "field1": { "query": "the quick brown fox", "type": "phrase", "slop": 2 } } }, "query_weight": 0.7, "rescore_query_weight": 1.2 } }, { "window_size": 10, "query": { "score_mode": "multiply", "rescore_query": { "function_score": { "script_score": { "script": "log10(doc['numeric'].value + 2)" } } } } } ] } ) self.assertEqual(collect_doc_classes(sq), {self.index.t}) sq = SearchQuery().post_filter(self.index.shirt.color == 'red') self.assert_expression( sq, { "post_filter": { "term": {"color": "red"} } } ) self.assertEqual(collect_doc_classes(sq), {self.index.shirt}) sq = ( SearchQuery() .filter(self.index.shirt.brand == 'gucci') .post_filter(self.index.shirt.color == 'red') .post_filter(self.index.shirt.model == 't-shirt') ) self.assert_expression( sq, { "query": { "filtered": { "filter": { "term": {"brand": "gucci"} } } }, "post_filter": { "bool": { "must": [ {"term": {"color": "red"}}, {"term": {"model": "t-shirt"}} ] } } } ) self.assertEqual(collect_doc_classes(sq), {self.index.shirt})
def test_highlight(self): sq = SearchQuery() sq = sq.highlight(fields={'content': {}}) self.assertEqual(collect_doc_classes(sq), set()) self.assert_expression( sq, { "highlight": { "fields": { "content": {} } } } ) sq = SearchQuery() sq = sq.highlight( fields=[self.index.test.content], pre_tags=['[em]'], post_tags=['[/em]'] ) self.assertEqual(collect_doc_classes(sq), {self.index.test}) self.assert_expression( sq, { "highlight": { "fields": [ { "content": {} } ], "pre_tags": ["[em]"], "post_tags": ["[/em]"] } } ) sq = SearchQuery() sq = sq.highlight( fields=[ self.index.test.content.highlight( matched_fields=[self.index.test.content, self.index.test.content.plain], type='fvh', ) ] ) self.assertEqual(collect_doc_classes(sq), {self.index.test}) self.assert_expression( sq, { "highlight": { "fields": [ { "content": { "matched_fields": ["content", "content.plain"], "type": "fvh" } } ] } } )
def test_combined_facet_filters(qf, compiler): sq = qf.apply(SearchQuery(), { 'a1': 'true', 'a18': '58084', 'a324': '57005', 'a8__gte': '2.71', }) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_bool.filter': agg.Filter(Bool.must( Term('attr.bool', 0b11), Term('attr.int', 0x12_0000e2e4), Term('attr.int', 0x144_0000dead), ), aggs={ 'qf.attr_bool': agg.Terms(Field('attr.bool'), size=100), }), 'qf.attr_bool.filter:1': agg.Filter(Bool.must( Term('attr.int', 0x12_0000e2e4), Term('attr.int', 0x144_0000dead), ), aggs={ 'qf.attr_bool:1': agg.Terms( Field('attr.bool'), size=2, include=[0b10, 0b11], ), }), 'qf.attr_int.filter': agg.Filter(Bool.must( Term('attr.bool', 0b11), Term('attr.int', 0x12_0000e2e4), Term('attr.int', 0x144_0000dead), ), aggs={ 'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000), }), 'qf.attr_int.filter:18': agg.Filter(Bool.must( Term('attr.bool', 0b11), Term('attr.int', 0x144_0000dead), ), aggs={ 'qf.attr_int:18': agg.Terms(Field('attr.int'), size=100), }), 'qf.attr_int.filter:324': agg.Filter(Bool.must( Term('attr.bool', 0b11), Term('attr.int', 0x12_0000e2e4), ), aggs={ 'qf.attr_int:324': agg.Terms(Field('attr.int'), size=100), }) }).post_filter(Term('attr.bool', 0b11)).post_filter( Term('attr.int', 0x12_0000e2e4)).post_filter( Term('attr.int', 0x144_0000dead)).filter( Range('attr.float', gte=0x8_402d70a4, lte=0x8_7f800000)).to_dict(compiler=compiler))
def test_attr_bool_facet_filter__unknown_param(bool_qf, compiler): sq = bool_qf.apply(SearchQuery(), {'b18': 'true'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_bool': agg.Terms(Field('attr.bool'), size=100) }).to_dict(compiler=compiler))
def test_attr_bool_facet_filter__multiple_selected_values(bool_qf, compiler): sq = bool_qf.apply(SearchQuery(), {'a1': ['true', 'false'], 'a2': 'true'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_bool.filter': agg.Filter( Bool.must( Terms('attr.bool', [0b11, 0b10]), Term('attr.bool', 0b101), ), aggs={'qf.attr_bool': agg.Terms(Field('attr.bool'), size=100)}), 'qf.attr_bool.filter:1': agg.Filter(Term('attr.bool', 0b101), aggs={ 'qf.attr_bool:1': agg.Terms(Field('attr.bool'), size=2, include=[0b10, 0b11]) }), 'qf.attr_bool.filter:2': agg.Filter(Terms('attr.bool', [0b11, 0b10]), aggs={ 'qf.attr_bool:2': agg.Terms(Field('attr.bool'), size=2, include=[0b100, 0b101]) }), }).post_filter( Bool.must( Terms('attr.bool', [0b11, 0b10]), Term('attr.bool', 0b101), )).to_dict(compiler=compiler)) qf_res = bool_qf.process_result( SearchResult( { 'aggregations': { 'qf.attr_bool.filter': { 'doc_count': 200, 'qf.attr_bool': { 'buckets': [ { 'key': 0b11, 'doc_count': 123, }, { 'key': 0b101, 'doc_count': 1 }, ] } }, 'qf.attr_bool.filter:1': { 'doc_count': 163, 'qf.attr_bool:1': { 'buckets': [ { 'key': 0b11, 'doc_count': 123, }, { 'key': 0b10, 'doc_count': 99 }, ] } }, 'qf.attr_bool.filter:2': { 'doc_count': 144, 'qf.attr_bool:2': { 'buckets': [ { 'key': 0b101, 'doc_count': 1 }, ] } }, } }, aggregations=sq.get_context().aggregations)) assert len(qf_res.attr_bool.facets) == 2 facet = qf_res.attr_bool.get_facet(1) assert len(facet.all_values) == 2 assert len(facet.selected_values) == 2 assert len(facet.values) == 0 assert facet.all_values[0] is facet.selected_values[0] assert facet.all_values[1] is facet.selected_values[1] assert facet.all_values[0].value is True assert facet.all_values[0].count == 123 assert facet.all_values[0].count_text == '123' assert facet.all_values[0].selected is True assert facet.all_values[1].value is False assert facet.all_values[1].count == 99 assert facet.all_values[1].count_text == '99' assert facet.all_values[1].selected is True facet = qf_res.attr_bool.get_facet(2) assert len(facet.all_values) == 1 assert len(facet.selected_values) == 1 assert len(facet.values) == 0 assert facet.all_values[0] is facet.selected_values[0] assert facet.all_values[0].value is True assert facet.all_values[0].count == 1 assert facet.all_values[0].count_text == '1' assert facet.all_values[0].selected is True
def test_multi_search(self): self.client.msearch = MagicMock( return_value={ u'responses': [{ u'_shards': { u'failed': 0, u'successful': 64, u'total': 64 }, u'hits': { u'hits': [], u'max_score': 0.0, u'total': 27802974 }, u'timed_out': False, u'took': 59 }, { u'_shards': { u'failed': 0, u'successful': 16, u'total': 16 }, u'hits': { u'hits': [{ u'_id': u'56565215', u'_index': u'ca', u'_type': u'product', u'_score': 1.0, u'_source': { u'name': u'Gold usb cable', u'price': { u'local': 92.421, u'unit': 5.67 }, u'status': 0 }, }], u'max_score': 1.0, u'total': 272 }, u'timed_out': False, u'took': 53 }] }) ProductDoc = self.index.product sq1 = SearchQuery(doc_cls=ProductDoc, search_type='count', routing=123) sq2 = (SearchQuery( index=self.cluster['us'], doc_cls=ProductDoc).filter(ProductDoc.status == 0).limit(1)) results = self.cluster.multi_search([sq1, sq2]) self.client.msearch.assert_called_with(body=[{ 'type': 'product', 'search_type': 'count', 'routing': 123 }, {}, { 'index': 'us', 'type': 'product' }, { 'query': { 'bool': { 'filter': { 'term': { 'status': 0 } } } }, 'size': 1 }]) self.assertIs(results[0], sq1.result) self.assertIs(results[1], sq2.result) self.assertEqual(results[0].total, 27802974) self.assertEqual(len(results[0].hits), 0) self.assertEqual(results[1].total, 272) self.assertEqual(len(results[1].hits), 1) doc = results[1].hits[0] self.assertEqual(doc._id, '56565215') self.assertEqual(doc._index, 'ca') self.assertEqual(doc._type, 'product') self.assertAlmostEqual(doc._score, 1.0) self.assertEqual(doc.name, 'Gold usb cable') self.assertAlmostEqual(doc.price.local, 92.421) self.assertAlmostEqual(doc.price.unit, 5.67) self.assertEqual(doc.status, 0)
def _apply_filter_expression(self, search_query: SearchQuery, expr: Expression, attr_id: int) -> SearchQuery: return search_query.filter(expr)
def test_attr_int_simple_filter(compiler): qf = QueryFilter() qf.add_filter(AttrIntSimpleFilter('attr_int', Field('attr.int'), alias='a')) sq = qf.apply(SearchQuery(), {}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'b18': '224'}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a18': '1234'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Term('attr.int', 0x12_000004d2)).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a18': ['1234', '5678']}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Terms('attr.int', [0x12_000004d2, 0x12_0000162e])).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a18': ['1234', '5678'], 'a324': '90'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Terms('attr.int', [0x12_000004d2, 0x12_0000162e])).filter( Term('attr.int', 0x144_0000005a)).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a18': '0x1234'}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a18-19': '1234'}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a2147483648': '1'}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a1': '2147483648'}) assert sq.to_dict(compiler=compiler) == {}
def test_attr_range_facet_filter__multiple_selected_filters( range_qf, compiler): sq = range_qf.apply(SearchQuery(), {'a8__gte': 2.71, 'a99__lte': 3.14}) assert_search_query( sq, SearchQuery().aggs({ 'qf.attr_range.filter': agg.Filter( Bool.must( Range('attr.float', gte=0x8_402d70a4, lte=0x8_7f800000), Bool.should( Range('attr.float', gte=0x63_00000000, lte=0x63_4048f5c3), Range('attr.float', gte=0x63_80000000, lte=0x63_ff800000))), aggs={ 'qf.attr_range': agg.Terms(script=Script('doc[params.field].value >>> 32', lang='painless', params={ 'field': 'attr.float', }), size=100) }), 'qf.attr_range:8': agg.Filter( Bool.must( Bool.should( Range('attr.float', gte=0x63_00000000, lte=0x63_4048f5c3), Range('attr.float', gte=0x63_80000000, lte=0x63_ff800000)), Range('attr.float', gte=0x8_00000000, lte=0x8_ffffffff))), 'qf.attr_range:99': agg.Filter( Bool.must( Range('attr.float', gte=0x8_402d70a4, lte=0x8_7f800000), Range('attr.float', gte=0x63_00000000, lte=0x63_ffffffff))), }).post_filter(Range('attr.float', gte=0x8_402d70a4, lte=0x8_7f800000)).post_filter( Bool.should( Range('attr.float', gte=0x63_00000000, lte=0x63_4048f5c3), Range('attr.float', gte=0x63_80000000, lte=0x63_ff800000))), compiler) qf_res = range_qf.process_results( SearchResult( { 'aggregations': { 'qf.attr_range.filter': { 'doc_count': 32, 'qf.attr_range': { 'buckets': [{ 'key': 8, 'doc_count': 32 }, { 'key': 99, 'doc_count': 18 }] } }, 'qf.attr_range:8': { 'doc_count': 84 }, 'qf.attr_range:99': { 'doc_count': 33 } } }, aggregations=sq.get_context().aggregations)) assert qf_res.attr_range.name == 'attr_range' assert qf_res.attr_range.alias == 'a' f = qf_res.attr_range.get_facet(8) assert f.attr_id == 8 assert f.count == 84 assert f.selected is True f = qf_res.attr_range.get_facet(99) assert f.attr_id == 99 assert f.count == 33 assert f.selected is True
def test_multi_search(self): self.client.msearch = MagicMock( return_value={ u'responses': [ { u'_shards': { u'failed': 0, u'successful': 64, u'total': 64 }, u'hits': { u'hits': [], u'max_score': 0.0, u'total': 27802974 }, u'timed_out': False, u'took': 59 }, { u'_shards': { u'failed': 0, u'successful': 16, u'total': 16 }, u'hits': { u'hits': [ { u'_id': u'56565215', u'_index': u'ca', u'_type': u'product', u'_score': 1.0, u'_source': { u'name': u'Gold usb cable', u'price': {u'local': 92.421, u'unit': 5.67}, u'status': 0 }, } ], u'max_score': 1.0, u'total': 272 }, u'timed_out': False, u'took': 53 } ] } ) ProductDoc = self.index.product sq1 = SearchQuery(doc_cls=ProductDoc, search_type='count', routing=123) sq2 = ( SearchQuery(index=self.cluster['us'], doc_cls=ProductDoc) .filter(ProductDoc.status == 0) .limit(1) ) results = self.cluster.multi_search([sq1, sq2]) self.client.msearch.assert_called_with( body=[ {'type': 'product', 'search_type': 'count', 'routing': 123}, {}, {'index': 'us', 'type': 'product'}, {'query': {'filtered': {'filter': {'term': {'status': 0}}}}, 'size': 1} ] ) self.assertIs(results[0], sq1.get_result()) self.assertIs(results[1], sq2.get_result()) self.assertEqual(results[0].total, 27802974) self.assertEqual(len(results[0].hits), 0) self.assertEqual(results[1].total, 272) self.assertEqual(len(results[1].hits), 1) doc = results[1].hits[0] self.assertEqual(doc._id, '56565215') self.assertEqual(doc._index, 'ca') self.assertEqual(doc._type, 'product') self.assertAlmostEqual(doc._score, 1.0) self.assertEqual(doc.name, 'Gold usb cable') self.assertAlmostEqual(doc.price.local, 92.421) self.assertAlmostEqual(doc.price.unit, 5.67) self.assertEqual(doc.status, 0)
def test_attr_int_facet_filter__multiple_selected_values(int_qf, compiler): sq = int_qf.apply(SearchQuery(), { 'a18': '58084', 'a324': ['57005', '48879'] }) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int.filter': agg.Filter(Bool.must( Term('attr.int', 0x12_0000e2e4), Terms('attr.int', [0x144_0000dead, 0x144_0000beef]), ), aggs={ 'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000), }), 'qf.attr_int.filter:18': agg.Filter(Terms('attr.int', [0x144_0000dead, 0x144_0000beef]), aggs={ 'qf.attr_int:18': agg.Terms(Field('attr.int'), size=100), }), 'qf.attr_int.filter:324': agg.Filter(Term('attr.int', 0x12_0000e2e4), aggs={ 'qf.attr_int:324': agg.Terms(Field('attr.int'), size=100), }) }).post_filter(Term('attr.int', 0x12_0000e2e4)).post_filter( Terms('attr.int', [0x144_0000dead, 0x1440000beef])).to_dict(compiler=compiler)) qf_res = int_qf.process_result( SearchResult( { 'aggregations': { 'qf.attr_int.filter': { 'doc_count': 404, 'qf.attr_int': { 'buckets': [{ 'key': 0x144_0000dead, 'doc_count': 1 }, { 'key': 0x12_0000e2e4, 'doc_count': 1 }, { 'key': 0x144_0000beef, 'doc_count': 1 }] } }, 'qf.attr_int.filter:18': { 'doc_count': 200, 'qf.attr_int:18': { 'buckets': [ { 'key': 0x12_0000e2e4, 'doc_count': 99 }, { 'key': 0x12_0000e7e5, 'doc_count': 88 }, ] } }, 'qf.attr_int.filter:324': { 'doc_count': 200, 'qf.attr_int:324': { 'buckets': [ { 'key': 0x144_0000dead, 'doc_count': 123 }, { 'key': 0x144_0000beef, 'doc_count': 1 }, ] } }, } }, aggregations=sq.get_context().aggregations))
def test_attr_int_facet_filter__unknown_param(int_qf, compiler): sq = int_qf.apply(SearchQuery(), {'b18': '224'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().aggs({ 'qf.attr_int': agg.Terms(Field('attr.int'), size=10_000) }).to_dict(compiler=compiler))
def test_attr_range_simple_filter(compiler): qf = QueryFilter() qf.add_filter( AttrRangeSimpleFilter('attr_float', Field('attr.float'), alias='a')) sq = qf.apply(SearchQuery(), {}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a8__gte': '3.14'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Range('attr.float', gte=0x8_4048f5c3, lte=0x8_7f800000)).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a8__gte': '-3.14'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Bool.should(Range('attr.float', gte=0x8_80000000, lte=0x8_c048f5c3), Range('attr.float', gte=0x8_00000000, lte=0x8_7f800000))).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a8__lte': '-2.71'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Range('attr.float', gte=0x8_c02d70a4, lte=0x8_ff800000)).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a8__gte': ['1', '2.71'], 'a8__lte': '3.14'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Range('attr.float', gte=0x8_402d70a4, lte=0x8_4048f5c3)).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a8__gte': '-3.14', 'a8__lte': '-2.71'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Range('attr.float', gte=0x8_c02d70a4, lte=0x8_c048f5c3)).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a8__gte': '-3.14', 'a8__lte': '3.14'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Bool.should(Range('attr.float', gte=0x8_80000000, lte=0x8_c048f5c3), Range('attr.float', gte=0x8_00000000, lte=0x8_4048f5c3))).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a8__gte': '3.14', 'a8__lte': '-3.14'}) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Bool.must(Range('attr.float', gte=0x8_4048f5c3, lte=0x8_7f800000), Range('attr.float', gte=0x8_c048f5c3, lte=0x8_ff800000))).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), { 'a8__gte': '2.71', 'a8__lte': '3.14', 'a99__lte': '99' }) assert sq.to_dict(compiler=compiler) == (SearchQuery().filter( Range('attr.float', gte=0x8_402d70a4, lte=0x8_4048f5c3)).filter( Bool.should( Range('attr.float', gte=0x63_00000000, lte=0x63_42c60000), Range('attr.float', gte=0x63_80000000, lte=0x63_ff800000))).to_dict(compiler=compiler)) sq = qf.apply(SearchQuery(), {'a99.9__gte': '99.9'}) assert sq.to_dict(compiler=compiler) == {} sq = qf.apply(SearchQuery(), {'a99__gte': '100ee2'}) assert sq.to_dict(compiler=compiler) == {}