def query_images(object_list): hit1 = set() image_set = set() print("11. object_list =", object_list) QI = Q('match_all') s1 = Search(index='idxo20') for objectk in object_list: print("objectk= ",objectk) QI = QI & Q("match", names=objectk) s1 = s1.query(QI).using(client) s1.execute() for hit in s1.scan() : print("33 ", hit.imgfile) image_set.add(hit.imgfile) print("image_set = {0}".format(image_set)) im = 0 for image in image_set : if im > 3 : break file, ext = os.path.splitext(image) image = file + '.png' print("66 image =", image) images_div.append(display_image(image)) im = im + 1 print("Please hit refresh...")
def get_imgfile(img_id): s = Search(index='idx1').query('match', image_id=img_id) s = s.using(client) s.execute() for hit in s.scan(): imgfile = hit.imagefile return imgfile
def get_imgfile(img_id): s = Search(index='idxi20').query('match', image_id=img_id) s = s.using(client) s.execute() for hit in s.scan(): imgfile = hit.imagefile #imagefile = hit.url #print("22 imgfile = ", imgfile) return imgfile
def test_count_prefetch(data_client, mocker): mocker.spy(data_client, "count") search = Search(using=data_client).index("git") search.execute() assert search.count() == 53 assert data_client.count.call_count == 0 search._response.hits.total.relation = "gte" assert search.count() == 53 assert data_client.count.call_count == 1
def query_imagesi(classnum_list, classval_list, upfilename): #print("In query_imagesi") hit1 = set() image_set = set() #print("11. classnum_list =", classnum_list) #print("12. classval_list =", classval_list) if (len(classnum_list) > 7): lenclassnum = 7 QI = Q('match_all') s1 = Search(index='adknum') #s1 = Search(index='vgnum') classn = 1 for class_num in classnum_list: if classn > 7: #can make this 5-- break classn = classn + 1 QI = QI & Q('bool', must=[Q("match", classnum=class_num)]) s1 = s1.query(QI).using(client) response = s1.execute() hit_num = 0 simDict = {} similarityClass = np.zeros(15) for hit in s1.scan(): print("123. hit.classnum: ", hit.classnum) print("124. hit.classval: ", hit.classval) lenimgclassnum = len(hit.classnum) simDict[hit.imgfile] = 1.0 #similarityClass[lenimgclassnum] # Compute similarity and choose top 4 rather than random 4 ii = 0 similarityImg = 0.0 for classi in classnum_list: jj = 0 for classj in hit.classnum: if classi == int(classj): similarityClass[ii] = classval_list[ii] / (abs( (classval_list[ii] - int(hit.classval[jj]))) + 10) #print("144. similarityClass[ii] = ", similarityClass[ii]) similarityImg = similarityImg + similarityClass[ii] break jj = jj + 1 simDict[hit.imgfile] = similarityImg #print("130. simDict[hit.imgfile] = ", simDict[hit.imgfile], similarityImg) ii = ii + 1 #for key in sorted(simDict.keys(), reverse=True) : kk = 0 for img in sorted(simDict, key=simDict.get, reverse=True): print("140. ", img, simDict[img]) image_set.add(img) #image_set.add(hit.imgfile) #pick top 4 images (hit_nums) kk = kk + 1 if kk > 3: break return display_image_set(image_set, upfilename, '')
def post(self): ts = self.args['_'] if abs(int(time() * 1000) - int(ts)) > 1800000: return {'success':0, 'message': '时间戳无效'}, 200 token = self.args['token'] appkey = self.args['appkey'] verify_token = flask_redis.get(appkey) if verify_token is None: return {'success': 0, 'message': 'token 无效'}, 200 else: verify_token = verify_token.decode('utf-8') if type(verify_token) == type(b'') else verify_token if verify_token != token: return {'success': 0, 'message': 'token 无效'}, 200 sign = self.args['sign'] if hash_sha256("{0},{1},{2}".format(ts, token, appkey)) != sign: return {'success': 0, 'message': 'sign 无效'}, 200 keyword = self.args['keyword'] query = Website.query.join(Token, Website.id==Token.website_id).filter(Token.appkey == appkey).first() domain = query.domain try: s = Search(using=client, index='suggest', doc_type='news') s = s.filter('term', website=domain).query('match', title=keyword) s = s[0:10] response = s.execute() return {'success': 1, 'data': response.to_dict()}, 200 except Exception as e: return {'success': 0, 'message': e}, 200
def test_search_can_be_limited_to_fields(data_client): s = Search(using=data_client).index('git').doc_type('repos').fields( 'organization') response = s.execute() assert response.hits.total == 1 assert response.hits[0] == {'organization': ['elasticsearch']}
def get(self, request): database_name = get_database_name(request.user) search_text = lower(request.GET["term"] or "") es = Elasticsearch(hosts=[{ "host": ELASTIC_SEARCH_HOST, "port": ELASTIC_SEARCH_PORT }]) search = Search(using=es, index=database_name, doc_type="reporter") search = search.extra(**{"size": "10"}) resp = [] if search_text: query_text_escaped = ElasticUtilsHelper().replace_special_chars( search_text) query_fields = [ "name", "name_value", "name_exact", "short_code", "short_code_exact", "short_code_value" ] search = search.query("query_string", query=query_text_escaped, fields=query_fields) search_results = search.execute() resp = [{ "id": result.short_code, "label": self.get_label(result) } for result in search_results.hits] return HttpResponse(json.dumps(resp))
def get(self, request): """ API to fetch store @params: store_id: integer @returns: Retrive store information """ result = {} store_id = int(request.query_params.get('store_id')) serializer = FetchStoreSerializer(data=request.query_params) if serializer.is_valid(): search = Search(using=self.es, index="store_data_index") search = search.filter("match", store_id=store_id) data = search.execute().to_dict() if data['hits']['hits']: result['status'] = True result['data'] = data['hits']['hits'][0]['_source'] else: result['status'] = False result['message'] = 'No data found for the give store id' status_code = status.HTTP_200_OK else: status_code = status.HTTP_400_BAD_REQUEST result = {'status': False, 'message': serializer.errors} return Response(result, status=status_code)
def test_search_type_count(data_client): s = Search(using=data_client, index='git') s.aggs.bucket('per_type', 'terms', field='_type') s = s.params(search_type='count') result = s.execute() assert [] == result.hits assert 2 == len(result.aggregations.per_type.buckets)
def get(self, request): keyWords = request.GET.get('s', '') kw = {'using': client, 'index': 'hfut_search', 'doc_type': 'hfut_type'} sugg = Search(**kw) sugg = sugg.suggest('my_suggest', keyWords, completion={ "field": "suggest", "size": common.FETCH_NUMBER_DEF['pageNum'] }) sugg = sugg.execute() options = sugg.suggest['my_suggest'][0].options reDatas = [match._source["title"] for match in options] return HttpResponse(json.dumps(reDatas), content_type="application/json")
def query_imageso(object_list): print("In query_imageso") hit1 = set() image_set = set() print("11. object_list =", object_list) QI = Q('match_all') s1 = Search(index='idx0') for name in object_list: print("name= ", name) QI = QI & Q("match", names=name) s1 = s1.query(QI).using(client) response = s1.execute() for hit in s1.scan(): image_set.add(hit.imgfile) return display_image_set(image_set, None, object_list)
def query_imagesi(classnum_list, upfilename): #Disabled -- #print("In query_imagesi") hit1 = set() image_set = set() #print("11. classnum_list =", classnum_list) QI = Q('match_all') s1 = Search(index='vgnum') classn = 1 for class_num in classnum_list: if classn > 7: #can make this 7-- break classn = classn + 1 print("class_num= ", class_num) QI = QI & Q('bool', must=[Q("match", classnum=class_num)]) s1 = s1.query(QI).using(client) response = s1.execute() for hit in s1.scan(): image_set.add(hit.imgfile) return display_image_set(image_set, upfilename, '')
def post(self): ts = self.args['_'] if abs(int(time() * 1000) - int(ts)) > 1800000: return {'success':0, 'message': '时间戳无效'}, 200 token = self.args['token'] appkey = self.args['appkey'] verify_token = flask_redis.get(appkey) if verify_token is None: return {'success': 0, 'message': 'token 无效'}, 200 else: verify_token = verify_token.decode('utf-8') if type(verify_token) == type(b'') else verify_token if verify_token != token: return {'success': 0, 'message': 'token 无效'}, 200 sign = self.args['sign'] if hash_sha256("{0},{1},{2}".format(ts, token, appkey)) != sign: return {'success': 0, 'message': 'sign 无效'}, 200 page = int(self.args['page']) size = int(self.args['size']) from_size = (page - 1) * size to_size = page * size channel = self.args['channel'] category = self.args['category'] location = self.args['location'] times = self.args['times'] and_ = self.args['and'] sort_field = self.args['s'] is_open = self.args['is_open'] o = self.args['o'] scope = self.args['scope'] if scope in ['content', 'tag', 'title', 'description', 'author', 'writings']: scope = [scope] else: scope = ['content', 'tag', 'title', 'description', 'author', 'writings'] keyword = self.args['keyword'] try: s = Search(using=client, index='gdszx', doc_type='culture') if times: s = s.filter('term', times=times) if category: s = s.filter('term', category=category) if location: s = s.filter('term', location=location) if channel: s = s.filter('term', channel=channel) if is_open == '0': s = s.filter('term', is_open=False) elif is_open == '1': s = s.filter('term', is_open=True) s = s.query('multi_match', query=keyword, fields=scope) if and_ != None and and_.strip() != '': for word in split(r'\s+', and_.strip()): s = s.query('multi_match', query=word, fields=scope) s = s.highlight('title', fragment_size=50).highlight('content', fragment_size=100) s.aggs.bucket('times_all', 'terms', field='times', size=10) s.aggs.bucket('channel_all', 'terms', field='channel', size=10) s.aggs.bucket('category_all', 'terms', field='category', size=10) s.aggs.bucket('location_all', 'terms', field='location', size=10) s = s.sort(o+sort_field) s = s[from_size:to_size] response = s.execute() return {'success': 1, 'data': response.to_dict()}, 200 except Exception as e: return {'success': 0, 'message': e}, 200
def post(self): ts = self.args['_'] if abs(int(time() * 1000) - int(ts)) > 1800000: return {'success':0, 'message': '时间戳无效'}, 200 token = self.args['token'] appkey = self.args['appkey'] verify_token = flask_redis.get(appkey) if verify_token is None: return {'success': 0, 'message': 'token 无效'}, 200 else: verify_token = verify_token.decode('utf-8') if type(verify_token) == type(b'') else verify_token if verify_token != token: return {'success': 0, 'message': 'token 无效'}, 200 sign = self.args['sign'] if hash_sha256("{0},{1},{2}".format(ts, token, appkey)) != sign: return {'success': 0, 'message': 'sign 无效'}, 200 query = Website.query.join(Token, Website.id==Token.website_id).filter(Token.appkey == appkey).first() domain = query.domain page = int(self.args['page']) size = int(self.args['size']) from_size = (page - 1) * size to_size = page * size origin = self.args['origin'] channel = self.args['channel'] category = self.args['category'] author = self.args['author'] editor = self.args['editor'] begin = self.args['from'] to = self.args['to'] has_pic = self.args['has_pic'] has_video = self.args['has_video'] v1 = self.args['v1'] v2 = self.args['v2'] v3 = self.args['v3'] v4 = self.args['v4'] v5 = self.args['v5'] v6 = self.args['v6'] not_ = self.args['not'] and_ = self.args['and'] sort_field = self.args['s'] o = self.args['o'] f = self.args['f'] l = self.args['l'] l = [] if l is None else l.split(',') scope = self.args['scope'] if scope in ['content', 'tag', 'title', 'description']: scope = [scope] else: scope = ['content', 'tag', 'title', 'description'] keyword = self.args['keyword'] try: s = Search(using=client, index='common', doc_type='search') s = s.filter('term', website=domain) if author: s = s.filter('term', author=author) if editor: s = s.filter('term', editor=editor) if origin: s = s.filter('term', origin=origin) if category: s = s.filter('term', category=category) if channel: s = s.filter('term', channel=channel) if v1: s = s.filter('term', reserved_1=v1) if v2: s = s.filter('term', reserved_2=v2) if v3: s = s.filter('term', reserved_3=v3) if v4: s = s.filter('term', reserved_4=v4) if v5: s = s.filter('term', reserved_5=v5) if v6: s = s.filter('term', reserved_6=v6) if has_pic == '0': s = s.filter('term', has_pic=False) elif has_pic == '1': s = s.filter('term', has_pic=True) if has_video == '0': s = s.filter('term', has_video=False) elif has_video == '1': s = s.filter('term', has_video=True) s = s.filter('range', pdate={'gte': begin, 'lte': to}) s = s.query('multi_match', query=keyword, fields=scope) if not_: s = s.exclude('multi_match', query=not_, fields=scope) if and_ != None and and_.strip() != '': for word in split(r'\s+', and_.strip()): s = s.query('multi_match', query=word, fields=scope) s = s.highlight('title', fragment_size=50).highlight('content', fragment_size=100).highlight('tag', fragment_size=50).highlight('description', fragment_size=100) if f == 'title': s = s.exclude('terms', title__raw=l) elif f == 'url': s = s.exclude('terms', url=l) s = s.sort(o+sort_field) s = s[from_size:to_size] response = s.execute() related = related_search(client, keyword) return {'success': 1, 'data': response.to_dict(), 'related': related}, 200 except Exception as e: return {'success': 0, 'message': e}, 200
def put(self) -> dict: """ search for a group in Elasticsearch Returns: dict -- search results """ args = self.parser.parse_args() # init search search: Search = Group.search() search_query: dict = { "bool": { "should": [ {"query_string": {"query": args["query"], "fields": ["*"]}}, { "nested": { "path": "topics", "score_mode": "avg", "query": { "bool": { "must": [ { "query_string": { "query": args["query"], "fields": ["*"], } } ] } }, } }, { "nested": { "path": "events", "score_mode": "avg", "query": { "bool": { "must": [ { "query_string": { "query": args["query"], "fields": ["*"], } } ] } }, } }, ], "must": [], } } # set event time filter if args["event_time_gte"] or args["event_time_lte"]: range_query: dict = {} if args["event_time_gte"]: range_query["gte"] = args["event_time_gte"] if args["event_time_lte"]: range_query["lte"] = args["event_time_lte"] search_query["bool"]["must"].append( { "nested": { "path": "events", "score_mode": "avg", "query": { "bool": {"must": [{"range": {"events.time": range_query}}]} }, } } ) # set geo_distance filter if args["geo_distance"] and args["geo_lat"] and args["geo_lon"]: search_query["bool"]["must"].append( { "nested": { "path": "events", "score_mode": "avg", "query": { "bool": { "must": [ { "geo_distance": { "distance": args["geo_distance"], "events.venue_location": { "lat": args["geo_lat"], "lon": args["geo_lon"], }, } } ] } }, } } ) # pagination strat_entry: int = args["page"] * args["limit"] end_entry: int = strat_entry + args["limit"] search = search[strat_entry:end_entry] # sort if args["sort"]: search = Search().sort(args["sort"]) # execute search search = search.query(Q(search_query)) # set highlight score search.highlight_options(order="score") # load response from elasticsearch results: Response = search.execute() # get response found_groups: List[dict] = [] map_center_lat: float = 0 map_center_lon: float = 0 for group in results.hits: group_dict: dict = {} if isinstance(group, Hit): group_object = Group.get_group(urlname=group.to_dict()["urlname"]) group_dict = group_object.to_json_dict(load_events=args["load_events"]) else: group_dict = group.to_json_dict(load_events=args["load_events"]) if "venue_location_average" in group_dict: map_center_lat = ( map_center_lat + group_dict["venue_location_average"]["lat"] ) map_center_lon = ( map_center_lon + group_dict["venue_location_average"]["lon"] ) else: map_center_lat = map_center_lat + group_dict["location"]["lat"] map_center_lon = map_center_lon + group_dict["location"]["lon"] # add group dict to array found_groups.append( {**group_dict,} ) if len(found_groups) > 0: map_center_lat = map_center_lat / len(found_groups) map_center_lon = map_center_lon / len(found_groups) return { "results": found_groups, "hits": results.hits.total["value"], "map_center": {"lat": map_center_lat, "lon": map_center_lon}, }