def query_images(object_list):
hit1 = set()
image_set = set()
print("11. object_list =", object_list)
QI = Q('match_all')
s1 = Search(index='idxo20')
for objectk in object_list:
print("objectk= ",objectk)
QI = QI & Q("match", names=objectk)
s1 = s1.query(QI).using(client)
s1.execute()
for hit in s1.scan() :
print("33 ", hit.imgfile)
image_set.add(hit.imgfile)
print("image_set = {0}".format(image_set))
im = 0
for image in image_set :
if im > 3 :
break
file, ext = os.path.splitext(image)
image = file + '.png'
print("66 image =", image)
images_div.append(display_image(image))
im = im + 1
print("Please hit refresh...")
def get_imgfile(img_id):
s = Search(index='idx1').query('match', image_id=img_id)
s = s.using(client)
s.execute()
for hit in s.scan():
imgfile = hit.imagefile
return imgfile
def get_imgfile(img_id):
s = Search(index='idxi20').query('match', image_id=img_id)
s = s.using(client)
s.execute()
for hit in s.scan():
imgfile = hit.imagefile
#imagefile = hit.url
#print("22 imgfile = ", imgfile)
return imgfile
def test_count_prefetch(data_client, mocker):
mocker.spy(data_client, "count")
search = Search(using=data_client).index("git")
search.execute()
assert search.count() == 53
assert data_client.count.call_count == 0
search._response.hits.total.relation = "gte"
assert search.count() == 53
assert data_client.count.call_count == 1
def query_imagesi(classnum_list, classval_list, upfilename):
#print("In query_imagesi")
hit1 = set()
image_set = set()
#print("11. classnum_list =", classnum_list)
#print("12. classval_list =", classval_list)
if (len(classnum_list) > 7):
lenclassnum = 7
QI = Q('match_all')
s1 = Search(index='adknum')
#s1 = Search(index='vgnum')
classn = 1
for class_num in classnum_list:
if classn > 7: #can make this 5--
break
classn = classn + 1
QI = QI & Q('bool', must=[Q("match", classnum=class_num)])
s1 = s1.query(QI).using(client)
response = s1.execute()
hit_num = 0
simDict = {}
similarityClass = np.zeros(15)
for hit in s1.scan():
print("123. hit.classnum: ", hit.classnum)
print("124. hit.classval: ", hit.classval)
lenimgclassnum = len(hit.classnum)
simDict[hit.imgfile] = 1.0
#similarityClass[lenimgclassnum]
# Compute similarity and choose top 4 rather than random 4
ii = 0
similarityImg = 0.0
for classi in classnum_list:
jj = 0
for classj in hit.classnum:
if classi == int(classj):
similarityClass[ii] = classval_list[ii] / (abs(
(classval_list[ii] - int(hit.classval[jj]))) + 10)
#print("144. similarityClass[ii] = ", similarityClass[ii])
similarityImg = similarityImg + similarityClass[ii]
break
jj = jj + 1
simDict[hit.imgfile] = similarityImg
#print("130. simDict[hit.imgfile] = ", simDict[hit.imgfile], similarityImg)
ii = ii + 1
#for key in sorted(simDict.keys(), reverse=True) :
kk = 0
for img in sorted(simDict, key=simDict.get, reverse=True):
print("140. ", img, simDict[img])
image_set.add(img)
#image_set.add(hit.imgfile)
#pick top 4 images (hit_nums)
kk = kk + 1
if kk > 3:
break
return display_image_set(image_set, upfilename, '')
def post(self):
ts = self.args['_']
if abs(int(time() * 1000) - int(ts)) > 1800000:
return {'success':0, 'message': '时间戳无效'}, 200
token = self.args['token']
appkey = self.args['appkey']
verify_token = flask_redis.get(appkey)
if verify_token is None:
return {'success': 0, 'message': 'token 无效'}, 200
else:
verify_token = verify_token.decode('utf-8') if type(verify_token) == type(b'') else verify_token
if verify_token != token:
return {'success': 0, 'message': 'token 无效'}, 200
sign = self.args['sign']
if hash_sha256("{0},{1},{2}".format(ts, token, appkey)) != sign:
return {'success': 0, 'message': 'sign 无效'}, 200
keyword = self.args['keyword']
query = Website.query.join(Token, Website.id==Token.website_id).filter(Token.appkey == appkey).first()
domain = query.domain
try:
s = Search(using=client, index='suggest', doc_type='news')
s = s.filter('term', website=domain).query('match', title=keyword)
s = s[0:10]
response = s.execute()
return {'success': 1, 'data': response.to_dict()}, 200
except Exception as e:
return {'success': 0, 'message': e}, 200
def test_search_can_be_limited_to_fields(data_client):
s = Search(using=data_client).index('git').doc_type('repos').fields(
'organization')
response = s.execute()
assert response.hits.total == 1
assert response.hits[0] == {'organization': ['elasticsearch']}
def get(self, request):
database_name = get_database_name(request.user)
search_text = lower(request.GET["term"] or "")
es = Elasticsearch(hosts=[{
"host": ELASTIC_SEARCH_HOST,
"port": ELASTIC_SEARCH_PORT
}])
search = Search(using=es, index=database_name, doc_type="reporter")
search = search.extra(**{"size": "10"})
resp = []
if search_text:
query_text_escaped = ElasticUtilsHelper().replace_special_chars(
search_text)
query_fields = [
"name", "name_value", "name_exact", "short_code",
"short_code_exact", "short_code_value"
]
search = search.query("query_string",
query=query_text_escaped,
fields=query_fields)
search_results = search.execute()
resp = [{
"id": result.short_code,
"label": self.get_label(result)
} for result in search_results.hits]
return HttpResponse(json.dumps(resp))
def get(self, request):
"""
API to fetch store
@params: store_id: integer
@returns: Retrive store information
"""
result = {}
store_id = int(request.query_params.get('store_id'))
serializer = FetchStoreSerializer(data=request.query_params)
if serializer.is_valid():
search = Search(using=self.es, index="store_data_index")
search = search.filter("match", store_id=store_id)
data = search.execute().to_dict()
if data['hits']['hits']:
result['status'] = True
result['data'] = data['hits']['hits'][0]['_source']
else:
result['status'] = False
result['message'] = 'No data found for the give store id'
status_code = status.HTTP_200_OK
else:
status_code = status.HTTP_400_BAD_REQUEST
result = {'status': False, 'message': serializer.errors}
return Response(result, status=status_code)
def test_search_type_count(data_client):
s = Search(using=data_client, index='git')
s.aggs.bucket('per_type', 'terms', field='_type')
s = s.params(search_type='count')
result = s.execute()
assert [] == result.hits
assert 2 == len(result.aggregations.per_type.buckets)
def test_search_type_count(data_client):
s = Search(using=data_client, index='git')
s.aggs.bucket('per_type', 'terms', field='_type')
s = s.params(search_type='count')
result = s.execute()
assert [] == result.hits
assert 2 == len(result.aggregations.per_type.buckets)
def get(self, request):
keyWords = request.GET.get('s', '')
kw = {'using': client, 'index': 'hfut_search', 'doc_type': 'hfut_type'}
sugg = Search(**kw)
sugg = sugg.suggest('my_suggest',
keyWords,
completion={
"field": "suggest",
"size": common.FETCH_NUMBER_DEF['pageNum']
})
sugg = sugg.execute()
options = sugg.suggest['my_suggest'][0].options
reDatas = [match._source["title"] for match in options]
return HttpResponse(json.dumps(reDatas),
content_type="application/json")
def query_imageso(object_list):
print("In query_imageso")
hit1 = set()
image_set = set()
print("11. object_list =", object_list)
QI = Q('match_all')
s1 = Search(index='idx0')
for name in object_list:
print("name= ", name)
QI = QI & Q("match", names=name)
s1 = s1.query(QI).using(client)
response = s1.execute()
for hit in s1.scan():
image_set.add(hit.imgfile)
return display_image_set(image_set, None, object_list)
def query_imagesi(classnum_list, upfilename): #Disabled --
#print("In query_imagesi")
hit1 = set()
image_set = set()
#print("11. classnum_list =", classnum_list)
QI = Q('match_all')
s1 = Search(index='vgnum')
classn = 1
for class_num in classnum_list:
if classn > 7: #can make this 7--
break
classn = classn + 1
print("class_num= ", class_num)
QI = QI & Q('bool', must=[Q("match", classnum=class_num)])
s1 = s1.query(QI).using(client)
response = s1.execute()
for hit in s1.scan():
image_set.add(hit.imgfile)
return display_image_set(image_set, upfilename, '')
def post(self):
ts = self.args['_']
if abs(int(time() * 1000) - int(ts)) > 1800000:
return {'success':0, 'message': '时间戳无效'}, 200
token = self.args['token']
appkey = self.args['appkey']
verify_token = flask_redis.get(appkey)
if verify_token is None:
return {'success': 0, 'message': 'token 无效'}, 200
else:
verify_token = verify_token.decode('utf-8') if type(verify_token) == type(b'') else verify_token
if verify_token != token:
return {'success': 0, 'message': 'token 无效'}, 200
sign = self.args['sign']
if hash_sha256("{0},{1},{2}".format(ts, token, appkey)) != sign:
return {'success': 0, 'message': 'sign 无效'}, 200
page = int(self.args['page'])
size = int(self.args['size'])
from_size = (page - 1) * size
to_size = page * size
channel = self.args['channel']
category = self.args['category']
location = self.args['location']
times = self.args['times']
and_ = self.args['and']
sort_field = self.args['s']
is_open = self.args['is_open']
o = self.args['o']
scope = self.args['scope']
if scope in ['content', 'tag', 'title', 'description', 'author', 'writings']:
scope = [scope]
else:
scope = ['content', 'tag', 'title', 'description', 'author', 'writings']
keyword = self.args['keyword']
try:
s = Search(using=client, index='gdszx', doc_type='culture')
if times:
s = s.filter('term', times=times)
if category:
s = s.filter('term', category=category)
if location:
s = s.filter('term', location=location)
if channel:
s = s.filter('term', channel=channel)
if is_open == '0':
s = s.filter('term', is_open=False)
elif is_open == '1':
s = s.filter('term', is_open=True)
s = s.query('multi_match', query=keyword, fields=scope)
if and_ != None and and_.strip() != '':
for word in split(r'\s+', and_.strip()):
s = s.query('multi_match', query=word, fields=scope)
s = s.highlight('title', fragment_size=50).highlight('content', fragment_size=100)
s.aggs.bucket('times_all', 'terms', field='times', size=10)
s.aggs.bucket('channel_all', 'terms', field='channel', size=10)
s.aggs.bucket('category_all', 'terms', field='category', size=10)
s.aggs.bucket('location_all', 'terms', field='location', size=10)
s = s.sort(o+sort_field)
s = s[from_size:to_size]
response = s.execute()
return {'success': 1, 'data': response.to_dict()}, 200
except Exception as e:
return {'success': 0, 'message': e}, 200
def post(self):
ts = self.args['_']
if abs(int(time() * 1000) - int(ts)) > 1800000:
return {'success':0, 'message': '时间戳无效'}, 200
token = self.args['token']
appkey = self.args['appkey']
verify_token = flask_redis.get(appkey)
if verify_token is None:
return {'success': 0, 'message': 'token 无效'}, 200
else:
verify_token = verify_token.decode('utf-8') if type(verify_token) == type(b'') else verify_token
if verify_token != token:
return {'success': 0, 'message': 'token 无效'}, 200
sign = self.args['sign']
if hash_sha256("{0},{1},{2}".format(ts, token, appkey)) != sign:
return {'success': 0, 'message': 'sign 无效'}, 200
query = Website.query.join(Token, Website.id==Token.website_id).filter(Token.appkey == appkey).first()
domain = query.domain
page = int(self.args['page'])
size = int(self.args['size'])
from_size = (page - 1) * size
to_size = page * size
origin = self.args['origin']
channel = self.args['channel']
category = self.args['category']
author = self.args['author']
editor = self.args['editor']
begin = self.args['from']
to = self.args['to']
has_pic = self.args['has_pic']
has_video = self.args['has_video']
v1 = self.args['v1']
v2 = self.args['v2']
v3 = self.args['v3']
v4 = self.args['v4']
v5 = self.args['v5']
v6 = self.args['v6']
not_ = self.args['not']
and_ = self.args['and']
sort_field = self.args['s']
o = self.args['o']
f = self.args['f']
l = self.args['l']
l = [] if l is None else l.split(',')
scope = self.args['scope']
if scope in ['content', 'tag', 'title', 'description']:
scope = [scope]
else:
scope = ['content', 'tag', 'title', 'description']
keyword = self.args['keyword']
try:
s = Search(using=client, index='common', doc_type='search')
s = s.filter('term', website=domain)
if author:
s = s.filter('term', author=author)
if editor:
s = s.filter('term', editor=editor)
if origin:
s = s.filter('term', origin=origin)
if category:
s = s.filter('term', category=category)
if channel:
s = s.filter('term', channel=channel)
if v1:
s = s.filter('term', reserved_1=v1)
if v2:
s = s.filter('term', reserved_2=v2)
if v3:
s = s.filter('term', reserved_3=v3)
if v4:
s = s.filter('term', reserved_4=v4)
if v5:
s = s.filter('term', reserved_5=v5)
if v6:
s = s.filter('term', reserved_6=v6)
if has_pic == '0':
s = s.filter('term', has_pic=False)
elif has_pic == '1':
s = s.filter('term', has_pic=True)
if has_video == '0':
s = s.filter('term', has_video=False)
elif has_video == '1':
s = s.filter('term', has_video=True)
s = s.filter('range', pdate={'gte': begin, 'lte': to})
s = s.query('multi_match', query=keyword, fields=scope)
if not_:
s = s.exclude('multi_match', query=not_, fields=scope)
if and_ != None and and_.strip() != '':
for word in split(r'\s+', and_.strip()):
s = s.query('multi_match', query=word, fields=scope)
s = s.highlight('title', fragment_size=50).highlight('content', fragment_size=100).highlight('tag', fragment_size=50).highlight('description', fragment_size=100)
if f == 'title':
s = s.exclude('terms', title__raw=l)
elif f == 'url':
s = s.exclude('terms', url=l)
s = s.sort(o+sort_field)
s = s[from_size:to_size]
response = s.execute()
related = related_search(client, keyword)
return {'success': 1, 'data': response.to_dict(), 'related': related}, 200
except Exception as e:
return {'success': 0, 'message': e}, 200
def put(self) -> dict:
"""
search for a group in Elasticsearch
Returns:
dict -- search results
"""
args = self.parser.parse_args()
# init search
search: Search = Group.search()
search_query: dict = {
"bool": {
"should": [
{"query_string": {"query": args["query"], "fields": ["*"]}},
{
"nested": {
"path": "topics",
"score_mode": "avg",
"query": {
"bool": {
"must": [
{
"query_string": {
"query": args["query"],
"fields": ["*"],
}
}
]
}
},
}
},
{
"nested": {
"path": "events",
"score_mode": "avg",
"query": {
"bool": {
"must": [
{
"query_string": {
"query": args["query"],
"fields": ["*"],
}
}
]
}
},
}
},
],
"must": [],
}
}
# set event time filter
if args["event_time_gte"] or args["event_time_lte"]:
range_query: dict = {}
if args["event_time_gte"]:
range_query["gte"] = args["event_time_gte"]
if args["event_time_lte"]:
range_query["lte"] = args["event_time_lte"]
search_query["bool"]["must"].append(
{
"nested": {
"path": "events",
"score_mode": "avg",
"query": {
"bool": {"must": [{"range": {"events.time": range_query}}]}
},
}
}
)
# set geo_distance filter
if args["geo_distance"] and args["geo_lat"] and args["geo_lon"]:
search_query["bool"]["must"].append(
{
"nested": {
"path": "events",
"score_mode": "avg",
"query": {
"bool": {
"must": [
{
"geo_distance": {
"distance": args["geo_distance"],
"events.venue_location": {
"lat": args["geo_lat"],
"lon": args["geo_lon"],
},
}
}
]
}
},
}
}
)
# pagination
strat_entry: int = args["page"] * args["limit"]
end_entry: int = strat_entry + args["limit"]
search = search[strat_entry:end_entry]
# sort
if args["sort"]:
search = Search().sort(args["sort"])
# execute search
search = search.query(Q(search_query))
# set highlight score
search.highlight_options(order="score")
# load response from elasticsearch
results: Response = search.execute()
# get response
found_groups: List[dict] = []
map_center_lat: float = 0
map_center_lon: float = 0
for group in results.hits:
group_dict: dict = {}
if isinstance(group, Hit):
group_object = Group.get_group(urlname=group.to_dict()["urlname"])
group_dict = group_object.to_json_dict(load_events=args["load_events"])
else:
group_dict = group.to_json_dict(load_events=args["load_events"])
if "venue_location_average" in group_dict:
map_center_lat = (
map_center_lat + group_dict["venue_location_average"]["lat"]
)
map_center_lon = (
map_center_lon + group_dict["venue_location_average"]["lon"]
)
else:
map_center_lat = map_center_lat + group_dict["location"]["lat"]
map_center_lon = map_center_lon + group_dict["location"]["lon"]
# add group dict to array
found_groups.append(
{**group_dict,}
)
if len(found_groups) > 0:
map_center_lat = map_center_lat / len(found_groups)
map_center_lon = map_center_lon / len(found_groups)
return {
"results": found_groups,
"hits": results.hits.total["value"],
"map_center": {"lat": map_center_lat, "lon": map_center_lon},
}
