def search_portrait(condition_num, query, sort, size):
user_result = []
index_name = portrait_index_name
index_type = portrait_index_type
search_result_max = get_evaluate_max()
if condition_num > 0:
result = es_user_portrait.search(index=index_name, doc_type=index_type, \
body={'query':{'bool':{'must':query}}, 'sort':[{sort:{'order':'desc'}}], 'size':size})['hits']['hits']
else:
result = es_user_portrait.search(index=index_name, doc_type=index_type, \
body={'query':{'match_all':{}}, 'sort':[{sort:{"order":"desc"}}], 'size':size})['hits']['hits']
if result:
for item in result:
user_dict = item['_source']
score = item['_score']
result_normal_activeness = math.log(user_dict['activeness'] / float(search_result_max['activeness']) * 9 + 1, 10)
result_normal_importance = math.log(user_dict['importance'] / float(search_result_max['importance']) * 9 + 1, 10)
result_normal_influence = math.log(user_dict['influence'] / float(search_result_max['influence']) * 9 + 1, 10)
result_normal_sensitive = math.log(user_dict['sensitive'] / float(search_result_max['sensitive']) * 9 + 1, 10)
user_dict['activeness'] = result_normal_activeness*100
user_dict['importance'] = result_normal_importance*100
user_dict['influence'] = result_normal_influence*100
user_dict['sensitive'] = result_normal_sensitive*100
user_result.append([user_dict['uid'], user_dict['uname'], user_dict['location'], user_dict['activeness'],\
user_dict['importance'], user_dict['influence'], user_dict['sensitive'], score])
return user_result
def search_portrait(condition_num, query, sort, size):
user_result = []
index_name = portrait_index_name
index_type = portrait_index_type
search_result_max = get_evaluate_max()
if condition_num > 0:
result = es_user_portrait.search(index=index_name, doc_type=index_type, \
body={'query':{'bool':{'must':query}}, 'sort':[{sort:{'order':'desc'}}], 'size':size})['hits']['hits']
else:
result = es_user_portrait.search(index=index_name, doc_type=index_type, \
body={'query':{'match_all':{}}, 'sort':[{sort:{"order":"desc"}}], 'size':size})['hits']['hits']
if result:
max_score = result[0]['_score']
for item in result:
user_dict = item['_source']
score = item['_score']
result_normal_activeness = math.log(user_dict['activeness'] / float(search_result_max['activeness']) * 9 + 1, 10)
result_normal_importance = math.log(user_dict['importance'] / float(search_result_max['importance']) * 9 + 1, 10)
result_normal_influence = math.log(user_dict['influence'] / float(search_result_max['influence']) * 9 + 1, 10)
result_normal_sensitive = math.log(user_dict['sensitive'] / float(search_result_max['sensitive']) * 9 + 1, 10)
user_dict['activeness'] = result_normal_activeness*100
user_dict['importance'] = result_normal_importance*100
user_dict['influence'] = result_normal_influence*100
user_dict['sensitive'] = result_normal_sensitive*100
user_dict['score'] = math.log(score/float(max_score)*9+1, 10)*100
user_result.append([user_dict['uid'], user_dict['uname'], user_dict['location'], user_dict['activeness'],\
user_dict['importance'], user_dict['influence'], user_dict['sensitive'], user_dict['score']])
return user_result
def show_social_sensing_task():
query_body = {
"query": {
"filtered": {
"filter": {
"term": {
"finish": "1"
}
}
}
},
"sort": {
"create_at": {
"order": "desc"
}
},
"size": 10000
}
results = []
search_results = es.search(index=index_manage_social_task,
doc_type=task_doc_type,
body=query_body)['hits']['hits']
if search_results:
for item in search_results:
results.append(item['_source']['task_name'])
return results
def get_evaluate_max():
max_result = {}
index_name = portrait_index_name
index_type = portrait_index_type
evaluate_index = ['activeness', 'importance', 'influence']
for evaluate in evaluate_index:
query_body = {
'query': {
'match_all': {}
},
'size': 1,
'sort': [{
evaluate: {
'order': 'desc'
}
}]
}
try:
result = es_user_portrait.search(index=index_name,
doc_type=index_type,
body=query_body)['hits']['hits']
except Exception, e:
raise e
max_evaluate = result[0]['_source'][evaluate]
max_result[evaluate] = max_evaluate
def ajax_show_task():
# show all working task
# "0": unfinish working task
# "1": finish working task
status = request.args.get("finish", "01")
user = request.args.get('user', '')
length = len(status)
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"create_by": user
}
}]
}
}
}
},
"sort": {
"create_at": {
"order": "desc"
}
},
"size": 10000
}
#if length == 2:
# category_list = [status[0], status[1]]
# query_body['query']['filtered']['filter']["bool"]["must"].append({"term":{"finish": category_list}})
if length == 1:
query_body['query']['filtered']['filter']['bool']['must'].append(
{"term": {
"finish": status
}})
#else:
# print "error"
try:
search_results = es.search(index=index_manage_sensing_task,
doc_type=task_doc_type,
body=query_body)['hits']['hits']
except:
search_results = []
results = []
if search_results:
for item in search_results:
item = item['_source']
history_status = json.loads(item['history_status'])
if history_status:
item['history_status'] = sorted(history_status,
key=lambda x: x,
reverse=True)
else:
item['history_status'] = []
results.append(item)
return json.dumps(results)
def get_evaluate_max():
max_result = {}
evaluate_index = ["importance", "influence"]
for evaluate in evaluate_index:
query_body = {"query": {"match_all": {}}, "size": 1, "sort": [{evaluate: {"order": "desc"}}]}
try:
result = es_user_portrait.search(index=portrait_index_name, doc_type=portrait_index_type, body=query_body)[
"hits"
]["hits"]
except Exception, e:
raise e
max_evaluate = result[0]["_source"][evaluate]
max_result[evaluate] = max_evaluate
def save_detect_single_task(input_dict):
results = {}
#step1: identify the seed user is in user_portrait
seed_user = input_dict['query_condition']['seed_user']
query = []
query_list = []
for user_item in seed_user:
query_list.append({'term': {user_item: seed_user[user_item]}})
query.append({'bool': {'should': query_list}})
try:
seed_user_result = es_user_portrait.search(index=portrait_index_name, doc_type=portrait_index_type, \
body={'query':{'bool':{'must':query}}, 'size':1})['hits']['hits']
except Exception, e:
raise e
def get_top_influence(key):
query_body = {
"query":{
"match_all": {}
},
"sort":{key:{"order":"desc"}},
"size": 1
}
search_result = es.search(index=portrait_index_name, doc_type=portrait_index_type, body=query_body)['hits']['hits']
if search_result:
result = search_result[0]['_source'][key]
return result
def save_detect_single_task(input_dict):
results = {}
#step1: identify the seed user is in user_portrait
seed_user = input_dict['query_condition']['seed_user']
query = []
query_list = []
for user_item in seed_user:
query_list.append({'term':{user_item: seed_user[user_item]}})
query.append({'bool':{'should': query_list}})
try:
seed_user_result = es_user_portrait.search(index=portrait_index_name, doc_type=portrait_index_type, \
body={'query':{'bool':{'must':query}}, 'size':1})['hits']['hits']
except Exception, e:
raise e
def get_evaluate_max():
max_result = {}
evaluate_index = ['influence', 'activeness', 'importance', 'sensitive']
for evaluate in evaluate_index:
query_body = {
'query':{
'match_all':{}
},
'size':1,
'sort':[{evaluate: {'order': 'desc'}}]
}
try:
result = es.search(index=portrait_index_name, doc_type=portrait_index_type, body=query_body)['hits']['hits']
except Exception, e:
raise e
max_evaluate = result[0]['_source'][evaluate]
max_result[evaluate] = max_evaluate
def ajax_show_task():
# show all working task
# "0": unfinish working task
# "1": finish working task
status = request.args.get("finish", "01")
user = request.args.get('user', '')
length = len(status)
query_body = {
"query":{
"filtered":{
"filter":{
"bool":{
"must":[
{"term":{"create_by": user}}
]
}
}
}
},
"sort": {"create_at": {"order": "desc"}},
"size": 10000
}
#if length == 2:
# category_list = [status[0], status[1]]
# query_body['query']['filtered']['filter']["bool"]["must"].append({"term":{"finish": category_list}})
if length == 1:
query_body['query']['filtered']['filter']['bool']['must'].append({"term":{"finish": status}})
#else:
# print "error"
try:
search_results = es.search(index=index_manage_sensing_task, doc_type=task_doc_type, body=query_body)['hits']['hits']
except:
search_results = []
results = []
if search_results:
for item in search_results:
item = item['_source']
history_status = json.loads(item['history_status'])
if history_status:
item['history_status'] = sorted(history_status, key=lambda x:x, reverse=True)
else:
item['history_status'] = []
results.append(item)
return json.dumps(results)
def get_evaluate_max():
max_result = {}
evaluate_index = ['activeness', 'importance', 'influence', 'sensitive']
for evaluate in evaluate_index:
query_body = {
'query':{
'match_all':{}
},
'size':1,
'sort':[{evaluate: {'order': 'desc'}}]
}
result = es_user_portrait.search(index=portrait_index_name, doc_type=portrait_index_type, body=query_body)['hits']['hits']
max_evaluate = result[0]['_source'][evaluate]
if max_evaluate != 0:
max_result[evaluate] = max_evaluate
else:
max_result[evaluate] = 99999
return max_result
def get_evaluate_max():
max_result = {}
evaluate_index = ['activeness', 'importance', 'influence', 'sensitive']
for evaluate in evaluate_index:
query_body = {
'query':{
'match_all':{}
},
'size':1,
'sort':[{evaluate: {'order': 'desc'}}]
}
result = es_user_portrait.search(index=portrait_index_name, doc_type=portrait_index_type, body=query_body)['hits']['hits']
max_evaluate = result[0]['_source'][evaluate]
if max_evaluate != 0:
max_result[evaluate] = max_evaluate
else:
max_result[evaluate] = 99999
return max_result
def show_social_sensing_task():
query_body = {
"query":{
"filtered":{
"filter":{
"term": {"finish": "1"}
}
}
},
"sort": {"create_at": {"order": "desc"}},
"size": 10000
}
results = []
search_results = es.search(index=index_manage_social_task, doc_type=task_doc_type, body=query_body)['hits']['hits']
if search_results:
for item in search_results:
results.append(item['_source']['task_name'])
return results
def get_top_all_influence(key, ts):
query_body = {
"query":{
"match_all": {}
},
"sort":{key:{"order":"desc"}},
"size": 1
}
index_name = "bci_" + ts2datetime(ts).replace('-','')
if not es.indices.exists(index=index_name):
index_name = "bci_" + ts2datetime(ts-DAY).replace('-','')
exist_es = es.indices.exists(index=index_name)
if exist_es:
search_result = es.search(index=index_name, doc_type="bci", body=query_body)['hits']['hits']
else:
search_result = {}
if search_result:
result = search_result[0]['_source'][key]
else:
result = 2000
return result
def search_task(task_name, submit_user, submit_date, state, status):
results = []
#query = [{"term": {"submit_user": submit_user}}]
query = []
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard': {'task_name': '*' + item + '*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_start = submit_date_ts
submit_date_end = submit_date_ts + DAY
query.append({
'range': {
'submit_date': {
'gte': submit_date_start,
'lt': submit_date_end
}
}
})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard': {'state': '*' + item + '*'}})
condition_num += 1
if status:
query.append({'match': {'status': status}})
condition_num += 1
if condition_num > 0:
query.append({'term': {'task_type': 'analysis'}})
try:
source = es_group_result.search(index=group_index_name,
doc_type=group_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'sort': [{
'count': {
'order': 'desc'
}
}],
'size':
MAX_VALUE
})
except Exception as e:
raise e
else:
query.append({'term': {'task_type': 'analysis'}})
source = es.search(index=group_index_name,
doc_type=group_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'sort': [{
'count': {
'order': 'desc'
}
}],
'size': MAX_VALUE
})
try:
task_dict_list = source['hits']['hits']
except:
return None
print 'step yes'
result = []
for task_dict in task_dict_list:
try:
state = task_dict['_source']['state']
except:
state = ''
try:
status = task_dict['_source']['status']
except:
status = 0
result.append([
task_dict['_source']['task_name'],
task_dict['_source']['submit_date'], task_dict['_source']['count'],
state, status
])
return result
def imagine(uid, query_fields_dict,index_name=portrait_index_name, doctype=portrait_index_type):
default_setting_dict = query_fields_dict
print query_fields_dict
personal_info = es.get(index=portrait_index_name, doc_type=portrait_index_type, id=uid, _source=True)['_source']
# tag
tag_dict = dict()
tag_dict_value = 0
if "tag" in query_fields_dict:
tag_dict_value = query_fields_dict["tag"]
query_fields_dict.pop("tag")
for key,value in personal_info.iteritems():
if "tag-" in key:
tag_dict[key] = value
print tag_dict, tag_dict_value
# size
sort_size = query_fields_dict["size"]
query_fields_dict.pop("size")
keys_list = []
for k, v in query_fields_dict.iteritems():
if v:
keys_list.append(k) #需要进行关联的键
search_dict = {} # 检索的属性字典
iter_list = []
tag_attri_vaule = []
# 对搜索的键值进行过滤，去掉无用的键
for iter_key in keys_list:
if iter_key in personal_info:
if not personal_info[iter_key] or not query_fields_dict[iter_key]:
query_fields_dict.pop(iter_key)
continue
else:
iter_list.append(iter_key)
temp = personal_info[iter_key]
search_dict[iter_key] = temp.split('&')
"""
query_fields_dict.pop(iter_key)
if tag_dict.get(iter_key,''):
tag_attri_vaule.append(iter_key+"-"+tag_dict[iter_key])
"""
if len(iter_list) == 0 and len(tag_dict) == 0:
return []
query_body = {
'query':{
'function_score':{
'query':{
'bool':{
'must':[
]
}
}
}
}
}
number = es.count(index=index_name, doc_type=doctype, body=query_body)['count']
query_body["size"] = sort_size+100
for (k,v) in query_fields_dict.items():
temp = {}
temp_list = []
if k in personal_info and v != 0:
for iter_key in search_dict[k]:
temp_list.append({'wildcard':{k:{'wildcard':'*'+iter_key+'*', 'boost': v}}})
query_body['query']['function_score']['query']['bool']['must'].append({'bool':{'should':temp_list}})
if tag_dict and tag_dict_value:
temp_list = []
for k,v in tag_dict.iteritems():
temp = {"term":{k:v}}
temp_list.append(temp)
query_body['query']['function_score']['query']['bool']['must'].append({'bool':{'should':temp_list}})
result = es.search(index=index_name, doc_type=doctype, body=query_body)['hits']['hits']
field_list = ['uid','uname', 'activeness','importance', 'influence', 'sensitive']
evaluate_index_list = ['activeness', 'importance', 'influence', 'sensitive']
return_list = []
count = 0
if len(result) > 1 and result:
if result[0]['_id'] != uid:
top_score = result[0]['_score']
else:
top_score = result[1]['_score']
#get evaluate max to normal
evaluate_max_dict = get_evaluate_max()
for item in result:
if uid == item['_id']:
score = item['_score']
continue
info = []
for field in field_list:
if field in evaluate_index_list:
value = item['_source'][field]
normal_value = math.log(value / float(evaluate_max_dict[field] )* 9 + 1, 10) * 100
else:
normal_value = item['_source'][field]
if not normal_value:
normal_value = item['_id']
info.append(normal_value)
info.append(item['_score']/float(top_score)*100)
return_list.append(info)
count += 1
if count == sort_size:
break
return_list.append(number)
temp_list = []
for field in field_list:
if field in evaluate_index_list:
value = personal_info[field]
normal_value = math.log(value / float(evaluate_max_dict[field]) * 9 + 1, 10) * 100
else:
normal_value = personal_info[field]
temp_list.append(normal_value)
results = []
results.append(temp_list)
results.extend(return_list)
return results
def imagine(uid,
query_fields_dict,
index_name=portrait_index_name,
doctype=portrait_index_type):
default_setting_dict = query_fields_dict
print query_fields_dict
personal_info = es.get(index=portrait_index_name,
doc_type=portrait_index_type,
id=uid,
_source=True)['_source']
# tag
tag_dict = dict()
tag_dict_value = 0
if "tag" in query_fields_dict:
tag_dict_value = query_fields_dict["tag"]
query_fields_dict.pop("tag")
for key, value in personal_info.iteritems():
if "tag-" in key:
tag_dict[key] = value
print tag_dict, tag_dict_value
# size
sort_size = query_fields_dict["size"]
query_fields_dict.pop("size")
keys_list = []
for k, v in query_fields_dict.iteritems():
if v:
keys_list.append(k) #需要进行关联的键
search_dict = {} # 检索的属性字典
iter_list = []
tag_attri_vaule = []
# 对搜索的键值进行过滤，去掉无用的键
for iter_key in keys_list:
if iter_key in personal_info:
if not personal_info[iter_key] or not query_fields_dict[iter_key]:
query_fields_dict.pop(iter_key)
continue
else:
iter_list.append(iter_key)
temp = personal_info[iter_key]
search_dict[iter_key] = temp.split('&')
"""
query_fields_dict.pop(iter_key)
if tag_dict.get(iter_key,''):
tag_attri_vaule.append(iter_key+"-"+tag_dict[iter_key])
"""
if len(iter_list) == 0 and len(tag_dict) == 0:
return []
query_body = {
'query': {
'function_score': {
'query': {
'bool': {
'must': []
}
}
}
}
}
number = es.count(index=index_name, doc_type=doctype,
body=query_body)['count']
query_body["size"] = sort_size + 100
for (k, v) in query_fields_dict.items():
temp = {}
temp_list = []
if k in personal_info and v != 0:
for iter_key in search_dict[k]:
temp_list.append({
'wildcard': {
k: {
'wildcard': '*' + iter_key + '*',
'boost': v
}
}
})
query_body['query']['function_score']['query']['bool'][
'must'].append({'bool': {
'should': temp_list
}})
if tag_dict and tag_dict_value:
temp_list = []
for k, v in tag_dict.iteritems():
temp = {"term": {k: v}}
temp_list.append(temp)
query_body['query']['function_score']['query']['bool']['must'].append(
{'bool': {
'should': temp_list
}})
result = es.search(index=index_name, doc_type=doctype,
body=query_body)['hits']['hits']
field_list = [
'uid', 'uname', 'activeness', 'importance', 'influence', 'sensitive'
]
evaluate_index_list = [
'activeness', 'importance', 'influence', 'sensitive'
]
return_list = []
count = 0
if len(result) > 1 and result:
if result[0]['_id'] != uid:
top_score = result[0]['_score']
else:
top_score = result[1]['_score']
#get evaluate max to normal
evaluate_max_dict = get_evaluate_max()
for item in result:
if uid == item['_id']:
score = item['_score']
continue
info = []
for field in field_list:
if field in evaluate_index_list:
value = item['_source'][field]
normal_value = math.log(
value / float(evaluate_max_dict[field]) * 9 + 1, 10) * 100
else:
normal_value = item['_source'][field]
if not normal_value:
normal_value = item['_id']
info.append(normal_value)
info.append(item['_score'] / float(top_score) * 100)
return_list.append(info)
count += 1
if count == sort_size:
break
return_list.append(number)
temp_list = []
for field in field_list:
if field in evaluate_index_list:
value = personal_info[field]
normal_value = math.log(
value / float(evaluate_max_dict[field]) * 9 + 1, 10) * 100
else:
normal_value = personal_info[field]
temp_list.append(normal_value)
results = []
results.append(temp_list)
results.extend(return_list)
return results
def delete_attribute(attribute_name):
status = False
try:
result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
except Exception, e:
raise e
return status
es.delete(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)
# delete attribute in user_portrait
query = []
attribute_value = result['attribute_value'].split('&')
attribute_name = "tag-" + attribute_name
for value in attribute_value:
query.append({'match':{attribute_name: value}})
try:
attribute_user_result = es_user_portrait.search(index=user_index_name, doc_type=user_index_type, \
body={'query':{'bool':{'must':query}}})['hits']['hits']
except:
attribute_user_result = []
if attribute_user_result==[]:
status = True
return status
bulk_action = []
for user_dict in attribute_user_result:
try:
user_item = user_dict['_source']
except:
next
user_item.pop(attribute_name)
user = user_item['uid']
action = {'index':{'_id':str(user)}}
bulk_action.extend([action, user_item])
def search_task(task_name, submit_user, submit_date, state, status):
results = []
# query = [{"term": {"submit_user": submit_user}}]
query = []
condition_num = 0
if task_name:
task_name_list = task_name.split(" ")
for item in task_name_list:
query.append({"wildcard": {"task_name": "*" + item + "*"}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_start = submit_date_ts
submit_date_end = submit_date_ts + DAY
query.append({"range": {"submit_date": {"gte": submit_date_start, "lt": submit_date_end}}})
condition_num += 1
if state:
state_list = state.split(" ")
for item in state_list:
query.append({"wildcard": {"state": "*" + item + "*"}})
condition_num += 1
if status:
query.append({"match": {"status": status}})
condition_num += 1
if condition_num > 0:
query.append({"term": {"task_type": "analysis"}})
try:
source = es_group_result.search(
index=group_index_name,
doc_type=group_index_type,
body={"query": {"bool": {"must": query}}, "sort": [{"count": {"order": "desc"}}], "size": MAX_VALUE},
)
except Exception as e:
raise e
else:
query.append({"term": {"task_type": "analysis"}})
source = es.search(
index=group_index_name,
doc_type=group_index_type,
body={"query": {"bool": {"must": query}}, "sort": [{"count": {"order": "desc"}}], "size": MAX_VALUE},
)
try:
task_dict_list = source["hits"]["hits"]
except:
return None
print "step yes"
result = []
for task_dict in task_dict_list:
try:
state = task_dict["_source"]["state"]
except:
state = ""
try:
status = task_dict["_source"]["status"]
except:
status = 0
result.append(
[
task_dict["_source"]["task_name"],
task_dict["_source"]["submit_date"],
task_dict["_source"]["count"],
state,
status,
]
)
return result