def get_group_list(task_name, submit_user):
results = []
query_body = {
"query": {"bool": {"must": [{"term": {"task_name": task_name}}, {"term": {"submit_user": submit_user}}]}}
}
es_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)["hits"][
"hits"
][0]["_source"]
uid_list = es_results["uid_list"]
user_portrait_attribute = es_user_portrait.mget(
index=portrait_index_name, doc_type=portrait_index_type, body={"ids": uid_list}
)["docs"]
evaluate_max = get_evaluate_max()
for item in user_portrait_attribute:
uid = item["_id"]
try:
source = item["_source"]
uname = source["uname"]
gender = source["gender"]
location = source["location"]
importance = source["importance"]
normal_importance = math.log(importance / evaluate_max["importance"] * 9 + 1, 10) * 100
influence = source["influence"]
normal_influence = math.log(influence / evaluate_max["influence"] * 9 + 1, 10) * 100
results.append([uid, uname, gender, location, normal_importance, normal_influence])
except:
results.append([uid, "", "", "", "", ""])
return results
def show_detect_task(submit_user):
results = []
query = [{
'match': {
'task_type': 'detect'
}
}, {
'term': {
'submit_user': submit_user
}
}]
try:
search_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool':{'must':query}}, 'sort':[{'submit_date': 'desc'}], 'size':MAX_VALUE})['hits']['hits']
except:
search_results = []
for group_item in search_results:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append(
[task_name, submit_user, submit_date, detect_type, state, process])
return results
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user):
results = []
query = [{'match':{'task_type': 'detect'}}]
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard':{'task_name': '*'+item+'*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_from = submit_date_ts
submit_date_to = submit_date_ts + DAY
query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard':{'state': '*'+item+'*'}})
condition_num += 1
if process:
query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}})
condition_num += 1
if detect_type:
detect_type_list = detect_type.split(',')
nest_body_list = []
for type_item in detect_type_list:
nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}})
query.append({'bool':{'should': nest_body_list}})
condition_num += 1
if submit_user:
query.append({'wildcard':{'submit_user': '******'+submit_user+'*'}})
condition_num += 1
try:
search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
except:
search_result = []
#get group information table
for group_item in search_result:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append([task_name, submit_user, submit_date, detect_type, state, process])
return results
def get_group_list(task_name, submit_user):
results = []
query_body = {
"query": {
"bool": {
"must": [{
"term": {
"task_name": task_name
}
}, {
"term": {
"submit_user": submit_user
}
}]
}
}
}
es_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body=query_body)["hits"]["hits"][0]["_source"]
uid_list = es_results['uid_list']
user_portrait_attribute = es_user_portrait.mget(
index=portrait_index_name,
doc_type=portrait_index_type,
body={'ids': uid_list})['docs']
evaluate_max = get_evaluate_max()
for item in user_portrait_attribute:
uid = item['_id']
try:
source = item['_source']
uname = source['uname']
gender = source['gender']
location = source['location']
importance = source['importance']
normal_importance = math.log(
importance / evaluate_max['importance'] * 9 + 1, 10) * 100
influence = source['influence']
normal_influence = math.log(
influence / evaluate_max['influence'] * 9 + 1, 10) * 100
results.append([
uid, uname, gender, location, normal_importance,
normal_influence
])
except:
results.append([uid, '', '', '', '', ''])
return results
def show_detect_task(submit_user):
results = []
query = [{'match':{'task_type': 'detect'}}, {'term': {'submit_user': submit_user}}]
try:
search_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool':{'must':query}}, 'sort':[{'submit_date': 'desc'}], 'size':MAX_VALUE})['hits']['hits']
except:
search_results = []
for group_item in search_results:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append([task_name, submit_user, submit_date, detect_type, state, process])
return results
def search_group_results(task_name, submit_user, module):
result = {}
# step1:identify the task_name exist
query_body = {
"query": {"bool": {"must": [{"term": {"task_name": task_name}}, {"term": {"submit_user": submit_user}}]}}
}
# try:
source = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)["hits"]["hits"][
0
]["_source"]
# except:
# return 'group task is not exist'
# step2: identify the task status=1(analysis completed)
status = source["status"]
if status != 1:
return "group task is not completed"
# step3:get module result
if module == "overview":
result["task_name"] = source["task_name"]
result["submit_date"] = ts2datetime(source["submit_date"])
result["state"] = source["state"]
result["submit_user"] = source["submit_user"]
result["density_star"] = source["density_star"]
result["activeness_star"] = source["activeness_star"]
result["influence_star"] = source["influence_star"]
result["importance_star"] = source["importance_star"]
result["tag_vector"] = json.loads(source["tag_vector"])
result["count"] = source["count"]
elif module == "basic":
result["politics"] = json.loads(source["politics"])
result["domain"] = json.loads(source["domain"])
result["topic"] = json.loads(source["topic"])
result["keywords"] = json.loads(source["keywords"])
result["sensitive_hashtag"] = json.loads(source["sensitive_hashtag"])
result["hashtag"] = json.loads(source["hashtag"])
result["sensitive_words"] = json.loads(source["sensitive_words"])
result["influence_his"] = json.loads(source["influence_his"])
result["sensitive_his"] = json.loads(source["sensitive_his"])
elif module == "activity":
result["activity_trend"] = json.loads(source["activity_trend"])
result["activity_time"] = json.loads(source["activity_time"])
result["activeness_trend"] = json.loads(source["activeness"])
result["activeness_his"] = json.loads(source["activeness_his"])
result["activeness_description"] = source["activeness_description"]
result["online_pattern"] = json.loads(source["online_pattern"])
elif module == "geo":
# result['activity_geo_disribution'] = json.loads(source['activity_geo_distribution'])
new_activity_geo_distribution = deal_geo_distribution(json.loads(source["activity_geo_distribution"]))
result["activity_geo_disribution"] = new_activity_geo_distribution
result["activiy_geo_vary"] = json.loads(source["activity_geo_vary"])
elif module == "preference":
result["sentiment_word"] = json.loads(source["sentiment_word"])
elif module == "influence":
result["influence_trend"] = json.loads(source["influence"])
result["influence_in_user"] = json.loads(source["influence_in_user"])
result["influence_out_user"] = json.loads(source["influence_out_user"])
elif module == "social":
result["in_density"] = source["in_density"]
result["in_inter_user_ratio"] = source["in_inter_user_ratio"]
result["in_inter_weibo_ratio"] = source["in_inter_weibo_ratio"]
result["social_in_record"] = json.loads(source["social_in_record"])
result["out_inter_user_ratio"] = source["out_inter_user_ratio"]
result["out_inter_weibo_ratio"] = source["out_inter_weibo_ratio"]
result["social_out_record"] = json.loads(source["social_out_record"])
result["density_description"] = source["density_description"]
result["mention"] = source["mention"]
elif module == "think":
result["sentiment_trend"] = json.loads(source["sentiment_trend"])
result["sentiment_pie"] = json.loads(source["sentiment_pie"])
result["character"] = json.loads(source["character"])
return result
def search_task(task_name, submit_user, submit_date, state, status):
results = []
# query = [{"term": {"submit_user": submit_user}}]
query = []
condition_num = 0
if task_name:
task_name_list = task_name.split(" ")
for item in task_name_list:
query.append({"wildcard": {"task_name": "*" + item + "*"}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_start = submit_date_ts
submit_date_end = submit_date_ts + DAY
query.append({"range": {"submit_date": {"gte": submit_date_start, "lt": submit_date_end}}})
condition_num += 1
if state:
state_list = state.split(" ")
for item in state_list:
query.append({"wildcard": {"state": "*" + item + "*"}})
condition_num += 1
if status:
query.append({"match": {"status": status}})
condition_num += 1
if condition_num > 0:
query.append({"term": {"task_type": "analysis"}})
try:
source = es_group_result.search(
index=group_index_name,
doc_type=group_index_type,
body={"query": {"bool": {"must": query}}, "sort": [{"count": {"order": "desc"}}], "size": MAX_VALUE},
)
except Exception as e:
raise e
else:
query.append({"term": {"task_type": "analysis"}})
source = es.search(
index=group_index_name,
doc_type=group_index_type,
body={"query": {"bool": {"must": query}}, "sort": [{"count": {"order": "desc"}}], "size": MAX_VALUE},
)
try:
task_dict_list = source["hits"]["hits"]
except:
return None
print "step yes"
result = []
for task_dict in task_dict_list:
try:
state = task_dict["_source"]["state"]
except:
state = ""
try:
status = task_dict["_source"]["status"]
except:
status = 0
result.append(
[
task_dict["_source"]["task_name"],
task_dict["_source"]["submit_date"],
task_dict["_source"]["count"],
state,
status,
]
)
return result
def search_detect_task(task_name, submit_date, state, process, detect_type,
submit_user):
results = []
query = [{'match': {'task_type': 'detect'}}]
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard': {'task_name': '*' + item + '*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_from = submit_date_ts
submit_date_to = submit_date_ts + DAY
query.append({
'range': {
'submit_date': {
'gte': submit_date_from,
'lt': submit_date_to
}
}
})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard': {'state': '*' + item + '*'}})
condition_num += 1
if process:
query.append({
'range': {
'detect_process': {
'from': int(process),
'to': MAX_PROCESS
}
}
})
condition_num += 1
if detect_type:
detect_type_list = detect_type.split(',')
nest_body_list = []
for type_item in detect_type_list:
nest_body_list.append(
{'wildcard': {
'detect_type': '*' + type_item + '*'
}})
query.append({'bool': {'should': nest_body_list}})
condition_num += 1
if submit_user:
query.append({'wildcard': {'submit_user': '******' + submit_user + '*'}})
condition_num += 1
try:
search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
except:
search_result = []
#get group information table
for group_item in search_result:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append(
[task_name, submit_user, submit_date, detect_type, state, process])
return results
def search_group_results(task_name, submit_user, module):
result = {}
#step1:identify the task_name exist
query_body = {
"query": {
"bool": {
"must": [{
"term": {
"task_name": task_name
}
}, {
"term": {
"submit_user": submit_user
}
}]
}
}
}
#try:
source = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body=query_body)["hits"]["hits"][0]["_source"]
#except:
# return 'group task is not exist'
#step2: identify the task status=1(analysis completed)
status = source['status']
if status != 1:
return 'group task is not completed'
#step3:get module result
if module == 'overview':
result['task_name'] = source['task_name']
result['submit_date'] = ts2datetime(source['submit_date'])
result['state'] = source['state']
result['submit_user'] = source['submit_user']
result['density_star'] = source['density_star']
result['activeness_star'] = source['activeness_star']
result['influence_star'] = source['influence_star']
result['importance_star'] = source['importance_star']
result['tag_vector'] = json.loads(source['tag_vector'])
result['count'] = source['count']
elif module == 'basic':
result['politics'] = json.loads(source['politics'])
result['domain'] = json.loads(source['domain'])
result['topic'] = json.loads(source['topic'])
result['keywords'] = json.loads(source['keywords'])
result['sensitive_hashtag'] = json.loads(source['sensitive_hashtag'])
result['hashtag'] = json.loads(source['hashtag'])
result['sensitive_words'] = json.loads(source['sensitive_words'])
result['influence_his'] = json.loads(source['influence_his'])
result['sensitive_his'] = json.loads(source['sensitive_his'])
elif module == 'activity':
result['activity_trend'] = json.loads(source['activity_trend'])
result['activity_time'] = json.loads(source['activity_time'])
result['activeness_trend'] = json.loads(source['activeness'])
result['activeness_his'] = json.loads(source['activeness_his'])
result['activeness_description'] = source['activeness_description']
result['online_pattern'] = json.loads(source['online_pattern'])
elif module == 'geo':
#result['activity_geo_disribution'] = json.loads(source['activity_geo_distribution'])
new_activity_geo_distribution = deal_geo_distribution(
json.loads(source['activity_geo_distribution']))
result['activity_geo_disribution'] = new_activity_geo_distribution
result['activiy_geo_vary'] = json.loads(source['activity_geo_vary'])
elif module == 'preference':
result['sentiment_word'] = json.loads(source['sentiment_word'])
elif module == 'influence':
result['influence_trend'] = json.loads(source['influence'])
result['influence_in_user'] = json.loads(source['influence_in_user'])
result['influence_out_user'] = json.loads(source['influence_out_user'])
elif module == 'social':
result['in_density'] = source['in_density']
result['in_inter_user_ratio'] = source['in_inter_user_ratio']
result['in_inter_weibo_ratio'] = source['in_inter_weibo_ratio']
result['social_in_record'] = json.loads(source['social_in_record'])
result['out_inter_user_ratio'] = source['out_inter_user_ratio']
result['out_inter_weibo_ratio'] = source['out_inter_weibo_ratio']
result['social_out_record'] = json.loads(source['social_out_record'])
result['density_description'] = source['density_description']
result['mention'] = source['mention']
elif module == 'think':
result['sentiment_trend'] = json.loads(source['sentiment_trend'])
result['sentiment_pie'] = json.loads(source['sentiment_pie'])
result['character'] = json.loads(source['character'])
return result
def search_task(task_name, submit_user, submit_date, state, status):
results = []
#query = [{"term": {"submit_user": submit_user}}]
query = []
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard': {'task_name': '*' + item + '*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_start = submit_date_ts
submit_date_end = submit_date_ts + DAY
query.append({
'range': {
'submit_date': {
'gte': submit_date_start,
'lt': submit_date_end
}
}
})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard': {'state': '*' + item + '*'}})
condition_num += 1
if status:
query.append({'match': {'status': status}})
condition_num += 1
if condition_num > 0:
query.append({'term': {'task_type': 'analysis'}})
try:
source = es_group_result.search(index=group_index_name,
doc_type=group_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'sort': [{
'count': {
'order': 'desc'
}
}],
'size':
MAX_VALUE
})
except Exception as e:
raise e
else:
query.append({'term': {'task_type': 'analysis'}})
source = es.search(index=group_index_name,
doc_type=group_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'sort': [{
'count': {
'order': 'desc'
}
}],
'size': MAX_VALUE
})
try:
task_dict_list = source['hits']['hits']
except:
return None
print 'step yes'
result = []
for task_dict in task_dict_list:
try:
state = task_dict['_source']['state']
except:
state = ''
try:
status = task_dict['_source']['status']
except:
status = 0
result.append([
task_dict['_source']['task_name'],
task_dict['_source']['submit_date'], task_dict['_source']['count'],
state, status
])
return result
