def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
# identify the user exist
# identify the attribute exist
# identify the attribute exist in user_portrait
# add attribute in user_portrait
# submit user should has power to change???without
try:
user_result = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es.get(index=attribute_index_name, doc_type=user_index_type, id=attribute_name)['_source']
except:
return 'no attribute'
attribute_value_list = json.loads(attribute_result['value'])
if attribute_value not in attribute_value_list:
return 'no attribute value'
if attribute_name in user_result:
return 'attribute exist'
add_attribute_dict = {attribute_name: attribute_value}
es.update(index=user_index_name, doc_type=user_index_type, id=uid, body={'doc':body})
status = True
return status
def get_group_list(task_name):
results = []
try:
es_results = es.get(index=index_name, doc_type=index_type, id=task_name)['_source']
except:
return results
#print 'es_result:', es_results['uid_list'], type(es_results['uid_list'])
uid_list = es_results['uid_list']
user_portrait_attribute = es.mget(index='user_portrait', doc_type='user', body={'ids':uid_list})['docs']
evaluate_max = get_evaluate_max()
for item in user_portrait_attribute:
uid = item['_id']
try:
source = item['_source']
uname = source['uname']
gender = source['gender']
location = source['location']
importance = source['importance']
normal_importance = math.log(importance / evaluate_max['importance'] * 9 + 1, 10) * 100
influence = source['influence']
normal_influence = math.log(influence / evaluate_max['influence'] * 9 + 1, 10) * 100
results.append([uid, uname, gender, location, normal_importance, normal_influence])
except:
results.append([uid])
return results
def ajax_get_group_detail():
task_name = request.args.get('task_name','') # task_name
user = request.args.get('user', '')
_id = user + '-' + task_name
portrait_detail = []
top_activeness = get_top_influence("activeness")
top_influence = get_top_influence("influence")
top_importance = get_top_influence("importance")
search_result = es.get(index=index_group_manage, doc_type=doc_type_group, id=_id).get('_source', {})
if search_result:
try:
uid_list = json.loads(search_result['uid_list'])
except:
uid_list = search_result['uid_list']
if uid_list:
search_results = es.mget(index=portrait_index_name, doc_type=portrait_index_type, body={"ids":uid_list}, fields=SOCIAL_SENSOR_INFO)['docs']
for item in search_results:
temp = []
if item['found']:
for iter_item in SOCIAL_SENSOR_INFO:
if iter_item == "topic_string":
temp.append(item["fields"][iter_item][0].split('&'))
temp.append(item["fields"][iter_item][0].split('&'))
elif iter_item == "activeness":
temp.append(math.log(item['fields']['activeness'][0]/float(top_activeness)*9+1, 10)*100)
elif iter_item == "importance":
temp.append(math.log(item['fields']['importance'][0]/float(top_importance)*9+1, 10)*100)
elif iter_item == "influence":
temp.append(math.log(item['fields']['influence'][0]/float(top_influence)*9+1, 10)*100)
else:
temp.append(item["fields"][iter_item][0])
portrait_detail.append(temp)
return json.dumps(portrait_detail)
def submit_task(input_data):
status = 0 # mark it can not submit
task_name = input_data['task_name']
try:
result = es.get(index=index_name, doc_type=index_type, id=task_name)
except:
status = 1
if status != 0 and 'uid_file' not in input_data:
r.lpush('group_task', json.dumps(input_data))
input_data['status'] = 0 # mark the task not compute
count = len(input_data['uid_list'])
input_data['count'] = count
uid_list_string = json.dumps(input_data['uid_list'])
es.index(index='group_result', doc_type='group', id=task_name, body=input_data)
elif status != 0 and 'uid_file' in input_data:
input_data['status'] = 0 # mark the task not compute
uid_file = input_data['uid_file']
uid_list = read_uid_file(uid_file)
input_data['count'] = len(uid_list)
input_data['uid_list'] = json.dumps(uid_list)
r.lpush('group_task', json.dumps(input_data))
es.index(index='group_result', doc_type='group', id=task_name, body=input_data)
delete_status = delete_uid_file(uid_file)
if delete_status == 0:
print 'fail delete uid file'
elif delete_status == 1:
print 'success delete uid file'
return status
def get_group_user_track(uid):
results = []
#step1:get user_portrait activity_geo_dict
try:
portrait_result = es_user_portrait.get(index=portrait_index_name, doc_type=portrait_index_type,\
id=uid, _source=False, fields=['activity_geo_dict'])
except:
portrait_result = {}
if portrait_result == {}:
return 'uid is not in user_portrait'
activity_geo_dict = json.loads(
portrait_result['fields']['activity_geo_dict'][0])
now_date_ts = datetime2ts(ts2datetime(int(time.time())))
start_ts = now_date_ts - DAY * len(activity_geo_dict)
#step2: iter date to get month track
for geo_item in activity_geo_dict:
iter_date = ts2datetime(start_ts)
sort_day_dict = sorted(geo_item.items(),
key=lambda x: x[1],
reverse=True)
if sort_day_dict:
results.append([iter_date, sort_day_dict[0][0]])
else:
results.append([iter_date, ''])
start_ts = start_ts + DAY
return results
def ajax_get_clustering_topic():
task_name = request.args.get('task_name', '') # task_name
user = request.args.get('user', '')
ts = int(request.args.get('ts', '')) # timestamp: 123456789
topic_list = []
_id = user + '-' + task_name
try:
task_detail = es.get(index=index_sensing_task, doc_type=_id,
id=ts)['_source']
except:
return json.dumps([])
burst_reason = task_detail['burst_reason']
if burst_reason:
topic_list = task_detail.get("clustering_topic", [])
if topic_list:
filter_list = []
topic_list = json.loads(topic_list)
for item in topic_list:
tmp = []
for word in item:
if len(word) > 3:
tmp.append(word)
filter_list.append(tmp)
return json.dumps(filter_list[:5])
def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
submit_user_tag = submit_user + "-tag"
id_attribute = submit_user + "-" + attribute_name
add_attribute_value = attribute_name + "-" + attribute_value
# identify the user exist
# identify the attribute exist
# identify the attribute exist in user_portrait
# add attribute in user_portrait
try:
user_result = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es_tag.get(index=attribute_index_name, doc_type=attribute_index_type, id=id_attribute)['_source']
except:
return 'no attribute'
attribute_value_list = attribute_result['attribute_value'].split('&')
if attribute_value not in attribute_value_list:
return 'no attribute value'
value_set = set()
for value in attribute_value_list:
value_set.add(attribute_name + '-' + value) #
submit_user_attribute = user_result.get(submit_user_tag, '') # 个人是否存在该管理员打上的个人标签
tmp_attribute_set = set(submit_user_attribute.split('&'))
attribute_exist = tmp_attribute_set & value_set
if attribute_exist:
return 'attribute exist'
user_result[submit_user_tag] = "&".join([submit_user_attribute, add_attribute_value])
es.index(index=user_index_name, doc_type=user_index_type, id=uid, body=user_result)
status = True
return status
def delete_attribute_portrait(uid, attribute_name, submit_user):
status = False
submit_user_tag = submit_user + "-tag"
id_attribute = submit_user + "-" + attribute_name
#identify the user exist
#identify the attribute value exist in es_user_portrait
#identify the submit_user have been admitted---without
try:
user_result = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
return 'no user'
submit_user_attribute = user_result.get(submit_user_tag, '')
if attribute_name not in submit_user_attribute:
return 'user have no attribtue'
try:
tmp_attribute = submit_user_attribute.split('&')
for item in tmp_attribute:
if attribute_name in item:
tmp_attribute.remove(item)
user_result[submit_user_tag] = "&".join(tmp_attribute)
es.index(index=user_index_name,
doc_type=user_index_type,
id=uid,
body=user_result)
status = True
except Exception, e:
raise e
def ajax_revise_task():
task_name = request.args.get('task_name', '') # must
finish = request.args.get("finish", "10")
stop_time = request.args.get('stop_time', '') # timestamp
user = request.args.get('user', '')
#now_ts = datetime2ts("2013-09-06")
_id = user + '-' + task_name
now_ts = time.time()
if stop_time and stop_time < now_ts:
return json.dumps([])
if task_name and user:
task_detail = es.get(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=_id)['_source']
if stop_time:
task_detail['stop_time'] = stop_time
if int(finish) == 0:
task_detail['finish'] = finish
task_detail['processing_status'] = "1" # 重启时将处理状态改为
if stop_time or int(finish) == 0:
es.index(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=_id,
body=task_detail)
return json.dumps(['1'])
return json.dumps([])
def add_tag2group(uid_list, attribute_name, attribute_value, submit_user):
id_attribute = submit_user + "-" + attribute_name
add_attribute = attribute_name + "-" + attribute_value
submit_user_tag = submit_user + "-tag"
status = False
#identify the attribute exist
#for uid in uid_list
#identify the attribute not in this user
#add tag to this user
try:
attribute_exist = es_tag.get(index=attribute_index_name, doc_type=attribute_index_type, id=id_attribute)['_source']
except:
return 'no attribute'
attribute_exist_value_list = attribute_exist['attribute_value'].split('&')
if attribute_value not in attribute_exist_value_list:
return 'no attribute value'
for uid in uid_list:
try:
user_exist = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
user_exist = {}
continue
submit_user_attribute = user_exist.get(submit_user_tag, '')
if user_exist and attribute_name not in submit_user_attribute:
user_exist[submit_user_tag] = "&".join([submit_user_attribute, add_attribute])
es.index(index=user_index_name, doc_type=user_index_type, id=uid, body=user_exist)
status = True
return status
def get_text_detail(task_name, ts, text_type, user, order, size=100):
results = []
_id = user + '-' + task_name
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=_id)["_source"]
social_sensors = json.loads(task_detail["social_sensors"])
#print social_sensors
if int(text_type) == 0: # 热门原创微博
results = get_origin_weibo_detail(ts, user, task_name, size, order, 1)
elif int(text_type) == 1: # 热门转发微博
results = get_origin_weibo_detail(ts, user, task_name, size, order, 2)
elif int(text_type) == 2: # 普通转发微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "message_type", 3)
elif int(text_type) == 3: # 普通评论微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "message_type", 2)
elif int(text_type) == 4: # 积极微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", "1")
elif int(text_type) == 5: # 中性微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", "0")
elif int(text_type) == 6: # 消极微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", ["2", "3", "4", "5", "6"])
elif int(text_type) == 7: # 敏感微博
results = get_origin_weibo_detail(ts, user, task_name, size, order, 3)
else:
print "error"
print '******************'
#print results
return results
def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
# identify the user exist
# identify the attribute exist
# identify the attribute exist in user_portrait
# add attribute in user_portrait
try:
user_result = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es_tag.get(index=attribute_index_name,
doc_type=attribute_index_type,
id=attribute_name)['_source']
except:
return 'no attribute'
attribute_value_list = attribute_result['attribute_value'].split('&')
if attribute_value not in attribute_value_list:
return 'no attribute value'
if attribute_name in user_result:
return 'attribute exist'
add_attribute_dict = {attribute_name: attribute_value}
es.update(index=user_index_name,
doc_type=user_index_type,
id=uid,
body={'doc': add_attribute_dict})
status = True
return status
def add_tag2group(uid_list, attribute_name, attribute_value):
status = False
#identify the attribute exist
#for uid in uid_list
#identify the attribute not in this user
#add tag to this user
try:
attribute_exist = es_tag.get(index=attribute_index_name,
doc_type=attribute_index_type,
id=attribute_name)['_source']
except:
return 'no attribute'
attribute_exist_value_list = attribute_exist['attribute_value'].split('&')
if attribute_value not in attribute_exist_value_list:
return 'no attribute value'
for uid in uid_list:
try:
user_exist = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
user_exist = {}
if user_exist and attribute_name not in user_exist:
add_attribute_dict = {attribute_name: attribute_value}
es.update(index=user_index_name,
doc_type=user_index_type,
id=uid,
body={'doc': add_attribute_dict})
status = True
return status
def change_attribute_portrait(uid, attribute_name, attribute_value,
submit_user):
status = False
#identify the user exist
#identify the attribute exist
#identify the attribute value exist
#identify the submit_user have been admitted----without
try:
user_exist = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es_tag.get(index=attribute_index_name,
doc_type=attribute_index_type,
id=attribute_name)['_source']
except:
return 'no attribute'
value_list = attribute_result['attribute_value'].split('&')
if attribute_value not in value_list:
return 'no attribute value'
change_attribute_dict = {attribute_name: attribute_value}
es.update(index=user_index_name,
doc_type=user_index_type,
id=uid,
body={'doc': change_attribute_dict})
status = True
return status
def ajax_get_group_detail():
task_name = request.args.get('task_name','') # task_name
user = request.args.get('user', '')
_id = user + '-' + task_name
portrait_detail = []
top_activeness = get_top_influence("activeness")
top_influence = get_top_influence("influence")
top_importance = get_top_influence("importance")
search_result = es.get(index=index_group_manage, doc_type=doc_type_group, id=_id).get('_source', {})
if search_result:
try:
uid_list = json.loads(search_result['uid_list'])
except:
uid_list = search_result['uid_list']
if uid_list:
search_results = es.mget(index=portrait_index_name, doc_type=portrait_index_type, body={"ids":uid_list}, fields=SOCIAL_SENSOR_INFO)['docs']
for item in search_results:
temp = []
if item['found']:
for iter_item in SOCIAL_SENSOR_INFO:
if iter_item == "topic_string":
temp.append(item["fields"][iter_item][0].split('&'))
temp.append(item["fields"][iter_item][0].split('&'))
elif iter_item == "activeness":
temp.append(math.ceil(item["fields"][iter_item][0]/float(top_activeness)*100))
elif iter_item == "importance":
temp.append(math.ceil(item["fields"][iter_item][0]/float(top_importance)*100))
elif iter_item == "influence":
temp.append(math.ceil(item["fields"][iter_item][0]/float(top_influence)*100))
else:
temp.append(item["fields"][iter_item][0])
portrait_detail.append(temp)
return json.dumps(portrait_detail)
def get_text_detail(task_name, ts, text_type, user, order, size=100):
results = []
_id = user + '-' + task_name
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=_id)["_source"]
social_sensors = json.loads(task_detail["social_sensors"])
#print social_sensors
if int(text_type) == 0: # 热门原创微博
results = get_origin_weibo_detail(ts, user, task_name, size, order, 1)
elif int(text_type) == 1: # 热门转发微博
results = get_origin_weibo_detail(ts, user, task_name, size, order, 2)
elif int(text_type) == 2: # 普通转发微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "message_type", 3)
elif int(text_type) == 3: # 普通评论微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "message_type", 2)
elif int(text_type) == 4: # 积极微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", "1")
elif int(text_type) == 5: # 中性微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", "0")
elif int(text_type) == 6: # 消极微博
results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", ["2", "3", "4", "5", "6"])
elif int(text_type) == 7: # 敏感微博
results = get_origin_weibo_detail(ts, user, task_name, size, order, 3)
else:
print "error"
return results
def delete_attribute(attribute_name):
status = False
try:
result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
except:
return status
attribute_value = json.loads(result['value'])
es.delete(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)
# delete attribute in user_portrait
query = []
for value in attribute_value:
query.append({'match':{attribute_name: value}})
try:
attribute_user_result = es.search(index=user_index_name, doc_type=user_index_type, \
body={'query':{'bool':{'must':query}}})['hits']['hits']
except:
attribute_user_result = []
if attribute_user_result==[]:
status = True
return status
bulk_action = []
for user_dict in attribute_user_result:
try:
user_item = user_dict['_source']
except:
next
user_item.pop(attribute)
user = user_item['uid']
action = {'index':{'_id':str(user)}}
bulk_action.extend([action, user_item])
es.bulk(bulk_action, index=user_index_name, doc_type=index_type)
status = True
return status
def change_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
submit_user_tag = submit_user + "-tag"
id_attribute = submit_user + "-" + attribute_name
#identify the user exist
#identify the attribute exist
#identify the attribute value exist
#identify the submit_user have been admitted----without
try:
user_result = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es_tag.get(index=attribute_index_name, doc_type=attribute_index_type, id=id_attribute)['_source']
except:
return 'no attribute'
value_list = attribute_result['attribute_value'].split('&')
if attribute_value not in value_list:
return 'no attribute value'
submit_user_attribute = user_result.get(submit_user_tag, '') # 个人是否存在该管理员打上的个人标签
if attribute_name not in submit_user_attribute:
return 'personal attribute no exist'
tmp_attribute_list = submit_user_attribute.split("&")
attribute_list = []
for item in tmp_attribute_list:
if attribute_name in item:
attribute_list.append(attribute_name + '-' + attribute_value)
else:
attribute_list.append(item)
user_result[submit_user_tag] = "&".join(attribute_list)
es.index(index=user_index_name, doc_type=user_index_type, id=uid, body=user_result)
status = True
return status
def search_identify_uid(uid):
result = 0
try:
user_dict = es_user_portrait.get(index=portrait_index_name, doc_type=portrait_index_type, id=uid)
result = 1
except:
result = 0
return result
def delete_attribute(attribute_name):
status = False
try:
result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
print 'result:', result
except Exception, e:
raise e
return status
def get_group_weibo(task_name, date):
group_weibo = []
#step1 : get group user list by task_name
group_index_name = 'group_result'
group_index_type = 'group'
try:
group_task = es.get(index=group_index_name, doc_type=group_index_type, id=task_name)['_source']
except Exception ,e:
raise e
def search_identify_uid(uid):
result = 0
try:
user_dict = es_user_portrait.get(index='user_portrait', doc_type='user', id=uid)
#print 'user_dict:', user_dict
result = 1
except:
result = 0
return result
def conclusion_on_influence(uid):
# test
index_name = copy_portrait_index_name
index_type = copy_portrait_index_type
total_number = es.count(index=copy_portrait_index_name, doc_type=copy_portrait_index_type)["count"]
try:
influ_result = es.get(index=index_name, doc_type=index_type, id=uid)["_source"]
except:
influ_result = {}
result = [0, 0, 0, 0, 0, 0, total_number] # aver_activeness, sorted, aver_influence, sorted
return result
aver_activeness = influ_result.get("aver_activeness", 0)
aver_influence = influ_result.get("aver_influence", 0)
aver_importance = influ_result.get("aver_importance", 0)
influence_query_body = {"query": {"match_all": {}}, "sort": {"aver_influence": {"order": "desc"}}, "size": 1}
top_influence = es.search(
index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=influence_query_body
)["hits"]["hits"][0]["sort"][0]
importance_query_body = {"query": {"match_all": {}}, "sort": {"aver_importance": {"order": "desc"}}, "size": 1}
top_importance = es.search(
index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=importance_query_body
)["hits"]["hits"][0]["sort"][0]
activeness_query_body = {"query": {"match_all": {}}, "sort": {"aver_activeness": {"order": "desc"}}, "size": 1}
top_activeness = es.search(
index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=activeness_query_body
)["hits"]["hits"][0]["sort"][0]
influence_query_body = {"query": {"filtered": {"filter": {"range": {"aver_influence": {"gt": aver_influence}}}}}}
activeness_query_body = {"query": {"filtered": {"filter": {"range": {"aver_activeness": {"gt": aver_activeness}}}}}}
importance_query_body = {"query": {"filtered": {"filter": {"range": {"aver_importance": {"gt": aver_importance}}}}}}
influence_count = es.count(
index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=influence_query_body
)["count"]
activeness_count = es.count(
index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=activeness_query_body
)["count"]
importance_count = es.count(
index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=importance_query_body
)["count"]
result = [
int(aver_activeness * 100.0 / top_activeness),
activeness_count,
int(aver_influence * 100.0 / top_influence),
influence_count,
int(aver_importance * 100.0 / top_importance),
importance_count,
total_number,
]
return result
def ajax_get_task_detail_info():
task_name = request.args.get('task_name','') # task_name
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)['_source']
task_detail["social_sensors"] = json.loads(task_detail["social_sensors"])
task_detail['keywords'] = json.loads(task_detail['keywords'])
task_detail["sensitive_words"]= json.loads(task_detail["sensitive_words"])
history_status = json.loads(task_detail['history_status'])
if history_status:
temp_list = []
temp_list.append(history_status[-1])
for item in history_status[:-1]:
if int(item[-1]) != 0:
temp_list.append(item)
sorted_list = sorted(temp_list, key=lambda x:x[0], reverse=True)
task_detail['history_status'] = sorted_list
else:
task_detail['history_status'] = history_status
task_detail['social_sensors_portrait'] = []
portrait_detail = []
if task_detail["social_sensors"]:
search_results = es.mget(index=portrait_index_name, doc_type=portrait_index_type, body={"ids": task_detail["social_sensors"]})['docs']
if search_results:
for item in search_results:
temp = []
if item['found']:
for iter_item in SOCIAL_SENSOR_INFO:
if iter_item == "topic_string":
temp.append(item["_source"][iter_item].split('&'))
elif iter_item == "influence":
top_influence = get_top_influence("influence")
influence = math.log(item["_source"][iter_item]/top_influence*9+1, 10)*100
if not influence:
influence = 0
temp.append(influence)
elif iter_item == "importance":
top_importance = get_top_influence("importance")
importance = math.log(item["_source"][iter_item]/top_importance*9+1, 10)*100
if not importance:
importance = 0
temp.append(importance)
elif iter_item == "activeness":
top_activeness = get_top_influence("activeness")
activeness = math.log(item["_source"][iter_item]/top_activeness*9+1, 10)*100
if not activeness:
activeness = 0
temp.append(activeness)
else:
temp.append(item["_source"][iter_item])
portrait_detail.append(temp)
if portrait_detail:
portrait_detail = sorted(portrait_detail, key=lambda x:x[5], reverse=True)
task_detail['social_sensors_portrait'] = portrait_detail
#print task_detail
return json.dumps(task_detail)
def get_attribute_value(attribute_name):
attribute_value_list = []
try:
attribute_result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
except:
return 'no attribute'
print 'attribute_result:', attribute_result
attribute_value_string = attribute_result['attribute_value']
attribute_value_list = attribute_value_string.split('&')
return attribute_value_list
def ajax_stop_task():
task_name = request.args.get('task_name','') # must
if task_name:
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)['_source']
#task_detail["finish"] = finish_signal
task_detail['processing_status'] = '0'
es.index(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name, body=task_detail)
return json.dumps(['1'])
else:
return json.dumps([])
def change_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
#identify the user exist
#identify the attribute exist
#identify the attribute value exist
#identify the submit_user have been admitted----without
try:
user_exist = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es.get(index=attribute_index_name, doc_type=user_index_type, id=uid)['_source']
except:
return 'no attribute'
value_list = json.loads(attribute_result['value'])
if attribute_value not in value_list:
return 'no attribute value'
es.update(index=user_index_name, doc_type=user_index_type, id=uid, body={'doc':body})
status = True
return status
def get_group_weibo(task_name, date):
group_weibo = []
#step1 : get group user list by task_name
group_index_name = 'group_result'
group_index_type = 'group'
try:
group_task = es.get(index=group_index_name,
doc_type=group_index_type,
id=task_name)['_source']
except Exception, e:
raise e
def ajax_stop_task():
task_name = request.args.get('task_name','') # must
user = request.args.get('user', '')
if task_name and user:
_id = user + "-" + task_name
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=_id)['_source']
#task_detail["finish"] = finish_signal
task_detail['processing_status'] = '0'
es.index(index=index_manage_sensing_task, doc_type=task_doc_type, id=_id, body=task_detail)
return json.dumps(['1'])
else:
return json.dumps([])
def ajax_get_keywords_list():
task_name = request.args.get('task_name','') # task_name
keywords = request.args.get('keywords', '') # warning keywords, seperate with ","
keywords_list = keywords.split(',')
ts = request.args.get('ts', '') # timestamp: 123456789
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)['_source']
start_time = task_detail['create_at']
results = aggregation_hot_keywords(start_time, ts, keywords_list)
return json.dumps(results)
def get_user_attribute_name(uid):
result = []
user_result = es.get(index=user_index_name, doc_type=user_index_type, \
id=uid)
try:
source = user_result['_source']
except:
source = {}
for key in source:
if key not in identify_attribute_list:
result.append(key)
return result
def get_user_attribute_name(uid):
result = []
user_result = es.get(index=user_index_name, doc_type=user_index_type, \
id=uid)
try:
source = user_result['_source']
except:
source = {}
for key in source:
if key not in identify_attribute_list:
result.append(key)
return result
def add_tag2group(uid_list, attribute_name, attribute_value):
status = False
#identify the attribute exist
#for uid in uid_list
#identify the attribute not in this user
#add tag to this user
try:
attribute_exist = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
except:
return 'no attribute'
attribute_exist_value_list = attribute_exist['attribute_value'].split('&')
if attribute_value not in attribute_exist_value_list:
return 'no attribute value'
for uid in uid_list:
try:
user_exist = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
user_exist = {}
if user_exist and attribute_name not in user_exist:
add_attribute_dict = {attribute_name: attribute_value}
es.update(index=user_index_name, doc_type=user_index_type, id=uid, body={'doc':add_attribute_dict})
status = True
return status
def get_sensor_detail(task_name, ts, user):
index_name = task_name
_id = user + "-" + task_name
task_detail = es.get(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=_id)["_source"]
social_sensors = json.loads(task_detail['social_sensors'])
portrait_detail = []
top_importance = get_top_influence("importance")
top_influence = get_top_influence("influence")
top_activeness = get_top_influence("activeness")
if social_sensors:
search_results = es.mget(index=portrait_index_name,
doc_type=portrait_index_type,
body={"ids": social_sensors},
fields=SOCIAL_SENSOR_INFO)['docs']
for item in search_results:
temp = []
if item['found']:
for iter_item in SOCIAL_SENSOR_INFO:
if iter_item == "topic_string":
temp.append(item["fields"][iter_item][0].split('&'))
elif iter_item == "activeness":
temp.append(
math.log(
item['fields']['activeness'][0] /
float(top_activeness) * 9 + 1, 10) * 100)
elif iter_item == "importance":
temp.append(
math.log(
item['fields']['importance'][0] /
float(top_importance) * 9 + 1, 10) * 100)
elif iter_item == "influence":
temp.append(
math.log(
item['fields']['influence'][0] /
float(top_influence) * 9 + 1, 10) * 100)
else:
temp.append(item["fields"][iter_item][0])
portrait_detail.append(temp)
portrait_detail = sorted(portrait_detail,
key=lambda x: x[5],
reverse=True)
else:
portrait_detail = []
return portrait_detail
def get_user_attribute_name(uid, submit_user):
result = []
user_result = es.get(index=user_index_name, doc_type=user_index_type, \
id=uid)
try:
source = user_result['_source']
except:
source = {}
submit_user_tag = submit_user + "-tag"
submit_user_attribute = source.get(submit_user_tag, '')
tag_list = submit_user_attribute.split('&')
for item in tag_list:
result.append(item.split('-')[0])
return result
def get_user_attribute_name(uid, submit_user):
result = []
user_result = es.get(index=user_index_name, doc_type=user_index_type, \
id=uid)
try:
source = user_result['_source']
except:
source = {}
submit_user_tag = submit_user + "-tag"
submit_user_attribute = source.get(submit_user_tag, '')
tag_list = submit_user_attribute.split('&')
for item in tag_list:
result.append(item.split('-')[0])
return result
def ajax_get_keywords_list():
task_name = request.args.get('task_name', '') # task_name
keywords = request.args.get('keywords',
'') # warning keywords, seperate with ","
keywords_list = keywords.split(',')
ts = request.args.get('ts', '') # timestamp: 123456789
task_detail = es.get(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=task_name)['_source']
start_time = task_detail['create_at']
results = aggregation_hot_keywords(start_time, ts, keywords_list)
return json.dumps(results)
def ajax_get_clustering_topic():
task_name = request.args.get('task_name','') # task_name
ts = int(request.args.get('ts', '')) # timestamp: 123456789
topic_list = []
try:
task_detail = es.get(index=index_sensing_task, doc_type=task_name, id=ts)['_source']
except:
return json.dumps([])
burst_reason = task_detail['burst_reason']
if burst_reason:
topic_list = task_detail.get("clustering_topic", [])
if topic_list:
topic_list = json.loads(topic_list)
return json.dumps(topic_list)
def submit_task(input_data):
status = 0 # mark it can not submit
task_name = input_data['task_name']
try:
result = es.get(index=index_name, doc_type=index_type, id=task_name)
except:
status = 1
if status != 0:
r.lpush('group_task', json.dumps(input_data))
input_data['status'] = 0 # mark the task not compute
count = len(input_data['uid_list'])
input_data['count'] = count
uid_list_string = json.dumps(input_data['uid_list'])
es.index(index='group_result', doc_type='group', id=task_name, body=input_data)
return status
def ajax_get_task_detail_info():
task_name = request.args.get('task_name', '') # task_name
user = request.args.get('user', 'admin')
_id = user + "-" + task_name
task_detail = es.get(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=_id)['_source']
task_detail["social_sensors"] = json.loads(task_detail["social_sensors"])
#task_detail['keywords'] = json.loads(task_detail['keywords'])
#task_detail["sensitive_words"]= json.loads(task_detail["sensitive_words"])
history_status = json.loads(task_detail['history_status'])
if history_status:
temp_list = []
"""
temp_list.append(history_status[-1])
print history_status
for item in history_status[:-1]:
temp_list.append(item)
"""
sorted_list = sorted(history_status, key=lambda x: x, reverse=True)
task_detail['history_status'] = sorted_list
else:
task_detail['history_status'] = []
task_detail['social_sensors_portrait'] = []
portrait_detail = []
if task_detail["social_sensors"]:
search_results = es.mget(index=portrait_index_name,
doc_type=portrait_index_type,
body={"ids":
task_detail["social_sensors"]})['docs']
if search_results:
for item in search_results:
temp = []
if item['found']:
for iter_item in SOCIAL_SENSOR_INFO:
if iter_item == "topic_string":
temp.append(item["_source"][iter_item].split('&'))
else:
temp.append(item["_source"][iter_item])
portrait_detail.append(temp)
if portrait_detail:
portrait_detail = sorted(portrait_detail,
key=lambda x: x[5],
reverse=True)
task_detail['social_sensors_portrait'] = portrait_detail
return json.dumps(task_detail)
def get_sort(uid):
try:
u_bci = es.get(index=BCI_INDEX_NAME, doc_type=BCI_INDEX_TYPE, id=uid,fields=['bci_week_ave'])['fields']['bci_week_ave'][0]
except:
return None
query_body={
'query':{
'filtered':{
'filter':{
'range':{'bci_week_ave':{'gte':u_bci}}
}
}
}
}
result = es.search(index=BCI_INDEX_NAME, doc_type=BCI_INDEX_TYPE,body=query_body)
return str(result['hits']['total'])
def new_get_sensitive_words(uid):
try:
user_portrait_result = es_user_portrait.get(index=portrait_index_name, doc_type=portrait_index_type,\
id=uid)['_source']
except:
user_portrait_result = {}
if user_portrait_result:
try:
sensitive_dict = json.loads(es_user_portrait['sensitive_dict'])
except:
sensitive_dict = {}
else:
sensitive_dict = {}
sort_sensitive_dict = sorted(sensitive_dict.items(), key=lambda x:x[1], reverse=True)
return sort_sensitive_dict
def new_get_sensitive_words(uid):
try:
user_portrait_result = es_user_portrait.get(index=portrait_index_name, doc_type=portrait_index_type,\
id=uid)['_source']
except:
user_portrait_result = {}
if user_portrait_result:
try:
sensitive_dict = json.loads(es_user_portrait['sensitive_dict'])
except:
sensitive_dict = {}
else:
sensitive_dict = {}
sort_sensitive_dict = sorted(sensitive_dict.items(), key=lambda x:x[1], reverse=True)
return sort_sensitive_dict
def ajax_get_clustering_topic():
task_name = request.args.get('task_name', '') # task_name
ts = int(request.args.get('ts', '')) # timestamp: 123456789
topic_list = []
try:
task_detail = es.get(index=index_sensing_task,
doc_type=task_name,
id=ts)['_source']
except:
return json.dumps([])
burst_reason = task_detail['burst_reason']
if burst_reason:
topic_list = task_detail.get("clustering_topic", [])
if topic_list:
topic_list = json.loads(topic_list)
return json.dumps(topic_list)
def delete_attribute_portrait(uid, attribute_name, submit_user):
status = False
#identify the user exist
#identify the attribute value exist in es_user_portrait
#identify the submit_user have been admitted---without
try:
user_exist = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
except:
return 'no user'
if attribute_name not in user_exist:
return 'user have no attribtue'
try:
del_attribute_value = user_exist.pop(attribute)
es.index(index=user_index_name, doc_type=user_index_type, body=user_exist)
status = True
except Exception, e:
raise e
def change_attribute(attribute_name, value, user, state):
status = False
# identify the attribute_name is in ES - custom attribute
try:
result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
except:
result = None
return status
value_list = value.split('&')
result['value'] = json.dumps(value_list)
result['user'] = user
result['state'] = state
now_ts = time.time()
now_date = ts2datetime(now_ts)
result['date'] = now_date
es.index(index=attribute_index_name, doc_type=attribute_index_type, body=result)
status = True
return status
def submit_attribute(attribute_name, attribute_value, submit_user, state):
status = False
#maybe there have to identify the user admitted to submit attribute
attribute_exist = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['docs']
try:
source = attribute_exist['_source']
except:
input_data = dict()
now_ts = time.time()
date = ts2datetime(now_ts)
input_data['name'] = attribute_name
input_data['value'] = json.dumps(attribute_value.split('&'))
input_data['user'] = submit_user
input_data['state'] = state
input_data['date'] = date
es.index(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name, body=input_data)
status = True
return status
def get_group_tag(group_name):
result = {}
order_result = []
#get group task uid list
#get user tag
#statistic tag
try:
group_task_result = es.get(index=group_index_name, doc_type=group_index_type, id=group_name)
except:
return 'no group task'
try:
uid_list = group_task_result['_source']['uid_list']
except:
return 'no user'
try:
user_result = es.mget(index=user_index_name, doc_type=user_index_type, body={'ids': uid_list})['docs']
except Exception, e:
raise e
def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
submit_user_tag = submit_user + "-tag"
id_attribute = submit_user + "-" + attribute_name
add_attribute_value = attribute_name + "-" + attribute_value
# identify the user exist
# identify the attribute exist
# identify the attribute exist in user_portrait
# add attribute in user_portrait
try:
user_result = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es_tag.get(index=attribute_index_name,
doc_type=attribute_index_type,
id=id_attribute)['_source']
except:
return 'no attribute'
attribute_value_list = attribute_result['attribute_value'].split('&')
if attribute_value not in attribute_value_list:
return 'no attribute value'
value_set = set()
for value in attribute_value_list:
value_set.add(attribute_name + '-' + value) #
submit_user_attribute = user_result.get(submit_user_tag,
'') # 个人是否存在该管理员打上的个人标签
tmp_attribute_set = set(submit_user_attribute.split('&'))
attribute_exist = tmp_attribute_set & value_set
if attribute_exist:
return 'attribute exist'
if submit_user_attribute:
user_result[submit_user_tag] = "&".join(
[submit_user_attribute, add_attribute_value])
else:
user_result[submit_user_tag] = add_attribute_value
es.index(index=user_index_name,
doc_type=user_index_type,
id=uid,
body=user_result)
status = True
return status
def ajax_imagine():
uid = request.args.get('uid', '') # uid
query_keywords = request.args.get('keywords', '') # 查询字段
submit_user = request.args.get('submit_user', '')
query_weight = request.args.get('weight', '') # 权重
size = request.args.get('size', 100)
keywords_list = query_keywords.split(',')
weight_list = query_weight.split(',')
if len(keywords_list) != len(weight_list):
return json.dumps([])
query_fields_dict = {}
for i in range(len(keywords_list)):
query_fields_dict[keywords_list[i]] = int(weight_list[i])
# 如果查询为空，获取上一次请求数据，再为空，默认领域搜索
if not query_fields_dict:
user_imagine_dict = {}
imagine_setting = R_ADMIN.hget(submit_user, "imagine_setting")
if not imagine_setting:
user_info = es_user_portrait.get(index="user_portrait_1222",
doc_type="user",
id=uid,
_source=False,
fields=["domain"])['_source']
user_domain = user_info['fields']['domain'][0]
query_fields_dict[user_domain] = 1
else:
query_fields_dict = json.loads(imagine_setting)
else:
R_ADMIN.hset(submit_user, "imagine_setting",
json.dumps(query_fields_dict))
query_fields_dict['size'] = int(size)
result = []
if uid and query_fields_dict:
result = imagine(submit_user, uid, query_fields_dict)
if result:
return json.dumps(result)
return json.dumps([])
def conclusion_on_activeness(uid):
# test
index_name = copy_portrait_index_name
index_type = copy_portrait_index_type
try:
influ_result = es.get(index=index_name, doc_type=index_type,
id=uid)['_source']
except:
influ_result = {}
result = activeness_conclusion_dict['0']
return result
# generate time series---keys
now_ts = time.time()
now_ts = datetime2ts('2013-09-12')
activeness_set = set()
for i in range(N):
ts = ts2datetime(now_ts - i * 3600 * 24)
activeness_set.add(pre_activeness + ts)
# 区分影响力和活跃度的keys
keys_set = set(influ_result.keys())
activeness_keys = keys_set & activeness_set
if activeness_keys:
activeness_value = []
for key in activeness_keys:
activeness_value.append(influ_result[key])
mean, std_var = level(activeness_value)
if mean < activeness_level[0]:
result = activeness_conclusion_dict['1']
elif mean >= activeness_level[0] and mean < activeness_level[1]:
result = activeness_conclusion_dict['2']
elif mean >= activeness_level[1] and mean < activeness_level[2]:
result = activeness_conclusion_dict["3"]
elif mean >= activeness_level[2] and mean < activeness_level[3]:
result = activeness_conclusion_dict["4"]
else:
result = activeness_conclusion_dict["5"]
else:
result = conclusion_dict['0']
return result
def get_text_detail(task_name, ts, text_type, size=100):
results = []
task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)["_source"]
keywords_list = json.loads(task_detail['keywords'])
sensitive_words_list = json.loads(task_detail['sensitive_words'])
social_sensors = json.loads(task_detail["social_sensors"])
#print social_sensors
if int(text_type) == 0: # 原创微博
results = get_origin_weibo_detail(ts, social_sensors, keywords_list, size)
elif int(text_type) == 1: # 转发微博
results = get_retweet_weibo_detail(ts, social_sensors, keywords_list, size, 3)
#results = get_origin_weibo_detail(ts, social_sensors, keywords_list, size, 3)
elif int(text_type) == 2: # 评论微博
results = get_retweet_weibo_detail(ts, social_sensors, keywords_list, size, 2)
#results = get_origin_weibo_detail(ts, social_sensors, keywords_list, size, 2)
elif int(text_type) == 3: # 积极微博
results = get_positive_weibo_detail(ts, social_sensors, keywords_list, size, "1")
elif int(text_type) == 4: # 中性微博
results = get_positive_weibo_detail(ts, social_sensors, keywords_list, size, "0")
elif int(text_type) == 5: # 消极微博
results = get_positive_weibo_detail(ts, social_sensors, keywords_list, size, "2")
elif int(text_type) == 6 and social_sensors: # 原创敏感微博
results = get_sensitive_weibo_detail(ts, social_sensors, sensitive_words_list, 1, size)
elif int(text_type) == 7 and social_sensors: # 转发敏感微博
results = get_sensitive_weibo_detail(ts, social_sensors, sensitive_words_list, 3, size)
elif int(text_type) == 8 and social_sensors: # 评论敏感微博
results = get_sensitive_weibo_detail(ts, social_sensors, sensitive_words_list, 2, size)
else:
print "input error"
return results
def conclusion_on_activeness(uid):
# test
index_name = copy_portrait_index_name
index_type = copy_portrait_index_type
try:
influ_result = es.get(index=index_name, doc_type=index_type, id=uid)["_source"]
except:
influ_result = {}
result = activeness_conclusion_dict["0"]
return result
# generate time series---keys
now_ts = time.time()
now_ts = datetime2ts("2013-09-12")
activeness_set = set()
for i in range(N):
ts = ts2datetime(now_ts - i * 3600 * 24)
activeness_set.add(pre_activeness + ts)
# 区分影响力和活跃度的keys
keys_set = set(influ_result.keys())
activeness_keys = keys_set & activeness_set
if activeness_keys:
activeness_value = []
for key in activeness_keys:
activeness_value.append(influ_result[key])
mean, std_var = level(activeness_value)
if mean < activeness_level[0]:
result = activeness_conclusion_dict["1"]
elif mean >= activeness_level[0] and mean < activeness_level[1]:
result = activeness_conclusion_dict["2"]
elif mean >= activeness_level[1] and mean < activeness_level[2]:
result = activeness_conclusion_dict["3"]
elif mean >= activeness_level[2] and mean < activeness_level[3]:
result = activeness_conclusion_dict["4"]
else:
result = activeness_conclusion_dict["5"]
else:
result = conclusion_dict["0"]
return result
def change_attribute_portrait(uid, attribute_name, attribute_value,
submit_user):
status = False
submit_user_tag = submit_user + "-tag"
id_attribute = submit_user + "-" + attribute_name
#identify the user exist
#identify the attribute exist
#identify the attribute value exist
#identify the submit_user have been admitted----without
try:
user_result = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
return 'no user'
try:
attribute_result = es_tag.get(index=attribute_index_name,
doc_type=attribute_index_type,
id=id_attribute)['_source']
except:
return 'no attribute'
value_list = attribute_result['attribute_value'].split('&')
if attribute_value not in value_list:
return 'no attribute value'
submit_user_attribute = user_result.get(submit_user_tag,
'') # 个人是否存在该管理员打上的个人标签
if attribute_name not in submit_user_attribute:
return 'personal attribute no exist'
tmp_attribute_list = submit_user_attribute.split("&")
attribute_list = []
for item in tmp_attribute_list:
if attribute_name in item:
attribute_list.append(attribute_name + '-' + attribute_value)
else:
attribute_list.append(item)
user_result[submit_user_tag] = "&".join(attribute_list)
es.index(index=user_index_name,
doc_type=user_index_type,
id=uid,
body=user_result)
status = True
return status
def add_tag2group(uid_list, attribute_name, attribute_value, submit_user):
id_attribute = submit_user + "-" + attribute_name
add_attribute = attribute_name + "-" + attribute_value
submit_user_tag = submit_user + "-tag"
status = False
#identify the attribute exist
#for uid in uid_list
#identify the attribute not in this user
#add tag to this user
try:
attribute_exist = es_tag.get(index=attribute_index_name,
doc_type=attribute_index_type,
id=id_attribute)['_source']
except:
return 'no attribute'
attribute_exist_value_list = attribute_exist['attribute_value'].split('&')
if attribute_value not in attribute_exist_value_list:
return 'no attribute value'
for uid in uid_list:
try:
user_exist = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
user_exist = {}
continue
submit_user_attribute = user_exist.get(submit_user_tag, '')
if user_exist and attribute_name not in submit_user_attribute:
if submit_user_attribute:
user_exist[submit_user_tag] = "&".join(
[submit_user_attribute, add_attribute])
else:
user_exist[submit_user_tag] = add_attribute
es.index(index=user_index_name,
doc_type=user_index_type,
id=uid,
body=user_exist)
status = True
return status
def delete_attribute_portrait(uid, attribute_name, submit_user):
status = False
#identify the user exist
#identify the attribute value exist in es_user_portrait
#identify the submit_user have been admitted---without
try:
user_exist = es.get(index=user_index_name,
doc_type=user_index_type,
id=uid)['_source']
except:
return 'no user'
if attribute_name not in user_exist:
return 'user have no attribtue'
try:
del_attribute_value = user_exist.pop(attribute_name)
es.index(index=user_index_name,
doc_type=user_index_type,
id=uid,
body=user_exist)
status = True
except Exception, e:
raise e
def get_sort(uid, fe):
result = {}
try:
u_bci = es.get(index=BCI_INDEX_NAME,
doc_type=BCI_INDEX_TYPE,
id=uid,
fields=['bci_week_ave'])['fields']['bci_week_ave'][0]
#u_bci = es.get(index='user_portrait_1222', doc_type='user', id=uid,fields=['bci_week_ave'])['fields']['bci_week_ave'][0]
result['in_score'] = u_bci
except:
result['in_score'] = ""
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'range': {
'bci_week_ave': {
'gte': u_bci
}
}
}, {
'term': {
'topic_string': fe
}
}]
}
}
}
}
}
result['in_top'] = es.search(index=BCI_INDEX_NAME,
doc_type=BCI_INDEX_TYPE,
body=query_body)['hits']['total']
print 'essearch'
print es.search(index=BCI_INDEX_NAME,
doc_type=BCI_INDEX_TYPE,
body=query_body)
try:
u_bci = es.get(index='bci_history',
doc_type='bci',
id=uid,
fields=['bci_week_ave'])['fields']['bci_week_ave'][0]
#u_bci = es.get(index='user_portrait_1222', doc_type='user', id=uid,fields=['bci_week_ave'])['fields']['bci_week_ave'][0]
print "trymax"
bci_max = get_max_value(es_user_profile, "bci_history", "bci")
print "max", bci_max
result['all_score'] = math.log(u_bci / float(bci_max) * 9 + 1,
10) * 100
except:
result['all_score'] = ""
result['all_top'] = ""
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'range': {
'bci_week_ave': {
'gte': u_bci
}
}
}]
}
}
}
}
}
result['all_top'] = es.search(index='bci_history',
doc_type='bci',
body=query_body)['hits']['total']
#result = es.search(index='user_portrait_1222', doc_type='user',body=query_body)
# return json.dumps([result['hits']['total'],u_bci])
return json.dumps(result)
def get_more_portrait(task_name, ts, user):
results = dict()
index_name = task_name
_id = user + "-" + task_name
task_detail = es.get(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=_id)["_source"]
task_name = task_detail['task_name']
social_sensors = json.loads(task_detail['social_sensors'])
history_status = json.loads(task_detail['history_status'])
important_user_set = set() # 重要人物列表
out_portrait_users = set() # 未入库
top_importance = get_top_influence("importance")
top_influence = get_top_influence("influence")
top_activeness = get_top_influence("activeness")
ts = int(ts)
time_series = history_status
if time_series:
flow_detail = es.mget(index=index_sensing_task,
doc_type=_id,
body={"ids": time_series})['docs']
else:
flow_detail = {}
if flow_detail:
for item in flow_detail:
item = item['_source']
timestamp = item['timestamp']
temp_important_user_list = json.loads(item['important_users'])
unfiltered_users = json.loads(item['unfilter_users'])
temp_out_portrait_users = set(unfiltered_users) - set(
temp_important_user_list)
important_user_set = important_user_set | set(
temp_important_user_list)
important_uid_list = list(important_user_set)
social_sensor_set = set(social_sensors)
user_detail_info = [] #
if important_uid_list:
important_uid_list = important_uid_list[:1000]
user_results = es.mget(index=portrait_index_name,
doc_type=portrait_index_type,
body={"ids": important_uid_list},
fields=[
'uid', 'uname', 'domain', 'topic_string',
"photo_url", 'importance', 'influence',
'activeness'
])['docs']
for item in user_results:
if item['found']:
temp = []
temp.append(item['fields']['uid'][0])
uname = item['fields']['uname'][0]
if not uname or uname == "未知":
uname = item['fields']['uid'][0]
temp.append(uname)
temp.append(item['fields']['photo_url'][0])
temp.append(item['fields']['domain'][0])
temp.append(item['fields']['topic_string'][0].split('&'))
temp.append(
math.log(
item['fields']['importance'][0] /
float(top_importance) * 9 + 1, 10) * 100)
temp.append(
math.log(
item['fields']['influence'][0] / float(top_influence) *
9 + 1, 10) * 100)
temp.append(
math.log(
item['fields']['activeness'][0] /
float(top_activeness) * 9 + 1, 10) * 100)
if item['fields']['uid'][0] in social_sensor_set:
temp.append(1)
else:
temp.append(0)
user_detail_info.append(temp)
if user_detail_info:
user_detail_info = sorted(user_detail_info,
key=lambda x: x[6],
reverse=True)
else:
user_detail_info = []
return user_detail_info
def get_more_out(task_name, ts, user):
results = dict()
index_name = task_name
_id = user + "-" + task_name
task_detail = es.get(index=index_manage_sensing_task,
doc_type=task_doc_type,
id=_id)["_source"]
task_name = task_detail['task_name']
social_sensors = json.loads(task_detail['social_sensors'])
history_status = json.loads(task_detail['history_status'])
important_user_set = set() # 重要人物列表
out_portrait_users = set() # 未入库
top_importance = get_top_influence("importance")
top_influence = get_top_influence("influence")
top_activeness = get_top_influence("activeness")
ts = int(ts)
time_series = history_status
if time_series:
flow_detail = es.mget(index=index_sensing_task,
doc_type=_id,
body={"ids": time_series})['docs']
else:
flow_detail = {}
if flow_detail:
for item in flow_detail:
item = item['_source']
timestamp = item['timestamp']
temp_important_user_list = json.loads(item['important_users'])
unfiltered_users = json.loads(item['unfilter_users'])
temp_out_portrait_users = set(unfiltered_users) - set(
temp_important_user_list)
important_user_set = important_user_set | set(
temp_important_user_list)
out_portrait_users = out_portrait_users | set(
temp_out_portrait_users)
out_portrait_users_list = list(out_portrait_users)
social_sensor_set = set(social_sensors)
out_user_detail_info = []
if out_portrait_users_list:
out_portrait_users_list = out_portrait_users_list[:1000]
profile_results = es_profile.mget(
index=profile_index_name,
doc_type=profile_index_type,
body={"ids": out_portrait_users_list})["docs"]
bci_index = "bci_" + ts2datetime(ts - DAY).replace('-', '')
influence_results = es.mget(index=bci_index,
doc_type="bci",
body={"ids": out_portrait_users_list},
fields=["user_index"])['docs']
bci_results = es_profile.mget(index="bci_history",
doc_type="bci",
body={"ids": out_portrait_users_list},
fields=['user_fansnum'])['docs']
top_influence = get_top_all_influence("user_index", ts)
count = 0
if profile_results:
for item in profile_results:
temp = []
if item['found']:
temp.append(item['_source']['uid'])
if item['_source']['nick_name']:
temp.append(item['_source']['nick_name'])
else:
temp.append(item['_source']['uid'])
temp.append(item['_source']['user_location'])
else:
temp.append(item['_id'])
temp.append(item['_id'])
temp.extend([''])
try:
user_fansnum = bci_results[count]["fields"][
"user_fansnum"][0]
except:
user_fansnum - 0
temp.append(user_fansnum)
temp_influ = influence_results[count]
if temp_influ.get('found', 0):
user_index = temp_influ['fields']['user_index'][0]
temp.append(
math.log(user_index / float(top_influence) * 9 + 1, 10)
* 100)
else:
temp.append(0)
count += 1
out_user_detail_info.append(temp)
if len(out_user_detail_info):
out_user_detail_info = sorted(out_user_detail_info,
key=lambda x: x[4],
reverse=True)
return out_user_detail_info
def new_get_user_portrait(uid, admin_user):
results = {}
try:
user_portrait_result = es_user_portrait.get(index=portrait_index_name, doc_type=portrait_index_type,\
id=uid)['_source']
except:
user_portrait_result = {}
if not user_portrait_result:
results['tag_remark'] = {}
results['attention_information'] = {}
results['tendency'] = {}
results['group_tag'] = []
else:
#step1: get attention_information
#sensitive words
try:
sensitive_words_dict = json.loads(user_portrait_result['sensitive_words_dict'])
except:
sensitive_words_dict = {}
sort_sensitive_words = sorted(sensitive_words_dict.items(), key=lambda x:x[1], reverse=True)
results['attention_information'] = {'sensitive_dict': sort_sensitive_words}
#keywords
sort_keywords = json.loads(user_portrait_result['keywords'])
results['attention_information']['keywords'] = sort_keywords
#hashtag
hashtag_dict = json.loads(user_portrait_result['hashtag_dict'])
sort_hashtag = sorted(hashtag_dict.items(), key=lambda x:x[1], reverse=True)
results['attention_information']['hashtag'] = sort_hashtag
#step2: get tendency_information
results['tendency'] = {'domain':user_portrait_result['domain']}
results['tendency']['topic'] = user_portrait_result['topic_string'].split('&')[0]
results['tendency']['character_sentiment'] = user_portrait_result['character_sentiment']
results['tendency']['character_text'] = user_portrait_result['character_text']
#step3: get tag_information
#tag
try:
admin_tag = user_portrait_result[admin_user + '-tag']
except:
admin_tag = {}
if not admin_tag:
results['tag_remark'] = {'tag': []}
else:
tag_list = admin_tag.split('&')
results['tag_remark'] = {'tag': tag_list}
#remark
try:
remark = user_portrait_result['remark']
except:
remark = ''
results['tag_remark']['remark'] = remark
#step4: get group_tag information
results['group_tag'] = []
try:
group_tag = user_portrait_result['group']
except:
group_tag = ''
if group_tag:
group_tag_list = group_tag.split('&')
for group_tag in group_tag_list:
group_tag_item_list = group_tag.split('-')
if group_tag_item_list[0] == admin_user:
results['group_tag'].append(group_tag_item_list[1])
return results
