def search_user_info(es,index_name,doc_type,uid,result_name):
try:
retweet_result = es.get(index=index_name, doc_type=doc_type, id=uid)['_source']
except:
return None
if retweet_result:
retweet_dict = json.loads(retweet_result[result_name])
sorted_list = sorted(retweet_dict.iteritems(),key=lambda x:x[1],reverse=True)[:20]
uid_list = [i[0] for i in sorted_list if i[0] != uid]
portrait_result = []
try:
user_result = es_user_profile.mget(index=profile_index_name, doc_type=profile_index_type, body={'ids':uid_list})['docs']
except:
user_result = []
try:
bci_history_result = es_bci_history.mget(index=bci_history_index_name, doc_type=bci_history_index_type, body={'ids':uid_list}, fields=fields)['docs']
except:
bci_history_result = []
#print bci_history_result
iter_count = 0
out_portrait_list = []
for out_user_item in user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
#location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = 'unknown'
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
if bci_history_item['found']==True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
influence = ''
#retweet_count = int(retweet_dict[uid])
count = retweet_dict[uid]
out_portrait_list.append({'uid':uid,'photo_url':photo_url,'count':count,'uname':uname,'influence':influence,'fansnum':fansnum, 'friendsnum':user_friendsnum,'weibo_count':user_weibo_count})#location,
iter_count += 1
return out_portrait_list
else:
return None
def search_fans(uid,top_count):
results = {}
now_ts = time.time()
now_date_ts = datetime2ts(ts2datetime(now_ts))
db_number = get_db_num(now_date_ts)
be_comment_index_name = be_comment_index_name_pre + str(db_number)
be_retweet_index_name = be_retweet_index_name_pre + str(db_number)
result = {}
be_retweet_inter_dict = {}
be_comment_inter_dict = {}
center_uid = uid
try:
be_retweet_result = es_retweet.get(index = be_retweet_index_name,doc_type=be_retweet_index_type,id=uid)['_source']
except:
be_retweet_result = {}
if be_retweet_result:
be_retweet_uid_dict = json.loads(be_retweet_result['uid_be_retweet'])
else:
be_retweet_uid_dict = {}
# print "be_retweet_uid_dict", be_retweet_uid_dict
try:
be_comment_result = es_be_comment.get(index=be_comment_index_name, doc_type=be_comment_index_type, id=uid)['_source']
except:
be_comment_result = {}
if be_comment_result:
be_comment_uid_dict = json.loads(be_comment_result['uid_be_comment'])
else:
be_comment_uid_dict = {}
# print "be_comment_uid_dict", be_comment_uid_dict
fans_result = union_dict(be_retweet_uid_dict,be_comment_uid_dict)
fans_user_set = set(fans_result.keys())
fans_list = list(fans_user_set)
# print "fans_list", fans_list
all_fans_dict = {}
for fans_user in fans_list:
if fans_user != center_uid:
all_fans_dict[fans_user] = fans_result[fans_user]
sort_all_fans_dict = sorted(all_fans_dict.items(), key=lambda x:x[1], reverse=True)
all_fans_uid_list=[]
all_fans_uid_list_all = [item[0] for item in sort_all_fans_dict]
count = 0
for i in all_fans_uid_list_all:
count += 1
all_fans_uid_list.append(i)
if count == 1000:
break
# print all_fans_uid_list
out_portrait_list = all_fans_uid_list
#use to get user information from user profile
out_portrait_result = {}
try:
out_user_result = es_user_profile.mget(index=profile_index_name, doc_type=profile_index_type, body={'ids':out_portrait_list})['docs']
except:
out_user_result = []
#add index from bci_history
try:
bci_history_result = es_bci_history.mget(index=bci_history_index_name, doc_type=bci_history_index_type, body={'ids': out_portrait_list}, fields=fields)['docs']
except:
bci_history_result = []
iter_count = 0
out_portrait_list = []
for out_user_item in out_user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = 'unknown'
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
# print bci_history_item
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
fans_count = int(all_fans_dict[uid])
out_portrait_list.append({'uid':uid,'photo_url':photo_url,'uname':uname, 'count':fans_count, 'fansnum':fansnum,'friendsnum': user_friendsnum,'weibo_count': user_weibo_count})
iter_count += 1
return out_portrait_list
def search_follower(uid, top_count):
results = {}
now_ts = time.time()
db_number = get_db_num(now_ts)
index_name = be_retweet_index_name_pre + str(db_number)
# return search_user_info(es_retweet,index_name,retweet_index_type,uid,'uid_be_retweet')
center_uid = uid
try:
retweet_result = es_retweet.get(index=index_name, doc_type=be_retweet_index_type, id=uid)['_source']
except:
return None
retweet_dict={}
if retweet_result:
retweet_dict_old = json.loads(retweet_result['uid_be_retweet'])
for key in retweet_dict_old:
retweet_dict[key]=int(retweet_dict_old[key])
sorted_list = sorted(retweet_dict.iteritems(),key=lambda x:x[1],reverse=True)[:20]
uid_list = [i[0] for i in sorted_list if i[0] != uid]
portrait_result = []
try:
user_result = es_user_profile.mget(index=profile_index_name, doc_type=profile_index_type, body={'ids':uid_list})['docs']
except:
user_result = []
try:
bci_history_result = es_bci_history.mget(index=bci_history_index_name, doc_type=bci_history_index_type, body={'ids':uid_list}, fields=fields)['docs']
except:
bci_history_result = []
# print bci_history_result
iter_count = 0
out_portrait_list = []
for out_user_item in user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
#location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = ''
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
if bci_history_item['found']==True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
influence = ''
#retweet_count = int(retweet_dict[uid])
print uid
count = retweet_dict[uid]
print count
out_portrait_list.append({'uid':uid,'photo_url':photo_url,'count':count,'uname':uname,'influence':influence,'fansnum':fansnum, 'friendsnum':user_friendsnum,'weibo_count':user_weibo_count})#location,
iter_count += 1
return out_portrait_list
else:
return None
def search_mention(now_ts, uid, top_count):
date = ts2datetime(now_ts)
#evaluate_max_dict = get_evaluate_max()
ts = datetime2ts(date)
stat_results = dict()
results = dict()
uid_dict = {}
for i in range(1,8):
ts = ts - DAY
try:
result_string = r_cluster.hget('at_' + str(ts), str(uid))
except:
result_string = ''
if not result_string:
continue
result_dict = json.loads(result_string)
for at_uname in result_dict:
try:
stat_results[at_uname] += result_dict[at_uname]
except:
stat_results[at_uname] = result_dict[at_uname]
sort_stat_results = sorted(stat_results.items(), key=lambda x:x[1], reverse=True)
# print sort_stat_results
out_portrait_list = []
out_list = stat_results.keys()
#use to get user information from user profile
out_query_list = [{'match':{'uname':item}} for item in out_list]
if len(out_query_list) != 0:
query = [{'bool':{'should': out_query_list}}]
try:
out_profile_result = es_user_profile.search(index=profile_index_name, doc_type=profile_index_type, body={'query':{'bool':{'must':query}}, 'size':100})['hits']['hits']
except:
out_profile_result = []
else:
out_profile_result = []
out_in_profile_list = []
bci_search_id_list = []
for out_item in out_profile_result:
source = out_item['_source']
uname = source['nick_name']
uid = source['uid']
location = source['location']
friendsnum = source['friendsnum']
out_portrait_list.append([uid, uname, stat_results[uname], '', location, friendsnum, ''])
out_in_profile_list.append(uname)
#use to search bci history
bci_search_id_list.append(uid)
out_out_profile_list = list(set(out_list) - set(out_in_profile_list))
for out_out_item in out_out_profile_list:
out_portrait_list.append(['', out_out_item, stat_results[out_out_item],'', '', '', ''])
#add index from bci_history
new_out_portrait_list = []
try:
bci_history_result = es_bci_history.mget(index=bci_history_index_name, doc_type=bci_history_index_type, body={'ids': bci_search_id_list}, fields=['user_fansnum', 'weibo_month_sum', 'user_friendsnum'])['docs']
except:
bci_history_result = []
iter_count = 0
for out_portrait_item in out_portrait_list:
append_dict = {}
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {}
new_out_portrait_item = out_portrait_item
append_dict['uid'] = out_portrait_item[0]
append_dict['uname'] = out_portrait_item[1]
append_dict['count'] = out_portrait_item[2]
if bci_history_item:
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields']['user_fansnum'][0]
user_weibo_count = bci_history_item['fields']['weibo_month_sum'][0]
user_friendsnum = bci_history_item['fields']['user_friendsnum'][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
append_dict['fansnum'] = fansnum
append_dict['weibo_count'] = user_weibo_count
append_dict['friendsnum'] = user_friendsnum
# new_out_portrait_item[3] = fansnum
# new_out_portrait_item[6] = user_weibo_count
# new_out_portrait_item[-2] = user_friendsnum
#new_out_portrait_list.append(new_out_portrait_item)
new_out_portrait_list.append(append_dict)
iter_count += 1
#print append_dict
return new_out_portrait_list # uid，名字，提及次数,粉丝数，注册地，关注数，微博数
def search_bidirect_interaction(uid, top_count):
now_ts = time.time()
now_date_ts = datetime2ts(ts2datetime(now_ts))
db_number = get_db_num(now_date_ts)
retweet_index_name = retweet_index_name_pre + str(db_number)
be_retweet_index_name = be_retweet_index_name_pre + str(db_number)
comment_index_name = comment_index_name_pre + str(db_number)
be_comment_index_name = be_comment_index_name_pre + str(db_number)
results = {}
retweet_inter_dict = {}
comment_inter_dict = {}
center_uid = uid
#bidirect interaction in retweet and be_retweet
try:
retweet_result = es_retweet.get(index=retweet_index_name, doc_type=retweet_index_type, id=uid)['_source']
except:
retweet_result = {}
if retweet_result:
retweet_uid_dict = json.loads(retweet_result['uid_retweet'])
else:
retweet_uid_dict = {}
retweet_uid_list = retweet_uid_dict.keys()
try:
be_retweet_result = es_retweet.get(index=be_retweet_index_name, doc_type=be_retweet_index_type, id=uid)['_source']
except:
be_retweet_result = {}
if be_retweet_result:
be_retweet_uid_dict = json.loads(be_retweet_result['uid_be_retweet'])
else:
be_retweet_uid_dict = {}
#bidirect interaction in comment and be_comment
try:
comment_result = es_comment.get(index=comment_index_name, doc_type=comment_index_type, id=uid)['_source']
except:
comment_result = {}
if comment_result:
comment_uid_dict = json.loads(comment_result['uid_comment'])
else:
comment_uid_dict = {}
comment_uid_list = comment_uid_dict.keys()
try:
be_comment_result = es_comment.get(index=be_coment_index_name, doc_type=be_comment_index_type, id=uid)['_source']
except:
be_comment_result = {}
if be_comment_result:
be_comment_uid_dict = json.loads(be_comment_result['uid_be_comment'])
else:
be_comment_uid_dict = {}
#get bidirect_interaction dict
#all_interaction_dict = union_dict(retweet_inter_dict, comment_inter_dict)
retweet_comment_result = union_dict(retweet_uid_dict, comment_uid_dict)
be_retweet_comment_result = union_dict(be_retweet_uid_dict, be_comment_uid_dict)
interaction_user_set = set(retweet_comment_result.keys()) & set(be_retweet_comment_result.keys())
interaction_user_list = list(interaction_user_set)
all_interaction_dict = {}
for interaction_user in interaction_user_list:
if interaction_user != center_uid:
all_interaction_dict[interaction_user] = retweet_comment_result[interaction_user] + be_retweet_comment_result[interaction_user]
sort_all_interaction_dict = sorted(all_interaction_dict.items(), key=lambda x:x[1], reverse=True)
#get in_portrait_list, in_portrait_results and out_portrait_list
all_interaction_uid_list = [item[0] for item in sort_all_interaction_dict]
#print all_interaction_uid_list
# if RUN_TYPE == 0:
# all_interaction_dict = {'2029036025':3,'1282005885':2,'2549228714':2,'1809833450':1}
# all_interaction_uid_list = ['2029036025', '1282005885', '2549228714', '1809833450']
out_portrait_list = all_interaction_uid_list
#use to get user information from user profile
out_portrait_result = {}
try:
out_user_result = es_user_profile.mget(index=profile_index_name, doc_type=profile_index_type, body={'ids':out_portrait_list})['docs']
except:
out_user_result = []
#add index from bci_history
try:
bci_history_result = es_bci_history.mget(index=bci_history_index_name, doc_type=bci_history_index_type, body={'ids': out_portrait_list}, fields=fields)['docs']
except:
bci_history_result = []
iter_count = 0
out_portrait_list = []
for out_user_item in out_user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = 'unknown'
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
# print bci_history_item
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
interaction_count = int(all_interaction_dict[uid])
out_portrait_list.append({'uid':uid,'photo_url':photo_url,'uname':uname, 'count':interaction_count, 'fansnum':fansnum,'friendsnum': user_friendsnum,'weibo_count': user_weibo_count})
iter_count += 1
return out_portrait_list
def search_fans(uid, top_count):
results = {}
now_ts = time.time()
now_date_ts = datetime2ts(ts2datetime(now_ts))
db_number = get_db_num(now_date_ts)
be_comment_index_name = be_comment_index_name_pre + str(db_number)
be_retweet_index_name = be_retweet_index_name_pre + str(db_number)
result = {}
be_retweet_inter_dict = {}
be_comment_inter_dict = {}
center_uid = uid
try:
be_retweet_result = es_retweet.get(index=be_retweet_index_name,
doc_type=be_retweet_index_type,
id=uid)['_source']
except:
be_retweet_result = {}
if be_retweet_result:
be_retweet_uid_dict = json.loads(be_retweet_result['uid_be_retweet'])
else:
be_retweet_uid_dict = {}
# # print "be_retweet_uid_dict", be_retweet_uid_dict
try:
be_comment_result = es_be_comment.get(index=be_comment_index_name,
doc_type=be_comment_index_type,
id=uid)['_source']
except:
be_comment_result = {}
if be_comment_result:
be_comment_uid_dict = json.loads(be_comment_result['uid_be_comment'])
else:
be_comment_uid_dict = {}
# # print "be_comment_uid_dict", be_comment_uid_dict
fans_result = union_dict(be_retweet_uid_dict, be_comment_uid_dict)
fans_user_set = set(fans_result.keys())
fans_list = list(fans_user_set)
# # print "fans_list", fans_list
all_fans_dict = {}
for fans_user in fans_list:
if fans_user != center_uid:
all_fans_dict[fans_user] = fans_result[fans_user]
sort_all_fans_dict = sorted(all_fans_dict.items(),
key=lambda x: x[1],
reverse=True)
all_fans_uid_list = []
all_fans_uid_list_all = [item[0] for item in sort_all_fans_dict]
count = 0
for i in all_fans_uid_list_all:
count += 1
all_fans_uid_list.append(i)
if count == 1000:
break
# # print all_fans_uid_list
out_portrait_list = all_fans_uid_list
#use to get user information from user profile
out_portrait_result = {}
try:
out_user_result = es_user_profile.mget(index=profile_index_name,
doc_type=profile_index_type,
body={'ids': out_portrait_list
})['docs']
except:
out_user_result = []
#add index from bci_history
try:
bci_history_result = es_bci_history.mget(
index=bci_history_index_name,
doc_type=bci_history_index_type,
body={'ids': out_portrait_list},
fields=fields)['docs']
except:
bci_history_result = []
iter_count = 0
out_portrait_list = []
for out_user_item in out_user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = 'unknown'
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
# # print bci_history_item
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
fans_count = int(all_fans_dict[uid])
out_portrait_list.append({
'uid': uid,
'photo_url': photo_url,
'uname': uname,
'count': fans_count,
'fansnum': fansnum,
'friendsnum': user_friendsnum,
'weibo_count': user_weibo_count
})
iter_count += 1
return out_portrait_list
def search_user_info(es, index_name, doc_type, uid, result_name):
try:
retweet_result = es.get(index=index_name, doc_type=doc_type,
id=uid)['_source']
except:
return None
if retweet_result:
retweet_dict = json.loads(retweet_result[result_name])
sorted_list = sorted(retweet_dict.iteritems(),
key=lambda x: x[1],
reverse=True)[:20]
uid_list = [i[0] for i in sorted_list if i[0] != uid]
portrait_result = []
try:
user_result = es_user_profile.mget(index=profile_index_name,
doc_type=profile_index_type,
body={'ids': uid_list})['docs']
except:
user_result = []
try:
bci_history_result = es_bci_history.mget(
index=bci_history_index_name,
doc_type=bci_history_index_type,
body={'ids': uid_list},
fields=fields)['docs']
except:
bci_history_result = []
## print bci_history_result
iter_count = 0
out_portrait_list = []
for out_user_item in user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
#location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = 'unknown'
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
influence = ''
#retweet_count = int(retweet_dict[uid])
count = retweet_dict[uid]
out_portrait_list.append({
'uid': uid,
'photo_url': photo_url,
'count': count,
'uname': uname,
'influence': influence,
'fansnum': fansnum,
'friendsnum': user_friendsnum,
'weibo_count': user_weibo_count
}) #location,
iter_count += 1
return out_portrait_list
else:
return None
def search_follower(uid, top_count):
results = {}
now_ts = time.time()
db_number = get_db_num(now_ts)
index_name = be_retweet_index_name_pre + str(db_number)
# return search_user_info(es_retweet,index_name,retweet_index_type,uid,'uid_be_retweet')
center_uid = uid
try:
retweet_result = es_retweet.get(index=index_name,
doc_type=be_retweet_index_type,
id=uid)['_source']
except:
return None
retweet_dict = {}
if retweet_result:
retweet_dict_old = json.loads(retweet_result['uid_be_retweet'])
for key in retweet_dict_old:
retweet_dict[key] = int(retweet_dict_old[key])
sorted_list = sorted(retweet_dict.iteritems(),
key=lambda x: x[1],
reverse=True)[:20]
uid_list = [i[0] for i in sorted_list if i[0] != uid]
portrait_result = []
try:
user_result = es_user_profile.mget(index=profile_index_name,
doc_type=profile_index_type,
body={'ids': uid_list})['docs']
except:
user_result = []
try:
bci_history_result = es_bci_history.mget(
index=bci_history_index_name,
doc_type=bci_history_index_type,
body={'ids': uid_list},
fields=fields)['docs']
except:
bci_history_result = []
# # print bci_history_result
iter_count = 0
out_portrait_list = []
for out_user_item in user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
#location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = ''
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
influence = ''
#retweet_count = int(retweet_dict[uid])
# print uid
count = retweet_dict[uid]
# print count
out_portrait_list.append({
'uid': uid,
'photo_url': photo_url,
'count': count,
'uname': uname,
'influence': influence,
'fansnum': fansnum,
'friendsnum': user_friendsnum,
'weibo_count': user_weibo_count
}) #location,
iter_count += 1
return out_portrait_list
else:
return None
def search_bidirect_interaction(uid, top_count):
now_ts = time.time()
now_date_ts = datetime2ts(ts2datetime(now_ts))
db_number = get_db_num(now_date_ts)
retweet_index_name = retweet_index_name_pre + str(db_number)
be_retweet_index_name = be_retweet_index_name_pre + str(db_number)
comment_index_name = comment_index_name_pre + str(db_number)
be_comment_index_name = be_comment_index_name_pre + str(db_number)
results = {}
retweet_inter_dict = {}
comment_inter_dict = {}
center_uid = uid
#bidirect interaction in retweet and be_retweet
try:
retweet_result = es_retweet.get(index=retweet_index_name,
doc_type=retweet_index_type,
id=uid)['_source']
except:
retweet_result = {}
if retweet_result:
retweet_uid_dict = json.loads(retweet_result['uid_retweet'])
else:
retweet_uid_dict = {}
retweet_uid_list = retweet_uid_dict.keys()
try:
be_retweet_result = es_retweet.get(index=be_retweet_index_name,
doc_type=be_retweet_index_type,
id=uid)['_source']
except:
be_retweet_result = {}
if be_retweet_result:
be_retweet_uid_dict = json.loads(be_retweet_result['uid_be_retweet'])
else:
be_retweet_uid_dict = {}
#bidirect interaction in comment and be_comment
try:
comment_result = es_comment.get(index=comment_index_name,
doc_type=comment_index_type,
id=uid)['_source']
except:
comment_result = {}
if comment_result:
comment_uid_dict = json.loads(comment_result['uid_comment'])
else:
comment_uid_dict = {}
comment_uid_list = comment_uid_dict.keys()
try:
be_comment_result = es_comment.get(index=be_coment_index_name,
doc_type=be_comment_index_type,
id=uid)['_source']
except:
be_comment_result = {}
if be_comment_result:
be_comment_uid_dict = json.loads(be_comment_result['uid_be_comment'])
else:
be_comment_uid_dict = {}
#get bidirect_interaction dict
#all_interaction_dict = union_dict(retweet_inter_dict, comment_inter_dict)
retweet_comment_result = union_dict(retweet_uid_dict, comment_uid_dict)
be_retweet_comment_result = union_dict(be_retweet_uid_dict,
be_comment_uid_dict)
interaction_user_set = set(retweet_comment_result.keys()) & set(
be_retweet_comment_result.keys())
interaction_user_list = list(interaction_user_set)
all_interaction_dict = {}
for interaction_user in interaction_user_list:
if interaction_user != center_uid:
all_interaction_dict[interaction_user] = retweet_comment_result[
interaction_user] + be_retweet_comment_result[interaction_user]
sort_all_interaction_dict = sorted(all_interaction_dict.items(),
key=lambda x: x[1],
reverse=True)
#get in_portrait_list, in_portrait_results and out_portrait_list
all_interaction_uid_list = [item[0] for item in sort_all_interaction_dict]
## print all_interaction_uid_list
# if RUN_TYPE == 0:
# all_interaction_dict = {'2029036025':3,'1282005885':2,'2549228714':2,'1809833450':1}
# all_interaction_uid_list = ['2029036025', '1282005885', '2549228714', '1809833450']
out_portrait_list = all_interaction_uid_list
#use to get user information from user profile
out_portrait_result = {}
try:
out_user_result = es_user_profile.mget(index=profile_index_name,
doc_type=profile_index_type,
body={'ids': out_portrait_list
})['docs']
except:
out_user_result = []
#add index from bci_history
try:
bci_history_result = es_bci_history.mget(
index=bci_history_index_name,
doc_type=bci_history_index_type,
body={'ids': out_portrait_list},
fields=fields)['docs']
except:
bci_history_result = []
iter_count = 0
out_portrait_list = []
for out_user_item in out_user_result:
uid = out_user_item['_id']
if out_user_item['found'] == True:
source = out_user_item['_source']
uname = source['nick_name']
photo_url = source['photo_url']
if uname == '':
uname = u'未知'
location = source['user_location']
friendsnum = source['friendsnum']
else:
uname = u'未知'
location = ''
friendsnum = ''
photo_url = 'unknown'
#add index from bci_history
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {'found': False}
# # print bci_history_item
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields'][fields[0]][0]
user_weibo_count = bci_history_item['fields'][fields[1]][0]
user_friendsnum = bci_history_item['fields'][fields[2]][0]
influence = bci_history_item['fields'][fields[3]][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
interaction_count = int(all_interaction_dict[uid])
out_portrait_list.append({
'uid': uid,
'photo_url': photo_url,
'uname': uname,
'count': interaction_count,
'fansnum': fansnum,
'friendsnum': user_friendsnum,
'weibo_count': user_weibo_count
})
iter_count += 1
return out_portrait_list
def search_mention(now_ts, uid, top_count):
date = ts2datetime(now_ts)
#evaluate_max_dict = get_evaluate_max()
ts = datetime2ts(date)
stat_results = dict()
results = dict()
uid_dict = {}
for i in range(1, 8):
ts = ts - DAY
try:
result_string = r_cluster.hget('at_' + str(ts), str(uid))
except:
result_string = ''
if not result_string:
continue
result_dict = json.loads(result_string)
for at_uname in result_dict:
try:
stat_results[at_uname] += result_dict[at_uname]
except:
stat_results[at_uname] = result_dict[at_uname]
sort_stat_results = sorted(stat_results.items(),
key=lambda x: x[1],
reverse=True)
# # print sort_stat_results
out_portrait_list = []
out_list = stat_results.keys()
#use to get user information from user profile
out_query_list = [{'match': {'uname': item}} for item in out_list]
if len(out_query_list) != 0:
query = [{'bool': {'should': out_query_list}}]
try:
out_profile_result = es_user_profile.search(
index=profile_index_name,
doc_type=profile_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'size': 100
})['hits']['hits']
except:
out_profile_result = []
else:
out_profile_result = []
out_in_profile_list = []
bci_search_id_list = []
for out_item in out_profile_result:
source = out_item['_source']
uname = source['nick_name']
uid = source['uid']
location = source['location']
friendsnum = source['friendsnum']
out_portrait_list.append(
[uid, uname, stat_results[uname], '', location, friendsnum, ''])
out_in_profile_list.append(uname)
#use to search bci history
bci_search_id_list.append(uid)
out_out_profile_list = list(set(out_list) - set(out_in_profile_list))
for out_out_item in out_out_profile_list:
out_portrait_list.append(
['', out_out_item, stat_results[out_out_item], '', '', '', ''])
#add index from bci_history
new_out_portrait_list = []
try:
bci_history_result = es_bci_history.mget(
index=bci_history_index_name,
doc_type=bci_history_index_type,
body={'ids': bci_search_id_list},
fields=['user_fansnum', 'weibo_month_sum',
'user_friendsnum'])['docs']
except:
bci_history_result = []
iter_count = 0
for out_portrait_item in out_portrait_list:
append_dict = {}
try:
bci_history_item = bci_history_result[iter_count]
except:
bci_history_item = {}
new_out_portrait_item = out_portrait_item
append_dict['uid'] = out_portrait_item[0]
append_dict['uname'] = out_portrait_item[1]
append_dict['count'] = out_portrait_item[2]
if bci_history_item:
if bci_history_item['found'] == True:
fansnum = bci_history_item['fields']['user_fansnum'][0]
user_weibo_count = bci_history_item['fields'][
'weibo_month_sum'][0]
user_friendsnum = bci_history_item['fields'][
'user_friendsnum'][0]
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
else:
fansnum = ''
user_weibo_count = ''
user_friendsnum = ''
append_dict['fansnum'] = fansnum
append_dict['weibo_count'] = user_weibo_count
append_dict['friendsnum'] = user_friendsnum
# new_out_portrait_item[3] = fansnum
# new_out_portrait_item[6] = user_weibo_count
# new_out_portrait_item[-2] = user_friendsnum
#new_out_portrait_list.append(new_out_portrait_item)
new_out_portrait_list.append(append_dict)
iter_count += 1
## print append_dict
return new_out_portrait_list # uid，名字，提及次数,粉丝数，注册地，关注数，微博数