def update_topic(uid_list=[]):
if not uid_list:
uid_list = load_uid_list()
tw_flow_text_index_list = get_twitter_flow_text_index_list(
load_timestamp(), TEST_MAX_FLOW_TEXT_DAYS)
user_topic_data = get_filter_keywords(tw_flow_text_index_list, uid_list)
user_topic_dict, user_topic_list = topic_classfiy(uid_list,
user_topic_data)
user_topic_string = {}
for uid, topic_list in user_topic_list.items():
li = []
for t in topic_list:
li.append(zh_data[name_list.index(t)].decode('utf8'))
user_topic_string[uid] = '&'.join(li)
user_topic = {}
for uid in uid_list:
if uid in user_topic_dict:
user_topic[uid] = {
'filter_keywords': json.dumps(user_topic_data[uid]),
'topic': json.dumps(user_topic_dict[uid]),
'topic_string': user_topic_string[uid]
}
else:
user_topic[uid] = {
'filter_keywords': json.dumps({}),
'topic': json.dumps({}),
'topic_string': ''
}
return save_data2es(user_topic)
def my_topic_classfiy(uid_list, datetime_list):
topic_dict_results = {}
topic_string_results = {}
#将处理后的结果保存到数据库中，并在处理前查询数据库中是否已经有了相应内容之前存储的结果，以提高效率
uids = uid_list
unresolved_uids = []
res = es.mget(index=tw_portrait_index_name, doc_type=tw_portrait_index_type, body={'ids': uids})['docs']
for r in res:
uid = r['_id']
if r.has_key('found'):
found = r['found']
if found and r['_source'].has_key('topic'):
topic = r['_source']['topic']
topic_string = r['_source']['topic_string']
topic_dict_results[uid] = json.loads(topic)
topic_string_results[uid] = [topic_ch2en_dict[ch_topic] for ch_topic in topic_string.split('&')]
else:
unresolved_uids.append(uid)
else: #es表中目前无任何记录
unresolved_uids.append(uid)
#未在数据库中的进行计算并存储
user_topic_dict = {}
user_topic_list = {}
if unresolved_uids:
tw_flow_text_index_list = []
for datetime in datetime_list:
tw_flow_text_index_list.append(flow_text_index_name_pre + datetime)
user_topic_data = get_filter_keywords(tw_flow_text_index_list, unresolved_uids)
user_topic_dict, user_topic_list = topic_classfiy(unresolved_uids, user_topic_data)
user_topic_string = {}
for uid, topic_list in user_topic_list.items():
li = []
for t in topic_list:
li.append(zh_data[name_list.index(t)].decode('utf8'))
user_topic_string[uid] = '&'.join(li)
user_topic = {}
for uid in unresolved_uids:
if uid in user_topic_dict:
user_topic[uid] = {
'filter_keywords': json.dumps(user_topic_data[uid]),
'topic': json.dumps(user_topic_dict[uid]),
'topic_string': user_topic_string[uid]
}
else:
user_topic[uid] = {
'filter_keywords': json.dumps({}),
'topic': json.dumps({}),
'topic_string': ''
}
save_data2es(user_topic)
#整合
user_topic_dict.update(topic_dict_results)
user_topic_list.update(topic_string_results)
return user_topic_dict, user_topic_list
