def update_topic(uid_list=[]): if not uid_list: uid_list = load_uid_list() tw_flow_text_index_list = get_twitter_flow_text_index_list( load_timestamp(), TEST_MAX_FLOW_TEXT_DAYS) user_topic_data = get_filter_keywords(tw_flow_text_index_list, uid_list) user_topic_dict, user_topic_list = topic_classfiy(uid_list, user_topic_data) user_topic_string = {} for uid, topic_list in user_topic_list.items(): li = [] for t in topic_list: li.append(zh_data[name_list.index(t)].decode('utf8')) user_topic_string[uid] = '&'.join(li) user_topic = {} for uid in uid_list: if uid in user_topic_dict: user_topic[uid] = { 'filter_keywords': json.dumps(user_topic_data[uid]), 'topic': json.dumps(user_topic_dict[uid]), 'topic_string': user_topic_string[uid] } else: user_topic[uid] = { 'filter_keywords': json.dumps({}), 'topic': json.dumps({}), 'topic_string': '' } return save_data2es(user_topic)
def my_topic_classfiy(uid_list, datetime_list): topic_dict_results = {} topic_string_results = {} #将处理后的结果保存到数据库中,并在处理前查询数据库中是否已经有了相应内容之前存储的结果,以提高效率 uids = uid_list unresolved_uids = [] res = es.mget(index=tw_portrait_index_name, doc_type=tw_portrait_index_type, body={'ids': uids})['docs'] for r in res: uid = r['_id'] if r.has_key('found'): found = r['found'] if found and r['_source'].has_key('topic'): topic = r['_source']['topic'] topic_string = r['_source']['topic_string'] topic_dict_results[uid] = json.loads(topic) topic_string_results[uid] = [topic_ch2en_dict[ch_topic] for ch_topic in topic_string.split('&')] else: unresolved_uids.append(uid) else: #es表中目前无任何记录 unresolved_uids.append(uid) #未在数据库中的进行计算并存储 user_topic_dict = {} user_topic_list = {} if unresolved_uids: tw_flow_text_index_list = [] for datetime in datetime_list: tw_flow_text_index_list.append(flow_text_index_name_pre + datetime) user_topic_data = get_filter_keywords(tw_flow_text_index_list, unresolved_uids) user_topic_dict, user_topic_list = topic_classfiy(unresolved_uids, user_topic_data) user_topic_string = {} for uid, topic_list in user_topic_list.items(): li = [] for t in topic_list: li.append(zh_data[name_list.index(t)].decode('utf8')) user_topic_string[uid] = '&'.join(li) user_topic = {} for uid in unresolved_uids: if uid in user_topic_dict: user_topic[uid] = { 'filter_keywords': json.dumps(user_topic_data[uid]), 'topic': json.dumps(user_topic_dict[uid]), 'topic_string': user_topic_string[uid] } else: user_topic[uid] = { 'filter_keywords': json.dumps({}), 'topic': json.dumps({}), 'topic_string': '' } save_data2es(user_topic) #整合 user_topic_dict.update(topic_dict_results) user_topic_list.update(topic_string_results) return user_topic_dict, user_topic_list