def test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts): status = False print 'start cron_text_attribute' uid_list = user_keywords_dict.keys() #get user flow information: hashtag, activity_geo, keywords print 'get flow result' flow_result = get_flow_information(uid_list) print 'flow result len:', len(flow_result) #get user profile information print 'get register result' register_result = get_profile_information(uid_list) print 'register result len:', len(register_result) #get user topic and domain by bulk action print 'get topic and domain, politics' topic_results_dict, topic_results_label = topic_classfiy( uid_list, user_keywords_dict) domain_results = domain_classfiy(uid_list, user_keywords_dict) politics_results = political_classify(uid_list, user_keywords_dict) print "topic: ", topic_results_dict, topic_results_label print "domain: ", domain_results print "politics:", politics_results #get user character attribute #print 'get character result' #type_mark = 0/1 for identify the task input status---just sentiment or text #character_start_time = ts2datetime(character_start_ts) #character_end_time = ts2datetime(character_start_ts + DAY * CHARACTER_TIME_GAP - DAY) #print 'character_start_time:', character_start_time #print 'character_end_time:', character_end_time #character_sentiment_result_dict = classify_sentiment(uid_list, user_weibo_dict, character_start_time, character_end_time, WEIBO_API_INPUT_TYPE) #character_text_result_dict = classify_topic(uid_list, user_keywords_dict) #print 'character result len:', len(character_sentiment_result_dict), len(character_text_result_dict) #get user fansnum max fansnum_max = get_fansnum_max() #get user activeness by bulk_action print 'get activeness results' activeness_results = get_activity_time(uid_list) print 'activeness result len:', len(activeness_results) #get user inlfuence by bulk action print 'get influence' influence_results = get_influence(uid_list) print 'influence results len:', len(influence_results) # compute text attribute bulk_action = [] count = 0 for user in uid_list: count += 1 results = {} #get user text attribute: online_pattern results['online_pattern'] = json.dumps(online_pattern_dict[user]) try: results['online_pattern_aggs'] = '&'.join( online_pattern_dict[user].keys()) except: results['online_pattern_aggs'] = '' results['uid'] = str(user) #add user flow information: hashtag, activity_geo, keywords flow_dict = flow_result[str(user)] results = dict(results, **flow_dict) #add user topic attribute user_topic_dict = topic_results_dict[user] user_label_dict = topic_results_label[user] results['topic'] = json.dumps( user_topic_dict) # {'topic1_en':pro1, 'topic2_en':pro2...} results['topic_string'] = topic_en2ch( user_label_dict) # 'topic1_ch&topic2_ch&topic3_ch' #add user domain attribute user_domain_dict = domain_results[user] domain_list = domain_en2ch(user_domain_dict) if domain_list: results['domain_list'] = json.dumps( domain_list) # [label1_en, label2_en, label3_en] results['domain'] = domain_list[0] # label_ch else: results['domain'] = "其他" results['domain_list'] = json.dumps(["其他"]) politics_label = politics_results[user] results['politics'] = politics_en2ch(politics_label) #add user character_sentiment attribute #character_sentiment = character_sentiment_result_dict[user] #results['character_sentiment'] = character_sentiment #add user character_text attribtue #character_text = character_text_result_dict[user] #results['character_text'] = character_text #add user profile attribute register_dict = register_result[str(user)] results = dict(results, **register_dict) #add user_evaluate attribute---importance results['importance'] = get_importance(results['domain'], results['topic_string'], results['fansnum'], fansnum_max) #add user_evaluate attribute---activeness user_activeness_time = activeness_results[user] user_activeness_geo = json.loads(results['activity_geo_dict'])[-1] results['activeness'] = get_activeness(user_activeness_geo, user_activeness_time) #add user_evaluate attribute---influence results['influence'] = influence_results[user] #bulk_action action = {'index': {'_id': str(user)}} bulk_action.extend([action, results]) status = save_user_results(bulk_action) print "status:", status return status
def test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts): status = False print 'start cron_text_attribute' uid_list = user_keywords_dict.keys() #get user flow information: hashtag, activity_geo, keywords print 'get flow result' flow_result = get_flow_information(uid_list) print 'flow result len:', len(flow_result) #get user profile information print 'get register result' register_result = get_profile_information(uid_list) print 'register result len:', len(register_result) #get user topic and domain by bulk action print 'get topic and domain, politics' topic_results_dict, topic_results_label = topic_classfiy(uid_list, user_keywords_dict) domain_results = domain_classfiy(uid_list, user_keywords_dict) politics_results = political_classify(uid_list, user_keywords_dict) print "topic: ", topic_results_dict, topic_results_label print "domain: ", domain_results print "politics:", politics_results #get user character attribute #print 'get character result' #type_mark = 0/1 for identify the task input status---just sentiment or text #character_start_time = ts2datetime(character_start_ts) #character_end_time = ts2datetime(character_start_ts + DAY * CHARACTER_TIME_GAP - DAY) #print 'character_start_time:', character_start_time #print 'character_end_time:', character_end_time #character_sentiment_result_dict = classify_sentiment(uid_list, user_weibo_dict, character_start_time, character_end_time, WEIBO_API_INPUT_TYPE) #character_text_result_dict = classify_topic(uid_list, user_keywords_dict) #print 'character result len:', len(character_sentiment_result_dict), len(character_text_result_dict) #get user fansnum max fansnum_max = get_fansnum_max() #get user activeness by bulk_action print 'get activeness results' activeness_results = get_activity_time(uid_list) print 'activeness result len:', len(activeness_results) #get user inlfuence by bulk action print 'get influence' influence_results = get_influence(uid_list) print 'influence results len:', len(influence_results) # compute text attribute bulk_action = [] count = 0 for user in uid_list: count += 1 results = {} #get user text attribute: online_pattern results['online_pattern'] = json.dumps(online_pattern_dict[user]) try: results['online_pattern_aggs'] = '&'.join(online_pattern_dict[user].keys()) except: results['online_pattern_aggs'] = '' results['uid'] = str(user) #add user flow information: hashtag, activity_geo, keywords flow_dict = flow_result[str(user)] results = dict(results, **flow_dict) #add user topic attribute user_topic_dict = topic_results_dict[user] user_label_dict = topic_results_label[user] results['topic'] = json.dumps(user_topic_dict) # {'topic1_en':pro1, 'topic2_en':pro2...} results['topic_string'] = topic_en2ch(user_label_dict) # 'topic1_ch&topic2_ch&topic3_ch' #add user domain attribute user_domain_dict = domain_results[user] domain_list = domain_en2ch(user_domain_dict) if domain_list: results['domain_list'] = json.dumps(domain_list) # [label1_en, label2_en, label3_en] results['domain'] = domain_list[0] # label_ch else: results['domain'] = "其他" results['domain_list'] = json.dumps(["其他"]) politics_label = politics_results[user] results['politics'] = politics_en2ch(politics_label) #add user character_sentiment attribute #character_sentiment = character_sentiment_result_dict[user] #results['character_sentiment'] = character_sentiment #add user character_text attribtue #character_text = character_text_result_dict[user] #results['character_text'] = character_text #add user profile attribute register_dict = register_result[str(user)] results = dict(results, **register_dict) #add user_evaluate attribute---importance results['importance'] = get_importance(results['domain'], results['topic_string'], results['fansnum'], fansnum_max) #add user_evaluate attribute---activeness user_activeness_time = activeness_results[user] user_activeness_geo = json.loads(results['activity_geo_dict'])[-1] results['activeness'] = get_activeness(user_activeness_geo, user_activeness_time) #add user_evaluate attribute---influence results['influence'] = influence_results[user] #bulk_action action = {'index':{'_id': str(user)}} bulk_action.extend([action, results]) status = save_user_results(bulk_action) print "status:", status return status
def deal_bulk_action(user_info_list, fansnum_max): start_ts = time.time() uid_list = user_info_list.keys() #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(uid_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(uid_list) #compute attribute--keywords, topic, online_pattern #get user topic results by bulk action topic_results_dict, topic_results_label = topic_classfiy(uid_list, user_keywords_dict) domain_results = domain_classfiy(uid_list, user_keywords_dict) politics_results = political_classify(uid_list, user_keywords_dict) #update school attribute---is_school/school_string/school_dict #school_results_dict = get_school(uid_list) #get bulk action bulk_action = [] for uid in uid_list: results = {} results['uid'] = uid #results['is_school'] = school_results_dict[uid]['is_school'] #results['school_string'] = school_results_dict[uid]['school_string'] #results['school_dict'] = school_results_dict[uid]['school_dict'] #print 'is_school, school_string, school_dict:', results['is_school'],type(results['is_school']) ,results['school_string'],type(results['school_string']), results['school_dict'], type(results['school_dict']) #add user topic attribute user_topic_dict = topic_results_dict[uid] user_label_dict = topic_results_label[uid] results['topic'] = json.dumps(user_topic_dict) results['topic_string'] = topic_en2ch(user_label_dict) #add user domain attribute user_domain_dict = domain_results[uid] domain_list = domain_en2ch(user_domain_dict) if domain_list: results['domain_list'] = json.dumps(domain_list) results['domain'] = domain_list[0] else: results['domain'] = "其他" results['domain_list'] = json.dumps(["其他"]) politics_label = politics_results[uid] results['politics'] = politics_en2ch(politics_label) #add user keywords attribute try: keywords_dict = user_keywords_dict[uid] except: keywords_dict = {} keywords_top50 = sorted(keywords_dict.items(), key=lambda x:x[1], reverse=True)[:50] keywords_top50_string = '&'.join([keyword_item[0] for keyword_item in keywords_top50]) results['keywords'] = json.dumps(keywords_top50) results['keywords_string'] = keywords_top50_string #add online_pattern try: user_online_pattern = json.dumps(online_pattern_dict[uid]) except: user_online_pattern = json.dumps({}) results['online_pattern'] = user_online_pattern try: results['online_pattern_aggs'] = '&'.join(user_online_pattern.keys()) except: results['online_pattern_aggs'] = '' #add user importance user_domain = user_info_list[uid]['domain'].encode('utf-8') user_fansnum = user_info_list[uid]['fansnum'] results['importance'] = get_importance(user_domain, results['topic_string'], user_fansnum, fansnum_max) # politics politics_label = politics_results[user] results['politics'] = politics_en2ch(politics_label) #bulk action action = {'update':{'_id': uid}} bulk_action.extend([action, {'doc': results}]) print 'bulk_action:', bulk_action #es_user_portrait.bulk(bulk_action, index=portrait_index_name, doc_type=portrait_index_type) end_ts = time.time() #log_should_delete #print '%s sec count %s' % (end_ts - start_ts, len(uid_list)) #log_should_delete start_ts = end_ts