def read_uid_list(): date = ts2datetime(time.time()-24*3600) date = date.replace('-','') sensitive_dict = r.hgetall('identify_in_sensitive_'+str(date)) influence_dict = r.hgetall('identify_in_influence_'+str(date)) uid_list = [] for uid in sensitive_dict: if sensitive_dict[uid] != '3': uid_list.append(uid) for uid in influence_dict: if influence_dict[uid] != '3': uid_list.append(uid) return uid_list
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') iter_user_list = [] mapping_dict = dict() #test count = 0 for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == '2': #imme #test #count += 1 #if count >= 3: # break iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: r.hmset('compute', mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text( iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset('compute', mapping_dict) #acquire bulk user weibo date if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text( iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict)
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') iter_user_list = [] mapping_dict = dict() for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == '2': iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: #mark status from 1 to 3 as identify_compute to computing r.hmset('compute', mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status==True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #deal user no weibo to compute portrait attribute if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set(user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([in_date, '2']) r.hmset(change_mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset('compute', mapping_dict) #acquire bulk user weibo date if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status==True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #deal user no weibo to compute portrait attribute if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set(user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([in_date, '2']) r.hmset(change_mapping_dict)
def scan_compute_redis(): hash_name = "compute" results = r.hgetall("compute") iter_user_list = [] mapping_dict = dict() for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == "2": iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, "3"]) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: # mark status from 2 to 3 as identify_compute to computing r.hmset("compute", mapping_dict) # acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list ) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list ) # compute text attribute compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts ) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset("compute", mapping_dict) # acquire bulk user weibo date if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list ) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list ) # compute text attribute compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts ) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict)
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') iter_user_list = [] mapping_dict = dict() #test count = 0 for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == '2': #imme #test #count += 1 #if count >= 3: # break iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: r.hmset('compute', mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text_sentiment(iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text(iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict) if compute_status==True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset('compute', mapping_dict) #acquire bulk user weibo date if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text_sentiment(iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text(iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict) if compute_status==True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict)
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') user_list = [] mapping_dict = dict() for uid in results: user_list = json.loads(results[uid]) print 'user_list:', user_list in_date = user_list[0] status = user_list[1] if status == '1': user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:2 computing print 'mapping_dict:', mapping_dict r.hmset('compute', mapping_dict) #acquire bulk user weibo data #user_weibo_dict = read_user_weibo(user_list) #compute text attribute #compute_status = compute2in(user_list, user_weibo_dict, status='insert') compute_status = False if compute_status==True: change_status_computed(mapping_dict)
def recommentation_in(input_ts, recomment_type, submit_user): date = ts2datetime(input_ts) recomment_results = [] # read from redis results = [] hash_name = 'recomment_'+str(date) + "_" + recomment_type identify_in_hashname = "identify_in_" + str(date) submit_user_recomment = "recomment_" + submit_user + "_" + str(date) # 用户自推荐名单 results = r.hgetall(hash_name) if not results: return [] # search from user_profile to rich the show information recommend_list = set(r.hkeys(hash_name)) identify_in_list = set(r.hkeys("compute")) submit_user_recomment = set(r.hkeys(submit_user_recomment)) recomment_results = list(recommend_list - identify_in_list) recomment_results = list(set(recomment_results) - submit_user_recomment) if recomment_results: results = get_user_detail(date, recomment_results, 'show_in', recomment_type) else: results = [] return results
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') iter_user_list = [] mapping_dict = dict() #test count = 0 for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == '1': #imme #test count += 1 iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: r.hmset('compute', mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status==True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #when uid user no weibo at latest week to change compute status to 1 if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set(user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([in_date, '1']) r.hmset('compute', change_mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset('compute', mapping_dict) #acquire bulk user weibo date print 'iter_user_list:', len(iter_user_list) if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list) #compute text attribute print 'user_weibo_dict:', len(user_weibo_dict) compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status==True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #when uid user no weibo at latest week to change compute status to 1 if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set(user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([in_date, '1']) r.hmset('compute', change_mapping_dict)
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') iter_user_list = [] mapping_dict = dict() for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == '2': iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing #revise identify_in_date influence_hashname = 'identify_in_influence_' + str(in_date) sensitive_hashname = 'identify_in_sensitive_' + str(in_date) manual_hashname = "identify_in_manual_" + str(in_date) tmp = r.hget(influence_hashname, uid) tmp1 = r.hget(sensitive_hashname, uid) if tmp: r.hset(influence_hashname, uid, '3') elif tmp1: r.hset(sensitive_hashname, uid, '3') else: r.hset(manual_hashname, uid, '3') if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: #mark status from 1 to 3 as identify_compute to computing r.hmset('compute', mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #deal user no weibo to compute portrait attribute if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set( user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps( [in_date, '2']) r.hmset(change_mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset('compute', mapping_dict) #acquire bulk user weibo date if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #deal user no weibo to compute portrait attribute if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set( user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([in_date, '2']) r.hmset(change_mapping_dict)
def scan_compute_redis(): hash_name = 'compute' results = r.hgetall('compute') iter_user_list = [] mapping_dict = dict() #test count = 0 for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] if status == '1': #imme #test count += 1 iter_user_list.append(uid) mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: r.hmset('compute', mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #when uid user no weibo at latest week to change compute status to 1 if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set( user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps( [in_date, '1']) r.hmset('compute', change_mapping_dict) iter_user_list = [] mapping_dict = {} if iter_user_list != [] and mapping_dict != {}: r.hmset('compute', mapping_dict) #acquire bulk user weibo date print 'iter_user_list:', len(iter_user_list) if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list) #compute text attribute print 'user_weibo_dict:', len(user_weibo_dict) compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #when uid user no weibo at latest week to change compute status to 1 if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set( user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([in_date, '1']) r.hmset('compute', change_mapping_dict)
def scan_compute_redis(): task_mark = 'user' hash_name = r_user_hash_name results = r.hgetall(hash_name) iter_user_list = [] mapping_dict = dict() verify_mark_dict = dict() relation_mark_dict = dict() submit_user_dict = dict() submit_ts_dict = dict() count = 0 for uid in results: user_list = json.loads(results[uid]) in_date = user_list[0] status = user_list[1] verify_mark = user_list[2] relation_list = user_list[3] submit_user = user_list[4] submit_ts = datetime2ts(in_date) verify_mark_dict[uid] = verify_mark relation_mark_dict[uid] = relation_list submit_user_dict[uid] = submit_user submit_ts_dict[uid] = submit_ts if status == '2': #imme #test count += 1 iter_user_list.append(uid) mapping_dict[uid] = json.dumps( [in_date, '3', verify_mark, relation_list, submit_user]) # mark status:3 computing if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0: r.hmset(r_user_hash_name, mapping_dict) #acquire bulk user weibo data if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list) #compute text attribute compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, relation_mark_dict, task_mark, submit_user_dict, submit_ts_dict) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #when uid user no weibo at latest week to change compute status to 1 if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set( user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([ in_date, '2', verify_mark_dict[change_user], relation_mark_dict[change_user], submit_user_dict[change_user], submit_ts_dict[change_user] ]) r.hmset(r_user_hash_name, change_mapping_dict) iter_user_list = [] mapping_dict = {} verify_mark_dict = dict() relation_mark_dict = dict() submit_user_dict = dict() submit_ts_dict = dict() if iter_user_list != [] and mapping_dict != {}: r.hmset(r_user_hash_name, mapping_dict) #acquire bulk user weibo date print 'iter_user_list:', len(iter_user_list) if WEIBO_API_INPUT_TYPE == 0: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment( iter_user_list) else: user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text( iter_user_list) #compute text attribute print 'user_weibo_dict:', len(user_weibo_dict) compute_status = test_cron_text_attribute_v2( user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, relation_mark_dict, task_mark, submit_user_dict, submit_ts_dict) if compute_status == True: change_status_computed(mapping_dict) else: change_status_compute_fail(mapping_dict) #when uid user no weibo at latest week to change compute status to 1 if len(user_keywords_dict) != len(iter_user_list): change_mapping_dict = dict() change_user_list = set(iter_user_list) - set( user_keywords_dict.keys()) for change_user in change_user_list: change_mapping_dict[change_user] = json.dumps([ in_date, '2', verify_mark_dict[change_user], relation_mark_dict[change_user], submit_user_dict[change_user], submit_ts_dict[change_user] ]) r.hmset(r_user_hash_name, change_mapping_dict)