def get_recommentation(admin_user): submit_recommentation_count = 0 compute_count = 0 search_date = ts2datetime(time.time() - DAY) submit_recomment_key = 'recomment_' + admin_user + '_' + search_date submit_user_recomment = set(R_RECOMMENTATION.hkeys(submit_recomment_key)) all_compute_set = set(R_RECOMMENTATION.hkeys('compute')) submit_recommentation_count = len(submit_user_recomment) compute_count = len(submit_user_recomment & all_compute_set) return submit_recommentation_count, compute_count
def filter_recommend(top_user_set): recommend_keys = r.hkeys('recommend') recommend_list = [] for key in recommend_keys: recommend_list.extend(json.loads(r.hget('recommend', key))) results = set(top_user_set) - set(recommend_list) return results
def get_recomment_history(admin_user, now_date): results = set() now_ts = datetime2ts(now_date) for i in range(RECOMMEND_IN_AUTO_DATE, 0, -1): iter_date = ts2datetime(now_ts - i * DAY) submit_user_recomment = 'recoment_' + admin_user + '_' + str(iter_date) recomment_user_list = set(R_RECOMMENTATION.hkeys(submit_user_recomment)) results = results | recomment_user_list return results
def update_recommentation_compute(admin_user): status = False #step1: update lastest 6 day compute count end_ts = datetime2ts(ts2datetime(time.time() - DAY)) all_compute_set = set(R_RECOMMENTATION.hkeys('compute')) bulk_action = [] for i in range(1, 6): iter_ts = end_ts - i* DAY iter_date = ts2datetime(iter_ts) submit_recomment_key = 'recomment_' + admin_user + '_' + iter_date submit_recomment_set = set(R_RECOMMENTATION.hkeys(submit_recomment_key)) compute_count = len(submit_recomment_set & all_compute_set) user_results = {'compute_count': compute_count} action = {'update': {'_id': admin_user + '_' + str(iter_ts)}} bulk_action.extend([action, {'doc': user_results}]) #step2: update bulk action #print 'bulk_action:', bulk_action es_operation.bulk(bulk_action, index=operation_index_name, doc_type=operation_index_type) status = True return status
def recommentation_in(input_ts, recomment_type, submit_user): date = ts2datetime(input_ts) recomment_results = [] # read from redis results = [] hash_name = 'recomment_'+str(date) + "_" + recomment_type identify_in_hashname = "identify_in_" + str(date) submit_user_recomment = "recomment_" + submit_user + "_" + str(date) # 用户自推荐名单 results = r.hgetall(hash_name) if not results: return [] # search from user_profile to rich the show information recommend_list = set(r.hkeys(hash_name)) identify_in_list = set(r.hkeys("compute")) submit_user_recomment = set(r.hkeys(submit_user_recomment)) recomment_results = list(recommend_list - identify_in_list) recomment_results = list(set(recomment_results) - submit_user_recomment) if recomment_results: results = get_user_detail(date, recomment_results, 'show_in', recomment_type) else: results = [] return results
def filter_rules(candidate_results): results = [] #rule1: activity count filter_result1 = filter_activity(candidate_results) #rule2: ip count filter_result2 = filter_ip(filter_result1) #rule3: retweet count & beretweeted count filter_result3 = filter_retweet_count(filter_result2) #rule4: mention count results = filter_mention(filter_result3) #rule5: compute count compute_uid_set = r.hkeys("compute") results = set(results) - set(compute_uid_set) return results
def update_recommentation_compute(admin_user): status = False #step1: update lastest 6 day compute count end_ts = datetime2ts(ts2datetime(time.time() - DAY)) all_compute_set = set(R_RECOMMENTATION.hkeys('compute')) bulk_action = [] for i in range(1, 6): iter_ts = end_ts - i * DAY iter_date = ts2datetime(iter_ts) submit_recomment_key = 'recomment_' + admin_user + '_' + iter_date submit_recomment_set = set( R_RECOMMENTATION.hkeys(submit_recomment_key)) compute_count = len(submit_recomment_set & all_compute_set) user_results = {'compute_count': compute_count} action = {'update': {'_id': admin_user + '_' + str(iter_ts)}} bulk_action.extend([action, {'doc': user_results}]) #step2: update bulk action #print 'bulk_action:', bulk_action es_operation.bulk(bulk_action, index=operation_index_name, doc_type=operation_index_type) status = True return status
def get_operate_information(): result = dict() now_ts = time.time() date = ts2datetime(now_ts - 24*3600) #test date = '2013-09-07' delete_date = ''.join(date.split('-')) #test #delete_date = '20150727' result['in_count'] = len(r_recomment.hkeys('recomment_'+str(date))) out_count_list = r_recomment.hget('recommend_delete_list', delete_date) #print 'out_count_list:', out_count_list if out_count_list: result['out_count'] = len(json.loads(out_count_list)) else: result['out_count'] = 0 compute_list = r_recomment.hkeys('compute') ''' if compute_list: result['compute'] = len(compute_list) ''' #print 'operate compute:', result return result
def get_operate_information(): result = dict() #run_type if RUN_TYPE == 1: now_ts = time.time() else: now_ts = datetime2ts(RUN_TEST_TIME) date = ts2datetime(now_ts - DAY) delete_date = ''.join(date.split('-')) result['in_count'] = len(r_recomment.hkeys('recomment_' + str(date))) out_count_list = r_recomment.hget('recommend_delete_list', delete_date) if out_count_list: result['out_count'] = len(json.loads(out_count_list)) else: result['out_count'] = 0 compute_list = r_recomment.hkeys('compute') if compute_list: result['compute'] = len(compute_list) else: result['compute'] = 0 return result
def createWordTree(): awords = [] sensitive_words = r.hkeys('sensitive_words') #for b in open('./../../sensitive_words.txt', 'rb'): # awords.append(b.strip()) awords = sensitive_words for word in awords: temp = wordTree for a in range(0,len(word)): index = ord(word[a]) if a < (len(word) - 1): if temp[index] == None: node = [[None for x in range(256)],0] temp[index] = node elif temp[index] == 1: node = [[None for x in range(256)],1] temp[index] = node temp = temp[index][0] else: temp[index] = 1