def get_recommentation(admin_user):
submit_recommentation_count = 0
compute_count = 0
search_date = ts2datetime(time.time() - DAY)
submit_recomment_key = 'recomment_' + admin_user + '_' + search_date
submit_user_recomment = set(R_RECOMMENTATION.hkeys(submit_recomment_key))
all_compute_set = set(R_RECOMMENTATION.hkeys('compute'))
submit_recommentation_count = len(submit_user_recomment)
compute_count = len(submit_user_recomment & all_compute_set)
return submit_recommentation_count, compute_count
def get_recommentation(admin_user):
submit_recommentation_count = 0
compute_count = 0
search_date = ts2datetime(time.time() - DAY)
submit_recomment_key = 'recomment_' + admin_user + '_' + search_date
submit_user_recomment = set(R_RECOMMENTATION.hkeys(submit_recomment_key))
all_compute_set = set(R_RECOMMENTATION.hkeys('compute'))
submit_recommentation_count = len(submit_user_recomment)
compute_count = len(submit_user_recomment & all_compute_set)
return submit_recommentation_count, compute_count
def filter_recommend(top_user_set):
recommend_keys = r.hkeys('recommend')
recommend_list = []
for key in recommend_keys:
recommend_list.extend(json.loads(r.hget('recommend', key)))
results = set(top_user_set) - set(recommend_list)
return results
def filter_recommend(top_user_set):
recommend_keys = r.hkeys('recommend')
recommend_list = []
for key in recommend_keys:
recommend_list.extend(json.loads(r.hget('recommend', key)))
results = set(top_user_set) - set(recommend_list)
return results
def get_recomment_history(admin_user, now_date):
results = set()
now_ts = datetime2ts(now_date)
for i in range(RECOMMEND_IN_AUTO_DATE, 0, -1):
iter_date = ts2datetime(now_ts - i * DAY)
submit_user_recomment = 'recoment_' + admin_user + '_' + str(iter_date)
recomment_user_list = set(R_RECOMMENTATION.hkeys(submit_user_recomment))
results = results | recomment_user_list
return results
def get_recomment_history(admin_user, now_date):
results = set()
now_ts = datetime2ts(now_date)
for i in range(RECOMMEND_IN_AUTO_DATE, 0, -1):
iter_date = ts2datetime(now_ts - i * DAY)
submit_user_recomment = 'recoment_' + admin_user + '_' + str(iter_date)
recomment_user_list = set(R_RECOMMENTATION.hkeys(submit_user_recomment))
results = results | recomment_user_list
return results
def update_recommentation_compute(admin_user):
status = False
#step1: update lastest 6 day compute count
end_ts = datetime2ts(ts2datetime(time.time() - DAY))
all_compute_set = set(R_RECOMMENTATION.hkeys('compute'))
bulk_action = []
for i in range(1, 6):
iter_ts = end_ts - i* DAY
iter_date = ts2datetime(iter_ts)
submit_recomment_key = 'recomment_' + admin_user + '_' + iter_date
submit_recomment_set = set(R_RECOMMENTATION.hkeys(submit_recomment_key))
compute_count = len(submit_recomment_set & all_compute_set)
user_results = {'compute_count': compute_count}
action = {'update': {'_id': admin_user + '_' + str(iter_ts)}}
bulk_action.extend([action, {'doc': user_results}])
#step2: update bulk action
#print 'bulk_action:', bulk_action
es_operation.bulk(bulk_action, index=operation_index_name, doc_type=operation_index_type)
status = True
return status
def recommentation_in(input_ts, recomment_type, submit_user):
date = ts2datetime(input_ts)
recomment_results = []
# read from redis
results = []
hash_name = 'recomment_'+str(date) + "_" + recomment_type
identify_in_hashname = "identify_in_" + str(date)
submit_user_recomment = "recomment_" + submit_user + "_" + str(date) # 用户自推荐名单
results = r.hgetall(hash_name)
if not results:
return []
# search from user_profile to rich the show information
recommend_list = set(r.hkeys(hash_name))
identify_in_list = set(r.hkeys("compute"))
submit_user_recomment = set(r.hkeys(submit_user_recomment))
recomment_results = list(recommend_list - identify_in_list)
recomment_results = list(set(recomment_results) - submit_user_recomment)
if recomment_results:
results = get_user_detail(date, recomment_results, 'show_in', recomment_type)
else:
results = []
return results
def filter_rules(candidate_results):
results = []
#rule1: activity count
filter_result1 = filter_activity(candidate_results)
#rule2: ip count
filter_result2 = filter_ip(filter_result1)
#rule3: retweet count & beretweeted count
filter_result3 = filter_retweet_count(filter_result2)
#rule4: mention count
results = filter_mention(filter_result3)
#rule5: compute count
compute_uid_set = r.hkeys("compute")
results = set(results) - set(compute_uid_set)
return results
def filter_rules(candidate_results):
results = []
#rule1: activity count
filter_result1 = filter_activity(candidate_results)
#rule2: ip count
filter_result2 = filter_ip(filter_result1)
#rule3: retweet count & beretweeted count
filter_result3 = filter_retweet_count(filter_result2)
#rule4: mention count
results = filter_mention(filter_result3)
#rule5: compute count
compute_uid_set = r.hkeys("compute")
results = set(results) - set(compute_uid_set)
return results
def update_recommentation_compute(admin_user):
status = False
#step1: update lastest 6 day compute count
end_ts = datetime2ts(ts2datetime(time.time() - DAY))
all_compute_set = set(R_RECOMMENTATION.hkeys('compute'))
bulk_action = []
for i in range(1, 6):
iter_ts = end_ts - i * DAY
iter_date = ts2datetime(iter_ts)
submit_recomment_key = 'recomment_' + admin_user + '_' + iter_date
submit_recomment_set = set(
R_RECOMMENTATION.hkeys(submit_recomment_key))
compute_count = len(submit_recomment_set & all_compute_set)
user_results = {'compute_count': compute_count}
action = {'update': {'_id': admin_user + '_' + str(iter_ts)}}
bulk_action.extend([action, {'doc': user_results}])
#step2: update bulk action
#print 'bulk_action:', bulk_action
es_operation.bulk(bulk_action,
index=operation_index_name,
doc_type=operation_index_type)
status = True
return status
def get_operate_information():
result = dict()
now_ts = time.time()
date = ts2datetime(now_ts - 24*3600)
#test
date = '2013-09-07'
delete_date = ''.join(date.split('-'))
#test
#delete_date = '20150727'
result['in_count'] = len(r_recomment.hkeys('recomment_'+str(date)))
out_count_list = r_recomment.hget('recommend_delete_list', delete_date)
#print 'out_count_list:', out_count_list
if out_count_list:
result['out_count'] = len(json.loads(out_count_list))
else:
result['out_count'] = 0
compute_list = r_recomment.hkeys('compute')
'''
if compute_list:
result['compute'] = len(compute_list)
'''
#print 'operate compute:', result
return result
def get_operate_information():
result = dict()
#run_type
if RUN_TYPE == 1:
now_ts = time.time()
else:
now_ts = datetime2ts(RUN_TEST_TIME)
date = ts2datetime(now_ts - DAY)
delete_date = ''.join(date.split('-'))
result['in_count'] = len(r_recomment.hkeys('recomment_' + str(date)))
out_count_list = r_recomment.hget('recommend_delete_list', delete_date)
if out_count_list:
result['out_count'] = len(json.loads(out_count_list))
else:
result['out_count'] = 0
compute_list = r_recomment.hkeys('compute')
if compute_list:
result['compute'] = len(compute_list)
else:
result['compute'] = 0
return result
def createWordTree():
awords = []
sensitive_words = r.hkeys('sensitive_words')
#for b in open('./../../sensitive_words.txt', 'rb'):
# awords.append(b.strip())
awords = sensitive_words
for word in awords:
temp = wordTree
for a in range(0,len(word)):
index = ord(word[a])
if a < (len(word) - 1):
if temp[index] == None:
node = [[None for x in range(256)],0]
temp[index] = node
elif temp[index] == 1:
node = [[None for x in range(256)],1]
temp[index] = node
temp = temp[index][0]
else:
temp[index] = 1
