def read_uid_list():
date = ts2datetime(time.time()-24*3600)
date = date.replace('-','')
sensitive_dict = r.hgetall('identify_in_sensitive_'+str(date))
influence_dict = r.hgetall('identify_in_influence_'+str(date))
uid_list = []
for uid in sensitive_dict:
if sensitive_dict[uid] != '3':
uid_list.append(uid)
for uid in influence_dict:
if influence_dict[uid] != '3':
uid_list.append(uid)
return uid_list
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
iter_user_list = []
mapping_dict = dict()
#test
count = 0
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == '2': #imme
#test
#count += 1
#if count >= 3:
# break
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date,
'3']) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text(
iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict,
character_start_ts, filter_keywords_dict)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo date
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts, filter_keywords_dict = read_flow_text(
iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict,
user_weibo_dict,
online_pattern_dict,
character_start_ts,
filter_keywords_dict)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
iter_user_list = []
mapping_dict = dict()
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == '2':
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
#mark status from 1 to 3 as identify_compute to computing
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts)
if compute_status==True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#deal user no weibo to compute portrait attribute
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([in_date, '2'])
r.hmset(change_mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo date
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts)
if compute_status==True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#deal user no weibo to compute portrait attribute
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([in_date, '2'])
r.hmset(change_mapping_dict)
def scan_compute_redis():
hash_name = "compute"
results = r.hgetall("compute")
iter_user_list = []
mapping_dict = dict()
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == "2":
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date, "3"]) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
# mark status from 2 to 3 as identify_compute to computing
r.hmset("compute", mapping_dict)
# acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list
)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list
)
# compute text attribute
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts
)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset("compute", mapping_dict)
# acquire bulk user weibo date
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list
)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list
)
# compute text attribute
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts
)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
iter_user_list = []
mapping_dict = dict()
#test
count = 0
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == '2': #imme
#test
#count += 1
#if count >= 3:
# break
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text_sentiment(iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text(iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict)
if compute_status==True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo date
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text_sentiment(iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict = read_flow_text(iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts,filter_keywords_dict)
if compute_status==True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
user_list = []
mapping_dict = dict()
for uid in results:
user_list = json.loads(results[uid])
print 'user_list:', user_list
in_date = user_list[0]
status = user_list[1]
if status == '1':
user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:2 computing
print 'mapping_dict:', mapping_dict
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
#user_weibo_dict = read_user_weibo(user_list)
#compute text attribute
#compute_status = compute2in(user_list, user_weibo_dict, status='insert')
compute_status = False
if compute_status==True:
change_status_computed(mapping_dict)
def recommentation_in(input_ts, recomment_type, submit_user):
date = ts2datetime(input_ts)
recomment_results = []
# read from redis
results = []
hash_name = 'recomment_'+str(date) + "_" + recomment_type
identify_in_hashname = "identify_in_" + str(date)
submit_user_recomment = "recomment_" + submit_user + "_" + str(date) # 用户自推荐名单
results = r.hgetall(hash_name)
if not results:
return []
# search from user_profile to rich the show information
recommend_list = set(r.hkeys(hash_name))
identify_in_list = set(r.hkeys("compute"))
submit_user_recomment = set(r.hkeys(submit_user_recomment))
recomment_results = list(recommend_list - identify_in_list)
recomment_results = list(set(recomment_results) - submit_user_recomment)
if recomment_results:
results = get_user_detail(date, recomment_results, 'show_in', recomment_type)
else:
results = []
return results
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
iter_user_list = []
mapping_dict = dict()
#test
count = 0
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == '1': #imme
#test
count += 1
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date, '3']) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts)
if compute_status==True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#when uid user no weibo at latest week to change compute status to 1
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([in_date, '1'])
r.hmset('compute', change_mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo date
print 'iter_user_list:', len(iter_user_list)
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(iter_user_list)
#compute text attribute
print 'user_weibo_dict:', len(user_weibo_dict)
compute_status = test_cron_text_attribute_v2(user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts)
if compute_status==True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#when uid user no weibo at latest week to change compute status to 1
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([in_date, '1'])
r.hmset('compute', change_mapping_dict)
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
iter_user_list = []
mapping_dict = dict()
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == '2':
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date,
'3']) # mark status:3 computing
#revise identify_in_date
influence_hashname = 'identify_in_influence_' + str(in_date)
sensitive_hashname = 'identify_in_sensitive_' + str(in_date)
manual_hashname = "identify_in_manual_" + str(in_date)
tmp = r.hget(influence_hashname, uid)
tmp1 = r.hget(sensitive_hashname, uid)
if tmp:
r.hset(influence_hashname, uid, '3')
elif tmp1:
r.hset(sensitive_hashname, uid, '3')
else:
r.hset(manual_hashname, uid, '3')
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
#mark status from 1 to 3 as identify_compute to computing
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict,
character_start_ts)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#deal user no weibo to compute portrait attribute
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(
user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps(
[in_date, '2'])
r.hmset(change_mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo date
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(user_keywords_dict,
user_weibo_dict,
online_pattern_dict,
character_start_ts)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#deal user no weibo to compute portrait attribute
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(
user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([in_date, '2'])
r.hmset(change_mapping_dict)
def scan_compute_redis():
hash_name = 'compute'
results = r.hgetall('compute')
iter_user_list = []
mapping_dict = dict()
#test
count = 0
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
if status == '1': #imme
#test
count += 1
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps([in_date,
'3']) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict,
character_start_ts)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#when uid user no weibo at latest week to change compute status to 1
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(
user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps(
[in_date, '1'])
r.hmset('compute', change_mapping_dict)
iter_user_list = []
mapping_dict = {}
if iter_user_list != [] and mapping_dict != {}:
r.hmset('compute', mapping_dict)
#acquire bulk user weibo date
print 'iter_user_list:', len(iter_user_list)
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list)
#compute text attribute
print 'user_weibo_dict:', len(user_weibo_dict)
compute_status = test_cron_text_attribute_v2(user_keywords_dict,
user_weibo_dict,
online_pattern_dict,
character_start_ts)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#when uid user no weibo at latest week to change compute status to 1
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(
user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([in_date, '1'])
r.hmset('compute', change_mapping_dict)
def scan_compute_redis():
task_mark = 'user'
hash_name = r_user_hash_name
results = r.hgetall(hash_name)
iter_user_list = []
mapping_dict = dict()
verify_mark_dict = dict()
relation_mark_dict = dict()
submit_user_dict = dict()
submit_ts_dict = dict()
count = 0
for uid in results:
user_list = json.loads(results[uid])
in_date = user_list[0]
status = user_list[1]
verify_mark = user_list[2]
relation_list = user_list[3]
submit_user = user_list[4]
submit_ts = datetime2ts(in_date)
verify_mark_dict[uid] = verify_mark
relation_mark_dict[uid] = relation_list
submit_user_dict[uid] = submit_user
submit_ts_dict[uid] = submit_ts
if status == '2': #imme
#test
count += 1
iter_user_list.append(uid)
mapping_dict[uid] = json.dumps(
[in_date, '3', verify_mark, relation_list,
submit_user]) # mark status:3 computing
if len(iter_user_list) % 100 == 0 and len(iter_user_list) != 0:
r.hmset(r_user_hash_name, mapping_dict)
#acquire bulk user weibo data
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list)
#compute text attribute
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict,
character_start_ts, relation_mark_dict, task_mark,
submit_user_dict, submit_ts_dict)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#when uid user no weibo at latest week to change compute status to 1
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(
user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([
in_date, '2', verify_mark_dict[change_user],
relation_mark_dict[change_user],
submit_user_dict[change_user],
submit_ts_dict[change_user]
])
r.hmset(r_user_hash_name, change_mapping_dict)
iter_user_list = []
mapping_dict = {}
verify_mark_dict = dict()
relation_mark_dict = dict()
submit_user_dict = dict()
submit_ts_dict = dict()
if iter_user_list != [] and mapping_dict != {}:
r.hmset(r_user_hash_name, mapping_dict)
#acquire bulk user weibo date
print 'iter_user_list:', len(iter_user_list)
if WEIBO_API_INPUT_TYPE == 0:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text_sentiment(
iter_user_list)
else:
user_keywords_dict, user_weibo_dict, online_pattern_dict, character_start_ts = read_flow_text(
iter_user_list)
#compute text attribute
print 'user_weibo_dict:', len(user_weibo_dict)
compute_status = test_cron_text_attribute_v2(
user_keywords_dict, user_weibo_dict, online_pattern_dict,
character_start_ts, relation_mark_dict, task_mark,
submit_user_dict, submit_ts_dict)
if compute_status == True:
change_status_computed(mapping_dict)
else:
change_status_compute_fail(mapping_dict)
#when uid user no weibo at latest week to change compute status to 1
if len(user_keywords_dict) != len(iter_user_list):
change_mapping_dict = dict()
change_user_list = set(iter_user_list) - set(
user_keywords_dict.keys())
for change_user in change_user_list:
change_mapping_dict[change_user] = json.dumps([
in_date, '2', verify_mark_dict[change_user],
relation_mark_dict[change_user],
submit_user_dict[change_user], submit_ts_dict[change_user]
])
r.hmset(r_user_hash_name, change_mapping_dict)
