def submit_task(input_data):
status = 0 # mark it can not submit
task_name = input_data['task_name']
submit_user = input_data['submit_user']
task_id = input_data['task_id']
try:
result = es_recommendation_result.get(
index=recommendation_index_name,
doc_type=recommendation_index_type,
id=task_id)['_source']
except:
status = 1
if status != 0 and 'uid_file' not in input_data:
input_data['status'] = 0 # mark the task not compute
count = len(json.loads(input_data['uid_list']))
input_data['count'] = count
input_data['submit_user'] = submit_user
# add_es_dict = {'task_information': input_data, 'query_condition':''}
es_recommendation_result.index(index=recommendation_index_name,
doc_type=recommendation_index_type,
id=task_id,
body=input_data)
if input_data['cal_style'] == 0:
group_analysis_queue_name = 'recommendation_in_now'
if input_data['cal_style'] == 1:
group_analysis_queue_name = 'recommendation_in_later'
r.lpush(group_analysis_queue_name, json.dumps(input_data))
return status
def identify_in(data, uid_list):
in_status = 1
compute_status = 0
compute_hash_name = 'compute'
compute_uid = r.hkeys(compute_hash_name)
for item in data:
date = item[0] # identify the date form '2013-09-01' with web
uid = item[1]
status = item[2]
relation_string = item[3]
recommend_style = item[4]
submit_user = item[5]
value_string = []
identify_in_hashname = "identify_in_" + str(date)
r.hset(identify_in_hashname, uid, in_status)
if status == '1':
in_date = date
compute_status = '1'
elif status == '2':
in_date = date
compute_status = '2'
r.hset(
compute_hash_name, uid,
json.dumps([
in_date, compute_status, relation_string, recommend_style,
submit_user, 0
]))
return True
def recommentation_in(input_ts, recomment_type, submit_user, node_type):
date = ts2datetime(input_ts)
recomment_results = []
# read from redis
results = []
hash_name = 'recomment_' + str(
date) + "_" + recomment_type + "_" + node_type
identify_in_hashname = "identify_in_" + str(date)
# submit_user_recomment = "recomment_" + submit_user + "_" + str(date) # 用户自推荐名单
results = r.hgetall(hash_name)
if not results:
return []
# search from user_profile to rich the show information
recommend_list = set(r.hkeys(hash_name))
identify_in_list = set(r.hkeys("compute"))
# submit_user_recomment = set(r.hkeys(submit_user_recomment))
recomment_results = list(recommend_list - identify_in_list)
# recomment_results = list(set(recomment_results) - submit_user_recomment)
if recomment_results:
results = get_user_detail(date, recomment_results[:1000], 'show_in',
recomment_type)
else:
results = []
return results
def ajax_update_user():
results = 0
date = request.args.get('date', '2016-11-27') # date = '2016-11-27'
node_type = request.args.get('node_type', '0') # '0':user '1'：org
uid = request.args.get('uid', '2298607941')
recommend_style = request.args.get('recommend_style', 'influence') #influence sensitive auto
submit_user = request.args.get('submit_user', 'admin')
relation_string = request.args.get('user_rel', 'friend') # split by ,
relation_list = relation_string.split(',')
r.rpush(uid, json.dumps([date, '1', node_type, relation_list, submit_user, recommend_style]))
return json.dumps(1)
def recommentation_in_auto(date, submit_user):
results = []
#run type
if RUN_TYPE == 1:
now_date = search_date
else:
now_date = ts2datetime(datetime2ts(RUN_TEST_TIME))
recomment_hash_name = 'recomment_' + now_date + '_auto'
# print recomment_hash_name,'============'
recomment_influence_hash_name = 'recomment_' + now_date + '_influence'
recomment_sensitive_hash_name = 'recomment_' + now_date + '_sensitive'
recomment_submit_hash_name = 'recomment_' + submit_user + '_' + now_date
recomment_compute_hash_name = 'compute'
# #step1: get auto
# auto_result = r.hget(recomment_hash_name, 'auto')
# if auto_result:
# auto_user_list = json.loads(auto_result)
# else:
# auto_user_list = []
#step2: get admin user result
admin_result = r.hget(recomment_hash_name, submit_user)
admin_user_list = []
if admin_result:
admin_result_dict = json.loads(admin_result)
else:
return None
final_result = []
#step3: get union user and filter compute/influence/sensitive
for k, v in admin_result_dict.iteritems():
admin_user_list = v
union_user_auto_set = set(admin_user_list)
influence_user = set(r.hkeys(recomment_influence_hash_name))
sensitive_user = set(r.hkeys(recomment_sensitive_hash_name))
compute_user = set(r.hkeys(recomment_compute_hash_name))
been_submit_user = set(r.hkeys(recomment_submit_hash_name))
filter_union_user = union_user_auto_set - (
influence_user | sensitive_user | compute_user | been_submit_user)
auto_user_list = list(filter_union_user)
#step4: get user detail
if auto_user_list == []:
return auto_user_list
results = get_user_detail(now_date, auto_user_list, 'show_in', 'auto')
for detail in results: #add root
re_detail = detail
re_detail.append(k)
final_result.append(re_detail)
return final_result
def ajax_show_user_task_status():
compute_name = 'compute'
result = r.hgetall(compute_name)
result_dict = {}
for k,v in result.iteritems():
detail = json.loads(v)
result_dict[k] = detail
return json.dumps(result_dict)
def recommentation_in(input_ts, recomment_type):
date = ts2datetime(input_ts)
recomment_results = []
results = []
hash_name = 'recomment_' + str(date) + "_" + recomment_type
identify_in_hashname = "identify_in_" + str(date)
results = r.hgetall(hash_name)
if not results:
return []
recommend_list = set(r.hkeys(hash_name))
identify_in_list = set(r.hkeys("compute"))
recomment_results = list(recommend_list - identify_in_list)[0:3]
if recomment_results:
results = get_user_detail(date, recomment_results)
else:
results = []
return results
def ajax_show_user_task_status():
node_type = request.args.get('node_type', '0') # '0':user '1'：org
compute_name = 'compute'
result = r.hgetall(compute_name)
# return json.dumps(result)
result_list = []
for k, v in result.iteritems():
detail = json.loads(v)
if detail[2] == node_type:
kv_list = [k]
kv_list.extend(detail)
result_list.append(kv_list)
else:
continue
return json.dumps(result_list)
def submit_identify_in_uid(input_data):
date = input_data['date']
submit_user = input_data['user']
operation_type = input_data['operation_type']
compute_status = input_data['compute_status']
relation_string = input_data['relation_string']
recommend_style = input_data['recommend_style']
hashname_submit = 'submit_recomment_' + date
hashname_influence = 'recomment_' + date + '_influence'
hashname_sensitive = 'recomment_' + date + '_sensitive'
compute_hash_name = 'compute'
# submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
auto_recomment_set = set(r.hkeys(hashname_influence)) | set(
r.hkeys(hashname_sensitive))
upload_data = input_data['upload_data']
line_list = upload_data.split('\n')
uid_list = []
invalid_uid_list = []
for line in line_list:
uid = line.split('\r')[0]
#if len(uid)==10:
# uid_list.append(uid)
if uid != '':
uid_list.append(uid)
if len(invalid_uid_list) != 0:
return False, 'invalid user info', invalid_uid_list
#identify the uid is not exist in user_portrait and compute
#step1: filter in user_portrait
new_uid_list = []
have_in_uid_list = []
try:
exist_portrait_result = es_user_profile.mget(
index=profile_index_name,
doc_type=profile_index_type,
body={'ids': uid_list},
_source=False)['docs']
except:
exist_portrait_result = []
if exist_portrait_result:
for exist_item in exist_portrait_result:
if exist_item['found'] == False:
new_uid_list.append(exist_item['_id'])
else:
have_in_uid_list.append(exist_item['_id'])
else:
new_uid_list = uid_list
#step2: filter in compute
new_uid_set = set(new_uid_list)
compute_set = set(r.hkeys('compute'))
in_uid_set = list(new_uid_set - compute_set)
print 'new_uid_set:', new_uid_set
print 'in_uid_set:', in_uid_set
if len(in_uid_set) == 0:
return False, 'all user in'
#identify the final add user
final_submit_user_list = []
for in_item in in_uid_set:
if in_item in auto_recomment_set:
tmp = json.loads(r.hget(hashname_submit, in_item))
recommentor_list = tmp['operation'].split('&')
recommentor_list.append(str(submit_user))
new_list = list(set(recommentor_list))
tmp['operation'] = '&'.join(new_list)
else:
tmp = {'system': '0', 'operation': submit_user}
if operation_type == 'submit':
r.hset(
compute_hash_name, in_item,
json.dumps([
in_date, compute_status, relation_string, recommend_style,
submit_user, 0
]))
r.hset(hashname_submit, in_item, json.dumps(tmp))
# r.hset(submit_user_recomment, in_item, '0')
final_submit_user_list.append(in_item)
return True, invalid_uid_list, have_in_uid_list, final_submit_user_list
sensitive_value = math.log(
sensitive_history_dict['fields'][sensitive_string][0] /
float(top_sensitive) * 9 + 1, 10) * 100
#print "sensitive_value", sensitive_value
else:
sensitive_value = 0
results.append([
uid, uname, location, fansnum, statusnum, influence,
sensitive_words, sensitive_value
])
else:
results.append(
[uid, uname, location, fansnum, statusnum, influence])
if auth:
hashname_submit = "submit_recomment_" + date
tmp_data = json.loads(r.hget(hashname_submit, uid))
recommend_list = (tmp_data['operation']).split('&')
admin_list = []
admin_list.append(tmp_data['system'])
admin_list.append(list(set(recommend_list)))
admin_list.append(len(recommend_list))
results[-1].extend(admin_list)
if status == 'show_compute':
in_date = json.loads(input_result[uid])[0]
compute_status = json.loads(input_result[uid])[1]
if compute_status == '1':
compute_status = '3'
results.append([
uid, uname, location, fansnum, statusnum, influence, in_date,
compute_status
])
def submit_identify_in_uid(input_data):
print input_data, '00000000000'
in_date = input_data['date']
submit_user = input_data['user']
operation_type = input_data['operation_type']
compute_status = input_data['compute_status']
relation_string = input_data['relation_string']
recommend_style = input_data['recommend_style']
node_type = input_data['node_type']
hashname_submit = 'submit_recomment_' + in_date
hashname_influence = 'recomment_' + in_date + '_influence'
hashname_sensitive = 'recomment_' + in_date + '_sensitive'
compute_hash_name = 'compute'
# submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
auto_recomment_set = set(r.hkeys(hashname_influence)) | set(
r.hkeys(hashname_sensitive))
upload_data = input_data['upload_data']
uid_list = []
invalid_uid_list = []
if recommend_style == 'upload':
line_list = upload_data
# print line_list,'====8888===='
for line in line_list:
uid = line.strip('\r')
# print len(str(uid)),'!!!0000000000999999999999'
if len(str(uid)) == 10:
uid_list.append(uid)
else:
invalid_uid_list.append(uid)
if recommend_style == 'write':
line_list = upload_data
# print line_list,'====8888===='
for line in line_list:
uid = line
if len(str(uid)) == 10:
uid_list.append(uid)
else:
invalid_uid_list.append(uid)
if len(invalid_uid_list) != 0:
return 0, 'invalid user info', invalid_uid_list
#identify the uid is not exist in user_portrait and compute
#step1: filter in user_portrait
new_uid_list = []
have_in_uid_list = []
try:
exist_portrait_result = es.mget(index=portrait_index_name,
doc_type=portrait_index_type,
body={'ids': uid_list},
_source=False)['docs']
except:
exist_portrait_result = []
if exist_portrait_result:
for exist_item in exist_portrait_result:
if exist_item['found'] == False:
new_uid_list.append(exist_item['_id'])
else:
have_in_uid_list.append(exist_item['_id'])
else:
new_uid_list = uid_list
#step2: filter in compute
new_uid_set = set(new_uid_list)
compute_set = set(r.hkeys('compute'))
in_uid_set = list(new_uid_set - compute_set)
print 'new_uid_set:', new_uid_set
print 'in_uid_set:', in_uid_set
if len(in_uid_set) == 0:
return 0, 'all user in'
#identify the final add user
final_submit_user_list = []
for in_item in in_uid_set:
# if in_item in auto_recomment_set:
# tmp = json.loads(r.hget(hashname_submit, in_item))
# recommentor_list = tmp['operation'].split('&')
# recommentor_list.append(str(submit_user))
# new_list = list(set(recommentor_list))
# tmp['operation'] = '&'.join(new_list)
# else:
# tmp = {'system':'0', 'operation':submit_user}
if operation_type == 'submit':
relation_list = relation_string.split(',')
r.hset(
compute_hash_name, in_item,
json.dumps([
in_date, compute_status, node_type, relation_list,
submit_user, recommend_style
]))
# r.hset(hashname_submit, in_item, json.dumps(tmp))
# r.hset(submit_user_recomment, in_item, '0')
final_submit_user_list.append(in_item)
return 1, invalid_uid_list, have_in_uid_list, final_submit_user_list