def tw_save_to_fans_follow_ES(xnr_user_no,uid,follow_type,trace_type):
try:
results = es_fb_tw.get(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\
id=xnr_user_no)
results = results["_source"]
except:
results = []
if follow_type == 'follow':
if trace_type == 'trace_follow':
try:
if results:
# 添加追随关注
try:
trace_follow_uids = results['trace_follow_list']
trace_follow_uids_set = set(trace_follow_uids)
trace_follow_uids_set.add(uid)
trace_follow_uids = list(trace_follow_uids_set)
except:
trace_follow_uids = [uid]
# # 添加普通关注
# try:
# followers_uids = results['followers_list']
# followers_uids_set = set(followers_uids)
# followers_uids_set.add(uid)
# followers_uids = list(followers_uids_set)
# except:
# followers_uids = [uid]
# results['followers_list'] = followers_uids
results['trace_follow_list'] = trace_follow_uids
es_fb_tw.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':results})
else:
results['trace_follow_list'] = [uid]
results['followers_list'] = [uid]
es_fb_tw.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
body=results,id=xnr_user_no)
except:
return False
elif follow_type == 'unfollow':
try:
followers_uids = results['trace_follow_pre_list']
followers_uids = list(set(followers_uids).difference(set([uid])))
results['trace_follow_pre_list'] = followers_uids
es_fb_tw.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':results})
except:
return False
return True
def get_un_trace_follow_operate(xnr_user_no,uid_string,nick_name_string):
mark = False
fail_nick_name_list = []
fail_uids = []
if uid_string:
uid_list = uid_string.encode('utf-8').split('，')
elif nick_name_string:
nick_name_list = nick_name_string.encode('utf-8').split('，')
uid_list = []
for nick_name in nick_name_list:
query_body = {
'query':{
'filtered':{
'filter':{
'term':{'nick_name':nick_name}
}
}
},
'_source':['uid']
}
try:
uid_results = es.search(index=facebook_user_index_name,doc_type=facebook_user_index_type,\
body=query_body)['hits']['hits']
uid_result = uid_result[0]['_source']
uid = uid_result['uid']
uid_list.append(uid)
except:
fail_nick_name_list.append(nick_name)
try:
result = es.get(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\
id=xnr_user_no)['_source']
trace_follow_list = result['trace_follow_list']
# 共同uids
comment_uids = list(set(trace_follow_list).intersection(set(uid_list)))
# 取消失败uid
fail_uids = list(set(comment_uids).difference(set(uid_list)))
# 求差
trace_follow_list = list(set(trace_follow_list).difference(set(uid_list)))
es.update(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':{'trace_follow_list':trace_follow_list}})
mark = True
except:
mark = False
return [mark,fail_uids,fail_nick_name_list]
def savedata2es(date, index_pre, index_type, data):
config = {
'facebook_feedback_like_':
['uid', 'root_uid', 'timestamp', 'text', 'root_text', 'root_mid'],
'facebook_feedback_comment_': [
'uid', 'root_uid', 'mid', 'timestamp', 'text', 'root_text',
'root_mid', 'comment_type'
],
'facebook_feedback_retweet_': [
'uid', 'root_uid', 'mid', 'timestamp', 'text', 'root_text',
'root_mid'
],
'facebook_feedback_private_':
['uid', 'root_uid', 'timestamp', 'text', 'root_text', 'private_type'],
'facebook_feedback_friends': ['uid', 'root_uid'],
'facebook_feedback_at_':
['uid', 'root_uid', 'mid', 'timestamp', 'text'],
}
if index_pre in [
'facebook_feedback_at_', 'facebook_feedback_comment_',
'facebook_feedback_retweet_', 'facebook_feedback_private_',
'facebook_feedback_like_'
]:
index_name = index_pre + date
search_index_name = index_pre + '*'
else:
index_name = index_pre
search_index_name = index_name
for d in data:
query_body = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": []
}
}
}
}
}
try:
for field in config[index_pre]:
query_body['query']['filtered']['filter']['bool'][
'must'].append({'term': {
field: d.get(field, '')
}})
query_result = es.search(search_index_name, index_type,
query_body)['hits']['hits']
if query_result:
print es.update(index=index_name,
doc_type=index_type,
body={'doc': d},
id=query_result[0]['_id'])
else:
print es.index(index_name, index_type, d)
except Exception, e:
EXCEPTION += '\n savedata2es Exception: ' + str(e)
def update_facebook_xnr_relations(root_uid,
uid,
data,
update_portrait_info=False):
# pingtaiguanzhu 决定了是否要在平台上真正关注该用户，涉及到更改该关系时，一定要指定该字段（1或0）。
'''
:param root_uid: xnr_uid
:param uid: user_uid
:param data: relation data. eg: data={'gensuiguanzhu': 1, 'pingtaiguanzhu': 1}
:param update_portrait_info: update or not
:return: update success or not
'''
xnr_user_no = fb_uid2xnr_user_no(root_uid)
if xnr_user_no:
data['platform'] = 'facebook'
data['xnr_no'] = xnr_user_no
data['xnr_uid'] = root_uid
data['uid'] = uid
'''
pingtaiguanzhu = data.get('pingtaiguanzhu', -1)
pingtaiguanzhu_state = load_pingtaiguanzhu_state(root_uid, uid)
if pingtaiguanzhu != pingtaiguanzhu_state:
if pingtaiguanzhu == 1:
'gaunzhu'
data['pingtaiguanzhu'] = 1
elif pingtaiguanzhu == 0:
'quxiao'
data['pingtaiguanzhu'] = 0
'''
try:
_id = '%s_%s' % (root_uid, uid)
user_exist = es_xnr_2.exists(
index=facebook_xnr_relations_index_name,
doc_type=facebook_xnr_relations_index_type,
id=_id)
if user_exist:
if update_portrait_info:
protrait_info = update_facebook_user_portrait_info(uid)
data.update(protrait_info)
es_result = es_xnr_2.update(
index=facebook_xnr_relations_index_name,
doc_type=facebook_xnr_relations_index_type,
id=_id,
body={'doc': data})
else:
protrait_info = update_facebook_user_portrait_info(uid)
data.update(protrait_info)
es_result = es_xnr_2.index(
index=facebook_xnr_relations_index_name,
doc_type=facebook_xnr_relations_index_type,
id=_id,
body=data)
print es_result
return True
except Exception, e:
print 'update_facebook_xnr_relations Error: ', str(e)
def publish_operate_timing():
query_body = {
'query':{
'filtered':{
'filter':{
'term':{'task_status':0}
}
}
},
'size':MAX_SEARCH_SIZE
}
results = es_xnr.search(index=fb_xnr_timing_list_index_name,doc_type=\
fb_xnr_timing_list_index_type,body=query_body)['hits']['hits']
#print 'results::',results
if results:
for result in results:
_id = result['_id']
result = result['_source']
timestamp_set = result['post_time']
print timestamp_set
if timestamp_set <= int(time.time()):
print '!!'
text = result['text'].encode('utf-8')
tweet_type = task_source_ch2en[result['task_source']]
xnr_user_no = result['xnr_user_no']
# try:
# p_url = result['p_url']
# except:
# p_url = ''
# try:
# rank = result['rank']
# except:
# rank = u'0'
# try:
# rankid = result['rankid']
# except:
# rankid = ''
#r_fid = result['fid']
es_get_result = es_xnr.get(index=fb_xnr_index_name,doc_type=fb_xnr_index_type,id=xnr_user_no)['_source']
fb_mail_account = es_get_result['fb_mail_account']
fb_phone_account = es_get_result['fb_phone_account']
password = es_get_result['password']
if fb_mail_account:
account_name = fb_mail_account
elif fb_phone_account:
account_name = fb_phone_account
else:
return False
mark = fb_publish(account_name, password, text, tweet_type, xnr_user_no)
if mark:
#task_id = xnr_user_no + '_' + r_fid
task_id = _id
# item_exist = es_xnr.get(index=fb_xnr_retweet_timing_list_index_name,doc_type=\
# fb_xnr_retweet_timing_list_index_type,id=task_id)['_source']
item_exist = {}
item_exist['task_status'] = 1
#item_exist['timstamp_post'] = int(time.time())
es_xnr.update(index=fb_xnr_timing_list_index_name,doc_type=\
fb_xnr_timing_list_index_type,id=task_id,body={'doc':item_exist})
else:
continue
def retweet_operate_timing():
query_body = {
'query':{
'filtered':{
'filter':{
'term':{'compute_status':0}
}
}
}
}
results = es_xnr.search(index=fb_xnr_retweet_timing_list_index_name,doc_type=\
fb_xnr_retweet_timing_list_index_type,body=query_body)['hits']['hits']
print 'results..',results
if results:
for result in results:
result = result['_source']
timestamp_set = result['timestamp_set']
if timestamp_set <= int(time.time()):
text = result['text'].encode('utf-8')
tweet_type = 'trace_follow_tweet'
xnr_user_no = result['xnr_user_no']
r_fid = result['fid']
uid = result['uid']
es_get_result = es_xnr.get(index=fb_xnr_index_name,doc_type=fb_xnr_index_type,id=xnr_user_no)['_source']
fb_mail_account = es_get_result['fb_mail_account']
fb_phone_account = es_get_result['fb_phone_account']
password = es_get_result['password']
if fb_mail_account:
account_name = fb_mail_account
elif fb_phone_account:
account_name = fb_phone_account
else:
return False
print 'text::',text
mark = fb_retweet(account_name, password, r_fid, uid, text, tweet_type, xnr_user_no)
print 'mark::',mark
if mark:
task_id = xnr_user_no + '_' + r_fid
# item_exist = es_xnr.get(index=fb_xnr_retweet_timing_list_index_name,doc_type=\
# fb_xnr_retweet_timing_list_index_type,id=task_id)['_source']
item_exist = {}
item_exist['compute_status'] = 1
#item_exist['timstamp_post'] = int(time.time())
es_xnr.update(index=fb_xnr_retweet_timing_list_index_name,doc_type=\
fb_xnr_retweet_timing_list_index_type,id=task_id,body={'doc':item_exist})
# # 保存微博
# try:
# save_mark = save_to_xnr_flow_text(tweet_type,xnr_user_no,text)
# except:
# print '保存微博过程遇到错误！'
# save_mark = False
else:
continue
def get_trace_follow_operate(xnr_user_no, uid_string, nick_name_string):
mark = False
fail_nick_name_list = []
if uid_string:
uid_list = uid_string.encode('utf-8').split('，')
elif nick_name_string:
nick_name_list = nick_name_string.encode('utf-8').split('，')
uid_list = []
for nick_name in nick_name_list:
query_body = {
'query': {
'filtered': {
'filter': {
'term': {
'nick_name': nick_name
}
}
}
},
'_source': ['uid']
}
try:
uid_results = es.search(index=twitter_user_index_name,doc_type=twitter_user_index_type,\
body=query_body)['hits']['hits']
uid_result = uid_result[0]['_source']
uid = uid_result['uid']
uid_list.append(uid)
except:
fail_nick_name_list.append(nick_name)
try:
result = es.get(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no)['_source']
try:
trace_follow_list = result['trace_follow_list']
except:
trace_follow_list = []
try:
followers_list = result['followers_list']
except:
followers_list = []
trace_follow_list = list(set(trace_follow_list) | set(uid_list))
followers_list = list(set(followers_list) | set(uid_list))
es.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body={'doc':{'trace_follow_list':trace_follow_list,'followers_list':followers_list}})
mark = True
except:
item_exists = {}
item_exists['xnr_user_no'] = xnr_user_no
item_exists['trace_follow_list'] = uid_list
item_exists['followers_list'] = uid_list
es.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\
id=xnr_user_no,body=item_exists)
mark = True
return [mark, fail_nick_name_list]
def update_twitter_xnr_relations(root_uid,
uid,
data,
update_portrait_info=False):
# pingtaiguanzhu 决定了是否要在平台上真正关注该用户，涉及到更改该关系时，一定要指定该字段（1或0）。
'''
:param root_uid: xnr_uid
:param uid: user_uid
:param data: relation data. eg: data={'gensuiguanzhu': 1, 'pingtaiguanzhu': 1}
:param update_portrait_info: update or not
:return: update success or not
'''
xnr_user_no = tw_uid2xnr_user_no(root_uid)
if xnr_user_no:
data['platform'] = 'twitter'
data['xnr_no'] = xnr_user_no
data['xnr_uid'] = root_uid
data['uid'] = uid
'''
pingtaiguanzhu = data.get('pingtaiguanzhu', -1)
pingtaiguanzhu_state = load_pingtaiguanzhu_state(root_uid, uid)
if pingtaiguanzhu != pingtaiguanzhu_state:
if pingtaiguanzhu == 1:
'gaunzhu'
data['pingtaiguanzhu'] = 1
elif pingtaiguanzhu == 0:
'quxiao'
data['pingtaiguanzhu'] = 0
'''
# kn push jinrong cloud
pingtaiguanzhu = data.get('pingtaiguanzhu', -1)
pingtaiguanzhu_state = load_twitter_pingtaiguanzhu_state(root_uid, uid)
tw_account, tw_password = load_twitter_user_passwd(root_uid)
params_dict = {}
params_dict['account'] = tw_account
params_dict['password'] = tw_password
params_dict['to_id'] = uid
params_dict['xnr_user_no'] = xnr_user_no
params_dict['root_uid'] = root_uid
# 根据screen_name关注
if pingtaiguanzhu != pingtaiguanzhu_state:
if pingtaiguanzhu == 1:
# 'gaunzhu'
params_dict['operate_type'] = 'follow'
data['pingtaiguanzhu'] = 1
ali_re.lpush(twitter_relation_params, json.dumps(params_dict))
print params_dict
print "push aliyun successful"
elif pingtaiguanzhu == 0:
# 'quxiao'
params_dict['operate_type'] = 'unfollow'
data['pingtaiguanzhu'] = 0
ali_re.lpush(twitter_relation_params, json.dumps(params_dict))
print "push aliyun successful"
else:
#params_dict['operate_type'] = 'follow'
#data['pingtaiguanzhu'] = 1
#ali_re.lpush(twitter_relation_params, json.dumps(params_dict))
#print "push aliyun successful"
pass
try:
_id = '%s_%s' % (root_uid, uid)
user_exist = es_xnr_2.exists(
index=twitter_xnr_relations_index_name,
doc_type=twitter_xnr_relations_index_type,
id=_id)
if user_exist:
if update_portrait_info:
protrait_info = update_twitter_user_portrait_info(uid)
data.update(protrait_info)
es_result = es_xnr_2.update(
index=twitter_xnr_relations_index_name,
doc_type=twitter_xnr_relations_index_type,
id=_id,
body={'doc': data})
else:
protrait_info = update_twitter_user_portrait_info(uid)
data.update(protrait_info)
es_result = es_xnr_2.index(
index=twitter_xnr_relations_index_name,
doc_type=twitter_xnr_relations_index_type,
id=_id,
body=data)
return True
except Exception, e:
print 'update_twitter_xnr_relations Error: ', str(e)
def publish_operate_timing():
query_body = {
'query': {
'filtered': {
'filter': {
'term': {
'task_status': 0
}
}
}
},
'size': MAX_SEARCH_SIZE
}
results = es_xnr.search(index=tw_xnr_timing_list_index_name,doc_type=\
tw_xnr_timing_list_index_type,body=query_body)['hits']['hits']
print '----------------------------------------------------------------------'
print 'results::', results
print '----------------------------------------------------------------------'
if results:
for result in results:
_id = result['_id']
result = result['_source']
timestamp_set = result['post_time']
#print timestamp_set
if timestamp_set <= int(time.time()):
print '!!'
text = result['text'].encode('utf-8')
# error
#tweet_type = task_source_ch2en[result['task_source']]
tweet_type = result['task_source']
print tweet_type
xnr_user_no = result['xnr_user_no']
# try:
# p_url = result['p_url']
# except:
# p_url = ''
# try:
# rank = result['rank']
# except:
# rank = u'0'
# try:
# rankid = result['rankid']
# except:
# rankid = ''
#r_tid = result['tid']
es_get_result = es_xnr.get(index=tw_xnr_index_name,
doc_type=tw_xnr_index_type,
id=xnr_user_no)['_source']
print '=---------------'
print es_get_result
print '=---------------'
tw_mail_account = es_get_result['tw_mail_account']
tw_phone_account = es_get_result['tw_phone_account']
password = es_get_result['password']
if tw_mail_account:
account_name = tw_mail_account
print account_name
elif tw_phone_account:
account_name = tw_phone_account
else:
return False
# kn 2019-6-27 push to aliyun
try:
task_detail = {}
task_detail['channel'] = "twitter"
task_detail['operate_type'] = "publish"
task_detail['text'] = text
task_detail['tweet_type'] = tweet_type
task_detail['xnr_user_no'] = xnr_user_no
mark = get_submit_tweet_tw(task_detail)
except Exception as e:
mark = 0
print e
# 2k19-6-27 之前的注销
# mark = tw_publish(account_name, password, text, tweet_type, xnr_user_no)
# 2019-6-27 之前的注销
if mark:
#task_id = xnr_user_no + '_' + r_tid
task_id = _id
# item_exist = es_xnr.get(index=tw_xnr_retweet_timing_list_index_name,doc_type=\
# tw_xnr_retweet_timing_list_index_type,id=task_id)['_source']
item_exist = {}
item_exist['task_status'] = 1
#item_exist['timstamp_post'] = int(time.time())
es_xnr.update(index=tw_xnr_timing_list_index_name,doc_type=\
tw_xnr_timing_list_index_type,id=task_id,body={'doc':item_exist})
else:
continue
EXCEPTION += '\n friends_exist Exception: ' + str(e)
try:
friends_list = es.get(index=fb_xnr_fans_followers_index_name,
doc_type=fb_xnr_fans_followers_index_type,
id=xnr_user_no)['_source']['fans_list']
except:
print es.index(fb_xnr_fans_followers_index_name,
fb_xnr_fans_followers_index_type,
body={'fans_list': []},
id=xnr_user_no)
friends_list = []
if not new_friends_list == friends_list:
print es.update(index=fb_xnr_fans_followers_index_name,
doc_type=fb_xnr_fans_followers_index_type,
body={'doc': {
'fans_list': new_friends_list
}},
id=xnr_user_no)
def main():
xnr_info_list = load_xnr_info()
date = ts2datetime(time.time())
for xnr_info in xnr_info_list:
try:
comment(xnr_info, date)
except Exception, e:
pass
try:
like(xnr_info, date)
def publish_operate_timing():
query_body = {
'query':{
'filtered':{
'filter':{
'term':{'task_status':0}
}
}
},
'size':MAX_SEARCH_SIZE
}
results = es_xnr.search(index=fb_xnr_timing_list_index_name,doc_type=\
fb_xnr_timing_list_index_type,body=query_body)['hits']['hits']
#print 'results::',results
print 'results::',results
if results:
for result in results:
print 'fb_retweet_in_time.publish_operate_timing'
print result
_id = result['_id']
result = result['_source']
xnr_user_no = result['xnr_user_no']
if xnr_user_no == 'null':
print xnr_user_no
continue
timestamp_set = result['post_time']
print timestamp_set
if timestamp_set <= int(time.time()):
print 'timestamp, time.time()====================================================================='
print timestamp_set, time.time()
print '!!'
text = result['text'].encode('utf-8')
#tweet_type = task_source_ch2en[result['task_source']]
tweet_type = result['task_source']
print tweet_type
# try:
# p_url = result['p_url']
# except:
# p_url = ''
# try:
# rank = result['rank']
# except:
# rank = u'0'
# try:
# rankid = result['rankid']
# except:
# rankid = ''
#r_fid = result['fid']
print '==========================================', xnr_user_no
es_get_result = es_xnr.get(index=fb_xnr_index_name,doc_type=fb_xnr_index_type,id=xnr_user_no)['_source']
fb_mail_account = es_get_result['fb_mail_account']
fb_phone_account = es_get_result['fb_phone_account']
password = es_get_result['password']
if fb_mail_account:
account_name = fb_mail_account
elif fb_phone_account:
account_name = fb_phone_account
else:
return False
# add to aliyun redis about some fb_publish need params kn
#mark = fb_publish(account_name, password, text, tweet_type, xnr_user_no)
if account_name:
try:
fb_tweet_params_dict = {}
fb_tweet_params_dict["account_name"] = account_name
fb_tweet_params_dict["password"] = password
fb_tweet_params_dict["text"] = text
fb_tweet_params_dict["tweet_type"] = tweet_type
fb_tweet_params_dict["xnr_user_no"] = xnr_user_no
ali_re.lpush(FB_TWEET_PARAMS, json.dumps(fb_tweet_params_dict))
mark = True
except Exception as e:
print e
else:
mark = False
if mark:
#task_id = xnr_user_no + '_' + r_fid
task_id = _id
# item_exist = es_xnr.get(index=fb_xnr_retweet_timing_list_index_name,doc_type=\
# fb_xnr_retweet_timing_list_index_type,id=task_id)['_source']
item_exist = {}
item_exist['task_status'] = 1
#item_exist['timstamp_post'] = int(time.time())
es_xnr.update(index=fb_xnr_timing_list_index_name,doc_type=\
fb_xnr_timing_list_index_type,id=task_id,body={'doc':item_exist})
else:
continue
