def tw_save_to_fans_follow_ES(xnr_user_no,uid,follow_type,trace_type): try: results = es_fb_tw.get(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\ id=xnr_user_no) results = results["_source"] except: results = [] if follow_type == 'follow': if trace_type == 'trace_follow': try: if results: # 添加追随关注 try: trace_follow_uids = results['trace_follow_list'] trace_follow_uids_set = set(trace_follow_uids) trace_follow_uids_set.add(uid) trace_follow_uids = list(trace_follow_uids_set) except: trace_follow_uids = [uid] # # 添加普通关注 # try: # followers_uids = results['followers_list'] # followers_uids_set = set(followers_uids) # followers_uids_set.add(uid) # followers_uids = list(followers_uids_set) # except: # followers_uids = [uid] # results['followers_list'] = followers_uids results['trace_follow_list'] = trace_follow_uids es_fb_tw.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\ id=xnr_user_no,body={'doc':results}) else: results['trace_follow_list'] = [uid] results['followers_list'] = [uid] es_fb_tw.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\ body=results,id=xnr_user_no) except: return False elif follow_type == 'unfollow': try: followers_uids = results['trace_follow_pre_list'] followers_uids = list(set(followers_uids).difference(set([uid]))) results['trace_follow_pre_list'] = followers_uids es_fb_tw.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\ id=xnr_user_no,body={'doc':results}) except: return False return True
def get_un_trace_follow_operate(xnr_user_no,uid_string,nick_name_string): mark = False fail_nick_name_list = [] fail_uids = [] if uid_string: uid_list = uid_string.encode('utf-8').split(',') elif nick_name_string: nick_name_list = nick_name_string.encode('utf-8').split(',') uid_list = [] for nick_name in nick_name_list: query_body = { 'query':{ 'filtered':{ 'filter':{ 'term':{'nick_name':nick_name} } } }, '_source':['uid'] } try: uid_results = es.search(index=facebook_user_index_name,doc_type=facebook_user_index_type,\ body=query_body)['hits']['hits'] uid_result = uid_result[0]['_source'] uid = uid_result['uid'] uid_list.append(uid) except: fail_nick_name_list.append(nick_name) try: result = es.get(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\ id=xnr_user_no)['_source'] trace_follow_list = result['trace_follow_list'] # 共同uids comment_uids = list(set(trace_follow_list).intersection(set(uid_list))) # 取消失败uid fail_uids = list(set(comment_uids).difference(set(uid_list))) # 求差 trace_follow_list = list(set(trace_follow_list).difference(set(uid_list))) es.update(index=fb_xnr_fans_followers_index_name,doc_type=fb_xnr_fans_followers_index_type,\ id=xnr_user_no,body={'doc':{'trace_follow_list':trace_follow_list}}) mark = True except: mark = False return [mark,fail_uids,fail_nick_name_list]
def savedata2es(date, index_pre, index_type, data): config = { 'facebook_feedback_like_': ['uid', 'root_uid', 'timestamp', 'text', 'root_text', 'root_mid'], 'facebook_feedback_comment_': [ 'uid', 'root_uid', 'mid', 'timestamp', 'text', 'root_text', 'root_mid', 'comment_type' ], 'facebook_feedback_retweet_': [ 'uid', 'root_uid', 'mid', 'timestamp', 'text', 'root_text', 'root_mid' ], 'facebook_feedback_private_': ['uid', 'root_uid', 'timestamp', 'text', 'root_text', 'private_type'], 'facebook_feedback_friends': ['uid', 'root_uid'], 'facebook_feedback_at_': ['uid', 'root_uid', 'mid', 'timestamp', 'text'], } if index_pre in [ 'facebook_feedback_at_', 'facebook_feedback_comment_', 'facebook_feedback_retweet_', 'facebook_feedback_private_', 'facebook_feedback_like_' ]: index_name = index_pre + date search_index_name = index_pre + '*' else: index_name = index_pre search_index_name = index_name for d in data: query_body = { "query": { "filtered": { "filter": { "bool": { "must": [] } } } } } try: for field in config[index_pre]: query_body['query']['filtered']['filter']['bool'][ 'must'].append({'term': { field: d.get(field, '') }}) query_result = es.search(search_index_name, index_type, query_body)['hits']['hits'] if query_result: print es.update(index=index_name, doc_type=index_type, body={'doc': d}, id=query_result[0]['_id']) else: print es.index(index_name, index_type, d) except Exception, e: EXCEPTION += '\n savedata2es Exception: ' + str(e)
def update_facebook_xnr_relations(root_uid, uid, data, update_portrait_info=False): # pingtaiguanzhu 决定了是否要在平台上真正关注该用户,涉及到更改该关系时,一定要指定该字段(1或0)。 ''' :param root_uid: xnr_uid :param uid: user_uid :param data: relation data. eg: data={'gensuiguanzhu': 1, 'pingtaiguanzhu': 1} :param update_portrait_info: update or not :return: update success or not ''' xnr_user_no = fb_uid2xnr_user_no(root_uid) if xnr_user_no: data['platform'] = 'facebook' data['xnr_no'] = xnr_user_no data['xnr_uid'] = root_uid data['uid'] = uid ''' pingtaiguanzhu = data.get('pingtaiguanzhu', -1) pingtaiguanzhu_state = load_pingtaiguanzhu_state(root_uid, uid) if pingtaiguanzhu != pingtaiguanzhu_state: if pingtaiguanzhu == 1: 'gaunzhu' data['pingtaiguanzhu'] = 1 elif pingtaiguanzhu == 0: 'quxiao' data['pingtaiguanzhu'] = 0 ''' try: _id = '%s_%s' % (root_uid, uid) user_exist = es_xnr_2.exists( index=facebook_xnr_relations_index_name, doc_type=facebook_xnr_relations_index_type, id=_id) if user_exist: if update_portrait_info: protrait_info = update_facebook_user_portrait_info(uid) data.update(protrait_info) es_result = es_xnr_2.update( index=facebook_xnr_relations_index_name, doc_type=facebook_xnr_relations_index_type, id=_id, body={'doc': data}) else: protrait_info = update_facebook_user_portrait_info(uid) data.update(protrait_info) es_result = es_xnr_2.index( index=facebook_xnr_relations_index_name, doc_type=facebook_xnr_relations_index_type, id=_id, body=data) print es_result return True except Exception, e: print 'update_facebook_xnr_relations Error: ', str(e)
def publish_operate_timing(): query_body = { 'query':{ 'filtered':{ 'filter':{ 'term':{'task_status':0} } } }, 'size':MAX_SEARCH_SIZE } results = es_xnr.search(index=fb_xnr_timing_list_index_name,doc_type=\ fb_xnr_timing_list_index_type,body=query_body)['hits']['hits'] #print 'results::',results if results: for result in results: _id = result['_id'] result = result['_source'] timestamp_set = result['post_time'] print timestamp_set if timestamp_set <= int(time.time()): print '!!' text = result['text'].encode('utf-8') tweet_type = task_source_ch2en[result['task_source']] xnr_user_no = result['xnr_user_no'] # try: # p_url = result['p_url'] # except: # p_url = '' # try: # rank = result['rank'] # except: # rank = u'0' # try: # rankid = result['rankid'] # except: # rankid = '' #r_fid = result['fid'] es_get_result = es_xnr.get(index=fb_xnr_index_name,doc_type=fb_xnr_index_type,id=xnr_user_no)['_source'] fb_mail_account = es_get_result['fb_mail_account'] fb_phone_account = es_get_result['fb_phone_account'] password = es_get_result['password'] if fb_mail_account: account_name = fb_mail_account elif fb_phone_account: account_name = fb_phone_account else: return False mark = fb_publish(account_name, password, text, tweet_type, xnr_user_no) if mark: #task_id = xnr_user_no + '_' + r_fid task_id = _id # item_exist = es_xnr.get(index=fb_xnr_retweet_timing_list_index_name,doc_type=\ # fb_xnr_retweet_timing_list_index_type,id=task_id)['_source'] item_exist = {} item_exist['task_status'] = 1 #item_exist['timstamp_post'] = int(time.time()) es_xnr.update(index=fb_xnr_timing_list_index_name,doc_type=\ fb_xnr_timing_list_index_type,id=task_id,body={'doc':item_exist}) else: continue
def retweet_operate_timing(): query_body = { 'query':{ 'filtered':{ 'filter':{ 'term':{'compute_status':0} } } } } results = es_xnr.search(index=fb_xnr_retweet_timing_list_index_name,doc_type=\ fb_xnr_retweet_timing_list_index_type,body=query_body)['hits']['hits'] print 'results..',results if results: for result in results: result = result['_source'] timestamp_set = result['timestamp_set'] if timestamp_set <= int(time.time()): text = result['text'].encode('utf-8') tweet_type = 'trace_follow_tweet' xnr_user_no = result['xnr_user_no'] r_fid = result['fid'] uid = result['uid'] es_get_result = es_xnr.get(index=fb_xnr_index_name,doc_type=fb_xnr_index_type,id=xnr_user_no)['_source'] fb_mail_account = es_get_result['fb_mail_account'] fb_phone_account = es_get_result['fb_phone_account'] password = es_get_result['password'] if fb_mail_account: account_name = fb_mail_account elif fb_phone_account: account_name = fb_phone_account else: return False print 'text::',text mark = fb_retweet(account_name, password, r_fid, uid, text, tweet_type, xnr_user_no) print 'mark::',mark if mark: task_id = xnr_user_no + '_' + r_fid # item_exist = es_xnr.get(index=fb_xnr_retweet_timing_list_index_name,doc_type=\ # fb_xnr_retweet_timing_list_index_type,id=task_id)['_source'] item_exist = {} item_exist['compute_status'] = 1 #item_exist['timstamp_post'] = int(time.time()) es_xnr.update(index=fb_xnr_retweet_timing_list_index_name,doc_type=\ fb_xnr_retweet_timing_list_index_type,id=task_id,body={'doc':item_exist}) # # 保存微博 # try: # save_mark = save_to_xnr_flow_text(tweet_type,xnr_user_no,text) # except: # print '保存微博过程遇到错误!' # save_mark = False else: continue
def get_trace_follow_operate(xnr_user_no, uid_string, nick_name_string): mark = False fail_nick_name_list = [] if uid_string: uid_list = uid_string.encode('utf-8').split(',') elif nick_name_string: nick_name_list = nick_name_string.encode('utf-8').split(',') uid_list = [] for nick_name in nick_name_list: query_body = { 'query': { 'filtered': { 'filter': { 'term': { 'nick_name': nick_name } } } }, '_source': ['uid'] } try: uid_results = es.search(index=twitter_user_index_name,doc_type=twitter_user_index_type,\ body=query_body)['hits']['hits'] uid_result = uid_result[0]['_source'] uid = uid_result['uid'] uid_list.append(uid) except: fail_nick_name_list.append(nick_name) try: result = es.get(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\ id=xnr_user_no)['_source'] try: trace_follow_list = result['trace_follow_list'] except: trace_follow_list = [] try: followers_list = result['followers_list'] except: followers_list = [] trace_follow_list = list(set(trace_follow_list) | set(uid_list)) followers_list = list(set(followers_list) | set(uid_list)) es.update(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\ id=xnr_user_no,body={'doc':{'trace_follow_list':trace_follow_list,'followers_list':followers_list}}) mark = True except: item_exists = {} item_exists['xnr_user_no'] = xnr_user_no item_exists['trace_follow_list'] = uid_list item_exists['followers_list'] = uid_list es.index(index=tw_xnr_fans_followers_index_name,doc_type=tw_xnr_fans_followers_index_type,\ id=xnr_user_no,body=item_exists) mark = True return [mark, fail_nick_name_list]
def update_twitter_xnr_relations(root_uid, uid, data, update_portrait_info=False): # pingtaiguanzhu 决定了是否要在平台上真正关注该用户,涉及到更改该关系时,一定要指定该字段(1或0)。 ''' :param root_uid: xnr_uid :param uid: user_uid :param data: relation data. eg: data={'gensuiguanzhu': 1, 'pingtaiguanzhu': 1} :param update_portrait_info: update or not :return: update success or not ''' xnr_user_no = tw_uid2xnr_user_no(root_uid) if xnr_user_no: data['platform'] = 'twitter' data['xnr_no'] = xnr_user_no data['xnr_uid'] = root_uid data['uid'] = uid ''' pingtaiguanzhu = data.get('pingtaiguanzhu', -1) pingtaiguanzhu_state = load_pingtaiguanzhu_state(root_uid, uid) if pingtaiguanzhu != pingtaiguanzhu_state: if pingtaiguanzhu == 1: 'gaunzhu' data['pingtaiguanzhu'] = 1 elif pingtaiguanzhu == 0: 'quxiao' data['pingtaiguanzhu'] = 0 ''' # kn push jinrong cloud pingtaiguanzhu = data.get('pingtaiguanzhu', -1) pingtaiguanzhu_state = load_twitter_pingtaiguanzhu_state(root_uid, uid) tw_account, tw_password = load_twitter_user_passwd(root_uid) params_dict = {} params_dict['account'] = tw_account params_dict['password'] = tw_password params_dict['to_id'] = uid params_dict['xnr_user_no'] = xnr_user_no params_dict['root_uid'] = root_uid # 根据screen_name关注 if pingtaiguanzhu != pingtaiguanzhu_state: if pingtaiguanzhu == 1: # 'gaunzhu' params_dict['operate_type'] = 'follow' data['pingtaiguanzhu'] = 1 ali_re.lpush(twitter_relation_params, json.dumps(params_dict)) print params_dict print "push aliyun successful" elif pingtaiguanzhu == 0: # 'quxiao' params_dict['operate_type'] = 'unfollow' data['pingtaiguanzhu'] = 0 ali_re.lpush(twitter_relation_params, json.dumps(params_dict)) print "push aliyun successful" else: #params_dict['operate_type'] = 'follow' #data['pingtaiguanzhu'] = 1 #ali_re.lpush(twitter_relation_params, json.dumps(params_dict)) #print "push aliyun successful" pass try: _id = '%s_%s' % (root_uid, uid) user_exist = es_xnr_2.exists( index=twitter_xnr_relations_index_name, doc_type=twitter_xnr_relations_index_type, id=_id) if user_exist: if update_portrait_info: protrait_info = update_twitter_user_portrait_info(uid) data.update(protrait_info) es_result = es_xnr_2.update( index=twitter_xnr_relations_index_name, doc_type=twitter_xnr_relations_index_type, id=_id, body={'doc': data}) else: protrait_info = update_twitter_user_portrait_info(uid) data.update(protrait_info) es_result = es_xnr_2.index( index=twitter_xnr_relations_index_name, doc_type=twitter_xnr_relations_index_type, id=_id, body=data) return True except Exception, e: print 'update_twitter_xnr_relations Error: ', str(e)
def publish_operate_timing(): query_body = { 'query': { 'filtered': { 'filter': { 'term': { 'task_status': 0 } } } }, 'size': MAX_SEARCH_SIZE } results = es_xnr.search(index=tw_xnr_timing_list_index_name,doc_type=\ tw_xnr_timing_list_index_type,body=query_body)['hits']['hits'] print '----------------------------------------------------------------------' print 'results::', results print '----------------------------------------------------------------------' if results: for result in results: _id = result['_id'] result = result['_source'] timestamp_set = result['post_time'] #print timestamp_set if timestamp_set <= int(time.time()): print '!!' text = result['text'].encode('utf-8') # error #tweet_type = task_source_ch2en[result['task_source']] tweet_type = result['task_source'] print tweet_type xnr_user_no = result['xnr_user_no'] # try: # p_url = result['p_url'] # except: # p_url = '' # try: # rank = result['rank'] # except: # rank = u'0' # try: # rankid = result['rankid'] # except: # rankid = '' #r_tid = result['tid'] es_get_result = es_xnr.get(index=tw_xnr_index_name, doc_type=tw_xnr_index_type, id=xnr_user_no)['_source'] print '=---------------' print es_get_result print '=---------------' tw_mail_account = es_get_result['tw_mail_account'] tw_phone_account = es_get_result['tw_phone_account'] password = es_get_result['password'] if tw_mail_account: account_name = tw_mail_account print account_name elif tw_phone_account: account_name = tw_phone_account else: return False # kn 2019-6-27 push to aliyun try: task_detail = {} task_detail['channel'] = "twitter" task_detail['operate_type'] = "publish" task_detail['text'] = text task_detail['tweet_type'] = tweet_type task_detail['xnr_user_no'] = xnr_user_no mark = get_submit_tweet_tw(task_detail) except Exception as e: mark = 0 print e # 2k19-6-27 之前的注销 # mark = tw_publish(account_name, password, text, tweet_type, xnr_user_no) # 2019-6-27 之前的注销 if mark: #task_id = xnr_user_no + '_' + r_tid task_id = _id # item_exist = es_xnr.get(index=tw_xnr_retweet_timing_list_index_name,doc_type=\ # tw_xnr_retweet_timing_list_index_type,id=task_id)['_source'] item_exist = {} item_exist['task_status'] = 1 #item_exist['timstamp_post'] = int(time.time()) es_xnr.update(index=tw_xnr_timing_list_index_name,doc_type=\ tw_xnr_timing_list_index_type,id=task_id,body={'doc':item_exist}) else: continue
EXCEPTION += '\n friends_exist Exception: ' + str(e) try: friends_list = es.get(index=fb_xnr_fans_followers_index_name, doc_type=fb_xnr_fans_followers_index_type, id=xnr_user_no)['_source']['fans_list'] except: print es.index(fb_xnr_fans_followers_index_name, fb_xnr_fans_followers_index_type, body={'fans_list': []}, id=xnr_user_no) friends_list = [] if not new_friends_list == friends_list: print es.update(index=fb_xnr_fans_followers_index_name, doc_type=fb_xnr_fans_followers_index_type, body={'doc': { 'fans_list': new_friends_list }}, id=xnr_user_no) def main(): xnr_info_list = load_xnr_info() date = ts2datetime(time.time()) for xnr_info in xnr_info_list: try: comment(xnr_info, date) except Exception, e: pass try: like(xnr_info, date)
def publish_operate_timing(): query_body = { 'query':{ 'filtered':{ 'filter':{ 'term':{'task_status':0} } } }, 'size':MAX_SEARCH_SIZE } results = es_xnr.search(index=fb_xnr_timing_list_index_name,doc_type=\ fb_xnr_timing_list_index_type,body=query_body)['hits']['hits'] #print 'results::',results print 'results::',results if results: for result in results: print 'fb_retweet_in_time.publish_operate_timing' print result _id = result['_id'] result = result['_source'] xnr_user_no = result['xnr_user_no'] if xnr_user_no == 'null': print xnr_user_no continue timestamp_set = result['post_time'] print timestamp_set if timestamp_set <= int(time.time()): print 'timestamp, time.time()=====================================================================' print timestamp_set, time.time() print '!!' text = result['text'].encode('utf-8') #tweet_type = task_source_ch2en[result['task_source']] tweet_type = result['task_source'] print tweet_type # try: # p_url = result['p_url'] # except: # p_url = '' # try: # rank = result['rank'] # except: # rank = u'0' # try: # rankid = result['rankid'] # except: # rankid = '' #r_fid = result['fid'] print '==========================================', xnr_user_no es_get_result = es_xnr.get(index=fb_xnr_index_name,doc_type=fb_xnr_index_type,id=xnr_user_no)['_source'] fb_mail_account = es_get_result['fb_mail_account'] fb_phone_account = es_get_result['fb_phone_account'] password = es_get_result['password'] if fb_mail_account: account_name = fb_mail_account elif fb_phone_account: account_name = fb_phone_account else: return False # add to aliyun redis about some fb_publish need params kn #mark = fb_publish(account_name, password, text, tweet_type, xnr_user_no) if account_name: try: fb_tweet_params_dict = {} fb_tweet_params_dict["account_name"] = account_name fb_tweet_params_dict["password"] = password fb_tweet_params_dict["text"] = text fb_tweet_params_dict["tweet_type"] = tweet_type fb_tweet_params_dict["xnr_user_no"] = xnr_user_no ali_re.lpush(FB_TWEET_PARAMS, json.dumps(fb_tweet_params_dict)) mark = True except Exception as e: print e else: mark = False if mark: #task_id = xnr_user_no + '_' + r_fid task_id = _id # item_exist = es_xnr.get(index=fb_xnr_retweet_timing_list_index_name,doc_type=\ # fb_xnr_retweet_timing_list_index_type,id=task_id)['_source'] item_exist = {} item_exist['task_status'] = 1 #item_exist['timstamp_post'] = int(time.time()) es_xnr.update(index=fb_xnr_timing_list_index_name,doc_type=\ fb_xnr_timing_list_index_type,id=task_id,body={'doc':item_exist}) else: continue