def follow_tweet_func(xnr_user_no, account_name, password, uid, trace_type):
sina_operate_api = SinaOperateAPI(account_name, password)
mark = sina_operate_api.followed(uid=uid)
save_type = 'followers'
follow_type = 'follow'
save_to_fans_follow_ES(xnr_user_no, uid, save_type, follow_type,
trace_type)
return mark
def follow_tweet_func(xnr_user_no, account_name, password, uid, trace_type):
xnr = SinaLauncher(account_name, password)
xnr.login()
user = SinaOperateAPI(xnr.uid)
user.r_mid = uid
mark = user.followed()
save_type = 'followers'
follow_type = 'follow'
save_to_fans_follow_ES(xnr_user_no, uid, save_type, follow_type,
trace_type)
return mark
def executeES(indexName, typeName, listData):
#current_time = int(time.time())
#indexName += '_' + ts2datetime(current_time)
#print 'listData:',listData
for list_data in listData:
data = {}
jsonData = json.loads(list_data)
for key, val in jsonData.items():
# print key, '====', val
data[key] = val
# data['update_time'] = current_time
if indexName != 'weibo_feedback_group':
xnr_user_no = uid2xnr_user_no(data["root_uid"])
sensor_mark = judge_sensing_sensor(xnr_user_no, data['uid'])
data['sensor_mark'] = sensor_mark
trace_follow_mark = judge_trace_follow(xnr_user_no, data['uid'])
data['trace_follow_mark'] = trace_follow_mark
data['sensitive_info'] = get_sensitive_info(
data['timestamp'], data['mid'])
data['sensitive_user'] = get_sensitive_user(
data['timestamp'], data['uid'])
if indexName == 'weibo_feedback_follow':
# 修改 _id、保存至fans_followers_es表
_id = data["root_uid"] + '_' + data["mid"]
xnr_user_no = uid2xnr_user_no(data["root_uid"])
save_type = 'followers'
follow_type = 'follow'
if xnr_user_no:
save_to_fans_follow_ES(xnr_user_no, data["uid"], save_type,
follow_type)
save_to_redis_fans_follow(xnr_user_no, data["uid"],
save_type)
# sensor_mark = judge_sensing_sensor(xnr_user_no,data['uid'])
# data['sensor_mark'] = sensor_mark
# trace_follow_mark = judge_trace_follow(xnr_user_no,data['uid'])
# data['trace_follow_mark'] = trace_follow_mark
print 'save to es!!!!', es.index(index=indexName,
doc_type=typeName,
id=_id,
body=data)
elif indexName == 'weibo_feedback_fans':
_id = data["root_uid"] + '_' + data["mid"]
xnr_user_no = uid2xnr_user_no(data["root_uid"])
save_type = 'fans'
follow_type = 'follow'
if xnr_user_no:
save_to_fans_follow_ES(xnr_user_no, data["uid"], save_type,
follow_type)
save_to_redis_fans_follow(xnr_user_no, data["uid"],
save_type)
# sensor_mark = judge_sensing_sensor(xnr_user_no,data['uid'])
# data['sensor_mark'] = sensor_mark
# trace_follow_mark = judge_trace_follow(xnr_user_no,data['uid'])
# data['trace_follow_mark'] = trace_follow_mark
try:
es.get(index=indexName, doc_type=typeName, id=_id)
except:
print 'save to es!!!!', es.index(index=indexName,
doc_type=typeName,
id=_id,
body=data)
elif indexName == 'weibo_feedback_comment':
indexName_date = indexName + '_' + ts2datetime(
data['timestamp'])
date_time = ts2datetime(data['timestamp'])
# print 'date!!!!!!!',date_time
# print 'indexName_date:::',indexName_date
mappings_func = weibo_feedback_comment_mappings
_id = data["mid"]
# print 'comment_id........',_id
mappings_func(date_time)
# print 'data:::',data
print 'save to es!!!!', es.index(index=indexName_date,
doc_type=typeName,
id=_id,
body=data)
elif indexName == 'weibo_feedback_retweet':
# indexName += '_' + ts2datetime(data['timestamp'])
indexName_date = indexName + '_' + ts2datetime(
data['timestamp'])
date_time = ts2datetime(data['timestamp'])
mappings_func = weibo_feedback_retweet_mappings
_id = data["mid"]
mappings_func(date_time)
print 'save to es!!!!', es.index(index=indexName_date,
doc_type=typeName,
id=_id,
body=data)
elif indexName == 'weibo_feedback_at':
# indexName += '_' + ts2datetime(data['timestamp'])
indexName_date = indexName + '_' + ts2datetime(
data['timestamp'])
date_time = ts2datetime(data['timestamp'])
mappings_func = weibo_feedback_at_mappings
_id = data["mid"]
mappings_func(date_time)
print 'save to es!!!!', es.index(index=indexName_date,
doc_type=typeName,
id=_id,
body=data)
elif indexName == 'weibo_feedback_like':
# indexName += '_' + ts2datetime(data['timestamp'])
indexName_date = indexName + '_' + ts2datetime(
data['timestamp'])
date_time = ts2datetime(data['timestamp'])
mappings_func = weibo_feedback_like_mappings
_id = data["mid"]
mappings_func(date_time)
print 'save to es!!!!', es.index(index=indexName_date,
doc_type=typeName,
id=_id,
body=data)
elif indexName == 'weibo_feedback_private':
# indexName += '_' + ts2datetime(data['timestamp'])
indexName_date = indexName + '_' + ts2datetime(
data['timestamp'])
date_time = ts2datetime(data['timestamp'])
mappings_func = weibo_feedback_private_mappings
_id = data["mid"]
mappings_func(date_time)
print 'save to es!!!!', es.index(index=indexName_date,
doc_type=typeName,
id=_id,
body=data)
else:
_id = data["mid"]
print 'save to es!!!!', es.index(index=indexName,
doc_type=typeName,
id=_id,
body=data)
# print 'data.........',data
# print 'indexName....',indexName
# print '_id......',_id
# #print 'typeName.....',typeName
# print 'es...',es
# print 'save to es!!!!',es.index(index=indexName, doc_type=typeName, id=_id, body=data)
print 'update %s ES done' % indexName
