def response(self, flow: http.HTTPFlow):
print("SearchInterceptor matched------------------------------")
form_data = flow.request.urlencoded_form
keyword = {'keyword': form_data['keyword']}
key_info = db.find_insert_keyword(keyword)
keyword_id = key_info['keyword_id']
if get_cur_keyword_id() == -1:
set_cur_keyword_id(keyword_id)
data = json.loads(flow.response.text)['data']
for v in data:
if v['type'] == 1:
aweme_info = v['aweme_info']
# print("----------------------nickname:%s,short_id:%s" %(aweme_info['author']['nickname'],aweme_info['author']['short_id']))
if aweme_info['author'].get('short_id'):
#-----------handle user-----------------
user_info = pack_user(aweme_info['author'])
# user_info['keyword_id'] = keyword_id
db.save_user(user_info, keyword_id)
#----------handle video-----------------
video = pack_video(aweme_info)
video['uid'] = user_info['uid']
db.save_video(video)
def response(self, flow: http.HTTPFlow):
print("UserInfoInterceptor matched------------------------------")
keyword_id = self.get_cur_keyword_id()
if keyword_id > 0:
user = json.loads(flow.response.text)['user']
if user['short_id']:
user_info = pack_user(user)
db.save_user(user_info, keyword_id)
def response(self, flow):
print("CommentsInterceptor matched------------------------------")
keyword_id = self.get_cur_keyword_id()
if keyword_id > 0:
for comment in json.loads(flow.response.text)['comments']:
if comment['user'].get('short_id'):
user_info = pack_user(comment['user'])
db.save_user(user_info, keyword_id)
cmts = pack_comment_1(comment)
db.save_comments(cmts)
def response(self, flow: http.HTTPFlow):
print("FansInterceptor matched------------------------------")
# print("-------------------------------\n" + str(flow.request.query) + "\n----------------------------")
# print("------------uid:" + str(flow.request.query['user_id']))
keyword_id = self.get_cur_keyword_id()
if keyword_id > 0:
user_id = flow.request.query['user_id']
for user in json.loads(flow.response.text)['followers']:
if user.get('short_id'):
user_info = pack_user(user)
print("*********************************************")
print(user_info)
print("*********************************************")
db.save_user(user_info, keyword_id)
fans = {}
fans['uid'] = user_id
fans['fid'] = user['uid']
db.save_fans(fans)