def start_app(self, short_id):
self.driver = webdriver.Remote('http://localhost:4723/wd/hub',
self.param)
self.go_search_page(short_id)
while True:
conn_db = Handle_DB()
short_id = conn_db.get_short_id()
print(short_id)
self.search_user(short_id)
def scheduler_parse_page(self):
conn_db = Handle_DB()
while True:
# 从数据库中获取分享ID
share_id = conn_db.get_share_id()
# Parse_Share_page(share_id).get_proxy()
# print(share_id)
# 如果取到数据，则解析
if share_id:
parse_data = Parse_Share_page()
personal_info = parse_data.parsed_data_insert_db(
conn_db, share_id)
else:
time.sleep(1)
def response(flow):
'''
这个函数是mitmdump的固定写法 启动方法 ：mitmdump -s py_path -p 8889
:param flow:抓取到的数据流
:return:
'''
mongod = Handle_DB()
if 'follower/list' in flow.request.url:
print(flow.response.text)
data = json.loads(flow.response.text)
followers = data['followers']
# 把获取的粉丝信息逐个存到数据库
for user in followers:
short_id = {}
if len(user['uid']) > 1:
short_id['share_id'] = user['uid']
mongod.insert_mitmdump_data(short_id)
print(short_id)
"com.ss.android.ugc.aweme:id/apx")):
self.driver.find_element_by_id(
"com.ss.android.ugc.aweme:id/apx").click()
if WebDriverWait(self.driver,
10).until(lambda x: x.find_element_by_id(
"com.ss.android.ugc.aweme:id/afo")):
self.driver.find_element_by_id(
"com.ss.android.ugc.aweme:id/afo").click()
self.search_user(short_id=short_id)
except:
print('没找到搜索按钮')
def start_app(self, short_id):
self.driver = webdriver.Remote('http://localhost:4723/wd/hub',
self.param)
self.go_search_page(short_id)
while True:
conn_db = Handle_DB()
short_id = conn_db.get_short_id()
print(short_id)
self.search_user(short_id)
if __name__ == '__main__':
is_first_run = True
conn_db = Handle_DB()
short_id = conn_db.get_short_id()
print(short_id)
Handle_App().start_app(short_id=short_id)
def scheduler_app(self):
is_first_run = True
conn_db = Handle_DB()
short_id = conn_db.get_short_id()
print(short_id)
Handle_App().start_app(short_id=short_id)
def __init__(self):
self.conn_db = Handle_DB()
# 定义标志位 如果多次没有拿到数据协程结束
self.flag_parse_timeout = 0
self.parse_data = Parse_Share_page()
print('这条用户信息已解析')
pass
else:
# print('解析用户信息')
# 解析分享页面
parse_data = Parse_Share_page()
personal_info = parse_data.start_request(share_id)
# 把解析到的个人数据存入到数据库
# print('personal_info', personal_info)
conn_db.insert_personal_info(personal_info)
conn_db.insert_shortid_to_mongo(personal_info['shortid'])
conn_db.insert_shareid_to_redis(personal_info['share_id'])
time.sleep(0.5)
if __name__ == '__main__':
conn_db = Handle_DB()
# while True:
# 从数据库中获取分享ID
share_id = conn_db.get_share_id()
# Parse_Share_page(share_id).get_proxy()
print(share_id)
# 如果取到数据，则解析
if share_id:
parse_data = Parse_Share_page()
personal_info = parse_data.parsed_data_insert_db(conn_db, share_id)
else:
print('没有分享ID')
time.sleep(1)