def request_data(url): response = "" proxies1 = get_proxies() proxy_pool = cycle(proxies1) time_out = BREAK_TIME while True: try: ua = UserAgent() user_agent = ua.random() headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'accept-encoding': 'gzip, deflate, sdch, br', 'accept-language': 'en-GB,en;q=0.8,en-US;q=0.6,ml;q=0.4', 'cache-control': 'max-age=0', 'upgrade-insecure-requests': '1', 'user-agent': str(user_agent) } if PROXY_ENABLED: proxy = next(proxy_pool) logger.info("Using proxy: " + proxy) response = requests.get(url, headers=headers, proxies={ "http": proxy, "https": proxy }, verify=False) else: response = requests.get(url, headers=headers, verify=False) logger.info("Response code: " + str(response.status_code)) logger.info("Response content size: " + str(len(response.content))) except: logger.info("Skipipng proxy. Connection error") continue if response and len(response.content) < 10000: # capcha check continue else: break return response
import requests import http.cookiejar as cookielib from fake_useragent import UserAgent shimoSession = requests.session() shimoSession, cookies = cookielib.LWPCookieJar(filename='shimo.txt') ua = UserAgent() header = { 'Referer': 'https://shimo.im/login?from=home', 'User-Agent': ua.random() } def shimoLogin(account, password): postUrl = 'https://shimo.im/lizard-api/auth/code/login' postData = { 'passport': account, 'password': password, } responseRes = shimoSession.post(postUrl, data=postData, headers=header) responseRes.cookies.save() if __name__ == '__main__': shimoSession.cookies.load() shimoLogin('', '')
ip, port = self.get_proxy() return ip, port # 释放代理 def close_proxy(self, ip): print('释放代理') close_url = 'https://proxy.qg.net/release?Key=F58B5B03A518E080&IP=' + ip requests.post(url=close_url) # selenium添加代理 # 实例化ChromeOptions对象 option = webdriver.ChromeOptions() # 设置UA option.add_argument('--user-agent=%s' % UserAgent.random()) # 设置屏幕宽度 option.add_argument('--window-size=%s' % random.choice(xy_window)) # 添加代理 ip, port = get_proxy() option.add_argument('--proxy-server=http://{}:{}'.format(ip, port)) # 以最高权限运行 option.add_argument('--no-sandbox') # 防止window.navigator.webdriver检测 option.add_experimental_option('excludeSwitches', ['enable-automation']) option.add_experimental_option('useAutomationExtension', False) # 就是这一行告诉chrome去掉了webdriver痕迹 option.add_argument("disable-blink-features=AutomationControlled") option.add_argument('lang=zh-CN,zh,zh-TW,en-US,en') # 生成浏览器对象