def request_data(url):
response = ""
proxies1 = get_proxies()
proxy_pool = cycle(proxies1)
time_out = BREAK_TIME
while True:
try:
ua = UserAgent()
user_agent = ua.random()
headers = {
'accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'accept-encoding': 'gzip, deflate, sdch, br',
'accept-language': 'en-GB,en;q=0.8,en-US;q=0.6,ml;q=0.4',
'cache-control': 'max-age=0',
'upgrade-insecure-requests': '1',
'user-agent': str(user_agent)
}
if PROXY_ENABLED:
proxy = next(proxy_pool)
logger.info("Using proxy: " + proxy)
response = requests.get(url,
headers=headers,
proxies={
"http": proxy,
"https": proxy
},
verify=False)
else:
response = requests.get(url, headers=headers, verify=False)
logger.info("Response code: " + str(response.status_code))
logger.info("Response content size: " + str(len(response.content)))
except:
logger.info("Skipipng proxy. Connection error")
continue
if response and len(response.content) < 10000: # capcha check
continue
else:
break
return response
import requests
import http.cookiejar as cookielib
from fake_useragent import UserAgent
shimoSession = requests.session()
shimoSession, cookies = cookielib.LWPCookieJar(filename='shimo.txt')
ua = UserAgent()
header = {
'Referer': 'https://shimo.im/login?from=home',
'User-Agent': ua.random()
}
def shimoLogin(account, password):
postUrl = 'https://shimo.im/lizard-api/auth/code/login'
postData = {
'passport': account,
'password': password,
}
responseRes = shimoSession.post(postUrl, data=postData, headers=header)
responseRes.cookies.save()
if __name__ == '__main__':
shimoSession.cookies.load()
shimoLogin('', '')
ip, port = self.get_proxy()
return ip, port
# 释放代理
def close_proxy(self, ip):
print('释放代理')
close_url = 'https://proxy.qg.net/release?Key=F58B5B03A518E080&IP=' + ip
requests.post(url=close_url)
# selenium添加代理
# 实例化ChromeOptions对象
option = webdriver.ChromeOptions()
# 设置UA
option.add_argument('--user-agent=%s' % UserAgent.random())
# 设置屏幕宽度
option.add_argument('--window-size=%s' % random.choice(xy_window))
# 添加代理
ip, port = get_proxy()
option.add_argument('--proxy-server=http://{}:{}'.format(ip, port))
# 以最高权限运行
option.add_argument('--no-sandbox')
# 防止window.navigator.webdriver检测
option.add_experimental_option('excludeSwitches', ['enable-automation'])
option.add_experimental_option('useAutomationExtension', False)
# 就是这一行告诉chrome去掉了webdriver痕迹
option.add_argument("disable-blink-features=AutomationControlled")
option.add_argument('lang=zh-CN,zh,zh-TW,en-US,en')
# 生成浏览器对象