def process_request(self, request, spider):
try:
if 'splash' in request.meta:
return
request.meta[
'proxy'] = "http://%s" % proxy_pool.random_choice_proxy(False)
except Exception as e:
log.error(e)
def process_request(self, request, spider):
try:
if 'splash' not in request.meta:
return
request.meta['splash']['args'][
'proxy'] = "http://%s" % proxy_pool.random_choice_proxy(False)
print("using proxy:" + request.meta['splash']['args']['proxy'])
except Exception as e:
log.error(e)
def phantomjs_opened(self):
capabilities = DesiredCapabilities.PHANTOMJS.copy()
proxy = proxy_pool.random_choice_proxy()
capabilities['proxy'] = {
'proxyType': 'MANUAL',
'ftpProxy': proxy,
'sslProxy': proxy,
'httpProxy': proxy,
'noProxy': None
}
# capabilities['phantomjs.cli.args'] = [
# '--proxy-auth=' + evar.get('WONDERPROXY_USER') + ':' + evar.get('WONDERPROXY_PASS')
# ]
driver = webdriver.PhantomJS(desired_capabilities=capabilities)
driver.set_page_load_timeout(120)
return driver
def download_img(img_url, file_path):
proxies = None
proxy = ''
if config.USE_PROXY:
proxy = proxy_pool.random_choice_proxy()
proxies = {
'http': "http://%s" % proxy,
}
try:
response = requests.get(img_url, stream=True, proxies=proxies)
if response.status_code == 200:
with open(file_path, 'wb') as f:
for chunk in response.iter_content(1024):
f.write(chunk)
else:
if config.USE_PROXY:
proxy_pool.add_failed_time(proxy)
except:
if config.USE_PROXY:
proxy_pool.add_failed_time(proxy)
def process_request(self, request, spider):
try:
request.meta[
'proxy'] = "http://%s" % proxy_pool.random_choice_proxy()
except Exception as e:
log.error(e)