def __open_browser(use_proxy: bool = False):
# TODO: add user agent
chrome_options = webdriver.ChromeOptions()
capabilities = webdriver.DesiredCapabilities.CHROME
if use_proxy:
random_proxy = Proxies.get_random_proxy()
# Parse Proxy
if '@' in random_proxy:
auth, ip_port = random_proxy.split('@')
user, pwd = auth.split(':')
ip, port = ip_port.split(':')
with zipfile.ZipFile(plugin_file, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js",
background_js % (ip, port, user, pwd))
chrome_options.add_extension(plugin_file)
else:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = random_proxy
prox.socks_proxy = random_proxy
prox.ssl_proxy = random_proxy
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
return webdriver.Chrome(chrome_options=chrome_options)
def prepare_desired_capabilities(self):
capabilities = DesiredCapabilities.FIREFOX.copy()
capabilities['javascriptEnabled'] = True
# capabilities['pageLoadStrategy'] = 'normal'
# Set proxy
proxy_string = configs.RANDOM_PROXY(return_tuple=False)
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = proxy_string
proxy.ssl_proxy = proxy_string
# proxy.ftp_proxy = proxy_string
# prox.socks_proxy = proxy_string
# proxy.add_to_capabilities(capabilities)
return capabilities
def get_tuned_driver(parser_name: str,
logger: 'Logger',
proxy_ip: Optional[str] = None,
proxy_port: Optional[str] = None,
headless: bool = True) -> 'WebDriver':
os.environ["DISPLAY"] = ':99'
chrome_options = Options()
capabilities = DesiredCapabilities.CHROME
capabilities['goog:loggingPrefs'] = {'browser': 'ALL'}
if proxy_ip and proxy_port:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = f"{proxy_ip}:{proxy_port}"
prox.ssl_proxy = f"{proxy_ip}:{proxy_port}"
try:
response = requests.get('https://google.com',
proxies={
'http': f'{proxy_ip}:{proxy_port}',
'https': f'{proxy_ip}:{proxy_port}',
})
except requests.RequestException:
update_proxy_status(proxy_ip, AccessStatus.fail)
raise
if response.status_code != 200:
update_proxy_status(proxy_ip, AccessStatus.fail)
logger.critical(f'proxy {proxy_ip}:{proxy_port} not work')
exit(-1)
update_proxy_status(proxy_ip, AccessStatus.success)
prox.add_to_capabilities(capabilities)
logger.info(f'{parser_name} use proxy: {proxy_ip}:{proxy_port}')
if headless:
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--headless")
chrome_options.add_argument("--remote-debugging-port=9222")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
driver = webdriver.Chrome(options=chrome_options,
desired_capabilities=capabilities)
else:
driver = webdriver.Chrome(options=chrome_options,
desired_capabilities=capabilities)
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option('prefs', prefs)
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_experimental_option('excludeSwitches',
['enable-automation'])
chrome_options.add_argument('start-maximized')
chrome_options.add_argument('incognito')
driver.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument", {
"source":
"""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
enumerable: false,
configurable: true
});
const newProto = navigator.__proto__;
delete newProto.webdriver;
navigator.__proto__ = newProto;
delete navigator.webdriver;
"""
})
driver.execute_cdp_cmd(
'Network.setUserAgentOverride', {
"userAgent":
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/83.0.4103.53 Safari/537.36'
})
driver.implicitly_wait(5)
return driver
from pathlib import Path
from selenium import webdriver
from selenium.webdriver import FirefoxProfile, Proxy
import os, time
from selenium.webdriver.common.proxy import ProxyType
from selenium.webdriver.firefox.options import Options
p = Proxy()
p.proxy_type = ProxyType.MANUAL
p.httpProxy = "1.1.1.1:8080"
x = Options()
x.accept_insecure_certs = True
# x.headless = True
# x.proxy = p
# x.accept_insecure_certs = True
# x.set_preference("browser.download.defaultFolder", str(Path(os.getcwd()).parent) + os.path.sep + "AutomationDownloads")
# myProxy = "86.111.144.194:3128"
# proxy = Proxy({
# 'proxyType': ProxyType.MANUAL,
# 'httpProxy': myProxy,
# 'ftpProxy': myProxy,
# 'sslProxy': myProxy,
# 'noProxy':''})