def __init__(self,
incognito=True,
headless=False,
disable_gpu=False) -> None:
options = ChromeOptions()
options.add_argument("disable-extensions")
if incognito:
options.add_argument("incognito")
if headless:
options.add_argument("headless")
if disable_gpu:
options.add_argument("disable-gpu")
if os.name == 'nt':
path_to_chrome = str(
Path('./ChromeDrivers/Windows/chromedriver.exe').absolute())
options.binary_location = str(
Path(
'/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe'
))
elif os.name == 'darwin':
path_to_chrome = str(
Path('./ChromeDrivers/Mac/chromedriver').absolute())
elif os.name == 'posix':
options.binary_location = "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"
path_to_chrome = str(
Path('./ChromeDrivers/Linux/chromedriver').absolute())
else:
raise UnrecognizedOSError('Unable to recogized Operating System')
self.browser = Chrome(path_to_chrome, options=options)
def create_webdriver(self, *args, **kwargs):
import selenium
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
selenium_version = tuple(map(int, selenium.__version__.split('.')))
browser, _, url = self.browser.partition('+')
if url:
kwargs.update({
'desired_capabilities':
getattr(DesiredCapabilities, browser.upper()),
'command_executor':
url,
})
browser = 'remote'
elif browser == 'chrome-headless':
from selenium.webdriver import ChromeOptions
browser = 'chrome'
options = ChromeOptions()
options.add_argument('headless')
options.add_argument('disable-gui')
options.add_argument('no-sandbox')
if os.environ.get('CHROME_BIN'):
options.binary_location = os.environ['CHROME_BIN']
kwargs['options'] = options
webdriver_cls = self.import_webdriver(browser)
if webdriver_cls:
return webdriver_cls(*args, **kwargs)
def setup_chrome_opt():
"""
Sets up options for Chrome webdriver. Based on defined `constants <constants.html>`_ it chooses where
to save data and where Chrome binary and Chrome driver can be found.
Sets following options:
* *download.default_directory: constants.saveDir*
* *download.prompt_for_download: False*
* *download.directory_upgrade: True*
* *safebrowsing.enaled: True*
:returns: Set Chrome webdriver object
"""
chrome_options = ChromeOptions()
chrome_options.binary_location = cs.GOOGLE_CHROME_BIN
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--no-sandbox')
prefs = {
'download.default_directory': cs.saveDir,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True
}
chrome_options.add_experimental_option('prefs', prefs)
#chrome_options.add_argument('headless')
driver = webdriver.Chrome(executable_path=cs.CHROMEDRIVER_PATH,
chrome_options=chrome_options)
return driver
def __init__(self) -> None:
logging.debug("driver.create/start")
options = ChromeOptions()
binary_location = os.environ.get("CHROME_BINARY_LOCATION", None)
if not (binary_location is None):
options.binary_location = binary_location
# 必須
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
options.add_argument("--single-process")
# options.add_argument("--disable-setuid-sandbox")
# options.add_argument("--disable-dev-shm-usage")
# エラーの許容
options.add_argument("--ignore-certificate-errors")
options.add_argument("--allow-running-insecure-content")
options.add_argument("--disable-web-security")
# headlessでは不要そうな機能
options.add_argument("--disable-desktop-notifications")
options.add_argument("--disable-extensions")
# UA
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" # noqa
options.add_argument("--user-agent=" + user_agent)
# 言語
options.add_argument("--lang=ja")
# 画像を読み込まないで軽くする
options.add_argument("--blink-settings=imagesEnabled=false")
# chromedriver生成
executable_path = os.environ.get("CHROME_DRIVER_LOCATION", None)
self.driver = Chrome(options=options, executable_path=executable_path)
logging.debug("driver.create/end")
def initialize(self):
proxylist = self.getproxy()
ip = proxylist[0]
port = proxylist[1]
proxy_ip_code = str(ip) + ':' + str(port)
proxy_ip = {"httpProxy": ""}
proxy_ip['httpProxy'] = proxy_ip_code
self.proxy = server.create_proxy(proxy_ip)
self.UA_code = self.ua_list()
self.UA = '--user-agent=' + self.UA_code
self.un_cookie = self.get_cookie(ip, port)
print(self.un_cookie)
url = self.get_txurl(ip, port)
option = ChromeOptions()
option.binary_location = r"C:\Users\Vilker\AppData\Local\Google\Chrome\Application\chrome.exe"
option.add_argument('--disable-infobars')
option.add_argument(self.UA)
#option.add_argument('--headless') # 无界面浏览开启
option.add_argument('--proxy-server={0}'.format(self.proxy.proxy))
self.proxy.new_har("vilker", options={'captureContent': True})
self.driver = Chrome(
options=option,
executable_path=
r'C:\Users\Vilker\AppData\Local\Programs\Python\chromedriver.exe')
self.driver.set_window_size(400, 600)
self.driver.get(url=url)
self.tx_main()
def __init__(self, incognito=True, headless=False, brave=False):
options = ChromeOptions()
options.add_argument("disable-extensions")
if incognito:
options.add_argument("incognito")
if headless:
options.add_argument("headless")
# options.add_argument("disable-gpu")
# options.add_argument('window-size=1200x1200')
# options.add_argument("remote-debugging-port=9222")
# options.add_argument("kiosk")
if os.name == 'nt':
# path_to_chrome = str(Path('./chromedriver.exe').relative_to('.'))
path_to_chrome = str(
Path('./ChromeDrivers/Windows/chromedriver.exe').absolute())
elif os.name == 'posix':
path_to_chrome = str(
Path('./ChromeDrivers/Mac/chromedriver').absolute())
else:
if brave:
options.binary_location = "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"
path_to_chrome = str(
Path('./ChromeDrivers/Linux/chromedriver').absolute())
self.browser = Chrome(path_to_chrome, options=options)
def get_distance():
URL = "https://voyager.jpl.nasa.gov/mission/status/"
chrome_bin = os.environ.get('GOOGLE_CHROME_SHIM', None)
opts = ChromeOptions()
opts.binary_location = chrome_bin
opts.add_argument('headless')
opts.add_argument('--disable-gpu')
opts.add_argument('--no-sandbox')
# Start the WebDriver and load the page
wd = Chrome(executable_path="chromedriver", chrome_options=opts)
wd.get(URL)
# Wait for the dynamically loaded elements to show up
WebDriverWait(wd, 10).until(
EC.visibility_of_element_located((By.ID, "voy1_km")))
# And grab the page HTML source
html_page = wd.page_source
wd.quit()
# parse html to get distance
soup = BeautifulSoup(html_page, "html.parser")
distance = soup.find(id="voy1_km").get_text()
sanitized = distance.split()[0]
no_commas = sanitized.replace(",", "")
return no_commas
def _process_chrome_options(self, opts):
if isinstance(opts, ChromeOptions):
options = opts
else:
options = ChromeOptions()
if 'args' in opts:
for arg in opts.pop('args'):
options.add_argument(arg)
if opts.pop('headless', None) or self.options.pop(
'headless', None):
options.headless = True
if 'binary' in opts or 'binary_location' in opts:
options.binary_location = opts.pop('binary') or opts.pop(
'binary_location')
if 'debugger_address' in opts:
options.debugger_address = opts.pop('debugger_address')
if 'extensions' in opts:
for path in opts.pop('extensions'):
options.add_extension(path)
if 'encoded_extensions' in opts:
for string in opts.pop('encoded_extensions'):
options.add_encoded_extension(string)
if 'experimental_options' in opts:
for name, value in opts.pop('experimental_options').items():
options.add_experimental_option(name, value)
self.selenium_opts['options'] = options
def get_driver(username, password):
options = ChromeOptions()
options.binary_location = GOOGLE_CHROME_PATH
options.add_argument('--headless')
options.add_argument("start-maximized") # open Browser in maximized mode
options.add_argument("disable-infobars") # disabling infobars
options.add_argument("--disable-extensions") # disabling extensions
options.add_argument("--no-sandbox")
options.add_argument(
"--disable-dev-shm-usage") # overcome limited resource problem
driver = Chrome(executable_path=DRIVER_PATH, chrome_options=options)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.implicitly_wait(10)
# Login
driver.get(LOGIN_URL)
usernameField = driver.find_element_by_xpath(USERNAME_PATH)
usernameField.send_keys(username)
passwordField = driver.find_element_by_xpath(PASSWORD_PATH)
passwordField.send_keys(password)
passwordField.send_keys(Keys.RETURN)
print("SUCCESS: login by", username)
return driver
def __init__(self):
# self.driver = webdriver.Chrome()
self.By = By
chrome_options = ChromeOptions()
# chrome_options.add_argument("--headless")
chrome_options.binary_location = r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"
self.driver = webdriver.Chrome(options=chrome_options)
def create_chrome_driver():
options = ChromeOptions()
options.binary_location = '/app/.apt/usr/bin/google-chrome'
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
return Chrome(options=options)
def _init_chrome(self, adblock: bool = False, incognito: bool = False) -> Chrome:
empty_dir(str(Path.cwd() / 'chrome-data'))
chr_opt = ChromeOptions()
chr_opt.headless = self.headless
chr_opt.add_argument("--no-sandbox")
chr_opt.add_argument('log-level=2')
chr_opt.add_argument('--disable-logging')
chr_opt.add_argument('--disable-remote-fonts')
chr_opt.add_argument("--user-data-dir=chrome-data")
chr_opt.add_argument("--remote-debugging-port=9222")
chr_opt.add_argument("--disable-infobars")
chr_opt.add_argument("--disable-dev-shm-usage")
chr_opt.add_argument("--ignore-certificate-errors")
chr_opt.add_argument("--disable-gpu")
if adblock and not self.headless: # headless + extensions = crash
chr_opt.add_extension(
r'C:\Users\Roel\PycharmProjects\scrapers\tripadvisor\driver\ublock.crx.crx')
if incognito:
chr_opt.add_argument('--incognito')
if 'posix' in os.name:
chr_opt.binary_location = '/home/vries274/scrapers/tripadvisor/chrome/chrome-linux/chrome'
chrome = Chrome(executable_path=self.CHR_PATH, options=chr_opt)
chrome.set_window_size(1920, 1080)
print('\n --- Browser started --- \n')
return chrome
def __init__(self,
user_id,
password,
driver_path=None,
headless=False,
poor=False,
gcf=False):
options = ChromeOptions()
if gcf:
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--window-size=480x270')
options.add_argument('--no-sandbox')
options.add_argument('--hide-scrollbars')
options.add_argument('--enable-logging')
options.add_argument('--log-level=0')
options.add_argument('--v=99')
options.add_argument('--single-process')
options.add_argument('--ignore-certificate-errors')
options.binary_location = os.getcwd() + "/headless-chromium"
self.driver = Chrome(os.getcwd() + "/chromedriver",
options=options)
else:
if poor:
options.add_argument("--disable-gpu")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--headless")
if headless:
options.add_argument("--headless")
if driver_path == 'remote': # リモート接続も可能（docker-seleniumの利用を想定）
self.driver = Remote(
command_executor='http://localhost:4444/wd/hub',
desired_capabilities=options.to_capabilities(),
options=options,
)
elif driver_path is not None:
self.driver = Chrome(executable_path=driver_path,
options=options)
else:
self.driver = Chrome(options=options)
if poor:
self.driver.set_window_size(480, 270)
print("Start Chrome Driver.")
print("Login to Zaim.")
self.driver.get("https://auth.zaim.net/")
time.sleep(1)
self.driver.find_element_by_id("UserEmail").send_keys(user_id)
self.driver.find_element_by_id("UserPassword").send_keys(
password, Keys.ENTER)
time.sleep(1)
print("Login Success.")
self.data = []
self.current = 0
def login_web():
chrome_options = ChromeOptions()
chrome_options.binary_location = r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'
driver = webdriver.Chrome(options=chrome_options)
driver.implicitly_wait(20)
login_page = LoginPage(driver)
yield driver, login_page
driver.quit()
def test_empty_chrome_options(self):
chrome_options = ChromeOptions()
chrome_options.binary_location = "./bin/headless-chromium"
title = request(examples.get_title,
"https://google.com",
chrome_options=chrome_options)
# works without chrome_options
self.assertTrue(type(title) is str)
self.assertTrue("Google" in title)
def get_driver(self, dcap, binary_location=None):
options = ChromeOptions()
if not self.load_images:
options.add_argument('disable-images')
if binary_location is not None:
options.binary_location = binary_location
return Chrome(executable_path=self.executable_path, desired_capabilities=dcap,
chrome_options=options)
def initDriver(cls):
options = ChromeOptions()
options.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
cls._driver = Chrome(
executable_path=
"/Users/seveniruby/projects/chromedriver/72/chromedriver",
options=options)
cls._driver.implicitly_wait(10)
def get_driver():
chrome_options = ChromeOptions()
chrome_options.binary_location = os.environ.get("GOOGLE_CHROME_BIN")
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-sandbox")
driver = Chrome(executable_path=os.environ.get("CHROMEDRIVER_PATH"),
chrome_options=chrome_options)
return driver
def __init__(self):
# 总页数
self.amount_page = 0
self.current_page = 0
self.amount_items = 0
self.current_item = 0
self.amount_img = 0
self.current_img = 0
self.delay_time = 1
option = ChromeOptions()
# 用来控制图片爬取暂停的标志
self.export_img_even = ''
self.error_message = ''
# 重定向
# prefs = {"profile.managed_default_content_settings.images": 2}
# option.add_experimental_option("prefs", prefs)
option.add_experimental_option('excludeSwitches',
['enable-automation'])
option.add_argument("disable-web-security")
# option.add_argument('--headless')
# option.add_argument('--disable-gpu')
path = r'chromedriver.exe'
option.binary_location = 'Chrome/Application/chrome.exe'
driver = webdriver.Chrome(executable_path=path, options=option)
driver.implicitly_wait(5)
self.driver = driver
self.login_url = 'https://login.tmall.com'
self.index_url = 'https://www.tmall.com'
self.index_url2 = 'https://www.tmall.com/'
self.shop_url = 'https://mai.taobao.com/seller_admin.htm'
self.items_url = 'https://ipublish.tmall.com/tmall/manager/render.htm'
self.detail_url = 'https://detail.tmall.com/item.htm'
# 二维码地址
self.twe_dimension_url = ''
# 登录状态
self.login = ''
self.login_name = ''
# 爬取详细页面的默认请求头
self.headers = {
"cache-control": "max-age=0",
"upgrade-insecure-requests": "1",
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
"sec-fetch-mode": "navigate",
"sec-fetch-user": "******",
"accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"sec-fetch-site": "none",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh-CN,zh;q=0.9"
}
def test_testerhome(self):
options = ChromeOptions()
options.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
self.driver = webdriver.Chrome(
executable_path=
"/Users/seveniruby/projects/chromedriver/72/chromedriver",
options=options)
self.driver.get("https://testerhome.com")
def __init__(self, *args, load_images=True, **kwargs):
super().__init__(*args, load_images=load_images, **kwargs)
options = ChromeOptions()
if 'Windows' == get_os():
options.binary_location = HEADLESS_CHROME_PATH
options.add_argument('no-sandbox')
options.add_argument('headless')
options.add_argument('disable-gpu')
if not load_images:
options.add_argument('disable-images')
self.options = options
def init_login():
chrome_options = ChromeOptions()
chrome_options.binary_location = r'C:\Users\Administrator\AppData\Local\Google\Chrome\Application\chrome.exe'
chrome_options.add_argument('--headless')
driver = Chrome(executable_path=r"C:\Python\Scripts\chromedriver.exe",
options=chrome_options)
driver.implicitly_wait(30)
# 初始化登录页面
login = LoginPage(driver)
yield driver, login
driver.quit()
def get_driver():
options = ChromeOptions()
options.binary_location = GOOGLE_CHROME_PATH
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
driver = Chrome(executable_path=DRIVER_PATH, chrome_options=options)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.implicitly_wait(10)
return driver
def manage_browser():
# 浏览器二进制路径配置
options = ChromeOptions()
options.binary_location = r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"
# 初始化浏览器
driver = webdriver.Chrome(options=options)
login_page = LoginPage(driver)
index_page = IndexPage(driver)
yield driver, login_page, index_page
driver.quit()
def __init__(self, start_url, source):
super().__init__(start_url, source)
chrome_options = ChromeOptions()
chrome_options.binary_location = "/usr/bin/google-chrome"
chrome_options.add_argument('user-agent=' +
UserAgent()['google chrome'])
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
self.web_driver_chrome = webdriver.Chrome(
executable_path=CHROME_DRIVER_PATH, options=chrome_options)
serialized_arr = self.read_data_from_json_file(JSON_FILE_PATH)
self.sent_ids = [] if serialized_arr is None else serialized_arr
def init_web():
# 设置无头浏览器
chrome_options = ChromeOptions()
chrome_options.binary_location = r'C:\Users\LittleStone\AppData\Local\Google\Chrome\Application\chrome.exe'
# 启动浏览器
driver = webdriver.Chrome(constant.DRIVER_PATH, options=chrome_options)
# 窗口最大化
driver.maximize_window()
# 设置隐式等待
driver.implicitly_wait(10)
yield driver
driver.quit()
def get_headless_option(self):
chrome_options = ChromeOptions()
for option in [
"--headless", "--disable-gpu", "--window-size=1366x768",
"--disable-application-cache", "--disable-infobars",
"--no-sandbox", "--hide-scrollbars", "--enable-logging",
"--log-level=0", "--single-process",
"--ignore-certificate-errors", "--homedir=/tmp"
]:
chrome_options.add_argument(option)
if os.getenv("ENV", "local") != "local":
chrome_options.binary_location = "/opt/python/bin/headless-chromium"
return chrome_options
def __init__(self, settings):
super().__init__(settings)
self.executable_path = settings['CHROME_EXECUTABLE_PATH']
self.dcap = DesiredCapabilities.CHROME.copy()
options = ChromeOptions()
if 'Windows' == get_os():
options.binary_location = settings["HEADLESS_CHROME_PATH"]
options.add_argument('no-sandbox')
options.add_argument('headless')
options.add_argument('disable-gpu')
if not settings.get("CHROME_LOAD_IMAGES", False):
options.add_argument('disable-images')
self.options = options
def fixture_class():
global driver
# chrome_options = Options() #设置无头浏览器
# chrome_options.add_argument('--headless')
# chrome_options.add_argument('--disable-gpu')
# driver = Chrome(options=chrome_options, service_log_path=r"D:\ChromeLog\log.log")
chrome_options = ChromeOptions()
chrome_options.binary_location=r"C:\Users\zjb\AppData\Local\Google\Chrome\Application\chrome.exe"
driver = Chrome(executable_path=r"D:\Python37\chromedriver.exe",chrome_options=chrome_options)
driver.maximize_window()
time.sleep(2)
yield driver
driver.quit()
def _create_chrome(cls,
config,
driver_path,
browser_bin_path,
svc_url,
proxy=None):
from selenium.webdriver import Chrome, ChromeOptions
caps = DesiredCapabilities.CHROME
caps.update(config["driverCapabilities"])
# if config["arjuna_options"]["BROWSER_PROXY_ON"]:
# proxy = Proxy()
# proxy_string = "{}.{}".format(
# config["arjuna_options"]["BROWSER_PROXY_HOST"],
# config["arjuna_options"]["BROWSER_PROXY_PORT"]
# )
# proxy.http_proxy = proxy_string
# proxy.ssl_proxy = proxy_string
# proxy.add_to_capabilities(caps)
from arjuna import log_debug
log_debug("Is proxy set for Chrome?: {}".format(proxy is not None))
if proxy is not None:
proxy.add_to_capabilities(caps)
caps['acceptInsecureCerts'] = True
options = ChromeOptions()
if browser_bin_path.lower() != "not_set":
options.binary_location = browser_bin_path
if cls.are_browser_prefs_set(config):
options.add_experimental_option("prefs",
config["browserPreferences"])
if config["arjuna_options"]["BROWSER_HEADLESS"]:
options.add_argument("--headless")
if cls.are_browser_args_set(config):
for arg in config["browserArgs"]:
options.add_argument(arg)
if cls.are_extensions_set(config):
for ext in config["browserExtensions"]:
options.add_extension(ext)
caps[ChromeOptions.KEY] = options.to_capabilities()[ChromeOptions.KEY]
from selenium import webdriver
return webdriver.Remote(svc_url, caps)
