def play(): # 画面上部「自動テストソフトソフトウェアによって制御されています」の表示を消す options = ChromeOptions() options.add_experimental_option( "excludeSwitches", ['enable-automation']) # ['disable-automation']にすると戻る # 拡張機能のCRXパス、ChromeDriverの絶対パスを指定しChromeを起動 options.add_extension('ChromeExtensions/2.0.0_0.crx') driver = Chrome('YOUR LOCATION', options=options) # ウィンドウを開く driver.maximize_window() driver.get(url) # 再生ページへ遷移 time.sleep(1) driver.find_element_by_link_text("すべて再生").click() # 全画面表示 time.sleep(1) pyautogui.press('f') # 起動時間を指定(秒) time.sleep(2350) # セッション終了 driver.close()
def create_driver(driver_path, **options): plugin_path = options.get('plugin_path', None) opt = ChromeOptions() if plugin_path: opt.add_extension(plugin_path) driver = webdriver.Chrome(driver_path, chrome_options=opt) return driver
def create_download_dir_capabilities_for_chrome(path_to_download, **extensions_files): """ Example use | ${capabilities} | create_download_dir_capabilities_for_chrome | Artifacts | | Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | gc | desired_capabilities=${capabilities} | | Click Element | //a[contains(@href,'sample.csv.zip')] | """ path_to_download_check = validate_create_artifacts_dir( path_to_download) chrome_options = ChromeOptions() prefs = { "download.default_directory": path_to_download_check, "directory_upgrade": "true" } chrome_options.add_experimental_option("prefs", prefs) chrome_options.add_argument("--disable-web-security") for single_extension in extensions_files: chrome_options.add_extension(single_extension) logger.info("Chrome Capabilities set download dir '" + path_to_download_check + "'") return chrome_options.to_capabilities()
def _init_chrome(self, adblock: bool = False, incognito: bool = False) -> Chrome: empty_dir(str(Path.cwd() / 'chrome-data')) chr_opt = ChromeOptions() chr_opt.headless = self.headless chr_opt.add_argument("--no-sandbox") chr_opt.add_argument('log-level=2') chr_opt.add_argument('--disable-logging') chr_opt.add_argument('--disable-remote-fonts') chr_opt.add_argument("--user-data-dir=chrome-data") chr_opt.add_argument("--remote-debugging-port=9222") chr_opt.add_argument("--disable-infobars") chr_opt.add_argument("--disable-dev-shm-usage") chr_opt.add_argument("--ignore-certificate-errors") chr_opt.add_argument("--disable-gpu") if adblock and not self.headless: # headless + extensions = crash chr_opt.add_extension( r'C:\Users\Roel\PycharmProjects\scrapers\tripadvisor\driver\ublock.crx.crx') if incognito: chr_opt.add_argument('--incognito') if 'posix' in os.name: chr_opt.binary_location = '/home/vries274/scrapers/tripadvisor/chrome/chrome-linux/chrome' chrome = Chrome(executable_path=self.CHR_PATH, options=chr_opt) chrome.set_window_size(1920, 1080) print('\n --- Browser started --- \n') return chrome
def _process_chrome_options(self, opts): if isinstance(opts, ChromeOptions): options = opts else: options = ChromeOptions() if 'args' in opts: for arg in opts.pop('args'): options.add_argument(arg) if opts.pop('headless', None) or self.options.pop( 'headless', None): options.headless = True if 'binary' in opts or 'binary_location' in opts: options.binary_location = opts.pop('binary') or opts.pop( 'binary_location') if 'debugger_address' in opts: options.debugger_address = opts.pop('debugger_address') if 'extensions' in opts: for path in opts.pop('extensions'): options.add_extension(path) if 'encoded_extensions' in opts: for string in opts.pop('encoded_extensions'): options.add_encoded_extension(string) if 'experimental_options' in opts: for name, value in opts.pop('experimental_options').items(): options.add_experimental_option(name, value) self.selenium_opts['options'] = options
def create_options(self): super().debug_begin() options = ChromeOptions() for file_name in self.__plugin_files: if '\\.' not in file_name: file_name += '.crx' options.add_extension(os.getcwd() + '/' + file_name) options.add_argument("--ignore-certificate-errors") options.add_argument("--start-maximized") options.add_argument("--disable-popup-blocking") options.add_argument("--enable-strict-powerful-feature-restrictions") options.add_argument("--disable-geolocation") if super()._language is not None: options.add_argument("--lang=" + super()._language) if super()._headless: options.add_argument("-headless") options.add_argument("--disable-gpu") if super()._use_proxy: # options.add_argument('--ignore-certificate-errors') pass super().debug_end() return options
def chrome(extension_zip): opts = ChromeOptions() # crbug.com/706008: No support for extensions in headless mode opts.add_extension(extension_zip) chrome = Chrome(options=opts) chrome.implicitly_wait(3) try: yield chrome finally: chrome.close()
def _install_crx_extension( extension_crx: str, chrome_options: webdriver.ChromeOptions ) -> None: """ Installs extensions to Chromium based browsers :param extension_crx: Absolute path to the CRX file for the extension :type extension_crx: str :param chrome_options: webdriver.ChromeOptions from the Selenium driver :type chrome_options: webdriver.ChromeOptions """ chrome_options.add_extension(extension_crx)
def _create_chrome(cls, config, driver_path, browser_bin_path, svc_url, proxy=None): from selenium.webdriver import Chrome, ChromeOptions caps = DesiredCapabilities.CHROME caps.update(config["driverCapabilities"]) # if config["arjuna_options"]["BROWSER_PROXY_ON"]: # proxy = Proxy() # proxy_string = "{}.{}".format( # config["arjuna_options"]["BROWSER_PROXY_HOST"], # config["arjuna_options"]["BROWSER_PROXY_PORT"] # ) # proxy.http_proxy = proxy_string # proxy.ssl_proxy = proxy_string # proxy.add_to_capabilities(caps) from arjuna import log_debug log_debug("Is proxy set for Chrome?: {}".format(proxy is not None)) if proxy is not None: proxy.add_to_capabilities(caps) caps['acceptInsecureCerts'] = True options = ChromeOptions() if browser_bin_path.lower() != "not_set": options.binary_location = browser_bin_path if cls.are_browser_prefs_set(config): options.add_experimental_option("prefs", config["browserPreferences"]) if config["arjuna_options"]["BROWSER_HEADLESS"]: options.add_argument("--headless") if cls.are_browser_args_set(config): for arg in config["browserArgs"]: options.add_argument(arg) if cls.are_extensions_set(config): for ext in config["browserExtensions"]: options.add_extension(ext) caps[ChromeOptions.KEY] = options.to_capabilities()[ChromeOptions.KEY] from selenium import webdriver return webdriver.Remote(svc_url, caps)
def generate_chrome_options( image=False, js=True, *, headless=True, headers='common_android', proxy=None, extension_path=None, gpu=False ): options = ChromeOptions() if headless: options.headless = True if not gpu: options.add_argument('--disable-gpu') ua = UserAgent() current_headers = getattr(ua, headers)() argument_ua = "user-agent={}".format(current_headers) options.add_argument(argument_ua) argument_dis_notifications = "--disable-notifications" options.add_argument(argument_dis_notifications) argument_exp_dis_image = ("prefs", {'profile.managed_default_content_settings.images': 2}) if not image: options.add_experimental_option(*argument_exp_dis_image) argument_exp_dis_js = ("prefs", {'profile.managed_default_content_settings.javascript': 2}) if not js: options.add_experimental_option(*argument_exp_dis_js) # extension_path = '/home/parallels/Downloads/GoogleHelper_1.2.0.crx' if extension_path: options.add_extension(extension_path) # proxy = "socks5://104.207.159.40:9100" if proxy: if proxy.get('password'): proxyauth_plugin_path = create_proxyauth_extension( proxy_host=proxy['ip'], proxy_port=int(proxy['port']), proxy_username=proxy['user'], proxy_password=proxy['password'], ) options.add_extension(proxyauth_plugin_path) else: argument_proxy = "--proxy-server={}://{}:{}".format( proxy['protocol'], proxy['ip'], proxy['port'] ) options.add_argument(argument_proxy) return options
def create_download_dir_capabilities_for_chrome(path_to_download, **extensions_files): """ Example use | ${capabilities} | create_download_dir_capabilities_for_chrome | Artifacts | | Open Browser Extension | https://support.spatialkey.com/spatialkey-sample-csv-data/ | gc | desired_capabilities=${capabilities} | | Click Element | //a[contains(@href,'sample.csv.zip')] | """ path_to_download_check = validate_create_artifacts_dir(path_to_download) chrome_options = ChromeOptions() prefs = {"download.default_directory": path_to_download_check, "directory_upgrade": "true"} chrome_options.add_experimental_option("prefs", prefs) chrome_options.add_argument("--disable-web-security") for single_extension in extensions_files: chrome_options.add_extension(single_extension) logger.info("Chrome Capabilities set download dir '" + path_to_download_check + "'") return chrome_options.to_capabilities()
def create_chrome(disableImage=True, mobile=False): options = ChromeOptions() chrome_prefs = {} if disableImage: chrome_prefs["profile.default_content_settings"] = {"images": 2} chrome_prefs["profile.managed_default_content_settings"] = {"images": 2} options.experimental_options["prefs"] = chrome_prefs options.add_argument('no-sandbox') if mobile: options.add_extension(SWITCHER_PATH) options.add_argument('user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"') options.add_argument('user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"') options.add_argument('disable-images') options.add_argument('headless') options.add_argument('disable-gpu') if platform.system() == 'Darwin': driver = Chrome(executable_path=CHROME_PATH, chrome_options=options) else: driver = Chrome(chrome_options=options) return driver
def _browser_options(self, option_instance: webdriver.ChromeOptions, options_config: dict): # 添加浏览器配置、扩展、实验性质参数等 """ 如果options_config内有值就添加配置: add_argument add_experimental_option add_encoded_extension add_extension :param option_instance: :param options_config: :return: """ # todo: add_argument 设置浏览器启动语言、隐藏自动化测试提示不生效 # todo: 可能是zh_CN 下次尝试一下下划线 options_initial: dict = options_config.pop('options') if options_initial: argument = options_initial.get("add_argument") experimental = options_initial.get("add_experimental_option") encoded = options_initial.get("add_encoded_extension") extension = options_initial.get("add_extension") if argument and len(argument) > 0: for arg in argument: option_instance.add_argument(arg) if experimental and len(experimental) > 0: for exper in experimental: option_instance.add_experimental_option(*exper) if encoded and len(encoded) > 0: for encod in encoded: option_instance.add_encoded_extension(encod) if extension and len(extension) > 0: for exten in extension: option_instance.add_extension(exten) return option_instance
def _create_chrome(cls, config, driver_path, browser_bin_path): from selenium.webdriver import Chrome, ChromeOptions caps = DesiredCapabilities.CHROME caps.update(config["driverCapabilities"]) if config["arjunaOptions"]["BROWSER_PROXY_ON"]: proxy = Proxy() proxy_string = "{}.{}".format( config["arjunaOptions"]["BROWSER_PROXY_HOST"], config["arjunaOptions"]["BROWSER_PROXY_PORT"]) proxy.http_proxy = proxy_string proxy.ssl_proxy = proxy_string proxy.add_to_capabilities(caps) options = ChromeOptions() if browser_bin_path.lower() != "not_set": options.binary_location = browser_bin_path if cls.are_browser_prefs_set(config): options.add_experimental_option("prefs", config["browserPreferences"]) if cls.are_browser_args_set(config): for arg in config["browserArgs"]: options.add_argument(arg) if cls.are_extensions_set(config): for ext in config["browserExtensions"]: options.add_extension(ext) caps[ChromeOptions.KEY] = options.to_capabilities()[ChromeOptions.KEY] return Chrome( executable_path=driver_path, desired_capabilities=caps, #service_args=["--verbose", "--log-path=/Users/rahulverma/Documents/____drivers/cd.log"] )
def open_with_extensions(): path_extentions = '3.2.0_0.crx' path_chromedriver = 'chromedriver_win32/chromedriver.exe' options = ChromeOptions() options.add_extension(path_extentions) try: driver = Chrome(executable_path=path_chromedriver, options=options) # driver = Chrome(executable_path=path_chromedriver) # driver = Chrome(options=options) driver.set_window_position(0, 0) # ブラウザの位置を左上に固定 driver.set_window_size(600, 740) # ブラウザのウィンドウサイズを固定 url = "http://tokidoki-web.com" driver.get(url) print("end") # # 拡張機能のアクションボタンをクリック # print("拡張機能ON") # # ディスプレイ位置から左550px、上80pxの位置で1回クリックして1秒待機 # pyautogui.click(550, 80, 1, 1, 'left') # # ポップアップ内の要素をクリック # time.sleep(2) # print("ポップアップ内クリック") # # ディスプレイ位置から左450px、上430pxの位置で1回クリックして1秒待機 # pyautogui.click(450, 430, 1, 1, 'left') time.sleep(1000) # ~いろいろな処理~ finally: os.kill(driver.service.process.pid, signal.SIGTERM)
from datetime import datetime from selenium import webdriver from bs4 import BeautifulSoup import requests from selenium.webdriver import Chrome, ChromeOptions options = ChromeOptions() options.add_extension(r"D:\Downloads\bypass-paywalls-chrome-master\bypass-paywalls-chrome-master.crx") counter = 1 o = 1 while counter < 600: url = "https://www.ft.com/search?q=bitcoin&page=" + str(counter) + "&contentType=article&dateTo=2017-12-31&dateFrom=2017-01-01&sort=date&expandRefinements=true" response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}) if not response: print("Wrong response: ", response) else: doc = response.text soup = BeautifulSoup(doc, "lxml") ul = soup.select("#site-content > div > ul > li") for li in ul: a = li.select_one("div > div > div > div.o-teaser__content > div.o-teaser__heading > a") link = a["href"] if link[0] == "/": link = "https://www.ft.com" + link
def set_driver(isHeadless=False, isManager=False, isSecret=False, isExtension=False, extension_path='', profile_path=''): options = ChromeOptions() user_agent = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36', ] if os.name == 'nt': # Windows driver_path = 'chromedriver.exe' elif os.name == 'posix': # Mac driver_path = 'chromedriver' if isHeadless: options.add_argument('--headless') options.add_argument('--single-process') if isExtension: if extension_path: options.add_extension(extension_path) else: options.add_argument('--disable-extensions') if isSecret: options.add_argument('--incognito') # シークレットモードの設定を付与 else: # プロファイル設定することで、初回手動でログインや拡張機能追加したものを2回目以降使用可能 # シークレットモードではプロファイル設定を使用できない # ヘッドレスモードではプロファイル設定、Chrome拡張機能を使用できない # 拡張機能を有効にして、以下のエラーが出た場合、その拡張機能は使用できない # failed to wait for extension background page to load # その場合は、プロファイル設定にて手動で機能を追加して、ヘッドレスモードかつ拡張機能Enableで使用する if (not isHeadless) or (not isExtension): # options.add_argument('--user-data-dir=' + profile_path) # options.add_argument('--profile-directory=Profile 1') if getattr(sys, 'frozen', False): directory_path = os.path.dirname(sys.executable) if '.app' in directory_path: idx = directory_path.find('.app') directory_path = directory_path[:idx] idx = directory_path.rfind('/') directory_path = directory_path[:idx] else: directory_path = os.getcwd() options.add_argument('--user-data-dir=' + os.path.join(directory_path, "profile")) options.add_argument('--disable-gpu') options.add_argument('--no-sandbox') options.add_argument('log-level=3') options.add_argument('--ignore-ssl-errors') options.add_argument( f'--user-agent={user_agent[random.randrange(0, len(user_agent), 1)]}') options.add_argument('--start-maximized') options.add_argument('--ignore-certificate-errors') options.add_argument('--allow-running-insecure-content') options.add_argument('--disable-web-security') options.add_argument('--disable-desktop-notifications') options.add_argument('--disable-application-cache') options.add_argument('--lang=ja') if isManager: # 自動取得 try: driver = Chrome(ChromeDriverManager().install(), options=options) except InvalidArgumentException as err: logger.error(err) logger.error('既存のブラウザを閉じで実行してください。') return None except Exception as err: logger.error(err) else: # 手動取得 try: if getattr(sys, 'frozen', False): directory_path = os.path.dirname(sys.executable) if '.app' in directory_path: idx = directory_path.find('.app') directory_path = directory_path[:idx] idx = directory_path.rfind('/') directory_path = directory_path[:idx] else: directory_path = os.getcwd() path = join(directory_path, driver_path) driver = Chrome(executable_path=path, options=options) except InvalidArgumentException as err: logger.error(err) logger.error('既存のブラウザを閉じで実行してください。') return None except WebDriverException as err: logger.error(err) logger.error('Chromeと同じバージョンのChrome Driverをダウンロードしてください。') return None return driver
def option_adding_xpath_helper(): options = ChromeOptions() options.add_argument("--disable-infobars") options.add_extension(r"chrome_extension_xpath_helper.crx") return options
class ChromeProxySetup(object): def __init__(self,proxy_host=None,proxy_port=None,proxy_auth_user=None,proxy_auth_pwd=None): self.proxy_host = proxy_host self.proxy_port = proxy_port self.proxy_auth_user = proxy_auth_user self.proxy_auth_pwd = proxy_auth_pwd self.chrome_options = ChromeOptions() self.chrome_options.add_argument('--disable-infobars') self.chrome_options.add_argument('--dns-prefetch-disabled') self.extension_file = '' def set_proxy(self): proxy = { 'host': self.proxy_host, 'port': self.proxy_port, 'user': self.proxy_auth_user, 'pass': self.proxy_auth_pwd } proxy_meta = proxy manifest_json = ''' { "version": "1.0.0", "manifest_version": 2, "name": "Chrome Proxy", "permissions": [ "proxy", "tabs", "unlimitedStorage", "storage", "<all_urls>", "webRequest", "webRequestBlocking" ], "background": { "scripts": ["background.js"] }, "minimum_chrome_version":"22.0.0" } ''' background_js = ''' var config = { mode:"fixed_servers", rules:{ singleProxy:{ scheme:"http", host:"%s", port:parseInt(%s) }, bypassList:["localhost"] } }; chrome.proxy.settings.set({value:config, scope:"regular"}, function() {}); function callbackFn(details) { return{ authCredentials:{ username: "******", password: "******" } }; } chrome.webRequest.onAuthRequired.addListener( callbackFn, {urls:["<all_urls>"]}, ["blocking"] ); chrome.runtime.onInstalled.addListener(function(){ chrome.tabs.reload() }) ''' % (self.proxy_host, self.proxy_port, self.proxy_auth_user, self.proxy_auth_pwd) proxy = proxy_meta proxy_zip_filename = ('%s.zip' %self.proxy_host) self.extension_file = os.path.join(BASE_DIR, proxy_zip_filename) if not exists(self.extension_file): with zipfile.ZipFile(self.extension_file, 'w') as prxtn: prxtn.writestr('manifest.json', manifest_json) prxtn.writestr('background.js', background_js) self.chrome_options.add_extension(self.extension_file) self.browser = webdriver.Chrome(chrome_options=self.chrome_options) return self.browser
zp.writestr("manifest.json", manifest_json) zp.writestr("background.js", background_js) return plugin_path proxyauth_plugin_path = create_proxyauth_extension( proxy_host='http-dyn.abuyun.com', proxy_port=9020, proxy_username='******', proxy_password='******') option = ChromeOptions() #option.add_argument("--proxy-server=http://202.20.16.82:10152") option.add_experimental_option('excludeSwitches', ['enable-automation']) option.add_extension(proxyauth_plugin_path) class webdriver(): def __init__(self, limit=15, timeout=10): self.limit = limit self.timeout = timeout self.create_driver() def create_driver(self): self.driver = Chrome('./chromedriver', options=option) self.driver.set_window_size(800, 900) self.driver.set_page_load_timeout(self.timeout) self.count = 0 def delete_driver(self):
# 添加UA option.add_argument(headers) # 指定浏览器分辨率 option.add_argument('window-size=1920x3000') # 无可视化界面浏览 option.add_argument('--headless') # 规避bug option.add_argument('--disable-gpu') # add_experimental_option()添加实验性质的设置参数 # 规避监测, 即启动开发者模式 option.add_experimental_option('excludeSwitches', ['enable-automation']) # 指定使用的浏览器位置 option.binary_location = c_path # option.add_extension()/option.add_encoded_extension()添加扩展应用 # 添加指定crx插件 option.add_extension(r'E:\temp\爬虫\crx\Adblock.crx') # e_path = r'E:\study\python\anaconda3\Scripts\chromedriver' master = webdriver.Chrome(options=option) master.get('https://kyfw.12306.cn/otn/login/init') # # 查看浏览历史 master.get('chrome://history/') sleep(5) master.quit()
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from selenium.common.exceptions import TimeoutException chrome_driver_path = os.environ["CHROME_DRIVER_PATH"] chrome_binary_location = os.environ["CHROME_BINARY_LOCATION"] line_url = "chrome-extension://ophjlpahpchlmihnnnihgmmeilfjmjjc/index.html" # confidential ################## login_e_mail = "*****@*****.**" login_pwd = "dummypass" ################################# options = ChromeOptions() options.add_extension("chrome_extensions/LINE-Chrome_2.4.3.0.crx") # options.add_argument("--headless") # headlessモードを使用する options.add_argument("--disable-gpu") # headlessモードで暫定的に必要なフラグ(そのうち不要になる) options.add_argument("--start-maximized") # 起動時にウィンドウを最大化する options.add_argument("--disable-dev-shm-usage") # options.add_argument("--no-sandbox") # options.add_argument("--blink-settings=imagesEnabled=false") options.binary_location = chrome_binary_location # ユーザプロファイルのフォルダ名(実行フォルダに作成されます) user_profile = 'UserProfile' # Optionでユーザプロファイルの場所を指定する options.add_argument('--user-data-dir=' + user_profile) try:
def sleep(n): for _ in range(1, n + 1): time.sleep(1) capa = DesiredCapabilities.CHROME capa["pageLoadStrategy"] = "none" chrome_options = ChromeOptions() chrome_options.add_argument("--start-maximized") chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']) chrome_options.add_experimental_option("useAutomationExtension", False) pluginfile = 'proxy_auth_plugin.zip' with zipfile.ZipFile(pluginfile, 'w') as zp: zp.writestr("manifest.json", manifest_json) zp.writestr("background.js", background_js) chrome_options.add_extension(pluginfile) driver = webdriver.Chrome(desired_capabilities=capa, options=chrome_options) driver.execute_cdp_cmd( "Page.addScriptToEvaluateOnNewDocument", { "source": """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }) """ }) driver.get("https://whatismyipaddress.com/") sleep(100000000) driver.quit()