def start_browser(request, browser, timeout): """ Setup and run browser :param request: pytest request :param browser: name of browser that will start :param timeout: page load timeout :return wd: web driver :return proxy: browsermob proxy """ chromedriver_path = '/home/zhukov/PycharmProjects/Otus/drivers/chromedriver' firefoxdriver_path = '/home/zhukov/PycharmProjects/Otus/drivers/geckodriver' url = urlparse(proxy.proxy).path if browser == "chrome": des_cap = DesiredCapabilities.CHROME des_cap['loggingPrefs'] = {'performance': 'ALL'} chrome_options().add_argument(argument='--proxy-server={}'.format(url)) options = chrome_options() options.headless = True wd = EventFiringWebDriver( webdriver.Chrome(options=options, executable_path=chromedriver_path, desired_capabilities=des_cap), OpencartListener()) # command_executor = 'http://*****:*****@hub.browserstack.com:80/wd/hub' # wd = EventFiringWebDriver(webdriver.Remote(command_executor, # desired_capabilities={"browserName": "chrome", # 'os': 'Linux', 'os_version': '16.04'}), # OpencartListener()) wd.implicitly_wait(int(timeout)) else: des_cap = DesiredCapabilities.FIREFOX des_cap['loggingPrefs'] = {'performance': 'ALL'} firefox_options().add_argument( argument='--proxy-server={}'.format(url)) options = firefox_options() options.headless = True wd = EventFiringWebDriver( webdriver.Firefox(options=options, executable_path=firefoxdriver_path, desired_capabilities=des_cap), OpencartListener()) # command_executor = 'http://*****:*****@hub.browserstack.com:80/wd/hub' # wd = EventFiringWebDriver(webdriver.Remote(command_executor, # desired_capabilities={"browserName": "firefox", # 'os': 'Windows', 'os_version': '10'}), # OpencartListener()) wd.implicitly_wait(int(timeout)) wd.maximize_window() request.addfinalizer(wd.quit) return wd, proxy
def get_firefox_driver(): profile = FirefoxProfile() profile.set_preference('browser.cache.disk.enable', False) profile.set_preference('browser.cache.memory.enable', False) profile.set_preference('browser.cache.offline.enable', False) profile.set_preference('network.cookie.cookieBehavior', 2) if os.environ.get("DEV"): profile.set_preference("network.proxy.type", 1) profile.set_preference("network.proxy.socks", "127.0.0.1") profile.set_preference("network.proxy.socks_port", 8050) profile.set_preference("network.proxy.socks_version", 5) profile.update_preferences() caps = DesiredCapabilities().FIREFOX caps["pageLoadStrategy"] = "none" options = firefox_options() options.headless = True options.add_argument('start-maximized') path = os.path.dirname(__file__) if platform.system() == "Linux": path = os.path.join(path, r"driver/geckodriver") else: path = os.path.join(path, r"driver/geckodriver_mac") driver = webdriver.Firefox(executable_path=path, firefox_options=options, desired_capabilities=caps, firefox_profile=profile) driver.set_window_size(1920, 1080) driver.implicitly_wait(1) return driver
def check_for_url(self): if self.url != "": if self.browser_type.get() == 0: ch_options = chrome_options() if self.headless.get() == 1: ch_options.add_argument("--headless") try: self.driver = webdriver.Chrome("chrome1/chromedriver.exe", options=ch_options) except: try: self.driver = webdriver.Chrome("chrome2/chromedriver.exe", options=ch_options) except: try: self.driver = webdriver.Chrome("chrome3/chromedriver.exe", options=ch_options) except: print("NO WORKING CHROME DRIVER") exit() else: ff_options = firefox_options() if self.headless.get() == 1: ff_options.add_argument("--headless") try: self.driver = webdriver.Firefox("firefox1/geckodriver.exe", options=ff_options) except: try: self.driver = webdriver.Firefox("firefox2/geckodriver.exe", options=ff_options) except: try: self.driver = webdriver.Firefox("firefox3/geckodriver.exe", options=ff_options) except: print("NO WORKING FIREFOX DRIVER") exit() self.driver.get(self.url) self.main_loop()
def choose_webdriver(browser): """Choosing availabe webdriver in the webdriver_manager module. Args: browser (str): browser's name. One of ['chrome', 'chromium', 'firefox', 'ie']. """ if browser == 'chrome': options = chrome_options() # silent mode options.add_argument('--headless') # no printing log in the terminal options.add_argument("--log-level=3") driver = webdriver.Chrome( executable_path=ChromeDriverManager().install(), chrome_options=options) elif browser == 'firefox': options = firefox_options() # silent mode options.headless = True driver = webdriver.Firefox( executable_path=GeckoDriverManager().install(), service_log_path=os.devnull, options=options) else: raise Exception('Your browser is not supported.') return (driver)
def start_browser(): if config.browser == 'firefox': options = firefox_options() options.headless = config.headless if config.system == "linux": browser = webdriver.Firefox( options=options, executable_path=r"./include/geckodriver") elif config.system == "windows": browser = webdriver.Firefox( options=options, executable_path=r".\include\geckodriver.exe") elif config.browser == 'chrome': options = chrome_options() if config.headless: options.add_argument('--headless') if config.system == "linux": browser = webdriver.Chrome( options=options, executable_path=r"./include/chromedriver") elif config.system == "windows": browser = webdriver.Chrome( options=options, executable_path=r".\include\chromedriver.exe") else: print("Error : wrong browser configured") exit(1) atexit.register(quit_browser, browser) return browser
def start_selenium(driver_type='firefox', seleniumwire_driver=False, timeout=60, is_headless=False, proxy=None, **kwargs): if platform.system() == 'Linux': display = Display(visible=0, size=(1280, 1024)) display.start() if seleniumwire_driver: driver_module = xhr_webdriver else: driver_module = webdriver if driver_type == 'firefox': driver_class = driver_module.Firefox driver_path = Path(__file__).parent / '_webdrivers' / 'gecko.exe' webdriver_manager = GeckoDriverManager() options = firefox_options() caps = webdriver.DesiredCapabilities.FIREFOX elif driver_type == 'chrome': driver_class = driver_module.Chrome driver_path = Path(__file__).parent / '_webdrivers' / 'chrome.exe' webdriver_manager = ChromeDriverManager() options = chrome_options() caps = webdriver.DesiredCapabilities.CHROME else: raise NotImplementedError( f"Запуск драйвера {driver_type} не реализован") if proxy: caps['proxy'] = { "proxyType": "MANUAL", "httpProxy": proxy, "sslProxy": proxy } driver_path = driver_path.resolve() if not driver_path.exists(): driver_path.parent.mkdir(exist_ok=True) cache_path = webdriver_manager.install() shutil.copy(cache_path, str(driver_path)) options.headless = is_headless driver = driver_class(options=options, executable_path=str(driver_path), capabilities=caps) if not is_headless: driver.maximize_window() driver.set_page_load_timeout(timeout) return driver
def grab_soup(url_, browser="firefox"): """ This function enables a driver (using Firefox or Chrome), goes to the URL, and retrieves the data after the JS is loaded. :param url_: url to go to to retrieve data :param browser: browser to use, defaults to firefox (requires geckodriver.exe on path) :return: soup - the data of the page driver - the browser (process) instance """ if browser == 'chrome': chromeOptions = chrome_options() chromeOptions.add_experimental_option( "prefs", { "download.default_directory": r"C:\Users\J_Ragbeer\Downloads", "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True }) chromeOptions.add_argument("--disable-gpu") chromeOptions.add_argument("--headless") chromeOptions.add_argument('--no-sandbox') chromeOptions.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome( "C:/Users/Julien/PycharmProjects/practice/chromedriver95", options=chromeOptions) else: firefoxOptions = firefox_options() firefoxOptions.set_preference("browser.download.folderList", 2) firefoxOptions.set_preference( "browser.download.manager.showWhenStarting", False) firefoxOptions.set_preference( "browser.download.dir", path.replace('/', '\\') + 'data\\downloads\\') firefoxOptions.set_preference( "browser.helperApps.neverAsk.saveToDisk", "application/octet-stream,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) driver = webdriver.Firefox(options=firefoxOptions) driver.get(url_) # go to the URL html = driver.page_source time.sleep( 1) # sleep for 1 second to ensure all JS scripts are loaded html = driver.execute_script( "return document.body.outerHTML;") # execute javascript code soup_ = bs.BeautifulSoup( html, 'lxml') # read the data as html (using lxml driver) return soup_, driver
def start(self): if self.browser == 'Chrome': browser_options = chrome_options() browser_options.add_argument("--headless") browser_options.add_argument('--no-sandbox') self.__driver = webdriver.Chrome(options=browser_options) elif self.browser == 'Firefox': browser_options = firefox_options() browser_options.add_argument("--headless") browser_options.add_argument('--no-sandbox') self.__driver = webdriver.Firefox(options=browser_options) else: raise Exception('Please set browser to browser=Firefox or Chrome.') logger.info(f'Using {self.browser}') time.sleep(1) logger.debug('Webdriver Started')
def make_driver(): driver = None if config.use_proxy and config.browser != 'chrome': raise Exception(f'proxy only supported in chrome') if config.browser == 'chrome': _chrome_options = chrome_options() _chrome_options.add_argument("--disable-extensions") _chrome_options.add_argument("--disable-gpu") _chrome_options.add_argument("--disable-notifications") _chrome_options.add_argument("--disable-logging") _chrome_options.add_experimental_option('excludeSwitches', ['enable-logging']) _chrome_options.add_experimental_option('prefs', { 'prefs': {'intl.accept_languages': 'en,en-US'}, 'profile.default_content_setting_values.notifications' : 2, 'profile.managed_default_content_settings.images': 2}) #_chrome_options.add_argument(f"user-data-dir={config.chrome_profile_path}") if config.use_proxy: host, port, protocol = config.proxy_list[0] _chrome_options.add_argument(f"--proxy-server={protocol}://{host}:{port}") helper.thread_safe_print(f'using proxy: {protocol}://{host}:{port}') if config.headless: _chrome_options.add_argument("--start-maximized") _chrome_options.add_argument("--headless") driver = webdriver.Chrome(executable_path=config.chrome_driver, options=_chrome_options) elif config.browser == 'firefox': _firefox_profile = FirefoxProfile(config.firefox_profile_path) _firefox_options = firefox_options() _firefox_options.headless = config.headless driver = webdriver.Firefox(_firefox_profile, options=_firefox_options) else: raise Exception(f'unknown brower "{config.browser}"') if not config.skip_login: login(driver) return driver
def get_driver(head=False): if head: # options = webdriver.FirefoxOptions().set_headless() # driver = webdriver.Firefox(firefox_options=options) options = firefox_options() options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument("--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36") driver = webdriver.Firefox(options=options) # driver = webdriver.Firefox() else: options = Options() options.add_argument('--headless') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument("--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36") driver = webdriver.Chrome(options=options) return driver
def init(): open("./file/DEBUG.txt", "w", encoding="utf-8").close() try: option = chrome_options() option.add_argument("--headless") v_driver = webdriver.Chrome(options=option) except: print_debug("[v_driver] don't detect Chrome", "yellow") try: option = firefox_options() option.add_argument("--headless") v_driver = webdriver.Opera(options=option) except: print_debug("[v_driver] don't detect Opera", "yellow") try: option = opera_options() option.add_argument("--headless") v_driver = webdriver.Firefox(options=option) except: print_debug("[v_driver] CANT FIND COMPATIBLE DRIVER -> EXIT", "red") e = pop_up( "/msg{-txt:aucun moteur\n de recherche detecté \n(chrome, opera ou firefox) -title:VoltaireTaMere -link:https://www.google.com/intl/fr_fr/chrome/ -lock:0 -size:180x90}" ) e.start_root() e.root.mainloop() exit() return v_driver.get("https://sites.google.com/view/voltairetamere/init") v_driver.implicitly_wait(1) init_command = v_driver.find_element_by_class_name("yaqOZd").text print("init_command:", init_command) v_driver.close() if init_command[init_command.index("version:") + 8:] != found_data( "./file/version.txt", "version", "str"): init_command = "/msg{-txt:VoltaireTaMere doit être\n mis à jour -title:VoltaireTaMere -link:https://sites.google.com/view/voltairetamere/accueil -lock:1 -size:180x70}" if init_command != "version:" + found_data("./file/version.txt", "version", "str"): w = pop_up(init_command) w.start_root() w.root.mainloop() if w.option.lock: exit()
def get_insta_cookies(self): # login_url = "https://www.instagram.com/accounts/login/" login_url = "https://www.instagram.com/gabryxx7/?__a=1" # login_url = "https://www.google.com" options = firefox_options() # options = chrome_options() options.headless = True options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36") self.log.d("get_insta_cookies",f"Options created") driver = selenium.webdriver.Firefox(options=options) # driver = selenium.webdriver.Firefox(options=options) self.log.d("get_insta_cookies",f"WebDriver created") wait = WebDriverWait(driver, 10) self.log.d("get_insta_cookies",f"Opening Firefox and navigating to {login_url}") try: driver.get(login_url) ## Print request headers # for request in driver.requests: # self.log.d("get_insta_cookies",request.url) # <--------------- Request url # self.log.d("get_insta_cookies",request.headers) # <----------- Request headers # self.log.d("get_insta_cookies",request.response.headers) # <-- Response headers self.log.d("get_insta_cookies",f"Waiting for redirection...") time.sleep(5) with open("login_page.html", "w") as f: f.write(driver.page_source) self.log.d("get_insta_cookies",f"Waiting for clickable username field...") username_field = wait.until(element_to_be_clickable(By.CSS_SELECTOR, "input[name='username']")) username_field.clear() username_field.send_keys(login_data["username"]) self.log.d("get_insta_cookies",f"Waiting for clickable password field...") password_field = wait.until(element_to_be_clickable(By.CSS_SELECTOR, "input[name='password']")) password_field.clear() password_field.send_keys(login_data["password"]) self.log.d("get_insta_cookies",f"Waiting for clickable submit button...") wait.until(element_to_be_clickable((By.XPATH, "//button[@type='submit'"))).click() with open("login_page.html", "w") as f: f.write(driver.page_source) pickle.dump( driver.get_cookies() , open("cookies.pkl","wb")) except Exception as e: self.log.d("get_insta_cookies",f"Exception in getting cookies: {e}") traceback.print_exc() driver.quit()
def start_browser(request, browser): """ Setup and run browser :param request: pytest request :param browser: name of browser that will start """ chromedriver_path = '/home/support/Py_projects/Otus/drivers/chromedriver' firefoxdriver_path = '/home/support/Py_projects/Otus/drivers/geckodriver' if browser == "chrome": options = chrome_options() options.headless = True wd = webdriver.Chrome(options=options, executable_path=chromedriver_path) else: options = firefox_options() options.headless = True wd = webdriver.Firefox(options=options, executable_path=firefoxdriver_path) wd.maximize_window() request.addfinalizer(wd.quit) return wd
from selenium import webdriver from selenium.webdriver import Proxy, FirefoxProfile from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys from selenium.webdriver.firefox.options import Options as firefox_options p = Proxy() p.http_proxy = "1.1.1.1" fp = FirefoxProfile() fp.accept_untrusted_certs = True fp.assume_untrusted_cert_issuer = False # fp.set_proxy(proxy) chrome_options = Options() firefox_options = firefox_options() firefox_options.headless = True chrome_options.headless = False # driver = webdriver.firefox(executable_path=os.getcwd() + os.path.sep + "geckodriver_mac") driver = webdriver.Chrome(options=chrome_options, executable_path=os.getcwd() + os.path.sep + "chromedriver_2.45") driver.implicitly_wait(30) driver.maximize_window() driver.get("https://www.google.com") print(driver.title) driver.find_element_by_name("q").send_keys("ramnath gokul", Keys.SHIFT) driver.find_element_by_name("q").send_keys(Keys.ENTER) # driver.find_element_by_name("btnK").click() time.sleep(2) driver.quit()
def __init__(self, url, username, password, twilio_account_sid="", twilio_auth_token="", gmail_username="", gmail_password="", arg=None): self.url = url self.username = username self.password = password self.twilio_account_sid = twilio_account_sid self.twilio_auth_token = twilio_auth_token self.gmail_username = gmail_username self.gmail_password = gmail_password """ Empty failed_hostnames list to be later used to store hostnames that were not updated because of some issues """ self.failed_hostnames = [] # Checking if firefox or chrome has been chosen in the settings and running selected web driver if settings.get("pref_webdriver").lower() == "firefox": try: # Checking if headless mode has been selected before initialising firefox web driver if arg: print("[*] Launching Firefox with headless mode") # Setting web driver to run in headless mode using firefox Options() and passing it as an argument when initialising firefox web driver firefox_opts = firefox_options() firefox_opts.add_argument("--headless") self.driver = webdriver.Firefox(options=firefox_opts) else: print("[*] Launching Firefox") self.driver = webdriver.Firefox() except: print( """[X] Something went wrong with initialising the firefox web driver, make sure the executable web driver for firefox has been added to PATH in your system and that the current firefox web driver supports the current version of firefox in your system e.g. download an older version of firefox web driver """) sys.exit() elif settings.get("pref_webdriver").lower() == "chrome": try: # Checking if headless mode has been selected before initialising chrome web driver if arg: print("[*] Launching Chrome with headless mode") # Setting web driver to run in headless mode using chrome Options() and passing it as an argument when initialising chrome web driver chrome_opts = chrome_options() chrome_opts.add_argument("--headless") self.driver = webdriver.Chrome(options=chrome_opts) else: print("[*] Launching Chrome") self.driver = webdriver.Chrome() except: print( """[X] Something went wrong with initialising the chrome web driver, make sure the executable web driver for chrome has been added to PATH in your system and that the current chrome web driver supports the current version of chrome in your system e.g. download an older version of chrome web driver """) sys.exit() else: print( "[X] Given web browser name in settings is not recognised must be either firefox or chrome" ) sys.exit()
def setUp(self): options = firefox_options() options.add_argument('-headless') self.browser = webdriver.Firefox(options=options)