def get_driver(self):
        """
        :return: Selenium driver
        """
        # self.driver_instance = singleton()
        # if self.driver_instance is not None:

        if TestData.BROWSER_TYPE.lower() == "chrome":
            return webdriver.Chrome()

        elif TestData.BROWSER_TYPE.lower() == "firefox":
            cap = DesiredCapabilities().FIREFOX
            cap["marionette"] = False
            return webdriver.Firefox(
                capabilities=cap,
                executable_path=TestData.FIREFOX_EXECUTABLE_PATH)
        elif TestData.BROWSER_TYPE.lower() == "safari":
            return webdriver.Safari()

        elif TestData.BROWSER_TYPE.lower() == "appium":
            self.dc = {}
            self.dc['udid'] = TestData.UDID
            self.dc['platformName'] = TestData.PLATFORM_NAME
            self.dc['deviceName'] = TestData.DEVICE_NAME
            self.dc['appActivity'] = TestData.APP_ACTIVITY
            self.dc['appPackage'] = TestData.APP_PACKAGE
            self.dc['noReset'] = TestData.NO_RESET
            self.dc['autoAcceptAlerts'] = TestData.AUTO_ACCEPET_ALERTS
            self.driver = webdriver.Remote(TestData.SERVER, self.dc)
            return self.driver

        elif TestData.BROWSER_TYPE.lower() == "api":
            return True
        else:
            raise Exception('There was problem with your driver')
Beispiel #2
0
    def driver_get(self, url):
        if 'login' in url or url == 'http://nightmareocykhgs.onion':
            if self.LOGIN:
                return
        print('driver get ', url)
        self.driver.get(url)
        try:
            time.sleep(10)
            self.driver.find_element_by_css_selector('body')
        except Exception as __:
            restart_tor()
            time.sleep(10)
            self.driver_get(url)
            return

        if 'main-frame-error' in self.driver.page_source:
            restart_tor()
            self.LOGIN = False
            self.driver.close()
            desired_capabilities = DesiredCapabilities().CHROME
            desired_capabilities['pageLoadStrategy'] = 'none'
            options = ChromeOptions()
            options.add_argument('--proxy-server=socks5://127.0.0.1:9050')
            self.driver = Chrome(options=options, desired_capabilities=desired_capabilities)
            if not self.LOGIN:
                self.login()
            self.driver_get(url)
            time.sleep(10)
            return
Beispiel #3
0
def get_firefox_driver():
    profile = FirefoxProfile()
    profile.set_preference('browser.cache.disk.enable', False)
    profile.set_preference('browser.cache.memory.enable', False)
    profile.set_preference('browser.cache.offline.enable', False)
    profile.set_preference('network.cookie.cookieBehavior', 2)
    if os.environ.get("DEV"):
        profile.set_preference("network.proxy.type", 1)
        profile.set_preference("network.proxy.socks", "127.0.0.1")
        profile.set_preference("network.proxy.socks_port", 8050)
        profile.set_preference("network.proxy.socks_version", 5)
    profile.update_preferences()
    caps = DesiredCapabilities().FIREFOX
    caps["pageLoadStrategy"] = "none"
    options = firefox_options()
    options.headless = True
    options.add_argument('start-maximized')
    path = os.path.dirname(__file__)
    if platform.system() == "Linux":
        path = os.path.join(path, r"driver/geckodriver")
    else:
        path = os.path.join(path, r"driver/geckodriver_mac")
    driver = webdriver.Firefox(executable_path=path,
                               firefox_options=options,
                               desired_capabilities=caps,
                               firefox_profile=profile)
    driver.set_window_size(1920, 1080)
    driver.implicitly_wait(1)
    return driver
Beispiel #4
0
def driver(config):
    driver = None

    if config['selenoid_host'] is not None:
        capabilities = {
            "browserName": "chrome",
            "browserVersion": "86.0",
            "selenoid:options": {
                "enableVNC": True,
                "enableVideo": False
            }
        }
        selenoid_url = 'http://' + config['selenoid_host'] + '/wd/hub'
        options = ChromeOptions()
        driver = webdriver.Remote(command_executor=selenoid_url,
                                  options=options,
                                  desired_capabilities=capabilities)
    else:
        options = ChromeOptions()
        driver = webdriver.Chrome(
            ChromeDriverManager().install(),
            options=options,
            desired_capabilities=DesiredCapabilities().CHROME)
    driver.maximize_window()
    driver.get(config['url'])
    yield driver
    driver.quit()
def init_driver(user_data_dir="", profile_dir="", headless=True):
    global driver, storage

    options = Options()

    if headless:
        options.add_argument("--headless")

        # disable image loading
        chrome_prefs = {}
        options.experimental_options["prefs"] = chrome_prefs
        chrome_prefs["profile.default_content_settings"] = {"images": 2}
        chrome_prefs["profile.managed_default_content_settings"] = {
            "images": 2
        }

    elif user_data_dir != "":
        options.add_argument(f"user-data-dir={user_data_dir}")
        options.add_argument(f"profile-directory={profile_dir}")

    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    options.set_capability('unhandledPromptBehavior', 'dismiss')

    # Page load strategy none doesnt wait for the page to fully load before continuing
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "none"

    driver = webdriver.Chrome(ChromeDriverManager().install(),
                              chrome_options=options,
                              desired_capabilities=caps)

    storage = LocalStorage(driver)
Beispiel #6
0
    def __init__(self):

        pf = {
            'browser.helperApps.neverAsk.saveToDisk':
            'application/x-netcdf, application/netcdf',
            'browser.download.manager.closeWhenDone': True,
            'browser.download.manager.showWhenStarting': False,
            'browser.download.folderList': 2,
            'browser.download.dir': AbstractBrowserBasedTest.DOWNLOAD_DIR_PATH
        }

        AbstractBrowserBasedTest._firefox_profile = FirefoxProfile()  # profile
        AbstractBrowserBasedTest._firefox_profile.set_preference(
            'extensions.logging.enabled', False)
        AbstractBrowserBasedTest._firefox_profile.set_preference(
            'network.dns.disableIPv6', False)

        for key, value in list(pf.items()):
            AbstractBrowserBasedTest._firefox_profile.set_preference(
                key, value)

        AbstractBrowserBasedTest._firefox_capabilities = DesiredCapabilities(
        ).FIREFOX
        AbstractBrowserBasedTest._firefox_capabilities['marionette'] = True
        AbstractBrowserBasedTest._firefox_capabilities[
            'moz:firefoxOptions'] = {
                'args': [self.__get_arg()]
            }

        urllib3.disable_warnings()
 def get_driver(self):
     # server = Server(r'/home/reggie/Documents/app/browsermob-proxy-2.1.4/bin/browsermob-proxy')
     server = Server(r'/root/browsermob-proxy-2.1.4/bin/browsermob-proxy')
     server.start()
     print('server started')
     # map={'http':'http://officepx.datayes.com','https':'http://officepx.datayes.com'}
     proxy = server.create_proxy()
     print('proxy created')
     caps = DesiredCapabilities().CHROME
     caps["pageLoadStrategy"] = "none"
     chrome_options = Options()
     chrome_options.add_argument('--ignore-ssl-errors=yes')
     chrome_options.add_argument('--ignore-certificate-errors')
     prefs = {'profile.managed_default_content_settings.images': 2}
     chrome_options.add_experimental_option('prefs', prefs)
     chrome_options.add_argument('--no-sandbox')
     chrome_options.add_argument('--headless')
     chrome_options.add_argument('--disable-gpu')
     chrome_options.add_argument('--disable-dev-shm-usage')
     chrome_options.add_experimental_option('excludeSwitches',
                                            ['enable-automation'])  # 以开发者模式
     chrome_options.add_argument('--proxy-server={0}'.format(proxy.proxy))
     drive = webdriver.Chrome(options=chrome_options,
                              desired_capabilities=caps)
     print('driver opened')
     return drive, server, proxy
Beispiel #8
0
def prepare_driver_chrome(url):
    """
    lunch selenium chrome driver in headless mode of input url
    :param url:the url for browsing
    :type url str
    :return:chrome driver with open
    :rtype:selenium.webdriver.chrome.webdriver.WebDriver
    """
    options = Options()
    #options.add_argument('-headless')
    options.add_argument("--lang=en-gb")
    options.add_argument('window-size=1920x1080')
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "normal"
    if platform.system() == 'Windows':
        driver = selenium.webdriver.Chrome(
            executable_path=
            'C:\\Users\\Ziv\\PycharmProjects\\TravelScanner\\chromedriver.exe',
            options=options,
            desired_capabilities=caps)
    else:
        driver = selenium.webdriver.Chrome(
            executable_path='/usr/bin/chromedriver',
            options=options,
            desired_capabilities=caps)
    try:
        driver.get(url)
    except Exception:
        traceback.print_exc()
    driver.implicitly_wait(0)
    return driver
Beispiel #9
0
    def __init__(
        self,
        user_agent=None,
        wait_time=2,
        fullscreen=False,
        options=None,
        headless=False,
        desired_capabilities=None,
        **kwargs,
    ):

        options = Options() if options is None else options

        if user_agent is not None:
            options.add_argument("--user-agent=" + user_agent)

        options.add_argument("--use-fake-device-for-media-stream")
        options.add_argument("--use-fake-ui-for-media-stream")
        if fullscreen:
            options.add_argument("--kiosk")

        if headless:
            # noinspection PyDeprecation
            # windows: chrome version >= 60
            options.set_headless()

        prefs = {
            "download": {
                "default_directory": _DOWNLOAD_PATH,
                "directory_upgrade": True,
                "extensions_to_open": "",
            }
        }

        options.add_experimental_option("prefs", prefs)

        chrome_capabilities = DesiredCapabilities().CHROME.copy()
        if desired_capabilities:
            chrome_capabilities.update(desired_capabilities)

        self.driver = Chrome(chrome_options=options,
                             desired_capabilities=chrome_capabilities,
                             **kwargs)
        self.element_class = WebDriverElement
        self._cookie_manager = CookieManager(self.driver)

        super(WebDriver, self).__init__(wait_time)
def gotoSymphonyEdge():
    cap = DesiredCapabilities().EDGE
    cap["platform"] = "ANY"
    driver = webdriver.Edge(executable_path=HOME + "/PycharmProjects/pythonProject2/SymphonyTest/driver/msedgedriver",
                            capabilities=cap)
    driver.get(symphonyURL)
    driver.maximize_window()
    return driver
Beispiel #11
0
    def __init__(self,
                 profile=None,
                 extensions=None,
                 user_agent=None,
                 profile_preferences=None,
                 fullscreen=False,
                 wait_time=2,
                 timeout=90,
                 capabilities=None,
                 headless=False,
                 incognito=False,
                 **kwargs):

        firefox_profile = FirefoxProfile(profile)
        firefox_profile.set_preference("extensions.logging.enabled", False)
        firefox_profile.set_preference("network.dns.disableIPv6", False)

        firefox_capabilities = DesiredCapabilities().FIREFOX
        firefox_capabilities["marionette"] = True

        firefox_options = Options()

        if capabilities:
            for key, value in capabilities.items():
                firefox_capabilities[key] = value

        if user_agent is not None:
            firefox_profile.set_preference("general.useragent.override",
                                           user_agent)

        if profile_preferences:
            for key, value in profile_preferences.items():
                firefox_profile.set_preference(key, value)

        if extensions:
            for extension in extensions:
                firefox_profile.add_extension(extension)

        if headless:
            firefox_options.add_argument("--headless")

        if incognito:
            firefox_options.add_argument("-private")

        self.driver = Firefox(firefox_profile,
                              capabilities=firefox_capabilities,
                              options=firefox_options,
                              timeout=timeout,
                              **kwargs)

        if fullscreen:
            self.driver.fullscreen_window()

        self.element_class = WebDriverElement

        self._cookie_manager = CookieManager(self.driver)

        super(WebDriver, self).__init__(wait_time)
Beispiel #12
0
    def setUp(self):        

        print("xxx xxx xxx BaseTestCase.setup xxx")
        #_download_dir = "/tmp"
        self._download_dir = tempfile.mkdtemp()
        print("...download_dir: {d}".format(d=self._download_dir))
        browser = config[BROWSER_SECTION][BROWSER_KEY]
        options = Options()
        #options.add_argument("--headless")
        options.add_argument("--foreground")
        #mode = "--headless"
        mode = "--foreground"

        if browser == 'chrome':
            #options.binary_location = "/usr/local/bin/chromedriver"
            chrome_options = webdriver.ChromeOptions()
            #chrome_options.binary_location = "/usr/local/bin/chromedriver"
            chrome_options.add_argument(mode)
            preferences = {"download.default_directory": self._download_dir,
                           "directory_upgrade": True,
                           "safebrowsing.enabled": True,
                           "prompt_for_download": "false"}
            chrome_options.add_experimental_option("prefs", preferences)
            self.driver = webdriver.Chrome(options=chrome_options)
        elif browser == 'firefox':
            firefox_profile = FirefoxProfile() # profile                                                                            
            firefox_profile.set_preference('extensions.logging.enabled', False)
            firefox_profile.set_preference('network.dns.disableIPv6', False)
            firefox_profile.set_preference('browser.download.dir', self._download_dir)
            firefox_profile.set_preference('browser.download.folderList', 2)
            firefox_profile.set_preference('browser.download.useDownloadDir', True)
            firefox_profile.set_preference('browser.download.panel.shown', False)
            firefox_profile.set_preference('browser.download.manager.showWhenStarting', False)
            firefox_profile.set_preference('browser.download.manager.showAlertOnComplete', False)
            firefox_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/x-netcdf");
            
            firefox_capabilities = DesiredCapabilities().FIREFOX
            firefox_capabilities['marionette'] = True
            #firefox_capabilities['moz:firefoxOptions'] = {'args': ['--headless']}

            options.binary_location = "/usr/local/bin/geckodriver"
            firefox_binary = FirefoxBinary("/usr/bin/firefox")
            #self.driver = webdriver.Firefox(firefox_profile=firefox_profile,
            #                                firefox_binary=firefox_binary,
            #                                options=options,
            #                                capabilities = firefox_capabilities)

            self.driver = webdriver.Firefox(firefox_profile=firefox_profile,
                                            firefox_binary=firefox_binary,
                                            executable_path="/usr/local/bin/geckodriver",
                                            options=options,
                                            capabilities = firefox_capabilities)

        self.driver.implicitly_wait(10)
        idp_server = self._get_idp_server()
        self.driver.get("https://{n}".format(n=idp_server))
        time.sleep(3)
Beispiel #13
0
def spawn_driver():
    caps = DesiredCapabilities().FIREFOX
    caps["pageLoadStrategy"] = "normal"
    options = Options()
    options.add_argument("--headless")
    driver_path = os.getcwd() + '\\geckodriver.exe'
    driver = webdriver.Firefox(options=options,
                               desired_capabilities=caps,
                               executable_path=driver_path)
    return driver
Beispiel #14
0
    def __init__(self,
                 profile=None,
                 extensions=None,
                 user_agent=None,
                 profile_preferences=None,
                 fullscreen=False,
                 wait_time=2,
                 timeout=90,
                 capabilities=None,
                 headless=False,
                 **kwargs):

        firefox_profile = FirefoxProfile(profile)
        firefox_profile.set_preference('extensions.logging.enabled', False)
        firefox_profile.set_preference('network.dns.disableIPv6', False)

        firefox_capabilities = DesiredCapabilities().FIREFOX
        firefox_capabilities["marionette"] = True

        if capabilities:
            for key, value in capabilities.items():
                firefox_capabilities[key] = value

        if user_agent is not None:
            firefox_profile.set_preference('general.useragent.override',
                                           user_agent)

        if profile_preferences:
            for key, value in profile_preferences.items():
                firefox_profile.set_preference(key, value)

        if extensions:
            for extension in extensions:
                firefox_profile.add_extension(extension)

        if headless:
            os.environ.update({"MOZ_HEADLESS": '1'})
            binary = FirefoxBinary()
            binary.add_command_line_options('-headless')
            kwargs['firefox_binary'] = binary

        self.driver = Firefox(firefox_profile,
                              capabilities=firefox_capabilities,
                              timeout=timeout,
                              **kwargs)

        if fullscreen:
            ActionChains(self.driver).send_keys(Keys.F11).perform()

        self.element_class = WebDriverElement

        self._cookie_manager = CookieManager(self.driver)

        super(WebDriver, self).__init__(wait_time)
Beispiel #15
0
def get_page_by_selenium(url):
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "none"
    if platform.system() == "Linux":
        path = r"./driver/chromedriver"
    else:
        path = r"./driver/chromedriver_mac"
    driver = webdriver.Chrome(path, desired_capabilities=caps)
    driver.implicitly_wait(1)
    driver.get(url)
    return driver
Beispiel #16
0
def start_firefox(proxy=None, user_agent=None):
    p = FirefoxProfile("E://profiles")
    p.accept_untrusted_certs = True
    if user_agent:
        p.set_preference("general.useragent.override", user_agent)
    capabilities = DesiredCapabilities().FIREFOX.copy()
    capabilities['acceptInsecureCerts'] = True

    if proxy:
        p.set_proxy(proxy)
    browser = Firefox(firefox_profile=p, capabilities=capabilities)
    return browser
    def __init__(self, *args, **kwargs):
        if 'positions' and 'properties' not in kwargs.keys(): raise Exception('Check Keyword arguments...')
        self.position = kwargs['positions']
        self.property = kwargs['properties']

        binary = FirefoxBinary(FIREFOX_PATH)
        cap = DesiredCapabilities().FIREFOX
        cap["marionette"] = False

        self.driver = webdriver.Firefox(executable_path = FIREFOX_PATH, firefox_binary = binary, capabilities = cap)
        self.driver.maximize_window()
        self.debug_mouse_position = debug_mouse_position
Beispiel #18
0
 def __init__(self):
     options = webdriver.FirefoxOptions()
     options.binary_location = os.getenv('FIREFOX_BIN')
     options.add_argument("--headless")
     options.add_argument('--disable-gpu')
     options.add_argument('--no-sandbox')
     cap = DesiredCapabilities().FIREFOX
     cap["marionette"] = True
     self.driver = webdriver.Firefox(
         capabilities=cap,
         executable_path=os.getenv('GECKODRIVER_PATH'),
         options=options)
    def get(self):
        options = FFOptions()
        options.headless = self.options.headless

        cap = DesiredCapabilities().FIREFOX
        sys.path.insert(0, self.driver_path)
        drv = webdriver.Firefox(options=options, capabilities=cap,
                                executable_path=self.driver_path)

        self.set_window_size(drv)
        self.set_window_position(drv)

        return drv
Beispiel #20
0
    def setup_for_firefox(self, mode):
        firefox_profile = FirefoxProfile()
        firefox_profile.set_preference('dom.disable_open_during_load', False)
        firefox_capabilities = DesiredCapabilities().FIREFOX
        firefox_capabilities['marionette'] = True
        firefox_capabilities['moz:firefoxOptions'] = {'args': ['--headless']}

        firefox_binary = FirefoxBinary(os.getenv("BROWSER_BINARY", "/usr/bin/firefox"))
        geckodriver_loc = os.getenv("BROWSER_DRIVER", "/usr/local/bin/geckodriver")
        self.driver = webdriver.Firefox(firefox_profile=firefox_profile,
                                        firefox_binary=firefox_binary,
                                        executable_path=geckodriver_loc,
                                        capabilities=firefox_capabilities)
Beispiel #21
0
def create_driver(tp, ip_port):
    port = utils.free_port()
    options = ChromeOptions()
    desired_capabilities = DesiredCapabilities().CHROME
    desired_capabilities['pageLoadStrategy'] = 'none'
    os.popen(
        'chrome.exe --remote-debugging-port={} --user-data-dir={} --proxy-server={}={}'
        .format(port, tempfile.mkdtemp(), tp, ip_port))
    options.add_experimental_option('debuggerAddress',
                                    '127.0.0.1:{}'.format(port))
    driver = Chrome(options=options, desired_capabilities=desired_capabilities)
    driver.set_window_position(0, 0)
    driver.set_window_size(700, 600)
    return driver
def setUp(browser):
    if browser == "chrome":
        driver = webdriver.Chrome()
        print("Launching Chrome browser")

    elif browser == "firefox":
        cap = DesiredCapabilities().FIREFOX
        cap["marionette"] = False
        driver = webdriver.Firefox(capabilities=cap)
        print("Launching Firefox browser")

    else:
        driver = webdriver.Ie()
        print("Launching IE browser")
    return driver
Beispiel #23
0
 def __init_driver(self, ip=None, headless=True):
     if ip == None: ip = self.ip
     chrome_option = webdriver.ChromeOptions()
     # proxies, ip = self.get_ip_proxy()
     chrome_option.add_argument('--proxy-server=http://%s' % ip)
     if headless is True:
         chrome_option.add_argument('--headless')
         chrome_option.add_argument("--no-sandbox")
     desired_caps = DesiredCapabilities().CHROME
     desired_caps['pageloadStrategy'] = 'normal'
     args = {
         'desired_capabilities': desired_caps,
         'chrome_options': chrome_option
     }
     driver = webdriver.Chrome(**args)
     return driver
Beispiel #24
0
    def init(self):
        path = "/Users/maiavinicius/PycharmProjects/minha_carteira/driver/chromedriver-mac"
        caps = DesiredCapabilities().CHROME
        # caps["pageLoadStrategy"] = "normal"  #  complete
        # caps["pageLoadStrategy"] = "eager"  # interactive
        # caps["pageLoadStrategy"] = "none"

        self.driver = webdriver.Chrome(path, desired_capabilities=caps)

        self._get(self.initial_url)
        self.login()
        self.wait_login()
        res = self.scrap()

        self.driver.close()

        return res
Beispiel #25
0
def init_browser():
    # Set up options
    opts = Options()
    opts.headless = True
    assert opts.headless  # Operating in headless mode
    # Set up capabilities
    capabilities = DesiredCapabilities().FIREFOX
    capabilities['marionette'] = False
    # Create browser instance
    browser = Firefox(
        options=opts,
        firefox_binary=FirefoxBinary(SELENIUM_BINARY_PATH),
        executable_path=SELENIUM_EXECUTABLE_PATH,
        capabilities=capabilities
    )
    browser.get(FACTORIAL_WEBSITE_INIT)
    return browser
Beispiel #26
0
def open_itax():
    cap = DesiredCapabilities().FIREFOX
    cap["marionette"] = False
    driver = webdriver.Firefox(capabilities=cap)
    driver.get('https://itax.kra.go.ke/')
    time.sleep(3)  # sleep for 3 seconds after page has loaded
    inputpin = driver.find_element_by_xpath('//*[@id="logid"]')
    inputpin.send_keys(secrets.pin)
    time.sleep(3)  # sleep for 10 seconds to see input
    signinbtn = driver.find_element_by_xpath(
        '//*[@id="normalDiv"]/table/tbody/tr[3]/td[2]/a')
    signinbtn.click()
    time.sleep(4)
    downloadimage(driver, '//*[@id="captcha_img"]')
    time.sleep(3)
    driver.find_element_by_xpath('//*[@id="captcahText"]').send_keys(
        readimage())
    time.sleep(5)
    driver.find_element_by_xpath('//*[@id="xxZTT9p2wQ"]').send_keys(
        secrets.password)
    time.sleep(3)
    driver.find_element_by_xpath('//*[@id="loginButton"]').click()
    time.sleep(8)
    action = ActionChains(driver)
    action.move_to_element(
        driver.find_element_by_xpath(
            '//*[@id="ddtopmenubar"]/ul/li[3]/a')).perform()
    # move to file nil returns
    filenil = driver.find_element_by_xpath('//*[@id="Returns"]/li[4]/a')
    # hover over the item
    action.move_to_element(filenil).perform()
    # now click on the item
    filenil.click()
    time.sleep(3)
    # find the  options
    driver.find_element_by_xpath('//*[@id="regType"]').click()
    # and hover over resident
    # click on it
    driver.find_element_by_xpath('//*[@id="regType"]/option[2]').click()
    time.sleep(2)
    # click the next button
    driver.find_element_by_xpath('//*[@id="btnSubmit"]')
    time.sleep(10)

    driver.quit()
Beispiel #27
0
    def __init__(self,
                 profile=None,
                 extensions=None,
                 user_agent=None,
                 profile_preferences=None,
                 fullscreen=False,
                 wait_time=2,
                 timeout=90,
                 capabilities=None):

        firefox_profile = FirefoxProfile(profile)
        firefox_profile.set_preference('extensions.logging.enabled', False)
        firefox_profile.set_preference('network.dns.disableIPv6', False)

        firefox_capabilities = DesiredCapabilities().FIREFOX
        firefox_capabilities["marionette"] = True

        if capabilities:
            for key, value in capabilities.items():
                firefox_capabilities[key] = value

        if user_agent is not None:
            firefox_profile.set_preference('general.useragent.override',
                                           user_agent)

        if profile_preferences:
            for key, value in profile_preferences.items():
                firefox_profile.set_preference(key, value)

        if extensions:
            for extension in extensions:
                firefox_profile.add_extension(extension)

        self.driver = Firefox(firefox_profile,
                              capabilities=firefox_capabilities,
                              timeout=timeout)

        if fullscreen:
            ActionChains(self.driver).send_keys(Keys.F11).perform()

        self.element_class = WebDriverElement

        self._cookie_manager = CookieManager(self.driver)

        super(WebDriver, self).__init__(wait_time)
    def process_request(self, request, spider):
        print(f"chrome is getting page")

        options = Options()
        options.add_argument('--headless')
        options.add_argument('--disable-gpu')

        # https://stackoverflow.max-everyday.com/2019/08/selenium-pageloadstrategy/
        caps = DesiredCapabilities().CHROME
        caps["pageLoadStrategy"] = "none"
        # full page load
        # caps["pageLoadStrategy"] = "normal"
        # interactive
        # caps["pageLoadStrategy"] = "eager"

        print(f"Creating driver for request")
        driver = webdriver.Chrome(desired_capabilities=caps,
                                  chrome_options=options)

        usedSelenium = request.meta.get('usedSelenium', False)
        if usedSelenium:
            try:
                driver.get(request.url)
                isRoot = request.meta.get('root', False)
                if isRoot:
                    spider.start_interaction(driver)
                else:
                    spider.start_sub_interaction(driver)

            except Exception as e:
                print(f"chrome getting page error, Exception = {e}")

            else:
                response = driver.page_source
                driver.quit()

                print(f"return html response")
                return HtmlResponse(url=request.url,
                                    body=response,
                                    request=request,
                                    encoding='utf-8',
                                    status=200)
Beispiel #29
0
def get_chrome_driver():
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "none"
    options = chrome_options()
    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36'
    options.add_argument('--headless')
    options.add_argument('--no-sandbox')
    if os.environ.get("DEV"):
        options.add_argument("--proxy-server=socks5://127.0.0.1:8050")
    options.add_argument('start-maximized')
    options.add_argument(f'user-agent={user_agent}')
    path = os.path.dirname(__file__)
    if platform.system() == "Linux":
        path = os.path.join(path, r"driver/chromedriver")
    else:
        path = os.path.join(path, r"driver/chromedriver_mac")
    driver = webdriver.Chrome(executable_path=path, desired_capabilities=caps, chrome_options=options)
    driver.set_window_size(1920, 1080)
    hide_webdriver(driver)
    return driver
    def get_local_driver(browser_name):
        browser_name = browser_name.lower()
        path = config.get_path_to_local_webdriver(browser_name)

        try:
            if browser_name == 'firefox':
                driver = webdriver.Firefox(executable_path=path)
            elif browser_name == 'chrome':
                driver = webdriver.Chrome(executable_path=path)
            elif browser_name == 'safari':
                caps = DesiredCapabilities().SAFARI
                caps["pageLoadStrategy"] = "normal"
                driver = webdriver.Safari(desired_capabilities=caps)
            else:
                sys.exit("Browser type incorrectly defined")
        except Exception as e:
            logging.error(e)
            raise e
        driver.maximize_window()
        return driver