コード例 #1
0
 def testCanNotChangeInitializedProxyType(self):
     proxy = Proxy(raw={'proxyType': 'direct'})
     try:
         proxy.proxy_type = ProxyType.SYSTEM
         raise Exception("Change of already initialized proxy type should raise exception")
     except Exception as e:
         pass
コード例 #2
0
 def getDriver(self, driverType='firefox'):
     try:
         if driverType == 'phantomjs':
             #capabilities = webdriver.DesiredCapabilities.PHANTOMJS.copy()
             #capabilities['platform'] = "LINUX"
             driver = webdriver.PhantomJS('/usr/local/share/phantomjs/bin/phantomjs')
         else:
             from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
             self.dis = Display(visible=0,size=(800,600))
             self.dis.start()
             prx = Proxy()
             prx.httpProxy= '199.200.120.36:8089' #'112.196.3.254:9064'  india
             prx.proxyType = ProxyType.MANUAL
             prof = FirefoxProfile('./ffprofile')
             driver = webdriver.Firefox(proxy=prx, firefox_profile=prof)
         driver.implicitly_wait(30)
         driver.set_page_load_timeout(30)
         print 'started the driver successfully...'
         return driver
     except Exception as e:
         print 'error has occured while starting the driver....'
         print type(e)
         print e
         self.quit()
         #sys.exit(0)
         return None
コード例 #3
0
def tt1():
    desired_capabilities = DesiredCapabilities.PHANTOMJS.copy()
    headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
               'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
               'Cache-Control': 'max-age=0',
               'Connection': 'keep-alive',
               'Host': 'www.dianping.com',
               }
    for key, value in headers.iteritems():
        desired_capabilities['phantomjs.page.customHeaders.{}'.format(key)] = value
    desired_capabilities[
        'phantomjs.page.customHeaders.User-Agent'] = \
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) ' \
        'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 ' \
        'Safari/604.1.38'
    ip_port = random.choice(redis_conn1())
    print ip_port
    proxy = Proxy(
        {
            'proxyType': ProxyType.MANUAL,
            'httpProxy': '%s' % ip_port  # 代理ip和端口
        }
    )

    proxy.add_to_capabilities(desired_capabilities)
    print desired_capabilities
    driver = webdriver.PhantomJS(desired_capabilities=desired_capabilities)
    driver.set_page_load_timeout(10)
    driver.get("http://www.dianping.com/shop/%s" % ['76964345', '15855144', ])
    list1 = driver.find_elements_by_xpath('//div[@class="comment-condition J-comment-condition Fix"]/div/span/a')
    for l in list1:
        print l.text
    if '403 Forbidden' in driver.page_source:
        print driver.page_source
    driver.close()
コード例 #4
0
ファイル: ff_profile_tests.py プロジェクト: Allariya/selenium
def test_autodetect_proxy_is_set_in_profile():
    profile = FirefoxProfile()
    proxy = Proxy()
    proxy.auto_detect = True

    profile.set_proxy(proxy)
    assert profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']
コード例 #5
0
ファイル: ff_profile_tests.py プロジェクト: Allariya/selenium
def test_pac_proxy_is_set_in_profile():
    profile = FirefoxProfile()
    proxy = Proxy()
    proxy.proxy_autoconfig_url = 'http://some.url:12345/path'

    profile.set_proxy(proxy)
    assert profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
    assert profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'
コード例 #6
0
ファイル: proxy_tests.py プロジェクト: Allariya/selenium
def testCanNotChangeInitializedProxyType():
    proxy = Proxy(raw={'proxyType': 'direct'})
    with pytest.raises(Exception):
        proxy.proxy_type = ProxyType.SYSTEM

    proxy = Proxy(raw={'proxyType': ProxyType.DIRECT})
    with pytest.raises(Exception):
        proxy.proxy_type = ProxyType.SYSTEM
コード例 #7
0
    def test_what_things_look_like(self):
        bmp_capabilities = copy.deepcopy(selenium.webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX)
        self.client.add_to_capabilities(bmp_capabilities)

        proxy_capabilities = copy.deepcopy(selenium.webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX)
        proxy = Proxy({'httpProxy': 'localhost:%d' % self.client.port})
        proxy.add_to_capabilities(proxy_capabilities)

        assert bmp_capabilities == proxy_capabilities
コード例 #8
0
    def test_sets_ssl_proxy(self):
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.ssl_proxy = 'https://test.hostname:1234'
        profile.set_proxy(proxy)
        assert profile.default_preferences["network.proxy.type"] == str(ProxyType.MANUAL['ff_value'])
        assert profile.default_preferences["network.proxy.ssl"] == '"test.hostname"'
        assert profile.default_preferences["network.proxy.ssl_port"] == '1234'
コード例 #9
0
ファイル: proxy_tests.py プロジェクト: Allariya/selenium
def testCanAddAutodetectProxyToDesiredCapabilities():
    proxy = Proxy()
    proxy.auto_detect = AUTODETECT_PROXY['autodetect']

    desired_capabilities = {}
    proxy.add_to_capabilities(desired_capabilities)

    proxy_capabilities = AUTODETECT_PROXY.copy()
    proxy_capabilities['proxyType'] = 'AUTODETECT'
    expected_capabilities = {'proxy': proxy_capabilities}
    assert expected_capabilities == desired_capabilities
コード例 #10
0
ファイル: proxy_tests.py プロジェクト: 217/selenium
    def testCanAddPACProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.proxy_autoconfig_url = self.PAC_PROXY['proxyAutoconfigUrl']

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.PAC_PROXY.copy()
        proxy_capabilities['proxyType'] = 'PAC'
        expected_capabilities = {'proxy': proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #11
0
ファイル: ff_profile_tests.py プロジェクト: 217/selenium
    def test_autodetect_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.auto_detect = True

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']
コード例 #12
0
ファイル: proxy_tests.py プロジェクト: RanchoLi/selenium
    def testCanAddPACProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.proxy_autoconfig_url = self.PAC_PROXY["proxyAutoconfigUrl"]

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.PAC_PROXY.copy()
        proxy_capabilities["proxyType"] = "PAC"
        expected_capabilities = {"proxy": proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #13
0
ファイル: proxy_tests.py プロジェクト: 217/selenium
    def testCanAddAutodetectProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.auto_detect = self.AUTODETECT_PROXY['autodetect']

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.AUTODETECT_PROXY.copy()
        proxy_capabilities['proxyType'] = 'AUTODETECT'
        expected_capabilities = {'proxy': proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #14
0
ファイル: proxy_tests.py プロジェクト: Allariya/selenium
def testCanAddPACProxyToDesiredCapabilities():
    proxy = Proxy()
    proxy.proxy_autoconfig_url = PAC_PROXY['proxyAutoconfigUrl']

    desired_capabilities = {}
    proxy.add_to_capabilities(desired_capabilities)

    proxy_capabilities = PAC_PROXY.copy()
    proxy_capabilities['proxyType'] = 'PAC'
    expected_capabilities = {'proxy': proxy_capabilities}
    assert expected_capabilities == desired_capabilities
コード例 #15
0
ファイル: proxy_tests.py プロジェクト: RanchoLi/selenium
    def testCanAddAutodetectProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.auto_detect = self.AUTODETECT_PROXY["autodetect"]

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.AUTODETECT_PROXY.copy()
        proxy_capabilities["proxyType"] = "AUTODETECT"
        expected_capabilities = {"proxy": proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #16
0
ファイル: ff_profile_tests.py プロジェクト: 217/selenium
    def test_pac_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.proxy_autoconfig_url = 'http://some.url:12345/path'

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
        assert self.profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'
コード例 #17
0
ファイル: proxy_tests.py プロジェクト: RanchoLi/selenium
    def testCanNotChangeInitializedProxyType(self):
        proxy = Proxy(raw={"proxyType": "direct"})
        try:
            proxy.proxy_type = ProxyType.SYSTEM
            raise Exception("Change of already initialized proxy type should raise exception")
        except Exception:
            pass

        proxy = Proxy(raw={"proxyType": ProxyType.DIRECT})
        try:
            proxy.proxy_type = ProxyType.SYSTEM
            raise Exception("Change of already initialized proxy type should raise exception")
        except Exception:
            pass
コード例 #18
0
    def testCanAddToDesiredCapabilities(self):
        desired_capabilities = {}
        proxy = Proxy()
        proxy.http_proxy = 'some.url:1234'

        proxy.add_to_capabilities(desired_capabilities)

        expected_capabilities = {
            'proxy': {
                'proxyType': 'manual',
                'httpProxy': 'some.url:1234'
            }
        }

        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #19
0
ファイル: ff_profile_tests.py プロジェクト: Allariya/selenium
def test_manual_proxy_is_set_in_profile():
    profile = FirefoxProfile()
    proxy = Proxy()
    proxy.no_proxy = 'localhost, foo.localhost'
    proxy.http_proxy = 'some.url:1234'
    proxy.ftp_proxy = None
    proxy.sslProxy = 'some2.url'

    profile.set_proxy(proxy)
    assert profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
    assert profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
    assert profile.default_preferences["network.proxy.http"] == 'some.url'
    assert profile.default_preferences["network.proxy.http_port"] == 1234
    assert profile.default_preferences["network.proxy.ssl"] == 'some2.url'
    assert "network.proxy.ssl_port" not in profile.default_preferences
    assert "network.proxy.ftp" not in profile.default_preferences
コード例 #20
0
ファイル: proxy_tests.py プロジェクト: 217/selenium
    def testCanAddManualProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.http_proxy = self.MANUAL_PROXY['httpProxy']
        proxy.ftp_proxy = self.MANUAL_PROXY['ftpProxy']
        proxy.no_proxy = self.MANUAL_PROXY['noProxy']
        proxy.sslProxy = self.MANUAL_PROXY['sslProxy']
        proxy.socksProxy = self.MANUAL_PROXY['socksProxy']
        proxy.socksUsername = self.MANUAL_PROXY['socksUsername']
        proxy.socksPassword = self.MANUAL_PROXY['socksPassword']

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.MANUAL_PROXY.copy()
        proxy_capabilities['proxyType'] = 'MANUAL'
        expected_capabilities = {'proxy': proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #21
0
ファイル: selenate.py プロジェクト: jsimnz/selenate
    def __init__(self, host="127.0.0.1", server="./selenium-server.jar"):
        if server and path.isfile(server) and not _server_started():
            self.selenium = _Selenium(server)

        proxy = Proxy({
            'proxyType': ProxyType.MANUAL,
            'httpProxy': host,
            'ftpProxy': host,
            'sslProxy': host,
            'noProxy': host 
        })
        caps = webdriver.DesiredCapabilities.FIREFOX
        proxy.add_to_capabilities(caps)
        try:
            self.driver = webdriver.Remote(desired_capabilities=caps)
        except URLError:
            raise SeleniumServerError
コード例 #22
0
ファイル: proxy_tests.py プロジェクト: RanchoLi/selenium
    def testCanAddManualProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.http_proxy = self.MANUAL_PROXY["httpProxy"]
        proxy.ftp_proxy = self.MANUAL_PROXY["ftpProxy"]
        proxy.no_proxy = self.MANUAL_PROXY["noProxy"]
        proxy.sslProxy = self.MANUAL_PROXY["sslProxy"]
        proxy.socksProxy = self.MANUAL_PROXY["socksProxy"]
        proxy.socksUsername = self.MANUAL_PROXY["socksUsername"]
        proxy.socksPassword = self.MANUAL_PROXY["socksPassword"]

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.MANUAL_PROXY.copy()
        proxy_capabilities["proxyType"] = "MANUAL"
        expected_capabilities = {"proxy": proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #23
0
ファイル: selenate.py プロジェクト: dkua/selenate
    def __init__(self, host="127.0.0.1", server="./selenium-server.jar"):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        port = sock.connect_ex(("127.0.0.1", 4444)) == 0
        if server and path.isfile(server) and not port:
            self.selenium = _Selenium(server)

        proxy = Proxy({
            'proxyType': ProxyType.MANUAL,
            'httpProxy': host,
            'ftpProxy': host,
            'sslProxy': host,
            'noProxy': host 
        })
        caps = webdriver.DesiredCapabilities.FIREFOX
        proxy.add_to_capabilities(caps)
        try:
            self.driver = webdriver.Remote(desired_capabilities=caps)
        except URLError:
            raise SeleniumServerError
コード例 #24
0
ファイル: proxy_tests.py プロジェクト: Allariya/selenium
def testCanInitEmptyProxy():
    proxy = Proxy()
    assert ProxyType.UNSPECIFIED == proxy.proxy_type
    assert '' == proxy.http_proxy
    assert '' == proxy.ftp_proxy
    assert '' == proxy.no_proxy
    assert '' == proxy.sslProxy
    assert '' == proxy.socksProxy
    assert '' == proxy.socksUsername
    assert '' == proxy.socksPassword
    assert proxy.auto_detect is False
    assert '' == proxy.proxy_autoconfig_url

    desired_capabilities = {}
    proxy.add_to_capabilities(desired_capabilities)

    proxy_capabilities = {}
    proxy_capabilities['proxyType'] = 'UNSPECIFIED'
    expected_capabilities = {'proxy': proxy_capabilities}
    assert expected_capabilities == desired_capabilities
コード例 #25
0
ファイル: proxy_tests.py プロジェクト: 217/selenium
    def testCanInitEmptyProxy(self):
        proxy = Proxy()

        self.assertEqual(ProxyType.UNSPECIFIED, proxy.proxy_type)
        self.assertEqual('', proxy.http_proxy)
        self.assertEqual('', proxy.ftp_proxy)
        self.assertEqual('', proxy.no_proxy)
        self.assertEqual('', proxy.sslProxy)
        self.assertEqual('', proxy.socksProxy)
        self.assertEqual('', proxy.socksUsername)
        self.assertEqual('', proxy.socksPassword)
        self.assertEqual(False, proxy.auto_detect)
        self.assertEqual('', proxy.proxy_autoconfig_url)

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = {}
        proxy_capabilities['proxyType'] = 'UNSPECIFIED'
        expected_capabilities = {'proxy': proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #26
0
ファイル: proxy_tests.py プロジェクト: Cpaniyadi/selenium
    def testCanAddToDesiredCapabilities(self):
        desired_capabilities = {}
        proxy = Proxy()
        proxy.http_proxy = 'some.url:1234'
        proxy.ftp_proxy = 'ftp.proxy:1234'
        proxy.no_proxy = 'localhost, foo.localhost'
        proxy.sslProxy = 'ssl.proxy:1234'
        proxy.autodetect = 'True'

        proxy.add_to_capabilities(desired_capabilities)

        expected_capabilities = {
            'proxy': {
                'proxyType': 'MANUAL',
                'httpProxy': 'some.url:1234',
                'ftpProxy': 'ftp.proxy:1234',
                'noProxy': 'localhost, foo.localhost',
                'sslProxy': 'ssl.proxy:1234',
                'autodetect': 'True'
            }
        }
        print 'descap', desired_capabilities

        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #27
0
def test_manual_proxy_is_set_in_profile():
    profile = FirefoxProfile()
    proxy = Proxy()
    proxy.no_proxy = 'localhost, foo.localhost'
    proxy.http_proxy = 'some.url:1234'
    proxy.ftp_proxy = None
    proxy.sslProxy = 'some2.url'

    profile.set_proxy(proxy)
    assert profile.default_preferences[
        "network.proxy.type"] == ProxyType.MANUAL['ff_value']
    assert profile.default_preferences[
        "network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
    assert profile.default_preferences["network.proxy.http"] == 'some.url'
    assert profile.default_preferences["network.proxy.http_port"] == 1234
    assert profile.default_preferences["network.proxy.ssl"] == 'some2.url'
    assert "network.proxy.ssl_port" not in profile.default_preferences
    assert "network.proxy.ftp" not in profile.default_preferences
コード例 #28
0
    def testCanNotChangeInitializedProxyType(self):
        proxy = Proxy(raw={'proxyType': 'direct'})
        try:
            proxy.proxy_type = ProxyType.SYSTEM
            raise Exception(
                "Change of already initialized proxy type should raise exception"
            )
        except Exception as e:
            pass

        proxy = Proxy(raw={'proxyType': ProxyType.DIRECT})
        try:
            proxy.proxy_type = ProxyType.SYSTEM
            raise Exception(
                "Change of already initialized proxy type should raise exception"
            )
        except Exception as e:
            pass
コード例 #29
0
    def __init__(self, Url, proxy_url):
        self.songUrl = Url
        self.id = int(self.songUrl.split('=')[1])
        self.lv = -1
        self.gender = -1

        self.homeUrl = 'https://music.163.com/#/user/home?id=' + str(self.id)
        self.followersUrl = 'https://music.163.com/#/user/follows?id=' + str(
            self.id)
        self.songRankUrl = 'https://music.163.com/#/user/songs/rank?id=' + str(
            self.id)

        self.recent_song_list = []
        self.follows_id_list = []
        self.playlists = []

        chrome_options = Options()
        chrome_options.add_argument('--headless')
        prox = Proxy()
        prox.proxy_type = ProxyType.MANUAL
        prox.http_proxy = proxy_url
        prox.ssl_proxy = proxy_url
        # prox.socks_proxy = proxy_url
        capabilities = webdriver.DesiredCapabilities.CHROME
        prox.add_to_capabilities(capabilities)
        if (config_is_ubuntu):
            chrome_options.add_argument('--no-sandbox')
            chrome_options.add_argument('--disable-dev-shm-usage')

        # chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
        # chrome_options.add_argument('user-agent={0}'.format(random.choice(uas)))
        # debug_print_thread("we are using proxy sever with url " + proxy_url)
        self.driver_home = webdriver.Chrome(config_chrome_path,
                                            options=chrome_options)
        self.driver_recent_songs = webdriver.Chrome(config_chrome_path,
                                                    options=chrome_options)
        #self.driver_recent_songs.set_page_load_timeout(10)
        self.driver_follows = webdriver.Chrome(config_chrome_path,
                                               options=chrome_options)
コード例 #30
0
    def get_browser(self, proxy):
        # 启动浏览器
        # 禁止加载image
        firefox_profile = webdriver.FirefoxProfile()
        #firefox_profile.set_preference('permissions.default.stylesheet', 2)
        #firefox_profile.set_preference('permissions.default.image', 2)
        #firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
        # 代理
        if proxy and proxy.is_valid():
            myProxy = '%s:%s' % (proxy.host, proxy.port)
            ff_proxy = Proxy({
                'proxyType': ProxyType.MANUAL,
                'httpProxy': myProxy,
                'ftpProxy': myProxy,
                'sslProxy': myProxy,
                'noProxy': ''
            })

            browser = webdriver.Firefox(firefox_profile=firefox_profile,
                                        proxy=ff_proxy)
        else:
            browser = webdriver.Firefox(firefox_profile=firefox_profile)

        return browser
コード例 #31
0
ファイル: basedriver.py プロジェクト: zsjtoby/cmdbac
 def save_screenshot(self, main_url, screenshot_path):
     try:
         from pyvirtualdisplay import Display
         display = Display(visible=0, size=(1024, 768))
         display.start()
         from selenium import webdriver
         try:
             if '127.0.0.1' in main_url or 'localhost' in main_url:
                 br = webdriver.Firefox()
             else:
                 from selenium.webdriver.common.proxy import Proxy, ProxyType
                 proxy = Proxy({
                     'proxyType': ProxyType.MANUAL,
                     'httpProxy': HTTP_PROXY
                 })
                 br = webdriver.Firefox(proxy=proxy)
         except Exception, e:
             LOG.exception(e)
             br = webdriver.Firefox()
         br.get(main_url)
         br.save_screenshot(screenshot_path)
         br.quit()
         display.stop()
         return screenshot_path
コード例 #32
0
def ready(proxy_ip_port=None):
    '''

    :param proxy_ip_port:
    :return:
    '''
    if system == 'windows':
        executable_path = 'D:/geckodriver/geckodriver.exe'
    elif system == 'linux':
        executable_path = '../temp_file/geckodriver'
    else:
        logger.warning('没有这个系统:%s,暂时默认为windows' % system)
        executable_path = 'D:/geckodriver/geckodriver.exe'

    fireFoxOptions = webdriver.FirefoxOptions()
    fireFoxOptions.set_headless()
    # fireFoxOptions.add_argument("--proxy-server=http://{}".format(random.choice(ip_list)))
    # driver = webdriver.Firefox( executable_path='temp_file/geckodriver',options=fireFoxOptions)
    if proxy_ip_port:
        proxy = Proxy({'httpProxy': proxy_ip_port})
    else:
        proxy = None
    if mode == 'product':
        driver = webdriver.Firefox(executable_path=executable_path, options=fireFoxOptions,
                                   proxy=proxy)
    elif mode == 'debug':
        if system == 'windows':
            binary_path = 'C:/Program Files (x86)/Mozilla Firefox/firefox.exe'
        else:
            binary_path = '/usr/bin/firefox'
        binary = FirefoxBinary(binary_path)
        driver = webdriver.Firefox(firefox_binary=binary, executable_path=executable_path, proxy=proxy)
    else:
        raise Exception('ready方法的参数里面没有这种mode:%s' % mode)
    driver.implicitly_wait(10)  # 设置寻找(包括异步加载的)element所等待的时间(如果不设置,则异步加载的element有可能会找不到)
    return driver
コード例 #33
0
 def __init__(self, *args, **kwargs):
     """
     Initializes spider. Starts xsession in virtual display and
     webdriver instance in this xsession. If PROXY_PARAMS defined in
     settings.py it'll specify proxy for webdriver instance.
     """
     super(IrrSpider, self).__init__(*args, **kwargs)
     settings = get_project_settings()
     self.xsession = Display(
         visible=settings.get('XSESSION_VISIBLE', False),
         size=settings.get('XSESSION_DISPLAY_RESOLUTION', (800, 600)),
     )
     self.xsession.start()
     proxy = settings.get('PROXY_PARAMS')
     if proxy:
         proxy = Proxy({
             'proxyType': ProxyType.MANUAL,
             'httpProxy': proxy,
             'ftpProxy': proxy,
             'sslProxy': proxy,
             'noProxy': proxy,
         })
     self.browser = webdriver.Firefox(proxy=proxy)
     self.browser.maximize_window()
コード例 #34
0
    def find_proxy(self, ip):
        random.shuffle(self.ip_list)
        proxy_ip = self.ip_list[0]
        proxy = Proxy({
            'proxyType': ProxyType.MANUAL,
            'httpProxy': proxy_ip
        })
        opt = webdriver.ChromeOptions()
        opt.add_argument('--proxy-server=http://'+proxy_ip)
        self.browser = webdriver.Chrome(self.chrome_driver, chrome_options=opt)
        flag = True
        locator = (By.LINK_TEXT, '丁香园')
        try:
            self.browser.get(ip)
            self.browser.find_element_by_id('head')
            WebDriverWait(self.browser, 10, 0.5).until(EC.presence_of_element_located(locator))

        except:
            flag = False
            self.browser.quit()
        finally:
            ip = proxy_ip
            return flag
        '''
コード例 #35
0
    def make_options(self, headless=True, proxing=False):
        '''Конфигурирование браузера'''
        dict_options = {}
        options = Options()
        if headless:
            options.add_argument("--headless")
        if self.binary_location:
            options.binary_location = (self.binary_location)
        options.page_load_strategy = 'eager'
        options.add_argument('--no-sandbox')
        options.add_argument('--disable-gpu')
        options.add_argument('user-agent={}'.format(load_ua()))
        dict_options['options'] = options

        if proxing:
            prox = Proxy()
            prox.proxy_type = ProxyType.MANUAL
            prox.http_proxy = 'socks5://127.0.0.1:9050'
            prox.ssl_proxy = 'socks5://127.0.0.1:9050'
            capabilities = webdriver.DesiredCapabilities.CHROME
            prox.add_to_capabilities(capabilities)
            dict_options['desired_capabilities'] = capabilities
        return dict_options
コード例 #36
0
def proxy_driver(PROXIES, ua):
    print('\n----CONNECTING TO PROXY----')
    options = webdriver.ChromeOptions()
    prox = Proxy()

    if len(PROXIES) < 1:
        print("--- Proxies used up (%s)" % len(PROXIES))
        PROXIES = get_proxies()

    pxy = PROXIES.pop()
    #pxy = '80.25.87.49:32850'
    print(f'Current Proxy ({pxy})')

    prox.proxy_type = ProxyType.MANUAL
    prox.autodetect = False
    prox.http_proxy = pxy
    prox.socks_proxy = pxy
    prox.ssl_proxy = pxy
    options.Proxy = prox

    print('\n----CHANGING USER AGENT----')
    userAgent = ua.random
    print(f'Current UserAgent\n{userAgent}')

    options.add_argument("--start-maximized")
    options.add_argument("--proxy-server=%s" % pxy)
    options.add_argument("user-agent={userAgent}")

    options.add_argument("ignore-certificate-errors")
    options.add_argument(
        "--disable-bundled-ppapi-flash")  # Disable internal Flash player
    options.add_argument(
        "--disable-plugins-discovery"
    )  # Disable external Flash player (by not allowing it to load)
    options.add_extension('mpbjkejclgfgadiemmefgebjfooflfhl.crx')
    driver = webdriver.Chrome('chromedriver.exe', options=options)
    return driver, PROXIES
コード例 #37
0
def __open_browser(context):
    chrm = context.config.userdata['chromedriver_path']

    try:
        # if there is a proxy, we'll use it.  Otherwise, we won't.
        requests.get("http://localhost:8888", timeout=0.01)

        # if there was no exception, we continue here.
        PROXY = "localhost:8888"

        proxy = Proxy()
        proxy.proxy_type = ProxyType.MANUAL
        proxy.http_proxy = PROXY

        capabilities = webdriver.DesiredCapabilities.CHROME
        proxy.add_to_capabilities(capabilities)

        if (chrm):
            context.driver = webdriver.Chrome(
                desired_capabilities=capabilities, executable_path=chrm)
        else:
            context.driver = webdriver.Chrome(
                desired_capabilities=capabilities)
        return context.driver
    except:
        if (chrm):
            context.driver = webdriver.Chrome(executable_path=chrm)
        else:
            # adding the service args as described below will cause Chromedriver
            # to create a log of the communication between it and the Chrome
            # browser.  It's eye-opening.
            #
            # for instance:
            #   [1568045962.076][INFO]: [e18882b1f2abbda89f232f777f98f686] COMMAND TypeElement {
            #      "id": "0.47079920350295135-1",
            #      "sessionId": "e18882b1f2abbda89f232f777f98f686",
            #      "text": "Byron",
            #      "value": [ "B", "y", "r", "o", "n" ]
            #   }
            #context.driver = webdriver.Chrome(service_args=["--verbose","--logepath=C:\\temp\\qc1.log"])
            context.driver = webdriver.Chrome()
        return context.driver
コード例 #38
0
    def test_manual_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.no_proxy = 'localhost, foo.localhost'
        proxy.http_proxy = 'some.url:1234'
        proxy.ftp_proxy = None
        proxy.sslProxy = 'some2.url'

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
        assert self.profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
        assert self.profile.default_preferences["network.proxy.http"] == 'some.url'
        assert self.profile.default_preferences["network.proxy.http_port"] == 1234
        assert self.profile.default_preferences["network.proxy.ssl"] == 'some2.url'
        assert "network.proxy.ssl_port" not in self.profile.default_preferences
        assert "network.proxy.ftp" not in self.profile.default_preferences
コード例 #39
0
def open_browser():
    try:
        # if there is a proxy, we'll use it.  Otherwise, we won't.
        requests.get("http://localhost:8888", timeout=0.01)

        # if there was no exception, we continue here.
        PROXY = "localhost:8888"

        proxy = Proxy()
        proxy.proxy_type = ProxyType.MANUAL
        proxy.http_proxy = PROXY

        capabilities = webdriver.DesiredCapabilities.CHROME
        proxy.add_to_capabilities(capabilities)

        driver = webdriver.Chrome(desired_capabilities=capabilities)
        return driver
    except:
        driver = webdriver.Chrome()
        return driver
コード例 #40
0
ファイル: webdriver.py プロジェクト: FrancoisLaruaz/Batch1
def getFirefoxDriver(host, port):
    try:
        path = 'C:\\webdrivers\\geckodriver.exe'
        fullproxy = host + ":" + port
        #https://stackoverflow.com/questions/17082425/running-selenium-webdriver-with-a-proxy-in-python
        firefox_capabilities = DesiredCapabilities.FIREFOX
        firefox_capabilities['marionette'] = True
        webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
            "httpProxy": fullproxy,
            "ftpProxy": fullproxy,
            "sslProxy": fullproxy,
            "noProxy": None,
            "proxyType": "MANUAL",
            "autodetect": False
        }
        prox = Proxy()
        prox.proxy_type = ProxyType.MANUAL
        prox.http_proxy = fullproxy
        prox.socks_proxy = fullproxy
        prox.ssl_proxy = fullproxy
        #prox.add_to_capabilities(firefox_capabilities)
        fp = webdriver.FirefoxProfile()
        fp.set_preference("network.proxy.type", 1)
        fp.set_preference("network.proxy.http", host)
        fp.set_preference("network.proxy.http_port", int(port))
        fp.set_preference("network.proxy.https", host)
        fp.set_preference("network.proxy.https_port", int(port))
        fp.set_preference("network.proxy.ssl", host)
        fp.set_preference("network.proxy.ssl_port", int(port))
        fp.set_preference("network.proxy.ftp", host)
        fp.set_preference("network.proxy.ftp_port", int(port))
        fp.set_preference("network.proxy.socks", host)
        fp.set_preference("network.proxy.socks_port", int(port))
        #fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")
        fp.update_preferences()
        driver = webdriver.Firefox(executable_path=path)

        return driver
    except Exception:
        LogError(traceback, "host = " + host + " and port = " + port)
    return getGoogleChromeDriver(host + ":" + port)
def driver(request):
    # wd = webdriver.Chrome()
    # wd = webdriver.Chrome(desired_capabilities={"chromeOptions": {"args": ["--start-fullscreen"]}})
    # wd = webdriver.Chrome(desired_capabilities={"proxy": {"proxyType": "MANUAL", "httpProxy": "localhost:8888"}})

    prox = Proxy()
    prox.proxy_type = ProxyType.MANUAL
    prox.http_proxy = "127.0.0.1:8888"
    # prox.socks_proxy = "127.0.0.1:8888" # Does not work with this
    prox.ssl_proxy = "127.0.0.1:8888"

    capabilities = webdriver.DesiredCapabilities.CHROME
    prox.add_to_capabilities(capabilities)

    wd = webdriver.Chrome(desired_capabilities=capabilities)

    # wd = webdriver.Firefox()
    # options = webdriver.FirefoxOptions()
    # options.binary_location = "C:\\Program Files\\Firefox Nightly\\firefox.exe"
    # options.add_argument("start-maximized")
    # wd = webdriver.Firefox(firefox_options=options)
    # new method
    # wd = webdriver.Firefox()
    # new method more obviously
    # wd = webdriver.Firefox(capabilities={"marionette": True})
    # old method
    # wd = webdriver.Firefox(capabilities={"marionette": False})
    # wd = webdriver.Edge(executable_path="C:\Webdrivers\msedgedriver")
    # wd = webdriver.Ie()
    # wd = webdriver.Ie(capabilities={"unexpectedAlertBehaviour": "dismiss"})
    # wd = webdriver.Ie(capabilities={"requireWindowFocus": True})
    # wd = webdriver.Ie(capabilities={"IntroduceInstabilityByIgnoringProtectedModeSettings": True, "requireWindowFocus": True, "unexpectedAlertBehaviour": "dismiss", "ignoreZoomSetting": True})
    # print(f'\nCAPABILITIES: {wd.capabilities}\nEND CAPABILITIES')
    print(f'\nWD capabilities: {wd.capabilities}\n')
    request.addfinalizer(wd.quit)
    return wd
コード例 #42
0
ファイル: main.py プロジェクト: starwayagency/sw_imp_exp
    def proxy_driver(self, p, ua):
        co = webdriver.ChromeOptions()
        co.add_argument('--no-sandbox')
        co.add_argument('--disable-dev-shm-usage')
        co.add_argument(f"user-agent={ua}")
        co.add_argument("--headless")
        co.add_argument(f"--window-size={choice(('1024,768', '1280,960', '1280,1024', '1600,1200', '1920,1080'))}")
        pxy = p
        prox = Proxy()
        prox.proxy_type = ProxyType.MANUAL
        prox.http_proxy = pxy
        prox.socks_proxy = pxy
        prox.ssl_proxy = pxy

        capabilities = webdriver.DesiredCapabilities.CHROME
        # capabilities['resolution'] = '1920x1080'
        # prox.add_to_capabilities(capabilities)

        driver = webdriver.Chrome('/home/jurgeon/projects/jcb/materials/jcb/chromedriver', options = co, desired_capabilities = capabilities)

        return driver
コード例 #43
0
ファイル: selenium.py プロジェクト: laurentb/weboob
    def _setup_driver(self):
        proxy = Proxy()
        proxy.proxy_type = ProxyType.DIRECT
        if 'http' in self.proxy:
            proxy.http_proxy = self.proxy['http']
        if 'https' in self.proxy:
            proxy.ssl_proxy = self.proxy['https']

        capa = self._build_capabilities()
        proxy.add_to_capabilities(capa)

        options = self._build_options()
        # TODO some browsers don't need headless
        # TODO handle different proxy setting?
        options.set_headless(self.HEADLESS)

        if self.DRIVER is webdriver.Firefox:
            if self.responses_dirname and not os.path.isdir(self.responses_dirname):
                os.makedirs(self.responses_dirname)

            options.profile = DirFirefoxProfile(self.responses_dirname)
            if self.responses_dirname:
                capa['profile'] = self.responses_dirname
            self.driver = self.DRIVER(options=options, capabilities=capa)
        elif self.DRIVER is webdriver.Chrome:
            self.driver = self.DRIVER(options=options, desired_capabilities=capa)
        elif self.DRIVER is webdriver.PhantomJS:
            if self.responses_dirname:
                if not os.path.isdir(self.responses_dirname):
                    os.makedirs(self.responses_dirname)
                log_path = os.path.join(self.responses_dirname, 'selenium.log')
            else:
                log_path = NamedTemporaryFile(prefix='weboob_selenium_', suffix='.log', delete=False).name

            self.driver = self.DRIVER(desired_capabilities=capa, service_log_path=log_path)
        else:
            raise NotImplementedError()

        if self.WINDOW_SIZE:
            self.driver.set_window_size(*self.WINDOW_SIZE)
コード例 #44
0
def main():
    """
        This is the entry for the command which makes it convenient to install the proxy certificate
    """
    setupLocalLogging()
    commandArgs = sys.argv[1:]

    proxyPort = findFreePort()

    proxyThread = threading.Thread(target=runProxy,
                                   args=[proxyPort],
                                   daemon=True)
    proxyThread.start()

    capabilities = webdriver.DesiredCapabilities.CHROME
    capabilities['loggingPrefs'] = {'browser': 'ALL'}
    proxyConfig = Proxy()
    proxyConfig.proxy_type = ProxyType.MANUAL
    proxyConfig.http_proxy = f"localhost:{proxyPort}"
    proxyConfig.add_to_capabilities(capabilities)

    chrome_options = webdriver.chrome.options.Options()
    if len(commandArgs) > 0:
        chrome_options.headless = True
    chrome_options.add_argument(f"--no-sandbox")

    driver = webdriver.Chrome(desired_capabilities=capabilities,
                              chrome_options=chrome_options)

    driver.get("http://mitm.it/")

    print(
        "Please kill the command with Ctrl-C or (Cmd-C on macOS) when you are finished installing the certificates. Timeout in 600 seconds..."
    )

    timeout = 600
    if len(commandArgs) > 0:
        timeout = int(str(commandArgs[0]))

    time.sleep(timeout)
コード例 #45
0
ファイル: test.py プロジェクト: ivanroma20/demo
def open_browser():
    try:
        # if there is a proxy, we'll use it.  Otherwise, we won't.
        requests.get("http://localhost:8888", timeout=0.01)

        # if there was no exception, we continue here.
        PROXY = "localhost:8888"

        proxy = Proxy()
        proxy.proxy_type = ProxyType.MANUAL
        proxy.http_proxy = PROXY

        capabilities = webdriver.DesiredCapabilities.CHROME
        proxy.add_to_capabilities(capabilities)

        driver = webdriver.Chrome(desired_capabilities=capabilities)
        return driver
    except:
        # if we got an exception while trying to hit a proxy URL,
        # it probably means the proxy isn't available, so we run
        # without a proxy, as follows.
        driver = webdriver.Chrome()
        return driver
コード例 #46
0
ファイル: footlocker.py プロジェクト: jamesbaker1/foot_sites
    def test2(self):
        driverLocation = '/Users/jamesbaker/Desktop/footlocker_notes/chromedriver'
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--no-sandbox')
        chrome_options.add_argument('--disable-dev-shm-usage')
        prox = Proxy()
        prox.proxy_type = ProxyType.MANUAL
        prox.http_proxy = "34.93.147.196:80"
        # prox.socks_proxy = "ip_addr:port"
        # prox.ssl_proxy = "ip_addr:port"
        capabilities = webdriver.DesiredCapabilities.CHROME
        prox.add_to_capabilities(capabilities)
        driver = webdriver.Chrome(driverLocation,
                                  chrome_options=chrome_options,
                                  desired_capabilities=capabilities)
        # driver.get("https://www.footlocker.com/product/nike-air-force-1-07-low-womens/O2132602.html")
        driver.get("https://www.google.com")

        print(driver.page_source)
        # elem = driver.find_element_by_xpath("//button[contains('Add To Cart')]")
        # elem = driver.find_element_by_css_selector(".Button.ProductDetails-form__action")
        # elem.click()

        # def test(self):
        #     driverLocation = '/Users/jamesbaker/Desktop/footlocker_notes/chromedriver'
        #     chrome_options = webdriver.ChromeOptions()
        PROXY = "smart-us.jeru035.com:23774:jerursvpking8588:rsvpking8588"
        PROXY = "jerursvpking8588:[email protected]:23774"
        PROXY = "174.127.155.118:32505"
        #     # chrome_options.add_argument('--headless')
        #     chrome_options.add_argument('window-size=1200x600') # optional
        #
        #     chrome_options.add_argument('--incognito')
        #     chrome_options.add_argument('--no-sandbox')
        #     chrome_options.add_argument('--disable-dev-shm-usage')
        chrome_options.add_argument('--proxy-server=%s' % PROXY)
コード例 #47
0
from selenium.webdriver.support.ui import WebDriverWait


proxy = "12.345.678.910:8080"
options = WebDriverWait.ChromeOptions()
options.add_argument('--proxy-server=%s' % proxy)
chrome = webdriver.Chrome(chrome_options=options)
chrome.get("https://www ... ")


#  or

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType

prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "ip_addr:port"
prox.socks_proxy = "ip_addr:port"
prox.ssl_proxy = "ip_addr:port"

capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)

driver = webdriver.Chrome(desired_capabilities=capabilities)


# or  Proxy for FireFox
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.http", "proxy.server.address")
コード例 #48
0
ファイル: tasks.py プロジェクト: thatandromeda/perma
def get_browser(user_agent, proxy_address, cert_path):
    """ Set up a Selenium browser with given user agent, proxy and SSL cert. """
    # PhantomJS
    if settings.CAPTURE_BROWSER == 'PhantomJS':
        desired_capabilities = dict(DesiredCapabilities.PHANTOMJS)
        desired_capabilities["phantomjs.page.settings.userAgent"] = user_agent
        browser = webdriver.PhantomJS(
            executable_path=getattr(settings, 'PHANTOMJS_BINARY', 'phantomjs'),
            desired_capabilities=desired_capabilities,
            service_args=[
                "--proxy=%s" % proxy_address,
                "--ssl-certificates-path=%s" % cert_path,
                "--ignore-ssl-errors=true",
                "--local-url-access=false",
                "--local-storage-path=.",
            ],
            service_log_path=settings.PHANTOMJS_LOG)

    # Firefox
    elif settings.CAPTURE_BROWSER == 'Firefox':
        desired_capabilities = dict(DesiredCapabilities.FIREFOX)
        proxy = Proxy({
            'proxyType': ProxyType.MANUAL,
            'httpProxy': proxy_address,
            'ftpProxy': proxy_address,
            'sslProxy': proxy_address,
        })
        proxy.add_to_capabilities(desired_capabilities)
        profile = webdriver.FirefoxProfile()
        profile.accept_untrusted_certs = True
        profile.assume_untrusted_cert_issuer = True
        browser = webdriver.Firefox(
            capabilities=desired_capabilities,
            firefox_profile=profile)

    # Chrome
    elif settings.CAPTURE_BROWSER == 'Chrome':
        # http://blog.likewise.org/2015/01/setting-up-chromedriver-and-the-selenium-webdriver-python-bindings-on-ubuntu-14-dot-04/
        download_dir = os.path.abspath('./downloads')
        os.mkdir(download_dir)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--proxy-server=%s' % proxy_address)
        chrome_options.add_argument('--test-type')
        chrome_options.add_experimental_option("prefs", {"profile.default_content_settings.popups": "0",
                                                         "download.default_directory": download_dir,
                                                         "download.prompt_for_download": "false"})
        desired_capabilities = chrome_options.to_capabilities()
        desired_capabilities["acceptSslCerts"] = True

        # for more detailed progress updates
        # desired_capabilities["loggingPrefs"] = {'performance': 'INFO'}
        # then:
        # performance_log = browser.get_log('performance')

        browser = webdriver.Chrome(desired_capabilities=desired_capabilities)

    else:
        assert False, "Invalid value for CAPTURE_BROWSER."

    browser.implicitly_wait(ELEMENT_DISCOVERY_TIMEOUT)
    browser.set_page_load_timeout(ROBOTS_TXT_TIMEOUT)

    return browser
コード例 #49
0
    def testCanAddAutodetectProxyToDesiredCapabilities(self):
        proxy = Proxy(raw=self.AUTODETECT_PROXY)

        self.assertEqual(ProxyType.AUTODETECT, proxy.proxy_type)
        self.assertEqual(self.AUTODETECT_PROXY['autodetect'],
                         proxy.auto_detect)
コード例 #50
0
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.common.proxy import ProxyType

proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': '114.235.82.16:8118'  # 代理ip和端口
})
# 新建一个代理IP对象
desired_capabilities = DesiredCapabilities.PHANTOMJS.copy()
# 加入代理IP
proxy.add_to_capabilities(desired_capabilities)
driver = webdriver.PhantomJS(executable_path='E:/phantomjs/bin/phantomjs.exe',
                             desired_capabilities=desired_capabilities)
# 测试一下,打开使用的代理IP地址信息
driver.get('http://www.whatismyip.com.tw/')
print(driver.page_source)
コード例 #51
0
 requesting the driver to refresh the page after 1-minute sleep...')
                    time.sleep(60)
            canvas = _wait_for_page_to_present(driver)
            continue


if __name__ == '__main__':
    logging_.initialize(module_name='game_abstract_crawler',
                        service_name='crawler')

    options = Options()
    options.headless = True
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('--window-size=800,600')

    proxy = Proxy()
    proxy.http_proxy = 'localhost:8080'
    proxy.https_proxy = 'localhost:8080'
    capabilities = DesiredCapabilities.CHROME
    proxy.add_to_capabilities(capabilities)

    with Chrome(options=options,
                desired_capabilities=capabilities) as driver:
        try:
            main(driver)
        except Exception as e:
            _get_screenshot(driver, '99-エラー.png')
            logging.exception('Abort with an unhandled exception.')
            raise
コード例 #52
0
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium import webdriver

from scroll import scroll

browser = None

proxy_address = "127.0.0.1:8118"
proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': proxy_address,
})

gecko = "./geckodriver"
firefox = "/usr/bin/firefox"

firefox_binary = FirefoxBinary(firefox)


def callback(driver):
    driver.quit()


def driver(binary=None, proxy=None):
    return webdriver.Firefox(executable_path=gecko,
                             firefox_binary=binary,
                             proxy=proxy)


if __name__ == "__main__":
コード例 #53
0
    def _search(self, start, visible=0):
        payload = {}
        try:
            payload['q'] = self.query.encode('utf8')  # query to lookup
        except:
            payload['q'] = self.query  # query to lookup
        payload['start'] = start  # start point
        payload['gl'] = self.country  # query from country
        payload['hl'] = self.language  # user query language
        payload['lr'] = 'lang_%s' % self.language  # restrict language pages
        payload['num'] = GoogleSeleniumPlus.PAGE_LIMIT
        payload['safe'] = 'off'

        params = urllib.urlencode(payload)

        display = Display(visible=visible, size=(800, 600))
        display.start()
        try:
            proxyInfo = ProxyManager.getNextProxy()

            myProxy = '%s:%s' % (proxyInfo.host, proxyInfo.port)

            proxy = Proxy({
                'proxyType': ProxyType.MANUAL,
                'httpProxy': myProxy,
                'ftpProxy': myProxy,
                'sslProxy': myProxy,
                'noProxy': ''  # set this value as desired
            })

            browser = webdriver.Firefox(proxy=proxy)
            browser.set_page_load_timeout(30)
            try:
                params = urllib.urlencode(payload)

                browser.implicitly_wait(10)

                browser.get('%s#%s' % (self.googleHost, params))

                app_logger.info(u"%s" % browser.current_url)

                h3List = browser.find_elements_by_xpath("//h3[@class='r']")

                results = []

                for h3 in h3List:
                    link = h3.find_element_by_tag_name('a')
                    results.append(link.get_attribute("href"))

                box = browser.find_element_by_id('lst-ib')

                partialQuery = ' '.join(self.query.split()[1:])

                for _letter in partialQuery:
                    box.send_keys(Keys.BACKSPACE)
                    randomSleep(0.03, 0.05)

                typeQuery(box, partialQuery)

                randomSleep(0.05, 0.25)
                print('-' * 80)

            finally:
                browser.close()

        except Exception as ex:
            raise ex

        finally:
            display.stop()

        if not results:
            ProxyManager.invalidateProxy()

        return results
コード例 #54
0
ファイル: parser.py プロジェクト: robbyb74/Scraper
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.proxy import Proxy, ProxyType

input_email = '*****@*****.**'
input_password = '******'

# Not working right now
myProxy = ''
proxyF = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': myProxy,
    'ftpProxy': myProxy,
    'sslProxy': myProxy,
    'noProxy': ''  # set this value as desired
})


def await_element(driver, selector):
    try:
        element_present = EC.presence_of_element_located(
            (By.CSS_SELECTOR, selector))
        WebDriverWait(driver, 180).until(element_present)
    except TimeoutException:
        raise


# Overwrite Selenium headers
コード例 #55
0
HTML_INJECTION_STORED_BLOG = '/htmli_stored.php'
IFRAME_INJECTION = '/iframei.php'
LDAP_INJECTION = ''
MAIL_HEADER_INJECTION = ''
OS_COMMAND_INJECTION = ''
OS_COMMAND_INJECTION_BLIND = ''
PHP_CODE_INJECTION = ''

# set zed attack proxy
zap = ZAPv2(apikey=APIKEY)
# Use the line below if ZAP is not listening on port 8080, for example, if listening on port 8090
# zap = ZAPv2(apikey=apikey, proxies={'http': 'http://127.0.0.1:8090', 'https': 'http://127.0.0.1:8090'})

# Set zap proxy with defined docker ip address
PROXY_ADDRESS = "172.17.0.6:8080"
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY_ADDRESS
proxy.ssl_proxy = PROXY_ADDRESS

# set desired capabilities for firefox client
capabilities = webdriver.DesiredCapabilities.FIREFOX
proxy.add_to_capabilities(capabilities)

driver = webdriver.Remote(command_executor=SELENIUM_HUB_ADDRESS,
                          desired_capabilities=capabilities)


def login_bwapp(target):
    driver.get(target + '/login.php')
    driver.find_element_by_id("login").send_keys("bee")
コード例 #56
0
    def start_client(self):
        capabilities = {}
        for c in self.capabilities:
            name, value = c.split(':')
            # handle integer capabilities
            if value.isdigit():
                value = int(value)
            # handle boolean capabilities
            elif value.lower() in ['true', 'false']:
                value = value.lower() == 'true'
            capabilities.update({name: value})
        if self.proxy_host and self.proxy_port:
            proxy = Proxy()
            proxy.http_proxy = '%s:%s' % (self.proxy_host, self.proxy_port)
            proxy.ssl_proxy = proxy.http_proxy
            proxy.add_to_capabilities(capabilities)
        profile = None

        if self.driver.upper() == 'REMOTE':
            capabilities.update(getattr(webdriver.DesiredCapabilities, self.browser_name.upper()))
            if json.loads(self.chrome_options) or self.extension_paths:
                capabilities = self.create_chrome_options(
                    self.chrome_options,
                    self.extension_paths).to_capabilities()
            if self.browser_name.upper() == 'FIREFOX':
                profile = self.create_firefox_profile(
                    self.firefox_preferences,
                    self.profile_path,
                    self.extension_paths)
            if self.browser_version:
                capabilities['version'] = self.browser_version
            capabilities['platform'] = self.platform.upper()
            executor = 'http://%s:%s/wd/hub' % (self.host, self.port)
            try:
                self.selenium = webdriver.Remote(command_executor=executor,
                                                 desired_capabilities=capabilities or None,
                                                 browser_profile=profile)
            except AttributeError:
                valid_browsers = [attr for attr in dir(webdriver.DesiredCapabilities) if not attr.startswith('__')]
                raise AttributeError("Invalid browser name: '%s'. Valid options are: %s" % (self.browser_name, ', '.join(valid_browsers)))

        elif self.driver.upper() == 'CHROME':
            options = None
            if self.chrome_options or self.extension_paths:
                options = self.create_chrome_options(
                    self.chrome_options,
                    self.extension_paths)
            if self.chrome_path:
                self.selenium = webdriver.Chrome(executable_path=self.chrome_path,
                                                 chrome_options=options,
                                                 desired_capabilities=capabilities or None)
            else:
                self.selenium = webdriver.Chrome(chrome_options=options,
                                                 desired_capabilities=capabilities or None)

        elif self.driver.upper() == 'FIREFOX':
            binary = self.firefox_path and FirefoxBinary(self.firefox_path) or None
            profile = self.create_firefox_profile(
                self.firefox_preferences,
                self.profile_path,
                self.extension_paths)
            self.selenium = webdriver.Firefox(
                firefox_binary=binary,
                firefox_profile=profile,
                capabilities=capabilities or None)
        elif self.driver.upper() == 'IE':
            self.selenium = webdriver.Ie()
        elif self.driver.upper() == 'PHANTOMJS':
            self.selenium = webdriver.PhantomJS()
        elif self.driver.upper() == 'OPERA':
            capabilities.update(webdriver.DesiredCapabilities.OPERA)
            self.selenium = webdriver.Opera(executable_path=self.opera_path,
                                            desired_capabilities=capabilities)
        elif self.driver.upper() == 'BROWSERSTACK':
            from cloud import BrowserStack
            self.cloud = BrowserStack()
            self.selenium = self.cloud.driver(
                self.test_id, capabilities, self.options)
        elif self.driver.upper() == 'SAUCELABS':
            from cloud import SauceLabs
            self.cloud = SauceLabs()
            self.selenium = self.cloud.driver(
                self.test_id, capabilities, self.options, self.keywords)
        else:
            self.selenium = getattr(webdriver, self.driver)()

        if self.event_listener is not None and not isinstance(self.selenium, EventFiringWebDriver):
            self.selenium = EventFiringWebDriver(self.selenium, self.event_listener())
コード例 #57
0
    def testCanAddPACProxyToDesiredCapabilities(self):
        proxy = Proxy(raw=self.PAC_PROXY)

        self.assertEqual(ProxyType.PAC, proxy.proxy_type)
        self.assertEqual(self.PAC_PROXY['proxyAutoconfigUrl'],
                         proxy.proxy_autoconfig_url)
コード例 #58
0
    def testCanAddManualProxyToDesiredCapabilities(self):
        proxy = Proxy()
        proxy.http_proxy = self.MANUAL_PROXY['httpProxy']
        proxy.ftp_proxy = self.MANUAL_PROXY['ftpProxy']
        proxy.no_proxy = self.MANUAL_PROXY['noProxy']
        proxy.sslProxy = self.MANUAL_PROXY['sslProxy']
        proxy.socksProxy = self.MANUAL_PROXY['socksProxy']
        proxy.socksUsername = self.MANUAL_PROXY['socksUsername']
        proxy.socksPassword = self.MANUAL_PROXY['socksPassword']

        desired_capabilities = {}
        proxy.add_to_capabilities(desired_capabilities)

        proxy_capabilities = self.MANUAL_PROXY.copy()
        proxy_capabilities['proxyType'] = 'MANUAL'
        expected_capabilities = {'proxy': proxy_capabilities}
        self.assertEqual(expected_capabilities, desired_capabilities)
コード例 #59
0
#! usr/bin/env/python3
# coding:utf-8
# @Time: 2019-04-03 15:55
# Author: turpure

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

prox = Proxy()
prox.proxy_type = ProxyType.MANUAL

prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
# prox.http_proxy = "127.0.0.1:1080"
# prox.socks_proxy = "127.0.0.1:1080"
prox.ssl_proxy = "175.155.249.57:8888"

capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)

capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)


def work():
    base_url = 'https://www.trackingmore.com/sprintpack-tracking/cn.html?number=0B048028400019917237E'
    options = webdriver.ChromeOptions()
    prefs = {