Exemplo n.º 1
0
def get_web_driver(url, has_proxy=True):
    """
    Selenium 使用代理请求
    :param url:
    :return driver:
    """

    if has_proxy:
        proxies = get_proxies()
        if proxies.has_key('http'):
            myProxy = proxies['http']
        elif proxies.has_key('https'):
            myProxy = proxies['https']

        proxy = Proxy({
           'proxyType': ProxyType.MANUAL,
            'httpProxy': myProxy,
            # 'ftpProxy': myProxy,
            # 'sslProxy': myProxy,
            # 'noProxy':d ''
        })
        print encode_wrap("使用代理:"), myProxy
        driver = webdriver.Firefox(proxy=proxy)
    else:
        driver = webdriver.Firefox()

    #driver.set_page_load_timeout(30)
    driver.get(url)

    return driver
Exemplo n.º 2
0
def get_session(url, has_proxy=True, cookie=None):
    """
    requests Session使用代理请求
    :param url:
    :param has_proxy:
    :param cookie:
    :return requests:
    """

    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36'}

    proxy = None
    if has_proxy:
        proxy = get_proxies()
    cf = ConfigParser.ConfigParser()
    cf.read('../config.ini')
    timeout = int(cf.get('web', 'timeout'))

    s = requests.Session()
    r = s.get(url, proxies=proxy, headers=headers, cookies=cookie,timeout=timeout)
    output = 'Code:{0}  Proxy:{1}'.format(r.status_code, proxy)
    print encode_wrap(output)

    if int(r.status_code) != 200:
        raise Exception('request fail')

    return r, s