Ejemplo n.º 1
0
 def resetProxies(self, httpProxyTuple):
     # for ntlm user and password are required
     self.hasNTLM = False
     if isinstance(httpProxyTuple,(tuple,list)) and len(httpProxyTuple) == 5:
         useOsProxy, _urlAddr, _urlPort, user, password = httpProxyTuple
         _proxyDirFmt = proxyDirFmt(httpProxyTuple)
         # only try ntlm if user and password are provided because passman is needed
         if user and not useOsProxy:
             for pluginXbrlMethod in pluginClassMethods("Proxy.HTTPNtlmAuthHandler"):
                 HTTPNtlmAuthHandler = pluginXbrlMethod()
                 if HTTPNtlmAuthHandler is not None:
                     self.hasNTLM = True
             if not self.hasNTLM: # try for python site-packages ntlm
                 try:
                     from ntlm import HTTPNtlmAuthHandler
                     self.hasNTLM = True
                 except ImportError:
                     pass
         if self.hasNTLM:    
             pwrdmgr = proxyhandlers.HTTPPasswordMgrWithDefaultRealm()
             pwrdmgr.add_password(None, _proxyDirFmt["http"], user, password)                
             self.proxy_handler = proxyhandlers.ProxyHandler({})
             self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler(pwrdmgr)
             self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler(pwrdmgr)
             self.ntlm_auth_handler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(pwrdmgr)            
             self.opener = proxyhandlers.build_opener(self.proxy_handler, self.ntlm_auth_handler, self.proxy_auth_handler, self.http_auth_handler)
     if not self.hasNTLM:
         self.proxy_handler = proxyhandlers.ProxyHandler(proxyDirFmt(httpProxyTuple))
         self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler()
         self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
         self.opener = proxyhandlers.build_opener(self.proxy_handler, self.proxy_auth_handler, self.http_auth_handler)
Ejemplo n.º 2
0
 def resetProxies(self, httpProxyTuple):
     # for ntlm user and password are required
     self.hasNTLM = False
     self._httpProxyTuple = httpProxyTuple  # save for resetting in noCertificateCheck setter
     if isinstance(httpProxyTuple,
                   (tuple, list)) and len(httpProxyTuple) == 5:
         useOsProxy, _urlAddr, _urlPort, user, password = httpProxyTuple
         _proxyDirFmt = proxyDirFmt(httpProxyTuple)
         # only try ntlm if user and password are provided because passman is needed
         if user and not useOsProxy:
             for pluginXbrlMethod in pluginClassMethods(
                     "Proxy.HTTPAuthenticate"):
                 pluginXbrlMethod(self.cntlr)
             for pluginXbrlMethod in pluginClassMethods(
                     "Proxy.HTTPNtlmAuthHandler"):
                 HTTPNtlmAuthHandler = pluginXbrlMethod()
                 if HTTPNtlmAuthHandler is not None:
                     self.hasNTLM = True
             if not self.hasNTLM:  # try for python site-packages ntlm
                 try:
                     from ntlm import HTTPNtlmAuthHandler
                     self.hasNTLM = True
                 except ImportError:
                     pass
         if self.hasNTLM:
             pwrdmgr = proxyhandlers.HTTPPasswordMgrWithDefaultRealm()
             pwrdmgr.add_password(None, _proxyDirFmt["http"], user,
                                  password)
             self.proxy_handler = proxyhandlers.ProxyHandler({})
             self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler(
                 pwrdmgr)
             self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler(
                 pwrdmgr)
             self.ntlm_auth_handler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(
                 pwrdmgr)
             proxyHandlers = [
                 self.proxy_handler, self.ntlm_auth_handler,
                 self.proxy_auth_handler, self.http_auth_handler
             ]
     if not self.hasNTLM:
         self.proxy_handler = proxyhandlers.ProxyHandler(
             proxyDirFmt(httpProxyTuple))
         self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler()
         self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
         proxyHandlers = [
             self.proxy_handler, self.proxy_auth_handler,
             self.http_auth_handler
         ]
     if ssl and self.noCertificateCheck:  # this is required in some Akamai environments, such as sec.gov
         context = ssl.create_default_context()
         context.check_hostname = False
         context.verify_mode = ssl.CERT_NONE
         proxyHandlers.append(proxyhandlers.HTTPSHandler(context=context))
     self.opener = proxyhandlers.build_opener(*proxyHandlers)
     self.opener.addheaders = [('User-Agent', self.httpUserAgent),
                               ('Accept-Encoding', 'gzip, deflate')]
Ejemplo n.º 3
0
    def request(self, url, params=None, decode=True):
        args = [url]

        try:
            if proxy['enable'] and self.site_name in url:
                opener = request.build_opener(
                    request.ProxyBasicAuthHandler(),
                    request.ProxyHandler(proxy['proxy_urls']))

                logger.info('proxy used for "%s"', url)
            else:
                opener = request.build_opener()

            # use cookies only for lostfilm site urls
            if self.site_name in url:
                if not params:
                    params = self.cookies
                else:
                    params.update(self.cookies)

            if params:
                args.append(parse.urlencode(params).encode('utf-8'))

            result = opener.open(*args).read()

            return result if not decode else result.decode('utf-8')

        except Exception as e:
            logger.error('%s url="%s" params="%s"' % (e, url, params))
Ejemplo n.º 4
0
def use_proxy(url):
    proxy_handler = request.ProxyHandler({'http': '122.96.59.103:82/'})
    proxy_auth_handler = request.ProxyBasicAuthHandler()
    #proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
    opener = request.build_opener(proxy_handler, proxy_auth_handler)
    response = opener.open(url)
    return response
Ejemplo n.º 5
0
def money_proxy_use():
    # 第一种方式付费代理发送请求
    # 代理IP
    money_proxy = {"http": "username:[email protected]:8080"}
    # 2.代理的处理器
    proxy_handler = request.ProxyHandler(money_proxy)

    # 3.通过处理器创建opener
    opener = request.build_opener(proxy_handler)
    # 4.open发送请求
    opener.open("http://www.baidu.com")

    # 第二种方式发送付费的IP代理
    user_name = "abcname"
    pwd = "123456"
    proxy_money = "158.63.66.77:8080"
    # 创建密码管理器,添加用户名和密码
    password_manager = request.HTTPPasswordMgrWithPriorAuth()
    # uri定位 uri>url
    # url 资源地位符
    password_manager.add_password(None, proxy_money, user_name, pwd)
    # 创建可以验证代理IP的处理器
    handle_auth_proxy = request.ProxyBasicAuthHandler(password_manager)
    # 根据处理器创建opener
    opener_auth = request.build_opener(handle_auth_proxy)
    # 发送请求
    response = opener.open("http://www.baid.com")
    print(response.read())
Ejemplo n.º 6
0
def proxy():
    proxy_handler = request.ProxyHandler(
        {'http': 'http://www.example.com:3128/'})
    proxy_auth_handler = request.ProxyBasicAuthHandler()
    proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
    opener = request.build_opener(proxy_handler, proxy_auth_handler)
    with opener.open('http://www.example.com/login.html') as f:
        pass
Ejemplo n.º 7
0
def get_url_Proxy():
    get_url = 'https://www.cnblogs.com/'
    proxy_handler = request.ProxyHandler({'http': '58.64.57.12:3128'})
    proxy_auth_handler = request.ProxyBasicAuthHandler()
    proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
    opener = request.build_opener(proxy_handler, proxy_auth_handler)
    with opener.open(get_url) as f:
        print(f.read().decode('utf-8'))
Ejemplo n.º 8
0
def testProxy():
    proxy_handler = request.ProxyHandler({'http': 'http://www.baidu.com/'})
    proxy_auth_handler = request.ProxyBasicAuthHandler()
    proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
    opener = request.build_opener(proxy_handler, proxy_auth_handler)
    with opener.open('http://www.baidu.com/') as f:
        print('Status:', f.status, f.reason)
        for k, v in f.getheaders():
            print('%s: %s' % (k, v))
        print('Data:', f.read())
Ejemplo n.º 9
0
def installopener():
    opener = urllib2.build_opener(
        urllib2.ProxyHandler(proxies_dict),
        urllib2.ProxyBasicAuthHandler(proxypwdmgr),
        urllib2.ProxyDigestAuthHandler(proxypwdmgr),
        urllib2.HTTPHandler,
    )
    # digest auth may not work with all proxies
    # http://bugs.python.org/issue16095
    # could add windows/nltm authentication here
    #opener=urllib2.build_opener(urllib2.ProxyHandler(proxies_dict), urllib2.HTTPHandler)

    urllib2.install_opener(opener)
Ejemplo n.º 10
0
def create_proxy_handler2():
    username = '******'
    password = '******'
    proxy_host = '123.158.63.130:8080'

    password_manager = request.HTTPPasswordMgrWithDefaultRealm()
    password_manager.add_password('',
                                  uri=proxy_host,
                                  user=username,
                                  passwd=password)
    handle_auth_proxy = request.ProxyBasicAuthHandler(password_manager)
    opener_auth = request.build_opener(handle_auth_proxy)

    response = opener_auth.open('http://www.baidu.com')
    print(response.read())
Ejemplo n.º 11
0
def get_content(spider_data):
    if spider_data['proxy_ip'] != '0.0.0.0':
        proxy_handler = request.ProxyHandler({'http': spider_data['proxy_ip']})
        proxy_auth_handler = request.ProxyBasicAuthHandler()
        opener = request.build_opener(proxy_handler, proxy_auth_handler)
        request.install_opener(opener)
    forged_header = {
        'User-Agent': spider_data['user_agent'],
        'Referer': spider_data['referer'],
        'Host': spider_data['host'],
        spider_data['request_type']: spider_data['url']
    }
    req = request.Request(spider_data['url'], headers=forged_header)
    html = request.urlopen(req, timeout=8)
    content = html.read()
    encoding = chardet.detect(content)['encoding']
    #html = request.urlopen(req, timeout=8)
    return content.decode(encoding)
Ejemplo n.º 12
0
def pwd_demo():
    # 私密代理授权的账户 密码
    user = '******'
    pwd = '1234'
    # 私密代理IP
    proxy_server = '119.129.99.29:1231'
    # 构建一个密码管理对象,保存用户名和密码
    pwdmgr = request.HTTPPasswordMgrWithDefaultRealm()
    # 添加账户信息
    pwdmgr.add_password(None, proxy_server, user=user, passwd=pwd)
    # 构建一个代理基础用户名/密码验证的ProxyBasicAuthHandler对象,参数是创建的密码管理对象
    proxy_handler = request.ProxyBasicAuthHandler(pwdmgr)
    # 通过build_opener方法使用Handler创建自定义opener
    opener = request.build_opener(proxy_handler)
    # 构造request请求
    req = request.Request("http://www.baidu.com")
    # 发送请求并打印响应
    response = opener.open(req)
    print(response.read().decode('utf-8'))
Ejemplo n.º 13
0
 def resetProxies(self, httpProxyTuple):
     try:
         from ntlm import HTTPNtlmAuthHandler
         self.hasNTLM = True
     except ImportError:
         self.hasNTLM = False
     self.proxy_handler = proxyhandlers.ProxyHandler(
         proxyDirFmt(httpProxyTuple))
     self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler()
     self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
     if self.hasNTLM:
         self.ntlm_auth_handler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler()
         self.opener = proxyhandlers.build_opener(self.proxy_handler,
                                                  self.ntlm_auth_handler,
                                                  self.proxy_auth_handler,
                                                  self.http_auth_handler)
     else:
         self.opener = proxyhandlers.build_opener(self.proxy_handler,
                                                  self.proxy_auth_handler,
                                                  self.http_auth_handler)
Ejemplo n.º 14
0
def use_proxy3():
    from urllib import request
    # case 3: authenticated proxy 2
    password = request.HTTPPasswordMgrWithDefaultRealm()
    proxy_auth = request.ProxyBasicAuthHandler(password)
    password.add_password(realm=None,
                          uri="http://127.0.0.1:1080",
                          user='******',
                          passwd='password')

    proxy_support = request.ProxyHandler({
        "http": "http://127.0.0.1:1080",
        "https": "http://127.0.0.1:1080"
    })

    opener = request.build_opener(proxy_support, proxy_auth)
    # request.install_opener(opener)
    # r2 = request.urlopen('http://www.twitter.com')
    # or this blow:
    r2 = opener.open('https://www.twitter.com')
    print(r2.read(100))
Ejemplo n.º 15
0
def main():
    name = "test"
    pswd = "123456"
    webserver = "192.168.78.1"

    # 很少用到这种情况
    # 创建一个密码管理对象,用来保存和http请求相关的授权账号信息
    pswd_mgr = urllib2.HTTPPasswordMgrWithPriorAuth()
    # 添加一个账号授权信息,第一个参数realm(域)如果没有就天None,然后依次是站点ip,用户名,密码
    pswd_mgr.add_password(None, webserver, name, pswd)

    # http基础验证处理器类
    http_auth_handler = urllib2.HTTPBasiAuthHandler(pswd_mgr)
    # 代理基础验证处理器类,(代理验证用ProxyHandler({"http":"user:pswd@ip:port"})更方便)
    proxy_auth_handler = urllib2.ProxyBasicAuthHandler(pswd_mgr)

    # build_opener可以添加多个处理器
    opener = urllib2.build_opener(http_auth_handler, proxy_auth_handler)
    request = urllib2.Request(f"http://{webserver}")
    response = opener.open(request)
    print(response.read())
Ejemplo n.º 16
0
    def opener(self, purl, puser=None, ppass=None, pscheme=('http', 'https')):
        if REDIRECT:
            h1, h2 = REDIRECT.BindableHTTPHandler, REDIRECT.BindableHTTPSHandler
        else:
            h1, h2 = ul2.HTTPHandler, ul2.HTTPSHandler

        handlers = [h1(), h2(), ul2.HTTPCookieProcessor(self.cookies)]
        if purl:
            handlers += [
                ul2.ProxyHandler({ps: purl
                                  for ps in pscheme}),
            ]
            #handlers += [ul2.ProxyHandler({ps:purl}) for ps in pscheme]

        if puser and ppass:
            pm = ul2.HTTPPasswordMgrWithDefaultRealm()
            pm.add_password(None, purl, puser, ppass)
            handlers += [
                ul2.ProxyBasicAuthHandler(pm),
            ]

        return ul2.build_opener(*handlers)
Ejemplo n.º 17
0
def getReq():
	global init

	if init:
		return 
	print('start setting the proxy! ')

	user = '******'
	passwd = 'maxzhang3'
	url = 'proxy.ict:8080'

	proxy_support =	request.ProxyHandler({'http':'http://%s:%s@%s' %(user,passwd,url)})
	passwdmgr= request.HTTPPasswordMgrWithDefaultRealm()
	passwdmgr.add_password(None,'hq.sinajs.cn','aspac\\axesr','maxzhang3')
	from ntlm import HTTPNtlmAuthHandler
	auth_NTLM = HTTPNtlmAuthHandler.ProxyNtlmAuthHandler(passwdmgr)
	proxy_auth_handler = request.ProxyBasicAuthHandler(passwdmgr)
	


	opener = request.build_opener(proxy_support)
	request.install_opener(opener)
	init = True
def soup1(url):
   
    try:
        username='******'
        password=getpass.getpass('input password:'******'www.someproxyaddress.com:8080'
        
        proxy_handler = u.ProxyHandler({'http':'%s'%(proxy)})
        password_mgr = u.HTTPPasswordMgrWithDefaultRealm()
        proxy_auth_handler = u.ProxyBasicAuthHandler(password_mgr)
        proxy_auth_handler.add_password(None, 'http://%s'%(proxy),'%s'%(username), '%s'%(password))
        
        opener = u.build_opener(proxy_handler,proxy_auth_handler)
        req = u.Request(url,headers={'User-Agent': 'Mozilla/5.0'})
        r = opener.open(req)
        result = r.read()
        soup=bs(result,'html.parser')
        print(url)

        return soup
    
      except Exception as e:
        print(e)
Ejemplo n.º 19
0
email = input('Email: ')
passwd = input('Password: '******'username', email),
    ('password', passwd),
    ('entry', 'mweibo'),
    ('client_id', ''),
    ('savestate', '1'),
    ('ec', ''),
    ('pagerefer', 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F%3Fjumpfrom%3Dweibocom&jumpfrom=weibocom')
])

req = request.Request('https://passport.weibo.cn/sso/login')
req.add_header('Origin', 'https://passport.weibo.cn')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F%3Fjumpfrom%3Dweibocom')

with request.urlopen(req, data=login_data.encode('utf-8')) as f:
    print('Status:', f.status, f.reason)
    for k, v in f.getheaders():
        print('%s: %s' % (k, v))
    print('Data:', f.read().decode('utf-8'))

# with proxy and proxy auth:

proxy_handler = request.ProxyHandler({'http': 'http://www.example.com:3128/'})
proxy_auth_handler = request.ProxyBasicAuthHandler()
proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
opener = request.build_opener(proxy_handler, proxy_auth_handler)
with opener.open('http://www.example.com/login.html') as f:
    pass
Ejemplo n.º 20
0
# 创一个cookie处理器
cookies = request.HTTPCookieProcessor(cookie)
# 以它为参数,创建opener对象
opener = request.build_opener(cookies)
# 使用这个opener 来发请求
res =opener.open('https://www.baidu.com/')
print(cookies.cookiejar)

#3. 设置代理
#运行爬虫的时候,经常会出现被封IP的情况,这时我们就需要使用ip代理来处理,urllib的IP代理的设置如下:

url ='http://httpbin.org/ip'
#代理地址
proxy ={'http':'172.0.0.1:3128'}
# 代理处理器
proxies =request.ProxyBasicAuthHandler(proxy)
# 创建opener对象
opener = request.build_opener(proxies)
re =opener.open(url)
print(re.read().decode())


"""
 urlib库中的类或或者方法,在发送网络请求后,都会返回一个urllib.response的对象。
 它包含了请求回来的数据结果。它包含了一些属性和方法,供我们处理返回的结果
read() 获取响应返回的数据,只能用一次 
readline() 读取一行 
info() 获取响应头信息 
geturl() 获取访问的url
getcode() 返回状态码 
Ejemplo n.º 21
0
from urllib import request
"""
request.HTTPPasswordMgrWithDefaultRealm  密码管理对象,用来保存和HTTP请求相关的授权信息
request.ProxyBasicAuthHandler  授权代理处理器
request.HTTPBasicAuthHandler   验证web客户端的授权处理器

"""

usn = 'test'
pwd = '123456'
web_server = '122.114.31.177:808'
# 构建一个密码管理对象
password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
# 添加授权用户信息
password_mgr.add_password(None, web_server, usn, pwd)
# 以密码管理对象为参数,创建auth handler
# 注意这里处理的是web服务器的验证信息,不是代理服务器的验证信息
httpauth_handler = request.HTTPBasicAuthHandler(password_mgr)
# 这个才是处理代理服务器验证信息的handler,可以同时作为创建opener的参数
proxyauth_handler = request.ProxyBasicAuthHandler(password_mgr)

opener = request.build_opener(httpauth_handler, proxyauth_handler)
request_obj = request.Request('http://' + web_server)
response = opener.open(request_obj)
print(response.read())
Ejemplo n.º 22
0
    ('client_id',''),
    ('savestate','1'),
    ('ec',''),
    ('pagerefer', 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F')
])

print(login_data)

req = request.Request('https://passport.weibo.cn/sso/login')
req.add_header('Origin', 'https://passport.weibo.cn')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F')

# 开始请求
with request.urlopen(req,data=login_data.encode('utf-8')) as f:
    print('Status:',f.status,f.reason)
    for k,v in f.getheaders():
        print('%s:%s' % (k,v))
    print('Data:',f.read().decode('utf-8'))
'''
'''
Handler
- 通过一个proxy去访问网站
'''

proxyHandler = request.ProxyHandler({'http': 'http://dev-proxy.oa.com:8080'})
proxyAuthHandler = request.ProxyBasicAuthHandler()
proxyAuthHandler.add_password('realm', 'host', 'username', 'password')
opener = request.build_opener(proxyHandler, proxyAuthHandler)
with opener.open('http://www.qq.com') as f:
    print(f.read().decode('utf-8'))
Ejemplo n.º 23
0
from urllib import request

#私密代理授权的账户
user = '******'

#私密代理授权的密码
password = '******'

#私密代理IP
proxyserver = '61.158.130:16816'

#构建一个密码管理对象,用来保存需要的用户名和密码
passwdmgr = request.HTTPPasswordMgrWithDefaultRealm()

#添加用户的信息,第一个参数realm是与远程服务器的域信息,一般下写None,后面三个参数分别是
passwdmgr.add_password(None, proxyserver, None, None)

#构建一个代理基础用户名/密码验证的ProxyBasicAuthHandler处理器对象,参数是创建的密码管理对象
#注意,这里不再使用普通ProxyHandler
proxyauth_handler = request.ProxyBasicAuthHandler(None)

opener = request.build_opener(proxyauth_handler)
opener.add_handler = []
request.install_opener(opener)
response = opener.open(request.Request('http://www.baidu.com/'))
print(response.read().decode('utf-8'))

print('popi')