def SetUrlHandler(self,
UsePoster=False,
UseCookie=False,
PacketDebug=False):
opener = urllib2._opener
if opener == None:
#opener = urllib2.OpenerDirector()
opener = urllib2.build_opener()
'''
opener.add_handler(urllib2.ProxyHandler())
opener.add_handler(urllib2.UnknownHandler())
opener.add_handler(urllib2.HTTPHandler())
opener.add_handler(urllib2.HTTPDefaultErrorHandler())
opener.add_handler(urllib2.HTTPSHandler())
opener.add_handler(urllib2.HTTPErrorProcessor())
'''
if UseCookie is True:
opener.add_handler(urllib2.HTTPCookieProcessor())
#opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
if PacketDebug is True:
opener.add_handler(urllib2.HTTPHandler(debuglevel=1))
opener.add_handler(urllib2.HTTPSHandler(debuglevel=1))
#opener = urllib2.build_opener(httpHandler, httpsHandler)
if UsePoster is True:
from poster.streaminghttp import StreamingHTTPHandler, StreamingHTTPRedirectHandler, StreamingHTTPSHandler
opener.add_handler(StreamingHTTPHandler())
opener.add_handler(StreamingHTTPRedirectHandler())
opener.add_handler(StreamingHTTPSHandler())
urllib2.install_opener(opener)
return opener
def set_handlers(self):
handlers = [
StreamingHTTPHandler(debuglevel=self.__loglevel),
StreamingHTTPRedirectHandler,
StreamingHTTPSHandler(debuglevel=self.__loglevel)
]
cookie = cookielib.MozillaCookieJar(self.__cookiefile)
handlers.append(urllib2.HTTPCookieProcessor(cookie))
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
self.__urllib = urllib2
return cookie
def set_handlers(self):
cookie_file = C.DEFAULT_COOKIEFILE
loglevel = C.DEFAULT_LOGLEVEL
handlers = [
StreamingHTTPHandler(debuglevel=loglevel),
StreamingHTTPRedirectHandler,
StreamingHTTPSHandler(debuglevel=loglevel)
]
cookie = cookielib.MozillaCookieJar(cookie_file)
handlers.append(urllib2.HTTPCookieProcessor(cookie))
opener = urllib2.build_opener(*handlers)
urllib2.install_opener(opener)
self.__urllib = urllib2
return cookie
def __init__(self,
proxy=None,
noverify=True,
follow_redirects=False,
headers={},
timeout=5,
cadata=None):
self.ctx = ssl.create_default_context(cadata=cadata)
if noverify:
self.ctx.check_hostname = False
self.ctx.verify_mode = ssl.CERT_NONE
self.proxy = None
tproxy = type(proxy)
if tproxy in (str, unicode):
proxyscheme = urlparse.urlparse(proxy)
scheme = proxyscheme.scheme.upper()
if scheme == 'SOCKS':
scheme = 'SOCKS5'
self.proxy = scheme, proxyscheme.hostname+(
':'+str(proxyscheme.port) if proxyscheme.port else ''), \
proxyscheme.username or None, \
proxyscheme.password or None
elif proxy in (True, None):
self.proxy = find_default_proxy()
elif hasattr(proxy, 'as_tuple'):
self.proxy = proxy.as_tuple()
else:
self.proxy = proxy
self.noverify = noverify
self.timeout = timeout
if not self.proxy:
handlers = [
StreamingHTTPHandler,
StreamingHTTPSHandler(context=self.ctx),
TCPReaderHandler(context=self.ctx)
]
else:
scheme, host, user, password = self.proxy
scheme = socks.PROXY_TYPES[scheme]
port = socks.DEFAULT_PORTS[scheme]
if ':' in host:
host, maybe_port = host.split(':')
try:
port = int(maybe_port)
except ValueError:
pass
sockshandler = SocksiPyHandler(
scheme,
host,
port,
user or None,
password or None,
context=self.ctx if self.noverify else None)
handlers = []
if scheme == socks.HTTP:
http_proxy = '{}:{}'.format(host, port)
if user and password:
http_proxy = '{}:{}@{}'.format(user, password, http_proxy)
http_proxy = 'http://' + http_proxy
handlers.append(urllib2.ProxyHandler({'http': http_proxy}))
handlers.append(StreamingHTTPHandler)
handlers.append(sockshandler)
if not follow_redirects:
handlers.append(NoRedirects)
handlers.append(UDPReaderHandler)
self.opener = urllib2.OpenerDirector()
for h in handlers:
if isinstance(h, (types.ClassType, type)):
h = h()
self.opener.add_handler(h)
if type(headers) == dict:
self.opener.addheaders = [(x, y) for x, y in headers.iteritems()]
else:
self.opener.addheaders = headers
def __init__(self, connection_class=VerifiedHTTPSConnection):
self.specialized_conn_class = connection_class
HTTPSHandler.__init__(self)
def __init__(self, debuglevel=0, key_file=None, cert_file=None):
self.key_file = key_file
self.cert_file = cert_file
StreamingHTTPSHandler.__init__(self, debuglevel)
def __init__(self, debuglevel=0, key_file=None, cert_file=None):
self.key_file = key_file
self.cert_file = cert_file
StreamingHTTPSHandler.__init__(self, debuglevel)
def __init__(self, connection_class=VerifiedHTTPSConnection):
self.specialized_conn_class = connection_class
HTTPSHandler.__init__(self)
def __init__(self,
proxy=None,
noverify=True,
follow_redirects=False,
headers={},
timeout=5,
cadata=None):
self.ctx = ssl.create_default_context()
if noverify:
self.ctx.check_hostname = False
self.ctx.verify_mode = ssl.CERT_NONE
if cadata:
self.ctx.load_verify_locations(None, None, cadata)
else:
self.ctx.load_default_certs()
self.proxy = proxy
self.noverify = noverify
self.timeout = timeout
if self.proxy is None or self.proxy is True:
handlers = [
urllib2.ProxyHandler(), StreamingHTTPHandler,
StreamingHTTPSHandler(context=self.ctx),
TCPReaderHandler(context=self.ctx)
]
elif self.proxy is False:
handlers = [
StreamingHTTPHandler,
StreamingHTTPSHandler(context=self.ctx),
TCPReaderHandler(context=self.ctx)
]
else:
proxyscheme = urlparse.urlparse(self.proxy)
scheme = proxyscheme.scheme.upper()
if scheme == 'SOCKS':
scheme = 'SOCKS5'
scheme = socks.PROXY_TYPES[scheme]
sockshandler = SocksiPyHandler(
scheme,
proxyscheme.hostname,
proxyscheme.port or socks.DEFAULT_PORTS[scheme],
username=proxyscheme.username or None,
password=proxyscheme.password or None,
context=self.ctx if self.noverify else None)
handlers = [sockshandler]
if not follow_redirects:
handlers.append(NoRedirects)
handlers.append(UDPReaderHandler)
self.opener = urllib2.build_opener(*handlers)
if type(headers) == dict:
self.opener.addheaders = [(x, y) for x, y in headers.iteritems()]
else:
self.opener.addheaders = headers
def make_opener(self, address, headers=None):
scheme = None
proxy_host = None
proxy_password_manager = None
http_password_manager = OptionalPasswordManager()
password_managers = []
if self.proxy == 'wpad':
proxy = get_proxy_for_address(address)
if proxy:
proxy = proxy[0]
else:
proxy = None
else:
proxy = self.proxy
if not proxy or proxy[0] == 'DIRECT' or self._is_direct(address):
handlers = [
StreamingHTTPHandler,
StreamingHTTPSHandler(context=self.ctx),
TCPReaderHandler(context=self.ctx)
]
else:
scheme, host, user, password = proxy
scheme = PROXY_TYPES[scheme]
port = DEFAULT_PORTS[scheme]
if ':' in host:
host, maybe_port = host.split(':')
try:
port = int(maybe_port)
except ValueError:
pass
proxy_host = host + ':' + str(port)
sockshandler = SocksiPyHandler(
scheme,
host,
port,
user or None,
password or None,
context=self.ctx if self.noverify else None)
handlers = []
if scheme == PROXY_SCHEME_HTTP:
http_proxy = proxy_host
handlers.append(
urllib2.ProxyHandler({'http': 'http://' + http_proxy}))
proxy_password_manager = ProxyPasswordManager(
'http', host, port, user, password)
for handler_klass in (ProxyAuthHandler,
urllib2.ProxyBasicAuthHandler,
urllib2.ProxyDigestAuthHandler):
if handler_klass is None:
continue
instance = handler_klass(proxy_password_manager)
if hasattr(instance, 'set_logger'):
instance.set_logger(logger)
handlers.append(instance)
password_managers.append(proxy_password_manager)
handlers.append(StreamingHTTPHandler)
handlers.append(sockshandler)
if self.follow_redirects:
handlers.append(urllib2.HTTPRedirectHandler)
else:
handlers.append(NoRedirects)
handlers.append(UDPReaderHandler)
for handler_klass in (urllib2.HTTPBasicAuthHandler,
urllib2.HTTPDigestAuthHandler, HTTPAuthHandler):
if handler_klass is None:
continue
instance = handler_klass(http_password_manager)
if hasattr(instance, 'set_logger'):
instance.set_logger(logger)
handlers.append(instance)
password_managers.append(http_password_manager)
context = HTTPContext.get_default()
handlers.append(context)
handlers.append(urllib2.HTTPDefaultErrorHandler)
handlers.append(urllib2.HTTPErrorProcessor)
opener = urllib2.OpenerDirector()
for h in handlers:
if isinstance(h, (types.ClassType, type)):
h = h()
opener.add_handler(h)
filter_headers = set()
if headers:
if isinstance(headers, dict):
filter_headers = set(headers.keys())
else:
filter_headers = set(x for x, _ in headers)
if isinstance(self.headers, dict):
opener.addheaders = [(x, y) for x, y in self.headers.iteritems()
if x not in filter_headers]
else:
opener.addheaders = self.headers
if headers:
if isinstance(headers, dict):
opener.addheaders.extend([
(x, y) for x, y in self.headers.iteritems()
])
else:
opener.addheaders.extend(headers)
return opener, scheme, proxy_host, password_managers, context
