def SetUrlHandler(self, UsePoster=False, UseCookie=False, PacketDebug=False): opener = urllib2._opener if opener == None: #opener = urllib2.OpenerDirector() opener = urllib2.build_opener() ''' opener.add_handler(urllib2.ProxyHandler()) opener.add_handler(urllib2.UnknownHandler()) opener.add_handler(urllib2.HTTPHandler()) opener.add_handler(urllib2.HTTPDefaultErrorHandler()) opener.add_handler(urllib2.HTTPSHandler()) opener.add_handler(urllib2.HTTPErrorProcessor()) ''' if UseCookie is True: opener.add_handler(urllib2.HTTPCookieProcessor()) #opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) if PacketDebug is True: opener.add_handler(urllib2.HTTPHandler(debuglevel=1)) opener.add_handler(urllib2.HTTPSHandler(debuglevel=1)) #opener = urllib2.build_opener(httpHandler, httpsHandler) if UsePoster is True: from poster.streaminghttp import StreamingHTTPHandler, StreamingHTTPRedirectHandler, StreamingHTTPSHandler opener.add_handler(StreamingHTTPHandler()) opener.add_handler(StreamingHTTPRedirectHandler()) opener.add_handler(StreamingHTTPSHandler()) urllib2.install_opener(opener) return opener
def set_handlers(self): handlers = [ StreamingHTTPHandler(debuglevel=self.__loglevel), StreamingHTTPRedirectHandler, StreamingHTTPSHandler(debuglevel=self.__loglevel) ] cookie = cookielib.MozillaCookieJar(self.__cookiefile) handlers.append(urllib2.HTTPCookieProcessor(cookie)) opener = urllib2.build_opener(*handlers) urllib2.install_opener(opener) self.__urllib = urllib2 return cookie
def set_handlers(self): cookie_file = C.DEFAULT_COOKIEFILE loglevel = C.DEFAULT_LOGLEVEL handlers = [ StreamingHTTPHandler(debuglevel=loglevel), StreamingHTTPRedirectHandler, StreamingHTTPSHandler(debuglevel=loglevel) ] cookie = cookielib.MozillaCookieJar(cookie_file) handlers.append(urllib2.HTTPCookieProcessor(cookie)) opener = urllib2.build_opener(*handlers) urllib2.install_opener(opener) self.__urllib = urllib2 return cookie
def __init__(self, proxy=None, noverify=True, follow_redirects=False, headers={}, timeout=5, cadata=None): self.ctx = ssl.create_default_context(cadata=cadata) if noverify: self.ctx.check_hostname = False self.ctx.verify_mode = ssl.CERT_NONE self.proxy = None tproxy = type(proxy) if tproxy in (str, unicode): proxyscheme = urlparse.urlparse(proxy) scheme = proxyscheme.scheme.upper() if scheme == 'SOCKS': scheme = 'SOCKS5' self.proxy = scheme, proxyscheme.hostname+( ':'+str(proxyscheme.port) if proxyscheme.port else ''), \ proxyscheme.username or None, \ proxyscheme.password or None elif proxy in (True, None): self.proxy = find_default_proxy() elif hasattr(proxy, 'as_tuple'): self.proxy = proxy.as_tuple() else: self.proxy = proxy self.noverify = noverify self.timeout = timeout if not self.proxy: handlers = [ StreamingHTTPHandler, StreamingHTTPSHandler(context=self.ctx), TCPReaderHandler(context=self.ctx) ] else: scheme, host, user, password = self.proxy scheme = socks.PROXY_TYPES[scheme] port = socks.DEFAULT_PORTS[scheme] if ':' in host: host, maybe_port = host.split(':') try: port = int(maybe_port) except ValueError: pass sockshandler = SocksiPyHandler( scheme, host, port, user or None, password or None, context=self.ctx if self.noverify else None) handlers = [] if scheme == socks.HTTP: http_proxy = '{}:{}'.format(host, port) if user and password: http_proxy = '{}:{}@{}'.format(user, password, http_proxy) http_proxy = 'http://' + http_proxy handlers.append(urllib2.ProxyHandler({'http': http_proxy})) handlers.append(StreamingHTTPHandler) handlers.append(sockshandler) if not follow_redirects: handlers.append(NoRedirects) handlers.append(UDPReaderHandler) self.opener = urllib2.OpenerDirector() for h in handlers: if isinstance(h, (types.ClassType, type)): h = h() self.opener.add_handler(h) if type(headers) == dict: self.opener.addheaders = [(x, y) for x, y in headers.iteritems()] else: self.opener.addheaders = headers
def __init__(self, connection_class=VerifiedHTTPSConnection): self.specialized_conn_class = connection_class HTTPSHandler.__init__(self)
def __init__(self, debuglevel=0, key_file=None, cert_file=None): self.key_file = key_file self.cert_file = cert_file StreamingHTTPSHandler.__init__(self, debuglevel)
def __init__(self, debuglevel=0, key_file=None, cert_file=None): self.key_file = key_file self.cert_file = cert_file StreamingHTTPSHandler.__init__(self, debuglevel)
def __init__(self, connection_class=VerifiedHTTPSConnection): self.specialized_conn_class = connection_class HTTPSHandler.__init__(self)
def __init__(self, proxy=None, noverify=True, follow_redirects=False, headers={}, timeout=5, cadata=None): self.ctx = ssl.create_default_context() if noverify: self.ctx.check_hostname = False self.ctx.verify_mode = ssl.CERT_NONE if cadata: self.ctx.load_verify_locations(None, None, cadata) else: self.ctx.load_default_certs() self.proxy = proxy self.noverify = noverify self.timeout = timeout if self.proxy is None or self.proxy is True: handlers = [ urllib2.ProxyHandler(), StreamingHTTPHandler, StreamingHTTPSHandler(context=self.ctx), TCPReaderHandler(context=self.ctx) ] elif self.proxy is False: handlers = [ StreamingHTTPHandler, StreamingHTTPSHandler(context=self.ctx), TCPReaderHandler(context=self.ctx) ] else: proxyscheme = urlparse.urlparse(self.proxy) scheme = proxyscheme.scheme.upper() if scheme == 'SOCKS': scheme = 'SOCKS5' scheme = socks.PROXY_TYPES[scheme] sockshandler = SocksiPyHandler( scheme, proxyscheme.hostname, proxyscheme.port or socks.DEFAULT_PORTS[scheme], username=proxyscheme.username or None, password=proxyscheme.password or None, context=self.ctx if self.noverify else None) handlers = [sockshandler] if not follow_redirects: handlers.append(NoRedirects) handlers.append(UDPReaderHandler) self.opener = urllib2.build_opener(*handlers) if type(headers) == dict: self.opener.addheaders = [(x, y) for x, y in headers.iteritems()] else: self.opener.addheaders = headers
def make_opener(self, address, headers=None): scheme = None proxy_host = None proxy_password_manager = None http_password_manager = OptionalPasswordManager() password_managers = [] if self.proxy == 'wpad': proxy = get_proxy_for_address(address) if proxy: proxy = proxy[0] else: proxy = None else: proxy = self.proxy if not proxy or proxy[0] == 'DIRECT' or self._is_direct(address): handlers = [ StreamingHTTPHandler, StreamingHTTPSHandler(context=self.ctx), TCPReaderHandler(context=self.ctx) ] else: scheme, host, user, password = proxy scheme = PROXY_TYPES[scheme] port = DEFAULT_PORTS[scheme] if ':' in host: host, maybe_port = host.split(':') try: port = int(maybe_port) except ValueError: pass proxy_host = host + ':' + str(port) sockshandler = SocksiPyHandler( scheme, host, port, user or None, password or None, context=self.ctx if self.noverify else None) handlers = [] if scheme == PROXY_SCHEME_HTTP: http_proxy = proxy_host handlers.append( urllib2.ProxyHandler({'http': 'http://' + http_proxy})) proxy_password_manager = ProxyPasswordManager( 'http', host, port, user, password) for handler_klass in (ProxyAuthHandler, urllib2.ProxyBasicAuthHandler, urllib2.ProxyDigestAuthHandler): if handler_klass is None: continue instance = handler_klass(proxy_password_manager) if hasattr(instance, 'set_logger'): instance.set_logger(logger) handlers.append(instance) password_managers.append(proxy_password_manager) handlers.append(StreamingHTTPHandler) handlers.append(sockshandler) if self.follow_redirects: handlers.append(urllib2.HTTPRedirectHandler) else: handlers.append(NoRedirects) handlers.append(UDPReaderHandler) for handler_klass in (urllib2.HTTPBasicAuthHandler, urllib2.HTTPDigestAuthHandler, HTTPAuthHandler): if handler_klass is None: continue instance = handler_klass(http_password_manager) if hasattr(instance, 'set_logger'): instance.set_logger(logger) handlers.append(instance) password_managers.append(http_password_manager) context = HTTPContext.get_default() handlers.append(context) handlers.append(urllib2.HTTPDefaultErrorHandler) handlers.append(urllib2.HTTPErrorProcessor) opener = urllib2.OpenerDirector() for h in handlers: if isinstance(h, (types.ClassType, type)): h = h() opener.add_handler(h) filter_headers = set() if headers: if isinstance(headers, dict): filter_headers = set(headers.keys()) else: filter_headers = set(x for x, _ in headers) if isinstance(self.headers, dict): opener.addheaders = [(x, y) for x, y in self.headers.iteritems() if x not in filter_headers] else: opener.addheaders = self.headers if headers: if isinstance(headers, dict): opener.addheaders.extend([ (x, y) for x, y in self.headers.iteritems() ]) else: opener.addheaders.extend(headers) return opener, scheme, proxy_host, password_managers, context