def process(self): parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] host = parsed[1].decode('ascii') if protocol in self.ports: port = self.ports[protocol] else: # handle pass if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' if protocol in self.protocols: factory = self.protocols[protocol] headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') headers.pop(b'user-agent', None) headers[b'user-agent'] = b'I2P' self.content.seek(0, 0) s = self.content.read() client = factory(self.method, rest, self.clientproto, headers, s, self) ep = self.endpointFactory(host, port) connectProtocol(ep, client.buildProtocol(ep))
def process(self): parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] host = parsed[1].decode('ascii') port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' class_ = self.protocols[protocol] headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') self.content.seek(0, 0) s = self.content.read() clientFactory = class_(self.method, rest, self.clientproto, headers, s, self) self.reactor.connectTCP(host, port, clientFactory)
def render(self, request): """ Render a request by forwarding it to the proxied server. """ if self.port == 80: host = self.host else: host = self.host + u":" + str(self.port) request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')]) request.content.seek(0, 0) qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b'?' + qs else: rest = self.path clientFactory = self.proxyClientFactoryClass( request.method, rest, request.clientproto, request.getAllHeaders(), request.content.read(), request) self.reactor.connectTCP(self.host, self.port, clientFactory) return NOT_DONE_YET
def process(self): try: parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] host = parsed[1].decode('ascii') port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' class_ = self.protocols[protocol] headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') self.content.seek(0, 0) s = self.content.read() clientFactory = class_(self.method, rest, self.clientproto, headers, s, self) log.msg(self.uri) m = hashlib.md5() if self.method == 'POST': self.content.seek(0) post_params = self.content.read() else: post_params = '' m.update(self.uri + post_params) self.cache_filename = os.path.join('cached_files', m.hexdigest() + ".cached") log.msg('Search for {}'.format(self.cache_filename)) if os.path.exists(self.cache_filename) and time.time() - os.path.getmtime(self.cache_filename) < 60: log.msg("Cache hit") data = open(self.cache_filename).read() self.write(data) log.msg('Cache is go to client') else: log.msg("Cache miss") self.reactor.connectTCP(host, port, clientFactory) except KeyError: log.msg("HTTPS is not supported at the moment!")
def process_prepare(self): # print self.method, self.uri, self.path, self.args, self.requestHeaders, self.responseHeaders, self.received_cookies, self.protocols, self.host, self.channel, self.content, self.cookies parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] or 'http' host = parsed[1].decode('ascii') port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' class_ = self.protocols[protocol] headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') self.content.seek(0, 0) s = self.content.read() clientFactory = class_(self.method, rest, self.clientproto, headers, s, self) return host, port, clientFactory
def __init__( self, url, user=None, password=None, allowNone=False, useDateTime=False, connectTimeout=30.0, reactor=reactor ): """ @param url: The URL to which to post method calls. Calls will be made over SSL if the scheme is HTTPS. If netloc contains username or password information, these will be used to authenticate, as long as the C{user} and C{password} arguments are not specified. @type url: L{bytes} """ scheme, netloc, path, params, query, fragment = urllib_parse.urlparse(url) netlocParts = netloc.split(b"@") if len(netlocParts) == 2: userpass = netlocParts.pop(0).split(b":") self.user = userpass.pop(0) try: self.password = userpass.pop(0) except: self.password = None else: self.user = self.password = None hostport = netlocParts[0].split(b":") self.host = hostport.pop(0) try: self.port = int(hostport.pop(0)) except: self.port = None self.path = path if self.path in [b"", None]: self.path = b"/" self.secure = scheme == b"https" if user is not None: self.user = user if password is not None: self.password = password self.allowNone = allowNone self.useDateTime = useDateTime self.connectTimeout = connectTimeout self._reactor = reactor
def __init__(self, url, user=None, password=None, allowNone=False, useDateTime=False, connectTimeout=30.0, reactor=reactor): """ @param url: The URL to which to post method calls. Calls will be made over SSL if the scheme is HTTPS. If netloc contains username or password information, these will be used to authenticate, as long as the C{user} and C{password} arguments are not specified. @type url: L{bytes} """ scheme, netloc, path, params, query, fragment = urllib_parse.urlparse( url) netlocParts = netloc.split(b'@') if len(netlocParts) == 2: userpass = netlocParts.pop(0).split(b':') self.user = userpass.pop(0) try: self.password = userpass.pop(0) except: self.password = None else: self.user = self.password = None hostport = netlocParts[0].split(b':') self.host = hostport.pop(0) try: self.port = int(hostport.pop(0)) except: self.port = None self.path = path if self.path in [b'', None]: self.path = b'/' self.secure = (scheme == b'https') if user is not None: self.user = user if password is not None: self.password = password self.allowNone = allowNone self.useDateTime = useDateTime self.connectTimeout = connectTimeout self._reactor = reactor
def render(self, request): if self.port == 80: host = self.host else: host = u"%s:%d" % (self.host, self.port) request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')]) gidd = request.getCookie(b'_gid') print('gid cookie:', gidd) for key in request.received_cookies.keys(): print("cookies:",key," ", request.received_cookies.get(key)) request.content.seek(0, 0) qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b'?' + qs else: rest = self.path clientFactory = self.proxyClientFactoryClass( request.method, rest, request.clientproto, request.getAllHeaders(), request.content.read(), request) self.reactor.connectTCP(self.host, self.port, clientFactory) return NOT_DONE_YET
def render(self, request): """ Render a request by forwarding it to the proxied server. """ # RFC 2616 tells us that we can omit the port if it's the default port, # but we have to provide it otherwise if self.port == 80: host = self.host else: host = self.host + u":" + str(self.port) request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')]) request.content.seek(0, 0) qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b'?' + qs else: rest = self.path clientFactory = self.proxyClientFactoryClass( request.method, rest, request.clientproto, request.getAllHeaders(), request.content.read(), request) self.reactor.connectTCP(self.host, self.port, clientFactory) return NOT_DONE_YET
def render(self, request): """ Render a request by forwarding it to the proxied server. """ # RFC 2616 tells us that we can omit the port if it's the default port, # but we have to provide it otherwise if self.port == 80: host = self.host else: host = u"%s:%d" % (self.host, self.port) request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')]) request.content.seek(0, 0) qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b'?' + qs else: rest = self.path clientFactory = self.proxyClientFactoryClass( request.method, rest, request.clientproto, request.getAllHeaders(), request.content.read(), request) self.reactor.connectTCP(self.host, self.port, clientFactory) return NOT_DONE_YET
def implReverseProxyRenderKeepHost(self, request): """ Actual resource rendering. If client connects over IPv4: query C{DNS} for the host's C{AAAA} record. Or if client connects over IPv6: use the hostname of the webproxy. Use resulting IPv6 address to establish the TCP connection, but keep the hostname as value of the C{Host} HTTP header. A C{X-Forwarded-For} header is added to avoid circular references. """ if isinstance(request.client, address.IPv6Address ) and not request.client.host.startswith("::ffff:"): self.host = ip426webproxy dns_result = yield lookupIPV6Address(self.host) self.host = socket.inet_ntop(socket.AF_INET6, dns_result[0][0].payload.address) request.content.seek(0, 0) qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b"?" + qs else: rest = self.path headers = request.getAllHeaders() headers[b"x-forwarded-for"] = request.getClientIP().encode("utf-8") clientFactory = self.proxyClientFactoryClass( request.method, rest, request.clientproto, headers, request.content.read(), request, ) self.reactor.connectTCP(self.host, self.port, clientFactory)
def process(self): log.msg('Processing: %s' % self.uri) parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] if protocol == b'': protocol = b'http' elif protocol == b'https': raise ConnectionRefusedError( 'The https protocol is not supported.') host = parsed[1].decode('ascii') if host is b'': raise ConnectionRefusedError('Host is empty.') port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') self.content.seek(0, 0) content = self.content.read() new_c_factory = self.factories[protocol] c_factory = new_c_factory(self.method, rest, self.clientproto, headers, content, self) self.client_endpoint = TCP4ClientEndpoint(self.reactor, host, port) d = self.client_endpoint.connect(c_factory) d.addCallback(lambda: self.got_protocol)
def process_prepare(self): parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] host = parsed[1].decode('ascii') port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' class_ = self.protocols[protocol] headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') self.content.seek(0, 0) s = self.content.read() clientFactory = class_(self.method, rest, self.clientproto, headers, s, self) # if self.uri == 'http://music.163.com/eapi/song/like': # print rest, headers, s # if self.uri == 'http://music.163.com/eapi/v1/playlist/manipulate/tracks': # print rest, headers, s return host, port, clientFactory
def process(self): log.msg('Processing: %s' % self.uri) parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] if protocol == b'': protocol = b'http' elif protocol == b'https': raise ConnectionRefusedError('The https protocol is not supported.') host = parsed[1].decode('ascii') if host is b'': raise ConnectionRefusedError('Host is empty.') port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') self.content.seek(0, 0) content = self.content.read() new_c_factory = self.factories[protocol] c_factory = new_c_factory(self.method, rest, self.clientproto, headers, content, self) self.client_endpoint = TCP4ClientEndpoint(self.reactor, host, port) d = self.client_endpoint.connect(c_factory) d.addCallback(lambda: self.got_protocol)
def render(self, request): """ Render a request by forwarding it to the proxied server. """ # RFC 2616 tells us that we can omit the port if it's the default port, # but we have to provide it otherwise self.logDebug("render", "Called...") if self.port == 80: host = self.host else: host = self.host + u":" + str(self.port) request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')]) request.content.seek(0, 0) qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b'?' + qs else: rest = self.path self.logDebug("render", "Creating factory") content = request.content.read() try: if request.getHeader("Accept-Encoding") == 'gzip': request.requestHeaders.removeHeader("accept-encoding") request.setHeader("accept-encoding", "identity") if request.getHeader( "Content-Type") == "application/vnd.ms-sync.wbxml": # its activesynccmd cmd = request.args['Cmd'][0] if cmd in self.commands: self.commands[cmd] = self.commands[cmd] + 1 else: self.commands[cmd] = 1 dump_file_prefix = self.uuid + "." + cmd + "." + str( self.commands[cmd]) cntnt = content FileDumper.dumpFile("request", dump_file_prefix, "headers", pprint.pformat(vars(request))) if (request.getHeader("Content-encoding") == 'gzip'): self.logDebug("render", "Request content seems gzipped") cntnt = zlib.decompress(cntnt, 16 + zlib.MAX_WBITS) FileDumper.dumpFile("request", dump_file_prefix, "wbxml", cntnt) xml = pnqwbxml2xml(cntnt) FileDumper.dumpFile("request", dump_file_prefix, "xml", xml) except: exc_type, exc_value, exc_traceback = sys.exc_info() exc = traceback.format_exception(exc_type, exc_value, exc_traceback) self.logDebug("render", "Something went seriously wrong!!!!") for e in exc: self.logDebug("render", e) clientFactory = self.proxyClientFactoryClass(self.uuid, request.method, rest, request.clientproto, request.getAllHeaders(), content, request) self.reactor.connectSSL(self.host, self.port, clientFactory, ssl.ClientContextFactory()) self.logDebug("render", "NOT_DONE_YET") return NOT_DONE_YET
def render(self, request): """ Render a request by forwarding it to the proxied server. """ self.log.info('{klass}.render(): forwarding incoming HTTP request ..', klass=self.__class__.__name__) # host request by client in incoming HTTP request requested_host = request.requestHeaders.getRawHeaders('Host')[0] # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host request.requestHeaders.setRawHeaders(b'X-Forwarded-Host', [requested_host.encode('ascii')]) # request.requestHeaders.setRawHeaders(b'X-Forwarded-Server', [requested_host.encode('ascii')]) # crossbar web transport listening IP/port server_port = request.getHost().port server_port = '{}'.format(server_port).encode('ascii') # RFC 2616 tells us that we can omit the port if it's the default port, # but we have to provide it otherwise if self.port == 80: host = self.host else: host = '%s:%d' % (self.host, self.port) request.requestHeaders.setRawHeaders(b'Host', [host.encode('utf8')]) # forward originating IP of incoming HTTP request client_ip = request.getClientAddress().host if client_ip: client_ip = client_ip.encode('ascii') request.requestHeaders.setRawHeaders(b'X-Forwarded-For', [client_ip]) request.requestHeaders.setRawHeaders(b'X-Real-IP', [client_ip]) # forward information of outside listening port and protocol (http vs https) if self._forwarded_port: request.requestHeaders.setRawHeaders( b'X-Forwarded-Port', ['{}'.format(self._forwarded_port).encode('ascii')]) else: request.requestHeaders.setRawHeaders(b'X-Forwarded-Port', [server_port]) if self._forwarded_proto: request.requestHeaders.setRawHeaders(b'X-Forwarded-Proto', [self._forwarded_proto]) else: request.requestHeaders.setRawHeaders( b'X-Forwarded-Proto', [('https' if server_port == 443 else 'http').encode('ascii')]) # rewind cursor to begin of request data request.content.seek(0, 0) # reapply query strings to forwarding HTTP request qs = urllib_parse.urlparse(request.uri)[4] if qs: rest = self.path + b'?' + qs else: rest = self.path self.log.info( 'forwarding HTTP request to "{rest}" with HTTP request headers {headers}', rest=rest, headers=request.getAllHeaders()) # now issue the forwarded request to the HTTP server that is being reverse-proxied clientFactory = self.proxyClientFactoryClass(request.method, rest, request.clientproto, request.getAllHeaders(), request.content.read(), request) self.reactor.connectTCP(self.host, self.port, clientFactory) # the proxy client request created ^ is taking care of actually finishing the request .. return NOT_DONE_YET
def process(self): method = self.method.decode('ascii') uri = self.uri.decode('ascii') if method == 'CONNECT': host, port = split_host_port(uri) port = int(port) else: parsed = urllib_parse.urlparse(self.uri) decoded = parsed[1].decode('ascii') host, port = split_host_port(decoded) if port is None: port = 80 rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' headers = self.getAllHeaders().copy() self.content.seek(0, 0) s = self.content.read() proxy_suggestion = (self.force_proxy or self.force_direct or pacparser.find_proxy('http://{}'.format(host))) proxy_suggestions = proxy_suggestion.split(";") parsed_proxy_suggestion = self.proxy_suggestion_parser.match( proxy_suggestions[0]) if parsed_proxy_suggestion: connect_method, destination = parsed_proxy_suggestion.groups() if connect_method == 'PROXY': proxy_host, proxy_port = destination.split(":") proxy_port = int(proxy_port) if method != 'CONNECT': clientFactory = proxy.ProxyClientFactory( self.method, self.uri, self.clientproto, headers, s, self, ) logger.info('%s %s; forwarding request to %s:%s', method, uri, proxy_host, proxy_port) else: self.transport.unregisterProducer() self.transport.pauseProducing() rawConnectionProtocol = portforward.Proxy() rawConnectionProtocol.transport = self.transport self.transport.protocol = rawConnectionProtocol clientFactory = CONNECTProtocolForwardFactory(host, port) clientFactory.setServer(rawConnectionProtocol) logger.info('%s %s; establishing tunnel through %s:%s', method, uri, proxy_host, proxy_port) self.reactor.connectTCP(proxy_host, proxy_port, clientFactory) return else: # can this be anything else? Let's fall back to the DIRECT # codepath. pass if method != 'CONNECT': if b'host' not in headers: headers[b'host'] = host.encode('ascii') clientFactory = proxy.ProxyClientFactory( self.method, rest, self.clientproto, headers, s, self, ) logger.info('%s %s; forwarding request', method, uri) self.reactor.connectTCP(host, port, clientFactory) else: # hack/trick to move responsibility for this connection # away from a HTTP protocol class hierarchy and to a # port forward hierarchy self.transport.unregisterProducer() self.transport.pauseProducing() rawConnectionProtocol = portforward.Proxy() rawConnectionProtocol.transport = self.transport self.transport.protocol = rawConnectionProtocol clientFactory = portforward.ProxyClientFactory() clientFactory.setServer(rawConnectionProtocol) clientFactory.protocol = CONNECTProtocolClient # we don't do connectSSL, as the handshake is taken # care of by the client, and we only forward it logger.info('%s %s; establishing tunnel to %s:%s', method, uri, host, port) self.reactor.connectTCP(host, port, clientFactory)
def process(self, args=None): if not self.has_valid_creds(): self.request_creds() return None parsed = urllib_parse.urlparse(self.uri) protocol = parsed[0] host = parsed[1].decode('ascii') port = 80 if protocol != b'': port = self.ports[protocol] if ':' in host: host, port = host.split(':') port = int(port) doSSL = False if self.method.upper() == b"CONNECT": # TODO: finish HTTPS support #self.setResponseCode(200) self.transport.write( "HTTP/1.1 200 Connection established\r\nConnection: close\n\n". encode("ascii")) self.transport.startTLS(MySSLContext2()) protocol = b"https" self.host = parsed.scheme #self.transport.write("HTTPS unsupported\r\n".encode("ascii")) self.finish() return else: if self.isSecure(): headers = self.getAllHeaders().copy() if "host" not in headers: self.setResponseCode(400) self.write("Malformed request\r\n".encode("ascii")) self.finish() return host = headers["host"] protocol = b"https" doSSL = True if protocol not in self.ports: self.setResponseCode(400) self.write("Unsupported protocol\r\n".encode("ascii")) self.finish() return None rest = urllib_parse.urlunparse((b'', b'') + parsed[2:]) if not rest: rest = rest + b'/' if protocol not in self.protocols: self.setResponseCode(400) self.write("Unsupported protocol\r\n".encode("ascii")) self.finish() return None class_ = self.protocols[protocol] headers = self.getAllHeaders().copy() if b'host' not in headers: headers[b'host'] = host.encode('ascii') # Add x-forwarded-for headers[b'X-Forwarded-For'] = self.getClientIP().encode('ascii') self.content.seek(0, 0) s = self.content.read() clientFactory = class_(self.method, rest, self.clientproto, headers, s, self) #print("About to connect upstream to {}:{}".format(host, port)) #print("Headers for upstream: {}".format(headers)) dec_path = self.path.decode("ascii", "ignore") if is_internal_page(dec_path): # 1st priority: internal pages self.serve_internal(dec_path, doSSL) elif should_block(dec_path): # 2nd priority: blocked page self.serve_blocked(doSSL) else: # 3rd priority: regular page if doSSL: print("Trying connect ssl to {} on {}".format(host, port)) self.reactor.connectSSL(host, port, clientFactory, ssl.ClientContextFactory()) else: self.reactor.connectTCP(host, port, clientFactory)