def process(self):
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
host = parsed[1].decode('ascii')
if protocol in self.ports:
port = self.ports[protocol]
else:
# handle
pass
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
if protocol in self.protocols:
factory = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
headers.pop(b'user-agent', None)
headers[b'user-agent'] = b'I2P'
self.content.seek(0, 0)
s = self.content.read()
client = factory(self.method, rest, self.clientproto, headers, s, self)
ep = self.endpointFactory(host, port)
connectProtocol(ep, client.buildProtocol(ep))
def process(self):
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
host = parsed[1].decode('ascii')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
class_ = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
s = self.content.read()
clientFactory = class_(self.method, rest, self.clientproto, headers,
s, self)
self.reactor.connectTCP(host, port, clientFactory)
def render(self, request):
"""
Render a request by forwarding it to the proxied server.
"""
if self.port == 80:
host = self.host
else:
host = self.host + u":" + str(self.port)
request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')])
request.content.seek(0, 0)
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b'?' + qs
else:
rest = self.path
clientFactory = self.proxyClientFactoryClass(
request.method, rest, request.clientproto,
request.getAllHeaders(), request.content.read(), request)
self.reactor.connectTCP(self.host, self.port, clientFactory)
return NOT_DONE_YET
def process(self):
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
host = parsed[1].decode('ascii')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
class_ = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
s = self.content.read()
clientFactory = class_(self.method, rest, self.clientproto, headers, s,
self)
self.reactor.connectTCP(host, port, clientFactory)
def process(self):
try:
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
host = parsed[1].decode('ascii')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
class_ = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
s = self.content.read()
clientFactory = class_(self.method, rest, self.clientproto, headers,
s, self)
log.msg(self.uri)
m = hashlib.md5()
if self.method == 'POST':
self.content.seek(0)
post_params = self.content.read()
else:
post_params = ''
m.update(self.uri + post_params)
self.cache_filename = os.path.join('cached_files', m.hexdigest() + ".cached")
log.msg('Search for {}'.format(self.cache_filename))
if os.path.exists(self.cache_filename) and time.time() - os.path.getmtime(self.cache_filename) < 60:
log.msg("Cache hit")
data = open(self.cache_filename).read()
self.write(data)
log.msg('Cache is go to client')
else:
log.msg("Cache miss")
self.reactor.connectTCP(host, port, clientFactory)
except KeyError:
log.msg("HTTPS is not supported at the moment!")
def process_prepare(self):
# print self.method, self.uri, self.path, self.args, self.requestHeaders, self.responseHeaders, self.received_cookies, self.protocols, self.host, self.channel, self.content, self.cookies
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0] or 'http'
host = parsed[1].decode('ascii')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
class_ = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
s = self.content.read()
clientFactory = class_(self.method, rest, self.clientproto, headers, s,
self)
return host, port, clientFactory
def __init__(
self, url, user=None, password=None, allowNone=False, useDateTime=False, connectTimeout=30.0, reactor=reactor
):
"""
@param url: The URL to which to post method calls. Calls will be made
over SSL if the scheme is HTTPS. If netloc contains username or
password information, these will be used to authenticate, as long as
the C{user} and C{password} arguments are not specified.
@type url: L{bytes}
"""
scheme, netloc, path, params, query, fragment = urllib_parse.urlparse(url)
netlocParts = netloc.split(b"@")
if len(netlocParts) == 2:
userpass = netlocParts.pop(0).split(b":")
self.user = userpass.pop(0)
try:
self.password = userpass.pop(0)
except:
self.password = None
else:
self.user = self.password = None
hostport = netlocParts[0].split(b":")
self.host = hostport.pop(0)
try:
self.port = int(hostport.pop(0))
except:
self.port = None
self.path = path
if self.path in [b"", None]:
self.path = b"/"
self.secure = scheme == b"https"
if user is not None:
self.user = user
if password is not None:
self.password = password
self.allowNone = allowNone
self.useDateTime = useDateTime
self.connectTimeout = connectTimeout
self._reactor = reactor
def __init__(self, url, user=None, password=None, allowNone=False,
useDateTime=False, connectTimeout=30.0, reactor=reactor):
"""
@param url: The URL to which to post method calls. Calls will be made
over SSL if the scheme is HTTPS. If netloc contains username or
password information, these will be used to authenticate, as long as
the C{user} and C{password} arguments are not specified.
@type url: L{bytes}
"""
scheme, netloc, path, params, query, fragment = urllib_parse.urlparse(
url)
netlocParts = netloc.split(b'@')
if len(netlocParts) == 2:
userpass = netlocParts.pop(0).split(b':')
self.user = userpass.pop(0)
try:
self.password = userpass.pop(0)
except:
self.password = None
else:
self.user = self.password = None
hostport = netlocParts[0].split(b':')
self.host = hostport.pop(0)
try:
self.port = int(hostport.pop(0))
except:
self.port = None
self.path = path
if self.path in [b'', None]:
self.path = b'/'
self.secure = (scheme == b'https')
if user is not None:
self.user = user
if password is not None:
self.password = password
self.allowNone = allowNone
self.useDateTime = useDateTime
self.connectTimeout = connectTimeout
self._reactor = reactor
def render(self, request):
if self.port == 80:
host = self.host
else:
host = u"%s:%d" % (self.host, self.port)
request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')])
gidd = request.getCookie(b'_gid')
print('gid cookie:', gidd)
for key in request.received_cookies.keys():
print("cookies:",key," ", request.received_cookies.get(key))
request.content.seek(0, 0)
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b'?' + qs
else:
rest = self.path
clientFactory = self.proxyClientFactoryClass(
request.method, rest, request.clientproto,
request.getAllHeaders(), request.content.read(), request)
self.reactor.connectTCP(self.host, self.port, clientFactory)
return NOT_DONE_YET
def render(self, request):
"""
Render a request by forwarding it to the proxied server.
"""
# RFC 2616 tells us that we can omit the port if it's the default port,
# but we have to provide it otherwise
if self.port == 80:
host = self.host
else:
host = self.host + u":" + str(self.port)
request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')])
request.content.seek(0, 0)
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b'?' + qs
else:
rest = self.path
clientFactory = self.proxyClientFactoryClass(
request.method, rest, request.clientproto,
request.getAllHeaders(), request.content.read(), request)
self.reactor.connectTCP(self.host, self.port, clientFactory)
return NOT_DONE_YET
def render(self, request):
"""
Render a request by forwarding it to the proxied server.
"""
# RFC 2616 tells us that we can omit the port if it's the default port,
# but we have to provide it otherwise
if self.port == 80:
host = self.host
else:
host = u"%s:%d" % (self.host, self.port)
request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')])
request.content.seek(0, 0)
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b'?' + qs
else:
rest = self.path
clientFactory = self.proxyClientFactoryClass(
request.method, rest, request.clientproto,
request.getAllHeaders(), request.content.read(), request)
self.reactor.connectTCP(self.host, self.port, clientFactory)
return NOT_DONE_YET
def implReverseProxyRenderKeepHost(self, request):
"""
Actual resource rendering.
If client connects over IPv4: query C{DNS} for the host's C{AAAA} record.
Or if client connects over IPv6: use the hostname of the webproxy.
Use resulting IPv6 address to establish the TCP connection,
but keep the hostname as value of the C{Host} HTTP header.
A C{X-Forwarded-For} header is added to avoid circular references.
"""
if isinstance(request.client, address.IPv6Address
) and not request.client.host.startswith("::ffff:"):
self.host = ip426webproxy
dns_result = yield lookupIPV6Address(self.host)
self.host = socket.inet_ntop(socket.AF_INET6,
dns_result[0][0].payload.address)
request.content.seek(0, 0)
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b"?" + qs
else:
rest = self.path
headers = request.getAllHeaders()
headers[b"x-forwarded-for"] = request.getClientIP().encode("utf-8")
clientFactory = self.proxyClientFactoryClass(
request.method,
rest,
request.clientproto,
headers,
request.content.read(),
request,
)
self.reactor.connectTCP(self.host, self.port, clientFactory)
def process(self):
log.msg('Processing: %s' % self.uri)
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
if protocol == b'':
protocol = b'http'
elif protocol == b'https':
raise ConnectionRefusedError(
'The https protocol is not supported.')
host = parsed[1].decode('ascii')
if host is b'':
raise ConnectionRefusedError('Host is empty.')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
content = self.content.read()
new_c_factory = self.factories[protocol]
c_factory = new_c_factory(self.method, rest, self.clientproto, headers,
content, self)
self.client_endpoint = TCP4ClientEndpoint(self.reactor, host, port)
d = self.client_endpoint.connect(c_factory)
d.addCallback(lambda: self.got_protocol)
def process_prepare(self):
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
host = parsed[1].decode('ascii')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
class_ = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
s = self.content.read()
clientFactory = class_(self.method, rest, self.clientproto, headers, s, self)
# if self.uri == 'http://music.163.com/eapi/song/like':
# print rest, headers, s
# if self.uri == 'http://music.163.com/eapi/v1/playlist/manipulate/tracks':
# print rest, headers, s
return host, port, clientFactory
def process(self):
log.msg('Processing: %s' % self.uri)
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
if protocol == b'':
protocol = b'http'
elif protocol == b'https':
raise ConnectionRefusedError('The https protocol is not supported.')
host = parsed[1].decode('ascii')
if host is b'':
raise ConnectionRefusedError('Host is empty.')
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
self.content.seek(0, 0)
content = self.content.read()
new_c_factory = self.factories[protocol]
c_factory = new_c_factory(self.method, rest, self.clientproto, headers, content, self)
self.client_endpoint = TCP4ClientEndpoint(self.reactor, host, port)
d = self.client_endpoint.connect(c_factory)
d.addCallback(lambda: self.got_protocol)
def render(self, request):
"""
Render a request by forwarding it to the proxied server.
"""
# RFC 2616 tells us that we can omit the port if it's the default port,
# but we have to provide it otherwise
self.logDebug("render", "Called...")
if self.port == 80:
host = self.host
else:
host = self.host + u":" + str(self.port)
request.requestHeaders.setRawHeaders(b"host", [host.encode('ascii')])
request.content.seek(0, 0)
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b'?' + qs
else:
rest = self.path
self.logDebug("render", "Creating factory")
content = request.content.read()
try:
if request.getHeader("Accept-Encoding") == 'gzip':
request.requestHeaders.removeHeader("accept-encoding")
request.setHeader("accept-encoding", "identity")
if request.getHeader(
"Content-Type") == "application/vnd.ms-sync.wbxml":
# its activesynccmd
cmd = request.args['Cmd'][0]
if cmd in self.commands:
self.commands[cmd] = self.commands[cmd] + 1
else:
self.commands[cmd] = 1
dump_file_prefix = self.uuid + "." + cmd + "." + str(
self.commands[cmd])
cntnt = content
FileDumper.dumpFile("request", dump_file_prefix, "headers",
pprint.pformat(vars(request)))
if (request.getHeader("Content-encoding") == 'gzip'):
self.logDebug("render", "Request content seems gzipped")
cntnt = zlib.decompress(cntnt, 16 + zlib.MAX_WBITS)
FileDumper.dumpFile("request", dump_file_prefix, "wbxml",
cntnt)
xml = pnqwbxml2xml(cntnt)
FileDumper.dumpFile("request", dump_file_prefix, "xml", xml)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_exception(exc_type, exc_value,
exc_traceback)
self.logDebug("render", "Something went seriously wrong!!!!")
for e in exc:
self.logDebug("render", e)
clientFactory = self.proxyClientFactoryClass(self.uuid, request.method,
rest, request.clientproto,
request.getAllHeaders(),
content, request)
self.reactor.connectSSL(self.host, self.port, clientFactory,
ssl.ClientContextFactory())
self.logDebug("render", "NOT_DONE_YET")
return NOT_DONE_YET
def render(self, request):
"""
Render a request by forwarding it to the proxied server.
"""
self.log.info('{klass}.render(): forwarding incoming HTTP request ..',
klass=self.__class__.__name__)
# host request by client in incoming HTTP request
requested_host = request.requestHeaders.getRawHeaders('Host')[0]
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host
request.requestHeaders.setRawHeaders(b'X-Forwarded-Host',
[requested_host.encode('ascii')])
# request.requestHeaders.setRawHeaders(b'X-Forwarded-Server', [requested_host.encode('ascii')])
# crossbar web transport listening IP/port
server_port = request.getHost().port
server_port = '{}'.format(server_port).encode('ascii')
# RFC 2616 tells us that we can omit the port if it's the default port,
# but we have to provide it otherwise
if self.port == 80:
host = self.host
else:
host = '%s:%d' % (self.host, self.port)
request.requestHeaders.setRawHeaders(b'Host', [host.encode('utf8')])
# forward originating IP of incoming HTTP request
client_ip = request.getClientAddress().host
if client_ip:
client_ip = client_ip.encode('ascii')
request.requestHeaders.setRawHeaders(b'X-Forwarded-For',
[client_ip])
request.requestHeaders.setRawHeaders(b'X-Real-IP', [client_ip])
# forward information of outside listening port and protocol (http vs https)
if self._forwarded_port:
request.requestHeaders.setRawHeaders(
b'X-Forwarded-Port',
['{}'.format(self._forwarded_port).encode('ascii')])
else:
request.requestHeaders.setRawHeaders(b'X-Forwarded-Port',
[server_port])
if self._forwarded_proto:
request.requestHeaders.setRawHeaders(b'X-Forwarded-Proto',
[self._forwarded_proto])
else:
request.requestHeaders.setRawHeaders(
b'X-Forwarded-Proto',
[('https' if server_port == 443 else 'http').encode('ascii')])
# rewind cursor to begin of request data
request.content.seek(0, 0)
# reapply query strings to forwarding HTTP request
qs = urllib_parse.urlparse(request.uri)[4]
if qs:
rest = self.path + b'?' + qs
else:
rest = self.path
self.log.info(
'forwarding HTTP request to "{rest}" with HTTP request headers {headers}',
rest=rest,
headers=request.getAllHeaders())
# now issue the forwarded request to the HTTP server that is being reverse-proxied
clientFactory = self.proxyClientFactoryClass(request.method, rest,
request.clientproto,
request.getAllHeaders(),
request.content.read(),
request)
self.reactor.connectTCP(self.host, self.port, clientFactory)
# the proxy client request created ^ is taking care of actually finishing the request ..
return NOT_DONE_YET
def process(self):
method = self.method.decode('ascii')
uri = self.uri.decode('ascii')
if method == 'CONNECT':
host, port = split_host_port(uri)
port = int(port)
else:
parsed = urllib_parse.urlparse(self.uri)
decoded = parsed[1].decode('ascii')
host, port = split_host_port(decoded)
if port is None:
port = 80
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
headers = self.getAllHeaders().copy()
self.content.seek(0, 0)
s = self.content.read()
proxy_suggestion = (self.force_proxy or self.force_direct
or pacparser.find_proxy('http://{}'.format(host)))
proxy_suggestions = proxy_suggestion.split(";")
parsed_proxy_suggestion = self.proxy_suggestion_parser.match(
proxy_suggestions[0])
if parsed_proxy_suggestion:
connect_method, destination = parsed_proxy_suggestion.groups()
if connect_method == 'PROXY':
proxy_host, proxy_port = destination.split(":")
proxy_port = int(proxy_port)
if method != 'CONNECT':
clientFactory = proxy.ProxyClientFactory(
self.method,
self.uri,
self.clientproto,
headers,
s,
self,
)
logger.info('%s %s; forwarding request to %s:%s', method,
uri, proxy_host, proxy_port)
else:
self.transport.unregisterProducer()
self.transport.pauseProducing()
rawConnectionProtocol = portforward.Proxy()
rawConnectionProtocol.transport = self.transport
self.transport.protocol = rawConnectionProtocol
clientFactory = CONNECTProtocolForwardFactory(host, port)
clientFactory.setServer(rawConnectionProtocol)
logger.info('%s %s; establishing tunnel through %s:%s',
method, uri, proxy_host, proxy_port)
self.reactor.connectTCP(proxy_host, proxy_port, clientFactory)
return
else:
# can this be anything else? Let's fall back to the DIRECT
# codepath.
pass
if method != 'CONNECT':
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
clientFactory = proxy.ProxyClientFactory(
self.method,
rest,
self.clientproto,
headers,
s,
self,
)
logger.info('%s %s; forwarding request', method, uri)
self.reactor.connectTCP(host, port, clientFactory)
else:
# hack/trick to move responsibility for this connection
# away from a HTTP protocol class hierarchy and to a
# port forward hierarchy
self.transport.unregisterProducer()
self.transport.pauseProducing()
rawConnectionProtocol = portforward.Proxy()
rawConnectionProtocol.transport = self.transport
self.transport.protocol = rawConnectionProtocol
clientFactory = portforward.ProxyClientFactory()
clientFactory.setServer(rawConnectionProtocol)
clientFactory.protocol = CONNECTProtocolClient
# we don't do connectSSL, as the handshake is taken
# care of by the client, and we only forward it
logger.info('%s %s; establishing tunnel to %s:%s', method, uri,
host, port)
self.reactor.connectTCP(host, port, clientFactory)
def process(self, args=None):
if not self.has_valid_creds():
self.request_creds()
return None
parsed = urllib_parse.urlparse(self.uri)
protocol = parsed[0]
host = parsed[1].decode('ascii')
port = 80
if protocol != b'':
port = self.ports[protocol]
if ':' in host:
host, port = host.split(':')
port = int(port)
doSSL = False
if self.method.upper() == b"CONNECT": # TODO: finish HTTPS support
#self.setResponseCode(200)
self.transport.write(
"HTTP/1.1 200 Connection established\r\nConnection: close\n\n".
encode("ascii"))
self.transport.startTLS(MySSLContext2())
protocol = b"https"
self.host = parsed.scheme
#self.transport.write("HTTPS unsupported\r\n".encode("ascii"))
self.finish()
return
else:
if self.isSecure():
headers = self.getAllHeaders().copy()
if "host" not in headers:
self.setResponseCode(400)
self.write("Malformed request\r\n".encode("ascii"))
self.finish()
return
host = headers["host"]
protocol = b"https"
doSSL = True
if protocol not in self.ports:
self.setResponseCode(400)
self.write("Unsupported protocol\r\n".encode("ascii"))
self.finish()
return None
rest = urllib_parse.urlunparse((b'', b'') + parsed[2:])
if not rest:
rest = rest + b'/'
if protocol not in self.protocols:
self.setResponseCode(400)
self.write("Unsupported protocol\r\n".encode("ascii"))
self.finish()
return None
class_ = self.protocols[protocol]
headers = self.getAllHeaders().copy()
if b'host' not in headers:
headers[b'host'] = host.encode('ascii')
# Add x-forwarded-for
headers[b'X-Forwarded-For'] = self.getClientIP().encode('ascii')
self.content.seek(0, 0)
s = self.content.read()
clientFactory = class_(self.method, rest, self.clientproto, headers, s,
self)
#print("About to connect upstream to {}:{}".format(host, port))
#print("Headers for upstream: {}".format(headers))
dec_path = self.path.decode("ascii", "ignore")
if is_internal_page(dec_path):
# 1st priority: internal pages
self.serve_internal(dec_path, doSSL)
elif should_block(dec_path):
# 2nd priority: blocked page
self.serve_blocked(doSSL)
else:
# 3rd priority: regular page
if doSSL:
print("Trying connect ssl to {} on {}".format(host, port))
self.reactor.connectSSL(host, port, clientFactory,
ssl.ClientContextFactory())
else:
self.reactor.connectTCP(host, port, clientFactory)
