def create_http_fwd_connection(self, destport, dest_addr='127.0.0.1', peer=None,
localport=None, trans=None, httpaddr='127.0.0.1',
**connection_kwargs):
"""
Create an http connection with port fowarding over this ssh session.
:param destport: port to forward (ie 80 for http)
:param dest_addr: addr used for remote request
:param peer: Remote host
:param localport: Local port used to forward
:param trans: ssh transport obj
:param httpaddr: addr used for http connection
:returns HTTP connection obj
"""
trans = trans or self.connection._transport
assert isinstance(trans, paramiko.Transport)
if peer is None:
peer = trans.getpeername()[0]
if localport is None:
localport = self._get_local_unused_port(start=9000)
self.debug('Making forwarded request from "localhost:{0}" to "{1}:{2}"'
.format(localport, peer, destport))
self.debug('open_channel(kind="{0}", dest_addr=("{1}", {2}), src_addr=("{3}", {4})'
.format('direct-tcpip', dest_addr, destport, peer, localport))
chan = trans.open_channel(kind='direct-tcpip', dest_addr=(dest_addr, destport),
src_addr=(peer, localport))
connection_kwargs = connection_kwargs or {}
connection_kwargs['host'] = httpaddr
connection_kwargs['port'] = destport
self.debug('Creating HTTP connection with kwargs:\n{0}\n'.format(connection_kwargs))
http = HTTPConnection(**connection_kwargs)
http.sock = chan
return http
def _resolve_http_redirect(self, url, header, depth=0):
"""this method sends http(s) requests and also resolves redirects"""
if depth > 10:
raise Exception("Redirected " + str(depth) + " times, giving up.")
parsed_url = urlparse(url, allow_fragments=True)
if (parsed_url.scheme == "http"):
conn = HTTPConnection(parsed_url.netloc)
elif (parsed_url.scheme == "https"):
conn = HTTPSConnection(parsed_url.netloc)
path = parsed_url.path
if (parsed_url.query):
path += "?" + parsed_url.query
try:
conn.request("GET", path, None, header)
except (OSError, UnicodeEncodeError):
conn.close()
raise Exception("OSError or UnicodeEncodeError")
try:
res = conn.getresponse()
except (HTTPException):
logging.error("HTTPException")
raise Exception("HTTPException")
if (conn.sock):
conn.sock.close()
conn.sock = None
if (res != None):
headers = dict(res.getheaders())
else:
raise Exception("Response empty")
loc_key = None
if ("Location" in headers):
loc_key = "Location"
elif ("location" in headers):
loc_key = "location"
if (loc_key != None and headers[loc_key] != url):
if ("IndexRedirect" in headers[loc_key]):
logging.error("IndexRedirect, exiting")
exit(1)
return self._resolve_http_redirect(
headers[loc_key],
header,
depth+1
)
else:
return res, headers
def _resolve_http_redirect(self, url, header, depth=0):
"""this method sends http(s) requests and also resolves redirects"""
if depth > 10:
raise Exception("Redirected " + str(depth) + " times, giving up.")
parsed_url = urlparse(url, allow_fragments=True)
if (parsed_url.scheme == "http"):
conn = HTTPConnection(parsed_url.netloc)
elif (parsed_url.scheme == "https"):
conn = HTTPSConnection(parsed_url.netloc)
path = parsed_url.path
if (parsed_url.query):
path += "?" + parsed_url.query
try:
conn.request("GET", path, None, header)
except (OSError, UnicodeEncodeError):
conn.close()
raise Exception("OSError or UnicodeEncodeError")
try:
res = conn.getresponse()
except (HTTPException):
logging.error("HTTPException")
raise Exception("HTTPException")
if (conn.sock):
conn.sock.close()
conn.sock = None
if (res != None):
headers = dict(res.getheaders())
else:
raise Exception("Response empty")
loc_key = None
if ("Location" in headers):
loc_key = "Location"
elif ("location" in headers):
loc_key = "location"
if (loc_key != None and headers[loc_key] != url):
if ("IndexRedirect" in headers[loc_key]):
logging.error("IndexRedirect, exiting")
exit(1)
return self._resolve_http_redirect(headers[loc_key], header,
depth + 1)
else:
return res, headers
def forward_hosts_http(self, addrs, host, port, environ, start_response):
try:
method, url, body, headers = copy_request(environ)
except:
start_response("400 Bad Request",
[("Content-Type", "text/plain; charset=utf-8")])
yield "Bad Request"
return
try:
u = urlparse.urlsplit(url)
if self.matcher.need_redirect(method, host):
start_response(
"%d %s" % (301, "Moved Permanently"),
[
("Location",
urlparse.urlunsplit(("https", u.netloc, u.path,
u.query, u.fragment))),
("Connection", "close"),
],
)
yield ""
return
set_forwarded_for(environ, headers)
http_conn = create_connection_hosts(addrs, port, self.timeout)
conn = HTTPConnection(host, port=port)
conn.sock = http_conn
path = urlparse.urlunsplit(("", "", u.path, u.query, ""))
conn.request(method, path, body, headers)
resp = conn.getresponse()
start_response("%d %s" % (resp.status, resp.reason),
resp.getheaders())
while True:
data = resp.read(CHUNKSIZE)
if not data:
break
yield data
conn.close()
except Exception, e:
log.error(
"[Exception][FireflyHTTPApplication.forward_hosts_http]: %s" %
str(e))
start_response("500 Internal Server Error",
[("Content-Type", "text/plain; charset=utf-8")])
yield "Internal Server Error"
return
def forward_hosts_http(self, addrs, host, port, environ, start_response):
try:
method, url, body, headers = copy_request(environ)
except:
start_response("400 Bad Request", [("Content-Type", "text/plain; charset=utf-8")])
yield "Bad Request"
return
try:
u = urlparse.urlsplit(url)
if self.matcher.need_redirect(method, host):
start_response(
"%d %s" % (301, "Moved Permanently"),
[
("Location", urlparse.urlunsplit(("https", u.netloc, u.path, u.query, u.fragment))),
("Connection", "close"),
],
)
yield ""
return
set_forwarded_for(environ, headers)
http_conn = create_connection_hosts(addrs, port, self.timeout)
conn = HTTPConnection(host, port=port)
conn.sock = http_conn
path = urlparse.urlunsplit(("", "", u.path, u.query, ""))
conn.request(method, path, body, headers)
resp = conn.getresponse()
start_response("%d %s" % (resp.status, resp.reason), resp.getheaders())
while True:
data = resp.read(CHUNKSIZE)
if not data:
break
yield data
conn.close()
except Exception, e:
log.error("[Exception][FireflyHTTPApplication.forward_hosts_http]: %s" % str(e))
start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")])
yield "Internal Server Error"
return
log.error("[Exception][http]: %s" % str(e))
start_response("400 Bad Request",
[("Content-Type", "text/plain; charset=utf-8")])
yield "Bad Request"
return
socksconn = self.connect_socks(host, port)
if not socksconn:
start_response("500 Internal Server Error",
[("Content-Type", "text/plain; charset=utf-8")])
yield "Internal Server Error"
return
try:
conn = HTTPConnection(host, port=port)
conn.sock = socksconn
set_forwarded_for(environ, headers)
u = urlparse.urlsplit(url)
path = urlparse.urlunsplit(("", "", u.path, u.query, ""))
conn.request(method, path, body, headers)
resp = conn.getresponse()
start_response("%d %s" % (resp.status, resp.reason),
resp.getheaders())
while True:
data = resp.read(CHUNKSIZE)
if not data:
break
yield data
conn.close()
except Exception, e:
log.error("[Exception][http]: %s" % str(e))
host, port = get_destination(environ)
except Exception, e:
log.error("[Exception][http]: %s" % str(e))
start_response("400 Bad Request", [("Content-Type", "text/plain; charset=utf-8")])
yield "Bad Request"
return
socksconn = self.connect_socks(host, port)
if not socksconn:
start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")])
yield "Internal Server Error"
return
try:
conn = HTTPConnection(host, port=port)
conn.sock = socksconn
set_forwarded_for(environ, headers)
u = urlparse.urlsplit(url)
path = urlparse.urlunsplit(("", "", u.path, u.query, ""))
conn.request(method, path, body, headers)
resp = conn.getresponse()
start_response("%d %s" % (resp.status, resp.reason), resp.getheaders())
while True:
data = resp.read(CHUNKSIZE)
if not data:
break
yield data
conn.close()
except Exception, e:
log.error("[Exception][http]: %s" % str(e))
start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")])
def resolve_http_redirect(self, url, header, dc_anchor, duck_s, r_type,
depth=0):
"""send request to url and return result"""
if (depth > 10):
raise Exception("Redirected " + depth + " times, giving up.")
header["Host"] = DUCKDUCKGO_HOST
parsed_url = urlparse(url, allow_fragments=True)
if (parsed_url.scheme == "http"):
conn = HTTPConnection(parsed_url.netloc)
elif (parsed_url.scheme == "https"):
conn = HTTPSConnection(parsed_url.netloc)
else:
conn = HTTPConnection(parsed_url.netloc)
path = parsed_url.path
if parsed_url.query:
path += "?" + parsed_url.query
if (r_type == "POST"):
if (dc_anchor == None and duck_s == None):
params = urlencode({
"q" : self.query,
"kl" : "us-en",
"b" : ""
}
)
else:
params = urlencode([
("q", self.query),
("s", duck_s),
("o", "json"),
("dc", dc_anchor),
("api", "d.js"),
("kl", "us-en")
]
)
header["Content-type"] = HEADER_CONTENT_TYPE
header["Accept"] = HEADER_ACCEPT
try:
conn.request("POST", path, params, header)
except (UnicodeEncodeError, OSError):
conn.close()
logging.error("Timeout, OSError")
raise Exception("oserror or unicodeencodeerror")
else:
try:
conn.request("GET", path, None, header)
except (UnicodeEncodeError, OSError):
conn.close()
logging.error("Timeout, OSError")
raise Exception("oserror or unicodeencodeerror")
try:
res = conn.getresponse()
except HTTPException:
logging.error("HTTPException")
raise Exception("HTTPException")
if (conn.sock):
conn.sock.close()
conn.sock = None
if (res != None):
headers = dict(res.getheaders())
else:
raise Exception("no header")
loc_key = None
if ("Location" in headers):
loc_key = "Location"
elif ("location" in headers):
loc_key = "location"
if (loc_key != None and headers[loc_key] != url):
if ("IndexRedirect" in headers[loc_key]):
print("IndexRedirect, exiting")
exit(1)
return self.resolve_http_redirect(
headers[loc_key],
header,
dc_anchor,
duck_s,
r_type,
depth+1
)
else:
return res, headers