def create_http_fwd_connection(self, destport, dest_addr='127.0.0.1', peer=None, localport=None, trans=None, httpaddr='127.0.0.1', **connection_kwargs): """ Create an http connection with port fowarding over this ssh session. :param destport: port to forward (ie 80 for http) :param dest_addr: addr used for remote request :param peer: Remote host :param localport: Local port used to forward :param trans: ssh transport obj :param httpaddr: addr used for http connection :returns HTTP connection obj """ trans = trans or self.connection._transport assert isinstance(trans, paramiko.Transport) if peer is None: peer = trans.getpeername()[0] if localport is None: localport = self._get_local_unused_port(start=9000) self.debug('Making forwarded request from "localhost:{0}" to "{1}:{2}"' .format(localport, peer, destport)) self.debug('open_channel(kind="{0}", dest_addr=("{1}", {2}), src_addr=("{3}", {4})' .format('direct-tcpip', dest_addr, destport, peer, localport)) chan = trans.open_channel(kind='direct-tcpip', dest_addr=(dest_addr, destport), src_addr=(peer, localport)) connection_kwargs = connection_kwargs or {} connection_kwargs['host'] = httpaddr connection_kwargs['port'] = destport self.debug('Creating HTTP connection with kwargs:\n{0}\n'.format(connection_kwargs)) http = HTTPConnection(**connection_kwargs) http.sock = chan return http
def _resolve_http_redirect(self, url, header, depth=0): """this method sends http(s) requests and also resolves redirects""" if depth > 10: raise Exception("Redirected " + str(depth) + " times, giving up.") parsed_url = urlparse(url, allow_fragments=True) if (parsed_url.scheme == "http"): conn = HTTPConnection(parsed_url.netloc) elif (parsed_url.scheme == "https"): conn = HTTPSConnection(parsed_url.netloc) path = parsed_url.path if (parsed_url.query): path += "?" + parsed_url.query try: conn.request("GET", path, None, header) except (OSError, UnicodeEncodeError): conn.close() raise Exception("OSError or UnicodeEncodeError") try: res = conn.getresponse() except (HTTPException): logging.error("HTTPException") raise Exception("HTTPException") if (conn.sock): conn.sock.close() conn.sock = None if (res != None): headers = dict(res.getheaders()) else: raise Exception("Response empty") loc_key = None if ("Location" in headers): loc_key = "Location" elif ("location" in headers): loc_key = "location" if (loc_key != None and headers[loc_key] != url): if ("IndexRedirect" in headers[loc_key]): logging.error("IndexRedirect, exiting") exit(1) return self._resolve_http_redirect( headers[loc_key], header, depth+1 ) else: return res, headers
def _resolve_http_redirect(self, url, header, depth=0): """this method sends http(s) requests and also resolves redirects""" if depth > 10: raise Exception("Redirected " + str(depth) + " times, giving up.") parsed_url = urlparse(url, allow_fragments=True) if (parsed_url.scheme == "http"): conn = HTTPConnection(parsed_url.netloc) elif (parsed_url.scheme == "https"): conn = HTTPSConnection(parsed_url.netloc) path = parsed_url.path if (parsed_url.query): path += "?" + parsed_url.query try: conn.request("GET", path, None, header) except (OSError, UnicodeEncodeError): conn.close() raise Exception("OSError or UnicodeEncodeError") try: res = conn.getresponse() except (HTTPException): logging.error("HTTPException") raise Exception("HTTPException") if (conn.sock): conn.sock.close() conn.sock = None if (res != None): headers = dict(res.getheaders()) else: raise Exception("Response empty") loc_key = None if ("Location" in headers): loc_key = "Location" elif ("location" in headers): loc_key = "location" if (loc_key != None and headers[loc_key] != url): if ("IndexRedirect" in headers[loc_key]): logging.error("IndexRedirect, exiting") exit(1) return self._resolve_http_redirect(headers[loc_key], header, depth + 1) else: return res, headers
def forward_hosts_http(self, addrs, host, port, environ, start_response): try: method, url, body, headers = copy_request(environ) except: start_response("400 Bad Request", [("Content-Type", "text/plain; charset=utf-8")]) yield "Bad Request" return try: u = urlparse.urlsplit(url) if self.matcher.need_redirect(method, host): start_response( "%d %s" % (301, "Moved Permanently"), [ ("Location", urlparse.urlunsplit(("https", u.netloc, u.path, u.query, u.fragment))), ("Connection", "close"), ], ) yield "" return set_forwarded_for(environ, headers) http_conn = create_connection_hosts(addrs, port, self.timeout) conn = HTTPConnection(host, port=port) conn.sock = http_conn path = urlparse.urlunsplit(("", "", u.path, u.query, "")) conn.request(method, path, body, headers) resp = conn.getresponse() start_response("%d %s" % (resp.status, resp.reason), resp.getheaders()) while True: data = resp.read(CHUNKSIZE) if not data: break yield data conn.close() except Exception, e: log.error( "[Exception][FireflyHTTPApplication.forward_hosts_http]: %s" % str(e)) start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")]) yield "Internal Server Error" return
def forward_hosts_http(self, addrs, host, port, environ, start_response): try: method, url, body, headers = copy_request(environ) except: start_response("400 Bad Request", [("Content-Type", "text/plain; charset=utf-8")]) yield "Bad Request" return try: u = urlparse.urlsplit(url) if self.matcher.need_redirect(method, host): start_response( "%d %s" % (301, "Moved Permanently"), [ ("Location", urlparse.urlunsplit(("https", u.netloc, u.path, u.query, u.fragment))), ("Connection", "close"), ], ) yield "" return set_forwarded_for(environ, headers) http_conn = create_connection_hosts(addrs, port, self.timeout) conn = HTTPConnection(host, port=port) conn.sock = http_conn path = urlparse.urlunsplit(("", "", u.path, u.query, "")) conn.request(method, path, body, headers) resp = conn.getresponse() start_response("%d %s" % (resp.status, resp.reason), resp.getheaders()) while True: data = resp.read(CHUNKSIZE) if not data: break yield data conn.close() except Exception, e: log.error("[Exception][FireflyHTTPApplication.forward_hosts_http]: %s" % str(e)) start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")]) yield "Internal Server Error" return
log.error("[Exception][http]: %s" % str(e)) start_response("400 Bad Request", [("Content-Type", "text/plain; charset=utf-8")]) yield "Bad Request" return socksconn = self.connect_socks(host, port) if not socksconn: start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")]) yield "Internal Server Error" return try: conn = HTTPConnection(host, port=port) conn.sock = socksconn set_forwarded_for(environ, headers) u = urlparse.urlsplit(url) path = urlparse.urlunsplit(("", "", u.path, u.query, "")) conn.request(method, path, body, headers) resp = conn.getresponse() start_response("%d %s" % (resp.status, resp.reason), resp.getheaders()) while True: data = resp.read(CHUNKSIZE) if not data: break yield data conn.close() except Exception, e: log.error("[Exception][http]: %s" % str(e))
host, port = get_destination(environ) except Exception, e: log.error("[Exception][http]: %s" % str(e)) start_response("400 Bad Request", [("Content-Type", "text/plain; charset=utf-8")]) yield "Bad Request" return socksconn = self.connect_socks(host, port) if not socksconn: start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")]) yield "Internal Server Error" return try: conn = HTTPConnection(host, port=port) conn.sock = socksconn set_forwarded_for(environ, headers) u = urlparse.urlsplit(url) path = urlparse.urlunsplit(("", "", u.path, u.query, "")) conn.request(method, path, body, headers) resp = conn.getresponse() start_response("%d %s" % (resp.status, resp.reason), resp.getheaders()) while True: data = resp.read(CHUNKSIZE) if not data: break yield data conn.close() except Exception, e: log.error("[Exception][http]: %s" % str(e)) start_response("500 Internal Server Error", [("Content-Type", "text/plain; charset=utf-8")])
def resolve_http_redirect(self, url, header, dc_anchor, duck_s, r_type, depth=0): """send request to url and return result""" if (depth > 10): raise Exception("Redirected " + depth + " times, giving up.") header["Host"] = DUCKDUCKGO_HOST parsed_url = urlparse(url, allow_fragments=True) if (parsed_url.scheme == "http"): conn = HTTPConnection(parsed_url.netloc) elif (parsed_url.scheme == "https"): conn = HTTPSConnection(parsed_url.netloc) else: conn = HTTPConnection(parsed_url.netloc) path = parsed_url.path if parsed_url.query: path += "?" + parsed_url.query if (r_type == "POST"): if (dc_anchor == None and duck_s == None): params = urlencode({ "q" : self.query, "kl" : "us-en", "b" : "" } ) else: params = urlencode([ ("q", self.query), ("s", duck_s), ("o", "json"), ("dc", dc_anchor), ("api", "d.js"), ("kl", "us-en") ] ) header["Content-type"] = HEADER_CONTENT_TYPE header["Accept"] = HEADER_ACCEPT try: conn.request("POST", path, params, header) except (UnicodeEncodeError, OSError): conn.close() logging.error("Timeout, OSError") raise Exception("oserror or unicodeencodeerror") else: try: conn.request("GET", path, None, header) except (UnicodeEncodeError, OSError): conn.close() logging.error("Timeout, OSError") raise Exception("oserror or unicodeencodeerror") try: res = conn.getresponse() except HTTPException: logging.error("HTTPException") raise Exception("HTTPException") if (conn.sock): conn.sock.close() conn.sock = None if (res != None): headers = dict(res.getheaders()) else: raise Exception("no header") loc_key = None if ("Location" in headers): loc_key = "Location" elif ("location" in headers): loc_key = "location" if (loc_key != None and headers[loc_key] != url): if ("IndexRedirect" in headers[loc_key]): print("IndexRedirect, exiting") exit(1) return self.resolve_http_redirect( headers[loc_key], header, dc_anchor, duck_s, r_type, depth+1 ) else: return res, headers