def proxy(url): log.info("Processing %s?%s", url, request.query_string) query_string = request.query_string.decode() if request.method == "GET": try: parsed_feed = feedcache.Cache(feed_db()).fetch( "?".join(filter(None, [url, query_string])), force_update=False ) # defaults: force_update = False, offline = False # return content finally: feed_db().sync() status = parsed_feed.get("status", 404) if status >= 400: # deal with errors response = ("External error", status, {}) elif status >= 300: # deal with redirects if status == 301: log.warn("Permanent redirect from %s to %s", url, parsed_feed.href) return redirect("/feed/{reurl}".format(reurl=parsed_feed.href)) else: etag = request.headers.get("IF_NONE_MATCH") modified = request.headers.get("IF_MODIFIED_SINCE") if False: pass if (etag and etag == parsed_feed.get("etag")) or ( modified and modified == parsed_feed.get("modified")): response = ("", 304, {}) else: if not parsed_feed["bozo"]: parsed_feed = copy.deepcopy( parsed_feed ) # if it's bozo copy fails and copy is not cached, # so we skip # deepcopy needed to avoid side effects on cache response = (_process(parsed_feed), 200, {}) if "headers" in parsed_feed: # some header rinsing for k, v in parsed_feed.headers.items(): # TODO: seems to work with all the hop by hop headers unset # or to default values, need to look into this k = k.lower() if (not is_hop_by_hop_header(k) and k != "content-length" and k != "content-encoding" and k != ""): # let django deal with these headers response[2][k] = v return response else: return ("POST not allowed for feeds", 405, {})
def application(environ, start_response): headers = list(EnvironHeaders(environ).items()) headers[:] = [(k, v) for k, v in headers if not is_hop_by_hop_header(k) and k.lower() not in ( 'content-length', 'host')] headers.append(('Connection', 'close')) if opts['host'] == '<auto>': headers.append(('Host', target.ascii_host)) elif opts['host'] is None: headers.append(('Host', environ['HTTP_HOST'])) else: headers.append(('Host', opts['host'])) headers.extend(opts['headers'].items()) remote_path = path if opts['remove_prefix']: remote_path = '%s/%s' % (target.path.rstrip('/'), remote_path[len(prefix):].lstrip('/')) content_length = environ.get('CONTENT_LENGTH') chunked = False if content_length not in ('', None): headers.append(('Content-Length', content_length)) elif content_length is not None: headers.append(('Transfer-Encoding', 'chunked')) chunked = True try: if target.scheme == 'http': con = httplib.HTTPConnection(target.ascii_host, target.port or 80, timeout=self.timeout) elif target.scheme == 'https': con = httplib.HTTPSConnection(target.ascii_host, target.port or 443, timeout=self.timeout, context=opts['ssl_context']) con.connect() con.putrequest(environ['REQUEST_METHOD'], url_quote(remote_path), skip_host=True) for k, v in headers: if k.lower() == 'connection': v = 'close' con.putheader(k, v) con.endheaders() stream = get_input_stream(environ) while 1: data = stream.read(self.chunk_size) if not data: break if chunked: con.send(b'%x\r\n%s\r\n' % (len(data), data)) else: con.send(data) resp = con.getresponse() except socket.error: from werkzeug.exceptions import BadGateway return BadGateway()(environ, start_response) start_response( '%d %s' % (resp.status, resp.reason), [(k.title(), v) for k, v in resp.getheaders() if not is_hop_by_hop_header(k)]) def read(): while 1: try: data = resp.read(self.chunk_size) except socket.error: break if not data: break yield data return read()
def application(environ, start_response): headers = list(EnvironHeaders(environ).items()) headers[:] = [(k, v) for k, v in headers if not is_hop_by_hop_header(k) and k.lower() not in ('content-length', 'host')] headers.append(('Connection', 'close')) if opts['host'] == '<auto>': headers.append(('Host', target.ascii_host)) elif opts['host'] is None: headers.append(('Host', environ['HTTP_HOST'])) else: headers.append(('Host', opts['host'])) headers.extend(opts['headers'].items()) remote_path = path if opts['remove_prefix']: remote_path = '%s/%s' % ( target.path.rstrip('/'), remote_path[len(prefix):].lstrip('/') ) content_length = environ.get('CONTENT_LENGTH') chunked = False if content_length not in ('', None): headers.append(('Content-Length', content_length)) elif content_length is not None: headers.append(('Transfer-Encoding', 'chunked')) chunked = True try: if target.scheme == 'http': con = httplib.HTTPConnection( target.ascii_host, target.port or 80, timeout=self.timeout) elif target.scheme == 'https': con = httplib.HTTPSConnection( target.ascii_host, target.port or 443, timeout=self.timeout, context=opts['ssl_context']) con.connect() con.putrequest(environ['REQUEST_METHOD'], url_quote(remote_path), skip_host=True) for k, v in headers: if k.lower() == 'connection': v = 'close' con.putheader(k, v) con.endheaders() stream = get_input_stream(environ) while 1: data = stream.read(self.chunk_size) if not data: break if chunked: con.send(b'%x\r\n%s\r\n' % (len(data), data)) else: con.send(data) resp = con.getresponse() except socket.error: from werkzeug.exceptions import BadGateway return BadGateway()(environ, start_response) start_response('%d %s' % (resp.status, resp.reason), [(k.title(), v) for k, v in resp.getheaders() if not is_hop_by_hop_header(k)]) def read(): while 1: try: data = resp.read(self.chunk_size) except socket.error: break if not data: break yield data return read()
def application(environ, start_response): headers = list(EnvironHeaders(environ).items()) headers[:] = [(k, v) for k, v in headers if not is_hop_by_hop_header(k) and k.lower() not in ( "content-length", "host")] headers.append(("Connection", "close")) if opts["host"] == "<auto>": headers.append(("Host", target.ascii_host)) elif opts["host"] is None: headers.append(("Host", environ["HTTP_HOST"])) else: headers.append(("Host", opts["host"])) headers.extend(opts["headers"].items()) remote_path = path if opts["remove_prefix"]: remote_path = "%s/%s" % ( target.path.rstrip("/"), remote_path[len(prefix):].lstrip("/"), ) content_length = environ.get("CONTENT_LENGTH") chunked = False if content_length not in ("", None): headers.append(("Content-Length", content_length)) elif content_length is not None: headers.append(("Transfer-Encoding", "chunked")) chunked = True try: if target.scheme == "http": con = httplib.HTTPConnection(target.ascii_host, target.port or 80, timeout=self.timeout) elif target.scheme == "https": con = httplib.HTTPSConnection( target.ascii_host, target.port or 443, timeout=self.timeout, context=opts["ssl_context"], ) con.connect() con.putrequest(environ["REQUEST_METHOD"], url_quote(remote_path), skip_host=True) for k, v in headers: if k.lower() == "connection": v = "close" con.putheader(k, v) con.endheaders() stream = get_input_stream(environ) while 1: data = stream.read(self.chunk_size) if not data: break if chunked: con.send(b"%x\r\n%s\r\n" % (len(data), data)) else: con.send(data) resp = con.getresponse() except socket.error: from werkzeug.exceptions import BadGateway return BadGateway()(environ, start_response) start_response( "%d %s" % (resp.status, resp.reason), [(k.title(), v) for k, v in resp.getheaders() if not is_hop_by_hop_header(k)], ) def read(): while 1: try: data = resp.read(self.chunk_size) except socket.error: break if not data: break yield data return read()