def proxy(url):
log.info("Processing %s?%s", url, request.query_string)
query_string = request.query_string.decode()
if request.method == "GET":
try:
parsed_feed = feedcache.Cache(feed_db()).fetch(
"?".join(filter(None, [url, query_string])),
force_update=False
) # defaults: force_update = False, offline = False
# return content
finally:
feed_db().sync()
status = parsed_feed.get("status", 404)
if status >= 400: # deal with errors
response = ("External error", status, {})
elif status >= 300: # deal with redirects
if status == 301:
log.warn("Permanent redirect from %s to %s", url,
parsed_feed.href)
return redirect("/feed/{reurl}".format(reurl=parsed_feed.href))
else:
etag = request.headers.get("IF_NONE_MATCH")
modified = request.headers.get("IF_MODIFIED_SINCE")
if False:
pass
if (etag and etag == parsed_feed.get("etag")) or (
modified and modified == parsed_feed.get("modified")):
response = ("", 304, {})
else:
if not parsed_feed["bozo"]:
parsed_feed = copy.deepcopy(
parsed_feed
) # if it's bozo copy fails and copy is not cached,
# so we skip
# deepcopy needed to avoid side effects on cache
response = (_process(parsed_feed), 200, {})
if "headers" in parsed_feed: # some header rinsing
for k, v in parsed_feed.headers.items():
# TODO: seems to work with all the hop by hop headers unset
# or to default values, need to look into this
k = k.lower()
if (not is_hop_by_hop_header(k) and k != "content-length"
and k != "content-encoding" and k != ""):
# let django deal with these headers
response[2][k] = v
return response
else:
return ("POST not allowed for feeds", 405, {})
def application(environ, start_response):
headers = list(EnvironHeaders(environ).items())
headers[:] = [(k, v) for k, v in headers
if not is_hop_by_hop_header(k) and k.lower() not in (
'content-length', 'host')]
headers.append(('Connection', 'close'))
if opts['host'] == '<auto>':
headers.append(('Host', target.ascii_host))
elif opts['host'] is None:
headers.append(('Host', environ['HTTP_HOST']))
else:
headers.append(('Host', opts['host']))
headers.extend(opts['headers'].items())
remote_path = path
if opts['remove_prefix']:
remote_path = '%s/%s' % (target.path.rstrip('/'),
remote_path[len(prefix):].lstrip('/'))
content_length = environ.get('CONTENT_LENGTH')
chunked = False
if content_length not in ('', None):
headers.append(('Content-Length', content_length))
elif content_length is not None:
headers.append(('Transfer-Encoding', 'chunked'))
chunked = True
try:
if target.scheme == 'http':
con = httplib.HTTPConnection(target.ascii_host,
target.port or 80,
timeout=self.timeout)
elif target.scheme == 'https':
con = httplib.HTTPSConnection(target.ascii_host,
target.port or 443,
timeout=self.timeout,
context=opts['ssl_context'])
con.connect()
con.putrequest(environ['REQUEST_METHOD'],
url_quote(remote_path),
skip_host=True)
for k, v in headers:
if k.lower() == 'connection':
v = 'close'
con.putheader(k, v)
con.endheaders()
stream = get_input_stream(environ)
while 1:
data = stream.read(self.chunk_size)
if not data:
break
if chunked:
con.send(b'%x\r\n%s\r\n' % (len(data), data))
else:
con.send(data)
resp = con.getresponse()
except socket.error:
from werkzeug.exceptions import BadGateway
return BadGateway()(environ, start_response)
start_response(
'%d %s' % (resp.status, resp.reason),
[(k.title(), v)
for k, v in resp.getheaders() if not is_hop_by_hop_header(k)])
def read():
while 1:
try:
data = resp.read(self.chunk_size)
except socket.error:
break
if not data:
break
yield data
return read()
def application(environ, start_response):
headers = list(EnvironHeaders(environ).items())
headers[:] = [(k, v) for k, v in headers
if not is_hop_by_hop_header(k) and
k.lower() not in ('content-length', 'host')]
headers.append(('Connection', 'close'))
if opts['host'] == '<auto>':
headers.append(('Host', target.ascii_host))
elif opts['host'] is None:
headers.append(('Host', environ['HTTP_HOST']))
else:
headers.append(('Host', opts['host']))
headers.extend(opts['headers'].items())
remote_path = path
if opts['remove_prefix']:
remote_path = '%s/%s' % (
target.path.rstrip('/'),
remote_path[len(prefix):].lstrip('/')
)
content_length = environ.get('CONTENT_LENGTH')
chunked = False
if content_length not in ('', None):
headers.append(('Content-Length', content_length))
elif content_length is not None:
headers.append(('Transfer-Encoding', 'chunked'))
chunked = True
try:
if target.scheme == 'http':
con = httplib.HTTPConnection(
target.ascii_host, target.port or 80,
timeout=self.timeout)
elif target.scheme == 'https':
con = httplib.HTTPSConnection(
target.ascii_host, target.port or 443,
timeout=self.timeout,
context=opts['ssl_context'])
con.connect()
con.putrequest(environ['REQUEST_METHOD'], url_quote(remote_path),
skip_host=True)
for k, v in headers:
if k.lower() == 'connection':
v = 'close'
con.putheader(k, v)
con.endheaders()
stream = get_input_stream(environ)
while 1:
data = stream.read(self.chunk_size)
if not data:
break
if chunked:
con.send(b'%x\r\n%s\r\n' % (len(data), data))
else:
con.send(data)
resp = con.getresponse()
except socket.error:
from werkzeug.exceptions import BadGateway
return BadGateway()(environ, start_response)
start_response('%d %s' % (resp.status, resp.reason),
[(k.title(), v) for k, v in resp.getheaders()
if not is_hop_by_hop_header(k)])
def read():
while 1:
try:
data = resp.read(self.chunk_size)
except socket.error:
break
if not data:
break
yield data
return read()
def application(environ, start_response):
headers = list(EnvironHeaders(environ).items())
headers[:] = [(k, v) for k, v in headers
if not is_hop_by_hop_header(k) and k.lower() not in (
"content-length", "host")]
headers.append(("Connection", "close"))
if opts["host"] == "<auto>":
headers.append(("Host", target.ascii_host))
elif opts["host"] is None:
headers.append(("Host", environ["HTTP_HOST"]))
else:
headers.append(("Host", opts["host"]))
headers.extend(opts["headers"].items())
remote_path = path
if opts["remove_prefix"]:
remote_path = "%s/%s" % (
target.path.rstrip("/"),
remote_path[len(prefix):].lstrip("/"),
)
content_length = environ.get("CONTENT_LENGTH")
chunked = False
if content_length not in ("", None):
headers.append(("Content-Length", content_length))
elif content_length is not None:
headers.append(("Transfer-Encoding", "chunked"))
chunked = True
try:
if target.scheme == "http":
con = httplib.HTTPConnection(target.ascii_host,
target.port or 80,
timeout=self.timeout)
elif target.scheme == "https":
con = httplib.HTTPSConnection(
target.ascii_host,
target.port or 443,
timeout=self.timeout,
context=opts["ssl_context"],
)
con.connect()
con.putrequest(environ["REQUEST_METHOD"],
url_quote(remote_path),
skip_host=True)
for k, v in headers:
if k.lower() == "connection":
v = "close"
con.putheader(k, v)
con.endheaders()
stream = get_input_stream(environ)
while 1:
data = stream.read(self.chunk_size)
if not data:
break
if chunked:
con.send(b"%x\r\n%s\r\n" % (len(data), data))
else:
con.send(data)
resp = con.getresponse()
except socket.error:
from werkzeug.exceptions import BadGateway
return BadGateway()(environ, start_response)
start_response(
"%d %s" % (resp.status, resp.reason),
[(k.title(), v)
for k, v in resp.getheaders() if not is_hop_by_hop_header(k)],
)
def read():
while 1:
try:
data = resp.read(self.chunk_size)
except socket.error:
break
if not data:
break
yield data
return read()