def _requests_fn(fn, url, parse_json=False, **kwargs):
"""Wraps requests.* and adds raise_for_status() and User-Agent."""
kwargs.setdefault('headers', {}).update(HEADERS)
resp = fn(url, **kwargs)
logging.info('Got %s headers:%s', resp.status_code, resp.headers)
type = content_type(resp)
if (type and type != 'text/html' and
(type.startswith('text/') or type.endswith('+json') or type.endswith('/json'))):
logging.info(resp.text)
if resp.status_code // 100 in (4, 5):
msg = 'Received %s from %s:\n%s' % (resp.status_code, url, resp.text)
logging.info(msg)
raise exc.HTTPBadGateway(msg)
if parse_json:
try:
return resp.json()
except ValueError:
msg = "Couldn't parse response as JSON"
logging.info(msg, exc_info=True)
raise exc.HTTPBadGateway(msg)
return resp
def __call__(self, req):
url = req.params.get('url')
if not url:
raise exc.HTTPBadRequest('No url parameter')
try:
r = urllib.urlopen(url)
except IOError, e:
raise exc.HTTPBadGateway('Error getting url: %s' % e)
def code2exception(code, detail):
"""Transforms a code + detail into a WebOb exception"""
if code == 400:
return exc.HTTPBadRequest(detail)
if code == 401:
return exc.HTTPUnauthorized(detail)
if code == 402:
return exc.HTTPPaymentRequired(detail)
if code == 403:
return exc.HTTPForbidden(detail)
if code == 404:
return exc.HTTPNotFound(detail)
if code == 405:
return exc.HTTPMethodNotAllowed(detail)
if code == 406:
return exc.HTTPNotAcceptable(detail)
if code == 407:
return exc.HTTPProxyAuthenticationRequired(detail)
if code == 408:
return exc.HTTPRequestTimeout(detail)
if code == 409:
return exc.HTTPConflict(detail)
if code == 410:
return exc.HTTPGone(detail)
if code == 411:
return exc.HTTPLengthRequired(detail)
if code == 412:
return exc.HTTPPreconditionFailed(detail)
if code == 413:
return exc.HTTPRequestEntityTooLarge(detail)
if code == 414:
return exc.HTTPRequestURITooLong(detail)
if code == 415:
return exc.HTTPUnsupportedMediaType(detail)
if code == 416:
return exc.HTTPRequestRangeNotSatisfiable(detail)
if code == 417:
return exc.HTTPExpectationFailed(detail)
if code == 500:
return exc.HTTPInternalServerError(detail)
if code == 501:
return exc.HTTPNotImplemented(detail)
if code == 502:
return exc.HTTPBadGateway(detail)
if code == 503:
return exc.HTTPServiceUnavailable(detail)
if code == 504:
return exc.HTTPGatewayTimeout(detail)
if code == 505:
return exc.HTTPVersionNotSupported(detail)
raise NotImplementedError(code)
class GetManifest(object):
def __init__(self):
pass
@wsgify
def __call__(self, req):
url = req.params.get('url')
if not url:
raise exc.HTTPBadRequest('No url parameter')
try:
r = urllib.urlopen(url)
except IOError, e:
raise exc.HTTPBadGateway('Error getting url: %s' % e)
if r.getcode() != 200:
raise exc.HTTPBadGateway('URL returned error code: %s' \
% r.getcode())
content_type = (r.headers.getheader('content-type') or 'application/octet-stream').lower()
if content_type != EXPECT_CONTENT_TYPE:
raise exc.HTTPBadGateway(
'You may only fetch URLs that have a Content-Type of %s'
% EXPECT_CONTENT_TYPE)
return Response(
body=r.read(),
content_type=content_type)
def __call__(self, environ, start_response):
scheme = environ['wsgi.url_scheme']
if scheme == 'http':
ConnClass = self.HTTPConnection
elif scheme == 'https':
ConnClass = self.HTTPSConnection
else:
raise ValueError("Unknown scheme: %r" % scheme)
if 'SERVER_NAME' not in environ:
host = environ.get('HTTP_HOST')
if not host:
raise ValueError(
"environ contains neither SERVER_NAME nor HTTP_HOST")
if ':' in host:
host, port = host.split(':', 1)
else:
if scheme == 'http':
port = '80'
else:
port = '443'
environ['SERVER_NAME'] = host
environ['SERVER_PORT'] = port
kw = {}
if ('webob.client.timeout' in environ
and self._timeout_supported(ConnClass)):
kw['timeout'] = environ['webob.client.timeout']
conn = ConnClass('%(SERVER_NAME)s:%(SERVER_PORT)s' % environ, **kw)
headers = {}
for key, value in environ.items():
if key.startswith('HTTP_'):
key = key[5:].replace('_', '-').title()
headers[key] = value
path = (url_quote(environ.get('SCRIPT_NAME', '')) +
url_quote(environ.get('PATH_INFO', '')))
if environ.get('QUERY_STRING'):
path += '?' + environ['QUERY_STRING']
try:
content_length = int(environ.get('CONTENT_LENGTH', '0'))
except ValueError:
content_length = 0
## FIXME: there is no streaming of the body, and that might be useful
## in some cases
if content_length:
body = environ['wsgi.input'].read(content_length)
else:
body = ''
headers['Content-Length'] = content_length
if environ.get('CONTENT_TYPE'):
headers['Content-Type'] = environ['CONTENT_TYPE']
if not path.startswith("/"):
path = "/" + path
try:
conn.request(environ['REQUEST_METHOD'], path, body, headers)
res = conn.getresponse()
except socket.timeout:
resp = exc.HTTPGatewayTimeout()
return resp(environ, start_response)
except (socket.error, socket.gaierror) as e:
if ((isinstance(e, socket.error) and e.args[0] == -2)
or (isinstance(e, socket.gaierror) and e.args[0] == 8)):
# Name or service not known
resp = exc.HTTPBadGateway(
"Name or service not known (bad domain name: %s)" %
environ['SERVER_NAME'])
return resp(environ, start_response)
elif e.args[0] in _e_refused: # pragma: no cover
# Connection refused
resp = exc.HTTPBadGateway("Connection refused")
return resp(environ, start_response)
raise
headers_out = self.parse_headers(res.msg)
status = '%s %s' % (res.status, res.reason)
start_response(status, headers_out)
length = res.getheader('content-length')
# FIXME: This shouldn't really read in all the content at once
if length is not None:
body = res.read(int(length))
else:
body = res.read()
conn.close()
return [body]
def __call__(self, environ, start_response):
method = environ['REQUEST_METHOD']
if (self.allowed_methods is not None
and method not in self.allowed_methods):
return exc.HTTPMethodNotAllowed()(environ, start_response)
if 'RAW_URI' in environ:
path_info = environ['RAW_URI']
elif 'REQUEST_URI' in environ:
path_info = environ['REQUEST_URI']
else:
if self.strip_script_name:
path_info = ''
else:
path_info = environ['SCRIPT_NAME']
path_info += environ['PATH_INFO']
if PY3:
path_info = url_quote(path_info.encode('latin-1'),
LOW_CHAR_SAFE)
query_string = environ['QUERY_STRING']
if query_string:
path_info += '?' + query_string
for key, dest in self.header_map.items():
value = environ.get(key)
if value:
environ['HTTP_%s' % dest] = value
host_uri = self.extract_uri(environ)
uri = host_uri + path_info
new_headers = {}
for k, v in environ.items():
if k.startswith('HTTP_'):
k = k[5:].replace('_', '-').title()
new_headers[k] = v
content_type = environ.get("CONTENT_TYPE")
if content_type and content_type is not None:
new_headers['Content-Type'] = content_type
content_length = environ.get('CONTENT_LENGTH')
transfer_encoding = environ.get('Transfer-Encoding', '').lower()
if not content_length and transfer_encoding != 'chunked':
new_headers['Transfer-Encoding'] = 'chunked'
elif content_length:
new_headers['Content-Length'] = content_length
if new_headers.get('Content-Length', '0') == '-1':
resp = exc.HTTPInternalServerError(detail=WEBOB_ERROR)
return resp(environ, start_response)
try:
response = self.process_request(uri, method, new_headers, environ)
except socket.timeout:
return exc.HTTPGatewayTimeout()(environ, start_response)
except (socket.error, socket.gaierror):
return exc.HTTPBadGateway()(environ, start_response)
except Exception as e:
self.logger.exception(e)
return exc.HTTPInternalServerError()(environ, start_response)
status, location, headerslist, app_iter = response
if location:
if self.strip_script_name:
prefix_path = environ['SCRIPT_NAME']
else:
prefix_path = None
new_location = rewrite_location(host_uri,
location,
prefix_path=prefix_path)
headers = []
for k, v in headerslist:
if k.lower() == 'location':
v = new_location
headers.append((k, v))
else:
headers = headerslist
start_response(status, headers)
if method == "HEAD":
return [six.b('')]
return app_iter
def __call__(self, environ, start_response):
scheme = environ["wsgi.url_scheme"]
if scheme == "http":
ConnClass = self.HTTPConnection
elif scheme == "https":
ConnClass = self.HTTPSConnection
else:
raise ValueError("Unknown scheme: %r" % scheme)
if "SERVER_NAME" not in environ:
host = environ.get("HTTP_HOST")
if not host:
raise ValueError(
"environ contains neither SERVER_NAME nor HTTP_HOST")
if ":" in host:
host, port = host.split(":", 1)
else:
if scheme == "http":
port = "80"
else:
port = "443"
environ["SERVER_NAME"] = host
environ["SERVER_PORT"] = port
kw = {}
if "webob.client.timeout" in environ and self._timeout_supported(
ConnClass):
kw["timeout"] = environ["webob.client.timeout"]
conn = ConnClass("%(SERVER_NAME)s:%(SERVER_PORT)s" % environ, **kw)
headers = {}
for key, value in environ.items():
if key.startswith("HTTP_"):
key = key[5:].replace("_", "-").title()
headers[key] = value
path = url_quote(environ.get("SCRIPT_NAME", "")) + url_quote(
environ.get("PATH_INFO", ""))
if environ.get("QUERY_STRING"):
path += "?" + environ["QUERY_STRING"]
try:
content_length = int(environ.get("CONTENT_LENGTH", "0"))
except ValueError:
content_length = 0
# FIXME: there is no streaming of the body, and that might be useful
# in some cases
if content_length:
body = environ["wsgi.input"].read(content_length)
else:
body = ""
headers["Content-Length"] = content_length
if environ.get("CONTENT_TYPE"):
headers["Content-Type"] = environ["CONTENT_TYPE"]
if not path.startswith("/"):
path = "/" + path
try:
conn.request(environ["REQUEST_METHOD"], path, body, headers)
res = conn.getresponse()
except socket.timeout:
resp = exc.HTTPGatewayTimeout()
return resp(environ, start_response)
except (OSError, socket.gaierror) as e:
if (isinstance(e, socket.error)
and e.args[0] == -2) or (isinstance(e, socket.gaierror)
and e.args[0] == 8):
# Name or service not known
resp = exc.HTTPBadGateway(
"Name or service not known (bad domain name: %s)" %
environ["SERVER_NAME"])
return resp(environ, start_response)
elif e.args[0] in _e_refused: # pragma: no cover
# Connection refused
resp = exc.HTTPBadGateway("Connection refused")
return resp(environ, start_response)
raise
headers_out = self.parse_headers(res.msg)
status = "%s %s" % (res.status, res.reason)
start_response(status, headers_out)
length = res.getheader("content-length")
# FIXME: This shouldn't really read in all the content at once
if length is not None:
body = res.read(int(length))
else:
body = res.read()
conn.close()
return [body]
def _error(resp):
msg = "Couldn't fetch %s as ActivityStreams 2" % url
logging.error(msg)
err = exc.HTTPBadGateway(msg)
err.response = resp
raise err