def __call__(self, environ, start_response):
try:
request = self.get_request(environ)
path, realpath, statinfo = self.check_path(request.path)
app = self.serve_file(path, realpath, statinfo, request)
if app:
return app(environ, start_response)
exc = get_http_exception(httplib.NOT_FOUND)
return exc(environ, start_response)
except webob.exc.HTTPException, exc:
return exc(environ, start_response)
def test_429_fault_json(self):
# Test fault serialized to JSON via file-extension and/or header.
requests = [
webob.Request.blank('/.json'),
webob.Request.blank('/', headers={"Accept": "application/json"}),
]
for request in requests:
exc = webob.exc.HTTPTooManyRequests
# NOTE(aloga): we intentionally pass an integer for the
# 'Retry-After' header. It should be then converted to a str
fault = wsgi.Fault(
exc(explanation='sorry', headers={'Retry-After': 4}))
response = request.get_response(fault)
expected = {
"overLimit": {
"message": "sorry",
"code": 429,
"retryAfter": "4",
},
}
actual = jsonutils.loads(response.body)
self.assertEqual(response.content_type, "application/json")
self.assertEqual(expected, actual)
def test_413_fault_json(self):
# Test fault serialized to JSON via file-extension and/or header.
requests = [
webob.Request.blank('/.json'),
webob.Request.blank('/', headers={"Accept": "application/json"}),
]
for request in requests:
exc = webob.exc.HTTPRequestEntityTooLarge
# NOTE(aloga): we intentionally pass an integer for the
# 'Retry-After' header. It should be then converted to a str
fault = wsgi.Fault(exc(explanation='sorry',
headers={'Retry-After': 4}))
response = request.get_response(fault)
expected = {
"overLimit": {
"message": "sorry",
"code": 413,
"retryAfter": "4",
},
}
actual = jsonutils.loads(response.body)
self.assertEqual(response.content_type, "application/json")
self.assertEqual(expected, actual)
def __call__(self, environ, start_response):
"""
Call the application and catch exceptions.
"""
app_iter = None
# Just call the application and send the output back
# unchanged and catch relevant HTTP exceptions
try:
app_iter = self.app(environ, start_response)
except (HTTPClientError, HTTPRedirection), exc:
app_iter = exc(environ, start_response)
def test_HTTPException():
import warnings
_called = []
_result = object()
def _response(environ, start_response):
_called.append((environ, start_response))
return _result
environ = {}
start_response = object()
exc = HTTPException('testing', _response)
ok_(exc.wsgi_response is _response)
result = exc(environ, start_response)
ok_(result is result)
assert_equal(_called, [(environ, start_response)])
def test_413_fault_json(self):
"""Test fault serialized to JSON via file-extension and/or header."""
requests = [webob.Request.blank("/.json"), webob.Request.blank("/", headers={"Accept": "application/json"})]
for request in requests:
exc = webob.exc.HTTPRequestEntityTooLarge
fault = wsgi.Fault(exc(explanation="sorry", headers={"Retry-After": 4}))
response = request.get_response(fault)
expected = {"overLimit": {"message": "sorry", "code": 413, "retryAfter": 4}}
actual = jsonutils.loads(response.body)
self.assertEqual(response.content_type, "application/json")
self.assertEqual(expected, actual)
def test_HTTPException():
_called = []
_result = object()
def _response(environ, start_response):
_called.append((environ, start_response))
return _result
environ = {}
start_response = object()
exc = HTTPException('testing', _response)
ok_(exc.wsgi_response is _response)
ok_(exc.exception is exc)
result = exc(environ, start_response)
ok_(result is result)
assert_equal(_called, [(environ, start_response)])
def __call__(self, environ, start_response):
# Note that catching the webob exception is for Python 2.4 support.
# In the brave new world of new-style exceptions (derived from object)
# multiple inheritance works like you'd expect: the NotAuthenticatedError
# is caught because it descends from the past and webob exceptions.
# In the old world (2.4-), the webob exception needs to be in the catch list
environ['paste.httpexceptions'] = self
environ.setdefault('paste.expected_exceptions',
[]).extend([paste.httpexceptions.HTTPException,
webob.exc.HTTPException])
try:
return self.application(environ, start_response)
except (paste.httpexceptions.HTTPException,
webob.exc.HTTPException), exc:
return exc(environ, start_response)
def exception_from_response(response):
"""Convert an OpenStack V2 Fault into a webob exception.
Since we are calling the OpenStack API we should process the Faults
produced by them. Extract the Fault information according to [1] and
convert it back to a webob exception.
[1] http://docs.openstack.org/developer/nova/v2/faults.html
:param response: a webob.Response containing an exception
:returns: a webob.exc.exception object
"""
exceptions = {
400: webob.exc.HTTPBadRequest,
401: webob.exc.HTTPUnauthorized,
403: webob.exc.HTTPForbidden,
404: webob.exc.HTTPNotFound,
405: webob.exc.HTTPMethodNotAllowed,
406: webob.exc.HTTPNotAcceptable,
409: webob.exc.HTTPConflict,
413: webob.exc.HTTPRequestEntityTooLarge,
415: webob.exc.HTTPUnsupportedMediaType,
429: webob.exc.HTTPTooManyRequests,
501: webob.exc.HTTPNotImplemented,
503: webob.exc.HTTPServiceUnavailable,
}
message = ('Unexpected API Error. Please report this at '
'http://bugs.launchpad.net/ooi/ and attach the ooi '
'API log if possible.')
code = response.status_int
exc = exceptions.get(code, webob.exc.HTTPInternalServerError)
if code in exceptions:
try:
message = response.json_body.popitem()[1].get("message")
except Exception:
LOG.exception("Unknown error happenened processing response %s"
% response)
exc = webob.exc.HTTPInternalServerError
else:
LOG.error("Nova returned an internal server error %s"
% response)
return exc(explanation=message)
def exception_from_response(response):
"""Convert an OpenStack V2 Fault into a webob exception.
Since we are calling the OpenStack API we should process the Faults
produced by them. Extract the Fault information according to [1] and
convert it back to a webob exception.
[1] http://docs.openstack.org/developer/nova/v2/faults.html
:param response: a webob.Response containing an exception
:returns: a webob.exc.exception object
"""
exceptions = {
400: webob.exc.HTTPBadRequest,
401: webob.exc.HTTPUnauthorized,
403: webob.exc.HTTPForbidden,
404: webob.exc.HTTPNotFound,
405: webob.exc.HTTPMethodNotAllowed,
406: webob.exc.HTTPNotAcceptable,
409: webob.exc.HTTPConflict,
413: webob.exc.HTTPRequestEntityTooLarge,
415: webob.exc.HTTPUnsupportedMediaType,
429: webob.exc.HTTPTooManyRequests,
501: webob.exc.HTTPNotImplemented,
503: webob.exc.HTTPServiceUnavailable,
}
message = ('Unexpected API Error. Please report this at '
'http://bugs.launchpad.net/ooi/ and attach the ooi '
'API log if possible.')
code = response.status_int
exc = exceptions.get(code, webob.exc.HTTPInternalServerError)
if code in exceptions:
try:
message = response.json_body.popitem()[1].get("message")
except Exception:
LOG.exception("Unknown error happenened processing response %s" %
response)
exc = webob.exc.HTTPInternalServerError
else:
LOG.error("Nova returned an internal server error %s" % response)
return exc(explanation=message)
def test_HTTPException():
import warnings
_called = []
_result = object()
def _response(environ, start_response):
_called.append((environ, start_response))
return _result
environ = {}
start_response = object()
exc = HTTPException('testing', _response)
ok_(exc.wsgi_response is _response)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
assert(exc.exception is exc)
assert(len(w) == 1)
result = exc(environ, start_response)
ok_(result is result)
assert_equal(_called, [(environ, start_response)])
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except ClientError as ce:
print 'ClientError:', ce, ce.status, ce.details
exc = {
400: webob.exc.HTTPBadRequest,
401: webob.exc.HTTPUnauthorized,
403: webob.exc.HTTPForbidden,
404: webob.exc.HTTPNotFound,
405: webob.exc.HTTPMethodNotAllowed,
406: webob.exc.HTTPNotAcceptable,
409: webob.exc.HTTPConflict,
413: webob.exc.HTTPRequestEntityTooLarge,
415: webob.exc.HTTPUnsupportedMediaType,
429: webob.exc.HTTPTooManyRequests,
501: webob.exc.HTTPNotImplemented,
503: webob.exc.HTTPServiceUnavailable,
}.get(ce.status, webob.exc.HTTPInternalServerError)
raise exc(explanation='{0}'.format(ce.message))
def __call__(self, environ, start_response):
"""
Call the application and catch exceptions.
"""
app_iter = None
# Just call the application and send the output back
# unchanged and catch relevant HTTP exceptions
try:
app_iter = self.app(environ, start_response)
except (HTTPClientError, HTTPRedirection) as exc:
app_iter = exc(environ, start_response)
# If an exception occours we get the exception information
# and prepare a traceback we can render
except Exception:
exc_type, exc_value, tb = sys.exc_info()
traceback = ['Traceback (most recent call last):']
traceback += format_tb(tb)
traceback.append('%s: %s' % (exc_type.__name__, exc_value))
# We might have not a stated response by now. Try to
# start one with the status code 500 or ignore any
# raised exception if the application already
# started one
try:
start_response('500 Internal Server Error',
[('Content-Type', 'text/plain')])
except Exception:
pass
traceback = '\n'.join(traceback)
logger.error(traceback)
yield traceback
for item in app_iter:
yield item
# Returned iterable might have a close function.
# If it exists it *must* be called
if hasattr(app_iter, 'close'):
app_iter.close()
def test_413_fault_xml(self):
requests = [
webob.Request.blank('/.xml'),
webob.Request.blank('/', headers={"Accept": "application/xml"}),
]
for request in requests:
exc = webob.exc.HTTPRequestEntityTooLarge
fault = faults.Fault(
exc(explanation='sorry', headers={'Retry-After': 4}))
response = request.get_response(fault)
expected = self._prepare_xml("""
<overLimit code="413" xmlns="%s">
<message>sorry</message>
<retryAfter>4</retryAfter>
</overLimit>
""" % common.XML_NS_V10)
actual = self._prepare_xml(response.body)
self.assertEqual(expected, actual)
self.assertEqual(response.content_type, "application/xml")
self.assertEqual(response.headers['Retry-After'], 4)
def test_413_fault_xml(self):
requests = [
webob.Request.blank('/.xml'),
webob.Request.blank('/', headers={"Accept": "application/xml"}),
]
for request in requests:
exc = webob.exc.HTTPRequestEntityTooLarge
fault = faults.Fault(exc(explanation='sorry',
headers={'Retry-After': 4}))
response = request.get_response(fault)
expected = self._prepare_xml("""
<overLimit code="413" xmlns="%s">
<message>sorry</message>
<retryAfter>4</retryAfter>
</overLimit>
""" % common.XML_NS_V10)
actual = self._prepare_xml(response.body)
self.assertEqual(expected, actual)
self.assertEqual(response.content_type, "application/xml")
self.assertEqual(response.headers['Retry-After'], 4)
def test_413_fault_json(self):
"""Test fault serialized to JSON via file-extension and/or header."""
requests = [
webob.Request.blank('/.json'),
webob.Request.blank('/', headers={"Accept": "application/json"}),
]
for request in requests:
exc = webob.exc.HTTPRequestEntityTooLarge
fault = faults.Fault(
exc(explanation='sorry', headers={'Retry-After': 4}))
response = request.get_response(fault)
expected = {
"overLimit": {
"message": "sorry",
"code": 413,
"retryAfter": 4,
},
}
actual = json.loads(response.body)
self.assertEqual(response.content_type, "application/json")
self.assertEqual(expected, actual)
def manage_http_exception(code, message):
exc = default_exceptions.get(code, webob.exc.HTTPInternalServerError)
return exc("%s" % message)
def application(environ, start_response):
"""Hand-rolled WSGI application so I can stream output.
...by returning a generator that yields the response body lines.
"""
request = webob.Request(environ)
headers = [('Content-Type', 'text/html')]
# validate request
if request.method not in ('GET', 'POST'):
return webob.exc.HTTPMethodNotAllowed()(environ, start_response)
url = request.params.get('url')
if not url:
return webob.exc.HTTPBadRequest('Missing required parameter: url')(
environ, start_response)
parsed = urlparse.urlparse(url)
if parsed.netloc in DOMAIN_BLACKLIST:
return webob.exc.HTTPBadRequest(
'Sorry, this content is not currently supported due to copyright.'
)(environ, start_response)
# check that our CPU credit balance isn't too low
try:
cloudwatch = boto.ec2.cloudwatch.connect_to_region(
'us-west-2',
aws_access_key_id=AWS_KEY_ID,
aws_secret_access_key=AWS_SECRET_KEY)
for metric in cloudwatch.list_metrics(metric_name='CPUCreditBalance'):
if metric.name == 'CPUCreditBalance':
stats = metric.query(
datetime.datetime.now() - datetime.timedelta(minutes=10),
datetime.datetime.now(), ['Average'])
if stats:
credit = stats[-1].get('Average')
if credit and credit <= 30:
msg = "Sorry, we're too busy right now. Please try again later!"
exc = webob.exc.HTTPServiceUnavailable(msg)
exc.html_template_obj = Template(HTML_HEADER + msg +
HTML_FOOTER)
return exc(environ, start_response)
except:
logging.exception("Couldn't fetch CPU credit balance from CloudWatch!")
write_fn = start_response('200 OK', headers)
def write(line):
write_fn(line.encode('utf-8'))
def run():
"""Generator that does all the work and yields the response body lines.
TODO: figure out how to catch and log stack traces when this function raises
an exception. Currently the log only gets the exception message. Wrapping
the call at the bottom in try/except doesn't work since it's a generator. :/
"""
yield HTML_HEADER
yield ('<div id="progress">\nFetching %s ...<br />' %
url).encode('utf-8')
# function to print out status while downloading
def download_progress_hook(progress):
status = progress.get('status')
if status == 'finished':
msg = '<br />Extracting audio (this can take a while)...\n'
elif status == 'error':
# we always get an 'error' progress when the video finishes downloading.
# not sure why. ignore it.
return
elif status == 'downloading':
p = lambda field: progress.get(field) or ''
try:
percent = float(p('_percent_str').strip('%') or '0')
except ValueError:
percent = 0
msg = (
'<span><progress max="100" value="%s"></progress><br /> '
'%s of %s at %s in %s...</span>\n' %
(percent, p('_downloaded_bytes_str'), p('_total_bytes_str')
or p('_total_bytes_estimate_str'), p('_speed_str'),
p('_eta_str')))
else:
msg = status + '<br />\n'
write(msg)
# fetch video info (resolves URL) to see if we've already downloaded it
options = {
'outtmpl':
u'/tmp/%(webpage_url)s',
'restrictfilenames':
True, # don't allow & or spaces in file names
'updatetime':
False, # don't set output file mtime to video mtime
'logger':
logging,
'logtostderr':
True,
'format':
'bestaudio/best',
'noplaylist':
True,
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'progress_hooks': [download_progress_hook],
}
ydl = youtube_dl.YoutubeDL(options)
with handle_errors(write):
info = ydl.extract_info(url, download=False)
# prepare_filename() returns the video filename, not the postprocessed one,
# so change the extension manually. the resulting filename will look like:
# '/tmp/https_-_www.youtube.com_watchv=6dyWlM4ej3Q.mp3'
#
# ext4 max filename length is 255 bytes, and huffduffer also silently
# truncates URLs to 255 chars total, so truncate before that if necessary.
filename_prefix = ydl.prepare_filename(info)[:245 - len(S3_BASE)]
options['outtmpl'] = filename_prefix.replace('%', '%%') + '.%(ext)s'
filename = filename_prefix + '.mp3'
s3 = boto.connect_s3(aws_access_key_id=AWS_KEY_ID,
aws_secret_access_key=AWS_SECRET_KEY)
bucket = s3.get_bucket(S3_BUCKET)
# strip the filename's path, scheme, and leading www., mobile, or m.
# the resulting S3 key will look like 'youtube.com_watchv=6dyWlM4ej3Q.mp3'
s3_key = re.sub('^https?_-_((www|m|mobile|player).)?', '',
os.path.basename(filename))
key = bucket.get_key(s3_key, validate=False)
if key.exists():
yield 'Already downloaded! <br />\n'
else:
# download video and extract mp3
yield 'Downloading...<br />\n'
with handle_errors(write):
youtube_dl.YoutubeDL(options).download([url])
# upload to S3
# http://docs.pythonboto.org/en/latest/s3_tut.html
yield 'Uploading to S3...<br />\n'
def upload_callback(sent, total):
write('<span><progress max="100" value="%s"></progress><br /> '
'%.2fMB of %.2fMB</span>\n' %
((sent * 100 / total), float(sent) / 1000000,
float(total) / 1000000))
key.set_contents_from_filename(filename, cb=upload_callback)
key.make_public()
os.remove(filename)
# get metadata, specifically last_modified
key = bucket.get_key(s3_key)
# generate description
description = info.get('description') or ''
footer = """\
Original video: %s
Downloaded by http://huffduff-video.snarfed.org/ on %s
Available for 30 days after download""" % (url, key.last_modified)
# last_modified format is RFC 7231, e.g. 'Fri, 22 Jul 2016 07:11:46 GMT'
if description:
footer = """
===
""" + footer
max_len = 1500 - len(footer)
if len(description) > max_len:
description = description[:max_len] + '...'
description += footer
# open 'Huffduff it' page
yield """\n<br />Opening Huffduffer dialog...
<script type="text/javascript">
window.location = "https://huffduffer.com/add?popup=true&%s";
</script>
""" % urllib.urlencode([(k, v.encode('utf-8')) for k, v in (
('bookmark[url]', (S3_BASE + s3_key)),
('bookmark[title]', info.get('title') or ''),
('bookmark[description]', description),
('bookmark[tags]', ','.join(info.get('categories') or [])),
)])
yield HTML_FOOTER
# alternative:
# http://themindfulbit.com/blog/optimizing-your-podcast-site-for-huffduffer
return run()