예제 #1
0
 def __call__(self, environ, start_response):
     try:
         request = self.get_request(environ)
         path, realpath, statinfo = self.check_path(request.path)
         app = self.serve_file(path, realpath, statinfo, request)
         if app:
             return app(environ, start_response)
         exc = get_http_exception(httplib.NOT_FOUND)
         return exc(environ, start_response)
     except webob.exc.HTTPException, exc:
         return exc(environ, start_response)
예제 #2
0
    def test_429_fault_json(self):
        # Test fault serialized to JSON via file-extension and/or header.
        requests = [
            webob.Request.blank('/.json'),
            webob.Request.blank('/', headers={"Accept": "application/json"}),
        ]

        for request in requests:
            exc = webob.exc.HTTPTooManyRequests
            # NOTE(aloga): we intentionally pass an integer for the
            # 'Retry-After' header. It should be then converted to a str
            fault = wsgi.Fault(
                exc(explanation='sorry', headers={'Retry-After': 4}))
            response = request.get_response(fault)

            expected = {
                "overLimit": {
                    "message": "sorry",
                    "code": 429,
                    "retryAfter": "4",
                },
            }
            actual = jsonutils.loads(response.body)

            self.assertEqual(response.content_type, "application/json")
            self.assertEqual(expected, actual)
예제 #3
0
    def test_413_fault_json(self):
        # Test fault serialized to JSON via file-extension and/or header.
        requests = [
            webob.Request.blank('/.json'),
            webob.Request.blank('/', headers={"Accept": "application/json"}),
        ]

        for request in requests:
            exc = webob.exc.HTTPRequestEntityTooLarge
            # NOTE(aloga): we intentionally pass an integer for the
            # 'Retry-After' header. It should be then converted to a str
            fault = wsgi.Fault(exc(explanation='sorry',
                        headers={'Retry-After': 4}))
            response = request.get_response(fault)

            expected = {
                "overLimit": {
                    "message": "sorry",
                    "code": 413,
                    "retryAfter": "4",
                },
            }
            actual = jsonutils.loads(response.body)

            self.assertEqual(response.content_type, "application/json")
            self.assertEqual(expected, actual)
예제 #4
0
    def __call__(self, environ, start_response):
        """
        Call the application and catch exceptions.
        """
        app_iter = None

        # Just call the application and send the output back
        #   unchanged and catch relevant HTTP exceptions
        try:
            app_iter = self.app(environ, start_response)
        except (HTTPClientError, HTTPRedirection), exc:        
            app_iter = exc(environ, start_response)            
예제 #5
0
파일: test_exc.py 프로젝트: dairiki/webob
def test_HTTPException():
    import warnings
    _called = []
    _result = object()
    def _response(environ, start_response):
        _called.append((environ, start_response))
        return _result
    environ = {}
    start_response = object()
    exc = HTTPException('testing', _response)
    ok_(exc.wsgi_response is _response)
    result = exc(environ, start_response)
    ok_(result is result)
    assert_equal(_called, [(environ, start_response)])
예제 #6
0
    def test_413_fault_json(self):
        """Test fault serialized to JSON via file-extension and/or header."""
        requests = [webob.Request.blank("/.json"), webob.Request.blank("/", headers={"Accept": "application/json"})]

        for request in requests:
            exc = webob.exc.HTTPRequestEntityTooLarge
            fault = wsgi.Fault(exc(explanation="sorry", headers={"Retry-After": 4}))
            response = request.get_response(fault)

            expected = {"overLimit": {"message": "sorry", "code": 413, "retryAfter": 4}}
            actual = jsonutils.loads(response.body)

            self.assertEqual(response.content_type, "application/json")
            self.assertEqual(expected, actual)
예제 #7
0
def test_HTTPException():
    _called = []
    _result = object()

    def _response(environ, start_response):
        _called.append((environ, start_response))
        return _result

    environ = {}
    start_response = object()
    exc = HTTPException('testing', _response)
    ok_(exc.wsgi_response is _response)
    ok_(exc.exception is exc)
    result = exc(environ, start_response)
    ok_(result is result)
    assert_equal(_called, [(environ, start_response)])
예제 #8
0
 def __call__(self, environ, start_response):
                 
     # Note that catching the webob exception is for Python 2.4 support.
     # In the brave new world of new-style exceptions (derived from object)
     # multiple inheritance works like you'd expect: the NotAuthenticatedError 
     # is caught because it descends from the past and webob exceptions.
     # In the old world (2.4-), the webob exception needs to be in the catch list
         
     environ['paste.httpexceptions'] = self
     environ.setdefault('paste.expected_exceptions',
                        []).extend([paste.httpexceptions.HTTPException,
                                    webob.exc.HTTPException])
     try:
         return self.application(environ, start_response)        
     except (paste.httpexceptions.HTTPException, 
             webob.exc.HTTPException), exc:        
         return exc(environ, start_response)
예제 #9
0
파일: helpers.py 프로젝트: openstack/ooi
def exception_from_response(response):
    """Convert an OpenStack V2 Fault into a webob exception.

    Since we are calling the OpenStack API we should process the Faults
    produced by them. Extract the Fault information according to [1] and
    convert it back to a webob exception.

    [1] http://docs.openstack.org/developer/nova/v2/faults.html

    :param response: a webob.Response containing an exception
    :returns: a webob.exc.exception object
    """
    exceptions = {
        400: webob.exc.HTTPBadRequest,
        401: webob.exc.HTTPUnauthorized,
        403: webob.exc.HTTPForbidden,
        404: webob.exc.HTTPNotFound,
        405: webob.exc.HTTPMethodNotAllowed,
        406: webob.exc.HTTPNotAcceptable,
        409: webob.exc.HTTPConflict,
        413: webob.exc.HTTPRequestEntityTooLarge,
        415: webob.exc.HTTPUnsupportedMediaType,
        429: webob.exc.HTTPTooManyRequests,
        501: webob.exc.HTTPNotImplemented,
        503: webob.exc.HTTPServiceUnavailable,
    }

    message = ('Unexpected API Error. Please report this at '
               'http://bugs.launchpad.net/ooi/ and attach the ooi '
               'API log if possible.')

    code = response.status_int
    exc = exceptions.get(code, webob.exc.HTTPInternalServerError)

    if code in exceptions:
        try:
            message = response.json_body.popitem()[1].get("message")
        except Exception:
            LOG.exception("Unknown error happenened processing response %s"
                          % response)
            exc = webob.exc.HTTPInternalServerError
    else:
        LOG.error("Nova returned an internal server error %s"
                  % response)

    return exc(explanation=message)
예제 #10
0
def exception_from_response(response):
    """Convert an OpenStack V2 Fault into a webob exception.

    Since we are calling the OpenStack API we should process the Faults
    produced by them. Extract the Fault information according to [1] and
    convert it back to a webob exception.

    [1] http://docs.openstack.org/developer/nova/v2/faults.html

    :param response: a webob.Response containing an exception
    :returns: a webob.exc.exception object
    """
    exceptions = {
        400: webob.exc.HTTPBadRequest,
        401: webob.exc.HTTPUnauthorized,
        403: webob.exc.HTTPForbidden,
        404: webob.exc.HTTPNotFound,
        405: webob.exc.HTTPMethodNotAllowed,
        406: webob.exc.HTTPNotAcceptable,
        409: webob.exc.HTTPConflict,
        413: webob.exc.HTTPRequestEntityTooLarge,
        415: webob.exc.HTTPUnsupportedMediaType,
        429: webob.exc.HTTPTooManyRequests,
        501: webob.exc.HTTPNotImplemented,
        503: webob.exc.HTTPServiceUnavailable,
    }

    message = ('Unexpected API Error. Please report this at '
               'http://bugs.launchpad.net/ooi/ and attach the ooi '
               'API log if possible.')

    code = response.status_int
    exc = exceptions.get(code, webob.exc.HTTPInternalServerError)

    if code in exceptions:
        try:
            message = response.json_body.popitem()[1].get("message")
        except Exception:
            LOG.exception("Unknown error happenened processing response %s" %
                          response)
            exc = webob.exc.HTTPInternalServerError
    else:
        LOG.error("Nova returned an internal server error %s" % response)

    return exc(explanation=message)
예제 #11
0
파일: test_exc.py 프로젝트: witsch/webob
def test_HTTPException():
    import warnings
    _called = []
    _result = object()
    def _response(environ, start_response):
        _called.append((environ, start_response))
        return _result
    environ = {}
    start_response = object()
    exc = HTTPException('testing', _response)
    ok_(exc.wsgi_response is _response)
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        assert(exc.exception is exc)
        assert(len(w) == 1)
    result = exc(environ, start_response)
    ok_(result is result)
    assert_equal(_called, [(environ, start_response)])
예제 #12
0
 def wrapper(*args, **kwargs):
     try:
         return f(*args, **kwargs)
     except ClientError as ce:
         print 'ClientError:', ce, ce.status, ce.details
         exc = {
             400: webob.exc.HTTPBadRequest,
             401: webob.exc.HTTPUnauthorized,
             403: webob.exc.HTTPForbidden,
             404: webob.exc.HTTPNotFound,
             405: webob.exc.HTTPMethodNotAllowed,
             406: webob.exc.HTTPNotAcceptable,
             409: webob.exc.HTTPConflict,
             413: webob.exc.HTTPRequestEntityTooLarge,
             415: webob.exc.HTTPUnsupportedMediaType,
             429: webob.exc.HTTPTooManyRequests,
             501: webob.exc.HTTPNotImplemented,
             503: webob.exc.HTTPServiceUnavailable,
         }.get(ce.status, webob.exc.HTTPInternalServerError)
         raise exc(explanation='{0}'.format(ce.message))
예제 #13
0
파일: app.py 프로젝트: jannis-a/coldsweat
    def __call__(self, environ, start_response):
        """
        Call the application and catch exceptions.
        """
        app_iter = None

        # Just call the application and send the output back
        #   unchanged and catch relevant HTTP exceptions
        try:
            app_iter = self.app(environ, start_response)
        except (HTTPClientError, HTTPRedirection) as exc:
            app_iter = exc(environ, start_response)
        # If an exception occours we get the exception information
        #   and prepare a traceback we can render
        except Exception:
            exc_type, exc_value, tb = sys.exc_info()
            traceback = ['Traceback (most recent call last):']
            traceback += format_tb(tb)
            traceback.append('%s: %s' % (exc_type.__name__, exc_value))
            # We might have not a stated response by now. Try to
            #   start one with the status code 500 or ignore any
            #   raised exception if the application already
            #   started one
            try:
                start_response('500 Internal Server Error',
                               [('Content-Type', 'text/plain')])
            except Exception:
                pass

            traceback = '\n'.join(traceback)
            logger.error(traceback)

            yield traceback

        for item in app_iter:
            yield item

        # Returned iterable might have a close function.
        #   If it exists it *must* be called
        if hasattr(app_iter, 'close'):
            app_iter.close()
예제 #14
0
    def test_413_fault_xml(self):
        requests = [
            webob.Request.blank('/.xml'),
            webob.Request.blank('/', headers={"Accept": "application/xml"}),
        ]

        for request in requests:
            exc = webob.exc.HTTPRequestEntityTooLarge
            fault = faults.Fault(
                exc(explanation='sorry', headers={'Retry-After': 4}))
            response = request.get_response(fault)

            expected = self._prepare_xml("""
                <overLimit code="413" xmlns="%s">
                    <message>sorry</message>
                    <retryAfter>4</retryAfter>
                </overLimit>
            """ % common.XML_NS_V10)
            actual = self._prepare_xml(response.body)

            self.assertEqual(expected, actual)
            self.assertEqual(response.content_type, "application/xml")
            self.assertEqual(response.headers['Retry-After'], 4)
예제 #15
0
    def test_413_fault_xml(self):
        requests = [
            webob.Request.blank('/.xml'),
            webob.Request.blank('/', headers={"Accept": "application/xml"}),
        ]

        for request in requests:
            exc = webob.exc.HTTPRequestEntityTooLarge
            fault = faults.Fault(exc(explanation='sorry',
                        headers={'Retry-After': 4}))
            response = request.get_response(fault)

            expected = self._prepare_xml("""
                <overLimit code="413" xmlns="%s">
                    <message>sorry</message>
                    <retryAfter>4</retryAfter>
                </overLimit>
            """ % common.XML_NS_V10)
            actual = self._prepare_xml(response.body)

            self.assertEqual(expected, actual)
            self.assertEqual(response.content_type, "application/xml")
            self.assertEqual(response.headers['Retry-After'], 4)
예제 #16
0
    def test_413_fault_json(self):
        """Test fault serialized to JSON via file-extension and/or header."""
        requests = [
            webob.Request.blank('/.json'),
            webob.Request.blank('/', headers={"Accept": "application/json"}),
        ]

        for request in requests:
            exc = webob.exc.HTTPRequestEntityTooLarge
            fault = faults.Fault(
                exc(explanation='sorry', headers={'Retry-After': 4}))
            response = request.get_response(fault)

            expected = {
                "overLimit": {
                    "message": "sorry",
                    "code": 413,
                    "retryAfter": 4,
                },
            }
            actual = json.loads(response.body)

            self.assertEqual(response.content_type, "application/json")
            self.assertEqual(expected, actual)
예제 #17
0
def manage_http_exception(code, message):
    exc = default_exceptions.get(code, webob.exc.HTTPInternalServerError)
    return exc("%s" % message)
예제 #18
0
def application(environ, start_response):
    """Hand-rolled WSGI application so I can stream output.

  ...by returning a generator that yields the response body lines.
  """
    request = webob.Request(environ)
    headers = [('Content-Type', 'text/html')]

    # validate request
    if request.method not in ('GET', 'POST'):
        return webob.exc.HTTPMethodNotAllowed()(environ, start_response)

    url = request.params.get('url')
    if not url:
        return webob.exc.HTTPBadRequest('Missing required parameter: url')(
            environ, start_response)

    parsed = urlparse.urlparse(url)
    if parsed.netloc in DOMAIN_BLACKLIST:
        return webob.exc.HTTPBadRequest(
            'Sorry, this content is not currently supported due to copyright.'
        )(environ, start_response)

    # check that our CPU credit balance isn't too low
    try:
        cloudwatch = boto.ec2.cloudwatch.connect_to_region(
            'us-west-2',
            aws_access_key_id=AWS_KEY_ID,
            aws_secret_access_key=AWS_SECRET_KEY)
        for metric in cloudwatch.list_metrics(metric_name='CPUCreditBalance'):
            if metric.name == 'CPUCreditBalance':
                stats = metric.query(
                    datetime.datetime.now() - datetime.timedelta(minutes=10),
                    datetime.datetime.now(), ['Average'])
                if stats:
                    credit = stats[-1].get('Average')
                    if credit and credit <= 30:
                        msg = "Sorry, we're too busy right now. Please try again later!"
                        exc = webob.exc.HTTPServiceUnavailable(msg)
                        exc.html_template_obj = Template(HTML_HEADER + msg +
                                                         HTML_FOOTER)
                        return exc(environ, start_response)
    except:
        logging.exception("Couldn't fetch CPU credit balance from CloudWatch!")

    write_fn = start_response('200 OK', headers)

    def write(line):
        write_fn(line.encode('utf-8'))

    def run():
        """Generator that does all the work and yields the response body lines.

    TODO: figure out how to catch and log stack traces when this function raises
    an exception. Currently the log only gets the exception message. Wrapping
    the call at the bottom in try/except doesn't work since it's a generator. :/
    """
        yield HTML_HEADER
        yield ('<div id="progress">\nFetching %s ...<br />' %
               url).encode('utf-8')

        # function to print out status while downloading
        def download_progress_hook(progress):
            status = progress.get('status')
            if status == 'finished':
                msg = '<br />Extracting audio (this can take a while)...\n'
            elif status == 'error':
                # we always get an 'error' progress when the video finishes downloading.
                # not sure why. ignore it.
                return
            elif status == 'downloading':
                p = lambda field: progress.get(field) or ''
                try:
                    percent = float(p('_percent_str').strip('%') or '0')
                except ValueError:
                    percent = 0
                msg = (
                    '<span><progress max="100" value="%s"></progress><br /> '
                    '%s of %s at %s in %s...</span>\n' %
                    (percent, p('_downloaded_bytes_str'), p('_total_bytes_str')
                     or p('_total_bytes_estimate_str'), p('_speed_str'),
                     p('_eta_str')))
            else:
                msg = status + '<br />\n'
            write(msg)

        # fetch video info (resolves URL) to see if we've already downloaded it
        options = {
            'outtmpl':
            u'/tmp/%(webpage_url)s',
            'restrictfilenames':
            True,  # don't allow & or spaces in file names
            'updatetime':
            False,  # don't set output file mtime to video mtime
            'logger':
            logging,
            'logtostderr':
            True,
            'format':
            'bestaudio/best',
            'noplaylist':
            True,
            'postprocessors': [{
                'key': 'FFmpegExtractAudio',
                'preferredcodec': 'mp3',
                'preferredquality': '192',
            }],
            'progress_hooks': [download_progress_hook],
        }
        ydl = youtube_dl.YoutubeDL(options)
        with handle_errors(write):
            info = ydl.extract_info(url, download=False)

        # prepare_filename() returns the video filename, not the postprocessed one,
        # so change the extension manually. the resulting filename will look like:
        #   '/tmp/https_-_www.youtube.com_watchv=6dyWlM4ej3Q.mp3'
        #
        # ext4 max filename length is 255 bytes, and huffduffer also silently
        # truncates URLs to 255 chars total, so truncate before that if necessary.
        filename_prefix = ydl.prepare_filename(info)[:245 - len(S3_BASE)]
        options['outtmpl'] = filename_prefix.replace('%', '%%') + '.%(ext)s'
        filename = filename_prefix + '.mp3'

        s3 = boto.connect_s3(aws_access_key_id=AWS_KEY_ID,
                             aws_secret_access_key=AWS_SECRET_KEY)
        bucket = s3.get_bucket(S3_BUCKET)
        # strip the filename's path, scheme, and leading www., mobile, or m.
        # the resulting S3 key will look like 'youtube.com_watchv=6dyWlM4ej3Q.mp3'
        s3_key = re.sub('^https?_-_((www|m|mobile|player).)?', '',
                        os.path.basename(filename))
        key = bucket.get_key(s3_key, validate=False)

        if key.exists():
            yield 'Already downloaded! <br />\n'
        else:
            # download video and extract mp3
            yield 'Downloading...<br />\n'
            with handle_errors(write):
                youtube_dl.YoutubeDL(options).download([url])

            # upload to S3
            # http://docs.pythonboto.org/en/latest/s3_tut.html
            yield 'Uploading to S3...<br />\n'

            def upload_callback(sent, total):
                write('<span><progress max="100" value="%s"></progress><br /> '
                      '%.2fMB of %.2fMB</span>\n' %
                      ((sent * 100 / total), float(sent) / 1000000,
                       float(total) / 1000000))

            key.set_contents_from_filename(filename, cb=upload_callback)
            key.make_public()
            os.remove(filename)

        # get metadata, specifically last_modified
        key = bucket.get_key(s3_key)
        # generate description
        description = info.get('description') or ''
        footer = """\
Original video: %s
Downloaded by http://huffduff-video.snarfed.org/ on %s
Available for 30 days after download""" % (url, key.last_modified)
        # last_modified format is RFC 7231, e.g. 'Fri, 22 Jul 2016 07:11:46 GMT'
        if description:
            footer = """

===
""" + footer

        max_len = 1500 - len(footer)
        if len(description) > max_len:
            description = description[:max_len] + '...'
        description += footer

        # open 'Huffduff it' page
        yield """\n<br />Opening Huffduffer dialog...
<script type="text/javascript">
window.location = "https://huffduffer.com/add?popup=true&%s";
</script>
""" % urllib.urlencode([(k, v.encode('utf-8')) for k, v in (
            ('bookmark[url]', (S3_BASE + s3_key)),
            ('bookmark[title]', info.get('title') or ''),
            ('bookmark[description]', description),
            ('bookmark[tags]', ','.join(info.get('categories') or [])),
        )])
        yield HTML_FOOTER

        # alternative:
        # http://themindfulbit.com/blog/optimizing-your-podcast-site-for-huffduffer

    return run()