Esempio n. 1
0
    def test_attachment(self):
        app = flask.Flask(__name__)
        with catch_warnings() as captured:
            with app.test_request_context():
                f = open(os.path.join(app.root_path, "static/index.html"))
                rv = flask.send_file(f, as_attachment=True)
                value, options = parse_options_header(rv.headers["Content-Disposition"])
                self.assert_equal(value, "attachment")
                rv.close()
            # mimetypes + etag
            self.assert_equal(len(captured), 2)

        with app.test_request_context():
            self.assert_equal(options["filename"], "index.html")
            rv = flask.send_file("static/index.html", as_attachment=True)
            value, options = parse_options_header(rv.headers["Content-Disposition"])
            self.assert_equal(value, "attachment")
            self.assert_equal(options["filename"], "index.html")
            rv.close()

        with app.test_request_context():
            rv = flask.send_file(StringIO("Test"), as_attachment=True, attachment_filename="index.txt", add_etags=False)
            self.assert_equal(rv.mimetype, "text/plain")
            value, options = parse_options_header(rv.headers["Content-Disposition"])
            self.assert_equal(value, "attachment")
            self.assert_equal(options["filename"], "index.txt")
            rv.close()
Esempio n. 2
0
    def test_attachment(self, catch_deprecation_warnings):
        app = flask.Flask(__name__)
        with catch_deprecation_warnings() as captured:
            with app.test_request_context():
                f = open(os.path.join(app.root_path, 'static/index.html'))
                rv = flask.send_file(f, as_attachment=True)
                value, options = parse_options_header(rv.headers['Content-Disposition'])
                assert value == 'attachment'
                rv.close()
            # mimetypes + etag
            assert len(captured) == 2

        with app.test_request_context():
            assert options['filename'] == 'index.html'
            rv = flask.send_file('static/index.html', as_attachment=True)
            value, options = parse_options_header(rv.headers['Content-Disposition'])
            assert value == 'attachment'
            assert options['filename'] == 'index.html'
            rv.close()

        with app.test_request_context():
            rv = flask.send_file(StringIO('Test'), as_attachment=True,
                                 attachment_filename='index.txt',
                                 add_etags=False)
            assert rv.mimetype == 'text/plain'
            value, options = parse_options_header(rv.headers['Content-Disposition'])
            assert value == 'attachment'
            assert options['filename'] == 'index.txt'
            rv.close()
Esempio n. 3
0
    def test_attachment(self, app, req_ctx):

        with open(os.path.join(app.root_path, 'static/index.html')) as f:
            rv = flask.send_file(f, as_attachment=True,
                                 attachment_filename='index.html')
            value, options = \
                parse_options_header(rv.headers['Content-Disposition'])
            assert value == 'attachment'
            assert options['filename'] == 'index.html'
            assert 'filename*' not in rv.headers['Content-Disposition']
            rv.close()

        rv = flask.send_file('static/index.html', as_attachment=True)
        value, options = parse_options_header(rv.headers['Content-Disposition'])
        assert value == 'attachment'
        assert options['filename'] == 'index.html'
        rv.close()

        rv = flask.send_file(StringIO('Test'), as_attachment=True,
                             attachment_filename='index.txt',
                             add_etags=False)
        assert rv.mimetype == 'text/plain'
        value, options = parse_options_header(rv.headers['Content-Disposition'])
        assert value == 'attachment'
        assert options['filename'] == 'index.txt'
        rv.close()
Esempio n. 4
0
 def test_parse_options_header(self):
     assert http.parse_options_header('something; foo="other\"thing"') == \
         ('something', {'foo': 'other"thing'})
     assert http.parse_options_header('something; foo="other\"thing"; meh=42') == \
         ('something', {'foo': 'other"thing', 'meh': '42'})
     assert http.parse_options_header('something; foo="other\"thing"; meh=42; bleh') == \
         ('something', {'foo': 'other"thing', 'meh': '42', 'bleh': None})
Esempio n. 5
0
    def test_attachment(self):
        app = flask.Flask(__name__)
        with catch_warnings() as captured:
            with app.test_request_context():
                f = open(os.path.join(app.root_path, 'static/index.html'))
                rv = flask.send_file(f, as_attachment=True)
                value, options = parse_options_header(rv.headers['Content-Disposition'])
                self.assert_equal(value, 'attachment')
            # mimetypes + etag
            self.assert_equal(len(captured), 2)

        with app.test_request_context():
            self.assert_equal(options['filename'], 'index.html')
            rv = flask.send_file('static/index.html', as_attachment=True)
            value, options = parse_options_header(rv.headers['Content-Disposition'])
            self.assert_equal(value, 'attachment')
            self.assert_equal(options['filename'], 'index.html')

        with app.test_request_context():
            rv = flask.send_file(StringIO('Test'), as_attachment=True,
                                 attachment_filename='index.txt',
                                 add_etags=False)
            self.assert_equal(rv.mimetype, 'text/plain')
            value, options = parse_options_header(rv.headers['Content-Disposition'])
            self.assert_equal(value, 'attachment')
            self.assert_equal(options['filename'], 'index.txt')
Esempio n. 6
0
 def test_parse_options_header_value_with_quotes(self):
     assert http.parse_options_header(
         'form-data; name="file"; filename="t\'es\'t.txt"'
     ) == ('form-data', {'name': 'file', 'filename': "t'es't.txt"})
     assert http.parse_options_header(
         'form-data; name="file"; filename*=UTF-8\'\'"\'🐍\'.txt"'
     ) == ('form-data', {'name': 'file', 'filename': u"'🐍'.txt"})
Esempio n. 7
0
 def test_parse_options_header_value_with_quotes(self):
     assert http.parse_options_header(
         'form-data; name="file"; filename="t\'es\'t.txt"'
     ) == ("form-data", {"name": "file", "filename": "t'es't.txt"})
     assert http.parse_options_header(
         "form-data; name=\"file\"; filename*=UTF-8''\"'🐍'.txt\""
     ) == ("form-data", {"name": "file", "filename": u"'🐍'.txt"})
Esempio n. 8
0
def parse_form_data(environ, stream_factory = None, charset = 'utf-8', errors = 'ignore', max_form_memory_size = None, max_content_length = None, cls = None, silent = True):
    content_type, extra = parse_options_header(environ.get('CONTENT_TYPE', ''))
    try:
        content_length = int(environ['CONTENT_LENGTH'])
    except (KeyError, ValueError):
        content_length = 0

    if cls is None:
        cls = MultiDict
    if max_content_length is not None and content_length > max_content_length:
        raise RequestEntityTooLarge()
    stream = _empty_stream
    files = ()
    if content_type == 'multipart/form-data':
        try:
            form, files = parse_multipart(environ['wsgi.input'], extra.get('boundary'), content_length, stream_factory, charset, errors, max_form_memory_size=max_form_memory_size)
        except ValueError as e:
            if not silent:
                raise 
            form = cls()
        else:
            form = cls(form)

    elif content_type == 'application/x-www-form-urlencoded' or content_type == 'application/x-url-encoded':
        if max_form_memory_size is not None and content_length > max_form_memory_size:
            raise RequestEntityTooLarge()
        form = url_decode(environ['wsgi.input'].read(content_length), charset, errors=errors, cls=cls)
    else:
        form = cls()
        stream = LimitedStream(environ['wsgi.input'], content_length)
    return (stream, form, cls(files))
Esempio n. 9
0
def handle_upload():
    """Handles the result of the upload.

    At this point in time, the file has already been uploaded by the Google App
    Engine API. All we have to do is return some response to the end user. This
    is why we never actually call something to save the file into a blobstore.
    """
    # Verify that the file has actually been uploaded.
    # 'filedata' is the id of the DOM element containing the file.
    filedata = request.files['filedata']
    if not filedata:
        return json.dumps({'message' : "No file."}), 400

    # The blob-key of the newly stored file is stored inside the header of the
    # file.
    header = filedata.headers['Content-Type']
    parsed_header = parse_options_header(header)
    blob_key = parsed_header[1]['blob-key']

    # Check for the file size, if it's too big, then Google App Engine can't
    # even return it, so we need to display an error message.
    # We will also delete the blob, in order to save storage space.
    blob = fileprocessor.get_blob(blob_key)
    if blob.size > MAX_FILE_SIZE:
        fileprocessor.delete_file(blob_key)
        return json.dumps({'message' : "Upload must be smaller than 30MB."}), 400

    # Save the blob key in a datastore so we can order any information we need.
    # save_file returns the base58 datastore key
    data_id = fileprocessor.save_file(blob_key)

    # Return a formatted url of the location of the file
    address = request.host_url + data_id
    return json.dumps({'data_id': data_id})
Esempio n. 10
0
 def _set_charset(self, charset):
     header = self.headers.get("content-type")
     ct, options = parse_options_header(header)
     if not ct:
         raise TypeError("Cannot set charset if Content-Type " "header is missing.")
     options["charset"] = charset
     self.headers["Content-Type"] = dump_options_header(ct, options)
Esempio n. 11
0
    def parse(self, stream, content_type, content_length, context=None):
        """Parse the `stream` as a multipart encoded form.

        :param stream: the stream to be parsed.
        :param content_type: the content type of the request payload.
        :param content_length: the content length of the request payload.
        :param context: a dictionary containing extra context data
                        that can be useful for parsing.
        """
        if content_length is None:
            raise BadRequest('MultiPartParser.parse() requires '
                             '`content_length` argument')

        _, options = parse_options_header(content_type)
        boundary = options.get('boundary')
        if boundary is None:
            raise BadRequest('Multipart data missing boundary '
                             'in Content-Type header')
        boundary = boundary.encode('ascii')

        parser = WerkzeugMultiPartParser(default_stream_factory)
        try:
            form, files = parser.parse(stream, boundary, content_length)
            return form.to_dict(), files.to_dict()
        except ValueError:
            raise BadRequest('Multipart data is invalid')
Esempio n. 12
0
def manager_humans_create():
    form = HumansForm()
    upload_url = blobstore.create_upload_url('/manager/humans/create/')
    if request.method == 'POST':
        if form.validate_on_submit():
            blob_key = None
            f = request.files['photo']
            if f:
                header = f.headers['Content-Type']
                parsed_header = parse_options_header(header)
                blob_key = parsed_header[1]['blob-key']

            humans = Humans(
                text=form.text.data,
                q_month=form.q_month.data,
                photo=blob_key
            )

            db.session.add(humans)
            db.session.commit()

            flash(u'게시글을 작성하였습니다.', 'success')
            return redirect(url_for('humans_list'))

    return render_template('manager/create.html', form=form, upload_url=upload_url)
Esempio n. 13
0
def generate_formdata(req, resp, params):
    """sets prarams['form'] to pass to every endpoint.
    """
    #print "here"
    form = dict()
    files = dict()
    if req.method == 'GET':
        di = parse_query_string(req.query_string)
        form = dict(di)
        params['form'], params['files'] = dict(form), dict(files)
    else:
        if 'json' in req.get_header('content-type', None):
            form = json.load(req.stream)
            params['form'], params['files'] = dict(form), dict(files)
        else:
            mimetype, options = parse_options_header(req.get_header('content-type'))
            data = req.stream.read()
            environ = {'wsgi.input': StringIO(data),
                       'CONTENT_LENGTH': str(len(data)),
                       'CONTENT_TYPE': req.get_header('content-type'),
                       'REQUEST_METHOD': 'POST'}
            stream, tempform, tempfiles = parse_form_data(environ)
            for item in tempform:
                form[item] = tempform[item]
            di = parse_query_string(req.query_string)
            for item in di:
                form[item] = di[item]
            for item in tempfiles:
                files[item] = tempfiles[item]
            params['form'], params['files'] = dict(form), dict(files)
    return True
Esempio n. 14
0
    def fix_headers(self, environ, headers, status=None):
        if self.fix_vary:
            header = headers.get('content-type', '')
            mimetype, options = parse_options_header(header)
            if mimetype not in ('text/html', 'text/plain', 'text/sgml'):
                headers.pop('vary', None)

        if self.fix_attach and 'content-disposition' in headers:
            pragma = parse_set_header(headers.get('pragma', ''))
            pragma.discard('no-cache')
            header = pragma.to_header()
            if not header:
                headers.pop('pragma', '')
            else:
                headers['Pragma'] = header
            header = headers.get('cache-control', '')
            if header:
                cc = parse_cache_control_header(header,
                                                cls=ResponseCacheControl)
                cc.no_cache = None
                cc.no_store = False
                header = cc.to_header()
                if not header:
                    headers.pop('cache-control', '')
                else:
                    headers['Cache-Control'] = header
Esempio n. 15
0
def generate_formdata(req, resp, params):
    form = dict()
    files = dict()
    if req.method == 'GET':
        di = parse_query_string(req.query_string)
        form = dict(di)
        params['form'], params['files'] = dict(form), dict(files)
    else:
        if 'json' in req.get_header('content-type', None):
            #if the method type is post "form" variable below can contain 
            #only either the post body or the quer parameter at once, modify 
            #umcomment the below commented line to store query parameters in "form"
            form = json.load(req.stream)
            #form = dict(parse_query_string(req.query_string))
            params['form'], params['files'] = dict(form), dict(files)
        else:
            mimetype, options = parse_options_header(req.get_header('content-type'))
            data = req.stream.read()
            environ = {'wsgi.input':StringIO(data),
                       'CONTENT_LENGTH': str(len(data)),
                       'CONTENT_TYPE': req.get_header('content-type'),
                       'REQUEST_METHOD': 'POST'}
            stream, tempform, tempfiles = parse_form_data(environ)
            for item in tempform:
                form[item] = tempform[item]
            di = parse_query_string(req.query_string)
            for item in di:
                form[item] = di[item]
            for item in tempfiles:
                files[item] = tempfiles[item]
            params['form'], params['files'] = dict(form), dict(files)
    #print form
    #print params
    return True
Esempio n. 16
0
    def _get_mimetype_params(self):

        def on_update(d):
            self.headers['Content-Type'] = dump_options_header(self.mimetype, d)

        d = parse_options_header(self.headers.get('content-type', ''))[1]
        return CallbackDict(d, on_update)
Esempio n. 17
0
 def register(self, identity: Identity, public_key: PKey):
     title = get_key_fingerprint(public_key)
     data = json.dumps({
         'title': title,
         'key': format_openssh_pubkey(public_key)
     })
     try:
         request(identity, self.LIST_URL, 'POST', data=data.encode())
     except urllib.request.HTTPError as e:
         if e.code != 422:
             raise
         content_type = e.headers.get('Content-Type')
         mimetype, options = parse_options_header(content_type)
         if mimetype != 'application/json':
             raise
         charset = options.get('charset', 'utf-8')
         response = json.loads(e.read().decode(charset))
         for error in response.get('errors', []):
             if not isinstance(error, dict):
                 continue
             elif error.get('field') != 'key':
                 continue
             message = error.get('message', '').strip().lower()
             if message != 'key is already in use':
                 continue
             raise DuplicatePublicKeyError(message)
         raise
Esempio n. 18
0
 def _set_charset(self, charset):
     header = self.headers.get('content-type')
     ct, options = parse_options_header(header)
     if not ct:
         raise TypeError('Cannot set charset if Content-Type header is missing.')
     options['charset'] = charset
     self.headers['Content-Type'] = dump_options_header(ct, options)
Esempio n. 19
0
def extract_page(url, ctype, data):
    """Worker-thread procedure: produce an ExtractedContent object from
       URL, CTYPE, and DATA.  Note that we intentionally do not care
       whether CTYPE is actually text/html.
    """
    charset = parse_options_header(ctype)[1].get("charset", "")
    return html_extractor.ExtractedContent(url, data, charset)
Esempio n. 20
0
def decode_json(response):
    if not response.error:
        content_type = parse_options_header(
            response.headers.get('Content-Type', 'application/octet-stream'))
        assert content_type[0] == 'application/json'
        return json.loads(response.body)
    return None
Esempio n. 21
0
 def get_part_charset(self, headers):
     # Figure out input charset for current part
     content_type = headers.get('content-type')
     if content_type:
         mimetype, ct_params = parse_options_header(content_type)
         return ct_params.get('charset', self.charset)
     return self.charset
Esempio n. 22
0
 def _get_charset(self):
     header = self.headers.get('content-type')
     if header:
         charset = parse_options_header(header)[1].get('charset')
         if charset:
             return charset
     return self.default_charset
Esempio n. 23
0
    def fix_headers(self, environ, headers, status=None):
        if self.fix_vary:
            header = headers.get("content-type", "")
            mimetype, options = parse_options_header(header)
            if mimetype not in ("text/html", "text/plain", "text/sgml"):
                headers.pop("vary", None)

        if self.fix_attach and "content-disposition" in headers:
            pragma = parse_set_header(headers.get("pragma", ""))
            pragma.discard("no-cache")
            header = pragma.to_header()
            if not header:
                headers.pop("pragma", "")
            else:
                headers["Pragma"] = header
            header = headers.get("cache-control", "")
            if header:
                cc = parse_cache_control_header(header, cls=ResponseCacheControl)
                cc.no_cache = None
                cc.no_store = False
                header = cc.to_header()
                if not header:
                    headers.pop("cache-control", "")
                else:
                    headers["Cache-Control"] = header
Esempio n. 24
0
def edit_contact(key_str):
    form_data = bottle.request.forms
    form = ContactForm(form_data.decode())

    if form.validate():
        # Check that a photo was uploaded
        if 'photo' in bottle.request.files:
            photo = bottle.request.files['photo']
            parsed_ct = parse_options_header(photo.content_type)
            blob_key = parsed_ct[1]['blob-key']
            photo_url = get_serving_url(blob_key, size=32, crop=True)
        else:
            # Set to None so that we don't get any funny business
            # on the call to create_contact
            photo_url = None
            blob_key = None

        # Retrieve contact's Datastore key
        key = ndb.Key(urlsafe=key_str)

        # Delete the record
        key.delete()

        # Retrieve the user's Datastore key string and create a contact linked
        # to that key
        user_key_str = bottle.request.session['user_key_str']
        key = create_contact(form, user_key=ndb.Key(urlsafe=user_key_str),
                             photo_url=photo_url, blob_key=blob_key)

        return refresh_path('/')

    else:
        return get_contact(key_str, form=form, upload_url=blobstore.create_upload_url("/edit/" + key_str))
Esempio n. 25
0
 def post(self, acct):
     #TODO graceful failure
     args = self.parser.parse_args()
     image = args['image']
     headers =  parse_options_header(image.headers['Content-Type'])
     img = UploadedImage(img=headers[1]['blob-key'])
     img.put()
     return img.to_json()
Esempio n. 26
0
 def authenticate(
     self,
     state,
     requested_redirect_url: str,
     wsgi_environ: Mapping[str, object]
 ) -> Identity:
     logger = self.logger.getChild('authenticate')
     req = Request(wsgi_environ, populate_request=False, shallow=True)
     args = cast(ImmutableMultiDict, req.args)
     try:
         code = args['code']
         if args['state'] != state:
             raise AuthenticationError()
     except KeyError:
         raise AuthenticationError()
     data = url_encode({
         'client_id': self.client_id,
         'client_secret': self.client_secret,
         'code': code,
         'redirect_uri': requested_redirect_url,
         'grant_type': 'authorization_code',
     }).encode()
     try:
         response = urllib.request.urlopen(self.access_token_url, data)
     except urllib.error.HTTPError as e:
         logger.debug('Response of POST %s (with/ %r): %s\n%s',
                      self.access_token_url, data, e.code, e.read())
         raise
     assert isinstance(response, http.client.HTTPResponse), \
         'isinstance(response, {0.__module__}.{0.__qualname__})'.format(
             type(response))
     headers = getattr(response, 'headers')  # workaround mypy
     content_type = headers['Content-Type']
     mimetype, options = parse_options_header(content_type)
     if mimetype == 'application/x-www-form-urlencoded':
         token_data = url_decode_stream(response)
     elif mimetype == 'application/json':
         charset = options.get('charset', 'utf-8')
         token_data = json.load(
             io.TextIOWrapper(cast(IO[bytes], response), encoding=charset)
         )
     else:
         response.close()
         raise AuthenticationError(
             '{} sent unsupported content type: {}'.format(
                 self.access_token_url,
                 content_type
             )
         )
     response.close()
     identity = self.determine_identity(token_data['access_token'])
     if self.authorize(identity):
         return identity
     raise AuthenticationError(
         self.unauthorized_identity_message_format.format(
             identity=identity, team=self
         )
     )
Esempio n. 27
0
def upload():
    """
    Endpoint after GCS upload
    :return: JSON --> filelink, filename, thumb
    """
    if request.method == 'POST':
        request_file = request.files['file']

        # Creates the options for the file
        header = request_file.headers['Content-Type']
        parsed_header = parse_options_header(header)

        # IF everything is OK, save the file
        if request_file:
            try:
                blob_key = parsed_header[1]['blob-key']
                blob_info = blobstore.get(blob_key)
                blob_key_object = blob_info._BlobInfo__key
                img_url = '/admin/file_serve/'+blob_key
                img_gallery = img_url

                # Check if is image and save a reference
                if blob_info.content_type in IMAGES_MIME:
                    img = Image(image_data=blob_info.open().read())

                    if img.height > 1600 or img.width > 1600:
                        img_gallery = get_serving_url(blob_key_object,
                                                      size=1600)

                        # Landscape
                        if img.height < img.width:
                            img_height = (img.height*1600)/img.width
                            img_width = 1600

                        # Portrait
                        else:
                            img_width = (img.width*1600)/img.height
                            img_height = 1600
                    else:
                        img_height = img.height
                        img_width = img.width

                    img_ref = ImageReference(filename=blob_info.filename,
                                             blob=blob_key_object,
                                             url=img_url,
                                             thumb=get_serving_url(blob_key_object, crop=True, size=200),
                                             gallery=img_gallery,
                                             height=img_height,
                                             width=img_width)
                    img_ref.put()

                return jsonify({"filelink": "/admin/file_serve/" + blob_key,
                                "filegallery": img_gallery,
                                "filename": "" + blob_info.filename,
                                "thumb": get_serving_url(blob_key, crop=True, size=200)})
            except Exception as e:
                logging.exception(e)
                return jsonify({"error": e})
Esempio n. 28
0
 def test_parse_options_header(self):
     assert http.parse_options_header(r'something; foo="other\"thing"') == ("something", {"foo": 'other"thing'})
     assert http.parse_options_header(r'something; foo="other\"thing"; meh=42') == (
         "something",
         {"foo": 'other"thing', "meh": "42"},
     )
     assert http.parse_options_header(r'something; foo="other\"thing"; meh=42; bleh') == (
         "something",
         {"foo": 'other"thing', "meh": "42", "bleh": None},
     )
     assert http.parse_options_header('something; foo="other;thing"; meh=42; bleh') == (
         "something",
         {"foo": "other;thing", "meh": "42", "bleh": None},
     )
     assert http.parse_options_header('something; foo="otherthing"; meh=; bleh') == (
         "something",
         {"foo": "otherthing", "meh": None, "bleh": None},
     )
Esempio n. 29
0
def get_blob_key(field_name, blob_key=None):
    try:
        upload_file = request.files[field_name]
        header = upload_file.headers["Content-Type"]
        parsed_header = parse_options_header(header)
        blob_key = parsed_header[1]["blob-key"]
    except:
        return None
    return blob_key
Esempio n. 30
0
def request(access_token, url: str, method: str='GET', data: bytes=None):
    """Make a request to GitHub API, and then return the parsed JSON result.

    :param access_token: api access token string,
                         or :class:`~geofront.identity.Identity` instance
    :type access_token: :class:`str`, :class:`~geofront.identity.Identity`
    :param url: the api url to request
    :type url: :class:`str`
    :param method: an optional http method.  ``'GET'`` by default
    :type method: :class:`str`
    :param data: an optional content body
    :type data: :class:`bytes`

    """
    if isinstance(access_token, Identity):
        access_token = access_token.access_token
    req = urllib.request.Request(
        url,
        headers={
            'Authorization': 'token ' + access_token,
            'Accept': 'application/json'
        },
        method=method,
        data=data
    )
    with contextlib.closing(urllib.request.urlopen(req)) as response:
        content_type = response.headers.get('Content-Type')
        mimetype, options = parse_options_header(content_type)
        assert mimetype == 'application/json' or method == 'DELETE', \
            'Content-Type of {} is not application/json but {}'.format(
                url,
                content_type
            )
        charset = options.get('charset', 'utf-8')
        io_wrapper = io.TextIOWrapper(response, encoding=charset)
        logger = logging.getLogger(__name__ + '.request')
        if logger.isEnabledFor(logging.DEBUG):
            read = io_wrapper.read()
            logger.debug(
                'HTTP/%d.%d %d %s\n%s\n\n%s',
                response.version // 10,
                response.version % 10,
                response.status,
                response.reason,
                '\n'.join('{}: {}'.format(k, v)
                          for k, v in response.headers.items()),
                read
            )
            if method == 'DELETE':
                return
            return json.loads(read)
        else:
            if method == 'DELETE':
                io_wrapper.read()
                return
            return json.load(io_wrapper)
Esempio n. 31
0
def parse_response(resp, content, strict=False, content_type=None):
    """Parse the response returned by :meth:`OAuthRemoteApp.http_request`.

    :param resp: response of http_request
    :param content: content of the response
    :param strict: strict mode for form urlencoded content
    :param content_type: assign a content type manually
    """
    if not content_type:
        content_type = resp.headers.get('content-type', 'application/json')
    ct, options = parse_options_header(content_type)

    if ct in ('application/json', 'text/javascript'):
        if not content:
            return {}
        return json.loads(content)

    if ct in ('application/xml', 'text/xml'):
        return get_etree().fromstring(content)

    if ct != 'application/x-www-form-urlencoded' and strict:
        return content
    charset = options.get('charset', 'utf-8')
    return url_decode(content, charset=charset).to_dict()
def parse_headers(content_disposition, location=None):
    """Build a ContentDisposition from header values.
    :type content_disposition: unicode|None
    :type location: unicode|None
    :rtype: ContentDisposition
    """

    if content_disposition is None:
        return ContentDisposition(location=location)

    if not PY3K:
        # parse_options_header uses urllib.unquote which have different implementation
        # if unicode is passed. However, there could be proper unicode which is handled
        # correctly
        try:
            content_disposition = content_disposition.encode('ascii')
        except UnicodeEncodeError:
            pass
    disposition, params = parse_options_header(
        normalize_ws(content_disposition))

    return ContentDisposition(disposition=disposition,
                              assocs=params,
                              location=location)
    def _run_task(self, messages):
        self.status = self.PROCESSING
        self.save(update_fields=['status'])
        dest_item = has_necessary_perm = False

        if 'destination' in self.data and self.data['destination']:
            _d = self.data.get('destination')
            dest_item = _resolve_url_to_asset(_d)
            if not dest_item.has_perm(self.user, PERM_CHANGE_ASSET):
                raise exceptions.PermissionDenied('user cannot update asset')
            else:
                has_necessary_perm = True

        if 'url' in self.data:
            # Retrieve file name from URL
            self._load_assets_from_url(
                messages=messages,
                url=self.data.get('url'),
                destination=dest_item,
                has_necessary_perm=has_necessary_perm,
            )
            return

        # Get filename
        try:
            filename = self.data['filename']
        except KeyError:
            filename = None

        if 'single_xls_url' in self.data:
            # Retrieve file name from URL
            # TODO: merge with `url` handling above; currently kept separate
            # because `_load_assets_from_url()` uses complex logic to deal with
            # multiple XLS files in a directory structure within a ZIP archive
            response = requests.get(self.data['single_xls_url'])
            response.raise_for_status()
            encoded_xls = to_str(base64.b64encode(response.content))

            # if filename is empty or None, try to retrieve
            # file name from the response headers
            if not filename:
                filename_from_header = parse_options_header(
                    response.headers['Content-Disposition'])

                try:
                    filename = filename_from_header[1]['filename']
                except (TypeError, IndexError, KeyError):
                    pass

            self.data['base64Encoded'] = encoded_xls

        if 'base64Encoded' in self.data:
            # When a file is uploaded as base64,
            # no name is provided in the encoded string
            # We should rely on self.data.get(:filename:)

            self._parse_b64_upload(
                base64_encoded_upload=self.data['base64Encoded'],
                filename=filename,
                messages=messages,
                library=self.data.get('library', False),
                destination=dest_item,
                has_necessary_perm=has_necessary_perm,
            )
            return

        raise Exception(
            'ImportTask data must contain `base64Encoded`, `url`, or '
            '`single_xls_url`')
Esempio n. 34
0
 async def decode(self, receive, content_type):
     mime_type, mime_options = parse_options_header(content_type)
     boundary = mime_options.get('boundary')
     if not boundary:
         raise BadRequest('Missing boundary')
     return FileStream(receive, boundary.encode())
Esempio n. 35
0
 def test_parse_options_header(self):
     assert http.parse_options_header(None) == \
         ('', {})
     assert http.parse_options_header("") == \
         ('', {})
     assert http.parse_options_header(r'something; foo="other\"thing"') == \
         ('something', {'foo': 'other"thing'})
     assert http.parse_options_header(r'something; foo="other\"thing"; meh=42') == \
         ('something', {'foo': 'other"thing', 'meh': '42'})
     assert http.parse_options_header(r'something; foo="other\"thing"; meh=42; bleh') == \
         ('something', {'foo': 'other"thing', 'meh': '42', 'bleh': None})
     assert http.parse_options_header('something; foo="other;thing"; meh=42; bleh') == \
         ('something', {'foo': 'other;thing', 'meh': '42', 'bleh': None})
     assert http.parse_options_header('something; foo="otherthing"; meh=; bleh') == \
         ('something', {'foo': 'otherthing', 'meh': None, 'bleh': None})
     # Issue #404
     assert http.parse_options_header('multipart/form-data; name="foo bar"; '
                                      'filename="bar foo"') == \
         ('multipart/form-data', {'name': 'foo bar', 'filename': 'bar foo'})
     # Examples from RFC
     assert http.parse_options_header('audio/*; q=0.2, audio/basic') == \
         ('audio/*', {'q': '0.2'})
     assert http.parse_options_header('audio/*; q=0.2, audio/basic', multiple=True) == \
         ('audio/*', {'q': '0.2'}, "audio/basic", {})
     assert http.parse_options_header(
         'text/plain; q=0.5, text/html\n        '
         'text/x-dvi; q=0.8, text/x-c',
         multiple=True) == \
         ('text/plain', {'q': '0.5'}, "text/html", {},
          "text/x-dvi", {'q': '0.8'}, "text/x-c", {})
     assert http.parse_options_header('text/plain; q=0.5, text/html\n'
                                      '        '
                                      'text/x-dvi; q=0.8, text/x-c') == \
         ('text/plain', {'q': '0.5'})
     # Issue #932
     assert http.parse_options_header(
         'form-data; '
         'name="a_file"; '
         'filename*=UTF-8\'\''
         '"%c2%a3%20and%20%e2%82%ac%20rates"') == \
         ('form-data', {'name': 'a_file',
                        'filename': u'\xa3 and \u20ac rates'})
     assert http.parse_options_header(
         'form-data; '
         'name*=UTF-8\'\'"%C5%AAn%C4%ADc%C5%8Dde%CC%BD"; '
         'filename="some_file.txt"') == \
         ('form-data', {'name': u'\u016an\u012dc\u014dde\u033d',
                        'filename': 'some_file.txt'})
Esempio n. 36
0
 def is_batch_input(self) -> bool:
     hv = parse_options_header(self.get(BATCH_HEADER))[0].lower()
     return hv == "true" if hv else None
Esempio n. 37
0
 def charset(self) -> Optional[str]:
     return parse_options_header(self.get('content-type'))[1].get(
         'charset', None)
Esempio n. 38
0
    def update_deposit(self, record: SWORDDeposit, replace: bool = True):
        """Update a deposit according to the request data

        :param replace: If ``True``, all previous data on the deposit is removed. If ``False``, the request augments
            data already provided.
        """

        content_disposition, content_disposition_options = parse_options_header(
            request.headers.get("Content-Disposition", ""))

        metadata_deposit = content_disposition_options.get(
            "metadata") == "true"
        by_reference_deposit = content_disposition_options.get(
            "by-reference") == "true"

        if metadata_deposit:
            if by_reference_deposit:  # pragma: nocover
                record.set_metadata(
                    request.json["metadata"],
                    self.metadata_class,
                    replace=replace,
                )
            else:
                record.set_metadata(
                    request.stream,
                    self.metadata_class,
                    request.content_type,
                    replace=replace,
                )
        elif replace:
            record.set_metadata(None, self.metadata_class, replace=replace)

        if by_reference_deposit:
            by_reference_schema = ByReferenceSchema(context={
                "url_adapter":
                current_app.create_url_adapter(request)
            }, )
            if metadata_deposit:
                by_reference = by_reference_schema.load(
                    request.json["by-reference"])
            else:
                by_reference = by_reference_schema.load(request.json)
            record.set_by_reference_files(
                by_reference["files"],
                dereference_policy=self.endpoint_options["dereference_policy"],
                request_url=request.url,
                replace=replace,
            )
        elif replace:
            record.set_by_reference_files(
                [],
                dereference_policy=self.endpoint_options["dereference_policy"],
                request_url=request.url,
                replace=replace,
            )

        if not (metadata_deposit or by_reference_deposit) and (
                request.content_type or request.content_length):
            self.ingest_file(record, request.stream, replace=replace)
        elif replace:
            self.ingest_file(record, None, replace=replace)

        self.update_deposit_status(record)

        record.commit()
        db.session.commit()
Esempio n. 39
0
 def test_content_type(self, response):
     assert parse_options_header(response.headers['content-type']) \
            == ('text/csv', {'charset': 'UTF-8'})
Esempio n. 40
0
def validate_content_type():
    if 'Content-Type' not in request.headers:
        raise InvalidInputException("Expects Content-Type to be application/json")
    content_type = http.parse_options_header(request.headers['Content-Type'])[0]
    if content_type != 'application/json':
        raise InvalidInputException("Expects Content-Type to be application/json")
Esempio n. 41
0
def parse_form_data(environ,
                    stream_factory=None,
                    charset='utf-8',
                    errors='ignore',
                    max_form_memory_size=None,
                    max_content_length=None,
                    cls=None,
                    silent=True):
    """Parse the form data in the environ and return it as tuple in the form
    ``(stream, form, files)``.  You should only call this method if the
    transport method is `POST` or `PUT`.
    
    If the mimetype of the data transmitted is `multipart/form-data` the
    files multidict will be filled with `FileStorage` objects.  If the
    mimetype is unknown the input stream is wrapped and returned as first
    argument, else the stream is empty.
    
    This function does not raise exceptions, even if the input data is
    malformed.
    
    Have a look at :ref:`dealing-with-request-data` for more details.
    
    .. versionadded:: 0.5
       The `max_form_memory_size`, `max_content_length` and
       `cls` parameters were added.
    
    .. versionadded:: 0.5.1
       The optional `silent` flag was added.
    
    :param environ: the WSGI environment to be used for parsing.
    :param stream_factory: An optional callable that returns a new read and
                           writeable file descriptor.  This callable works
                           the same as :meth:`~BaseResponse._get_file_stream`.
    :param charset: The character set for URL and url encoded form data.
    :param errors: The encoding error behavior.
    :param max_form_memory_size: the maximum number of bytes to be accepted for
                           in-memory stored form data.  If the data
                           exceeds the value specified an
                           :exc:`~exceptions.RequestURITooLarge`
                           exception is raised.
    :param max_content_length: If this is provided and the transmitted data
                               is longer than this value an
                               :exc:`~exceptions.RequestEntityTooLarge`
                               exception is raised.
    :param cls: an optional dict class to use.  If this is not specified
                       or `None` the default :class:`MultiDict` is used.
    :param silent: If set to False parsing errors will not be caught.
    :return: A tuple in the form ``(stream, form, files)``.
    """
    content_type, extra = parse_options_header(environ.get('CONTENT_TYPE', ''))
    try:
        content_length = int(environ['CONTENT_LENGTH'])
    except (KeyError, ValueError):
        content_length = 0

    if cls is None:
        cls = MultiDict
    if max_content_length is not None and content_length > max_content_length:
        raise RequestEntityTooLarge()
    stream = _empty_stream
    files = ()
    if content_type == 'multipart/form-data':
        try:
            form, files = parse_multipart(
                environ['wsgi.input'],
                extra.get('boundary'),
                content_length,
                stream_factory,
                charset,
                errors,
                max_form_memory_size=max_form_memory_size)
        except ValueError as e:
            if not silent:
                raise
            form = cls()
        else:
            form = cls(form)

    elif content_type == 'application/x-www-form-urlencoded' or content_type == 'application/x-url-encoded':
        if max_form_memory_size is not None and content_length > max_form_memory_size:
            raise RequestEntityTooLarge()
        form = url_decode(environ['wsgi.input'].read(content_length),
                          charset,
                          errors=errors,
                          cls=cls)
    else:
        form = cls()
        stream = LimitedStream(environ['wsgi.input'], content_length)
    return (stream, form, cls(files))
from werkzeug.http import parse_options_header
import requests


def parse_content_type(content_type):
    if isinstance(content_type, (tuple, list)):
        content_type = content_type[0].lower()
    return content_type


url = 'https://boingboing.net/feed'
# url = 'http://qz.com/feed/atom/'

r = requests.get(url)

headers = r.headers.get('content-type')

print(headers)

parsed = parse_options_header(headers)
print(parsed)

print(type(parsed))

ct = parse_content_type(parsed)
print(ct)
print(type(ct))
Esempio n. 43
0
 def charset(self) -> Optional[str]:
     _, options = parse_options_header(self.get('content-type'))
     charset = options.get('charset', None)
     assert charset is None or isinstance(charset, str)
     return charset
Esempio n. 44
0
 def _parsed_mimetype(self):
     name, options = parse_options_header(self.content_type)
     return name, options
Esempio n. 45
0
def request(access_token, url: str, method: str = 'GET', data: bytes = None):
    """Make a request to GitHub API, and then return the parsed JSON result.

    :param access_token: api access token string,
                         or :class:`~geofront.identity.Identity` instance
    :type access_token: :class:`str`, :class:`~geofront.identity.Identity`
    :param url: the api url to request
    :type url: :class:`str`
    :param method: an optional http method.  ``'GET'`` by default
    :type method: :class:`str`
    :param data: an optional content body
    :type data: :class:`bytes`

    """
    logger = logging.getLogger(__name__ + '.request')
    if isinstance(access_token, Identity):
        access_token = access_token.access_token
    logger.debug('access_token: %r', access_token)
    req = urllib.request.Request(url,
                                 headers={
                                     'Authorization': 'Bearer ' + access_token,
                                     'Accept': 'application/json'
                                 },
                                 method=method,
                                 data=data)
    try:
        with contextlib.closing(urllib.request.urlopen(req)) as response:
            content_type = response.headers.get('Content-Type')
            mimetype, options = parse_options_header(content_type)
            assert mimetype == 'application/json' or method == 'DELETE', \
                'Content-Type of {} is not application/json but {}'.format(
                    url,
                    content_type
                )
            charset = options.get('charset', 'utf-8')
            io_wrapper = io.TextIOWrapper(response, encoding=charset)
            if logger.isEnabledFor(logging.DEBUG):
                read = io_wrapper.read()
                logger.debug(
                    'HTTP/%d.%d %d %s\n%s\n\n%s', response.version // 10,
                    response.version % 10, response.status, response.reason,
                    '\n'.join('{}: {}'.format(k, v)
                              for k, v in response.headers.items()), read)
                if method == 'DELETE':
                    return
                return json.loads(read)
            else:
                if method == 'DELETE':
                    io_wrapper.read()
                    return
                return json.load(io_wrapper)
    except urllib.error.HTTPError as e:
        if logger.isEnabledFor(logging.DEBUG):
            f = io.BytesIO()
            shutil.copyfileobj(e, f)
            logger.debug(
                'HTTP/%d.%d %d %s\n%s\n\n%r', e.version // 10, e.version % 10,
                e.status, e.reason,
                '\n'.join('{}: {}'.format(k, v) for k, v in e.headers.items()),
                f.getvalue())
            f.seek(0)
            logger.debug(str(e), exc_info=1)
            raise urllib.error.HTTPError(e.url, e.code, e.reason, e.headers,
                                         f) from e
        else:
            raise
Esempio n. 46
0
def parse_multipart(file,
                    boundary,
                    content_length,
                    stream_factory=None,
                    charset='utf-8',
                    errors='ignore',
                    buffer_size=10240,
                    max_form_memory_size=None):
    if stream_factory is None:
        stream_factory = default_stream_factory
    if not boundary:
        raise ValueError('Missing boundary')
    if not is_valid_multipart_boundary(boundary):
        raise ValueError('Invalid boundary: %s' % boundary)
    if len(boundary) > buffer_size:
        raise ValueError('Boundary longer than buffer size')
    total_content_length = content_length
    next_part = '--' + boundary
    last_part = next_part + '--'
    form = []
    files = []
    in_memory = 0
    file = LimitedStream(file, content_length)
    iterator = chain(make_line_iter(file, buffer_size=buffer_size),
                     _empty_string_iter)
    try:
        terminator = _find_terminator(iterator)
        if terminator != next_part:
            raise ValueError('Expected boundary at start of multipart data')
        while terminator != last_part:
            headers = parse_multipart_headers(iterator)
            disposition = headers.get('content-disposition')
            if disposition is None:
                raise ValueError('Missing Content-Disposition header')
            disposition, extra = parse_options_header(disposition)
            name = extra.get('name')
            transfer_encoding = headers.get('content-transfer-encoding')
            try_decode = transfer_encoding is not None and transfer_encoding in _supported_multipart_encodings
            filename = extra.get('filename')
            if filename is None:
                is_file = False
                container = []
                _write = container.append
                guard_memory = max_form_memory_size is not None
            else:
                content_type = headers.get('content-type')
                content_type = parse_options_header(
                    content_type)[0] or 'text/plain'
                is_file = True
                guard_memory = False
                if filename is not None:
                    filename = _fix_ie_filename(
                        _decode_unicode(filename, charset, errors))
                try:
                    content_length = int(headers['content-length'])
                except (KeyError, ValueError):
                    content_length = 0

                container = stream_factory(total_content_length, content_type,
                                           filename, content_length)
                _write = container.write
            buf = ''
            for line in iterator:
                if not line:
                    raise ValueError('unexpected end of stream')
                if line[:2] == '--':
                    terminator = line.rstrip()
                    if terminator in (next_part, last_part):
                        break
                if try_decode:
                    try:
                        line = line.decode(transfer_encoding)
                    except:
                        raise ValueError(
                            'could not decode transfer encoded chunk')

                if buf:
                    _write(buf)
                    buf = ''
                if line[-2:] == '\r\n':
                    buf = '\r\n'
                    cutoff = -2
                else:
                    buf = line[-1]
                    cutoff = -1
                _write(line[:cutoff])
                if guard_memory:
                    in_memory += len(line)
                    if in_memory > max_form_memory_size:
                        from werkzeug.exceptions import RequestEntityTooLarge
                        raise RequestEntityTooLarge()
            else:
                raise ValueError('unexpected end of part')

            if is_file:
                container.seek(0)
                files.append(
                    (name,
                     FileStorage(container, filename, name, content_type,
                                 content_length, headers)))
            else:
                form.append(
                    (name, _decode_unicode(''.join(container), charset,
                                           errors)))

    finally:
        file.exhaust()

    return (form, files)
Esempio n. 47
0
    def parse(self, file, boundary, content_length):
        next_part = '--' + boundary
        last_part = next_part + '--'

        form = []
        files = []
        in_memory = 0

        iterator = chain(make_line_iter(file, limit=content_length,
                                        buffer_size=self.buffer_size),
                         _empty_string_iter)

        terminator = self._find_terminator(iterator)
        if terminator != next_part:
            self.fail('Expected boundary at start of multipart data')

        while terminator != last_part:
            headers = parse_multipart_headers(iterator)

            disposition = headers.get('content-disposition')
            if disposition is None:
                self.fail('Missing Content-Disposition header')
            disposition, extra = parse_options_header(disposition)
            transfer_encoding = self.get_part_encoding(headers)
            name = extra.get('name')
            filename = extra.get('filename')
            part_charset = self.get_part_charset(headers)

            # if no content type is given we stream into memory.  A list is
            # used as a temporary container.
            if filename is None:
                is_file = False
                container = []
                _write = container.append
                guard_memory = self.max_form_memory_size is not None

            # otherwise we parse the rest of the headers and ask the stream
            # factory for something we can write in.
            else:
                is_file = True
                guard_memory = False
                filename, container = self.start_file_streaming(
                    filename, headers, content_length)
                _write = container.write

            buf = ''
            for line in iterator:
                if not line:
                    self.fail('unexpected end of stream')

                if line[:2] == '--':
                    terminator = line.rstrip()
                    if terminator in (next_part, last_part):
                        break

                if transfer_encoding is not None:
                    try:
                        line = line.decode(transfer_encoding)
                    except Exception:
                        self.fail('could not decode transfer encoded chunk')

                # we have something in the buffer from the last iteration.
                # this is usually a newline delimiter.
                if buf:
                    _write(buf)
                    buf = ''

                # If the line ends with windows CRLF we write everything except
                # the last two bytes.  In all other cases however we write
                # everything except the last byte.  If it was a newline, that's
                # fine, otherwise it does not matter because we will write it
                # the next iteration.  this ensures we do not write the
                # final newline into the stream.  That way we do not have to
                # truncate the stream.  However we do have to make sure that
                # if something else than a newline is in there we write it
                # out.
                if line[-2:] == '\r\n':
                    buf = '\r\n'
                    cutoff = -2
                else:
                    buf = line[-1]
                    cutoff = -1
                _write(line[:cutoff])

                # if we write into memory and there is a memory size limit we
                # count the number of bytes in memory and raise an exception if
                # there is too much data in memory.
                if guard_memory:
                    in_memory += len(line)
                    if in_memory > self.max_form_memory_size:
                        self.in_memory_threshold_reached(in_memory)
            else: # pragma: no cover
                raise ValueError('unexpected end of part')

            # if we have a leftover in the buffer that is not a newline
            # character we have to flush it, otherwise we will chop of
            # certain values.
            if buf not in ('', '\r', '\n', '\r\n'):
                _write(buf)

            if is_file:
                container.seek(0)
                files.append((name, FileStorage(container, filename, name,
                                                headers=headers)))
            else:
                form.append((name, _decode_unicode(''.join(container),
                                                   part_charset, self.errors)))

        return self.cls(form), self.cls(files)
Esempio n. 48
0
 def test_parse_options_header(self):
     assert http.parse_options_header(None) == ("", {})
     assert http.parse_options_header("") == ("", {})
     assert http.parse_options_header(r'something; foo="other\"thing"') == (
         "something",
         {
             "foo": 'other"thing'
         },
     )
     assert http.parse_options_header(
         r'something; foo="other\"thing"; meh=42') == (
             "something",
             {
                 "foo": 'other"thing',
                 "meh": "42"
             },
         )
     assert http.parse_options_header(
         r'something; foo="other\"thing"; meh=42; bleh') == ("something", {
             "foo": 'other"thing',
             "meh": "42",
             "bleh": None
         })
     assert http.parse_options_header(
         'something; foo="other;thing"; meh=42; bleh') == ("something", {
             "foo": "other;thing",
             "meh": "42",
             "bleh": None
         })
     assert http.parse_options_header(
         'something; foo="otherthing"; meh=; bleh') == (
             "something",
             {
                 "foo": "otherthing",
                 "meh": None,
                 "bleh": None
             },
         )
     # Issue #404
     assert http.parse_options_header(
         'multipart/form-data; name="foo bar"; filename="bar foo"') == (
             "multipart/form-data", {
                 "name": "foo bar",
                 "filename": "bar foo"
             })
     # Examples from RFC
     assert http.parse_options_header("audio/*; q=0.2, audio/basic") == (
         "audio/*",
         {
             "q": "0.2"
         },
     )
     assert http.parse_options_header("audio/*; q=0.2, audio/basic",
                                      multiple=True) == ("audio/*", {
                                          "q": "0.2"
                                      }, "audio/basic", {})
     assert http.parse_options_header(
         "text/plain; q=0.5, text/html\n        text/x-dvi; q=0.8, text/x-c",
         multiple=True,
     ) == (
         "text/plain",
         {
             "q": "0.5"
         },
         "text/html",
         {},
         "text/x-dvi",
         {
             "q": "0.8"
         },
         "text/x-c",
         {},
     )
     assert http.parse_options_header(
         "text/plain; q=0.5, text/html\n        text/x-dvi; q=0.8, text/x-c"
     ) == ("text/plain", {
         "q": "0.5"
     })
     # Issue #932
     assert http.parse_options_header(
         "form-data; name=\"a_file\"; filename*=UTF-8''"
         '"%c2%a3%20and%20%e2%82%ac%20rates"') == ("form-data", {
             "name":
             "a_file",
             "filename":
             "\xa3 and \u20ac rates"
         })
     assert http.parse_options_header(
         "form-data; name*=UTF-8''\"%C5%AAn%C4%ADc%C5%8Dde%CC%BD\"; "
         'filename="some_file.txt"') == (
             "form-data",
             {
                 "name": "\u016an\u012dc\u014dde\u033d",
                 "filename": "some_file.txt"
             },
         )
Esempio n. 49
0
def parse_link_header(link_header):
    href, options = parse_options_header(link_header)
    assert href.startswith('<')
    assert href.endswith('>')
    assert list(options.keys()) == ['rel']
    return href[1:-1], options['rel']
Esempio n. 50
0
 def test_parse_options_header(self):
     assert http.parse_options_header(None) == \
         ('', {})
     assert http.parse_options_header("") == \
         ('', {})
     assert http.parse_options_header(r'something; foo="other\"thing"') == \
         ('something', {'foo': 'other"thing'})
     assert http.parse_options_header(r'something; foo="other\"thing"; meh=42') == \
         ('something', {'foo': 'other"thing', 'meh': '42'})
     assert http.parse_options_header(r'something; foo="other\"thing"; meh=42; bleh') == \
         ('something', {'foo': 'other"thing', 'meh': '42', 'bleh': None})
     assert http.parse_options_header('something; foo="other;thing"; meh=42; bleh') == \
         ('something', {'foo': 'other;thing', 'meh': '42', 'bleh': None})
     assert http.parse_options_header('something; foo="otherthing"; meh=; bleh') == \
         ('something', {'foo': 'otherthing', 'meh': None, 'bleh': None})
     # Issue #404
     assert http.parse_options_header('multipart/form-data; name="foo bar"; filename="bar foo"') == \
         ('multipart/form-data', {'name': 'foo bar', 'filename': 'bar foo'})
     # Examples from RFC
     assert http.parse_options_header('audio/*; q=0.2, audio/basic') == \
         ('audio/*', {'q': '0.2'})
     assert http.parse_options_header('audio/*; q=0.2, audio/basic', multiple=True) == \
         ('audio/*', {'q': '0.2'}, "audio/basic", {})
     assert http.parse_options_header(
         'text/plain; q=0.5, text/html\n        '
         'text/x-dvi; q=0.8, text/x-c',
         multiple=True) == \
         ('text/plain', {'q': '0.5'}, "text/html", {},
          "text/x-dvi", {'q': '0.8'}, "text/x-c", {})
     assert http.parse_options_header('text/plain; q=0.5, text/html\n        text/x-dvi; q=0.8, text/x-c') == \
         ('text/plain', {'q': '0.5'})
Esempio n. 51
0
 def content_type(self) -> str:
     return parse_options_header(self.get('content-type'))[0].lower()
Esempio n. 52
0
    def process_request(self, request):
        # make sure these fields are always set
        request.grip = GripData()
        request.wscontext = None

        # old
        request.grip_proxied = False
        request.grip_signed = False

        proxied = False
        signed = False

        grip_sig_header = request.META.get('HTTP_GRIP_SIG')
        if grip_sig_header:
            proxies = _get_proxies()

            all_proxies_have_keys = True
            for entry in proxies:
                if 'key' not in entry:
                    all_proxies_have_keys = False
                    break

            if all_proxies_have_keys:
                # if all proxies have keys, then don't
                #   consider the request to be proxied unless
                #   one of them signed it
                for entry in proxies:
                    if validate_sig(grip_sig_header, entry['key']):
                        proxied = True
                        signed = True
                        break
            else:
                # if even one proxy doesn't have a key, then
                #   don't require verification in order to
                #   consider the request to have been proxied
                proxied = True

        # parse Grip-Feature
        hvalue = request.META.get('HTTP_GRIP_FEATURE')
        if hvalue:
            parsed = parse_options_header(hvalue, multiple=True)
            features = set()
            for n in range(0, len(parsed), 2):
                features.add(parsed[n])
            request.grip.features = features

        # parse Grip-Last
        hvalue = request.META.get('HTTP_GRIP_LAST')
        if hvalue:
            parsed = parse_options_header(hvalue, multiple=True)

            last = {}
            for n in range(0, len(parsed), 2):
                channel = parsed[n]
                params = parsed[n + 1]
                last[channel] = params.get('last-id', '')

            request.grip.last = last

        if hasattr(request, 'content_type'):
            content_type = request.content_type
        else:
            content_type = ''
            hvalue = request.META.get('CONTENT_TYPE')
            if hvalue:
                parsed = parse_options_header(hvalue)
                content_type = parsed[0]

        # detect WebSocket-Over-HTTP request
        wscontext = None
        if (request.method == 'POST'
                and content_type == 'application/websocket-events'):
            cid = request.META.get('HTTP_CONNECTION_ID')
            meta = {}
            for k, v in six.iteritems(request.META):
                if k.startswith('HTTP_META_'):
                    meta[_convert_header_name(k[10:])] = v
            body = request.body
            if isinstance(body, six.text_type):
                body = body.encode('utf-8')
            try:
                events = decode_websocket_events(body)
            except:
                return HttpResponseBadRequest(
                    'Error parsing WebSocket events.\n')

            wscontext = WebSocketContext(cid,
                                         meta,
                                         events,
                                         grip_prefix=_get_prefix())

        request.grip.proxied = proxied
        request.grip.signed = signed
        request.wscontext = wscontext

        # old
        request.grip_proxied = proxied
        request.grip_signed = signed
Esempio n. 53
0
 def content_encoding(self) -> str:
     return parse_options_header(self.get('content-encoding'))[0].lower()
Esempio n. 54
0
 def _parse_content_type(self):
     if not hasattr(self, '_parsed_content_type'):
         self._parsed_content_type = \
             parse_options_header(self.content_type)
Esempio n. 55
0
def aws_invoke(app,
               gateway_input,
               server_name='localhost',
               server_port='5000',
               http_protocol='HTTP/1.1',
               TLS=True,
               block_headers=True):
    headers = CaseInsensitiveDict(gateway_input.get('headers', {}))
    requestContext = gateway_input.get('requestContext')
    queryStringParameters = gateway_input.get('queryStringParameters', {})
    clientIp = headers.get('x-forwarded-for')
    if clientIp is None:
        clientIp = requestContext.get(
            'identity',
            {}).get('sourceIp') if requestContext is not None else ''
    else:
        clientIp = clientIp.split(',')[0]
    environ = {
        'REQUEST_METHOD':
        gateway_input.get('httpMethod', 'GET').upper(),
        'SCRIPT_NAME':
        '',
        'PATH_INFO':
        gateway_input.get('path', '/'),
        'QUERY_STRING':
        urlencode(queryStringParameters)
        if queryStringParameters is not None else '',
        'SERVER_NAME':
        headers.get('host', server_name),
        'SERVER_PORT':
        headers.get('x-forwarded-port', server_port),
        'SERVER_PROTOCOL':
        http_protocol,
        'SERVER_SOFTWARE':
        'flask-serverless',
        'REMOTE_ADDR':
        clientIp,
        'wsgi.version': (1, 0),
        'wsgi.url_scheme':
        headers.get('x-forwarded-proto', 'https' if TLS else 'http'),
        'wsgi.input':
        None,
        'wsgi.errors':
        sys.stderr,
        'wsgi.multiprocess':
        True,
        'wsgi.multithread':
        False,
        'wsgi.run_once':
        True,
        'HTTP_X_AWS_PATH':
        requestContext['path']
    }

    if environ['REQUEST_METHOD'] == 'POST' or environ[
            'REQUEST_METHOD'] == 'PUT':
        contentType = headers.get('content-type', 'application/octet-stream')
        parsedContentType = parse_options_header(contentType)
        raw = gateway_input.get('body')
        if raw is None or gateway_input.get('isBase64Encoded', False):
            body = b64decode(raw) if raw is not None else None
        else:
            body = raw.encode(parsedContentType[1].get('charset', 'utf-8'))
        add_body(environ, body, contentType)

    add_headers(environ, headers, block_headers)

    response = Response.from_app(app.wsgi_app, environ)

    gateway_output = {
        'headers': dict(response.headers),
        'statusCode': response.status_code,
    }

    compressed = response.headers.get('Content-Encoding') == 'gzip'

    responseType = parse_options_header(
        response.headers.get('Content-Type', 'application/octet-stream'))
    if not compressed and ('charset' in responseType[1] or responseType[0]
                           in textTypes or responseType[0][0:5] == 'text/'):
        gateway_output['body'] = response.data.decode(responseType[1].get(
            'charset', 'utf-8'))
        gateway_output['isBase64Encoded'] = False
    else:
        gateway_output['body'] = b64encode(response.data).decode('utf-8')
        gateway_output['isBase64Encoded'] = True

    return gateway_output
Esempio n. 56
0
File: base.py Progetto: ccns1/ccns11
 def mimetype(self) -> str:
     """Returns the mimetype parsed from the Content-Type header."""
     return parse_options_header(self.headers.get("Content-Type"))[0]
Esempio n. 57
0
    def parse_lines(self, file, boundary, content_length, cap_at_buffer=True):
        """Generate parts of
        ``('begin_form', (headers, name))``
        ``('begin_file', (headers, name, filename))``
        ``('cont', bytestring)``
        ``('end', None)``

        Always obeys the grammar
        parts = ( begin_form cont* end |
                  begin_file cont* end )*
        """
        next_part = b"--" + boundary
        last_part = next_part + b"--"

        iterator = chain(
            make_line_iter(
                file,
                limit=content_length,
                buffer_size=self.buffer_size,
                cap_at_buffer=cap_at_buffer,
            ),
            _empty_string_iter,
        )

        terminator = self._find_terminator(iterator)

        if terminator == last_part:
            return
        elif terminator != next_part:
            self.fail("Expected boundary at start of multipart data")

        while terminator != last_part:
            headers = parse_multipart_headers(iterator)

            disposition = headers.get("content-disposition")
            if disposition is None:
                self.fail("Missing Content-Disposition header")
            disposition, extra = parse_options_header(disposition)
            transfer_encoding = self.get_part_encoding(headers)
            name = extra.get("name")

            # Accept filename* to support non-ascii filenames as per rfc2231
            filename = extra.get("filename") or extra.get("filename*")

            # if no content type is given we stream into memory.  A list is
            # used as a temporary container.
            if filename is None:
                yield _begin_form, (headers, name)

            # otherwise we parse the rest of the headers and ask the stream
            # factory for something we can write in.
            else:
                yield _begin_file, (headers, name, filename)

            buf = b""
            for line in iterator:
                if not line:
                    self.fail("unexpected end of stream")

                if line[:2] == b"--":
                    terminator = line.rstrip()
                    if terminator in (next_part, last_part):
                        break

                if transfer_encoding is not None:
                    if transfer_encoding == "base64":
                        transfer_encoding = "base64_codec"
                    try:
                        line = codecs.decode(line, transfer_encoding)
                    except Exception:
                        self.fail("could not decode transfer encoded chunk")

                # we have something in the buffer from the last iteration.
                # this is usually a newline delimiter.
                if buf:
                    yield _cont, buf
                    buf = b""

                # If the line ends with windows CRLF we write everything except
                # the last two bytes.  In all other cases however we write
                # everything except the last byte.  If it was a newline, that's
                # fine, otherwise it does not matter because we will write it
                # the next iteration.  this ensures we do not write the
                # final newline into the stream.  That way we do not have to
                # truncate the stream.  However we do have to make sure that
                # if something else than a newline is in there we write it
                # out.
                if line[-2:] == b"\r\n":
                    buf = b"\r\n"
                    cutoff = -2
                else:
                    buf = line[-1:]
                    cutoff = -1
                yield _cont, line[:cutoff]

            else:  # pragma: no cover
                raise ValueError("unexpected end of part")

            # if we have a leftover in the buffer that is not a newline
            # character we have to flush it, otherwise we will chop of
            # certain values.
            if buf not in (b"", b"\r", b"\n", b"\r\n"):
                yield _cont, buf

            yield _end, None
Esempio n. 58
0
File: util.py Progetto: m42e/pb
def _content_type():
    content_type = http.parse_options_header(
        request.headers.get('Content-Type', ''))
    if content_type:
        content_type, _ = content_type
        return content_type
Esempio n. 59
0
def sparql(branch_or_ref):
    """Process a SPARQL query (Select or Update).

    Returns:
        HTTP Response with query result: If query was a valid select query.
        HTTP Response 200: If request contained a valid update query.
        HTTP Response 406: If accept header is not acceptable.
    """
    quit = current_app.config['quit']
    default_branch = quit.config.getDefaultBranch()

    if not branch_or_ref and not quit.repository.is_empty:
        branch_or_ref = default_branch

    logger.debug("Request method: {}".format(request.method))

    query = None
    if request.method == "GET":
        default_graph = request.args.get('default-graph-uri', None)
        named_graph = request.args.get('named-graph-uri', None)
        query = request.args.get('query', None)
    elif request.method == "POST":
        if 'Content-Type' in request.headers:
            contentMimeType, options = parse_options_header(
                request.headers['Content-Type'])
            if contentMimeType == "application/x-www-form-urlencoded":
                if 'query' in request.form:
                    default_graph = request.form.get('default-graph-uri', None)
                    named_graph = request.form.get('named-graph-uri', None)
                    query = request.form.get('query', None)
                elif 'update' in request.form:
                    default_graph = request.form.get('using-graph-uri', None)
                    named_graph = request.form.get('using-named-graph-uri',
                                                   None)
                    query = request.form.get('update', None)
            elif contentMimeType == "application/sparql-query":
                default_graph = request.args.get('default-graph-uri', None)
                named_graph = request.args.get('named-graph-uri', None)
                query = request.data.decode("utf-8")
            elif contentMimeType == "application/sparql-update":
                default_graph = request.args.get('using-graph-uri', None)
                named_graph = request.args.get('using-named-graph-uri', None)
                query = request.data.decode("utf-8")

    if 'Accept' in request.headers:
        logger.info('Received query via {}: {} with accept header: {}'.format(
            request.method, query, request.headers['Accept']))
        mimetype = parse_accept_header(request.headers['Accept']).best
    else:
        logger.info('Received query via {}: {} with no accept header.'.format(
            request.method, query))
        mimetype = '*/*'

    if query is None:
        if mimetype == 'text/html':
            return render_template('sparql.html', current_ref=branch_or_ref)
        else:
            return make_response(
                'No Query was specified or the Content-Type is not set according'
                + 'to the SPARQL 1.1 standard', 400)

    try:
        queryType, parsedQuery = parse_query_type(query, quit.config.namespace)
        graph = quit.instance(branch_or_ref)
    except UnSupportedQueryType as e:
        logger.exception(e)
        return make_response('Unsupported Query Type', 400)
    except Exception as e:
        logger.exception(e)
        return make_response('No branch or reference given.', 400)

    if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere']:
        res = graph.update(parsedQuery)

        try:
            ref = request.values.get('ref', None) or default_branch
            ref = 'refs/heads/{}'.format(ref)
            quit.commit(graph,
                        res,
                        'New Commit from QuitStore',
                        branch_or_ref,
                        ref,
                        query=query)
            return '', 200
        except Exception as e:
            # query ok, but unsupported query type or other problem during commit
            logger.exception(e)
            return make_response('Error after executing the update query.',
                                 400)
    elif queryType in [
            'SelectQuery', 'DescribeQuery', 'AskQuery', 'ConstructQuery'
    ]:
        res = graph.query(parsedQuery)
    else:
        logger.debug("Unsupported Type: {}".format(queryType))
        return make_response("Unsupported Query Type: {}".format(queryType),
                             400)

    try:
        if queryType in ['SelectQuery', 'AskQuery']:
            return create_result_response(res, resultSetMimetypes[mimetype])
        elif queryType in ['ConstructQuery', 'DescribeQuery']:
            return create_result_response(res, rdfMimetypes[mimetype])
    except KeyError as e:
        return make_response("Mimetype: {} not acceptable".format(mimetype),
                             406)
Esempio n. 60
0
 def _get_mimetype_and_options(headers: http.Headers) -> typing.Tuple[str, dict]:
     return parse_options_header(headers.get('Content-Type'))