def test_attachment(self):
app = flask.Flask(__name__)
with catch_warnings() as captured:
with app.test_request_context():
f = open(os.path.join(app.root_path, "static/index.html"))
rv = flask.send_file(f, as_attachment=True)
value, options = parse_options_header(rv.headers["Content-Disposition"])
self.assert_equal(value, "attachment")
rv.close()
# mimetypes + etag
self.assert_equal(len(captured), 2)
with app.test_request_context():
self.assert_equal(options["filename"], "index.html")
rv = flask.send_file("static/index.html", as_attachment=True)
value, options = parse_options_header(rv.headers["Content-Disposition"])
self.assert_equal(value, "attachment")
self.assert_equal(options["filename"], "index.html")
rv.close()
with app.test_request_context():
rv = flask.send_file(StringIO("Test"), as_attachment=True, attachment_filename="index.txt", add_etags=False)
self.assert_equal(rv.mimetype, "text/plain")
value, options = parse_options_header(rv.headers["Content-Disposition"])
self.assert_equal(value, "attachment")
self.assert_equal(options["filename"], "index.txt")
rv.close()
def test_attachment(self, catch_deprecation_warnings):
app = flask.Flask(__name__)
with catch_deprecation_warnings() as captured:
with app.test_request_context():
f = open(os.path.join(app.root_path, 'static/index.html'))
rv = flask.send_file(f, as_attachment=True)
value, options = parse_options_header(rv.headers['Content-Disposition'])
assert value == 'attachment'
rv.close()
# mimetypes + etag
assert len(captured) == 2
with app.test_request_context():
assert options['filename'] == 'index.html'
rv = flask.send_file('static/index.html', as_attachment=True)
value, options = parse_options_header(rv.headers['Content-Disposition'])
assert value == 'attachment'
assert options['filename'] == 'index.html'
rv.close()
with app.test_request_context():
rv = flask.send_file(StringIO('Test'), as_attachment=True,
attachment_filename='index.txt',
add_etags=False)
assert rv.mimetype == 'text/plain'
value, options = parse_options_header(rv.headers['Content-Disposition'])
assert value == 'attachment'
assert options['filename'] == 'index.txt'
rv.close()
def test_attachment(self, app, req_ctx):
with open(os.path.join(app.root_path, 'static/index.html')) as f:
rv = flask.send_file(f, as_attachment=True,
attachment_filename='index.html')
value, options = \
parse_options_header(rv.headers['Content-Disposition'])
assert value == 'attachment'
assert options['filename'] == 'index.html'
assert 'filename*' not in rv.headers['Content-Disposition']
rv.close()
rv = flask.send_file('static/index.html', as_attachment=True)
value, options = parse_options_header(rv.headers['Content-Disposition'])
assert value == 'attachment'
assert options['filename'] == 'index.html'
rv.close()
rv = flask.send_file(StringIO('Test'), as_attachment=True,
attachment_filename='index.txt',
add_etags=False)
assert rv.mimetype == 'text/plain'
value, options = parse_options_header(rv.headers['Content-Disposition'])
assert value == 'attachment'
assert options['filename'] == 'index.txt'
rv.close()
def test_parse_options_header(self):
assert http.parse_options_header('something; foo="other\"thing"') == \
('something', {'foo': 'other"thing'})
assert http.parse_options_header('something; foo="other\"thing"; meh=42') == \
('something', {'foo': 'other"thing', 'meh': '42'})
assert http.parse_options_header('something; foo="other\"thing"; meh=42; bleh') == \
('something', {'foo': 'other"thing', 'meh': '42', 'bleh': None})
def test_attachment(self):
app = flask.Flask(__name__)
with catch_warnings() as captured:
with app.test_request_context():
f = open(os.path.join(app.root_path, 'static/index.html'))
rv = flask.send_file(f, as_attachment=True)
value, options = parse_options_header(rv.headers['Content-Disposition'])
self.assert_equal(value, 'attachment')
# mimetypes + etag
self.assert_equal(len(captured), 2)
with app.test_request_context():
self.assert_equal(options['filename'], 'index.html')
rv = flask.send_file('static/index.html', as_attachment=True)
value, options = parse_options_header(rv.headers['Content-Disposition'])
self.assert_equal(value, 'attachment')
self.assert_equal(options['filename'], 'index.html')
with app.test_request_context():
rv = flask.send_file(StringIO('Test'), as_attachment=True,
attachment_filename='index.txt',
add_etags=False)
self.assert_equal(rv.mimetype, 'text/plain')
value, options = parse_options_header(rv.headers['Content-Disposition'])
self.assert_equal(value, 'attachment')
self.assert_equal(options['filename'], 'index.txt')
def test_parse_options_header_value_with_quotes(self):
assert http.parse_options_header(
'form-data; name="file"; filename="t\'es\'t.txt"'
) == ('form-data', {'name': 'file', 'filename': "t'es't.txt"})
assert http.parse_options_header(
'form-data; name="file"; filename*=UTF-8\'\'"\'🐍\'.txt"'
) == ('form-data', {'name': 'file', 'filename': u"'🐍'.txt"})
def test_parse_options_header_value_with_quotes(self):
assert http.parse_options_header(
'form-data; name="file"; filename="t\'es\'t.txt"'
) == ("form-data", {"name": "file", "filename": "t'es't.txt"})
assert http.parse_options_header(
"form-data; name=\"file\"; filename*=UTF-8''\"'🐍'.txt\""
) == ("form-data", {"name": "file", "filename": u"'🐍'.txt"})
def parse_form_data(environ, stream_factory = None, charset = 'utf-8', errors = 'ignore', max_form_memory_size = None, max_content_length = None, cls = None, silent = True):
content_type, extra = parse_options_header(environ.get('CONTENT_TYPE', ''))
try:
content_length = int(environ['CONTENT_LENGTH'])
except (KeyError, ValueError):
content_length = 0
if cls is None:
cls = MultiDict
if max_content_length is not None and content_length > max_content_length:
raise RequestEntityTooLarge()
stream = _empty_stream
files = ()
if content_type == 'multipart/form-data':
try:
form, files = parse_multipart(environ['wsgi.input'], extra.get('boundary'), content_length, stream_factory, charset, errors, max_form_memory_size=max_form_memory_size)
except ValueError as e:
if not silent:
raise
form = cls()
else:
form = cls(form)
elif content_type == 'application/x-www-form-urlencoded' or content_type == 'application/x-url-encoded':
if max_form_memory_size is not None and content_length > max_form_memory_size:
raise RequestEntityTooLarge()
form = url_decode(environ['wsgi.input'].read(content_length), charset, errors=errors, cls=cls)
else:
form = cls()
stream = LimitedStream(environ['wsgi.input'], content_length)
return (stream, form, cls(files))
def handle_upload():
"""Handles the result of the upload.
At this point in time, the file has already been uploaded by the Google App
Engine API. All we have to do is return some response to the end user. This
is why we never actually call something to save the file into a blobstore.
"""
# Verify that the file has actually been uploaded.
# 'filedata' is the id of the DOM element containing the file.
filedata = request.files['filedata']
if not filedata:
return json.dumps({'message' : "No file."}), 400
# The blob-key of the newly stored file is stored inside the header of the
# file.
header = filedata.headers['Content-Type']
parsed_header = parse_options_header(header)
blob_key = parsed_header[1]['blob-key']
# Check for the file size, if it's too big, then Google App Engine can't
# even return it, so we need to display an error message.
# We will also delete the blob, in order to save storage space.
blob = fileprocessor.get_blob(blob_key)
if blob.size > MAX_FILE_SIZE:
fileprocessor.delete_file(blob_key)
return json.dumps({'message' : "Upload must be smaller than 30MB."}), 400
# Save the blob key in a datastore so we can order any information we need.
# save_file returns the base58 datastore key
data_id = fileprocessor.save_file(blob_key)
# Return a formatted url of the location of the file
address = request.host_url + data_id
return json.dumps({'data_id': data_id})
def _set_charset(self, charset):
header = self.headers.get("content-type")
ct, options = parse_options_header(header)
if not ct:
raise TypeError("Cannot set charset if Content-Type " "header is missing.")
options["charset"] = charset
self.headers["Content-Type"] = dump_options_header(ct, options)
def parse(self, stream, content_type, content_length, context=None):
"""Parse the `stream` as a multipart encoded form.
:param stream: the stream to be parsed.
:param content_type: the content type of the request payload.
:param content_length: the content length of the request payload.
:param context: a dictionary containing extra context data
that can be useful for parsing.
"""
if content_length is None:
raise BadRequest('MultiPartParser.parse() requires '
'`content_length` argument')
_, options = parse_options_header(content_type)
boundary = options.get('boundary')
if boundary is None:
raise BadRequest('Multipart data missing boundary '
'in Content-Type header')
boundary = boundary.encode('ascii')
parser = WerkzeugMultiPartParser(default_stream_factory)
try:
form, files = parser.parse(stream, boundary, content_length)
return form.to_dict(), files.to_dict()
except ValueError:
raise BadRequest('Multipart data is invalid')
def manager_humans_create():
form = HumansForm()
upload_url = blobstore.create_upload_url('/manager/humans/create/')
if request.method == 'POST':
if form.validate_on_submit():
blob_key = None
f = request.files['photo']
if f:
header = f.headers['Content-Type']
parsed_header = parse_options_header(header)
blob_key = parsed_header[1]['blob-key']
humans = Humans(
text=form.text.data,
q_month=form.q_month.data,
photo=blob_key
)
db.session.add(humans)
db.session.commit()
flash(u'게시글을 작성하였습니다.', 'success')
return redirect(url_for('humans_list'))
return render_template('manager/create.html', form=form, upload_url=upload_url)
def generate_formdata(req, resp, params):
"""sets prarams['form'] to pass to every endpoint.
"""
#print "here"
form = dict()
files = dict()
if req.method == 'GET':
di = parse_query_string(req.query_string)
form = dict(di)
params['form'], params['files'] = dict(form), dict(files)
else:
if 'json' in req.get_header('content-type', None):
form = json.load(req.stream)
params['form'], params['files'] = dict(form), dict(files)
else:
mimetype, options = parse_options_header(req.get_header('content-type'))
data = req.stream.read()
environ = {'wsgi.input': StringIO(data),
'CONTENT_LENGTH': str(len(data)),
'CONTENT_TYPE': req.get_header('content-type'),
'REQUEST_METHOD': 'POST'}
stream, tempform, tempfiles = parse_form_data(environ)
for item in tempform:
form[item] = tempform[item]
di = parse_query_string(req.query_string)
for item in di:
form[item] = di[item]
for item in tempfiles:
files[item] = tempfiles[item]
params['form'], params['files'] = dict(form), dict(files)
return True
def fix_headers(self, environ, headers, status=None):
if self.fix_vary:
header = headers.get('content-type', '')
mimetype, options = parse_options_header(header)
if mimetype not in ('text/html', 'text/plain', 'text/sgml'):
headers.pop('vary', None)
if self.fix_attach and 'content-disposition' in headers:
pragma = parse_set_header(headers.get('pragma', ''))
pragma.discard('no-cache')
header = pragma.to_header()
if not header:
headers.pop('pragma', '')
else:
headers['Pragma'] = header
header = headers.get('cache-control', '')
if header:
cc = parse_cache_control_header(header,
cls=ResponseCacheControl)
cc.no_cache = None
cc.no_store = False
header = cc.to_header()
if not header:
headers.pop('cache-control', '')
else:
headers['Cache-Control'] = header
def generate_formdata(req, resp, params):
form = dict()
files = dict()
if req.method == 'GET':
di = parse_query_string(req.query_string)
form = dict(di)
params['form'], params['files'] = dict(form), dict(files)
else:
if 'json' in req.get_header('content-type', None):
#if the method type is post "form" variable below can contain
#only either the post body or the quer parameter at once, modify
#umcomment the below commented line to store query parameters in "form"
form = json.load(req.stream)
#form = dict(parse_query_string(req.query_string))
params['form'], params['files'] = dict(form), dict(files)
else:
mimetype, options = parse_options_header(req.get_header('content-type'))
data = req.stream.read()
environ = {'wsgi.input':StringIO(data),
'CONTENT_LENGTH': str(len(data)),
'CONTENT_TYPE': req.get_header('content-type'),
'REQUEST_METHOD': 'POST'}
stream, tempform, tempfiles = parse_form_data(environ)
for item in tempform:
form[item] = tempform[item]
di = parse_query_string(req.query_string)
for item in di:
form[item] = di[item]
for item in tempfiles:
files[item] = tempfiles[item]
params['form'], params['files'] = dict(form), dict(files)
#print form
#print params
return True
def _get_mimetype_params(self):
def on_update(d):
self.headers['Content-Type'] = dump_options_header(self.mimetype, d)
d = parse_options_header(self.headers.get('content-type', ''))[1]
return CallbackDict(d, on_update)
def register(self, identity: Identity, public_key: PKey):
title = get_key_fingerprint(public_key)
data = json.dumps({
'title': title,
'key': format_openssh_pubkey(public_key)
})
try:
request(identity, self.LIST_URL, 'POST', data=data.encode())
except urllib.request.HTTPError as e:
if e.code != 422:
raise
content_type = e.headers.get('Content-Type')
mimetype, options = parse_options_header(content_type)
if mimetype != 'application/json':
raise
charset = options.get('charset', 'utf-8')
response = json.loads(e.read().decode(charset))
for error in response.get('errors', []):
if not isinstance(error, dict):
continue
elif error.get('field') != 'key':
continue
message = error.get('message', '').strip().lower()
if message != 'key is already in use':
continue
raise DuplicatePublicKeyError(message)
raise
def _set_charset(self, charset):
header = self.headers.get('content-type')
ct, options = parse_options_header(header)
if not ct:
raise TypeError('Cannot set charset if Content-Type header is missing.')
options['charset'] = charset
self.headers['Content-Type'] = dump_options_header(ct, options)
def extract_page(url, ctype, data):
"""Worker-thread procedure: produce an ExtractedContent object from
URL, CTYPE, and DATA. Note that we intentionally do not care
whether CTYPE is actually text/html.
"""
charset = parse_options_header(ctype)[1].get("charset", "")
return html_extractor.ExtractedContent(url, data, charset)
def decode_json(response):
if not response.error:
content_type = parse_options_header(
response.headers.get('Content-Type', 'application/octet-stream'))
assert content_type[0] == 'application/json'
return json.loads(response.body)
return None
def get_part_charset(self, headers):
# Figure out input charset for current part
content_type = headers.get('content-type')
if content_type:
mimetype, ct_params = parse_options_header(content_type)
return ct_params.get('charset', self.charset)
return self.charset
def _get_charset(self):
header = self.headers.get('content-type')
if header:
charset = parse_options_header(header)[1].get('charset')
if charset:
return charset
return self.default_charset
def fix_headers(self, environ, headers, status=None):
if self.fix_vary:
header = headers.get("content-type", "")
mimetype, options = parse_options_header(header)
if mimetype not in ("text/html", "text/plain", "text/sgml"):
headers.pop("vary", None)
if self.fix_attach and "content-disposition" in headers:
pragma = parse_set_header(headers.get("pragma", ""))
pragma.discard("no-cache")
header = pragma.to_header()
if not header:
headers.pop("pragma", "")
else:
headers["Pragma"] = header
header = headers.get("cache-control", "")
if header:
cc = parse_cache_control_header(header, cls=ResponseCacheControl)
cc.no_cache = None
cc.no_store = False
header = cc.to_header()
if not header:
headers.pop("cache-control", "")
else:
headers["Cache-Control"] = header
def edit_contact(key_str):
form_data = bottle.request.forms
form = ContactForm(form_data.decode())
if form.validate():
# Check that a photo was uploaded
if 'photo' in bottle.request.files:
photo = bottle.request.files['photo']
parsed_ct = parse_options_header(photo.content_type)
blob_key = parsed_ct[1]['blob-key']
photo_url = get_serving_url(blob_key, size=32, crop=True)
else:
# Set to None so that we don't get any funny business
# on the call to create_contact
photo_url = None
blob_key = None
# Retrieve contact's Datastore key
key = ndb.Key(urlsafe=key_str)
# Delete the record
key.delete()
# Retrieve the user's Datastore key string and create a contact linked
# to that key
user_key_str = bottle.request.session['user_key_str']
key = create_contact(form, user_key=ndb.Key(urlsafe=user_key_str),
photo_url=photo_url, blob_key=blob_key)
return refresh_path('/')
else:
return get_contact(key_str, form=form, upload_url=blobstore.create_upload_url("/edit/" + key_str))
def post(self, acct):
#TODO graceful failure
args = self.parser.parse_args()
image = args['image']
headers = parse_options_header(image.headers['Content-Type'])
img = UploadedImage(img=headers[1]['blob-key'])
img.put()
return img.to_json()
def authenticate(
self,
state,
requested_redirect_url: str,
wsgi_environ: Mapping[str, object]
) -> Identity:
logger = self.logger.getChild('authenticate')
req = Request(wsgi_environ, populate_request=False, shallow=True)
args = cast(ImmutableMultiDict, req.args)
try:
code = args['code']
if args['state'] != state:
raise AuthenticationError()
except KeyError:
raise AuthenticationError()
data = url_encode({
'client_id': self.client_id,
'client_secret': self.client_secret,
'code': code,
'redirect_uri': requested_redirect_url,
'grant_type': 'authorization_code',
}).encode()
try:
response = urllib.request.urlopen(self.access_token_url, data)
except urllib.error.HTTPError as e:
logger.debug('Response of POST %s (with/ %r): %s\n%s',
self.access_token_url, data, e.code, e.read())
raise
assert isinstance(response, http.client.HTTPResponse), \
'isinstance(response, {0.__module__}.{0.__qualname__})'.format(
type(response))
headers = getattr(response, 'headers') # workaround mypy
content_type = headers['Content-Type']
mimetype, options = parse_options_header(content_type)
if mimetype == 'application/x-www-form-urlencoded':
token_data = url_decode_stream(response)
elif mimetype == 'application/json':
charset = options.get('charset', 'utf-8')
token_data = json.load(
io.TextIOWrapper(cast(IO[bytes], response), encoding=charset)
)
else:
response.close()
raise AuthenticationError(
'{} sent unsupported content type: {}'.format(
self.access_token_url,
content_type
)
)
response.close()
identity = self.determine_identity(token_data['access_token'])
if self.authorize(identity):
return identity
raise AuthenticationError(
self.unauthorized_identity_message_format.format(
identity=identity, team=self
)
)
def upload():
"""
Endpoint after GCS upload
:return: JSON --> filelink, filename, thumb
"""
if request.method == 'POST':
request_file = request.files['file']
# Creates the options for the file
header = request_file.headers['Content-Type']
parsed_header = parse_options_header(header)
# IF everything is OK, save the file
if request_file:
try:
blob_key = parsed_header[1]['blob-key']
blob_info = blobstore.get(blob_key)
blob_key_object = blob_info._BlobInfo__key
img_url = '/admin/file_serve/'+blob_key
img_gallery = img_url
# Check if is image and save a reference
if blob_info.content_type in IMAGES_MIME:
img = Image(image_data=blob_info.open().read())
if img.height > 1600 or img.width > 1600:
img_gallery = get_serving_url(blob_key_object,
size=1600)
# Landscape
if img.height < img.width:
img_height = (img.height*1600)/img.width
img_width = 1600
# Portrait
else:
img_width = (img.width*1600)/img.height
img_height = 1600
else:
img_height = img.height
img_width = img.width
img_ref = ImageReference(filename=blob_info.filename,
blob=blob_key_object,
url=img_url,
thumb=get_serving_url(blob_key_object, crop=True, size=200),
gallery=img_gallery,
height=img_height,
width=img_width)
img_ref.put()
return jsonify({"filelink": "/admin/file_serve/" + blob_key,
"filegallery": img_gallery,
"filename": "" + blob_info.filename,
"thumb": get_serving_url(blob_key, crop=True, size=200)})
except Exception as e:
logging.exception(e)
return jsonify({"error": e})
def test_parse_options_header(self):
assert http.parse_options_header(r'something; foo="other\"thing"') == ("something", {"foo": 'other"thing'})
assert http.parse_options_header(r'something; foo="other\"thing"; meh=42') == (
"something",
{"foo": 'other"thing', "meh": "42"},
)
assert http.parse_options_header(r'something; foo="other\"thing"; meh=42; bleh') == (
"something",
{"foo": 'other"thing', "meh": "42", "bleh": None},
)
assert http.parse_options_header('something; foo="other;thing"; meh=42; bleh') == (
"something",
{"foo": "other;thing", "meh": "42", "bleh": None},
)
assert http.parse_options_header('something; foo="otherthing"; meh=; bleh') == (
"something",
{"foo": "otherthing", "meh": None, "bleh": None},
)
def get_blob_key(field_name, blob_key=None):
try:
upload_file = request.files[field_name]
header = upload_file.headers["Content-Type"]
parsed_header = parse_options_header(header)
blob_key = parsed_header[1]["blob-key"]
except:
return None
return blob_key
def request(access_token, url: str, method: str='GET', data: bytes=None):
"""Make a request to GitHub API, and then return the parsed JSON result.
:param access_token: api access token string,
or :class:`~geofront.identity.Identity` instance
:type access_token: :class:`str`, :class:`~geofront.identity.Identity`
:param url: the api url to request
:type url: :class:`str`
:param method: an optional http method. ``'GET'`` by default
:type method: :class:`str`
:param data: an optional content body
:type data: :class:`bytes`
"""
if isinstance(access_token, Identity):
access_token = access_token.access_token
req = urllib.request.Request(
url,
headers={
'Authorization': 'token ' + access_token,
'Accept': 'application/json'
},
method=method,
data=data
)
with contextlib.closing(urllib.request.urlopen(req)) as response:
content_type = response.headers.get('Content-Type')
mimetype, options = parse_options_header(content_type)
assert mimetype == 'application/json' or method == 'DELETE', \
'Content-Type of {} is not application/json but {}'.format(
url,
content_type
)
charset = options.get('charset', 'utf-8')
io_wrapper = io.TextIOWrapper(response, encoding=charset)
logger = logging.getLogger(__name__ + '.request')
if logger.isEnabledFor(logging.DEBUG):
read = io_wrapper.read()
logger.debug(
'HTTP/%d.%d %d %s\n%s\n\n%s',
response.version // 10,
response.version % 10,
response.status,
response.reason,
'\n'.join('{}: {}'.format(k, v)
for k, v in response.headers.items()),
read
)
if method == 'DELETE':
return
return json.loads(read)
else:
if method == 'DELETE':
io_wrapper.read()
return
return json.load(io_wrapper)
def parse_response(resp, content, strict=False, content_type=None):
"""Parse the response returned by :meth:`OAuthRemoteApp.http_request`.
:param resp: response of http_request
:param content: content of the response
:param strict: strict mode for form urlencoded content
:param content_type: assign a content type manually
"""
if not content_type:
content_type = resp.headers.get('content-type', 'application/json')
ct, options = parse_options_header(content_type)
if ct in ('application/json', 'text/javascript'):
if not content:
return {}
return json.loads(content)
if ct in ('application/xml', 'text/xml'):
return get_etree().fromstring(content)
if ct != 'application/x-www-form-urlencoded' and strict:
return content
charset = options.get('charset', 'utf-8')
return url_decode(content, charset=charset).to_dict()
def parse_headers(content_disposition, location=None):
"""Build a ContentDisposition from header values.
:type content_disposition: unicode|None
:type location: unicode|None
:rtype: ContentDisposition
"""
if content_disposition is None:
return ContentDisposition(location=location)
if not PY3K:
# parse_options_header uses urllib.unquote which have different implementation
# if unicode is passed. However, there could be proper unicode which is handled
# correctly
try:
content_disposition = content_disposition.encode('ascii')
except UnicodeEncodeError:
pass
disposition, params = parse_options_header(
normalize_ws(content_disposition))
return ContentDisposition(disposition=disposition,
assocs=params,
location=location)
def _run_task(self, messages):
self.status = self.PROCESSING
self.save(update_fields=['status'])
dest_item = has_necessary_perm = False
if 'destination' in self.data and self.data['destination']:
_d = self.data.get('destination')
dest_item = _resolve_url_to_asset(_d)
if not dest_item.has_perm(self.user, PERM_CHANGE_ASSET):
raise exceptions.PermissionDenied('user cannot update asset')
else:
has_necessary_perm = True
if 'url' in self.data:
# Retrieve file name from URL
self._load_assets_from_url(
messages=messages,
url=self.data.get('url'),
destination=dest_item,
has_necessary_perm=has_necessary_perm,
)
return
# Get filename
try:
filename = self.data['filename']
except KeyError:
filename = None
if 'single_xls_url' in self.data:
# Retrieve file name from URL
# TODO: merge with `url` handling above; currently kept separate
# because `_load_assets_from_url()` uses complex logic to deal with
# multiple XLS files in a directory structure within a ZIP archive
response = requests.get(self.data['single_xls_url'])
response.raise_for_status()
encoded_xls = to_str(base64.b64encode(response.content))
# if filename is empty or None, try to retrieve
# file name from the response headers
if not filename:
filename_from_header = parse_options_header(
response.headers['Content-Disposition'])
try:
filename = filename_from_header[1]['filename']
except (TypeError, IndexError, KeyError):
pass
self.data['base64Encoded'] = encoded_xls
if 'base64Encoded' in self.data:
# When a file is uploaded as base64,
# no name is provided in the encoded string
# We should rely on self.data.get(:filename:)
self._parse_b64_upload(
base64_encoded_upload=self.data['base64Encoded'],
filename=filename,
messages=messages,
library=self.data.get('library', False),
destination=dest_item,
has_necessary_perm=has_necessary_perm,
)
return
raise Exception(
'ImportTask data must contain `base64Encoded`, `url`, or '
'`single_xls_url`')
async def decode(self, receive, content_type):
mime_type, mime_options = parse_options_header(content_type)
boundary = mime_options.get('boundary')
if not boundary:
raise BadRequest('Missing boundary')
return FileStream(receive, boundary.encode())
def test_parse_options_header(self):
assert http.parse_options_header(None) == \
('', {})
assert http.parse_options_header("") == \
('', {})
assert http.parse_options_header(r'something; foo="other\"thing"') == \
('something', {'foo': 'other"thing'})
assert http.parse_options_header(r'something; foo="other\"thing"; meh=42') == \
('something', {'foo': 'other"thing', 'meh': '42'})
assert http.parse_options_header(r'something; foo="other\"thing"; meh=42; bleh') == \
('something', {'foo': 'other"thing', 'meh': '42', 'bleh': None})
assert http.parse_options_header('something; foo="other;thing"; meh=42; bleh') == \
('something', {'foo': 'other;thing', 'meh': '42', 'bleh': None})
assert http.parse_options_header('something; foo="otherthing"; meh=; bleh') == \
('something', {'foo': 'otherthing', 'meh': None, 'bleh': None})
# Issue #404
assert http.parse_options_header('multipart/form-data; name="foo bar"; '
'filename="bar foo"') == \
('multipart/form-data', {'name': 'foo bar', 'filename': 'bar foo'})
# Examples from RFC
assert http.parse_options_header('audio/*; q=0.2, audio/basic') == \
('audio/*', {'q': '0.2'})
assert http.parse_options_header('audio/*; q=0.2, audio/basic', multiple=True) == \
('audio/*', {'q': '0.2'}, "audio/basic", {})
assert http.parse_options_header(
'text/plain; q=0.5, text/html\n '
'text/x-dvi; q=0.8, text/x-c',
multiple=True) == \
('text/plain', {'q': '0.5'}, "text/html", {},
"text/x-dvi", {'q': '0.8'}, "text/x-c", {})
assert http.parse_options_header('text/plain; q=0.5, text/html\n'
' '
'text/x-dvi; q=0.8, text/x-c') == \
('text/plain', {'q': '0.5'})
# Issue #932
assert http.parse_options_header(
'form-data; '
'name="a_file"; '
'filename*=UTF-8\'\''
'"%c2%a3%20and%20%e2%82%ac%20rates"') == \
('form-data', {'name': 'a_file',
'filename': u'\xa3 and \u20ac rates'})
assert http.parse_options_header(
'form-data; '
'name*=UTF-8\'\'"%C5%AAn%C4%ADc%C5%8Dde%CC%BD"; '
'filename="some_file.txt"') == \
('form-data', {'name': u'\u016an\u012dc\u014dde\u033d',
'filename': 'some_file.txt'})
def is_batch_input(self) -> bool:
hv = parse_options_header(self.get(BATCH_HEADER))[0].lower()
return hv == "true" if hv else None
def charset(self) -> Optional[str]:
return parse_options_header(self.get('content-type'))[1].get(
'charset', None)
def update_deposit(self, record: SWORDDeposit, replace: bool = True):
"""Update a deposit according to the request data
:param replace: If ``True``, all previous data on the deposit is removed. If ``False``, the request augments
data already provided.
"""
content_disposition, content_disposition_options = parse_options_header(
request.headers.get("Content-Disposition", ""))
metadata_deposit = content_disposition_options.get(
"metadata") == "true"
by_reference_deposit = content_disposition_options.get(
"by-reference") == "true"
if metadata_deposit:
if by_reference_deposit: # pragma: nocover
record.set_metadata(
request.json["metadata"],
self.metadata_class,
replace=replace,
)
else:
record.set_metadata(
request.stream,
self.metadata_class,
request.content_type,
replace=replace,
)
elif replace:
record.set_metadata(None, self.metadata_class, replace=replace)
if by_reference_deposit:
by_reference_schema = ByReferenceSchema(context={
"url_adapter":
current_app.create_url_adapter(request)
}, )
if metadata_deposit:
by_reference = by_reference_schema.load(
request.json["by-reference"])
else:
by_reference = by_reference_schema.load(request.json)
record.set_by_reference_files(
by_reference["files"],
dereference_policy=self.endpoint_options["dereference_policy"],
request_url=request.url,
replace=replace,
)
elif replace:
record.set_by_reference_files(
[],
dereference_policy=self.endpoint_options["dereference_policy"],
request_url=request.url,
replace=replace,
)
if not (metadata_deposit or by_reference_deposit) and (
request.content_type or request.content_length):
self.ingest_file(record, request.stream, replace=replace)
elif replace:
self.ingest_file(record, None, replace=replace)
self.update_deposit_status(record)
record.commit()
db.session.commit()
def test_content_type(self, response):
assert parse_options_header(response.headers['content-type']) \
== ('text/csv', {'charset': 'UTF-8'})
def validate_content_type():
if 'Content-Type' not in request.headers:
raise InvalidInputException("Expects Content-Type to be application/json")
content_type = http.parse_options_header(request.headers['Content-Type'])[0]
if content_type != 'application/json':
raise InvalidInputException("Expects Content-Type to be application/json")
def parse_form_data(environ,
stream_factory=None,
charset='utf-8',
errors='ignore',
max_form_memory_size=None,
max_content_length=None,
cls=None,
silent=True):
"""Parse the form data in the environ and return it as tuple in the form
``(stream, form, files)``. You should only call this method if the
transport method is `POST` or `PUT`.
If the mimetype of the data transmitted is `multipart/form-data` the
files multidict will be filled with `FileStorage` objects. If the
mimetype is unknown the input stream is wrapped and returned as first
argument, else the stream is empty.
This function does not raise exceptions, even if the input data is
malformed.
Have a look at :ref:`dealing-with-request-data` for more details.
.. versionadded:: 0.5
The `max_form_memory_size`, `max_content_length` and
`cls` parameters were added.
.. versionadded:: 0.5.1
The optional `silent` flag was added.
:param environ: the WSGI environment to be used for parsing.
:param stream_factory: An optional callable that returns a new read and
writeable file descriptor. This callable works
the same as :meth:`~BaseResponse._get_file_stream`.
:param charset: The character set for URL and url encoded form data.
:param errors: The encoding error behavior.
:param max_form_memory_size: the maximum number of bytes to be accepted for
in-memory stored form data. If the data
exceeds the value specified an
:exc:`~exceptions.RequestURITooLarge`
exception is raised.
:param max_content_length: If this is provided and the transmitted data
is longer than this value an
:exc:`~exceptions.RequestEntityTooLarge`
exception is raised.
:param cls: an optional dict class to use. If this is not specified
or `None` the default :class:`MultiDict` is used.
:param silent: If set to False parsing errors will not be caught.
:return: A tuple in the form ``(stream, form, files)``.
"""
content_type, extra = parse_options_header(environ.get('CONTENT_TYPE', ''))
try:
content_length = int(environ['CONTENT_LENGTH'])
except (KeyError, ValueError):
content_length = 0
if cls is None:
cls = MultiDict
if max_content_length is not None and content_length > max_content_length:
raise RequestEntityTooLarge()
stream = _empty_stream
files = ()
if content_type == 'multipart/form-data':
try:
form, files = parse_multipart(
environ['wsgi.input'],
extra.get('boundary'),
content_length,
stream_factory,
charset,
errors,
max_form_memory_size=max_form_memory_size)
except ValueError as e:
if not silent:
raise
form = cls()
else:
form = cls(form)
elif content_type == 'application/x-www-form-urlencoded' or content_type == 'application/x-url-encoded':
if max_form_memory_size is not None and content_length > max_form_memory_size:
raise RequestEntityTooLarge()
form = url_decode(environ['wsgi.input'].read(content_length),
charset,
errors=errors,
cls=cls)
else:
form = cls()
stream = LimitedStream(environ['wsgi.input'], content_length)
return (stream, form, cls(files))
from werkzeug.http import parse_options_header
import requests
def parse_content_type(content_type):
if isinstance(content_type, (tuple, list)):
content_type = content_type[0].lower()
return content_type
url = 'https://boingboing.net/feed'
# url = 'http://qz.com/feed/atom/'
r = requests.get(url)
headers = r.headers.get('content-type')
print(headers)
parsed = parse_options_header(headers)
print(parsed)
print(type(parsed))
ct = parse_content_type(parsed)
print(ct)
print(type(ct))
def charset(self) -> Optional[str]:
_, options = parse_options_header(self.get('content-type'))
charset = options.get('charset', None)
assert charset is None or isinstance(charset, str)
return charset
def _parsed_mimetype(self):
name, options = parse_options_header(self.content_type)
return name, options
def request(access_token, url: str, method: str = 'GET', data: bytes = None):
"""Make a request to GitHub API, and then return the parsed JSON result.
:param access_token: api access token string,
or :class:`~geofront.identity.Identity` instance
:type access_token: :class:`str`, :class:`~geofront.identity.Identity`
:param url: the api url to request
:type url: :class:`str`
:param method: an optional http method. ``'GET'`` by default
:type method: :class:`str`
:param data: an optional content body
:type data: :class:`bytes`
"""
logger = logging.getLogger(__name__ + '.request')
if isinstance(access_token, Identity):
access_token = access_token.access_token
logger.debug('access_token: %r', access_token)
req = urllib.request.Request(url,
headers={
'Authorization': 'Bearer ' + access_token,
'Accept': 'application/json'
},
method=method,
data=data)
try:
with contextlib.closing(urllib.request.urlopen(req)) as response:
content_type = response.headers.get('Content-Type')
mimetype, options = parse_options_header(content_type)
assert mimetype == 'application/json' or method == 'DELETE', \
'Content-Type of {} is not application/json but {}'.format(
url,
content_type
)
charset = options.get('charset', 'utf-8')
io_wrapper = io.TextIOWrapper(response, encoding=charset)
if logger.isEnabledFor(logging.DEBUG):
read = io_wrapper.read()
logger.debug(
'HTTP/%d.%d %d %s\n%s\n\n%s', response.version // 10,
response.version % 10, response.status, response.reason,
'\n'.join('{}: {}'.format(k, v)
for k, v in response.headers.items()), read)
if method == 'DELETE':
return
return json.loads(read)
else:
if method == 'DELETE':
io_wrapper.read()
return
return json.load(io_wrapper)
except urllib.error.HTTPError as e:
if logger.isEnabledFor(logging.DEBUG):
f = io.BytesIO()
shutil.copyfileobj(e, f)
logger.debug(
'HTTP/%d.%d %d %s\n%s\n\n%r', e.version // 10, e.version % 10,
e.status, e.reason,
'\n'.join('{}: {}'.format(k, v) for k, v in e.headers.items()),
f.getvalue())
f.seek(0)
logger.debug(str(e), exc_info=1)
raise urllib.error.HTTPError(e.url, e.code, e.reason, e.headers,
f) from e
else:
raise
def parse_multipart(file,
boundary,
content_length,
stream_factory=None,
charset='utf-8',
errors='ignore',
buffer_size=10240,
max_form_memory_size=None):
if stream_factory is None:
stream_factory = default_stream_factory
if not boundary:
raise ValueError('Missing boundary')
if not is_valid_multipart_boundary(boundary):
raise ValueError('Invalid boundary: %s' % boundary)
if len(boundary) > buffer_size:
raise ValueError('Boundary longer than buffer size')
total_content_length = content_length
next_part = '--' + boundary
last_part = next_part + '--'
form = []
files = []
in_memory = 0
file = LimitedStream(file, content_length)
iterator = chain(make_line_iter(file, buffer_size=buffer_size),
_empty_string_iter)
try:
terminator = _find_terminator(iterator)
if terminator != next_part:
raise ValueError('Expected boundary at start of multipart data')
while terminator != last_part:
headers = parse_multipart_headers(iterator)
disposition = headers.get('content-disposition')
if disposition is None:
raise ValueError('Missing Content-Disposition header')
disposition, extra = parse_options_header(disposition)
name = extra.get('name')
transfer_encoding = headers.get('content-transfer-encoding')
try_decode = transfer_encoding is not None and transfer_encoding in _supported_multipart_encodings
filename = extra.get('filename')
if filename is None:
is_file = False
container = []
_write = container.append
guard_memory = max_form_memory_size is not None
else:
content_type = headers.get('content-type')
content_type = parse_options_header(
content_type)[0] or 'text/plain'
is_file = True
guard_memory = False
if filename is not None:
filename = _fix_ie_filename(
_decode_unicode(filename, charset, errors))
try:
content_length = int(headers['content-length'])
except (KeyError, ValueError):
content_length = 0
container = stream_factory(total_content_length, content_type,
filename, content_length)
_write = container.write
buf = ''
for line in iterator:
if not line:
raise ValueError('unexpected end of stream')
if line[:2] == '--':
terminator = line.rstrip()
if terminator in (next_part, last_part):
break
if try_decode:
try:
line = line.decode(transfer_encoding)
except:
raise ValueError(
'could not decode transfer encoded chunk')
if buf:
_write(buf)
buf = ''
if line[-2:] == '\r\n':
buf = '\r\n'
cutoff = -2
else:
buf = line[-1]
cutoff = -1
_write(line[:cutoff])
if guard_memory:
in_memory += len(line)
if in_memory > max_form_memory_size:
from werkzeug.exceptions import RequestEntityTooLarge
raise RequestEntityTooLarge()
else:
raise ValueError('unexpected end of part')
if is_file:
container.seek(0)
files.append(
(name,
FileStorage(container, filename, name, content_type,
content_length, headers)))
else:
form.append(
(name, _decode_unicode(''.join(container), charset,
errors)))
finally:
file.exhaust()
return (form, files)
def parse(self, file, boundary, content_length):
next_part = '--' + boundary
last_part = next_part + '--'
form = []
files = []
in_memory = 0
iterator = chain(make_line_iter(file, limit=content_length,
buffer_size=self.buffer_size),
_empty_string_iter)
terminator = self._find_terminator(iterator)
if terminator != next_part:
self.fail('Expected boundary at start of multipart data')
while terminator != last_part:
headers = parse_multipart_headers(iterator)
disposition = headers.get('content-disposition')
if disposition is None:
self.fail('Missing Content-Disposition header')
disposition, extra = parse_options_header(disposition)
transfer_encoding = self.get_part_encoding(headers)
name = extra.get('name')
filename = extra.get('filename')
part_charset = self.get_part_charset(headers)
# if no content type is given we stream into memory. A list is
# used as a temporary container.
if filename is None:
is_file = False
container = []
_write = container.append
guard_memory = self.max_form_memory_size is not None
# otherwise we parse the rest of the headers and ask the stream
# factory for something we can write in.
else:
is_file = True
guard_memory = False
filename, container = self.start_file_streaming(
filename, headers, content_length)
_write = container.write
buf = ''
for line in iterator:
if not line:
self.fail('unexpected end of stream')
if line[:2] == '--':
terminator = line.rstrip()
if terminator in (next_part, last_part):
break
if transfer_encoding is not None:
try:
line = line.decode(transfer_encoding)
except Exception:
self.fail('could not decode transfer encoded chunk')
# we have something in the buffer from the last iteration.
# this is usually a newline delimiter.
if buf:
_write(buf)
buf = ''
# If the line ends with windows CRLF we write everything except
# the last two bytes. In all other cases however we write
# everything except the last byte. If it was a newline, that's
# fine, otherwise it does not matter because we will write it
# the next iteration. this ensures we do not write the
# final newline into the stream. That way we do not have to
# truncate the stream. However we do have to make sure that
# if something else than a newline is in there we write it
# out.
if line[-2:] == '\r\n':
buf = '\r\n'
cutoff = -2
else:
buf = line[-1]
cutoff = -1
_write(line[:cutoff])
# if we write into memory and there is a memory size limit we
# count the number of bytes in memory and raise an exception if
# there is too much data in memory.
if guard_memory:
in_memory += len(line)
if in_memory > self.max_form_memory_size:
self.in_memory_threshold_reached(in_memory)
else: # pragma: no cover
raise ValueError('unexpected end of part')
# if we have a leftover in the buffer that is not a newline
# character we have to flush it, otherwise we will chop of
# certain values.
if buf not in ('', '\r', '\n', '\r\n'):
_write(buf)
if is_file:
container.seek(0)
files.append((name, FileStorage(container, filename, name,
headers=headers)))
else:
form.append((name, _decode_unicode(''.join(container),
part_charset, self.errors)))
return self.cls(form), self.cls(files)
def test_parse_options_header(self):
assert http.parse_options_header(None) == ("", {})
assert http.parse_options_header("") == ("", {})
assert http.parse_options_header(r'something; foo="other\"thing"') == (
"something",
{
"foo": 'other"thing'
},
)
assert http.parse_options_header(
r'something; foo="other\"thing"; meh=42') == (
"something",
{
"foo": 'other"thing',
"meh": "42"
},
)
assert http.parse_options_header(
r'something; foo="other\"thing"; meh=42; bleh') == ("something", {
"foo": 'other"thing',
"meh": "42",
"bleh": None
})
assert http.parse_options_header(
'something; foo="other;thing"; meh=42; bleh') == ("something", {
"foo": "other;thing",
"meh": "42",
"bleh": None
})
assert http.parse_options_header(
'something; foo="otherthing"; meh=; bleh') == (
"something",
{
"foo": "otherthing",
"meh": None,
"bleh": None
},
)
# Issue #404
assert http.parse_options_header(
'multipart/form-data; name="foo bar"; filename="bar foo"') == (
"multipart/form-data", {
"name": "foo bar",
"filename": "bar foo"
})
# Examples from RFC
assert http.parse_options_header("audio/*; q=0.2, audio/basic") == (
"audio/*",
{
"q": "0.2"
},
)
assert http.parse_options_header("audio/*; q=0.2, audio/basic",
multiple=True) == ("audio/*", {
"q": "0.2"
}, "audio/basic", {})
assert http.parse_options_header(
"text/plain; q=0.5, text/html\n text/x-dvi; q=0.8, text/x-c",
multiple=True,
) == (
"text/plain",
{
"q": "0.5"
},
"text/html",
{},
"text/x-dvi",
{
"q": "0.8"
},
"text/x-c",
{},
)
assert http.parse_options_header(
"text/plain; q=0.5, text/html\n text/x-dvi; q=0.8, text/x-c"
) == ("text/plain", {
"q": "0.5"
})
# Issue #932
assert http.parse_options_header(
"form-data; name=\"a_file\"; filename*=UTF-8''"
'"%c2%a3%20and%20%e2%82%ac%20rates"') == ("form-data", {
"name":
"a_file",
"filename":
"\xa3 and \u20ac rates"
})
assert http.parse_options_header(
"form-data; name*=UTF-8''\"%C5%AAn%C4%ADc%C5%8Dde%CC%BD\"; "
'filename="some_file.txt"') == (
"form-data",
{
"name": "\u016an\u012dc\u014dde\u033d",
"filename": "some_file.txt"
},
)
def parse_link_header(link_header):
href, options = parse_options_header(link_header)
assert href.startswith('<')
assert href.endswith('>')
assert list(options.keys()) == ['rel']
return href[1:-1], options['rel']
def test_parse_options_header(self):
assert http.parse_options_header(None) == \
('', {})
assert http.parse_options_header("") == \
('', {})
assert http.parse_options_header(r'something; foo="other\"thing"') == \
('something', {'foo': 'other"thing'})
assert http.parse_options_header(r'something; foo="other\"thing"; meh=42') == \
('something', {'foo': 'other"thing', 'meh': '42'})
assert http.parse_options_header(r'something; foo="other\"thing"; meh=42; bleh') == \
('something', {'foo': 'other"thing', 'meh': '42', 'bleh': None})
assert http.parse_options_header('something; foo="other;thing"; meh=42; bleh') == \
('something', {'foo': 'other;thing', 'meh': '42', 'bleh': None})
assert http.parse_options_header('something; foo="otherthing"; meh=; bleh') == \
('something', {'foo': 'otherthing', 'meh': None, 'bleh': None})
# Issue #404
assert http.parse_options_header('multipart/form-data; name="foo bar"; filename="bar foo"') == \
('multipart/form-data', {'name': 'foo bar', 'filename': 'bar foo'})
# Examples from RFC
assert http.parse_options_header('audio/*; q=0.2, audio/basic') == \
('audio/*', {'q': '0.2'})
assert http.parse_options_header('audio/*; q=0.2, audio/basic', multiple=True) == \
('audio/*', {'q': '0.2'}, "audio/basic", {})
assert http.parse_options_header(
'text/plain; q=0.5, text/html\n '
'text/x-dvi; q=0.8, text/x-c',
multiple=True) == \
('text/plain', {'q': '0.5'}, "text/html", {},
"text/x-dvi", {'q': '0.8'}, "text/x-c", {})
assert http.parse_options_header('text/plain; q=0.5, text/html\n text/x-dvi; q=0.8, text/x-c') == \
('text/plain', {'q': '0.5'})
def content_type(self) -> str:
return parse_options_header(self.get('content-type'))[0].lower()
def process_request(self, request):
# make sure these fields are always set
request.grip = GripData()
request.wscontext = None
# old
request.grip_proxied = False
request.grip_signed = False
proxied = False
signed = False
grip_sig_header = request.META.get('HTTP_GRIP_SIG')
if grip_sig_header:
proxies = _get_proxies()
all_proxies_have_keys = True
for entry in proxies:
if 'key' not in entry:
all_proxies_have_keys = False
break
if all_proxies_have_keys:
# if all proxies have keys, then don't
# consider the request to be proxied unless
# one of them signed it
for entry in proxies:
if validate_sig(grip_sig_header, entry['key']):
proxied = True
signed = True
break
else:
# if even one proxy doesn't have a key, then
# don't require verification in order to
# consider the request to have been proxied
proxied = True
# parse Grip-Feature
hvalue = request.META.get('HTTP_GRIP_FEATURE')
if hvalue:
parsed = parse_options_header(hvalue, multiple=True)
features = set()
for n in range(0, len(parsed), 2):
features.add(parsed[n])
request.grip.features = features
# parse Grip-Last
hvalue = request.META.get('HTTP_GRIP_LAST')
if hvalue:
parsed = parse_options_header(hvalue, multiple=True)
last = {}
for n in range(0, len(parsed), 2):
channel = parsed[n]
params = parsed[n + 1]
last[channel] = params.get('last-id', '')
request.grip.last = last
if hasattr(request, 'content_type'):
content_type = request.content_type
else:
content_type = ''
hvalue = request.META.get('CONTENT_TYPE')
if hvalue:
parsed = parse_options_header(hvalue)
content_type = parsed[0]
# detect WebSocket-Over-HTTP request
wscontext = None
if (request.method == 'POST'
and content_type == 'application/websocket-events'):
cid = request.META.get('HTTP_CONNECTION_ID')
meta = {}
for k, v in six.iteritems(request.META):
if k.startswith('HTTP_META_'):
meta[_convert_header_name(k[10:])] = v
body = request.body
if isinstance(body, six.text_type):
body = body.encode('utf-8')
try:
events = decode_websocket_events(body)
except:
return HttpResponseBadRequest(
'Error parsing WebSocket events.\n')
wscontext = WebSocketContext(cid,
meta,
events,
grip_prefix=_get_prefix())
request.grip.proxied = proxied
request.grip.signed = signed
request.wscontext = wscontext
# old
request.grip_proxied = proxied
request.grip_signed = signed
def content_encoding(self) -> str:
return parse_options_header(self.get('content-encoding'))[0].lower()
def _parse_content_type(self):
if not hasattr(self, '_parsed_content_type'):
self._parsed_content_type = \
parse_options_header(self.content_type)
def aws_invoke(app,
gateway_input,
server_name='localhost',
server_port='5000',
http_protocol='HTTP/1.1',
TLS=True,
block_headers=True):
headers = CaseInsensitiveDict(gateway_input.get('headers', {}))
requestContext = gateway_input.get('requestContext')
queryStringParameters = gateway_input.get('queryStringParameters', {})
clientIp = headers.get('x-forwarded-for')
if clientIp is None:
clientIp = requestContext.get(
'identity',
{}).get('sourceIp') if requestContext is not None else ''
else:
clientIp = clientIp.split(',')[0]
environ = {
'REQUEST_METHOD':
gateway_input.get('httpMethod', 'GET').upper(),
'SCRIPT_NAME':
'',
'PATH_INFO':
gateway_input.get('path', '/'),
'QUERY_STRING':
urlencode(queryStringParameters)
if queryStringParameters is not None else '',
'SERVER_NAME':
headers.get('host', server_name),
'SERVER_PORT':
headers.get('x-forwarded-port', server_port),
'SERVER_PROTOCOL':
http_protocol,
'SERVER_SOFTWARE':
'flask-serverless',
'REMOTE_ADDR':
clientIp,
'wsgi.version': (1, 0),
'wsgi.url_scheme':
headers.get('x-forwarded-proto', 'https' if TLS else 'http'),
'wsgi.input':
None,
'wsgi.errors':
sys.stderr,
'wsgi.multiprocess':
True,
'wsgi.multithread':
False,
'wsgi.run_once':
True,
'HTTP_X_AWS_PATH':
requestContext['path']
}
if environ['REQUEST_METHOD'] == 'POST' or environ[
'REQUEST_METHOD'] == 'PUT':
contentType = headers.get('content-type', 'application/octet-stream')
parsedContentType = parse_options_header(contentType)
raw = gateway_input.get('body')
if raw is None or gateway_input.get('isBase64Encoded', False):
body = b64decode(raw) if raw is not None else None
else:
body = raw.encode(parsedContentType[1].get('charset', 'utf-8'))
add_body(environ, body, contentType)
add_headers(environ, headers, block_headers)
response = Response.from_app(app.wsgi_app, environ)
gateway_output = {
'headers': dict(response.headers),
'statusCode': response.status_code,
}
compressed = response.headers.get('Content-Encoding') == 'gzip'
responseType = parse_options_header(
response.headers.get('Content-Type', 'application/octet-stream'))
if not compressed and ('charset' in responseType[1] or responseType[0]
in textTypes or responseType[0][0:5] == 'text/'):
gateway_output['body'] = response.data.decode(responseType[1].get(
'charset', 'utf-8'))
gateway_output['isBase64Encoded'] = False
else:
gateway_output['body'] = b64encode(response.data).decode('utf-8')
gateway_output['isBase64Encoded'] = True
return gateway_output
def mimetype(self) -> str:
"""Returns the mimetype parsed from the Content-Type header."""
return parse_options_header(self.headers.get("Content-Type"))[0]
def parse_lines(self, file, boundary, content_length, cap_at_buffer=True):
"""Generate parts of
``('begin_form', (headers, name))``
``('begin_file', (headers, name, filename))``
``('cont', bytestring)``
``('end', None)``
Always obeys the grammar
parts = ( begin_form cont* end |
begin_file cont* end )*
"""
next_part = b"--" + boundary
last_part = next_part + b"--"
iterator = chain(
make_line_iter(
file,
limit=content_length,
buffer_size=self.buffer_size,
cap_at_buffer=cap_at_buffer,
),
_empty_string_iter,
)
terminator = self._find_terminator(iterator)
if terminator == last_part:
return
elif terminator != next_part:
self.fail("Expected boundary at start of multipart data")
while terminator != last_part:
headers = parse_multipart_headers(iterator)
disposition = headers.get("content-disposition")
if disposition is None:
self.fail("Missing Content-Disposition header")
disposition, extra = parse_options_header(disposition)
transfer_encoding = self.get_part_encoding(headers)
name = extra.get("name")
# Accept filename* to support non-ascii filenames as per rfc2231
filename = extra.get("filename") or extra.get("filename*")
# if no content type is given we stream into memory. A list is
# used as a temporary container.
if filename is None:
yield _begin_form, (headers, name)
# otherwise we parse the rest of the headers and ask the stream
# factory for something we can write in.
else:
yield _begin_file, (headers, name, filename)
buf = b""
for line in iterator:
if not line:
self.fail("unexpected end of stream")
if line[:2] == b"--":
terminator = line.rstrip()
if terminator in (next_part, last_part):
break
if transfer_encoding is not None:
if transfer_encoding == "base64":
transfer_encoding = "base64_codec"
try:
line = codecs.decode(line, transfer_encoding)
except Exception:
self.fail("could not decode transfer encoded chunk")
# we have something in the buffer from the last iteration.
# this is usually a newline delimiter.
if buf:
yield _cont, buf
buf = b""
# If the line ends with windows CRLF we write everything except
# the last two bytes. In all other cases however we write
# everything except the last byte. If it was a newline, that's
# fine, otherwise it does not matter because we will write it
# the next iteration. this ensures we do not write the
# final newline into the stream. That way we do not have to
# truncate the stream. However we do have to make sure that
# if something else than a newline is in there we write it
# out.
if line[-2:] == b"\r\n":
buf = b"\r\n"
cutoff = -2
else:
buf = line[-1:]
cutoff = -1
yield _cont, line[:cutoff]
else: # pragma: no cover
raise ValueError("unexpected end of part")
# if we have a leftover in the buffer that is not a newline
# character we have to flush it, otherwise we will chop of
# certain values.
if buf not in (b"", b"\r", b"\n", b"\r\n"):
yield _cont, buf
yield _end, None
def _content_type():
content_type = http.parse_options_header(
request.headers.get('Content-Type', ''))
if content_type:
content_type, _ = content_type
return content_type
def sparql(branch_or_ref):
"""Process a SPARQL query (Select or Update).
Returns:
HTTP Response with query result: If query was a valid select query.
HTTP Response 200: If request contained a valid update query.
HTTP Response 406: If accept header is not acceptable.
"""
quit = current_app.config['quit']
default_branch = quit.config.getDefaultBranch()
if not branch_or_ref and not quit.repository.is_empty:
branch_or_ref = default_branch
logger.debug("Request method: {}".format(request.method))
query = None
if request.method == "GET":
default_graph = request.args.get('default-graph-uri', None)
named_graph = request.args.get('named-graph-uri', None)
query = request.args.get('query', None)
elif request.method == "POST":
if 'Content-Type' in request.headers:
contentMimeType, options = parse_options_header(
request.headers['Content-Type'])
if contentMimeType == "application/x-www-form-urlencoded":
if 'query' in request.form:
default_graph = request.form.get('default-graph-uri', None)
named_graph = request.form.get('named-graph-uri', None)
query = request.form.get('query', None)
elif 'update' in request.form:
default_graph = request.form.get('using-graph-uri', None)
named_graph = request.form.get('using-named-graph-uri',
None)
query = request.form.get('update', None)
elif contentMimeType == "application/sparql-query":
default_graph = request.args.get('default-graph-uri', None)
named_graph = request.args.get('named-graph-uri', None)
query = request.data.decode("utf-8")
elif contentMimeType == "application/sparql-update":
default_graph = request.args.get('using-graph-uri', None)
named_graph = request.args.get('using-named-graph-uri', None)
query = request.data.decode("utf-8")
if 'Accept' in request.headers:
logger.info('Received query via {}: {} with accept header: {}'.format(
request.method, query, request.headers['Accept']))
mimetype = parse_accept_header(request.headers['Accept']).best
else:
logger.info('Received query via {}: {} with no accept header.'.format(
request.method, query))
mimetype = '*/*'
if query is None:
if mimetype == 'text/html':
return render_template('sparql.html', current_ref=branch_or_ref)
else:
return make_response(
'No Query was specified or the Content-Type is not set according'
+ 'to the SPARQL 1.1 standard', 400)
try:
queryType, parsedQuery = parse_query_type(query, quit.config.namespace)
graph = quit.instance(branch_or_ref)
except UnSupportedQueryType as e:
logger.exception(e)
return make_response('Unsupported Query Type', 400)
except Exception as e:
logger.exception(e)
return make_response('No branch or reference given.', 400)
if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere']:
res = graph.update(parsedQuery)
try:
ref = request.values.get('ref', None) or default_branch
ref = 'refs/heads/{}'.format(ref)
quit.commit(graph,
res,
'New Commit from QuitStore',
branch_or_ref,
ref,
query=query)
return '', 200
except Exception as e:
# query ok, but unsupported query type or other problem during commit
logger.exception(e)
return make_response('Error after executing the update query.',
400)
elif queryType in [
'SelectQuery', 'DescribeQuery', 'AskQuery', 'ConstructQuery'
]:
res = graph.query(parsedQuery)
else:
logger.debug("Unsupported Type: {}".format(queryType))
return make_response("Unsupported Query Type: {}".format(queryType),
400)
try:
if queryType in ['SelectQuery', 'AskQuery']:
return create_result_response(res, resultSetMimetypes[mimetype])
elif queryType in ['ConstructQuery', 'DescribeQuery']:
return create_result_response(res, rdfMimetypes[mimetype])
except KeyError as e:
return make_response("Mimetype: {} not acceptable".format(mimetype),
406)
def _get_mimetype_and_options(headers: http.Headers) -> typing.Tuple[str, dict]:
return parse_options_header(headers.get('Content-Type'))
