def test_base64_spaces(self):
result = parse_data_uri("data:text/plain;base64,SGVsb%20G8sIH%0A%20%20"
"dvcm%20%20%20xk%20Lg%3D%0A%3D")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.data, b"Hello, world.")
result = parse_data_uri("data:text/plain;base64,SGVsb G8sIH\n "
"dvcm xk Lg%3D\n%3D")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.data, b"Hello, world.")
def test_base64_spaces(self):
result = parse_data_uri("data:text/plain;base64,SGVsb%20G8sIH%0A%20%20"
"dvcm%20%20%20xk%20Lg%3D%0A%3D")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.data, b"Hello, world.")
result = parse_data_uri("data:text/plain;base64,SGVsb G8sIH\n "
"dvcm xk Lg%3D\n%3D")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.data, b"Hello, world.")
def create_column():
try:
# Accept incoming msg
if (request.is_json):
# Create object
post = {
'title': request.json.get('title'),
'author': request.json.get('author'),
'base64s': request.json.get('base64s'),
'content': request.json.get('content'),
}
try:
post_document = {
'title': request.json.get('title'),
'author': request.json.get('author'),
'content': request.json.get('content'),
'date': datetime.today(),
'imgurl': [],
}
if post['base64s']:
# Connect to S3
s3 = client(
's3',
aws_access_key_id=app.config['AWS_S3_KEY'],
aws_secret_access_key=app.config['AWS_S3_SECRET'])
# Convert base64 img to real img
for base64 in post['base64s']:
result = parse_data_uri(base64)
if (result[0] == 'image/jpeg'
or result[0] == 'image/png'):
new_id = uuid4()
extension = result[0][result[0].index('/') + 1:]
file_name = f'img/{new_id}.{extension}'
# Upload to aws S3
s3.put_object(
Body=result[2],
Bucket=app.config['BUCKET_COLUMN'],
Key=file_name,
ContentType=result[0],
)
post_document['imgurl'].append(file_name)
else:
raise WE.BadRequest(
'The img should be jpeg or png')
# Insert the column into DB
insert_single_doc(app.config['MONGO_COLLECTION_COLUMN'],
post_document)
return '', 204
except ValueError as e:
raise ValueError(e.args[0])
else:
raise WE.NotAcceptable(
'Request should have MIME type of application/json')
except Exception as e:
print(e)
WE.abort(500)
def download_request(self, request, spider):
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse) and
uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)
def download_request(self, request, spider):
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse) and
uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)
def test_mediatype_parameters(self):
result = parse_data_uri('data:text/plain;'
'foo=%22foo;bar%5C%22%22;'
'charset=utf-8;'
'bar=%22foo;%5C%22foo%20;/%20,%22,'
'%CE%8E%CE%A3%CE%8E')
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.media_type_parameters,
{"charset": "utf-8",
"foo": 'foo;bar"',
"bar": 'foo;"foo ;/ ,'})
self.assertEqual(result.data, b"\xce\x8e\xce\xa3\xce\x8e")
def test_mediatype_parameters(self):
result = parse_data_uri('data:text/plain;'
'foo=%22foo;bar%5C%22%22;'
'charset=utf-8;'
'bar=%22foo;%5C%22foo%20;/%20,%22,'
'%CE%8E%CE%A3%CE%8E')
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.media_type_parameters,
{"charset": "utf-8",
"foo": 'foo;bar"',
"bar": 'foo;"foo ;/ ,'})
self.assertEqual(result.data, b"\xce\x8e\xce\xa3\xce\x8e")
def predict():
data = {"success": False}
if flask.request.method == "POST":
imgData = flask.request.data
parsed = parse_data_uri(imgData)
if parsed:
# read the image in PIL format
image = Image.open(io.BytesIO(parsed.data))
# preprocess the image prior to classification
image = prepare_image(image, (28, 28))
# classify the input image with the model
preds = model.predict(image)
# decode the prediction results
label = int(np.argmax(preds))
probability = preds[0][label] * 100
# form the response
result = {"label": label, "probability": float(probability)}
data["prediction"] = result
data["success"] = True
return flask.jsonify(data)
def uri_to_img(data_uri: str) -> Image:
"""
Converts a data URI to a PIL.Image.
"""
return Image.open(io.BytesIO(parse_data_uri(data_uri).data)).convert("RGB")
def test_default_mediatype_charset(self):
result = parse_data_uri("data:,A%20brief%20note")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.media_type_parameters, {"charset": "US-ASCII"})
self.assertEqual(result.data, b"A brief note")
def test_default_mediatype_charset(self):
result = parse_data_uri("data:,A%20brief%20note")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.media_type_parameters, {"charset": "US-ASCII"})
self.assertEqual(result.data, b"A brief note")
def test_unicode_uri(self):
result = parse_data_uri(u"data:,é")
self.assertEqual(result.data, u"é".encode('utf-8'))
uri = [
'data:image/webp;base64,UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA',
'data:image/webp;base64,UklGRhoAAABXRUJQVlA4TA0AAAAvAAAAEAcQERGIiP4HAA==',
'data:image/webp;base64,UklGRkoAAABXRUJQVlA4WAoAAAAQAAAAAAAAAAAAQUxQSAwAAAARBxAR/Q9ERP8DAABWUDggGAAAABQBAJ0BKgEAAQAAAP4AAA3AAP7mtQAAAA==',
'data:image/webp;base64,UklGRlIAAABXRUJQVlA4WAoAAAASAAAAAAAAAAAAQU5JTQYAAAD/////AABBTk1GJgAAAAAAAAAAAAAAAAAAAGQAAABWUDhMDQAAAC8AAAAQBxAREYiI/gcA',
'data:image/webp;base64,UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAAwA0JaQAA3AA/vuUAAA'
]
#有scrapy的话，用parse_data_uri解码base64_url(含data字符)
from w3lib.url import parse_data_uri
for i in range(5):
base64_str = uri[i].split(',')[-1]
if len(base64_str) % 4 != 0:
uri[i] += '=' * (4 - len(base64_str) % 4)
ret = parse_data_uri(uri[i]).data
with open(f'{i}.webp', 'wb') as f:
f.write(ret)
#用base64解码base64_url
# import base64
# for i in range(5):
# data = uri[i].split(',')[-1]
# if len(data)%4 != 0: #base64_url不足4的倍数，手动添加"="补足
# data += '='*(4-len(data)%4)
# ret = base64.b64decode(data) ##如果base64_url有特殊符号“+”，"/"就用base64.urlsafe_b64decode(data)
# with open(f'{i}.webp','wb') as f:
# f.write(ret)
def _check_uri(self, attribute, value):
try:
parse_data_uri(value)
except ValueError as e:
if not is_url(value):
raise ValueError("uri must be a URL or a data URI") from e
def test_wrong_scheme(self):
with self.assertRaises(ValueError):
parse_data_uri("http://example.com/")
def image_input(image_request: pb.Image) -> Image:
img_bytes = parse_data_uri(image_request.image_data_url).data
img = Image.open(io.BytesIO(img_bytes)).convert("RGB")
return img
def test_bytes_uri(self):
result = parse_data_uri(b"data:,A%20brief%20note")
self.assertEqual(result.data, b"A brief note")
def test_missing_comma(self):
with self.assertRaises(ValueError):
parse_data_uri("data:A%20brief%20note")
def test_text_charset(self):
result = parse_data_uri("data:text/plain;charset=iso-8859-7,%be%d3%be")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.media_type_parameters,
{"charset": "iso-8859-7"})
self.assertEqual(result.data, b"\xbe\xd3\xbe")
def test_bytes_uri(self):
result = parse_data_uri(b"data:,A%20brief%20note")
self.assertEqual(result.data, b"A brief note")
def test_base64(self):
result = parse_data_uri("data:text/plain;base64,"
"SGVsbG8sIHdvcmxkLg%3D%3D")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.data, b"Hello, world.")
def test_unicode_uri(self):
result = parse_data_uri(u"data:,é")
self.assertEqual(result.data, u"é".encode('utf-8'))
def test_text_charset(self):
result = parse_data_uri("data:text/plain;charset=iso-8859-7,%be%d3%be")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.media_type_parameters,
{"charset": "iso-8859-7"})
self.assertEqual(result.data, b"\xbe\xd3\xbe")
def test_unicode_uri(self):
result = parse_data_uri("data:,é")
self.assertEqual(result.data, "é".encode())
def test_wrong_base64_param(self):
with self.assertRaises(ValueError):
parse_data_uri("data:text/plain;baes64,SGVsbG8sIHdvcmxkLg%3D%3D")
def test_base64(self):
result = parse_data_uri("data:text/plain;base64,"
"SGVsbG8sIHdvcmxkLg%3D%3D")
self.assertEqual(result.media_type, "text/plain")
self.assertEqual(result.data, b"Hello, world.")
def test_missing_comma(self):
with self.assertRaises(ValueError):
parse_data_uri("data:A%20brief%20note")
def test_wrong_base64_param(self):
with self.assertRaises(ValueError):
parse_data_uri("data:text/plain;baes64,SGVsbG8sIHdvcmxkLg%3D%3D")
def test_missing_scheme(self):
with self.assertRaises(ValueError):
parse_data_uri("text/plain,A%20brief%20note")
def test_missing_scheme(self):
with self.assertRaises(ValueError):
parse_data_uri("text/plain,A%20brief%20note")
def test_wrong_scheme(self):
with self.assertRaises(ValueError):
parse_data_uri("http://example.com/")
def test_scheme_case_insensitive(self):
result = parse_data_uri("DATA:,A%20brief%20note")
self.assertEqual(result.data, b"A brief note")
result = parse_data_uri("DaTa:,A%20brief%20note")
self.assertEqual(result.data, b"A brief note")
def test_scheme_case_insensitive(self):
result = parse_data_uri("DATA:,A%20brief%20note")
self.assertEqual(result.data, b"A brief note")
result = parse_data_uri("DaTa:,A%20brief%20note")
self.assertEqual(result.data, b"A brief note")
