def subTest(self, test_idx, test_name, chunksize, original_data,
content_type, multipart_data, multipart_filename):
print(test_idx, '; name: ', test_name, '; data_size: ',
len(original_data), '; chunksize: ', chunksize)
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
target = ValueTarget()
parser.register('file', target)
remaining = len(multipart_data)
offset = 0
while (remaining):
step_size = min(remaining, chunksize)
parser.data_received(multipart_data[offset:offset + step_size])
offset += step_size
remaining -= step_size
self.assertEqual(offset, len(multipart_data))
self.assertEqual(target.multipart_filename, multipart_filename)
self.assertEqual(target._started, True)
self.assertEqual(target._finished, True)
result = target.value
self.assertEqual(len(result), len(original_data))
self.assertEqual(result, original_data)
def test_file_content_single():
filenames = (
'file.txt',
'image-600x400.png',
'image-2560x1600.png',
'empty.html',
'hyphen-hyphen.txt',
'LF.txt',
'CRLF.txt',
'1M.dat',
'1M-1.dat',
'1M+1.dat',
)
for filename in filenames:
with open_dataset(filename) as dataset_:
expected_value = dataset_.read()
content_type, body = encoded_dataset(filename)
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
parser.register(filename, target)
parser.data_received(body)
assert target.value == expected_value
def test_break_chunk_at_boundary():
expected_first_value = 'hello' * 500
expected_second_value = 'hello' * 500
first = ValueTarget()
second = ValueTarget()
encoder = MultipartEncoder(fields={
'first': 'hello' * 500,
'second': 'hello' * 500
})
body = encoder.to_string()
boundary = encoder.boundary.encode('utf-8')
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('first', first)
parser.register('second', second)
index = body[50:].index(boundary) + 5
parser.data_received(body[:index])
parser.data_received(body[index:])
assert first.value == expected_first_value.encode('utf-8')
assert second.value == expected_second_value.encode('utf-8')
def test_multiple_files():
txt_filename = 'file.txt'
png_filename = 'image-600x400.png'
with open_dataset(txt_filename) as dataset_:
expected_txt = dataset_.read()
with open_dataset(png_filename) as dataset_:
expected_png = dataset_.read()
txt_target = ValueTarget()
png_target = ValueTarget()
with open_dataset(txt_filename) as txt_file, open_dataset(
png_filename) as png_file:
encoder = MultipartEncoder(
fields={
txt_filename: (txt_filename, txt_file, 'application/plain'),
png_filename: (png_filename, png_file, 'image/png'),
})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register(txt_filename, txt_target)
parser.register(png_filename, png_target)
parser.data_received(encoder.to_string())
assert txt_target.value == expected_txt
assert png_target.value == expected_png
def test_case_insensitive_content_disposition_header():
content_disposition_header = 'Content-Disposition'
for header in (
content_disposition_header,
content_disposition_header.lower(),
content_disposition_header.upper(),
):
data = b'''\
--1234
{header}: form-data; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b'\n', b'\r\n').replace(b'{header}',
header.encode('utf-8'))
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b'Foo'
def test_special_filenames():
filenames = [
'a;b.txt',
'a"b.txt',
'a";b.txt',
'a;"b.txt',
'a";";.txt',
'a\\"b.txt',
'a\\b.txt',
]
for filename in filenames:
data = ('''\
--1234
Content-Disposition: form-data; name=files; filename={}
Foo
--1234--'''.format(filename).replace('\n', '\r\n').encode('utf-8'))
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b'Foo'
def test_multiple_targets():
filename = 'image-600x400.png'
with open_dataset(filename) as dataset_:
expected_data = dataset_.read()
value_target = ValueTarget()
sha256_target = SHA256Target()
with open_dataset(filename) as file_:
encoder = MultipartEncoder(
fields={filename: (filename, file_, 'image/png')})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register(filename, value_target)
parser.register(filename, sha256_target)
assert not value_target.value
assert sha256_target.value == hashlib.sha256(b'').hexdigest()
parser.data_received(encoder.to_string())
assert value_target.value == expected_data
assert sha256_target.value == hashlib.sha256(expected_data).hexdigest()
def send():
url = f'{host}/files/fileSearchContext.mask'
headers = {'Accept-Encoding': "gzip", 'Transfer-Encoding': "gzip"}
extension = os.path.splitext(file_path)[1]
os.makedirs(folder_name, exist_ok=True)
with open(file_path, 'rb') as f:
if len(media_type) > 0:
encoder = MultipartEncoder(fields={
'context': ('context', context, 'application/json'),
'file': ('file', f, media_type)
})
else:
encoder = MultipartEncoder(fields={
'context': ('context', context, 'application/json'),
'file': ('file', f)
})
with session.post(url, data=encoder, stream=True,
headers={'Content-Type': encoder.content_type}) as r:
if r.status_code >= 300:
raise Exception(f"Failed with status {r.status_code}:\n\n{r.json()}")
parser = StreamingFormDataParser(headers=r.headers)
parser.register('file', FileTarget(f'{folder_name}/masked{extension}'))
parser.register('results', FileTarget(f'{folder_name}/results.json'))
for chunk in r.iter_content(chunk_size):
parser.data_received(chunk)
def test_multiple_files(self):
txt_filename = 'file.txt'
png_filename = 'image-600x400.png'
with open(data_file_path(txt_filename), 'rb') as file_:
expected_txt = file_.read()
with open(data_file_path(png_filename), 'rb') as file_:
expected_png = file_.read()
txt_target = ValueTarget()
png_target = ValueTarget()
with open(data_file_path(txt_filename), 'rb') as txt_file, \
open(data_file_path(png_filename), 'rb') as png_file:
encoder = MultipartEncoder(
fields={
txt_filename: (txt_filename, txt_file,
'application/plain'),
png_filename: (png_filename, png_file, 'image/png')
})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register(txt_filename, txt_target)
parser.register(png_filename, png_target)
parser.data_received(encoder.to_string())
self.assertEqual(txt_target.value, expected_txt)
self.assertEqual(png_target.value, expected_png)
def test_directory_upload(tmp_path):
data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234
Content-Disposition: form-data; name="files"; filename="cd.txt"
Bar
--1234--'''.replace(b'\n', b'\r\n')
target = DirectoryTarget(tmp_path)
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
with open(tmp_path / 'ab.txt') as file:
assert file.read() == 'Foo'
with open(tmp_path / 'cd.txt') as file:
assert file.read() == 'Bar'
assert target.multipart_filenames == ['ab.txt', 'cd.txt']
assert tmp_path
assert target._started
assert target._finished
def test_register_after_data_received():
encoder = MultipartEncoder(fields={'name': 'hello'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.data_received(encoder.to_string())
with pytest.raises(ParseFailedException):
parser.register('name', ValueTarget())
def test_parameter_contains_crlf():
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello\r\nworld'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
assert target.value == b'hello\r\nworld'
def test_basic_single(self):
target = CustomTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
data = encoder.to_string()
self.assertRaises(ValueError, parser.data_received, data)
def test_basic_single(self):
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
self.assertEqual(target.value, b'hello world')
def test_parameter_starts_with_crlf(self):
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': '\r\nworld'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
self.assertEqual(target.value, b'\r\nworld')
def test_custom_target_exception():
target = CustomTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
data = encoder.to_string()
with pytest.raises(ValueError):
parser.data_received(data)
def test_basic_single():
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
assert target.value == b'hello world'
assert target._started
assert target._finished
def test_filename_passed_to_target():
filename = 'file.txt'
content_type, body = encoded_dataset(filename)
target = ValueTarget()
assert not target.multipart_filename
parser = StreamingFormDataParser(headers={'Content-Type': content_type})
parser.register(filename, target)
parser.data_received(body)
assert target.multipart_filename == filename
def test_target_raises_exception():
filename = 'file.txt'
content_type, body = encoded_dataset(filename)
class BadTarget(BaseTarget):
def data_received(self, data):
raise ValueError()
target = BadTarget()
parser = StreamingFormDataParser(headers={'Content-Type': content_type})
parser.register(filename, target)
with pytest.raises(ValueError):
parser.data_received(body)
def test_case_insensitive_content_type(self):
content_type_header = 'Content-Type'
for header_key in (content_type_header, content_type_header.lower(),
content_type_header.upper(), 'cOnTeNt-tYPe'):
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={header_key: encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
self.assertEqual(target.value, b'hello world')
def test_missing_headers():
data = '''\
--1234
Foo
--1234--'''.replace('\n', '\r\n').encode('utf-8')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b''
def test_line_does_not_end_with_correct_linebreak(self):
data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo--1234--'''.replace(b'\n', b'\r\n')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
self.assertEqual(target.value, b'Foo')
def cgi_upload() -> MethodResponse:
LOGGER.info('Upload file begin')
# wsgiref does not support async reading from environ['wsgi.input']
# It blocks forever in read(size) call.
use_async_implementation = config.WEB_SERVER != 'wsgiref'
if use_async_implementation:
size = 0
file = NullTarget() if config.DISABLE_STORAGE else StorageFileTarget()
parser = StreamingFormDataParser(headers=bottle.request.headers)
parser.register('file', file)
while True:
LOGGER.debug('Read async chunk...')
buffer = bottle.request.environ['wsgi.input']
chunk = buffer.read(64 * 1024)
if not chunk:
break
LOGGER.debug('Got async chunk from network: %d bytes', len(chunk))
parser.data_received(chunk)
size += len(chunk)
LOGGER.info('Uploaded request size: %s bytes', size)
else:
size = 0
files: bottle.FormsDict = bottle.request.files
upload = files.file # pylint: disable=no-member
if upload is None:
raise Exception('ERROR! "file" multipart field was not found')
original_filename = upload.raw_filename
body = upload.file
with STORAGE.open_file_writer(original_filename) as writer:
while True:
LOGGER.debug('Read synchronous chunk...')
chunk = body.read(64 * 1024)
if not chunk:
break
LOGGER.debug('Got synchronous chunk from network: %d bytes',
len(chunk))
if not config.DISABLE_STORAGE:
writer.write(chunk)
size += len(chunk)
LOGGER.info('Uploaded file size: %d bytes', size)
return 'OK'
class UploadHandler(RequestHandler):
def prepare(self):
self.value = ValueTarget()
self.file_ = FileTarget('/tmp/file-{}.dat'.format(int(time())))
self._parser = StreamingFormDataParser(headers=self.request.headers)
self._parser.register('name', self.value)
self._parser.register('file', self.file_)
def data_received(self, chunk):
self._parser.data_received(chunk)
def post(self):
self.render('upload.html',
name=self.value.value,
filename=self.file_.filename)
def test_without_name_parameter():
data = b'''\
--1234
Content-Disposition: form-data; filename="ab.txt"
Foo
--1234--'''.replace(b'\n', b'\r\n')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b''
class UploadHandler(RequestHandler):
def prepare(self):
self.value = ValueTarget()
name = 'uploaded-file-tornado-{}.dat'.format(int(time()))
self.file_ = FileTarget(os.path.join(tempfile.gettempdir(), name))
self._parser = StreamingFormDataParser(headers=self.request.headers)
self._parser.register('name', self.value)
self._parser.register('file', self.file_)
def data_received(self, chunk):
self._parser.data_received(chunk)
def post(self):
self.render('upload.html', name=self.value.value,
filename=self.file_.filename)
def test_file_content_varying_chunk_size():
with open_dataset('file.txt') as dataset_:
expected_value = dataset_.read()
content_type, body = encoded_dataset('file.txt')
for index in range(len(body)):
txt = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
parser.register('file.txt', txt)
parser.data_received(body[:index])
parser.data_received(body[index:])
assert txt.value == expected_value
def test_invalid_content_disposition(self):
data = b'''\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b'\n', b'\r\n')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
self.assertRaises(ParseFailedException, parser.data_received, data)
self.assertEqual(target.value, b'')
def test_unquoted_names(self):
data = b'''\
--1234
Content-Disposition: form-data; name=files; filename=ab.txt
Foo
--1234--'''.replace(b'\n', b'\r\n')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
self.assertEqual(target.value, b'Foo')
def test_large_file(self):
for filename in ['image-500k.png', 'image-2560x1600.png',
'image-600x400.png', 'image-high-res.jpg']:
with open_dataset(filename) as dataset_:
expected_value = dataset_.read()
content_type, body = encoded_dataset(filename)
value = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
parser.register(filename, value)
parser.data_received(body)
self.assertEqual(value.value, expected_value)
def cgi_upload():
log('Upload file begin')
use_async_implementation = True
if use_async_implementation:
size = 0
file = NullTarget() if config.DISABLE_STORAGE else StorageFileTarget()
parser = StreamingFormDataParser(headers=bottle.request.headers)
parser.register('file', file)
while True:
chunk = bottle.request.environ['wsgi.input'].read(64 * 1024)
if not chunk:
break
parser.data_received(chunk)
size += len(chunk)
log('Uploaded request size: ' + str(size))
else:
size = 0
upload = bottle.request.files.get('file')
if upload is None:
raise Exception('ERROR! "file" multipart field was not found')
original_filename = upload.raw_filename
body = upload.file
with storage.open_file_writer(original_filename) as writer:
while True:
chunk = body.read(64 * 1024)
if not chunk:
break
if not config.DISABLE_STORAGE:
writer.write(chunk)
size += len(chunk)
log('Uploaded file size: ' + str(size))
return 'OK'
