def prepare(self):
if self.request.method == "POST":
self.request.connection.set_max_body_size(self.max_upload_size)
tmpname = self.file_manager.gen_temp_upload_path()
self._targets = {
'root': ValueTarget(),
'print': ValueTarget(),
'path': ValueTarget(),
}
self._file = FileTarget(tmpname)
self._parser = StreamingFormDataParser(self.request.headers)
self._parser.register('file', self._file)
for name, target in self._targets.items():
self._parser.register(name, target)
def test_break_chunk_at_boundary():
expected_first_value = 'hello' * 500
expected_second_value = 'hello' * 500
first = ValueTarget()
second = ValueTarget()
encoder = MultipartEncoder(
fields={'first': 'hello' * 500, 'second': 'hello' * 500}
)
body = encoder.to_string()
boundary = encoder.boundary.encode('utf-8')
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type}
)
parser.register('first', first)
parser.register('second', second)
index = body[50:].index(boundary) + 5
parser.data_received(body[:index])
parser.data_received(body[index:])
assert first.value == expected_first_value.encode('utf-8')
assert second.value == expected_second_value.encode('utf-8')
def test_multiple_targets():
filename = 'image-600x400.png'
with open_dataset(filename) as dataset_:
expected_data = dataset_.read()
value_target = ValueTarget()
sha256_target = SHA256Target()
with open_dataset(filename) as file_:
encoder = MultipartEncoder(
fields={filename: (filename, file_, 'image/png')}
)
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type}
)
parser.register(filename, value_target)
parser.register(filename, sha256_target)
assert not value_target.value
assert sha256_target.value == hashlib.sha256(b'').hexdigest()
parser.data_received(encoder.to_string())
assert value_target.value == expected_data
assert sha256_target.value == hashlib.sha256(expected_data).hexdigest()
def prepare(self):
self.request.connection.set_max_body_size(10_000_000_000)
self.streaming_parser = StreamingFormDataParser(self.request.headers)
self.uploaded_file_tmp = tempfile.NamedTemporaryFile(prefix='upload_')
self.uploaded_file = HashedFileTarget(self.uploaded_file_tmp.name)
self.streaming_parser.register('rpz_file', self.uploaded_file)
self.rpz_url = ValueTarget()
self.streaming_parser.register('rpz_url', self.rpz_url)
self.not_permanent = ValueTarget()
self.streaming_parser.register('not_permanent', self.not_permanent)
self.sent_xsrf_token = ValueTarget()
self.streaming_parser.register('_xsrf', self.sent_xsrf_token)
def test_target_raises_exception(self):
filename = 'file.txt'
content_type, body = encoded_dataset(filename)
class BadTarget(BaseTarget):
def data_received(self, data):
raise ValueError()
target = BadTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
parser.register(filename, target)
self.assertRaises(ValueError, parser.data_received, body)
class UploadHandler(RequestHandler):
def prepare(self):
self.value = ValueTarget()
self.file_ = FileTarget('/tmp/file-{}.dat'.format(int(time())))
self._parser = StreamingFormDataParser(headers=self.request.headers)
self._parser.register('name', self.value)
self._parser.register('file', self.file_)
def data_received(self, chunk):
self._parser.data_received(chunk)
def post(self):
self.render('upload.html',
name=self.value.value,
filename=self.file_.filename)
def test_incorrect_content_type():
for value in (
'multipart/mixed; boundary=1234',
'multipart/form-data',
'multipart/form-data; delimiter=1234',
):
with pytest.raises(ParseFailedException):
StreamingFormDataParser({'Content-Type': value})
class UploadHandler(RequestHandler):
def prepare(self):
self.value = ValueTarget()
name = 'uploaded-file-tornado-{}.dat'.format(int(time()))
self.file_ = FileTarget(os.path.join(tempfile.gettempdir(), name))
self._parser = StreamingFormDataParser(headers=self.request.headers)
self._parser.register('name', self.value)
self._parser.register('file', self.file_)
def data_received(self, chunk):
self._parser.data_received(chunk)
def post(self):
self.render('upload.html', name=self.value.value,
filename=self.file_.filename)
def test_invalid_content_disposition(self):
data = b'''\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b'\n', b'\r\n')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
self.assertRaises(ParseFailedException, parser.data_received, data)
self.assertEqual(target.value, b'')
def test_file_target_exceeds_max_size(self):
data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b'\n', b'\r\n')
target = FileTarget('/tmp/file.txt', validator=MaxSizeValidator(1))
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
self.assertRaises(ValidationError, parser.data_received, data)
self.assertTrue(target._started)
self.assertTrue(target._finished)
def prepare(self) -> None:
super(FileUploadHandler, self).prepare()
if self.request.method == "POST":
assert isinstance(self.request.connection, HTTP1Connection)
self.request.connection.set_max_body_size(self.max_upload_size)
tmpname = self.file_manager.gen_temp_upload_path()
self._targets = {
'root': ValueTarget(),
'print': ValueTarget(),
'path': ValueTarget(),
'checksum': ValueTarget(),
}
self._file = FileTarget(tmpname)
self._sha256_target = SHA256Target()
self._parser = StreamingFormDataParser(self.request.headers)
self._parser.register('file', self._file)
self._parser.register('file', self._sha256_target)
for name, target in self._targets.items():
self._parser.register(name, target)
def test_multiple_files(self):
txt_filename = 'file.txt'
png_filename = 'image-600x400.png'
with open(data_file_path(txt_filename), 'rb') as file_:
expected_txt = file_.read()
with open(data_file_path(png_filename), 'rb') as file_:
expected_png = file_.read()
txt_target = ValueTarget()
png_target = ValueTarget()
with open(data_file_path(txt_filename), 'rb') as txt_file, \
open(data_file_path(png_filename), 'rb') as png_file:
encoder = MultipartEncoder(
fields={
txt_filename: (txt_filename, txt_file,
'application/plain'),
png_filename: (png_filename, png_file, 'image/png')
})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register(txt_filename, txt_target)
parser.register(png_filename, png_target)
parser.data_received(encoder.to_string())
self.assertEqual(txt_target.value, expected_txt)
self.assertEqual(png_target.value, expected_png)
def subTest(self, test_idx, test_name, chunksize, original_data,
content_type, multipart_data, multipart_filename):
print(test_idx, '; name: ', test_name, '; data_size: ',
len(original_data), '; chunksize: ', chunksize)
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
target = ValueTarget()
parser.register('file', target)
remaining = len(multipart_data)
offset = 0
while (remaining):
step_size = min(remaining, chunksize)
parser.data_received(multipart_data[offset:offset + step_size])
offset += step_size
remaining -= step_size
self.assertEqual(offset, len(multipart_data))
self.assertEqual(target.multipart_filename, multipart_filename)
self.assertEqual(target._started, True)
self.assertEqual(target._finished, True)
result = target.value
self.assertEqual(len(result), len(original_data))
self.assertEqual(result, original_data)
def test_directory_upload(tmp_path):
data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"
Foo
--1234
Content-Disposition: form-data; name="files"; filename="cd.txt"
Bar
--1234--'''.replace(b'\n', b'\r\n')
target = DirectoryTarget(tmp_path)
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
with open(tmp_path / 'ab.txt') as file:
assert file.read() == 'Foo'
with open(tmp_path / 'cd.txt') as file:
assert file.read() == 'Bar'
assert target.multipart_filenames == ['ab.txt', 'cd.txt']
assert tmp_path
assert target._started
assert target._finished
def test_chunked_multiple():
expected_first_value = 'foo' * 1000
expected_second_value = 'bar' * 1000
expected_third_value = 'baz' * 1000
first = ValueTarget()
second = ValueTarget()
third = ValueTarget()
encoder = MultipartEncoder(
fields={
'first': expected_first_value,
'second': expected_second_value,
'third': expected_third_value,
}
)
body = encoder.to_string()
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type}
)
parser.register('first', first)
parser.register('second', second)
parser.register('third', third)
chunks = []
size = 100
while len(body):
chunks.append(body[:size])
body = body[size:]
for chunk in chunks:
parser.data_received(chunk)
assert first.value == expected_first_value.encode('utf-8')
assert second.value == expected_second_value.encode('utf-8')
assert third.value == expected_third_value.encode('utf-8')
def test_file_content_single():
filenames = (
'file.txt',
'image-600x400.png',
'image-2560x1600.png',
'empty.html',
'hyphen-hyphen.txt',
'LF.txt',
'CRLF.txt',
'1M.dat',
'1M-1.dat',
'1M+1.dat',
)
for filename in filenames:
with open_dataset(filename) as dataset_:
expected_value = dataset_.read()
content_type, body = encoded_dataset(filename)
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': content_type})
parser.register(filename, target)
parser.data_received(body)
assert target.value == expected_value
def test_special_filenames():
filenames = [
'a;b.txt',
'a"b.txt',
'a";b.txt',
'a;"b.txt',
'a";";.txt',
'a\\"b.txt',
'a\\b.txt',
]
for filename in filenames:
data = ('''\
--1234
Content-Disposition: form-data; name=files; filename={}
Foo
--1234--'''.format(filename).replace('\n', '\r\n').encode('utf-8'))
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b'Foo'
def test_multiple_files():
txt_filename = 'file.txt'
png_filename = 'image-600x400.png'
with open_dataset(txt_filename) as dataset_:
expected_txt = dataset_.read()
with open_dataset(png_filename) as dataset_:
expected_png = dataset_.read()
txt_target = ValueTarget()
png_target = ValueTarget()
with open_dataset(txt_filename) as txt_file, open_dataset(
png_filename) as png_file:
encoder = MultipartEncoder(
fields={
txt_filename: (txt_filename, txt_file, 'application/plain'),
png_filename: (png_filename, png_file, 'image/png'),
})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register(txt_filename, txt_target)
parser.register(png_filename, png_target)
parser.data_received(encoder.to_string())
assert txt_target.value == expected_txt
assert png_target.value == expected_png
def test_case_insensitive_content_disposition_header():
content_disposition_header = 'Content-Disposition'
for header in (
content_disposition_header,
content_disposition_header.lower(),
content_disposition_header.upper(),
):
data = b'''\
--1234
{header}: form-data; name="files"; filename="ab.txt"
Foo
--1234--'''.replace(b'\n', b'\r\n').replace(b'{header}',
header.encode('utf-8'))
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b'Foo'
def test_basic_multiple():
first = ValueTarget()
second = ValueTarget()
third = ValueTarget()
encoder = MultipartEncoder(
fields={'first': 'foo', 'second': 'bar', 'third': 'baz'}
)
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type}
)
parser.register('first', first)
parser.register('second', second)
parser.register('third', third)
parser.data_received(encoder.to_string())
assert first.value == b'foo'
assert second.value == b'bar'
assert third.value == b'baz'
def cgi_upload():
log('Upload file begin')
use_async_implementation = True
if use_async_implementation:
size = 0
file = NullTarget() if config.DISABLE_STORAGE else StorageFileTarget()
parser = StreamingFormDataParser(headers=bottle.request.headers)
parser.register('file', file)
while True:
chunk = bottle.request.environ['wsgi.input'].read(64 * 1024)
if not chunk:
break
parser.data_received(chunk)
size += len(chunk)
log('Uploaded request size: ' + str(size))
else:
size = 0
upload = bottle.request.files.get('file')
if upload is None:
raise Exception('ERROR! "file" multipart field was not found')
original_filename = upload.raw_filename
body = upload.file
with storage.open_file_writer(original_filename) as writer:
while True:
chunk = body.read(64 * 1024)
if not chunk:
break
if not config.DISABLE_STORAGE:
writer.write(chunk)
size += len(chunk)
log('Uploaded file size: ' + str(size))
return 'OK'
def test_register_after_data_received():
encoder = MultipartEncoder(fields={'name': 'hello'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.data_received(encoder.to_string())
with pytest.raises(ParseFailedException):
parser.register('name', ValueTarget())
def upload_page():
value = ValueTarget()
name = 'uploaded-file-tornado-{}.dat'.format(int(time()))
file = FileTarget(os.path.join(tempfile.gettempdir(), name))
parser = StreamingFormDataParser(headers=bottle.request.headers)
parser.register('name', value)
parser.register('file', file)
while True:
chunk = bottle.request.environ['wsgi.input'].read(8192)
if not chunk:
break
parser.data_received(chunk)
return {'name': value.value, 'filename': file.filename}
def test_parameter_contains_crlf():
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello\r\nworld'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
assert target.value == b'hello\r\nworld'
def test_basic_single(self):
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
self.assertEqual(target.value, b'hello world')
def test_parameter_starts_with_crlf(self):
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': '\r\nworld'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
self.assertEqual(target.value, b'\r\nworld')
def test_custom_target_exception():
target = CustomTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
data = encoder.to_string()
with pytest.raises(ValueError):
parser.data_received(data)
def test_basic_single():
target = ValueTarget()
encoder = MultipartEncoder(fields={'value': 'hello world'})
parser = StreamingFormDataParser(
headers={'Content-Type': encoder.content_type})
parser.register('value', target)
parser.data_received(encoder.to_string())
assert target.value == b'hello world'
assert target._started
assert target._finished
def test_filename_passed_to_target():
filename = 'file.txt'
content_type, body = encoded_dataset(filename)
target = ValueTarget()
assert not target.multipart_filename
parser = StreamingFormDataParser(headers={'Content-Type': content_type})
parser.register(filename, target)
parser.data_received(body)
assert target.multipart_filename == filename
def mask(filename):
basename = ntpath.basename(filename)
process_files = [(filename, 'masked')]
for file_name, masked_folder in process_files:
files = {'file': open(file_name, 'rb'), 'context': context}
logging.info(f"POST: sending '{file_name}' to {url}")
with s.post(url, files=files, stream=True) as r:
if r.status_code >= 300:
logging.info(
f"Failed with status {r.status_code}:\n\n{r.json()}")
break
logging.info(
f"Extracting 'masked_{basename}' and 'masked_{basename}_results.json' into {masked_folder}."
)
parser = StreamingFormDataParser(headers=r.headers)
parser.register('file',
FileTarget(f'{masked_folder}/masked_{basename}'))
parser.register(
'results',
FileTarget(f'{masked_folder}/masked_{basename}_results.json'))
for chunk in r.iter_content(4096):
parser.data_received(chunk)
def test_missing_headers():
data = '''\
--1234
Foo
--1234--'''.replace('\n', '\r\n').encode('utf-8')
target = ValueTarget()
parser = StreamingFormDataParser(
headers={'Content-Type': 'multipart/form-data; boundary=1234'})
parser.register('files', target)
parser.data_received(data)
assert target.value == b''