def prepare(self): if self.request.method == "POST": self.request.connection.set_max_body_size(self.max_upload_size) tmpname = self.file_manager.gen_temp_upload_path() self._targets = { 'root': ValueTarget(), 'print': ValueTarget(), 'path': ValueTarget(), } self._file = FileTarget(tmpname) self._parser = StreamingFormDataParser(self.request.headers) self._parser.register('file', self._file) for name, target in self._targets.items(): self._parser.register(name, target)
def test_break_chunk_at_boundary(): expected_first_value = 'hello' * 500 expected_second_value = 'hello' * 500 first = ValueTarget() second = ValueTarget() encoder = MultipartEncoder( fields={'first': 'hello' * 500, 'second': 'hello' * 500} ) body = encoder.to_string() boundary = encoder.boundary.encode('utf-8') parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type} ) parser.register('first', first) parser.register('second', second) index = body[50:].index(boundary) + 5 parser.data_received(body[:index]) parser.data_received(body[index:]) assert first.value == expected_first_value.encode('utf-8') assert second.value == expected_second_value.encode('utf-8')
def test_multiple_targets(): filename = 'image-600x400.png' with open_dataset(filename) as dataset_: expected_data = dataset_.read() value_target = ValueTarget() sha256_target = SHA256Target() with open_dataset(filename) as file_: encoder = MultipartEncoder( fields={filename: (filename, file_, 'image/png')} ) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type} ) parser.register(filename, value_target) parser.register(filename, sha256_target) assert not value_target.value assert sha256_target.value == hashlib.sha256(b'').hexdigest() parser.data_received(encoder.to_string()) assert value_target.value == expected_data assert sha256_target.value == hashlib.sha256(expected_data).hexdigest()
def prepare(self): self.request.connection.set_max_body_size(10_000_000_000) self.streaming_parser = StreamingFormDataParser(self.request.headers) self.uploaded_file_tmp = tempfile.NamedTemporaryFile(prefix='upload_') self.uploaded_file = HashedFileTarget(self.uploaded_file_tmp.name) self.streaming_parser.register('rpz_file', self.uploaded_file) self.rpz_url = ValueTarget() self.streaming_parser.register('rpz_url', self.rpz_url) self.not_permanent = ValueTarget() self.streaming_parser.register('not_permanent', self.not_permanent) self.sent_xsrf_token = ValueTarget() self.streaming_parser.register('_xsrf', self.sent_xsrf_token)
def test_target_raises_exception(self): filename = 'file.txt' content_type, body = encoded_dataset(filename) class BadTarget(BaseTarget): def data_received(self, data): raise ValueError() target = BadTarget() parser = StreamingFormDataParser( headers={'Content-Type': content_type}) parser.register(filename, target) self.assertRaises(ValueError, parser.data_received, body)
class UploadHandler(RequestHandler): def prepare(self): self.value = ValueTarget() self.file_ = FileTarget('/tmp/file-{}.dat'.format(int(time()))) self._parser = StreamingFormDataParser(headers=self.request.headers) self._parser.register('name', self.value) self._parser.register('file', self.file_) def data_received(self, chunk): self._parser.data_received(chunk) def post(self): self.render('upload.html', name=self.value.value, filename=self.file_.filename)
def test_incorrect_content_type(): for value in ( 'multipart/mixed; boundary=1234', 'multipart/form-data', 'multipart/form-data; delimiter=1234', ): with pytest.raises(ParseFailedException): StreamingFormDataParser({'Content-Type': value})
class UploadHandler(RequestHandler): def prepare(self): self.value = ValueTarget() name = 'uploaded-file-tornado-{}.dat'.format(int(time())) self.file_ = FileTarget(os.path.join(tempfile.gettempdir(), name)) self._parser = StreamingFormDataParser(headers=self.request.headers) self._parser.register('name', self.value) self._parser.register('file', self.file_) def data_received(self, chunk): self._parser.data_received(chunk) def post(self): self.render('upload.html', name=self.value.value, filename=self.file_.filename)
def test_invalid_content_disposition(self): data = b'''\ --1234 Content-Disposition: invalid; name="files"; filename="ab.txt" Foo --1234--'''.replace(b'\n', b'\r\n') target = ValueTarget() parser = StreamingFormDataParser( headers={'Content-Type': 'multipart/form-data; boundary=1234'}) parser.register('files', target) self.assertRaises(ParseFailedException, parser.data_received, data) self.assertEqual(target.value, b'')
def test_file_target_exceeds_max_size(self): data = b'''\ --1234 Content-Disposition: form-data; name="files"; filename="ab.txt" Foo --1234--'''.replace(b'\n', b'\r\n') target = FileTarget('/tmp/file.txt', validator=MaxSizeValidator(1)) parser = StreamingFormDataParser( headers={'Content-Type': 'multipart/form-data; boundary=1234'}) parser.register('files', target) self.assertRaises(ValidationError, parser.data_received, data) self.assertTrue(target._started) self.assertTrue(target._finished)
def prepare(self) -> None: super(FileUploadHandler, self).prepare() if self.request.method == "POST": assert isinstance(self.request.connection, HTTP1Connection) self.request.connection.set_max_body_size(self.max_upload_size) tmpname = self.file_manager.gen_temp_upload_path() self._targets = { 'root': ValueTarget(), 'print': ValueTarget(), 'path': ValueTarget(), 'checksum': ValueTarget(), } self._file = FileTarget(tmpname) self._sha256_target = SHA256Target() self._parser = StreamingFormDataParser(self.request.headers) self._parser.register('file', self._file) self._parser.register('file', self._sha256_target) for name, target in self._targets.items(): self._parser.register(name, target)
def test_multiple_files(self): txt_filename = 'file.txt' png_filename = 'image-600x400.png' with open(data_file_path(txt_filename), 'rb') as file_: expected_txt = file_.read() with open(data_file_path(png_filename), 'rb') as file_: expected_png = file_.read() txt_target = ValueTarget() png_target = ValueTarget() with open(data_file_path(txt_filename), 'rb') as txt_file, \ open(data_file_path(png_filename), 'rb') as png_file: encoder = MultipartEncoder( fields={ txt_filename: (txt_filename, txt_file, 'application/plain'), png_filename: (png_filename, png_file, 'image/png') }) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register(txt_filename, txt_target) parser.register(png_filename, png_target) parser.data_received(encoder.to_string()) self.assertEqual(txt_target.value, expected_txt) self.assertEqual(png_target.value, expected_png)
def subTest(self, test_idx, test_name, chunksize, original_data, content_type, multipart_data, multipart_filename): print(test_idx, '; name: ', test_name, '; data_size: ', len(original_data), '; chunksize: ', chunksize) parser = StreamingFormDataParser( headers={'Content-Type': content_type}) target = ValueTarget() parser.register('file', target) remaining = len(multipart_data) offset = 0 while (remaining): step_size = min(remaining, chunksize) parser.data_received(multipart_data[offset:offset + step_size]) offset += step_size remaining -= step_size self.assertEqual(offset, len(multipart_data)) self.assertEqual(target.multipart_filename, multipart_filename) self.assertEqual(target._started, True) self.assertEqual(target._finished, True) result = target.value self.assertEqual(len(result), len(original_data)) self.assertEqual(result, original_data)
def test_directory_upload(tmp_path): data = b'''\ --1234 Content-Disposition: form-data; name="files"; filename="ab.txt" Foo --1234 Content-Disposition: form-data; name="files"; filename="cd.txt" Bar --1234--'''.replace(b'\n', b'\r\n') target = DirectoryTarget(tmp_path) parser = StreamingFormDataParser( headers={'Content-Type': 'multipart/form-data; boundary=1234'}) parser.register('files', target) parser.data_received(data) with open(tmp_path / 'ab.txt') as file: assert file.read() == 'Foo' with open(tmp_path / 'cd.txt') as file: assert file.read() == 'Bar' assert target.multipart_filenames == ['ab.txt', 'cd.txt'] assert tmp_path assert target._started assert target._finished
def test_chunked_multiple(): expected_first_value = 'foo' * 1000 expected_second_value = 'bar' * 1000 expected_third_value = 'baz' * 1000 first = ValueTarget() second = ValueTarget() third = ValueTarget() encoder = MultipartEncoder( fields={ 'first': expected_first_value, 'second': expected_second_value, 'third': expected_third_value, } ) body = encoder.to_string() parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type} ) parser.register('first', first) parser.register('second', second) parser.register('third', third) chunks = [] size = 100 while len(body): chunks.append(body[:size]) body = body[size:] for chunk in chunks: parser.data_received(chunk) assert first.value == expected_first_value.encode('utf-8') assert second.value == expected_second_value.encode('utf-8') assert third.value == expected_third_value.encode('utf-8')
def test_file_content_single(): filenames = ( 'file.txt', 'image-600x400.png', 'image-2560x1600.png', 'empty.html', 'hyphen-hyphen.txt', 'LF.txt', 'CRLF.txt', '1M.dat', '1M-1.dat', '1M+1.dat', ) for filename in filenames: with open_dataset(filename) as dataset_: expected_value = dataset_.read() content_type, body = encoded_dataset(filename) target = ValueTarget() parser = StreamingFormDataParser( headers={'Content-Type': content_type}) parser.register(filename, target) parser.data_received(body) assert target.value == expected_value
def test_special_filenames(): filenames = [ 'a;b.txt', 'a"b.txt', 'a";b.txt', 'a;"b.txt', 'a";";.txt', 'a\\"b.txt', 'a\\b.txt', ] for filename in filenames: data = ('''\ --1234 Content-Disposition: form-data; name=files; filename={} Foo --1234--'''.format(filename).replace('\n', '\r\n').encode('utf-8')) target = ValueTarget() parser = StreamingFormDataParser( headers={'Content-Type': 'multipart/form-data; boundary=1234'}) parser.register('files', target) parser.data_received(data) assert target.value == b'Foo'
def test_multiple_files(): txt_filename = 'file.txt' png_filename = 'image-600x400.png' with open_dataset(txt_filename) as dataset_: expected_txt = dataset_.read() with open_dataset(png_filename) as dataset_: expected_png = dataset_.read() txt_target = ValueTarget() png_target = ValueTarget() with open_dataset(txt_filename) as txt_file, open_dataset( png_filename) as png_file: encoder = MultipartEncoder( fields={ txt_filename: (txt_filename, txt_file, 'application/plain'), png_filename: (png_filename, png_file, 'image/png'), }) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register(txt_filename, txt_target) parser.register(png_filename, png_target) parser.data_received(encoder.to_string()) assert txt_target.value == expected_txt assert png_target.value == expected_png
def test_case_insensitive_content_disposition_header(): content_disposition_header = 'Content-Disposition' for header in ( content_disposition_header, content_disposition_header.lower(), content_disposition_header.upper(), ): data = b'''\ --1234 {header}: form-data; name="files"; filename="ab.txt" Foo --1234--'''.replace(b'\n', b'\r\n').replace(b'{header}', header.encode('utf-8')) target = ValueTarget() parser = StreamingFormDataParser( headers={'Content-Type': 'multipart/form-data; boundary=1234'}) parser.register('files', target) parser.data_received(data) assert target.value == b'Foo'
def test_basic_multiple(): first = ValueTarget() second = ValueTarget() third = ValueTarget() encoder = MultipartEncoder( fields={'first': 'foo', 'second': 'bar', 'third': 'baz'} ) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type} ) parser.register('first', first) parser.register('second', second) parser.register('third', third) parser.data_received(encoder.to_string()) assert first.value == b'foo' assert second.value == b'bar' assert third.value == b'baz'
def cgi_upload(): log('Upload file begin') use_async_implementation = True if use_async_implementation: size = 0 file = NullTarget() if config.DISABLE_STORAGE else StorageFileTarget() parser = StreamingFormDataParser(headers=bottle.request.headers) parser.register('file', file) while True: chunk = bottle.request.environ['wsgi.input'].read(64 * 1024) if not chunk: break parser.data_received(chunk) size += len(chunk) log('Uploaded request size: ' + str(size)) else: size = 0 upload = bottle.request.files.get('file') if upload is None: raise Exception('ERROR! "file" multipart field was not found') original_filename = upload.raw_filename body = upload.file with storage.open_file_writer(original_filename) as writer: while True: chunk = body.read(64 * 1024) if not chunk: break if not config.DISABLE_STORAGE: writer.write(chunk) size += len(chunk) log('Uploaded file size: ' + str(size)) return 'OK'
def test_register_after_data_received(): encoder = MultipartEncoder(fields={'name': 'hello'}) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.data_received(encoder.to_string()) with pytest.raises(ParseFailedException): parser.register('name', ValueTarget())
def upload_page(): value = ValueTarget() name = 'uploaded-file-tornado-{}.dat'.format(int(time())) file = FileTarget(os.path.join(tempfile.gettempdir(), name)) parser = StreamingFormDataParser(headers=bottle.request.headers) parser.register('name', value) parser.register('file', file) while True: chunk = bottle.request.environ['wsgi.input'].read(8192) if not chunk: break parser.data_received(chunk) return {'name': value.value, 'filename': file.filename}
def test_parameter_contains_crlf(): target = ValueTarget() encoder = MultipartEncoder(fields={'value': 'hello\r\nworld'}) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register('value', target) parser.data_received(encoder.to_string()) assert target.value == b'hello\r\nworld'
def test_basic_single(self): target = ValueTarget() encoder = MultipartEncoder(fields={'value': 'hello world'}) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register('value', target) parser.data_received(encoder.to_string()) self.assertEqual(target.value, b'hello world')
def test_parameter_starts_with_crlf(self): target = ValueTarget() encoder = MultipartEncoder(fields={'value': '\r\nworld'}) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register('value', target) parser.data_received(encoder.to_string()) self.assertEqual(target.value, b'\r\nworld')
def test_custom_target_exception(): target = CustomTarget() encoder = MultipartEncoder(fields={'value': 'hello world'}) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register('value', target) data = encoder.to_string() with pytest.raises(ValueError): parser.data_received(data)
def test_basic_single(): target = ValueTarget() encoder = MultipartEncoder(fields={'value': 'hello world'}) parser = StreamingFormDataParser( headers={'Content-Type': encoder.content_type}) parser.register('value', target) parser.data_received(encoder.to_string()) assert target.value == b'hello world' assert target._started assert target._finished
def test_filename_passed_to_target(): filename = 'file.txt' content_type, body = encoded_dataset(filename) target = ValueTarget() assert not target.multipart_filename parser = StreamingFormDataParser(headers={'Content-Type': content_type}) parser.register(filename, target) parser.data_received(body) assert target.multipart_filename == filename
def mask(filename): basename = ntpath.basename(filename) process_files = [(filename, 'masked')] for file_name, masked_folder in process_files: files = {'file': open(file_name, 'rb'), 'context': context} logging.info(f"POST: sending '{file_name}' to {url}") with s.post(url, files=files, stream=True) as r: if r.status_code >= 300: logging.info( f"Failed with status {r.status_code}:\n\n{r.json()}") break logging.info( f"Extracting 'masked_{basename}' and 'masked_{basename}_results.json' into {masked_folder}." ) parser = StreamingFormDataParser(headers=r.headers) parser.register('file', FileTarget(f'{masked_folder}/masked_{basename}')) parser.register( 'results', FileTarget(f'{masked_folder}/masked_{basename}_results.json')) for chunk in r.iter_content(4096): parser.data_received(chunk)
def test_missing_headers(): data = '''\ --1234 Foo --1234--'''.replace('\n', '\r\n').encode('utf-8') target = ValueTarget() parser = StreamingFormDataParser( headers={'Content-Type': 'multipart/form-data; boundary=1234'}) parser.register('files', target) parser.data_received(data) assert target.value == b''