def check_read_file_with_chunks(self, file_size, read_size):
#print "===== Testing with file_size=",file_size,"read_size=",read_size
boundary="deterministic-test-boundary"
a_file = LargeFileMock(file_size)
parts = {'some_field': 'this is the value...',
'some_file': a_file.read(),
}
expected_bytes = encode_multipart_formdata(parts, boundary)[0]
content_length = len(expected_bytes)
# Now read from our encoder :
a_file = LargeFileMock(file_size)
parts = {'some_field': 'this is the value...',
'some_file': a_file,
}
encoder = MultipartEncoder(parts, boundary=boundary)
raw_bytes_count = 0
while True:
data = encoder.read(read_size)
if not data:
break
#print "read",len(data),"bytes : ",repr(data)
assert data == expected_bytes[raw_bytes_count:raw_bytes_count+len(data)]
raw_bytes_count += len(data)
#if raw_bytes_count != content_length:
# print "Test failed with file_size=",file_size,"and read_size=",read_size
assert raw_bytes_count == content_length
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_reads_file_from_url_wrapper(self):
s = requests.Session()
recorder = get_betamax(s)
url = ('https://stxnext.com/static/img/logo.830ebe551641.svg')
with recorder.use_cassette('file_for_download'):
m = MultipartEncoder([('field', 'foo'),
('file', FileFromURLWrapper(url))])
assert m.read() is not None
def test_handles_empty_unicode_values(self):
"""Verify that the Encoder can handle empty unicode strings.
See https://github.com/requests/toolbelt/issues/46 for
more context.
"""
fields = [(b'test'.decode('utf-8'), b''.decode('utf-8'))]
m = MultipartEncoder(fields=fields)
assert len(m.read()) > 0
def test_reads_file_from_url_wrapper(self):
s = requests.Session()
recorder = get_betamax(s)
url = ('https://stxnext.com/static/img/logo.830ebe551641.svg')
with recorder.use_cassette(
'file_for_download'):
m = MultipartEncoder(
[('field', 'foo'), ('file', FileFromURLWrapper(url))])
assert m.read() is not None
def test_handles_empty_unicode_values(self):
"""Verify that the Encoder can handle empty unicode strings.
See https://github.com/requests/toolbelt/issues/46 for
more context.
"""
fields = [(b'test'.decode('utf-8'), b''.decode('utf-8'))]
m = MultipartEncoder(fields=fields)
assert len(m.read()) > 0
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_accepts_custom_content_type(self):
"""Verify that the Encoder handles custom content-types.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [(b'test'.decode('utf-8'),
(b'filename'.decode('utf-8'), b'filecontent',
b'application/json'.decode('utf-8')))]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('Content-Type: application/json\r\n') > 0
def test_accepts_custom_headers(self):
"""Verify that the Encoder handles custom headers.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [(b'test'.decode('utf-8'),
(b'filename'.decode('utf-8'), b'filecontent',
b'application/json'.decode('utf-8'), {
'X-My-Header': 'my-value'
}))]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('X-My-Header: my-value\r\n') > 0
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
read_size = 1024 * 1024 * 128
while True:
read = encoder.read(read_size)
if not read:
break
assert encoder._buffer.tell() <= read_size
def test_accepts_custom_content_type(self):
"""Verify that the Encoder handles custom content-types.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [
(b'test'.decode('utf-8'), (b'filename'.decode('utf-8'),
b'filecontent',
b'application/json'.decode('utf-8')))
]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('Content-Type: application/json\r\n') > 0
def test_accepts_custom_headers(self):
"""Verify that the Encoder handles custom headers.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [
(b'test'.decode('utf-8'), (b'filename'.decode('utf-8'),
b'filecontent',
b'application/json'.decode('utf-8'),
{'X-My-Header': 'my-value'}))
]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('X-My-Header: my-value\r\n') > 0
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {
'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
read_size = 1024 * 1024 * 128
while True:
read = encoder.read(read_size)
if not read:
break
assert encoder._buffer.tell() <= read_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {"test": "t" * 8100}
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = encoder.len
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {
'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = encoder.len
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 8100
}
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {"test": "t" * 100}
for x in range(30):
fields['f%d' % x] = ('test',
open('tests/test_multipart_encoder.py', 'rb'))
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 100
}
for x in range(30):
fields['f%d' % x] = (
'test', open('tests/test_multipart_encoder.py', 'rb')
)
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {
'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = len(encoder)
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_length_is_correct(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert len(encoded) == len(self.instance)
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'),
('file', ('filename', fd, 'text/plain'))])
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([('field', s.decode('utf-8'))])
assert m.read() is not None
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {"test": "t" * 100}
for x in range(30):
fields['f%d' % x] = ('test',
open('tests/test_multipart_encoder.py', 'rb'))
m = MultipartEncoder(fields=fields)
total_size = len(m)
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {"test": "t" * 8100}
m = MultipartEncoder(fields=fields)
total_size = len(m)
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_handles_empty_unicode_values(self):
"""Verify that the Encoder can handle empty unicode strings.
See https://github.com/sigmavirus24/requests-toolbelt/issues/46 for
more context.
"""
fields = [(b'test'.decode('utf-8'), b''.decode('utf-8'))]
m = MultipartEncoder(fields=fields)
assert len(m.read()) > 0
def test_no_parts(self):
fields = []
boundary = '--90967316f8404798963cce746a4f4ef9'
m = MultipartEncoder(fields=fields, boundary=boundary)
output = m.read().decode('utf-8')
assert output == '----90967316f8404798963cce746a4f4ef9--\r\n'
def test_read(self):
new_encoder = MultipartEncoder(self.fields, self.boundary)
assert new_encoder.read() == self.monitor.read()
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'),
('file', ('filename', fd, 'text/plain'))])
assert m.read() is not None
datarobot_api_token = os.getenv("DATAROBOT_API_TOKEN")
datarobot_endpoint = os.getenv("DATAROBOT_ENDPOINT")
dataset_file_path = Path(".").parent.joinpath("data", "auto-mpg.csv")
# Step 1: Upload a dataset
form_data = {
"file": (dataset_file_path.name, dataset_file_path.open("rb")),
"projectName": "Auto MPG PY",
}
encoder = MultipartEncoder(fields=form_data)
headers = {}
headers.update({"Authorization": f"Bearer {datarobot_api_token}"})
headers.update({"Content-Type": encoder.content_type})
response = requests.post(f"{datarobot_endpoint}/projects/",
headers=headers,
data=encoder.read())
try:
assert response.status_code == 202
except AssertionError:
print(f"Status code: {response.status_code}, Reason: {response.reason}, "
f"Details: {response.content}")
else:
# if you are not using app2.datatrobot.com endpoints
# change `location` to `Location`
while True:
project_id = (requests.get(response.headers["location"],
headers=headers).json().get("id", None))
if project_id is not None:
print(f"Project setup complete. Project ID: {project_id}")
break
else:
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = filepost.encode_multipart_formdata(self.parts,
self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {
'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = encoder.len
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_length_is_correct(self):
encoded = filepost.encode_multipart_formdata(self.parts,
self.boundary)[0]
assert len(encoded) == self.instance.len
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_file_from_url_wrapper(self):
s = requests.Session()
recorder = get_betamax(s)
url = ('https://stxnext.com/static/img/logo.830ebe551641.svg')
with recorder.use_cassette('file_for_download'):
m = MultipartEncoder([('field', 'foo'),
('file', FileFromURLWrapper(url))])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'),
('file', ('filename', fd, 'text/plain'))])
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([('field', s.decode('utf-8'))])
assert m.read() is not None
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {"test": "t" * 100}
for x in range(30):
fields['f%d' % x] = ('test',
open('tests/test_multipart_encoder.py', 'rb'))
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {"test": "t" * 8100}
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_handles_empty_unicode_values(self):
"""Verify that the Encoder can handle empty unicode strings.
See https://github.com/requests/toolbelt/issues/46 for
more context.
"""
fields = [(b'test'.decode('utf-8'), b''.decode('utf-8'))]
m = MultipartEncoder(fields=fields)
assert len(m.read()) > 0
def test_accepts_custom_content_type(self):
"""Verify that the Encoder handles custom content-types.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [(b'test'.decode('utf-8'),
(b'filename'.decode('utf-8'), b'filecontent',
b'application/json'.decode('utf-8')))]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('Content-Type: application/json\r\n') > 0
def test_accepts_custom_headers(self):
"""Verify that the Encoder handles custom headers.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [(b'test'.decode('utf-8'),
(b'filename'.decode('utf-8'), b'filecontent',
b'application/json'.decode('utf-8'), {
'X-My-Header': 'my-value'
}))]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('X-My-Header: my-value\r\n') > 0
def test_no_parts(self):
fields = []
boundary = '--90967316f8404798963cce746a4f4ef9'
m = MultipartEncoder(fields=fields, boundary=boundary)
output = m.read().decode('utf-8')
assert output == '----90967316f8404798963cce746a4f4ef9--\r\n'
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = filepost.encode_multipart_formdata(self.parts,
self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = encoder.len
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_length_is_correct(self):
encoded = filepost.encode_multipart_formdata(self.parts,
self.boundary)[0]
assert len(encoded) == self.instance.len
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder(
[('field', 'foo'), ('file', ('filename', fd, 'text/plain'))]
)
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([
('field', s.decode('utf-8'))
])
assert m.read() is not None
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 100
}
for x in range(30):
fields['f%d' % x] = (
'test', open('tests/test_multipart_encoder.py', 'rb')
)
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 8100
}
m = MultipartEncoder(fields=fields)
total_size = m.len
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_handles_empty_unicode_values(self):
"""Verify that the Encoder can handle empty unicode strings.
See https://github.com/requests/toolbelt/issues/46 for
more context.
"""
fields = [(b'test'.decode('utf-8'), b''.decode('utf-8'))]
m = MultipartEncoder(fields=fields)
assert len(m.read()) > 0
def test_accepts_custom_content_type(self):
"""Verify that the Encoder handles custom content-types.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [
(b'test'.decode('utf-8'), (b'filename'.decode('utf-8'),
b'filecontent',
b'application/json'.decode('utf-8')))
]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('Content-Type: application/json\r\n') > 0
def test_accepts_custom_headers(self):
"""Verify that the Encoder handles custom headers.
See https://github.com/requests/toolbelt/issues/52
"""
fields = [
(b'test'.decode('utf-8'), (b'filename'.decode('utf-8'),
b'filecontent',
b'application/json'.decode('utf-8'),
{'X-My-Header': 'my-value'}))
]
m = MultipartEncoder(fields=fields)
output = m.read().decode('utf-8')
assert output.index('X-My-Header: my-value\r\n') > 0
def test_read(self):
new_encoder = MultipartEncoder(self.fields, self.boundary)
assert new_encoder.read() == self.monitor.read()
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([('field', s.decode('utf-8'))])
assert m.read() is not None
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock(123456789)
parts = {'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = len(encoder)
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_streams_its_data_with_correct_length(self):
for i in range(0, 100): # or more than 100 to increase fuzzing strength
file_size = random.randint(0, 12345)
if random.random() < 0.1:
file_size = 0 # sometimes we check with an empty file
self.check_read_file_with_chunks(file_size, read_size=1)
self.check_read_file_with_chunks(file_size, read_size=2)
self.check_read_file_with_chunks(file_size, read_size=3)
read_size = random.randint(0, 2*file_size)
self.check_read_file_with_chunks(file_size, read_size=1)
for read_size in range(file_size - 10, file_size + 200):
if read_size < -1 or read_size == 0:
continue
self.check_read_file_with_chunks(file_size, read_size)
def check_read_file_with_chunks(self, file_size, read_size):
#print "===== Testing with file_size=",file_size,"read_size=",read_size
boundary="deterministic-test-boundary"
a_file = LargeFileMock(file_size)
parts = {'some_field': 'this is the value...',
'some_file': a_file.read(),
}
expected_bytes = encode_multipart_formdata(parts, boundary)[0]
content_length = len(expected_bytes)
# Now read from our encoder :
a_file = LargeFileMock(file_size)
parts = {'some_field': 'this is the value...',
'some_file': a_file,
}
encoder = MultipartEncoder(parts, boundary=boundary)
raw_bytes_count = 0
while True:
data = encoder.read(read_size)
if not data:
break
#print "read",len(data),"bytes : ",repr(data)
assert data == expected_bytes[raw_bytes_count:raw_bytes_count+len(data)]
raw_bytes_count += len(data)
#if raw_bytes_count != content_length:
# print "Test failed with file_size=",file_size,"and read_size=",read_size
assert raw_bytes_count == content_length
def test_length_is_correct(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert len(encoded) == len(self.instance)
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder(
[('field', 'foo'), ('file', ('filename', fd, 'text/plain'))]
)
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([
('field', s.decode('utf-8'))
])
assert m.read() is not None
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 100
}
for x in range(30):
fields['f%d' % x] = (
'test', open('tests/test_multipart_encoder.py', 'rb')
)
m = MultipartEncoder(fields=fields)
total_size = len(m)
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 8100
}
m = MultipartEncoder(fields=fields)
total_size = len(m)
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_no_parts(self):
fields = []
boundary = '--90967316f8404798963cce746a4f4ef9'
m = MultipartEncoder(fields=fields, boundary=boundary)
output = m.read().decode('utf-8')
assert output == '----90967316f8404798963cce746a4f4ef9--\r\n'
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder(
[('field', 'foo'), ('file', ('filename', fd, 'text/plain'))]
)
assert m.read() is not None
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
read_size = 1024 * 1024 * 128
while True:
read = encoder.read(read_size)
if not read:
break
assert encoder._buffer.tell() <= read_size
def test_length_is_correct(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert len(encoded) == len(self.instance)
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder(
[('field', 'foo'), ('file', ('filename', fd, 'text/plain'))]
)
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([
('field', s.decode('utf-8'))
])
assert m.read() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([
('field', s.decode('utf-8'))
])
assert m.read() is not None
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {
'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
read_size = 1024 * 1024 * 128
while True:
read = encoder.read(read_size)
if not read:
break
assert encoder._buffer.tell() <= read_size
def test_length_is_correct(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert len(encoded) == len(self.instance)
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n').encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'),
('file', ('filename', fd, 'text/plain'))])
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([('field', s.decode('utf-8'))])
assert m.read() is not None
class TestMultipartEncoder(unittest.TestCase):
def setUp(self):
self.parts = [('field', 'value'), ('other_field', 'other_value')]
self.boundary = 'this-is-a-boundary'
self.instance = MultipartEncoder(self.parts, boundary=self.boundary)
def test_to_string(self):
assert self.instance.to_string() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="other_field"\r\n\r\n'
'other_value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_content_type(self):
expected = 'multipart/form-data; boundary=this-is-a-boundary'
assert self.instance.content_type == expected
def test_encodes_data_the_same(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert encoded == self.instance.read()
def test_streams_its_data(self):
large_file = LargeFileMock()
parts = {'some field': 'value',
'some file': large_file,
}
encoder = MultipartEncoder(parts)
total_size = len(encoder)
read_size = 1024 * 1024 * 128
already_read = 0
while True:
read = encoder.read(read_size)
already_read += len(read)
if not read:
break
assert encoder._buffer.tell() <= read_size
assert already_read == total_size
def test_length_is_correct(self):
encoded = encode_multipart_formdata(self.parts, self.boundary)[0]
assert len(encoded) == len(self.instance)
def test_encodes_with_readable_data(self):
s = io.BytesIO(b'value')
m = MultipartEncoder([('field', s)], boundary=self.boundary)
assert m.read() == (
'--this-is-a-boundary\r\n'
'Content-Disposition: form-data; name="field"\r\n\r\n'
'value\r\n'
'--this-is-a-boundary--\r\n'
).encode()
def test_reads_open_file_objects(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.read() is not None
def test_reads_open_file_objects_with_a_specified_filename(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder(
[('field', 'foo'), ('file', ('filename', fd, 'text/plain'))]
)
assert m.read() is not None
def test_reads_open_file_objects_using_to_string(self):
with open('setup.py', 'rb') as fd:
m = MultipartEncoder([('field', 'foo'), ('file', fd)])
assert m.to_string() is not None
def test_handles_encoded_unicode_strings(self):
m = MultipartEncoder([
('field',
b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3')
])
assert m.read() is not None
def test_handles_uncode_strings(self):
s = b'this is a unicode string: \xc3\xa9 \xc3\xa1 \xc7\xab \xc3\xb3'
m = MultipartEncoder([
('field', s.decode('utf-8'))
])
assert m.read() is not None
def test_regresion_1(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 100
}
for x in range(30):
fields['f%d' % x] = (
'test', open('tests/test_multipart_encoder.py', 'rb')
)
m = MultipartEncoder(fields=fields)
total_size = len(m)
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
def test_regression_2(self):
"""Ensure issue #31 doesn't ever happen again."""
fields = {
"test": "t" * 8100
}
m = MultipartEncoder(fields=fields)
total_size = len(m)
blocksize = 8192
read_so_far = 0
while True:
data = m.read(blocksize)
if not data:
break
read_so_far += len(data)
assert read_so_far == total_size
