def test_limit_multiread():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj, 7)
assert stream.read(4) == 'this'
assert stream.read(3) == ' is'
assert stream.read(2) == ''
assert 7 == stream.tell()
def test_limit_multiread():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj, 7)
assert stream.read(4) == b"this"
assert stream.read(3) == b" is"
assert stream.read(2) == b""
assert 7 == stream.tell()
def stream_upload_chunk(self,
uuid,
offset,
length,
in_fp,
storage_metadata,
content_type=None):
if length == 0:
return 0, storage_metadata, None
upload_blob_path = self._upload_blob_path_from_uuid(uuid)
new_metadata = copy.deepcopy(storage_metadata)
total_bytes_written = 0
while True:
current_length = length - total_bytes_written
max_length = (min(current_length, _MAX_BLOCK_SIZE)
if length != READ_UNTIL_END else _MAX_BLOCK_SIZE)
if max_length <= 0:
break
limited = LimitingStream(in_fp, max_length, seekable=False)
# Note: Azure fails if a zero-length block is uploaded, so we read all the data here,
# and, if there is none, terminate early.
block_data = b""
for chunk in iter(lambda: limited.read(4096), b""):
block_data += chunk
if len(block_data) == 0:
break
block_index = len(new_metadata[_BLOCKS_KEY])
block_id = format(block_index, "05")
new_metadata[_BLOCKS_KEY].append(block_id)
try:
self._blob_service.put_block(
self._azure_container,
upload_blob_path,
block_data,
block_id,
validate_content=True,
)
except AzureException as ae:
logger.exception(
"Exception when trying to stream_upload_chunk block %s for %s",
block_id, uuid)
return total_bytes_written, new_metadata, ae
bytes_written = len(block_data)
total_bytes_written += bytes_written
if bytes_written == 0 or bytes_written < max_length:
break
if content_type is not None:
new_metadata[_CONTENT_TYPE_KEY] = content_type
return total_bytes_written, new_metadata, None
def test_non_filelike_obj_read_limit():
from gunicorn.http.body import Body
content = b"this will not really be a real fileobj"
fileobj = BytesIO(content)
body = Body(fileobj)
ls = LimitingStream(body)
assert ls.readable()
assert ls.read(-1) == content
assert len(content) == ls.tell()
def test_seek_to_tell():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj, 3)
stream.seek(stream.tell())
assert stream.read(4) == b"thi"
assert 3 == stream.tell()
def test_seek_to_tell():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj, 3)
stream.seek(stream.tell())
assert stream.read(4) == 'thi'
assert 3 == stream.tell()
def test_seek_pastlimit():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj, 3)
stream.seek(4)
assert stream.read(1) == b""
assert 3 == stream.tell()
def test_seek_pastlimit():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj, 3)
stream.seek(4)
assert stream.read(1) == ''
assert 3 == stream.tell()
def test_seek():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj)
stream.seek(2)
assert stream.read(2) == 'is'
assert 4 == stream.tell()
def test_seek_withlimit():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj, 3)
stream.seek(2)
assert stream.read(2) == 'i'
assert 3 == stream.tell()
def test_seek_withlimit():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj, 3)
stream.seek(2)
assert stream.read(2) == b"i"
assert 3 == stream.tell()
def test_seek():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj)
stream.seek(2)
assert stream.read(2) == b"is"
assert 4 == stream.tell()
def test_simplelimit_readdefined():
fileobj = StringIO("this is a cool test")
stream = LimitingStream(fileobj, 4)
assert stream.read(2) == "th"
assert 2 == stream.tell()
def test_non_filelike_obj_read():
from gunicorn.http.body import Body
content = b"this will not really be a real fileobj"
fileobj = BytesIO(content)
# Limited
body1 = Body(fileobj)
ls1 = LimitingStream(body1, 4)
assert ls1.readable()
resp1 = b""
while True:
buf = ls1.read(-1)
if not buf:
break
resp1 += buf
assert resp1 == content[:4]
assert ls1.read(1) == b""
assert 4 == ls1.tell()
# Non limited
fileobj.seek(0)
body2 = Body(fileobj)
ls2 = LimitingStream(body2)
resp2 = b""
while True:
buf = ls2.read(-1)
if not buf:
break
resp2 += buf
assert resp2 == content
assert ls2.read(1) == b""
assert len(content) == ls2.tell()
def test_nolimit():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj)
assert stream.read(-1) == b"this is a cool test"
assert len(b"this is a cool test") == stream.tell()
def test_nolimit_readdefined():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj, -1)
assert stream.read(2) == 'th'
assert 2 == stream.tell()
def test_simplelimit():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj, 4)
assert stream.read(-1) == 'this'
assert 4 == stream.tell()
def test_nolimit():
fileobj = StringIO('this is a cool test')
stream = LimitingStream(fileobj)
assert stream.read(-1) == 'this is a cool test'
assert len('this is a cool test') == stream.tell()
def test_simplelimit():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj, 4)
assert stream.read(-1) == b"this"
assert 4 == stream.tell()
def test_limit_multiread2():
fileobj = StringIO("this is a cool test")
stream = LimitingStream(fileobj, 7)
assert stream.read(4) == "this"
assert stream.read(-1) == " is"
assert 7 == stream.tell()
def test_nolimit_readdefined():
fileobj = BytesIO(b"this is a cool test")
stream = LimitingStream(fileobj, -1)
assert stream.read(2) == b"th"
assert 2 == stream.tell()