def test_readinto(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'0123456789')
with azure.open(a, 'rb') as f:
buffer = bytearray(6)
l = f.readinto(buffer)
assert l == 6
assert buffer == b'012345'
buffer = bytearray(6)
l = f.readinto(buffer)
assert l == 4
assert buffer == b'6789\x00\x00'
buffer = bytearray(6)
l = f.readinto(buffer)
assert buffer == b'\x00\x00\x00\x00\x00\x00'
assert l == 0
with azure.open(a, 'rb') as f:
buffer = bytearray(6)
l = f.readinto(buffer)
assert l == 6
assert buffer == b'012345'
l = f.readinto(buffer)
assert l == 4
assert buffer == b'678945' # 45 from previous buffer fill should not be overwritten
def test_seek(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'123')
with azure.open(a) as f:
# assert False
with pytest.raises(ValueError):
f.seek(1000)
with pytest.raises(ValueError):
f.seek(-1)
with pytest.raises(ValueError):
f.seek(-5, 2)
with pytest.raises(ValueError):
f.seek(0, 10)
f.seek(0)
assert f.read(1) == b'1'
f.seek(0)
assert f.read(1) == b'1'
f.seek(3)
assert f.read(1) == b''
f.seek(-1, 2)
assert f.read(1) == b'3'
f.seek(-1, 1)
f.seek(-1, 1)
assert f.read(1) == b'2'
for i in range(4):
assert f.seek(i) == i
def test_write_empty(azure):
with azure_teardown(azure):
with azure.open(a, mode='wb') as f:
f.write(b'')
with azure.open(a, mode='rb') as f:
assert f.read() == b''
def test_skip_existing_block(azure):
with azure.open(a, mode='wb') as f:
f.write(b'0' * 15)
with pytest.raises((IOError, RuntimeError)):
with azure.open(a, mode='ab') as f:
assert f.tell() == 15
f.loc = 5 # not a user method
f.write(b'blah')
def test_du(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'123')
with azure.open(b, 'wb') as f:
f.write(b'4567')
assert azure.du(test_dir) == {a: 3, b: 4}
assert azure.du(test_dir, total=True) == 3 + 4
def test_TextIOWrapper(azure):
with azure_teardown(azure):
with azure.open(a, mode='wb') as f:
f.write(b'1,2\n3,4\n5,6')
with azure.open(a, mode='rb') as f:
ff = io.TextIOWrapper(f)
data = list(ff)
assert data == ['1,2\n', '3,4\n', '5,6']
def test_readline(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'\n'.join([b'123', b'456', b'789']))
with azure.open(a) as f:
assert f.readline() == b'123\n'
assert f.readline() == b'456\n'
assert f.readline() == b'789'
assert f.readline() == b''
def __ready_and_read_file_for_cache_test(azure, data=b'0123456789abcdef'):
with azure.open(a, 'wb') as f:
f.write(data)
f = azure.open(a, 'rb', blocksize=4)
# start cache @ 2
f.seek(2)
# end cache @ 6
f.read(4)
return f
def test_df(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'a' * 10)
with azure.open(b, 'wb') as f:
f.write(b'a' * 10)
result = azure.df(test_dir)
assert result['fileCount'] > 0
assert result['spaceConsumed'] > 0
def test_array(azure):
from array import array
data = array('B', [65] * 1000)
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(data)
with azure.open(a, 'rb') as f:
out = f.read()
assert out == b'A' * 1000
def test_fooable(azure):
with azure_teardown(azure):
azure.touch(a)
with azure.open(a, mode='rb') as f:
assert f.readable()
assert f.seekable()
assert not f.writable()
with azure.open(a, mode='wb') as f:
assert not f.readable()
assert not f.seekable()
assert f.writable()
def test_gzip(azure):
import gzip
data = b'name,amount\nAlice,100\nBob,200'
with azure_teardown(azure):
with azure.open(a, mode='wb') as f:
with gzip.GzipFile(fileobj=f) as g:
g.write(b'name,amount\nAlice,100\nBob,200')
with azure.open(a) as f:
with gzip.GzipFile(fileobj=f) as g:
bytes = g.read()
assert bytes == data
def test_simple(azure):
with azure_teardown(azure):
data = b'a' * (2**16)
with azure.open(a, 'wb') as f:
assert f.blocksize == 4 * 2**20
l = f.write(data)
assert l == len(data)
with azure.open(a, 'rb') as f:
out = f.read(len(data))
assert len(data) == len(out)
assert out == data
def test_append(azure):
with azure_teardown(azure):
with azure.open(a, mode='ab') as f:
f.write(b'123')
with azure.open(a, mode='ab') as f:
f.write(b'456')
with azure.open(a, mode='rb') as f:
assert f.read() == b'123456'
with azure.open(a, mode='ab') as f:
f.write(b'789')
with azure.open(a, mode='rb') as f:
assert f.read() == b'123456789'
def test_concat(azure):
with azure.open(a, 'wb') as f:
f.write(b'hello ')
with azure.open(b, 'wb') as f:
f.write(b'world')
try:
azure.rm(c)
except:
pass
azure.concat(c, [a, b])
out = azure.cat(c)
azure.rm(c)
assert out == b'hello world'
def test_info_invalidate_cache(azure, second_azure):
with azure_teardown(azure):
# construct initial cache and ensure the file doesn't already exist
assert not azure.exists(a, invalidate_cache=False)
assert not second_azure.exists(a, invalidate_cache=False)
with azure.open(a, 'wb') as f:
f.write(b'a' * 5)
# verify that it works in the fs that did the write and not on the other
info = azure.info(a, invalidate_cache=False)
with pytest.raises(FileNotFoundError):
second_azure.info(a, invalidate_cache=False)
# then invalidate
second_info = second_azure.info(a, invalidate_cache=True)
assert info['length'] == 5
assert info['name'] == a
assert info['type'] == 'FILE'
assert info['length'] == second_info['length']
assert info['name'] == second_info['name']
assert info['type'] == second_info['type']
# assure that the cache was properly repopulated on the info call
assert second_azure.info(test_dir,
invalidate_cache=False)['type'] == 'DIRECTORY'
def setup_tree(azure):
for directory in ['', 'data/a', 'data/b']:
azure.mkdir(test_dir / directory)
for filename in ['x.csv', 'y.csv', 'z.txt']:
with azure.open(test_dir / directory / filename, 'wb') as f:
f.write(b'123456')
azure.mkdir(test_dir / 'data/empty')
azure.mkdir(test_dir / 'data/single/single')
with azure.open(test_dir / 'data/single/single'/ 'single.txt', 'wb') as f:
f.write(b'123456')
try:
yield
finally:
for path in azure.ls(test_dir, invalidate_cache=False):
if azure.exists(path, invalidate_cache=False):
azure.rm(path, recursive=True)
def test_cat(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'0123456789')
assert azure.cat(a) == b'0123456789'
with pytest.raises(IOError):
azure.cat(b)
def test_chmod(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.info(a)['permission'] == '770'
azure.chmod(a, '0555')
assert azure.info(a)['permission'] == '555'
with pytest.raises((OSError, IOError)):
with azure.open(a, 'ab') as f:
try:
f.write(b'data')
except Exception as e:
print(e)
raise e
azure.chmod(a, '0770')
azure.rm(a)
azure.mkdir(test_dir / 'deep')
azure.touch(test_dir / 'deep' / 'file')
azure.chmod(test_dir / 'deep', '660')
with pytest.raises((OSError, IOError)):
azure.ls(test_dir / 'deep')
azure.chmod(test_dir / 'deep', '770')
def test_write_in_read_mode(azure):
with azure_teardown(azure):
azure.touch(a)
with azure.open(a, 'rb') as f:
with pytest.raises(ValueError):
f.write(b'123')
def test_read_delimited_block(azure):
fn = '/tmp/test/a'
delimiter = b'\n'
data = delimiter.join([b'123', b'456', b'789'])
with azure_teardown(azure):
with azure.open(fn, 'wb') as f:
f.write(data)
#TODO: add E2E validation with the transfer client once delimiters are hooked up
assert azure.read_block(fn, 1, 2) == b'23'
assert azure.read_block(fn, 0, 1, delimiter=b'\n') == b'1'
assert azure.read_block(fn, 0, 2, delimiter=b'\n') == b'12'
assert azure.read_block(fn, 0, 3, delimiter=b'\n') == b'123'
assert azure.read_block(fn, 0, 4, delimiter=b'\n') == b'123\n'
assert azure.read_block(fn, 0, 5, delimiter=b'\n') == b'123\n'
assert azure.read_block(fn, 0, 8, delimiter=b'\n') == b'123\n456\n'
assert azure.read_block(fn, 0, 100, delimiter=b'\n') == b'123\n456\n'
assert azure.read_block(fn, 1, 1, delimiter=b'\n') == b'2'
assert azure.read_block(fn, 1, 5, delimiter=b'\n') == b'23\n'
assert azure.read_block(fn, 1, 8, delimiter=b'\n') == b'23\n456\n'
azure.rm(fn)
# test the negative cases of just the util read block
with io.BytesIO(bytearray([1] * 2**22)) as data:
with pytest.raises(IndexError):
utils.read_block(data, 0, 2**22, delimiter=b'\n')
# ensure it throws if the new line is past 4MB
data.seek(2**22)
data.write(b'\n')
data.seek(0)
with pytest.raises(IndexError):
utils.read_block(data, 0, 1 + 2**22, delimiter=b'\n')
def setup_file(azure):
tmp = str(working_dir() / 'foo' / 'bar')
with azure.open(tmp, 'wb') as f:
f.write('123456'.encode())
try:
yield tmp
finally:
azure.rm(tmp)
def test_tail_head(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'0123456789')
assert azure.tail(a, 3) == b'789'
assert azure.head(a, 3) == b'012'
assert azure.tail(a, 100) == b'0123456789'
def test_closed(azure):
with azure_teardown(azure):
azure.touch(a)
f = azure.open(a, mode='rb')
assert not f.closed
f.close()
assert f.closed
def test_file_creation_open(azure):
with azure_teardown(azure):
if azure.exists(a):
azure.rm(a)
assert not azure.exists(a)
f = azure.open(a, "wb")
assert azure.exists(a)
f.close()
assert azure.info(a)['length'] == 0
def test_concat(azure):
aplus = a + "+file1"
bplus = b + "+file2"
cplus = c + "+res"
with azure.open(aplus, 'wb') as f:
f.write(b'hello ')
with azure.open(bplus, 'wb') as f:
f.write(b'world')
try:
azure.rm(cplus)
except:
pass
azure.concat(cplus, [aplus, bplus])
out = azure.cat(cplus)
azure.rm(cplus)
assert out == b'hello world'
def test_write_blocks(azure):
with azure_teardown(azure):
with azure.open(a, mode='wb', blocksize=5) as f:
f.write(b'000')
assert f.buffer.tell() == 3
f.write(b'000') # forces flush
assert f.buffer.tell() == 1
f.write(b'000')
assert f.tell() == 9
assert azure.du(a)[a] == 9
def test_info(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'a' * 5)
info = azure.info(a, invalidate_cache=False)
assert info['length'] == 5
assert info['name'] == a
assert info['type'] == 'FILE'
assert azure.info(test_dir, invalidate_cache=True)['type'] == 'DIRECTORY'
def test_filename_specialchar(azure):
with azure_teardown(azure):
with azure.open(specialCharFile, 'wb') as f:
f.write(b'0123456789')
with azure.open(specialCharFile, 'rb') as f:
assert len(f.read(4)) == 4
assert len(f.read(4)) == 4
assert len(f.read(4)) == 2
with azure.open(specialCharFile, 'rb') as f:
assert len(f.read()) == 10
with azure.open(specialCharFile, 'rb') as f:
assert f.tell() == 0
f.seek(3)
assert f.read(4) == b'3456'
assert f.tell() == 7
assert f.read(4) == b'789'
assert f.tell() == 10
def test_full_read(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'0123456789')
with azure.open(a, 'rb') as f:
assert len(f.read(4)) == 4
assert len(f.read(4)) == 4
assert len(f.read(4)) == 2
with azure.open(a, 'rb') as f:
assert len(f.read()) == 10
with azure.open(a, 'rb') as f:
assert f.tell() == 0
f.seek(3)
assert f.read(4) == b'3456'
assert f.tell() == 7
assert f.read(4) == b'789'
assert f.tell() == 10