def test_cat(azure):
with azure_teardown(azure):
with azure.open(a, 'wb') as f:
f.write(b'0123456789')
assert azure.cat(a) == b'0123456789'
with pytest.raises(IOError):
azure.cat(b)
def test_DatalakeBadOffsetExceptionRecovery(azure):
from azure.datalake.store.core import _put_data_with_retry
data = b'abc'
_put_data_with_retry(azure.azure, 'CREATE', a, data=data)
_put_data_with_retry(azure.azure, 'APPEND', a, data=data, offset=len(data))
_put_data_with_retry(azure.azure, 'APPEND', a, data=data, offset=len(data))
assert azure.cat(a) == data*2
_put_data_with_retry(azure.azure, 'APPEND', a, data=data)
assert azure.cat(a) == data*3
def write_delimited_data(azure, delimiter):
data = delimiter.join([b'123', b'456', b'789'])
data2 = data + delimiter
with azure.open(a, 'wb', delimiter=delimiter, blocksize=6) as f:
f.write(b'123' + delimiter)
assert f.buffer.tell() == 3 + len(delimiter)
f.write(b'456' + delimiter) # causes flush
assert azure.cat(a) == b'123' + delimiter
f.write(b'789')
# close causes forced flush
assert azure.cat(a) == data
def write_delimited_data(azure, delimiter):
data = delimiter.join([b'123', b'456', b'789'])
with azure.open(a, 'wb', delimiter=delimiter, blocksize=6) as f:
f.write(b'123' + delimiter)
assert f.buffer.tell() == 3 + len(delimiter)
f.write(b'456' + delimiter) # causes flush, but will only write till 123 + delimiter
assert f.buffer.tell() == 3 + len(delimiter) # buffer will have b'456' + delimiter
f.buffer, temp_buffer = io.BytesIO(), f.buffer # Emptry buffer so flush doesn't write any more data
f.loc, temp_loc = 3 + len(delimiter), f.loc # Fix location.
f.flush(force=True) # To Sync metadata. Force is needed as there is no data in buffer
assert azure.cat(a) == b'123' + delimiter
f.buffer = temp_buffer
f.loc = temp_loc
# close causes forced flush
f.write(b'789')
assert azure.cat(a) == data
def test_put(azure):
data = b'1234567890' * 100
with azure_teardown(azure):
with tmpfile() as fn:
with open(fn, 'wb') as f:
f.write(data)
azure.put(fn, a)
assert azure.cat(a) == data
def test_upload_many(local_files, azure):
with azure_teardown(azure):
bigfile, littlefile, emptyfile, a, b, c = local_files
root = os.path.dirname(bigfile)
# single thread
up = ADLUploader(azure, test_dir, root, nthreads=1, overwrite=True)
assert azure.info(test_dir / 'littlefile')['length'] == 10
assert azure.cat(test_dir / 'nested1/nested2/a') == b'0123456789'
assert len(azure.du(test_dir, deep=True)) == 6
assert azure.du(test_dir, deep=True, total=True) == 10000 + 40
def test_concat(azure):
with azure.open(a, 'wb') as f:
f.write(b'hello ')
with azure.open(b, 'wb') as f:
f.write(b'world')
try:
azure.rm(c)
except:
pass
azure.concat(c, [a, b])
out = azure.cat(c)
azure.rm(c)
assert out == b'hello world'
def test_concat(azure):
aplus = a + "+file1"
bplus = b + "+file2"
cplus = c + "+res"
with azure.open(aplus, 'wb') as f:
f.write(b'hello ')
with azure.open(bplus, 'wb') as f:
f.write(b'world')
try:
azure.rm(cplus)
except:
pass
azure.concat(cplus, [aplus, bplus])
out = azure.cat(cplus)
azure.rm(cplus)
assert out == b'hello world'
