def test_cat(azure): with azure_teardown(azure): with azure.open(a, 'wb') as f: f.write(b'0123456789') assert azure.cat(a) == b'0123456789' with pytest.raises(IOError): azure.cat(b)
def test_DatalakeBadOffsetExceptionRecovery(azure): from azure.datalake.store.core import _put_data_with_retry data = b'abc' _put_data_with_retry(azure.azure, 'CREATE', a, data=data) _put_data_with_retry(azure.azure, 'APPEND', a, data=data, offset=len(data)) _put_data_with_retry(azure.azure, 'APPEND', a, data=data, offset=len(data)) assert azure.cat(a) == data*2 _put_data_with_retry(azure.azure, 'APPEND', a, data=data) assert azure.cat(a) == data*3
def write_delimited_data(azure, delimiter): data = delimiter.join([b'123', b'456', b'789']) data2 = data + delimiter with azure.open(a, 'wb', delimiter=delimiter, blocksize=6) as f: f.write(b'123' + delimiter) assert f.buffer.tell() == 3 + len(delimiter) f.write(b'456' + delimiter) # causes flush assert azure.cat(a) == b'123' + delimiter f.write(b'789') # close causes forced flush assert azure.cat(a) == data
def write_delimited_data(azure, delimiter): data = delimiter.join([b'123', b'456', b'789']) with azure.open(a, 'wb', delimiter=delimiter, blocksize=6) as f: f.write(b'123' + delimiter) assert f.buffer.tell() == 3 + len(delimiter) f.write(b'456' + delimiter) # causes flush, but will only write till 123 + delimiter assert f.buffer.tell() == 3 + len(delimiter) # buffer will have b'456' + delimiter f.buffer, temp_buffer = io.BytesIO(), f.buffer # Emptry buffer so flush doesn't write any more data f.loc, temp_loc = 3 + len(delimiter), f.loc # Fix location. f.flush(force=True) # To Sync metadata. Force is needed as there is no data in buffer assert azure.cat(a) == b'123' + delimiter f.buffer = temp_buffer f.loc = temp_loc # close causes forced flush f.write(b'789') assert azure.cat(a) == data
def test_put(azure): data = b'1234567890' * 100 with azure_teardown(azure): with tmpfile() as fn: with open(fn, 'wb') as f: f.write(data) azure.put(fn, a) assert azure.cat(a) == data
def test_upload_many(local_files, azure): with azure_teardown(azure): bigfile, littlefile, emptyfile, a, b, c = local_files root = os.path.dirname(bigfile) # single thread up = ADLUploader(azure, test_dir, root, nthreads=1, overwrite=True) assert azure.info(test_dir / 'littlefile')['length'] == 10 assert azure.cat(test_dir / 'nested1/nested2/a') == b'0123456789' assert len(azure.du(test_dir, deep=True)) == 6 assert azure.du(test_dir, deep=True, total=True) == 10000 + 40
def test_concat(azure): with azure.open(a, 'wb') as f: f.write(b'hello ') with azure.open(b, 'wb') as f: f.write(b'world') try: azure.rm(c) except: pass azure.concat(c, [a, b]) out = azure.cat(c) azure.rm(c) assert out == b'hello world'
def test_concat(azure): aplus = a + "+file1" bplus = b + "+file2" cplus = c + "+res" with azure.open(aplus, 'wb') as f: f.write(b'hello ') with azure.open(bplus, 'wb') as f: f.write(b'world') try: azure.rm(cplus) except: pass azure.concat(cplus, [aplus, bplus]) out = azure.cat(cplus) azure.rm(cplus) assert out == b'hello world'