def setup_dir(azure):
d = str(working_dir() / 'foo')
azure.mkdir(d)
try:
yield d
finally:
azure.rm(d, recursive=True)
def test_chmod(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.info(a)['permission'] == '770'
azure.chmod(a, '0555')
assert azure.info(a)['permission'] == '555'
with pytest.raises((OSError, IOError)):
with azure.open(a, 'ab') as f:
try:
f.write(b'data')
except Exception as e:
print(e)
raise e
azure.chmod(a, '0770')
azure.rm(a)
azure.mkdir(test_dir / 'deep')
azure.touch(test_dir / 'deep' / 'file')
azure.chmod(test_dir / 'deep', '660')
with pytest.raises((OSError, IOError)):
azure.ls(test_dir / 'deep')
azure.chmod(test_dir / 'deep', '770')
def test_glob_walk_invalidate_cache(azure):
with azure_teardown(azure):
azure.mkdir(test_dir / 'c')
azure.mkdir(test_dir / 'c' / 'd')
filenames = ['a', 'a1', 'a2', 'a3', 'b1', 'c/x1', 'c/x2', 'c/d/x3']
filenames = [test_dir / s for s in filenames]
for fn in filenames:
azure.touch(fn)
assert set(azure.glob(test_dir / 'a*')) == {
posix(test_dir / 'a'),
posix(test_dir / 'a1'),
posix(test_dir / 'a2'),
posix(test_dir / 'a3')}
assert set(azure.glob(test_dir / 'c' / '*')) == {
posix(test_dir / 'c' / 'x1'),
posix(test_dir / 'c' / 'x2')}
assert (set(azure.glob(test_dir / 'c')) ==
set(azure.glob(test_dir / 'c' / '')))
assert set(azure.glob(test_dir / 'a')) == {posix(test_dir / 'a')}
assert set(azure.glob(test_dir / 'a1')) == {posix(test_dir / 'a1')}
assert set(azure.glob(test_dir / '*')) == {
posix(test_dir / 'a'),
posix(test_dir / 'a1'),
posix(test_dir / 'a2'),
posix(test_dir / 'a3'),
posix(test_dir / 'b1')}
assert set(azure.walk(test_dir, invalidate_cache=True)) == {
posix(test_dir / 'a'),
posix(test_dir / 'a1'),
posix(test_dir / 'a2'),
posix(test_dir / 'a3'),
posix(test_dir / 'b1'),
posix(test_dir / 'c' / 'x1'),
posix(test_dir / 'c' / 'x2'),
posix(test_dir / 'c' / 'd' / 'x3')}
assert set(azure.walk(test_dir / 'c', invalidate_cache=True)) == {
posix(test_dir / 'c' / 'x1'),
posix(test_dir / 'c' / 'x2'),
posix(test_dir / 'c' / 'd' / 'x3')}
assert set(azure.walk(test_dir / 'c', invalidate_cache=True)) == set(azure.walk(test_dir / 'c', invalidate_cache=True))
# test glob and walk with details=True
glob_details = azure.glob(test_dir / '*', details=True, invalidate_cache=True)
# validate that the objects are subscriptable
assert glob_details[0]['name'] is not None
assert glob_details[0]['type'] is not None
walk_details = azure.walk(test_dir, details=True, invalidate_cache=True)
assert walk_details[0]['name'] is not None
assert walk_details[0]['type'] is not None
def setup_tree(azure):
for directory in ['', 'data/a', 'data/b']:
azure.mkdir(test_dir / directory)
for filename in ['x.csv', 'y.csv', 'z.txt']:
with azure.open(test_dir / directory / filename, 'wb') as f:
f.write(b'123456')
try:
yield
finally:
for path in azure.ls(test_dir, invalidate_cache=False):
if azure.exists(path, invalidate_cache=False):
azure.rm(path, recursive=True)
def test_upload_overwrite(local_files, azure):
bigfile, littlefile, emptyfile, a, b, c = local_files
with azure_teardown(azure):
# create the folder that we want to make sure the overwrite
# test fails on if it doesn't already exist
if not azure.exists(test_dir):
azure.mkdir(test_dir)
with pytest.raises(OSError) as e:
ADLUploader(azure, test_dir, littlefile, nthreads=1)
assert test_dir.as_posix() in str(e)
def test_ls_batched(azure):
test_dir = working_dir() / 'abc'
azure.mkdir(test_dir)
with azure_teardown(azure):
test_size = 10
assert azure._ls(test_dir, batch_size=10) == []
create_files(azure, number_of_files = 10, prefix='123', root_path=test_dir)
with pytest.raises(ValueError):
assert len(azure._ls(test_dir, batch_size=1)) == test_size
assert len(azure._ls(test_dir, batch_size=9)) == test_size
assert len(azure._ls(test_dir, batch_size=10)) == test_size
assert len(azure._ls(test_dir, batch_size=11)) == test_size
assert len(azure._ls(test_dir, batch_size=2)) == test_size
assert len(azure._ls(test_dir, batch_size=100)) == test_size
assert len(azure._ls(test_dir)) == test_size
def __test_retry_error(azure,
error_status,
total_tries,
is_exception_expected=False,
last_try_status=200,
body=""):
mock_url_a = mock_url + a
while total_tries > 1:
responses.add(responses.PUT,
mock_url_a,
body=body,
status=error_status)
total_tries -= 1
responses.add(responses.PUT, mock_url_a, body="", status=last_try_status)
# teardown not required in mock tests
try:
azure.mkdir(a)
assert not is_exception_expected
except DatalakeRESTException:
assert is_exception_expected
