def test_set_expiry(azure):
with azure_teardown(azure):
azure.touch(a)
# first get the existing expiry, which should be never
initial_expiry = azure.info(a)['msExpirationTime']
# this future time gives the milliseconds since the epoch that have occured as of 01/31/2030 at noon
epoch_time = datetime.datetime.utcfromtimestamp(0)
final_time = datetime.datetime(2030, 1, 31, 12)
time_in_milliseconds = (final_time - epoch_time).total_seconds() * 1000
azure.set_expiry(a, 'Absolute', time_in_milliseconds)
cur_expiry = azure.info(a)['msExpirationTime']
assert time_in_milliseconds == cur_expiry
assert initial_expiry != cur_expiry
# now set it back to never expire and validate it is the same
azure.set_expiry(a, 'NeverExpire')
cur_expiry = azure.info(a)['msExpirationTime']
assert initial_expiry == cur_expiry
# now validate the fail cases
# bad enum
with pytest.raises(ValueError):
azure.set_expiry(a, 'BadEnumValue')
# missing time
with pytest.raises(ValueError):
azure.set_expiry(a, 'Absolute')
def test_chown(azure):
with azure_teardown(azure):
azure.touch(a)
# fake IDs
user_id = '470c0ccf-c91a-4597-98cd-48507d2f1486'
group_id = '6b190b7a-0acf-43c8-ab14-965f5aea6243'
# Account doesn't have permission to change owner
owner = azure.info(a)['owner']
with pytest.raises(PermissionError):
azure.chown(a, owner=user_id)
assert owner == azure.info(a)['owner']
# Account doesn't have permission to change group
group = azure.info(a)['group']
with pytest.raises(PermissionError):
azure.chown(a, group=group_id)
assert group == azure.info(a)['group']
# Account doesn't have permission to change owner/group
with pytest.raises(PermissionError):
azure.chown(a, owner=owner, group=group)
assert owner == azure.info(a)['owner']
assert group == azure.info(a)['group']
def test_rm(azure):
with azure_teardown(azure):
assert not azure.exists(a, invalidate_cache=False)
azure.touch(a)
assert azure.exists(a, invalidate_cache=False)
azure.rm(a)
assert not azure.exists(a, invalidate_cache=False)
def test_chmod(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.info(a)['permission'] == '770'
azure.chmod(a, '0555')
assert azure.info(a)['permission'] == '555'
with pytest.raises((OSError, IOError)):
with azure.open(a, 'ab') as f:
try:
f.write(b'data')
except Exception as e:
print(e)
raise e
azure.chmod(a, '0770')
azure.rm(a)
azure.mkdir(test_dir / 'deep')
azure.touch(test_dir / 'deep' / 'file')
azure.chmod(test_dir / 'deep', '660')
with pytest.raises((OSError, IOError)):
azure.ls(test_dir / 'deep')
azure.chmod(test_dir / 'deep', '770')
def test_rm(azure):
with azure_teardown(azure):
assert not azure.exists(a)
azure.touch(a)
assert azure.exists(a)
azure.rm(a)
assert not azure.exists(a)
def test_write_in_read_mode(azure):
with azure_teardown(azure):
azure.touch(a)
with azure.open(a, 'rb') as f:
with pytest.raises(ValueError):
f.write(b'123')
def test_set_expiry(azure):
with azure_teardown(azure):
# this future time gives the milliseconds since the epoch that have occured as of 01/31/2030 at noon
epoch_time = datetime.datetime.utcfromtimestamp(0)
final_time = datetime.datetime(2030, 1, 31, 12)
time_in_milliseconds = (final_time - epoch_time).total_seconds() * 1000
# create the file
azure.touch(a)
# first get the existing expiry, which should be never
initial_expiry = azure.info(a,
invalidate_cache=True)['msExpirationTime']
azure.set_expiry(a, 'Absolute', time_in_milliseconds)
cur_expiry = azure.info(a, invalidate_cache=True)['msExpirationTime']
# this is a range of +- 100ms because the service does a best effort to set it precisely, but there is
# no guarantee that the expiry will be to the exact millisecond
assert time_in_milliseconds - 100 <= cur_expiry <= time_in_milliseconds + 100
assert initial_expiry != cur_expiry
# now set it back to never expire and validate it is the same
azure.set_expiry(a, 'NeverExpire')
cur_expiry = azure.info(a)['msExpirationTime']
assert initial_expiry == cur_expiry
# now validate the fail cases
# bad enum
with pytest.raises(ValueError):
azure.set_expiry(a, 'BadEnumValue')
# missing time
with pytest.raises(ValueError):
azure.set_expiry(a, 'Absolute')
def test_glob_walk_invalidate_cache(azure):
with azure_teardown(azure):
azure.mkdir(test_dir / 'c')
azure.mkdir(test_dir / 'c' / 'd')
filenames = ['a', 'a1', 'a2', 'a3', 'b1', 'c/x1', 'c/x2', 'c/d/x3']
filenames = [test_dir / s for s in filenames]
for fn in filenames:
azure.touch(fn)
assert set(azure.glob(test_dir / 'a*')) == {
posix(test_dir / 'a'),
posix(test_dir / 'a1'),
posix(test_dir / 'a2'),
posix(test_dir / 'a3')}
assert set(azure.glob(test_dir / 'c' / '*')) == {
posix(test_dir / 'c' / 'x1'),
posix(test_dir / 'c' / 'x2')}
assert (set(azure.glob(test_dir / 'c')) ==
set(azure.glob(test_dir / 'c' / '')))
assert set(azure.glob(test_dir / 'a')) == {posix(test_dir / 'a')}
assert set(azure.glob(test_dir / 'a1')) == {posix(test_dir / 'a1')}
assert set(azure.glob(test_dir / '*')) == {
posix(test_dir / 'a'),
posix(test_dir / 'a1'),
posix(test_dir / 'a2'),
posix(test_dir / 'a3'),
posix(test_dir / 'b1')}
assert set(azure.walk(test_dir, invalidate_cache=True)) == {
posix(test_dir / 'a'),
posix(test_dir / 'a1'),
posix(test_dir / 'a2'),
posix(test_dir / 'a3'),
posix(test_dir / 'b1'),
posix(test_dir / 'c' / 'x1'),
posix(test_dir / 'c' / 'x2'),
posix(test_dir / 'c' / 'd' / 'x3')}
assert set(azure.walk(test_dir / 'c', invalidate_cache=True)) == {
posix(test_dir / 'c' / 'x1'),
posix(test_dir / 'c' / 'x2'),
posix(test_dir / 'c' / 'd' / 'x3')}
assert set(azure.walk(test_dir / 'c', invalidate_cache=True)) == set(azure.walk(test_dir / 'c', invalidate_cache=True))
# test glob and walk with details=True
glob_details = azure.glob(test_dir / '*', details=True, invalidate_cache=True)
# validate that the objects are subscriptable
assert glob_details[0]['name'] is not None
assert glob_details[0]['type'] is not None
walk_details = azure.walk(test_dir, details=True, invalidate_cache=True)
assert walk_details[0]['name'] is not None
assert walk_details[0]['type'] is not None
def test_move(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.exists(a)
assert not azure.exists(b)
azure.mv(a, b)
assert not azure.exists(a)
assert azure.exists(b)
def test_closed(azure):
with azure_teardown(azure):
azure.touch(a)
f = azure.open(a, mode='rb')
assert not f.closed
f.close()
assert f.closed
def test_copy(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.exists(a, invalidate_cache=False)
assert not azure.exists(b, invalidate_cache=False)
azure.cp(a, b)
assert azure.exists(a, invalidate_cache=False)
assert azure.exists(b, invalidate_cache=False)
def test_copy(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.exists(a)
assert not azure.exists(b)
azure.cp(a, b)
assert azure.exists(a)
assert azure.exists(b)
def test_ls_touch(azure):
with azure_teardown(azure):
assert not azure.ls(test_dir, invalidate_cache=False)
azure.touch(a)
azure.touch(b)
L = azure.ls(test_dir, True, invalidate_cache=False)
assert set(d['name'] for d in L) == set([a, b])
L = azure.ls(test_dir, False, invalidate_cache=False)
assert set(L) == set([a, b])
def test_upload_overwrite(local_files, azure):
bigfile, littlefile, emptyfile, a, b, c = local_files
with azure_teardown(azure):
# make the file already exist.
azure.touch('/{}/littlefile'.format(test_dir.as_posix()))
with pytest.raises(OSError) as e:
ADLUploader(azure, test_dir, littlefile, nthreads=1)
assert test_dir.as_posix() in str(e)
def test_ls_touch_invalidate_cache(azure, second_azure):
with azure_teardown(azure):
assert not azure.ls(test_dir, invalidate_cache=False)
assert not second_azure.ls(test_dir, invalidate_cache=False)
azure.touch(a)
azure.touch(b)
L = azure.ls(test_dir, True, invalidate_cache=False)
assert not second_azure.ls(test_dir, invalidate_cache=False)
L_second = second_azure.ls(test_dir, True, invalidate_cache=True)
assert set(d['name'] for d in L) == set([a, b])
assert L == L_second
def test_acl_management(azure):
user_id = '470c0ccf-c91a-4597-98cd-48507d2f1486'
acl_to_add = 'user:{}:rwx'.format(user_id)
acl_to_modify = 'user:{}:-w-'.format(user_id)
acl_to_remove = 'user:{}'.format(user_id)
with azure_teardown(azure):
azure.touch(a)
# get initial ACLs
initial_acls = azure.get_acl_status(a)
acl_len = len(initial_acls['entries'])
# set the full acl
new_acl = ','.join(initial_acls['entries'])
new_acl += ',{}'.format(acl_to_add)
azure.set_acl(a, new_acl)
# get the ACL and ensure it has an additional entry
new_acls = azure.get_acl_status(a)
assert acl_len + 2 == len(new_acls['entries'])
# assert that the entry is there
assert acl_to_add in new_acls['entries']
# set the specific ACL entry
azure.modify_acl_entries(a, acl_to_modify)
# get and validate that it was changed
new_acls = azure.get_acl_status(a)
assert acl_len + 2 == len(new_acls['entries'])
# assert that the entry is there
assert acl_to_modify in new_acls['entries']
# remove the ACL entry
azure.remove_acl_entries(a, acl_to_remove)
# get and validate that it was changed
new_acls = azure.get_acl_status(a)
assert acl_len + 1 == len(new_acls['entries'])
# assert that the entry is there
assert acl_to_modify not in new_acls['entries']
# remove the full acl and validate the lengths
azure.remove_default_acl(a)
# get and validate that it was changed
new_acls = azure.get_acl_status(a)
assert 4 == len(new_acls['entries'])
# remove the full acl and validate the lengths
azure.remove_acl(a)
# get and validate that it was changed
new_acls = azure.get_acl_status(a)
assert 3 == len(new_acls['entries'])
def test_set_acl(azure):
with azure_teardown(azure):
acluser = AZURE_ACL_TEST_APPID
azure.touch(a)
set_acl_base ="user::rwx,group::rwx,other::---,"
permission = "rwx"
azure.set_acl(a, acl_spec=set_acl_base + "user:"******":"+permission)
current_acl = azure.get_acl_status(a)
aclspec = [s for s in current_acl['entries'] if acluser in s][0]
assert len(current_acl['entries']) == 5
assert aclspec.split(':')[-1] == permission
def test_fooable(azure):
with azure_teardown(azure):
azure.touch(a)
with azure.open(a, mode='rb') as f:
assert f.readable()
assert f.seekable()
assert not f.writable()
with azure.open(a, mode='wb') as f:
assert not f.readable()
assert not f.seekable()
assert f.writable()
def test_remove_acl_entries(azure):
with azure_teardown(azure):
acluser = AZURE_ACL_TEST_APPID
azure.touch(a)
permission = "rwx"
azure.modify_acl_entries(a, acl_spec="user:"******":"+permission)
current_acl = azure.get_acl_status(a)
aclspec = [s for s in current_acl['entries'] if acluser in s]
assert aclspec != []
azure.remove_acl_entries(a, acl_spec="user:" + acluser)
current_acl = azure.get_acl_status(a)
aclspec = [s for s in current_acl['entries'] if acluser in s]
assert aclspec == []
def test_modify_acl_entries(azure):
with azure_teardown(azure):
acluser = AZURE_ACL_TEST_APPID
azure.touch(a)
permission = "---"
azure.modify_acl_entries(a, acl_spec="user:"******":"+permission)
current_acl = azure.get_acl_status(a)
aclspec = [s for s in current_acl['entries'] if acluser in s][0]
assert aclspec.split(':')[-1] == permission
permission = "rwx"
azure.modify_acl_entries(a, acl_spec="user:"******":" + permission)
current_acl = azure.get_acl_status(a)
aclspec = [s for s in current_acl['entries'] if acluser in s][0]
assert aclspec.split(':')[-1] == permission
def test_exists_remove_invalidate_cache(azure, second_azure):
with azure_teardown(azure):
# test to ensure file does not exist up front, cache doesn't matter
assert not azure.exists(a, invalidate_cache=False)
assert not second_azure.exists(a, invalidate_cache=False)
azure.touch(a)
# now ensure that it exists in the client that did the work, but not in the other
assert azure.exists(a, invalidate_cache=False)
assert not second_azure.exists(a, invalidate_cache=False)
# now, with cache invalidation it should exist
assert second_azure.exists(a, invalidate_cache=True)
azure.rm(a)
# same idea with remove. It should no longer exist (cache invalidated or not) in client 1, but still exist in client 2
assert not azure.exists(a, invalidate_cache=False)
assert second_azure.exists(a, invalidate_cache=False)
# now ensure it does not exist when we do invalidate the cache
assert not second_azure.exists(a, invalidate_cache=True)
def test_touch_exists(azure):
with azure_teardown(azure):
azure.touch(a)
assert azure.exists(a)