def setup_dir(azure): d = str(working_dir() / 'foo') azure.mkdir(d) try: yield d finally: azure.rm(d, recursive=True)
def test_chmod(azure): with azure_teardown(azure): azure.touch(a) assert azure.info(a)['permission'] == '770' azure.chmod(a, '0555') assert azure.info(a)['permission'] == '555' with pytest.raises((OSError, IOError)): with azure.open(a, 'ab') as f: try: f.write(b'data') except Exception as e: print(e) raise e azure.chmod(a, '0770') azure.rm(a) azure.mkdir(test_dir / 'deep') azure.touch(test_dir / 'deep' / 'file') azure.chmod(test_dir / 'deep', '660') with pytest.raises((OSError, IOError)): azure.ls(test_dir / 'deep') azure.chmod(test_dir / 'deep', '770')
def test_glob_walk_invalidate_cache(azure): with azure_teardown(azure): azure.mkdir(test_dir / 'c') azure.mkdir(test_dir / 'c' / 'd') filenames = ['a', 'a1', 'a2', 'a3', 'b1', 'c/x1', 'c/x2', 'c/d/x3'] filenames = [test_dir / s for s in filenames] for fn in filenames: azure.touch(fn) assert set(azure.glob(test_dir / 'a*')) == { posix(test_dir / 'a'), posix(test_dir / 'a1'), posix(test_dir / 'a2'), posix(test_dir / 'a3')} assert set(azure.glob(test_dir / 'c' / '*')) == { posix(test_dir / 'c' / 'x1'), posix(test_dir / 'c' / 'x2')} assert (set(azure.glob(test_dir / 'c')) == set(azure.glob(test_dir / 'c' / ''))) assert set(azure.glob(test_dir / 'a')) == {posix(test_dir / 'a')} assert set(azure.glob(test_dir / 'a1')) == {posix(test_dir / 'a1')} assert set(azure.glob(test_dir / '*')) == { posix(test_dir / 'a'), posix(test_dir / 'a1'), posix(test_dir / 'a2'), posix(test_dir / 'a3'), posix(test_dir / 'b1')} assert set(azure.walk(test_dir, invalidate_cache=True)) == { posix(test_dir / 'a'), posix(test_dir / 'a1'), posix(test_dir / 'a2'), posix(test_dir / 'a3'), posix(test_dir / 'b1'), posix(test_dir / 'c' / 'x1'), posix(test_dir / 'c' / 'x2'), posix(test_dir / 'c' / 'd' / 'x3')} assert set(azure.walk(test_dir / 'c', invalidate_cache=True)) == { posix(test_dir / 'c' / 'x1'), posix(test_dir / 'c' / 'x2'), posix(test_dir / 'c' / 'd' / 'x3')} assert set(azure.walk(test_dir / 'c', invalidate_cache=True)) == set(azure.walk(test_dir / 'c', invalidate_cache=True)) # test glob and walk with details=True glob_details = azure.glob(test_dir / '*', details=True, invalidate_cache=True) # validate that the objects are subscriptable assert glob_details[0]['name'] is not None assert glob_details[0]['type'] is not None walk_details = azure.walk(test_dir, details=True, invalidate_cache=True) assert walk_details[0]['name'] is not None assert walk_details[0]['type'] is not None
def setup_tree(azure): for directory in ['', 'data/a', 'data/b']: azure.mkdir(test_dir / directory) for filename in ['x.csv', 'y.csv', 'z.txt']: with azure.open(test_dir / directory / filename, 'wb') as f: f.write(b'123456') try: yield finally: for path in azure.ls(test_dir, invalidate_cache=False): if azure.exists(path, invalidate_cache=False): azure.rm(path, recursive=True)
def test_upload_overwrite(local_files, azure): bigfile, littlefile, emptyfile, a, b, c = local_files with azure_teardown(azure): # create the folder that we want to make sure the overwrite # test fails on if it doesn't already exist if not azure.exists(test_dir): azure.mkdir(test_dir) with pytest.raises(OSError) as e: ADLUploader(azure, test_dir, littlefile, nthreads=1) assert test_dir.as_posix() in str(e)
def test_ls_batched(azure): test_dir = working_dir() / 'abc' azure.mkdir(test_dir) with azure_teardown(azure): test_size = 10 assert azure._ls(test_dir, batch_size=10) == [] create_files(azure, number_of_files = 10, prefix='123', root_path=test_dir) with pytest.raises(ValueError): assert len(azure._ls(test_dir, batch_size=1)) == test_size assert len(azure._ls(test_dir, batch_size=9)) == test_size assert len(azure._ls(test_dir, batch_size=10)) == test_size assert len(azure._ls(test_dir, batch_size=11)) == test_size assert len(azure._ls(test_dir, batch_size=2)) == test_size assert len(azure._ls(test_dir, batch_size=100)) == test_size assert len(azure._ls(test_dir)) == test_size
def __test_retry_error(azure, error_status, total_tries, is_exception_expected=False, last_try_status=200, body=""): mock_url_a = mock_url + a while total_tries > 1: responses.add(responses.PUT, mock_url_a, body=body, status=error_status) total_tries -= 1 responses.add(responses.PUT, mock_url_a, body="", status=last_try_status) # teardown not required in mock tests try: azure.mkdir(a) assert not is_exception_expected except DatalakeRESTException: assert is_exception_expected