async def test_file_and_directory_error_with_slash_non_empty_file_only(
router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]],
cloud_scheme: str):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, cloud_scheme)
await write_file(fs, f'{src_base}not-empty-file-w-slash/', b'not-empty')
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}', recursive=True))
with pytest.raises(FileAndDirectoryError):
await collect_files(await
fs.listfiles(f'{src_base}not-empty-file-w-slash/',
recursive=True))
for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR,
Transfer.INFER_DEST):
with pytest.raises(FileAndDirectoryError):
dest_base = await fresh_dir(fs, bases, cloud_scheme)
await Copier.copy(
fs, sema,
Transfer(f'{src_base}not-empty-file-w-slash/',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
with pytest.raises(FileAndDirectoryError):
dest_base = await fresh_dir(fs, bases, cloud_scheme)
await Copier.copy(
fs, sema,
Transfer(f'{src_base}',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
async def test_file_and_directory_error_with_slash_non_empty_file(
router_filesystem):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, 'gs')
await write_file(fs, f'{src_base}not-empty/', b'not-empty')
await write_file(fs, f'{src_base}not-empty/bar', b'bar')
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}'))
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}', recursive=True))
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}not-empty/'))
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}not-empty/',
recursive=True))
for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR,
Transfer.INFER_DEST):
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}not-empty/bar',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
if transfer_type == Transfer.DEST_DIR:
exp_dest = f'{dest_base}bar'
await expect_file(fs, exp_dest, 'bar')
assert not await fs.isfile(f'{dest_base}not-empty/')
assert not await fs.isdir(f'{dest_base}not-empty/')
x = await collect_files(await fs.listfiles(f'{dest_base}'))
assert x == [f'{dest_base}bar'], x
else:
await expect_file(fs, dest_base.rstrip('/'), 'bar')
with pytest.raises(FileAndDirectoryError):
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}not-empty/',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
with pytest.raises(FileAndDirectoryError):
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
async def test_file_and_directory_error_with_slash_empty_file(
router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]],
cloud_scheme: str):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, cloud_scheme)
await write_file(fs, f'{src_base}empty/', '')
await write_file(fs, f'{src_base}empty/foo', b'foo')
await collect_files(await fs.listfiles(f'{src_base}'))
await collect_files(await fs.listfiles(f'{src_base}', recursive=True))
await collect_files(await fs.listfiles(f'{src_base}empty/'))
await collect_files(await fs.listfiles(f'{src_base}empty/',
recursive=True))
for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR,
Transfer.INFER_DEST):
dest_base = await fresh_dir(fs, bases, cloud_scheme)
await Copier.copy(
fs, sema,
Transfer(f'{src_base}',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
dest_base = await fresh_dir(fs, bases, cloud_scheme)
await Copier.copy(
fs, sema,
Transfer(f'{src_base}empty/',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
await collect_files(await fs.listfiles(f'{dest_base}'))
await collect_files(await fs.listfiles(f'{dest_base}', recursive=True))
if transfer_type == Transfer.DEST_DIR:
exp_dest = f'{dest_base}empty/foo'
await expect_file(fs, exp_dest, 'foo')
assert not await fs.isfile(f'{dest_base}empty/')
assert await fs.isdir(f'{dest_base}empty/')
await collect_files(await fs.listfiles(f'{dest_base}empty/'))
await collect_files(await fs.listfiles(f'{dest_base}empty/',
recursive=True))
else:
exp_dest = f'{dest_base}foo'
await expect_file(fs, exp_dest, 'foo')
async def test_file_overwrite_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
with RaisesOrGS(dest_base, IsADirectoryError):
await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_IS_TARGET))
async def test_copy_file_src_trailing_slash(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
with pytest.raises(FileNotFoundError):
await Copier.copy(fs, sema, Transfer(f'{src_base}a/', dest_base))
async def test_copy_multiple_dest_target_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await create_test_file(fs, 'src', src_base, 'b')
with RaisesOrGS(dest_base, NotADirectoryError):
await fs.copy(sema, Transfer([f'{src_base}a', f'{src_base}b'], dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_IS_TARGET))
async def test_copy_file_dest_target_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_DIR))
await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_copy_file_dest_target_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}a', treat_dest_as=Transfer.DEST_IS_TARGET))
await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_copy_file_dest_target_directory_doesnt_exist(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
# SourceCopier._copy_file creates destination directories as needed
await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}x', treat_dest_as=Transfer.DEST_DIR))
await expect_file(fs, f'{dest_base}x/a', 'src/a')
async def test_copy_rename_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await Copier.copy(fs, sema, Transfer(f'{src_base}a', f'{dest_base}x'))
await expect_file(fs, f'{dest_base}x', 'src/a')
async def test_copy_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/')))
await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_copy_multiple_dest_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await create_test_file(fs, 'src', src_base, 'b')
await create_test_file(fs, 'dest', dest_base, 'x')
with RaisesOrGS(dest_base, NotADirectoryError):
await fs.copy(sema, Transfer([f'{src_base}a', f'{src_base}b'], f'{dest_base}x'))
async def test_copy_rename_dir_dest_is_target(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_dir(fs, 'src', src_base, 'a/')
await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}x', treat_dest_as=Transfer.DEST_IS_TARGET))
await expect_file(fs, f'{dest_base}x/file1', 'src/a/file1')
await expect_file(fs, f'{dest_base}x/subdir/file2', 'src/a/subdir/file2')
async def test_overwrite_rename_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await create_test_file(fs, 'dest', dest_base, 'x')
await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}x'))
await expect_file(fs, f'{dest_base}x', 'src/a')
async def test_copy_src_parts(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_dir(fs, 'src', src_base, 'a/')
await fs.copy(sema, Transfer([f'{src_base}a/file1', f'{src_base}a/subdir'], dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_DIR))
await expect_file(fs, f'{dest_base}file1', 'src/a/file1')
await expect_file(fs, f'{dest_base}subdir/file2', 'src/a/subdir/file2')
async def test_copy_rename_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_dir(fs, 'src', src_base, 'a/')
await Copier.copy(fs, sema, Transfer(f'{src_base}a', f'{dest_base}x'))
await expect_file(fs, f'{dest_base}x/file1', 'src/a/file1')
await expect_file(fs, f'{dest_base}x/subdir/file2', 'src/a/subdir/file2')
async def test_copy_file_dest_trailing_slash_target_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await Copier.copy(
fs, sema,
Transfer(f'{src_base}a', dest_base, treat_dest_as=Transfer.DEST_DIR))
await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_overwrite_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_dir(fs, 'src', src_base, 'a/')
await create_test_dir(fs, 'dest', dest_base, 'a/')
await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/')))
await expect_file(fs, f'{dest_base}a/file1', 'src/a/file1')
await expect_file(fs, f'{dest_base}a/subdir/file2', 'src/a/subdir/file2')
await expect_file(fs, f'{dest_base}a/file3', 'dest/a/file3')
async def test_copy_dest_target_file_is_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
with RaisesOrObjectStore(dest_base, IsADirectoryError):
await Copier.copy(
fs, sema,
Transfer(f'{src_base}a',
dest_base.rstrip('/'),
treat_dest_as=Transfer.DEST_IS_TARGET))
async def test_file_and_directory_error(router_filesystem):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, 'gs')
dest_base = await fresh_dir(fs, bases, 'file')
await create_test_file(fs, 'src', src_base, 'a')
await create_test_file(fs, 'src', src_base, 'a/subfile')
with pytest.raises(FileAndDirectoryError):
await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/')))
async def test_copy_multiple(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_file(fs, 'src', src_base, 'a')
await create_test_file(fs, 'src', src_base, 'b')
await Copier.copy(
fs, sema,
Transfer([f'{src_base}a', f'{src_base}b'], dest_base.rstrip('/')))
await expect_file(fs, f'{dest_base}a', 'src/a')
await expect_file(fs, f'{dest_base}b', 'src/b')
async def test_file_and_directory_error_with_slash_non_empty_file_only(
router_filesystem):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, 'gs')
await write_file(fs, f'{src_base}not-empty-file-w-slash/', b'not-empty')
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}'))
with pytest.raises(FileAndDirectoryError):
await collect_files(await fs.listfiles(f'{src_base}', recursive=True))
with pytest.raises(FileAndDirectoryError):
await collect_files(await
fs.listfiles(f'{src_base}not-empty-file-w-slash/'))
with pytest.raises(FileAndDirectoryError):
await collect_files(await
fs.listfiles(f'{src_base}not-empty-file-w-slash/',
recursive=True))
for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR,
Transfer.INFER_DEST):
with pytest.raises(FileAndDirectoryError):
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}not-empty-file-w-slash/',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
with pytest.raises(FileAndDirectoryError):
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
async def test_copy_large_file(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
# mainly needs to be larger than the transfer block size (8K)
contents = secrets.token_bytes(1_000_000)
async with await fs.create(f'{src_base}a') as f:
await f.write(contents)
await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/')))
async with await fs.open(f'{dest_base}a') as f:
copy_contents = await f.read()
assert copy_contents == contents
async def test_file_and_directory_error(
router_filesystem: Tuple[asyncio.Semaphore, AsyncFS,
Dict[str, str]], cloud_scheme: str):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, cloud_scheme)
dest_base = await fresh_dir(fs, bases, 'file')
await create_test_file(fs, 'src', src_base, 'a')
await create_test_file(fs, 'src', src_base, 'a/subfile')
with pytest.raises(FileAndDirectoryError):
await Copier.copy(fs, sema,
Transfer(f'{src_base}a', dest_base.rstrip('/')))
async def test_file_and_directory_error_with_slash_empty_file_only(
router_filesystem):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, 'gs')
await write_file(fs, f'{src_base}empty-only/', '')
await collect_files(await fs.listfiles(f'{src_base}'))
await collect_files(await fs.listfiles(f'{src_base}', recursive=True))
await collect_files(await fs.listfiles(f'{src_base}empty-only/'))
await collect_files(await fs.listfiles(f'{src_base}empty-only/',
recursive=True))
for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR,
Transfer.INFER_DEST):
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}empty-only/',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
# We ignore empty directories when copying
with pytest.raises(FileNotFoundError):
await collect_files(await fs.listfiles(f'{dest_base}empty-only/'))
with pytest.raises(FileNotFoundError):
await collect_files(await fs.listfiles(f'{dest_base}empty-only/',
recursive=True))
dest_base = await fresh_dir(fs, bases, 'gs')
await fs.copy(
sema,
Transfer(f'{src_base}',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
async def test_overwrite_rename_dir(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
await create_test_dir(fs, 'src', src_base, 'a/')
await create_test_dir(fs, 'dest', dest_base, 'x/')
await Copier.copy(
fs, sema,
Transfer(f'{src_base}a',
f'{dest_base}x',
treat_dest_as=Transfer.DEST_IS_TARGET))
await expect_file(fs, f'{dest_base}x/file1', 'src/a/file1')
await expect_file(fs, f'{dest_base}x/subdir/file2', 'src/a/subdir/file2')
await expect_file(fs, f'{dest_base}x/file3', 'dest/x/file3')
async def run_test_spec(sema, fs, spec, src_base, dest_base):
await create_test_data(fs, 'src', src_base, 'a', spec['src_type'])
await create_test_data(fs, 'dest', dest_base, 'a', spec['dest_type'])
src = f'{src_base}a'
if spec['src_trailing_slash']:
src = src + '/'
dest_basename = spec['dest_basename']
if dest_basename:
dest = f'{dest_base}{dest_basename}'
else:
dest = dest_base
if spec['dest_trailing_slash']:
if not dest.endswith('/'):
dest = dest + '/'
else:
dest = dest.rstrip('/')
result = None
exc_type = None
try:
await fs.copy(sema,
Transfer(src, dest, treat_dest_as=spec['treat_dest_as']))
except Exception as e:
exc_type = type(e)
if exc_type not in (NotADirectoryError, IsADirectoryError,
FileNotFoundError):
raise
result = {'exception': exc_type.__name__}
if exc_type is None:
files = {}
async for entry in await fs.listfiles(dest_base, recursive=True):
url = await entry.url()
assert not url.endswith('/')
file = remove_prefix(url, dest_base.rstrip('/'))
async with await fs.open(url) as f:
contents = (await f.read()).decode('utf-8')
files[file] = contents
result = {'files': files}
return result
async def test_file_and_directory_error_with_slash_non_empty_file_only_for_google_non_recursive(
router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]]):
sema, fs, bases = router_filesystem
src_base = await fresh_dir(fs, bases, 'gs')
await write_file(fs, f'{src_base}empty-only/', '')
await collect_files(await fs.listfiles(f'{src_base}'))
await collect_files(await fs.listfiles(f'{src_base}empty-only/'))
for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR,
Transfer.INFER_DEST):
dest_base = await fresh_dir(fs, bases, 'gs')
await Copier.copy(
fs, sema,
Transfer(f'{src_base}empty-only/',
dest_base.rstrip('/'),
treat_dest_as=transfer_type))
# We ignore empty directories when copying
with pytest.raises(FileNotFoundError):
await collect_files(await fs.listfiles(f'{dest_base}empty-only/'))
async def test_copy_doesnt_exist(copy_test_context):
sema, fs, src_base, dest_base = copy_test_context
with pytest.raises(FileNotFoundError):
await Copier.copy(fs, sema, Transfer(f'{src_base}a', dest_base))
