async def test_file_and_directory_error_with_slash_non_empty_file_only( router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]], cloud_scheme: str): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, cloud_scheme) await write_file(fs, f'{src_base}not-empty-file-w-slash/', b'not-empty') with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}', recursive=True)) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}not-empty-file-w-slash/', recursive=True)) for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR, Transfer.INFER_DEST): with pytest.raises(FileAndDirectoryError): dest_base = await fresh_dir(fs, bases, cloud_scheme) await Copier.copy( fs, sema, Transfer(f'{src_base}not-empty-file-w-slash/', dest_base.rstrip('/'), treat_dest_as=transfer_type)) with pytest.raises(FileAndDirectoryError): dest_base = await fresh_dir(fs, bases, cloud_scheme) await Copier.copy( fs, sema, Transfer(f'{src_base}', dest_base.rstrip('/'), treat_dest_as=transfer_type))
async def test_file_and_directory_error_with_slash_non_empty_file( router_filesystem): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, 'gs') await write_file(fs, f'{src_base}not-empty/', b'not-empty') await write_file(fs, f'{src_base}not-empty/bar', b'bar') with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}')) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}', recursive=True)) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}not-empty/')) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}not-empty/', recursive=True)) for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR, Transfer.INFER_DEST): dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}not-empty/bar', dest_base.rstrip('/'), treat_dest_as=transfer_type)) if transfer_type == Transfer.DEST_DIR: exp_dest = f'{dest_base}bar' await expect_file(fs, exp_dest, 'bar') assert not await fs.isfile(f'{dest_base}not-empty/') assert not await fs.isdir(f'{dest_base}not-empty/') x = await collect_files(await fs.listfiles(f'{dest_base}')) assert x == [f'{dest_base}bar'], x else: await expect_file(fs, dest_base.rstrip('/'), 'bar') with pytest.raises(FileAndDirectoryError): dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}not-empty/', dest_base.rstrip('/'), treat_dest_as=transfer_type)) with pytest.raises(FileAndDirectoryError): dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}', dest_base.rstrip('/'), treat_dest_as=transfer_type))
async def test_file_and_directory_error_with_slash_empty_file( router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]], cloud_scheme: str): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, cloud_scheme) await write_file(fs, f'{src_base}empty/', '') await write_file(fs, f'{src_base}empty/foo', b'foo') await collect_files(await fs.listfiles(f'{src_base}')) await collect_files(await fs.listfiles(f'{src_base}', recursive=True)) await collect_files(await fs.listfiles(f'{src_base}empty/')) await collect_files(await fs.listfiles(f'{src_base}empty/', recursive=True)) for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR, Transfer.INFER_DEST): dest_base = await fresh_dir(fs, bases, cloud_scheme) await Copier.copy( fs, sema, Transfer(f'{src_base}', dest_base.rstrip('/'), treat_dest_as=transfer_type)) dest_base = await fresh_dir(fs, bases, cloud_scheme) await Copier.copy( fs, sema, Transfer(f'{src_base}empty/', dest_base.rstrip('/'), treat_dest_as=transfer_type)) await collect_files(await fs.listfiles(f'{dest_base}')) await collect_files(await fs.listfiles(f'{dest_base}', recursive=True)) if transfer_type == Transfer.DEST_DIR: exp_dest = f'{dest_base}empty/foo' await expect_file(fs, exp_dest, 'foo') assert not await fs.isfile(f'{dest_base}empty/') assert await fs.isdir(f'{dest_base}empty/') await collect_files(await fs.listfiles(f'{dest_base}empty/')) await collect_files(await fs.listfiles(f'{dest_base}empty/', recursive=True)) else: exp_dest = f'{dest_base}foo' await expect_file(fs, exp_dest, 'foo')
async def test_file_overwrite_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') with RaisesOrGS(dest_base, IsADirectoryError): await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_IS_TARGET))
async def test_copy_file_src_trailing_slash(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') with pytest.raises(FileNotFoundError): await Copier.copy(fs, sema, Transfer(f'{src_base}a/', dest_base))
async def test_copy_multiple_dest_target_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await create_test_file(fs, 'src', src_base, 'b') with RaisesOrGS(dest_base, NotADirectoryError): await fs.copy(sema, Transfer([f'{src_base}a', f'{src_base}b'], dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_IS_TARGET))
async def test_copy_file_dest_target_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_DIR)) await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_copy_file_dest_target_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}a', treat_dest_as=Transfer.DEST_IS_TARGET)) await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_copy_file_dest_target_directory_doesnt_exist(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') # SourceCopier._copy_file creates destination directories as needed await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}x', treat_dest_as=Transfer.DEST_DIR)) await expect_file(fs, f'{dest_base}x/a', 'src/a')
async def test_copy_rename_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await Copier.copy(fs, sema, Transfer(f'{src_base}a', f'{dest_base}x')) await expect_file(fs, f'{dest_base}x', 'src/a')
async def test_copy_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'))) await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_copy_multiple_dest_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await create_test_file(fs, 'src', src_base, 'b') await create_test_file(fs, 'dest', dest_base, 'x') with RaisesOrGS(dest_base, NotADirectoryError): await fs.copy(sema, Transfer([f'{src_base}a', f'{src_base}b'], f'{dest_base}x'))
async def test_copy_rename_dir_dest_is_target(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_dir(fs, 'src', src_base, 'a/') await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}x', treat_dest_as=Transfer.DEST_IS_TARGET)) await expect_file(fs, f'{dest_base}x/file1', 'src/a/file1') await expect_file(fs, f'{dest_base}x/subdir/file2', 'src/a/subdir/file2')
async def test_overwrite_rename_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await create_test_file(fs, 'dest', dest_base, 'x') await fs.copy(sema, Transfer(f'{src_base}a', f'{dest_base}x')) await expect_file(fs, f'{dest_base}x', 'src/a')
async def test_copy_src_parts(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_dir(fs, 'src', src_base, 'a/') await fs.copy(sema, Transfer([f'{src_base}a/file1', f'{src_base}a/subdir'], dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_DIR)) await expect_file(fs, f'{dest_base}file1', 'src/a/file1') await expect_file(fs, f'{dest_base}subdir/file2', 'src/a/subdir/file2')
async def test_copy_rename_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_dir(fs, 'src', src_base, 'a/') await Copier.copy(fs, sema, Transfer(f'{src_base}a', f'{dest_base}x')) await expect_file(fs, f'{dest_base}x/file1', 'src/a/file1') await expect_file(fs, f'{dest_base}x/subdir/file2', 'src/a/subdir/file2')
async def test_copy_file_dest_trailing_slash_target_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await Copier.copy( fs, sema, Transfer(f'{src_base}a', dest_base, treat_dest_as=Transfer.DEST_DIR)) await expect_file(fs, f'{dest_base}a', 'src/a')
async def test_overwrite_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_dir(fs, 'src', src_base, 'a/') await create_test_dir(fs, 'dest', dest_base, 'a/') await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'))) await expect_file(fs, f'{dest_base}a/file1', 'src/a/file1') await expect_file(fs, f'{dest_base}a/subdir/file2', 'src/a/subdir/file2') await expect_file(fs, f'{dest_base}a/file3', 'dest/a/file3')
async def test_copy_dest_target_file_is_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') with RaisesOrObjectStore(dest_base, IsADirectoryError): await Copier.copy( fs, sema, Transfer(f'{src_base}a', dest_base.rstrip('/'), treat_dest_as=Transfer.DEST_IS_TARGET))
async def test_file_and_directory_error(router_filesystem): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, 'gs') dest_base = await fresh_dir(fs, bases, 'file') await create_test_file(fs, 'src', src_base, 'a') await create_test_file(fs, 'src', src_base, 'a/subfile') with pytest.raises(FileAndDirectoryError): await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/')))
async def test_copy_multiple(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_file(fs, 'src', src_base, 'a') await create_test_file(fs, 'src', src_base, 'b') await Copier.copy( fs, sema, Transfer([f'{src_base}a', f'{src_base}b'], dest_base.rstrip('/'))) await expect_file(fs, f'{dest_base}a', 'src/a') await expect_file(fs, f'{dest_base}b', 'src/b')
async def test_file_and_directory_error_with_slash_non_empty_file_only( router_filesystem): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, 'gs') await write_file(fs, f'{src_base}not-empty-file-w-slash/', b'not-empty') with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}')) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}', recursive=True)) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}not-empty-file-w-slash/')) with pytest.raises(FileAndDirectoryError): await collect_files(await fs.listfiles(f'{src_base}not-empty-file-w-slash/', recursive=True)) for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR, Transfer.INFER_DEST): with pytest.raises(FileAndDirectoryError): dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}not-empty-file-w-slash/', dest_base.rstrip('/'), treat_dest_as=transfer_type)) with pytest.raises(FileAndDirectoryError): dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}', dest_base.rstrip('/'), treat_dest_as=transfer_type))
async def test_copy_large_file(copy_test_context): sema, fs, src_base, dest_base = copy_test_context # mainly needs to be larger than the transfer block size (8K) contents = secrets.token_bytes(1_000_000) async with await fs.create(f'{src_base}a') as f: await f.write(contents) await fs.copy(sema, Transfer(f'{src_base}a', dest_base.rstrip('/'))) async with await fs.open(f'{dest_base}a') as f: copy_contents = await f.read() assert copy_contents == contents
async def test_file_and_directory_error( router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]], cloud_scheme: str): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, cloud_scheme) dest_base = await fresh_dir(fs, bases, 'file') await create_test_file(fs, 'src', src_base, 'a') await create_test_file(fs, 'src', src_base, 'a/subfile') with pytest.raises(FileAndDirectoryError): await Copier.copy(fs, sema, Transfer(f'{src_base}a', dest_base.rstrip('/')))
async def test_file_and_directory_error_with_slash_empty_file_only( router_filesystem): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, 'gs') await write_file(fs, f'{src_base}empty-only/', '') await collect_files(await fs.listfiles(f'{src_base}')) await collect_files(await fs.listfiles(f'{src_base}', recursive=True)) await collect_files(await fs.listfiles(f'{src_base}empty-only/')) await collect_files(await fs.listfiles(f'{src_base}empty-only/', recursive=True)) for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR, Transfer.INFER_DEST): dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}empty-only/', dest_base.rstrip('/'), treat_dest_as=transfer_type)) # We ignore empty directories when copying with pytest.raises(FileNotFoundError): await collect_files(await fs.listfiles(f'{dest_base}empty-only/')) with pytest.raises(FileNotFoundError): await collect_files(await fs.listfiles(f'{dest_base}empty-only/', recursive=True)) dest_base = await fresh_dir(fs, bases, 'gs') await fs.copy( sema, Transfer(f'{src_base}', dest_base.rstrip('/'), treat_dest_as=transfer_type))
async def test_overwrite_rename_dir(copy_test_context): sema, fs, src_base, dest_base = copy_test_context await create_test_dir(fs, 'src', src_base, 'a/') await create_test_dir(fs, 'dest', dest_base, 'x/') await Copier.copy( fs, sema, Transfer(f'{src_base}a', f'{dest_base}x', treat_dest_as=Transfer.DEST_IS_TARGET)) await expect_file(fs, f'{dest_base}x/file1', 'src/a/file1') await expect_file(fs, f'{dest_base}x/subdir/file2', 'src/a/subdir/file2') await expect_file(fs, f'{dest_base}x/file3', 'dest/x/file3')
async def run_test_spec(sema, fs, spec, src_base, dest_base): await create_test_data(fs, 'src', src_base, 'a', spec['src_type']) await create_test_data(fs, 'dest', dest_base, 'a', spec['dest_type']) src = f'{src_base}a' if spec['src_trailing_slash']: src = src + '/' dest_basename = spec['dest_basename'] if dest_basename: dest = f'{dest_base}{dest_basename}' else: dest = dest_base if spec['dest_trailing_slash']: if not dest.endswith('/'): dest = dest + '/' else: dest = dest.rstrip('/') result = None exc_type = None try: await fs.copy(sema, Transfer(src, dest, treat_dest_as=spec['treat_dest_as'])) except Exception as e: exc_type = type(e) if exc_type not in (NotADirectoryError, IsADirectoryError, FileNotFoundError): raise result = {'exception': exc_type.__name__} if exc_type is None: files = {} async for entry in await fs.listfiles(dest_base, recursive=True): url = await entry.url() assert not url.endswith('/') file = remove_prefix(url, dest_base.rstrip('/')) async with await fs.open(url) as f: contents = (await f.read()).decode('utf-8') files[file] = contents result = {'files': files} return result
async def test_file_and_directory_error_with_slash_non_empty_file_only_for_google_non_recursive( router_filesystem: Tuple[asyncio.Semaphore, AsyncFS, Dict[str, str]]): sema, fs, bases = router_filesystem src_base = await fresh_dir(fs, bases, 'gs') await write_file(fs, f'{src_base}empty-only/', '') await collect_files(await fs.listfiles(f'{src_base}')) await collect_files(await fs.listfiles(f'{src_base}empty-only/')) for transfer_type in (Transfer.DEST_IS_TARGET, Transfer.DEST_DIR, Transfer.INFER_DEST): dest_base = await fresh_dir(fs, bases, 'gs') await Copier.copy( fs, sema, Transfer(f'{src_base}empty-only/', dest_base.rstrip('/'), treat_dest_as=transfer_type)) # We ignore empty directories when copying with pytest.raises(FileNotFoundError): await collect_files(await fs.listfiles(f'{dest_base}empty-only/'))
async def test_copy_doesnt_exist(copy_test_context): sema, fs, src_base, dest_base = copy_test_context with pytest.raises(FileNotFoundError): await Copier.copy(fs, sema, Transfer(f'{src_base}a', dest_base))