def test_isfile():
fs = LocalFileSystem()
with filetexts(files, mode="b"):
for f in files.keys():
assert fs.isfile(f)
assert fs.isfile("file://" + f)
assert not fs.isfile("not-a-file")
assert not fs.isfile("file://not-a-file")
def test_recursive_get_put(tmpdir):
tmpdir = str(tmpdir)
fs = LocalFileSystem()
fs.mkdir(tmpdir + "/a1/a2/a3")
fs.touch(tmpdir + "/a1/a2/a3/afile")
fs.touch(tmpdir + "/a1/afile")
fs.get("file://{0}/a1".format(tmpdir), tmpdir + "/b1", recursive=True)
assert fs.isfile(tmpdir + "/b1/afile")
assert fs.isfile(tmpdir + "/b1/a2/a3/afile")
fs.put(tmpdir + "/b1", "file://{0}/c1".format(tmpdir), recursive=True)
assert fs.isfile(tmpdir + "/c1/afile")
assert fs.isfile(tmpdir + "/c1/a2/a3/afile")
def test_recursive_get_put(tmpdir):
tmpdir = str(tmpdir)
fs = LocalFileSystem()
fs.mkdir(tmpdir + '/a1/a2/a3')
fs.touch(tmpdir + '/a1/a2/a3/afile')
fs.touch(tmpdir + '/a1/afile')
fs.get('file://{0}/a1'.format(tmpdir), tmpdir + '/b1', recursive=True)
assert fs.isfile(tmpdir + '/b1/afile')
assert fs.isfile(tmpdir + '/b1/a2/a3/afile')
fs.put(tmpdir + '/b1', 'file://{0}/c1'.format(tmpdir), recursive=True)
assert fs.isfile(tmpdir + '/c1/afile')
assert fs.isfile(tmpdir + '/c1/a2/a3/afile')
def _resolve_single_pattern_locally(
base_path: str,
pattern: str,
allowed_extensions: Optional[List[str]] = None) -> List[Path]:
"""
Return the absolute paths to all the files that match the given patterns.
It also supports absolute paths in patterns.
If an URL is passed, it is returned as is.
"""
pattern = os.path.join(base_path, pattern)
data_files_ignore = FILES_TO_IGNORE
fs = LocalFileSystem()
glob_iter = [
PurePath(filepath) for filepath in fs.glob(pattern)
if fs.isfile(filepath)
]
matched_paths = [
Path(filepath).resolve() for filepath in glob_iter
if filepath.name not in data_files_ignore
and not filepath.name.startswith(".")
]
if allowed_extensions is not None:
out = [
filepath for filepath in matched_paths
if any(suffix[1:] in allowed_extensions
for suffix in filepath.suffixes)
]
if len(out) < len(matched_paths):
invalid_matched_files = list(set(matched_paths) - set(out))
logger.info(
f"Some files matched the pattern '{pattern}' at {Path(base_path).resolve()} but don't have valid data file extensions: {invalid_matched_files}"
)
else:
out = matched_paths
if not out and not contains_wildcards(pattern):
error_msg = f"Unable to find '{pattern}' at {Path(base_path).resolve()}"
if allowed_extensions is not None:
error_msg += f" with any supported extension {list(allowed_extensions)}"
raise FileNotFoundError(error_msg)
return sorted(out)
def test_isfile():
fs = LocalFileSystem()
with filetexts(files, mode='b'):
for f in files.keys():
assert fs.isfile(f)
assert not fs.isfile('not-a-file')
