def ListFiles(root_path, recurse=False, filepattern=None, sort=False):
"""Return list of file paths under given root path.
Directories are intentionally excluded from results. The root_path
argument can be a local directory path, a Google storage directory URI,
or a Colossus (/cns) directory path.
Args:
root_path: A local path, CNS path, or GS path to directory.
recurse: Look for files in subdirectories, as well
filepattern: glob pattern to match against basename of file
sort: If True then do a default sort on paths
Returns:
List of paths to files that matched
"""
uri_type = GetUriType(root_path)
if TYPE_GS == uri_type:
return gslib.ListFiles(root_path, recurse=recurse,
filepattern=filepattern, sort=sort)
if TYPE_LOCAL == uri_type:
return filelib.ListFiles(root_path, recurse=recurse,
filepattern=filepattern, sort=sort)
raise NotSupportedForType(uri_type)
def testListFiles(self):
gs_path = 'gs://bucket/some/path'
local_path = '/some/local/path'
http_path = 'http://host.domain/some/path'
result = 'TheResult'
patt = 'TheFilePattern'
self.mox.StubOutWithMock(gslib, 'ListFiles')
self.mox.StubOutWithMock(filelib, 'ListFiles')
# Set up the test replay script.
# Run 1, local.
filelib.ListFiles(local_path,
recurse=True,
filepattern=None,
sort=False).AndReturn(result)
# Run 2, GS.
gslib.ListFiles(gs_path, recurse=False, filepattern=patt,
sort=True).AndReturn(result)
# Run 4, HTTP.
self.mox.ReplayAll()
# Run the test verification.
self.assertEquals(result, urilib.ListFiles(local_path, recurse=True))
self.assertEquals(
result, urilib.ListFiles(gs_path, filepattern=patt, sort=True))
self.assertRaises(urilib.NotSupportedForType, urilib.ListFiles,
http_path)
self.mox.VerifyAll()
def testIntegrationScript(self):
dir1 = None
dir2 = None
try:
dir1 = utils.CreateTmpDir('filelib_unittest1-')
dir2 = utils.CreateTmpDir('filelib_unittest2-')
self._SetUpTempdir(dir1)
dir1_file1 = os.path.join(dir1, self.FILE1)
dir1_file2 = os.path.join(dir1, self.FILE2)
dir1_subfile = os.path.join(dir1, self.SUBFILE)
dir1_top_files = [dir1_file1, dir1_file2]
dir1_deep_files = dir1_top_files + [dir1_subfile]
dir2_file1 = os.path.join(dir2, self.FILE1)
dir2_file2 = os.path.join(dir2, self.FILE2)
dir2_subdir = os.path.join(dir2, self.SUBDIR)
dir2_subfile = os.path.join(dir2, self.SUBFILE)
dir2_top_files = [dir2_file1, dir2_file2]
dir2_deep_files = dir2_top_files + [dir2_subfile]
# Test Exists.
for dir1_path in dir1_deep_files:
self.assertTrue(filelib.Exists(dir1_path))
for dir2_path in dir2_deep_files:
self.assertFalse(filelib.Exists(dir2_path))
# Test ListFiles with various options.
self.assertEqual(set(dir1_top_files),
set(filelib.ListFiles(dir1)))
self.assertEqual(set(dir1_deep_files),
set(filelib.ListFiles(dir1, recurse=True)))
self.assertEqual(sorted(dir1_deep_files),
filelib.ListFiles(dir1, recurse=True, sort=True))
self.assertEqual(set([dir1_file1, dir1_subfile]),
set(filelib.ListFiles(dir1, recurse=True,
filepattern=self.FILE_GLOB)))
# Test CopyFiles from dir1 to dir2.
self.assertEqual(set(dir2_deep_files),
set(filelib.CopyFiles(dir1, dir2)))
for dir2_path in dir2_deep_files:
self.assertTrue(filelib.Exists(dir2_path))
# Test Cmp.
self.assertTrue(filelib.Cmp(dir1_file1, dir2_file1))
self.assertTrue(filelib.Cmp(dir2_file2, dir1_file2))
self.assertFalse(filelib.Cmp(dir1_file2, dir2_file1))
# Test RemoveDirContents.
filelib.RemoveDirContents(dir2_subdir)
self.assertTrue(filelib.Exists(dir2_subdir, as_dir=True))
self.assertFalse(filelib.Exists(dir2_subfile))
filelib.RemoveDirContents(dir2)
self.assertTrue(filelib.Exists(dir2, as_dir=True))
for dir2_path in dir2_deep_files:
self.assertFalse(filelib.Exists(dir2_path))
filelib.RemoveDirContents(dir1)
self.assertTrue(filelib.Exists(dir1, as_dir=True))
for dir1_path in dir1_deep_files:
self.assertFalse(filelib.Exists(dir1_path))
finally:
for d in (dir1, dir2):
if d and os.path.isdir(d):
shutil.rmtree(d)