def ListFiles(root_path, recurse=False, filepattern=None, sort=False): """Return list of file paths under given root path. Directories are intentionally excluded from results. The root_path argument can be a local directory path, a Google storage directory URI, or a Colossus (/cns) directory path. Args: root_path: A local path, CNS path, or GS path to directory. recurse: Look for files in subdirectories, as well filepattern: glob pattern to match against basename of file sort: If True then do a default sort on paths Returns: List of paths to files that matched """ uri_type = GetUriType(root_path) if TYPE_GS == uri_type: return gslib.ListFiles(root_path, recurse=recurse, filepattern=filepattern, sort=sort) if TYPE_LOCAL == uri_type: return filelib.ListFiles(root_path, recurse=recurse, filepattern=filepattern, sort=sort) raise NotSupportedForType(uri_type)
def testListFiles(self): gs_path = 'gs://bucket/some/path' local_path = '/some/local/path' http_path = 'http://host.domain/some/path' result = 'TheResult' patt = 'TheFilePattern' self.mox.StubOutWithMock(gslib, 'ListFiles') self.mox.StubOutWithMock(filelib, 'ListFiles') # Set up the test replay script. # Run 1, local. filelib.ListFiles(local_path, recurse=True, filepattern=None, sort=False).AndReturn(result) # Run 2, GS. gslib.ListFiles(gs_path, recurse=False, filepattern=patt, sort=True).AndReturn(result) # Run 4, HTTP. self.mox.ReplayAll() # Run the test verification. self.assertEquals(result, urilib.ListFiles(local_path, recurse=True)) self.assertEquals( result, urilib.ListFiles(gs_path, filepattern=patt, sort=True)) self.assertRaises(urilib.NotSupportedForType, urilib.ListFiles, http_path) self.mox.VerifyAll()
def testIntegrationScript(self): dir1 = None dir2 = None try: dir1 = utils.CreateTmpDir('filelib_unittest1-') dir2 = utils.CreateTmpDir('filelib_unittest2-') self._SetUpTempdir(dir1) dir1_file1 = os.path.join(dir1, self.FILE1) dir1_file2 = os.path.join(dir1, self.FILE2) dir1_subfile = os.path.join(dir1, self.SUBFILE) dir1_top_files = [dir1_file1, dir1_file2] dir1_deep_files = dir1_top_files + [dir1_subfile] dir2_file1 = os.path.join(dir2, self.FILE1) dir2_file2 = os.path.join(dir2, self.FILE2) dir2_subdir = os.path.join(dir2, self.SUBDIR) dir2_subfile = os.path.join(dir2, self.SUBFILE) dir2_top_files = [dir2_file1, dir2_file2] dir2_deep_files = dir2_top_files + [dir2_subfile] # Test Exists. for dir1_path in dir1_deep_files: self.assertTrue(filelib.Exists(dir1_path)) for dir2_path in dir2_deep_files: self.assertFalse(filelib.Exists(dir2_path)) # Test ListFiles with various options. self.assertEqual(set(dir1_top_files), set(filelib.ListFiles(dir1))) self.assertEqual(set(dir1_deep_files), set(filelib.ListFiles(dir1, recurse=True))) self.assertEqual(sorted(dir1_deep_files), filelib.ListFiles(dir1, recurse=True, sort=True)) self.assertEqual(set([dir1_file1, dir1_subfile]), set(filelib.ListFiles(dir1, recurse=True, filepattern=self.FILE_GLOB))) # Test CopyFiles from dir1 to dir2. self.assertEqual(set(dir2_deep_files), set(filelib.CopyFiles(dir1, dir2))) for dir2_path in dir2_deep_files: self.assertTrue(filelib.Exists(dir2_path)) # Test Cmp. self.assertTrue(filelib.Cmp(dir1_file1, dir2_file1)) self.assertTrue(filelib.Cmp(dir2_file2, dir1_file2)) self.assertFalse(filelib.Cmp(dir1_file2, dir2_file1)) # Test RemoveDirContents. filelib.RemoveDirContents(dir2_subdir) self.assertTrue(filelib.Exists(dir2_subdir, as_dir=True)) self.assertFalse(filelib.Exists(dir2_subfile)) filelib.RemoveDirContents(dir2) self.assertTrue(filelib.Exists(dir2, as_dir=True)) for dir2_path in dir2_deep_files: self.assertFalse(filelib.Exists(dir2_path)) filelib.RemoveDirContents(dir1) self.assertTrue(filelib.Exists(dir1, as_dir=True)) for dir1_path in dir1_deep_files: self.assertFalse(filelib.Exists(dir1_path)) finally: for d in (dir1, dir2): if d and os.path.isdir(d): shutil.rmtree(d)