def testListFiles(self):
files = [
'%s/some/path' % self.bucket_uri,
'%s/some/file/path' % self.bucket_uri,
]
directories = [
'%s/some/dir/' % self.bucket_uri,
'%s/some/dir/path/' % self.bucket_uri,
]
gs_uri = '%s/**' % self.bucket_uri
cmd = [self.gsutil, 'ls', gs_uri]
# Prepare cmd_result for a good run.
# Fake a trailing empty line.
output = '\n'.join(files + directories + [''])
cmd_result_ok = cros_test_lib.EasyAttr(output=output, returncode=0)
# Prepare exception for a run that finds nothing.
stderr = 'CommandException: One or more URLs matched no objects.\n'
cmd_result_empty = cros_build_lib.CommandResult(error=stderr)
empty_exception = cros_build_lib.RunCommandError(
stderr, cmd_result_empty)
# Prepare exception for a run that triggers a GS failure.
error = cros_build_lib.RunCommandError(GS_RETRY_FAILURE,
self.cmd_result)
# Set up the test replay script.
# Run 1, runs ok.
cros_build_lib.RunCommand(
cmd, redirect_stdout=True,
redirect_stderr=True).AndReturn(cmd_result_ok)
# Run 2, runs ok, sorts files.
cros_build_lib.RunCommand(
cmd, redirect_stdout=True,
redirect_stderr=True).AndReturn(cmd_result_ok)
# Run 3, finds nothing.
cros_build_lib.RunCommand(
cmd, redirect_stdout=True,
redirect_stderr=True).AndRaise(empty_exception)
# Run 4, failure in GS.
for _ix in xrange(gslib.RETRY_ATTEMPTS + 1):
cros_build_lib.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True).AndRaise(error)
self.mox.ReplayAll()
# Run the test verification.
result = gslib.ListFiles(self.bucket_uri, recurse=True)
self.assertEqual(files, result)
result = gslib.ListFiles(self.bucket_uri, recurse=True, sort=True)
self.assertEqual(sorted(files), result)
result = gslib.ListFiles(self.bucket_uri, recurse=True)
self.assertEqual([], result)
self.assertRaises(gslib.GSLibError,
gslib.ListFiles,
self.bucket_uri,
recurse=True)
self.mox.VerifyAll()
def testListFiles(self):
gs_path = 'gs://bucket/some/path'
local_path = '/some/local/path'
http_path = 'http://host.domain/some/path'
result = 'TheResult'
patt = 'TheFilePattern'
self.mox.StubOutWithMock(gslib, 'ListFiles')
self.mox.StubOutWithMock(filelib, 'ListFiles')
# Set up the test replay script.
# Run 1, local.
filelib.ListFiles(local_path,
recurse=True,
filepattern=None,
sort=False).AndReturn(result)
# Run 2, GS.
gslib.ListFiles(gs_path, recurse=False, filepattern=patt,
sort=True).AndReturn(result)
# Run 4, HTTP.
self.mox.ReplayAll()
# Run the test verification.
self.assertEquals(result, urilib.ListFiles(local_path, recurse=True))
self.assertEquals(
result, urilib.ListFiles(gs_path, filepattern=patt, sort=True))
self.assertRaises(urilib.NotSupportedForType, urilib.ListFiles,
http_path)
self.mox.VerifyAll()
def ListFiles(root_path, recurse=False, filepattern=None, sort=False):
"""Return list of file paths under given root path.
Directories are intentionally excluded from results. The root_path
argument can be a local directory path, a Google storage directory URI,
or a Colossus (/cns) directory path.
Args:
root_path: A local path, CNS path, or GS path to directory.
recurse: Look for files in subdirectories, as well
filepattern: glob pattern to match against basename of file
sort: If True then do a default sort on paths
Returns:
List of paths to files that matched
"""
uri_type = GetUriType(root_path)
if TYPE_GS == uri_type:
return gslib.ListFiles(root_path, recurse=recurse,
filepattern=filepattern, sort=sort)
if TYPE_LOCAL == uri_type:
return filelib.ListFiles(root_path, recurse=recurse,
filepattern=filepattern, sort=sort)
raise NotSupportedForType(uri_type)