def testRemove(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
path1 = 'gs://bucket/some/gs/path'
path2 = 'gs://bucket/some/other/path'
# Set up the test replay script.
# Run 1, one path.
utils.RunCommand([self.gsutil, 'rm', path1],
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
# Run 2, two paths.
utils.RunCommand([self.gsutil, 'rm', path1, path2],
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
# Run 3, one path, recursive.
utils.RunCommand([self.gsutil, 'rm', '-R', path1],
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
self.mox.ReplayAll()
# Run the test verification.
gslib.Remove(path1)
gslib.Remove(path1, path2)
gslib.Remove(path1, recurse=True)
self.mox.VerifyAll()
def testCopy(self):
src_path = '/path/to/some/file'
dest_path = 'gs://bucket/some/gs/path'
self.mox.StubOutWithMock(utils, 'RunCommand')
# Set up the test replay script.
# Run 1, success.
cmd = [self.gsutil, 'cp', src_path, dest_path]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
# Run 2, failure.
for _ix in xrange(gslib.RETRY_ATTEMPTS + 1):
cmd = [self.gsutil, 'cp', src_path, dest_path]
utils.RunCommand(
cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(
utils.CommandFailedException(GS_RETRY_FAILURE))
self.mox.ReplayAll()
# Run the test verification.
gslib.Copy(src_path, dest_path)
self.assertRaises(gslib.CopyFail, gslib.Copy, src_path, dest_path)
self.mox.VerifyAll()
def _MD5Sum(self, file_path):
"""Use RunCommand to get the md5sum of a file."""
md5_path = '/usr/bin/md5sum'
if not os.path.exists(md5_path):
self.fail('%s is required to test MD5 logic.')
cmd = [md5_path, file_path]
return utils.RunCommand(cmd, redirect_stdout=True).split(' ')[0]
def _CreateArchive(self, archive_file, hashes, hash_names):
"""Take the hash strings and bundle them in the signer request format.
Take the contents of an array of strings, and put them into a specified
file in .tar.bz2 format. Each string is named with a specified name in
the tar file.
The number of hashes and number of hash_names must be equal. The
archive_file will be created or overridden as needed. It's up to
the caller to ensure it's cleaned up.
Args:
archive_file: Name of file to put the tar contents into.
hashes: List of hashes to sign, stored in strings.
hash_names: File names expected in the signer request.
"""
try:
tmp_dir = tempfile.mkdtemp()
# Copy hash files into tmp_dir with standard hash names.
for h, hash_name in zip(hashes, hash_names):
with open(os.path.join(tmp_dir, hash_name), 'wb') as f:
f.write(h)
cmd = ['tar', '-cjf', archive_file] + hash_names
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
cwd=tmp_dir)
finally:
# Cleanup.
shutil.rmtree(tmp_dir)
def testFileSize(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = '%s/%s' % (self.bucket_uri, 'some/file/path')
# Set up the test replay script.
cmd = [self.gsutil, '-d', 'stat', gs_uri]
size = 96
output = '\n'.join([
'header: x-goog-generation: 1386322968237000',
'header: x-goog-metageneration: 1',
'header: x-goog-stored-content-encoding: identity',
'header: x-goog-stored-content-length: %d' % size,
'header: Content-Type: application/octet-stream'
])
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(
cros_test_lib.EasyAttr(output=output))
self.mox.ReplayAll()
# Run the test verification.
result = gslib.FileSize(gs_uri)
self.assertEqual(size, result)
self.mox.VerifyAll()
def testRunGsutilCommand(self):
args = ['TheCommand', 'Arg1', 'Arg2']
cmd = [self.gsutil] + args
self.mox.StubOutWithMock(utils, 'RunCommand')
# Set up the test replay script.
# Run 1.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(1)
# Run 2.
utils.RunCommand(cmd,
redirect_stdout=False,
redirect_stderr=True,
return_result=True).AndReturn(2)
# Run 3.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True,
error_ok=True).AndReturn(3)
# Run 4.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(
utils.CommandFailedException())
# Run 5.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(
OSError(errno.ENOENT, 'errmsg'))
self.mox.ReplayAll()
# Run the test verification.
self.assertEqual(1, gslib.RunGsutilCommand(args))
self.assertEqual(2, gslib.RunGsutilCommand(args,
redirect_stdout=False))
self.assertEqual(3, gslib.RunGsutilCommand(args, error_ok=True))
self.assertRaises(gslib.GSLibError, gslib.RunGsutilCommand, args)
self.assertRaises(gslib.GsutilMissingError, gslib.RunGsutilCommand,
args)
self.mox.VerifyAll()
def testDeleteBucket(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
# Set up the test replay script.
cmd = [self.gsutil, 'rm', '%s/*' % self.bucket_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
error_ok=True,
return_result=True)
cmd = [self.gsutil, 'rb', self.bucket_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
self.mox.ReplayAll()
# Run the test verification.
gslib.DeleteBucket(self.bucket_name)
self.mox.VerifyAll()
def testListDirs(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
files = [
'%s/some/path' % self.bucket_uri,
'%s/some/file/path' % self.bucket_uri,
]
directories = [
'%s/some/dir/' % self.bucket_uri,
'%s/some/dir/path/' % self.bucket_uri,
]
gs_uri = '%s/**' % self.bucket_uri
cmd = [self.gsutil, 'ls', gs_uri]
# Prepare cmd_result for a good run.
# Fake a trailing empty line.
output = '\n'.join(files + directories + [''])
cmd_result = cros_test_lib.EasyAttr(output=output, returncode=0)
# Set up the test replay script.
# Run 1.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(cmd_result)
# Run 2.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(cmd_result)
self.mox.ReplayAll()
# Run the test verification.
result = gslib.ListDirs(self.bucket_uri, recurse=True)
self.assertEqual(directories, result)
result = gslib.ListDirs(self.bucket_uri, recurse=True, sort=True)
self.assertEqual(sorted(directories), result)
self.mox.VerifyAll()
def _TestCatWithHeaders(self, gs_uri, cmd_output, cmd_error):
self.mox.StubOutWithMock(utils, 'RunCommand')
# Set up the test replay script.
# Run 1, versioning not enabled in bucket, one line of output.
cmd = ['gsutil', '-d', 'cat', gs_uri]
cmd_result = cros_test_lib.EasyAttr(output=cmd_output,
error=cmd_error,
cmdstr=' '.join(cmd))
cmd[0] = mox.IsA(str)
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(cmd_result)
self.mox.ReplayAll()
def testRemoveNoMatch(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
path = 'gs://bucket/some/gs/path'
# Set up the test replay script.
cmd = [self.gsutil, 'rm', path]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
self.mox.ReplayAll()
# Run the test verification.
gslib.Remove(path, ignore_no_match=True)
self.mox.VerifyAll()
def testStatFail(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
path = 'gs://bucket/some/gs/path'
# Set up the test replay script.
cmd = [self.gsutil, 'stat', path]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(
utils.CommandFailedException())
self.mox.ReplayAll()
# Run the test verification.
self.assertRaises(gslib.StatFail, gslib.Stat, path)
self.mox.VerifyAll()
def testStat(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
path = 'gs://bucket/some/gs/path'
# Set up the test replay script.
cmd = [self.gsutil, 'stat', path]
result = cros_test_lib.EasyAttr(error='', output='')
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(result)
self.mox.ReplayAll()
# Run the test verification.
self.assertIs(gslib.Stat(path), None)
self.mox.VerifyAll()
def testMove(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
src_path = 'gs://bucket/some/gs/path'
dest_path = '/some/other/path'
# Set up the test replay script.
cmd = [self.gsutil, 'mv', src_path, dest_path]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
self.mox.ReplayAll()
# Run the test verification.
gslib.Move(src_path, dest_path)
self.mox.VerifyAll()
def testSetACL(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = 'gs://bucket/foo/bar/somefile'
acl_file = '/some/gs/acl/file'
# Set up the test replay script.
cmd = [self.gsutil, 'setacl', acl_file, gs_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True)
self.mox.ReplayAll()
# Run the test verification.
gslib.SetACL(gs_uri, acl_file)
self.mox.VerifyAll()
def testSetACLFail(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = 'gs://bucket/foo/bar/somefile'
acl_file = '/some/gs/acl/file'
# Set up the test replay script. (Multiple times because of retry logic)
cmd = [self.gsutil, 'setacl', acl_file, gs_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).MultipleTimes().AndRaise(
utils.CommandFailedException())
self.mox.ReplayAll()
# Run the test verification.
self.assertRaises(gslib.AclFail, gslib.SetACL, gs_uri, acl_file)
self.mox.VerifyAll()
def testFileSizeNoSuchFile(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = '%s/%s' % (self.bucket_uri, 'some/file/path')
# Set up the test replay script.
cmd = [self.gsutil, '-d', 'stat', gs_uri]
for _ in xrange(0, gslib.RETRY_ATTEMPTS + 1):
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(
utils.CommandFailedException)
self.mox.ReplayAll()
# Run the test verification.
self.assertRaises(gslib.URIError, gslib.FileSize, gs_uri)
self.mox.VerifyAll()
def testDeleteBucketFail(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
# Set up the test replay script.
cmd = [self.gsutil, 'rm', '%s/*' % self.bucket_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
error_ok=True,
return_result=True).AndRaise(
utils.CommandFailedException(GS_DONE_FAILURE))
self.mox.ReplayAll()
# Run the test verification.
self.assertRaises(gslib.BucketOperationError, gslib.DeleteBucket,
self.bucket_name)
self.mox.VerifyAll()
def testRemoveFail(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
path = 'gs://bucket/some/gs/path'
# Set up the test replay script.
cmd = [self.gsutil, 'rm', path]
for _ix in xrange(gslib.RETRY_ATTEMPTS + 1):
utils.RunCommand(
cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True,
).AndRaise(utils.CommandFailedException(GS_RETRY_FAILURE))
self.mox.ReplayAll()
# Run the test verification.
self.assertRaises(gslib.RemoveFail, gslib.Remove, path)
self.mox.VerifyAll()
def testCreateBucketFail(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
# Set up the test replay script.
cmd = [self.gsutil, 'mb', self.bucket_uri]
for _ix in xrange(gslib.RETRY_ATTEMPTS + 1):
utils.RunCommand(
cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(
utils.CommandFailedException(GS_RETRY_FAILURE))
self.mox.ReplayAll()
# Run the test verification.
self.assertRaises(gslib.BucketOperationError, gslib.CreateBucket,
self.bucket_name)
self.mox.VerifyAll()
def _PrepareGenerator(self):
"""Download, and extract au-generator.zip into self.generator_dir."""
if self._au_generator_uri_override:
generator_uri = self._au_generator_uri_override
else:
generator_uri = gspaths.ChromeosReleases.GeneratorUri(
self.payload.tgt_image.channel, self.payload.tgt_image.board,
self.payload.tgt_image.version, self.payload.tgt_image.bucket)
logging.info('Preparing au-generator.zip from %s.', generator_uri)
# Extract zipped delta generator files to the expected directory.
tmp_zip = self.cache.GetFileInTempFile(generator_uri)
utils.RunCommand(
['unzip', '-o', '-d', self.generator_dir, tmp_zip.name],
redirect_stdout=True,
redirect_stderr=True)
tmp_zip.close()
def testMD5SumAccessOK(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = 'gs://bucket/foo/bar/somefile'
crc32c = 'c96fd51e'
crc32c_64 = base64.b64encode(base64.b16decode(crc32c, casefold=True))
md5_sum = 'b026324c6904b2a9cb4b88d6d61c81d1'
md5_sum_64 = base64.b64encode(base64.b16decode(md5_sum, casefold=True))
output = '\n'.join([
'%s:' % gs_uri,
' Creation time: Tue, 04 Mar 2014 19:55:26 GMT',
' Content-Language: en',
' Content-Length: 2',
' Content-Type: application/octet-stream',
' Hash (crc32c): %s' % crc32c_64,
' Hash (md5): %s' % md5_sum_64,
' ETag: CMi938jU+bwCEAE=',
' Generation: 1393962926989000',
' Metageneration: 1',
' ACL: [',
' {',
' "entity": "project-owners-134157665460",',
' "projectTeam": {',
' "projectNumber": "134157665460",',
' "team": "owners"',
' },',
' "role": "OWNER"',
' }',
']',
])
# Set up the test replay script.
cmd = [self.gsutil, 'ls', '-L', gs_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
error_ok=True,
return_result=True).AndReturn(
cros_test_lib.EasyAttr(output=output))
self.mox.ReplayAll()
# Run the test verification.
result = gslib.MD5Sum(gs_uri)
self.assertEqual(md5_sum, result)
self.mox.VerifyAll()
def testCreateArchive(self):
"""Test that we can correctly archive up hash values for the signer."""
client = self.createStandardClient()
tmp_dir = None
hashes = ['Hash 1', 'Hash 2', 'Hash 3']
try:
with tempfile.NamedTemporaryFile() as archive_file:
client._CreateArchive(archive_file.name, hashes,
self.hash_names)
# Make sure the archive file created exists
self.assertTrue(os.path.exists(archive_file.name))
tmp_dir = tempfile.mkdtemp()
cmd = ['tar', '-xjf', archive_file.name]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
cwd=tmp_dir)
# Check that the expected (and only the expected) contents are present
extracted_file_names = os.listdir(tmp_dir)
self.assertEquals(len(extracted_file_names),
len(self.hash_names))
for name in self.hash_names:
self.assertTrue(name in extracted_file_names)
# Make sure each file has the expected contents
for h, hash_name in zip(hashes, self.hash_names):
with open(os.path.join(tmp_dir, hash_name), 'r') as f:
self.assertEqual([h], f.readlines())
finally:
# Clean up at the end of the test
if tmp_dir:
shutil.rmtree(tmp_dir)
def testFileTimestamp(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = '%s/%s' % (self.bucket_uri, 'some/file/path')
# Set up the test replay script.
cmd = [self.gsutil, 'ls', '-l', gs_uri]
output = '\n'.join([
' 96 2012-05-17T14:00:33 gs://bucket/chromeos.bin.md5',
'TOTAL: 1 objects, 96 bytes (96.0 B)',
])
cmd_result = cros_test_lib.EasyAttr(output=output)
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(cmd_result)
self.mox.ReplayAll()
# Run the test verification.
result = gslib.FileTimestamp(gs_uri)
self.assertEqual(datetime.datetime(2012, 5, 17, 14, 0, 33), result)
self.mox.VerifyAll()
def RunGsutilCommand(args,
redirect_stdout=True,
redirect_stderr=True,
failed_exception=GSLibError,
generation=None,
headers=None,
get_headers_from_stdout=False,
**kwargs):
"""Run gsutil with given args through RunCommand with given options.
Generally this method is intended for use within this module, see the various
command-specific wrappers provided for convenience. However, it can be called
directly if 'gsutil' needs to be called in specific way.
A few of the options for RunCommand have their default values switched for
this function. Those options are called out explicitly as options here, while
addition RunCommand options can be used through extra_run_command_opts.
Args:
args: List of arguments to use with 'gsutil'.
redirect_stdout: Boolean option passed directly to RunCommand.
redirect_stderr: Boolean option passed directly to RunCommand.
failed_exception: Exception class to raise if CommandFailedException is
caught. It should be GSLibError or a subclass.
generation: Only run the specified command if the generation matches.
(See "Conditional Updates Using Object Versioning" in the gsutil docs.)
headers: Fill in this dictionary with header values captured from stderr.
get_headers_from_stdout: Whether header information is to be parsed from
stdout (default: stderr).
kwargs: Additional options to pass directly to RunCommand, beyond the
explicit ones above. See RunCommand itself.
Returns:
Anything that RunCommand returns, which should be a CommandResult object.
Raises:
GsutilMissingError is the gsutil utility cannot be found.
GSLibError (or whatever is in failed_exception) if RunCommand failed (and
error_ok was not True).
"""
# The -d flag causes gsutil to dump various metadata, including user
# credentials. We therefore don't allow users to pass it in directly.
assert '-d' not in args, 'Cannot pass in the -d flag directly'
gsutil = FindGsUtil()
if generation is not None:
args = ['-h', 'x-goog-if-generation-match:%s' % generation] + args
if headers is not None:
args.insert(0, '-d')
assert redirect_stderr
cmd = [gsutil] + args
run_opts = {
'redirect_stdout': redirect_stdout,
'redirect_stderr': redirect_stderr,
}
run_opts.update(kwargs)
# Always use RunCommand with return_result on, which will be the default
# behavior for RunCommand itself someday.
run_opts['return_result'] = True
try:
result = utils.RunCommand(cmd, **run_opts)
except OSError as e:
if e.errno == errno.ENOENT:
raise GsutilMissingError()
raise
except utils.CommandFailedException as e:
# If headers is set, we have to hide the output here because it may contain
# credentials that we don't want to show in buildbot logs.
raise failed_exception('%r failed' % cmd if headers else e)
if headers is not None and result is not None:
assert redirect_stdout if get_headers_from_stdout else redirect_stderr
# Parse headers that look like this:
# header: x-goog-generation: 1359148994758000
# header: x-goog-metageneration: 1
headers_source = result.output if get_headers_from_stdout else result.error
for line in headers_source.splitlines():
if line.startswith('header: '):
header, _, value = line.partition(': ')[-1].partition(': ')
headers[header.replace('x-goog-', '')] = value
# Strip out stderr entirely to avoid showing credentials in logs; for
# commands that dump credentials to stdout, clobber that as well.
result.error = '<stripped>'
if get_headers_from_stdout:
result.output = '<stripped>'
return result
def testListFiles(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
files = [
'%s/some/path' % self.bucket_uri,
'%s/some/file/path' % self.bucket_uri,
]
directories = [
'%s/some/dir/' % self.bucket_uri,
'%s/some/dir/path/' % self.bucket_uri,
]
gs_uri = '%s/**' % self.bucket_uri
cmd = [self.gsutil, 'ls', gs_uri]
# Prepare cmd_result for a good run.
# Fake a trailing empty line.
output = '\n'.join(files + directories + [''])
cmd_result_ok = cros_test_lib.EasyAttr(output=output, returncode=0)
# Prepare exception for a run that finds nothing.
stderr = 'CommandException: One or more URLs matched no objects.\n'
empty_exception = utils.CommandFailedException(stderr)
# Prepare exception for a run that triggers a GS failure.
failure_exception = utils.CommandFailedException(GS_RETRY_FAILURE)
# Set up the test replay script.
# Run 1, runs ok.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(cmd_result_ok)
# Run 2, runs ok, sorts files.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndReturn(cmd_result_ok)
# Run 3, finds nothing.
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(empty_exception)
# Run 4, failure in GS.
for _ix in xrange(gslib.RETRY_ATTEMPTS + 1):
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
return_result=True).AndRaise(failure_exception)
self.mox.ReplayAll()
# Run the test verification.
result = gslib.ListFiles(self.bucket_uri, recurse=True)
self.assertEqual(files, result)
result = gslib.ListFiles(self.bucket_uri, recurse=True, sort=True)
self.assertEqual(sorted(files), result)
result = gslib.ListFiles(self.bucket_uri, recurse=True)
self.assertEqual([], result)
self.assertRaises(gslib.GSLibError,
gslib.ListFiles,
self.bucket_uri,
recurse=True)
self.mox.VerifyAll()
