def testCopyAndMD5Sum(self):
"""Higher-level functional test. Test MD5Sum OK."""
with gs.TemporaryURL('chromite.gslib.md5') as tempuri:
local_path = self.populateUri(tempuri)
local_md5 = gslib.filelib.MD5Sum(local_path)
gs_md5 = gslib.MD5Sum(tempuri)
self.assertEqual(gs_md5, local_md5)
def testMD5SumAccessError(self):
gs_uri = 'gs://bucket/foo/bar/somefile'
crc32c = 'c96fd51e'
crc32c_64 = base64.b64encode(base64.b16decode(crc32c, casefold=True))
md5_sum = 'b026324c6904b2a9cb4b88d6d61c81d1'
md5_sum_64 = base64.b64encode(base64.b16decode(md5_sum, casefold=True))
output = '\n'.join([
'%s:' % gs_uri,
' Creation time: Tue, 04 Mar 2014 19:55:26 GMT',
' Content-Language: en',
' Content-Length: 2',
' Content-Type: application/octet-stream',
' Hash (crc32c): %s' % crc32c_64,
' Hash (md5): %s' % md5_sum_64,
' ETag: CMi938jU+bwCEAE=',
' Generation: 1393962926989000',
' Metageneration: 1',
' ACL: ACCESS DENIED. Note: you need OWNER '
'permission',
' on the object to read its ACL.',
])
# Set up the test replay script.
cmd = [self.gsutil, 'ls', '-L', gs_uri]
cros_build_lib.RunCommand(
cmd, redirect_stdout=True, redirect_stderr=True,
error_code_ok=True).AndReturn(
cros_test_lib.EasyAttr(output=output))
self.mox.ReplayAll()
# Run the test verification.
result = gslib.MD5Sum(gs_uri)
self.assertEqual(md5_sum, result)
self.mox.VerifyAll()
def MD5Sum(uri):
"""Compute or retrieve MD5 sum of uri.
Supported for: local files, GS files.
Args:
uri: The /unix/path or gs:// uri to compute the md5sum on.
Returns:
A string representing the md5sum of the file/uri passed in.
None if we do not understand the uri passed in or cannot compute
the md5sum.
"""
uri_type = GetUriType(uri)
if uri_type == TYPE_LOCAL:
return filelib.MD5Sum(uri)
elif uri_type == TYPE_GS:
try:
return gslib.MD5Sum(uri)
except gslib.GSLibError:
return None
# Colossus does not have a command for getting MD5 sum. We could
# copy the file to local disk and calculate it, but it seems better
# to explicitly say it is not supported.
raise NotSupportedForType(uri_type)
def testMD5SumBadBucket(self):
"""Higher-level functional test. Test MD5Sum bad bucket:
1) Make up random, non-existent gs bucket and path
2) Ask for MD5Sum. Make sure it fails, with exception
"""
gs_path = 'gs://lokijuhygtfrdesxcv/awsedrftgyhujikol'
gs_md5 = gslib.MD5Sum(gs_path)
self.assertTrue(gs_md5 is None)
def testMD5SumBadPath(self):
"""Higher-level functional test. Test MD5Sum bad path:
1) Make up random, non-existent gs path
2) Ask for MD5Sum. Make sure it fails, but with no exeption.
"""
gs_path = 'gs://chromeos-releases/awsedrftgyhujikol'
gs_md5 = gslib.MD5Sum(gs_path)
self.assertTrue(gs_md5 is None)
def testMD5SumAccessOK(self):
self.mox.StubOutWithMock(utils, 'RunCommand')
gs_uri = 'gs://bucket/foo/bar/somefile'
crc32c = 'c96fd51e'
crc32c_64 = base64.b64encode(base64.b16decode(crc32c, casefold=True))
md5_sum = 'b026324c6904b2a9cb4b88d6d61c81d1'
md5_sum_64 = base64.b64encode(base64.b16decode(md5_sum, casefold=True))
output = '\n'.join([
'%s:' % gs_uri,
' Creation time: Tue, 04 Mar 2014 19:55:26 GMT',
' Content-Language: en',
' Content-Length: 2',
' Content-Type: application/octet-stream',
' Hash (crc32c): %s' % crc32c_64,
' Hash (md5): %s' % md5_sum_64,
' ETag: CMi938jU+bwCEAE=',
' Generation: 1393962926989000',
' Metageneration: 1',
' ACL: [',
' {',
' "entity": "project-owners-134157665460",',
' "projectTeam": {',
' "projectNumber": "134157665460",',
' "team": "owners"',
' },',
' "role": "OWNER"',
' }',
']',
])
# Set up the test replay script.
cmd = [self.gsutil, 'ls', '-L', gs_uri]
utils.RunCommand(cmd,
redirect_stdout=True,
redirect_stderr=True,
error_ok=True,
return_result=True).AndReturn(
cros_test_lib.EasyAttr(output=output))
self.mox.ReplayAll()
# Run the test verification.
result = gslib.MD5Sum(gs_uri)
self.assertEqual(md5_sum, result)
self.mox.VerifyAll()
def testMD5Sum(self):
gs_path = 'gs://bucket/some/path'
local_path = '/some/local/path'
http_path = 'http://host.domain/some/path'
self.mox.StubOutWithMock(gslib, 'MD5Sum')
self.mox.StubOutWithMock(filelib, 'MD5Sum')
# Set up the test replay script.
# Run 1, GS.
gslib.MD5Sum(gs_path).AndReturn('TheResult')
# Run 3, local file.
filelib.MD5Sum(local_path).AndReturn('TheResult')
self.mox.ReplayAll()
# Run the test verification.
self.assertEquals('TheResult', urilib.MD5Sum(gs_path))
self.assertEquals('TheResult', urilib.MD5Sum(local_path))
self.assertRaises(urilib.NotSupportedForType, urilib.MD5Sum, http_path)
self.mox.VerifyAll()
def testCmp(self):
uri1 = 'gs://some/gs/path'
uri2 = 'gs://some/other/path'
local_path = '/some/local/path'
md5 = 'TheMD5Sum'
self.mox.StubOutWithMock(gslib, 'MD5Sum')
self.mox.StubOutWithMock(gslib.filelib, 'MD5Sum')
# Set up the test replay script.
# Run 1, same md5, both GS.
gslib.MD5Sum(uri1).AndReturn(md5)
gslib.MD5Sum(uri2).AndReturn(md5)
# Run 2, different md5, both GS.
gslib.MD5Sum(uri1).AndReturn(md5)
gslib.MD5Sum(uri2).AndReturn('Other' + md5)
# Run 3, same md5, one GS on local.
gslib.MD5Sum(uri1).AndReturn(md5)
gslib.filelib.MD5Sum(local_path).AndReturn(md5)
# Run 4, different md5, one GS on local.
gslib.MD5Sum(uri1).AndReturn(md5)
gslib.filelib.MD5Sum(local_path).AndReturn('Other' + md5)
# Run 5, missing file, both GS.
gslib.MD5Sum(uri1).AndReturn(None)
# Run 6, args are None.
gslib.filelib.MD5Sum(None).AndReturn(None)
self.mox.ReplayAll()
# Run the test verification.
self.assertTrue(gslib.Cmp(uri1, uri2))
self.assertFalse(gslib.Cmp(uri1, uri2))
self.assertTrue(gslib.Cmp(uri1, local_path))
self.assertFalse(gslib.Cmp(uri1, local_path))
self.assertFalse(gslib.Cmp(uri1, uri2))
self.assertFalse(gslib.Cmp(None, None))
self.mox.VerifyAll()
