def test_repo_matches_manifest(self):
"""Ensure any new top-level files are present in the manifest."""
p = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.communicate()
if p.returncode != 0:
unittest.skip('Test only runs from git repository.')
manifest_lines = ['gslib', 'third_party', 'MANIFEST.in']
with open(os.path.join(GSUTIL_DIR, 'MANIFEST.in'), 'r') as fp:
for line in fp:
if line.startswith('include '):
manifest_lines.append(line.split()[-1])
p = subprocess.Popen(['git', 'ls-tree', '--name-only', 'HEAD'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, _) = p.communicate()
git_top_level_files = stdout.splitlines()
for filename in git_top_level_files:
if filename.endswith('.pyc'):
# Ignore compiled code.
continue
if filename in ('.gitmodules', '.gitignore', '.travis.yml'):
# We explicitly drop these files when building the gsutil tarball.
# If we add any other files to this list, the tarball script must
# also be updated or we could break the gsutil update command.
continue
if filename not in manifest_lines:
self.fail('Found file %s not present in MANIFEST.in, which would '
'break gsutil update.' % filename)
def test_rewrite_key_rotation_bucket_subdir(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
object_contents = 'bar'
rotate_subdir = suri(bucket_uri, 'bar')
object_uri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo/bar',
contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY1)
object_uri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='bar/foo',
contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY2)
object_uri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='bar/baz',
contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY3)
object_uri4 = self.CreateObject(bucket_uri=bucket_uri,
object_name='bar/qux',
contents=object_contents)
# Rotate subdir keys to TEST_ENCRYPTION_KEY3.
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)
]
self.AssertNObjectsInBucket(bucket_uri, 4)
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
return_stderr=True)
self.assertIn('Rotating', stderr) # Object 2.
self.assertIn('Skipping %s' % suri(object_uri3), stderr)
self.assertIn('Encrypting', stderr) # Object 4.
# First subdir should be unaffected.
self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)
for object_uri_str in (suri(object_uri2), suri(object_uri3),
suri(object_uri4)):
self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3)
# Remove encryption in subdir.
boto_config_for_test2 = [('GSUtil', 'decryption_key1',
TEST_ENCRYPTION_KEY3)]
with SetBotoConfigForTest(boto_config_for_test2):
stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
return_stderr=True)
self.assertIn('Decrypting', stderr)
# First subdir should be unaffected.
self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)
for object_uri_str in (suri(object_uri2), suri(object_uri3),
suri(object_uri4)):
self.AssertObjectUnencrypted(object_uri_str)
def test_rewrite_key_rotation_with_storage_class_change(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Rotate key and change storage class to nearline.
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)
]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-s', 'nearline', '-k',
suri(object_uri)],
return_stderr=True)
self.assertIn('Rotating', stderr)
self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)
stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
self.assertRegexpMatchesWithFlags(
stdout,
r'Storage class:\s+NEARLINE',
flags=re.IGNORECASE,
msg=('Storage class appears not to have been changed.'))
def test_rewrite_key_rotation_single_object(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Rotate key.
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)
]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Rotating', stderr)
self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)
# Remove encryption.
boto_config_for_test2 = [('GSUtil', 'decryption_key1',
TEST_ENCRYPTION_KEY2)]
with SetBotoConfigForTest(boto_config_for_test2):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Decrypting', stderr)
self.AssertObjectUnencrypted(suri(object_uri))
def test_stat_encrypted_object(self):
"""Tests stat command with an encrypted object."""
if self.test_api == ApiSelector.XML:
return unittest.skip(
'gsutil does not support encryption with the XML API')
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=TEST_ENCRYPTION_CONTENT1,
encryption_key=TEST_ENCRYPTION_KEY1)
# Stat object with key should return unencrypted hashes.
with SetBotoConfigForTest([('GSUtil', 'encryption_key',
TEST_ENCRYPTION_KEY1)]):
stdout = self.RunGsUtil(['stat', suri(object_uri)],
return_stdout=True)
self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
# Stat object without key should return encrypted hashes.
stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn('encrypted', stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def test_rewrite_bucket_recursive(self):
"""Tests rewrite command recursively on a bucket."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
self._test_rewrite_key_rotation_bucket(
bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)])
def test_stat_encrypted_object(self):
"""Tests stat command with an encrypted object."""
if self.test_api == ApiSelector.XML:
return unittest.skip("gsutil does not support encryption with the XML API")
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(
bucket_uri=bucket_uri,
object_name="foo",
contents=TEST_ENCRYPTION_CONTENT1,
encryption_key=TEST_ENCRYPTION_KEY1,
)
# Stat object with key should return unencrypted hashes.
with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]):
stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True)
self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
# Stat object without key should return encrypted hashes.
stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True)
self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn("encrypted", stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def test_rewrite_to_same_storage_class_is_skipped(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar')
stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)],
return_stderr=True)
self.assertIn('Skipping %s' % suri(object_uri), stderr)
def testSetContentTypeFromFile(self):
"""Tests that content type is correctly determined for symlinks."""
if IS_WINDOWS:
return unittest.skip('use_magicfile features not available on Windows')
surprise_html = '<html><body>And you thought I was just text!</body></html>'
temp_dir_path = self.CreateTempDir()
txt_file_path = self.CreateTempFile(
tmpdir=temp_dir_path, contents=surprise_html,
file_name='html_in_disguise.txt')
link_name = 'link_to_realfile' # Notice no file extension was supplied.
os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name)
# Content-type of a symlink should be obtained from the link's target.
dst_obj_metadata_mock = mock.MagicMock(contentType=None)
src_url_stub = mock.MagicMock(
object_name=temp_dir_path + os.path.sep + link_name,
**{'IsFileUrl.return_value': True,
'IsStream.return_value': False,
'IsFifo.return_value': False})
# The file command should detect HTML in the real file.
with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]):
_SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
self.assertEqual('text/html; charset=us-ascii', dst_obj_metadata_mock.contentType)
dst_obj_metadata_mock = mock.MagicMock(contentType=None)
# The mimetypes module should guess based on the real file's extension.
with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]):
_SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
self.assertEqual('text/plain', dst_obj_metadata_mock.contentType)
def test_parallel_rewrite_bucket_flat_wildcard(self):
"""Tests parallel rewrite command with a flat wildcard on a bucket."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
self._test_rewrite_key_rotation_bucket(
bucket_uri, ['-m', 'rewrite', '-k', suri(bucket_uri, '**')])
def test_rewrite_key_rotation_single_object(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Rotate key.
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
return_stderr=True)
self.assertIn('Rotating', stderr)
self.AssertObjectUsesEncryptionKey(suri(object_uri),
TEST_ENCRYPTION_KEY2)
# Remove encryption.
boto_config_for_test2 = [
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2)]
with SetBotoConfigForTest(boto_config_for_test2):
stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
return_stderr=True)
self.assertIn('Decrypting', stderr)
self.AssertObjectUnencrypted(suri(object_uri))
def test_compose_with_encryption(self):
"""Tests composing encrypted objects."""
if self.test_api == ApiSelector.XML:
return unittest.skip(
'gsutil does not support encryption with the XML API')
bucket_uri = self.CreateBucket()
object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo',
encryption_key=TEST_ENCRYPTION_KEY1)
object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Compose without correct key should fail.
stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
suri(bucket_uri, 'obj')], expected_status=1,
return_stderr=True)
self.assertIn('is encrypted by a customer-supplied encryption key', stderr)
# Compose with different encryption key should fail; source and destination
# encryption keys must match.
with SetBotoConfigForTest([
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]):
stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
suri(bucket_uri, 'obj')], expected_status=1,
return_stderr=True)
self.assertIn('provided encryption key is incorrect', stderr)
with SetBotoConfigForTest(
[('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]):
self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
suri(bucket_uri, 'obj')])
def test_rewrite_bucket_recursive(self):
"""Tests rewrite command recursively on a bucket."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
self._test_rewrite_key_rotation_bucket(
bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)])
def test_rewrite_to_same_storage_class_is_skipped(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar')
stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)],
return_stderr=True)
self.assertIn('Skipping %s' % suri(object_uri), stderr)
def test_compose_with_encryption(self):
"""Tests composing encrypted objects."""
if self.test_api == ApiSelector.XML:
return unittest.skip(
'gsutil does not support encryption with the XML API')
bucket_uri = self.CreateBucket()
object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo',
encryption_key=TEST_ENCRYPTION_KEY1)
object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Compose without correct key should fail.
stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
suri(bucket_uri, 'obj')], expected_status=1,
return_stderr=True)
self.assertIn('is encrypted by a customer-supplied encryption key', stderr)
# Compose with different encryption key should fail; source and destination
# encryption keys must match.
with SetBotoConfigForTest([
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]):
stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
suri(bucket_uri, 'obj')], expected_status=1,
return_stderr=True)
self.assertIn('is encrypted by a different customer-supplied key', stderr)
with SetBotoConfigForTest(
[('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]):
self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
suri(bucket_uri, 'obj')])
def test_parallel_rewrite_bucket_flat_wildcard(self):
"""Tests parallel rewrite command with a flat wildcard on a bucket."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
self._test_rewrite_key_rotation_bucket(
bucket_uri, ['-m', 'rewrite', '-k', suri(bucket_uri, '**')])
def testSetContentTypeFromFile(self):
"""Tests that content type is correctly determined for symlinks."""
if system_util.IS_WINDOWS:
return unittest.skip('use_magicfile features not available on Windows')
surprise_html = b'<html><body>And you thought I was just text!</body></html>'
temp_dir_path = self.CreateTempDir()
txt_file_path = self.CreateTempFile(tmpdir=temp_dir_path,
contents=surprise_html,
file_name='html_in_disguise.txt')
link_name = 'link_to_realfile' # Notice no file extension was supplied.
os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name)
# Content-type of a symlink should be obtained from the link's target.
dst_obj_metadata_mock = mock.MagicMock(contentType=None)
src_url_stub = mock.MagicMock(object_name=temp_dir_path + os.path.sep +
link_name,
**{
'IsFileUrl.return_value': True,
'IsStream.return_value': False,
'IsFifo.return_value': False
})
# The file command should detect HTML in the real file.
with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]):
_SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
self.assertEqual('text/html; charset=us-ascii',
dst_obj_metadata_mock.contentType)
dst_obj_metadata_mock = mock.MagicMock(contentType=None)
# The mimetypes module should guess based on the real file's extension.
with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]):
_SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
self.assertEqual('text/plain', dst_obj_metadata_mock.contentType)
def test_rewrite_unintentional_key_rotation_fails(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
encrypted_obj_uri = self.CreateObject(
contents='bar', encryption_key=TEST_ENCRYPTION_KEY1)
unencrypted_obj_uri = self.CreateObject(contents='bar')
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)
]
with SetBotoConfigForTest(boto_config_for_test):
# Executing rewrite without the -k flag should fail if your boto file has
# a different encryption_key than was last used to encrypt the object.
stderr = self.RunGsUtil(
['rewrite', '-s', 'dra',
suri(encrypted_obj_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('EncryptionException', stderr)
# Should also fail for a previously unencrypted object.
stderr = self.RunGsUtil(
['rewrite', '-s', 'dra',
suri(unencrypted_obj_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('EncryptionException', stderr)
def test_rewrite_with_no_encryption_key_operates_on_unencrypted_objects(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
# Since the introduction of default KMS keys for GCS buckets, rewriting
# with no explicitly specified CSEK/CMEK can still result in the rewritten
# objects being encrypted. Before KMS support, this would always result in
# decrypted objects. With this new possibility, we want to always rewrite
# every specified object when no encryption_key was set in the boto config,
# since we don't know if the operation will end up decrypting the object or
# implicitly encrypting it with the bucket's default KMS key.
key_fqn = self.authorize_project_to_use_testing_kms_key()
# Create an unencrypted object.
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(
bucket_uri=bucket_uri, object_name='foo', contents='foo')
# Set the bucket's default KMS key.
self.RunGsUtil(['kms', 'encryption', '-k', key_fqn, suri(bucket_uri)])
# Rewriting with no encryption_key should rewrite the object, resulting in
# the bucket's default KMS key being used to encrypt it.
with SetBotoConfigForTest([('GSUtil', 'encryption_key', None)]):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Rewriting', stderr)
self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)
def test_rewrite_key_rotation_bucket_subdir(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
object_contents = 'bar'
rotate_subdir = suri(bucket_uri, 'bar')
object_uri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo/bar',
contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY1)
object_uri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='bar/foo',
contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY2)
object_uri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='bar/baz',
contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY3)
object_uri4 = self.CreateObject(bucket_uri=bucket_uri,
object_name='bar/qux',
contents=object_contents)
# Rotate subdir keys to TEST_ENCRYPTION_KEY3.
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)]
self.AssertNObjectsInBucket(bucket_uri, 4)
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
return_stderr=True)
self.assertIn('Rotating', stderr) # Object 2.
self.assertIn('Skipping %s' % suri(object_uri3), stderr)
self.assertIn('Encrypting', stderr) # Object 4.
# First subdir should be unaffected.
self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)
for object_uri_str in (suri(object_uri2), suri(object_uri3),
suri(object_uri4)):
self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3)
# Remove encryption in subdir.
boto_config_for_test2 = [
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]
with SetBotoConfigForTest(boto_config_for_test2):
stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
return_stderr=True)
self.assertIn('Decrypting', stderr)
# First subdir should be unaffected.
self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)
for object_uri_str in (suri(object_uri2), suri(object_uri3),
suri(object_uri4)):
self.AssertObjectUnencrypted(object_uri_str)
def test_turning_on(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('XML API has no concept of Bucket Policy Only')
bucket_uri = self.CreateBucket()
self.RunGsUtil(self._set_bpo_cmd + ['on', suri(bucket_uri)])
self._AssertEnabled(bucket_uri, True)
def test_rewrite_with_no_value_for_minus_s(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'],
return_stderr=True, expected_status=1)
self.assertIn('CommandException', stderr)
self.assertIn('expects at least one URL', stderr)
def test_rewrite_with_no_value_for_minus_s(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'],
return_stderr=True, expected_status=1)
self.assertIn('CommandException', stderr)
self.assertIn('expects at least one URL', stderr)
def testRetryableErrorMediaCollection(self):
"""Tests that retryable errors are collected on JSON media operations."""
# Retryable errors will only be collected with the JSON API.
if self.test_api != ApiSelector.JSON:
return unittest.skip('Retryable errors are only collected in JSON')
boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB))]
bucket_uri = self.CreateBucket()
# For the resumable upload exception, we need to ensure at least one
# callback occurs.
halt_size = START_CALLBACK_PER_BYTES * 2
fpath = self.CreateTempFile(contents='a' * halt_size)
# Test that the retry function for data transfers catches and logs an error.
test_callback_file = self.CreateTempFile(contents=pickle.dumps(
_ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError,
('unused', 'unused', 'unused'))))
with SetBotoConfigForTest(boto_config_for_test):
metrics_list = self._RunGsUtilWithAnalyticsOutput(
['cp', '--testcallbackfile', test_callback_file,
fpath, suri(bucket_uri)])
self._CheckParameterValue('Event Category',
metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
self._CheckParameterValue('Event Action', 'BadStatusCodeError',
metrics_list)
self._CheckParameterValue('Retryable Errors', '1', metrics_list)
self._CheckParameterValue('Num Retryable Service Errors', '1',
metrics_list)
# Test that the ResumableUploadStartOverException in copy_helper is caught.
test_callback_file = self.CreateTempFile(
contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
with SetBotoConfigForTest(boto_config_for_test):
metrics_list = self._RunGsUtilWithAnalyticsOutput(
['cp', '--testcallbackfile', test_callback_file,
fpath, suri(bucket_uri)])
self._CheckParameterValue(
'Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
self._CheckParameterValue(
'Event Action', 'ResumableUploadStartOverException', metrics_list)
self._CheckParameterValue('Retryable Errors', '1', metrics_list)
self._CheckParameterValue(
'Num Retryable Service Errors', '1', metrics_list)
# Test retryable error collection in a multithread/multiprocess situation.
test_callback_file = self.CreateTempFile(
contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
with SetBotoConfigForTest(boto_config_for_test):
metrics_list = self._RunGsUtilWithAnalyticsOutput(
['-m', 'cp', '--testcallbackfile',
test_callback_file, fpath, suri(bucket_uri)])
self._CheckParameterValue('Event Category',
metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
self._CheckParameterValue(
'Event Action', 'ResumableUploadStartOverException', metrics_list)
self._CheckParameterValue('Retryable Errors', '1', metrics_list)
self._CheckParameterValue(
'Num Retryable Service Errors', '1', metrics_list)
def testRetryableErrorMediaCollection(self):
"""Tests that retryable errors are collected on JSON media operations."""
# Retryable errors will only be collected with the JSON API.
if self.test_api != ApiSelector.JSON:
return unittest.skip('Retryable errors are only collected in JSON')
boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB))]
bucket_uri = self.CreateBucket()
# For the resumable upload exception, we need to ensure at least one
# callback occurs.
halt_size = START_CALLBACK_PER_BYTES * 2
fpath = self.CreateTempFile(contents='a' * halt_size)
# Test that the retry function for data transfers catches and logs an error.
test_callback_file = self.CreateTempFile(contents=pickle.dumps(
_ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError,
('unused', 'unused', 'unused'))))
with SetBotoConfigForTest(boto_config_for_test):
metrics_list = self._RunGsUtilWithAnalyticsOutput(
['cp', '--testcallbackfile', test_callback_file,
fpath, suri(bucket_uri)])
self._CheckParameterValue('Event Category',
metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
self._CheckParameterValue('Event Action', 'BadStatusCodeError',
metrics_list)
self._CheckParameterValue('Retryable Errors', '1', metrics_list)
self._CheckParameterValue('Num Retryable Service Errors', '1',
metrics_list)
# Test that the ResumableUploadStartOverException in copy_helper is caught.
test_callback_file = self.CreateTempFile(
contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
with SetBotoConfigForTest(boto_config_for_test):
metrics_list = self._RunGsUtilWithAnalyticsOutput(
['cp', '--testcallbackfile', test_callback_file,
fpath, suri(bucket_uri)])
self._CheckParameterValue(
'Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
self._CheckParameterValue(
'Event Action', 'ResumableUploadStartOverException', metrics_list)
self._CheckParameterValue('Retryable Errors', '1', metrics_list)
self._CheckParameterValue(
'Num Retryable Service Errors', '1', metrics_list)
# Test retryable error collection in a multithread/multiprocess situation.
test_callback_file = self.CreateTempFile(
contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
with SetBotoConfigForTest(boto_config_for_test):
metrics_list = self._RunGsUtilWithAnalyticsOutput(
['-m', 'cp', '--testcallbackfile',
test_callback_file, fpath, suri(bucket_uri)])
self._CheckParameterValue('Event Category',
metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
self._CheckParameterValue(
'Event Action', 'ResumableUploadStartOverException', metrics_list)
self._CheckParameterValue('Retryable Errors', '1', metrics_list)
self._CheckParameterValue(
'Num Retryable Service Errors', '1', metrics_list)
def test_turning_off_on_enabled_buckets(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('XML API has no concept of Bucket Policy Only')
bucket_uri = self.CreateBucket(bucket_policy_only=True,
prefer_json_api=True)
self._AssertEnabled(bucket_uri, True)
self.RunGsUtil(self._set_bpo_cmd + ['off', suri(bucket_uri)])
self._AssertEnabled(bucket_uri, False)
def test_turning_on(self):
if self.test_api == ApiSelector.XML:
return unittest.skip(
'XML API has no concept of Uniform bucket-level access')
bucket_uri = self.CreateBucket()
self.RunGsUtil(self._set_ubla_cmd + ['on', suri(bucket_uri)])
self._AssertEnabled(bucket_uri, True)
def test_set_valid_acl_bucket(self):
"""Ensures that valid canned and XML ACLs work with get/set."""
if self._ServiceAccountCredentialsPresent():
# See comments in _ServiceAccountCredentialsPresent
unittest.skip('Canned ACLs orphan service account permissions.')
bucket_uri = suri(self.CreateBucket())
acl_string = self.RunGsUtil(self._get_acl_prefix + [bucket_uri],
return_stdout=True)
inpath = self.CreateTempFile(contents=acl_string)
self.RunGsUtil(self._set_acl_prefix + ['public-read', bucket_uri])
acl_string2 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri],
return_stdout=True)
self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri])
acl_string3 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri],
return_stdout=True)
self.assertNotEqual(acl_string, acl_string2)
self.assertEqual(acl_string, acl_string3)
def test_set_lifecycle_wildcard(self):
"""Tests setting lifecycle with a wildcarded bucket URI."""
if self.test_api == ApiSelector.XML:
# This test lists buckets with wildcards, but it is possible that another
# test being run in parallel (in the same project) deletes a bucket after
# it is listed in this test. This causes the subsequent XML metadata get
# for the lifecycle configuration to fail on that just-deleted bucket,
# even though that bucket is not used directly in this test.
return unittest.skip(
'XML wildcard behavior can cause test to flake '
'if a bucket in the same project is deleted '
'during execution.')
random_prefix = self.MakeRandomTestString()
bucket1_name = self.MakeTempName('bucket', prefix=random_prefix)
bucket2_name = self.MakeTempName('bucket', prefix=random_prefix)
bucket1_uri = self.CreateBucket(bucket_name=bucket1_name)
bucket2_uri = self.CreateBucket(bucket_name=bucket2_name)
# This just double checks that the common prefix of the two buckets is what
# we think it should be (based on implementation detail of CreateBucket).
# We want to be careful when setting a wildcard on buckets to make sure we
# don't step outside the test buckets to affect other buckets.
common_prefix = posixpath.commonprefix(
[suri(bucket1_uri), suri(bucket2_uri)])
self.assertTrue(
common_prefix.startswith(
'gs://%sgsutil-test-test-set-lifecycle-wildcard-' %
random_prefix))
wildcard = '%s*' % common_prefix
fpath = self.CreateTempFile(
contents=self.lifecycle_doc.encode('ascii'))
# Use @Retry as hedge against bucket listing eventual consistency.
expected = set([
'Setting lifecycle configuration on %s/...' % suri(bucket1_uri),
'Setting lifecycle configuration on %s/...' % suri(bucket2_uri)
])
actual = set()
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard],
return_stderr=True)
actual.update(stderr.splitlines())
self.assertEqual(expected, actual)
self.assertEqual(stderr.count('Setting lifecycle configuration'),
2)
_Check1()
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True)
self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True)
self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
def test_uppercase_header(self):
"""Tests setting custom metadata with an uppercase value."""
if self.test_api == ApiSelector.XML:
return unittest.skip('XML header keys are case-insensitive.')
objuri = self.CreateObject(contents='foo')
self.RunGsUtil([
'setmeta', '-h', 'x-%s-meta-CaSe:SeNsItIvE' % self.provider_custom_meta,
suri(objuri)])
stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True)
self.assertRegexpMatches(stdout, ur'CaSe:\s+SeNsItIvE')
def test_repo_matches_manifest(self):
"""Ensure any new top-level files are present in the manifest."""
p = subprocess.Popen(['git', 'branch'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.communicate()
if p.returncode != 0:
unittest.skip('Test only runs from git repository.')
return
manifest_lines = ['gslib', 'third_party', 'MANIFEST.in']
manifest_file = os.path.join(GSUTIL_DIR, 'MANIFEST.in')
try:
with open(manifest_file, 'r') as fp:
for line in fp:
if line.startswith('include '):
manifest_lines.append(line.split()[-1])
except IOError:
# If manifest file is not readable (example: Travis CI), skip the test.
unittest.skip('Test must be able to read manifest file.')
return
p = subprocess.Popen(['git', 'ls-tree', '--name-only', 'HEAD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, _) = p.communicate()
git_top_level_files = stdout.splitlines()
for filename in git_top_level_files:
if filename.endswith('.pyc'):
# Ignore compiled code.
continue
if filename in ('.gitmodules', '.gitignore', '.travis.yml'):
# We explicitly drop these files when building the gsutil tarball.
# If we add any other files to this list, the tarball script must
# also be updated or we could break the gsutil update command.
continue
if filename not in manifest_lines:
self.fail(
'Found file %s not present in MANIFEST.in, which would '
'break gsutil update.' % filename)
def test_uppercase_header(self):
"""Tests setting custom metadata with an uppercase value."""
if self.test_api == ApiSelector.XML:
return unittest.skip('XML header keys are case-insensitive.')
objuri = self.CreateObject(contents='foo')
self.RunGsUtil([
'setmeta', '-h', 'x-%s-meta-CaSe:SeNsItIvE' % self.provider_custom_meta,
suri(objuri)])
stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True)
self.assertRegexpMatches(stdout, ur'CaSe:\s+SeNsItIvE')
def test_rewrite_generation_url(self):
"""Tests that rewrite fails on a URL that includes a generation."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
generation = object_uri.generation
stderr = self.RunGsUtil(
['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)],
return_stderr=True, expected_status=1)
self.assertIn('"rewrite" called on URL with generation', stderr)
def test_rewrite_generation_url(self):
"""Tests that rewrite fails on a URL that includes a generation."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
generation = object_uri.generation
stderr = self.RunGsUtil(
['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)],
return_stderr=True, expected_status=1)
self.assertIn('"rewrite" called on URL with generation', stderr)
def test_turning_off_on_enabled_buckets(self):
if self.test_api == ApiSelector.XML:
return unittest.skip(
'XML API has no concept of Uniform bucket-level access')
# TODO(mynameisrafe): Replace bucket_policy_only with uniform_bucket_level_access when the property is live.
bucket_uri = self.CreateBucket(bucket_policy_only=True,
prefer_json_api=True)
self._AssertEnabled(bucket_uri, True)
self.RunGsUtil(self._set_ubla_cmd + ['off', suri(bucket_uri)])
self._AssertEnabled(bucket_uri, False)
def testRetryableErrorMetadataCollection(self):
"""Tests that retryable errors are collected on JSON metadata operations."""
# Retryable errors will only be collected with the JSON API.
if self.test_api != ApiSelector.JSON:
return unittest.skip('Retryable errors are only collected in JSON')
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo', contents=b'bar')
# Set the command name to rsync in order to collect PerformanceSummary info.
self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync'
# Generate a JSON API instance to test with, because the RunGsUtil method
# may use the XML API.
gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
RetryableErrorsQueue(), self.default_provider)
# Don't wait for too many retries or for long periods between retries to
# avoid long tests.
gsutil_api.api_client.num_retries = 2
gsutil_api.api_client.max_retry_wait = 1
# Throw an error when transferring metadata.
key = object_uri.get_key()
src_obj_metadata = apitools_messages.Object(name=key.name,
bucket=key.bucket.name,
contentType=key.content_type)
dst_obj_metadata = apitools_messages.Object(
bucket=src_obj_metadata.bucket,
name=self.MakeTempName('object'),
contentType=src_obj_metadata.contentType)
with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
side_effect=socket.error()):
_TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata,
dst_obj_metadata)
if six.PY2:
self.assertEqual(self.collector.retryable_errors['SocketError'], 1)
else:
# In PY3, socket.* errors are deprecated aliases for OSError
self.assertEqual(self.collector.retryable_errors['OSError'], 1)
# Throw an error when removing a bucket.
with mock.patch.object(
http_wrapper, '_MakeRequestNoRetry',
side_effect=apitools_exceptions.HttpError('unused', 'unused',
'unused')):
_TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name,
object_uri.object_name)
self.assertEqual(self.collector.retryable_errors['HttpError'], 1)
# Check that the number of each kind of retryable error was logged.
self.assertEqual(
self.collector.perf_sum_params.num_retryable_network_errors, 1)
self.assertEqual(
self.collector.perf_sum_params.num_retryable_service_errors, 1)
def test_set_lifecycle_wildcard(self):
"""Tests setting lifecycle with a wildcarded bucket URI."""
if self.test_api == ApiSelector.XML:
# This test lists buckets with wildcards, but it is possible that another
# test being run in parallel (in the same project) deletes a bucket after
# it is listed in this test. This causes the subsequent XML metadata get
# for the lifecycle configuration to fail on that just-deleted bucket,
# even though that bucket is not used directly in this test.
return unittest.skip('XML wildcard behavior can cause test to flake '
'if a bucket in the same project is deleted '
'during execution.')
random_prefix = self.MakeRandomTestString()
bucket1_name = self.MakeTempName('bucket', prefix=random_prefix)
bucket2_name = self.MakeTempName('bucket', prefix=random_prefix)
bucket1_uri = self.CreateBucket(bucket_name=bucket1_name)
bucket2_uri = self.CreateBucket(bucket_name=bucket2_name)
# This just double checks that the common prefix of the two buckets is what
# we think it should be (based on implementation detail of CreateBucket).
# We want to be careful when setting a wildcard on buckets to make sure we
# don't step outside the test buckets to affect other buckets.
common_prefix = posixpath.commonprefix(
[suri(bucket1_uri), suri(bucket2_uri)])
self.assertTrue(
common_prefix.startswith(
'gs://%sgsutil-test-test-set-lifecycle-wildcard-' % random_prefix))
wildcard = '%s*' % common_prefix
fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii'))
# Use @Retry as hedge against bucket listing eventual consistency.
expected = set([
'Setting lifecycle configuration on %s/...' % suri(bucket1_uri),
'Setting lifecycle configuration on %s/...' % suri(bucket2_uri)
])
actual = set()
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard],
return_stderr=True)
actual.update(stderr.splitlines())
self.assertEqual(expected, actual)
self.assertEqual(stderr.count('Setting lifecycle configuration'), 2)
_Check1()
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True)
self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True)
self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
def test_rewrite_with_same_key_and_storage_class_is_skipped(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='foo',
encryption_key=TEST_ENCRYPTION_KEY1,
storage_class='standard')
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', '-s', 'standard', suri(object_uri)],
return_stderr=True)
self.assertIn('Skipping %s' % suri(object_uri), stderr)
def test_mv_early_deletion_warning(self):
"""Tests that mv on a recent nearline object warns about early deletion."""
if self.test_api == ApiSelector.XML:
return unittest.skip('boto does not return object storage class')
bucket_uri = self.CreateBucket(storage_class='NEARLINE')
object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='obj')
stderr = self.RunGsUtil(['mv', suri(object_uri), suri(bucket_uri, 'foo')],
return_stderr=True)
self.assertIn(
'Warning: moving nearline object %s may incur an early deletion '
'charge, because the original object is less than 30 days old '
'according to the local system time.' % suri(object_uri), stderr)
def test_mv_early_deletion_warning(self):
"""Tests that mv on a recent nearline object warns about early deletion."""
if self.test_api == ApiSelector.XML:
return unittest.skip('boto does not return object storage class')
bucket_uri = self.CreateBucket(storage_class='NEARLINE')
object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='obj')
stderr = self.RunGsUtil(['mv', suri(object_uri), suri(bucket_uri, 'foo')],
return_stderr=True)
self.assertIn(
'Warning: moving nearline object %s may incur an early deletion '
'charge, because the original object is less than 30 days old '
'according to the local system time.' % suri(object_uri), stderr)
def test_rewrite_with_same_key_and_storage_class_is_skipped(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='foo',
encryption_key=TEST_ENCRYPTION_KEY1,
storage_class='standard')
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', '-s', 'standard', suri(object_uri)],
return_stderr=True)
self.assertIn('Skipping %s' % suri(object_uri), stderr)
def test_rewrite_with_only_storage_class_change(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar')
# Change storage class to nearline.
stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)],
return_stderr=True)
self.assertIn('Rewriting', stderr)
stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
self.assertRegexpMatchesWithFlags(
stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,
msg=('Storage class appears not to have been changed.'))
def test_rewrite_stdin_args(self):
"""Tests rewrite with arguments supplied on stdin."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
stdin_arg = suri(object_uri)
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg)
self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2)
def test_rewrite_missing_decryption_key(self):
"""Tests that rewrite fails when no decryption key matches."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(object_name='foo', contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
return_stderr=True, expected_status=1)
self.assertIn('No decryption key matches object %s' %
suri(object_uri), stderr)
def test_rewrite_with_only_storage_class_change(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar')
# Change storage class to nearline.
stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)],
return_stderr=True)
self.assertIn('Rewriting', stderr)
stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
self.assertRegexpMatchesWithFlags(
stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,
msg=('Storage class appears not to have been changed.'))
def test_rewrite_missing_decryption_key(self):
"""Tests that rewrite fails when no decryption key matches."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(object_name='foo', contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
return_stderr=True, expected_status=1)
self.assertIn('No decryption key matches object %s' %
suri(object_uri), stderr)
def test_rewrite_stdin_args(self):
"""Tests rewrite with arguments supplied on stdin."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
stdin_arg = suri(object_uri)
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg)
self.AssertObjectUsesEncryptionKey(stdin_arg, TEST_ENCRYPTION_KEY2)
def testRetryableErrorMetadataCollection(self):
"""Tests that retryable errors are collected on JSON metadata operations."""
# Retryable errors will only be collected with the JSON API.
if self.test_api != ApiSelector.JSON:
return unittest.skip('Retryable errors are only collected in JSON')
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo', contents='bar')
# Set the command name to rsync in order to collect PerformanceSummary info.
self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync'
# Generate a JSON API instance to test with, because the RunGsUtil method
# may use the XML API.
gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
RetryableErrorsQueue(), self.default_provider)
# Don't wait for too many retries or for long periods between retries to
# avoid long tests.
gsutil_api.api_client.num_retries = 2
gsutil_api.api_client.max_retry_wait = 1
# Throw an error when transferring metadata.
key = object_uri.get_key()
src_obj_metadata = apitools_messages.Object(name=key.name,
bucket=key.bucket.name,
contentType=key.content_type)
dst_obj_metadata = apitools_messages.Object(
bucket=src_obj_metadata.bucket,
name=self.MakeTempName('object'),
contentType=src_obj_metadata.contentType)
with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
side_effect=socket.error()):
_TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata,
dst_obj_metadata)
self.assertEqual(self.collector.retryable_errors['SocketError'], 1)
# Throw an error when removing a bucket.
with mock.patch.object(
http_wrapper, '_MakeRequestNoRetry',
side_effect=apitools_exceptions.HttpError('unused', 'unused',
'unused')):
_TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name,
object_uri.object_name)
self.assertEqual(self.collector.retryable_errors['HttpError'], 1)
# Check that the number of each kind of retryable error was logged.
self.assertEqual(
self.collector.perf_sum_params.num_retryable_network_errors, 1)
self.assertEqual(
self.collector.perf_sum_params.num_retryable_service_errors, 1)
def test_rewrite_seek_ahead(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Remove encryption
boto_config_for_test = [
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1),
('GSUtil', 'task_estimation_threshold', '1'),
('GSUtil', 'task_estimation_force', 'True')]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['-m', 'rewrite', '-k', suri(object_uri)],
return_stderr=True)
self.assertIn(
'Estimated work for this command: objects: 1, total size: 3', stderr)
def test_rewrite_seek_ahead(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Remove encryption
boto_config_for_test = [
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1),
('GSUtil', 'task_estimation_threshold', '1'),
('GSUtil', 'task_estimation_force', 'True')]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(['-m', 'rewrite', '-k', suri(object_uri)],
return_stderr=True)
self.assertIn(
'Estimated work for this command: objects: 1, total size: 3', stderr)
def test_stat_encrypted_object_wildcard(self):
"""Tests stat command with a mix of encrypted and unencrypted objects."""
if self.test_api == ApiSelector.XML:
return unittest.skip(
'gsutil does not support encryption with the XML API')
bucket_uri = self.CreateBucket()
object1_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo1',
contents=TEST_ENCRYPTION_CONTENT1,
encryption_key=TEST_ENCRYPTION_KEY1)
object2_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo2',
contents=TEST_ENCRYPTION_CONTENT2,
encryption_key=TEST_ENCRYPTION_KEY2)
object3_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo3',
contents=TEST_ENCRYPTION_CONTENT3)
stat_string = suri(object1_uri)[:-2] + '*'
# Stat 3 objects, two encrypted each with a different key, and one
# unencrypted. Should result in two unencrypted listing and one encrypted
# listing.
with SetBotoConfigForTest([('GSUtil', 'encryption_key',
TEST_ENCRYPTION_KEY1)]):
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _StatExpectMixed():
"""Runs stat and validates output."""
stdout, stderr = self.RunGsUtil(['stat', stat_string],
return_stdout=True,
return_stderr=True)
self.assertIn(suri(object1_uri), stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'),
stdout)
self.assertIn(suri(object2_uri), stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout)
self.assertIn('encrypted', stdout)
self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64.decode('ascii'),
stdout)
self.assertIn(suri(object3_uri), stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout)
_StatExpectMixed()
def test_rewrite_with_nonkey_transform_works_when_key_is_unchanged(self):
# Tests that when a valid transformation flag aside from "-k" is supplied,
# the "-k" flag is not supplied, and the encryption key previously used to
# encrypt the target object matches the encryption_key in the user's boto
# config file (via hash comparison), that the rewrite command properly
# passes the same tuple for decryption and encryption, in addition to
# performing the other desired transformations.
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True)
self.assertIn('Rewriting', stderr)
def test_stat_encrypted_object_wildcard(self):
"""Tests stat command with a mix of encrypted and unencrypted objects."""
if self.test_api == ApiSelector.XML:
return unittest.skip("gsutil does not support encryption with the XML API")
bucket_uri = self.CreateBucket()
object1_uri = self.CreateObject(
bucket_uri=bucket_uri,
object_name="foo1",
contents=TEST_ENCRYPTION_CONTENT1,
encryption_key=TEST_ENCRYPTION_KEY1,
)
object2_uri = self.CreateObject(
bucket_uri=bucket_uri,
object_name="foo2",
contents=TEST_ENCRYPTION_CONTENT2,
encryption_key=TEST_ENCRYPTION_KEY2,
)
object3_uri = self.CreateObject(bucket_uri=bucket_uri, object_name="foo3", contents=TEST_ENCRYPTION_CONTENT3)
stat_string = suri(object1_uri)[:-2] + "*"
# Stat 3 objects, two encrypted each with a different key, and one
# unencrypted. Should result in two unencrypted listing and one encrypted
# listing.
with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]):
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _StatExpectMixed():
"""Runs stat and validates output."""
stdout = self.RunGsUtil(["stat", stat_string], return_stdout=True)
self.assertIn(suri(object1_uri), stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
self.assertIn(suri(object2_uri), stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout)
self.assertIn("encrypted", stdout)
self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64, stdout)
self.assertIn(suri(object3_uri), stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout)
_StatExpectMixed()
def test_cat_encrypted_object(self):
if self.test_api == ApiSelector.XML:
return unittest.skip(
'gsutil does not support encryption with the XML API')
object_contents = '0123456789'
object_uri = self.CreateObject(object_name='foo', contents=object_contents,
encryption_key=TEST_ENCRYPTION_KEY1)
stderr = self.RunGsUtil(['cat', suri(object_uri)], expected_status=1,
return_stderr=True)
self.assertIn('No decryption key matches object', stderr)
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stdout = self.RunGsUtil(['cat', suri(object_uri)], return_stdout=True)
self.assertEqual(stdout, object_contents)
stdout = self.RunGsUtil(['cat', '-r 1-3', suri(object_uri)],
return_stdout=True)
self.assertEqual(stdout, '123')
def test_rewrite_to_kms_then_unencrypted(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
key_fqn = self.authorize_project_to_use_testing_kms_key()
object_uri = self.CreateObject(contents='foo')
boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Encrypting', stderr)
self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)
# Rewrite back to unencrypted and make sure no KMS key was used.
boto_config_for_test = [('GSUtil', 'encryption_key', None)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Decrypting', stderr)
self.AssertObjectUnencrypted(suri(object_uri))
def test_rewrite_to_kms_then_csek(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
key_fqn = self.authorize_project_to_use_testing_kms_key()
object_uri = self.CreateObject(contents='foo')
boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Encrypting', stderr)
self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)
# Rewrite from CMEK to CSEK encryption.
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-k', suri(object_uri)], return_stderr=True)
self.assertIn('Rotating', stderr)
self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1)
def test_rewrite_overwrite_acl(self):
"""Tests rewrite with the -O flag."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)])
stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)],
return_stdout=True)
self.assertIn('allUsers', stdout)
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
self.RunGsUtil(['rewrite', '-k', '-O', suri(object_uri)])
self.AssertObjectUsesEncryptionKey(suri(object_uri), TEST_ENCRYPTION_KEY2)
stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)],
return_stdout=True)
self.assertNotIn('allUsers', stdout)
def test_rewrite_key_rotation_with_storage_class_change(self):
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
object_uri = self.CreateObject(contents='bar',
encryption_key=TEST_ENCRYPTION_KEY1)
# Rotate key and change storage class to nearline.
boto_config_for_test = [
('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
with SetBotoConfigForTest(boto_config_for_test):
stderr = self.RunGsUtil(
['rewrite', '-s', 'nearline', '-k', suri(object_uri)],
return_stderr=True)
self.assertIn('Rotating', stderr)
self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)
stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
self.assertRegexpMatchesWithFlags(
stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,
msg=('Storage class appears not to have been changed.'))
def testRetryableErrorMetadataCollection(self):
"""Tests that retryable errors are collected on JSON metadata operations."""
# Retryable errors will only be collected with the JSON API.
if self.test_api != ApiSelector.JSON:
return unittest.skip('Retryable errors are only collected in JSON')
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents='bar')
# Generate a JSON API instance because the RunGsUtil method uses the XML
# API.
gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
RetryableErrorsQueue(), self.default_provider)
# Don't wait for too many retries or for long periods between retries to
# avoid long tests.
gsutil_api.api_client.num_retries = 2
gsutil_api.api_client.max_retry_wait = 1
# Throw an error when transferring metadata.
key = object_uri.get_key()
src_obj_metadata = apitools_messages.Object(name=key.name,
bucket=key.bucket.name,
contentType=key.content_type)
dst_obj_metadata = apitools_messages.Object(
bucket=src_obj_metadata.bucket,
name=self.MakeTempName('object'),
contentType=src_obj_metadata.contentType)
with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
side_effect=socket.error()):
_TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata,
dst_obj_metadata)
self.assertEqual(self.collector.retryable_errors['SocketError'], 1)
# Throw an error when removing a bucket.
with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
side_effect=ValueError()):
_TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name,
object_uri.object_name)
self.assertEqual(self.collector.retryable_errors['ValueError'], 1)