def test_repo_matches_manifest(self): """Ensure any new top-level files are present in the manifest.""" p = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() if p.returncode != 0: unittest.skip('Test only runs from git repository.') manifest_lines = ['gslib', 'third_party', 'MANIFEST.in'] with open(os.path.join(GSUTIL_DIR, 'MANIFEST.in'), 'r') as fp: for line in fp: if line.startswith('include '): manifest_lines.append(line.split()[-1]) p = subprocess.Popen(['git', 'ls-tree', '--name-only', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, _) = p.communicate() git_top_level_files = stdout.splitlines() for filename in git_top_level_files: if filename.endswith('.pyc'): # Ignore compiled code. continue if filename in ('.gitmodules', '.gitignore', '.travis.yml'): # We explicitly drop these files when building the gsutil tarball. # If we add any other files to this list, the tarball script must # also be updated or we could break the gsutil update command. continue if filename not in manifest_lines: self.fail('Found file %s not present in MANIFEST.in, which would ' 'break gsutil update.' % filename)
def test_rewrite_key_rotation_bucket_subdir(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() object_contents = 'bar' rotate_subdir = suri(bucket_uri, 'bar') object_uri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/bar', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY2) object_uri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/baz', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY3) object_uri4 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/qux', contents=object_contents) # Rotate subdir keys to TEST_ENCRYPTION_KEY3. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1) ] self.AssertNObjectsInBucket(bucket_uri, 4) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir], return_stderr=True) self.assertIn('Rotating', stderr) # Object 2. self.assertIn('Skipping %s' % suri(object_uri3), stderr) self.assertIn('Encrypting', stderr) # Object 4. # First subdir should be unaffected. self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1) for object_uri_str in (suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3) # Remove encryption in subdir. boto_config_for_test2 = [('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir], return_stderr=True) self.assertIn('Decrypting', stderr) # First subdir should be unaffected. self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1) for object_uri_str in (suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUnencrypted(object_uri_str)
def test_rewrite_key_rotation_with_storage_class_change(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Rotate key and change storage class to nearline. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1) ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-s', 'nearline', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rotating', stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2) stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertRegexpMatchesWithFlags( stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE, msg=('Storage class appears not to have been changed.'))
def test_rewrite_key_rotation_single_object(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Rotate key. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1) ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rotating', stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2) # Remove encryption. boto_config_for_test2 = [('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2)] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Decrypting', stderr) self.AssertObjectUnencrypted(suri(object_uri))
def test_stat_encrypted_object(self): """Tests stat command with an encrypted object.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1) # Stat object with key should return unencrypted hashes. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout) # Stat object without key should return encrypted hashes. stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn('encrypted', stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def test_rewrite_bucket_recursive(self): """Tests rewrite command recursively on a bucket.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() self._test_rewrite_key_rotation_bucket( bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)])
def test_stat_encrypted_object(self): """Tests stat command with an encrypted object.""" if self.test_api == ApiSelector.XML: return unittest.skip("gsutil does not support encryption with the XML API") bucket_uri = self.CreateBucket() object_uri = self.CreateObject( bucket_uri=bucket_uri, object_name="foo", contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1, ) # Stat object with key should return unencrypted hashes. with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]): stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout) # Stat object without key should return encrypted hashes. stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn("encrypted", stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def test_rewrite_to_same_storage_class_is_skipped(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar') stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)], return_stderr=True) self.assertIn('Skipping %s' % suri(object_uri), stderr)
def testSetContentTypeFromFile(self): """Tests that content type is correctly determined for symlinks.""" if IS_WINDOWS: return unittest.skip('use_magicfile features not available on Windows') surprise_html = '<html><body>And you thought I was just text!</body></html>' temp_dir_path = self.CreateTempDir() txt_file_path = self.CreateTempFile( tmpdir=temp_dir_path, contents=surprise_html, file_name='html_in_disguise.txt') link_name = 'link_to_realfile' # Notice no file extension was supplied. os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name) # Content-type of a symlink should be obtained from the link's target. dst_obj_metadata_mock = mock.MagicMock(contentType=None) src_url_stub = mock.MagicMock( object_name=temp_dir_path + os.path.sep + link_name, **{'IsFileUrl.return_value': True, 'IsStream.return_value': False, 'IsFifo.return_value': False}) # The file command should detect HTML in the real file. with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]): _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual('text/html; charset=us-ascii', dst_obj_metadata_mock.contentType) dst_obj_metadata_mock = mock.MagicMock(contentType=None) # The mimetypes module should guess based on the real file's extension. with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]): _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual('text/plain', dst_obj_metadata_mock.contentType)
def test_parallel_rewrite_bucket_flat_wildcard(self): """Tests parallel rewrite command with a flat wildcard on a bucket.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() self._test_rewrite_key_rotation_bucket( bucket_uri, ['-m', 'rewrite', '-k', suri(bucket_uri, '**')])
def test_rewrite_key_rotation_single_object(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Rotate key. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rotating', stderr) self.AssertObjectUsesEncryptionKey(suri(object_uri), TEST_ENCRYPTION_KEY2) # Remove encryption. boto_config_for_test2 = [ ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2)] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Decrypting', stderr) self.AssertObjectUnencrypted(suri(object_uri))
def test_compose_with_encryption(self): """Tests composing encrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo', encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Compose without correct key should fail. stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj')], expected_status=1, return_stderr=True) self.assertIn('is encrypted by a customer-supplied encryption key', stderr) # Compose with different encryption key should fail; source and destination # encryption keys must match. with SetBotoConfigForTest([ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]): stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj')], expected_status=1, return_stderr=True) self.assertIn('provided encryption key is incorrect', stderr) with SetBotoConfigForTest( [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj')])
def test_compose_with_encryption(self): """Tests composing encrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo', encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Compose without correct key should fail. stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj')], expected_status=1, return_stderr=True) self.assertIn('is encrypted by a customer-supplied encryption key', stderr) # Compose with different encryption key should fail; source and destination # encryption keys must match. with SetBotoConfigForTest([ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]): stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj')], expected_status=1, return_stderr=True) self.assertIn('is encrypted by a different customer-supplied key', stderr) with SetBotoConfigForTest( [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj')])
def testSetContentTypeFromFile(self): """Tests that content type is correctly determined for symlinks.""" if system_util.IS_WINDOWS: return unittest.skip('use_magicfile features not available on Windows') surprise_html = b'<html><body>And you thought I was just text!</body></html>' temp_dir_path = self.CreateTempDir() txt_file_path = self.CreateTempFile(tmpdir=temp_dir_path, contents=surprise_html, file_name='html_in_disguise.txt') link_name = 'link_to_realfile' # Notice no file extension was supplied. os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name) # Content-type of a symlink should be obtained from the link's target. dst_obj_metadata_mock = mock.MagicMock(contentType=None) src_url_stub = mock.MagicMock(object_name=temp_dir_path + os.path.sep + link_name, **{ 'IsFileUrl.return_value': True, 'IsStream.return_value': False, 'IsFifo.return_value': False }) # The file command should detect HTML in the real file. with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]): _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual('text/html; charset=us-ascii', dst_obj_metadata_mock.contentType) dst_obj_metadata_mock = mock.MagicMock(contentType=None) # The mimetypes module should guess based on the real file's extension. with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]): _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual('text/plain', dst_obj_metadata_mock.contentType)
def test_rewrite_unintentional_key_rotation_fails(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') encrypted_obj_uri = self.CreateObject( contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) unencrypted_obj_uri = self.CreateObject(contents='bar') boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1) ] with SetBotoConfigForTest(boto_config_for_test): # Executing rewrite without the -k flag should fail if your boto file has # a different encryption_key than was last used to encrypt the object. stderr = self.RunGsUtil( ['rewrite', '-s', 'dra', suri(encrypted_obj_uri)], return_stderr=True, expected_status=1) self.assertIn('EncryptionException', stderr) # Should also fail for a previously unencrypted object. stderr = self.RunGsUtil( ['rewrite', '-s', 'dra', suri(unencrypted_obj_uri)], return_stderr=True, expected_status=1) self.assertIn('EncryptionException', stderr)
def test_rewrite_with_no_encryption_key_operates_on_unencrypted_objects(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') # Since the introduction of default KMS keys for GCS buckets, rewriting # with no explicitly specified CSEK/CMEK can still result in the rewritten # objects being encrypted. Before KMS support, this would always result in # decrypted objects. With this new possibility, we want to always rewrite # every specified object when no encryption_key was set in the boto config, # since we don't know if the operation will end up decrypting the object or # implicitly encrypting it with the bucket's default KMS key. key_fqn = self.authorize_project_to_use_testing_kms_key() # Create an unencrypted object. bucket_uri = self.CreateBucket() object_uri = self.CreateObject( bucket_uri=bucket_uri, object_name='foo', contents='foo') # Set the bucket's default KMS key. self.RunGsUtil(['kms', 'encryption', '-k', key_fqn, suri(bucket_uri)]) # Rewriting with no encryption_key should rewrite the object, resulting in # the bucket's default KMS key being used to encrypt it. with SetBotoConfigForTest([('GSUtil', 'encryption_key', None)]): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rewriting', stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)
def test_rewrite_key_rotation_bucket_subdir(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() object_contents = 'bar' rotate_subdir = suri(bucket_uri, 'bar') object_uri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/bar', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY2) object_uri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/baz', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY3) object_uri4 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/qux', contents=object_contents) # Rotate subdir keys to TEST_ENCRYPTION_KEY3. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)] self.AssertNObjectsInBucket(bucket_uri, 4) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir], return_stderr=True) self.assertIn('Rotating', stderr) # Object 2. self.assertIn('Skipping %s' % suri(object_uri3), stderr) self.assertIn('Encrypting', stderr) # Object 4. # First subdir should be unaffected. self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1) for object_uri_str in (suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3) # Remove encryption in subdir. boto_config_for_test2 = [ ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir], return_stderr=True) self.assertIn('Decrypting', stderr) # First subdir should be unaffected. self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1) for object_uri_str in (suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUnencrypted(object_uri_str)
def test_turning_on(self): if self.test_api == ApiSelector.XML: return unittest.skip('XML API has no concept of Bucket Policy Only') bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_bpo_cmd + ['on', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, True)
def test_rewrite_with_no_value_for_minus_s(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'], return_stderr=True, expected_status=1) self.assertIn('CommandException', stderr) self.assertIn('expects at least one URL', stderr)
def testRetryableErrorMediaCollection(self): """Tests that retryable errors are collected on JSON media operations.""" # Retryable errors will only be collected with the JSON API. if self.test_api != ApiSelector.JSON: return unittest.skip('Retryable errors are only collected in JSON') boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB))] bucket_uri = self.CreateBucket() # For the resumable upload exception, we need to ensure at least one # callback occurs. halt_size = START_CALLBACK_PER_BYTES * 2 fpath = self.CreateTempFile(contents='a' * halt_size) # Test that the retry function for data transfers catches and logs an error. test_callback_file = self.CreateTempFile(contents=pickle.dumps( _ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError, ('unused', 'unused', 'unused')))) with SetBotoConfigForTest(boto_config_for_test): metrics_list = self._RunGsUtilWithAnalyticsOutput( ['cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri)]) self._CheckParameterValue('Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'BadStatusCodeError', metrics_list) self._CheckParameterValue('Retryable Errors', '1', metrics_list) self._CheckParameterValue('Num Retryable Service Errors', '1', metrics_list) # Test that the ResumableUploadStartOverException in copy_helper is caught. test_callback_file = self.CreateTempFile( contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404))) with SetBotoConfigForTest(boto_config_for_test): metrics_list = self._RunGsUtilWithAnalyticsOutput( ['cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri)]) self._CheckParameterValue( 'Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list) self._CheckParameterValue( 'Event Action', 'ResumableUploadStartOverException', metrics_list) self._CheckParameterValue('Retryable Errors', '1', metrics_list) self._CheckParameterValue( 'Num Retryable Service Errors', '1', metrics_list) # Test retryable error collection in a multithread/multiprocess situation. test_callback_file = self.CreateTempFile( contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404))) with SetBotoConfigForTest(boto_config_for_test): metrics_list = self._RunGsUtilWithAnalyticsOutput( ['-m', 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri)]) self._CheckParameterValue('Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list) self._CheckParameterValue( 'Event Action', 'ResumableUploadStartOverException', metrics_list) self._CheckParameterValue('Retryable Errors', '1', metrics_list) self._CheckParameterValue( 'Num Retryable Service Errors', '1', metrics_list)
def test_turning_off_on_enabled_buckets(self): if self.test_api == ApiSelector.XML: return unittest.skip('XML API has no concept of Bucket Policy Only') bucket_uri = self.CreateBucket(bucket_policy_only=True, prefer_json_api=True) self._AssertEnabled(bucket_uri, True) self.RunGsUtil(self._set_bpo_cmd + ['off', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, False)
def test_turning_on(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of Uniform bucket-level access') bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_ubla_cmd + ['on', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, True)
def test_set_valid_acl_bucket(self): """Ensures that valid canned and XML ACLs work with get/set.""" if self._ServiceAccountCredentialsPresent(): # See comments in _ServiceAccountCredentialsPresent unittest.skip('Canned ACLs orphan service account permissions.') bucket_uri = suri(self.CreateBucket()) acl_string = self.RunGsUtil(self._get_acl_prefix + [bucket_uri], return_stdout=True) inpath = self.CreateTempFile(contents=acl_string) self.RunGsUtil(self._set_acl_prefix + ['public-read', bucket_uri]) acl_string2 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri], return_stdout=True) self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri]) acl_string3 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertEqual(acl_string, acl_string3)
def test_set_lifecycle_wildcard(self): """Tests setting lifecycle with a wildcarded bucket URI.""" if self.test_api == ApiSelector.XML: # This test lists buckets with wildcards, but it is possible that another # test being run in parallel (in the same project) deletes a bucket after # it is listed in this test. This causes the subsequent XML metadata get # for the lifecycle configuration to fail on that just-deleted bucket, # even though that bucket is not used directly in this test. return unittest.skip( 'XML wildcard behavior can cause test to flake ' 'if a bucket in the same project is deleted ' 'during execution.') random_prefix = self.MakeRandomTestString() bucket1_name = self.MakeTempName('bucket', prefix=random_prefix) bucket2_name = self.MakeTempName('bucket', prefix=random_prefix) bucket1_uri = self.CreateBucket(bucket_name=bucket1_name) bucket2_uri = self.CreateBucket(bucket_name=bucket2_name) # This just double checks that the common prefix of the two buckets is what # we think it should be (based on implementation detail of CreateBucket). # We want to be careful when setting a wildcard on buckets to make sure we # don't step outside the test buckets to affect other buckets. common_prefix = posixpath.commonprefix( [suri(bucket1_uri), suri(bucket2_uri)]) self.assertTrue( common_prefix.startswith( 'gs://%sgsutil-test-test-set-lifecycle-wildcard-' % random_prefix)) wildcard = '%s*' % common_prefix fpath = self.CreateTempFile( contents=self.lifecycle_doc.encode('ascii')) # Use @Retry as hedge against bucket listing eventual consistency. expected = set([ 'Setting lifecycle configuration on %s/...' % suri(bucket1_uri), 'Setting lifecycle configuration on %s/...' % suri(bucket2_uri) ]) actual = set() @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard], return_stderr=True) actual.update(stderr.splitlines()) self.assertEqual(expected, actual) self.assertEqual(stderr.count('Setting lifecycle configuration'), 2) _Check1() stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
def test_uppercase_header(self): """Tests setting custom metadata with an uppercase value.""" if self.test_api == ApiSelector.XML: return unittest.skip('XML header keys are case-insensitive.') objuri = self.CreateObject(contents='foo') self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-CaSe:SeNsItIvE' % self.provider_custom_meta, suri(objuri)]) stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True) self.assertRegexpMatches(stdout, ur'CaSe:\s+SeNsItIvE')
def test_repo_matches_manifest(self): """Ensure any new top-level files are present in the manifest.""" p = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() if p.returncode != 0: unittest.skip('Test only runs from git repository.') return manifest_lines = ['gslib', 'third_party', 'MANIFEST.in'] manifest_file = os.path.join(GSUTIL_DIR, 'MANIFEST.in') try: with open(manifest_file, 'r') as fp: for line in fp: if line.startswith('include '): manifest_lines.append(line.split()[-1]) except IOError: # If manifest file is not readable (example: Travis CI), skip the test. unittest.skip('Test must be able to read manifest file.') return p = subprocess.Popen(['git', 'ls-tree', '--name-only', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, _) = p.communicate() git_top_level_files = stdout.splitlines() for filename in git_top_level_files: if filename.endswith('.pyc'): # Ignore compiled code. continue if filename in ('.gitmodules', '.gitignore', '.travis.yml'): # We explicitly drop these files when building the gsutil tarball. # If we add any other files to this list, the tarball script must # also be updated or we could break the gsutil update command. continue if filename not in manifest_lines: self.fail( 'Found file %s not present in MANIFEST.in, which would ' 'break gsutil update.' % filename)
def test_rewrite_generation_url(self): """Tests that rewrite fails on a URL that includes a generation.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) generation = object_uri.generation stderr = self.RunGsUtil( ['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)], return_stderr=True, expected_status=1) self.assertIn('"rewrite" called on URL with generation', stderr)
def test_turning_off_on_enabled_buckets(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of Uniform bucket-level access') # TODO(mynameisrafe): Replace bucket_policy_only with uniform_bucket_level_access when the property is live. bucket_uri = self.CreateBucket(bucket_policy_only=True, prefer_json_api=True) self._AssertEnabled(bucket_uri, True) self.RunGsUtil(self._set_ubla_cmd + ['off', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, False)
def testRetryableErrorMetadataCollection(self): """Tests that retryable errors are collected on JSON metadata operations.""" # Retryable errors will only be collected with the JSON API. if self.test_api != ApiSelector.JSON: return unittest.skip('Retryable errors are only collected in JSON') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'bar') # Set the command name to rsync in order to collect PerformanceSummary info. self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync' # Generate a JSON API instance to test with, because the RunGsUtil method # may use the XML API. gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), RetryableErrorsQueue(), self.default_provider) # Don't wait for too many retries or for long periods between retries to # avoid long tests. gsutil_api.api_client.num_retries = 2 gsutil_api.api_client.max_retry_wait = 1 # Throw an error when transferring metadata. key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type) dst_obj_metadata = apitools_messages.Object( bucket=src_obj_metadata.bucket, name=self.MakeTempName('object'), contentType=src_obj_metadata.contentType) with mock.patch.object(http_wrapper, '_MakeRequestNoRetry', side_effect=socket.error()): _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata, dst_obj_metadata) if six.PY2: self.assertEqual(self.collector.retryable_errors['SocketError'], 1) else: # In PY3, socket.* errors are deprecated aliases for OSError self.assertEqual(self.collector.retryable_errors['OSError'], 1) # Throw an error when removing a bucket. with mock.patch.object( http_wrapper, '_MakeRequestNoRetry', side_effect=apitools_exceptions.HttpError('unused', 'unused', 'unused')): _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name, object_uri.object_name) self.assertEqual(self.collector.retryable_errors['HttpError'], 1) # Check that the number of each kind of retryable error was logged. self.assertEqual( self.collector.perf_sum_params.num_retryable_network_errors, 1) self.assertEqual( self.collector.perf_sum_params.num_retryable_service_errors, 1)
def test_set_lifecycle_wildcard(self): """Tests setting lifecycle with a wildcarded bucket URI.""" if self.test_api == ApiSelector.XML: # This test lists buckets with wildcards, but it is possible that another # test being run in parallel (in the same project) deletes a bucket after # it is listed in this test. This causes the subsequent XML metadata get # for the lifecycle configuration to fail on that just-deleted bucket, # even though that bucket is not used directly in this test. return unittest.skip('XML wildcard behavior can cause test to flake ' 'if a bucket in the same project is deleted ' 'during execution.') random_prefix = self.MakeRandomTestString() bucket1_name = self.MakeTempName('bucket', prefix=random_prefix) bucket2_name = self.MakeTempName('bucket', prefix=random_prefix) bucket1_uri = self.CreateBucket(bucket_name=bucket1_name) bucket2_uri = self.CreateBucket(bucket_name=bucket2_name) # This just double checks that the common prefix of the two buckets is what # we think it should be (based on implementation detail of CreateBucket). # We want to be careful when setting a wildcard on buckets to make sure we # don't step outside the test buckets to affect other buckets. common_prefix = posixpath.commonprefix( [suri(bucket1_uri), suri(bucket2_uri)]) self.assertTrue( common_prefix.startswith( 'gs://%sgsutil-test-test-set-lifecycle-wildcard-' % random_prefix)) wildcard = '%s*' % common_prefix fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii')) # Use @Retry as hedge against bucket listing eventual consistency. expected = set([ 'Setting lifecycle configuration on %s/...' % suri(bucket1_uri), 'Setting lifecycle configuration on %s/...' % suri(bucket2_uri) ]) actual = set() @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard], return_stderr=True) actual.update(stderr.splitlines()) self.assertEqual(expected, actual) self.assertEqual(stderr.count('Setting lifecycle configuration'), 2) _Check1() stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
def test_rewrite_with_same_key_and_storage_class_is_skipped(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='foo', encryption_key=TEST_ENCRYPTION_KEY1, storage_class='standard') boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', '-s', 'standard', suri(object_uri)], return_stderr=True) self.assertIn('Skipping %s' % suri(object_uri), stderr)
def test_mv_early_deletion_warning(self): """Tests that mv on a recent nearline object warns about early deletion.""" if self.test_api == ApiSelector.XML: return unittest.skip('boto does not return object storage class') bucket_uri = self.CreateBucket(storage_class='NEARLINE') object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='obj') stderr = self.RunGsUtil(['mv', suri(object_uri), suri(bucket_uri, 'foo')], return_stderr=True) self.assertIn( 'Warning: moving nearline object %s may incur an early deletion ' 'charge, because the original object is less than 30 days old ' 'according to the local system time.' % suri(object_uri), stderr)
def test_rewrite_with_only_storage_class_change(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar') # Change storage class to nearline. stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True) self.assertIn('Rewriting', stderr) stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertRegexpMatchesWithFlags( stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE, msg=('Storage class appears not to have been changed.'))
def test_rewrite_stdin_args(self): """Tests rewrite with arguments supplied on stdin.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) stdin_arg = suri(object_uri) boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg) self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2)
def test_rewrite_missing_decryption_key(self): """Tests that rewrite fails when no decryption key matches.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(object_name='foo', contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)], return_stderr=True, expected_status=1) self.assertIn('No decryption key matches object %s' % suri(object_uri), stderr)
def test_rewrite_stdin_args(self): """Tests rewrite with arguments supplied on stdin.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) stdin_arg = suri(object_uri) boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg) self.AssertObjectUsesEncryptionKey(stdin_arg, TEST_ENCRYPTION_KEY2)
def testRetryableErrorMetadataCollection(self): """Tests that retryable errors are collected on JSON metadata operations.""" # Retryable errors will only be collected with the JSON API. if self.test_api != ApiSelector.JSON: return unittest.skip('Retryable errors are only collected in JSON') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents='bar') # Set the command name to rsync in order to collect PerformanceSummary info. self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync' # Generate a JSON API instance to test with, because the RunGsUtil method # may use the XML API. gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), RetryableErrorsQueue(), self.default_provider) # Don't wait for too many retries or for long periods between retries to # avoid long tests. gsutil_api.api_client.num_retries = 2 gsutil_api.api_client.max_retry_wait = 1 # Throw an error when transferring metadata. key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type) dst_obj_metadata = apitools_messages.Object( bucket=src_obj_metadata.bucket, name=self.MakeTempName('object'), contentType=src_obj_metadata.contentType) with mock.patch.object(http_wrapper, '_MakeRequestNoRetry', side_effect=socket.error()): _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata, dst_obj_metadata) self.assertEqual(self.collector.retryable_errors['SocketError'], 1) # Throw an error when removing a bucket. with mock.patch.object( http_wrapper, '_MakeRequestNoRetry', side_effect=apitools_exceptions.HttpError('unused', 'unused', 'unused')): _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name, object_uri.object_name) self.assertEqual(self.collector.retryable_errors['HttpError'], 1) # Check that the number of each kind of retryable error was logged. self.assertEqual( self.collector.perf_sum_params.num_retryable_network_errors, 1) self.assertEqual( self.collector.perf_sum_params.num_retryable_service_errors, 1)
def test_rewrite_seek_ahead(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Remove encryption boto_config_for_test = [ ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1), ('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['-m', 'rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn( 'Estimated work for this command: objects: 1, total size: 3', stderr)
def test_stat_encrypted_object_wildcard(self): """Tests stat command with a mix of encrypted and unencrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo1', contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1) object2_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo2', contents=TEST_ENCRYPTION_CONTENT2, encryption_key=TEST_ENCRYPTION_KEY2) object3_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo3', contents=TEST_ENCRYPTION_CONTENT3) stat_string = suri(object1_uri)[:-2] + '*' # Stat 3 objects, two encrypted each with a different key, and one # unencrypted. Should result in two unencrypted listing and one encrypted # listing. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _StatExpectMixed(): """Runs stat and validates output.""" stdout, stderr = self.RunGsUtil(['stat', stat_string], return_stdout=True, return_stderr=True) self.assertIn(suri(object1_uri), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) self.assertIn(suri(object2_uri), stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout) self.assertIn('encrypted', stdout) self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64.decode('ascii'), stdout) self.assertIn(suri(object3_uri), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout) _StatExpectMixed()
def test_rewrite_with_nonkey_transform_works_when_key_is_unchanged(self): # Tests that when a valid transformation flag aside from "-k" is supplied, # the "-k" flag is not supplied, and the encryption key previously used to # encrypt the target object matches the encryption_key in the user's boto # config file (via hash comparison), that the rewrite command properly # passes the same tuple for decryption and encryption, in addition to # performing the other desired transformations. if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True) self.assertIn('Rewriting', stderr)
def test_stat_encrypted_object_wildcard(self): """Tests stat command with a mix of encrypted and unencrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip("gsutil does not support encryption with the XML API") bucket_uri = self.CreateBucket() object1_uri = self.CreateObject( bucket_uri=bucket_uri, object_name="foo1", contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1, ) object2_uri = self.CreateObject( bucket_uri=bucket_uri, object_name="foo2", contents=TEST_ENCRYPTION_CONTENT2, encryption_key=TEST_ENCRYPTION_KEY2, ) object3_uri = self.CreateObject(bucket_uri=bucket_uri, object_name="foo3", contents=TEST_ENCRYPTION_CONTENT3) stat_string = suri(object1_uri)[:-2] + "*" # Stat 3 objects, two encrypted each with a different key, and one # unencrypted. Should result in two unencrypted listing and one encrypted # listing. with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]): # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _StatExpectMixed(): """Runs stat and validates output.""" stdout = self.RunGsUtil(["stat", stat_string], return_stdout=True) self.assertIn(suri(object1_uri), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout) self.assertIn(suri(object2_uri), stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout) self.assertIn("encrypted", stdout) self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64, stdout) self.assertIn(suri(object3_uri), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout) _StatExpectMixed()
def test_cat_encrypted_object(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') object_contents = '0123456789' object_uri = self.CreateObject(object_name='foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) stderr = self.RunGsUtil(['cat', suri(object_uri)], expected_status=1, return_stderr=True) self.assertIn('No decryption key matches object', stderr) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stdout = self.RunGsUtil(['cat', suri(object_uri)], return_stdout=True) self.assertEqual(stdout, object_contents) stdout = self.RunGsUtil(['cat', '-r 1-3', suri(object_uri)], return_stdout=True) self.assertEqual(stdout, '123')
def test_rewrite_to_kms_then_unencrypted(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') key_fqn = self.authorize_project_to_use_testing_kms_key() object_uri = self.CreateObject(contents='foo') boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Encrypting', stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn) # Rewrite back to unencrypted and make sure no KMS key was used. boto_config_for_test = [('GSUtil', 'encryption_key', None)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Decrypting', stderr) self.AssertObjectUnencrypted(suri(object_uri))
def test_rewrite_to_kms_then_csek(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') key_fqn = self.authorize_project_to_use_testing_kms_key() object_uri = self.CreateObject(contents='foo') boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Encrypting', stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn) # Rewrite from CMEK to CSEK encryption. boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rotating', stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1)
def test_rewrite_overwrite_acl(self): """Tests rewrite with the -O flag.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)]) stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)], return_stdout=True) self.assertIn('allUsers', stdout) boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['rewrite', '-k', '-O', suri(object_uri)]) self.AssertObjectUsesEncryptionKey(suri(object_uri), TEST_ENCRYPTION_KEY2) stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)], return_stdout=True) self.assertNotIn('allUsers', stdout)
def test_rewrite_key_rotation_with_storage_class_change(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents='bar', encryption_key=TEST_ENCRYPTION_KEY1) # Rotate key and change storage class to nearline. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-s', 'nearline', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rotating', stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2) stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertRegexpMatchesWithFlags( stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE, msg=('Storage class appears not to have been changed.'))
def testRetryableErrorMetadataCollection(self): """Tests that retryable errors are collected on JSON metadata operations.""" # Retryable errors will only be collected with the JSON API. if self.test_api != ApiSelector.JSON: return unittest.skip('Retryable errors are only collected in JSON') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents='bar') # Generate a JSON API instance because the RunGsUtil method uses the XML # API. gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), RetryableErrorsQueue(), self.default_provider) # Don't wait for too many retries or for long periods between retries to # avoid long tests. gsutil_api.api_client.num_retries = 2 gsutil_api.api_client.max_retry_wait = 1 # Throw an error when transferring metadata. key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type) dst_obj_metadata = apitools_messages.Object( bucket=src_obj_metadata.bucket, name=self.MakeTempName('object'), contentType=src_obj_metadata.contentType) with mock.patch.object(http_wrapper, '_MakeRequestNoRetry', side_effect=socket.error()): _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata, dst_obj_metadata) self.assertEqual(self.collector.retryable_errors['SocketError'], 1) # Throw an error when removing a bucket. with mock.patch.object(http_wrapper, '_MakeRequestNoRetry', side_effect=ValueError()): _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name, object_uri.object_name) self.assertEqual(self.collector.retryable_errors['ValueError'], 1)