Beispiel #1
0
  def test_repo_matches_manifest(self):
    """Ensure any new top-level files are present in the manifest."""
    p = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    p.communicate()
    if p.returncode != 0:
      unittest.skip('Test only runs from git repository.')

    manifest_lines = ['gslib', 'third_party', 'MANIFEST.in']

    with open(os.path.join(GSUTIL_DIR, 'MANIFEST.in'), 'r') as fp:
      for line in fp:
        if line.startswith('include '):
          manifest_lines.append(line.split()[-1])

    p = subprocess.Popen(['git', 'ls-tree', '--name-only', 'HEAD'],
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (stdout, _) = p.communicate()
    git_top_level_files = stdout.splitlines()

    for filename in git_top_level_files:
      if filename.endswith('.pyc'):
        # Ignore compiled code.
        continue
      if filename in ('.gitmodules', '.gitignore', '.travis.yml'):
        # We explicitly drop these files when building the gsutil tarball.
        # If we add any other files to this list, the tarball script must
        # also be updated or we could break the gsutil update command.
        continue
      if filename not in manifest_lines:
        self.fail('Found file %s not present in MANIFEST.in, which would '
                  'break gsutil update.' % filename)
Beispiel #2
0
    def test_rewrite_key_rotation_bucket_subdir(self):
        if self.test_api == ApiSelector.XML:
            return unittest.skip('Rewrite API is only supported in JSON.')
        bucket_uri = self.CreateBucket()
        object_contents = 'bar'
        rotate_subdir = suri(bucket_uri, 'bar')
        object_uri1 = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='foo/bar',
                                        contents=object_contents,
                                        encryption_key=TEST_ENCRYPTION_KEY1)
        object_uri2 = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='bar/foo',
                                        contents=object_contents,
                                        encryption_key=TEST_ENCRYPTION_KEY2)
        object_uri3 = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='bar/baz',
                                        contents=object_contents,
                                        encryption_key=TEST_ENCRYPTION_KEY3)
        object_uri4 = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='bar/qux',
                                        contents=object_contents)

        # Rotate subdir keys to TEST_ENCRYPTION_KEY3.
        boto_config_for_test = [
            ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3),
            ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2),
            ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)
        ]

        self.AssertNObjectsInBucket(bucket_uri, 4)

        with SetBotoConfigForTest(boto_config_for_test):
            stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
                                    return_stderr=True)
            self.assertIn('Rotating', stderr)  # Object 2.
            self.assertIn('Skipping %s' % suri(object_uri3), stderr)
            self.assertIn('Encrypting', stderr)  # Object 4.

        # First subdir should be unaffected.
        self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)

        for object_uri_str in (suri(object_uri2), suri(object_uri3),
                               suri(object_uri4)):
            self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3)

        # Remove encryption in subdir.
        boto_config_for_test2 = [('GSUtil', 'decryption_key1',
                                  TEST_ENCRYPTION_KEY3)]

        with SetBotoConfigForTest(boto_config_for_test2):
            stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
                                    return_stderr=True)
            self.assertIn('Decrypting', stderr)

        # First subdir should be unaffected.
        self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)

        for object_uri_str in (suri(object_uri2), suri(object_uri3),
                               suri(object_uri4)):
            self.AssertObjectUnencrypted(object_uri_str)
Beispiel #3
0
    def test_rewrite_key_rotation_with_storage_class_change(self):
        if self.test_api == ApiSelector.XML:
            return unittest.skip('Rewrite API is only supported in JSON.')
        object_uri = self.CreateObject(contents='bar',
                                       encryption_key=TEST_ENCRYPTION_KEY1)

        # Rotate key and change storage class to nearline.
        boto_config_for_test = [
            ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
            ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)
        ]
        with SetBotoConfigForTest(boto_config_for_test):
            stderr = self.RunGsUtil(
                ['rewrite', '-s', 'nearline', '-k',
                 suri(object_uri)],
                return_stderr=True)
            self.assertIn('Rotating', stderr)

        self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)
        stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
        self.assertRegexpMatchesWithFlags(
            stdout,
            r'Storage class:\s+NEARLINE',
            flags=re.IGNORECASE,
            msg=('Storage class appears not to have been changed.'))
Beispiel #4
0
    def test_rewrite_key_rotation_single_object(self):
        if self.test_api == ApiSelector.XML:
            return unittest.skip('Rewrite API is only supported in JSON.')
        object_uri = self.CreateObject(contents='bar',
                                       encryption_key=TEST_ENCRYPTION_KEY1)

        # Rotate key.
        boto_config_for_test = [
            ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
            ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)
        ]

        with SetBotoConfigForTest(boto_config_for_test):
            stderr = self.RunGsUtil(
                ['rewrite', '-k', suri(object_uri)], return_stderr=True)
            self.assertIn('Rotating', stderr)

        self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)

        # Remove encryption.
        boto_config_for_test2 = [('GSUtil', 'decryption_key1',
                                  TEST_ENCRYPTION_KEY2)]
        with SetBotoConfigForTest(boto_config_for_test2):
            stderr = self.RunGsUtil(
                ['rewrite', '-k', suri(object_uri)], return_stderr=True)
            self.assertIn('Decrypting', stderr)

        self.AssertObjectUnencrypted(suri(object_uri))
Beispiel #5
0
    def test_stat_encrypted_object(self):
        """Tests stat command with an encrypted object."""
        if self.test_api == ApiSelector.XML:
            return unittest.skip(
                'gsutil does not support encryption with the XML API')
        bucket_uri = self.CreateBucket()
        object_uri = self.CreateObject(bucket_uri=bucket_uri,
                                       object_name='foo',
                                       contents=TEST_ENCRYPTION_CONTENT1,
                                       encryption_key=TEST_ENCRYPTION_KEY1)

        # Stat object with key should return unencrypted hashes.
        with SetBotoConfigForTest([('GSUtil', 'encryption_key',
                                    TEST_ENCRYPTION_KEY1)]):
            stdout = self.RunGsUtil(['stat', suri(object_uri)],
                                    return_stdout=True)
            self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
            self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
            self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)

        # Stat object without key should return encrypted hashes.
        stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
        self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
        self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
        self.assertIn('encrypted', stdout)
        self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
 def test_rewrite_bucket_recursive(self):
   """Tests rewrite command recursively on a bucket."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   bucket_uri = self.CreateBucket()
   self._test_rewrite_key_rotation_bucket(
       bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)])
    def test_stat_encrypted_object(self):
        """Tests stat command with an encrypted object."""
        if self.test_api == ApiSelector.XML:
            return unittest.skip("gsutil does not support encryption with the XML API")
        bucket_uri = self.CreateBucket()
        object_uri = self.CreateObject(
            bucket_uri=bucket_uri,
            object_name="foo",
            contents=TEST_ENCRYPTION_CONTENT1,
            encryption_key=TEST_ENCRYPTION_KEY1,
        )

        # Stat object with key should return unencrypted hashes.
        with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]):
            stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True)
            self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
            self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
            self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)

        # Stat object without key should return encrypted hashes.
        stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True)
        self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
        self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
        self.assertIn("encrypted", stdout)
        self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
 def test_rewrite_to_same_storage_class_is_skipped(self):
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar')
   stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)],
                           return_stderr=True)
   self.assertIn('Skipping %s' % suri(object_uri), stderr)
  def testSetContentTypeFromFile(self):
    """Tests that content type is correctly determined for symlinks."""
    if IS_WINDOWS:
      return unittest.skip('use_magicfile features not available on Windows')

    surprise_html = '<html><body>And you thought I was just text!</body></html>'
    temp_dir_path = self.CreateTempDir()
    txt_file_path = self.CreateTempFile(
        tmpdir=temp_dir_path, contents=surprise_html,
        file_name='html_in_disguise.txt')
    link_name = 'link_to_realfile'  # Notice no file extension was supplied.
    os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name)
    # Content-type of a symlink should be obtained from the link's target.
    dst_obj_metadata_mock = mock.MagicMock(contentType=None)
    src_url_stub = mock.MagicMock(
        object_name=temp_dir_path + os.path.sep + link_name,
        **{'IsFileUrl.return_value': True,
           'IsStream.return_value': False,
           'IsFifo.return_value': False})

    # The file command should detect HTML in the real file.
    with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]):
      _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
    self.assertEqual('text/html; charset=us-ascii', dst_obj_metadata_mock.contentType)

    dst_obj_metadata_mock = mock.MagicMock(contentType=None)
    # The mimetypes module should guess based on the real file's extension.
    with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]):
      _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
    self.assertEqual('text/plain', dst_obj_metadata_mock.contentType)
Beispiel #10
0
 def test_parallel_rewrite_bucket_flat_wildcard(self):
   """Tests parallel rewrite command with a flat wildcard on a bucket."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   bucket_uri = self.CreateBucket()
   self._test_rewrite_key_rotation_bucket(
       bucket_uri, ['-m', 'rewrite', '-k', suri(bucket_uri, '**')])
Beispiel #11
0
  def test_rewrite_key_rotation_single_object(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar',
                                   encryption_key=TEST_ENCRYPTION_KEY1)

    # Rotate key.
    boto_config_for_test = [
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]

    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
                              return_stderr=True)
      self.assertIn('Rotating', stderr)

    self.AssertObjectUsesEncryptionKey(suri(object_uri),
                                       TEST_ENCRYPTION_KEY2)

    # Remove encryption.
    boto_config_for_test2 = [
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2)]
    with SetBotoConfigForTest(boto_config_for_test2):
      stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
                              return_stderr=True)
      self.assertIn('Decrypting', stderr)

    self.AssertObjectUnencrypted(suri(object_uri))
  def test_compose_with_encryption(self):
    """Tests composing encrypted objects."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip(
          'gsutil does not support encryption with the XML API')
    bucket_uri = self.CreateBucket()
    object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo',
                                    encryption_key=TEST_ENCRYPTION_KEY1)
    object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='bar',
                                    encryption_key=TEST_ENCRYPTION_KEY1)

    # Compose without correct key should fail.
    stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
                             suri(bucket_uri, 'obj')], expected_status=1,
                            return_stderr=True)
    self.assertIn('is encrypted by a customer-supplied encryption key', stderr)

    # Compose with different encryption key should fail; source and destination
    # encryption keys must match.
    with SetBotoConfigForTest([
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]):
      stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
                               suri(bucket_uri, 'obj')], expected_status=1,
                              return_stderr=True)
      self.assertIn('provided encryption key is incorrect', stderr)

    with SetBotoConfigForTest(
        [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]):
      self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
                      suri(bucket_uri, 'obj')])
Beispiel #13
0
 def test_rewrite_bucket_recursive(self):
   """Tests rewrite command recursively on a bucket."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   bucket_uri = self.CreateBucket()
   self._test_rewrite_key_rotation_bucket(
       bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)])
Beispiel #14
0
 def test_rewrite_to_same_storage_class_is_skipped(self):
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar')
   stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)],
                           return_stderr=True)
   self.assertIn('Skipping %s' % suri(object_uri), stderr)
Beispiel #15
0
  def test_compose_with_encryption(self):
    """Tests composing encrypted objects."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip(
          'gsutil does not support encryption with the XML API')
    bucket_uri = self.CreateBucket()
    object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='foo',
                                    encryption_key=TEST_ENCRYPTION_KEY1)
    object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents='bar',
                                    encryption_key=TEST_ENCRYPTION_KEY1)

    # Compose without correct key should fail.
    stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
                             suri(bucket_uri, 'obj')], expected_status=1,
                            return_stderr=True)
    self.assertIn('is encrypted by a customer-supplied encryption key', stderr)

    # Compose with different encryption key should fail; source and destination
    # encryption keys must match.
    with SetBotoConfigForTest([
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]):
      stderr = self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
                               suri(bucket_uri, 'obj')], expected_status=1,
                              return_stderr=True)
      self.assertIn('is encrypted by a different customer-supplied key', stderr)

    with SetBotoConfigForTest(
        [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]):
      self.RunGsUtil(['compose', suri(object_uri1), suri(object_uri2),
                      suri(bucket_uri, 'obj')])
 def test_parallel_rewrite_bucket_flat_wildcard(self):
   """Tests parallel rewrite command with a flat wildcard on a bucket."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   bucket_uri = self.CreateBucket()
   self._test_rewrite_key_rotation_bucket(
       bucket_uri, ['-m', 'rewrite', '-k', suri(bucket_uri, '**')])
  def testSetContentTypeFromFile(self):
    """Tests that content type is correctly determined for symlinks."""
    if system_util.IS_WINDOWS:
      return unittest.skip('use_magicfile features not available on Windows')

    surprise_html = b'<html><body>And you thought I was just text!</body></html>'
    temp_dir_path = self.CreateTempDir()
    txt_file_path = self.CreateTempFile(tmpdir=temp_dir_path,
                                        contents=surprise_html,
                                        file_name='html_in_disguise.txt')
    link_name = 'link_to_realfile'  # Notice no file extension was supplied.
    os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name)
    # Content-type of a symlink should be obtained from the link's target.
    dst_obj_metadata_mock = mock.MagicMock(contentType=None)
    src_url_stub = mock.MagicMock(object_name=temp_dir_path + os.path.sep +
                                  link_name,
                                  **{
                                      'IsFileUrl.return_value': True,
                                      'IsStream.return_value': False,
                                      'IsFifo.return_value': False
                                  })

    # The file command should detect HTML in the real file.
    with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]):
      _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
    self.assertEqual('text/html; charset=us-ascii',
                     dst_obj_metadata_mock.contentType)

    dst_obj_metadata_mock = mock.MagicMock(contentType=None)
    # The mimetypes module should guess based on the real file's extension.
    with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]):
      _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock)
    self.assertEqual('text/plain', dst_obj_metadata_mock.contentType)
Beispiel #18
0
    def test_rewrite_unintentional_key_rotation_fails(self):
        if self.test_api == ApiSelector.XML:
            return unittest.skip('Rewrite API is only supported in JSON.')
        encrypted_obj_uri = self.CreateObject(
            contents='bar', encryption_key=TEST_ENCRYPTION_KEY1)
        unencrypted_obj_uri = self.CreateObject(contents='bar')

        boto_config_for_test = [
            ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
            ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)
        ]
        with SetBotoConfigForTest(boto_config_for_test):
            # Executing rewrite without the -k flag should fail if your boto file has
            # a different encryption_key than was last used to encrypt the object.
            stderr = self.RunGsUtil(
                ['rewrite', '-s', 'dra',
                 suri(encrypted_obj_uri)],
                return_stderr=True,
                expected_status=1)
            self.assertIn('EncryptionException', stderr)

            # Should also fail for a previously unencrypted object.
            stderr = self.RunGsUtil(
                ['rewrite', '-s', 'dra',
                 suri(unencrypted_obj_uri)],
                return_stderr=True,
                expected_status=1)
            self.assertIn('EncryptionException', stderr)
  def test_rewrite_with_no_encryption_key_operates_on_unencrypted_objects(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    # Since the introduction of default KMS keys for GCS buckets, rewriting
    # with no explicitly specified CSEK/CMEK can still result in the rewritten
    # objects being encrypted. Before KMS support, this would always result in
    # decrypted objects. With this new possibility, we want to always rewrite
    # every specified object when no encryption_key was set in the boto config,
    # since we don't know if the operation will end up decrypting the object or
    # implicitly encrypting it with the bucket's default KMS key.

    key_fqn = self.authorize_project_to_use_testing_kms_key()

    # Create an unencrypted object.
    bucket_uri = self.CreateBucket()
    object_uri = self.CreateObject(
        bucket_uri=bucket_uri, object_name='foo', contents='foo')

    # Set the bucket's default KMS key.
    self.RunGsUtil(['kms', 'encryption', '-k', key_fqn, suri(bucket_uri)])

    # Rewriting with no encryption_key should rewrite the object, resulting in
    # the bucket's default KMS key being used to encrypt it.
    with SetBotoConfigForTest([('GSUtil', 'encryption_key', None)]):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', suri(object_uri)], return_stderr=True)
    self.assertIn('Rewriting', stderr)
    self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)
  def test_rewrite_key_rotation_bucket_subdir(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    bucket_uri = self.CreateBucket()
    object_contents = 'bar'
    rotate_subdir = suri(bucket_uri, 'bar')
    object_uri1 = self.CreateObject(bucket_uri=bucket_uri,
                                    object_name='foo/bar',
                                    contents=object_contents,
                                    encryption_key=TEST_ENCRYPTION_KEY1)
    object_uri2 = self.CreateObject(bucket_uri=bucket_uri,
                                    object_name='bar/foo',
                                    contents=object_contents,
                                    encryption_key=TEST_ENCRYPTION_KEY2)
    object_uri3 = self.CreateObject(bucket_uri=bucket_uri,
                                    object_name='bar/baz',
                                    contents=object_contents,
                                    encryption_key=TEST_ENCRYPTION_KEY3)
    object_uri4 = self.CreateObject(bucket_uri=bucket_uri,
                                    object_name='bar/qux',
                                    contents=object_contents)

    # Rotate subdir keys to TEST_ENCRYPTION_KEY3.
    boto_config_for_test = [
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)]

    self.AssertNObjectsInBucket(bucket_uri, 4)

    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
                              return_stderr=True)
      self.assertIn('Rotating', stderr)  # Object 2.
      self.assertIn('Skipping %s' % suri(object_uri3), stderr)
      self.assertIn('Encrypting', stderr)  # Object 4.

    # First subdir should be unaffected.
    self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)

    for object_uri_str in (suri(object_uri2), suri(object_uri3),
                           suri(object_uri4)):
      self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3)

    # Remove encryption in subdir.
    boto_config_for_test2 = [
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]

    with SetBotoConfigForTest(boto_config_for_test2):
      stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir],
                              return_stderr=True)
      self.assertIn('Decrypting', stderr)

    # First subdir should be unaffected.
    self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1)

    for object_uri_str in (suri(object_uri2), suri(object_uri3),
                           suri(object_uri4)):
      self.AssertObjectUnencrypted(object_uri_str)
Beispiel #21
0
  def test_turning_on(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('XML API has no concept of Bucket Policy Only')

    bucket_uri = self.CreateBucket()
    self.RunGsUtil(self._set_bpo_cmd + ['on', suri(bucket_uri)])

    self._AssertEnabled(bucket_uri, True)
Beispiel #22
0
  def test_rewrite_with_no_value_for_minus_s(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'],
                            return_stderr=True, expected_status=1)

    self.assertIn('CommandException', stderr)
    self.assertIn('expects at least one URL', stderr)
  def test_rewrite_with_no_value_for_minus_s(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'],
                            return_stderr=True, expected_status=1)

    self.assertIn('CommandException', stderr)
    self.assertIn('expects at least one URL', stderr)
Beispiel #24
0
  def testRetryableErrorMediaCollection(self):
    """Tests that retryable errors are collected on JSON media operations."""
    # Retryable errors will only be collected with the JSON API.
    if self.test_api != ApiSelector.JSON:
      return unittest.skip('Retryable errors are only collected in JSON')

    boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB))]
    bucket_uri = self.CreateBucket()
    # For the resumable upload exception, we need to ensure at least one
    # callback occurs.
    halt_size = START_CALLBACK_PER_BYTES * 2
    fpath = self.CreateTempFile(contents='a' * halt_size)

    # Test that the retry function for data transfers catches and logs an error.
    test_callback_file = self.CreateTempFile(contents=pickle.dumps(
        _ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError,
                                     ('unused', 'unused', 'unused'))))
    with SetBotoConfigForTest(boto_config_for_test):
      metrics_list = self._RunGsUtilWithAnalyticsOutput(
          ['cp', '--testcallbackfile', test_callback_file,
           fpath, suri(bucket_uri)])
      self._CheckParameterValue('Event Category',
                                metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
      self._CheckParameterValue('Event Action', 'BadStatusCodeError',
                                metrics_list)
      self._CheckParameterValue('Retryable Errors', '1', metrics_list)
      self._CheckParameterValue('Num Retryable Service Errors', '1',
                                metrics_list)

    # Test that the ResumableUploadStartOverException in copy_helper is caught.
    test_callback_file = self.CreateTempFile(
        contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
    with SetBotoConfigForTest(boto_config_for_test):
      metrics_list = self._RunGsUtilWithAnalyticsOutput(
          ['cp', '--testcallbackfile', test_callback_file,
           fpath, suri(bucket_uri)])
      self._CheckParameterValue(
          'Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
      self._CheckParameterValue(
          'Event Action', 'ResumableUploadStartOverException', metrics_list)
      self._CheckParameterValue('Retryable Errors', '1', metrics_list)
      self._CheckParameterValue(
          'Num Retryable Service Errors', '1', metrics_list)

    # Test retryable error collection in a multithread/multiprocess situation.
    test_callback_file = self.CreateTempFile(
        contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
    with SetBotoConfigForTest(boto_config_for_test):
      metrics_list = self._RunGsUtilWithAnalyticsOutput(
          ['-m', 'cp', '--testcallbackfile',
           test_callback_file, fpath, suri(bucket_uri)])
      self._CheckParameterValue('Event Category',
                                metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
      self._CheckParameterValue(
          'Event Action', 'ResumableUploadStartOverException', metrics_list)
      self._CheckParameterValue('Retryable Errors', '1', metrics_list)
      self._CheckParameterValue(
          'Num Retryable Service Errors', '1', metrics_list)
  def testRetryableErrorMediaCollection(self):
    """Tests that retryable errors are collected on JSON media operations."""
    # Retryable errors will only be collected with the JSON API.
    if self.test_api != ApiSelector.JSON:
      return unittest.skip('Retryable errors are only collected in JSON')

    boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB))]
    bucket_uri = self.CreateBucket()
    # For the resumable upload exception, we need to ensure at least one
    # callback occurs.
    halt_size = START_CALLBACK_PER_BYTES * 2
    fpath = self.CreateTempFile(contents='a' * halt_size)

    # Test that the retry function for data transfers catches and logs an error.
    test_callback_file = self.CreateTempFile(contents=pickle.dumps(
        _ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError,
                                     ('unused', 'unused', 'unused'))))
    with SetBotoConfigForTest(boto_config_for_test):
      metrics_list = self._RunGsUtilWithAnalyticsOutput(
          ['cp', '--testcallbackfile', test_callback_file,
           fpath, suri(bucket_uri)])
      self._CheckParameterValue('Event Category',
                                metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
      self._CheckParameterValue('Event Action', 'BadStatusCodeError',
                                metrics_list)
      self._CheckParameterValue('Retryable Errors', '1', metrics_list)
      self._CheckParameterValue('Num Retryable Service Errors', '1',
                                metrics_list)

    # Test that the ResumableUploadStartOverException in copy_helper is caught.
    test_callback_file = self.CreateTempFile(
        contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
    with SetBotoConfigForTest(boto_config_for_test):
      metrics_list = self._RunGsUtilWithAnalyticsOutput(
          ['cp', '--testcallbackfile', test_callback_file,
           fpath, suri(bucket_uri)])
      self._CheckParameterValue(
          'Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
      self._CheckParameterValue(
          'Event Action', 'ResumableUploadStartOverException', metrics_list)
      self._CheckParameterValue('Retryable Errors', '1', metrics_list)
      self._CheckParameterValue(
          'Num Retryable Service Errors', '1', metrics_list)

    # Test retryable error collection in a multithread/multiprocess situation.
    test_callback_file = self.CreateTempFile(
        contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404)))
    with SetBotoConfigForTest(boto_config_for_test):
      metrics_list = self._RunGsUtilWithAnalyticsOutput(
          ['-m', 'cp', '--testcallbackfile',
           test_callback_file, fpath, suri(bucket_uri)])
      self._CheckParameterValue('Event Category',
                                metrics._GA_ERRORRETRY_CATEGORY, metrics_list)
      self._CheckParameterValue(
          'Event Action', 'ResumableUploadStartOverException', metrics_list)
      self._CheckParameterValue('Retryable Errors', '1', metrics_list)
      self._CheckParameterValue(
          'Num Retryable Service Errors', '1', metrics_list)
Beispiel #26
0
  def test_turning_off_on_enabled_buckets(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('XML API has no concept of Bucket Policy Only')
    bucket_uri = self.CreateBucket(bucket_policy_only=True,
                                   prefer_json_api=True)
    self._AssertEnabled(bucket_uri, True)

    self.RunGsUtil(self._set_bpo_cmd + ['off', suri(bucket_uri)])
    self._AssertEnabled(bucket_uri, False)
    def test_turning_on(self):
        if self.test_api == ApiSelector.XML:
            return unittest.skip(
                'XML API has no concept of Uniform bucket-level access')

        bucket_uri = self.CreateBucket()
        self.RunGsUtil(self._set_ubla_cmd + ['on', suri(bucket_uri)])

        self._AssertEnabled(bucket_uri, True)
Beispiel #28
0
  def test_set_valid_acl_bucket(self):
    """Ensures that valid canned and XML ACLs work with get/set."""
    if self._ServiceAccountCredentialsPresent():
      # See comments in _ServiceAccountCredentialsPresent
      unittest.skip('Canned ACLs orphan service account permissions.')
    bucket_uri = suri(self.CreateBucket())
    acl_string = self.RunGsUtil(self._get_acl_prefix + [bucket_uri],
                                return_stdout=True)
    inpath = self.CreateTempFile(contents=acl_string)
    self.RunGsUtil(self._set_acl_prefix + ['public-read', bucket_uri])
    acl_string2 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri],
                                 return_stdout=True)
    self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri])
    acl_string3 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri],
                                 return_stdout=True)

    self.assertNotEqual(acl_string, acl_string2)
    self.assertEqual(acl_string, acl_string3)
    def test_set_lifecycle_wildcard(self):
        """Tests setting lifecycle with a wildcarded bucket URI."""
        if self.test_api == ApiSelector.XML:
            # This test lists buckets with wildcards, but it is possible that another
            # test being run in parallel (in the same project) deletes a bucket after
            # it is listed in this test. This causes the subsequent XML metadata get
            # for the lifecycle configuration to fail on that just-deleted bucket,
            # even though that bucket is not used directly in this test.
            return unittest.skip(
                'XML wildcard behavior can cause test to flake '
                'if a bucket in the same project is deleted '
                'during execution.')

        random_prefix = self.MakeRandomTestString()
        bucket1_name = self.MakeTempName('bucket', prefix=random_prefix)
        bucket2_name = self.MakeTempName('bucket', prefix=random_prefix)
        bucket1_uri = self.CreateBucket(bucket_name=bucket1_name)
        bucket2_uri = self.CreateBucket(bucket_name=bucket2_name)
        # This just double checks that the common prefix of the two buckets is what
        # we think it should be (based on implementation detail of CreateBucket).
        # We want to be careful when setting a wildcard on buckets to make sure we
        # don't step outside the test buckets to affect other buckets.
        common_prefix = posixpath.commonprefix(
            [suri(bucket1_uri), suri(bucket2_uri)])
        self.assertTrue(
            common_prefix.startswith(
                'gs://%sgsutil-test-test-set-lifecycle-wildcard-' %
                random_prefix))
        wildcard = '%s*' % common_prefix

        fpath = self.CreateTempFile(
            contents=self.lifecycle_doc.encode('ascii'))

        # Use @Retry as hedge against bucket listing eventual consistency.
        expected = set([
            'Setting lifecycle configuration on %s/...' % suri(bucket1_uri),
            'Setting lifecycle configuration on %s/...' % suri(bucket2_uri)
        ])
        actual = set()

        @Retry(AssertionError, tries=3, timeout_secs=1)
        def _Check1():
            stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard],
                                    return_stderr=True)
            actual.update(stderr.splitlines())
            self.assertEqual(expected, actual)
            self.assertEqual(stderr.count('Setting lifecycle configuration'),
                             2)

        _Check1()

        stdout = self.RunGsUtil(
            ['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True)
        self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
        stdout = self.RunGsUtil(
            ['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True)
        self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
Beispiel #30
0
 def test_uppercase_header(self):
   """Tests setting custom metadata with an uppercase value."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('XML header keys are case-insensitive.')
   objuri = self.CreateObject(contents='foo')
   self.RunGsUtil([
       'setmeta', '-h', 'x-%s-meta-CaSe:SeNsItIvE' % self.provider_custom_meta,
       suri(objuri)])
   stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True)
   self.assertRegexpMatches(stdout, ur'CaSe:\s+SeNsItIvE')
Beispiel #31
0
    def test_repo_matches_manifest(self):
        """Ensure any new top-level files are present in the manifest."""
        p = subprocess.Popen(['git', 'branch'],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        p.communicate()
        if p.returncode != 0:
            unittest.skip('Test only runs from git repository.')
            return

        manifest_lines = ['gslib', 'third_party', 'MANIFEST.in']

        manifest_file = os.path.join(GSUTIL_DIR, 'MANIFEST.in')

        try:
            with open(manifest_file, 'r') as fp:
                for line in fp:
                    if line.startswith('include '):
                        manifest_lines.append(line.split()[-1])
        except IOError:
            # If manifest file is not readable (example: Travis CI), skip the test.
            unittest.skip('Test must be able to read manifest file.')
            return

        p = subprocess.Popen(['git', 'ls-tree', '--name-only', 'HEAD'],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        (stdout, _) = p.communicate()
        git_top_level_files = stdout.splitlines()

        for filename in git_top_level_files:
            if filename.endswith('.pyc'):
                # Ignore compiled code.
                continue
            if filename in ('.gitmodules', '.gitignore', '.travis.yml'):
                # We explicitly drop these files when building the gsutil tarball.
                # If we add any other files to this list, the tarball script must
                # also be updated or we could break the gsutil update command.
                continue
            if filename not in manifest_lines:
                self.fail(
                    'Found file %s not present in MANIFEST.in, which would '
                    'break gsutil update.' % filename)
 def test_uppercase_header(self):
   """Tests setting custom metadata with an uppercase value."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('XML header keys are case-insensitive.')
   objuri = self.CreateObject(contents='foo')
   self.RunGsUtil([
       'setmeta', '-h', 'x-%s-meta-CaSe:SeNsItIvE' % self.provider_custom_meta,
       suri(objuri)])
   stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True)
   self.assertRegexpMatches(stdout, ur'CaSe:\s+SeNsItIvE')
Beispiel #33
0
 def test_rewrite_generation_url(self):
   """Tests that rewrite fails on a URL that includes a generation."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   generation = object_uri.generation
   stderr = self.RunGsUtil(
       ['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)],
       return_stderr=True, expected_status=1)
   self.assertIn('"rewrite" called on URL with generation', stderr)
 def test_rewrite_generation_url(self):
   """Tests that rewrite fails on a URL that includes a generation."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   generation = object_uri.generation
   stderr = self.RunGsUtil(
       ['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)],
       return_stderr=True, expected_status=1)
   self.assertIn('"rewrite" called on URL with generation', stderr)
    def test_turning_off_on_enabled_buckets(self):
        if self.test_api == ApiSelector.XML:
            return unittest.skip(
                'XML API has no concept of Uniform bucket-level access')
        # TODO(mynameisrafe): Replace bucket_policy_only with uniform_bucket_level_access  when the property is live.
        bucket_uri = self.CreateBucket(bucket_policy_only=True,
                                       prefer_json_api=True)
        self._AssertEnabled(bucket_uri, True)

        self.RunGsUtil(self._set_ubla_cmd + ['off', suri(bucket_uri)])
        self._AssertEnabled(bucket_uri, False)
Beispiel #36
0
  def testRetryableErrorMetadataCollection(self):
    """Tests that retryable errors are collected on JSON metadata operations."""
    # Retryable errors will only be collected with the JSON API.
    if self.test_api != ApiSelector.JSON:
      return unittest.skip('Retryable errors are only collected in JSON')

    bucket_uri = self.CreateBucket()
    object_uri = self.CreateObject(bucket_uri=bucket_uri,
                                   object_name='foo', contents=b'bar')
    # Set the command name to rsync in order to collect PerformanceSummary info.
    self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync'
    # Generate a JSON API instance to test with, because the RunGsUtil method
    # may use the XML API.
    gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
                            RetryableErrorsQueue(), self.default_provider)
    # Don't wait for too many retries or for long periods between retries to
    # avoid long tests.
    gsutil_api.api_client.num_retries = 2
    gsutil_api.api_client.max_retry_wait = 1

    # Throw an error when transferring metadata.
    key = object_uri.get_key()
    src_obj_metadata = apitools_messages.Object(name=key.name,
                                                bucket=key.bucket.name,
                                                contentType=key.content_type)
    dst_obj_metadata = apitools_messages.Object(
        bucket=src_obj_metadata.bucket,
        name=self.MakeTempName('object'),
        contentType=src_obj_metadata.contentType)
    with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
                           side_effect=socket.error()):
      _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata,
                        dst_obj_metadata)
    if six.PY2:
      self.assertEqual(self.collector.retryable_errors['SocketError'], 1)
    else:
      # In PY3, socket.* errors are deprecated aliases for OSError
      self.assertEqual(self.collector.retryable_errors['OSError'], 1)

    # Throw an error when removing a bucket.
    with mock.patch.object(
        http_wrapper, '_MakeRequestNoRetry',
        side_effect=apitools_exceptions.HttpError('unused', 'unused',
                                                  'unused')):
      _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name,
                        object_uri.object_name)
    self.assertEqual(self.collector.retryable_errors['HttpError'], 1)

    # Check that the number of each kind of retryable error was logged.
    self.assertEqual(
        self.collector.perf_sum_params.num_retryable_network_errors, 1)
    self.assertEqual(
        self.collector.perf_sum_params.num_retryable_service_errors, 1)
  def test_set_lifecycle_wildcard(self):
    """Tests setting lifecycle with a wildcarded bucket URI."""
    if self.test_api == ApiSelector.XML:
      # This test lists buckets with wildcards, but it is possible that another
      # test being run in parallel (in the same project) deletes a bucket after
      # it is listed in this test. This causes the subsequent XML metadata get
      # for the lifecycle configuration to fail on that just-deleted bucket,
      # even though that bucket is not used directly in this test.
      return unittest.skip('XML wildcard behavior can cause test to flake '
                           'if a bucket in the same project is deleted '
                           'during execution.')

    random_prefix = self.MakeRandomTestString()
    bucket1_name = self.MakeTempName('bucket', prefix=random_prefix)
    bucket2_name = self.MakeTempName('bucket', prefix=random_prefix)
    bucket1_uri = self.CreateBucket(bucket_name=bucket1_name)
    bucket2_uri = self.CreateBucket(bucket_name=bucket2_name)
    # This just double checks that the common prefix of the two buckets is what
    # we think it should be (based on implementation detail of CreateBucket).
    # We want to be careful when setting a wildcard on buckets to make sure we
    # don't step outside the test buckets to affect other buckets.
    common_prefix = posixpath.commonprefix(
        [suri(bucket1_uri), suri(bucket2_uri)])
    self.assertTrue(
        common_prefix.startswith(
            'gs://%sgsutil-test-test-set-lifecycle-wildcard-' % random_prefix))
    wildcard = '%s*' % common_prefix

    fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii'))

    # Use @Retry as hedge against bucket listing eventual consistency.
    expected = set([
        'Setting lifecycle configuration on %s/...' % suri(bucket1_uri),
        'Setting lifecycle configuration on %s/...' % suri(bucket2_uri)
    ])
    actual = set()

    @Retry(AssertionError, tries=3, timeout_secs=1)
    def _Check1():
      stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard],
                              return_stderr=True)
      actual.update(stderr.splitlines())
      self.assertEqual(expected, actual)
      self.assertEqual(stderr.count('Setting lifecycle configuration'), 2)

    _Check1()

    stdout = self.RunGsUtil(
        ['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True)
    self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
    stdout = self.RunGsUtil(
        ['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True)
    self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
Beispiel #38
0
  def test_rewrite_with_same_key_and_storage_class_is_skipped(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='foo',
                                   encryption_key=TEST_ENCRYPTION_KEY1,
                                   storage_class='standard')

    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', '-s', 'standard', suri(object_uri)],
          return_stderr=True)
    self.assertIn('Skipping %s' % suri(object_uri), stderr)
Beispiel #39
0
  def test_mv_early_deletion_warning(self):
    """Tests that mv on a recent nearline object warns about early deletion."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip('boto does not return object storage class')

    bucket_uri = self.CreateBucket(storage_class='NEARLINE')
    object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='obj')
    stderr = self.RunGsUtil(['mv', suri(object_uri), suri(bucket_uri, 'foo')],
                            return_stderr=True)
    self.assertIn(
        'Warning: moving nearline object %s may incur an early deletion '
        'charge, because the original object is less than 30 days old '
        'according to the local system time.' % suri(object_uri), stderr)
Beispiel #40
0
  def test_mv_early_deletion_warning(self):
    """Tests that mv on a recent nearline object warns about early deletion."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip('boto does not return object storage class')

    bucket_uri = self.CreateBucket(storage_class='NEARLINE')
    object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='obj')
    stderr = self.RunGsUtil(['mv', suri(object_uri), suri(bucket_uri, 'foo')],
                            return_stderr=True)
    self.assertIn(
        'Warning: moving nearline object %s may incur an early deletion '
        'charge, because the original object is less than 30 days old '
        'according to the local system time.' % suri(object_uri), stderr)
  def test_rewrite_with_same_key_and_storage_class_is_skipped(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='foo',
                                   encryption_key=TEST_ENCRYPTION_KEY1,
                                   storage_class='standard')

    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', '-s', 'standard', suri(object_uri)],
          return_stderr=True)
    self.assertIn('Skipping %s' % suri(object_uri), stderr)
  def test_rewrite_with_only_storage_class_change(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar')

    # Change storage class to nearline.
    stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)],
                            return_stderr=True)
    self.assertIn('Rewriting', stderr)

    stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
    self.assertRegexpMatchesWithFlags(
        stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,
        msg=('Storage class appears not to have been changed.'))
  def test_rewrite_stdin_args(self):
    """Tests rewrite with arguments supplied on stdin."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar',
                                   encryption_key=TEST_ENCRYPTION_KEY1)
    stdin_arg = suri(object_uri)

    boto_config_for_test = [
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg)
    self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2)
 def test_rewrite_missing_decryption_key(self):
   """Tests that rewrite fails when no decryption key matches."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(object_name='foo', contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   boto_config_for_test = [
       ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
       ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]
   with SetBotoConfigForTest(boto_config_for_test):
     stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
                             return_stderr=True, expected_status=1)
     self.assertIn('No decryption key matches object %s' %
                   suri(object_uri), stderr)
Beispiel #45
0
  def test_rewrite_with_only_storage_class_change(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar')

    # Change storage class to nearline.
    stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)],
                            return_stderr=True)
    self.assertIn('Rewriting', stderr)

    stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
    self.assertRegexpMatchesWithFlags(
        stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,
        msg=('Storage class appears not to have been changed.'))
Beispiel #46
0
 def test_rewrite_missing_decryption_key(self):
   """Tests that rewrite fails when no decryption key matches."""
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(object_name='foo', contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   boto_config_for_test = [
       ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
       ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)]
   with SetBotoConfigForTest(boto_config_for_test):
     stderr = self.RunGsUtil(['rewrite', '-k', suri(object_uri)],
                             return_stderr=True, expected_status=1)
     self.assertIn('No decryption key matches object %s' %
                   suri(object_uri), stderr)
Beispiel #47
0
  def test_rewrite_stdin_args(self):
    """Tests rewrite with arguments supplied on stdin."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar',
                                   encryption_key=TEST_ENCRYPTION_KEY1)
    stdin_arg = suri(object_uri)

    boto_config_for_test = [
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg)
    self.AssertObjectUsesEncryptionKey(stdin_arg, TEST_ENCRYPTION_KEY2)
Beispiel #48
0
  def testRetryableErrorMetadataCollection(self):
    """Tests that retryable errors are collected on JSON metadata operations."""
    # Retryable errors will only be collected with the JSON API.
    if self.test_api != ApiSelector.JSON:
      return unittest.skip('Retryable errors are only collected in JSON')

    bucket_uri = self.CreateBucket()
    object_uri = self.CreateObject(bucket_uri=bucket_uri,
                                   object_name='foo', contents='bar')
    # Set the command name to rsync in order to collect PerformanceSummary info.
    self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync'
    # Generate a JSON API instance to test with, because the RunGsUtil method
    # may use the XML API.
    gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
                            RetryableErrorsQueue(), self.default_provider)
    # Don't wait for too many retries or for long periods between retries to
    # avoid long tests.
    gsutil_api.api_client.num_retries = 2
    gsutil_api.api_client.max_retry_wait = 1

    # Throw an error when transferring metadata.
    key = object_uri.get_key()
    src_obj_metadata = apitools_messages.Object(name=key.name,
                                                bucket=key.bucket.name,
                                                contentType=key.content_type)
    dst_obj_metadata = apitools_messages.Object(
        bucket=src_obj_metadata.bucket,
        name=self.MakeTempName('object'),
        contentType=src_obj_metadata.contentType)
    with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
                           side_effect=socket.error()):
      _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata,
                        dst_obj_metadata)
    self.assertEqual(self.collector.retryable_errors['SocketError'], 1)

    # Throw an error when removing a bucket.
    with mock.patch.object(
        http_wrapper, '_MakeRequestNoRetry',
        side_effect=apitools_exceptions.HttpError('unused', 'unused',
                                                  'unused')):
      _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name,
                        object_uri.object_name)
    self.assertEqual(self.collector.retryable_errors['HttpError'], 1)

    # Check that the number of each kind of retryable error was logged.
    self.assertEqual(
        self.collector.perf_sum_params.num_retryable_network_errors, 1)
    self.assertEqual(
        self.collector.perf_sum_params.num_retryable_service_errors, 1)
 def test_rewrite_seek_ahead(self):
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   # Remove encryption
   boto_config_for_test = [
       ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1),
       ('GSUtil', 'task_estimation_threshold', '1'),
       ('GSUtil', 'task_estimation_force', 'True')]
   with SetBotoConfigForTest(boto_config_for_test):
     stderr = self.RunGsUtil(['-m', 'rewrite', '-k', suri(object_uri)],
                             return_stderr=True)
     self.assertIn(
         'Estimated work for this command: objects: 1, total size: 3', stderr)
Beispiel #50
0
 def test_rewrite_seek_ahead(self):
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   # Remove encryption
   boto_config_for_test = [
       ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1),
       ('GSUtil', 'task_estimation_threshold', '1'),
       ('GSUtil', 'task_estimation_force', 'True')]
   with SetBotoConfigForTest(boto_config_for_test):
     stderr = self.RunGsUtil(['-m', 'rewrite', '-k', suri(object_uri)],
                             return_stderr=True)
     self.assertIn(
         'Estimated work for this command: objects: 1, total size: 3', stderr)
    def test_stat_encrypted_object_wildcard(self):
        """Tests stat command with a mix of encrypted and unencrypted objects."""
        if self.test_api == ApiSelector.XML:
            return unittest.skip(
                'gsutil does not support encryption with the XML API')
        bucket_uri = self.CreateBucket()
        object1_uri = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='foo1',
                                        contents=TEST_ENCRYPTION_CONTENT1,
                                        encryption_key=TEST_ENCRYPTION_KEY1)
        object2_uri = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='foo2',
                                        contents=TEST_ENCRYPTION_CONTENT2,
                                        encryption_key=TEST_ENCRYPTION_KEY2)
        object3_uri = self.CreateObject(bucket_uri=bucket_uri,
                                        object_name='foo3',
                                        contents=TEST_ENCRYPTION_CONTENT3)

        stat_string = suri(object1_uri)[:-2] + '*'

        # Stat 3 objects, two encrypted each with a different key, and one
        # unencrypted. Should result in two unencrypted listing and one encrypted
        # listing.
        with SetBotoConfigForTest([('GSUtil', 'encryption_key',
                                    TEST_ENCRYPTION_KEY1)]):
            # Use @Retry as hedge against bucket listing eventual consistency.
            @Retry(AssertionError, tries=3, timeout_secs=1)
            def _StatExpectMixed():
                """Runs stat and validates output."""
                stdout, stderr = self.RunGsUtil(['stat', stat_string],
                                                return_stdout=True,
                                                return_stderr=True)
                self.assertIn(suri(object1_uri), stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
                self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'),
                              stdout)
                self.assertIn(suri(object2_uri), stdout)
                self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout)
                self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout)
                self.assertIn('encrypted', stdout)
                self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64.decode('ascii'),
                              stdout)
                self.assertIn(suri(object3_uri), stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout)

            _StatExpectMixed()
Beispiel #52
0
 def test_rewrite_with_nonkey_transform_works_when_key_is_unchanged(self):
   # Tests that when a valid transformation flag aside from "-k" is supplied,
   # the "-k" flag is not supplied, and the encryption key previously used to
   # encrypt the target object matches the encryption_key in the user's boto
   # config file (via hash comparison), that the rewrite command properly
   # passes the same tuple for decryption and encryption, in addition to
   # performing the other desired transformations.
   if self.test_api == ApiSelector.XML:
     return unittest.skip('Rewrite API is only supported in JSON.')
   object_uri = self.CreateObject(contents='bar',
                                  encryption_key=TEST_ENCRYPTION_KEY1)
   boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
   with SetBotoConfigForTest(boto_config_for_test):
     stderr = self.RunGsUtil(
         ['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True)
     self.assertIn('Rewriting', stderr)
    def test_stat_encrypted_object_wildcard(self):
        """Tests stat command with a mix of encrypted and unencrypted objects."""
        if self.test_api == ApiSelector.XML:
            return unittest.skip("gsutil does not support encryption with the XML API")
        bucket_uri = self.CreateBucket()
        object1_uri = self.CreateObject(
            bucket_uri=bucket_uri,
            object_name="foo1",
            contents=TEST_ENCRYPTION_CONTENT1,
            encryption_key=TEST_ENCRYPTION_KEY1,
        )
        object2_uri = self.CreateObject(
            bucket_uri=bucket_uri,
            object_name="foo2",
            contents=TEST_ENCRYPTION_CONTENT2,
            encryption_key=TEST_ENCRYPTION_KEY2,
        )
        object3_uri = self.CreateObject(bucket_uri=bucket_uri, object_name="foo3", contents=TEST_ENCRYPTION_CONTENT3)

        stat_string = suri(object1_uri)[:-2] + "*"

        # Stat 3 objects, two encrypted each with a different key, and one
        # unencrypted. Should result in two unencrypted listing and one encrypted
        # listing.
        with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]):
            # Use @Retry as hedge against bucket listing eventual consistency.
            @Retry(AssertionError, tries=3, timeout_secs=1)
            def _StatExpectMixed():
                """Runs stat and validates output."""
                stdout = self.RunGsUtil(["stat", stat_string], return_stdout=True)
                self.assertIn(suri(object1_uri), stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
                self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
                self.assertIn(suri(object2_uri), stdout)
                self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout)
                self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout)
                self.assertIn("encrypted", stdout)
                self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64, stdout)
                self.assertIn(suri(object3_uri), stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout)
                self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout)

            _StatExpectMixed()
Beispiel #54
0
  def test_cat_encrypted_object(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip(
          'gsutil does not support encryption with the XML API')
    object_contents = '0123456789'
    object_uri = self.CreateObject(object_name='foo', contents=object_contents,
                                   encryption_key=TEST_ENCRYPTION_KEY1)

    stderr = self.RunGsUtil(['cat', suri(object_uri)], expected_status=1,
                            return_stderr=True)
    self.assertIn('No decryption key matches object', stderr)

    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]

    with SetBotoConfigForTest(boto_config_for_test):
      stdout = self.RunGsUtil(['cat', suri(object_uri)], return_stdout=True)
      self.assertEqual(stdout, object_contents)
      stdout = self.RunGsUtil(['cat', '-r 1-3', suri(object_uri)],
                              return_stdout=True)
      self.assertEqual(stdout, '123')
  def test_rewrite_to_kms_then_unencrypted(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    key_fqn = self.authorize_project_to_use_testing_kms_key()
    object_uri = self.CreateObject(contents='foo')

    boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', suri(object_uri)], return_stderr=True)
    self.assertIn('Encrypting', stderr)
    self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)

    # Rewrite back to unencrypted and make sure no KMS key was used.
    boto_config_for_test = [('GSUtil', 'encryption_key', None)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', suri(object_uri)], return_stderr=True)
    self.assertIn('Decrypting', stderr)
    self.AssertObjectUnencrypted(suri(object_uri))
  def test_rewrite_to_kms_then_csek(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    key_fqn = self.authorize_project_to_use_testing_kms_key()
    object_uri = self.CreateObject(contents='foo')

    boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', suri(object_uri)], return_stderr=True)
    self.assertIn('Encrypting', stderr)
    self.AssertObjectUsesCMEK(suri(object_uri), key_fqn)

    # Rewrite from CMEK to CSEK encryption.
    boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-k', suri(object_uri)], return_stderr=True)
    self.assertIn('Rotating', stderr)
    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1)
Beispiel #57
0
  def test_rewrite_overwrite_acl(self):
    """Tests rewrite with the -O flag."""
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar',
                                   encryption_key=TEST_ENCRYPTION_KEY1)
    self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)])
    stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)],
                            return_stdout=True)
    self.assertIn('allUsers', stdout)

    boto_config_for_test = [
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      self.RunGsUtil(['rewrite', '-k', '-O', suri(object_uri)])
    self.AssertObjectUsesEncryptionKey(suri(object_uri), TEST_ENCRYPTION_KEY2)
    stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)],
                            return_stdout=True)
    self.assertNotIn('allUsers', stdout)
  def test_rewrite_key_rotation_with_storage_class_change(self):
    if self.test_api == ApiSelector.XML:
      return unittest.skip('Rewrite API is only supported in JSON.')
    object_uri = self.CreateObject(contents='bar',
                                   encryption_key=TEST_ENCRYPTION_KEY1)

    # Rotate key and change storage class to nearline.
    boto_config_for_test = [
        ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
        ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
    with SetBotoConfigForTest(boto_config_for_test):
      stderr = self.RunGsUtil(
          ['rewrite', '-s', 'nearline', '-k', suri(object_uri)],
          return_stderr=True)
      self.assertIn('Rotating', stderr)

    self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2)
    stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True)
    self.assertRegexpMatchesWithFlags(
        stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE,
        msg=('Storage class appears not to have been changed.'))
Beispiel #59
0
  def testRetryableErrorMetadataCollection(self):
    """Tests that retryable errors are collected on JSON metadata operations."""
    # Retryable errors will only be collected with the JSON API.
    if self.test_api != ApiSelector.JSON:
      return unittest.skip('Retryable errors are only collected in JSON')

    bucket_uri = self.CreateBucket()
    object_uri = self.CreateObject(bucket_uri=bucket_uri,
                                   object_name='foo',
                                   contents='bar')
    # Generate a JSON API instance because the RunGsUtil method uses the XML
    # API.
    gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
                            RetryableErrorsQueue(), self.default_provider)
    # Don't wait for too many retries or for long periods between retries to
    # avoid long tests.
    gsutil_api.api_client.num_retries = 2
    gsutil_api.api_client.max_retry_wait = 1

    # Throw an error when transferring metadata.
    key = object_uri.get_key()
    src_obj_metadata = apitools_messages.Object(name=key.name,
                                                bucket=key.bucket.name,
                                                contentType=key.content_type)
    dst_obj_metadata = apitools_messages.Object(
        bucket=src_obj_metadata.bucket,
        name=self.MakeTempName('object'),
        contentType=src_obj_metadata.contentType)
    with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
                           side_effect=socket.error()):
      _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata,
                        dst_obj_metadata)
    self.assertEqual(self.collector.retryable_errors['SocketError'], 1)

    # Throw an error when removing a bucket.
    with mock.patch.object(http_wrapper, '_MakeRequestNoRetry',
                           side_effect=ValueError()):
      _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name,
                        object_uri.object_name)
    self.assertEqual(self.collector.retryable_errors['ValueError'], 1)