Ejemplo n.º 1
0
  def test_RewriteTrackerFile(self):
    """Tests Rewrite tracker file functions."""
    tracker_file_name = GetRewriteTrackerFilePath('bk1', 'obj1', 'bk2', 'obj2',
                                                  self.test_api)
    # Should succeed regardless of whether it exists.
    DeleteTrackerFile(tracker_file_name)
    src_obj_metadata = apitools_messages.Object(
        bucket='bk1', name='obj1', etag='etag1', md5Hash='12345')
    src_obj2_metadata = apitools_messages.Object(
        bucket='bk1', name='obj1', etag='etag2', md5Hash='67890')
    dst_obj_metadata = apitools_messages.Object(
        bucket='bk2', name='obj2')
    rewrite_token = 'token1'
    self.assertIsNone(ReadRewriteTrackerFile(tracker_file_name,
                                             src_obj_metadata))
    rewrite_params_hash = HashRewriteParameters(
        src_obj_metadata, dst_obj_metadata, 'full')
    WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash,
                            rewrite_token)
    self.assertEqual(
        ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash),
        rewrite_token)

    # Tracker file for an updated source object (with non-matching etag/md5)
    # should return None.
    rewrite_params_hash2 = HashRewriteParameters(
        src_obj2_metadata, dst_obj_metadata, 'full')

    self.assertIsNone(ReadRewriteTrackerFile(tracker_file_name,
                                             rewrite_params_hash2))
    DeleteTrackerFile(tracker_file_name)
    
Ejemplo n.º 2
0
    def _test_rewrite_resume_or_restart(self,
                                        initial_dec_key,
                                        initial_enc_key,
                                        new_dec_key=None,
                                        new_enc_key=None):
        """Tests that the rewrite command restarts if the object's key changed.

    Args:
      initial_dec_key: Initial key the object is encrypted with, used as
          decryption key in the first rewrite call.
      initial_enc_key: Initial encryption key to rewrite the object with,
          used as encryption key in the first rewrite call.
      new_dec_key: Decryption key for the second rewrite call; if specified,
          object will be overwritten with a new encryption key in between
          the first and second rewrite calls, and this key will be used for
          the second rewrite call.
      new_enc_key: Encryption key for the second rewrite call; if specified,
          this key will be used for the second rewrite call, otherwise the
          initial key will be used.

    Returns:
      None
    """
        if self.test_api == ApiSelector.XML:
            return unittest.skip('Rewrite API is only supported in JSON.')
        bucket_uri = self.CreateBucket()
        # maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we
        # need 2 response from the service: 1 success, 1 failure prior to
        # completion.
        object_uri = self.CreateObject(bucket_uri=bucket_uri,
                                       object_name='foo',
                                       contents=('12' * ONE_MIB) + 'bar',
                                       prefer_json_api=True,
                                       encryption_key=initial_dec_key)
        gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
                                DiscardMessagesQueue(), self.default_provider)
        with SetBotoConfigForTest([('GSUtil', 'decryption_key1',
                                    initial_dec_key)]):
            src_obj_metadata = gsutil_api.GetObjectMetadata(
                object_uri.bucket_name,
                object_uri.object_name,
                provider=self.default_provider,
                fields=['bucket', 'contentType', 'etag', 'name'])
        dst_obj_metadata = src_obj_metadata
        tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket,
                                                      src_obj_metadata.name,
                                                      dst_obj_metadata.bucket,
                                                      dst_obj_metadata.name,
                                                      self.test_api)
        decryption_tuple = CryptoKeyWrapperFromKey(initial_dec_key)
        decryption_tuple2 = CryptoKeyWrapperFromKey(new_dec_key
                                                    or initial_dec_key)
        encryption_tuple = CryptoKeyWrapperFromKey(initial_enc_key)
        encryption_tuple2 = CryptoKeyWrapperFromKey(new_enc_key
                                                    or initial_enc_key)

        try:
            try:
                gsutil_api.CopyObject(
                    src_obj_metadata,
                    dst_obj_metadata,
                    progress_callback=HaltingRewriteCallbackHandler(ONE_MIB *
                                                                    2).call,
                    max_bytes_per_call=ONE_MIB,
                    decryption_tuple=decryption_tuple,
                    encryption_tuple=encryption_tuple)
                self.fail('Expected RewriteHaltException.')
            except RewriteHaltException:
                pass

            # Tracker file should be left over.
            self.assertTrue(os.path.exists(tracker_file_name))

            if new_dec_key:
                # Recreate the object with a different encryption key.
                self.CreateObject(bucket_uri=bucket_uri,
                                  object_name='foo',
                                  contents=('12' * ONE_MIB) + 'bar',
                                  prefer_json_api=True,
                                  encryption_key=new_dec_key,
                                  gs_idempotent_generation=urigen(object_uri))

            with SetBotoConfigForTest([('GSUtil', 'decryption_key1',
                                        new_dec_key or initial_dec_key)]):
                original_md5 = gsutil_api.GetObjectMetadata(
                    src_obj_metadata.bucket,
                    src_obj_metadata.name,
                    fields=['customerEncryption', 'md5Hash']).md5Hash

            if new_dec_key or new_enc_key:
                # Keys changed, rewrite should be restarted.
                progress_callback = EnsureRewriteRestartCallbackHandler(
                    ONE_MIB).call
            else:
                # Keys are the same, rewrite should be resumed.
                progress_callback = EnsureRewriteResumeCallbackHandler(
                    ONE_MIB * 2).call

            # Now resume. Callback ensures the appropriate resume/restart behavior.
            gsutil_api.CopyObject(src_obj_metadata,
                                  dst_obj_metadata,
                                  progress_callback=progress_callback,
                                  max_bytes_per_call=ONE_MIB,
                                  decryption_tuple=decryption_tuple2,
                                  encryption_tuple=encryption_tuple2)

            # Copy completed; tracker file should be deleted.
            self.assertFalse(os.path.exists(tracker_file_name))

            final_enc_key = new_enc_key or initial_enc_key

            with SetBotoConfigForTest([('GSUtil', 'encryption_key',
                                        final_enc_key)]):
                self.assertEqual(
                    original_md5,
                    gsutil_api.GetObjectMetadata(
                        dst_obj_metadata.bucket,
                        dst_obj_metadata.name,
                        fields=['customerEncryption', 'md5Hash']).md5Hash,
                    'Error: Rewritten object\'s hash doesn\'t match source object.'
                )
        finally:
            # Clean up if something went wrong.
            DeleteTrackerFile(tracker_file_name)