def test_RewriteTrackerFile(self):
"""Tests Rewrite tracker file functions."""
tracker_file_name = GetRewriteTrackerFilePath('bk1', 'obj1', 'bk2', 'obj2',
self.test_api)
# Should succeed regardless of whether it exists.
DeleteTrackerFile(tracker_file_name)
src_obj_metadata = apitools_messages.Object(
bucket='bk1', name='obj1', etag='etag1', md5Hash='12345')
src_obj2_metadata = apitools_messages.Object(
bucket='bk1', name='obj1', etag='etag2', md5Hash='67890')
dst_obj_metadata = apitools_messages.Object(
bucket='bk2', name='obj2')
rewrite_token = 'token1'
self.assertIsNone(ReadRewriteTrackerFile(tracker_file_name,
src_obj_metadata))
rewrite_params_hash = HashRewriteParameters(
src_obj_metadata, dst_obj_metadata, 'full')
WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash,
rewrite_token)
self.assertEqual(
ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash),
rewrite_token)
# Tracker file for an updated source object (with non-matching etag/md5)
# should return None.
rewrite_params_hash2 = HashRewriteParameters(
src_obj2_metadata, dst_obj_metadata, 'full')
self.assertIsNone(ReadRewriteTrackerFile(tracker_file_name,
rewrite_params_hash2))
DeleteTrackerFile(tracker_file_name)
def _test_rewrite_resume_or_restart(self,
initial_dec_key,
initial_enc_key,
new_dec_key=None,
new_enc_key=None):
"""Tests that the rewrite command restarts if the object's key changed.
Args:
initial_dec_key: Initial key the object is encrypted with, used as
decryption key in the first rewrite call.
initial_enc_key: Initial encryption key to rewrite the object with,
used as encryption key in the first rewrite call.
new_dec_key: Decryption key for the second rewrite call; if specified,
object will be overwritten with a new encryption key in between
the first and second rewrite calls, and this key will be used for
the second rewrite call.
new_enc_key: Encryption key for the second rewrite call; if specified,
this key will be used for the second rewrite call, otherwise the
initial key will be used.
Returns:
None
"""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
# maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we
# need 2 response from the service: 1 success, 1 failure prior to
# completion.
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=('12' * ONE_MIB) + 'bar',
prefer_json_api=True,
encryption_key=initial_dec_key)
gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
DiscardMessagesQueue(), self.default_provider)
with SetBotoConfigForTest([('GSUtil', 'decryption_key1',
initial_dec_key)]):
src_obj_metadata = gsutil_api.GetObjectMetadata(
object_uri.bucket_name,
object_uri.object_name,
provider=self.default_provider,
fields=['bucket', 'contentType', 'etag', 'name'])
dst_obj_metadata = src_obj_metadata
tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket,
src_obj_metadata.name,
dst_obj_metadata.bucket,
dst_obj_metadata.name,
self.test_api)
decryption_tuple = CryptoKeyWrapperFromKey(initial_dec_key)
decryption_tuple2 = CryptoKeyWrapperFromKey(new_dec_key
or initial_dec_key)
encryption_tuple = CryptoKeyWrapperFromKey(initial_enc_key)
encryption_tuple2 = CryptoKeyWrapperFromKey(new_enc_key
or initial_enc_key)
try:
try:
gsutil_api.CopyObject(
src_obj_metadata,
dst_obj_metadata,
progress_callback=HaltingRewriteCallbackHandler(ONE_MIB *
2).call,
max_bytes_per_call=ONE_MIB,
decryption_tuple=decryption_tuple,
encryption_tuple=encryption_tuple)
self.fail('Expected RewriteHaltException.')
except RewriteHaltException:
pass
# Tracker file should be left over.
self.assertTrue(os.path.exists(tracker_file_name))
if new_dec_key:
# Recreate the object with a different encryption key.
self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=('12' * ONE_MIB) + 'bar',
prefer_json_api=True,
encryption_key=new_dec_key,
gs_idempotent_generation=urigen(object_uri))
with SetBotoConfigForTest([('GSUtil', 'decryption_key1',
new_dec_key or initial_dec_key)]):
original_md5 = gsutil_api.GetObjectMetadata(
src_obj_metadata.bucket,
src_obj_metadata.name,
fields=['customerEncryption', 'md5Hash']).md5Hash
if new_dec_key or new_enc_key:
# Keys changed, rewrite should be restarted.
progress_callback = EnsureRewriteRestartCallbackHandler(
ONE_MIB).call
else:
# Keys are the same, rewrite should be resumed.
progress_callback = EnsureRewriteResumeCallbackHandler(
ONE_MIB * 2).call
# Now resume. Callback ensures the appropriate resume/restart behavior.
gsutil_api.CopyObject(src_obj_metadata,
dst_obj_metadata,
progress_callback=progress_callback,
max_bytes_per_call=ONE_MIB,
decryption_tuple=decryption_tuple2,
encryption_tuple=encryption_tuple2)
# Copy completed; tracker file should be deleted.
self.assertFalse(os.path.exists(tracker_file_name))
final_enc_key = new_enc_key or initial_enc_key
with SetBotoConfigForTest([('GSUtil', 'encryption_key',
final_enc_key)]):
self.assertEqual(
original_md5,
gsutil_api.GetObjectMetadata(
dst_obj_metadata.bucket,
dst_obj_metadata.name,
fields=['customerEncryption', 'md5Hash']).md5Hash,
'Error: Rewritten object\'s hash doesn\'t match source object.'
)
finally:
# Clean up if something went wrong.
DeleteTrackerFile(tracker_file_name)