def testValidateParallelCompositeTrackerData(self):
tempdir = self.CreateTempDir()
fpath = os.path.join(tempdir, 'foo')
random_prefix = '123'
old_enc_key = '456'
bucket_url = StorageUrlFromString('gs://foo')
objects = [
ObjectFromTracker('obj1', '42'),
ObjectFromTracker('obj2', '314159')
]
WriteParallelUploadTrackerFile(fpath,
random_prefix,
objects,
encryption_key_sha256=old_enc_key)
# Test the permissions
if os.name == 'posix':
mode = oct(stat.S_IMODE(os.stat(fpath).st_mode))
# Assert that only user has read/write permission
self.assertEqual(oct(0o600), mode)
# Mock command object since Valdiate will call Apply() to delete the
# existing components.
class MockCommandObject(object):
delete_called = False
# We call Apply with parallel_operations_override, which expects this enum
# class to exist.
class ParallelOverrideReason(object):
SPEED = 'speed'
def Apply(self, *unused_args, **unused_kwargs):
self.delete_called = True
def MockDeleteFunc():
pass
def MockDeleteExceptionHandler():
pass
command_obj = MockCommandObject()
# Validate with correct key should succeed.
(actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData(
fpath, old_enc_key, random_prefix, objects, old_enc_key,
bucket_url, command_obj, self.logger, MockDeleteFunc,
MockDeleteExceptionHandler)
self.assertEqual(False, command_obj.delete_called)
self.assertEqual(random_prefix, actual_prefix)
self.assertEqual(objects, actual_objects)
new_enc_key = '789'
command_obj = MockCommandObject()
(actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData(
fpath, old_enc_key, random_prefix, objects, new_enc_key,
bucket_url, command_obj, self.logger, MockDeleteFunc,
MockDeleteExceptionHandler)
self.assertEqual(True, command_obj.delete_called)
self.assertEqual(None, actual_prefix)
self.assertEqual([], actual_objects)
def testWriteComponentToParallelUploadTrackerFile(self):
tracker_file_lock = parallelism_framework_util.CreateLock()
fpath = self.CreateTempFile(file_name='foo')
random_prefix = '123'
enc_key = '456'
objects = [ObjectFromTracker('obj1', '42'),
ObjectFromTracker('obj2', '314159')]
WriteParallelUploadTrackerFile(fpath, random_prefix, objects,
encryption_key_sha256=enc_key)
new_object = ObjectFromTracker('obj3', '43')
try:
WriteComponentToParallelUploadTrackerFile(
fpath, tracker_file_lock, new_object, self.logger,
encryption_key_sha256=None)
self.fail('Expected CommandException due to different encryption key')
except CommandException as e:
self.assertIn('does not match encryption key', str(e))
WriteComponentToParallelUploadTrackerFile(
fpath, tracker_file_lock, new_object, self.logger,
encryption_key_sha256='456')
(actual_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(
fpath, self.logger)
self.assertEqual(enc_key, actual_key)
self.assertEqual(random_prefix, actual_prefix)
self.assertEqual(objects + [new_object], actual_objects)
def testParallelUploadTrackerFileNoEncryption(self):
fpath = self.CreateTempFile(file_name='foo')
random_prefix = '123'
objects = [ObjectFromTracker('obj1', '42'),
ObjectFromTracker('obj2', '314159')]
WriteParallelUploadTrackerFile(fpath, random_prefix, objects)
(enc_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(
fpath, self.logger)
self.assertEqual(random_prefix, actual_prefix)
self.assertEqual(None, enc_key)
self.assertEqual(objects, actual_objects)
def testValidateParallelCompositeTrackerData(self):
fpath = self.CreateTempFile(file_name='foo')
random_prefix = '123'
old_enc_key = '456'
bucket_url = StorageUrlFromString('gs://foo')
objects = [
ObjectFromTracker('obj1', '42'),
ObjectFromTracker('obj2', '314159')
]
WriteParallelUploadTrackerFile(fpath,
random_prefix,
objects,
encryption_key_sha256=old_enc_key)
# Mock command object since Valdiate will call Apply() to delete the
# existing components.
class MockCommandObject(object):
delete_called = False
def Apply(self, *unused_args, **unused_kwargs):
self.delete_called = True
def MockDeleteFunc():
pass
def MockDeleteExceptionHandler():
pass
command_obj = MockCommandObject()
# Validate with correct key should succeed.
(actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData(
fpath, old_enc_key, random_prefix, objects, old_enc_key,
bucket_url, command_obj, self.logger, MockDeleteFunc,
MockDeleteExceptionHandler)
self.assertEqual(False, command_obj.delete_called)
self.assertEqual(random_prefix, actual_prefix)
self.assertEqual(objects, actual_objects)
new_enc_key = '789'
command_obj = MockCommandObject()
(actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData(
fpath, old_enc_key, random_prefix, objects, new_enc_key,
bucket_url, command_obj, self.logger, MockDeleteFunc,
MockDeleteExceptionHandler)
self.assertEqual(True, command_obj.delete_called)
self.assertEqual(None, actual_prefix)
self.assertEqual([], actual_objects)