def testValidateParallelCompositeTrackerData(self): tempdir = self.CreateTempDir() fpath = os.path.join(tempdir, 'foo') random_prefix = '123' old_enc_key = '456' bucket_url = StorageUrlFromString('gs://foo') objects = [ ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159') ] WriteParallelUploadTrackerFile(fpath, random_prefix, objects, encryption_key_sha256=old_enc_key) # Test the permissions if os.name == 'posix': mode = oct(stat.S_IMODE(os.stat(fpath).st_mode)) # Assert that only user has read/write permission self.assertEqual(oct(0o600), mode) # Mock command object since Valdiate will call Apply() to delete the # existing components. class MockCommandObject(object): delete_called = False # We call Apply with parallel_operations_override, which expects this enum # class to exist. class ParallelOverrideReason(object): SPEED = 'speed' def Apply(self, *unused_args, **unused_kwargs): self.delete_called = True def MockDeleteFunc(): pass def MockDeleteExceptionHandler(): pass command_obj = MockCommandObject() # Validate with correct key should succeed. (actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData( fpath, old_enc_key, random_prefix, objects, old_enc_key, bucket_url, command_obj, self.logger, MockDeleteFunc, MockDeleteExceptionHandler) self.assertEqual(False, command_obj.delete_called) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(objects, actual_objects) new_enc_key = '789' command_obj = MockCommandObject() (actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData( fpath, old_enc_key, random_prefix, objects, new_enc_key, bucket_url, command_obj, self.logger, MockDeleteFunc, MockDeleteExceptionHandler) self.assertEqual(True, command_obj.delete_called) self.assertEqual(None, actual_prefix) self.assertEqual([], actual_objects)
def testWriteComponentToParallelUploadTrackerFile(self): tracker_file_lock = parallelism_framework_util.CreateLock() fpath = self.CreateTempFile(file_name='foo') random_prefix = '123' enc_key = '456' objects = [ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159')] WriteParallelUploadTrackerFile(fpath, random_prefix, objects, encryption_key_sha256=enc_key) new_object = ObjectFromTracker('obj3', '43') try: WriteComponentToParallelUploadTrackerFile( fpath, tracker_file_lock, new_object, self.logger, encryption_key_sha256=None) self.fail('Expected CommandException due to different encryption key') except CommandException as e: self.assertIn('does not match encryption key', str(e)) WriteComponentToParallelUploadTrackerFile( fpath, tracker_file_lock, new_object, self.logger, encryption_key_sha256='456') (actual_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile( fpath, self.logger) self.assertEqual(enc_key, actual_key) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(objects + [new_object], actual_objects)
def testParallelUploadTrackerFileNoEncryption(self): fpath = self.CreateTempFile(file_name='foo') random_prefix = '123' objects = [ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159')] WriteParallelUploadTrackerFile(fpath, random_prefix, objects) (enc_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile( fpath, self.logger) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(None, enc_key) self.assertEqual(objects, actual_objects)
def testValidateParallelCompositeTrackerData(self): fpath = self.CreateTempFile(file_name='foo') random_prefix = '123' old_enc_key = '456' bucket_url = StorageUrlFromString('gs://foo') objects = [ ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159') ] WriteParallelUploadTrackerFile(fpath, random_prefix, objects, encryption_key_sha256=old_enc_key) # Mock command object since Valdiate will call Apply() to delete the # existing components. class MockCommandObject(object): delete_called = False def Apply(self, *unused_args, **unused_kwargs): self.delete_called = True def MockDeleteFunc(): pass def MockDeleteExceptionHandler(): pass command_obj = MockCommandObject() # Validate with correct key should succeed. (actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData( fpath, old_enc_key, random_prefix, objects, old_enc_key, bucket_url, command_obj, self.logger, MockDeleteFunc, MockDeleteExceptionHandler) self.assertEqual(False, command_obj.delete_called) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(objects, actual_objects) new_enc_key = '789' command_obj = MockCommandObject() (actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData( fpath, old_enc_key, random_prefix, objects, new_enc_key, bucket_url, command_obj, self.logger, MockDeleteFunc, MockDeleteExceptionHandler) self.assertEqual(True, command_obj.delete_called) self.assertEqual(None, actual_prefix) self.assertEqual([], actual_objects)
def testReadGsutil416ParallelUploadTrackerFile(self): """Tests the parallel upload tracker file format prior to gsutil 4.17.""" random_prefix = '123' objects = ['obj1', '42', 'obj2', '314159'] contents = '\n'.join([random_prefix] + objects) + '\n' fpath = self.CreateTempFile(file_name='foo', contents=contents) expected_objects = [ObjectFromTracker(objects[2 * i], objects[2 * i + 1]) for i in range(0, len(objects) / 2)] (_, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile( fpath, self.logger) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(expected_objects, actual_objects)
def test_FilterExistingComponentsNonVersioned(self): """Tests upload with a variety of component states.""" mock_api = MockCloudApi() bucket_name = self.MakeTempName('bucket') tracker_file = self.CreateTempFile(file_name='foo', contents='asdf') tracker_file_lock = CreateLock() # dst_obj_metadata used for passing content-type. empty_object = apitools_messages.Object() # Already uploaded, contents still match, component still used. fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1', contents='1') fpath_uploaded_correctly_url = StorageUrlFromString( str(fpath_uploaded_correctly)) object_uploaded_correctly_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_uploaded_correctly)) with open(fpath_uploaded_correctly) as f_in: fpath_uploaded_correctly_md5 = CalculateB64EncodedMd5FromContents( f_in) mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( bucket=bucket_name, name=fpath_uploaded_correctly, md5Hash=fpath_uploaded_correctly_md5), contents='1') args_uploaded_correctly = PerformParallelUploadFileToObjectArgs( fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url, object_uploaded_correctly_url, '', empty_object, tracker_file, tracker_file_lock, None) # Not yet uploaded, but needed. fpath_not_uploaded = self.CreateTempFile(file_name='foo2', contents='2') fpath_not_uploaded_url = StorageUrlFromString(str(fpath_not_uploaded)) object_not_uploaded_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_not_uploaded)) args_not_uploaded = PerformParallelUploadFileToObjectArgs( fpath_not_uploaded, 0, 1, fpath_not_uploaded_url, object_not_uploaded_url, '', empty_object, tracker_file, tracker_file_lock, None) # Already uploaded, but contents no longer match. Even though the contents # differ, we don't delete this since the bucket is not versioned and it # will be overwritten anyway. fpath_wrong_contents = self.CreateTempFile(file_name='foo4', contents='4') fpath_wrong_contents_url = StorageUrlFromString( str(fpath_wrong_contents)) object_wrong_contents_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_wrong_contents)) with open(self.CreateTempFile(contents='_')) as f_in: fpath_wrong_contents_md5 = CalculateB64EncodedMd5FromContents(f_in) mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( bucket=bucket_name, name=fpath_wrong_contents, md5Hash=fpath_wrong_contents_md5), contents='1') args_wrong_contents = PerformParallelUploadFileToObjectArgs( fpath_wrong_contents, 0, 1, fpath_wrong_contents_url, object_wrong_contents_url, '', empty_object, tracker_file, tracker_file_lock, None) # Exists in tracker file, but component object no longer exists. fpath_remote_deleted = self.CreateTempFile(file_name='foo5', contents='5') fpath_remote_deleted_url = StorageUrlFromString( str(fpath_remote_deleted)) args_remote_deleted = PerformParallelUploadFileToObjectArgs( fpath_remote_deleted, 0, 1, fpath_remote_deleted_url, '', '', empty_object, tracker_file, tracker_file_lock, None) # Exists in tracker file and already uploaded, but no longer needed. fpath_no_longer_used = self.CreateTempFile(file_name='foo6', contents='6') with open(fpath_no_longer_used) as f_in: file_md5 = CalculateB64EncodedMd5FromContents(f_in) mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( bucket=bucket_name, name='foo6', md5Hash=file_md5), contents='6') dst_args = { fpath_uploaded_correctly: args_uploaded_correctly, fpath_not_uploaded: args_not_uploaded, fpath_wrong_contents: args_wrong_contents, fpath_remote_deleted: args_remote_deleted } existing_components = [ ObjectFromTracker(fpath_uploaded_correctly, ''), ObjectFromTracker(fpath_wrong_contents, ''), ObjectFromTracker(fpath_remote_deleted, ''), ObjectFromTracker(fpath_no_longer_used, '') ] bucket_url = StorageUrlFromString('%s://%s' % (self.default_provider, bucket_name)) (components_to_upload, uploaded_components, existing_objects_to_delete) = (FilterExistingComponents( dst_args, existing_components, bucket_url, mock_api)) for arg in [ args_not_uploaded, args_wrong_contents, args_remote_deleted ]: self.assertTrue(arg in components_to_upload) self.assertEqual(1, len(uploaded_components)) self.assertEqual(args_uploaded_correctly.dst_url.url_string, uploaded_components[0].url_string) self.assertEqual(1, len(existing_objects_to_delete)) no_longer_used_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_no_longer_used)) self.assertEqual(no_longer_used_url.url_string, existing_objects_to_delete[0].url_string)
def test_FilterExistingComponentsVersioned(self): """Tests upload with versionined parallel components.""" mock_api = MockCloudApi() bucket_name = self.MakeTempName('bucket') mock_api.MockCreateVersionedBucket(bucket_name) # dst_obj_metadata used for passing content-type. empty_object = apitools_messages.Object() tracker_file = self.CreateTempFile(file_name='foo', contents='asdf') tracker_file_lock = CreateLock() # Already uploaded, contents still match, component still used. fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1', contents='1') fpath_uploaded_correctly_url = StorageUrlFromString( str(fpath_uploaded_correctly)) with open(fpath_uploaded_correctly) as f_in: fpath_uploaded_correctly_md5 = CalculateB64EncodedMd5FromContents( f_in) object_uploaded_correctly = mock_api.MockCreateObjectWithMetadata( apitools_messages.Object(bucket=bucket_name, name=fpath_uploaded_correctly, md5Hash=fpath_uploaded_correctly_md5), contents='1') object_uploaded_correctly_url = StorageUrlFromString( '%s://%s/%s#%s' % (self.default_provider, bucket_name, fpath_uploaded_correctly, object_uploaded_correctly.generation)) args_uploaded_correctly = PerformParallelUploadFileToObjectArgs( fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url, object_uploaded_correctly_url, object_uploaded_correctly.generation, empty_object, tracker_file, tracker_file_lock, None) # Duplicate object name in tracker file, but uploaded correctly. fpath_duplicate = fpath_uploaded_correctly fpath_duplicate_url = StorageUrlFromString(str(fpath_duplicate)) duplicate_uploaded_correctly = mock_api.MockCreateObjectWithMetadata( apitools_messages.Object(bucket=bucket_name, name=fpath_duplicate, md5Hash=fpath_uploaded_correctly_md5), contents='1') duplicate_uploaded_correctly_url = StorageUrlFromString( '%s://%s/%s#%s' % (self.default_provider, bucket_name, fpath_uploaded_correctly, duplicate_uploaded_correctly.generation)) args_duplicate = PerformParallelUploadFileToObjectArgs( fpath_duplicate, 0, 1, fpath_duplicate_url, duplicate_uploaded_correctly_url, duplicate_uploaded_correctly.generation, empty_object, tracker_file, tracker_file_lock, None) # Already uploaded, but contents no longer match. fpath_wrong_contents = self.CreateTempFile(file_name='foo4', contents='4') fpath_wrong_contents_url = StorageUrlFromString( str(fpath_wrong_contents)) with open(self.CreateTempFile(contents='_')) as f_in: fpath_wrong_contents_md5 = CalculateB64EncodedMd5FromContents(f_in) object_wrong_contents = mock_api.MockCreateObjectWithMetadata( apitools_messages.Object(bucket=bucket_name, name=fpath_wrong_contents, md5Hash=fpath_wrong_contents_md5), contents='_') wrong_contents_url = StorageUrlFromString( '%s://%s/%s#%s' % (self.default_provider, bucket_name, fpath_wrong_contents, object_wrong_contents.generation)) args_wrong_contents = PerformParallelUploadFileToObjectArgs( fpath_wrong_contents, 0, 1, fpath_wrong_contents_url, wrong_contents_url, '', empty_object, tracker_file, tracker_file_lock, None) dst_args = { fpath_uploaded_correctly: args_uploaded_correctly, fpath_wrong_contents: args_wrong_contents } existing_components = [ ObjectFromTracker(fpath_uploaded_correctly, object_uploaded_correctly_url.generation), ObjectFromTracker(fpath_duplicate, duplicate_uploaded_correctly_url.generation), ObjectFromTracker(fpath_wrong_contents, wrong_contents_url.generation) ] bucket_url = StorageUrlFromString('%s://%s' % (self.default_provider, bucket_name)) (components_to_upload, uploaded_components, existing_objects_to_delete) = (FilterExistingComponents( dst_args, existing_components, bucket_url, mock_api)) self.assertEqual([args_wrong_contents], components_to_upload) self.assertEqual(args_uploaded_correctly.dst_url.url_string, uploaded_components[0].url_string) expected_to_delete = [(args_wrong_contents.dst_url.object_name, args_wrong_contents.dst_url.generation), (args_duplicate.dst_url.object_name, args_duplicate.dst_url.generation)] for uri in existing_objects_to_delete: self.assertTrue((uri.object_name, uri.generation) in expected_to_delete) self.assertEqual(len(expected_to_delete), len(existing_objects_to_delete))