def test_skip_versioning(self):
"""Tests the --skip_versioning flag works correctly."""
self.parsed_args.skip_versioning = True
details = bucket_details.BucketDetails(
conf=self.parsed_args, source_bucket=self.source_bucket)
self.assertEqual(self.parsed_args.location, details.location)
self.assertEqual(self.parsed_args.storage_class, details.storage_class)
self.assertEqual(self.source_bucket.get_iam_policy(),
details.iam_policy)
self.assertEqual(self.source_bucket.acl.get_entities(),
details.acl_entities)
self.assertEqual(self.source_bucket.default_object_acl.get_entities(),
details.default_obj_acl_entities)
self.assertEqual(self.source_bucket.requester_pays,
details.requester_pays)
self.assertEqual(self.source_bucket.cors, details.cors)
self.assertEqual(self.source_bucket.default_kms_key_name,
details.default_kms_key_name)
self.assertEqual(self.source_bucket.labels, details.labels)
self.assertEqual(self.source_bucket.lifecycle_rules,
details.lifecycle_rules)
self.assertEqual(self.source_bucket.get_logging(), details.logging)
self.assertIsNone(details.versioning_enabled)
self.assertEqual(self.source_bucket.list_notifications(),
details.notifications)
details.versioning_enabled = 'test'
self.assertIsNone(details.versioning_enabled)
def main(config, parsed_args, cloud_logger):
"""Main entry point for the bucket mover tool
Args:
config: A Configuration object with all of the config values needed for the script to run
parsed_args: the configargparser parsing of command line options
cloud_logger: A GCP logging client instance
"""
cloud_logger.log_text("Starting GCS Bucket Mover")
_print_config_details(cloud_logger, config)
source_bucket = config.source_storage_client.lookup_bucket( # pylint: disable=no-member
config.bucket_name)
# Get copies of all of the source bucket's IAM, ACLs and settings so they
# can be copied over to the target project bucket
source_bucket_details = bucket_details.BucketDetails(
conf=parsed_args, source_bucket=source_bucket)
_check_bucket_lock(cloud_logger, config, source_bucket,
source_bucket_details)
sts_client = discovery.build(
'storagetransfer', 'v1', credentials=config.target_project_credentials)
if config.is_rename:
_rename_bucket(cloud_logger, config, source_bucket,
source_bucket_details, sts_client)
else:
_move_bucket(cloud_logger, config, source_bucket, source_bucket_details,
sts_client)
cloud_logger.log_text('Completed GCS Bucket Mover')
def test_create_bucket_default_obj_acl_skip(
self, mock_update_notifications, mock_update_iam_policies,
mock_update_acl_entities, mock_create_bucket_api_call,
mock_add_target_project_to_kms_key, mock_bucket):
"""Tests that the create bucket happens without the notifications set"""
self.parsed_args = common.get_mock_args()
self.source_bucket = common.get_mock_source_bucket()
details = bucket_details.BucketDetails(
conf=self.parsed_args, source_bucket=self.source_bucket)
details.notifications = None
mock_bucket_instance = mock.MagicMock()
mock_bucket.return_value = mock_bucket_instance
bucket_mover_service._create_bucket(mock.MagicMock(), mock.MagicMock(),
mock.MagicMock(), mock.MagicMock(),
details)
self.assertEqual(details.default_kms_key_name,
mock_bucket_instance.default_kms_key_name)
mock_add_target_project_to_kms_key.assert_called_once()
mock_bucket_instance.enable_logging.assert_called_once_with(
details.logging['logBucket'], details.logging['logObjectPrefix'])
mock_create_bucket_api_call.assert_called_once()
mock_update_iam_policies.assert_called_once()
self.assertEqual(2, mock_update_acl_entities.call_count)
mock_bucket_instance.acl.save.assert_called_once()
mock_bucket_instance.default_object_acl.save.assert_called_once()
mock_update_notifications.assert_not_called()
def move_bucket(config, parsed_args, cloud_logger):
"""Main entry point for the bucket mover script
Args:
config: A Configuration object with all of the config values needed for the script to run
parsed_args: the configargparser parsing of command line options
cloud_logger: A GCP logging client instance
"""
cloud_logger.log_text("Starting GCS Bucket Mover")
_print_and_log(
cloud_logger,
'Using the following service accounts for GCS credentials: ')
_print_and_log(cloud_logger, 'Source Project - {}'.format(
config.source_project_credentials.service_account_email)) # pylint: disable=no-member
_print_and_log(cloud_logger, 'Target Project - {}'.format(
config.target_project_credentials.service_account_email)) # pylint: disable=no-member
source_bucket = config.source_storage_client.lookup_bucket( # pylint: disable=no-member
config.bucket_name)
# Get copies of all of the source bucket's IAM, ACLs and settings so they can be copied over to
# the target project bucket
source_bucket_details = bucket_details.BucketDetails(
conf=parsed_args, source_bucket=source_bucket)
_check_bucket_lock(cloud_logger, config, source_bucket)
target_temp_bucket = _create_temp_target_bucket(cloud_logger, config,
source_bucket_details)
# Create STS client
sts_client = discovery.build(
'storagetransfer', 'v1', credentials=config.target_project_credentials)
sts_account_email = _assign_sts_permissions(cloud_logger, sts_client,
config, target_temp_bucket)
_run_and_wait_for_sts_job(sts_client, config.target_project,
config.bucket_name, config.temp_bucket_name,
cloud_logger)
_delete_empty_source_bucket(cloud_logger, source_bucket)
_recreate_source_bucket(cloud_logger, config, source_bucket_details)
_assign_sts_permissions_to_new_bucket(cloud_logger, sts_account_email,
config)
_run_and_wait_for_sts_job(sts_client, config.target_project,
config.temp_bucket_name, config.bucket_name,
cloud_logger)
_delete_empty_temp_bucket(cloud_logger, target_temp_bucket)
_remove_sts_permissions(cloud_logger, sts_account_email, config)
cloud_logger.log_text('Completed GCS Bucket Mover')
def test_default_constructor(self):
"""Tests the default object is successfully created."""
details = bucket_details.BucketDetails(
conf=self.parsed_args, source_bucket=self.source_bucket)
# Test that the properties are set to the parsed_args/source_bucket values
self.assertEqual(self.parsed_args.location, details.location)
self.assertEqual(self.parsed_args.storage_class, details.storage_class)
self.assertEqual(self.source_bucket.get_iam_policy(),
details.iam_policy)
self.assertEqual(self.source_bucket.acl.get_entities(),
details.acl_entities)
self.assertEqual(self.source_bucket.default_object_acl.get_entities(),
details.default_obj_acl_entities)
self.assertEqual(self.source_bucket.requester_pays,
details.requester_pays)
self.assertEqual(self.source_bucket.cors, details.cors)
self.assertEqual(self.source_bucket.default_kms_key_name,
details.default_kms_key_name)
self.assertEqual(self.source_bucket.labels, details.labels)
self.assertEqual(self.source_bucket.lifecycle_rules,
details.lifecycle_rules)
self.assertEqual(self.source_bucket.get_logging(), details.logging)
self.assertEqual(self.source_bucket.versioning_enabled,
details.versioning_enabled)
self.assertEqual(self.source_bucket.list_notifications(),
details.notifications)
# Test that the bucket properties can be overridden
value = 'test'
details.iam_policy = details.acl_entities = details.default_obj_acl_entities = value
details.requester_pays = details.cors = details.default_kms_key_name = value
details.labels = details.lifecycle_rules = details.logging = value
details.versioning_enabled = details.notifications = value
self.assertEqual(value, details.iam_policy)
self.assertEqual(value, details.acl_entities)
self.assertEqual(value, details.default_obj_acl_entities)
self.assertEqual(value, details.requester_pays)
self.assertEqual(value, details.cors)
self.assertEqual(value, details.default_kms_key_name)
self.assertEqual(value, details.labels)
self.assertEqual(value, details.lifecycle_rules)
self.assertEqual(value, details.logging)
self.assertEqual(value, details.versioning_enabled)
self.assertEqual(value, details.notifications)
def test_skip_everything(self):
"""Tests the object constructor when the skip_everything flag is True."""
self.parsed_args.skip_everything = True
details = bucket_details.BucketDetails(
conf=self.parsed_args, source_bucket=self.source_bucket)
self.assertEqual(self.parsed_args.location, details.location)
self.assertEqual(self.parsed_args.storage_class, details.storage_class)
self.assertIsNone(details.iam_policy)
self.assertIsNone(details.acl_entities)
self.assertIsNone(details.default_obj_acl_entities)
self.assertIsNone(details.requester_pays)
self.assertIsNone(details.cors)
self.assertIsNone(details.default_kms_key_name)
self.assertEqual(details.labels, {})
self.assertIsNone(details.lifecycle_rules)
self.assertIsNone(details.logging)
self.assertIsNone(details.versioning_enabled)
self.assertListEqual(details.notifications, [])
# Test that the bucket properties cannot be overridden
value = 'test'
details.iam_policy = details.acl_entities = details.default_obj_acl_entities = value
details.requester_pays = details.cors = details.default_kms_key_name = value
details.labels = details.lifecycle_rules = details.logging = value
details.versioning_enabled = details.notifications = value
self.assertIsNone(details.iam_policy)
self.assertIsNone(details.acl_entities)
self.assertIsNone(details.default_obj_acl_entities)
self.assertIsNone(details.requester_pays)
self.assertIsNone(details.cors)
self.assertIsNone(details.default_kms_key_name)
self.assertEqual(details.labels, {})
self.assertIsNone(details.lifecycle_rules)
self.assertIsNone(details.logging)
self.assertIsNone(details.versioning_enabled)
self.assertListEqual(details.notifications, [])
def move_bucket(conf):
"""Main entry point for the bucket mover script
Args:
conf: the argparser parsing of command line options
"""
# Load the environment config values set in config.sh and create the storage clients.
config = configuration.Configuration.from_conf(conf)
print('Using the following service accounts for GCS credentials: ')
print('Source Project - {}'.format(
config.source_project_credentials.service_account_email)) # pylint: disable=no-member
print('Target Project - {}\n'.format(
config.target_project_credentials.service_account_email)) # pylint: disable=no-member
# Get copies of all of the source bucket's IAM, ACLs and settings so they can be copied over to
# the target project bucket
source_bucket = config.source_storage_client.lookup_bucket( # pylint: disable=no-member
config.bucket_name)
source_bucket_details = bucket_details.BucketDetails(
conf=conf, source_bucket=source_bucket)
with yaspin(text='Create temp target bucket') as spinner:
target_temp_bucket = _create_bucket(spinner, config,
config.target_storage_client,
config.temp_bucket_name,
source_bucket_details)
spinner.write('{} Bucket {} created in target project {}'.format(
_CHECKMARK, config.temp_bucket_name, config.target_project))
# Create STS client
sts_client = discovery.build('storagetransfer',
'v1',
credentials=config.target_project_credentials)
spinner_text = 'Assign STS permissions to source/temp buckets'
with yaspin(text=spinner_text) as spinner:
sts_account_email = _get_sts_iam_account_email(sts_client,
config.target_project)
spinner.write('\nSTS service account for IAM usage: {}'.format(
sts_account_email))
_assign_sts_iam_roles(sts_account_email, config.source_storage_client,
config.source_project, config.bucket_name, True)
_assign_sts_iam_roles(sts_account_email, config.target_storage_client,
config.target_project, target_temp_bucket.name,
True)
spinner.ok(_CHECKMARK)
_run_and_wait_for_sts_job(sts_client, config.target_project,
config.bucket_name, config.temp_bucket_name)
with yaspin(text='Delete empty source bucket') as spinner:
source_bucket.delete()
spinner.ok(_CHECKMARK)
with yaspin(text='Re-create source bucket in target project') as spinner:
_create_bucket(spinner, config, config.target_storage_client,
config.bucket_name, source_bucket_details)
spinner.ok(_CHECKMARK)
with yaspin(text='Assign STS permissions to new source bucket') as spinner:
_assign_sts_iam_roles(sts_account_email, config.target_storage_client,
config.target_project, config.bucket_name, False)
spinner.ok(_CHECKMARK)
_run_and_wait_for_sts_job(sts_client, config.target_project,
config.temp_bucket_name, config.bucket_name)
with yaspin(text='Delete empty temp bucket') as spinner:
target_temp_bucket.delete()
spinner.ok(_CHECKMARK)
spinner_text = 'Remove STS permissions from new source bucket'
with yaspin(text=spinner_text) as spinner:
_remove_sts_iam_roles(sts_account_email, config.target_storage_client,
config.bucket_name)
spinner.ok(_CHECKMARK)
