def pull_shared_backups(self):
account_id = self.account_id
s3_client = AwsHelper.boto3_client('s3')
for src_account_id in RuntimeConfig.get_source_backup_accounts(self):
try:
bucket_name = self.get_remote_bucket_name(src_account_id)
path = f"backups/shared/{account_id}/{self.get_engine_type()}/"
path_processed = f"backups/shared/{account_id}/{self.get_engine_type()}-processed"
path_failed = f"backups/shared/{account_id}/{self.get_engine_type()}-failed"
bucket_loc = s3_client.get_bucket_location(Bucket=bucket_name)
bucket_region = bucket_loc['LocationConstraint']
if bucket_region == 'EU':
bucket_region = 'eu-west-1'
elif bucket_region is None:
bucket_region = 'us-east-1'
regional_client = AwsHelper.boto3_client('s3', region_name=bucket_region)
shared_backups = regional_client.list_objects_v2(Bucket=bucket_name, Prefix=path)
if 'Contents' in shared_backups:
all_backups = shared_backups['Contents']
else:
self.logger.info(f"No shared backups of type {self.get_engine_type()} found to pull")
all_backups = {}
while 'NextContinuationToken' in shared_backups:
shared_backups = regional_client.list_objects_v2(
Bucket=bucket_name, Delimiter='/',
Prefix=path, ContinuationToken=shared_backups['NextContinuationToken']
)
all_backups.extend(shared_backups['Contents'])
for backup_object in all_backups:
try:
serialised_shared_backup = regional_client.get_object(
Bucket=bucket_name,
Key=backup_object['Key'])['Body'].read()
shared_backup = yaml.load(serialised_shared_backup)
new_backup_id = self.copy_shared_backup(src_account_id, shared_backup)
new_backup = shared_backup.cross_account_copy(new_backup_id)
self.tag_backup_resource(new_backup)
self.store_backup_data(new_backup)
regional_client.delete_object(Bucket=bucket_name, Key=backup_object['Key'])
self.logger.info(f"Removed s3://{bucket_name}/{backup_object['Key']}")
regional_client.put_object(
Bucket=bucket_name,
Key=f"{path_processed}/{shared_backup.name}.yaml",
Body=yaml.dump(shared_backup, default_flow_style=False)
)
self.logger.info(
f"Moved shared backup info to s3://{bucket_name}/{path_processed}/{shared_backup.name}.yaml")
self.snspublisher.notify({
'Operation': 'PullSharedBackup',
'Status': 'OK',
'BackupType': self.get_engine_type(),
'SourceAccount': src_account_id,
'Backup': shared_backup.name
})
except Exception as e:
backup_name = backup_object['Key'].split('/')[-1].replace('.yaml', '')
self.logger.exception(f"Failed to copy shared backup s3://{bucket_name}/{backup_object['Key']}")
self.snspublisher_error.notify({
'Operation': 'PullSharedBackup',
'Status': 'ERROR',
'ExceptionInfo': e.__dict__,
'BackupType': self.get_engine_type(),
'SourceAccount': src_account_id,
'BackupS3Location': backup_object['Key'],
'NewS3Location': f"{path_failed}/{backup_name}.yaml",
'Bucket': bucket_name
})
regional_client.put_object(
Bucket=bucket_name,
Key=f"{path_failed}/{backup_name}.yaml",
Body=yaml.dump(shared_backup, default_flow_style=False)
)
self.logger.info(
f"Failed share backup operation | backup info moved to s3://{bucket_name}/{path_failed}/{shared_backup.name}.yaml ")
except Exception as e:
self.snspublisher_error.notify({
'Operation': 'PullSharedBackupsFromAccount',
'Status': 'ERROR',
'ExceptionInfo': e.__dict__,
'BackupType': self.get_engine_type(),
'SourceAccount': src_account_id,
})
self.logger.exception("Failed to pull shared backups")
def _get_shelvery_shared_backups(self):
account_id = RuntimeConfig.get_aws_account_id()
for account_id in RuntimeConfig.get_source_backup_accounts(self):
bucket = self._get_data_bucket(account_id)
path = f"{S3_DATA_PREFIX}/{self.get_engine_type()}/shared/{account_id}"
