def test_NameTransformed(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
ec2client = AwsHelper.boto3_client('ec2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
snapshot_id = backup.backup_id
self.created_snapshots.append(snapshot_id)
# wait for snapshot to become available
snapshots = ec2client.describe_snapshots(SnapshotIds=[snapshot_id])['Snapshots']
self.assertTrue(len(snapshots) == 1)
self.assertTrue(snapshots[0]['VolumeId'] == self.volume['VolumeId'])
d_tags = dict(map(lambda x: (x['Key'], x['Value']), snapshots[0]['Tags']))
self.assertTrue(d_tags['Name'].startswith(NAME_TRANSFORMED))
print(f"required: {backup.date_created.strftime(BackupResource.TIMESTAMP_FORMAT)}-{backup.retention_type}")
print(f"actual: {d_tags['Name']}")
self.assertTrue(d_tags['Name'].endswith(f"{backup.date_created.strftime(BackupResource.TIMESTAMP_FORMAT)}-{backup.retention_type}"))
valid = True
self.assertTrue(valid)
def test_CreateBackups(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
ec2client = AwsHelper.boto3_client('ec2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
snapshot_id = backup.backup_id
self.created_snapshots.append(snapshot_id)
snapshots = ec2client.describe_snapshots(
SnapshotIds=[snapshot_id])['Snapshots']
self.assertTrue(len(snapshots) == 1)
self.assertTrue(
snapshots[0]['VolumeId'] == self.volume['VolumeId'])
d_tags = dict(
map(lambda x: (x['Key'], x['Value']),
snapshots[0]['Tags']))
marker_tag = f"{RuntimeConfig.get_tag_prefix()}:{BackupResource.BACKUP_MARKER_TAG}"
self.assertTrue(marker_tag in d_tags)
self.assertTrue(
f"{RuntimeConfig.get_tag_prefix()}:entity_id" in d_tags)
self.assertTrue(
d_tags[f"{RuntimeConfig.get_tag_prefix()}:entity_id"] ==
self.volume['VolumeId'])
valid = True
self.assertTrue(valid)
def test_CreateSharingInfo(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
os.environ["shelvery_share_aws_account_ids"] = str(self.share_with_id)
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
finally:
del os.environ["shelvery_share_aws_account_ids"]
valid = False
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
account_id = ebs_backups_engine.account_id
s3path = f"{S3_DATA_PREFIX}/shared/{self.share_with_id}/{ebs_backups_engine.get_engine_type()}/{backup.name}.yaml"
s3bucket = ShelveryEngine.get_local_bucket_name()
bucket = boto3.resource('s3').Bucket(s3bucket)
object = bucket.Object(s3path)
content = object.get()['Body'].read()
restored_br = yaml.load(content)
engine_backup = ebs_backups_engine.get_backup_resource(backup.region, backup.backup_id)
self.assertEquals(restored_br.backup_id, engine_backup.backup_id)
self.assertEquals(restored_br.name, engine_backup.name)
self.assertEquals(restored_br.region, engine_backup.region)
for tag in ['name','date_created','entity_id','region','retention_type']:
self.assertEquals(
restored_br.tags[f"{RuntimeConfig.get_tag_prefix()}:{tag}"],
engine_backup.tags[f"{RuntimeConfig.get_tag_prefix()}:{tag}"]
)
valid = True
self.assertTrue(valid)
def test_CleanBackups(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
ec2client = boto3.client('ec2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
snapshot_id = backup.backup_id
snapshots = ec2client.describe_snapshots(SnapshotIds=[snapshot_id])['Snapshots']
self.assertEqual(len(snapshots), 1)
ec2client.create_tags(
Resources=[snapshot_id],
Tags=[{'Key': f"{RuntimeConfig.get_tag_prefix()}:date_created",
'Value': datetime(1990, 1, 1).strftime(BackupResource.TIMESTAMP_FORMAT)
}]
)
ebs_backups_engine.clean_backups()
with self.assertRaises(botocore.exceptions.ClientError) as context:
ec2client.describe_snapshots(SnapshotIds=[snapshot_id])['Snapshots']
self.assertTrue('does not exist' in context.exception.response['Error']['Message'])
self.assertEqual('InvalidSnapshot.NotFound', context.exception.response['Error']['Code'])
valid = True
self.assertTrue(valid)
def test_ShareBackups(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
os.environ["shelvery_share_aws_account_ids"] = str(self.share_with_id)
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
finally:
del os.environ["shelvery_share_aws_account_ids"]
ec2 = boto3.resource('ec2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
print(f"Testing snap {backup.entity_id} if shared with {self.share_with_id}")
snapshot_id = backup.backup_id
self.created_snapshots.append(snapshot_id)
snapshot = ec2.Snapshot(snapshot_id)
attr = snapshot.describe_attribute(Attribute='createVolumePermission')
print(f"CreateVolumePermissions: {attr}")
userlist = attr['CreateVolumePermissions']
self.assertTrue(str(self.share_with_id) in list(map(lambda x: x['UserId'], userlist)))
valid = True
self.assertTrue(valid)
def test_CopyBackups(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
os.environ['shelvery_dr_regions'] = 'us-west-2'
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
finally:
del os.environ["shelvery_dr_regions"]
ec2dr_region = AwsHelper.boto3_client('ec2', region_name='us-west-2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
print(
f"Testing snap {backup.entity_id} if copied to region us-west-2"
)
snapshot_id = backup.backup_id
self.created_snapshots.append(snapshot_id)
drsnapshot = ec2dr_region.describe_snapshots(Filters=[{
'Name':
f"tag:{RuntimeConfig.get_tag_prefix()}:entity_id",
'Values': [self.volume['VolumeId']]
}])['Snapshots'][0]
drsnapshot_dtags = dict(
map(lambda x: (x['Key'], x['Value']), drsnapshot['Tags']))
tag_key = f"{RuntimeConfig.get_tag_prefix()}:dr_copy"
tag_value = 'true'
self.assertTrue(tag_key in drsnapshot_dtags)
self.assertEquals(drsnapshot_dtags[tag_key], tag_value)
tag_key = f"{RuntimeConfig.get_tag_prefix()}:dr_source_backup"
tag_value = f"us-east-1:{snapshot_id}"
self.assertTrue(tag_key in drsnapshot_dtags)
self.assertEquals(drsnapshot_dtags[tag_key], tag_value)
tag_key = f"{RuntimeConfig.get_tag_prefix()}:region"
tag_value = 'us-west-2'
self.assertTrue(tag_key in drsnapshot_dtags)
self.assertEquals(drsnapshot_dtags[tag_key], tag_value)
self.regional_snapshots['us-west-2'].append(
drsnapshot['SnapshotId'])
valid = True
self.assertTrue(valid)
def test_CreateEbsBackup(self):
print(f"ebs - Running backup test")
ebs_backup_engine = ShelveryEBSBackup()
print(f"ebs - Shelvery backup initialised")
backups = initCreateBackups(ebs_backup_engine)
ec2_client = AwsHelper.boto3_client('ec2')
valid = False
# validate there is
for backup in backups:
#Get source snapshot
source_snapshot = ec2_client.describe_snapshots(
Filters=[{
'Name': 'tag:Name',
'Values': [backup.name]
}])
#Get snapshot id and add to created snapshot list for removal in teardown later
dest_snapshot_id = source_snapshot['Snapshots'][0]['SnapshotId']
self.created_snapshots.append(dest_snapshot_id)
valid = compareBackups(self=self,
backup=backup,
backup_engine=ebs_backup_engine)
self.assertTrue(valid)
def test_CleanBackupData(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
ec2client = boto3.client('ec2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
snapshot_id = backup.backup_id
snapshots = ec2client.describe_snapshots(SnapshotIds=[snapshot_id])['Snapshots']
self.assertEqual(len(snapshots), 1)
ec2client.create_tags(
Resources=[snapshot_id],
Tags=[{'Key': f"{RuntimeConfig.get_tag_prefix()}:date_created",
'Value': datetime(2000, 1, 1).strftime(BackupResource.TIMESTAMP_FORMAT)
}]
)
ebs_backups_engine.clean_backups()
with self.assertRaises(botocore.exceptions.ClientError) as context:
ec2client.describe_snapshots(SnapshotIds=[snapshot_id])['Snapshots']
self.assertTrue('does not exist' in context.exception.response['Error']['Message'])
self.assertEqual('InvalidSnapshot.NotFound', context.exception.response['Error']['Code'])
account_id = ebs_backups_engine.account_id
s3path = f"{S3_DATA_PREFIX}/{ebs_backups_engine.get_engine_type()}/removed/{backup.name}.yaml"
s3bucket = ShelveryEngine.get_local_bucket_name()
bucket = boto3.resource('s3').Bucket(s3bucket)
object = bucket.Object(s3path)
content = object.get()['Body'].read()
restored_br = yaml.load(content)
self.assertEquals(restored_br.backup_id, backup.backup_id)
self.assertEquals(restored_br.name, backup.name)
self.assertEquals(restored_br.region, backup.region)
self.assertIsNotNone(restored_br.date_deleted)
self.assertEquals(restored_br.date_created, datetime(2000, 1, 1))
valid = True
self.assertTrue(valid)
def get_shelvery_instance(cls, type: str) -> ShelveryEngine:
if type == 'ebs':
return ShelveryEBSBackup()
if type == 'rds':
return ShelveryRDSBackup()
if type == 'ami':
raise Exception("AMIs not supported yet")
def test_Cleanup(self):
print(f"ebs - Running cleanup test")
ebs_backup_engine = ShelveryEBSBackup()
backups = initCleanup(ebs_backup_engine)
ec2_client = AwsHelper.boto3_client('ec2')
valid = False
for backup in backups:
valid = ebsCleanupBackups(self=self,
backup=backup,
backup_engine=ebs_backup_engine,
service_client=ec2_client)
self.assertTrue(valid)
def test_ShareEbsBackup(self):
print(f"ebs - Running share backup test")
ebs_backup_engine = ShelveryEBSBackup()
print("Creating shared backups")
backups = initShareBackups(ebs_backup_engine, str(self.share_with_id))
print("Shared backups created")
valid = False
# validate there is
for backup in backups:
valid = ec2ShareBackups(self=self, backup=backup)
self.assertTrue(valid)
def get_shelvery_instance(cls, type: str) -> ShelveryEngine:
if type == 'ebs':
return ShelveryEBSBackup()
if type == 'rds':
return ShelveryRDSBackup()
if type == 'rds_cluster':
return ShelveryRDSClusterBackup()
if type == 'ec2ami':
return ShelveryEC2AMIBackup()
if type == 'redshift':
return ShelveryRedshiftBackup()
def test_CreateBackupData(self):
ebs_backups_engine = ShelveryEBSBackup()
try:
backups = ebs_backups_engine.create_backups()
except Exception as e:
print(e)
print(f"Failed with {e}")
traceback.print_exc(file=sys.stdout)
raise e
ec2client = AwsHelper.boto3_client('ec2')
valid = False
# validate there is
for backup in backups:
if backup.entity_id == self.volume['VolumeId']:
snapshot_id = backup.backup_id
self.created_snapshots.append(snapshot_id)
# wait for snapshot to become available
ebs_backups_engine.wait_backup_available(backup.region, backup.backup_id, None, None)
# allow buffer period for engine to write data to s3
time.sleep(20)
# this is the backup that gets stored in s3
engine_backup = ebs_backups_engine.get_backup_resource(backup.region, backup.backup_id)
# verify the s3 data
account_id = ebs_backups_engine.account_id
s3path = f"{S3_DATA_PREFIX}/{ebs_backups_engine.get_engine_type()}/{engine_backup.name}.yaml"
s3bucket = ebs_backups_engine.get_local_bucket_name()
print(f"Usingbucket {s3bucket}")
print(f"Using path {s3path}")
bucket = boto3.resource('s3').Bucket(s3bucket)
object = bucket.Object(s3path)
content = object.get()['Body'].read()
restored_br = yaml.load(content, Loader=yaml.Loader)
self.assertEqual(restored_br.backup_id, engine_backup.backup_id)
self.assertEqual(restored_br.name, engine_backup.name)
self.assertEqual(restored_br.region, engine_backup.region)
print(f"Tags restored: \n{yaml.dump(restored_br.tags)}\n")
print(f"Tags backup: \n{yaml.dump(engine_backup.tags)}\n")
self.assertEqual(restored_br.tags['Name'], engine_backup.tags['Name'])
for tag in ['name','date_created','entity_id','region','retention_type']:
self.assertEqual(
restored_br.tags[f"{RuntimeConfig.get_tag_prefix()}:{tag}"],
engine_backup.tags[f"{RuntimeConfig.get_tag_prefix()}:{tag}"]
)
valid = True
self.assertTrue(valid)