def populate_volume_information(self, backups):
volume_ids = []
volumes = {}
local_region = boto3.session.Session().region_name
# create list of all volume ids
for backup in backups:
if backup.entity_id not in volume_ids:
volume_ids.append(backup.entity_id)
# populate map volumeid->volume if present
for volume_id in volume_ids:
try:
volume = self.ec2client.describe_volumes(
VolumeIds=[volume_id])['Volumes'][0]
d_tags = dict(
map(lambda t: (t['Key'], t['Value']), volume['Tags']))
volumes[volume_id] = EntityResource(volume_id, local_region,
volume['CreateTime'],
d_tags)
except ClientError as e:
if 'InvalidVolume.NotFound' in str(e):
volumes[volume_id] = EntityResource.empty()
volumes[volume_id].resource_id = volume_id
else:
raise e
# add info to backup resource objects
for backup in backups:
if backup.entity_id in volumes:
backup.entity_resource = volumes[backup.entity_id]
def populate_snap_entity_resource(self, all_snapshots):
instance_ids = []
for snap in all_snapshots:
if snap['DBInstanceIdentifier'] not in instance_ids:
instance_ids.append(snap['DBInstanceIdentifier'])
entities = {}
rds_client = AwsHelper.boto3_client('rds',
arn=self.role_arn,
external_id=self.role_external_id)
local_region = boto3.session.Session().region_name
for instance_id in instance_ids:
try:
rds_instance = rds_client.describe_db_instances(
DBInstanceIdentifier=instance_id)['DBInstances'][0]
tags = rds_client.list_tags_for_resource(
ResourceName=rds_instance['DBInstanceArn'])['TagList']
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
rds_entity = EntityResource(instance_id, local_region,
rds_instance['InstanceCreateTime'],
d_tags)
entities[instance_id] = rds_entity
except ClientError as e:
if 'DBInstanceNotFoundFault' in str(type(e)):
entities[instance_id] = EntityResource.empty()
entities[instance_id].resource_id = instance_id
else:
raise e
for snap in all_snapshots:
if snap['DBInstanceIdentifier'] in entities:
snap['EntityResource'] = entities[snap['DBInstanceIdentifier']]
def populate_snap_entity_resource(self, all_snapshots):
cluster_ids = []
for snap in all_snapshots:
if snap['DBClusterIdentifier'] not in cluster_ids:
cluster_ids.append(snap['DBClusterIdentifier'])
entities = {}
rds_client = AwsHelper.boto3_client('rds',
arn=self.role_arn,
external_id=self.role_external_id)
local_region = boto3.session.Session().region_name
for cluster_id in cluster_ids:
try:
self.logger.info(
f"Collecting tags from DB cluster {cluster_id} ...")
rds_instance = rds_client.describe_db_clusters(
DBClusterIdentifier=cluster_id)['DBClusters'][0]
tags = rds_instance['TagList']
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
rds_entity = EntityResource(cluster_id, local_region,
rds_instance['ClusterCreateTime'],
d_tags)
entities[cluster_id] = rds_entity
except ClientError as e:
if 'DBClusterNotFoundFault' in str(type(e)):
entities[cluster_id] = EntityResource.empty()
entities[cluster_id].resource_id = cluster_id
else:
raise e
for snap in all_snapshots:
if snap['DBClusterIdentifier'] in entities:
snap['EntityResource'] = entities[snap['DBClusterIdentifier']]
def populate_snap_entity_resource(self, all_snapshots):
cluster_ids = []
for snap in all_snapshots:
if snap['DBClusterIdentifier'] not in cluster_ids:
cluster_ids.append(snap['DBClusterIdentifier'])
entities = {}
rds_client = boto3.client('rds')
local_region = boto3.session.Session().region_name
for cluster_id in cluster_ids:
try:
rds_instance = rds_client.describe_db_clusters(DBClusterIdentifier=cluster_id)['DBClusters'][0]
tags = rds_client.list_tags_for_resource(ResourceName=rds_instance['DBClusterArn'])['TagList']
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
rds_entity = EntityResource(cluster_id,
local_region,
rds_instance['ClusterCreateTime'],
d_tags)
entities[cluster_id] = rds_entity
except ClientError as e:
if 'DBClusterNotFoundFault' in str(type(e)):
entities[cluster_id] = EntityResource.empty()
entities[cluster_id].resource_id = cluster_id
else:
raise e
for snap in all_snapshots:
if snap['DBClusterIdentifier'] in entities:
snap['EntityResource'] = entities[snap['DBClusterIdentifier']]
def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
# region and api client
local_region = boto3.session.Session().region_name
rds_client = AwsHelper.boto3_client('rds',
arn=self.role_arn,
external_id=self.role_external_id)
# list of models returned from api
db_cluster_entities = []
db_clusters = self.get_all_clusters(rds_client)
# collect tags in check if instance tagged with marker tag
for instance in db_clusters:
tags = instance['TagList']
# convert api response to dictionary
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
# check if marker tag is present
if tag_name in d_tags and d_tags[
tag_name] in SHELVERY_DO_BACKUP_TAGS:
resource = EntityResource(instance['DBClusterIdentifier'],
local_region,
instance['ClusterCreateTime'],
d_tags)
db_cluster_entities.append(resource)
return db_cluster_entities
def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
# region and api client
local_region = boto3.session.Session().region_name
rds_client = boto3.client('rds')
# list of models returned from api
db_entities = []
db_instances = self.get_all_instances(rds_client)
# collect tags in check if instance tagged with marker tag
for instance in db_instances:
tags = rds_client.list_tags_for_resource(
ResourceName=instance['DBInstanceArn'])['TagList']
# convert api response to dictionary
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
# check if marker tag is present
if tag_name in d_tags:
resource = EntityResource(instance['DBInstanceIdentifier'],
local_region,
instance['InstanceCreateTime'],
d_tags)
db_entities.append(resource)
return db_entities
def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
"""Get all instances that contain `tag_name` as a tag."""
clusters = self.collect_clusters(tag_name)
# TODO: To get the cluster's creation time, we need to query the "events" with the
# cluster ID.
entities = []
for cluster in clusters:
if cluster['ClusterStatus'] != 'available':
self.logger.info(
f"Skipping cluster '{cluster['ClusterIdentifier']}' as its status is '{cluster['ClusterStatus']}'."
)
continue
d_tags = BackupResource.dict_from_boto3_tags(cluster['Tags'])
entity = EntityResource(
resource_id=cluster['ClusterIdentifier'],
resource_region=self.region,
date_created=f"{datetime.datetime.utcnow():%Y-%m-%d %H:%M:%S}",
tags=d_tags)
entities.append(entity)
return entities
def get_existing_backups(self,
backup_tag_prefix: str) -> List[BackupResource]:
"""
Collect existing backups on system of given type, marked with given tag
"""
local_region = boto3.session.Session().region_name
marker_tag = f"{backup_tag_prefix}:{BackupResource.BACKUP_MARKER_TAG}"
response = self.redshift_client.describe_cluster_snapshots(
SnapshotType='manual',
TagKeys=[marker_tag],
TagValues=SHELVERY_DO_BACKUP_TAGS)
snapshots = response['Snapshots']
backups = []
for snap in snapshots:
cluster_id = snap['ClusterIdentifier']
d_tags = BackupResource.dict_from_boto3_tags(snap['Tags'])
create_time = snap['ClusterCreateTime']
redshift_entity = EntityResource(cluster_id, local_region,
create_time, d_tags)
backup_id = f"arn:aws:redshift:{local_region}:{snap['OwnerAccount']}"
backup_id = f"{backup_id}:snapshot:{snap['ClusterIdentifier']}/{snap['SnapshotIdentifier']}"
backup_resource = BackupResource.construct(backup_tag_prefix,
backup_id, d_tags)
backup_resource.entity_resource = redshift_entity
backup_resource.entity_id = redshift_entity.resource_id
backups.append(backup_resource)
return backups
def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
# region and api client
local_region = boto3.session.Session().region_name
rds_client = boto3.client('rds')
# list of models returned from api
db_entities = []
db_instances = self.get_all_instances(rds_client)
# collect tags in check if instance tagged with marker tag
for instance in db_instances:
tags = rds_client.list_tags_for_resource(
ResourceName=instance['DBInstanceArn'])['TagList']
# convert api response to dictionary
d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
if 'DBClusterIdentifier' in instance:
self.logger.info(
f"Skipping RDS Instance {instance['DBInstanceIdentifier']} skipped as it is part"
f" of cluster {instance['DBClusterIdentifier']}")
continue
# check if marker tag is present
if tag_name in d_tags and d_tags[
tag_name] in SHELVERY_DO_BACKUP_TAGS:
resource = EntityResource(instance['DBInstanceIdentifier'],
local_region,
instance['InstanceCreateTime'],
d_tags)
db_entities.append(resource)
return db_entities
def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
volumes = self.collect_volumes(tag_name)
return list(
map(
lambda vol: EntityResource(
resource_id=vol['VolumeId'],
resource_region=self.region,
date_created=vol['CreateTime'],
tags=dict(
map(lambda t: (t['Key'], t['Value']), vol['Tags']))),
volumes))
def _convert_instances_to_entities(instances):
"""
Params:
instances: a list of Reservations (i.e. the response from `aws ec2 describe-instances`)
"""
local_region = boto3.session.Session().region_name
entities = []
for reservation in instances['Reservations']:
for instance in reservation['Instances']:
tags = {}
if 'Tags' in instance:
tags = dict(map(lambda tag: (tag['Key'], tag['Value']), instance['Tags']))
entities.append(EntityResource(resource_id=instance['InstanceId'], resource_region=local_region, date_created=instance['LaunchTime'], tags=tags))
return entities