def execute(self):
self._logger_.info("{}, version {}",
str(self.__class__).split(".")[-1],
self.properties[ACTION_VERSION])
self._logger_.debug("Implementation {}", __name__)
self._logger_.info(INF_BACKUP, self.config_table)
scan_args = {"TableName": self.config_table}
backup_config_items = []
dynamodb_client = get_client_with_retries("dynamodb", ["scan"],
context=self._context_,
session=self._session_,
logger=self._logger_)
# get all configuration items
while True:
if self.time_out():
break
resp = dynamodb_client.scan_with_retries(**scan_args)
for item in resp.get("Items", []):
if self.debug:
self._logger_.debug(json.dumps(item))
backup_config_items.append(item)
if "LastEvaluatedKey" in resp:
scan_args["ExclusiveStartKey"] = resp["LastEvaluatedKey"]
else:
break
s3_client = get_client_with_retries("s3",
methods=["put_object"],
context=self._context_,
session=self._session_,
logger=self._logger_)
# create name of object in s3
dt = self._datetime_.now()
backup_object_key = BACKUP_OBJECT_KEY_TEMPLATE.format(
self.S3Prefix, dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second)
backup_data = json.dumps(backup_config_items, indent=3)
resp = s3_client.put_object_with_retries(Body=backup_data,
Bucket=self.S3Bucket,
Key=backup_object_key)
if self.debug:
self._logger_.debug(resp)
return {
"backed-up-config-items": len(backup_config_items),
"backup-name": backup_object_key,
"backup-bucket": self.S3Bucket
}
def start_instances(self, kwargs):
self._init_scheduler(kwargs)
methods = ["start_db_instance",
"add_tags_to_resource",
"remove_tags_from_resource"]
client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region)
start_tags = self._validate_rds_tag_values(kwargs[schedulers.PARAM_CONFIG].started_tags)
stop_tags_keys = [t["Key"] for t in self._config.stopped_tags]
started_instances = kwargs["started_instances"]
for inst in started_instances:
arn = RDS_DB_ARN.format(inst.region, inst.account, inst.id)
try:
client.start_db_instance_with_retries(DBInstanceIdentifier=inst.id)
try:
if stop_tags_keys is not None and len(stop_tags_keys):
self._logger.info(INF_REMOVE_KEYS, "stop",
",".join(["\"{}\"".format(k) for k in stop_tags_keys]), arn)
client.remove_tags_from_resource_with_retries(ResourceName=arn, TagKeys=stop_tags_keys)
if start_tags is not None and len(start_tags) > 0:
self._logger.info(INF_ADD_TAGS, "start", str(start_tags), arn)
client.add_tags_to_resource_with_retries(ResourceName=arn, Tags=start_tags)
except Exception as ex:
self._logger.warning(WARN_TAGGING_STARTED, inst.id, str(ex))
yield inst.id, InstanceSchedule.STATE_RUNNING
except Exception as ex:
self._logger.error(ERR_STARTING_INSTANCE, inst.instance_str, str(ex))
return
def start_instances(self, kwargs):
def is_in_starting_state(state):
return (state & 0xFF) in Ec2Service.EC2_STARTING_STATES
self._init_scheduler(kwargs)
instances_to_start = kwargs[schedulers.PARAM_STARTED_INSTANCES]
start_tags = kwargs[schedulers.PARAM_CONFIG].started_tags
if start_tags is None:
start_tags = []
start_tags_key_names = [t["Key"] for t in start_tags]
stop_tags_keys = [{"Key": t["Key"]} for t in kwargs[schedulers.PARAM_CONFIG].stopped_tags if
t["Key"] not in start_tags_key_names]
client = get_client_with_retries("ec2", ["start_instances", "describe_instances", "create_tags", "delete_tags"],
context=self._context, session=self._session, region=self._region)
for instance_batch in self.instance_batches(instances_to_start, START_BATCH_SIZE):
instance_ids = [i.id for i in list(instance_batch)]
try:
start_resp = client.start_instances_with_retries(InstanceIds=instance_ids)
instances_starting = [i["InstanceId"] for i in start_resp.get("StartingInstances", []) if
is_in_starting_state(i.get("CurrentState", {}).get("Code", ""))]
get_status_count = 0
if len(instances_starting) < len(instance_ids):
time.sleep(5)
instances_starting = [i["InstanceId"] for i in self.get_instance_status(client, instance_ids) if
is_in_starting_state(i.get("State", {}).get("Code", ""))]
if len(instances_starting) == len(instance_ids):
break
get_status_count += 1
if get_status_count > 3:
for i in instance_ids:
if i not in instances_starting:
self._logger.warning(WARNING_INSTANCE_NOT_STARTING, i)
break
if len(instances_starting) > 0:
try:
if stop_tags_keys is not None and len(stop_tags_keys) > 0:
self._logger.info(INFO_REMOVING_KEYS, "stop",
",".join(["\"{}\"".format(k["Key"]) for k in stop_tags_keys]),
",".join(instances_starting))
client.delete_tags_with_retries(Resources=instances_starting, Tags=stop_tags_keys)
if len(start_tags) > 0:
self._logger.info(INF_ADD_KEYS, "start", str(start_tags), ",".join(instances_starting))
client.create_tags_with_retries(Resources=instances_starting, Tags=start_tags)
except Exception as ex:
self._logger.warning(WARN_STARTED_INSTANCES_TAGGING, ','.join(instances_starting), str(ex))
for i in instances_starting:
yield i, InstanceSchedule.STATE_RUNNING
except Exception as ex:
self._logger.error(ERR_STARTING_INSTANCES, ",".join(instance_ids), str(ex))
def write(self, data, key):
s3_client = get_client_with_retries("s3", ["put_object"],
context=self._context,
logger=self._logger)
s3_client.put_object_with_retries(Bucket=os.getenv(ENV_REPORT_BUCKET),
Key=key,
Body=data)
def set_lambda_logs_retention_period(self):
"""
Aligns retention period for default Lambda log streams with settings
:return:
"""
if self.context is None:
return True
loggroup = self.context.log_group_name
log_client = get_client_with_retries(
"logs", ["delete_retention_policy", "put_retention_policy"],
context=self.context)
retention_days = self.arguments.get("log_retention_days", 30)
try:
if retention_days is None:
self._logger.info(INFO_DELETE_LOG_RETENTION_POLICY, loggroup)
log_client.delete_retention_policy_with_retries(loggroup)
return True
else:
self._logger.info(INFO_SET_LOG_RETENTION_POLICY, loggroup,
retention_days)
log_client.put_retention_policy_with_retries(
logGroupName=loggroup, retentionInDays=int(retention_days))
return True
except Exception as ex:
self._logger.warning(ERR_SETTING_RETENTION_LAMBDA_LOGGROUP,
self.context.log_group_name, str(ex))
return True
def stop_instances(self, kwargs):
self._init_scheduler(kwargs)
methods = ["stop_db_instance",
"stop_db_cluster",
"describe_db_snapshots",
"delete_db_snapshot",
"add_tags_to_resource",
"remove_tags_from_resource"]
client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region)
stopped_instances = kwargs["stopped_instances"]
for rds_resource in stopped_instances:
try:
if rds_resource.is_cluster:
client.stop_db_cluster_with_retries(DBClusterIdentifier=rds_resource.id)
self._logger.info(INF_STOPPED_RESOURCE, "cluster", rds_resource.id)
else:
self._stop_instance(client, rds_resource)
self._tag_stopped_resource(client, rds_resource)
yield rds_resource.id, InstanceSchedule.STATE_STOPPED
except Exception as ex:
self._logger.error(ERR_STOPPING_INSTANCE, "cluster" if rds_resource.is_cluster else "instance",
rds_resource.instance_str, str(ex))
def start_instances(self, kwargs):
self._init_scheduler(kwargs)
methods = [
"start_db_instance", "start_db_cluster", "add_tags_to_resource",
"remove_tags_from_resource"
]
client = get_client_with_retries("rds",
methods,
context=self._context,
session=self._session,
region=self._region)
started_instances = kwargs["started_instances"]
for rds_resource in started_instances:
try:
if rds_resource.is_cluster:
client.start_db_cluster_with_retries(
DBClusterIdentifier=rds_resource.id)
else:
client.start_db_instance_with_retries(
DBInstanceIdentifier=rds_resource.id)
self._tag_started_instances(client, rds_resource)
yield rds_resource.id, InstanceSchedule.STATE_RUNNING
except Exception as ex:
self._logger.error(
ERR_STARTING_INSTANCE,
"cluster" if rds_resource.is_cluster else "instance",
rds_resource.instance_str, str(ex))
return
def _get_cloudwatch_events_client(context):
"""
Builds client for making boto3 CloudWatch rule api calls with retries
:param context:
:return:
"""
return boto_retry.get_client_with_retries("events", ["enable_rule", "disable_rule", "list_rules", "put_rule"], context)
def tag_shared_snapshots(snapshot_data, snap_ids):
if self.accounts_with_create_permissions not in ["", None] and self.tag_shared_snapshots:
for account in self.accounts_with_create_permissions:
session_for_tagging = self.get_action_session(account=account,
param_name=PARAM_SHARED_ACCOUNT_TAGGING_ROLENAME,
logger=self._logger_)
if session_for_tagging is None:
self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account)
continue
try:
ec2_client = get_client_with_retries(service_name="ec2",
methods=[
"create_tags",
"delete_tags"
],
context=self._context_,
region=self._region_,
session=session_for_tagging,
logger=self._logger_)
for snap_id in snap_ids:
tags = snapshot_data.get(snap_id, {}).get("tags", None)
if tags is not None:
self._logger_.info(INFO_SET_SNAPSHOT_TAGS_SHARED, safe_json(tags, indent=3), snap_id, account,
self._region_)
tagging.set_ec2_tags(ec2_client=ec2_client,
resource_ids=[snap_id],
tags=tags,
logger=self._logger_)
except Exception as ex:
raise Exception(ERR_SETTING_SHARED_TAGS.format(account, str(ex)))
def delete_templates(self):
s3_client = get_client_with_retries("s3", ["delete_object"],
context=self.context)
s3_key = ""
try:
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_INTERNAL, False):
self._logger.info(INF_DELETING_ACTION_TEMPLATE,
action_name)
s3_key = S3_KEY_ACTION_CONFIGURATION_TEMPLATE.format(
action_name)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
except Exception as ex:
self._logger.error(ERR_DELETE_CONFIG_ITEM, s3_key,
self.configuration_bucket, str(ex))
self._logger.info(INF_DELETE_ALL_ACTIONS_TEMPLATE)
for key in [
S3_KEY_ACTIONS_HTML_PAGE,
S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS,
S3_KEY_ACCOUNT_CONFIG_CREATE_ALL,
S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE
]:
try:
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=key)
except Exception as ex:
self._logger.error(ERR_DELETE_CONFIG_ITEM, key,
self.configuration_bucket, str(ex))
def instance_tags(self):
if self._instance_tags is None:
tag_client = get_client_with_retries("resourcegroupstaggingapi",
methods=["get_resources"],
session=self._session,
context=self._context,
region=self._region)
args = {
"TagFilters": [{"Key": self._tagname}],
"ResourcesPerPage": 50,
"ResourceTypeFilters": ["rds:db"]
}
self._instance_tags = {}
while True:
resp = tag_client.get_resources_with_retries(**args)
for resource in resp.get("ResourceTagMappingList", []):
self._instance_tags[resource["ResourceARN"]] = {tag["Key"]: tag["Value"]
for tag in resource.get("Tags", {})
if tag["Key"] in ["Name", self._tagname]}
if resp.get("PaginationToken", "") != "":
args["PaginationToken"] = resp["PaginationToken"]
else:
break
return self._instance_tags
def metrics_client(self):
if self._metrics_client is None:
self._metrics_client = boto_retry.get_client_with_retries(
"cloudwatch", ["put_metric_data"],
context=self._context,
logger=self._logger)
return self._metrics_client
def delete_templates(self):
"""
Deletes cross-account role and configuration templates
:return:
"""
s3_client = get_client_with_retries("s3", ["delete_object"],
context=self.context)
s3_key = ""
try:
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_INTERNAL, False):
self._logger.info(ERR_DELETING_ACTION_TEMPLATE,
action_name)
s3_key = S3_CONFIGURATION_TEMPLATE.format(action_name)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
s3_key = S3_ROLES_TEMPLATE.format(action_name)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
except Exception as ex:
self._logger.error(ERR_DELETE_TEMPLATE_, s3_key, str(ex))
try:
self._logger.info(INF_DELETE_ALL_ACTIONS_TEMPLATE, s3_key)
s3_key = S3_ROLES_TEMPLATE.format(ALL_ACTIONS_TEMPLATE_NAME)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
except Exception as ex:
self._logger.error(ERR_DELETE_TEMPLATE_, s3_key, str(ex))
def stop_instances(self, **kwargs):
def does_snapshot_exist(name):
try:
resp = client.describe_db_snapshots_with_retries(DBSnapshotIdentifier=name, SnapshotType="manual")
snapshot = resp.get("DBSnapshots", None)
return snapshot is not None
except Exception as e:
if type(e).__name__ == "DBSnapshotNotFoundFault":
return False
else:
raise ex
self._init_scheduler(kwargs)
methods = ["stop_db_instance",
"describe_db_snapshots",
"delete_db_snapshot",
"add_tags_to_resource",
"remove_tags_from_resource"]
client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region)
stop_tags = self._config.stopped_tags
start_tags_keys = [t["Key"] for t in self._config.started_tags]
stopped_instances = kwargs["stopped_instances"]
action = ""
for inst in stopped_instances:
try:
arn = RDS_DB_ARN.format(inst.region, inst.account, inst.id)
snapshot_name = "{}-stopped-{}".format(self._stack_name, inst.id).replace(" ", "")
action = "describe_db_snapshots"
if does_snapshot_exist(snapshot_name):
action = "delete_db_snapshot"
client.delete_db_snapshot_with_retries(DBSnapshotIdentifier=snapshot_name)
self._logger.info(INF_DELETE_SNAPSHOT, snapshot_name)
action = "stop_db_instance"
client.stop_db_instance_with_retries(DBInstanceIdentifier=inst.id, DBSnapshotIdentifier=snapshot_name)
self._logger.info(INF_STOPPED_INSTANCE, inst.id, snapshot_name)
yield inst.id, InstanceSchedule.STATE_STOPPED
except Exception as ex:
self._logger.error(ERR_STOPPING_INSTANCE, inst.instance_str, action, str(ex))
return
try:
if start_tags_keys is not None and len(start_tags_keys):
client.remove_tags_from_resource_with_retries(ResourceName=arn, TagKeys=start_tags_keys)
if stop_tags is not None and len(stop_tags) > 0:
client.add_tags_to_resource_with_retries(ResourceName=arn, Tags=stop_tags)
except Exception as ex:
self._logger.warning(WARN_TAGGING_STOPPED, inst.id, str(ex))
def _dynamodb_client(self):
"""
Returns boto3 dynamodb client
:return:
"""
if self._client is None:
self._client = boto_retry.get_client_with_retries("dynamodb", ["batch_write_item"], context=self._context)
return self._client
def ec2_client(self):
if self._ec2_client is None:
methods = ["create_tags",
"delete_tags"]
self._ec2_client = get_client_with_retries("ec2", methods, region=self._region_,
session=self._session_, logger=self._logger_)
return self._ec2_client
def get_item_resource_data(item, context):
global _kms_client
resource_data = item.get(TASK_TR_RESOURCES, "{}")
if item.get(TASK_TR_ENCRYPTED_RESOURCES):
if _kms_client is None:
_kms_client = boto_retry.get_client_with_retries("kms", ["decrypt"], context=context)
resource_data = _kms_client.decrypt(CiphertextBlob=base64.b64decode(resource_data))["Plaintext"]
return resource_data if type(resource_data) in [dict, list] else json.loads(resource_data)
def get_schedulable_instances(self, kwargs):
self._session = kwargs[schedulers.PARAM_SESSION]
context = kwargs[schedulers.PARAM_CONTEXT]
region = kwargs[schedulers.PARAM_REGION]
account = kwargs[schedulers.PARAM_ACCOUNT]
self._logger = kwargs[schedulers.PARAM_LOGGER]
tagname = kwargs[schedulers.PARAM_CONFIG].tag_name
config = kwargs[schedulers.PARAM_CONFIG]
self.schedules_with_hibernation = [
s.name for s in config.schedules.values() if s.hibernate
]
client = get_client_with_retries("ec2", ["describe_instances"],
context=context,
session=self._session,
region=region)
def is_in_schedulable_state(ec2_inst):
state = ec2_inst["state"] & 0xFF
return state in Ec2Service.EC2_SCHEDULABLE_STATES
jmes = "Reservations[*].Instances[*].{InstanceId:InstanceId, EbsOptimized:EbsOptimized, Tags:Tags, " \
"InstanceType:InstanceType,State:State}[]" + \
"|[?Tags]|[?contains(Tags[*].Key, '{}')]".format(tagname)
args = {}
number_of_instances = 0
instances = []
done = False
self._logger.info(INF_FETCHING_INSTANCES, account, region)
while not done:
ec2_resp = client.describe_instances_with_retries(**args)
for reservation_inst in jmespath.search(jmes, ec2_resp):
inst = self._select_instance_data(instance=reservation_inst,
tagname=tagname,
config=config)
number_of_instances += 1
if is_in_schedulable_state(inst):
instances.append(inst)
self._logger.debug(DEBUG_SELECTED_INSTANCE,
inst[schedulers.INST_ID],
inst[schedulers.INST_STATE_NAME])
else:
self._logger.debug(DEBUG_SKIPPED_INSTANCE,
inst[schedulers.INST_ID],
inst[schedulers.INST_STATE_NAME])
if "NextToken" in ec2_resp:
args["NextToken"] = ec2_resp["NextToken"]
else:
done = True
self._logger.info(INF_FETCHED_INSTANCES, number_of_instances,
len(instances))
return instances
def lambda_client(self):
"""
Get the lambda client
:return: lambda client
"""
if self._lambda_client is None:
self._lambda_client = get_client_with_retries(
"lambda", ["invoke"], context=self._context)
return self._lambda_client
def purge_table(table_name, key_name):
db = get_client_with_retries(
"dynamodb", ["scan", "delete_item", "batch_write_item"],
session=boto3.Session())
def keys_to_delete():
key_ids = []
delete_args = {
"TableName": table_name,
"Select": "SPECIFIC_ATTRIBUTES",
"ConsistentRead": True,
"AttributesToGet": [key_name]
}
while True:
resp = db.scan_with_retries(**delete_args)
for item in resp.get("Items", []):
key_ids.append(item[key_name]["S"])
delete_args["ExclusiveStartKey"] = resp.get(
"LastEvaluatedKey", None)
if delete_args["ExclusiveStartKey"] is None:
break
return key_ids
keys = keys_to_delete()
print("{} items to delete".format(len(keys)))
# delete items in batches of max 25 items
while True:
delete_requests = []
while len(keys) > 0:
delete_requests.append(
{'DeleteRequest': {
'Key': {
key_name: {
"S": keys.pop(0)
}
}
}})
if len(keys) == 0 or len(delete_requests) == 25:
db.batch_write_item_with_retries(
RequestItems={table_name: delete_requests})
print(".", end="")
time.sleep(1)
delete_requests = []
time.sleep(1)
keys = keys_to_delete()
if len(keys) == 0:
break
print("")
def s3_client(self):
"""
Returns S3 client for handling configuration files
:return: S3 client
"""
if self._s3_client is None:
# IMPORTANT, list_objects and list_objects_v2 require s3:ListBucket permission !!
self._s3_client = get_client_with_retries(
"s3", ["list_objects_v2", "get_object"])
return self._s3_client
def ec2_client(self):
if self._ec2_client is None:
self._ec2_client = get_client_with_retries(
"ec2",
methods=["delete_snapshot"],
region=self._region_,
context=self._context_,
session=self._session_,
logger=self._logger_)
return self._ec2_client
def metrics_client(self):
if self._metrics_client is None:
methods = ["get_metric_data"]
self._metrics_client = get_client_with_retries(
"cloudwatch",
methods,
region=self._region_,
session=self._session_,
logger=self._logger_)
return self._metrics_client
def client(self):
if self._client is None:
self._client = get_client_with_retries("dynamodb",
methods=[
"batch_write_item"
],
context=self._context_,
session=self._session_,
logger=self._logger_)
return self._client
def get_ec2_client():
if client is not None:
return client
methods = ["create_tags", "delete_tags"]
return get_client_with_retries("ec2",
methods=methods,
region=region,
session=self._session_,
logger=self._logger_)
def stack_resources(self):
"""
Reads the action stack resources
:return: Stack resources for the action
"""
if self._stack_resources is None:
self._stack_resources = {}
# test if this action has additional stack resources
resources = self.action_properties.get(
actions.ACTION_STACK_RESOURCES, {})
if resources:
# name of the class
class_name = self.action_properties[
actions.ACTION_CLASS_NAME][0:-len("Action")]
# actual resource names is name of class + name from class properties
logical_resource_names = [
class_name + resource_name for resource_name in resources
]
cfn = get_client_with_retries("cloudformation",
["list_stack_resources"],
context=self._context)
args = {"StackName": self.stack_id}
while True:
# get the stack resources
cfn_resp = cfn.list_stack_resources_with_retries(**args)
for res in cfn_resp.get("StackResourceSummaries", []):
# actual name
logical_resource_id = res["LogicalResourceId"]
# test if this resource is an resource from the action properties
if logical_resource_id in logical_resource_names:
self._stack_resources[
logical_resource_id[len(class_name):]] = {
i: res[i]
for i in [
"LogicalResourceId",
"PhysicalResourceId", "ResourceType"
]
}
# test if we've found the number of resources that we declared, in that case no need to read more
if len(list(self._stack_resources.keys())) == len(
resources):
return self._stack_resources
# continuation if > 100 resources in stack
if "NextToken" in cfn_resp:
args["NextToken"] = cfn_resp["NextToken"]
else:
break
return self._stack_resources
def this_account(self):
"""
Returns the AWS account number
:return: AWS account number
"""
if self._this_account is None:
client = boto_retry.get_client_with_retries(
"sts", ["get_caller_identity"], context=self._context)
self._this_account = client.get_caller_identity_with_retries(
)["Account"]
return self._this_account
def stack_resources(self):
if self._stack_resources is None:
cfn = get_client_with_retries("cloudformation",
["list_stack_resources"])
resp = cfn.list_stack_resources_with_retries(
StackName=self.stack_name)
self._stack_resources = {
r["LogicalResourceId"]: r["PhysicalResourceId"]
for r in resp.get("StackResourceSummaries", [])
if r.get("PhysicalResourceId", None) is not None
}
return self._stack_resources
def get_aws_account(sts=None):
"""
Returns the current AWS account
:param sts: Optional sts reused sts client
:return:
"""
client = sts if sts is not None else get_client_with_retries(
"sts", ["get_caller_identity"])
if getattr(client, "get_caller_identity_with_retries", None):
return client.get_caller_identity_with_retries()["Account"]
else:
return client.get_caller_identity()["Account"]
def ec2_source_client(self):
if self._ec2_source_client is None:
methods = ["create_tags", "delete_tags", "delete_snapshot"]
self._ec2_source_client = get_client_with_retries(
"ec2",
methods=methods,
region=self.source_region,
context=self._context_,
session=self._session_,
logger=self._logger_)
return self._ec2_source_client
