def execute(self): self._logger_.info("{}, version {}", str(self.__class__).split(".")[-1], self.properties[ACTION_VERSION]) self._logger_.debug("Implementation {}", __name__) self._logger_.info(INF_BACKUP, self.config_table) scan_args = {"TableName": self.config_table} backup_config_items = [] dynamodb_client = get_client_with_retries("dynamodb", ["scan"], context=self._context_, session=self._session_, logger=self._logger_) # get all configuration items while True: if self.time_out(): break resp = dynamodb_client.scan_with_retries(**scan_args) for item in resp.get("Items", []): if self.debug: self._logger_.debug(json.dumps(item)) backup_config_items.append(item) if "LastEvaluatedKey" in resp: scan_args["ExclusiveStartKey"] = resp["LastEvaluatedKey"] else: break s3_client = get_client_with_retries("s3", methods=["put_object"], context=self._context_, session=self._session_, logger=self._logger_) # create name of object in s3 dt = self._datetime_.now() backup_object_key = BACKUP_OBJECT_KEY_TEMPLATE.format( self.S3Prefix, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) backup_data = json.dumps(backup_config_items, indent=3) resp = s3_client.put_object_with_retries(Body=backup_data, Bucket=self.S3Bucket, Key=backup_object_key) if self.debug: self._logger_.debug(resp) return { "backed-up-config-items": len(backup_config_items), "backup-name": backup_object_key, "backup-bucket": self.S3Bucket }
def start_instances(self, kwargs): self._init_scheduler(kwargs) methods = ["start_db_instance", "add_tags_to_resource", "remove_tags_from_resource"] client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region) start_tags = self._validate_rds_tag_values(kwargs[schedulers.PARAM_CONFIG].started_tags) stop_tags_keys = [t["Key"] for t in self._config.stopped_tags] started_instances = kwargs["started_instances"] for inst in started_instances: arn = RDS_DB_ARN.format(inst.region, inst.account, inst.id) try: client.start_db_instance_with_retries(DBInstanceIdentifier=inst.id) try: if stop_tags_keys is not None and len(stop_tags_keys): self._logger.info(INF_REMOVE_KEYS, "stop", ",".join(["\"{}\"".format(k) for k in stop_tags_keys]), arn) client.remove_tags_from_resource_with_retries(ResourceName=arn, TagKeys=stop_tags_keys) if start_tags is not None and len(start_tags) > 0: self._logger.info(INF_ADD_TAGS, "start", str(start_tags), arn) client.add_tags_to_resource_with_retries(ResourceName=arn, Tags=start_tags) except Exception as ex: self._logger.warning(WARN_TAGGING_STARTED, inst.id, str(ex)) yield inst.id, InstanceSchedule.STATE_RUNNING except Exception as ex: self._logger.error(ERR_STARTING_INSTANCE, inst.instance_str, str(ex)) return
def start_instances(self, kwargs): def is_in_starting_state(state): return (state & 0xFF) in Ec2Service.EC2_STARTING_STATES self._init_scheduler(kwargs) instances_to_start = kwargs[schedulers.PARAM_STARTED_INSTANCES] start_tags = kwargs[schedulers.PARAM_CONFIG].started_tags if start_tags is None: start_tags = [] start_tags_key_names = [t["Key"] for t in start_tags] stop_tags_keys = [{"Key": t["Key"]} for t in kwargs[schedulers.PARAM_CONFIG].stopped_tags if t["Key"] not in start_tags_key_names] client = get_client_with_retries("ec2", ["start_instances", "describe_instances", "create_tags", "delete_tags"], context=self._context, session=self._session, region=self._region) for instance_batch in self.instance_batches(instances_to_start, START_BATCH_SIZE): instance_ids = [i.id for i in list(instance_batch)] try: start_resp = client.start_instances_with_retries(InstanceIds=instance_ids) instances_starting = [i["InstanceId"] for i in start_resp.get("StartingInstances", []) if is_in_starting_state(i.get("CurrentState", {}).get("Code", ""))] get_status_count = 0 if len(instances_starting) < len(instance_ids): time.sleep(5) instances_starting = [i["InstanceId"] for i in self.get_instance_status(client, instance_ids) if is_in_starting_state(i.get("State", {}).get("Code", ""))] if len(instances_starting) == len(instance_ids): break get_status_count += 1 if get_status_count > 3: for i in instance_ids: if i not in instances_starting: self._logger.warning(WARNING_INSTANCE_NOT_STARTING, i) break if len(instances_starting) > 0: try: if stop_tags_keys is not None and len(stop_tags_keys) > 0: self._logger.info(INFO_REMOVING_KEYS, "stop", ",".join(["\"{}\"".format(k["Key"]) for k in stop_tags_keys]), ",".join(instances_starting)) client.delete_tags_with_retries(Resources=instances_starting, Tags=stop_tags_keys) if len(start_tags) > 0: self._logger.info(INF_ADD_KEYS, "start", str(start_tags), ",".join(instances_starting)) client.create_tags_with_retries(Resources=instances_starting, Tags=start_tags) except Exception as ex: self._logger.warning(WARN_STARTED_INSTANCES_TAGGING, ','.join(instances_starting), str(ex)) for i in instances_starting: yield i, InstanceSchedule.STATE_RUNNING except Exception as ex: self._logger.error(ERR_STARTING_INSTANCES, ",".join(instance_ids), str(ex))
def write(self, data, key): s3_client = get_client_with_retries("s3", ["put_object"], context=self._context, logger=self._logger) s3_client.put_object_with_retries(Bucket=os.getenv(ENV_REPORT_BUCKET), Key=key, Body=data)
def set_lambda_logs_retention_period(self): """ Aligns retention period for default Lambda log streams with settings :return: """ if self.context is None: return True loggroup = self.context.log_group_name log_client = get_client_with_retries( "logs", ["delete_retention_policy", "put_retention_policy"], context=self.context) retention_days = self.arguments.get("log_retention_days", 30) try: if retention_days is None: self._logger.info(INFO_DELETE_LOG_RETENTION_POLICY, loggroup) log_client.delete_retention_policy_with_retries(loggroup) return True else: self._logger.info(INFO_SET_LOG_RETENTION_POLICY, loggroup, retention_days) log_client.put_retention_policy_with_retries( logGroupName=loggroup, retentionInDays=int(retention_days)) return True except Exception as ex: self._logger.warning(ERR_SETTING_RETENTION_LAMBDA_LOGGROUP, self.context.log_group_name, str(ex)) return True
def stop_instances(self, kwargs): self._init_scheduler(kwargs) methods = ["stop_db_instance", "stop_db_cluster", "describe_db_snapshots", "delete_db_snapshot", "add_tags_to_resource", "remove_tags_from_resource"] client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region) stopped_instances = kwargs["stopped_instances"] for rds_resource in stopped_instances: try: if rds_resource.is_cluster: client.stop_db_cluster_with_retries(DBClusterIdentifier=rds_resource.id) self._logger.info(INF_STOPPED_RESOURCE, "cluster", rds_resource.id) else: self._stop_instance(client, rds_resource) self._tag_stopped_resource(client, rds_resource) yield rds_resource.id, InstanceSchedule.STATE_STOPPED except Exception as ex: self._logger.error(ERR_STOPPING_INSTANCE, "cluster" if rds_resource.is_cluster else "instance", rds_resource.instance_str, str(ex))
def start_instances(self, kwargs): self._init_scheduler(kwargs) methods = [ "start_db_instance", "start_db_cluster", "add_tags_to_resource", "remove_tags_from_resource" ] client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region) started_instances = kwargs["started_instances"] for rds_resource in started_instances: try: if rds_resource.is_cluster: client.start_db_cluster_with_retries( DBClusterIdentifier=rds_resource.id) else: client.start_db_instance_with_retries( DBInstanceIdentifier=rds_resource.id) self._tag_started_instances(client, rds_resource) yield rds_resource.id, InstanceSchedule.STATE_RUNNING except Exception as ex: self._logger.error( ERR_STARTING_INSTANCE, "cluster" if rds_resource.is_cluster else "instance", rds_resource.instance_str, str(ex)) return
def _get_cloudwatch_events_client(context): """ Builds client for making boto3 CloudWatch rule api calls with retries :param context: :return: """ return boto_retry.get_client_with_retries("events", ["enable_rule", "disable_rule", "list_rules", "put_rule"], context)
def tag_shared_snapshots(snapshot_data, snap_ids): if self.accounts_with_create_permissions not in ["", None] and self.tag_shared_snapshots: for account in self.accounts_with_create_permissions: session_for_tagging = self.get_action_session(account=account, param_name=PARAM_SHARED_ACCOUNT_TAGGING_ROLENAME, logger=self._logger_) if session_for_tagging is None: self._logger_.error(ERR_TAGS_NOT_SET_IN_ACCOUNT, account) continue try: ec2_client = get_client_with_retries(service_name="ec2", methods=[ "create_tags", "delete_tags" ], context=self._context_, region=self._region_, session=session_for_tagging, logger=self._logger_) for snap_id in snap_ids: tags = snapshot_data.get(snap_id, {}).get("tags", None) if tags is not None: self._logger_.info(INFO_SET_SNAPSHOT_TAGS_SHARED, safe_json(tags, indent=3), snap_id, account, self._region_) tagging.set_ec2_tags(ec2_client=ec2_client, resource_ids=[snap_id], tags=tags, logger=self._logger_) except Exception as ex: raise Exception(ERR_SETTING_SHARED_TAGS.format(account, str(ex)))
def delete_templates(self): s3_client = get_client_with_retries("s3", ["delete_object"], context=self.context) s3_key = "" try: for action_name in actions.all_actions(): action_properties = actions.get_action_properties(action_name) if not action_properties.get(actions.ACTION_INTERNAL, False): self._logger.info(INF_DELETING_ACTION_TEMPLATE, action_name) s3_key = S3_KEY_ACTION_CONFIGURATION_TEMPLATE.format( action_name) s3_client.delete_object_with_retries( Bucket=self.configuration_bucket, Key=s3_key) except Exception as ex: self._logger.error(ERR_DELETE_CONFIG_ITEM, s3_key, self.configuration_bucket, str(ex)) self._logger.info(INF_DELETE_ALL_ACTIONS_TEMPLATE) for key in [ S3_KEY_ACTIONS_HTML_PAGE, S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS, S3_KEY_ACCOUNT_CONFIG_CREATE_ALL, S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE ]: try: s3_client.delete_object_with_retries( Bucket=self.configuration_bucket, Key=key) except Exception as ex: self._logger.error(ERR_DELETE_CONFIG_ITEM, key, self.configuration_bucket, str(ex))
def instance_tags(self): if self._instance_tags is None: tag_client = get_client_with_retries("resourcegroupstaggingapi", methods=["get_resources"], session=self._session, context=self._context, region=self._region) args = { "TagFilters": [{"Key": self._tagname}], "ResourcesPerPage": 50, "ResourceTypeFilters": ["rds:db"] } self._instance_tags = {} while True: resp = tag_client.get_resources_with_retries(**args) for resource in resp.get("ResourceTagMappingList", []): self._instance_tags[resource["ResourceARN"]] = {tag["Key"]: tag["Value"] for tag in resource.get("Tags", {}) if tag["Key"] in ["Name", self._tagname]} if resp.get("PaginationToken", "") != "": args["PaginationToken"] = resp["PaginationToken"] else: break return self._instance_tags
def metrics_client(self): if self._metrics_client is None: self._metrics_client = boto_retry.get_client_with_retries( "cloudwatch", ["put_metric_data"], context=self._context, logger=self._logger) return self._metrics_client
def delete_templates(self): """ Deletes cross-account role and configuration templates :return: """ s3_client = get_client_with_retries("s3", ["delete_object"], context=self.context) s3_key = "" try: for action_name in actions.all_actions(): action_properties = actions.get_action_properties(action_name) if not action_properties.get(actions.ACTION_INTERNAL, False): self._logger.info(ERR_DELETING_ACTION_TEMPLATE, action_name) s3_key = S3_CONFIGURATION_TEMPLATE.format(action_name) s3_client.delete_object_with_retries( Bucket=self.configuration_bucket, Key=s3_key) s3_key = S3_ROLES_TEMPLATE.format(action_name) s3_client.delete_object_with_retries( Bucket=self.configuration_bucket, Key=s3_key) except Exception as ex: self._logger.error(ERR_DELETE_TEMPLATE_, s3_key, str(ex)) try: self._logger.info(INF_DELETE_ALL_ACTIONS_TEMPLATE, s3_key) s3_key = S3_ROLES_TEMPLATE.format(ALL_ACTIONS_TEMPLATE_NAME) s3_client.delete_object_with_retries( Bucket=self.configuration_bucket, Key=s3_key) except Exception as ex: self._logger.error(ERR_DELETE_TEMPLATE_, s3_key, str(ex))
def stop_instances(self, **kwargs): def does_snapshot_exist(name): try: resp = client.describe_db_snapshots_with_retries(DBSnapshotIdentifier=name, SnapshotType="manual") snapshot = resp.get("DBSnapshots", None) return snapshot is not None except Exception as e: if type(e).__name__ == "DBSnapshotNotFoundFault": return False else: raise ex self._init_scheduler(kwargs) methods = ["stop_db_instance", "describe_db_snapshots", "delete_db_snapshot", "add_tags_to_resource", "remove_tags_from_resource"] client = get_client_with_retries("rds", methods, context=self._context, session=self._session, region=self._region) stop_tags = self._config.stopped_tags start_tags_keys = [t["Key"] for t in self._config.started_tags] stopped_instances = kwargs["stopped_instances"] action = "" for inst in stopped_instances: try: arn = RDS_DB_ARN.format(inst.region, inst.account, inst.id) snapshot_name = "{}-stopped-{}".format(self._stack_name, inst.id).replace(" ", "") action = "describe_db_snapshots" if does_snapshot_exist(snapshot_name): action = "delete_db_snapshot" client.delete_db_snapshot_with_retries(DBSnapshotIdentifier=snapshot_name) self._logger.info(INF_DELETE_SNAPSHOT, snapshot_name) action = "stop_db_instance" client.stop_db_instance_with_retries(DBInstanceIdentifier=inst.id, DBSnapshotIdentifier=snapshot_name) self._logger.info(INF_STOPPED_INSTANCE, inst.id, snapshot_name) yield inst.id, InstanceSchedule.STATE_STOPPED except Exception as ex: self._logger.error(ERR_STOPPING_INSTANCE, inst.instance_str, action, str(ex)) return try: if start_tags_keys is not None and len(start_tags_keys): client.remove_tags_from_resource_with_retries(ResourceName=arn, TagKeys=start_tags_keys) if stop_tags is not None and len(stop_tags) > 0: client.add_tags_to_resource_with_retries(ResourceName=arn, Tags=stop_tags) except Exception as ex: self._logger.warning(WARN_TAGGING_STOPPED, inst.id, str(ex))
def _dynamodb_client(self): """ Returns boto3 dynamodb client :return: """ if self._client is None: self._client = boto_retry.get_client_with_retries("dynamodb", ["batch_write_item"], context=self._context) return self._client
def ec2_client(self): if self._ec2_client is None: methods = ["create_tags", "delete_tags"] self._ec2_client = get_client_with_retries("ec2", methods, region=self._region_, session=self._session_, logger=self._logger_) return self._ec2_client
def get_item_resource_data(item, context): global _kms_client resource_data = item.get(TASK_TR_RESOURCES, "{}") if item.get(TASK_TR_ENCRYPTED_RESOURCES): if _kms_client is None: _kms_client = boto_retry.get_client_with_retries("kms", ["decrypt"], context=context) resource_data = _kms_client.decrypt(CiphertextBlob=base64.b64decode(resource_data))["Plaintext"] return resource_data if type(resource_data) in [dict, list] else json.loads(resource_data)
def get_schedulable_instances(self, kwargs): self._session = kwargs[schedulers.PARAM_SESSION] context = kwargs[schedulers.PARAM_CONTEXT] region = kwargs[schedulers.PARAM_REGION] account = kwargs[schedulers.PARAM_ACCOUNT] self._logger = kwargs[schedulers.PARAM_LOGGER] tagname = kwargs[schedulers.PARAM_CONFIG].tag_name config = kwargs[schedulers.PARAM_CONFIG] self.schedules_with_hibernation = [ s.name for s in config.schedules.values() if s.hibernate ] client = get_client_with_retries("ec2", ["describe_instances"], context=context, session=self._session, region=region) def is_in_schedulable_state(ec2_inst): state = ec2_inst["state"] & 0xFF return state in Ec2Service.EC2_SCHEDULABLE_STATES jmes = "Reservations[*].Instances[*].{InstanceId:InstanceId, EbsOptimized:EbsOptimized, Tags:Tags, " \ "InstanceType:InstanceType,State:State}[]" + \ "|[?Tags]|[?contains(Tags[*].Key, '{}')]".format(tagname) args = {} number_of_instances = 0 instances = [] done = False self._logger.info(INF_FETCHING_INSTANCES, account, region) while not done: ec2_resp = client.describe_instances_with_retries(**args) for reservation_inst in jmespath.search(jmes, ec2_resp): inst = self._select_instance_data(instance=reservation_inst, tagname=tagname, config=config) number_of_instances += 1 if is_in_schedulable_state(inst): instances.append(inst) self._logger.debug(DEBUG_SELECTED_INSTANCE, inst[schedulers.INST_ID], inst[schedulers.INST_STATE_NAME]) else: self._logger.debug(DEBUG_SKIPPED_INSTANCE, inst[schedulers.INST_ID], inst[schedulers.INST_STATE_NAME]) if "NextToken" in ec2_resp: args["NextToken"] = ec2_resp["NextToken"] else: done = True self._logger.info(INF_FETCHED_INSTANCES, number_of_instances, len(instances)) return instances
def lambda_client(self): """ Get the lambda client :return: lambda client """ if self._lambda_client is None: self._lambda_client = get_client_with_retries( "lambda", ["invoke"], context=self._context) return self._lambda_client
def purge_table(table_name, key_name): db = get_client_with_retries( "dynamodb", ["scan", "delete_item", "batch_write_item"], session=boto3.Session()) def keys_to_delete(): key_ids = [] delete_args = { "TableName": table_name, "Select": "SPECIFIC_ATTRIBUTES", "ConsistentRead": True, "AttributesToGet": [key_name] } while True: resp = db.scan_with_retries(**delete_args) for item in resp.get("Items", []): key_ids.append(item[key_name]["S"]) delete_args["ExclusiveStartKey"] = resp.get( "LastEvaluatedKey", None) if delete_args["ExclusiveStartKey"] is None: break return key_ids keys = keys_to_delete() print("{} items to delete".format(len(keys))) # delete items in batches of max 25 items while True: delete_requests = [] while len(keys) > 0: delete_requests.append( {'DeleteRequest': { 'Key': { key_name: { "S": keys.pop(0) } } }}) if len(keys) == 0 or len(delete_requests) == 25: db.batch_write_item_with_retries( RequestItems={table_name: delete_requests}) print(".", end="") time.sleep(1) delete_requests = [] time.sleep(1) keys = keys_to_delete() if len(keys) == 0: break print("")
def s3_client(self): """ Returns S3 client for handling configuration files :return: S3 client """ if self._s3_client is None: # IMPORTANT, list_objects and list_objects_v2 require s3:ListBucket permission !! self._s3_client = get_client_with_retries( "s3", ["list_objects_v2", "get_object"]) return self._s3_client
def ec2_client(self): if self._ec2_client is None: self._ec2_client = get_client_with_retries( "ec2", methods=["delete_snapshot"], region=self._region_, context=self._context_, session=self._session_, logger=self._logger_) return self._ec2_client
def metrics_client(self): if self._metrics_client is None: methods = ["get_metric_data"] self._metrics_client = get_client_with_retries( "cloudwatch", methods, region=self._region_, session=self._session_, logger=self._logger_) return self._metrics_client
def client(self): if self._client is None: self._client = get_client_with_retries("dynamodb", methods=[ "batch_write_item" ], context=self._context_, session=self._session_, logger=self._logger_) return self._client
def get_ec2_client(): if client is not None: return client methods = ["create_tags", "delete_tags"] return get_client_with_retries("ec2", methods=methods, region=region, session=self._session_, logger=self._logger_)
def stack_resources(self): """ Reads the action stack resources :return: Stack resources for the action """ if self._stack_resources is None: self._stack_resources = {} # test if this action has additional stack resources resources = self.action_properties.get( actions.ACTION_STACK_RESOURCES, {}) if resources: # name of the class class_name = self.action_properties[ actions.ACTION_CLASS_NAME][0:-len("Action")] # actual resource names is name of class + name from class properties logical_resource_names = [ class_name + resource_name for resource_name in resources ] cfn = get_client_with_retries("cloudformation", ["list_stack_resources"], context=self._context) args = {"StackName": self.stack_id} while True: # get the stack resources cfn_resp = cfn.list_stack_resources_with_retries(**args) for res in cfn_resp.get("StackResourceSummaries", []): # actual name logical_resource_id = res["LogicalResourceId"] # test if this resource is an resource from the action properties if logical_resource_id in logical_resource_names: self._stack_resources[ logical_resource_id[len(class_name):]] = { i: res[i] for i in [ "LogicalResourceId", "PhysicalResourceId", "ResourceType" ] } # test if we've found the number of resources that we declared, in that case no need to read more if len(list(self._stack_resources.keys())) == len( resources): return self._stack_resources # continuation if > 100 resources in stack if "NextToken" in cfn_resp: args["NextToken"] = cfn_resp["NextToken"] else: break return self._stack_resources
def this_account(self): """ Returns the AWS account number :return: AWS account number """ if self._this_account is None: client = boto_retry.get_client_with_retries( "sts", ["get_caller_identity"], context=self._context) self._this_account = client.get_caller_identity_with_retries( )["Account"] return self._this_account
def stack_resources(self): if self._stack_resources is None: cfn = get_client_with_retries("cloudformation", ["list_stack_resources"]) resp = cfn.list_stack_resources_with_retries( StackName=self.stack_name) self._stack_resources = { r["LogicalResourceId"]: r["PhysicalResourceId"] for r in resp.get("StackResourceSummaries", []) if r.get("PhysicalResourceId", None) is not None } return self._stack_resources
def get_aws_account(sts=None): """ Returns the current AWS account :param sts: Optional sts reused sts client :return: """ client = sts if sts is not None else get_client_with_retries( "sts", ["get_caller_identity"]) if getattr(client, "get_caller_identity_with_retries", None): return client.get_caller_identity_with_retries()["Account"] else: return client.get_caller_identity()["Account"]
def ec2_source_client(self): if self._ec2_source_client is None: methods = ["create_tags", "delete_tags", "delete_snapshot"] self._ec2_source_client = get_client_with_retries( "ec2", methods=methods, region=self.source_region, context=self._context_, session=self._session_, logger=self._logger_) return self._ec2_source_client