def validate_tagfilter(tag_filter, action_name):
"""
Tests if tags are supported by the resources for the action. If this is nit the case then the use of tag filters is
not possible and an exception is raised
:param tag_filter: Tag filter value
:param action_name: Name of the action
:return: Filter if tags are supported and the filter can be used, otherwise an exception is raised
"""
if tag_filter is None or tag_filter == "":
return None
action_properties = actions.get_action_properties(action_name)
resource_supports_tags = action_properties.get(
actions.ACTION_RESOURCES) in services.create_service(
action_properties[actions.ACTION_SERVICE]).resources_with_tags
# resource does not allow tags, so tagfilters can not be used
if not resource_supports_tags:
raise ValueError(
MSG_NO_TAG_FILTER.format(
action_properties[actions.ACTION_RESOURCES], tag_filter,
action_name))
# destructive actions can deny use of wildcards for tag name
if not action_properties.get(actions.ACTION_ALLOW_TAGFILTER_WILDCARD,
True):
if "".join([s.strip() for s in tag_filter.split("=")[0:1]
]) in ["*", "**", "*="]:
raise ValueError(
MSG_NO_WILDCARS_TAG_FILTER_ALLOWED.format(
tag_filter, action_name))
return tag_filter
def generate_html_actions_page(html_file, region):
with open(html_file) as f:
html_template = "".join(f.readlines())
bucket = os.getenv(handlers.ENV_CONFIG_BUCKET)
stack = os.getenv(handlers.ENV_STACK_NAME)
action_groups = {}
for a in actions.all_actions():
ap = actions.get_action_properties(a)
if ap.get(actions.ACTION_INTERNAL):
continue
href = CFN_CONSOLE_URL_TEMPLATE.format(
region, region,
urllib.parse.quote(ap.get(actions.PARAM_DESCRIPTION, "")), region,
bucket, a)
group_name = group_name_from_action_name(a)
if group_name not in action_groups:
action_groups[group_name] = {}
action_groups[group_name][a] = (href, ap.get(actions.ACTION_TITLE))
action_list = ""
for g in sorted(action_groups.keys()):
actions_list = ""
for a in sorted(action_groups[g].keys()):
actions_list += HTML_ACTION_LIST_ITEM.format(
action_groups[g][a][0], action_groups[g][a][1])
action_list += HTML_GROUP_LIST_ITEM.format(g, actions_list)
action_list = HTML_ACTIONS_GROUPS_LISTS.format(action_list)
return html_template.replace("%actions%",
action_list).replace("%stack%", stack)
def _add_actions_permissions(self, role_actions):
"""
Adds permissions to cross account role statement for list of actions
:param role_actions: List of actions
:return:
"""
def action_select_resources_permissions(action_props):
return services.get_resource_describe_permissions(action_props[actions.ACTION_SERVICE],
action_props[actions.ACTION_RESOURCES])
action_role = self.template["Resources"]["ActionRole"]
policy_document = action_role["Properties"]["Policies"][0]["PolicyDocument"]
policy_document_statement = []
policy_document["Statement"] = policy_document_statement
for action_name in role_actions:
action_properties = actions.get_action_properties(action_name)
if action_properties.get(actions.ACTION_INTERNAL, False):
continue
# get permissions from action properties
action_permissions = action_properties.get(actions.ACTION_PERMISSIONS, [])
# get the permissions to retrieve the resources for that action
# with possible additional permissions to retrieve tags
action_permissions += action_select_resources_permissions(action_properties)
if len(action_permissions) is not 0:
policy_document_statement.append({
"Sid": re.sub("[^0-9A-Za-z]", "", action_name + str(uuid.uuid4())),
"Effect": "Allow",
"Resource": "*",
"Action": list(set(action_permissions))
})
def delete_templates(self):
s3_client = get_client_with_retries("s3", ["delete_object"],
context=self.context)
s3_key = ""
try:
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_INTERNAL, False):
self._logger.info(INF_DELETING_ACTION_TEMPLATE,
action_name)
s3_key = S3_KEY_ACTION_CONFIGURATION_TEMPLATE.format(
action_name)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
except Exception as ex:
self._logger.error(ERR_DELETE_CONFIG_ITEM, s3_key,
self.configuration_bucket, str(ex))
self._logger.info(INF_DELETE_ALL_ACTIONS_TEMPLATE)
for key in [
S3_KEY_ACTIONS_HTML_PAGE,
S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS,
S3_KEY_ACCOUNT_CONFIG_CREATE_ALL,
S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE
]:
try:
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=key)
except Exception as ex:
self._logger.error(ERR_DELETE_CONFIG_ITEM, key,
self.configuration_bucket, str(ex))
def add_actions_permissions(template, all_actions):
def action_select_resources_permissions(action_properties):
return services.get_resource_describe_permissions(
action_properties[actions.ACTION_SERVICE],
action_properties[actions.ACTION_RESOURCES])
scheduler_role = template["Resources"]["SchedulerRole"]
action_statement = scheduler_role["Properties"]["Policies"][0][
"PolicyDocument"]["Statement"]
for action_name in all_actions:
action_properties = actions.get_action_properties(action_name)
# get permissions from action properties
action_permissions = action_properties.get(actions.ACTION_PERMISSIONS,
[])
# get the permissions to retrieve the resources for that action
# with possible additional permissions to retrieve tags
action_permissions.append(
action_select_resources_permissions(action_properties))
if len(action_permissions) is not 0:
statements = build_action_policy_statement(action_name,
action_permissions)
action_statement += statements
def build_template(self, action_name):
"""
Build a cloudformation template for the action to create tasks for that action
:param action_name: name of the action
:return: template as dictionary
"""
self.action_name = action_name
self.action_properties = actions.get_action_properties(self.action_name)
self.action_class = actions.get_action_class(self.action_name)
self.has_completion_logic = getattr(self.action_class, handlers.COMPLETION_METHOD, None)
self.has_cross_account_parameter = self.action_properties.get(actions.ACTION_CROSS_ACCOUNT, True)
self.is_regional_service = services.get_service_class(self.action_properties[actions.ACTION_SERVICE]).is_regional()
self.has_regions_parameter = self.is_regional_service and self.action_properties.get(actions.ACTION_MULTI_REGION, True)
self.use_intervals = actions.ACTION_TRIGGER_INTERVAL[0] in self.action_properties.get(actions.ACTION_TRIGGERS,
actions.ACTION_TRIGGER_BOTH)
self.interval_min = self.action_properties.get(actions.ACTION_MIN_INTERVAL_MIN, 0)
self.use_events = actions.ACTION_TRIGGER_EVENTS[0] in self.action_properties.get(actions.ACTION_TRIGGERS,
actions.ACTION_TRIGGER_BOTH)
self._setup_template()
self._setup_common_parameters()
self._setup_action_parameters()
self._setup_resources()
self._setup_outputs()
for p in self._template_parameters:
description = self._template_parameters[p]["Description"]
if description not in ["", None] and not description.endswith("."):
self._template_parameters[p]["Description"] += "."
return self._template
def delete_templates(self):
"""
Deletes cross-account role and configuration templates
:return:
"""
s3_client = get_client_with_retries("s3", ["delete_object"],
context=self.context)
s3_key = ""
try:
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_INTERNAL, False):
self._logger.info(ERR_DELETING_ACTION_TEMPLATE,
action_name)
s3_key = S3_CONFIGURATION_TEMPLATE.format(action_name)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
s3_key = S3_ROLES_TEMPLATE.format(action_name)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
except Exception as ex:
self._logger.error(ERR_DELETE_TEMPLATE_, s3_key, str(ex))
try:
self._logger.info(INF_DELETE_ALL_ACTIONS_TEMPLATE, s3_key)
s3_key = S3_ROLES_TEMPLATE.format(ALL_ACTIONS_TEMPLATE_NAME)
s3_client.delete_object_with_retries(
Bucket=self.configuration_bucket, Key=s3_key)
except Exception as ex:
self._logger.error(ERR_DELETE_TEMPLATE_, s3_key, str(ex))
def _add_action_condition_parameter(self, action_name):
action_properties = actions.get_action_properties(action_name)
self.parameters[action_name] = {
"Type": "String",
"AllowedValues": [YES, NO],
"Default": NO,
"Description": "{}".format(action_name)
}
self.parameter_labels[action_name] = {
"default": action_properties[actions.ACTION_TITLE]
}
group_name = group_name_from_action_name(action_name)
group = [
g for g in self.parameter_groups
if g["Label"]["default"] == group_name
]
if not group:
group = {"Label": {"default": group_name}, "Parameters": []}
self.parameter_groups.insert(-1, group)
else:
group = group[0]
group["Parameters"].append(action_name)
self.conditions[action_name + "Condition"] = {
"Fn::Equals": [{
"Ref": action_name
}, YES]
}
def verify_cross_account_roles(self, this_account, roles, action_name):
"""
Tests if cross account roles have a valid format and removes roles with duplicated account numbers
:param this_account: The account that is used for the scheduler, none of resources are not processed for that account
:param roles: List of cross account role arns
:param action_name: Name of the action
:return: List of verified roles
"""
result = []
accounts = [this_account] if this_account is not None else []
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_CROSS_ACCOUNT, True):
if len(roles) > 0:
raise ValueError(
ERR_NO_CROSS_ACCOUNT_OPERATIONS.format(action_name))
if this_account is None:
raise ValueError(
ERR_THIS_ACCOUNT_MUST_BE_TRUE.format(
action_name, configuration.CONFIG_THIS_ACCOUNT))
for role in set(roles):
if not TaskConfiguration.is_valid_role_arn(role):
raise ValueError(MSG_ARN_FORMAT_INVALID.format(role))
account = AwsService.account_from_role_arn(role)
if account not in accounts:
accounts.append(account)
result.append(role)
else:
msg = WARN_OVERLAPPING_ROLES.format(account, role)
self._warn(msg)
return result
def validate_regions(
self,
regions,
action_name,
):
action_properties = actions.get_action_properties(action_name)
service_name = action_properties[actions.ACTION_SERVICE]
if self.service_is_regional(service_name) and action_properties.get(
actions.ACTION_MULTI_REGION, True):
if regions is None or len(regions) == 0:
return [boto3.Session().region_name]
else:
available_regions = self.service_regions(service_name)
if len(regions) == 1 and list(regions)[0] == "*":
return available_regions
for region in regions:
if region not in available_regions:
raise ValueError(
MSG_BAD_REGION.format(region, service_name,
",".join(available_regions)))
return list(regions)
else:
if regions is not None and len(regions) != 0:
msg = WARN_NOT_REGIONAL_SERVICE.format(",".join(regions),
service_name,
action_name)
self._warn(msg)
return []
def add_actions_permissions(template, all_actions):
def action_select_resources_permissions(action_prop):
return services.get_resource_describe_permissions(
action_prop[actions.ACTION_SERVICE],
[action_prop[actions.ACTION_RESOURCES]])
ops_automator_role = template["Resources"][LAMBDA_ROLE]
action_statement = ops_automator_role["Properties"]["Policies"][0][
"PolicyDocument"]["Statement"]
required_actions = set()
for action_name in all_actions:
action_properties = actions.get_action_properties(action_name)
# get permissions from action properties
action_permissions = action_properties.get(actions.ACTION_PERMISSIONS,
[])
# get the permissions to retrieve the resources for that action
# with possible additional permissions to retrieve tags
action_permissions += list(
action_select_resources_permissions(action_properties))
if len(action_permissions) is not 0:
required_actions.update(action_permissions)
# if using these lines individual statemens are built for every action
# statements = build_action_policy_statement(action_name, action_permissions)
# action_statement += statements
action_statement += build_action_policy_statement("ActionPermissions",
required_actions)
def _add_actions_permissions(self,
role_actions,
with_conditional_param=False):
def action_select_resources_permissions(action_props):
return services.get_resource_describe_permissions(
action_props[actions.ACTION_SERVICE],
[action_props[actions.ACTION_RESOURCES]])
for action_name in sorted(role_actions):
action_properties = actions.get_action_properties(action_name)
if action_properties.get(actions.ACTION_INTERNAL, False):
continue
if with_conditional_param:
self._add_action_condition_parameter(action_name)
# get permissions from action properties
action_permissions = action_properties.get(
actions.ACTION_PERMISSIONS, [])
# get the permissions to retrieve the resources for that action
# with possible additional permissions to retrieve tags
action_permissions += action_select_resources_permissions(
action_properties)
if action_properties.get(
actions.ACTION_SERVICE
) in services.aws_service.SERVICES_SUPPORTED_BY_RESOURCEGROUP_TAGGING_API:
action_permissions.append("tag:GetResources")
if len(action_permissions) > 0:
policy = {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Resource":
"*",
"Action":
list(set(action_permissions))
}]
},
"PolicyName": action_name,
"Roles": [{
"Ref": ACTION_ROLE
}]
}
}
if with_conditional_param:
policy["Condition"] = action_name + "Condition"
self.resources[action_name + "Policy"] = policy
def __init__(self, event, context):
"""
Initializes handler.
:param event: Event to handle
:param context: Context if run within Lambda environment
"""
self._context = context
self._event = event
self.action_id = self._event[handlers.TASK_TR_ID]
self.task = self._event[handlers.TASK_TR_NAME]
self.task_timezone = self._event.get(handlers.TASK_TR_TIMEZONE, None)
self.has_completion = self._event[handlers.TASK_TR_HAS_COMPLETION]
self.action_parameters = self._event.get(handlers.TASK_TR_PARAMETERS,
{})
self.dryrun = self._event.get(handlers.TASK_TR_DRYRUN)
self.interval = self._event.get(handlers.TASK_TR_INTERVAL, None)
self.metrics = self._event.get(handlers.TASK_TR_METRICS, False)
self.debug = self._event.get(handlers.TASK_TR_DEBUG)
self.started_at = int(self._event.get(handlers.TASK_TR_STARTED_TS, 0))
self.start_result = self._event.get(handlers.TASK_TR_START_RESULT,
None)
self.session = services.get_session(
self._event.get(handlers.TASK_TR_ASSUMED_ROLE))
self.stack_name = os.getenv(handlers.ENV_STACK_NAME)
self.stack_id = os.getenv(handlers.ENV_STACK_ID)
self.action = event[handlers.TASK_TR_ACTION]
self.tagfilter = event.get(handlers.TASK_TR_TAGFILTER, "")
self.action_properties = actions.get_action_properties(self.action)
self.action_class = actions.get_action_class(self.action)
self._stack_resources = None
self.timeout = int(
self._event[handlers.TASK_TR_TIMEOUT]) * 60 if self._event.get(
handlers.TASK_TR_TIMEOUT, None) not in [None, "None"] else 0
self.execution_log_stream = self._event.get(
handlers.TASK_TR_EXECUTION_LOGSTREAM)
self.assumed_role = self._event.get(handlers.TASK_TR_ASSUMED_ROLE,
None)
self.events = self._event.get(handlers.TASK_TR_EVENTS, {})
if isinstance(self.events, str):
self.events = json.loads(
self._event.get(handlers.TASK_TR_EVENTS,
"{}").replace("u'", '"').replace("'", '"'))
self._action_resources = None
self._s3_client = None
self._action_instance = None
self._action_class = None
self._action_arguments = None
self._timer = None
self._timeout_event = None
self.__logger = None
self.__action_tracking = None
def validate_events(events, action_name):
# get properties for action for the task and the actions parameter definitions
action_properties = actions.get_action_properties(action_name)
action_service = action_properties[actions.ACTION_SERVICE].lower()
for event in events:
event_service = event.split(":")[0].lower()
if event_service != action_service:
raise ValueError(
MSG_EVENT_SERVICE_NOT_ACTION_SERVICE.format(
action_service, action_name, event_service, event))
return events
def __init__(self, event, context):
"""
Initializes handler.
:param event: Event to handle
:param context: Context if run within Lambda environment
"""
self._context = context
self._event = event
self._action_tracking = TaskTrackingTable(context)
self.action_id = self._event[tracking.TASK_TR_ID]
self.task = self._event[tracking.TASK_TR_NAME]
self.action = self._event[tracking.TASK_TR_ACTION]
self.test_completion_method = getattr(
actions.get_action_class(self.action), handlers.COMPLETION_METHOD,
None)
self.action_parameters = json.loads(
self._event.get(tracking.TASK_TR_PARAMETERS, "{}"))
self.action_resources = json.loads(
self._event.get(tracking.TASK_TR_RESOURCES, "{}"))
self.dryrun = self._event.get(tracking.TASK_TR_DRYRUN)
self.debug = self._event.get(tracking.TASK_TR_DEBUG)
self.started_at = float(self._event.get(tracking.TASK_TR_STARTED_TS,
0))
self.start_result = self._event.get(tracking.TASK_TR_START_RESULT,
None)
self.session = AwsService.get_session(
self._event.get(tracking.TASK_TR_ASSUMED_ROLE))
self.stack_name = os.getenv(handlers.ENV_STACK_NAME)
self.stack_id = os.getenv(handlers.ENV_STACK_ID)
self.action = event[tracking.TASK_TR_ACTION]
self.action_properties = actions.get_action_properties(self.action)
self.action_class = actions.get_action_class(self.action)
self._stack_resources = None
self.timeout = self._event.get(tracking.TASK_TR_TIMEOUT)
self.execution_log_stream = self._event.get(
tracking.TASK_TR_EXECUTION_LOGSTREAM)
# setup logging
if self.execution_log_stream is None:
dt = datetime.utcnow()
self.execution_log_stream = LOG_STREAM.format(
self._event[tracking.TASK_TR_NAME], dt.year, dt.month, dt.day,
dt.hour, dt.minute, self.action_id)
else:
self.execution_log_stream = self.execution_log_stream
debug = event[tracking.TASK_TR_DEBUG]
self._logger = Logger(logstream=self.execution_log_stream,
buffersize=40 if debug else 20,
context=context,
debug=debug)
def verify_internal(internal, action_name):
"""
Tests if the tasks that are not internal do not use actions that are marked as internal. If an internal
action is used for a task that is not internal an exception is raised.
:param internal: Value of task internal attribute
:param action_name: name of the action
:return: Validated internal setting
"""
action_properties = actions.get_action_properties(action_name)
action_is_internal = action_properties.get(actions.ACTION_INTERNAL,
False)
if not internal and action_is_internal:
raise ValueError(MSG_ACTION_ONLY_INTERNAL, action_name)
return internal
def add_additional_lambda_functions(template, all_actions):
additional_lambda_sizes = {}
for action_name in all_actions:
action_properties = actions.get_action_properties(action_name)
memory_requirements = get_action_memory_size(action_name,
action_properties)
if memory_requirements != actions.LAMBDA_DEFAULT_MEMORY:
if memory_requirements not in additional_lambda_sizes:
additional_lambda_sizes[memory_requirements] = [action_name]
else:
additional_lambda_sizes[memory_requirements].append(
action_name)
if len(additional_lambda_sizes) > 0:
scheduler_role = template["Resources"]["SchedulerRole"]
action_statement = scheduler_role["Properties"]["Policies"][0][
"PolicyDocument"]["Statement"]
# get item in statement that give persmission to lambda to invoke
temp = [
s for s in action_statement
if s.get("Sid", "") == "SchedulerInvoke"
]
if len(temp) == 0:
raise Exception(
"Can not find statement with Sid named \"SchedulerInvoke\"")
scheduler_invoke = temp[0]
default = template["Resources"]["SchedulerDefault"]
for memory_size in additional_lambda_sizes:
new_name = "Scheduler{:0>04d}".format(memory_size)
new_lambda = copy.deepcopy(default)
new_lambda["Properties"]["MemorySize"] = memory_size
new_lambda["Properties"]["Description"] = \
default["Properties"]["Description"].replace("(default)", "({} MB".format(memory_size)) \
+ " for execution of actions: {})".format(",".join(additional_lambda_sizes[memory_size]))
new_lambda["Properties"]["FunctionName"]["Fn::Join"][1][
-1] = new_name
template["Resources"][new_name] = new_lambda
# add permission to invoke this lambda
new_resource = copy.deepcopy(scheduler_invoke["Resource"][0])
new_resource["Fn::Join"][1][-1]["Fn::Join"][1][-1] = new_name
scheduler_invoke["Resource"].append(new_resource)
def get_action(name, context=None, log_this_call=True):
"""
Gets the details of the specified action
:param name: Name of the action, raises an exception if the action does not exist
:param context: Lambda context
:param log_this_call: switch
:return: Details of the specified action.
"""
with _get_logger(context=context) as logger:
if log_this_call:
logger.info("get_action")
all_actions = actions.all_actions()
if name not in all_actions:
raise_value_error(ERR_ACTION_DOES_NOT_EXIST, name, ",".join(all_actions))
return actions.get_action_properties(name)
def add_actions_stack_resource_permissions(template, all_actions):
scheduler_role = template["Resources"]["SchedulerRole"]
action_statement = scheduler_role["Properties"]["Policies"][0][
"PolicyDocument"]["Statement"]
for action_name in all_actions:
action_properties = actions.get_action_properties(action_name)
# get permissions from action properties
action_permissions = action_properties.get(
actions.ACTION_STACK_RESOURCES_PERMISSIONS, [])
if len(action_permissions) is not 0:
statements = build_action_policy_statement(action_name,
action_permissions)
action_statement += statements
def task_account_region_sub_tasks(task):
action_properties = actions.get_action_properties(
task[handlers.TASK_ACTION])
aggregation_level = action_properties[actions.ACTION_AGGREGATION]
# property may be a lambda function, call the function with parameters of task as lambda parameters
if types.FunctionType == type(aggregation_level):
aggregation_level = aggregation_level(task.get("parameters", {}))
if aggregation_level == actions.ACTION_AGGREGATION_TASK:
yield {
handlers.TASK_THIS_ACCOUNT: task[handlers.TASK_THIS_ACCOUNT],
handlers.TASK_ACCOUNTS: task[handlers.TASK_ACCOUNTS],
handlers.TASK_REGIONS: task[handlers.TASK_REGIONS]
}
else:
if task[handlers.TASK_THIS_ACCOUNT]:
if aggregation_level == actions.ACTION_AGGREGATION_ACCOUNT:
yield {
handlers.TASK_THIS_ACCOUNT: True,
handlers.TASK_ACCOUNTS: [],
handlers.TASK_REGIONS: task[handlers.TASK_REGIONS]
}
else:
for region in task.get(handlers.TASK_REGIONS, [None]):
yield {
handlers.TASK_THIS_ACCOUNT: True,
handlers.TASK_ACCOUNTS: [],
handlers.TASK_REGIONS: [region]
}
for account in task.get(handlers.TASK_ACCOUNTS, []):
if aggregation_level == actions.ACTION_AGGREGATION_ACCOUNT:
yield {
handlers.TASK_THIS_ACCOUNT: False,
handlers.TASK_ACCOUNTS: [account],
handlers.TASK_REGIONS: task[handlers.TASK_REGIONS]
}
else:
for region in task.get(handlers.TASK_REGIONS, [None]):
yield {
handlers.TASK_THIS_ACCOUNT: False,
handlers.TASK_ACCOUNTS: [account],
handlers.TASK_REGIONS: [region]
}
def _is_waitlisted(self, item):
"""
Test if there is a max concurrency level for the tasks action. If this is the case then a concurrency key is retrieved
from the action and it is used to update the counter in the concurrency table for that key. The updated counter is tested
against the max concurrency level for the tasks action
:param item: task item
:return: True if counter for tasks action concurrency key > mac concurrency level, False if it is less or equal or the
action has no max concurrency level
"""
action = item[tracking.TASK_TR_ACTION]
action_properies = actions.get_action_properties(action)
# test if there are concurrency restrictions
max_action_concurrency = action_properies.get(
actions.ACTION_MAX_CONCURRENCY)
if max_action_concurrency is None:
return False
# get the key for the tasks action
concurrency_key = TaskTrackingHandler._get_action_concurrency_key(item)
# enter the waiting list for that key
count = self._enter_waiting_list(concurrency_key)
# set status to waiting if count > max concurrency level
status = tracking.STATUS_WAITING if count > max_action_concurrency else None
# store the concurrency key twice, the concurrency id is used for the index in the GSI and is removed after the
# action is handled so it does not longer show in the GSI, but we keep another copy in the task tracking table that
# we need to decrement the counter in the waiting list and possible start waiting instances with the same key
self.tracking_table.update_action(
item[tracking.TASK_TR_ID], status, {
tracking.TASK_TR_CONCURRENCY_KEY: concurrency_key,
tracking.TASK_TR_CONCURRENCY_ID: concurrency_key
})
if count > max_action_concurrency:
self._logger.info(INFO_WAITING, item[tracking.TASK_TR_ACTION],
concurrency_key, count, max_action_concurrency,
item[tracking.TASK_TR_ID])
return True
return False
def build_template(self, action_name):
"""
Build a cloudformation template for the action to create tasks for that action
:param action_name: name of the action
:return: template as dictionary
"""
self.action_name = action_name
self.action_properties = actions.get_action_properties(self.action_name)
self.action_class = actions.get_action_class(self.action_name)
self.has_timeout_parameter = getattr(self.action_class, handlers.COMPLETION_METHOD, None)
self.has_cross_account_parameter = self.action_properties.get(actions.ACTION_CROSS_ACCOUNT, True)
self.is_regional_service = services.get_service_class(self.action_properties[actions.ACTION_SERVICE]).is_regional()
self.has_regions_parameter = self.is_regional_service and self.action_properties.get(actions.ACTION_MULTI_REGION, True)
self._setup_template()
self._setup_common_parameters()
self._setup_action_parameters()
self._setup_resources()
return self._template
def action_stack_resources(self):
if self._action_stack_resources is None:
self._action_stack_resources = {}
if self._action_stack is not None:
# test if this action has additional stack resources
resources = actions.get_action_properties(
self.action_name).get(actions.ACTION_STACK_RESOURCES, {})
if resources:
# name of the class
class_name = self.action_properties[
actions.ACTION_CLASS_NAME][0:-len("Action")]
# actual resource names is name of class + name from class properties
logical_resource_names = [
class_name + resource_name
for resource_name in resources
]
for res in list(
self.action_stack.stack_resources.values()):
# actual name
logical_resource_id = res["LogicalResourceId"]
# test if this resource is an resource from the action properties
if logical_resource_id in logical_resource_names:
self._action_stack_resources[
logical_resource_id[len(class_name):]] = {
i: res[i]
for i in [
"LogicalResourceId",
"PhysicalResourceId", "ResourceType"
]
}
if len(list(self._action_stack_resources.keys())
) == len(resources):
return self._action_stack_resources
return self._action_stack_resources
def __init__(self, event, context):
self._context = context
self._event = event
self.task = event[handlers.HANDLER_EVENT_TASK]
# setup logging
classname = self.__class__.__name__
dt = datetime.utcnow()
logstream = LOG_STREAM.format(classname, self.task[handlers.TASK_NAME],
dt.year, dt.month, dt.day)
debug = event[handlers.HANDLER_EVENT_TASK].get(handlers.TASK_DEBUG,
False)
self._logger = Logger(logstream=logstream,
context=context,
buffersize=40 if debug else 20,
debug=debug)
self._sts = None
self._dynamodb = boto3.client("dynamodb")
self.select_args = event.get(handlers.HANDLER_SELECT_ARGUMENTS, {})
self.task_dt = event[handlers.HANDLER_EVENT_TASK_DT]
self.action_properties = actions.get_action_properties(
self.task[handlers.TASK_ACTION])
self.action_class = actions.get_action_class(
self.task[handlers.TASK_ACTION])
self.task_parameters = self.task.get(handlers.TASK_PARAMETERS, {})
self.aggregation_level = self.action_properties.get(
actions.ACTION_AGGREGATION, actions.ACTION_AGGREGATION_RESOURCE)
self.service = self.action_properties[actions.ACTION_SERVICE]
self.resource_name = self.action_properties[actions.ACTION_RESOURCES]
self.keep_tags = self.action_properties.get(
actions.ACTION_KEEP_RESOURCE_TAGS, True)
self.source = self._event.get(handlers.HANDLER_EVENT_SOURCE,
handlers.UNKNOWN_SOURCE)
def verify_timeout(action_name, timeout):
completion_method = getattr(actions.get_action_class(action_name),
handlers.COMPLETION_METHOD, None)
if completion_method is None and timeout is not None:
raise ValueError(ERR_TIMEOUT_NOT_ALLOWED.format(action_name))
if completion_method is None:
return None
if timeout is None:
action_properties = actions.get_action_properties(action_name)
return action_properties.get(
actions.ACTION_COMPLETION_TIMEOUT_MINUTES,
actions.DEFAULT_COMPLETION_TIMEOUT_MINUTES_DEFAULT)
try:
result = int(str(timeout).partition(".")[0])
if result > 0:
return result
else:
raise ValueError(ERR_TIMEOUT_MUST_BE_GREATER_0.format(result))
except ValueError as ex:
raise ValueError(ERR_INVALID_NUMERIC_TIMEOUT.format(timeout, ex))
def generate_templates(self):
"""
Generates configuration and cross-account role templates
:return:
"""
def generate_configuration_template(s3, builder, action):
configuration_template = S3_KEY_ACTION_CONFIGURATION_TEMPLATE.format(action)
self._logger.info(INF_CREATE_ACTION_TEMPLATE, action, configuration_template)
template = json.dumps(builder.build_template(action), indent=3)
s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=configuration_template)
def generate_all_actions_cross_account_role_template_parameterized(s3, builder, all_act, template_description):
self._logger.info(INF_CREATE_ALL_ACTIONS_CROSS_ROLES_TEMPLATE, S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS)
template = builder.build_template(action_list=all_act, description=template_description, with_conditional_params=True)
if self.optimize_cross_account_template:
template = CrossAccountRoleBuilder.compress_template(template)
template_json = json.dumps(template, indent=3)
s3.put_object_with_retries(Body=template_json, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS)
# noinspection PyUnusedLocal
def generate_all_actions_cross_account_role_template(s3, builder, all_act, template_description):
self._logger.info(INF_CREATE_ALL_ACTIONS_CROSS_ROLES_TEMPLATE, S3_KEY_ACCOUNT_CONFIG_CREATE_ALL)
template = json.dumps(
builder.build_template(action_list=all_act, description=template_description, with_conditional_params=False),
indent=3)
s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_CONFIG_CREATE_ALL)
def generate_forward_events_template(s3):
self._logger.info(INF_CREATE_EVENT_FORWARD_TEMPLATE, S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE)
template = build_events_forward_template(template_filename="./cloudformation/{}".format(FORWARD_EVENTS_TEMPLATE),
script_filename="./forward-events.py",
stack=self.stack_name,
event_role_arn=self.events_forward_role,
ops_automator_topic_arn=self.ops_automator_topic_arn,
version=self.stack_version)
s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE)
def generate_scenario_templates(s3):
self._logger.info("Creating task scenarios templates")
for template_name, template in list(builders.build_scenario_templates(templates_dir="./cloudformation/scenarios",
stack=self.stack_name)):
self._logger.info(INF_SCENARIO_TEMPLATE, template_name, S3_KEY_SCENARIO_TEMPLATE_BUCKET)
s3.put_object_with_retries(Body=template,
Bucket=self.configuration_bucket,
Key=S3_KEY_SCENARIO_TEMPLATE_KEY.format(template_name))
def generate_custom_resource_builder(s3):
self._logger.info("Create custom resource builder script {}", S3_KEY_CUSTOM_RESOURCE_BUILDER)
with open("./build_task_custom_resource.py", "rt") as f:
script_text = "".join(f.readlines())
script_text = script_text.replace("%stack%", self.stack_name)
script_text = script_text.replace("%account%", self.account)
script_text = script_text.replace("%region%", self.region)
script_text = script_text.replace("%config_table%", os.getenv("CONFIG_TABLE"))
s3.put_object_with_retries(Body=script_text, Bucket=self.configuration_bucket, Key=S3_KEY_CUSTOM_RESOURCE_BUILDER)
def generate_actions_html_page(s3):
self._logger.info("Generating Actions HTML page {}", S3_KEY_ACTIONS_HTML_PAGE)
html = builders.generate_html_actions_page(html_file="./builders/actions.html", region=self.region)
s3.put_object_with_retries(Body=html, Bucket=self.configuration_bucket, Key=S3_KEY_ACTIONS_HTML_PAGE,
ContentType="text/html")
self._logger.info(INF_GENERATING_TEMPLATES, self.configuration_bucket)
try:
stack = os.getenv(handlers.ENV_STACK_NAME, "")
s3_client = get_client_with_retries("s3", ["put_object"], context=self.context)
config_template_builder = ActionTemplateBuilder(self.context,
service_token_arn="arn:aws:region:account:function:used-for-debug-only",
ops_automator_role=self.automator_role_arn,
use_ecs=self.use_ecs)
role_template_builder = CrossAccountRoleBuilder(self.automator_role_arn, stack)
all_actions = []
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_INTERNAL, False):
generate_configuration_template(s3_client, config_template_builder, action_name)
# Enable to generate a template for every individual action
# description = TEMPLATE_DESC_CROSS_ACCOUNT_ACTION.format(action_name, stack, account)
# generate_action_cross_account_role_template(s3_client, role_template_builder, action_name, description)
all_actions.append(action_name)
if len(all_actions) > 0:
description = TEMPLATE_DESC_ALL_ACTIONS_PARAMETERS.format(stack, self.account)
generate_all_actions_cross_account_role_template_parameterized(s3_client, role_template_builder, all_actions,
description)
# enable to generate a template with all actions enabled
# description = TEMPLATE_DESC_ALL_ACTIONS.format(stack, account)
# generate_all_actions_cross_account_role_template(s3_client, role_template_builder, all_actions, description)
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if action_properties.get(actions.ACTION_EVENTS, None) is not None:
generate_forward_events_template(s3_client)
break
generate_actions_html_page(s3_client)
generate_scenario_templates(s3_client)
generate_custom_resource_builder(s3_client)
except Exception as ex:
self._logger.error(ERR_BUILDING_TEMPLATES, str(ex), full_stack())
def __init__(self, arguments, action_parameters):
self._assumed_role_ = None
self._context_ = None
self._debug_ = None
self._dryrun_ = None
self._event_ = None
self._events_ = None
self._logger_ = None
self._resources_ = None
self._session_ = None
self._stack_ = None
self._stack_id_ = None
self._stack_resources_ = None
self._start_result_ = None
self._started_at_ = None
self._tagfilter_ = None
self._task_ = None
self._task_id_ = None
self._task_timezone_ = None
self._timeout_ = None
self._timeout_event_ = None
self._datetime_ = date_time_provider()
for a in arguments:
setattr(self, "_{}_".format(a), arguments[a])
if not services.get_service_class(
self._event_.get(actions.ACTION_SERVICE)).is_regional():
self._region_ = self._session_.region_name
else:
action_properties = actions.get_action_properties(
self._event_[actions.ACTION])
aggregation_level = action_properties.get(
actions.ACTION_AGGREGATION,
actions.ACTION_AGGREGATION_RESOURCE)
if aggregation_level is not None and isinstance(
aggregation_level, types.FunctionType):
aggregation_level = aggregation_level(action_parameters)
if aggregation_level in [
actions.ACTION_AGGREGATION_REGION,
actions.ACTION_AGGREGATION_RESOURCE
]:
if isinstance(self._resources_, list):
if len(self._resources_) > 0:
self._region_ = self._resources_[0]["Region"]
else:
if self._resources_ is not None:
self._region_ = self._resources_["Region"]
if self._region_ is None:
self._region_ = self._session_.region_name
else:
self._region_ = self._session_.region_name
self._account_ = self.get_account_for_task()
if self._debug_ is None:
self._debug_ = False
if self._dryrun_ is None:
self._dryrun_ = False
for ap in action_parameters:
setattr(self, "_{}_".format(pascal_to_snake_case(ap)),
action_parameters[ap])
def _is_wait_listed(self, item):
"""
Test if there is a max concurrency level for the tasks action. If this is the case then a concurrency key is retrieved
from the action and it is used to update the counter in the concurrency table for that key. The updated counter is tested
against the max concurrency level for the tasks action
:param item: task item
:return: True if counter for tasks action concurrency key > mac concurrency level, False if it is less or equal or the
action has no max concurrency level
"""
action = item.get(handlers.TASK_TR_ACTION, None)
if action is None:
return False
action_properties = actions.get_action_properties(action)
# test if there are concurrency restrictions
max_action_concurrency = action_properties.get(
actions.ACTION_MAX_CONCURRENCY)
# no maximum
if max_action_concurrency in [None, 0]:
return False
# property may be a lambda function, call the function with parameters of task as lambda parameters
if types.FunctionType == type(max_action_concurrency):
parameters = item[handlers.TASK_TR_PARAMETERS]
max_action_concurrency = max_action_concurrency(parameters)
if max_action_concurrency in [None, 0]:
return False
# get the key for the tasks action
concurrency_key = self._get_action_concurrency_key(item)
# enter the waiting list for that key
count = int(self._enter_waiting_list(concurrency_key))
# set status to waiting if count > max concurrency level
status = handlers.STATUS_WAITING if count >= int(
max_action_concurrency) else None
# store the concurrency key twice, the concurrency id is used for the index in the GSI and is removed after the
# action is handled so it does not longer show in the GSI, but we keep another copy in the task tracking table that
# we need to decrement the counter in the waiting list and possible start waiting instances with the same key
self.tracking_table.update_task(item[handlers.TASK_TR_ID],
task=item[handlers.TASK_TR_NAME],
task_metrics=item.get(
handlers.TASK_TR_METRICS, False),
status=status,
status_data={
handlers.TASK_TR_CONCURRENCY_KEY:
concurrency_key,
handlers.TASK_TR_CONCURRENCY_ID:
concurrency_key
})
if count > max_action_concurrency:
self._logger.debug(DEBUG_WAITING, item[handlers.TASK_TR_ACTION],
concurrency_key, count, max_action_concurrency,
item[handlers.TASK_TR_ID])
return True
return False
def __init__(self, event, context, logger=None, tracking_store=None):
def log_stream_name():
classname = self.__class__.__name__
dt = datetime.utcnow()
account = self._event.get(handlers.HANDLER_SELECT_ARGUMENTS,
{}).get(handlers.HANDLER_EVENT_ACCOUNT,
"")
regions = self._event.get(handlers.HANDLER_SELECT_ARGUMENTS,
{}).get(handlers.HANDLER_EVENT_REGIONS,
[])
if account is not None and len(regions) > 0:
account_and_region = "-".join([account, regions[0]]) + "-"
else:
region = ""
if self.sub_task is not None:
account = ""
if self._this_account:
if len(self._accounts) == 0:
account = os.getenv(
handlers.ENV_OPS_AUTOMATOR_ACCOUNT)
elif len(self._accounts) == 1:
account = self._accounts[0]
region = self._regions[0] if len(
self._regions) == 1 else ""
if account != "":
if region not in ["", None]:
account_and_region = "-".join([account, region]) + "-"
else:
account_and_region = account
else:
account_and_region = ""
return LOG_STREAM.format(classname, self.task[handlers.TASK_NAME],
account_and_region, dt.year, dt.month,
dt.day)
self._context = context
self._event = event
self.task = event[handlers.HANDLER_EVENT_TASK]
self.sub_task = event.get(handlers.HANDLER_EVENT_SUB_TASK, None)
self.use_custom_select = event.get(
handlers.HANDLER_EVENT_CUSTOM_SELECT, True)
# the job id is used to correlate all generated tasks for the selected resources
self.task_group = self._event.get(handlers.HANDLER_EVENT_TASK_GROUP,
None)
if self.task_group is None:
self.task_group = str(uuid.uuid4())
debug = event[handlers.HANDLER_EVENT_TASK].get(handlers.TASK_DEBUG,
False)
if logger is None:
self._logger = QueuedLogger(logstream=log_stream_name(),
context=context,
buffersize=50 if debug else 20,
debug=debug)
else:
self._logger = logger
self._sts = None
self.select_args = event.get(handlers.HANDLER_SELECT_ARGUMENTS, {})
self.task_dt = event[handlers.HANDLER_EVENT_TASK_DT]
self.action_properties = actions.get_action_properties(
self.task[handlers.TASK_ACTION])
self.action_class = actions.get_action_class(
self.task[handlers.TASK_ACTION])
self.task_parameters = self.task.get(handlers.TASK_PARAMETERS, {})
self.metrics = self.task.get(handlers.TASK_METRICS, False)
self.service = self.action_properties[actions.ACTION_SERVICE]
self.keep_tags = self.action_properties.get(
actions.ACTION_KEEP_RESOURCE_TAGS, True)
self.source = self._event.get(handlers.HANDLER_EVENT_SOURCE,
handlers.UNKNOWN_SOURCE)
self.run_local = handlers.running_local(self._context)
self._timer = None
self._timeout_event = self._timeout_event = threading.Event()
self.aggregation_level = self.action_properties.get(
actions.ACTION_AGGREGATION, actions.ACTION_AGGREGATION_RESOURCE)
if self.aggregation_level is not None and isinstance(
self.aggregation_level, types.FunctionType):
self.aggregation_level = self.aggregation_level(
self.task_parameters)
self.batch_size = self.action_properties.get(actions.ACTION_BATCH_SIZE)
if self.batch_size is not None and isinstance(self.batch_size,
types.FunctionType):
self.batch_size = self.batch_size(self.task_parameters)
self.actions_tracking = TaskTrackingTable(
self._context,
logger=self._logger) if tracking_store is None else tracking_store
def get_action(name, context=None, log_this_call=True):
"""
Gets the details of the specified action
:param name: Name of the action, raises an exception if the action does not exist
:param context: Lambda context
:param log_this_call: switch
:return: Details of the specified action. This dictionary can contain the following actions:
Constants used below can be found in actions/__init__.py
-ACTION_SERVICE: Name of the service of the resources of this action
-ACTION_RESOURCES: Name of the resources for this action
-ACTION_AGGREGATION: Possible values are:
ACTION_AGGREGATION_RESOURCE: resources are not aggregated, execution of the action for each individual resource.
ACTION_AGGREGATION_ACCOUNT: resources are aggregated per account, execution of the action for the list of resources
in that account
ACTION_AGGREGATION_TASK: resources are aggregated per task, single execution of the action for list of all resources
in all accounts
-ACTION_SELECT_EXPRESSION: Optional JMES path to map/select attributes of and filtering of resources
-ACTION_BATCH_SIZE: Optional batch size for aggregated resources.
-ACTION_PERMISSIONS: Optional, permissions required for the action
-ACTION_MEMORY: Optional memory requirement for lambda function to run action, default is size of the scheduler lambda function
-ACTION_CROSS_ACCOUNT: Optional, cross account operations supported by action, default is True
-ACTION_EVENT_FILTER: Optional, regex filter which type of source events are supported by the
action, default is None (all events)
-ACTION_TITLE: Optional, title to be used in UI
-ACTION_DESCRIPTION: Optional, description or url to be used in UI
-ACTION_AUTHOR: Optional, author of the action
-ACTION_VERSION: Optional, implementation version of the action
-ACTION_MULTI_REGION: Optional, True if the action can execute in multiple regions (default)
-ACTION_INTERNAL: Optional, True if the service can only be used in internal tasks
-ACTION_PARAM_STACK_RESOURCES: Optional, cloudformation snippet of resources owen and used by action implementation
-ACTION_STACK_RESOURCES_PERMISSIONS: Optional, list of permissions for action stack resources
-ACTION_PARAMETERS: Parameters for the action:
-PARAM_ALLOWED_VALUES: allowed valued for a parameter (optional)
-PARAM_DEFAULT: default value for a parameter (optional)
-PARAM_MAX_LEN: max length for a string parameter (optional)
-PARAM_MAX_VALUE: max value for a numeric parameter (optional)
-PARAM_MIN_LEN: min length for a string parameter (optional)
-PARAM_MIN_VALUE: # min value for a numeric parameter (optional)
-PARAM_PATTERN: allowed pattern for a string parameter (optional)
-PARAM_REQUIRED: true if parameter is required (default=False)
-PARAM_TYPE: (Python) type name of a parameter
-PARAM_DESCRIBE_PARAMETER: name of a parameter if it must be used as a parameter in the describe method for a resource
-PARAM_DESCRIPTION: user readable description for parameter
-PARAM_LABEL: label for parameter
"""
with _get_logger(context=context) as logger:
if log_this_call:
logger.info("get_action")
all_actions = actions.all_actions()
if name not in all_actions:
raise ValueError(ERR_ACTION_DOES_NOT_EXIST.format(name, ",".join(all_actions)))
return safe_json(actions.get_action_properties(name))
