def action_validate_parameters(parameters, task_settings, logger):
valid_regions = services.get_session().get_available_regions(
"ec2", "aws")
region = parameters.get(PARAM_DESTINATION_REGION)
if region not in valid_regions:
raise_value_error(ERR_INVALID_DESTINATION_REGION, region,
",".join(valid_regions))
if parameters.get(PARAM_DELETE_AFTER_COPY, False) and parameters.get(
PARAM_COPIED_SNAPSHOTS) != COPIED_OWNED_BY_ACCOUNT:
raise_value_error(ERR_CANNOT_DELETE_SHARED_SNAPSHOTS)
kms_key_id = parameters.get(PARAM_KMS_KEY_ID, None)
if not parameters[PARAM_ENCRYPTED] and kms_key_id not in ["", None]:
raise_value_error(ERR_KMS_KEY_ONLY_IF_ENCRYPTED, PARAM_KMS_KEY_ID)
if kms_key_id not in ["", None]:
if re.match(KMS_KEY_ID_PATTERN, kms_key_id) is None:
raise_value_error(ERR_INVALID_KMS_ID_ARN, kms_key_id)
destination_region = parameters[PARAM_DESTINATION_REGION]
if kms_key_id.split(":")[3] != destination_region:
raise_value_error(ERR_KMS_KEY_NOT_IN_REGION, kms_key_id,
destination_region)
return parameters
def action_validate_parameters(parameters, task_settings, logger):
retention_days = parameters.get(PARAM_RETENTION_DAYS)
retention_count = parameters.get(PARAM_RETENTION_COUNT)
if not retention_count and not retention_days:
raise_value_error(ERR_RETENTION_PARAM_NONE, PARAM_RETENTION_COUNT,
PARAM_RETENTION_DAYS)
if retention_days and retention_count:
raise_value_error(ERR_RETENTION_PARAM_BOTH, PARAM_RETENTION_COUNT,
PARAM_RETENTION_DAYS)
return parameters
def action_validate_parameters(parameters, task_settings, __):
high = int(parameters.get(PARAM_CPU_PERC_HIGH))
low = int(parameters.get(PARAM_CPU_PERC_LOW))
if high <= low:
raise_value_error(ERR_PARAM_HIGH_LOW.format(PARAM_CPU_PERC_HIGH, PARAM_CPU_PERC_LOW))
cpu_high_tags = parameters.get(PARAM_CPU_HIGH_TAGS)
cpu_low_tags = parameters.get(PARAM_CPU_LOW_TAGS)
if cpu_high_tags is None and cpu_low_tags is None:
raise_value_error(ERR_TAG_BOTH_EMPTY.format(PARAM_CPU_LOW_TAGS, PARAM_CPU_HIGH_TAGS))
interval_cron = task_settings.get(CONFIG_INTERVAL, None)
if interval_cron is None:
raise_value_error(ERR_MISSING_INTERVAL.format(actions.ACTION_PARAM_INTERVAL))
e = CronExpression(interval_cron)
last = None
for i in e.within_next(timespan=timedelta(days=365),
start_dt=date_time_provider().now().replace(hour=0,
minute=0,
second=0,
microsecond=0)):
if last is not None:
between = i - last
if between > timedelta(minutes=DAY_IN_MINUTES):
raise_value_error(ERR_INTERVAL_TOO_LONG)
last = i
return parameters
def get_action(name, context=None, log_this_call=True):
"""
Gets the details of the specified action
:param name: Name of the action, raises an exception if the action does not exist
:param context: Lambda context
:param log_this_call: switch
:return: Details of the specified action.
"""
with _get_logger(context=context) as logger:
if log_this_call:
logger.info("get_action")
all_actions = actions.all_actions()
if name not in all_actions:
raise_value_error(ERR_ACTION_DOES_NOT_EXIST, name, ",".join(all_actions))
return actions.get_action_properties(name)
def action_validate_parameters(parameters, task_settings, logger):
mode = parameters.get(PARAM_RESIZE_MODE)
if mode == RESIZE_BY_SPECIFIED_TYPE:
instance_types = parameters.get(PARAM_INSTANCE_TYPES, [])
if len(instance_types) == 0:
raise_value_error(ERR_NO_TYPE_IN_SPECIFIED_MODE,
PARAM_RESIZE_MODE.format(mode))
valid_types = services.ec2_service.Ec2Service.valid_instance_types(
)
if valid_types not in [None, []]:
for inst_type in [e.strip() for e in instance_types]:
if inst_type not in valid_types:
raise_value_error(
ERR_INVALID_INSTANCE_TYPE.format(inst_type))
else:
scaling_range = parameters.get(PARAM_SCALING_RANGE, [])
if len(scaling_range) < 2:
raise_value_error(
ERR_AT_LEAST_TWO_TYPES.format(PARAM_SCALING_RANGE))
assumed_type = parameters.get(PARAM_ASSUMED_TYPE)
if assumed_type is not None:
if assumed_type not in scaling_range:
raise_value_error(ERR_ASSUMED_NOT_IN_SCALING_RANGE,
PARAM_ASSUMED_TYPE, PARAM_SCALING_RANGE)
scale_up_filter = parameters.get(PARAM_TAGFILTER_SCALE_UP)
scale_down_filter = parameters.get(PARAM_TAGFILTER_SCALE_DOWN)
if scale_up_filter is None and scale_down_filter is None:
raise_value_error(ERR_BOTH_SCALING_FILTERS_EMPTY,
PARAM_TAGFILTER_SCALE_UP,
PARAM_TAGFILTER_SCALE_DOWN, mode)
ActionEc2EventBase.check_tag_filters_and_tags(
parameters, task_settings, [PARAM_RESIZED_INSTANCE_TAGS], logger)
return parameters
def delete_task(name, exception_if_not_exists=False, context=None):
"""
Deletes the specified task
:param name: Name of the task to be deleted, if the task does not exist an exception is raised
:param exception_if_not_exists: if set to True raises an exception if the item does not exist
:param context: Lambda context
:return: Deleted task item
"""
with _get_logger(context=context) as logger:
logger.info("delete_task")
config = TaskConfiguration(context=context, logger=logger)
if exception_if_not_exists:
item = config.get_config_item(name)
if item is None:
raise_value_error(ERR_TASK_DOES_NOT_EXIST, name)
else:
item = {"Name": name}
config.delete_config_item(name)
return item
def update_task(name, context=None, **kwargs):
"""
Updates the specified task. An exception is raised when the action does not exist.
:param name: Name of the task. This name overwrites the name in kwargs if it is used there
:param kwargs: Task parameters dictionary, see create_task for details.
:param context: Lambda context
:return: Updated task item
"""
with _get_logger(context=context) as logger:
config = TaskConfiguration(context=context, logger=logger)
if name is None or len(name) == 0:
raise_value_error(ERR_NO_TASK_NAME)
item = config.get_config_item(name)
if item is None:
raise_value_error(ERR_TASK_DOES_NOT_EXIST, name)
# copy to avoid side effects when modifying arguments
args = copy.deepcopy(kwargs)
args[configuration.CONFIG_TASK_NAME] = name
stack_id = item.get(configuration.CONFIG_STACK_ID)
if stack_id is not None:
args[configuration.CONFIG_STACK_ID] = stack_id
item = config.put_config_item(**args)
return item
def process_and_select_resource(service, logger, resource_name, resource,
context, task, task_assumed_role):
def get_snapshot_tags(client, snap_id):
try:
resp = client.describe_snapshots_with_retries(
RestorableByUserIds=["self"], SnapshotIds=[snap_id])
list_of_tags = resp.get("Snapshots", [{}])[0].get("Tags", [])
return {
tag["Key"].strip(): tag.get("Value", "").strip()
for tag in list_of_tags
}, True
except Exception as ex:
if getattr(ex, "response",
{}).get("Error",
{}).get("Code",
"") == "InvalidSnapshot.NotFound":
return {}, False
else:
raise ex
def mark_as_being_selected_for_copy(client, snapshot):
try:
tag_name = Ec2CopySnapshotAction.marker_tag_copied_to(
task[handlers.TASK_NAME])
# Serial number for copy. This is stored in the tag of the snapshot anf the stored resources in the task
# Before starting a copy there will be check if these match to avoid double copied of a snapshot
copy_serial = str(uuid.uuid4())
tag_data = {
tag_name:
safe_json({
TAG_REGION:
task.get(handlers.TASK_PARAMETERS,
{}).get(PARAM_DESTINATION_REGION, ""),
COPY_SERIAL_NUMBER:
copy_serial,
TAG_COPY_SNAPSHOT_ID:
""
})
}
client.create_tags_with_retries(
Resources=[snapshot["SnapshotId"]],
Tags=tag_key_value_list(tag_data))
# store the copy serial number as part of the selected resource
resource[COPY_SERIAL_NUMBER] = copy_serial
except Exception as ex:
logger.warning(WARN_SETTING_COPIED_TAG, snapshot["SnapshotId"],
ex)
# source snapshot
snapshot_id = resource["SnapshotId"]
# owner of the snapshot
snapshot_owner = resource["OwnerId"]
parameters = task.get(handlers.TASK_PARAMETERS, {})
# copy owned, shared or both
copied_snapshot_types = parameters[PARAM_COPIED_SNAPSHOTS]
this_account = task.get(TASK_THIS_ACCOUNT, False)
accounts = task.get(TASK_ACCOUNTS, [])
if this_account and len(accounts) == 0:
account = os.getenv(handlers.ENV_OPS_AUTOMATOR_ACCOUNT)
elif not this_account and len(accounts) == 1:
account = accounts[0]
else:
account = services.account_from_role_arn(task_assumed_role)
if copied_snapshot_types == COPIED_OWNED_BY_ACCOUNT and account != snapshot_owner:
logger.debug(DEBUG_ONLY_COPY_OWNED_SNAPSHOTS, snapshot_id,
snapshot_owner, PARAM_COPIED_SNAPSHOTS, account)
return None
if copied_snapshot_types == COPIED_SNAPSHOTS_SHARED_TO_ACCOUNT and account == snapshot_owner:
logger.debug(DEBUG_ONLY_COPY_SHARED_SNAPSHOTS, snapshot_id,
snapshot_owner, PARAM_COPIED_SNAPSHOTS, account)
return None
copy_from_accounts = parameters.get(PARAM_COPY_FROM_OWNER_ACCOUNTS,
None)
if copy_from_accounts not in [None, []]:
if copied_snapshot_types == COPIED_OWNED_BY_ACCOUNT:
raise_value_error(ERR_ACCOUNTS_BUT_NOT_SHARED,
PARAM_COPY_FROM_OWNER_ACCOUNTS,
PARAM_COPIED_SNAPSHOTS)
if snapshot_owner != account and snapshot_owner not in [
a.strip() for a in copy_from_accounts
]:
logger.debug(DEBUG_SHARED_SNAPSHOT_OWNER_NOT_IN_LIST,
snapshot_id, snapshot_owner,
",".join(copy_from_accounts))
return None
# name of tag that is used to mark snapshots being copied
copied_tag_name = Ec2CopySnapshotAction.marker_tag_copied_to(
task[handlers.TASK_NAME])
if copied_tag_name in resource.get("Tags", {}):
# noinspection PyBroadException
try:
logger.debug(
"Snapshot already copied or being copied, copy data is:\n {}",
safe_json(
json.loads(
resource.get("Tags", {}).get(copied_tag_name,
{}))))
except Exception:
pass
return None
# ec2 client for getting most current tag values and setting tags
ec2 = get_client_with_retries(
service_name="ec2",
methods=["create_tags", "describe_snapshots"],
region=resource["Region"],
context=context,
session=service.session,
logger=logger)
# get the most current tags as they might be changed by overlapping copy tasks
tags, snapshot_found = get_snapshot_tags(ec2, snapshot_id)
# snapshot no longer there
if not snapshot_found:
logger.debug("Snapshot {} not longer available", snapshot_id)
return None
if copied_tag_name in tags:
return None
mark_as_being_selected_for_copy(ec2, resource)
return resource
def describe(self, service_resource, region=None, tags=False, tags_as_dict=None, as_tuple=None,
select=None, filter_func=None, context=None, select_on_tag=None, tag_roles=None, **describe_args):
"""
This method is used to retrieve service resources, specified by their name, from a service
:param filter_func: function for additional filtering of resources
:param service_resource: Name of the service resource, not case sensitive, use camel or snake case
:param region: Region from where resources are retrieved, if None then the current region is used
:param tags: Set to True to return tags with the resource
:param tags_as_dict: Set to True to return tags as python dictionaries
:param as_tuple: Set to true to return results as immutable named dictionaries instead of dictionaries
:param select: JMES path to select resources and select/transform attributes of returned resources
:param select_on_tag: only include resources that have a tag with this name
:param tag_roles: optional roles used to assume to select tags for a resource as this may be required by shared resources
from another account
:param describe_args: Parameters passed to the boto "describe" function
:param context: Lambda context
:return: Service resources of the specified resource type for the service.
"""
def use_tuple():
"""
Tests if resources should be returned as named tuples
:return: True for tuples, False for dictionaries
"""
return (as_tuple is not None and as_tuple) or (as_tuple is None and self._as_tuple)
def tags_as_dictionary():
"""
Tests if tags should be returned as python dictionaries
:return: True for dictionaries, False for original tag format
"""
return tags_as_dict if tags_as_dict is not None else self._tags_as_dict
# normalize resource name
self._resource_name = self._get_resource_name(service_resource)
# get the name of the boto3 method to retrieve this resource type
describe_func_name = self.describe_resources_function_name(self._resource_name)
# get additional parameters for boto describe method and map parameter names
if describe_args is None:
function_args = {}
else:
function_args = self._map_describe_function_parameters(self._resource_name, describe_args)
# get method from boto service client
if self._service_retry_strategy is not None:
method_names = [describe_func_name]
describe_func_name = describe_func_name + boto_retry.DEFAULT_SUFFIX
else:
method_names = None
client = self.service_client(region=region, method_names=method_names)
describe_func = getattr(client, describe_func_name, None)
if describe_func is None:
raise_value_error(ERR_NO_BOTO_SERVICE_METHOD, self.service_name, describe_func_name)
self._cached_tags = None
next_token = self._next_token_result_name(self._resource_name)
self._tags_as_dict = tags_as_dictionary()
self._use_tuple = use_tuple()
self._describe_args = describe_args
self._select_on_tag = select_on_tag
self._tags = tags
self._tag_roles = tag_roles
self._context = context
done = False
while not done:
# call boto method to retrieve until no more resources are retrieved
try:
resp = describe_func(**function_args)
except Exception as ex:
expected_exceptions = describe_args.get(boto_retry.EXPECTED_EXCEPTIONS, [])
if type(ex).__name__ in expected_exceptions or getattr(ex, "response", {}).get("Error", {}) \
.get("Code", "") in expected_exceptions:
done = True
continue
else:
raise ex
# extract resources from result and transform to requested output format
resources_data = self._extract_resources(resp=resp, select=select)
self._use_cached_tags = self.__class__.use_cached_tags(self._resource_name, len(resources_data))
for obj in resources_data:
if filter_func is not None and not filter_func(obj):
continue
# annotate additional account and region attributes
obj["AwsAccount"] = self.aws_account
obj["Region"] = self.service_client(region).meta.region_name if self.is_regional() else None
obj["Service"] = self.service_name
obj["ResourceTypeName"] = self._resource_name
# yield the transformed resource
transformed = self._transform_returned_resource(self.service_client(region=region), resource=obj)
if select_on_tag is None or select_on_tag in transformed.get("Tags", {}):
yield transformed
# if there are set the continuation token parameter for the next call to the value of the results continuation token
# test if more resources are available
if next_token in resp and resp[next_token] not in ["", False, None]:
self.set_continuation_call_parameters(function_args, next_token, resp)
else:
# all resources retrieved
done = True
def create_task(context=None, **kwargs):
"""
Creates a new task
:param kwargs: Task parameters
:param context: Lambda context
Constants can be found in configuration/__init__.py
-CONFIG_ACTION_NAME: Name of the action executed by the task, exception is raised if not specified or action does not
exist (mandatory, string)
-CONFIG_DEBUG: Set to True to log additional debug information for this task (optional, default False, boolean)
-CONFIG_DESCRIPTION: Task description(optional, default None, string)
-CONFIG_ACCOUNTS: List of accounts to execute task for
-CONFIG_CROSS_ACCOUNT_ROLE_NAME: Name of alternative cross account role to use instead of default role in external accounts
-CONFIG_ENABLED: Set to True to enable execution of task, False to suspend executions (optional, default True, boolean)
-CONFIG_INTERNAL: Flag to indicate task is used for internal tats of the scheduler (optional, default False, boolean)
-CONFIG_INTERVAL: Cron expression to schedule time/date based execution of task (optional, default "", string)
-CONFIG_TASK_TIMEOUT: Timeout in minutes for task to complete (optional, default is action's value or global timeout, number)
-CONFIG_TASK_NAME: Name of the task, exception is raised if not specified or name does already exist (mandatory, string)
-CONFIG_PARAMETERS: dictionary with names and values passed to the executed action of this task(optional,default {}, dictionary)
-CONFIG_THIS_ACCOUNT: Set to True to run tasks for resources in the account of the (optional, default True, boolean)
-CONFIG_TIMEZONE: Timezone for time/date based tasks for this task (optional, default UTC, string)
-CONFIG_TAG_FILTER: Tag filter used to select resources for the task instead of name of task in the list of values for the
automation tag. Only allowed if selected resources support tags (optional, default "", string)
-CONFIG_REGIONS: Regions in which to run the task. Use "*" for all regions in which the service for this tasks action
is available. If no regions are specified the region in which the scheduler is installed is used as default. Specifying one
or more regions for services tha are not region specific will generate a warning when processing the task. (optional,
default current region, List<string>)
-CONFIG_STACK_ID: Id of the stack if the task is created as part of a cloudformation template (optional, default None, string)
-CONFIG_DRYRUN: Dryrun parameter passed to the executed action (optional, default False, boolean)
-CONFIG_TASK_METRICS: Task metrics parameter passed to the executed action (optional, default False, boolean)
-CONFIG_EVENTS: List of resource events that trigger the task to be executed (optional, default, List<string>)
-CONFIG_DRYRUN: Dryrun parameter passed to the executed action (optional, default False, boolean)
-CONFIG_EVENTS: Resource events that trigger the task (optional, default None, dict)
-CONFIG_EVENT_SOURCE_TAG_FILTER: Filter for tags of source resource of events (optional, default None, string)
-CONFIG_EVENT_SCOPES: Scope to select resource events for that trigger the task (optional, default None, dict)
-CONFIG_SELECT_Size: Size to use for selecting resources (option, default is "Standard")
-CONFIG_EXECUTE_SIZE: Size to use for executing task action(option, default is "Standard")
-CONFIG_COMPLETION_SIZE: Size to use for executing task completion logic (option, default is "Standard")
-CONFIG_TASK_METRICS: Flag to indicate if metrics should be generated for the task
:return: Item created in the task configuration
"""
with _get_logger(context=context) as logger:
logger.info("create_task")
config = TaskConfiguration(context=context, logger=logger)
name = kwargs.get(configuration.CONFIG_TASK_NAME)
if name is None or len(name) == 0:
raise_value_error(ERR_NO_TASK_NAME)
item = config.get_config_item(name)
if item is not None:
raise_value_error(ERR_TASK_DOES_ALREADY_EXIST, name)
new_item = config.put_config_item(**kwargs)
return new_item
