def __init__(self,
role_arn=None,
session=None,
tags_as_dict=True,
as_named_tuple=False,
service_retry_strategy=None):
"""
:param role_arn: Optional (cross account) role to use to retrieve services
:param session: Optional session to use to retrieve services
:param tags_as_dict: Set to True true to convert resource tags to dictionaries
:param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary
:param service_retry_strategy: Retry strategy for service
:param service_retry_strategy: service retry strategy for making boto api calls
"""
custom_resource_paths = {
r: r[0].lower() + r[1:]
for r in RESOURCE_NAMES
}
custom_resource_paths[LOG_EVENTS] = "events"
AwsService.__init__(self,
service_name='logs',
resource_names=RESOURCE_NAMES,
role_arn=role_arn,
session=session,
tags_as_dict=tags_as_dict,
as_named_tuple=as_named_tuple,
custom_result_paths=custom_resource_paths,
mapped_parameters=MAPPED_PARAMETERS,
next_token_argument=NEXT_TOKEN_ARGUMENT,
next_token_result=NEXT_TOKEN_RESULT,
service_retry_strategy=service_retry_strategy)
def __init__(self,
role_arn=None,
session=None,
tags_as_dict=True,
as_named_tuple=False,
service_retry_strategy=None):
"""
:param role_arn: Optional (cross account) role to use to retrieve services
:param session: Optional session to use to retrieve services
:param tags_as_dict: Set to True true to convert resource tags to dictionaries
:param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary
:param service_retry_strategy: service retry strategy for making boto api calls
"""
AwsService.__init__(self,
service_name="dynamodb",
resource_names=RESOURCE_NAMES,
resources_with_tags=RESOURCES_WITH_TAGS,
role_arn=role_arn,
session=session,
tags_as_dict=tags_as_dict,
as_named_tuple=as_named_tuple,
custom_result_paths=CUSTOM_RESULT_PATHS,
mapped_parameters=MAPPED_PARAMETERS,
next_token_argument=NEXT_TOKEN_ARGUMENT,
next_token_result=NEXT_TOKEN_RESULT,
service_retry_strategy=service_retry_strategy)
def __init__(self,
role_arn=None,
session=None,
as_named_tuple=False,
service_retry_strategy=None):
"""
:param role_arn: Optional (cross account) role to use to retrieve services
:param session: Optional session to use to retrieve services
:param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary
"""
AwsService.__init__(self,
service_name='TestService',
resource_names=actions.ops_automator_test_action.
TEST_RESOURCE_NAMES,
role_arn=role_arn,
session=session,
tags_as_dict=False,
resources_with_tags=actions.
ops_automator_test_action.TEST_RESOURCE_NAMES,
as_named_tuple=as_named_tuple,
service_retry_strategy=service_retry_strategy)
self._args = None
self._test_data = None
self._region = None
self._tags = None
def __init__(self,
role_arn=None,
session=None,
tags_as_dict=True,
as_named_tuple=False,
service_retry_strategy=None):
"""
:param role_arn: Optional (cross account) role to use to retrieve services
:param session: Optional session to use to retrieve services
:param tags_as_dict: Set to True true to convert resource tags to dictionaries
:param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary
:param service_retry_strategy: service retry strategy for making boto api calls
"""
AwsService.__init__(self,
service_name='s3',
resource_names=RESOURCE_NAMES,
resources_with_tags=RESOURCES_WITH_TAGS,
role_arn=role_arn,
session=session,
tags_as_dict=tags_as_dict,
as_named_tuple=as_named_tuple,
custom_result_paths=CUSTOM_RESULT_PATHS,
service_retry_strategy=service_retry_strategy)
self._continuation_data = CONTINUATION_DATA
def set_continuation_call_parameters(self, function_args, next_token, resp):
if self._resource_name in MULTI_ELEMENT_CONTINUATION_MARKERS:
for marker in MULTI_ELEMENT_CONTINUATION_MARKERS[self._resource_name]:
if marker[1] in resp:
function_args[marker[0]] = resp[marker[1]]
else:
function_args.pop(marker[0], None)
else:
AwsService.set_continuation_call_parameters(self, function_args, next_token, resp)
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name=resource_name)
if resource_name in [PRODUCTS,
PRODUCTS_AS_ADMIN]:
s = s.replace("describe_", "search_")
elif resource_name in [ACCEPTED_PORTFOLIO_SHARES,
CONSTRAINTS_FOR_PORTFOLIO,
LAUNCH_PATHS,
PORTFOLIO_ACCESS,
PORTFOLIOS,
PORTFOLIOS_FOR_PRODUCT,
PRINCIPALS_FOR_PORTFOLIO,
PROVISIONING_ARTIFACTS,
RECORD_HISTORY]:
s = s.replace("describe_", "list_")
elif resource_name == PROVISIONED_PRODUCTS:
s = s.replace("describe_", "scan_")
return s
def verify_cross_account_roles(self, this_account, roles, action_name):
"""
Tests if cross account roles have a valid format and removes roles with duplicated account numbers
:param this_account: The account that is used for the scheduler, none of resources are not processed for that account
:param roles: List of cross account role arns
:param action_name: Name of the action
:return: List of verified roles
"""
result = []
accounts = [this_account] if this_account is not None else []
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_CROSS_ACCOUNT, True):
if len(roles) > 0:
raise ValueError(
ERR_NO_CROSS_ACCOUNT_OPERATIONS.format(action_name))
if this_account is None:
raise ValueError(
ERR_THIS_ACCOUNT_MUST_BE_TRUE.format(
action_name, configuration.CONFIG_THIS_ACCOUNT))
for role in set(roles):
if not TaskConfiguration.is_valid_role_arn(role):
raise ValueError(MSG_ARN_FORMAT_INVALID.format(role))
account = AwsService.account_from_role_arn(role)
if account not in accounts:
accounts.append(account)
result.append(role)
else:
msg = WARN_OVERLAPPING_ROLES.format(account, role)
self._warn(msg)
return result
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name=resource_name)
if resource_name in [
ACCOUNT_LIMIT,
CHANGE_INFO,
GEO_LOCATION,
HEALTH_CHECK,
HEALTH_CHECK_COUNT,
HEALTH_CHECK_LAST_FAILURE_REASON,
HEALTH_CHECK_STATUS,
HOSTED_ZONE,
HOSTED_ZONE_COUNT,
HOSTED_ZONE_LIMIT,
QUERY_LOGGING_CONFIG,
REUSABLE_DELEGATION_SET,
RESOURCE_RECORD_SET_LIMIT,
TRAFFIC_POLICY,
TRAFFIC_POLICY_INSTANCE,
TRAFFIC_POLICY_INSTANCE_COUNT
]:
s = s.replace("describe_", "get_")
else:
s = s.replace("describe_", "list_")
return s
def _map_describe_function_parameters(self, resources, args):
if len(args) == 0:
return args
temp = AwsService._map_describe_function_parameters(
self, resources, args)
# for this service arguments start with lowercase
translated = {b[0].lower() + b[1:]: temp[b] for b in temp}
return translated
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name)
return s.replace("describe_", "list_")
def __init__(self,
role_arn=None,
session=None,
as_named_tuple=False,
service_retry_strategy=None):
"""
:param role_arn: Optional (cross account) role to use to retrieve services
:param session: Optional session to use to retrieve services
:param as_named_tuple: Set to True to return resources as named tuples instead of a dictionary
"""
AwsService.__init__(self,
service_name='time',
resource_names=[],
role_arn=role_arn,
session=session,
tags_as_dict=False,
as_named_tuple=as_named_tuple,
service_retry_strategy=service_retry_strategy)
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name)
if resource_name in [FUNCTION, ALIAS, EVENT_SOURCE_MAPPING, FUNCTION_CONFIGURATION, POLICY]:
return s.replace("describe_", "get_")
return s.replace("describe_", "list_")
def add_task_action(self, task, assumed_role, action_resources, task_datetime, source):
"""
Creates and adds a new action to be written to the tracking table. Note that the items are kept in an internal
buffer and written in batches to the dynamodb table when the instance goes out of scope or the close method
is called explicitly.
:param task: Task that executes the action
:param assumed_role: Role to assume to execute the action
:param action_resources: Resources on which the action is performed
:param task_datetime: Time the task was scheduled for
:param source of event that started the task
test run their actions
:return: Created item
"""
item = {
TASK_TR_ID: str(uuid.uuid4()),
TASK_TR_NAME: task[handlers.TASK_NAME],
TASK_TR_ACTION: task[handlers.TASK_ACTION],
TASK_TR_CREATED: datetime.now().isoformat(),
TASK_TR_CREATED_TS: int(time()),
TASK_TR_SOURCE: source,
TASK_TR_DT: task_datetime,
TASK_TR_RESOURCES: safe_json(action_resources),
TASK_TR_STATUS: STATUS_PENDING,
TASK_TR_DEBUG: task[handlers.TASK_DEBUG],
TASK_TR_DRYRUN: task[handlers.TASK_DRYRUN],
TASK_TR_INTERNAL: task[handlers.TASK_INTERNAL],
TASK_TR_TIMEOUT: task[handlers.TASK_TIMOUT]
}
if assumed_role is not None:
item[TASK_TR_ASSUMED_ROLE] = assumed_role
item[TASK_TR_ACCOUNT] = AwsService.account_from_role_arn(assumed_role)
else:
item[TASK_TR_ACCOUNT] = AwsService.get_aws_account()
if len(task[handlers.TASK_PARAMETERS]) > 0:
item[TASK_TR_PARAMETERS] = task[handlers.TASK_PARAMETERS]
if item[TASK_TR_PARAMETERS]:
item[TASK_TR_PARAMETERS] = safe_json(item[TASK_TR_PARAMETERS])
self._new_action_items.append(item)
return item
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name)
return s.replace("describe_", "list_") if resource_name in [
TAGS_OF_RESOURCE, TABLES, BACKUPS
] else s
def _extract_resources(self, resp, select):
# the RDS API returns an ARN in the DBSnapshotIdentifierfield for shared snapshots. This overloaded method will
# extract the expected DBSnapshotIdentifier from the property
resources = AwsService._extract_resources(self, resp, select)
if self._resource_name != DB_SNAPSHOTS:
return resources
for r in resources:
if "DBSnapshotIdentifier" in r and r["DBSnapshotIdentifier"].startswith("arn:aws:rds:"):
r["DBSnapshotIdentifier"] = r["DBSnapshotIdentifier"].split(":")[-1]
return sorted(resources, key=lambda rc: rc.get("DBSnapshotArn", ""))
def __init__(self, event, context):
"""
Initializes handler.
:param event: Event to handle
:param context: Context if run within Lambda environment
"""
self._context = context
self._event = event
self._action_tracking = TaskTrackingTable(context)
self.action_id = self._event[tracking.TASK_TR_ID]
self.task = self._event[tracking.TASK_TR_NAME]
self.action = self._event[tracking.TASK_TR_ACTION]
self.test_completion_method = getattr(
actions.get_action_class(self.action), handlers.COMPLETION_METHOD,
None)
self.action_parameters = json.loads(
self._event.get(tracking.TASK_TR_PARAMETERS, "{}"))
self.action_resources = json.loads(
self._event.get(tracking.TASK_TR_RESOURCES, "{}"))
self.dryrun = self._event.get(tracking.TASK_TR_DRYRUN)
self.debug = self._event.get(tracking.TASK_TR_DEBUG)
self.started_at = float(self._event.get(tracking.TASK_TR_STARTED_TS,
0))
self.start_result = self._event.get(tracking.TASK_TR_START_RESULT,
None)
self.session = AwsService.get_session(
self._event.get(tracking.TASK_TR_ASSUMED_ROLE))
self.stack_name = os.getenv(handlers.ENV_STACK_NAME)
self.stack_id = os.getenv(handlers.ENV_STACK_ID)
self.action = event[tracking.TASK_TR_ACTION]
self.action_properties = actions.get_action_properties(self.action)
self.action_class = actions.get_action_class(self.action)
self._stack_resources = None
self.timeout = self._event.get(tracking.TASK_TR_TIMEOUT)
self.execution_log_stream = self._event.get(
tracking.TASK_TR_EXECUTION_LOGSTREAM)
# setup logging
if self.execution_log_stream is None:
dt = datetime.utcnow()
self.execution_log_stream = LOG_STREAM.format(
self._event[tracking.TASK_TR_NAME], dt.year, dt.month, dt.day,
dt.hour, dt.minute, self.action_id)
else:
self.execution_log_stream = self.execution_log_stream
debug = event[tracking.TASK_TR_DEBUG]
self._logger = Logger(logstream=self.execution_log_stream,
buffersize=40 if debug else 20,
context=context,
debug=debug)
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name)
if resource_name in [
CLUSTERS_ARNS, CONTAINER_INSTANCES_ARNS, SERVICES_ARNS,
TASK_DEFINITION_FAMILIES, TASK_DEFINITIONS_ARNS, TASK_ARNS
]:
return s.replace("describe_", "list_")
return s
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(
self, resource_name=resource_name)
if resource_name in [
ALLOWED_NODE_TYPE_MODIFICATIONS, TAGS_FOR_RESOURCE
]:
s = s.replace("describe_", "list_")
return s
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name)
if resource_name in [KEY]:
return s
if resource_name in [
KEY_POLICY, KEY_ROTATION_STATUS, PARAMETERS_FOR_INPUT, KEY
]:
return s.replace("describe_", "get_")
return s.replace("describe_", "list_")
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(self, resource_name)
if resource_name in [BUCKETS, MULTIPART_UPLOADS, OBJECT_VERSIONS, OBJECTS, PARTS]:
s = s.replace("describe_", "list_")
if resource_name == OBJECTS:
s += "_v2"
else:
s = s.replace("describe_", "get_")
return s
def describe_resources_function_name(self, resource_name):
"""
Returns the name of the boto client method call to retrieve the specified resource.
:param resource_name:
:return: Name of the boto3 client function to retrieve the specified resource type
"""
s = AwsService.describe_resources_function_name(
self, resource_name=resource_name)
if resource_name in [
CHANGE_SETS_SUMMARY, RESOURCES_SUMMARY, STACKS_SUMMARY
]:
s = s.replace("describe_", "list_")[0:-len("_Summary")]
return s
def _account_service_sessions(self, service_name):
"""
Returns a list of service instances for each handled account/role
:return:
"""
account = self._event.get(handlers.HANDLER_SELECT_ARGUMENTS,
{}).get(handlers.HANDLER_EVENT_ACCOUNT)
retry_strategy = get_default_retry_strategy(service=service_name,
context=self._context)
if account is not None:
if account == AwsService.get_aws_account():
yield services.create_service(service_name=service_name)
else:
for role in self.task[actions.ACTION_CROSS_ACCOUNT]:
if AwsService.account_from_role_arn(role) == account:
yield services.create_service(
service_name=service_name,
role_arn=role,
service_retry_strategy=retry_strategy)
else:
self._logger.error(MSG_NO_CROSS_ACCOUNT_ROLE,
self.task[handlers.TASK_NAME], account)
else:
if self.task.get(handlers.TASK_THIS_ACCOUNT, True):
yield services.create_service(
service_name=service_name,
service_retry_strategy=retry_strategy)
for role in self.task.get(handlers.TASK_CROSS_ACCOUNT_ROLES, []):
yield services.create_service(
service_name=service_name,
role_arn=role,
service_retry_strategy=retry_strategy)
def _map_describe_function_parameters(self, resources, args):
"""
Maps the parameter names passed to the service class describe call to names used to make the call the the boto
service client describe call
:param resources: Name of the resource type
:param args: parameters to be mapped
:return: mapped parameters
"""
if len(args) == 0:
return args
temp = AwsService._map_describe_function_parameters(
self, resources, args)
# for this service arguments start with lowercase
translated = {b[0].lower() + b[1:]: temp[b] for b in temp}
return translated
def _map_describe_function_parameters(self, resources, args):
"""
Maps the parameter names passed to the service class describe call to names used to make the call the the boto
service client describe call
:param resources: Name of the resource type
:param args: parameters to be mapped
:return: mapped parameters
"""
translated_args = args.copy()
for arg in translated_args:
if arg == "MaxResults" and resources in self._continuation_data:
del translated_args[arg]
translated_args[self._continuation_data[resources][2]] = args[arg]
return AwsService._map_describe_function_parameters(self, resources, translated_args)
def _transform_returned_resource(self, client, resource, resource_name, tags, tags_as_dict, use_tuple, **kwargs):
"""
This method takes the resource from the boto "describe" method and transforms them into the requested
output format of the service class describe function
:param client: boto client for the service that can be used to retrieve additional attributes, eg tags
:param resource: The resource returned from the boto call
:param resource_name: Tha name of the resource type
:param tags: Set true true if the tags must be retrieved for this resource
:param tags_as_dict: Set to true to convert the tags into Python dictionaries
:param use_tuple: Set to true to return the resources as un-mutable named tuples instead of dictionaries
:param kwargs: Additional service specific arguments for the transformation
:return: The transformed resources
"""
temp = {i: resource[i] for i in resource if i not in ["ResponseMetadata", "nextToken"]}
return AwsService._transform_returned_resource(self, client, temp, resource_name, tags, tags_as_dict, use_tuple)
def _transform_returned_resource(self,
client,
resource,
use_cached_tags=False):
"""
This method takes the resource from the boto "describe" method and transforms them into the requested
output format of the service class describe function
:param client: boto client for the service that can be used to retrieve additional attributes, eg tags
:param resource: The resource returned from the boto call
:return: The transformed resources
"""
temp = {
i: resource[i]
for i in resource if i not in ["ResponseMetadata", "nextToken"]
}
return AwsService._transform_returned_resource(self, client, temp)
def _transform_returned_resource(self,
client,
resource,
use_cached_tags=False):
if self._resource_name in [
INSTANCE_ATTRIBUTE, IMAGE_ATTRIBUTE, INTERFACE_ATTRIBUTE,
SNAPSHOT_ATTRIBUTE, FLEET_REQUEST_HISTORY, VOLUME_ATTRIBUTE,
VPC_ATTRIBUTE
]:
temp = {
r: resource[r]
for r in resource if resource[r] is not None
}
for r in temp:
if isinstance(temp[r], dict) and "Value" in temp[r]:
temp[r] = temp[r]["Value"]
else:
temp = resource
return AwsService._transform_returned_resource(self, client, temp)
def _transform_returned_resource(self, client, resource, resource_name,
tags_as_dict, use_tuple, **kwargs):
"""
This method takes the resource from the boto "describe" method and transforms them into the requested
output format of the service class describe function
:param client: boto client for the service that can be used to retrieve additional attributes, eg tags
:param resource: The resource returned from the boto call
:param resource_name: Tha name of the resource type
:param tags: Set true true if the tags must be retrieved for this resource
:param tags_as_dict: Set to true to convert the tags into Python dictionaries
:param use_tuple: Set to true to return the resources as un-mutable named tuples instead of dictionaries
:param kwargs: Additional service specific arguments for the transformation
:return: The transformed service resource
"""
if resource_name in [
INSTANCE_ATTRIBUTE, IMAGE_ATTRIBUTE, INTERFACE_ATTRIBUTE,
SNAPSHOT_ATTRIBUTE, FLEET_REQUEST_HISTORY, VOLUME_ATTRIBUTE,
VPC_ATTRIBUTE
]:
temp = {
r: resource[r]
for r in resource if resource[r] is not None
}
for r in temp:
if isinstance(temp[r], dict) and "Value" in temp[r]:
temp[r] = temp[r]["Value"]
else:
temp = resource
return AwsService._transform_returned_resource(
self,
client,
temp,
resource_name=resource_name,
tags_as_dict=tags_as_dict,
use_tuple=use_tuple,
**kwargs)
def _simulate_stream_processing(table_action, new_item, old_item=None):
# if not running in lambda environment create event that normally results from dynamodb inserts and pass directly
# to the main lambda handler to simulate an event triggered by the dynamodb stream
if old_item is None:
old_item = {}
account = AwsService.get_aws_account()
region = boto3.Session().region_name
table = os.environ.get(handlers.ENV_ACTION_TRACKING_TABLE)
event = {
"Records": [
{
"eventName": table_action,
"eventSourceARN": "arn:aws:dynamodb:{}:{}:table/{}/stream/{}".format(region, account, table,
datetime.utcnow().isoformat()),
"eventSource": "aws:dynamodb",
"dynamodb": {
"NewImage": {n: TaskTrackingTable.typed_item(new_item[n]) for n in new_item},
"OldImage": {o: TaskTrackingTable.typed_item(old_item[o]) for o in old_item}
}
}]
}
main.lambda_handler(event, None)
def _get_tags_for_resource(self, client, resource):
if self._use_cached_tags:
arn = "arn:aws:s3:::{}".format(resource["Name"])
return self.cached_tags(resource_name="").get(arn)
else:
return AwsService._get_tags_for_resource(self, client, resource)
