def handle_request(self):
"""
Handles the custom resource request from cloudformation
:return:
"""
start = datetime.now()
self._logger.info("Cloudformation request is {}",
safe_json(self._event, indent=2))
try:
result = CustomResource.handle_request(self)
return safe_dict({
"result":
result,
"datetime":
datetime.now().isoformat(),
"running-time": (datetime.now() - start).total_seconds()
})
except Exception as ex:
self._logger.error(ERR_HANDLING_SETUP_REQUEST, ex, full_stack())
raise ex
finally:
self._logger.flush()
def lambda_handler(event, context):
dt = datetime.utcnow()
log_stream_name = LOG_STREAM.format("OpsAutomatorMain", dt.year, dt.month,
dt.day)
with outputs.queued_logger.QueuedLogger(logstream=log_stream_name,
context=context,
buffersize=20) as logger:
for handler_name in handlers.all_handlers():
try:
if not handlers.get_class_for_handler(
handler_name).is_handling_request(event, context):
continue
except Exception as ex:
logger.error(ERR_IS_HANDLING, handler_name,
safe_json(event, indent=2), ex)
break
if context is not None and os.getenv(ENV_DEBUG_MAIN_EVENT_HANDLER,
"false").lower() == "true":
print(("Handler is {}".format(handler_name)))
print(("Event is {}".format(safe_json(event, indent=3))))
handler = handlers.create_handler(handler_name, event, context)
try:
logger.debug(DEBUG_HANDLER_INFO, handler_name)
result = handler.handle_request()
return safe_dict(result)
except Exception as e:
logger.error(ERR_HANDLING_REQUEST, safe_json(event, indent=2),
handler_name, e, full_stack())
finally:
if len(boto_retry.statistics) > 0:
logger.info(MSG_BOTO_STATS,
safe_json(boto_retry.statistics, indent=3))
boto_retry.clear_statistics()
return
else:
logger.debug(MSG_NO_REQUEST_HANDLER, safe_json(event, indent=2))
def _log_failed_call_exception(self, name, arguments, exception, retry_wait=0):
"""
Logs all details of failed boto3 calls
:param name: Name of the method
:param arguments: arguments used in the call
:param exception: raised exception
:param retry_wait period for retry if can be retried
:return:
"""
# logger information, method and arguments
s = "Boto3 method call: {}\nArguments:\n{}\n".format(name, safe_json(arguments, indent=3))
d = getattr(exception, "__dict__", None)
if d is not None:
s += safe_json(d, indent=3) + "\n"
if retry_wait == 0:
# exception and stack trace
s += full_stack()
else:
s += "Call will be retried in {} seconds".format(retry_wait)
if self.logger is None:
print(s)
else:
try:
if retry_wait == 0:
self.logger.error(s)
else:
self.logger.warning(s)
except Exception as ex:
print("Logging to stream failed, {}".format(ex))
print(s)
finally:
if self.logger is not None:
self.logger.flush()
def generate_templates(self):
"""
Generates configuration and cross-account role templates
:return:
"""
def generate_configuration_template(s3, builder, action):
configuration_template = S3_KEY_ACTION_CONFIGURATION_TEMPLATE.format(action)
self._logger.info(INF_CREATE_ACTION_TEMPLATE, action, configuration_template)
template = json.dumps(builder.build_template(action), indent=3)
s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=configuration_template)
def generate_all_actions_cross_account_role_template_parameterized(s3, builder, all_act, template_description):
self._logger.info(INF_CREATE_ALL_ACTIONS_CROSS_ROLES_TEMPLATE, S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS)
template = builder.build_template(action_list=all_act, description=template_description, with_conditional_params=True)
if self.optimize_cross_account_template:
template = CrossAccountRoleBuilder.compress_template(template)
template_json = json.dumps(template, indent=3)
s3.put_object_with_retries(Body=template_json, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS)
# noinspection PyUnusedLocal
def generate_all_actions_cross_account_role_template(s3, builder, all_act, template_description):
self._logger.info(INF_CREATE_ALL_ACTIONS_CROSS_ROLES_TEMPLATE, S3_KEY_ACCOUNT_CONFIG_CREATE_ALL)
template = json.dumps(
builder.build_template(action_list=all_act, description=template_description, with_conditional_params=False),
indent=3)
s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_CONFIG_CREATE_ALL)
def generate_forward_events_template(s3):
self._logger.info(INF_CREATE_EVENT_FORWARD_TEMPLATE, S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE)
template = build_events_forward_template(template_filename="./cloudformation/{}".format(FORWARD_EVENTS_TEMPLATE),
script_filename="./forward-events.py",
stack=self.stack_name,
event_role_arn=self.events_forward_role,
ops_automator_topic_arn=self.ops_automator_topic_arn,
version=self.stack_version)
s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE)
def generate_scenario_templates(s3):
self._logger.info("Creating task scenarios templates")
for template_name, template in list(builders.build_scenario_templates(templates_dir="./cloudformation/scenarios",
stack=self.stack_name)):
self._logger.info(INF_SCENARIO_TEMPLATE, template_name, S3_KEY_SCENARIO_TEMPLATE_BUCKET)
s3.put_object_with_retries(Body=template,
Bucket=self.configuration_bucket,
Key=S3_KEY_SCENARIO_TEMPLATE_KEY.format(template_name))
def generate_custom_resource_builder(s3):
self._logger.info("Create custom resource builder script {}", S3_KEY_CUSTOM_RESOURCE_BUILDER)
with open("./build_task_custom_resource.py", "rt") as f:
script_text = "".join(f.readlines())
script_text = script_text.replace("%stack%", self.stack_name)
script_text = script_text.replace("%account%", self.account)
script_text = script_text.replace("%region%", self.region)
script_text = script_text.replace("%config_table%", os.getenv("CONFIG_TABLE"))
s3.put_object_with_retries(Body=script_text, Bucket=self.configuration_bucket, Key=S3_KEY_CUSTOM_RESOURCE_BUILDER)
def generate_actions_html_page(s3):
self._logger.info("Generating Actions HTML page {}", S3_KEY_ACTIONS_HTML_PAGE)
html = builders.generate_html_actions_page(html_file="./builders/actions.html", region=self.region)
s3.put_object_with_retries(Body=html, Bucket=self.configuration_bucket, Key=S3_KEY_ACTIONS_HTML_PAGE,
ContentType="text/html")
self._logger.info(INF_GENERATING_TEMPLATES, self.configuration_bucket)
try:
stack = os.getenv(handlers.ENV_STACK_NAME, "")
s3_client = get_client_with_retries("s3", ["put_object"], context=self.context)
config_template_builder = ActionTemplateBuilder(self.context,
service_token_arn="arn:aws:region:account:function:used-for-debug-only",
ops_automator_role=self.automator_role_arn,
use_ecs=self.use_ecs)
role_template_builder = CrossAccountRoleBuilder(self.automator_role_arn, stack)
all_actions = []
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if not action_properties.get(actions.ACTION_INTERNAL, False):
generate_configuration_template(s3_client, config_template_builder, action_name)
# Enable to generate a template for every individual action
# description = TEMPLATE_DESC_CROSS_ACCOUNT_ACTION.format(action_name, stack, account)
# generate_action_cross_account_role_template(s3_client, role_template_builder, action_name, description)
all_actions.append(action_name)
if len(all_actions) > 0:
description = TEMPLATE_DESC_ALL_ACTIONS_PARAMETERS.format(stack, self.account)
generate_all_actions_cross_account_role_template_parameterized(s3_client, role_template_builder, all_actions,
description)
# enable to generate a template with all actions enabled
# description = TEMPLATE_DESC_ALL_ACTIONS.format(stack, account)
# generate_all_actions_cross_account_role_template(s3_client, role_template_builder, all_actions, description)
for action_name in actions.all_actions():
action_properties = actions.get_action_properties(action_name)
if action_properties.get(actions.ACTION_EVENTS, None) is not None:
generate_forward_events_template(s3_client)
break
generate_actions_html_page(s3_client)
generate_scenario_templates(s3_client)
generate_custom_resource_builder(s3_client)
except Exception as ex:
self._logger.error(ERR_BUILDING_TEMPLATES, str(ex), full_stack())
def _start_task_execution(self,
task_item,
action=handlers.HANDLER_ACTION_EXECUTE):
"""
Creates an instance of the lambda function that executes the tasks action. It first checks is the action has specific memory
requirements and based on this it creates a copy of this instance or one configured for the required memory. All
information for executing the action is passed in the event.
:param task_item: Task item for which action is executed
:return:
"""
try:
self._logger.debug(
"Entering start_task_execution ({}) with task {}", action,
safe_json(task_item, indent=3))
# Create event for execution of the action and set its action so that is picked up by the execution handler
event = {i: task_item.get(i) for i in task_item}
event[handlers.HANDLER_EVENT_ACTION] = action
self._logger.debug(DEBUG_ACTION,
task_item[handlers.TASK_TR_ACTION],
task_item[handlers.TASK_TR_NAME],
task_item[handlers.TASK_TR_ID])
self._logger.debug(
DEBUG_ACTION_PARAMETERS,
safe_json(task_item.get(handlers.TASK_TR_PARAMETERS, {}),
indent=3))
# get memory allocation for executing the task
lambda_size = handlers.TASK_TR_COMPLETION_SIZE \
if action == handlers.HANDLER_ACTION_TEST_COMPLETION \
else handlers.TASK_TR_EXECUTE_SIZE
execute_lambda_size = task_item.get(lambda_size,
actions.ACTION_SIZE_STANDARD)
if execute_lambda_size == actions.ACTION_USE_ECS:
ecs_memory = task_item.get(
handlers.TASK_EXECUTE_ECS_MEMORY
if action == handlers.HANDLER_ACTION_EXECUTE else
handlers.TASK_COMPLETION_ECS_MEMORY, None)
else:
ecs_memory = None
if not handlers.running_local(self._context):
self._logger.debug(DEBUG_MEMORY_SIZE, execute_lambda_size)
if execute_lambda_size != actions.ACTION_USE_ECS:
# create event payload
payload = str.encode(safe_json(event))
# determine which lambda to execute on
function_name = "{}-{}-{}".format(
os.getenv(handlers.ENV_STACK_NAME),
os.getenv(handlers.ENV_LAMBDA_NAME),
execute_lambda_size)
self._logger.debug(
"Running execution of task on lambda function {}",
function_name)
self._logger.debug(DEBUG_LAMBDA_FUNCTION_, function_name,
payload)
# start lambda function
lambda_client = boto_retry.get_client_with_retries(
"lambda", ["invoke"],
context=self._context,
logger=self._logger)
resp = lambda_client.invoke_with_retries(
FunctionName=function_name,
InvocationType="Event",
LogType="None",
Payload=payload)
task_info = {
"id": task_item[handlers.TASK_TR_ID],
"task": task_item[handlers.TASK_TR_NAME],
"action": task_item[handlers.TASK_TR_ACTION],
"payload": payload,
"status-code": resp["StatusCode"]
}
self._logger.debug(DEBUG_LAMBDA,
safe_json(task_info, indent=2))
self.invoked_lambda_functions.append(task_info)
else:
# run as ECS job
ecs_args = {
"subnets": os.getenv('AWSVPC_SUBNETS'),
"securitygroups": os.getenv('AWSVPC_SECURITYGROUPS'),
"assignpublicip": os.getenv('AWSVPC_ASSIGNPUBLICIP'),
handlers.HANDLER_EVENT_ACTION: action,
handlers.TASK_NAME: task_item[handlers.TASK_TR_NAME],
handlers.TASK_TR_ID: task_item[handlers.TASK_TR_ID]
}
self._logger.debug(DEBUG_RUNNING_ECS_TASK, action,
task_item[handlers.TASK_TR_NAME])
handlers.run_as_ecs_job(ecs_args,
ecs_memory_size=ecs_memory,
context=self._context,
logger=self._logger)
else:
lambda_handler(event, self._context)
ResultNotifications(context=self._context,
logger=self._logger).publish_started(task_item)
except Exception as ex:
self._logger.error(ERR_RUNNING_TASK, task_item, str(ex),
full_stack())
def handle_request(self):
"""
Handles action execute requests, creates an instance of the required action class and executes the action on the
resources passed in the event.
:return:
"""
# get class of the action, this class is needed by the _logger property
self._action_class = actions.get_action_class(self.action)
try:
self._action_arguments = {
actions.ACTION_PARAM_CONTEXT: self._context,
actions.ACTION_PARAM_EVENT: self._event,
actions.ACTION_PARAM_SESSION: self.session,
actions.ACTION_PARAM_RESOURCES: self.action_resources,
actions.ACTION_PARAM_INTERVAL: self.interval,
actions.ACTION_PARAM_DEBUG: self.debug,
actions.ACTION_PARAM_DRYRUN: self.dryrun,
actions.ACTION_PARAM_TASK_ID: self.action_id,
actions.ACTION_PARAM_TASK: self.task,
actions.ACTION_PARAM_TASK_TIMEZONE: self.task_timezone,
actions.ACTION_PARAM_STACK: self.stack_name,
actions.ACTION_PARAM_STACK_ID: self.stack_id,
actions.ACTION_PARAM_STACK_RESOURCES: self.stack_resources,
actions.ACTION_PARAM_ASSUMED_ROLE: self.assumed_role,
actions.ACTION_PARAM_STARTED_AT: self.started_at,
actions.ACTION_PARAM_TAGFILTER: self.tagfilter,
actions.ACTION_PARAM_TIMEOUT: self.timeout,
actions.ACTION_PARAM_TAG_FILTER: self.tagfilter,
actions.ACTION_PARAM_EVENTS: self.events
}
# called after initialization other arguments as it is using these to construct the logger
self._action_arguments[actions.ACTION_PARAM_LOGGER] = self._logger
if self._context is not None:
self._timeout_event = threading.Event()
self._action_arguments[
actions.ACTION_PARAM_TIMEOUT_EVENT] = self._timeout_event
# create the instance of the action class
self._action_instance = self._action_class(self._action_arguments,
self.action_parameters)
self._logger.info(INF_START_EXEC,
self._event[handlers.HANDLER_EVENT_ACTION],
self.action_id)
if self._event[
handlers.
HANDLER_EVENT_ACTION] == handlers.HANDLER_ACTION_EXECUTE:
return self._handle_task_execution()
elif self._event[
handlers.
HANDLER_EVENT_ACTION] == handlers.HANDLER_ACTION_TEST_COMPLETION:
return self._handle_test_task_completion()
raise Exception(
ERR_INVALID_ACTION.format(
self._event[handlers.HANDLER_EVENT_ACTION]))
except Exception as ex:
self._logger.error(ERR_EXECUTION_TASK,
self._event[handlers.HANDLER_EVENT_ACTION],
self.task, str(ex),
("\n" + full_stack()) if self.debug else "")
self._action_tracking.update_task(
action_id=self.action_id,
task=self.task,
task_metrics=self.metrics,
status=handlers.STATUS_FAILED,
status_data={handlers.TASK_TR_ERROR: str(ex)})
finally:
self._logger.info(INF_FINISH_EXEC,
self._event[handlers.HANDLER_EVENT_ACTION],
self.action_id)
self._logger.flush()
