def handle_request(self): """ Handles the custom resource request from cloudformation :return: """ start = datetime.now() self._logger.info("Cloudformation request is {}", safe_json(self._event, indent=2)) try: result = CustomResource.handle_request(self) return safe_dict({ "result": result, "datetime": datetime.now().isoformat(), "running-time": (datetime.now() - start).total_seconds() }) except Exception as ex: self._logger.error(ERR_HANDLING_SETUP_REQUEST, ex, full_stack()) raise ex finally: self._logger.flush()
def lambda_handler(event, context): dt = datetime.utcnow() log_stream_name = LOG_STREAM.format("OpsAutomatorMain", dt.year, dt.month, dt.day) with outputs.queued_logger.QueuedLogger(logstream=log_stream_name, context=context, buffersize=20) as logger: for handler_name in handlers.all_handlers(): try: if not handlers.get_class_for_handler( handler_name).is_handling_request(event, context): continue except Exception as ex: logger.error(ERR_IS_HANDLING, handler_name, safe_json(event, indent=2), ex) break if context is not None and os.getenv(ENV_DEBUG_MAIN_EVENT_HANDLER, "false").lower() == "true": print(("Handler is {}".format(handler_name))) print(("Event is {}".format(safe_json(event, indent=3)))) handler = handlers.create_handler(handler_name, event, context) try: logger.debug(DEBUG_HANDLER_INFO, handler_name) result = handler.handle_request() return safe_dict(result) except Exception as e: logger.error(ERR_HANDLING_REQUEST, safe_json(event, indent=2), handler_name, e, full_stack()) finally: if len(boto_retry.statistics) > 0: logger.info(MSG_BOTO_STATS, safe_json(boto_retry.statistics, indent=3)) boto_retry.clear_statistics() return else: logger.debug(MSG_NO_REQUEST_HANDLER, safe_json(event, indent=2))
def _log_failed_call_exception(self, name, arguments, exception, retry_wait=0): """ Logs all details of failed boto3 calls :param name: Name of the method :param arguments: arguments used in the call :param exception: raised exception :param retry_wait period for retry if can be retried :return: """ # logger information, method and arguments s = "Boto3 method call: {}\nArguments:\n{}\n".format(name, safe_json(arguments, indent=3)) d = getattr(exception, "__dict__", None) if d is not None: s += safe_json(d, indent=3) + "\n" if retry_wait == 0: # exception and stack trace s += full_stack() else: s += "Call will be retried in {} seconds".format(retry_wait) if self.logger is None: print(s) else: try: if retry_wait == 0: self.logger.error(s) else: self.logger.warning(s) except Exception as ex: print("Logging to stream failed, {}".format(ex)) print(s) finally: if self.logger is not None: self.logger.flush()
def generate_templates(self): """ Generates configuration and cross-account role templates :return: """ def generate_configuration_template(s3, builder, action): configuration_template = S3_KEY_ACTION_CONFIGURATION_TEMPLATE.format(action) self._logger.info(INF_CREATE_ACTION_TEMPLATE, action, configuration_template) template = json.dumps(builder.build_template(action), indent=3) s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=configuration_template) def generate_all_actions_cross_account_role_template_parameterized(s3, builder, all_act, template_description): self._logger.info(INF_CREATE_ALL_ACTIONS_CROSS_ROLES_TEMPLATE, S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS) template = builder.build_template(action_list=all_act, description=template_description, with_conditional_params=True) if self.optimize_cross_account_template: template = CrossAccountRoleBuilder.compress_template(template) template_json = json.dumps(template, indent=3) s3.put_object_with_retries(Body=template_json, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_CONFIG_WITH_PARAMS) # noinspection PyUnusedLocal def generate_all_actions_cross_account_role_template(s3, builder, all_act, template_description): self._logger.info(INF_CREATE_ALL_ACTIONS_CROSS_ROLES_TEMPLATE, S3_KEY_ACCOUNT_CONFIG_CREATE_ALL) template = json.dumps( builder.build_template(action_list=all_act, description=template_description, with_conditional_params=False), indent=3) s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_CONFIG_CREATE_ALL) def generate_forward_events_template(s3): self._logger.info(INF_CREATE_EVENT_FORWARD_TEMPLATE, S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE) template = build_events_forward_template(template_filename="./cloudformation/{}".format(FORWARD_EVENTS_TEMPLATE), script_filename="./forward-events.py", stack=self.stack_name, event_role_arn=self.events_forward_role, ops_automator_topic_arn=self.ops_automator_topic_arn, version=self.stack_version) s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_ACCOUNT_EVENTS_FORWARD_TEMPLATE) def generate_scenario_templates(s3): self._logger.info("Creating task scenarios templates") for template_name, template in list(builders.build_scenario_templates(templates_dir="./cloudformation/scenarios", stack=self.stack_name)): self._logger.info(INF_SCENARIO_TEMPLATE, template_name, S3_KEY_SCENARIO_TEMPLATE_BUCKET) s3.put_object_with_retries(Body=template, Bucket=self.configuration_bucket, Key=S3_KEY_SCENARIO_TEMPLATE_KEY.format(template_name)) def generate_custom_resource_builder(s3): self._logger.info("Create custom resource builder script {}", S3_KEY_CUSTOM_RESOURCE_BUILDER) with open("./build_task_custom_resource.py", "rt") as f: script_text = "".join(f.readlines()) script_text = script_text.replace("%stack%", self.stack_name) script_text = script_text.replace("%account%", self.account) script_text = script_text.replace("%region%", self.region) script_text = script_text.replace("%config_table%", os.getenv("CONFIG_TABLE")) s3.put_object_with_retries(Body=script_text, Bucket=self.configuration_bucket, Key=S3_KEY_CUSTOM_RESOURCE_BUILDER) def generate_actions_html_page(s3): self._logger.info("Generating Actions HTML page {}", S3_KEY_ACTIONS_HTML_PAGE) html = builders.generate_html_actions_page(html_file="./builders/actions.html", region=self.region) s3.put_object_with_retries(Body=html, Bucket=self.configuration_bucket, Key=S3_KEY_ACTIONS_HTML_PAGE, ContentType="text/html") self._logger.info(INF_GENERATING_TEMPLATES, self.configuration_bucket) try: stack = os.getenv(handlers.ENV_STACK_NAME, "") s3_client = get_client_with_retries("s3", ["put_object"], context=self.context) config_template_builder = ActionTemplateBuilder(self.context, service_token_arn="arn:aws:region:account:function:used-for-debug-only", ops_automator_role=self.automator_role_arn, use_ecs=self.use_ecs) role_template_builder = CrossAccountRoleBuilder(self.automator_role_arn, stack) all_actions = [] for action_name in actions.all_actions(): action_properties = actions.get_action_properties(action_name) if not action_properties.get(actions.ACTION_INTERNAL, False): generate_configuration_template(s3_client, config_template_builder, action_name) # Enable to generate a template for every individual action # description = TEMPLATE_DESC_CROSS_ACCOUNT_ACTION.format(action_name, stack, account) # generate_action_cross_account_role_template(s3_client, role_template_builder, action_name, description) all_actions.append(action_name) if len(all_actions) > 0: description = TEMPLATE_DESC_ALL_ACTIONS_PARAMETERS.format(stack, self.account) generate_all_actions_cross_account_role_template_parameterized(s3_client, role_template_builder, all_actions, description) # enable to generate a template with all actions enabled # description = TEMPLATE_DESC_ALL_ACTIONS.format(stack, account) # generate_all_actions_cross_account_role_template(s3_client, role_template_builder, all_actions, description) for action_name in actions.all_actions(): action_properties = actions.get_action_properties(action_name) if action_properties.get(actions.ACTION_EVENTS, None) is not None: generate_forward_events_template(s3_client) break generate_actions_html_page(s3_client) generate_scenario_templates(s3_client) generate_custom_resource_builder(s3_client) except Exception as ex: self._logger.error(ERR_BUILDING_TEMPLATES, str(ex), full_stack())
def _start_task_execution(self, task_item, action=handlers.HANDLER_ACTION_EXECUTE): """ Creates an instance of the lambda function that executes the tasks action. It first checks is the action has specific memory requirements and based on this it creates a copy of this instance or one configured for the required memory. All information for executing the action is passed in the event. :param task_item: Task item for which action is executed :return: """ try: self._logger.debug( "Entering start_task_execution ({}) with task {}", action, safe_json(task_item, indent=3)) # Create event for execution of the action and set its action so that is picked up by the execution handler event = {i: task_item.get(i) for i in task_item} event[handlers.HANDLER_EVENT_ACTION] = action self._logger.debug(DEBUG_ACTION, task_item[handlers.TASK_TR_ACTION], task_item[handlers.TASK_TR_NAME], task_item[handlers.TASK_TR_ID]) self._logger.debug( DEBUG_ACTION_PARAMETERS, safe_json(task_item.get(handlers.TASK_TR_PARAMETERS, {}), indent=3)) # get memory allocation for executing the task lambda_size = handlers.TASK_TR_COMPLETION_SIZE \ if action == handlers.HANDLER_ACTION_TEST_COMPLETION \ else handlers.TASK_TR_EXECUTE_SIZE execute_lambda_size = task_item.get(lambda_size, actions.ACTION_SIZE_STANDARD) if execute_lambda_size == actions.ACTION_USE_ECS: ecs_memory = task_item.get( handlers.TASK_EXECUTE_ECS_MEMORY if action == handlers.HANDLER_ACTION_EXECUTE else handlers.TASK_COMPLETION_ECS_MEMORY, None) else: ecs_memory = None if not handlers.running_local(self._context): self._logger.debug(DEBUG_MEMORY_SIZE, execute_lambda_size) if execute_lambda_size != actions.ACTION_USE_ECS: # create event payload payload = str.encode(safe_json(event)) # determine which lambda to execute on function_name = "{}-{}-{}".format( os.getenv(handlers.ENV_STACK_NAME), os.getenv(handlers.ENV_LAMBDA_NAME), execute_lambda_size) self._logger.debug( "Running execution of task on lambda function {}", function_name) self._logger.debug(DEBUG_LAMBDA_FUNCTION_, function_name, payload) # start lambda function lambda_client = boto_retry.get_client_with_retries( "lambda", ["invoke"], context=self._context, logger=self._logger) resp = lambda_client.invoke_with_retries( FunctionName=function_name, InvocationType="Event", LogType="None", Payload=payload) task_info = { "id": task_item[handlers.TASK_TR_ID], "task": task_item[handlers.TASK_TR_NAME], "action": task_item[handlers.TASK_TR_ACTION], "payload": payload, "status-code": resp["StatusCode"] } self._logger.debug(DEBUG_LAMBDA, safe_json(task_info, indent=2)) self.invoked_lambda_functions.append(task_info) else: # run as ECS job ecs_args = { "subnets": os.getenv('AWSVPC_SUBNETS'), "securitygroups": os.getenv('AWSVPC_SECURITYGROUPS'), "assignpublicip": os.getenv('AWSVPC_ASSIGNPUBLICIP'), handlers.HANDLER_EVENT_ACTION: action, handlers.TASK_NAME: task_item[handlers.TASK_TR_NAME], handlers.TASK_TR_ID: task_item[handlers.TASK_TR_ID] } self._logger.debug(DEBUG_RUNNING_ECS_TASK, action, task_item[handlers.TASK_TR_NAME]) handlers.run_as_ecs_job(ecs_args, ecs_memory_size=ecs_memory, context=self._context, logger=self._logger) else: lambda_handler(event, self._context) ResultNotifications(context=self._context, logger=self._logger).publish_started(task_item) except Exception as ex: self._logger.error(ERR_RUNNING_TASK, task_item, str(ex), full_stack())
def handle_request(self): """ Handles action execute requests, creates an instance of the required action class and executes the action on the resources passed in the event. :return: """ # get class of the action, this class is needed by the _logger property self._action_class = actions.get_action_class(self.action) try: self._action_arguments = { actions.ACTION_PARAM_CONTEXT: self._context, actions.ACTION_PARAM_EVENT: self._event, actions.ACTION_PARAM_SESSION: self.session, actions.ACTION_PARAM_RESOURCES: self.action_resources, actions.ACTION_PARAM_INTERVAL: self.interval, actions.ACTION_PARAM_DEBUG: self.debug, actions.ACTION_PARAM_DRYRUN: self.dryrun, actions.ACTION_PARAM_TASK_ID: self.action_id, actions.ACTION_PARAM_TASK: self.task, actions.ACTION_PARAM_TASK_TIMEZONE: self.task_timezone, actions.ACTION_PARAM_STACK: self.stack_name, actions.ACTION_PARAM_STACK_ID: self.stack_id, actions.ACTION_PARAM_STACK_RESOURCES: self.stack_resources, actions.ACTION_PARAM_ASSUMED_ROLE: self.assumed_role, actions.ACTION_PARAM_STARTED_AT: self.started_at, actions.ACTION_PARAM_TAGFILTER: self.tagfilter, actions.ACTION_PARAM_TIMEOUT: self.timeout, actions.ACTION_PARAM_TAG_FILTER: self.tagfilter, actions.ACTION_PARAM_EVENTS: self.events } # called after initialization other arguments as it is using these to construct the logger self._action_arguments[actions.ACTION_PARAM_LOGGER] = self._logger if self._context is not None: self._timeout_event = threading.Event() self._action_arguments[ actions.ACTION_PARAM_TIMEOUT_EVENT] = self._timeout_event # create the instance of the action class self._action_instance = self._action_class(self._action_arguments, self.action_parameters) self._logger.info(INF_START_EXEC, self._event[handlers.HANDLER_EVENT_ACTION], self.action_id) if self._event[ handlers. HANDLER_EVENT_ACTION] == handlers.HANDLER_ACTION_EXECUTE: return self._handle_task_execution() elif self._event[ handlers. HANDLER_EVENT_ACTION] == handlers.HANDLER_ACTION_TEST_COMPLETION: return self._handle_test_task_completion() raise Exception( ERR_INVALID_ACTION.format( self._event[handlers.HANDLER_EVENT_ACTION])) except Exception as ex: self._logger.error(ERR_EXECUTION_TASK, self._event[handlers.HANDLER_EVENT_ACTION], self.task, str(ex), ("\n" + full_stack()) if self.debug else "") self._action_tracking.update_task( action_id=self.action_id, task=self.task, task_metrics=self.metrics, status=handlers.STATUS_FAILED, status_data={handlers.TASK_TR_ERROR: str(ex)}) finally: self._logger.info(INF_FINISH_EXEC, self._event[handlers.HANDLER_EVENT_ACTION], self.action_id) self._logger.flush()