Exemplo n.º 1
0
def get_resource_stack(tested_action, create_resource_stack_func, use_existing=False, region_name=None):
    stack_region = region_name if region_name is not None else region()
    resource_stack_name = resources_stack_name(tested_action)
    resource_stack = Stack(resource_stack_name, owned=False, region=stack_region)
    if not use_existing or not resource_stack.is_stack_present():
        resource_stack = create_resource_stack_func(resource_stack_name)
        assert (resource_stack is not None)
        resource_stack.owned = True
    return resource_stack
Exemplo n.º 2
0
 def create_resource_stack(cls, resource_stack_name):
     try:
         cls.logger.test("Creating test resources stack {}", resource_stack_name)
         resource_stack = Stack(resource_stack_name, region=region())
         resource_stack.create_stack(template_file=template_path(__file__, TEST_RESOURCES_TEMPLATE),
                                     iam_capability=True,
                                     params={})
         return resource_stack
     except Exception as ex:
         cls.logger.test("Error creating stack {}, {}", resource_stack_name, ex)
         return None
Exemplo n.º 3
0
def get_task_runner(tested_action, use_existing_action_stack=False, interval=None):
    stack_name = action_stack_name(tested_action)
    if not use_existing_action_stack:
        action_stack = Stack(stack_name=stack_name, owned=True, region=region())
        if action_stack.is_stack_present():
            action_stack.delete_stack(1800)
    task_runner = TaskTestRunner(action_name=tested_action,
                                 action_stack_name=stack_name,
                                 task_list_tag_name=tasklist_tagname(tested_action),
                                 tested_region=region())
    task_runner.create_stack()
    return task_runner
Exemplo n.º 4
0
 def create_remote_resource_stack(cls, resource_stack_name):
     try:
         cls.logger.test("Creating test remote resources stack {}", resource_stack_name)
         role = cls.task_runner.action_stack.stack_resources[testing.OPS_AUTOMATOR_ROLE_NAME]["PhysicalResourceId"]
         resource_stack = Stack(resource_stack_name, region=remote_region())
         resource_stack.create_stack(template_file=template_path(__file__, TEST_REMOTE_RESOURCES_TEMPLATE),
                                     iam_capability=True,
                                     params={"TaskRole": role})
         return resource_stack
     except Exception as ex:
         cls.logger.test("Error creating stack {}, {}", resource_stack_name, ex)
         return None
Exemplo n.º 5
0
 def create_resource_stack(cls, resource_stack_name):
     try:
         cls.logger.test("Creating test resources stack {}", resource_stack_name)
         resource_stack = Stack(resource_stack_name, region=region())
         default_vpc_id = cls.ec2.get_default_vpc()
         assert default_vpc_id is not None
         resource_stack.create_stack(template_file=template_path(__file__, TEST_RESOURCES_TEMPLATE),
                                     timeout=1200,
                                     iam_capability=True,
                                     params={
                                         "VpcId": default_vpc_id["VpcId"],
                                     })
         return resource_stack
     except Exception as ex:
         cls.logger.test("Error creating stack {}, {}", resource_stack_name, ex)
         return None
 def create_resource_stack(cls, resource_stack_name):
     try:
         cls.logger.test("Creating test resources stack {}", resource_stack_name)
         ami = Ec2(region()).latest_aws_linux_image["ImageId"]
         resource_stack = Stack(resource_stack_name, region=region())
         resource_stack.create_stack(template_file=template_path(__file__, TEST_RESOURCES_TEMPLATE),
                                     timeout=1200,
                                     iam_capability=True,
                                     params={
                                         "InstanceAmi": ami,
                                         "InstanceType": TEST_INSTANCE_TYPES[0]
                                     })
         return resource_stack
     except Exception as ex:
         cls.logger.test("Error creating stack {}, {}", resource_stack_name, ex)
         return None
Exemplo n.º 7
0
 def create_resource_stack(cls, resource_stack_name):
     try:
         cls.logger.test("Creating test resources stack {}", resource_stack_name)
         ami = Ec2(region()).latest_aws_linux_image["ImageId"]
         resource_stack = Stack(resource_stack_name, region=region())
         resource_stack.create_stack(template_file=template_path(__file__, TEST_RESOURCES_TEMPLATE), iam_capability=True,
                                     params={
                                         "InstanceAmi": ami,
                                         "InstanceType": "t2.micro",
                                         "TaskListTagName": tasklist_tagname(TESTED_ACTION),
                                         "TaskListTagValue": ",".join(cls.get_methods())
                                     })
         return resource_stack
     except Exception as ex:
         cls.logger.test("Error creating stack {}, {}", resource_stack_name, ex)
         return None
Exemplo n.º 8
0
 def create_resource_stack(cls, resource_stack_name):
     try:
         cls.logger.test("Creating test resources stack {}",
                         resource_stack_name)
         resource_stack = Stack(resource_stack_name, region=region())
         resource_stack.create_stack(template_file=template_path(
             __file__, TEST_RESOURCES_TEMPLATE),
                                     iam_capability=False,
                                     params={
                                         "IndexName":
                                         INDEX_NAME,
                                         "TaskListTagName":
                                         tasklist_tagname(TESTED_ACTION),
                                         "TaskListTagValue":
                                         "/".join(cls.get_methods())
                                     })
         return resource_stack
     except Exception as ex:
         cls.logger.test("Error creating stack {}, {}", resource_stack_name,
                         ex)
         return None
    def _ensure_action_stack(self):

        if self.action_stack_template == {}:
            return

        if self._action_stack is None:
            self._action_stack = Stack(self.action_stack_name,
                                       region=self.test_region,
                                       owned=False)
            if not self._action_stack.is_stack_present():
                self._action_stack = Stack(self.action_stack_name,
                                           region=self.test_region,
                                           owned=True)
                self.logger.test(
                    "Creating action resources stack {} in region {}",
                    self.action_stack_name, self.test_region)
                if len(self.action_stack_template) > 0:
                    self._action_stack.create_stack(json.dumps(
                        self.action_stack_template),
                                                    iam_capability=True)
            else:
                self.logger.test("Using existing action stack {}",
                                 self.action_stack_name)
class TaskTestRunner(object):
    def __init__(self,
                 action_name,
                 tested_region=None,
                 test_region=None,
                 action_stack_name=None,
                 logger=None,
                 task_list_tag_name=None,
                 debug=False):

        self.debug = debug
        self.logger = logger if logger is not None else ConsoleLogger(
            debug=self.debug)
        if getattr(self.logger, "test") is None:
            setattr(self.logger, "test", getattr(self.logger, "info"))

        self.context = None

        self.action_name = action_name
        self.action_class = actions.get_action_class(self.action_name)
        self.action_properties = actions.get_action_properties(
            self.action_name)

        self.tested_region = tested_region if tested_region is not None else boto3.Session(
        ).region_name
        self.test_region = test_region if test_region is not None else boto3.Session(
        ).region_name

        if task_list_tag_name is not None:
            os.environ[handlers.ENV_AUTOMATOR_TAG_NAME] = task_list_tag_name

        self.interval = None

        os.environ[
            handlers.ENV_OPS_AUTOMATOR_ACCOUNT] = services.get_aws_account()

        if action_stack_name is None:
            self.action_stack_name = testing.action_stack_name(
                self.action_name)
        else:
            self.action_stack_name = action_stack_name

        self._action_stack = None

        os.environ[handlers.ENV_STACK_NAME] = self.action_stack_name

        actions.set_report_output_provider(create_output_writer)

        self._action_stack_resources = None

        self._assumed_role = None
        self._tested_account = None
        self._session = None

        self._cloudformation_client = None
        self._action_stack_template = None
        self._events = None
        self._tag_filter = None

        self.results = None
        self.parameters = None
        self.task_name = None
        self.action_select_parameters = None
        self.run_after_select = None

        self.max_concurrency = None
        self.concurrency_key = None
        self.log_subject = None
        self.run_in_regions = None

        self.run_after_select = None

        self.executed_tasks = []

    @property
    def action_stack_template(self):
        if self._action_stack_template is None:

            self._action_stack_template = copy.deepcopy(testing.TEMPLATE)
            self._action_stack_template["Resources"][
                testing.OPS_AUTOMATOR_ROLE_NAME]["Properties"][
                    "RoleName"] = testing.assumed_test_role_name(
                        self.action_name)
            if len(
                    self.action_properties.get(actions.ACTION_STACK_RESOURCES,
                                               {})) > 0:
                self._add_action_stack_resources_to_action_stack_template()

            # noinspection PyTypeChecker,PyPep8
            self._action_stack_template["Resources"][
                testing.OPS_AUTOMATOR_ROLE_NAME]["Properties"][
                    "AssumeRolePolicyDocument"]["Statement"][0]["Principal"][
                        "AWS"] = boto3.client(
                            "sts").get_caller_identity()["Arn"]
            self._add_actions_permissions_to_action_stack_template()

            self._action_stack_template[
                "Description"] = "Stack containing resources for test of Ops Automator action {}".format(
                    self.action_name)

        return self._action_stack_template

    @property
    def tested_account(self):
        if self._tested_account is None:
            if self._assumed_role not in [None, ""]:
                self._tested_account = self._assumed_role = services.account_from_role_arn(
                    self._assumed_role)
            else:
                self._tested_account = boto3.client(
                    "sts").get_caller_identity()['Account']
        return self._tested_account

    @property
    def action_stack(self):
        if self._action_stack is None:
            self._ensure_action_stack()
        return self._action_stack

    @property
    def action_stack_resources(self):
        if self._action_stack_resources is None:

            self._action_stack_resources = {}
            if self._action_stack is not None:
                # test if this action has additional stack resources
                resources = actions.get_action_properties(
                    self.action_name).get(actions.ACTION_STACK_RESOURCES, {})
                if resources:
                    # name of the class
                    class_name = self.action_properties[
                        actions.ACTION_CLASS_NAME][0:-len("Action")]
                    # actual resource names is name of class + name from class properties
                    logical_resource_names = [
                        class_name + resource_name
                        for resource_name in resources
                    ]

                    for res in self.action_stack.stack_resources.values():
                        # actual name
                        logical_resource_id = res["LogicalResourceId"]
                        # test if this resource is an resource from the action properties
                        if logical_resource_id in logical_resource_names:
                            self._action_stack_resources[
                                logical_resource_id[len(class_name):]] = {
                                    i: res[i]
                                    for i in [
                                        "LogicalResourceId",
                                        "PhysicalResourceId", "ResourceType"
                                    ]
                                }

                            if len(self._action_stack_resources.keys()) == len(
                                    resources):
                                return self._action_stack_resources

        return self._action_stack_resources

    def _build_config_item(self):

        item = {
            configuration.CONFIG_ACTION_NAME:
            self.action_name,
            configuration.CONFIG_PARAMETERS:
            self.parameters,
            configuration.CONFIG_ENABLED:
            True,
            configuration.CONFIG_DEBUG:
            self.debug,
            configuration.CONFIG_THIS_ACCOUNT:
            False,
            configuration.CONFIG_ACCOUNTS: [self.tested_account],
            configuration.CONFIG_TASK_CROSS_ACCOUNT_ROLE_NAME:
            testing.assumed_test_role_name(self.action_name),
            configuration.CONFIG_DRYRUN:
            False,
            configuration.CONFIG_TASK_NOTIFICATIONS:
            False,
            configuration.CONFIG_INTERNAL:
            self.action_properties.get(actions.ACTION_INTERNAL, False),
            configuration.CONFIG_TASK_NAME:
            self.task_name,
            configuration.CONFIG_TAG_FILTER:
            self._tag_filter,
            configuration.CONFIG_TIMEZONE:
            "UTC",
            configuration.CONFIG_REGIONS:
            self.run_in_regions,
            configuration.CONFIG_TASK_METRICS:
            False,
            configuration.CONFIG_EVENTS:
            self._events
        }

        if getattr(self.action_class, handlers.COMPLETION_METHOD,
                   None) is not None:
            timeout_value = self.parameters.get(actions.ACTION_PARAM_TIMEOUT)
            if timeout_value is None:
                timeout_value = self.action_properties.get(
                    actions.ACTION_COMPLETION_TIMEOUT_MINUTES,
                    actions.DEFAULT_COMPLETION_TIMEOUT_MINUTES_DEFAULT)
            item[configuration.CONFIG_TASK_TIMEOUT] = timeout_value

        use_intervals = actions.ACTION_TRIGGER_INTERVAL[
            0] in self.action_properties.get(actions.ACTION_TRIGGERS,
                                             actions.ACTION_TRIGGER_BOTH)
        if use_intervals:
            item[
                configuration.
                CONFIG_INTERVAL] = self.interval if self.interval is not None else "0 0 * * ?"

        return item

    def _build_tasks_for_selected_resources(self):
        try:

            configured_task = configuration.task_configuration.TaskConfiguration(self.context, self.logger) \
                .configuration_item_to_task(self._build_config_item())

            select_event = {
                handlers.HANDLER_EVENT_ACTION:
                handlers.HANDLER_ACTION_SELECT_RESOURCES,
                handlers.HANDLER_EVENT_CUSTOM_SELECT:
                True,
                handlers.HANDLER_EVENT_SOURCE:
                "{}-test".format(
                    configured_task[handlers.TASK_ACTION]).lower(),
                handlers.HANDLER_EVENT_TASK:
                configured_task,
                handlers.HANDLER_EVENT_TASK_DT:
                datetime.now().isoformat(),
                handlers.HANDLER_SELECT_ARGUMENTS:
                self.action_select_parameters,
                handlers.HANDLER_EVENT_TASK_GROUP:
                self.task_name
            }

            self.task_tracking = TaskTracker()
            select_handler = SelectResourcesHandler(select_event, self.context,
                                                    self.logger,
                                                    self.task_tracking)
            select_handler.handle_request()
            if self.run_after_select is not None:
                self.run_after_select(self.task_tracking)
            return self.task_tracking.task_items
        except Exception as ex:
            self.logger.test("Error building tasks for resources, {}", ex)
            raise ex

    def _get_tasks_to_execute(self):

        task_items = self._build_tasks_for_selected_resources()

        for item in task_items:
            event = {i: item.get(i) for i in item}
            event[handlers.
                  HANDLER_EVENT_ACTION] = handlers.HANDLER_ACTION_EXECUTE
            event[actions.ACTION_SERVICE] = actions.get_action_properties(
                item[handlers.TASK_TR_ACTION]).get(actions.ACTION_SERVICE)
            action_argument = {
                actions.ACTION_PARAM_CONTEXT:
                self.context,
                actions.ACTION_PARAM_EVENT:
                event,
                actions.ACTION_PARAM_SESSION:
                services.get_session(
                    role_arn=item.get(handlers.TASK_TR_ASSUMED_ROLE, None)),
                actions.ACTION_PARAM_RESOURCES:
                handlers.get_item_resource_data(item, context=self.context),
                actions.ACTION_PARAM_DEBUG:
                item[handlers.TASK_TR_DEBUG],
                actions.ACTION_PARAM_DRYRUN:
                item[handlers.TASK_TR_DRYRUN],
                actions.ACTION_PARAM_TASK_ID:
                item[handlers.TASK_TR_ID],
                actions.ACTION_PARAM_TASK:
                item[handlers.TASK_TR_NAME],
                actions.ACTION_PARAM_TASK_TIMEZONE:
                item[handlers.TASK_TR_TIMEZONE],
                actions.ACTION_PARAM_STACK:
                self.action_stack.stack_name
                if self.action_stack is not None else None,
                actions.ACTION_PARAM_STACK_ID:
                self.action_stack.stack_id
                if self.action_stack is not None else None,
                actions.ACTION_PARAM_STACK_RESOURCES:
                self.action_stack_resources,
                actions.ACTION_PARAM_ASSUMED_ROLE:
                item.get(handlers.TASK_TR_ASSUMED_ROLE),
                actions.ACTION_PARAM_STARTED_AT:
                item[handlers.TASK_TR_STARTED_TS],
                actions.ACTION_PARAM_TAGFILTER:
                item[handlers.TASK_TR_TAGFILTER],
                actions.ACTION_PARAM_TIMEOUT:
                item[handlers.TASK_TR_TIMEOUT],
                actions.ACTION_PARAM_LOGGER:
                self.logger,
                actions.ACTION_PARAM_EVENTS:
                self._events,
                actions.ACTION_PARAM_HAS_COMPLETION:
                item[handlers.TASK_TR_HAS_COMPLETION],
                actions.ACTION_PARAM_INTERVAL:
                item[handlers.TASK_INTERVAL]
            }

            if self._assumed_role is None:
                self._assumed_role = action_argument[
                    actions.ACTION_PARAM_ASSUMED_ROLE]

            action_instance = self.action_class(
                action_argument, item.get(handlers.TASK_TR_PARAMETERS, {}))

            self.verify_log_subject(action_argument)
            self.verify_concurrency(action_argument, item)

            yield action_instance

    def verify_concurrency(self, action_argument, item):
        concurrency_key_method = getattr(
            self.action_class, handlers.ACTION_CONCURRENCY_KEY_METHOD, None)
        # prepare parameters for calling static function that returns the concurrency key
        if concurrency_key_method is not None:
            get_key_params = {
                actions.ACTION_PARAM_RESOURCES:
                action_argument[actions.ACTION_PARAM_RESOURCES],
                actions.ACTION_PARAM_ACCOUNT:
                item[handlers.TASK_TR_ACCOUNT],
                actions.ACTION_PARAM_STACK:
                action_argument[actions.ACTION_PARAM_STACK],
                actions.ACTION_PARAM_STACK_ID:
                action_argument[actions.ACTION_PARAM_STACK_ID],
                actions.ACTION_PARAM_TASK_ID:
                action_argument[actions.ACTION_PARAM_STACK_ID],
                actions.ACTION_PARAM_TASK:
                item[handlers.TASK_TR_NAME]
            }
            get_key_params.update(item.get(handlers.TASK_TR_PARAMETERS))
            key = concurrency_key_method(get_key_params)
            assert (key is not None)
            self.logger.test("Action concurrency key is ", key)
            self.concurrency_key = key

        # test if there are concurrency restrictions
        max_action_concurrency = self.action_properties.get(
            actions.ACTION_MAX_CONCURRENCY)
        # has maximum method
        if max_action_concurrency is not None and types.FunctionType == type(
                max_action_concurrency):
            # noinspection PyBroadException,PyPep8
            try:
                parameters = item[handlers.TASK_TR_PARAMETERS]
                max_action_concurrency = max_action_concurrency(parameters)
            except:
                max_action_concurrency = None
            assert (max_action_concurrency is not None)
            self.logger.test("Concurrency method returns {}",
                             max_action_concurrency)
        self.max_concurrency = max_action_concurrency

    def verify_log_subject(self, action_argument):
        if callable(getattr(self.action_class, "action_logging_subject",
                            None)):
            # noinspection PyBroadException,PyPep8
            try:
                action_log_subject = self.action_class.action_logging_subject(
                    action_argument, self.parameters)
            except:
                action_log_subject = None
            assert (action_log_subject is not None)
            self.logger.test("Action log subject is {}", action_log_subject)
            self.log_subject = action_log_subject

    def _add_action_stack_resources_to_action_stack_template(self):
        def fix_resource_references(resources, old, new):
            def update_list(l, old_name, new_name):
                for item in l:
                    if isinstance(item, dict):
                        fix_resource_references(item, old_name, new_name)
                    elif isinstance(item, list):
                        update_list(item, old_name, new_name)

            for key in resources:
                val = resources[key]
                if key == "Ref" and val == old:
                    resources[key] = new
                if isinstance(val, dict):
                    fix_resource_references(val, old, new)

                elif isinstance(val, list):
                    update_list(val, old, new)

        template_resources = self.action_stack_template["Resources"]

        action_statement = template_resources[testing.OPS_AUTOMATOR_ROLE_NAME][
            "Properties"]["Policies"][0]["PolicyDocument"]["Statement"]

        stack_resources = self.action_properties.get(
            actions.ACTION_STACK_RESOURCES)
        stack_resource_permissions = self.action_properties.get(
            actions.ACTION_STACK_RESOURCES_PERMISSIONS, {})

        if stack_resources:

            resource_names = []

            action_resources_to_add = {}
            # get additional resources and build new dict with prefixed names
            for resource_name in stack_resources:
                prefixed_resource_name = self.action_name + resource_name
                resource_names.append((resource_name, prefixed_resource_name))
                action_resources_to_add[
                    prefixed_resource_name] = stack_resources[resource_name]

            # fix names of prefixed resource names in references
            for resource_name in resource_names:
                # references in stack resources
                fix_resource_references(action_resources_to_add,
                                        resource_name[0], resource_name[1])
                # references in list of permissions for stack resources
                for i in stack_resource_permissions["Resource"]:
                    if isinstance(i, dict):
                        fix_resource_references(i, resource_name[0],
                                                resource_name[1])

            # add the resources for this action to the template
            for resource_name in action_resources_to_add:
                template_resources[resource_name] = action_resources_to_add[
                    resource_name]

            if len(stack_resource_permissions) != 0:
                # statements = build_action_policy_statement(action_name, stack_resource_permissions)
                stack_resource_permissions["Sid"] = re.sub(
                    "[^0-9A-Za-z]", "", self.action_name + "Resources")
                action_statement.append(stack_resource_permissions)

    def _add_actions_permissions_to_action_stack_template(self):
        def build_action_policy_statement(action_name,
                                          added_action_permissions):
            statements = []

            if len(added_action_permissions) > 0:
                statements.append({
                    "Sid":
                    re.sub("[^0-9A-Za-z]", "", action_name),
                    "Effect":
                    "Allow",
                    "Resource":
                    "*",
                    "Action":
                    sorted(list(set(added_action_permissions)))
                })
            return statements

        def action_select_resources_permissions(action_prop):
            return services.get_resource_describe_permissions(
                action_prop[actions.ACTION_SERVICE],
                [action_prop[actions.ACTION_RESOURCES]])

        # noinspection PyTypeChecker
        ops_automator_role = self.action_stack_template["Resources"][
            "OpsAutomatorRole"]
        action_statement = ops_automator_role["Properties"]["Policies"][0][
            "PolicyDocument"]["Statement"]

        required_actions = set()

        action_permissions = self.action_properties.get(
            actions.ACTION_PERMISSIONS, [])
        action_permissions += list(
            action_select_resources_permissions(self.action_properties))

        if len(action_permissions) != 0:
            required_actions.update(action_permissions)

        action_statement += build_action_policy_statement(
            "ActionPermissions", required_actions)

    def _ensure_action_stack(self):

        if self.action_stack_template == {}:
            return

        if self._action_stack is None:
            self._action_stack = Stack(self.action_stack_name,
                                       region=self.test_region,
                                       owned=False)
            if not self._action_stack.is_stack_present():
                self._action_stack = Stack(self.action_stack_name,
                                           region=self.test_region,
                                           owned=True)
                self.logger.test(
                    "Creating action resources stack {} in region {}",
                    self.action_stack_name, self.test_region)
                if len(self.action_stack_template) > 0:
                    self._action_stack.create_stack(json.dumps(
                        self.action_stack_template),
                                                    iam_capability=True)
            else:
                self.logger.test("Using existing action stack {}",
                                 self.action_stack_name)

    def run(self,
            parameters,
            complete_check_polling_interval=60,
            task_timeout=None,
            task_name=None,
            datetime_delta=None,
            events=None,
            tag_filter=None,
            run_in_regions=None,
            action_select_params=None,
            debug=False,
            run_after_select=None):

        self.results = []
        self.executed_tasks = []
        self.parameters = parameters
        self.action_select_parameters = action_select_params if action_select_params is not None else {}
        self.task_name = task_name if task_name is not None else "{}-test".format(
            self.action_name).lower()
        self._ensure_action_stack()

        self.run_after_select = run_after_select

        self.context = Context()

        self._events = events if events is not None else {}
        self._tag_filter = tag_filter

        save_debug = self.debug
        self.debug = debug
        self.logger._debug = self.debug

        self.interval = parameters.get(actions.ACTION_PARAM_INTERVAL, None)

        self.run_in_regions = run_in_regions if run_in_regions is not None else [
            self.test_region
        ]

        try:
            if datetime_delta is not None:
                set_datetime_delta(datetime_delta)
                actions.set_date_time_provider(DatetimeProvider)
                self.logger.test(
                    "Setting simulated test execution date and time to {}",
                    actions.date_time_provider().now())

            for executed_task in self._get_tasks_to_execute():
                try:
                    self.executed_tasks.append(executed_task)
                    self.logger.test(
                        "Start execution of action {} using assumed role {} in region {}",
                        self.action_name, self._assumed_role,
                        self.tested_region)
                    start_result = executed_task.execute()
                    if not executed_task.get(
                            actions.ACTION_PARAM_HAS_COMPLETION, False):
                        self.logger.test("Action completed with result {}",
                                         safe_json(start_result, indent=3))
                        setattr(executed_task, handlers.TASK_TR_RESULT,
                                start_result)
                        setattr(executed_task, handlers.TASK_TR_STATUS,
                                handlers.STATUS_COMPLETED)
                    else:
                        self.logger.test("Waiting for task to complete")
                        setattr(executed_task, handlers.TASK_TR_START_RESULT,
                                start_result)

                        # noinspection PyProtectedMember
                        timeout = executed_task._timeout_
                        if timeout is None:
                            timeout = task_timeout * 60 if task_timeout is not None else 60
                        timeout *= 60
                        with Timer(timeout_seconds=timeout) as timer:
                            while True:
                                is_completed = getattr(
                                    executed_task, handlers.COMPLETION_METHOD,
                                    None)
                                if is_completed is None:
                                    raise Exception(
                                        "Tested action needs completion but does not implement the required {} method",
                                        handlers.COMPLETION_METHOD)
                                complete_result = executed_task.is_completed(
                                    start_result)
                                if complete_result is not None:
                                    self.logger.test(
                                        "Action completed with result {}",
                                        safe_json(complete_result, indent=3))
                                    setattr(executed_task,
                                            handlers.TASK_TR_STATUS,
                                            handlers.STATUS_COMPLETED)
                                    setattr(executed_task,
                                            handlers.TASK_TR_RESULT,
                                            complete_result)
                                    break
                                if timer.timeout:
                                    self.logger.test("Action timed out")
                                    setattr(executed_task,
                                            handlers.TASK_TR_STATUS,
                                            handlers.STATUS_TIMED_OUT)
                                    setattr(executed_task,
                                            handlers.TASK_TR_ERROR, "Timeout")
                                    break
                                self.logger.test(
                                    "Action not completed yet, waiting {} seconds",
                                    complete_check_polling_interval)
                                time.sleep(complete_check_polling_interval)
                    self.results.append(executed_task)

                except Exception as ex:
                    self.logger.test("Action failed {}", str(ex))
                    setattr(executed_task, handlers.TASK_TR_STATUS,
                            handlers.STATUS_FAILED)
                    setattr(executed_task, handlers.TASK_TR_ERROR, str(ex))
                    self.results.append(executed_task)

        finally:
            if datetime_delta is not None:
                actions.reset_date_provider()
            self.debug = save_debug
            self.logger._debug = self.debug

        return self.results

    def create_stack(self):
        self._ensure_action_stack()

    def cleanup(self, keep_action_stack=False):
        # noinspection PyBroadException,PyPep8
        try:
            if self._action_stack is not None:
                if self.action_stack.owned and (not keep_action_stack):
                    self.logger.test("Deleting stack {}",
                                     self.action_stack_name)
                    self.action_stack.delete_stack(empty_bucket_resources=True)
        except:
            pass

    def success(self, expected_executed_tasks=None, results=None):
        checked_results = results if results is not None else self.results
        if expected_executed_tasks is not None and expected_executed_tasks != len(
                checked_results):
            return False
        return all([
            getattr(a, handlers.TASK_TR_STATUS) == handlers.STATUS_COMPLETED
            for a in self.results
        ])