Esempio n. 1
0
 def hash_role(hash_obj, role_path):
     # Role content is easy to hash - the hash of the role content with the
     # hash of any role dependencies it has
     hash_dir(hash_obj, role_path)
     for dependency in get_dependencies_for_role(role_path):
         if dependency:
             dependency_path = resolve_role_to_path(dependency)
             hash_role(hash_obj, dependency_path)
     # However tasks within that role might reference files outside of the
     # role, like source code
     loader = DataLoader()
     var_man = VariableManager(loader=loader)
     play = Play.load(generate_playbook_for_role(service_name, config_vars, role)[0],
                      variable_manager=var_man, loader=loader)
     play_context = PlayContext(play=play)
     inv_man = InventoryManager(loader, sources=['%s,' % service_name])
     host = Host(service_name)
     iterator = PlayIterator(inv_man, play, play_context, var_man, config_vars)
     while True:
         _, task = iterator.get_next_task_for_host(host)
         if task is None: break
         if task.action in FILE_COPY_MODULES:
             src = task.args.get('src')
             if src is not None:
                 if not os.path.exists(src) or not src.startswith(('/', '..')): continue
                 src = os.path.realpath(src)
                 if os.path.isfile(src):
                     hash_file(hash_obj, src)
                 else:
                     hash_dir(hash_obj, src)
Esempio n. 2
0
 def test_iterating_states_deprecation_instance_attr(self):
     iterator = PlayIterator(MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock())
     assert iterator.ITERATING_SETUP == IteratingStates.SETUP
     assert iterator.ITERATING_TASKS == IteratingStates.TASKS
     assert iterator.ITERATING_RESCUE == IteratingStates.RESCUE
     assert iterator.ITERATING_ALWAYS == IteratingStates.ALWAYS
     assert iterator.ITERATING_COMPLETE == IteratingStates.COMPLETE
Esempio n. 3
0
 def test_failed_states_deprecation_instance_attr(self):
     iterator = PlayIterator(MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock())
     assert iterator.FAILED_NONE == FailedStates.NONE
     assert iterator.FAILED_SETUP == FailedStates.SETUP
     assert iterator.FAILED_TASKS == FailedStates.TASKS
     assert iterator.FAILED_RESCUE == FailedStates.RESCUE
     assert iterator.FAILED_ALWAYS == FailedStates.ALWAYS
Esempio n. 4
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        # Fork # of forks, # of hosts or serial, whichever is lowest
        contenders =  [self._options.forks, play.serial, len(self._inventory.get_hosts(play.hosts))]
        contenders =  [ v for v in contenders if v is not None and v > 0 ]
        self._initialize_processes(min(contenders))

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)

        play_context = PlayContext(new_play, self._options, self.passwords, self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play.handlers)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" % new_play.strategy, obj=play._ds)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done = self._start_at_done,
        )

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if getattr(self._options, 'start_at_task', None) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(iterator, play_context)
        self._cleanup_processes()
        return play_return
Esempio n. 5
0
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml':
            """
            - debug: msg="this is a role task"
            """,
        })

        p = Playbook.load('test_play.yml', loader=fake_loader)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            all_vars=dict(),
        )

        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNone(task)
Esempio n. 6
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(loader=self._loader,
                                                   play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)

        play_context = PlayContext(new_play, self._options, self.passwords,
                                   self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play.handlers)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               new_play.strategy,
                               obj=play._ds)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
        )

        # and run the play using the strategy
        return strategy.run(iterator, play_context)
Esempio n. 7
0
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        p = Playbook.load('test_play.yml', loader=fake_loader)

        hosts = []
        for i in range(0, 10):
            host  = MagicMock()
            host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        connection_info = ConnectionInformation(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            connection_info=connection_info,
            all_vars=dict(),
        )

        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNone(task)
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        connection_info = ConnectionInformation(play, self._options)
        self._callback.set_connection_info(connection_info)

        # run final validation on the play now, to make sure fields are templated
        # FIXME: is this even required? Everything is validated and merged at the
        #        task level, so else in the play needs to be templated
        #all_vars = self._vmw.get_vars(loader=self._dlw, play=play)
        #all_vars = self._vmw.get_vars(loader=self._loader, play=play)
        #play.post_validate(all_vars=all_vars)

        self._callback.playbook_on_play_start(play.name)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(play.handlers)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               play.strategy,
                               obj=play._ds)

        # build the iterator
        iterator = PlayIterator(inventory=self._inventory, play=play)

        # and run the play using the strategy
        return strategy.run(iterator, connection_info)
Esempio n. 9
0
    def test_play_iterator_add_tasks(self):
        fake_loader = DictDataLoader({
            'test_play.yml':
            """
            - hosts: all
              gather_facts: no
              tasks:
              - debug: msg="dummy task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # test the high-level add_tasks() method
        s = HostState(blocks=[0, 1, 2])
        itr._insert_tasks_into_state = MagicMock(return_value=s)
        itr.add_tasks(hosts[0], [MagicMock(), MagicMock(), MagicMock()])
        self.assertEqual(itr._host_states[hosts[0].name], s)

        # now actually test the lower-level method that does the work
        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # iterate past first task
        _, task = itr.get_next_task_for_host(hosts[0])
        while (task and task.action != 'debug'):
            _, task = itr.get_next_task_for_host(hosts[0])

        if task is None:
            raise Exception(
                "iterated past end of play while looking for place to insert tasks"
            )

        # get the current host state and copy it so we can mutate it
        s = itr.get_host_state(hosts[0])
        s_copy = s.copy()

        # assert with an empty task list, or if we're in a failed state, we simply return the state as-is
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
        self.assertEqual(res_state, s_copy)

        s_copy.fail_state = itr.FAILED_TASKS
        res_state = itr._insert_tasks_into_state(s_copy,
                                                 task_list=[MagicMock()])
        self.assertEqual(res_state, s_copy)

        # but if we've failed with a rescue/always block
        mock_task = MagicMock()
        s_copy.run_state = itr.ITERATING_RESCUE
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
        self.assertEqual(res_state, s_copy)
        self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
        itr._host_states[hosts[0].name] = res_state
        (next_state, next_task) = itr.get_next_task_for_host(hosts[0],
                                                             peek=True)
        self.assertEqual(next_task, mock_task)
        itr._host_states[hosts[0].name] = s

        # test a regular insertion
        s_copy = s.copy()
        res_state = itr._insert_tasks_into_state(s_copy,
                                                 task_list=[MagicMock()])
Esempio n. 10
0
    def test_play_iterator_nested_blocks(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: false
              tasks:
              - block:
                - block:
                  - block:
                    - block:
                      - block:
                        - debug: msg="this is the first task"
                        - ping:
                      rescue:
                      - block:
                        - block:
                          - block:
                            - block:
                              - debug: msg="this is the rescue task"
                  always:
                  - block:
                    - block:
                      - block:
                        - block:
                          - debug: msg="this is the always task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # get the first task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the first task'))
        # fail the host
        itr.mark_host_failed(hosts[0])
        # get the resuce task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the rescue task'))
        # get the always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the always task'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)
Esempio n. 11
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(play=play)
        warn_if_reserved(all_vars)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)
        new_play.handlers = new_play.compile_roles_handlers(
        ) + new_play.handlers

        self.hostvars = HostVars(
            inventory=self._inventory,
            variable_manager=self._variable_manager,
            loader=self._loader,
        )

        # Fork # of forks, # of hosts or serial, whichever is lowest
        num_hosts = len(
            self._inventory.get_hosts(new_play.hosts,
                                      ignore_restrictions=True))

        max_serial = 0
        if new_play.serial:
            # the play has not been post_validated here, so we may need
            # to convert the scalar value to a list at this point
            serial_items = new_play.serial
            if not isinstance(serial_items, list):
                serial_items = [serial_items]
            max_serial = max([pct_to_int(x, num_hosts) for x in serial_items])

        contenders = [self._options.forks, max_serial, num_hosts]
        contenders = [v for v in contenders if v is not None and v > 0]
        self._initialize_processes(min(contenders))

        play_context = PlayContext(new_play, self._options, self.passwords,
                                   self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               new_play.strategy,
                               obj=play._ds)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done=self._start_at_done,
        )

        # Because the TQM may survive multiple play runs, we start by marking
        # any hosts as failed in the iterator here which may have been marked
        # as failed in previous runs. Then we clear the internal list of failed
        # hosts so we know what failed this round.
        for host_name in self._failed_hosts.keys():
            host = self._inventory.get_host(host_name)
            iterator.mark_host_failed(host)

        self.clear_failed_hosts()

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if getattr(self._options, 'start_at_task',
                   None) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(iterator, play_context)

        # now re-save the hosts that failed from the iterator to our internal list
        for host_name in iterator.get_failed_hosts():
            self._failed_hosts[host_name] = True

        strategy.cleanup()
        self._cleanup_processes()
        return play_return
Esempio n. 12
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(play=play)
        warn_if_reserved(all_vars)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)
        new_play.handlers = new_play.compile_roles_handlers(
        ) + new_play.handlers

        self.hostvars = HostVars(
            inventory=self._inventory,
            variable_manager=self._variable_manager,
            loader=self._loader,
        )

        play_context = PlayContext(new_play, self.passwords,
                                   self._connection_lockfile.fileno())
        if (self._stdout_callback
                and hasattr(self._stdout_callback, 'set_play_context')):
            self._stdout_callback.set_play_context(play_context)

        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done=self._start_at_done,
        )

        # adjust to # of workers to configured forks or size of batch, whatever is lower
        self._initialize_processes(min(self._forks, iterator.batch_size))

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               new_play.strategy,
                               obj=play._ds)

        # Because the TQM may survive multiple play runs, we start by marking
        # any hosts as failed in the iterator here which may have been marked
        # as failed in previous runs. Then we clear the internal list of failed
        # hosts so we know what failed this round.
        for host_name in self._failed_hosts.keys():
            host = self._inventory.get_host(host_name)
            iterator.mark_host_failed(host)

        self.clear_failed_hosts()

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if context.CLIARGS.get(
                'start_at_task'
        ) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(iterator, play_context)

        # now re-save the hosts that failed from the iterator to our internal list
        for host_name in iterator.get_failed_hosts():
            self._failed_hosts[host_name] = True

        strategy.cleanup()
        self._cleanup_processes()
        return play_return
Esempio n. 13
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)
        new_play.handlers = new_play.compile_roles_handlers() + new_play.handlers

        self.hostvars = HostVars(
            inventory=self._inventory,
            variable_manager=self._variable_manager,
            loader=self._loader,
        )

        # Fork # of forks, # of hosts or serial, whichever is lowest
        num_hosts = len(self._inventory.get_hosts(new_play.hosts, ignore_restrictions=True))

        max_serial = 0
        if new_play.serial:
            # the play has not been post_validated here, so we may need
            # to convert the scalar value to a list at this point
            serial_items = new_play.serial
            if not isinstance(serial_items, list):
                serial_items = [serial_items]
            max_serial = max([pct_to_int(x, num_hosts) for x in serial_items])

        contenders = [self._options.forks, max_serial, num_hosts]
        contenders = [v for v in contenders if v is not None and v > 0]
        self._initialize_processes(min(contenders))

        play_context = PlayContext(new_play, self._options, self.passwords, self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" % new_play.strategy, obj=play._ds)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done = self._start_at_done,
        )

        # Because the TQM may survive multiple play runs, we start by marking
        # any hosts as failed in the iterator here which may have been marked
        # as failed in previous runs. Then we clear the internal list of failed
        # hosts so we know what failed this round.
        for host_name in self._failed_hosts.keys():
            host = self._inventory.get_host(host_name)
            iterator.mark_host_failed(host)

        self.clear_failed_hosts()

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if getattr(self._options, 'start_at_task', None) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(iterator, play_context)

        # now re-save the hosts that failed from the iterator to our internal list
        for host_name in iterator.get_failed_hosts():
            self._failed_hosts[host_name] = True

        strategy.cleanup()
        self._cleanup_processes()
        return play_return
Esempio n. 14
0
    def test_noop(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: localhost
              gather_facts: no
              tasks:
              - name: simple_function
                debug:
                  msg : 'hello world'
                tags:
                 - functional

              - name: call_simple_function
                call_task:
                  name : "simple_function"

            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        inventory = MagicMock()
        inventory.hosts = {}
        hosts = []

        host = MagicMock()
        host.name = host.get_name.return_value = 'localhost'
        hosts.append(host)
        inventory.hosts[host.name] = host

        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        mock_var_manager._fact_cache['localhost'] = dict()

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        tqm = TaskQueueManager(
            inventory=inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            passwords=None,
            forks=5,
        )
        tqm._initialize_processes(3)
        strategy = StrategyModule(tqm)
        strategy._hosts_cache = [h.name for h in hosts]
        strategy._hosts_cache_all = [h.name for h in hosts]

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
    def test_play_iterator_add_tasks(self):
        fake_loader = DictDataLoader({
            'test_play.yml': """
            - hosts: all
              gather_facts: no
              tasks:
              - debug: msg="dummy task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # test the high-level add_tasks() method
        s = HostState(blocks=[0,1,2])
        itr._insert_tasks_into_state = MagicMock(return_value=s)
        itr.add_tasks(hosts[0], [3,4,5])
        self.assertEqual(itr._host_states[hosts[0].name], s)

        # now actually test the lower-level method that does the work
        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # iterate past first task
        _, task = itr.get_next_task_for_host(hosts[0])
        while(task and task.action != 'debug'):
            _, task = itr.get_next_task_for_host(hosts[0])

        if task is None:
            raise Exception("iterated past end of play while looking for place to insert tasks")

        # get the current host state and copy it so we can mutate it
        s = itr.get_host_state(hosts[0])
        s_copy = s.copy()

        # assert with an empty task list, or if we're in a failed state, we simply return the state as-is
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
        self.assertEqual(res_state, s_copy)

        s_copy.fail_state = itr.FAILED_TASKS
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
        self.assertEqual(res_state, s_copy)

        # but if we've failed with a rescue/always block
        mock_task = MagicMock()
        s_copy.run_state = itr.ITERATING_RESCUE
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
        self.assertEqual(res_state, s_copy)
        self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
        itr._host_states[hosts[0].name] = res_state
        (next_state, next_task) = itr.get_next_task_for_host(hosts[0], peek=True)
        self.assertEqual(next_task, mock_task)
        itr._host_states[hosts[0].name] = s

        # test a regular insertion
        s_copy = s.copy()
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
    def test_noop(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: no
              tasks:
                - block:
                   - block:
                     - name: task1
                       debug: msg='task1'
                       failed_when: inventory_hostname == 'host01'

                     - name: task2
                       debug: msg='task2'

                     rescue:
                       - name: rescue1
                         debug: msg='rescue1'

                       - name: rescue2
                         debug: msg='rescue2'
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 2):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        mock_options = MagicMock()
        mock_options.module_path = None

        tqm = TaskQueueManager(
            inventory=inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=None,
        )
        tqm._initialize_processes(3)
        strategy = StrategyModule(tqm)

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'meta')

        # debug: task1, debug: task1
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'debug')
        self.assertEqual(host2_task.action, 'debug')
        self.assertEqual(host1_task.name, 'task1')
        self.assertEqual(host2_task.name, 'task1')

        # mark the second host failed
        itr.mark_host_failed(hosts[1])

        # debug: task2, meta: noop
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'debug')
        self.assertEqual(host2_task.action, 'meta')
        self.assertEqual(host1_task.name, 'task2')
        self.assertEqual(host2_task.name, '')

        # meta: noop, debug: rescue1
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'debug')
        self.assertEqual(host1_task.name, '')
        self.assertEqual(host2_task.name, 'rescue1')

        # meta: noop, debug: rescue2
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'debug')
        self.assertEqual(host1_task.name, '')
        self.assertEqual(host2_task.name, 'rescue2')

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'meta')

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'meta')

        # end of iteration
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNone(host1_task)
        self.assertIsNone(host2_task)
Esempio n. 17
0
    def test_play_iterator(self):
        # import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - name: role task
              debug: msg="this is a role task"
            - block:
              - name: role block task
                debug: msg="inside block in role"
              always:
              - name: role always task
                debug: msg="always task in block in role"
            - include: foo.yml
            - name: role task after include
              debug: msg="after include in role"
            - block:
              - name: starting role nested block 1
                debug:
              - block:
                - name: role nested block 1 task 1
                  debug:
                - name: role nested block 1 task 2
                  debug:
                - name: role nested block 1 task 3
                  debug:
              - name: end of role nested block 1
                debug:
              - name: starting role nested block 2
                debug:
              - block:
                - name: role nested block 2 task 1
                  debug:
                - name: role nested block 2 task 2
                  debug:
                - name: role nested block 2 task 3
                  debug:
              - name: end of role nested block 2
                debug:
            """,
            '/etc/ansible/roles/test_role/tasks/foo.yml': """
            - name: role included task
              debug: msg="this is task in an include from a role"
            """
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.name, "role task")
        self.assertIsNotNone(task._role)
        # role block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role block task")
        self.assertIsNotNone(task._role)
        # role block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role always task")
        self.assertIsNotNone(task._role)
        # role include task
        # (host_state, task) = itr.get_next_task_for_host(hosts[0])
        # self.assertIsNotNone(task)
        # self.assertEqual(task.action, 'debug')
        # self.assertEqual(task.name, "role included task")
        # self.assertIsNotNone(task._role)
        # role task after include
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role task after include")
        self.assertIsNotNone(task._role)
        # role nested block tasks
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 2")
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
Esempio n. 18
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        if play.vars_prompt:
            for var in play.vars_prompt:
                if 'name' not in var:
                    raise AnsibleError("'vars_prompt' item is missing 'name:'",
                                       obj=play._ds)

                vname = var['name']
                prompt = var.get("prompt", vname)
                default = var.get("default", None)
                private = var.get("private", True)

                confirm = var.get("confirm", False)
                encrypt = var.get("encrypt", None)
                salt_size = var.get("salt_size", None)
                salt = var.get("salt", None)

                if vname not in play.vars:
                    self.send_callback('v2_playbook_on_vars_prompt', vname,
                                       private, prompt, encrypt, confirm,
                                       salt_size, salt, default)
                    play.vars[vname] = self._do_var_prompt(
                        vname, private, prompt, encrypt, confirm, salt_size,
                        salt, default)

        all_vars = self._variable_manager.get_vars(loader=self._loader,
                                                   play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)

        connection_info = ConnectionInformation(new_play, self._options,
                                                self.passwords)
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_connection_info'):
                callback_plugin.set_connection_info(connection_info)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play.handlers)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               new_play.strategy,
                               obj=play._ds)

        # build the iterator
        iterator = PlayIterator(inventory=self._inventory,
                                play=new_play,
                                connection_info=connection_info,
                                all_vars=all_vars)

        # and run the play using the strategy
        return strategy.run(iterator, connection_info)
Esempio n. 19
0
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        p = Playbook.load('test_play.yml', loader=fake_loader)

        hosts = []
        for i in range(0, 10):
            host  = MagicMock()
            host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)
Esempio n. 20
0
class TaskQueueManager:
    '''
    This class handles the multiprocessing requirements of Ansible by
    creating a pool of worker forks, a result handler fork, and a
    manager object with shared datastructures/queues for coordinating
    work between all processes.

    The queue manager is responsible for loading the play strategy plugin,
    which dispatches the Play's tasks to hosts.
    '''

    RUN_OK = 0
    RUN_ERROR = 1
    RUN_FAILED_HOSTS = 2
    RUN_UNREACHABLE_HOSTS = 4
    RUN_FAILED_BREAK_PLAY = 8
    RUN_UNKNOWN_ERROR = 255

    def __init__(self,
                 inventory,
                 variable_manager,
                 loader,
                 options,
                 passwords,
                 stdout_callback=None,
                 run_additional_callbacks=True,
                 run_tree=False):

        self._inventory = inventory
        self._variable_manager = variable_manager
        self._loader = loader
        self._options = options
        self._stats = AggregateStats()
        self.passwords = passwords
        self._stdout_callback = stdout_callback
        self._run_additional_callbacks = run_additional_callbacks
        self._run_tree = run_tree

        self._iterator = None

        self._callbacks_loaded = False
        self._callback_plugins = []
        self._start_at_done = False

        # make sure the module path (if specified) is parsed and
        # added to the module_loader object
        if options.module_path is not None:
            for path in options.module_path.split(os.pathsep):
                module_loader.add_directory(path)

        # a special flag to help us exit cleanly
        self._terminated = False

        # this dictionary is used to keep track of notified handlers
        self._notified_handlers = dict()
        self._listening_handlers = dict()

        # dictionaries to keep track of failed/unreachable hosts
        self._failed_hosts = dict()
        self._unreachable_hosts = dict()

        # the "queue" for the background thread to use
        self._queued_tasks = deque()
        self._queued_tasks_lock = threading.Lock()

        # the background queuing thread
        self._queue_thread = None

        self._workers = []
        self._final_q = multiprocessing.Queue()

        # A temporary file (opened pre-fork) used by connection
        # plugins for inter-process locking.
        self._connection_lockfile = tempfile.TemporaryFile()

    def _queue_thread_main(self):

        # create a dummy object with plugin loaders set as an easier
        # way to share them with the forked processes
        shared_loader_obj = SharedPluginLoaderObj()

        display.debug("queuing thread starting")
        while not self._terminated:
            available_workers = []
            for idx, entry in enumerate(self._workers):
                (worker_prc, _) = entry
                if worker_prc is None or not worker_prc.is_alive():
                    available_workers.append(idx)

            if len(available_workers) == 0:
                time.sleep(0.01)
                continue

            for worker_idx in available_workers:
                try:
                    self._queued_tasks_lock.acquire()
                    (host, task, task_vars,
                     play_context) = self._queued_tasks.pop()
                except IndexError:
                    break
                finally:
                    self._queued_tasks_lock.release()

                if task.action not in action_write_locks.action_write_locks:
                    display.debug('Creating lock for %s' % task.action)
                    action_write_locks.action_write_locks[
                        task.action] = multiprocessing.Lock()

                try:
                    worker_prc = WorkerProcess(
                        self._final_q,
                        self._iterator._play,
                        host,
                        task,
                        task_vars,
                        play_context,
                        self._loader,
                        self._variable_manager,
                        shared_loader_obj,
                    )
                    self._workers[worker_idx][0] = worker_prc
                    worker_prc.start()
                    display.debug("worker is %d (out of %d available)" %
                                  (worker_idx + 1, len(self._workers)))

                except (EOFError, IOError, AssertionError) as e:
                    # most likely an abort
                    display.debug("got an error while queuing: %s" % e)
                    break

        display.debug("queuing thread exiting")

    def queue_task(self, host, task, task_vars, play_context):
        self._queued_tasks_lock.acquire()
        self._queued_tasks.append((host, task, task_vars, play_context))
        self._queued_tasks_lock.release()

    def queue_multiple_tasks(self, items, play_context):
        for item in items:
            (host, task, task_vars) = item
            self._queued_tasks_lock.acquire()
            self._queued_tasks.append((host, task, task_vars, play_context))
            self._queued_tasks_lock.release()

    def _initialize_processes(self, num):
        self._workers = []

        for i in range(num):
            rslt_q = multiprocessing.Queue()
            self._workers.append([None, rslt_q])

    def _initialize_notified_handlers(self, play):
        '''
        Clears and initializes the shared notified handlers dict with entries
        for each handler in the play, which is an empty array that will contain
        inventory hostnames for those hosts triggering the handler.
        '''

        # Zero the dictionary first by removing any entries there.
        # Proxied dicts don't support iteritems, so we have to use keys()
        self._notified_handlers.clear()
        self._listening_handlers.clear()

        def _process_block(b):
            temp_list = []
            for t in b.block:
                if isinstance(t, Block):
                    temp_list.extend(_process_block(t))
                else:
                    temp_list.append(t)
            return temp_list

        handler_list = []
        for handler_block in play.handlers:
            handler_list.extend(_process_block(handler_block))

        # then initialize it with the given handler list
        for handler in handler_list:
            if handler not in self._notified_handlers:
                self._notified_handlers[handler] = []
            if handler.listen:
                listeners = handler.listen
                if not isinstance(listeners, list):
                    listeners = [listeners]
                for listener in listeners:
                    if listener not in self._listening_handlers:
                        self._listening_handlers[listener] = []
                    self._listening_handlers[listener].append(
                        handler.get_name())

    def load_callbacks(self):
        '''
        Loads all available callbacks, with the exception of those which
        utilize the CALLBACK_TYPE option. When CALLBACK_TYPE is set to 'stdout',
        only one such callback plugin will be loaded.
        '''

        if self._callbacks_loaded:
            return

        stdout_callback_loaded = False
        if self._stdout_callback is None:
            self._stdout_callback = C.DEFAULT_STDOUT_CALLBACK

        if isinstance(self._stdout_callback, CallbackBase):
            stdout_callback_loaded = True
        elif isinstance(self._stdout_callback, string_types):
            if self._stdout_callback not in callback_loader:
                raise AnsibleError(
                    "Invalid callback for stdout specified: %s" %
                    self._stdout_callback)
            else:
                self._stdout_callback = callback_loader.get(
                    self._stdout_callback)
                stdout_callback_loaded = True
        else:
            raise AnsibleError(
                "callback must be an instance of CallbackBase or the name of a callback plugin"
            )

        for callback_plugin in callback_loader.all(class_only=True):
            if hasattr(callback_plugin, 'CALLBACK_VERSION'
                       ) and callback_plugin.CALLBACK_VERSION >= 2.0:
                # we only allow one callback of type 'stdout' to be loaded, so check
                # the name of the current plugin and type to see if we need to skip
                # loading this callback plugin
                callback_type = getattr(callback_plugin, 'CALLBACK_TYPE', None)
                callback_needs_whitelist = getattr(callback_plugin,
                                                   'CALLBACK_NEEDS_WHITELIST',
                                                   False)
                (callback_name, _) = os.path.splitext(
                    os.path.basename(callback_plugin._original_path))
                if callback_type == 'stdout':
                    if callback_name != self._stdout_callback or stdout_callback_loaded:
                        continue
                    stdout_callback_loaded = True
                elif callback_name == 'tree' and self._run_tree:
                    pass
                elif not self._run_additional_callbacks or (
                        callback_needs_whitelist and
                    (C.DEFAULT_CALLBACK_WHITELIST is None
                     or callback_name not in C.DEFAULT_CALLBACK_WHITELIST)):
                    continue

            self._callback_plugins.append(callback_plugin())

        self._callbacks_loaded = True

    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        if self._queue_thread is None:
            self._queue_thread = threading.Thread(
                target=self._queue_thread_main)
            self._queue_thread.start()

        all_vars = self._variable_manager.get_vars(loader=self._loader,
                                                   play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)
        new_play.handlers = new_play.compile_roles_handlers(
        ) + new_play.handlers

        self.hostvars = HostVars(
            inventory=self._inventory,
            variable_manager=self._variable_manager,
            loader=self._loader,
        )

        # Fork # of forks, # of hosts or serial, whichever is lowest
        num_hosts = len(self._inventory.get_hosts(new_play.hosts))

        max_serial = 0
        if new_play.serial:
            # the play has not been post_validated here, so we may need
            # to convert the scalar value to a list at this point
            serial_items = new_play.serial
            if not isinstance(serial_items, list):
                serial_items = [serial_items]
            max_serial = max([pct_to_int(x, num_hosts) for x in serial_items])

        contenders = [self._options.forks, max_serial, num_hosts]
        contenders = [v for v in contenders if v is not None and v > 0]
        self._initialize_processes(min(contenders))

        play_context = PlayContext(new_play, self._options, self.passwords,
                                   self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               new_play.strategy,
                               obj=play._ds)

        # build the iterator
        self._iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done=self._start_at_done,
        )

        # Because the TQM may survive multiple play runs, we start by marking
        # any hosts as failed in the iterator here which may have been marked
        # as failed in previous runs. Then we clear the internal list of failed
        # hosts so we know what failed this round.
        for host_name in self._failed_hosts.keys():
            host = self._inventory.get_host(host_name)
            self._iterator.mark_host_failed(host)

        self.clear_failed_hosts()

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if getattr(self._options, 'start_at_task',
                   None) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(self._iterator, play_context)

        # now re-save the hosts that failed from the iterator to our internal list
        for host_name in self._iterator.get_failed_hosts():
            self._failed_hosts[host_name] = True

        self._cleanup_processes()
        return play_return

    def cleanup(self):
        display.debug("RUNNING CLEANUP")
        self.terminate()
        self._final_q.close()
        self._cleanup_processes()

    def _cleanup_processes(self):
        for (worker_prc, rslt_q) in self._workers:
            rslt_q.close()
            if worker_prc and worker_prc.is_alive():
                try:
                    worker_prc.terminate()
                except AttributeError:
                    pass

    def clear_failed_hosts(self):
        self._failed_hosts = dict()

    def get_inventory(self):
        return self._inventory

    def get_variable_manager(self):
        return self._variable_manager

    def get_loader(self):
        return self._loader

    def get_workers(self):
        return self._workers[:]

    def terminate(self):
        self._terminated = True

    def has_dead_workers(self):

        # [<WorkerProcess(WorkerProcess-2, stopped[SIGKILL])>,
        # <WorkerProcess(WorkerProcess-2, stopped[SIGTERM])>

        defunct = False
        for idx, x in enumerate(self._workers):
            if hasattr(x[0], 'exitcode'):
                if x[0].exitcode in [-9, -15]:
                    defunct = True
        return defunct

    def send_callback(self, method_name, *args, **kwargs):
        for callback_plugin in [self._stdout_callback
                                ] + self._callback_plugins:
            # a plugin that set self.disabled to True will not be called
            # see osx_say.py example for such a plugin
            if getattr(callback_plugin, 'disabled', False):
                continue

            # try to find v2 method, fallback to v1 method, ignore callback if no method found
            methods = []
            for possible in [method_name, 'v2_on_any']:
                gotit = getattr(callback_plugin, possible, None)
                if gotit is None:
                    gotit = getattr(callback_plugin,
                                    possible.replace('v2_', ''), None)
                if gotit is not None:
                    methods.append(gotit)

            for method in methods:
                try:
                    # temporary hack, required due to a change in the callback API, so
                    # we don't break backwards compatibility with callbacks which were
                    # designed to use the original API
                    # FIXME: target for removal and revert to the original code here after a year (2017-01-14)
                    if method_name == 'v2_playbook_on_start':
                        import inspect
                        (f_args, f_varargs, f_keywords,
                         f_defaults) = inspect.getargspec(method)
                        if 'playbook' in f_args:
                            method(*args, **kwargs)
                        else:
                            method()
                    else:
                        method(*args, **kwargs)
                except Exception as e:
                    # TODO: add config toggle to make this fatal or not?
                    display.warning(
                        u"Failure using method (%s) in callback plugin (%s): %s"
                        % (to_text(method_name), to_text(callback_plugin),
                           to_text(e)))
                    from traceback import format_tb
                    from sys import exc_info
                    display.debug('Callback Exception: \n' +
                                  ' '.join(format_tb(exc_info()[2])))
Esempio n. 21
0
    def test_play_iterator(self):
        # import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml':
            """
            - name: role task
              debug: msg="this is a role task"
            - block:
              - name: role block task
                debug: msg="inside block in role"
              always:
              - name: role always task
                debug: msg="always task in block in role"
            - include: foo.yml
            - name: role task after include
              debug: msg="after include in role"
            - block:
              - name: starting role nested block 1
                debug:
              - block:
                - name: role nested block 1 task 1
                  debug:
                - name: role nested block 1 task 2
                  debug:
                - name: role nested block 1 task 3
                  debug:
              - name: end of role nested block 1
                debug:
              - name: starting role nested block 2
                debug:
              - block:
                - name: role nested block 2 task 1
                  debug:
                - name: role nested block 2 task 2
                  debug:
                - name: role nested block 2 task 3
                  debug:
              - name: end of role nested block 2
                debug:
            """,
            '/etc/ansible/roles/test_role/tasks/foo.yml':
            """
            - name: role included task
              debug: msg="this is task in an include from a role"
            """
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.name, "role task")
        self.assertIsNotNone(task._role)
        # role block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role block task")
        self.assertIsNotNone(task._role)
        # role block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role always task")
        self.assertIsNotNone(task._role)
        # role include task
        # (host_state, task) = itr.get_next_task_for_host(hosts[0])
        # self.assertIsNotNone(task)
        # self.assertEqual(task.action, 'debug')
        # self.assertEqual(task.name, "role included task")
        # self.assertIsNotNone(task._role)
        # role task after include
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role task after include")
        self.assertIsNotNone(task._role)
        # role nested block tasks
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 2")
        self.assertIsNotNone(task._role)
        # implicit meta: role_complete
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args,
                         dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args,
                         dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
    def test_play_iterator_nested_blocks(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              tasks:
              - block:
                - block:
                  - block:
                    - block:
                      - block:
                        - debug: msg="this is the first task"
                        - ping:
                      rescue:
                      - block:
                        - block:
                          - block:
                            - block:
                              - debug: msg="this is the rescue task"
                  always:
                  - block:
                    - block:
                      - block:
                        - block:
                          - debug: msg="this is the always task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # get the first task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the first task'))
        # fail the host
        itr.mark_host_failed(hosts[0])
        # get the resuce task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the rescue task'))
        # get the always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the always task'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # lookup up an original task
        target_task = p._entries[0].tasks[0].block[0]
        task_copy = target_task.copy(exclude_block=True)
        found_task = itr.get_original_task(hosts[0], task_copy)
        self.assertEqual(target_task, found_task)

        bad_task = Task()
        found_task = itr.get_original_task(hosts[0], bad_task)
        self.assertIsNone(found_task)

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
    def test_play_iterator(self):
        #import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # lookup up an original task
        target_task = p._entries[0].tasks[0].block[0]
        task_copy = target_task.copy(exclude_block=True)
        found_task = itr.get_original_task(hosts[0], task_copy)
        self.assertEqual(target_task, found_task)

        bad_task = Task()
        found_task = itr.get_original_task(hosts[0], bad_task)
        self.assertIsNone(found_task)

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
Esempio n. 25
0
    def run(self, play):
        '''
        Iterates over the roles/tasks in a play, using the given (or default)
        strategy for queueing tasks. The default is the linear strategy, which
        operates like classic Ansible by keeping all hosts in lock-step with
        a given task (meaning no hosts move on to the next task until all hosts
        are done with the current task).
        '''

        if not self._callbacks_loaded:
            self.load_callbacks()

        all_vars = self._variable_manager.get_vars(loader=self._loader,
                                                   play=play)
        templar = Templar(loader=self._loader, variables=all_vars)

        new_play = play.copy()
        new_play.post_validate(templar)

        class HostVarsManager(SyncManager):
            pass

        hostvars = HostVars(
            play=new_play,
            inventory=self._inventory,
            variable_manager=self._variable_manager,
            loader=self._loader,
        )

        HostVarsManager.register(
            'hostvars',
            callable=lambda: hostvars,
            # FIXME: this is the list of exposed methods to the DictProxy object, plus our
            #        special ones (set_variable_manager/set_inventory). There's probably a better way
            #        to do this with a proper BaseProxy/DictProxy derivative
            exposed=('set_variable_manager', 'set_inventory', '__contains__',
                     '__delitem__', '__getitem__', '__len__', '__setitem__',
                     'clear', 'copy', 'get', 'has_key', 'items', 'keys', 'pop',
                     'popitem', 'setdefault', 'update', 'values'),
        )
        self._hostvars_manager = HostVarsManager()
        self._hostvars_manager.start()

        # Fork # of forks, # of hosts or serial, whichever is lowest
        contenders = [
            self._options.forks, play.serial,
            len(self._inventory.get_hosts(new_play.hosts))
        ]
        contenders = [v for v in contenders if v is not None and v > 0]
        self._initialize_processes(min(contenders))

        play_context = PlayContext(new_play, self._options, self.passwords,
                                   self._connection_lockfile.fileno())
        for callback_plugin in self._callback_plugins:
            if hasattr(callback_plugin, 'set_play_context'):
                callback_plugin.set_play_context(play_context)

        self.send_callback('v2_playbook_on_play_start', new_play)

        # initialize the shared dictionary containing the notified handlers
        self._initialize_notified_handlers(new_play.handlers)

        # load the specified strategy (or the default linear one)
        strategy = strategy_loader.get(new_play.strategy, self)
        if strategy is None:
            raise AnsibleError("Invalid play strategy specified: %s" %
                               new_play.strategy,
                               obj=play._ds)

        # build the iterator
        iterator = PlayIterator(
            inventory=self._inventory,
            play=new_play,
            play_context=play_context,
            variable_manager=self._variable_manager,
            all_vars=all_vars,
            start_at_done=self._start_at_done,
        )

        # during initialization, the PlayContext will clear the start_at_task
        # field to signal that a matching task was found, so check that here
        # and remember it so we don't try to skip tasks on future plays
        if getattr(self._options, 'start_at_task',
                   None) is not None and play_context.start_at_task is None:
            self._start_at_done = True

        # and run the play using the strategy and cleanup on way out
        play_return = strategy.run(iterator, play_context)
        self._cleanup_processes()
        self._hostvars_manager.shutdown()
        return play_return