예제 #1
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook

        # first, we use the original parent method to correctly load the object
        # via the munge/load_data system we normally use for other playbook objects
        new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = new_obj.include
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)

        # finally, playbook includes can specify a list of variables, which are simply
        # used to update the vars of each play in the playbook
        for entry in pb._entries:
            entry.vars.update(new_obj.vars)

        return pb
예제 #2
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = new_obj.include
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:
            entry.vars.update(new_obj.vars)
            entry.tags = list(set(entry.tags).union(new_obj.tags))

        return pb
예제 #3
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)

        all_vars = self.vars.copy()
        if variable_manager:
            all_vars.update(variable_manager.get_vars(loader=loader))

        templar = Templar(loader=loader, variables=all_vars)

        try:
            forward_conditional = False
            if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
                return None
        except AnsibleError:
            # conditional evaluation raised an error, so we set a flag to indicate
            # we need to forward the conditionals on to the included play(s)
            forward_conditional = True

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = templar.template(new_obj.include)
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:
            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(file_name)

            # Check to see if we need to forward the conditionals on to the included
            # plays. If so, we can take a shortcut here and simply prepend them to
            # those attached to each block (if any)
            if forward_conditional:
                for task_block in entry.pre_tasks + entry.roles + entry.tasks + entry.post_tasks:
                    task_block.when = self.when[:] + task_block.when

        return pb
예제 #4
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook
        from ansible.playbook.play import Play

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)

        all_vars = self.vars.copy()
        if variable_manager:
            all_vars.update(variable_manager.get_vars())

        templar = Templar(loader=loader, variables=all_vars)

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = templar.template(new_obj.import_playbook)
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:

            # conditional includes on a playbook need a marker to skip gathering
            if new_obj.when and isinstance(entry, Play):
                entry._included_conditional = new_obj.when[:]

            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(file_name)

            # Check to see if we need to forward the conditionals on to the included
            # plays. If so, we can take a shortcut here and simply prepend them to
            # those attached to each block (if any)
            if new_obj.when:
                for task_block in (entry.pre_tasks + entry.roles + entry.tasks + entry.post_tasks):
                    task_block._attributes['when'] = new_obj.when[:] + task_block.when[:]

        return pb
    def test_playbook_executor__get_serialized_batches(self):
        fake_loader = DictDataLoader({
            'no_serial.yml': '''
            - hosts: all
              gather_facts: no
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_int.yml': '''
            - hosts: all
              gather_facts: no
              serial: 2
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_pct.yml': '''
            - hosts: all
              gather_facts: no
              serial: 20%
              tasks:
              - debug: var=inventory_hostname
            ''',
        })

        mock_inventory = MagicMock()
        mock_var_manager = MagicMock()

        # fake out options to use the syntax CLI switch, which will ensure
        # the PlaybookExecutor doesn't create a TaskQueueManager
        mock_options = MagicMock()
        mock_options.syntax.value = True

        pbe = PlaybookExecutor(
            playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml'],
            inventory=mock_inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=[],
        )

        playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']])

        playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])

        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
예제 #6
0
def create_playbook(playbook_path):
    """
    Load the given playbook and return the ansible playbook.

    :param playbook_path: Path to the playbook file.
    :type playbook_path: :class:`str`
    :returns: The loaded ansible playbook.
    :rtype: :class:`ansible.playbook.Playbook`
    """
    try:
        playbook = Playbook.load(
            playbook_path,
            loader=LOADER,
            variable_manager=VARIABLE_MANAGER,
        )

        scrape_playbook(playbook)
        return playbook
    except Exception as error:  # pylint: disable=broad-except
        logging.exception(
            "Unexpected error scrapping playbook %r: %r",
            playbook_path, error
        )

        raise argparse.ArgumentTypeError(error)
예제 #7
0
 def test_basic_playbook(self):
     fake_loader = DictDataLoader({
         "test_file.yml":"""
         - hosts: all
         """,
     })
     p = Playbook.load("test_file.yml", loader=fake_loader)
     plays = p.get_plays()
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)

        all_vars = dict()
        if variable_manager:
            all_vars = variable_manager.get_vars(loader=loader)

        templar = Templar(loader=loader, variables=all_vars)
        if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
            return None

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = new_obj.include
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:
            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(file_name)

        return pb
예제 #9
0
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        p = Playbook.load('test_play.yml', loader=fake_loader)

        hosts = []
        for i in range(0, 10):
            host  = MagicMock()
            host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        connection_info = ConnectionInformation(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            connection_info=connection_info,
            all_vars=dict(),
        )

        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNone(task)
예제 #10
0
    def run(self):

        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        signal.signal(signal.SIGINT, self._cleanup)

        result = 0
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
        
                # FIXME: playbook entries are just plays, so we should rename them
                for play in pb.get_entries():
                    self._inventory.remove_restriction()

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                    new_play = play.copy()
                    new_play.post_validate(all_vars, fail_on_undefined=False)

                    for batch in self._get_serialized_batches(new_play):
                        if len(batch) == 0:
                            self._tqm._callback.playbook_on_play_start(new_play.name)
                            self._tqm._callback.playbook_on_no_hosts_matched()
                            result = 0
                            break
                        # restrict the inventory to the hosts in the serialized batch
                        self._inventory.restrict_to_hosts(batch)
                        # and run it...
                        result = self._tqm.run(play=play)
                        if result != 0:
                            break

                    if result != 0:
                        # FIXME: do something here, to signify the playbook execution failed
                        self._cleanup()
                        return result
        except:
            self._cleanup()
            raise

        self._cleanup()
        return result
    def test_playbook_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        p = Playbook.load('test_play.yml', loader=fake_loader)

        hosts = []
        for i in range(0, 10):
            host  = MagicMock()
            host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts

        itr = PlaybookIterator(inventory, None, p)
        task = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        task = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        task = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        task = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNotNone(task)
        task = itr.get_next_task_for_host(hosts[0])
        print(task)
        self.assertIsNone(task)
def build_plays(loader, variable_manager, playbook_path, plays=[], hosts=[]):
    playbook = Playbook.load(playbook_path, variable_manager, loader)
    plays = []
    
    for play in playbook.get_plays():
        if play.get_name() in plays:
            plays.append(play)
            continue
        
        if play._ds['hosts'] in hosts:
            plays.append(play)
            continue

        for piece in play._ds['hosts']:
            if piece in hosts:
                plays.append(play)
                break

    return plays
예제 #13
0
    def run(self, playbook):
        pb = Playbook.load(playbook, variable_manager=self.variable_manager, loader=self.loader)
        # only support one playbook.yml
        play = pb.get_plays()[0]

        tqm = None
        try:
            tqm = TaskQueueManager(
                inventory=self.inventory,
                variable_manager=self.variable_manager,
                loader=self.loader,
                options=self.options,
                passwords={'conn_pass': None, 'become_pass': None},
                stdout_callback=self.callback,
            )
            return tqm.run(play), self.callback.result
        finally:
            if tqm is not None:
                tqm.cleanup()
예제 #14
0
파일: duploades.py 프로젝트: djaodjin/drop
def _execute_playbook(playbook_name):
    """
    Execute a playbook stored in the *share_dir*.
    """
    install_dir = os.path.dirname(os.path.dirname(sys.executable))
    share_dir = os.path.join(install_dir, 'share', 'dws')
    playbook_path = os.path.join(share_dir, 'playbooks', playbook_name)
    if not os.path.exists(playbook_path):
        # When running directly from within src_dir.
        share_dir = os.path.join(install_dir, 'share')
        playbook_path = os.path.join(share_dir, 'playbooks', playbook_name)
    sysconf_dir = os.path.join(install_dir, 'etc')
    Options = namedtuple('Options', ['connection', 'module_path', 'forks',
        'become', 'become_method', 'become_user', 'check'])
    options = Options(connection='local',
        module_path=os.path.dirname(tero.__file__), forks=100, become=None,
        become_method=None, become_user=None, check=False)
    passwords = dict(vault_pass='******')
    loader = DataLoader()
    variable_manager = VariableManager()
    inventory = Inventory(loader=loader, variable_manager=variable_manager,
        host_list=os.path.join(sysconf_dir, 'ansible', 'hosts'))
    variable_manager.set_inventory(inventory)
    playbook = Playbook.load(playbook_path,
        variable_manager=variable_manager, loader=loader)
    tqm = None
    try:
        tqm = TaskQueueManager(
            inventory=inventory,
            variable_manager=variable_manager,
            loader=loader,
            options=options,
            passwords=passwords)
        for play in playbook.get_plays():
            result = tqm.run(play)
    finally:
        if tqm is not None:
            tqm.cleanup()
예제 #15
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        result = 0
        entrylist = []
        entry = {}
        try:
            # preload become/connection/shell to set config defs cached
            list(connection_loader.all(class_only=True))
            list(shell_loader.all(class_only=True))
            list(become_loader.all(class_only=True))

            for playbook in self._playbooks:

                # deal with FQCN
                resource = _get_collection_playbook_path(playbook)
                if resource is not None:
                    playbook_path = resource[1]
                    playbook_collection = resource[2]
                else:
                    playbook_path = playbook
                    # not fqcn, but might still be colleciotn playbook
                    playbook_collection = _get_collection_name_from_path(
                        playbook)

                if playbook_collection:
                    display.warning(
                        "running playbook inside collection {0}".format(
                            playbook_collection))
                    AnsibleCollectionConfig.default_collection = playbook_collection
                else:
                    AnsibleCollectionConfig.default_collection = None

                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)
                # FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))

                if self._tqm is None:  # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []
                else:
                    # make sure the tqm has callbacks loaded
                    self._tqm.load_callbacks()
                    self._tqm.send_callback('v2_playbook_on_start', pb)

                i = 1
                plays = pb.get_plays()
                display.vv(u'%d plays in %s' %
                           (len(plays), to_text(playbook_path)))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    # Allow variables to be used in vars_prompt fields.
                    all_vars = self._variable_manager.get_vars(play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    setattr(play, 'vars_prompt',
                            templar.template(play.vars_prompt))

                    # FIXME: this should be a play 'sub object' like loop_control
                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            vname = var['name']
                            prompt = var.get("prompt", vname)
                            default = var.get("default", None)
                            private = boolean(var.get("private", True))
                            confirm = boolean(var.get("confirm", False))
                            encrypt = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt = var.get("salt", None)
                            unsafe = var.get("unsafe", None)

                            if vname not in self._variable_manager.extra_vars:
                                if self._tqm:
                                    self._tqm.send_callback(
                                        'v2_playbook_on_vars_prompt', vname,
                                        private, prompt, encrypt, confirm,
                                        salt_size, salt, default, unsafe)
                                    play.vars[vname] = display.do_var_prompt(
                                        vname, private, prompt, encrypt,
                                        confirm, salt_size, salt, default,
                                        unsafe)
                                else:  # we are either in --list-<option> or syntax check
                                    play.vars[vname] = default

                    # Post validate so any play level variables are templated
                    all_vars = self._variable_manager.get_vars(play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    play.post_validate(templar)

                    if context.CLIARGS['syntax']:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(play)

                    else:
                        self._tqm._unreachable_hosts.update(
                            self._unreachable_hosts)

                        previously_failed = len(self._tqm._failed_hosts)
                        previously_unreachable = len(
                            self._tqm._unreachable_hosts)

                        break_play = False
                        # we are actually running plays
                        batches = self._get_serialized_batches(play)
                        if len(batches) == 0:
                            self._tqm.send_callback(
                                'v2_playbook_on_play_start', play)
                            self._tqm.send_callback(
                                'v2_playbook_on_no_hosts_matched')
                        for batch in batches:
                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            try:
                                result = self._tqm.run(play=play)
                            except AnsibleEndPlay as e:
                                result = e.result
                                break_play = True
                                break

                            # break the play if the result equals the special return code
                            if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
                                result = self._tqm.RUN_FAILED_HOSTS
                                break_play = True

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
                                (previously_failed + previously_unreachable)

                            if len(batch) == failed_hosts_count:
                                break_play = True
                                break

                            # update the previous counts so they don't accumulate incorrectly
                            # over multiple serial batches
                            previously_failed += len(
                                self._tqm._failed_hosts) - previously_failed
                            previously_unreachable += len(
                                self._tqm._unreachable_hosts
                            ) - previously_unreachable

                            # save the unreachable hosts from this batch
                            self._unreachable_hosts.update(
                                self._tqm._unreachable_hosts)

                        if break_play:
                            break

                    i = i + 1  # per play

                if entry:
                    entrylist.append(entry)  # per playbook

                # send the stats callback for this playbook
                if self._tqm is not None:
                    if C.RETRY_FILES_ENABLED:
                        retries = set(self._tqm._failed_hosts.keys())
                        retries.update(self._tqm._unreachable_hosts.keys())
                        retries = sorted(retries)
                        if len(retries) > 0:
                            if C.RETRY_FILES_SAVE_PATH:
                                basedir = C.RETRY_FILES_SAVE_PATH
                            elif playbook_path:
                                basedir = os.path.dirname(
                                    os.path.abspath(playbook_path))
                            else:
                                basedir = '~/'

                            (retry_name, _) = os.path.splitext(
                                os.path.basename(playbook_path))
                            filename = os.path.join(basedir,
                                                    "%s.retry" % retry_name)
                            if self._generate_retry_inventory(
                                    filename, retries):
                                display.display(
                                    "\tto retry, use: --limit @%s\n" %
                                    filename)

                    self._tqm.send_callback('v2_playbook_on_stats',
                                            self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._tqm.cleanup()
            if self._loader:
                self._loader.cleanup_all_tmp_files()

        if context.CLIARGS['syntax']:
            display.display("No issues encountered")
            return result

        if context.CLIARGS['start_at_task'] and not self._tqm._start_at_done:
            display.error(
                "No matching task \"%s\" found."
                " Note: --start-at-task can only follow static includes." %
                context.CLIARGS['start_at_task'])

        return result
예제 #16
0
    def test_play_iterator(self):
        # import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml':
            """
            - name: role task
              debug: msg="this is a role task"
            - block:
              - name: role block task
                debug: msg="inside block in role"
              always:
              - name: role always task
                debug: msg="always task in block in role"
            - include: foo.yml
            - name: role task after include
              debug: msg="after include in role"
            - block:
              - name: starting role nested block 1
                debug:
              - block:
                - name: role nested block 1 task 1
                  debug:
                - name: role nested block 1 task 2
                  debug:
                - name: role nested block 1 task 3
                  debug:
              - name: end of role nested block 1
                debug:
              - name: starting role nested block 2
                debug:
              - block:
                - name: role nested block 2 task 1
                  debug:
                - name: role nested block 2 task 2
                  debug:
                - name: role nested block 2 task 3
                  debug:
              - name: end of role nested block 2
                debug:
            """,
            '/etc/ansible/roles/test_role/tasks/foo.yml':
            """
            - name: role included task
              debug: msg="this is task in an include from a role"
            """
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.name, "role task")
        self.assertIsNotNone(task._role)
        # role block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role block task")
        self.assertIsNotNone(task._role)
        # role block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role always task")
        self.assertIsNotNone(task._role)
        # role include task
        # (host_state, task) = itr.get_next_task_for_host(hosts[0])
        # self.assertIsNotNone(task)
        # self.assertEqual(task.action, 'debug')
        # self.assertEqual(task.name, "role included task")
        # self.assertIsNotNone(task._role)
        # role task after include
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role task after include")
        self.assertIsNotNone(task._role)
        # role nested block tasks
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 2")
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args,
                         dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args,
                         dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
예제 #17
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        signal.signal(signal.SIGINT, self._cleanup)

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)

                if self._tqm is None:  # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []

                i = 1
                plays = pb.get_plays()
                self._display.vv('%d plays in %s' %
                                 (len(plays), playbook_path))

                for play in plays:
                    self._inventory.remove_restriction()

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(
                        loader=self._loader, play=play)
                    new_play = play.copy()
                    new_play.post_validate(all_vars, fail_on_undefined=False)

                    if self._tqm is None:
                        # we are just doing a listing

                        pname = new_play.get_name().strip()
                        if pname == 'PLAY: <no name specified>':
                            pname = 'PLAY: #%d' % i
                        p = {'name': pname}

                        if self._options.listhosts:
                            p['pattern'] = play.hosts
                            p['hosts'] = set(
                                self._inventory.get_hosts(new_play.hosts))

                        #TODO: play tasks are really blocks, need to figure out how to get task objects from them
                        elif self._options.listtasks:
                            p['tasks'] = []
                            for task in play.get_tasks():
                                p['tasks'].append(task)
                                #p['tasks'].append({'name': task.get_name().strip(), 'tags': task.tags})

                        elif self._options.listtags:
                            p['tags'] = set(new_play.tags)
                            for task in play.get_tasks():
                                p['tags'].update(task)
                                #p['tags'].update(task.tags)
                        entry['plays'].append(p)

                    else:
                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback(
                                    'v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback(
                                    'v2_playbook_on_no_hosts_matched')
                                result = 0
                                break
                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)
                            if result != 0:
                                break

                        if result != 0:
                            break

                    i = i + 1  # per play

                if entry:
                    entrylist.append(entry)  # per playbook

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._cleanup()

        # FIXME: this stat summary stuff should be cleaned up and moved
        #        to a new method, if it even belongs here...
        self._display.banner("PLAY RECAP")

        hosts = sorted(self._tqm._stats.processed.keys())
        for h in hosts:
            t = self._tqm._stats.summarize(h)

            self._display.display(
                "%s : %s %s %s %s" %
                (hostcolor(h, t), colorize('ok', t['ok'], 'green'),
                 colorize('changed', t['changed'], 'yellow'),
                 colorize('unreachable', t['unreachable'],
                          'red'), colorize('failed', t['failures'], 'red')),
                screen_only=True)

            self._display.display(
                "%s : %s %s %s %s" %
                (hostcolor(h, t, False), colorize('ok', t['ok'], None),
                 colorize('changed', t['changed'], None),
                 colorize('unreachable', t['unreachable'],
                          None), colorize('failed', t['failures'], None)),
                log_only=True)

        self._display.display("", screen_only=True)
        # END STATS STUFF

        return result
예제 #18
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook
        from ansible.playbook.play import Play

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude,
                        self).load_data(ds, variable_manager, loader)

        all_vars = self.vars.copy()
        if variable_manager:
            all_vars.update(variable_manager.get_vars())

        templar = Templar(loader=loader, variables=all_vars)

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = templar.template(new_obj.import_playbook)

        # check for FQCN
        resource = _get_collection_playbook_path(file_name)
        if resource is not None:
            playbook = resource[1]
            playbook_collection = resource[2]
        else:
            # not FQCN try path
            playbook = file_name
            if not os.path.isabs(playbook):
                playbook = os.path.join(basedir, playbook)

            # might still be collection playbook
            playbook_collection = _get_collection_name_from_path(playbook)

        if playbook_collection:
            # it is a collection playbook, setup default collections
            AnsibleCollectionConfig.default_collection = playbook_collection
        else:
            # it is NOT a collection playbook, setup adjecent paths
            AnsibleCollectionConfig.playbook_paths.append(
                os.path.dirname(
                    os.path.abspath(
                        to_bytes(playbook, errors='surrogate_or_strict'))))

        pb._load_playbook_data(file_name=playbook,
                               variable_manager=variable_manager,
                               vars=self.vars.copy())

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:

            # conditional includes on a playbook need a marker to skip gathering
            if new_obj.when and isinstance(entry, Play):
                entry._included_conditional = new_obj.when[:]

            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(playbook)

            # Check to see if we need to forward the conditionals on to the included
            # plays. If so, we can take a shortcut here and simply prepend them to
            # those attached to each block (if any)
            if new_obj.when:
                for task_block in (entry.pre_tasks + entry.roles +
                                   entry.tasks + entry.post_tasks):
                    task_block._when = new_obj.when[:] + task_block.when[:]

        return pb
예제 #19
0
    def test_play_iterator_nested_blocks(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: false
              tasks:
              - block:
                - block:
                  - block:
                    - block:
                      - block:
                        - debug: msg="this is the first task"
                        - ping:
                      rescue:
                      - block:
                        - block:
                          - block:
                            - block:
                              - debug: msg="this is the rescue task"
                  always:
                  - block:
                    - block:
                      - block:
                        - block:
                          - debug: msg="this is the always task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # get the first task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the first task'))
        # fail the host
        itr.mark_host_failed(hosts[0])
        # get the resuce task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the rescue task'))
        # get the always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the always task'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)
예제 #20
0
 def test_empty_playbook(self):
     fake_loader = DictDataLoader({})
     p = Playbook(loader=fake_loader)
예제 #21
0
    def test_get_serialized_batches(self):
        fake_loader = DictDataLoader({
            'no_serial.yml':
            '''
            - hosts: all
              gather_facts: no
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_int.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: 2
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_pct.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: 20%
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [1, 2, 3]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list_mixed.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [1, "20%", -1]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_no_unreachable.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [[2, 1]]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_pct_no_unreachable.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [[20%, 1]]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list_no_unreachable.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [[1, 1], 2, [3, 0]]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list_no_unreachable_mixed.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [1, ["20%", 1], [-1, 1]]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list_no_unreachable_inv.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: [[1, 1], [2, "not_valid"], [2, 1], [2, 10], [2, 0], 1]
              tasks:
              - debug: var=inventory_hostname
            ''',
        })

        mock_inventory = MagicMock()
        mock_var_manager = MagicMock()

        templar = Templar(loader=fake_loader)

        pbe = PlaybookExecutor(
            playbooks=[
                'no_serial.yml', 'serial_int.yml', 'serial_pct.yml',
                'serial_list.yml', 'serial_list_mixed.yml',
                'serial_no_unreachable.yml', 'serial_pct_no_unreachable.yml',
                'serial_list_no_unreachable.yml',
                'serial_list_no_unreachable_mixed.yml',
                'serial_list_no_unreachable_inv.yml'
            ],
            inventory=mock_inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            passwords=[],
        )

        playbook = Playbook.load(pbe._playbooks[0],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(pbe._get_serialized_batches(play), ([[
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]], [0]))

        playbook = Playbook.load(pbe._playbooks[1],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
              ['host6', 'host7'], ['host8', 'host9']], [0, 0, 0, 0, 0]))

        playbook = Playbook.load(pbe._playbooks[2],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
              ['host6', 'host7'], ['host8', 'host9']], [0, 0, 0, 0, 0]))

        playbook = Playbook.load(pbe._playbooks[3],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'],
              ['host6', 'host7', 'host8'], ['host9']], [0, 0, 0, 0, 0]))

        playbook = Playbook.load(pbe._playbooks[4],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0'], ['host1', 'host2'],
              ['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
              ], [0, 0, 0]))

        # Test when serial percent is under 1.0
        playbook = Playbook.load(pbe._playbooks[2],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
        self.assertEqual(pbe._get_serialized_batches(play),
                         ([['host0'], ['host1'], ['host2']], [0, 0, 0]))

        # Test when there is a remainder for serial as a percent
        playbook = Playbook.load(pbe._playbooks[2],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9', 'host10'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
              ['host6', 'host7'], ['host8', 'host9'], ['host10']
              ], [0, 0, 0, 0, 0, 0]))
        playbook = Playbook.load(pbe._playbooks[5],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
              ['host6', 'host7'], ['host8', 'host9']], [1, 1, 1, 1, 1]))

        playbook = Playbook.load(pbe._playbooks[6],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
              ['host6', 'host7'], ['host8', 'host9']], [1, 1, 1, 1, 1]))

        playbook = Playbook.load(pbe._playbooks[7],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'],
              ['host6', 'host7', 'host8'], ['host9']], [1, 0, 0, 0, 0]))

        playbook = Playbook.load(pbe._playbooks[8],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            ([['host0'], ['host1', 'host2'],
              ['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
              ], [0, 1, 0]))

        playbook = Playbook.load(pbe._playbooks[9],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(pbe._get_serialized_batches(play),
                         ([['host0'], ['host1', 'host2'], ['host3', 'host4'],
                           ['host5', 'host6'], ['host7', 'host8'], ['host9']
                           ], [1, 0, 1, 0, 0, 0]))
    def run(self):

        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        signal.signal(signal.SIGINT, self._cleanup)

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
                self._inventory.set_playbook_basedir(os.path.dirname(playbook_path))

                if self._tqm is None: # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []

                i = 1
                plays = pb.get_plays()
                self._display.vv('%d plays in %s' % (len(plays), playbook_path))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            vname     = var['name']
                            prompt    = var.get("prompt", vname)
                            default   = var.get("default", None)
                            private   = var.get("private", True)
                            confirm   = var.get("confirm", False)
                            encrypt   = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt      = var.get("salt", None)

                            if vname not in play.vars:
                                if self._tqm:
                                    self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default)
                                play.vars[vname] = self._do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        # make sure the tqm has callbacks loaded
                        self._tqm.load_callbacks()
                        self._tqm._unreachable_hosts.update(self._unreachable_hosts)

                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback('v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                                break

                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts)
                            if new_play.any_errors_fatal and failed_hosts_count > 0:
                                break
                            elif new_play.max_fail_percentage is not None and \
                               int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
                                break
                            elif len(batch) == failed_hosts_count:
                                break

                            # clear the failed hosts dictionaires in the TQM for the next batch
                            self._unreachable_hosts.update(self._tqm._unreachable_hosts)
                            self._tqm.clear_failed_hosts()

                        # if the last result wasn't zero or 3 (some hosts were unreachable),
                        # break out of the serial batch loop
                        if result not in (0, 3):
                            break

                    i = i + 1 # per play

                if entry:
                    entrylist.append(entry) # per playbook

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._cleanup()

        if self._options.syntax:
            self.display.display("No issues encountered")
            return result

        # FIXME: this stat summary stuff should be cleaned up and moved
        #        to a new method, if it even belongs here...
        self._display.banner("PLAY RECAP")

        hosts = sorted(self._tqm._stats.processed.keys())
        for h in hosts:
            t = self._tqm._stats.summarize(h)

            self._display.display(u"%s : %s %s %s %s" % (
                hostcolor(h, t),
                colorize(u'ok', t['ok'], 'green'),
                colorize(u'changed', t['changed'], 'yellow'),
                colorize(u'unreachable', t['unreachable'], 'red'),
                colorize(u'failed', t['failures'], 'red')),
                screen_only=True
            )

            self._display.display(u"%s : %s %s %s %s" % (
                hostcolor(h, t, False),
                colorize(u'ok', t['ok'], None),
                colorize(u'changed', t['changed'], None),
                colorize(u'unreachable', t['unreachable'], None),
                colorize(u'failed', t['failures'], None)),
                log_only=True
            )

        self._display.display("", screen_only=True)
        # END STATS STUFF

        return result
예제 #23
0
    def test_get_serialized_batches(self):
        fake_loader = DictDataLoader({
            'no_serial.yml': '''
            - hosts: all
              gather_facts: no
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_int.yml': '''
            - hosts: all
              gather_facts: no
              serial: 2
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_pct.yml': '''
            - hosts: all
              gather_facts: no
              serial: 20%
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list.yml': '''
            - hosts: all
              gather_facts: no
              serial: [1, 2, 3]
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_list_mixed.yml': '''
            - hosts: all
              gather_facts: no
              serial: [1, "20%", -1]
              tasks:
              - debug: var=inventory_hostname
            ''',
        })

        mock_inventory = MagicMock()
        mock_var_manager = MagicMock()

        # fake out options to use the syntax CLI switch, which will ensure
        # the PlaybookExecutor doesn't create a TaskQueueManager
        mock_options = MagicMock()
        mock_options.syntax.value = True

        templar = Templar(loader=fake_loader)

        pbe = PlaybookExecutor(
            playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml', 'serial_list.yml', 'serial_list_mixed.yml'],
            inventory=mock_inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=[],
        )

        playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']])

        playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'], ['host6', 'host7'], ['host8', 'host9']]
        )

        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'], ['host6', 'host7'], ['host8', 'host9']]
        )

        playbook = Playbook.load(pbe._playbooks[3], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'], ['host6', 'host7', 'host8'], ['host9']]
        )

        playbook = Playbook.load(pbe._playbooks[4], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']])

        # Test when serial percent is under 1.0
        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
        self.assertEqual(pbe._get_serialized_batches(play), [['host0'], ['host1'], ['host2']])

        # Test when there is a remainder for serial as a percent
        playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
        play = playbook.get_plays()[0]
        play.post_validate(templar)
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9', 'host10']
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'], ['host6', 'host7'], ['host8', 'host9'], ['host10']]
        )
예제 #24
0
 def test_empty_playbook(self):
     p = Playbook()
예제 #25
0
    def run(self):

        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        signal.signal(signal.SIGINT, self._cleanup)

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)

                if self._tqm is None: # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []

                i = 1
                plays = pb.get_plays()
                self._display.vv('%d plays in %s' % (len(plays), playbook_path))

                for play in plays:
                    self._inventory.remove_restriction()

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing

                        pname =  new_play.get_name().strip()
                        if pname == 'PLAY: <no name specified>':
                            pname = 'PLAY: #%d' % i
                        p = { 'name': pname }

                        if self._options.listhosts:
                            p['pattern']=play.hosts
                            p['hosts']=set(self._inventory.get_hosts(new_play.hosts))

                        #TODO: play tasks are really blocks, need to figure out how to get task objects from them
                        elif self._options.listtasks:
                            p['tasks'] = []
                            for task in play.get_tasks():
                               p['tasks'].append(task)
                               #p['tasks'].append({'name': task.get_name().strip(), 'tags': task.tags})

                        elif self._options.listtags:
                            p['tags'] = set(new_play.tags)
                            for task in play.get_tasks():
                                p['tags'].update(task)
                                #p['tags'].update(task.tags)
                        entry['plays'].append(p)

                    else:
                        # make sure the tqm has callbacks loaded
                        self._tqm.load_callbacks()

                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback('v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                                break
                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)
                            # if the last result wasn't zero, break out of the serial batch loop
                            if result != 0:
                                break

                        # if the last result wasn't zero, break out of the play loop
                        if result != 0:
                            break

                    i = i + 1 # per play

                if entry:
                    entrylist.append(entry) # per playbook

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._cleanup()

        if self._options.syntax:
            self.display.display("No issues encountered")
            return result

        # FIXME: this stat summary stuff should be cleaned up and moved
        #        to a new method, if it even belongs here...
        self._display.banner("PLAY RECAP")

        hosts = sorted(self._tqm._stats.processed.keys())
        for h in hosts:
            t = self._tqm._stats.summarize(h)

            self._display.display("%s : %s %s %s %s" % (
                hostcolor(h, t),
                colorize('ok', t['ok'], 'green'),
                colorize('changed', t['changed'], 'yellow'),
                colorize('unreachable', t['unreachable'], 'red'),
                colorize('failed', t['failures'], 'red')),
                screen_only=True
            )

            self._display.display("%s : %s %s %s %s" % (
                hostcolor(h, t, False),
                colorize('ok', t['ok'], None),
                colorize('changed', t['changed'], None),
                colorize('unreachable', t['unreachable'], None),
                colorize('failed', t['failures'], None)),
                log_only=True
            )

        self._display.display("", screen_only=True)
        # END STATS STUFF

        return result
예제 #26
0
def execute_playbook(play_book, host_list=[]):
    host_list = host_list
    # since the API is constructed for CLI it expects certain options to always be set in the context object
    context.CLIARGS = ImmutableDict(connection='smart',
                                    module_path=None,
                                    become=None,
                                    become_method=None,
                                    become_user=None,
                                    check=False,
                                    forks=4)

    sources = ','.join(host_list)
    if len(host_list) == 1:
        sources += ','

    # initialize needed objects
    loader = DataLoader(
    )  # Takes care of finding and reading yaml, json and ini files
    passwords = dict(vault_pass='******')

    # Instantiate our ResultsCollectorJSONCallback for handling results as they come in. Ansible expects this to be one of its main display outlets
    results_callback = ResultsCollectorJSONCallback()

    inventory = InventoryManager(loader=loader, sources=sources)

    variable_manager = VariableManager(loader=loader, inventory=inventory)

    tqm = TaskQueueManager(
        inventory=inventory,
        variable_manager=variable_manager,
        loader=loader,
        passwords=passwords,
        stdout_callback=
        results_callback,  # Use our custom callback instead of the ``default`` callback plugin, which prints to stdout
    )

    pbex = PlaybookExecutor(playbooks=[play_book],
                            inventory=inventory,
                            variable_manager=variable_manager,
                            loader=loader,
                            passwords=passwords)
    playbook = Playbook.load(pbex._playbooks[0],
                             variable_manager=variable_manager,
                             loader=loader)
    play = playbook.get_plays()[0]

    # Actually run it
    try:
        result = tqm.run(
            play
        )  # most interesting data for a play is actually sent to the callback's methods
    finally:
        # we always need to cleanup child procs and the structures we use to communicate with them
        tqm.cleanup()
        if loader:
            loader.cleanup_all_tmp_files()

    # Remove ansible tmpdir
    shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)

    results_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
    for host, result in results_callback.host_ok.items():
        results_raw['success'][host] = result._result

    for host, result in results_callback.host_failed.items():
        results_raw['failed'][host] = result._result

    for host, result in results_callback.host_unreachable.items():
        results_raw['unreachable'][host] = result._result
    return results_raw
예제 #27
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''
        result = 0
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)
                self._inventory.set_playbook_basedir(
                    os.path.dirname(playbook_path))

                i = 1

                # make sure the tqm has callbacks loaded
                self._tqm.load_callbacks()
                self._tqm.send_callback('v2_playbook_on_start', pb)

                for play in pb.get_plays():
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the
                    # inventory
                    self._inventory.remove_restriction()

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the
                    # original object.
                    all_vars = self._variable_manager.get_vars(
                        loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    self._tqm._unreachable_hosts.update(
                        self._unreachable_hosts)

                    # we are actually running plays
                    for batch in self._get_serialized_batches(new_play):
                        if len(batch) == 0:
                            self._tqm.send_callback(
                                'v2_playbook_on_play_start', new_play)
                            self._tqm.send_callback(
                                'v2_playbook_on_no_hosts_matched')
                            break
                        # restrict the inventory to the hosts in the serialized
                        # batch
                        self._inventory.restrict_to_hosts(batch)
                        # and run it...
                        result = self._tqm.run(play=play)

                        # check the number of failures here, to see if they're above the maximum
                        # failure percentage allowed, or if any errors are fatal. If either of those
                        # conditions are met, we break out, otherwise we only break out if the entire
                        # batch failed
                        failed_hosts_count = len(
                            self._tqm._failed_hosts) + len(
                                self._tqm._unreachable_hosts)
                        if new_play.any_errors_fatal and failed_hosts_count > 0:
                            break
                        elif new_play.max_fail_percentage is not None and \
                                (int((new_play.max_fail_percentage) / 100.0 * len(batch)) >
                                 int((len(batch) - failed_hosts_count) / len(batch) * 100.0)):
                            break
                        elif len(batch) == failed_hosts_count:
                            break

                        # clear the failed hosts dictionaires in the TQM for
                        # the next batch
                        self._unreachable_hosts.update(
                            self._tqm._unreachable_hosts)
                        self._tqm.clear_failed_hosts()

                    # if the last result wasn't zero or 3 (some hosts were unreachable),
                    # break out of the serial batch loop
                    if result not in (0, 3):
                        break

                    i = i + 1  # per play

                self._tqm.send_callback('v2_playbook_on_stats',
                                        self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook
                # file name loop
                if result != 0:
                    break

        finally:
            if self._tqm is not None:
                self._tqm.cleanup()

        return result
options = Options(listtags=False, listtasks=False, listhosts=False, syntax=False, connection='ssh', module_path=None, forks=100, remote_user='******', private_key_file=None, ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None, scp_extra_args=None, become=True, become_method=None, become_user='******', verbosity=None, check=False)

variable_manager.extra_vars = {'hosts': 'mywebserver'} # This can accomodate various other command line arguments.`

passwords = {}

pbex = PlaybookExecutor(playbooks=[playbook_path], inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords)
print pbex._playbooks

taskData = []
conditionalData = []

print "\nParsed tasks....."
# Extract data from compiled playbook
for playbook_path in pbex._playbooks:
  pb = Playbook.load(playbook_path, variable_manager=pbex._variable_manager, loader=pbex._loader)
  plays = pb.get_plays()
  for play in plays:
    #print play.get_name()
    tasklist = play.get_tasks() # play.get_tasks() returns a merged list of task blocks (pre, post, etc)
    for taskBlocks in tasklist:
        for task in taskBlocks:
          hashedTask = hash(task)
          changed = Task._get_parent_attribute(task, "changed_when")
          taskData.append(hashedTask)
          if changed is not None:
              conditionalData.append(hashedTask)
          print task, hashedTask, changed


# Build graph
예제 #29
0
def execute_playbook(play_book, host_list=[]):
    host_list = host_list
    # since the API is constructed for CLI it expects certain options to always be set in the context object
    context.CLIARGS = ImmutableDict(
        connection='paramiko',
        module_path=[
            '/home/koshik/Documents/projects/devops/scripts/aenv/lib/python3.6/site-packages/ansible'
        ],
        forks=10,
        become=None,
        become_method=None,
        become_user=None,
        check=False,
        diff=False)

    sources = ','.join(host_list)
    if len(host_list) == 1:
        sources += ','

    # initialize needed objects
    loader = DataLoader(
    )  # Takes care of finding and reading yaml, json and ini files
    passwords = dict(vault_pass='******')

    # Instantiate our ResultsCollectorJSONCallback for handling results as they come in. Ansible expects this to be one of its main display outlets
    results_callback = ResultsCollectorJSONCallback()

    inventory = InventoryManager(loader=loader, sources=sources)

    variable_manager = VariableManager(loader=loader, inventory=inventory)

    tqm = TaskQueueManager(
        inventory=inventory,
        variable_manager=variable_manager,
        loader=loader,
        passwords=passwords,
        stdout_callback=
        results_callback,  # Use our custom callback instead of the ``default`` callback plugin, which prints to stdout
    )

    pbex = PlaybookExecutor(playbooks=[play_book],
                            inventory=inventory,
                            variable_manager=variable_manager,
                            loader=loader,
                            passwords=passwords)
    playbook = Playbook.load(pbex._playbooks[0],
                             variable_manager=variable_manager,
                             loader=loader)
    play = playbook.get_plays()[0]

    # Actually run it
    try:
        result = tqm.run(
            play
        )  # most interesting data for a play is actually sent to the callback's methods
    finally:
        # we always need to cleanup child procs and the structures we use to communicate with them
        tqm.cleanup()
        if loader:
            loader.cleanup_all_tmp_files()

    # Remove ansible tmpdir
    shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)

    if (results_callback.host_ok.items()):
        print("********* SUCCESS ***********")
        return results_callback.host_ok.items()

    if (results_callback.host_failed.items()):
        print("********* FAILED *******")
        for host, result in results_callback.host_failed.items():
            print('{0} >>> {1}'.format(host, result._result['msg']))

    if (results_callback.host_unreachable.items()):
        print("********* HOST DOWN *********")
        for host, result in results_callback.host_unreachable.items():
            print('{0} >>> {1}'.format(host, result._result['msg']))
예제 #30
0
    def parse(self, *args, **kwargs) -> PlaybookNode:
        """
        Loop through the playbook and generate the graph.

        The graph is drawn following this order (https://docs.ansible.com/ansible/2.4/playbooks_reuse_roles.html#using-roles)
        for each play:
            add pre_tasks
            add roles
                if  include_role_tasks
                    add role_tasks
            add tasks
            add post_tasks
        :return:
        """

        playbook = Playbook.load(
            self.playbook_filename,
            loader=self.data_loader,
            variable_manager=self.variable_manager,
        )
        # the root node
        playbook_root_node = PlaybookNode(self.playbook_filename,
                                          raw_object=playbook)
        # loop through the plays
        for play in playbook.get_plays():

            # the load basedir is relative to the playbook path
            if play._included_path is not None:
                self.data_loader.set_basedir(play._included_path)
            else:
                self.data_loader.set_basedir(playbook._basedir)
            display.vvv(
                f"Loader basedir set to {self.data_loader.get_basedir()}")

            play_vars = self.variable_manager.get_vars(play)
            play_hosts = [
                h.get_name() for h in self.inventory_manager.get_hosts(
                    self.template(play.hosts, play_vars))
            ]
            play_name = f"Play: {clean_name(play.get_name())} ({len(play_hosts)})"
            play_name = self.template(play_name, play_vars)

            display.v(f"Parsing {play_name}")

            play_node = PlayNode(play_name, hosts=play_hosts, raw_object=play)
            playbook_root_node.add_node("plays", play_node)

            # loop through the pre_tasks
            display.v("Parsing pre_tasks...")
            for pre_task_block in play.pre_tasks:
                self._include_tasks_in_blocks(
                    current_play=play,
                    parent_nodes=[play_node],
                    block=pre_task_block,
                    play_vars=play_vars,
                    node_type="pre_task",
                )

            # loop through the roles
            display.v("Parsing roles...")

            for role in play.get_roles():
                # Don't insert tasks from ``import/include_role``, preventing duplicate graphing
                if role.from_include:
                    continue

                # the role object doesn't inherit the tags from the play. So we add it manually.
                role.tags = role.tags + play.tags
                if not role.evaluate_tags(only_tags=self.tags,
                                          skip_tags=self.skip_tags,
                                          all_vars=play_vars):
                    display.vv(
                        f"The role '{role.get_name()}' is skipped due to the tags."
                    )
                    # Go to the next role
                    continue

                if self.group_roles_by_name:
                    # If we are grouping roles, we use the hash of role name as the node id
                    role_node_id = "role_" + hash_value(role.get_name())
                else:
                    # Otherwise, a random id is used
                    role_node_id = generate_id("role_")
                role_node = RoleNode(
                    clean_name(role.get_name()),
                    node_id=role_node_id,
                    raw_object=role,
                    parent=play_node,
                )
                # edge from play to role
                play_node.add_node("roles", role_node)

                if self.include_role_tasks:
                    # loop through the tasks of the roles
                    for block in role.compile(play):
                        self._include_tasks_in_blocks(
                            current_play=play,
                            parent_nodes=[role_node],
                            block=block,
                            play_vars=play_vars,
                            node_type="task",
                        )
                    # end of roles loop

            # loop through the tasks
            display.v("Parsing tasks...")
            for task_block in play.tasks:
                self._include_tasks_in_blocks(
                    current_play=play,
                    parent_nodes=[play_node],
                    block=task_block,
                    play_vars=play_vars,
                    node_type="task",
                )

            # loop through the post_tasks
            display.v("Parsing post_tasks...")
            for post_task_block in play.post_tasks:
                self._include_tasks_in_blocks(
                    current_play=play,
                    parent_nodes=[play_node],
                    block=post_task_block,
                    play_vars=play_vars,
                    node_type="post_task",
                )
            # Summary
            display.v(
                f"{len(play_node.pre_tasks)} pre_task(s) added to the graph.")
            display.v(f"{len(play_node.roles)} role(s) added to the play")
            display.v(f"{len(play_node.tasks)} task(s) added to the play")
            display.v(
                f"{len(play_node.post_tasks)} post_task(s) added to the play")
            # moving to the next play

        return playbook_root_node
예제 #31
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)
                # FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))

                if self._tqm is None:  # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []
                else:
                    # make sure the tqm has callbacks loaded
                    self._tqm.load_callbacks()
                    self._tqm.send_callback('v2_playbook_on_start', pb)

                i = 1
                plays = pb.get_plays()
                display.vv(u'%d plays in %s' %
                           (len(plays), to_text(playbook_path)))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    # Doing this before vars_prompt to allow for using variables in prompt.
                    all_vars = self._variable_manager.get_vars(play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if play.vars_prompt:
                        for var in new_play.vars_prompt:
                            vname = var['name']
                            prompt = var.get("prompt", vname)
                            default = var.get("default", None)
                            private = boolean(var.get("private", True))
                            confirm = boolean(var.get("confirm", False))
                            encrypt = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt = var.get("salt", None)

                            if vname not in self._variable_manager.extra_vars:
                                if self._tqm:
                                    self._tqm.send_callback(
                                        'v2_playbook_on_vars_prompt', vname,
                                        private, prompt, encrypt, confirm,
                                        salt_size, salt, default)
                                    play.vars[vname] = display.do_var_prompt(
                                        vname, private, prompt, encrypt,
                                        confirm, salt_size, salt, default)
                                else:  # we are either in --list-<option> or syntax check
                                    play.vars[vname] = default

                        # Post validating again in case variables were entered in the prompt.
                        all_vars = self._variable_manager.get_vars(play=play)
                        templar = Templar(loader=self._loader,
                                          variables=all_vars)
                        new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        self._tqm._unreachable_hosts.update(
                            self._unreachable_hosts)

                        previously_failed = len(self._tqm._failed_hosts)
                        previously_unreachable = len(
                            self._tqm._unreachable_hosts)

                        break_play = False
                        # we are actually running plays
                        batches = self._get_serialized_batches(new_play)
                        if len(batches) == 0:
                            self._tqm.send_callback(
                                'v2_playbook_on_play_start', new_play)
                            self._tqm.send_callback(
                                'v2_playbook_on_no_hosts_matched')
                        for batch in batches:
                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)

                            # break the play if the result equals the special return code
                            if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
                                result = self._tqm.RUN_FAILED_HOSTS
                                break_play = True

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
                                (previously_failed + previously_unreachable)

                            if len(batch) == failed_hosts_count:
                                break_play = True
                                break

                            # update the previous counts so they don't accumulate incorrectly
                            # over multiple serial batches
                            previously_failed += len(
                                self._tqm._failed_hosts) - previously_failed
                            previously_unreachable += len(
                                self._tqm._unreachable_hosts
                            ) - previously_unreachable

                            # save the unreachable hosts from this batch
                            self._unreachable_hosts.update(
                                self._tqm._unreachable_hosts)

                        if break_play:
                            break

                    i = i + 1  # per play

                if entry:
                    entrylist.append(entry)  # per playbook

                # send the stats callback for this playbook
                if self._tqm is not None:
                    if C.RETRY_FILES_ENABLED:
                        retries = set(self._tqm._failed_hosts.keys())
                        retries.update(self._tqm._unreachable_hosts.keys())
                        retries = sorted(retries)
                        if len(retries) > 0:
                            if C.RETRY_FILES_SAVE_PATH:
                                basedir = C.RETRY_FILES_SAVE_PATH
                            elif playbook_path:
                                basedir = os.path.dirname(
                                    os.path.abspath(playbook_path))
                            else:
                                basedir = '~/'

                            (retry_name, _) = os.path.splitext(
                                os.path.basename(playbook_path))
                            filename = os.path.join(basedir,
                                                    "%s.retry" % retry_name)
                            if self._generate_retry_inventory(
                                    filename, retries):
                                display.display(
                                    "\tto retry, use: --limit @%s\n" %
                                    filename)

                    self._tqm.send_callback('v2_playbook_on_stats',
                                            self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._tqm.cleanup()
            if self._loader:
                self._loader.cleanup_all_tmp_files()

        if self._options.syntax:
            display.display("No issues encountered")
            return result

        return result
예제 #32
0
def get_playbook(self, *args, **kwargs):
    inject_ansible_paths(self.master.ctrl)
    from ansible.playbook import Play, Playbook
    import ansible.errors

    (options, loader, inventory, variable_manager) = self.get_ansible_variablemanager(**kwargs)
    playbooks_directory = get_playbooks_directory(self.master.main_config)
    playbook = kwargs.pop('playbook', None)
    if playbook is None:
        for instance_id in (self.uid, self.id):
            playbook_path = os.path.join(playbooks_directory, '%s.yml' % instance_id)
            if os.path.exists(playbook_path):
                playbook = playbook_path
                break
        if 'playbook' in self.config:
            if playbook is not None and playbook != self.config['playbook']:
                log.warning("Instance '%s' has the 'playbook' option set, but there is also a playbook at the default location '%s', which differs from '%s'." % (self.config_id, playbook, self.config['playbook']))
            playbook = self.config['playbook']
    if playbook is not None:
        log.info("Using playbook at '%s'." % playbook)
    roles = kwargs.pop('roles', None)
    if roles is None and 'roles' in self.config:
        roles = self.config['roles']
    if roles is not None and playbook is not None:
        log.error("You can't use a playbook and the 'roles' options at the same time for instance '%s'." % self.config_id)
        sys.exit(1)
    if playbook is None and roles is None:
        return None
    skip_host_check = kwargs.pop('skip_host_check', False)
    try:
        if roles is None:
            pb = Playbook.load(playbook, variable_manager=variable_manager, loader=loader)
            plays = pb.get_plays()
        else:
            if isinstance(roles, basestring):
                roles = roles.split()
            data = {
                'hosts': [self.uid],
                'roles': roles}
            plays = [Play.load(data, variable_manager=variable_manager, loader=loader)]
            pb = Playbook(loader=loader)
            pb._entries.extend(plays)
    except ansible.errors.AnsibleError as e:
        log.error("AnsibleError: %s" % e)
        sys.exit(1)
    for play in plays:
        if play._attributes.get('remote_user') is None:
            play._attributes['remote_user'] = self.config.get('user', 'root')
        if self.config.get('sudo'):
            play._attributes['sudo'] = self.config.get('sudo')
        if not skip_host_check:
            hosts = play._attributes.get('hosts', None)
            if isinstance(hosts, basestring):
                hosts = hosts.split(':')
            if hosts is None:
                hosts = {}
            if self.uid not in hosts:
                log.warning("The host '%s' is not in the list of hosts (%s) of '%s'.", self.uid, ','.join(hosts), playbook)
                if not yesno("Do you really want to apply '%s' to the host '%s'?" % (playbook, self.uid)):
                    sys.exit(1)
        play._attributes['hosts'] = [self.uid]
    return pb
예제 #33
0
    def run_playbook(self, playbook, workname, work_uuid, username, describe, yaml_content, is_checksyntax=False):
        """

        :param playbooks: playbook路径的列表
        :return:
        """
        if is_checksyntax:
            self._tqm = None
        else:
            # self.callback = PlayBookResultsCollector()
            # self.callback = ResultCallback(work_uuid, workname, 'playbook', self.options, describe,
            #                                yaml_content=yaml_content, username=username)
            self.callback = JsonCallback(work_uuid, workname, 'playbook', self.options, describe,
                                         yaml_content=yaml_content, username=username)
            self._tqm = TaskQueueManager(inventory=self.inventory,
                                         variable_manager=self.variable_manager,
                                         loader=self.loader,
                                         options=self.options,
                                         passwords=self.passwords,
                                         stdout_callback=self.callback)

        check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE)

        try:
            pb = Playbook.load(playbook, variable_manager=self.variable_manager, loader=self.loader)
            if self._tqm is not None:
                self._tqm.load_callbacks()
                self._tqm.send_callback('v2_playbook_on_start', pb)

            plays = pb.get_plays()
            for play in plays:
                if play._included_path is not None:
                    self.loader.set_basedir(play._included_path)
                else:
                    self.loader.set_basedir(pb._basedir)
                self.inventory.remove_restriction()

                # Allow variables to be used in vars_prompt fields.
                all_vars = self.variable_manager.get_vars(play=play)
                templar = Templar(loader=self.loader, variables=all_vars)
                play.post_validate(templar)

                if is_checksyntax or self._tqm is None:
                    return True

                batches = self._get_serialized_batches(play)
                if len(batches) == 0:
                    self._tqm.send_callback('v2_playbook_on_play_start', play)
                    self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                    continue
                for batch in batches:
                    self.inventory.restrict_to_hosts(batch)
                    try:
                        self._tqm.run(play)
                    except Exception as e:
                        print(e)

            self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
            self._tqm.cleanup()
            self.loader.cleanup_all_tmp_files()
            return True
        except Exception as e:
            print(e)
            return False
예제 #34
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        signal.signal(signal.SIGINT, self._cleanup)

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)
                self._inventory.set_playbook_basedir(
                    os.path.dirname(playbook_path))

                if self._tqm is None:  # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []

                i = 1
                plays = pb.get_plays()
                self._display.vv('%d plays in %s' %
                                 (len(plays), playbook_path))

                for play in plays:
                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            if 'name' not in var:
                                raise AnsibleError(
                                    "'vars_prompt' item is missing 'name:'",
                                    obj=play._ds)

                            vname = var['name']
                            prompt = var.get("prompt", vname)
                            default = var.get("default", None)
                            private = var.get("private", True)

                            confirm = var.get("confirm", False)
                            encrypt = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt = var.get("salt", None)

                            if vname not in play.vars:
                                self._tqm.send_callback(
                                    'v2_playbook_on_vars_prompt', vname,
                                    private, prompt, encrypt, confirm,
                                    salt_size, salt, default)
                                play.vars[vname] = self._do_var_prompt(
                                    vname, private, prompt, encrypt, confirm,
                                    salt_size, salt, default)

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(
                        loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        # make sure the tqm has callbacks loaded
                        self._tqm.load_callbacks()

                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback(
                                    'v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback(
                                    'v2_playbook_on_no_hosts_matched')
                                break
                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)
                            # if the last result wasn't zero, break out of the serial batch loop
                            if result != 0:
                                break

                        # if the last result wasn't zero, break out of the play loop
                        if result != 0:
                            break

                    i = i + 1  # per play

                if entry:
                    entrylist.append(entry)  # per playbook

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._cleanup()

        if self._options.syntax:
            self.display.display("No issues encountered")
            return result

        # FIXME: this stat summary stuff should be cleaned up and moved
        #        to a new method, if it even belongs here...
        self._display.banner("PLAY RECAP")

        hosts = sorted(self._tqm._stats.processed.keys())
        for h in hosts:
            t = self._tqm._stats.summarize(h)

            self._display.display(
                "%s : %s %s %s %s" %
                (hostcolor(h, t), colorize('ok', t['ok'], 'green'),
                 colorize('changed', t['changed'], 'yellow'),
                 colorize('unreachable', t['unreachable'],
                          'red'), colorize('failed', t['failures'], 'red')),
                screen_only=True)

            self._display.display(
                "%s : %s %s %s %s" %
                (hostcolor(h, t, False), colorize('ok', t['ok'], None),
                 colorize('changed', t['changed'], None),
                 colorize('unreachable', t['unreachable'],
                          None), colorize('failed', t['failures'], None)),
                log_only=True)

        self._display.display("", screen_only=True)
        # END STATS STUFF

        return result
    def test_noop(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: all
              gather_facts: no
              tasks:
                - block:
                   - block:
                     - name: task1
                       debug: msg='task1'
                       failed_when: inventory_hostname == 'host01'

                     - name: task2
                       debug: msg='task2'

                     rescue:
                       - name: rescue1
                         debug: msg='rescue1'

                       - name: rescue2
                         debug: msg='rescue2'
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 2):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        mock_options = MagicMock()
        mock_options.module_path = None

        tqm = TaskQueueManager(
            inventory=inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=None,
        )
        tqm._initialize_processes(3)
        strategy = StrategyModule(tqm)

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'meta')

        # debug: task1, debug: task1
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'debug')
        self.assertEqual(host2_task.action, 'debug')
        self.assertEqual(host1_task.name, 'task1')
        self.assertEqual(host2_task.name, 'task1')

        # mark the second host failed
        itr.mark_host_failed(hosts[1])

        # debug: task2, meta: noop
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'debug')
        self.assertEqual(host2_task.action, 'meta')
        self.assertEqual(host1_task.name, 'task2')
        self.assertEqual(host2_task.name, '')

        # meta: noop, debug: rescue1
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'debug')
        self.assertEqual(host1_task.name, '')
        self.assertEqual(host2_task.name, 'rescue1')

        # meta: noop, debug: rescue2
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'debug')
        self.assertEqual(host1_task.name, '')
        self.assertEqual(host2_task.name, 'rescue2')

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'meta')

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNotNone(host1_task)
        self.assertIsNotNone(host2_task)
        self.assertEqual(host1_task.action, 'meta')
        self.assertEqual(host2_task.action, 'meta')

        # end of iteration
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
        self.assertIsNone(host1_task)
        self.assertIsNone(host2_task)
예제 #36
0
    def run(self):
        ''' create and execute the single task playbook '''

        super(AdHocCLI, self).run()

        # only thing left should be host pattern
        pattern = to_text(self.args[0], errors='surrogate_or_strict')

        sshpass = None
        becomepass = None

        self.normalize_become_options()
        (sshpass, becomepass) = self.ask_passwords()
        passwords = {'conn_pass': sshpass, 'become_pass': becomepass}

        # dynamically load any plugins
        get_all_plugin_loaders()

        loader, inventory, variable_manager = self._play_prereqs(self.options)

        try:
            hosts = CLI.get_host_list(inventory, self.options.subset, pattern)
        except AnsibleError:
            if self.options.subset:
                raise
            else:
                hosts = []
                display.warning("No hosts matched, nothing to do")

        if self.options.listhosts:
            display.display('  hosts (%d):' % len(hosts))
            for host in hosts:
                display.display('    %s' % host)
            return 0

        if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args:
            err = "No argument passed to %s module" % self.options.module_name
            if pattern.endswith(".yml"):
                err = err + ' (did you mean to run ansible-playbook?)'
            raise AnsibleOptionsError(err)

        play_ds = self._play_ds(pattern, self.options.seconds,
                                self.options.poll_interval)
        play = Play().load(play_ds,
                           variable_manager=variable_manager,
                           loader=loader)

        # used in start callback
        playbook = Playbook(loader)
        playbook._entries.append(play)
        playbook._file_name = '__adhoc_playbook__'

        if self.callback:
            cb = self.callback
        elif self.options.one_line:
            cb = 'oneline'
        # Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks'
        elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
            cb = C.DEFAULT_STDOUT_CALLBACK
        else:
            cb = 'minimal'

        run_tree = False
        if self.options.tree:
            C.DEFAULT_CALLBACK_WHITELIST.append('tree')
            C.TREE_DIR = self.options.tree
            run_tree = True

        # now create a task queue manager to execute the play
        self._tqm = None
        try:
            self._tqm = TaskQueueManager(
                inventory=inventory,
                variable_manager=variable_manager,
                loader=loader,
                options=self.options,
                passwords=passwords,
                stdout_callback=cb,
                run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
                run_tree=run_tree,
            )

            self._tqm.send_callback('v2_playbook_on_start', playbook)

            result = self._tqm.run(play)

            self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
        finally:
            if self._tqm:
                self._tqm.cleanup()
            if loader:
                loader.cleanup_all_tmp_files()

        return result
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # lookup up an original task
        target_task = p._entries[0].tasks[0].block[0]
        task_copy = target_task.copy(exclude_block=True)
        found_task = itr.get_original_task(hosts[0], task_copy)
        self.assertEqual(target_task, found_task)

        bad_task = Task()
        found_task = itr.get_original_task(hosts[0], bad_task)
        self.assertIsNone(found_task)

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
예제 #38
0
    def test_play_iterator_add_tasks(self):
        fake_loader = DictDataLoader({
            'test_play.yml': """
            - hosts: all
              gather_facts: no
              tasks:
              - debug: msg="dummy task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # test the high-level add_tasks() method
        s = HostState(blocks=[0,1,2])
        itr._insert_tasks_into_state = MagicMock(return_value=s)
        itr.add_tasks(hosts[0], [3,4,5])
        self.assertEqual(itr._host_states[hosts[0].name], s)

        # now actually test the lower-level method that does the work
        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # iterate past first task
        _, task = itr.get_next_task_for_host(hosts[0])
        while(task and task.action != 'debug'):
            _, task = itr.get_next_task_for_host(hosts[0])

        if task is None:
            raise Exception("iterated past end of play while looking for place to insert tasks")

        # get the current host state and copy it so we can mutate it
        s = itr.get_host_state(hosts[0])
        s_copy = s.copy()

        # assert with an empty task list, or if we're in a failed state, we simply return the state as-is
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
        self.assertEqual(res_state, s_copy)

        s_copy.fail_state = itr.FAILED_TASKS
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
        self.assertEqual(res_state, s_copy)

        # but if we've failed with a rescue/always block
        mock_task = MagicMock()
        s_copy.run_state = itr.ITERATING_RESCUE
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
        self.assertEqual(res_state, s_copy)
        self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
        itr._host_states[hosts[0].name] = res_state
        (next_state, next_task) = itr.get_next_task_for_host(hosts[0], peek=True)
        self.assertEqual(next_task, mock_task)
        itr._host_states[hosts[0].name] = s

        # test a regular insertion
        s_copy = s.copy()
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
예제 #39
0
    def test_play_iterator_add_tasks(self):
        fake_loader = DictDataLoader({
            'test_play.yml':
            """
            - hosts: all
              gather_facts: no
              tasks:
              - debug: msg="dummy task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # test the high-level add_tasks() method
        s = HostState(blocks=[0, 1, 2])
        itr._insert_tasks_into_state = MagicMock(return_value=s)
        itr.add_tasks(hosts[0], [MagicMock(), MagicMock(), MagicMock()])
        self.assertEqual(itr._host_states[hosts[0].name], s)

        # now actually test the lower-level method that does the work
        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # iterate past first task
        _, task = itr.get_next_task_for_host(hosts[0])
        while (task and task.action != 'debug'):
            _, task = itr.get_next_task_for_host(hosts[0])

        if task is None:
            raise Exception(
                "iterated past end of play while looking for place to insert tasks"
            )

        # get the current host state and copy it so we can mutate it
        s = itr.get_host_state(hosts[0])
        s_copy = s.copy()

        # assert with an empty task list, or if we're in a failed state, we simply return the state as-is
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
        self.assertEqual(res_state, s_copy)

        s_copy.fail_state = itr.FAILED_TASKS
        res_state = itr._insert_tasks_into_state(s_copy,
                                                 task_list=[MagicMock()])
        self.assertEqual(res_state, s_copy)

        # but if we've failed with a rescue/always block
        mock_task = MagicMock()
        s_copy.run_state = itr.ITERATING_RESCUE
        res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
        self.assertEqual(res_state, s_copy)
        self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
        itr._host_states[hosts[0].name] = res_state
        (next_state, next_task) = itr.get_next_task_for_host(hosts[0],
                                                             peek=True)
        self.assertEqual(next_task, mock_task)
        itr._host_states[hosts[0].name] = s

        # test a regular insertion
        s_copy = s.copy()
        res_state = itr._insert_tasks_into_state(s_copy,
                                                 task_list=[MagicMock()])
예제 #40
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        signal.signal(signal.SIGINT, self._cleanup)

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)
                self._inventory.set_playbook_basedir(
                    os.path.dirname(playbook_path))

                if self._tqm is None:  # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []

                i = 1
                plays = pb.get_plays()
                self._display.vv('%d plays in %s' %
                                 (len(plays), playbook_path))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            vname = var['name']
                            prompt = var.get("prompt", vname)
                            default = var.get("default", None)
                            private = var.get("private", True)
                            confirm = var.get("confirm", False)
                            encrypt = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt = var.get("salt", None)

                            if vname not in play.vars:
                                if self._tqm:
                                    self._tqm.send_callback(
                                        'v2_playbook_on_vars_prompt', vname,
                                        private, prompt, encrypt, confirm,
                                        salt_size, salt, default)
                                play.vars[vname] = self._do_var_prompt(
                                    vname, private, prompt, encrypt, confirm,
                                    salt_size, salt, default)

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(
                        loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        # make sure the tqm has callbacks loaded
                        self._tqm.load_callbacks()
                        self._tqm._unreachable_hosts.update(
                            self._unreachable_hosts)

                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback(
                                    'v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback(
                                    'v2_playbook_on_no_hosts_matched')
                                break

                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(
                                self._tqm._failed_hosts) + len(
                                    self._tqm._unreachable_hosts)
                            if new_play.any_errors_fatal and failed_hosts_count > 0:
                                break
                            elif new_play.max_fail_percentage is not None and \
                               int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
                                break
                            elif len(batch) == failed_hosts_count:
                                break

                            # clear the failed hosts dictionaires in the TQM for the next batch
                            self._unreachable_hosts.update(
                                self._tqm._unreachable_hosts)
                            self._tqm.clear_failed_hosts()

                        # if the last result wasn't zero or 3 (some hosts were unreachable),
                        # break out of the serial batch loop
                        if result not in (0, 3):
                            break

                    i = i + 1  # per play

                if entry:
                    entrylist.append(entry)  # per playbook

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._cleanup()

        if self._options.syntax:
            self.display.display("No issues encountered")
            return result

        # FIXME: this stat summary stuff should be cleaned up and moved
        #        to a new method, if it even belongs here...
        self._display.banner("PLAY RECAP")

        hosts = sorted(self._tqm._stats.processed.keys())
        for h in hosts:
            t = self._tqm._stats.summarize(h)

            self._display.display(
                u"%s : %s %s %s %s" %
                (hostcolor(h, t), colorize(u'ok', t['ok'], 'green'),
                 colorize(u'changed', t['changed'], 'yellow'),
                 colorize(u'unreachable', t['unreachable'],
                          'red'), colorize(u'failed', t['failures'], 'red')),
                screen_only=True)

            self._display.display(
                u"%s : %s %s %s %s" %
                (hostcolor(h, t, False), colorize(u'ok', t['ok'], None),
                 colorize(u'changed', t['changed'], None),
                 colorize(u'unreachable', t['unreachable'],
                          None), colorize(u'failed', t['failures'], None)),
                log_only=True)

        self._display.display("", screen_only=True)
        # END STATS STUFF

        return result
예제 #41
0
    def run(self):
        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path,
                                   variable_manager=self._variable_manager,
                                   loader=self._loader)
                self._inventory.set_playbook_basedir(
                    os.path.dirname(playbook_path))

                if self._tqm is None:  # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []
                else:
                    # make sure the tqm has callbacks loaded
                    self._tqm.load_callbacks()
                    self._tqm.send_callback('v2_playbook_on_start', pb)

                i = 1
                plays = pb.get_plays()
                display.vv(u'%d plays in %s' %
                           (len(plays), to_unicode(playbook_path)))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            vname = var['name']
                            prompt = var.get("prompt", vname)
                            default = var.get("default", None)
                            private = var.get("private", True)
                            confirm = var.get("confirm", False)
                            encrypt = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt = var.get("salt", None)

                            if vname not in self._variable_manager.extra_vars:
                                if self._tqm:
                                    self._tqm.send_callback(
                                        'v2_playbook_on_vars_prompt', vname,
                                        private, prompt, encrypt, confirm,
                                        salt_size, salt, default)
                                    play.vars[vname] = display.do_var_prompt(
                                        vname, private, prompt, encrypt,
                                        confirm, salt_size, salt, default)
                                else:  # we are either in --list-<option> or syntax check
                                    play.vars[vname] = default

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(
                        loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        self._tqm._unreachable_hosts.update(
                            self._unreachable_hosts)

                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback(
                                    'v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback(
                                    'v2_playbook_on_no_hosts_matched')
                                break

                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(
                                self._tqm._failed_hosts) + len(
                                    self._tqm._unreachable_hosts)
                            if new_play.max_fail_percentage is not None and \
                               int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
                                break
                            elif len(batch) == failed_hosts_count:
                                break

                            # clear the failed hosts dictionaires in the TQM for the next batch
                            self._unreachable_hosts.update(
                                self._tqm._unreachable_hosts)
                            self._tqm.clear_failed_hosts()

                        # if the last result wasn't zero or 3 (some hosts were unreachable),
                        # break out of the serial batch loop
                        if result not in (0, 3):
                            break

                    i = i + 1  # per play

                if entry:
                    entrylist.append(entry)  # per playbook

                # send the stats callback for this playbook
                if self._tqm is not None:
                    if C.RETRY_FILES_ENABLED:
                        retries = set(self._tqm._failed_hosts.keys())
                        retries.update(self._tqm._unreachable_hosts.keys())
                        retries = sorted(retries)
                        if len(retries) > 0:
                            if C.RETRY_FILES_SAVE_PATH:
                                basedir = C.shell_expand(
                                    C.RETRY_FILES_SAVE_PATH)
                            else:
                                basedir = os.path.dirname(playbook_path)

                            (retry_name, _) = os.path.splitext(
                                os.path.basename(playbook_path))
                            filename = os.path.join(basedir,
                                                    "%s.retry" % retry_name)
                            if self._generate_retry_inventory(
                                    filename, retries):
                                display.display(
                                    "\tto retry, use: --limit @%s\n" %
                                    filename)

                    self._tqm.send_callback('v2_playbook_on_stats',
                                            self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._tqm.cleanup()

        if self._options.syntax:
            display.display("No issues encountered")
            return result

        return result
예제 #42
0
            if self._inventory else [],
            ['--user', self._username] if self._username else [],
            ['--become'] if self._become else [],
            ['--become-user', self._become_user] if self._become_user else [],
            ['--forks', str(self._forks)] if self._forks else [],
            [
                '--ssh-extra-args',
                '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
            ] if not self._modifyKnownHosts
            else []['--extra-vars',
                    json.dumps(self._extra_vars)] if self._extra_vars else [],
            [self._playbook_file.name],
        ]

        cmd = reduce(operator.concat, cmd_builder_monoid)
        Subprocess(cmd, **self._subprocess_kwargs)

        ################################################################


if __name__ == '__main__':
    loader = DataLoader()
    variable_manager = VariableManager()
    playbook = Playbook(loader)
    Inventory(loader,
              variable_manager,
              host_list={
                  'a': ['foo', 'bar'],
                  'b': ['foo', 'baz']
              })
예제 #43
0
파일: adhoc.py 프로젝트: srivanib95/ansible
    def run(self):
        ''' create and execute the single task playbook '''

        super(AdHocCLI, self).run()

        # only thing left should be host pattern
        pattern = to_text(context.CLIARGS['args'],
                          errors='surrogate_or_strict')

        sshpass = None
        becomepass = None

        (sshpass, becomepass) = self.ask_passwords()
        passwords = {'conn_pass': sshpass, 'become_pass': becomepass}

        # get basic objects
        loader, inventory, variable_manager = self._play_prereqs()

        try:
            hosts = self.get_host_list(inventory, context.CLIARGS['subset'],
                                       pattern)
        except AnsibleError:
            if context.CLIARGS['subset']:
                raise
            else:
                hosts = []
                display.warning("No hosts matched, nothing to do")

        if context.CLIARGS['listhosts']:
            display.display('  hosts (%d):' % len(hosts))
            for host in hosts:
                display.display('    %s' % host)
            return 0

        if context.CLIARGS[
                'module_name'] in C.MODULE_REQUIRE_ARGS and not context.CLIARGS[
                    'module_args']:
            err = "No argument passed to %s module" % context.CLIARGS[
                'module_name']
            if pattern.endswith(".yml"):
                err = err + ' (did you mean to run ansible-playbook?)'
            raise AnsibleOptionsError(err)

        # Avoid modules that don't work with ad-hoc
        if context.CLIARGS['module_name'] in ('import_playbook', ):
            raise AnsibleOptionsError(
                "'%s' is not a valid action for ad-hoc commands" %
                context.CLIARGS['module_name'])

        play_ds = self._play_ds(pattern, context.CLIARGS['seconds'],
                                context.CLIARGS['poll_interval'])
        play = Play().load(play_ds,
                           variable_manager=variable_manager,
                           loader=loader)

        # used in start callback
        playbook = Playbook(loader)
        playbook._entries.append(play)
        playbook._file_name = '__adhoc_playbook__'

        if self.callback:
            cb = self.callback
        elif context.CLIARGS['one_line']:
            cb = 'oneline'
        # Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks'
        elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
            cb = C.DEFAULT_STDOUT_CALLBACK
        else:
            cb = 'minimal'

        run_tree = False
        if context.CLIARGS['tree']:
            C.CALLBACKS_ENABLED.append('tree')
            C.TREE_DIR = context.CLIARGS['tree']
            run_tree = True

        # now create a task queue manager to execute the play
        self._tqm = None
        try:
            self._tqm = TaskQueueManager(
                inventory=inventory,
                variable_manager=variable_manager,
                loader=loader,
                passwords=passwords,
                stdout_callback=cb,
                run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
                run_tree=run_tree,
                forks=context.CLIARGS['forks'],
            )

            self._tqm.load_callbacks()
            self._tqm.send_callback('v2_playbook_on_start', playbook)

            result = self._tqm.run(play)

            self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
        finally:
            if self._tqm:
                self._tqm.cleanup()
            if loader:
                loader.cleanup_all_tmp_files()

        return result
예제 #44
0
    def run(self):
        ''' create and execute the single task playbook '''

        super(AdHocCLI, self).run()

        # only thing left should be host pattern
        pattern = to_text(self.args[0], errors='surrogate_or_strict')

        sshpass = None
        becomepass = None

        self.normalize_become_options()
        (sshpass, becomepass) = self.ask_passwords()
        passwords = {'conn_pass': sshpass, 'become_pass': becomepass}

        # dynamically load any plugins
        get_all_plugin_loaders()

        loader, inventory, variable_manager = self._play_prereqs(self.options)

        try:
            hosts = CLI.get_host_list(inventory, self.options.subset, pattern)
        except AnsibleError:
            if self.options.subset:
                raise
            else:
                hosts = []
                display.warning("No hosts matched, nothing to do")

        if self.options.listhosts:
            display.display('  hosts (%d):' % len(hosts))
            for host in hosts:
                display.display('    %s' % host)
            return 0

        if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args:
            err = "No argument passed to %s module" % self.options.module_name
            if pattern.endswith(".yml"):
                err = err + ' (did you mean to run ansible-playbook?)'
            raise AnsibleOptionsError(err)

        # Avoid modules that don't work with ad-hoc
        if self.options.module_name.startswith(('include', 'import_')):
            raise AnsibleOptionsError("'%s' is not a valid action for ad-hoc commands" % self.options.module_name)

        play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval)
        play = Play().load(play_ds, variable_manager=variable_manager, loader=loader)

        # used in start callback
        playbook = Playbook(loader)
        playbook._entries.append(play)
        playbook._file_name = '__adhoc_playbook__'

        if self.callback:
            cb = self.callback
        elif self.options.one_line:
            cb = 'oneline'
        # Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks'
        elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
            cb = C.DEFAULT_STDOUT_CALLBACK
        else:
            cb = 'minimal'

        run_tree = False
        if self.options.tree:
            C.DEFAULT_CALLBACK_WHITELIST.append('tree')
            C.TREE_DIR = self.options.tree
            run_tree = True

        # now create a task queue manager to execute the play
        self._tqm = None
        try:
            self._tqm = TaskQueueManager(
                inventory=inventory,
                variable_manager=variable_manager,
                loader=loader,
                options=self.options,
                passwords=passwords,
                stdout_callback=cb,
                run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
                run_tree=run_tree,
            )

            self._tqm.send_callback('v2_playbook_on_start', playbook)

            result = self._tqm.run(play)

            self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
        finally:
            if self._tqm:
                self._tqm.cleanup()
            if loader:
                loader.cleanup_all_tmp_files()

        return result
예제 #45
0
    def test_playbook_executor__get_serialized_batches(self):
        fake_loader = DictDataLoader({
            'no_serial.yml':
            '''
            - hosts: all
              gather_facts: no
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_int.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: 2
              tasks:
              - debug: var=inventory_hostname
            ''',
            'serial_pct.yml':
            '''
            - hosts: all
              gather_facts: no
              serial: 20%
              tasks:
              - debug: var=inventory_hostname
            ''',
        })

        mock_inventory = MagicMock()
        mock_var_manager = MagicMock()

        # fake out options to use the syntax CLI switch, which will ensure
        # the PlaybookExecutor doesn't create a TaskQueueManager
        mock_options = MagicMock()
        mock_options.syntax.value = True

        pbe = PlaybookExecutor(
            playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml'],
            inventory=mock_inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            options=mock_options,
            passwords=[],
        )

        playbook = Playbook.load(pbe._playbooks[0],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(pbe._get_serialized_batches(play), [[
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]])

        playbook = Playbook.load(pbe._playbooks[1],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
             ['host6', 'host7'], ['host8', 'host9']])

        playbook = Playbook.load(pbe._playbooks[2],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
             ['host6', 'host7'], ['host8', 'host9']])

        # Test when serial percent is under 1.0
        playbook = Playbook.load(pbe._playbooks[2],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
        self.assertEqual(pbe._get_serialized_batches(play),
                         [['host0'], ['host1'], ['host2']])

        # Test when there is a remainder for serial as a percent
        playbook = Playbook.load(pbe._playbooks[2],
                                 variable_manager=mock_var_manager,
                                 loader=fake_loader)
        play = playbook.get_plays()[0]
        mock_inventory.get_hosts.return_value = [
            'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
            'host7', 'host8', 'host9', 'host10'
        ]
        self.assertEqual(
            pbe._get_serialized_batches(play),
            [['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
             ['host6', 'host7'], ['host8', 'host9'], ['host10']])
예제 #46
0
 def _run_playbook(self, yamlfile, is_checksyntax=False):
     if not is_checksyntax :
         if self.options.flush_cache:
             self._flush_cache()
                 
         self._loading_callback(yamlfile)
         # self._unreachable_hosts = dict()
         
         tqm = TaskQueueManager(
             inventory=self.inventory,
             variable_manager=self.variable_manager,
             loader=self.loader,
             options=self.options,
             passwords=self.passwords,
             stdout_callback=self.callback,
         )
     
         check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE)
     else :
         tqm = None
         
     try :   
         pb = Playbook.load(
             yamlfile,
             variable_manager=self.variable_manager,
             loader=self.loader
         )
         self.inventory.set_playbook_basedir(yamlfile)
 
         if tqm is not None :
             tqm.load_callbacks()
             tqm.send_callback('v2_playbook_on_start', pb)
             self.logger.debug(self.log_prefix + '任务开始执行')
 
         plays = pb.get_plays()
 
         for play in plays:
             if play._included_path is not None:
                 self.loader.set_basedir(play._included_path)
             else:
                 self.loader.set_basedir(pb._basedir)
 
             self.inventory.remove_restriction()
             all_vars = self.variable_manager.get_vars(loader=self.loader, play=play)
 
             templar = Templar(loader=self.loader, variables=all_vars)
             new_play = play.copy()
             new_play.post_validate(templar)
             
             if is_checksyntax or tqm is None:
                 return True
 
             # tqm._unreachable_hosts.update(self._unreachable_hosts)
                         
             batches = self._get_serialized_batches(new_play)
             if len(batches) == 0:
                 tqm.send_callback('v2_playbook_on_play_start', new_play)
                 tqm.send_callback('v2_playbook_on_no_hosts_matched')
                 self.logger.debug(self.log_prefix + '任务开始执行,但没有匹配主机')
                 # 需要写日志,需要写该模块
                 continue
                            
             for batch in batches:
                 self.inventory.restrict_to_hosts(batch)
                 try :
                     tqm.run(play)
                 except Exception as e:
                     print(e)
                     pass
                                 
             # self._unreachable_hosts.update(tqm._unreachable_hosts)
 
         tqm.send_callback('v2_playbook_on_stats', tqm._stats)
         self.logger.debug(self.log_prefix + '任务执行完比')
         # self.host_list = self.inventory.get_hosts(play.hosts)
                     
         tqm.cleanup()
         self.loader.cleanup_all_tmp_files()
         return True
     except Exception as e:
         print(e)
         if is_checksyntax :
             return False
         else :
             return False
예제 #47
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude,
                        self).load_data(ds, variable_manager, loader)

        all_vars = self.vars.copy()
        if variable_manager:
            all_vars.update(variable_manager.get_vars(loader=loader))

        templar = Templar(loader=loader, variables=all_vars)

        try:
            forward_conditional = False
            if not new_obj.evaluate_conditional(templar=templar,
                                                all_vars=all_vars):
                return None
        except AnsibleError:
            # conditional evaluation raised an error, so we set a flag to indicate
            # we need to forward the conditionals on to the included play(s)
            forward_conditional = True

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = templar.template(new_obj.include)
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name,
                               variable_manager=variable_manager)

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:
            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(file_name)

            # Check to see if we need to forward the conditionals on to the included
            # plays. If so, we can take a shortcut here and simply prepend them to
            # those attached to each block (if any)
            if forward_conditional:
                for task_block in entry.tasks:
                    task_block.when = self.when[:] + task_block.when

        return pb
예제 #48
0
    def run(self):

        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''
        result = 0
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
                self._inventory.set_playbook_basedir(os.path.dirname(playbook_path))

                i = 1

                # make sure the tqm has callbacks loaded
                self._tqm.load_callbacks()
                self._tqm.send_callback('v2_playbook_on_start', pb)

                for play in pb.get_plays():
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)


                    self._tqm._unreachable_hosts.update(self._unreachable_hosts)

                    # we are actually running plays
                    for batch in self._get_serialized_batches(new_play):
                        if len(batch) == 0:
                            self._tqm.send_callback('v2_playbook_on_play_start', new_play)
                            self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                            break
                        # restrict the inventory to the hosts in the serialized batch
                        self._inventory.restrict_to_hosts(batch)
                        # and run it...
                        result = self._tqm.run(play=play)

                        # check the number of failures here, to see if they're above the maximum
                        # failure percentage allowed, or if any errors are fatal. If either of those
                        # conditions are met, we break out, otherwise we only break out if the entire
                        # batch failed
                        failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts)
                        if new_play.any_errors_fatal and failed_hosts_count > 0:
                            break
                        elif new_play.max_fail_percentage is not None and \
                           int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
                            break
                        elif len(batch) == failed_hosts_count:
                            break

                        # clear the failed hosts dictionaires in the TQM for the next batch
                        self._unreachable_hosts.update(self._tqm._unreachable_hosts)
                        self._tqm.clear_failed_hosts()

                    # if the last result wasn't zero or 3 (some hosts were unreachable),
                    # break out of the serial batch loop
                    if result not in (0, 3):
                        break

                    i = i + 1 # per play

                self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

        finally:
            if self._tqm is not None:
                self._cleanup()

        return result
예제 #49
0
    def test_play_iterator(self):
        # import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - name: role task
              debug: msg="this is a role task"
            - block:
              - name: role block task
                debug: msg="inside block in role"
              always:
              - name: role always task
                debug: msg="always task in block in role"
            - include: foo.yml
            - name: role task after include
              debug: msg="after include in role"
            - block:
              - name: starting role nested block 1
                debug:
              - block:
                - name: role nested block 1 task 1
                  debug:
                - name: role nested block 1 task 2
                  debug:
                - name: role nested block 1 task 3
                  debug:
              - name: end of role nested block 1
                debug:
              - name: starting role nested block 2
                debug:
              - block:
                - name: role nested block 2 task 1
                  debug:
                - name: role nested block 2 task 2
                  debug:
                - name: role nested block 2 task 3
                  debug:
              - name: end of role nested block 2
                debug:
            """,
            '/etc/ansible/roles/test_role/tasks/foo.yml': """
            - name: role included task
              debug: msg="this is task in an include from a role"
            """
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.name, "role task")
        self.assertIsNotNone(task._role)
        # role block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role block task")
        self.assertIsNotNone(task._role)
        # role block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role always task")
        self.assertIsNotNone(task._role)
        # role include task
        # (host_state, task) = itr.get_next_task_for_host(hosts[0])
        # self.assertIsNotNone(task)
        # self.assertEqual(task.action, 'debug')
        # self.assertEqual(task.name, "role included task")
        # self.assertIsNotNone(task._role)
        # role task after include
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role task after include")
        self.assertIsNotNone(task._role)
        # role nested block tasks
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 1 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "starting role nested block 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 1")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 2")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "role nested block 2 task 3")
        self.assertIsNotNone(task._role)
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.name, "end of role nested block 2")
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
# Object supplying methods to load JSON or YAML from file or string, into a Python data structure.
loader = DataLoader()

# https://github.com/ansible/ansible/blob/devel/lib/ansible/inventory/__init__.py
#
# Builds the inventory as a data structure (including groups), object also caches host and group vars.
# Loader supplies JSON/YAML parsing methods, variable_manager allows object to copy (cache) vars data
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list='localhost')

# Here we simply copy back the inventory data into the central variable_manager.
variable_manager.set_inventory(inventory)

# https://github.com/ansible/ansible/blob/devel/lib/ansible/playbook/__init__.py
#
# We use the Playbook object to load and then create Play objects from YAML files.  Playbook
# will load and parse the YAML data into the Playbook.entries[] list.
# Playbook.load() is a static method that calls Playbook.__init__
playbook = Playbook.load("test/site.yml", variable_manager, loader)

# Now we have the playbook loaded we can loop through the plays.  A play is a list of roles
# and/or tasks/handler blocks to execute on a set of hosts.
# see definition: https://github.com/ansible/ansible/blob/devel/lib/ansible/playbook/play.py
for play in playbook.get_plays():
    for block in play.compile():
        if block.has_tasks():
            for task in block.block:
                if task._role in play.get_roles():
                    changed = str(task._get_parent_attribute('changed_when'))
                    failed = str(task._get_parent_attribute('failed_when'))
                    print("PLAY: %s, ROLE: %s, %s, changed_when: %s, failed_when: %s" % (play.get_name(), task._role, task, changed, failed))
예제 #51
0
    def run(self):

        '''
        Run the given playbook, based on the settings in the play which
        may limit the runs to serialized groups, etc.
        '''

        result = 0
        entrylist = []
        entry = {}
        try:
            for playbook_path in self._playbooks:
                pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
                self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))

                if self._tqm is None: # we are doing a listing
                    entry = {'playbook': playbook_path}
                    entry['plays'] = []
                else:
                    # make sure the tqm has callbacks loaded
                    self._tqm.load_callbacks()
                    self._tqm.send_callback('v2_playbook_on_start', pb)

                i = 1
                plays = pb.get_plays()
                display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path)))

                for play in plays:
                    if play._included_path is not None:
                        self._loader.set_basedir(play._included_path)
                    else:
                        self._loader.set_basedir(pb._basedir)

                    # clear any filters which may have been applied to the inventory
                    self._inventory.remove_restriction()

                    if play.vars_prompt:
                        for var in play.vars_prompt:
                            vname     = var['name']
                            prompt    = var.get("prompt", vname)
                            default   = var.get("default", None)
                            private   = var.get("private", True)
                            confirm   = var.get("confirm", False)
                            encrypt   = var.get("encrypt", None)
                            salt_size = var.get("salt_size", None)
                            salt      = var.get("salt", None)

                            if vname not in self._variable_manager.extra_vars:
                                if self._tqm:
                                    self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default)
                                    play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)
                                else: # we are either in --list-<option> or syntax check
                                    play.vars[vname] = default

                    # Create a temporary copy of the play here, so we can run post_validate
                    # on it without the templating changes affecting the original object.
                    all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                    templar = Templar(loader=self._loader, variables=all_vars)
                    new_play = play.copy()
                    new_play.post_validate(templar)

                    if self._options.syntax:
                        continue

                    if self._tqm is None:
                        # we are just doing a listing
                        entry['plays'].append(new_play)

                    else:
                        self._tqm._unreachable_hosts.update(self._unreachable_hosts)

                        previously_failed = len(self._tqm._failed_hosts)
                        previously_unreachable = len(self._tqm._unreachable_hosts)

                        break_play = False
                        # we are actually running plays
                        for batch in self._get_serialized_batches(new_play):
                            if len(batch) == 0:
                                self._tqm.send_callback('v2_playbook_on_play_start', new_play)
                                self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                                break

                            # restrict the inventory to the hosts in the serialized batch
                            self._inventory.restrict_to_hosts(batch)
                            # and run it...
                            result = self._tqm.run(play=play)

                            # break the play if the result equals the special return code
                            if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
                                result = self._tqm.RUN_FAILED_HOSTS
                                break_play = True

                            # check the number of failures here, to see if they're above the maximum
                            # failure percentage allowed, or if any errors are fatal. If either of those
                            # conditions are met, we break out, otherwise we only break out if the entire
                            # batch failed
                            failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
                                                 (previously_failed + previously_unreachable)

                            if len(batch) == failed_hosts_count:
                                break_play = True
                                break

                            # update the previous counts so they don't accumulate incorrectly
                            # over multiple serial batches
                            previously_failed += len(self._tqm._failed_hosts) - previously_failed
                            previously_unreachable += len(self._tqm._unreachable_hosts) - previously_unreachable

                            # save the unreachable hosts from this batch
                            self._unreachable_hosts.update(self._tqm._unreachable_hosts)

                        if break_play:
                            break

                    i = i + 1 # per play

                if entry:
                    entrylist.append(entry) # per playbook

                # send the stats callback for this playbook
                if self._tqm is not None:
                    if C.RETRY_FILES_ENABLED:
                        retries = set(self._tqm._failed_hosts.keys())
                        retries.update(self._tqm._unreachable_hosts.keys())
                        retries = sorted(retries)
                        if len(retries) > 0:
                            if C.RETRY_FILES_SAVE_PATH:
                                basedir = C.shell_expand(C.RETRY_FILES_SAVE_PATH)
                            elif playbook_path:
                                basedir = os.path.dirname(playbook_path)
                            else:
                                basedir = '~/'

                            (retry_name, _) = os.path.splitext(os.path.basename(playbook_path))
                            filename = os.path.join(basedir, "%s.retry" % retry_name)
                            if self._generate_retry_inventory(filename, retries):
                                display.display("\tto retry, use: --limit @%s\n" % filename)

                    self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)

                # if the last result wasn't zero, break out of the playbook file name loop
                if result != 0:
                    break

            if entrylist:
                return entrylist

        finally:
            if self._tqm is not None:
                self._tqm.cleanup()
            if self._loader:
                self._loader.cleanup_all_tmp_files()

        if self._options.syntax:
            display.display("No issues encountered")
            return result

        return result
예제 #52
0
    def test_noop(self):
        fake_loader = DictDataLoader({
            "test_play.yml":
            """
            - hosts: localhost
              gather_facts: no
              tasks:
              - name: simple_function
                debug:
                  msg : 'hello world'
                tags:
                 - functional

              - name: call_simple_function
                call_task:
                  name : "simple_function"

            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml',
                          loader=fake_loader,
                          variable_manager=mock_var_manager)

        inventory = MagicMock()
        inventory.hosts = {}
        hosts = []

        host = MagicMock()
        host.name = host.get_name.return_value = 'localhost'
        hosts.append(host)
        inventory.hosts[host.name] = host

        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        mock_var_manager._fact_cache['localhost'] = dict()

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        tqm = TaskQueueManager(
            inventory=inventory,
            variable_manager=mock_var_manager,
            loader=fake_loader,
            passwords=None,
            forks=5,
        )
        tqm._initialize_processes(3)
        strategy = StrategyModule(tqm)
        strategy._hosts_cache = [h.name for h in hosts]
        strategy._hosts_cache_all = [h.name for h in hosts]

        # implicit meta: flush_handlers
        hosts_left = strategy.get_hosts_left(itr)
        hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
        host1_task = hosts_tasks[0][1]
        host2_task = hosts_tasks[1][1]
예제 #53
0
    def test_play_iterator_nested_blocks(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              tasks:
              - block:
                - block:
                  - block:
                    - block:
                      - block:
                        - debug: msg="this is the first task"
                        - ping:
                      rescue:
                      - block:
                        - block:
                          - block:
                            - block:
                              - debug: msg="this is the rescue task"
                  always:
                  - block:
                    - block:
                      - block:
                        - block:
                          - debug: msg="this is the always task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # get the first task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the first task'))
        # fail the host
        itr.mark_host_failed(hosts[0])
        # get the resuce task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the rescue task'))
        # get the always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg='this is the always task'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)
예제 #54
0
    def test_play_iterator(self):
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        p = Playbook.load('test_play.yml', loader=fake_loader)

        hosts = []
        for i in range(0, 10):
            host  = MagicMock()
            host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)
예제 #55
0
    def test_play_iterator(self):
        #import epdb; epdb.st()
        fake_loader = DictDataLoader({
            "test_play.yml": """
            - hosts: all
              gather_facts: false
              roles:
              - test_role
              pre_tasks:
              - debug: msg="this is a pre_task"
              tasks:
              - debug: msg="this is a regular task"
              - block:
                - debug: msg="this is a block task"
                - block:
                  - debug: msg="this is a sub-block in a block"
                rescue:
                - debug: msg="this is a rescue task"
                - block:
                  - debug: msg="this is a sub-block in a rescue"
                always:
                - debug: msg="this is an always task"
                - block:
                  - debug: msg="this is a sub-block in an always"
              post_tasks:
              - debug: msg="this is a post_task"
            """,
            '/etc/ansible/roles/test_role/tasks/main.yml': """
            - debug: msg="this is a role task"
            """,
        })

        mock_var_manager = MagicMock()
        mock_var_manager._fact_cache = dict()
        mock_var_manager.get_vars.return_value = dict()

        p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)

        hosts = []
        for i in range(0, 10):
            host = MagicMock()
            host.name = host.get_name.return_value = 'host%02d' % i
            hosts.append(host)

        mock_var_manager._fact_cache['host00'] = dict()

        inventory = MagicMock()
        inventory.get_hosts.return_value = hosts
        inventory.filter_hosts.return_value = hosts

        play_context = PlayContext(play=p._entries[0])

        itr = PlayIterator(
            inventory=inventory,
            play=p._entries[0],
            play_context=play_context,
            variable_manager=mock_var_manager,
            all_vars=dict(),
        )

        # lookup up an original task
        target_task = p._entries[0].tasks[0].block[0]
        task_copy = target_task.copy(exclude_block=True)
        found_task = itr.get_original_task(hosts[0], task_copy)
        self.assertEqual(target_task, found_task)

        bad_task = Task()
        found_task = itr.get_original_task(hosts[0], bad_task)
        self.assertIsNone(found_task)

        # pre task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # role task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNotNone(task._role)
        # regular play task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertIsNone(task._role)
        # block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a block task"))
        # sub-block task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
        # mark the host failed
        itr.mark_host_failed(hosts[0])
        # block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a rescue task"))
        # sub-block rescue task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
        # block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is an always task"))
        # sub-block always task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # post task
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'debug')
        # implicit meta: flush_handlers
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNotNone(task)
        self.assertEqual(task.action, 'meta')
        # end of iteration
        (host_state, task) = itr.get_next_task_for_host(hosts[0])
        self.assertIsNone(task)

        # host 0 shouldn't be in the failed hosts, as the error
        # was handled by a rescue block
        failed_hosts = itr.get_failed_hosts()
        self.assertNotIn(hosts[0], failed_hosts)
예제 #56
0
    def load_data(self, ds, basedir, variable_manager=None, loader=None):
        '''
        Overrides the base load_data(), as we're actually going to return a new
        Playbook() object rather than a PlaybookInclude object
        '''

        # import here to avoid a dependency loop
        from ansible.playbook import Playbook
        from ansible.playbook.play import Play

        # first, we use the original parent method to correctly load the object
        # via the load_data/preprocess_data system we normally use for other
        # playbook objects
        new_obj = super(PlaybookInclude,
                        self).load_data(ds, variable_manager, loader)

        all_vars = self.vars.copy()
        if variable_manager:
            all_vars.update(variable_manager.get_vars())

        templar = Templar(loader=loader, variables=all_vars)

        # then we use the object to load a Playbook
        pb = Playbook(loader=loader)

        file_name = templar.template(new_obj.import_playbook)
        if not os.path.isabs(file_name):
            file_name = os.path.join(basedir, file_name)

        pb._load_playbook_data(file_name=file_name,
                               variable_manager=variable_manager,
                               vars=self.vars.copy())

        # finally, update each loaded playbook entry with any variables specified
        # on the included playbook and/or any tags which may have been set
        for entry in pb._entries:

            # conditional includes on a playbook need a marker to skip gathering
            if new_obj.when and isinstance(entry, Play):
                entry._included_conditional = new_obj.when[:]

            temp_vars = entry.vars.copy()
            temp_vars.update(new_obj.vars)
            param_tags = temp_vars.pop('tags', None)
            if param_tags is not None:
                entry.tags.extend(param_tags.split(','))
            entry.vars = temp_vars
            entry.tags = list(set(entry.tags).union(new_obj.tags))
            if entry._included_path is None:
                entry._included_path = os.path.dirname(file_name)

            # Check to see if we need to forward the conditionals on to the included
            # plays. If so, we can take a shortcut here and simply prepend them to
            # those attached to each block (if any)
            if new_obj.when:
                for task_block in (entry.pre_tasks + entry.roles +
                                   entry.tasks + entry.post_tasks):
                    task_block._attributes[
                        'when'] = new_obj.when[:] + task_block.when[:]

        return pb