def find_plugin(self, name, suffixes=None): ''' Find a plugin named name ''' if not suffixes: if self.class_name: suffixes = ['.py'] else: suffixes = ['.py', ''] potential_names = frozenset('%s%s' % (name, s) for s in suffixes) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] found = None for path in [ p for p in self._get_paths() if p not in self._searched_paths ]: if os.path.isdir(path): try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: d = Display() d.warning("Error accessing plugin paths: %s" % str(e)) for full_path in (f for f in full_paths if os.path.isfile(f)): for suffix in suffixes: if full_path.endswith(suffix): full_name = os.path.basename(full_path) break else: # Yes, this is a for-else: http://bit.ly/1ElPkyg continue if full_name not in self._plugin_path_cache: self._plugin_path_cache[full_name] = full_path self._searched_paths.add(path) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] # if nothing is found, try finding alias/deprecated if not name.startswith('_'): for alias_name in ('_%s' % n for n in potential_names): # We've already cached all the paths at this point if alias_name in self._plugin_path_cache: if not os.path.islink(self._plugin_path_cache[alias_name]): d = Display() d.warning('%s has been deprecated, which means ' 'it is kept for backwards compatibility ' 'but usage is discouraged. The module ' 'documentation details page may explain ' 'more about this rationale.' % name.lstrip('_')) return self._plugin_path_cache[alias_name] return None
def test_warning_no_color(capsys, mocker, warning_message): warning_message, expected_warning_message = warning_message mocker.patch('ansible.utils.color.ANSIBLE_COLOR', False) d = Display() d.warning(warning_message) out, err = capsys.readouterr() assert d._warns == {expected_warning_message: 1} assert err == expected_warning_message
def find_plugin(self, name, suffixes=None): ''' Find a plugin named name ''' if not suffixes: if self.class_name: suffixes = ['.py'] else: suffixes = ['.py', ''] potential_names = frozenset('%s%s' % (name, s) for s in suffixes) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] found = None for path in [p for p in self._get_paths() if p not in self._searched_paths]: if os.path.isdir(path): try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: d = Display() d.warning("Error accessing plugin paths: %s" % str(e)) for full_path in (f for f in full_paths if os.path.isfile(f)): for suffix in suffixes: if full_path.endswith(suffix): full_name = os.path.basename(full_path) break else: # Yes, this is a for-else: http://bit.ly/1ElPkyg continue if full_name not in self._plugin_path_cache: self._plugin_path_cache[full_name] = full_path self._searched_paths.add(path) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] # if nothing is found, try finding alias/deprecated if not name.startswith('_'): for alias_name in ('_%s' % n for n in potential_names): # We've already cached all the paths at this point if alias_name in self._plugin_path_cache: if not os.path.islink(self._plugin_path_cache[alias_name]): d = Display() d.warning('%s has been deprecated, which means ' 'it is kept for backwards compatibility ' 'but usage is discouraged. The module ' 'documentation details page may explain ' 'more about this rationale.' % name.lstrip('_')) return self._plugin_path_cache[alias_name] return None
def test_warning(capsys, mocker, warning_message): warning_message, expected_warning_message = warning_message mocker.patch('ansible.utils.color.ANSIBLE_COLOR', True) mocker.patch('ansible.utils.color.parsecolor', return_value=u'1;35') # value for 'bright purple' d = Display() d.warning(warning_message) out, err = capsys.readouterr() assert d._warns == {expected_warning_message: 1} assert err == '\x1b[1;35m{0}\x1b[0m\n\x1b[1;35m\x1b[0m'.format( expected_warning_message.rstrip('\n'))
def find_plugin(self, name, suffixes=None): ''' Find a plugin named name ''' if not suffixes: if self.class_name: suffixes = ['.py'] else: suffixes = ['.py', ''] potential_names = frozenset('%s%s' % (name, s) for s in suffixes) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] found = None for path in [ p for p in self._get_paths() if p not in self._searched_paths ]: if os.path.isdir(path): try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: d = Display() d.warning("Error accessing plugin paths: %s" % str(e)) for full_path in (f for f in full_paths if os.path.isfile(f)): for suffix in suffixes: if full_path.endswith(suffix): full_name = os.path.basename(full_path) break else: # Yes, this is a for-else: http://bit.ly/1ElPkyg continue if full_name not in self._plugin_path_cache: self._plugin_path_cache[full_name] = full_path self._searched_paths.add(path) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] # if nothing is found, try finding alias/deprecated if not name.startswith('_'): for alias_name in ('_%s' % n for n in potential_names): # We've already cached all the paths at this point if alias_name in self._plugin_path_cache: return self._plugin_path_cache[alias_name] return None
def find_plugin(self, name, suffixes=None): ''' Find a plugin named name ''' if not suffixes: if self.class_name: suffixes = ['.py'] else: suffixes = ['.py', ''] potential_names = frozenset('%s%s' % (name, s) for s in suffixes) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] found = None for path in [p for p in self._get_paths() if p not in self._searched_paths]: if os.path.isdir(path): try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: d = Display() d.warning("Error accessing plugin paths: %s" % str(e)) for full_path in (f for f in full_paths if os.path.isfile(f)): for suffix in suffixes: if full_path.endswith(suffix): full_name = os.path.basename(full_path) break else: # Yes, this is a for-else: http://bit.ly/1ElPkyg continue if full_name not in self._plugin_path_cache: self._plugin_path_cache[full_name] = full_path self._searched_paths.add(path) for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] # if nothing is found, try finding alias/deprecated if not name.startswith('_'): for alias_name in ('_%s' % n for n in potential_names): # We've already cached all the paths at this point if alias_name in self._plugin_path_cache: return self._plugin_path_cache[alias_name] return None
JINJA2_OVERRIDE = '#jinja2:' from jinja2 import __version__ as j2_version USE_JINJA2_NATIVE = False if C.DEFAULT_JINJA2_NATIVE: try: from jinja2.nativetypes import NativeEnvironment as Environment from ansible.template.native_helpers import ansible_native_concat as j2_concat from ansible.template.native_helpers import NativeJinjaText USE_JINJA2_NATIVE = True except ImportError: from jinja2 import Environment from jinja2.utils import concat as j2_concat display.warning( 'jinja2_native requires Jinja 2.10 and above. ' 'Version detected: %s. Falling back to default.' % j2_version ) else: from jinja2 import Environment from jinja2.utils import concat as j2_concat JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin')) JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end')) RANGE_TYPE = type(range(0)) def generate_ansible_template_vars(path, dest_path=None): b_path = to_bytes(path)
class Grapher(object): """ Main class to make the graph """ DEFAULT_GRAPH_ATTR = { "ratio": "fill", "rankdir": "LR", "concentrate": "true", "ordering": "in" } DEFAULT_EDGE_ATTR = {"sep": "10", "esep": "5"} def __init__(self, data_loader, inventory_manager, variable_manager, playbook_filename, options, graph=None): """ Main grapher responsible to parse the playbook and draw graph :param data_loader: :type data_loader: ansible.parsing.dataloader.DataLoader :param inventory_manager: :type inventory_manager: ansible.inventory.manager.InventoryManager :param variable_manager: :type variable_manager: ansible.vars.manager.VariableManager :param options Command line options :type options: optparse.Values :param playbook_filename: :type playbook_filename: str :param graph: :type graph: Digraph """ self.options = options self.variable_manager = variable_manager self.inventory_manager = inventory_manager self.data_loader = data_loader self.playbook_filename = playbook_filename self.options.output_filename = self.options.output_filename self.rendered_file_path = None self.display = Display(verbosity=options.verbosity) if self.options.tags is None: self.options.tags = ["all"] if self.options.skip_tags is None: self.options.skip_tags = [] self.graph_representation = GraphRepresentation() self.playbook = Playbook.load(self.playbook_filename, loader=self.data_loader, variable_manager=self.variable_manager) if graph is None: self.graph = CustomDigrah(edge_attr=self.DEFAULT_EDGE_ATTR, graph_attr=self.DEFAULT_GRAPH_ATTR, format="svg") def template(self, data, variables, fail_on_undefined=False): """ Template the data using Jinja. Return data if an error occurs during the templating :param fail_on_undefined: :type fail_on_undefined: bool :param data: :type data: Union[str, ansible.parsing.yaml.objects.AnsibleUnicode] :param variables: :type variables: dict :return: """ try: templar = Templar(loader=self.data_loader, variables=variables) return templar.template(data, fail_on_undefined=fail_on_undefined) except AnsibleError as ansible_error: # Sometime we need to export if fail_on_undefined: raise self.display.warning(ansible_error) return data def make_graph(self): """ Loop through the playbook and make the graph. The graph is drawn following this order (https://docs.ansible.com/ansible/2.4/playbooks_reuse_roles.html#using-roles) for each play: draw pre_tasks draw roles if include_role_tasks draw role_tasks draw tasks draw post_tasks :return: :rtype: """ # the root node self.graph.node(self.playbook_filename, style="dotted", id="root_node") # loop through the plays for play_counter, play in enumerate(self.playbook.get_plays(), 1): # the load basedir is relative to the playbook path if play._included_path is not None: self.data_loader.set_basedir(play._included_path) else: self.data_loader.set_basedir(self.playbook._basedir) self.display.vvv("Loader basedir set to {}".format( self.data_loader.get_basedir())) play_vars = self.variable_manager.get_vars(play) play_hosts = [ h.get_name() for h in self.inventory_manager.get_hosts( self.template(play.hosts, play_vars)) ] play_name = "Play #{}: {} ({})".format(play_counter, clean_name(play.get_name()), len(play_hosts)) play_name = self.template(play_name, play_vars) self.display.banner("Graphing " + play_name) play_id = "play_" + str(uuid.uuid4()) self.graph_representation.add_node(play_id) with self.graph.subgraph(name=play_name) as play_subgraph: color, play_font_color = get_play_colors(play) # play node play_subgraph.node(play_name, id=play_id, style="filled", shape="box", color=color, fontcolor=play_font_color, tooltip=" ".join(play_hosts)) # edge from root node to plays play_edge_id = "edge_" + str(uuid.uuid4()) play_subgraph.edge(self.playbook_filename, play_name, id=play_edge_id, style="bold", label=str(play_counter), color=color, fontcolor=color) # loop through the pre_tasks self.display.v("Graphing pre_tasks...") nb_pre_tasks = 0 for pre_task_block in play.pre_tasks: nb_pre_tasks = self._include_tasks_in_blocks( current_play=play, graph=play_subgraph, parent_node_name=play_name, parent_node_id=play_id, block=pre_task_block, color=color, current_counter=nb_pre_tasks, play_vars=play_vars, node_name_prefix="[pre_task] ") # loop through the roles self.display.v("Graphing roles...") role_number = 0 for role in play.get_roles(): # Don't insert tasks from ``import/include_role``, preventing # duplicate graphing if role.from_include: continue role_number += 1 role_name = "[role] " + clean_name(role.get_name()) # the role object doesn't inherit the tags from the play. So we add it manually role.tags = role.tags + play.tags role_not_tagged = "" if not role.evaluate_tags(only_tags=self.options.tags, skip_tags=self.options.skip_tags, all_vars=play_vars): role_not_tagged = NOT_TAGGED with self.graph.subgraph(name=role_name, node_attr={}) as role_subgraph: current_counter = role_number + nb_pre_tasks role_id = "role_" + str(uuid.uuid4()) + role_not_tagged role_subgraph.node(role_name, id=role_id) edge_id = "edge_" + str(uuid.uuid4()) + role_not_tagged # edge from play to role role_subgraph.edge(play_name, role_name, label=str(current_counter), color=color, fontcolor=color, id=edge_id) self.graph_representation.add_link(play_id, edge_id) self.graph_representation.add_link(edge_id, role_id) # loop through the tasks of the roles if self.options.include_role_tasks: role_tasks_counter = 0 for block in role.compile(play): role_tasks_counter = self._include_tasks_in_blocks( current_play=play, graph=role_subgraph, parent_node_name=role_name, parent_node_id=role_id, block=block, color=color, play_vars=play_vars, current_counter=role_tasks_counter, node_name_prefix="[task] ") role_tasks_counter += 1 self.display.v( "{} roles added to the graph".format(role_number)) # loop through the tasks self.display.v("Graphing tasks...") nb_tasks = 0 for task_block in play.tasks: nb_tasks = self._include_tasks_in_blocks( current_play=play, graph=play_subgraph, parent_node_name=play_name, parent_node_id=play_id, block=task_block, color=color, current_counter=role_number + nb_pre_tasks, play_vars=play_vars, node_name_prefix="[task] ") # loop through the post_tasks self.display.v("Graphing post_tasks...") for post_task_block in play.post_tasks: self._include_tasks_in_blocks( current_play=play, graph=play_subgraph, parent_node_name=play_name, parent_node_id=play_id, block=post_task_block, color=color, current_counter=nb_tasks, play_vars=play_vars, node_name_prefix="[post_task] ") self.display.banner("Done graphing {}".format(play_name)) self.display.display("") # just an empty line # moving to the next play def render_graph(self): """ Render the graph :return: The rendered file path :rtype: str """ self.rendered_file_path = self.graph.render( cleanup=not self.options.save_dot_file, filename=self.options.output_filename) if self.options.save_dot_file: # add .gv extension. The render doesn't add an extension final_name = self.options.output_filename + ".dot" os.rename(self.options.output_filename, final_name) self.display.display( "Graphviz dot file has been exported to {}".format(final_name)) return self.rendered_file_path def post_process_svg(self): """ Post process the rendered svg :return The post processed file path :rtype: str :return: """ post_processor = PostProcessor(svg_path=self.rendered_file_path) post_processor.post_process( graph_representation=self.graph_representation) post_processor.write() self.display.display("The graph has been exported to {}".format( self.rendered_file_path)) return self.rendered_file_path def _include_tasks_in_blocks(self, current_play, graph, parent_node_name, parent_node_id, block, color, current_counter, play_vars=None, node_name_prefix=""): """ Recursively read all the tasks of the block and add it to the graph FIXME: This function needs some refactoring. Thinking of a BlockGrapher to handle this :param current_play: :type current_play: ansible.playbook.play.Play :param graph: :type graph: :param parent_node_name: :type parent_node_name: str :param parent_node_id: :type parent_node_id: str :param block: :type block: Union[Block,TaskInclude] :param color: :type color: str :param current_counter: :type current_counter: int :param play_vars: :type play_vars: dict :param node_name_prefix: :type node_name_prefix: str :return: :rtype: """ loop_counter = current_counter # loop through the tasks for counter, task_or_block in enumerate(block.block, 1): if isinstance(task_or_block, Block): loop_counter = self._include_tasks_in_blocks( current_play=current_play, graph=graph, parent_node_name=parent_node_name, parent_node_id=parent_node_id, block=task_or_block, color=color, current_counter=loop_counter, play_vars=play_vars, node_name_prefix=node_name_prefix) elif isinstance( task_or_block, TaskInclude ): # include, include_tasks, include_role are dynamic # So we need to process it explicitly because Ansible does it during th execution of the playbook task_vars = self.variable_manager.get_vars(play=current_play, task=task_or_block) if isinstance(task_or_block, IncludeRole): self.display.v( "An 'include_role' found. Including tasks from '{}'". format(task_or_block.args["name"])) # here we have an include_role. The class IncludeRole is a subclass of TaskInclude. # We do this because the management of an include_role is different. # See :func:`~ansible.playbook.included_file.IncludedFile.process_include_results` from line 155 my_blocks, _ = task_or_block.get_block_list( play=current_play, loader=self.data_loader, variable_manager=self.variable_manager) else: self.display.v( "An 'include_tasks' found. Including tasks from '{}'". format(task_or_block.get_name())) templar = Templar(loader=self.data_loader, variables=task_vars) try: include_file = handle_include_path( original_task=task_or_block, loader=self.data_loader, templar=templar) except AnsibleUndefinedVariable as e: # TODO: mark this task with some special shape or color self.display.warning( "Unable to translate the include task '{}' due to an undefined variable: {}. " "Some variables are available only during the real execution." .format(task_or_block.get_name(), str(e))) loop_counter += 1 self._include_task(task_or_block, loop_counter, task_vars, graph, node_name_prefix, color, parent_node_id, parent_node_name) continue data = self.data_loader.load_from_file(include_file) if data is None: self.display.warning( "file %s is empty and had no tasks to include" % include_file) continue elif not isinstance(data, list): raise AnsibleParserError( "included task files must contain a list of tasks", obj=data) # get the blocks from the include_tasks my_blocks = load_list_of_blocks( data, play=current_play, variable_manager=self.variable_manager, role=task_or_block._role, loader=self.data_loader, parent_block=task_or_block) for b in my_blocks: # loop through the blocks inside the included tasks or role loop_counter = self._include_tasks_in_blocks( current_play=current_play, graph=graph, parent_node_name=parent_node_name, parent_node_id=parent_node_id, block=b, color=color, current_counter=loop_counter, play_vars=task_vars, node_name_prefix=node_name_prefix) else: # check if this task comes from a role and we dont want to include role's task if has_role_parent( task_or_block) and not self.options.include_role_tasks: # skip role's task self.display.vv( "The task '{}' has a role as parent and include_role_tasks is false. " "It will be skipped.".format(task_or_block.get_name())) continue self._include_task(task_or_block=task_or_block, loop_counter=loop_counter + 1, play_vars=play_vars, graph=graph, node_name_prefix=node_name_prefix, color=color, parent_node_id=parent_node_id, parent_node_name=parent_node_name) loop_counter += 1 return loop_counter def _include_task(self, task_or_block, loop_counter, play_vars, graph, node_name_prefix, color, parent_node_id, parent_node_name): """ Include the task in the graph :return: :rtype: """ self.display.vv("Adding the task '{}' to the graph".format( task_or_block.get_name())) # check if the task should be included tagged = '' if not task_or_block.evaluate_tags(only_tags=self.options.tags, skip_tags=self.options.skip_tags, all_vars=play_vars): self.display.vv( "The task '{}' should not be executed. It will be marked as NOT_TAGGED" .format(task_or_block.get_name())) tagged = NOT_TAGGED task_edge_label = str(loop_counter) if len(task_or_block.when) > 0: when = "".join(map(str, task_or_block.when)) task_edge_label += " [when: " + when + "]" task_name = clean_name( node_name_prefix + self.template(task_or_block.get_name(), play_vars)) # get prefix id from node_name id_prefix = node_name_prefix.replace("[", "").replace("]", "").replace(" ", "_") task_id = id_prefix + str(uuid.uuid4()) + tagged edge_id = "edge_" + str(uuid.uuid4()) + tagged graph.node(task_name, shape="octagon", id=task_id) graph.edge(parent_node_name, task_name, label=task_edge_label, color=color, fontcolor=color, style="bold", id=edge_id) self.graph_representation.add_link(parent_node_id, edge_id) self.graph_representation.add_link(edge_id, task_id)
def defined(value, test_value=None, var_type=None, fail_action=None, var_name=None, run_tests=False): """ defined - Ansible test plugin to test if a variable is defined and not none Arista.avd.defined will test value if defined and is not none and return true or false. If test_value is supplied, the value must also pass == test_value to return true. If var_type is supplied, the value must also be of the specified class/type If fail_action is 'warning' a warning will be emitted on failure. If fail_action is 'error' an error will be emitted on failure and the task will fail. If var_name is supplied it will be used in the warning and error messages to ease troubleshooting. Examples: 1. Test if var is defined and not none: {% if spanning_tree is arista.avd.defined %} ... {% endif %} 2. Test if variable is defined, not none and has value "something" {% if extremely_long_variable_name is arista.avd.defined("something") %} ... {% endif %} 3. Test if variable is defined and of not print a warning message with the variable name {% if my_dict.my_list[12].my_var is arista.avd.defined(fail_action='warning', var_name='my_dict.my_list[12].my_var' %} Parameters ---------- value : any Value to test from ansible test_value : any, optional Value to test in addition of defined and not none, by default None var_type : ['float', 'int', 'str', 'list', 'dict', 'tuple', 'bool'], optional Type or Class to test for fail_action : ['warning', 'error'], optional Optional action if test fails to emit a Warning or Error var_name : <string>, optional Optional string to use as variable name in warning or error messages Returns ------- boolean True if variable matches criteria, False in other cases. """ display = Display() if isinstance(value, Undefined) or value is None: # Invalid value - return false if str(fail_action).lower() == 'warning': display._warns = {} if var_name is not None: display.warning( f"{var_name} was expected but not set. Output may be incorrect or incomplete!" ) else: display.warning( "A variable was expected but not set. Output may be incorrect or incomplete!" ) elif str(fail_action).lower() == 'error': if var_name is not None: raise AnsibleError(f"{var_name} was expected but not set!") else: raise AnsibleError("A variable was expected but not set!") if run_tests: return False, display._warns return False elif test_value is not None and value != test_value: # Valid value but not matching the optional argument if str(fail_action).lower() == 'warning': display._warns = {} if var_name is not None: display.warning( f"{var_name} was set to {value} but we expected {test_value}. Output may be incorrect or incomplete!" ) else: display.warning( f"A variable was set to {value} but we expected {test_value}. Output may be incorrect or incomplete!" ) elif str(fail_action).lower() == 'error': if var_name is not None: raise AnsibleError( f"{var_name} was set to {value} but we expected {test_value}!" ) else: raise AnsibleError( f"A variable was set to {value} but we expected {test_value}!" ) if run_tests: return False, display._warns return False elif str(var_type).lower() in [ 'float', 'int', 'str', 'list', 'dict', 'tuple', 'bool' ] and str(var_type).lower() != type(value).__name__: # Invalid class - return false if str(fail_action).lower() == 'warning': display._warns = {} if var_name is not None: display.warning( f"{var_name} was a {type(value).__name__} but we expected a {str(var_type).lower()}. Output may be incorrect or incomplete!" ) else: display.warning( f"A variable was a {type(value).__name__} but we expected a {str(var_type).lower()}. Output may be incorrect or incomplete!" ) elif str(fail_action).lower() == 'error': if var_name is not None: raise AnsibleError( f"{var_name} was a {type(value).__name__} but we expected a {str(var_type).lower()}!" ) else: raise AnsibleError( f"A variable was a {type(value).__name__} but we expected a {str(var_type).lower()}!" ) if run_tests: return False, display._warns return False else: # Valid value and is matching optional argument if provided - return true return True
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): NAME = "oci" LIFECYCLE_ACTIVE_STATE = "ACTIVE" LIFECYCLE_RUNNING_STATE = "RUNNING" LIFECYCLE_ATTACHED_STATE = "ATTACHED" def __init__(self): super(InventoryModule, self).__init__() self.inventory = None self.config = {} self.compartments = None self._region_subscriptions = None self.regions = {} self.params = { "config_file": os.path.join(os.path.expanduser("~"), ".oci", "config"), "profile": "DEFAULT", "user": None, "fingerprint": None, "key_file": None, "tenancy": None, "region": None, "pass_phrase": None, # other options "compartment_ocid": None, "compartment_name": None, "parent_compartment_ocid": None, "fetch_hosts_from_subcompartments": False, "debug": False, "hostname_format": "public_ip", "sanitize_names": True, "replace_dash_in_names": False, "max_thread_count": 50, "freeform_tags": None, "defined_tags": None, "regions": None, "strict_hostname_checking": "no", "filters": None, } self.group_prefix = "oci_" self.display = Display() def _get_config_file(self): """ :param config_data: contents of the inventory config file """ # Preference order: .oci.yml > environment variable > settings from config file. if self.get_option("config_file") is not None: self.params["config_file"] = os.path.expanduser( self.get_option("config_file") ) elif "OCI_CONFIG_FILE" in os.environ: self.params["config_file"] = os.path.expanduser( os.path.expandvars(os.environ.get("OCI_CONFIG_FILE")) ) if self.get_option("config_profile") is not None: self.params["profile"] = self.get_option("config_profile") elif "OCI_CONFIG_PROFILE" in os.environ: self.params["profile"] = os.environ.get("OCI_CONFIG_PROFILE") def read_config(self): self._get_config_file() # Read values from config file if os.path.isfile(to_bytes(self.params["config_file"])): self.config = oci.config.from_file( file_location=self.params["config_file"], profile_name=self.params["profile"], ) self.config["additional_user_agent"] = ( oci_config_utils.inventory_agent_name + oci_common_utils.__version__ ) self.log(self.config["additional_user_agent"]) for setting in self.config: self.params[setting] = self.config[setting] def read_settings_config(self, boolean_options, dict_options): if self.settings_config.has_section("oci"): for option in self.settings_config.options("oci"): if option in boolean_options: self.params[option] = self.settings_config.getboolean("oci", option) elif option in dict_options: self.params[option] = json.loads( self.settings_config.get("oci", option) ) else: self.params[option] = self.settings_config.get("oci", option) def log(self, *args, **kwargs): if self.params["debug"]: self.display.warning(*args, **kwargs) pass def setup_clients(self, regions): """ :param regions: A list of regions to create clients """ self.regions = regions self.identity_client = self.create_service_client(IdentityClient) self._compute_clients = dict( (region, self.create_service_client(ComputeClient, region=region)) for region in self.regions ) self._virtual_nw_clients = dict( (region, self.create_service_client(VirtualNetworkClient, region=region)) for region in self.regions ) def create_service_client(self, service_client_class, region=None): if not region: region = self.params["region"] params = dict(self.params, region=region) kwargs = {} if self._is_instance_principal_auth(): kwargs["signer"] = self.create_instance_principal_signer() # Create service client class with the signer. client = service_client_class(params, **kwargs) return client def _is_instance_principal_auth(self): # check if auth is set to `instance_principal`. return self.get_option("instance_principal_authentication") @staticmethod def create_instance_principal_signer(): try: signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() except Exception as ex: raise Exception( "Failed retrieving certificates from localhost. Instance principal based authentication is only" "possible from within OCI compute instances. Exception: {0}".format( str(ex) ) ) return signer @contextmanager def pool(self, **kwargs): pool = ThreadPool(**kwargs) try: yield pool finally: pool.close() # wait for all the instances to be processed pool.join() # terminate the pool pool.terminate() def _get_instances_by_region(self, regions): """ :param regions: a list of regions in which to describe instances :return A list of instance dictionaries """ self.setup_clients(regions) self.display.warning("Building inventory.") # Compartments(including the root compartment) from which the instances are to be retrieved. self.compartments = dict( (compartment.id, compartment) for compartment in self.get_compartments( compartment_ocid=self.params["compartment_ocid"], parent_compartment_ocid=self.params["parent_compartment_ocid"], compartment_name=self.params["compartment_name"], ) ) if not self.compartments: self.display.warning("No compartments matching the criteria.") return all_instances = self.get_instances(self.compartments) instance_inventories = [ self.build_inventory_for_instance(instance, region) for region in all_instances for instance in all_instances[region] ] return instance_inventories def get_instances(self, compartment_ocids): """Get and return instances from all the specified compartments and regions. :param compartment_ocids: List of compartment ocid's to fetch the instances from :return: dict with region as key and list of instances of the region as value """ instances = defaultdict(list) if self.get_option("enable_parallel_processing"): for region in self.regions: num_threads = min( len(compartment_ocids), self.params["max_thread_count"] ) self.display.warning( "Parallel processing enabled. Getting instances from {0} in {1} threads.".format( region, num_threads ) ) with self.pool(processes=num_threads) as pool: get_filtered_instances_for_region = partial( self.get_filtered_instances, region=region ) lists_of_instances = pool.map( get_filtered_instances_for_region, compartment_ocids ) for sublist in lists_of_instances: instances[region].extend(sublist) else: for region in self.regions: for compartment_ocid in compartment_ocids: instances[region].extend( self.get_filtered_instances(compartment_ocid, region) ) return instances def get_compartments( self, compartment_ocid=None, parent_compartment_ocid=None, compartment_name=None, fetch_hosts_from_subcompartments=True, ): """ Get the compartments based on the parameters passed. When compartment_name is None, all the compartments including the root compartment is returned. When compartment_name is passed, the compartment with that name and its hierarchy of compartments are returned if fetch_hosts_from_subcompartments is true. The tenancy is returned when compartment_name is the tenancy name. :param str compartment_ocid: (optional) OCID of the compartment. If None, root compartment is assumed to be parent. :param str parent_compartment_ocid: (optional) OCID of the parent compartment. If None, root compartment is assumed to be parent. :param str compartment_name: (optional) Name of the compartment. If None and :attr:`compartment_ocid` is not set, all the compartments including the root compartment are returned. :param str fetch_hosts_from_subcompartments: (optional) Only applicable when compartment_name is specified. When set to true, the entire hierarchy of compartments of the given compartment is returned. :raises ServiceError: When the Service returned an Error response :raises MaximumWaitTimeExceededError: When maximum wait time is exceeded while invoking target_fn :return: list of :class:`~oci.identity.models.Compartment` """ if compartment_ocid: try: compartment_with_ocid = oci_utils.call_with_backoff( self.identity_client.get_compartment, compartment_id=compartment_ocid, ).data except ServiceError as se: if se.status == 404: raise Exception( "Compartment with OCID {0} either does not exist or " "you do not have permission to access it.".format( compartment_ocid ) ) else: if not fetch_hosts_from_subcompartments: return [compartment_with_ocid] return self.get_sub_compartments(compartment_with_ocid) if not self.params["tenancy"]: raise Exception( "Tenancy OCID required to get the compartments in the tenancy." ) try: tenancy = oci_utils.call_with_backoff( self.identity_client.get_compartment, compartment_id=self.params["tenancy"], ).data except ServiceError as se: if se.status == 404: raise Exception( "Either tenancy ocid is invalid or need inspect permission on root compartment to get the " "compartments in the tenancy." ) all_compartments = [tenancy] + [ compartment for compartment in oci_utils.list_all_resources( target_fn=self.identity_client.list_compartments, compartment_id=self.params["tenancy"], compartment_id_in_subtree=True, ) if self.filter_resource( compartment, lifecycle_state=self.LIFECYCLE_ACTIVE_STATE ) ] # return all the compartments if compartment_name is not passed if not compartment_name: return all_compartments if compartment_name == tenancy.name: # return all the compartments when fetch_hosts_from_subcompartments is true if fetch_hosts_from_subcompartments: return all_compartments else: return [tenancy] if not parent_compartment_ocid: parent_compartment_ocid = tenancy.id compartment_with_name = None for compartment in all_compartments: if ( compartment.name == compartment_name and compartment.compartment_id == parent_compartment_ocid ): compartment_with_name = compartment break if not compartment_with_name: raise Exception( "Compartment with name {0} not found.".format(compartment_name) ) if not fetch_hosts_from_subcompartments: return [compartment_with_name] return self.get_sub_compartments(compartment_with_name) @staticmethod def filter_resource(resource, **kwargs): for key, val in six.iteritems(kwargs): if getattr(resource, key, None) != val: return False return True def get_sub_compartments(self, root): # OCI SDK does not support fetching sub-compartments for non root compartments # So traverse the compartment tree to fetch all the sub compartments compartments = [] queue = deque() queue.append(root) while len(queue) > 0: parent_compartment = queue.popleft() compartments.append(parent_compartment) child_compartments = [ compartment for compartment in oci_utils.list_all_resources( target_fn=self.identity_client.list_compartments, compartment_id=parent_compartment.id, ) if self.filter_resource( compartment, lifecycle_state=self.LIFECYCLE_ACTIVE_STATE ) ] for child_compartment in child_compartments: queue.append(child_compartment) return compartments def build_inventory_for_instance(self, instance, region): """Build and return inventory for an instance""" try: instance_inventory = {} compute_client = self.get_compute_client_for_region(region) virtual_nw_client = self.get_virtual_nw_client_for_region(region) compartment = self.compartments[instance.compartment_id] instance_vars = to_dict(instance) common_groups = set(["all_hosts"]) # Group by availability domain ad = self.sanitize(instance.availability_domain) common_groups.add(ad) # Group by compartments compartment_name = self.sanitize(compartment.name) common_groups.add(compartment_name) # Group by region region_grp = self.sanitize("region_" + region) common_groups.add(region_grp) # Group by freeform tags tag_key=value for key in instance.freeform_tags: tag_group_name = self.sanitize( "tag_" + key + "=" + instance.freeform_tags[key] ) common_groups.add(tag_group_name) # Group by defined tags for namespace in instance.defined_tags: for key in instance.defined_tags[namespace]: defined_tag_group_name = self.sanitize( namespace + "#" + key + "=" + instance.defined_tags[namespace][key] ) common_groups.add(defined_tag_group_name) vnic_attachments = [ vnic_attachment for vnic_attachment in oci_utils.list_all_resources( target_fn=compute_client.list_vnic_attachments, compartment_id=compartment.id, instance_id=instance.id, ) if self.filter_resource( vnic_attachment, lifecycle_state=self.LIFECYCLE_ATTACHED_STATE ) ] for vnic_attachment in vnic_attachments: vnic = oci_utils.call_with_backoff( virtual_nw_client.get_vnic, vnic_id=vnic_attachment.vnic_id ).data self.log( "VNIC {0} is attached to instance {1}.".format( vnic.id, vnic_attachment.instance_id ) ) subnet = oci_utils.call_with_backoff( virtual_nw_client.get_subnet, subnet_id=vnic.subnet_id ).data if instance_vars.get("id") == vnic_attachment.instance_id: instance_vars.update({"vcn_id": subnet.vcn_id}) instance_vars.update({"vnic_id": vnic.id}) instance_vars.update({"subnet_id": vnic.subnet_id}) instance_vars.update({"public_ip": vnic.public_ip}) instance_vars.update({"private_ip": vnic.private_ip}) host_name = self.get_host_name(vnic, region=region) # Skip host which is not addressable using hostname_format if not host_name: if self.params["strict_hostname_checking"] == "yes": raise Exception( "Instance with OCID: {0} does not have a valid hostname.".format( vnic_attachment.instance_id ) ) self.log( "Skipped instance with OCID:" + vnic_attachment.instance_id ) return None host_name = self.sanitize(host_name) groups = set(common_groups) self.display.warning( "Creating inventory for host {0}.".format(host_name) ) self.create_instance_inventory_for_host( instance_inventory, host_name, vars=instance_vars, groups=groups ) self.log("Final inventory for {0}.".format(str(instance_inventory))) return instance_inventory except ServiceError as ex: if ex.status == 401: self.log(ex) raise self.log(ex) def create_instance_inventory_for_host( self, instance_inventory, host_name, vars, groups ): instance_inventory.setdefault(host_name, {"groups": {}, "vars": {}}) instance_inventory[host_name]["vars"] = vars for group in groups: instance_inventory[host_name]["groups"].setdefault(group, {"children": []}) return instance_inventory def sanitize(self, word): # regex represents an invalid non-alphanumeric character except UNDERSCORE, HASH, EQUALS and DOT regex = r"[^A-Za-z0-9_#=." if self.params["sanitize_names"]: if not self.params["replace_dash_in_names"]: # Add DASH as a valid character in regex. regex += r"-" # Replace all invalid characters with UNDERSCORE return re.sub(regex + "]", "_", word) return word def get_filtered_instances(self, compartment_ocid, region): try: compute_client = self.get_compute_client_for_region(region) instances = oci_utils.list_all_resources( target_fn=compute_client.list_instances, compartment_id=compartment_ocid, lifecycle_state="RUNNING", ) self.log( "All RUNNING instances from compartment {0}:{1}".format( compartment_ocid, instances ) ) # Data is cached so filter using all of the data not using the API if "display_name" in self.filters and self.filters["display_name"]: instances = [ instance for instance in instances if (instance.display_name == self.filters["display_name"]) ] if ( "availability_domain" in self.filters and self.filters["availability_domain"] ): instances = [ instance for instance in instances if ( instance.availability_domain == self.filters["availability_domain"] ) ] if "lifecycle_state" in self.filters and self.filters["lifecycle_state"]: instances = [ instance for instance in instances if (instance.lifecycle_state == self.filters["lifecycle_state"]) ] if "freeform_tags" in self.filters and self.filters["freeform_tags"]: instances = [ instance for instance in instances if all( instance.freeform_tags.get(key) == value for key, value in six.iteritems(self.filters["freeform_tags"]) ) ] self.display.warning( "Instances in compartment {0} which match all the freeform tags: {1}".format( compartment_ocid, instances ) ) if "defined_tags" in self.filters and self.filters["defined_tags"]: instances = [ instance for instance in instances if all( (instance.defined_tags.get(namespace, {})).get(key) == value for namespace in self.filters["defined_tags"] for key, value in six.iteritems( self.filters["defined_tags"][namespace] ) ) ] self.display.warning( "Instances in compartment {0} which match all the freeform & defined tags: {1}".format( compartment_ocid, instances ) ) return instances except ServiceError as ex: if ex.status == 401: self.display.warning(ex) raise self.display.warning(ex) return [] def get_compute_client_for_region(self, region): if region not in self._compute_clients: raise ValueError( "Could not fetch the compute client for region {0}.".format(region) ) return self._compute_clients[region] def get_virtual_nw_client_for_region(self, region): if region not in self._compute_clients: raise ValueError( "Could not fetch the virtual network for region {0}.".format(region) ) return self._virtual_nw_clients[region] def get_host_name(self, vnic, region): virtual_nw_client = self.get_virtual_nw_client_for_region(region) if self.params["hostname_format"] == "fqdn": subnet = oci_utils.call_with_backoff( virtual_nw_client.get_subnet, subnet_id=vnic.subnet_id ).data vcn = oci_utils.call_with_backoff( virtual_nw_client.get_vcn, vcn_id=subnet.vcn_id ).data oraclevcn_domain_name = ".oraclevcn.com" if not (vnic.hostname_label or subnet.dns_label or vcn.dns_label): return None fqdn = ( vnic.hostname_label + "." + subnet.dns_label + "." + vcn.dns_label + oraclevcn_domain_name ) self.display.warning("FQDN for VNIC: {0} is {1}.".format(vnic.id, fqdn)) return fqdn elif self.params["hostname_format"] == "private_ip": self.display.warning( "Private IP for VNIC: {0} is {1}.".format(vnic.id, vnic.private_ip) ) return vnic.private_ip return vnic.public_ip def _query(self, regions): """ :param regions: a list of regions to query """ return self._get_instances_by_region(regions) def _populate(self, instance_inventories, hostnames): for instance_inventory in instance_inventories: if instance_inventory: for host_name, host_inventory in six.iteritems(instance_inventory): if not hostnames or host_name in hostnames: for group in host_inventory["groups"]: self.inventory.add_group(group) self.inventory.add_host(host_name, group=group) self.inventory.set_variable( host_name, host_inventory["vars"]["display_name"], host_inventory["vars"], ) self.inventory.add_child("all", host_name) self._set_composite_vars( self.get_option("compose"), host_inventory["vars"], host_name, ) self._add_host_to_composed_groups( self.get_option("groups"), host_inventory["vars"], host_name, ) self._add_host_to_keyed_groups( self.get_option("keyed_groups"), host_inventory["vars"], host_name, ) def verify_file(self, path): """ :param loader: an ansible.parsing.dataloader.DataLoader object :param path: the path to the inventory config file :return the contents of the config file """ if super(InventoryModule, self).verify_file(path): if path.endswith(".oci.yml") or path.endswith(".oci.yaml"): return True def _get_query_options(self, config_data): """ :param config_data: contents of the inventory config file :return A list of regions to query a list of possible hostnames """ options = { "regions": {"type_to_be": list, "value": config_data.get("regions", [])}, "compartments": { "type_to_be": list, "value": config_data.get("compartments", []), }, "filters": {"type_to_be": list, "value": config_data.get("filters", [])}, "hostnames": { "type_to_be": list, "value": config_data.get("hostnames", []), }, } # validate the options for name in options: options[name]["value"] = self._validate_option( name, options[name]["type_to_be"], options[name]["value"] ) filters = dict((key, d[key]) for d in options["filters"]["value"] for key in d) self.filters = filters for item in options["compartments"]["value"]: self.display.warning(" --- compartments item: {0} ".format(item)) if "compartment_ocid" in item: self.params["compartment_ocid"] = item["compartment_ocid"] if "parent_compartment_ocid" in item: self.params["parent_compartment_ocid"] = item["parent_compartment_ocid"] if "compartment_name" in item: self.params["compartment_name"] = item["compartment_name"] regions = options["regions"]["value"] hostnames = options["hostnames"]["value"] return regions, filters, hostnames def _validate_option(self, name, desired_type, option_value): """ :param name: the option name :param desired_type: the class the option needs to be :param option: the value the user has provided :return The option of the correct class """ if isinstance(option_value, string_types) and desired_type == list: option_value = [option_value] if option_value is None: option_value = desired_type() if not isinstance(option_value, desired_type): raise AnsibleParserError( "The option %s (%s) must be a %s" % (name, option_value, desired_type) ) return option_value def parse(self, inventory, loader, path, cache=True): super(InventoryModule, self).parse(inventory, loader, path) config_data = self._read_config_data(path) # read oci config self.read_config() regions, filters, hostnames = self._get_query_options(config_data) cache_key = self.get_cache_key(path) if not regions: regions = [self.params["region"]] # false when refresh_cache if cache: cache = self.get_option("cache") # Generate inventory cache_needs_update = False if cache: try: cached_results = self._cache[cache_key] except Exception: # cache expired or cache file doesn't exist cache_needs_update = True else: self.display.warning("Using cached results") self._populate(cached_results, hostnames) results = None if not cache or cache_needs_update: results = self._query(regions) self._populate(results, hostnames) # update the cached inventory if cache_needs_update or (not cache and self.get_option("cache")): self._cache[cache_key] = results
def main(argv=sys.argv[1:]): display = Display() host_list = ['spark-node1', 'spark-node2', 'spark-node3'] # initialize needed objects Options = namedtuple('Options', ['connection', 'module_path', 'log_path', 'forks', 'become', 'become_method', 'become_user', 'verbosity', 'check']) variable_manager = VariableManager() loader = DataLoader() options = Options(connection='ssh', module_path='/path/to/mymodules', log_path='./log', forks=100, become=None, become_method=None, become_user=None, verbosity=None, check=False) # create inventory and pass to var manager inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=host_list) variable_manager.set_inventory(inventory) # inventory_list = inventory.get_hosts() # display.display('hosts: %s' % dir(inventory_list[0])) # h = inventory_list[0].get_name() # display.display('hosts: %s' % inventory_list[0].get_name()) # display.display('hosts: %s' % variable_manager.get_vars(loader)) # display.display('host list: %s' % inventory_list) display.warning('Running Ansible embedded') # create play with tasks play_source = dict(name="Ansible Play", hosts=host_list, gather_facts='no', tasks=[ dict(action=dict(module='command', args='uname -a'), register='shell_out'), dict(action=dict(module='debug', args=dict(msg='{{shell_out.stdout}}'))) ] ) play = Play().load(play_source, variable_manager=variable_manager, loader=loader) callback = ResultsCollector() # actually run it try: tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords={}, stdout_callback=callback, ) result = tqm.run(play) if result != 0: print "ERROR" finally: if tqm is not None: tqm.cleanup() print "UP ***********" for host, result in callback.host_ok.items(): print '{}: {}'.format(host, result._result['msg']) print "FAILED *******" for host, result in callback.host_failed.items(): print '{}: {}'.format(host, result._result['msg']) print "DOWN *********" for host, result in callback.host_unreachable.items(): print '{}: {}'.format(host, result._result['msg'])