def _rolepath(basedir, role): role_path = None possible_paths = [ # if included from a playbook path_dwim(basedir, os.path.join('roles', role)), path_dwim(basedir, role), # if included from roles/[role]/meta/main.yml path_dwim( basedir, os.path.join('..', '..', '..', 'roles', role) ), path_dwim(basedir, os.path.join('..', '..', role)) ] if C.DEFAULT_ROLES_PATH: search_locations = C.DEFAULT_ROLES_PATH if isinstance(search_locations, basestring): search_locations = search_locations.split(os.pathsep) for loc in search_locations: loc = os.path.expanduser(loc) possible_paths.append(path_dwim(loc, role)) for path_option in possible_paths: if os.path.isdir(path_option): role_path = path_option break if role_path: role_modules = os.path.join(role_path, 'library') if os.path.exists(role_modules): module_loader.add_directory(role_modules) return role_path
def __init__(self, inventory, variable_manager, loader, display, options, passwords, stdout_callback=None): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._callbacks_loaded = False self._callback_plugins = [] self._start_at_done = False # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() self._final_q = multiprocessing.Queue() # create the pool of worker threads, based on the number of forks specified try: fileno = sys.stdin.fileno() except ValueError: fileno = None # A temporary file (opened pre-fork) used by connection # plugins for inter-process locking. self._connection_lockfile = tempfile.TemporaryFile() #FIXME: should this move to 'run' and get serial and play pattern applied as limiter? # Treat "forks" config parameter as max value. Only create number of workers # equal to number of hosts in inventory if less than max value. num_workers = min(self._options.forks, len(self._inventory.list_hosts())) self._workers = [] for i in range(num_workers): main_q = multiprocessing.Queue() rslt_q = multiprocessing.Queue() prc = WorkerProcess(self, main_q, rslt_q, loader) prc.start() self._workers.append((prc, main_q, rslt_q)) self._result_prc = ResultProcess(self._final_q, self._workers) self._result_prc.start()
def _rolepath(basedir, role): role_path = None possible_paths = [ # if included from a playbook path_dwim(basedir, os.path.join('roles', role)), path_dwim(basedir, role), # if included from roles/[role]/meta/main.yml path_dwim( basedir, os.path.join('..', '..', '..', 'roles', role) ), path_dwim(basedir, os.path.join('..', '..', role)) ] if C.DEFAULT_ROLES_PATH: search_locations = C.DEFAULT_ROLES_PATH.split(os.pathsep) for loc in search_locations: loc = os.path.expanduser(loc) possible_paths.append(path_dwim(loc, role)) for path_option in possible_paths: if os.path.isdir(path_option): role_path = path_option break if role_path: role_modules = os.path.join(role_path, 'library') if os.path.exists(role_modules): module_loader.add_directory(role_modules) return role_path
def __init__(self, playbooks, inventory, variable_manager, loader, display, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self.passwords = passwords # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, display=display, options=options, passwords=self.passwords)
def set_basedir(loader, basedir): if version_info < (2, 4): from ansible.plugins import module_utils_loader, module_loader else: from ansible.plugins.loader import module_utils_loader, module_loader module_loader.add_directory(os.path.join(basedir, 'library')) loader.set_basedir(basedir) module_utils_loader.add_directory(os.path.join(basedir, 'module_utils'))
def __init__(self, inventory, variable_manager, loader, options, passwords, stdout_callback=None, run_additional_callbacks=True, run_tree=False): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._run_additional_callbacks = run_additional_callbacks self._run_tree = run_tree self._iterator = None self._callbacks_loaded = False self._callback_plugins = [] self._start_at_done = False # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() self._listening_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() # the "queue" for the background thread to use self._queued_tasks = deque() self._queued_tasks_lock = threading.Lock() # the background queuing thread self._queue_thread = None self._workers = [] self._final_q = multiprocessing.Queue() # A temporary file (opened pre-fork) used by connection # plugins for inter-process locking. self._connection_lockfile = tempfile.TemporaryFile()
def __init__(self, inventory, variable_manager, loader, display, options, passwords, stdout_callback=None): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._callbacks_loaded = False self._callback_plugins = [] self._start_at_done = False # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() self._final_q = multiprocessing.Queue() # create the pool of worker threads, based on the number of forks specified try: fileno = sys.stdin.fileno() except ValueError: fileno = None # A temporary file (opened pre-fork) used by connection # plugins for inter-process locking. self._connection_lockfile = tempfile.TemporaryFile() self._workers = [] for i in range(self._options.forks): main_q = multiprocessing.Queue() rslt_q = multiprocessing.Queue() prc = WorkerProcess(self, main_q, rslt_q, loader) prc.start() self._workers.append((prc, main_q, rslt_q)) self._result_prc = ResultProcess(self._final_q, self._workers) self._result_prc.start()
def list_modules(self): modules = set() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) module_paths = module_loader._get_paths() for path in module_paths: if path is not None: modules.update(self._find_modules_in_path(path)) return modules
def __init__(self, inventory, variable_manager, loader, display, options, passwords, stdout_callback=None): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._callbacks_loaded = False self._callback_plugins = [] self._start_at_done = False # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() # create the pool of worker threads, based on the number of forks specified pool = Process.Pool(self._options.forks) result = Process.Result() self._final_q = result.output # A temporary file (opened pre-fork) used by connection # plugins for inter-process locking. self._connection_lockfile = tempfile.TemporaryFile() self._workers = [] for i in range(self._options.forks): worker = Process.Worker(result.input, pool, loader) self._workers.append((worker, worker.input, worker.output)) self._result_prc = result
def _load_role_data(self, role_include, parent_role=None): self._role_name = role_include.role self._role_path = role_include.get_role_path() self._role_params = role_include.get_role_params() self._variable_manager = role_include.get_variable_manager() self._loader = role_include.get_loader() if parent_role: self.add_parent(parent_role) # copy over all field attributes, except for when and tags, which # are special cases and need to preserve pre-existing values for (attr_name, _) in iteritems(self._get_base_attributes()): if attr_name not in ('when', 'tags'): setattr(self, attr_name, getattr(role_include, attr_name)) current_when = getattr(self, 'when')[:] current_when.extend(role_include.when) setattr(self, 'when', current_when) current_tags = getattr(self, 'tags')[:] current_tags.extend(role_include.tags) setattr(self, 'tags', current_tags) # load the role's files, if they exist library = os.path.join(self._role_path, 'library') if os.path.isdir(library): module_loader.add_directory(library) metadata = self._load_role_yaml('meta') if metadata: self._metadata = RoleMetadata.load(metadata, owner=self, loader=self._loader) self._dependencies = self._load_dependencies() task_data = self._load_role_yaml('tasks') if task_data: self._task_blocks = load_list_of_blocks(task_data, role=self, loader=self._loader) handler_data = self._load_role_yaml('handlers') if handler_data: self._handler_blocks = load_list_of_blocks(handler_data, role=self, loader=self._loader) # vars and default vars are regular dictionaries self._role_vars = self._load_role_yaml('vars') if not isinstance(self._role_vars, (dict, NoneType)): raise AnsibleParserError("The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) elif self._role_vars is None: self._role_vars = dict() self._default_vars = self._load_role_yaml('defaults') if not isinstance(self._default_vars, (dict, NoneType)): raise AnsibleParserError("The default/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) elif self._default_vars is None: self._default_vars = dict()
def _load_role_data(self, role_include, parent_role=None): self._role_name = role_include.role self._role_path = role_include.get_role_path() self._role_params = role_include.get_role_params() self._variable_manager = role_include.get_variable_manager() self._loader = role_include.get_loader() if parent_role: self.add_parent(parent_role) current_when = getattr(self, 'when')[:] current_when.extend(role_include.when) setattr(self, 'when', current_when) current_tags = getattr(self, 'tags')[:] current_tags.extend(role_include.tags) setattr(self, 'tags', current_tags) # save the current base directory for the loader and set it to the current role path #cur_basedir = self._loader.get_basedir() #self._loader.set_basedir(self._role_path) # load the role's files, if they exist library = os.path.join(self._role_path, 'library') if os.path.isdir(library): module_loader.add_directory(library) metadata = self._load_role_yaml('meta') if metadata: self._metadata = RoleMetadata.load(metadata, owner=self, loader=self._loader) self._dependencies = self._load_dependencies() task_data = self._load_role_yaml('tasks') if task_data: self._task_blocks = load_list_of_blocks(task_data, role=self, loader=self._loader) handler_data = self._load_role_yaml('handlers') if handler_data: self._handler_blocks = load_list_of_blocks(handler_data, role=self, loader=self._loader) # vars and default vars are regular dictionaries self._role_vars = self._load_role_yaml('vars') if not isinstance(self._role_vars, (dict, NoneType)): raise AnsibleParserError("The vars/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) elif self._role_vars is None: self._role_vars = dict() self._default_vars = self._load_role_yaml('defaults') if not isinstance(self._default_vars, (dict, NoneType)): raise AnsibleParserError("The default/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) elif self._default_vars is None: self._default_vars = dict()
def __init__(self, filename): super(RoleFile, self).__init__(filename) self.version = None parentdir = os.path.dirname(os.path.abspath(filename)) while parentdir != os.path.dirname(parentdir): meta_file = os.path.join(parentdir, "meta", "main.yml") if os.path.exists(meta_file): self.version = find_version(meta_file) if self.version: break parentdir = os.path.dirname(parentdir) role_modules = os.path.join(parentdir, 'library') if os.path.exists(role_modules): module_loader.add_directory(role_modules)
def __init__(self, inventory, variable_manager, loader, display, options, passwords, stdout_callback=None): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._callbacks_loaded = False self._callback_plugins = [] # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() self._final_q = multiprocessing.Queue() # create the pool of worker threads, based on the number of forks specified try: fileno = sys.stdin.fileno() except ValueError: fileno = None self._workers = [] for i in range(self._options.forks): main_q = multiprocessing.Queue() rslt_q = multiprocessing.Queue() prc = WorkerProcess(self, main_q, rslt_q, loader) prc.start() self._workers.append((prc, main_q, rslt_q)) self._result_prc = ResultProcess(self._final_q, self._workers) self._result_prc.start()
def __init__(self, inventory, variable_manager, loader, options, passwords, stdout_callback=None, run_additional_callbacks=True, run_tree=False): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._run_additional_callbacks = run_additional_callbacks self._run_tree = run_tree self._callbacks_loaded = False self._callback_plugins = [] self._start_at_done = False # make sure any module paths (if specified) are added to the module_loader if isinstance(options.module_path, list): for path in options.module_path: if path is not None: module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() self._listening_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() self._final_q = multiprocessing.Queue() # A temporary file (opened pre-fork) used by connection # plugins for inter-process locking. self._connection_lockfile = tempfile.TemporaryFile()
def __init__(self, inventory, variable_manager, loader, options, passwords, stdout_callback=None, run_additional_callbacks=True, run_tree=False): self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self._stats = AggregateStats() self.passwords = passwords self._stdout_callback = stdout_callback self._run_additional_callbacks = run_additional_callbacks self._run_tree = run_tree self._callbacks_loaded = False self._callback_plugins = [] self._start_at_done = False self._result_prc = None self._prst = [] # all information in processes # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) # a special flag to help us exit cleanly self._terminated = False # this dictionary is used to keep track of notified handlers self._notified_handlers = dict() # dictionaries to keep track of failed/unreachable hosts self._failed_hosts = dict() self._unreachable_hosts = dict() self._final_q = multiprocessing.Queue() # A temporary file (opened pre-fork) used by connection # plugins for inter-process locking. self._connection_lockfile = tempfile.TemporaryFile()
def _load_library_if_exists(path): if os.path.exists(path): module_loader.add_directory(path)
def run(self): super(DocCLI, self).run() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.pager(self.get_module_list_text()) return 0 if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: try: filename = module_loader.find_plugin(module) if filename is None: self.display.warning( "module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in self.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring( filename, verbose=(self.options.verbosity > 0)) except: self.display.vvv(traceback.print_exc()) self.display.error( "module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: all_keys = [] for (k, v) in iteritems(doc['options']): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime( '%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs if self.options.show_snippet: text += DocCLI.get_snippet_text(doc) else: text += DocCLI.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. raise AnsibleError("Parsing produced an empty object.") except Exception as e: self.display.vvv(traceback.print_exc()) raise AnsibleError( "module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e))) self.pager(text) return 0
def run(self): if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) CLI.pager(self.get_module_list_text()) return 0 if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: filename = module_loader.find_plugin(module) if filename is None: self.display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in self.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename) except: self.display.vvv(traceback.print_exc()) self.display.error("module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: all_keys = [] for (k,v) in doc['options'].iteritems(): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs if self.options.show_snippet: text += DocCLI.get_snippet_text(doc) else: text += DocCLI.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. self.display.warning("module %s missing documentation (or could not parse documentation)\n" % module) CLI.pager(text) return 0
def run(self): super(DocCLI, self).run() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.pager(self.get_module_list_text()) return 0 if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: try: # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs filename = module_loader.find_plugin(module, mod_type='.py') if filename is None: display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in C.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename, verbose=(self.options.verbosity > 0)) except: display.vvv(traceback.print_exc()) display.error("module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: # is there corresponding action plugin? if module in action_loader: doc['action'] = True else: doc['action'] = False all_keys = [] for (k,v) in iteritems(doc['options']): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs if self.options.show_snippet: text += self.get_snippet_text(doc) else: text += self.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. raise AnsibleError("Parsing produced an empty object.") except Exception as e: display.vvv(traceback.print_exc()) raise AnsibleError("module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e))) if text: self.pager(text) return 0
def run(self): super(DocCLI, self).run() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.pager(self.get_module_list_text()) return 0 # process all modules if self.options.all_modules: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.args = sorted( set(self.module_list) - module_docs.BLACKLIST_MODULES) if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: try: # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs filename = module_loader.find_plugin(module, mod_type='.py', ignore_deprecated=True) if filename is None: display.warning( "module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in C.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs, metadata = module_docs.get_docstring( filename, verbose=(self.options.verbosity > 0)) except: display.vvv(traceback.format_exc()) display.error( "module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: # is there corresponding action plugin? if module in action_loader: doc['action'] = True else: doc['action'] = False all_keys = [] for (k, v) in iteritems(doc['options']): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime( '%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs doc['metadata'] = metadata if 'metadata_version' in doc['metadata']: del doc['metadata']['metadata_version'] if 'version' in doc['metadata']: del doc['metadata']['metadata_version'] if self.options.show_snippet: text += self.get_snippet_text(doc) else: text += self.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. raise AnsibleError("Parsing produced an empty object.") except Exception as e: display.vvv(traceback.format_exc()) raise AnsibleError( "module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e))) if text: self.pager(text) return 0