def _validate_docs(self): doc_info = self._get_docs() try: doc = yaml.safe_load(doc_info['DOCUMENTATION']['value']) except yaml.YAMLError as e: doc = None # This offsets the error line number to where the # DOCUMENTATION starts so we can just go to that line in the # module e.problem_mark.line += ( doc_info['DOCUMENTATION']['lineno'] - 1 ) e.problem_mark.name = '%s.DOCUMENTATION' % self.name self.traces.append(e) self.errors.append('DOCUMENTATION is not valid YAML. Line %d ' 'column %d' % (e.problem_mark.line + 1, e.problem_mark.column + 1)) except AttributeError: self.errors.append('No DOCUMENTATION provided') else: with CaptureStd(): try: get_docstring(self.path, verbose=True) except AssertionError: fragment = doc['extends_documentation_fragment'] self.errors.append('DOCUMENTATION fragment missing: %s' % fragment) except Exception as e: self.traces.append(e) self.errors.append('Unknown DOCUMENTATION error, see ' 'TRACE') self._validate_docs_schema(doc) self._check_version_added(doc) self._check_for_new_args(doc) if not bool(doc_info['EXAMPLES']['value']): self.errors.append('No EXAMPLES provided') if not bool(doc_info['RETURN']['value']): if self._is_new_module(): self.errors.append('No RETURN documentation provided') else: self.warnings.append('No RETURN provided') else: try: yaml.safe_load(doc_info['RETURN']['value']) except yaml.YAMLError as e: e.problem_mark.line += ( doc_info['RETURN']['lineno'] - 1 ) e.problem_mark.name = '%s.RETURN' % self.name self.errors.append('RETURN is not valid YAML. Line %d ' 'column %d' % (e.problem_mark.line + 1, e.problem_mark.column + 1)) self.traces.append(e)
def _validate_docs(self): doc_info = self._get_docs() try: doc = yaml.safe_load(doc_info["DOCUMENTATION"]["value"]) except yaml.YAMLError as e: doc = None # This offsets the error line number to where the # DOCUMENTATION starts so we can just go to that line in the # module e.problem_mark.line += doc_info["DOCUMENTATION"]["lineno"] - 1 e.problem_mark.name = "%s.DOCUMENTATION" % self.name self.traces.append(e) self.errors.append( "DOCUMENTATION is not valid YAML. Line %d " "column %d" % (e.problem_mark.line + 1, e.problem_mark.column + 1) ) except AttributeError: self.errors.append("No DOCUMENTATION provided") else: with CaptureStd(): try: get_docstring(self.path, verbose=True) except AssertionError: fragment = doc["extends_documentation_fragment"] self.errors.append("DOCUMENTATION fragment missing: %s" % fragment) except Exception as e: self.traces.append(e) self.errors.append("Unknown DOCUMENTATION error, see " "TRACE") self._validate_docs_schema(doc) self._check_version_added(doc) self._check_for_new_args(doc) if not bool(doc_info["EXAMPLES"]["value"]): self.errors.append("No EXAMPLES provided") if not bool(doc_info["RETURN"]["value"]): if self._is_new_module(): self.errors.append("No RETURN documentation provided") else: self.warnings.append("No RETURN provided") else: try: yaml.safe_load(doc_info["RETURN"]["value"]) except yaml.YAMLError as e: e.problem_mark.line += doc_info["RETURN"]["lineno"] - 1 e.problem_mark.name = "%s.RETURN" % self.name self.errors.append( "RETURN is not valid YAML. Line %d " "column %d" % (e.problem_mark.line + 1, e.problem_mark.column + 1) ) self.traces.append(e)
def _check_for_new_args(self, doc): if self._is_new_module(): return with CaptureStd(): try: existing = module_loader.find_plugin(self.name, mod_type='.py') existing_doc, _, _ = get_docstring(existing, verbose=True) existing_options = existing_doc.get('options', {}) except AssertionError: fragment = doc['extends_documentation_fragment'] self.warnings.append('Pre-existing DOCUMENTATION fragment ' 'missing: %s' % fragment) return except Exception as e: self.warning_traces.append(e) self.warnings.append('Unknown pre-existing DOCUMENTATION ' 'error, see TRACE. Submodule refs may ' 'need updated') return try: mod_version_added = StrictVersion( str(existing_doc.get('version_added', '0.0')) ) except ValueError: mod_version_added = StrictVersion('0.0') options = doc.get('options', {}) should_be = '.'.join(ansible_version.split('.')[:2]) strict_ansible_version = StrictVersion(should_be) for option, details in options.iteritems(): new = not bool(existing_options.get(option)) if not new: continue try: version_added = StrictVersion( str(details.get('version_added', '0.0')) ) except ValueError: version_added = details.get('version_added', '0.0') self.errors.append('version_added for new option (%s) ' 'is not a valid version number: %r' % (option, version_added)) continue except: # If there is any other exception it should have been caught # in schema validation, so we won't duplicate errors by # listing it again continue if (strict_ansible_version != mod_version_added and (version_added < strict_ansible_version or strict_ansible_version < version_added)): self.errors.append('version_added for new option (%s) should ' 'be %s. Currently %s' % (option, should_be, version_added))
def get_documents(): for root, dirs, files in os.walk(os.path.dirname( ansible.modules.__file__)): for f in files: if f == '__init__.py' or not f.endswith('py'): continue documentation = get_docstring(os.path.join(root, f))[0] if documentation is None: continue yield documentation
def process_module(fname, template, outputname): print MODULEDIR + fname if LooseVersion(ansible_version) >= LooseVersion('2.3.0'): doc, examples, returndocs, _metadata = module_docs.get_docstring( MODULEDIR + fname) else: doc, examples, returndocs = module_docs.get_docstring(MODULEDIR + fname) all_keys = [] if 'version_added' not in doc: sys.stderr.write( "*** ERROR: missing version_added in: %s ***\n".format(fname)) sys.exit(1) added = 0 if doc['version_added'] == 'historical': del doc['version_added'] else: added = doc['version_added'] # don't show version added information if it's too old to be called out if added: added_tokens = str(added).split(".") added = added_tokens[0] + "." + added_tokens[1] added_float = float(added) for (k, v) in doc['options'].iteritems(): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = fname doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = examples # plain text # here is where we build the table of contents... text = template.render(doc) write_data(text, outputname, fname, OUTPUTDIR)
def main(): doc_cli = DocCLI([]) module_paths = module_loader._get_paths() module_keys = ('module', 'short_description', 'options', 'deprecated') for path in module_paths: doc_cli.find_modules(path) result = { 'modules': [], 'directives': defaultdict(list), 'lookup_plugins': [], } for module in sorted(set(doc_cli.module_list)): if module in module_docs.BLACKLIST_MODULES: continue filename = module_loader.find_plugin(module, mod_type='.py') if not filename: continue if filename.endswith('.ps1'): continue if os.path.isdir(filename): continue try: doc, plain_examples, return_docs = module_docs.get_docstring(filename) filtered_doc = {key: doc.get(key) for key in module_keys} result['modules'].append(filtered_doc) except Exception: pass for aclass in (Play, Role, Block, Task): aobj = aclass() name = aclass.__name__ for attr in aobj.__dict__['_attributes']: if 'private' in attr and attr.private: continue direct_target = result['directives'][attr] direct_target.append(name) if attr == 'action': local_action = result['directives']['local_action'] local_action.append(name) result['directives']['with_'] = ['Task'] for lookup in lookup_loader.all(): name = os.path.splitext(os.path.basename(lookup._original_path))[0] result['lookup_plugins'].append(name) fn = os.path.join(__path__, '../data/ansible-data.json') with codecs.open(fn, 'wb', encoding='utf-8') as f: json.dump(result, f, ensure_ascii=False, indent=2)
def process_module(fname, template, outputname): print MODULEDIR + fname if LooseVersion(ansible_version) >= LooseVersion('2.3.0'): doc, examples, returndocs, _metadata = module_docs.get_docstring(MODULEDIR + fname) else: doc, examples, returndocs = module_docs.get_docstring(MODULEDIR + fname) all_keys = [] if 'version_added' not in doc: sys.stderr.write("*** ERROR: missing version_added in: %s ***\n".format(fname)) sys.exit(1) added = 0 if doc['version_added'] == 'historical': del doc['version_added'] else: added = doc['version_added'] # don't show version added information if it's too old to be called out if added: added_tokens = str(added).split(".") added = added_tokens[0] + "." + added_tokens[1] added_float = float(added) for (k, v) in doc['options'].iteritems(): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = fname doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = examples # plain text # here is where we build the table of contents... text = template.render(doc) write_data(text, outputname, fname, OUTPUTDIR)
def main(): print "%s" % sys.argv[1] path = sys.argv[1] if path[-1] != '/': path = path + '/' all_docs = [] errors = [] for (dirpath, dirnames, ansible_modules) in walk(path): if dirpath == path: for mod in ansible_modules: try: try: doc, plainexamples, returndocs = module_docs.get_docstring( path + mod) except ValueError: doc, plainexamples = module_docs.get_docstring(path + mod) except Exception as e: print "error : %s" % str(e) try: examples_list = plainexamples.split('\n') doc['examples'] = examples_list except: errors.append(doc) errors.append('error-examples') continue all_docs.append(doc) except: errors.append(mod) errors.append('unknown') continue print "results : %s" % all_docs print "errors : %s" % errors
def helpdefault(self, module_name): if module_name in self.modules: in_path = module_loader.find_plugin(module_name) if in_path: oc, a, _, _ = module_docs.get_docstring(in_path) if oc: display.display(oc['short_description']) display.display('Parameters:') for opt in oc['options'].keys(): display.display(' ' + stringc(opt, C.COLOR_HIGHLIGHT) + ' ' + oc['options'][opt]['description'][0]) else: display.error('No documentation found for %s.' % module_name) else: display.error('%s is not a valid command, use ? to list all valid commands.' % module_name)
def helpdefault(self, module_name): if module_name in self.modules: in_path = module_loader.find_plugin(module_name) if in_path: oc, a, _ = module_docs.get_docstring(in_path) if oc: display.display(oc['short_description']) display.display('Parameters:') for opt in oc['options'].keys(): display.display(' ' + stringc(opt, C.COLOR_HIGHLIGHT) + ' ' + oc['options'][opt]['description'][0]) else: display.error('No documentation found for %s.' % module_name) else: display.error('%s is not a valid command, use ? to list all valid commands.' % module_name)
def process_doc(module): doc, examples, returndocs, metadata = get_docstring(module) if not doc: print("No docstring retrieved for {}".format(module), file=sys.stderr) return doc['options'] = sorted(doc['options'].items()) if examples and filter(lambda x: x.replace('\n', ''), examples): examples = filter(lambda x: x, examples) examples_list = examples.split('\n') doc['examples'] = examples_list if returndocs: doc['returndocs'] = returndocs print('Retrieved docstring for {}'.format(doc['module'])) return doc
def get_module_list_text(self): tty_size = 0 if os.isatty(0): tty_size = struct.unpack( 'HHHH', fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)))[1] columns = max(60, tty_size) displace = max(len(x) for x in self.module_list) linelimit = columns - displace - 5 text = [] deprecated = [] for module in sorted(set(self.module_list)): if module in module_docs.BLACKLIST_MODULES: continue filename = module_loader.find_plugin(module) if filename is None: continue if filename.endswith(".ps1"): continue if os.path.isdir(filename): continue try: doc, plainexamples, returndocs = module_docs.get_docstring( filename) desc = self.tty_ify(doc.get('short_description', '?')).strip() if len(desc) > linelimit: desc = desc[:linelimit] + '...' if module.startswith('_'): # Handle deprecated deprecated.append( "%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc)) else: text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) except: raise AnsibleError( "module %s has a documentation error formatting or is missing documentation\n" % module) if len(deprecated) > 0: text.append("\nDEPRECATED:") text.extend(deprecated) return "\n".join(text)
def get_module_doc(module): ''' Takes a module name and returns a dict made from the DOCSTRING of the module. We also add some useful fields. ''' filename = utils.plugins.module_finder.find_plugin(module) if filename is None: pass if any(filename.endswith(x) for x in BLACKLIST_EXTS): pass try: doc, plainexamples = module_docs.get_docstring(filename) except: traceback.print_exc() sys.stderr.write( "ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) pass if doc is not None: all_keys = [] options = [] for (k, v) in doc['options'].iteritems(): all_keys.append(k) option = {'name': k} for key, value in v.iteritems(): if key == 'description': value = ''.join(value) if key == 'required': if value == True: value = 'yes' else: value = 'no' params = {key: value} option.update(params) if 'required' not in option.keys(): option['required'] = 'no' options.append(option) doc['options'] = options doc['description'] = str('\n'.join(doc['description'])) doc['option_keys'] = all_keys doc['filename'] = filename doc['module_path'] = '%s/%s' % (os.path.split( re.sub('^%s' % PACKAGE_PATH, '', filename))[0], doc['module']) doc['docuri'] = doc['module'].replace('_', '-') else: sys.stderr.write( "ERROR: module %s missing documentation (or could not parse documentation)\n" % module) return unformat(doc)
def main(): doc_cli = DocCLI([]) module_paths = module_loader._get_paths() module_keys = ('module', 'short_description', 'options', 'deprecated') for path in module_paths: doc_cli.find_modules(path) result = {'modules': [], 'directives': {}, 'lookup_plugins':[]} for module in sorted(set(doc_cli.module_list)): if module in module_docs.BLACKLIST_MODULES: continue filename = module_loader.find_plugin(module, mod_type='.py') if filename is None: continue if filename.endswith(".ps1"): continue if os.path.isdir(filename): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename) filtered_doc = {key: doc.get(key, None) for key in module_keys} result['modules'].append(filtered_doc) except: pass for aclass in (Play, Role, Block, Task): aobj = aclass() name = type(aobj).__name__ for attr in aobj.__dict__['_attributes']: if 'private' in attr and attr.private: continue direct_target = result['directives'].setdefault(attr, []) direct_target.append(name) if attr == 'action': local_action = result['directives'].setdefault('local_action', []) local_action.append(name) result['directives']['with_'] = ['Task'] for lookup in lookup_loader.all(): name = os.path.splitext(os.path.basename(lookup._original_path))[0] result['lookup_plugins'].append(name) print(json.dumps(result))
def get_module_list_text(self): columns = display.columns displace = max(len(x) for x in self.module_list) linelimit = columns - displace - 5 text = [] deprecated = [] for module in sorted(set(self.module_list)): if module in module_docs.BLACKLIST_MODULES: continue # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs filename = module_loader.find_plugin(module, mod_type='.py', ignore_deprecated=True) if filename is None: continue if filename.endswith(".ps1"): continue if os.path.isdir(filename): continue try: doc, plainexamples, returndocs, metadata = module_docs.get_docstring( filename) desc = self.tty_ify(doc.get('short_description', '?')).strip() if len(desc) > linelimit: desc = desc[:linelimit] + '...' if module.startswith('_'): # Handle deprecated deprecated.append( "%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc)) else: text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) except: raise AnsibleError( "module %s has a documentation error formatting or is missing documentation\n" % module) if len(deprecated) > 0: text.append("\nDEPRECATED:") text.extend(deprecated) return "\n".join(text)
def get_docs(module_dir=None, warnings=False): requires_others = {} for module in get_modules(module_dir): filename = utils.plugins.module_finder.find_plugin(module) if filename is None: if warnings: sys.stderr.write("WARNING: module %s not found in %s\n" % (module, print_paths(utils.plugins.module_finder))) continue try: doc, plainexamples = module_docs.get_docstring(filename) except: traceback.print_exc() sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) continue if doc is not None: if module == "file": file_module_options = doc['options'] yield module, doc elif 'options' in doc: # Minor inconsistency fix if 'free-form' in doc['options']: doc['options']['free_form'] = doc['options'].pop('free-form') # Others actually means file options if 'others' in doc['options']: requires_others[module] = doc else: yield module, doc else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. sys.stderr.write("ERROR: module %s missing documentation (or could not parse documentation)\n" % module) # Merge file options into modules that require 'others' for module, doc in requires_others.iteritems(): # Remove others doc['options'].pop('others') for key, value in file_module_options.iteritems(): if not key in doc: doc['options'][key] = value yield module, doc
def get_module_doc(module): ''' Takes a module name and returns a dict made from the DOCSTRING of the module. We also add some useful fields. ''' filename = utils.plugins.module_finder.find_plugin(module) if filename is None: pass if any(filename.endswith(x) for x in BLACKLIST_EXTS): pass try: doc, plainexamples = module_docs.get_docstring(filename) except: traceback.print_exc() sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) pass if doc is not None: all_keys = [] options = [] for (k,v) in doc['options'].iteritems(): all_keys.append(k) option = {'name':k} for key, value in v.iteritems(): if key == 'description': value = ''.join(value) if key == 'required': if value == True: value = 'yes' else: value = 'no' params = {key: value} option.update(params) if 'required' not in option.keys(): option['required'] = 'no' options.append(option) doc['options'] = options doc['description'] = str('\n'.join(doc['description'])) doc['option_keys'] = all_keys doc['filename'] = filename doc['module_path'] = '%s/%s' % (os.path.split(re.sub('^%s' % PACKAGE_PATH, '', filename))[0], doc['module']) doc['docuri'] = doc['module'].replace('_', '-') else: sys.stderr.write("ERROR: module %s missing documentation (or could not parse documentation)\n" % module) return unformat(doc)
def validate(self): if set([self.basename, self.name]) & set(BLACKLIST_MODULES): return doc, examples, ret = get_docstring(self.path) if not bool(doc): self.errors.append('Invalid or no DOCUMENTATION provided') if not bool(examples): self.errors.append('No EXAMPLES provided') if not bool(ret): self.warnings.append('No RETURN provided') if not self._just_docs(): module_utils = self._find_module_utils() main = self._find_main_call() for mu in module_utils: if mu < main - 10: self.errors.append('module_utils import not near main()') self._find_has_import()
def get_module_list_text(self): columns = display.columns displace = max(len(x) for x in self.module_list) linelimit = columns - displace - 5 text = [] deprecated = [] for module in sorted(set(self.module_list)): if module in module_docs.BLACKLIST_MODULES: continue # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs filename = module_loader.find_plugin(module, mod_type='.py') if filename is None: continue if filename.endswith(".ps1"): continue if os.path.isdir(filename): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename) desc = self.tty_ify(doc.get('short_description', '?')).strip() if len(desc) > linelimit: desc = desc[:linelimit] + '...' if module.startswith('_'): # Handle deprecated deprecated.append("%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc)) else: text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc)) except: raise AnsibleError("module %s has a documentation error formatting or is missing documentation\n" % module) if len(deprecated) > 0: text.append("\nDEPRECATED:") text.extend(deprecated) return "\n".join(text)
def generate(path, args): doc, examples, _ = get_docstring(path) prefix = args.snipmate and '\t' or '' if doc != None: print "snippet %s" % doc['module'] print "%s- name: ${1:task_description}" % prefix print "%s %s: " % (prefix, doc['module']) count = 1 if 'options' in doc: for opt in doc['options']: count += 1 optional = not doc['options'][opt].get('required', False) if 'default' in doc['options'][opt]: value = "${%d:%s}" % (count, doc['options'][opt]['default']) else: value = "${%d}" % (count) print "%s %s%s: %s" % (prefix, optional and '#' or '', opt, value) if args.ultisnips: print 'endsnippet' print
def module_args(self, module_name): in_path = module_loader.find_plugin(module_name) oc, a, _ = module_docs.get_docstring(in_path) return oc['options'].keys()
def run(self): if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) CLI.pager(self.get_module_list_text()) return 0 if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: filename = module_loader.find_plugin(module) if filename is None: self.display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in self.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename) except: self.display.vvv(traceback.print_exc()) self.display.error("module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: all_keys = [] for (k,v) in doc['options'].iteritems(): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs if self.options.show_snippet: text += DocCLI.get_snippet_text(doc) else: text += DocCLI.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. self.display.warning("module %s missing documentation (or could not parse documentation)\n" % module) CLI.pager(text) return 0
def _get_docstring(self): self._docstring = module_docs.get_docstring(self.path)
if len(choices) > 0: return ":#{0}".format("|".join(str(v) for v in choices)) if required: return ":#REQUIRED" return "" ansible_path = os.path.dirname(os.path.dirname(module_docs.__file__)) modules_path = os.path.join(ansible_path, 'modules') for (dirpath, dirnames, filesname) in os.walk(modules_path): for f in filesname: if f.endswith('.py') and not f.startswith('__init__'): module = os.path.join(dirpath, f) docstring = module_docs.get_docstring(module) mod = docstring[0] if mod == None: continue name = mod['module'] print "snippet\t\t{0}".format(name) print "options\t\thead" print "\t{0}:".format(name) # iterate module parameters paramIndex = 1 for option in mod['options'].iteritems(): parameters = option[1]
return ":#{0}".format("|".join(str(v) for v in choices)) if required: return ":#REQUIRED" return "" ansible_path = os.path.dirname(os.path.dirname(module_docs.__file__)) modules_path = os.path.join(ansible_path, 'modules') for (dirpath, dirnames, filesname) in os.walk(modules_path): for f in filesname: if f.endswith('.py') and not f.startswith('__init__'): module = os.path.join(dirpath, f) docstring = module_docs.get_docstring(module) mod = docstring[0] if mod == None: continue name = mod['module'] print "snippet\t\t{0}".format(name) print "options\t\thead" print "\t{0}:".format(name) # iterate module parameters paramIndex = 1 for option in mod['options'].iteritems(): parameters = option[1]
libpath = os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, 'library' )) fname_list = [] for d in os.listdir(libpath): for f in os.listdir(os.path.join(libpath, d)): fname_list.append(os.path.join(libpath, d, f)) for fname in fname_list: if os.path.isdir(fname): continue doc, examples = get_docstring(fname) if doc != None: print "snippet %s" % (doc['module']) print " - name: ${1:task_description}" print " %s:" % (doc['module']), if 'options' in doc: count = 1 for o in doc['options']: if 'required' in doc['options'][o] and doc['options'][o]['required']: count += 1 if 'default' in doc['options'][o] and doc['options'][o]['default']: value = "${%d:%s}" % (count, doc['options'][o]['default']) else: value = "${%d}" % (count) print "%s=%s" % (o, value), print "#",
def process_module(module, options, env, template, outputname, module_map, aliases): fname = module_map[module] if isinstance(fname, dict): return "SKIPPED" basename = os.path.basename(fname) deprecated = False # ignore files with extensions if not basename.endswith(".py"): return elif module.startswith("_"): if os.path.islink(fname): return # ignore, its an alias deprecated = True module = module.replace("_","",1) print("rendering: %s" % module) # use ansible core library to parse out doc metadata YAML and plaintext examples doc, examples, returndocs = module_docs.get_docstring(fname, verbose=options.verbose) # crash if module is missing documentation and not explicitly hidden from docs index if doc is None: if module in module_docs.BLACKLIST_MODULES: return "SKIPPED" else: sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) sys.exit(1) if deprecated and 'deprecated' not in doc: sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module)) sys.exit(1) if "/core/" in fname: doc['core'] = True else: doc['core'] = False if module in aliases: doc['aliases'] = aliases[module] all_keys = [] if not 'version_added' in doc: sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module) sys.exit(1) added = 0 if doc['version_added'] == 'historical': del doc['version_added'] else: added = doc['version_added'] # don't show version added information if it's too old to be called out if too_old(added): del doc['version_added'] if 'options' in doc and doc['options']: for (k,v) in doc['options'].iteritems(): # don't show version added information if it's too old to be called out if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']): del doc['options'][k]['version_added'] all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = fname doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['ansible_version'] = options.ansible_version doc['plainexamples'] = examples #plain text if returndocs: doc['returndocs'] = yaml.safe_load(returndocs) else: doc['returndocs'] = None # here is where we build the table of contents... try: text = template.render(doc) except Exception as e: raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e))) write_data(text, options, outputname, module) return doc['short_description']
def run(self): super(DocCLI, self).run() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.pager(self.get_module_list_text()) return 0 # process all modules if self.options.all_modules: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.args = sorted( set(self.module_list) - module_docs.BLACKLIST_MODULES) if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: try: # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs filename = module_loader.find_plugin(module, mod_type='.py', ignore_deprecated=True) if filename is None: display.warning( "module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in C.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs, metadata = module_docs.get_docstring( filename, verbose=(self.options.verbosity > 0)) except: display.vvv(traceback.format_exc()) display.error( "module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: # is there corresponding action plugin? if module in action_loader: doc['action'] = True else: doc['action'] = False all_keys = [] for (k, v) in iteritems(doc['options']): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime( '%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs doc['metadata'] = metadata if 'metadata_version' in doc['metadata']: del doc['metadata']['metadata_version'] if 'version' in doc['metadata']: del doc['metadata']['metadata_version'] if self.options.show_snippet: text += self.get_snippet_text(doc) else: text += self.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. raise AnsibleError("Parsing produced an empty object.") except Exception as e: display.vvv(traceback.format_exc()) raise AnsibleError( "module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e))) if text: self.pager(text) return 0
if os.path.isfile(fullpath): module_list.append(fullpath) # Use module_docs lib to get all parameter info mdict = {} for module in sorted(module_list): doc = None plainexamples = None mname = os.path.basename(module) if mname in module_docs.BLACKLIST_MODULES: continue try: doc, plainexamples = module_docs.get_docstring(module) except AssertionError, e: pass except SyntaxError, e: pass except Exception, e: pass if doc: module_name = os.path.basename(module) mdict[module_name] = {} mdict[module_name]['params'] = {} mdict[module_name]['version_added'] = doc['version_added'] mdict[module_name]['module'] = doc['module'] for optkey in doc['options'].keys():
def run(self): super(DocCLI, self).run() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.pager(self.get_module_list_text()) return 0 if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: try: filename = module_loader.find_plugin(module) if filename is None: self.display.warning( "module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in self.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring( filename, verbose=(self.options.verbosity > 0)) except: self.display.vvv(traceback.print_exc()) self.display.error( "module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: all_keys = [] for (k, v) in iteritems(doc['options']): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime( '%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs if self.options.show_snippet: text += DocCLI.get_snippet_text(doc) else: text += DocCLI.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. raise AnsibleError("Parsing produced an empty object.") except Exception as e: self.display.vvv(traceback.print_exc()) raise AnsibleError( "module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e))) self.pager(text) return 0
from ansible.utils import module_docs from os import walk path = "/home/markus/projects/other/ansible/lib/ansible/modules/cloud/amazon" if path[-1] != '/': path = path + '/' all_docs = [] errors = [] for (dirpath, dirnames, ansible_modules) in walk(path): if dirpath == path: for mod in ansible_modules: print(mod) try: try: doc, plainexamples, returndocs = module_docs.get_docstring( path + mod) except ValueError: print("XXX") doc, plainexamples = module_docs.get_docstring(path + mod) print("ZZZ") print(plainexamples) except Exception as e: print("YYY") print(str(e)) # module.fail_json(msg='error', error=str(e)) try: examples_list = plainexamples.split('\n') doc['examples'] = examples_list except:
def run(self): super(DocCLI, self).run() if self.options.module_path is not None: for i in self.options.module_path.split(os.pathsep): module_loader.add_directory(i) # list modules if self.options.list_dir: paths = module_loader._get_paths() for path in paths: self.find_modules(path) self.pager(self.get_module_list_text()) return 0 if len(self.args) == 0: raise AnsibleOptionsError("Incorrect options passed") # process command line module list text = '' for module in self.args: try: # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs filename = module_loader.find_plugin(module, mod_type='.py') if filename is None: display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader))) continue if any(filename.endswith(x) for x in C.BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename, verbose=(self.options.verbosity > 0)) except: display.vvv(traceback.print_exc()) display.error("module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module) continue if doc is not None: # is there corresponding action plugin? if module in action_loader: doc['action'] = True else: doc['action'] = False all_keys = [] for (k,v) in iteritems(doc['options']): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs if self.options.show_snippet: text += self.get_snippet_text(doc) else: text += self.get_man_text(doc) else: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. raise AnsibleError("Parsing produced an empty object.") except Exception as e: display.vvv(traceback.print_exc()) raise AnsibleError("module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e))) if text: self.pager(text) return 0
def main(): p = optparse.OptionParser( version=version("%prog"), usage='usage: %prog [options] [module...]', description='Show Ansible module documentation', ) p.add_option("-M", "--module-path", action="store", dest="module_path", default=MODULEDIR, help="Ansible modules/ directory") p.add_option('-v', action='version', help='Show version number and exit') (options, args) = p.parse_args() if options.module_path is not None: for i in options.module_path.split(os.pathsep): utils.plugins.module_finder.add_directory(i) def print_paths(finder): ''' Returns a string suitable for printing of the search path ''' # Uses a list to get the order right ret = [] for i in finder._get_paths(): if i not in ret: ret.append(i) return os.pathsep.join(ret) has_error = False for module in args: filename = utils.plugins.module_finder.find_plugin(module) if filename is None: sys.stderr.write( "module %s not found in %s\n" % (module, print_paths(utils.plugins.module_finder))) continue if any(filename.endswith(x) for x in BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring( filename) except: traceback.print_exc() sys.stderr.write( "ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) has_error = True continue if doc is None: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. sys.stderr.write( "ERROR: module %s missing documentation (or could not parse documentation)\n" % module) has_error = True else: all_keys = [] for (k, v) in doc['options'].iteritems(): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs text = get_man_text(doc) try: text.decode('ascii') except (UnicodeDecodeError, UnicodeEncodeError): traceback.print_exc() sys.stderr.write( "ERROR: module %s has an documentation encoding error\n" % module) has_error = True if has_error: sys.exit(1) else: sys.exit(0)
def main(): argument_spec = { "path": { "required": True, "type": 'str' }, "exclusion_filters": { "required": False, "type": 'list' } } module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False) root_path = module.params['path'] root_path = os.path.abspath(root_path) if not os.path.isdir(root_path): module.fail_json(msg=root_path + " is not a directory or not exists.") exclusion_filters = module.params['exclusion_filters'] doc_list = list() for root, dirs, files in os.walk(root_path): for file_name in sorted(files): # Skips all non python files if not fnmatch(file_name, "*.py"): continue # Skips according exclusion_filters if check_exclusion(file_name, exclusion_filters): continue try: module_file_name = os.path.normpath( os.path.join(root, file_name)) # Tuple size may vary according to the Ansible version docstring = module_docs.get_docstring(module_file_name) doc = docstring[0] plainexamples = docstring[1] returndocs = docstring[2] if doc: doc = format_dict(doc) if plainexamples: doc['examples'] = plainexamples.split('\n') if returndocs and returndocs.strip() != '#': try: doc['returns'] = yaml.load(returndocs) except Exception as e: errors.append( "{0} - Failed yaml.load(doc['returns']) - {1}". format(file_name, e.args[0])) doc_list.append(doc) else: errors.append(file_name + " - No docs found") except Exception as ex: errors.append("{0} - {1}".format(file_name, ex.args[0])) module.exit_json(modules_docs=doc_list, errors=errors)
def process_module(module, options, env, template, outputname, module_map, aliases): fname = module_map[module] if isinstance(fname, dict): return "SKIPPED" basename = os.path.basename(fname) deprecated = False # ignore files with extensions if not basename.endswith(".py"): return elif module.startswith("_"): if os.path.islink(fname): return # ignore, its an alias deprecated = True module = module.replace("_","",1) print("rendering: %s" % module) # use ansible core library to parse out doc metadata YAML and plaintext examples doc, examples, returndocs = module_docs.get_docstring(fname, verbose=options.verbose) # crash if module is missing documentation and not explicitly hidden from docs index if doc is None: if module in module_docs.BLACKLIST_MODULES: return "SKIPPED" else: sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) sys.exit(1) if deprecated and 'deprecated' not in doc: sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module)) sys.exit(1) if "/core/" in fname: doc['core'] = True else: doc['core'] = False if module in aliases: doc['aliases'] = aliases[module] all_keys = [] if not 'version_added' in doc: sys.stderr.write("*** ERROR: missing version_added in: %s ***\n" % module) sys.exit(1) added = 0 if doc['version_added'] == 'historical': del doc['version_added'] else: added = doc['version_added'] # don't show version added information if it's too old to be called out if too_old(added): del doc['version_added'] if 'options' in doc and doc['options']: for (k,v) in iteritems(doc['options']): # don't show version added information if it's too old to be called out if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']): del doc['options'][k]['version_added'] all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = fname doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['ansible_version'] = options.ansible_version doc['plainexamples'] = examples #plain text if returndocs: try: doc['returndocs'] = yaml.safe_load(returndocs) except: print("could not load yaml: %s" % returndocs) raise else: doc['returndocs'] = None # here is where we build the table of contents... try: text = template.render(doc) except Exception, e: raise AnsibleError("Failed to render doc for %s: %s" % (fname, str(e)))
def main(): p = optparse.OptionParser( version=version("%prog"), usage='usage: %prog [options] [module...]', description='Show Ansible module documentation', ) p.add_option("-M", "--module-path", action="store", dest="module_path", default=MODULEDIR, help="Ansible modules/ directory") p.add_option('-v', action='version', help='Show version number and exit') (options, args) = p.parse_args() if options.module_path is not None: for i in options.module_path.split(os.pathsep): utils.plugins.module_finder.add_directory(i) def print_paths(finder): ''' Returns a string suitable for printing of the search path ''' # Uses a list to get the order right ret = [] for i in finder._get_paths(): if i not in ret: ret.append(i) return os.pathsep.join(ret) has_error = False for module in args: filename = utils.plugins.module_finder.find_plugin(module) if filename is None: sys.stderr.write("module %s not found in %s\n" % (module, print_paths(utils.plugins.module_finder))) continue if any(filename.endswith(x) for x in BLACKLIST_EXTS): continue try: doc, plainexamples, returndocs = module_docs.get_docstring(filename) except: traceback.print_exc() sys.stderr.write("ERROR: module %s has a documentation error formatting or is missing documentation\n" % module) has_error = True continue if doc is None: # this typically means we couldn't even parse the docstring, not just that the YAML is busted, # probably a quoting issue. sys.stderr.write("ERROR: module %s missing documentation (or could not parse documentation)\n" % module) has_error = True else: all_keys = [] for (k,v) in doc['options'].iteritems(): all_keys.append(k) all_keys = sorted(all_keys) doc['option_keys'] = all_keys doc['filename'] = filename doc['docuri'] = doc['module'].replace('_', '-') doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d') doc['plainexamples'] = plainexamples doc['returndocs'] = returndocs text = get_man_text(doc) try: text.decode('ascii') except (UnicodeDecodeError, UnicodeEncodeError): traceback.print_exc() sys.stderr.write("ERROR: module %s has an documentation encoding error\n" % module) has_error = True if has_error: sys.exit(1) else: sys.exit(0)
def validate(self): super(ModuleValidator, self).validate() # Blacklists -- these files are not checked if not frozenset((self.basename, self.name)).isdisjoint(self.BLACKLIST): return for pat in self.BLACKLIST_PATTERNS: if fnmatch(self.basename, pat): return # if self._powershell_module(): # self.warnings.append('Cannot check powershell modules at this ' # 'time. Skipping') # return if not self._python_module() and not self._powershell_module(): self.errors.append('Official Ansible modules must have a .py ' 'extension for python modules or a .ps1 ' 'for powershell modules') self._python_module_override = True if self._python_module() and self.ast is None: self.errors.append('Python SyntaxError while parsing module') return if self._python_module(): sys_stdout = sys.stdout sys_stderr = sys.stderr sys.stdout = sys.stderr = StringIO() setattr(sys.stdout, 'encoding', sys_stdout.encoding) setattr(sys.stderr, 'encoding', sys_stderr.encoding) try: doc, examples, ret = get_docstring(self.path, verbose=True) trace = None except: doc = None _, examples, ret = self._get_docs() trace = traceback.format_exc() finally: sys.stdout = sys_stdout sys.stderr = sys_stderr if trace: self.traces.append(trace) if not bool(doc): self.errors.append('Invalid or no DOCUMENTATION provided') else: self._check_version_added(doc) if not bool(examples): self.errors.append('No EXAMPLES provided') if not bool(ret): if self._is_new_module(): self.errors.append('No RETURN provided') else: self.warnings.append('No RETURN provided') else: try: yaml.safe_load(ret) except: self.errors.append('RETURN is not valid YAML') self.traces.append(traceback.format_exc()) if self._python_module() and not self._just_docs(): self._check_for_sys_exit() self._find_json_import() self._find_requests_import() main = self._find_main_call() self._find_module_utils(main) self._find_has_import() self._check_for_tabs() self._find_redeclarations() if self._powershell_module(): self._find_ps_replacers() self._find_ps_docs_py_file() self._check_for_gpl3_header() if not self._just_docs(): self._check_interpreter(powershell=self._powershell_module())
def main(): argument_spec = { "path": { "required": True, "type": 'str' }, "exclusion_filters": { "required": False, "type": 'list' } } module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False) root_path = module.params['path'] root_path = os.path.abspath(root_path) if not os.path.isdir(root_path): module.fail_json(msg=root_path + " is not a directory or not exists.") exclusion_filters = module.params['exclusion_filters'] doc_list = list() for root, dirs, files in os.walk(root_path): for file_name in sorted(files): # Skips all non python files if not fnmatch(file_name, "*.py"): continue # Skips according exclusion_filters if check_exclusion(file_name, exclusion_filters): continue try: module_file_name = os.path.normpath(os.path.join(root, file_name)) # Tuple size may vary according to the Ansible version docstring = module_docs.get_docstring(module_file_name) doc = docstring[0] plainexamples = docstring[1] returndocs = docstring[2] if doc: doc = format_dict(doc) if plainexamples: doc['examples'] = plainexamples.split('\n') if returndocs and returndocs.strip() != '#': try: doc['returns'] = yaml.load(returndocs) except Exception as e: errors.append("{0} - Failed yaml.load(doc['returns']) - {1}".format(file_name, e.args[0])) doc_list.append(doc) else: errors.append(file_name + " - No docs found") except Exception as ex: errors.append("{0} - {1}".format(file_name, ex.args[0])) module.exit_json(modules_docs=doc_list, errors=errors)