def __init__(self,name,files_registry): self.config = SingleConfig() self.log = SingleLog() self._files_registry = files_registry self._all_annotations = self.config.get_annotations_definition() if name in self._all_annotations.keys(): self._annotation_definition = self._all_annotations[name] # check for allow multiple if "allow_multiple" in self._annotation_definition.keys() and self._annotation_definition["allow_multiple"] == True: self._allow_multiple = True else: self._allow_multiple = False self._role_items = {} self._all_items = {} self._duplucate_items = {} if self._annotation_definition is not None: self._find_annotation() if "special" in self._annotation_definition.keys() and self._annotation_definition["special"]: if name == "tag": self._find_tags()
def __init__(self, doc_parser): self.config = SingleConfig() self.log = SingleLog() self.log.info("Using template dir: " + self.config.get_template_base_dir()) self._parser = doc_parser self._scan_template()
def __init__(self): self.config = SingleConfig() self.log = SingleLog(self.config.debug_level) args = self._cli_args() self._parse_args(args) doc_parser = Parser() doc_generator = Generator(doc_parser) doc_generator.render()
def test_scan_template(self, tmpdir): config = SingleConfig() config.template_dir = os.path.realpath( os.path.dirname(os.path.realpath(__file__)) + "../../") config.template = "test-template" doc_generator = Generator({}) print(doc_generator.template_files) assert doc_generator.template_files.sort() == [ 'Readme.md.j2', 'sub_dir/_sample_include.md.j2', 'sub_dir/subdir.md.j2' ].sort()
def test_get_files(): config = SingleConfig() config.set_base_dir(sample_project) file_registry = Registry() fr_items = file_registry.get_files() assert "_ansible_playbook_" in fr_items assert "role1" in fr_items assert_file = project_dir+"/test/test-project/test.yaml" assert assert_file in fr_items[PLAYBOOK_ROLE_NAME] assert_role_file = project_dir+"/test/test-project/roles/role1/tasks/test.yaml" assert assert_role_file in fr_items["role1"] print("")
def test_render(self, tmpdir): config = SingleConfig() config.template_dir = os.path.realpath( os.path.dirname(os.path.realpath(__file__)) + "../../") config.template = "test-template" config.output_dir = str(tmpdir) + "/generated_doc" doc_parser = MocParser() doc_generator = Generator(doc_parser) doc_generator.render() rendered_file = open(str(tmpdir) + "/generated_doc/Readme.md", "r") line = rendered_file.read() print(line)
class Annotation: _files_registry = None # reference to current file registry _annotation_definition = None # current annotation definition _all_annotations = None # reference to defined annotation definition _allow_multiple = None _current_file = None _current_file_pos = None _current_role = None _current_line = None _file_handler = None _all_items = {} _duplucate_items = {} _role_items = {} def __init__(self,name,files_registry): self.config = SingleConfig() self.log = SingleLog() self._files_registry = files_registry self._all_annotations = self.config.get_annotations_definition() if name in self._all_annotations.keys(): self._annotation_definition = self._all_annotations[name] # check for allow multiple if "allow_multiple" in self._annotation_definition.keys() and self._annotation_definition["allow_multiple"] == True: self._allow_multiple = True else: self._allow_multiple = False self._role_items = {} self._all_items = {} self._duplucate_items = {} if self._annotation_definition is not None: self._find_annotation() if "special" in self._annotation_definition.keys() and self._annotation_definition["special"]: if name == "tag": self._find_tags() def get_details(self): return { "all": self._all_items, "duplicates": self._duplucate_items, "role_items": self._role_items, } def _find_annotation(self): regex = "(\#\ *\@"+self._annotation_definition["name"]+"\ +.*)" for role, files_in_role in self._files_registry.get_files().items(): for file in files_in_role: # reset stats self._current_line = 1 self._current_file = file self._current_role = role self._file_handler = open(file, encoding='utf8') self._current_file_pos = 0 while True: line = self._file_handler.readline() if not line: break if re.match(regex, line): item = self._get_annotation_data(line,self._annotation_definition["name"]) # print(item.get_obj()) self._populate_item(item) self._current_line += 1 self._file_handler.close() def _populate_item(self,item): if self._allow_multiple: # all items if item.key not in self._all_items.keys(): self._all_items[item.key] = [] self._all_items[item.key].append(item.get_obj()) if item.role not in self._role_items.keys(): self._role_items[item.role] = {} if item.key not in self._role_items[item.role].keys(): self._role_items[item.role][item.key] = [] self._role_items[item.role][item.key].append(item.get_obj()) else: if item.key not in self._all_items.keys(): self._all_items[item.key] = item.get_obj() else: # add to duplicates # print("Dup:" + str(item.key)) if item.key not in self._duplucate_items.keys(): self._duplucate_items[item.key] = [] self._duplucate_items[item.key].append(self._all_items[item.key]) self._duplucate_items[item.key].append(item.get_obj()) # role items if item.role not in self._role_items.keys(): self._role_items[item.role] = {} if item.key not in self._role_items[item.role].keys(): self._role_items[item.role][item.key] = item.get_obj() def _get_annotation_data(self,line,name): """ make some string conversion on a line in order to get the relevant data :param line: """ item = AnnotationItem() # fill some more data if self._current_file: # item.file = self._current_file item.file = self._current_file[len(self.config.get_base_dir()) +1 :] if self._current_role: item.role = self._current_role if self._current_line: item.line = self._current_line # step1 remove the annotation # reg1 = "(\#\ *\@"++"\ *)" reg1 = "(\#\ *\@"+name+"\ *)" line1 = re.sub(reg1, '', line).strip() # print("line1: '"+line1+"'") # step2 split annotation and comment by # parts = line1.split("#") # step3 take the main key value from the annotation subparts = parts[0].split(":",1) key = str(subparts[0].strip()) if key.strip() == "": key = "_unset_" item.key = key if len(subparts)>1: item.value = subparts[1].strip() # step4 check for multiline description multiline = "" stars_with_annotation = '(\#\ *[\@][\w]+)' current_file_position = self._file_handler.tell() while True: next_line = self._file_handler.readline() if not next_line.strip(): self._file_handler.seek(current_file_position) break # match if annotation in line if re.match(stars_with_annotation, next_line): self._file_handler.seek(current_file_position) break # match if empty line or commented empty line test_line = next_line.replace("#", "").strip() if len(test_line) == 0: self._file_handler.seek(current_file_position) break # match if does not start with comment test_line2 = next_line.strip() if test_line2[:1] != "#": self._file_handler.seek(current_file_position) break if name == "example": multiline += next_line.replace("#", "", 1) else: multiline += " "+test_line.strip() # step5 take the description, there is something after # if len(parts) > 1: desc = parts[1].strip() desc += " "+multiline.strip() item.desc = desc.strip() elif multiline != "": item.desc = multiline.strip() # step5, check for @example example = "" if name != "example": current_file_position = self._file_handler.tell() example_regex = "(\#\ *\@example\ +.*)" while True: next_line = self._file_handler.readline() if not next_line: self._file_handler.seek(current_file_position) break # exit if next annotation is not @example if re.match(stars_with_annotation, next_line): if "@example" not in next_line: self._file_handler.seek(current_file_position) break if re.match(example_regex, next_line): example = self._get_annotation_data(next_line,"example") # pprint.pprint(example.get_obj()) self._file_handler.seek(current_file_position) break if example != "": item.example = example.get_obj() return item def _find_tags(self): for role, files_in_role in self._files_registry.get_files().items(): for file in files_in_role: with open(file, 'r',encoding='utf8') as yaml_file: try: data = yaml.load(yaml_file, yaml.Loader) tags_found = Annotation.find_tag("tags",data) for tag in tags_found: if tag not in self.config.excluded_tags: item = AnnotationItem() # item.file = file item.file = file[len(self.config.get_base_dir()) +1 :] item.role = role item.key = tag # self._populate_item(item) if tag not in self._all_items.keys(): self._all_items[tag] = item.get_obj() # if already in all tags, check if its from the same role # if not, add to duplicate elif role != self._all_items[tag]["role"]: if tag not in self._duplucate_items.keys(): self._duplucate_items[tag] = [] self._duplucate_items[tag].append(self._all_items[tag]) self._duplucate_items[tag].append(item.get_obj()) # per role if role not in self._role_items.keys(): self._role_items[role] = {} if tag not in self._role_items[role].keys(): self._role_items[role][tag] = item.get_obj() except yaml.YAMLError as exc: print(exc) @staticmethod def find_tag(key,data): r = [] if isinstance(data,list): for d in data: tmp_r = Annotation.find_tag(key, d) if tmp_r: r = Annotation.merge_list_no_duplicates(r,tmp_r) elif isinstance(data,dict): for k,d in data.items(): if k == key: if isinstance(d,str): d = [d] r = r + d else: tmp_r = Annotation.find_tag(key, d) if tmp_r: r = Annotation.merge_list_no_duplicates(r,tmp_r) return r @staticmethod def merge_list_no_duplicates(a1,a2): """ merge two lists by overwriting the original with the second one if the key exists :param a1: :param a2: :return: """ r = [] if isinstance(a1,list) and isinstance(a2,list): r = a1 for i in a2: if i not in a1: r.append(i) return r
def __init__(self): self.config = SingleConfig() self.log = SingleLog() self._files_registry = Registry() self._populate_doc_data()
class Parser: log = None config = None _files_registry = None _annotation_data = {} _annotation_objs = {} def __init__(self): self.config = SingleConfig() self.log = SingleLog() self._files_registry = Registry() self._populate_doc_data() def _populate_doc_data(self): """ Generate the documentation data object """ for annotaion in self.config.get_annotations_names(special=True, automatic=True): self.log.info('Finding annotations for: @' + annotaion) self._annotation_objs[annotaion] = Annotation( name=annotaion, files_registry=self._files_registry) self._annotation_data[annotaion] = self._annotation_objs[ annotaion].get_details() def get_data(self): return self._annotation_data def get_annotations(self): return self._annotation_data.keys() def get_roles(self, exclude_playbook=True): roles = list(self._files_registry.get_files().keys()) if PLAYBOOK_ROLE_NAME in roles and exclude_playbook: roles.remove(PLAYBOOK_ROLE_NAME) return roles def include(self, filename): base = self.config.get_base_dir() base += '/' + filename base = os.path.abspath(base) self.log.debug('try to include:' + base) if os.path.isfile(base): text_file = open(base, 'r') lines = text_file.readlines() out = '' for line in lines: out += line return out else: # return "[include] file: "+base+" not found" return '' def is_role(self): return self.config.is_role def get_name(self): return self.config.project_name def cli_print_section(self): return self.config.use_print_template def _get_annotation(self, name, role='all', return_keys=False, return_item=None, return_multi=False): if name in self._annotation_objs.keys(): data = self._annotation_objs[name].get_details() if role == 'all': r_data = data['all'] elif role in data['role_items'].keys(): r_data = data['role_items'][role] elif role == 'play' and PLAYBOOK_ROLE_NAME in data[ 'role_items'].keys(): r_data = data['role_items'][PLAYBOOK_ROLE_NAME] else: r_data = {} if return_keys: print(list(r_data.keys())) return list(r_data.keys()) elif isinstance(return_item, str): if return_item in r_data.keys(): return r_data[return_item] else: return '' elif return_multi and self.allow_multiple(name): return r_data.items() else: if self.allow_multiple(name): # return r_data r = [] for k, v in r_data.items(): for item in v: r.append(item) return r else: r = [] for k, v in r_data.items(): r.append(v) return r else: return None def get_type(self, name, role='all'): return self._get_annotation(name, role) def get_multi_type(self, name, role='all'): return self._get_annotation(name, role, return_multi=True) def get_keys(self, name, role='all'): return self._get_annotation(name, role, True) def get_item(self, name, key, role='all'): return self._get_annotation(name, role, False, key) def get_duplicates(self, name): if name in self._annotation_objs.keys(): data = self._annotation_objs[name].get_details() return data['duplicates'].items() def has_items(self, name, role='all'): if len(self._get_annotation(name, role)) > 0: return True else: return False def allow_multiple(self, name): if name in self.config.annotations: if 'allow_multiple' in self.config.annotations[name].keys( ) and self.config.annotations[name]['allow_multiple']: return True return False def cli_left_space(self, item1='', l=25): item1 = item1.ljust(l) return item1 def capitalize(self, s): return s.capitalize() def fprn(self, string, re='Playbook'): if string == '_ansible_playbook_': return re else: return string def about(self, l='md'): if l == 'md': return 'Documentation generated using: [' + self.config.autodoc_name + '](' + self.config.autodoc_url + ')' def test(self): return 'test()'
def test_get_data(self): config = SingleConfig() doc_parser = Parser() print(doc_parser.get_data())
#!/usr/bin/python3 # -*- coding: utf-8 -*- import os import pprint from ansibleautodoc.Config import SingleConfig, Config from ansibleautodoc.Annotation import Annotation from ansibleautodoc.FileRegistry import Registry project_dir = os.path.realpath( os.path.dirname(os.path.realpath(__file__)) + '../../../') sample_project = os.path.realpath( os.path.dirname(os.path.realpath(__file__)) + '../../test-project') config = SingleConfig() config.set_base_dir(sample_project) fr = Registry() fr._doc = { '_ansible_playbook_': [sample_project + '/test.yaml'], 'role1': [sample_project + '/roles/role1/tasks/test.yaml'] } def test_get_details(): print() annotation = Annotation('meta', files_registry=fr) items = annotation.get_details() item1 = items['all']['key1']
def test_is_role(): conf = SingleConfig() conf.set_base_dir(sample_project) assert conf.is_role == False conf.set_base_dir(sample_project + '/roles/role1') assert conf.is_role == True
def __init__(self): self.config = SingleConfig() self.log = SingleLog() self._scan_for_yamls_in_project()
class Registry: _doc = {} log = None config = None def __init__(self): self.config = SingleConfig() self.log = SingleLog() self._scan_for_yamls_in_project() def get_files(self): """ :return objcect structured as: { "role_name":["/abs_path/to_file","/abs_path/to_file2"], "role2_name:["abs/path/2"] } :param :return: """ return self._doc def _scan_for_yamls_in_project(self): """ Search for Yaml files depending if we are scanning a playbook or a role :return: """ base_dir = self.config.get_base_dir() base_dir_roles = base_dir + '/roles' if not self.config.is_role: self.log.debug('Scan for playbook files: ' + base_dir) self._scan_for_yamls(base_dir, is_role=False) self.log.debug('Scan for roles in the project: ' + base_dir_roles) for entry in os.scandir(base_dir_roles): try: is_dir = entry.is_dir(follow_symlinks=False) except OSError as error: print('Error calling is_dir():', error, file=sys.stderr) continue if is_dir: self._scan_for_yamls(entry.path) else: # it is a role self.log.debug('Scan for files in a role: ' + base_dir) self._scan_for_yamls(base_dir) def _scan_for_yamls(self, base, is_role=True): """ Search for the yaml files in each project/role root and append to the corresponding object :param base: directory in witch we are searching :param is_role: is this a role directory :return: None """ extensions = YAML_EXTENSIONS base_dir = base for extension in extensions: for filename in glob.iglob(base_dir + '/**/*.' + extension, recursive=True): if self._is_excluded_yaml_file(filename, base_dir, is_role=is_role): self.log.trace('Excluding: ' + filename) else: if not is_role: self.log.trace('Adding to playbook: ' + filename) self.add_role_file(filename, PLAYBOOK_ROLE_NAME) else: role_dir = os.path.basename(base_dir) self.log.trace('Adding to role:' + role_dir + ' => ' + filename) self.add_role_file(filename, role_dir) def _is_excluded_yaml_file(self, file, role_base_dir=None, is_role=True): """ sub method for handling file exclusions based on the full path starts with :param file: :param role_base_dir: :param is_role: :return: """ if is_role: base_dir = role_base_dir excluded = self.config.excluded_roles_dirs.copy() else: base_dir = self.config.get_base_dir() excluded = self.config.excluded_playbook_dirs.copy() excluded.append('roles') is_filtered = False for excluded_dir in excluded: if file.startswith(base_dir + '/' + excluded_dir): return True return is_filtered def add_role_file(self, path, role_name): self.log.trace('add_role_file(' + path + ',' + role_name + ')') if role_name not in self._doc.keys(): self._doc[role_name] = [] self._doc[role_name].append(path)
class AnsibleAutodoc: def __init__(self): self.config = SingleConfig() self.log = SingleLog(self.config.debug_level) args = self._cli_args() self._parse_args(args) doc_parser = Parser() doc_generator = Generator(doc_parser) doc_generator.render() def _cli_args(self): """ use argparse for parsing CLI arguments :return: args objec """ usage = '''ansible-autodoc [project_directory] [options]''' parser = argparse.ArgumentParser( description= 'Generate documentation from annotated playbooks and roles using templates', usage=usage) # parser.add_argument("-f", "--force", help="Force online list", action="store_true") # parser.add_argument('-t','--type', nargs='+', help='<Required> Set flag', required=True) # parser.add_argument('-t','--type', nargs='+', help='<Required> Set flag') parser.add_argument('project_dir', nargs='?', default=os.getcwd(), help="Project directory to scan, " "if empty current working will be used.") parser.add_argument('-C', "--conf", nargs='?', default="", help="specify an configuration file") parser.add_argument('-o', action="store", dest="output", type=str, help='Define the destination ' 'folder of your documenation') parser.add_argument('-y', action='store_true', help='overwrite the output without asking') parser.add_argument('-D', "--dry", action='store_true', help='Dry runt without writing') parser.add_argument("--sample-config", action='store_true', help='Print the sample configuration yaml file') parser.add_argument( '-p', nargs='?', default="_unset_", help='use print template instead of writing to files, ' 'sections: all, info, tags, todo, var') parser.add_argument('-V', "--version", action='store_true', help='Get versions') debug_level = parser.add_mutually_exclusive_group() debug_level.add_argument('-v', action='store_true', help='Set debug level to info') debug_level.add_argument('-vv', action='store_true', help='Set debug level to debug') debug_level.add_argument('-vvv', action='store_true', help='Set debug level to trace') return parser.parse_args() def _parse_args(self, args): """ Use an args object to apply all the configuration combinations to the config object :param args: :return: None """ self.config.set_base_dir(os.path.abspath(args.project_dir)) # search for config file if args.conf != "": conf_file = os.path.abspath(args.conf) if os.path.isfile(conf_file) and os.path.basename( conf_file) == self.config.config_file_name: self.config.load_config_file(conf_file) # re apply log level based on config self.log.set_level(self.config.debug_level) else: self.log.warn("No configuration file found: " + conf_file) else: conf_file = self.config.get_base_dir( ) + "/" + self.config.config_file_name if os.path.isfile(conf_file): self.config.load_config_file(conf_file) # re apply log level based on config self.log.set_level(self.config.debug_level) # sample configuration if args.sample_config: print(self.config.sample_config) sys.exit() # version if args.version: print(__version__) sys.exit() # Debug levels if args.v is True: self.log.set_level("info") elif args.vv is True: self.log.set_level("debug") elif args.vvv is True: self.log.set_level("trace") # need to send the message after the log levels have been set self.log.debug("using configuration file: " + conf_file) # Overwrite if args.y is True: self.config.template_overwrite = True # Dry run if args.dry is True: self.config.dry_run = True if self.log.log_level > 1: self.log.set_level(1) self.log.info( "Running in Dry mode: Therefore setting log level at least to INFO" ) # Print template if args.p == "_unset_": pass elif args.p is None: self.config.use_print_template = "all" else: self.config.use_print_template = args.p # output dir if args.output is not None: self.config.output_dir = os.path.abspath(args.output) # some debug self.log.debug(args) self.log.info("Using base dir: " + self.config.get_base_dir()) if self.config.is_role: self.log.info("This is detected as: ROLE ") elif self.config.is_role is not None and not self.config.is_role: self.log.info("This is detected as: PLAYBOOK ") else: self.log.error([ self.config.get_base_dir() + "/roles", self.config.get_base_dir() + "/tasks" ], "No ansible root project found, checked for: ") sys.exit(1)
class Generator: template_files = [] extension = "j2" _parser = None def __init__(self, doc_parser): self.config = SingleConfig() self.log = SingleLog() self.log.info("Using template dir: " + self.config.get_template_base_dir()) self._parser = doc_parser self._scan_template() def _scan_template(self): """ Search for Jinja2 (.j2) files to apply to the destination :return: None """ base_dir = self.config.get_template_base_dir() for file in glob.iglob(base_dir + '/**/*.' + self.extension, recursive=True): relative_file = file[len(base_dir) + 1:] if ntpath.basename(file)[:1] != "_": self.log.trace("[GENERATOR] found template file: " + relative_file) self.template_files.append(relative_file) else: self.log.debug("[GENERATOR] ignoring template file: " + relative_file) def _create_dir(self, dir): if not self.config.dry_run: os.makedirs(dir, exist_ok=True) else: self.log.info("[GENERATOR][DRY] Creating dir: " + dir) def _write_doc(self): files_to_overwite = [] for file in self.template_files: doc_file = self.config.get_output_dir( ) + "/" + file[:-len(self.extension) - 1] if os.path.isfile(doc_file): files_to_overwite.append(doc_file) if len(files_to_overwite ) > 0 and self.config.template_overwrite is False: SingleLog.print("This files will be overwritten:", files_to_overwite) if not self.config.dry_run: resulst = FileUtils.query_yes_no("do you want to continue?") if resulst != "yes": sys.exit() for file in self.template_files: doc_file = self.config.get_output_dir( ) + "/" + file[:-len(self.extension) - 1] source_file = self.config.get_template_base_dir() + "/" + file self.log.trace("[GENERATOR] Writing doc output to: " + doc_file + " from: " + source_file) # make sure the directory exists self._create_dir(os.path.dirname(os.path.realpath(doc_file))) if os.path.exists(source_file) and os.path.isfile(source_file): with open(source_file, 'r') as template: data = template.read() if data is not None: try: data = Environment( loader=FileSystemLoader( self.config.get_template_base_dir()), lstrip_blocks=True, trim_blocks=True).from_string(data).render( self._parser.get_data(), r=self._parser) if not self.config.dry_run: with open(doc_file, 'w') as outfile: outfile.write(data) self.log.info("Writing to: " + doc_file) else: self.log.info("[GENERATOR][DRY] Writing to: " + doc_file) except jinja2.exceptions.UndefinedError as e: self.log.error( "Jinja2 templating error: <" + str(e) + "> when loading file: \"" + file + "\", run in debug mode to see full except") if self.log.log_level < 1: raise except UnicodeEncodeError as e: self.log.error( "At the moment I'm unable to print special chars: <" + str(e) + ">, run in debug mode to see full except") if self.log.log_level < 1: raise sys.exit() def print_to_cli(self): for file in self.template_files: source_file = self.config.get_template_base_dir() + "/" + file with open(source_file, 'r') as template: data = template.read() if data is not None: try: data = Environment( loader=FileSystemLoader( self.config.get_template_base_dir()), lstrip_blocks=True, trim_blocks=True).from_string(data).render( self._parser.get_data(), r=self._parser) print(data) except jinja2.exceptions.UndefinedError as e: self.log.error( "Jinja2 templating error: <" + str(e) + "> when loading file: \"" + file + "\", run in debug mode to see full except") if self.log.log_level < 1: raise except UnicodeEncodeError as e: self.log.error( "At the moment I'm unable to print special chars: <" + str(e) + ">, run in debug mode to see full except") if self.log.log_level < 1: raise except: print("Unexpected error:", sys.exc_info()[0]) raise def render(self): if self.config.use_print_template: self.print_to_cli() else: self.log.info("Using output dir: " + self.config.get_output_dir()) self._write_doc()