class Annotation:
_files_registry = None # reference to current file registry
_annotation_definition = None # current annotation definition
_all_annotations = None # reference to defined annotation definition
_allow_multiple = None
_current_file = None
_current_file_pos = None
_current_role = None
_current_line = None
_file_handler = None
_all_items = {}
_duplucate_items = {}
_role_items = {}
def __init__(self,name,files_registry):
self.config = SingleConfig()
self.log = SingleLog()
self._files_registry = files_registry
self._all_annotations = self.config.get_annotations_definition()
if name in self._all_annotations.keys():
self._annotation_definition = self._all_annotations[name]
# check for allow multiple
if "allow_multiple" in self._annotation_definition.keys() and self._annotation_definition["allow_multiple"] == True:
self._allow_multiple = True
else:
self._allow_multiple = False
self._role_items = {}
self._all_items = {}
self._duplucate_items = {}
if self._annotation_definition is not None:
self._find_annotation()
if "special" in self._annotation_definition.keys() and self._annotation_definition["special"]:
if name == "tag":
self._find_tags()
def get_details(self):
return {
"all": self._all_items,
"duplicates": self._duplucate_items,
"role_items": self._role_items,
}
def _find_annotation(self):
regex = "(\#\ *\@"+self._annotation_definition["name"]+"\ +.*)"
for role, files_in_role in self._files_registry.get_files().items():
for file in files_in_role:
# reset stats
self._current_line = 1
self._current_file = file
self._current_role = role
self._file_handler = open(file, encoding='utf8')
self._current_file_pos = 0
while True:
line = self._file_handler.readline()
if not line:
break
if re.match(regex, line):
item = self._get_annotation_data(line,self._annotation_definition["name"])
# print(item.get_obj())
self._populate_item(item)
self._current_line += 1
self._file_handler.close()
def _populate_item(self,item):
if self._allow_multiple:
# all items
if item.key not in self._all_items.keys():
self._all_items[item.key] = []
self._all_items[item.key].append(item.get_obj())
if item.role not in self._role_items.keys():
self._role_items[item.role] = {}
if item.key not in self._role_items[item.role].keys():
self._role_items[item.role][item.key] = []
self._role_items[item.role][item.key].append(item.get_obj())
else:
if item.key not in self._all_items.keys():
self._all_items[item.key] = item.get_obj()
else:
# add to duplicates
# print("Dup:" + str(item.key))
if item.key not in self._duplucate_items.keys():
self._duplucate_items[item.key] = []
self._duplucate_items[item.key].append(self._all_items[item.key])
self._duplucate_items[item.key].append(item.get_obj())
# role items
if item.role not in self._role_items.keys():
self._role_items[item.role] = {}
if item.key not in self._role_items[item.role].keys():
self._role_items[item.role][item.key] = item.get_obj()
def _get_annotation_data(self,line,name):
"""
make some string conversion on a line in order to get the relevant data
:param line:
"""
item = AnnotationItem()
# fill some more data
if self._current_file:
# item.file = self._current_file
item.file = self._current_file[len(self.config.get_base_dir()) +1 :]
if self._current_role:
item.role = self._current_role
if self._current_line:
item.line = self._current_line
# step1 remove the annotation
# reg1 = "(\#\ *\@"++"\ *)"
reg1 = "(\#\ *\@"+name+"\ *)"
line1 = re.sub(reg1, '', line).strip()
# print("line1: '"+line1+"'")
# step2 split annotation and comment by #
parts = line1.split("#")
# step3 take the main key value from the annotation
subparts = parts[0].split(":",1)
key = str(subparts[0].strip())
if key.strip() == "":
key = "_unset_"
item.key = key
if len(subparts)>1:
item.value = subparts[1].strip()
# step4 check for multiline description
multiline = ""
stars_with_annotation = '(\#\ *[\@][\w]+)'
current_file_position = self._file_handler.tell()
while True:
next_line = self._file_handler.readline()
if not next_line.strip():
self._file_handler.seek(current_file_position)
break
# match if annotation in line
if re.match(stars_with_annotation, next_line):
self._file_handler.seek(current_file_position)
break
# match if empty line or commented empty line
test_line = next_line.replace("#", "").strip()
if len(test_line) == 0:
self._file_handler.seek(current_file_position)
break
# match if does not start with comment
test_line2 = next_line.strip()
if test_line2[:1] != "#":
self._file_handler.seek(current_file_position)
break
if name == "example":
multiline += next_line.replace("#", "", 1)
else:
multiline += " "+test_line.strip()
# step5 take the description, there is something after #
if len(parts) > 1:
desc = parts[1].strip()
desc += " "+multiline.strip()
item.desc = desc.strip()
elif multiline != "":
item.desc = multiline.strip()
# step5, check for @example
example = ""
if name != "example":
current_file_position = self._file_handler.tell()
example_regex = "(\#\ *\@example\ +.*)"
while True:
next_line = self._file_handler.readline()
if not next_line:
self._file_handler.seek(current_file_position)
break
# exit if next annotation is not @example
if re.match(stars_with_annotation, next_line):
if "@example" not in next_line:
self._file_handler.seek(current_file_position)
break
if re.match(example_regex, next_line):
example = self._get_annotation_data(next_line,"example")
# pprint.pprint(example.get_obj())
self._file_handler.seek(current_file_position)
break
if example != "":
item.example = example.get_obj()
return item
def _find_tags(self):
for role, files_in_role in self._files_registry.get_files().items():
for file in files_in_role:
with open(file, 'r',encoding='utf8') as yaml_file:
try:
data = yaml.load(yaml_file, yaml.Loader)
tags_found = Annotation.find_tag("tags",data)
for tag in tags_found:
if tag not in self.config.excluded_tags:
item = AnnotationItem()
# item.file = file
item.file = file[len(self.config.get_base_dir()) +1 :]
item.role = role
item.key = tag
# self._populate_item(item)
if tag not in self._all_items.keys():
self._all_items[tag] = item.get_obj()
# if already in all tags, check if its from the same role
# if not, add to duplicate
elif role != self._all_items[tag]["role"]:
if tag not in self._duplucate_items.keys():
self._duplucate_items[tag] = []
self._duplucate_items[tag].append(self._all_items[tag])
self._duplucate_items[tag].append(item.get_obj())
# per role
if role not in self._role_items.keys():
self._role_items[role] = {}
if tag not in self._role_items[role].keys():
self._role_items[role][tag] = item.get_obj()
except yaml.YAMLError as exc:
print(exc)
@staticmethod
def find_tag(key,data):
r = []
if isinstance(data,list):
for d in data:
tmp_r = Annotation.find_tag(key, d)
if tmp_r:
r = Annotation.merge_list_no_duplicates(r,tmp_r)
elif isinstance(data,dict):
for k,d in data.items():
if k == key:
if isinstance(d,str):
d = [d]
r = r + d
else:
tmp_r = Annotation.find_tag(key, d)
if tmp_r:
r = Annotation.merge_list_no_duplicates(r,tmp_r)
return r
@staticmethod
def merge_list_no_duplicates(a1,a2):
"""
merge two lists by overwriting the original with the second one if the key exists
:param a1:
:param a2:
:return:
"""
r = []
if isinstance(a1,list) and isinstance(a2,list):
r = a1
for i in a2:
if i not in a1:
r.append(i)
return r
class Registry:
_doc = {}
log = None
config = None
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog()
self._scan_for_yamls_in_project()
def get_files(self):
"""
:return objcect structured as:
{
"role_name":["/abs_path/to_file","/abs_path/to_file2"],
"role2_name:["abs/path/2"]
}
:param
:return:
"""
return self._doc
def _scan_for_yamls_in_project(self):
"""
Search for Yaml files depending if we are scanning a playbook or a role
:return:
"""
base_dir = self.config.get_base_dir()
base_dir_roles = base_dir + '/roles'
if not self.config.is_role:
self.log.debug('Scan for playbook files: ' + base_dir)
self._scan_for_yamls(base_dir, is_role=False)
self.log.debug('Scan for roles in the project: ' + base_dir_roles)
for entry in os.scandir(base_dir_roles):
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError as error:
print('Error calling is_dir():', error, file=sys.stderr)
continue
if is_dir:
self._scan_for_yamls(entry.path)
else: # it is a role
self.log.debug('Scan for files in a role: ' + base_dir)
self._scan_for_yamls(base_dir)
def _scan_for_yamls(self, base, is_role=True):
"""
Search for the yaml files in each project/role root and append to the corresponding object
:param base: directory in witch we are searching
:param is_role: is this a role directory
:return: None
"""
extensions = YAML_EXTENSIONS
base_dir = base
for extension in extensions:
for filename in glob.iglob(base_dir + '/**/*.' + extension,
recursive=True):
if self._is_excluded_yaml_file(filename,
base_dir,
is_role=is_role):
self.log.trace('Excluding: ' + filename)
else:
if not is_role:
self.log.trace('Adding to playbook: ' + filename)
self.add_role_file(filename, PLAYBOOK_ROLE_NAME)
else:
role_dir = os.path.basename(base_dir)
self.log.trace('Adding to role:' + role_dir + ' => ' +
filename)
self.add_role_file(filename, role_dir)
def _is_excluded_yaml_file(self, file, role_base_dir=None, is_role=True):
"""
sub method for handling file exclusions based on the full path starts with
:param file:
:param role_base_dir:
:param is_role:
:return:
"""
if is_role:
base_dir = role_base_dir
excluded = self.config.excluded_roles_dirs.copy()
else:
base_dir = self.config.get_base_dir()
excluded = self.config.excluded_playbook_dirs.copy()
excluded.append('roles')
is_filtered = False
for excluded_dir in excluded:
if file.startswith(base_dir + '/' + excluded_dir):
return True
return is_filtered
def add_role_file(self, path, role_name):
self.log.trace('add_role_file(' + path + ',' + role_name + ')')
if role_name not in self._doc.keys():
self._doc[role_name] = []
self._doc[role_name].append(path)
class Parser:
log = None
config = None
_files_registry = None
_annotation_data = {}
_annotation_objs = {}
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog()
self._files_registry = Registry()
self._populate_doc_data()
def _populate_doc_data(self):
"""
Generate the documentation data object
"""
for annotaion in self.config.get_annotations_names(special=True,
automatic=True):
self.log.info('Finding annotations for: @' + annotaion)
self._annotation_objs[annotaion] = Annotation(
name=annotaion, files_registry=self._files_registry)
self._annotation_data[annotaion] = self._annotation_objs[
annotaion].get_details()
def get_data(self):
return self._annotation_data
def get_annotations(self):
return self._annotation_data.keys()
def get_roles(self, exclude_playbook=True):
roles = list(self._files_registry.get_files().keys())
if PLAYBOOK_ROLE_NAME in roles and exclude_playbook:
roles.remove(PLAYBOOK_ROLE_NAME)
return roles
def include(self, filename):
base = self.config.get_base_dir()
base += '/' + filename
base = os.path.abspath(base)
self.log.debug('try to include:' + base)
if os.path.isfile(base):
text_file = open(base, 'r')
lines = text_file.readlines()
out = ''
for line in lines:
out += line
return out
else:
# return "[include] file: "+base+" not found"
return ''
def is_role(self):
return self.config.is_role
def get_name(self):
return self.config.project_name
def cli_print_section(self):
return self.config.use_print_template
def _get_annotation(self,
name,
role='all',
return_keys=False,
return_item=None,
return_multi=False):
if name in self._annotation_objs.keys():
data = self._annotation_objs[name].get_details()
if role == 'all':
r_data = data['all']
elif role in data['role_items'].keys():
r_data = data['role_items'][role]
elif role == 'play' and PLAYBOOK_ROLE_NAME in data[
'role_items'].keys():
r_data = data['role_items'][PLAYBOOK_ROLE_NAME]
else:
r_data = {}
if return_keys:
print(list(r_data.keys()))
return list(r_data.keys())
elif isinstance(return_item, str):
if return_item in r_data.keys():
return r_data[return_item]
else:
return ''
elif return_multi and self.allow_multiple(name):
return r_data.items()
else:
if self.allow_multiple(name):
# return r_data
r = []
for k, v in r_data.items():
for item in v:
r.append(item)
return r
else:
r = []
for k, v in r_data.items():
r.append(v)
return r
else:
return None
def get_type(self, name, role='all'):
return self._get_annotation(name, role)
def get_multi_type(self, name, role='all'):
return self._get_annotation(name, role, return_multi=True)
def get_keys(self, name, role='all'):
return self._get_annotation(name, role, True)
def get_item(self, name, key, role='all'):
return self._get_annotation(name, role, False, key)
def get_duplicates(self, name):
if name in self._annotation_objs.keys():
data = self._annotation_objs[name].get_details()
return data['duplicates'].items()
def has_items(self, name, role='all'):
if len(self._get_annotation(name, role)) > 0:
return True
else:
return False
def allow_multiple(self, name):
if name in self.config.annotations:
if 'allow_multiple' in self.config.annotations[name].keys(
) and self.config.annotations[name]['allow_multiple']:
return True
return False
def cli_left_space(self, item1='', l=25):
item1 = item1.ljust(l)
return item1
def capitalize(self, s):
return s.capitalize()
def fprn(self, string, re='Playbook'):
if string == '_ansible_playbook_':
return re
else:
return string
def about(self, l='md'):
if l == 'md':
return 'Documentation generated using: [' + self.config.autodoc_name + '](' + self.config.autodoc_url + ')'
def test(self):
return 'test()'
class AnsibleAutodoc:
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog(self.config.debug_level)
args = self._cli_args()
self._parse_args(args)
doc_parser = Parser()
doc_generator = Generator(doc_parser)
doc_generator.render()
def _cli_args(self):
"""
use argparse for parsing CLI arguments
:return: args objec
"""
usage = '''ansible-autodoc [project_directory] [options]'''
parser = argparse.ArgumentParser(
description=
'Generate documentation from annotated playbooks and roles using templates',
usage=usage)
# parser.add_argument("-f", "--force", help="Force online list", action="store_true")
# parser.add_argument('-t','--type', nargs='+', help='<Required> Set flag', required=True)
# parser.add_argument('-t','--type', nargs='+', help='<Required> Set flag')
parser.add_argument('project_dir',
nargs='?',
default=os.getcwd(),
help="Project directory to scan, "
"if empty current working will be used.")
parser.add_argument('-C',
"--conf",
nargs='?',
default="",
help="specify an configuration file")
parser.add_argument('-o',
action="store",
dest="output",
type=str,
help='Define the destination '
'folder of your documenation')
parser.add_argument('-y',
action='store_true',
help='overwrite the output without asking')
parser.add_argument('-D',
"--dry",
action='store_true',
help='Dry runt without writing')
parser.add_argument("--sample-config",
action='store_true',
help='Print the sample configuration yaml file')
parser.add_argument(
'-p',
nargs='?',
default="_unset_",
help='use print template instead of writing to files, '
'sections: all, info, tags, todo, var')
parser.add_argument('-V',
"--version",
action='store_true',
help='Get versions')
debug_level = parser.add_mutually_exclusive_group()
debug_level.add_argument('-v',
action='store_true',
help='Set debug level to info')
debug_level.add_argument('-vv',
action='store_true',
help='Set debug level to debug')
debug_level.add_argument('-vvv',
action='store_true',
help='Set debug level to trace')
return parser.parse_args()
def _parse_args(self, args):
"""
Use an args object to apply all the configuration combinations to the config object
:param args:
:return: None
"""
self.config.set_base_dir(os.path.abspath(args.project_dir))
# search for config file
if args.conf != "":
conf_file = os.path.abspath(args.conf)
if os.path.isfile(conf_file) and os.path.basename(
conf_file) == self.config.config_file_name:
self.config.load_config_file(conf_file)
# re apply log level based on config
self.log.set_level(self.config.debug_level)
else:
self.log.warn("No configuration file found: " + conf_file)
else:
conf_file = self.config.get_base_dir(
) + "/" + self.config.config_file_name
if os.path.isfile(conf_file):
self.config.load_config_file(conf_file)
# re apply log level based on config
self.log.set_level(self.config.debug_level)
# sample configuration
if args.sample_config:
print(self.config.sample_config)
sys.exit()
# version
if args.version:
print(__version__)
sys.exit()
# Debug levels
if args.v is True:
self.log.set_level("info")
elif args.vv is True:
self.log.set_level("debug")
elif args.vvv is True:
self.log.set_level("trace")
# need to send the message after the log levels have been set
self.log.debug("using configuration file: " + conf_file)
# Overwrite
if args.y is True:
self.config.template_overwrite = True
# Dry run
if args.dry is True:
self.config.dry_run = True
if self.log.log_level > 1:
self.log.set_level(1)
self.log.info(
"Running in Dry mode: Therefore setting log level at least to INFO"
)
# Print template
if args.p == "_unset_":
pass
elif args.p is None:
self.config.use_print_template = "all"
else:
self.config.use_print_template = args.p
# output dir
if args.output is not None:
self.config.output_dir = os.path.abspath(args.output)
# some debug
self.log.debug(args)
self.log.info("Using base dir: " + self.config.get_base_dir())
if self.config.is_role:
self.log.info("This is detected as: ROLE ")
elif self.config.is_role is not None and not self.config.is_role:
self.log.info("This is detected as: PLAYBOOK ")
else:
self.log.error([
self.config.get_base_dir() + "/roles",
self.config.get_base_dir() + "/tasks"
], "No ansible root project found, checked for: ")
sys.exit(1)
