def __init__(self, doc_parser):
self.config = SingleConfig()
self.log = SingleLog()
self.log.info("Using template dir: " +
self.config.get_template_base_dir())
self._parser = doc_parser
self._scan_template()
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog(self.config.debug_level)
args = self._cli_args()
self._parse_args(args)
doc_parser = Parser()
doc_generator = Generator(doc_parser)
doc_generator.render()
def test_print_in_info(self,capsys):
log = SingleLog()
log.set_level('info')
log.debug('test message')
log.info('test message')
captured = capsys.readouterr()
assert captured.out == '*INFO*: test message\n'
def test_print_in_info(self, capsys):
log = SingleLog()
log.set_level("info")
log.debug("test message")
log.info("test message")
captured = capsys.readouterr()
assert captured.out == "*INFO*: test message\n"
def __init__(self,name,files_registry):
self.config = SingleConfig()
self.log = SingleLog()
self._files_registry = files_registry
self._all_annotations = self.config.get_annotations_definition()
if name in self._all_annotations.keys():
self._annotation_definition = self._all_annotations[name]
# check for allow multiple
if "allow_multiple" in self._annotation_definition.keys() and self._annotation_definition["allow_multiple"] == True:
self._allow_multiple = True
else:
self._allow_multiple = False
self._role_items = {}
self._all_items = {}
self._duplucate_items = {}
if self._annotation_definition is not None:
self._find_annotation()
if "special" in self._annotation_definition.keys() and self._annotation_definition["special"]:
if name == "tag":
self._find_tags()
def _write_doc(self):
files_to_overwite = []
for file in self.template_files:
doc_file=self.config.get_output_dir()+'/'+file[:-len(self.extension)-1]
if os.path.isfile(doc_file):
files_to_overwite.append(doc_file)
if len(files_to_overwite) > 0 and self.config.template_overwrite is False:
SingleLog.print('This files will be overwritten:',files_to_overwite)
if not self.config.dry_run:
resulst = FileUtils.query_yes_no('do you want to continue?')
if resulst != 'yes':
sys.exit()
for file in self.template_files:
doc_file = self.config.get_output_dir()+'/'+file[:-len(self.extension)-1]
source_file = self.config.get_template_base_dir()+'/'+file
self.log.trace('[GENERATOR] Writing doc output to: '+doc_file+' from: '+source_file)
# make sure the directory exists
self._create_dir(os.path.dirname(os.path.realpath(doc_file)))
if os.path.exists(source_file) and os.path.isfile(source_file):
with open(source_file, 'r') as template:
data = template.read()
if data is not None:
try:
data = Environment(loader=FileSystemLoader(self.config.get_template_base_dir()),lstrip_blocks=True, trim_blocks=True).from_string(data).render(self._parser.get_data(),r=self._parser)
if not self.config.dry_run:
with open(doc_file, 'w') as outfile:
outfile.write(data)
self.log.info('Writing to: '+doc_file)
else:
self.log.info('[GENERATOR][DRY] Writing to: '+doc_file)
except jinja2.exceptions.UndefinedError as e:
self.log.error('Jinja2 templating error: <'+str(e)+"> when loading file: \""+file+"\", run in debug mode to see full except")
if self.log.log_level < 1:
raise
except UnicodeEncodeError as e:
self.log.error("At the moment I'm unable to print special chars: <"+str(e)+'>, run in debug mode to see full except')
if self.log.log_level < 1:
raise
sys.exit()
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog()
self._files_registry = Registry()
self._populate_doc_data()
class Parser:
log = None
config = None
_files_registry = None
_annotation_data = {}
_annotation_objs = {}
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog()
self._files_registry = Registry()
self._populate_doc_data()
def _populate_doc_data(self):
"""
Generate the documentation data object
"""
for annotaion in self.config.get_annotations_names(special=True,
automatic=True):
self.log.info('Finding annotations for: @' + annotaion)
self._annotation_objs[annotaion] = Annotation(
name=annotaion, files_registry=self._files_registry)
self._annotation_data[annotaion] = self._annotation_objs[
annotaion].get_details()
def get_data(self):
return self._annotation_data
def get_annotations(self):
return self._annotation_data.keys()
def get_roles(self, exclude_playbook=True):
roles = list(self._files_registry.get_files().keys())
if PLAYBOOK_ROLE_NAME in roles and exclude_playbook:
roles.remove(PLAYBOOK_ROLE_NAME)
return roles
def include(self, filename):
base = self.config.get_base_dir()
base += '/' + filename
base = os.path.abspath(base)
self.log.debug('try to include:' + base)
if os.path.isfile(base):
text_file = open(base, 'r')
lines = text_file.readlines()
out = ''
for line in lines:
out += line
return out
else:
# return "[include] file: "+base+" not found"
return ''
def is_role(self):
return self.config.is_role
def get_name(self):
return self.config.project_name
def cli_print_section(self):
return self.config.use_print_template
def _get_annotation(self,
name,
role='all',
return_keys=False,
return_item=None,
return_multi=False):
if name in self._annotation_objs.keys():
data = self._annotation_objs[name].get_details()
if role == 'all':
r_data = data['all']
elif role in data['role_items'].keys():
r_data = data['role_items'][role]
elif role == 'play' and PLAYBOOK_ROLE_NAME in data[
'role_items'].keys():
r_data = data['role_items'][PLAYBOOK_ROLE_NAME]
else:
r_data = {}
if return_keys:
print(list(r_data.keys()))
return list(r_data.keys())
elif isinstance(return_item, str):
if return_item in r_data.keys():
return r_data[return_item]
else:
return ''
elif return_multi and self.allow_multiple(name):
return r_data.items()
else:
if self.allow_multiple(name):
# return r_data
r = []
for k, v in r_data.items():
for item in v:
r.append(item)
return r
else:
r = []
for k, v in r_data.items():
r.append(v)
return r
else:
return None
def get_type(self, name, role='all'):
return self._get_annotation(name, role)
def get_multi_type(self, name, role='all'):
return self._get_annotation(name, role, return_multi=True)
def get_keys(self, name, role='all'):
return self._get_annotation(name, role, True)
def get_item(self, name, key, role='all'):
return self._get_annotation(name, role, False, key)
def get_duplicates(self, name):
if name in self._annotation_objs.keys():
data = self._annotation_objs[name].get_details()
return data['duplicates'].items()
def has_items(self, name, role='all'):
if len(self._get_annotation(name, role)) > 0:
return True
else:
return False
def allow_multiple(self, name):
if name in self.config.annotations:
if 'allow_multiple' in self.config.annotations[name].keys(
) and self.config.annotations[name]['allow_multiple']:
return True
return False
def cli_left_space(self, item1='', l=25):
item1 = item1.ljust(l)
return item1
def capitalize(self, s):
return s.capitalize()
def fprn(self, string, re='Playbook'):
if string == '_ansible_playbook_':
return re
else:
return string
def about(self, l='md'):
if l == 'md':
return 'Documentation generated using: [' + self.config.autodoc_name + '](' + self.config.autodoc_url + ')'
def test(self):
return 'test()'
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog()
self._scan_for_yamls_in_project()
class Registry:
_doc = {}
log = None
config = None
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog()
self._scan_for_yamls_in_project()
def get_files(self):
"""
:return objcect structured as:
{
"role_name":["/abs_path/to_file","/abs_path/to_file2"],
"role2_name:["abs/path/2"]
}
:param
:return:
"""
return self._doc
def _scan_for_yamls_in_project(self):
"""
Search for Yaml files depending if we are scanning a playbook or a role
:return:
"""
base_dir = self.config.get_base_dir()
base_dir_roles = base_dir + '/roles'
if not self.config.is_role:
self.log.debug('Scan for playbook files: ' + base_dir)
self._scan_for_yamls(base_dir, is_role=False)
self.log.debug('Scan for roles in the project: ' + base_dir_roles)
for entry in os.scandir(base_dir_roles):
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError as error:
print('Error calling is_dir():', error, file=sys.stderr)
continue
if is_dir:
self._scan_for_yamls(entry.path)
else: # it is a role
self.log.debug('Scan for files in a role: ' + base_dir)
self._scan_for_yamls(base_dir)
def _scan_for_yamls(self, base, is_role=True):
"""
Search for the yaml files in each project/role root and append to the corresponding object
:param base: directory in witch we are searching
:param is_role: is this a role directory
:return: None
"""
extensions = YAML_EXTENSIONS
base_dir = base
for extension in extensions:
for filename in glob.iglob(base_dir + '/**/*.' + extension,
recursive=True):
if self._is_excluded_yaml_file(filename,
base_dir,
is_role=is_role):
self.log.trace('Excluding: ' + filename)
else:
if not is_role:
self.log.trace('Adding to playbook: ' + filename)
self.add_role_file(filename, PLAYBOOK_ROLE_NAME)
else:
role_dir = os.path.basename(base_dir)
self.log.trace('Adding to role:' + role_dir + ' => ' +
filename)
self.add_role_file(filename, role_dir)
def _is_excluded_yaml_file(self, file, role_base_dir=None, is_role=True):
"""
sub method for handling file exclusions based on the full path starts with
:param file:
:param role_base_dir:
:param is_role:
:return:
"""
if is_role:
base_dir = role_base_dir
excluded = self.config.excluded_roles_dirs.copy()
else:
base_dir = self.config.get_base_dir()
excluded = self.config.excluded_playbook_dirs.copy()
excluded.append('roles')
is_filtered = False
for excluded_dir in excluded:
if file.startswith(base_dir + '/' + excluded_dir):
return True
return is_filtered
def add_role_file(self, path, role_name):
self.log.trace('add_role_file(' + path + ',' + role_name + ')')
if role_name not in self._doc.keys():
self._doc[role_name] = []
self._doc[role_name].append(path)
class AnsibleAutodoc:
def __init__(self):
self.config = SingleConfig()
self.log = SingleLog(self.config.debug_level)
args = self._cli_args()
self._parse_args(args)
doc_parser = Parser()
doc_generator = Generator(doc_parser)
doc_generator.render()
def _cli_args(self):
"""
use argparse for parsing CLI arguments
:return: args objec
"""
usage = '''ansible-autodoc [project_directory] [options]'''
parser = argparse.ArgumentParser(
description=
'Generate documentation from annotated playbooks and roles using templates',
usage=usage)
# parser.add_argument("-f", "--force", help="Force online list", action="store_true")
# parser.add_argument('-t','--type', nargs='+', help='<Required> Set flag', required=True)
# parser.add_argument('-t','--type', nargs='+', help='<Required> Set flag')
parser.add_argument('project_dir',
nargs='?',
default=os.getcwd(),
help="Project directory to scan, "
"if empty current working will be used.")
parser.add_argument('-C',
"--conf",
nargs='?',
default="",
help="specify an configuration file")
parser.add_argument('-o',
action="store",
dest="output",
type=str,
help='Define the destination '
'folder of your documenation')
parser.add_argument('-y',
action='store_true',
help='overwrite the output without asking')
parser.add_argument('-D',
"--dry",
action='store_true',
help='Dry runt without writing')
parser.add_argument("--sample-config",
action='store_true',
help='Print the sample configuration yaml file')
parser.add_argument(
'-p',
nargs='?',
default="_unset_",
help='use print template instead of writing to files, '
'sections: all, info, tags, todo, var')
parser.add_argument('-V',
"--version",
action='store_true',
help='Get versions')
debug_level = parser.add_mutually_exclusive_group()
debug_level.add_argument('-v',
action='store_true',
help='Set debug level to info')
debug_level.add_argument('-vv',
action='store_true',
help='Set debug level to debug')
debug_level.add_argument('-vvv',
action='store_true',
help='Set debug level to trace')
return parser.parse_args()
def _parse_args(self, args):
"""
Use an args object to apply all the configuration combinations to the config object
:param args:
:return: None
"""
self.config.set_base_dir(os.path.abspath(args.project_dir))
# search for config file
if args.conf != "":
conf_file = os.path.abspath(args.conf)
if os.path.isfile(conf_file) and os.path.basename(
conf_file) == self.config.config_file_name:
self.config.load_config_file(conf_file)
# re apply log level based on config
self.log.set_level(self.config.debug_level)
else:
self.log.warn("No configuration file found: " + conf_file)
else:
conf_file = self.config.get_base_dir(
) + "/" + self.config.config_file_name
if os.path.isfile(conf_file):
self.config.load_config_file(conf_file)
# re apply log level based on config
self.log.set_level(self.config.debug_level)
# sample configuration
if args.sample_config:
print(self.config.sample_config)
sys.exit()
# version
if args.version:
print(__version__)
sys.exit()
# Debug levels
if args.v is True:
self.log.set_level("info")
elif args.vv is True:
self.log.set_level("debug")
elif args.vvv is True:
self.log.set_level("trace")
# need to send the message after the log levels have been set
self.log.debug("using configuration file: " + conf_file)
# Overwrite
if args.y is True:
self.config.template_overwrite = True
# Dry run
if args.dry is True:
self.config.dry_run = True
if self.log.log_level > 1:
self.log.set_level(1)
self.log.info(
"Running in Dry mode: Therefore setting log level at least to INFO"
)
# Print template
if args.p == "_unset_":
pass
elif args.p is None:
self.config.use_print_template = "all"
else:
self.config.use_print_template = args.p
# output dir
if args.output is not None:
self.config.output_dir = os.path.abspath(args.output)
# some debug
self.log.debug(args)
self.log.info("Using base dir: " + self.config.get_base_dir())
if self.config.is_role:
self.log.info("This is detected as: ROLE ")
elif self.config.is_role is not None and not self.config.is_role:
self.log.info("This is detected as: PLAYBOOK ")
else:
self.log.error([
self.config.get_base_dir() + "/roles",
self.config.get_base_dir() + "/tasks"
], "No ansible root project found, checked for: ")
sys.exit(1)
class Generator:
template_files = []
extension = "j2"
_parser = None
def __init__(self, doc_parser):
self.config = SingleConfig()
self.log = SingleLog()
self.log.info("Using template dir: " +
self.config.get_template_base_dir())
self._parser = doc_parser
self._scan_template()
def _scan_template(self):
"""
Search for Jinja2 (.j2) files to apply to the destination
:return: None
"""
base_dir = self.config.get_template_base_dir()
for file in glob.iglob(base_dir + '/**/*.' + self.extension,
recursive=True):
relative_file = file[len(base_dir) + 1:]
if ntpath.basename(file)[:1] != "_":
self.log.trace("[GENERATOR] found template file: " +
relative_file)
self.template_files.append(relative_file)
else:
self.log.debug("[GENERATOR] ignoring template file: " +
relative_file)
def _create_dir(self, dir):
if not self.config.dry_run:
os.makedirs(dir, exist_ok=True)
else:
self.log.info("[GENERATOR][DRY] Creating dir: " + dir)
def _write_doc(self):
files_to_overwite = []
for file in self.template_files:
doc_file = self.config.get_output_dir(
) + "/" + file[:-len(self.extension) - 1]
if os.path.isfile(doc_file):
files_to_overwite.append(doc_file)
if len(files_to_overwite
) > 0 and self.config.template_overwrite is False:
SingleLog.print("This files will be overwritten:",
files_to_overwite)
if not self.config.dry_run:
resulst = FileUtils.query_yes_no("do you want to continue?")
if resulst != "yes":
sys.exit()
for file in self.template_files:
doc_file = self.config.get_output_dir(
) + "/" + file[:-len(self.extension) - 1]
source_file = self.config.get_template_base_dir() + "/" + file
self.log.trace("[GENERATOR] Writing doc output to: " + doc_file +
" from: " + source_file)
# make sure the directory exists
self._create_dir(os.path.dirname(os.path.realpath(doc_file)))
if os.path.exists(source_file) and os.path.isfile(source_file):
with open(source_file, 'r') as template:
data = template.read()
if data is not None:
try:
data = Environment(
loader=FileSystemLoader(
self.config.get_template_base_dir()),
lstrip_blocks=True,
trim_blocks=True).from_string(data).render(
self._parser.get_data(), r=self._parser)
if not self.config.dry_run:
with open(doc_file, 'w') as outfile:
outfile.write(data)
self.log.info("Writing to: " + doc_file)
else:
self.log.info("[GENERATOR][DRY] Writing to: " +
doc_file)
except jinja2.exceptions.UndefinedError as e:
self.log.error(
"Jinja2 templating error: <" + str(e) +
"> when loading file: \"" + file +
"\", run in debug mode to see full except")
if self.log.log_level < 1:
raise
except UnicodeEncodeError as e:
self.log.error(
"At the moment I'm unable to print special chars: <"
+ str(e) +
">, run in debug mode to see full except")
if self.log.log_level < 1:
raise
sys.exit()
def print_to_cli(self):
for file in self.template_files:
source_file = self.config.get_template_base_dir() + "/" + file
with open(source_file, 'r') as template:
data = template.read()
if data is not None:
try:
data = Environment(
loader=FileSystemLoader(
self.config.get_template_base_dir()),
lstrip_blocks=True,
trim_blocks=True).from_string(data).render(
self._parser.get_data(), r=self._parser)
print(data)
except jinja2.exceptions.UndefinedError as e:
self.log.error(
"Jinja2 templating error: <" + str(e) +
"> when loading file: \"" + file +
"\", run in debug mode to see full except")
if self.log.log_level < 1:
raise
except UnicodeEncodeError as e:
self.log.error(
"At the moment I'm unable to print special chars: <"
+ str(e) +
">, run in debug mode to see full except")
if self.log.log_level < 1:
raise
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def render(self):
if self.config.use_print_template:
self.print_to_cli()
else:
self.log.info("Using output dir: " + self.config.get_output_dir())
self._write_doc()
def test_default_log_level(self):
log = SingleLog()
assert log.log_level == 1
def test_print_list(self,capsys):
SingleLog.print('msg',['item1'])
captured = capsys.readouterr()
assert captured.out == 'msg\n [0]: item1\n'
def test_set_log_level_debug(self):
log = SingleLog()
log.set_level(0)
log.set_level('debug')
assert log.log_level == 0