def build_class_database(include_dirs, input_dirs):
"""
Create the class database.
Returns a dict() of DatabaseItem objects. The key is the class name e.g., Diffusion.
Inputs:
include_dirs[str,list]: A space separated str or a list of include directories.
input_dirs[str,list]: A space separated str or a list of input file directories.
"""
# Build the lists from strings
if isinstance(include_dirs, str):
include_dirs = [mooseutils.eval_path(x) for x in include_dirs.split()]
if isinstance(input_dirs, str):
input_dirs = [mooseutils.eval_path(x) for x in input_dirs.split()]
# Locate filenames
headers = _locate_filenames(include_dirs, '.h')
inputs = _locate_filenames(input_dirs, '.i')
# Create the database
objects = dict()
_process(objects, headers, DEFINITION_RE, _match_definition)
_process(objects, headers, CHILD_RE, _match_child)
_process(objects, inputs, INPUT_RE, _match_input)
return objects
def build_class_database(include_dirs=None, input_dirs=None):
"""
Create the class database.
Returns a dict() of DatabaseItem objects. The key is the class name e.g., Diffusion.
Inputs:
include_dirs[list]: A space separated str or a list of include directories.
input_dirs[list]: A space separated str or a list of input file directories.
"""
# Handle environment variables
if include_dirs:
include_dirs = [mooseutils.eval_path(x) for x in include_dirs]
else:
include_dirs = [MooseDocs.ROOT_DIR]
if input_dirs:
input_dirs = [mooseutils.eval_path(x) for x in input_dirs]
else:
input_dirs = [MooseDocs.ROOT_DIR]
# Locate filenames
headers = _locate_filenames(include_dirs, '.h')
inputs = _locate_filenames(input_dirs, '.i')
# Create the database
objects = dict()
_process(objects, headers, DEFINITION_RE, _match_definition)
_process(objects, headers, CHILD_RE, _match_child)
_process(objects, inputs, INPUT_RE, _match_input)
return objects
def execute(self, **kwargs):
"""Perform app syntax checking"""
# Check that the supplied content dir exists
content_dir = mooseutils.eval_path(self.content_directory)
if not os.path.isdir(content_dir):
content_dir = os.path.join(self.working_dir, content_dir)
if not os.path.isdir(content_dir):
raise NotADirectoryError("'content_directory' input is not a directory: {}".format(content_dir))
# Populate the available list of files
file_cache = mooseutils.git_ls_files(content_dir)
# Check that the supplied exe dir exists
exe_dir = mooseutils.eval_path(self.exe_directory)
if not os.path.isdir(exe_dir):
exe_dir = os.path.join(self.working_dir, exe_dir)
if not os.path.isdir(exe_dir):
raise NotADirectoryError("'exe_directory' input is not a directory: {}".format(exe_dir))
# Locate the executable
exe = mooseutils.find_moose_executable(exe_dir, name=self.exe_name, show_error=False)
if exe is None:
raise OSError("An executable was not found in '{}' with a name '{}'.".format(exe_dir, self.exe_name))
# Determine the application type (e.g., MooseTestApp)
if self.app_types is None:
out = subprocess.check_output([exe, '--type'], encoding='utf-8')
match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$', out, flags=re.MULTILINE)
if match:
self.app_types = [match.group("type").replace('TestApp', 'App')]
# Build syntax tree if not provided
if self.app_syntax is None:
# Get the removed/alias information
remove = self._loadYamlFiles(self.remove)
alias = self._loadYamlFiles(self.alias)
unregister = self._loadYamlFiles(self.unregister)
# Build the complete syntax tree
self.app_syntax = moosesyntax.get_moose_syntax_tree(exe, remove=remove,
alias=alias, unregister=unregister)
# Perform the checks
kwargs.setdefault('syntax_prefix', mooseutils.eval_path(self.syntax_prefix))
kwargs.setdefault('object_prefix', mooseutils.eval_path(self.object_prefix))
kwargs.setdefault('allow_test_objects', self.allow_test_objects)
logger = check_syntax(self.app_syntax, self.app_types, file_cache, **kwargs)
return logger
def __initApplicationSyntax(self):
"""Initialize the application syntax."""
start = time.time()
LOG.info("Reading MOOSE application syntax...")
exe = mooseutils.eval_path(self['executable'])
exe = mooseutils.find_moose_executable(exe, show_error=False)
if exe is None:
LOG.error("Failed to locate a valid executable in %s.", self['executable'])
else:
try:
self._app_syntax = app_syntax(exe,
alias=self['alias'],
remove=self['remove'],
hide=self['hide'],
allow_test_objects=self['allow-test-objects'])
out = mooseutils.runExe(exe, ['--type'])
match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$', out, flags=re.MULTILINE)
if match:
self._app_type = match.group("type")
else:
msg = "Failed to determine application type by running the following:\n"
msg += " {} --type".format(exe)
LOG.error(msg)
except Exception as e: #pylint: disable=broad-except
msg = "Failed to load application executable from '%s', " \
"application syntax is being disabled:\n%s"
self.setActive(False)
LOG.error(msg, self.get('executable'), e.message)
LOG.info("MOOSE application syntax complete [%s sec.]", time.time() - start)
def __init__(self, *args, **kwargs):
command.CommandExtension.__init__(self, *args, **kwargs)
self._app_type = None
self._app_syntax = None
if not self['disable'] and self['executable'] is not None:
LOG.info("Reading MOOSE application syntax.")
exe = mooseutils.eval_path(self['executable'])
exe = mooseutils.find_moose_executable(exe, show_error=False)
if exe is None:
LOG.error("Failed to locate a valid executable in %s.",
self['executable'])
else:
try:
self._app_syntax = app_syntax(
exe,
alias=self['alias'],
remove=self['remove'],
hide=self['hide'],
allow_test_objects=self['allow-test-objects'])
out = mooseutils.runExe(exe, ['--type'])
match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$',
out,
flags=re.MULTILINE)
if match:
self._app_type = match.group("type")
else:
msg = "Failed to determine application type by running the following:\n"
msg += " {} --type".format(exe)
LOG.error(msg)
except Exception as e: #pylint: disable=broad-except
msg = "Failed to load application executable from '%s', " \
"application syntax is being disabled:\n%s"
LOG.error(msg, self['executable'], e.message)
LOG.info("Building MOOSE class database.")
self._database = common.build_class_database(self['includes'],
self['inputs'])
# Cache the syntax entries, search the tree is very slow
if self._app_syntax:
self._cache = dict()
self._object_cache = dict()
self._syntax_cache = dict()
for node in anytree.PreOrderIter(self._app_syntax):
if not node.removed:
self._cache[node.fullpath] = node
if node.alias:
self._cache[node.alias] = node
if isinstance(node, syntax.ObjectNode):
self._object_cache[node.fullpath] = node
if node.alias:
self._object_cache[node.alias] = node
elif isinstance(node, syntax.SyntaxNode):
self._syntax_cache[node.fullpath] = node
if node.alias:
self._syntax_cache[node.alias] = node
def execute(self, **kwargs):
"""
Computes the percent complete, number missing items, and the pass/fail status
"""
# Extract configuration parameters
specs = self.specs or 'tests'
# Get complete directory paths
root_dir = mooseutils.git_root_dir()
directories = [mooseutils.eval_path(d) for d in (self.directories or [root_dir])]
for i, d in enumerate(directories):
if not os.path.isdir(d):
directories[i] = os.path.join(root_dir, d)
if not os.path.isdir(directories[i]):
raise NotADirectoryError("Supplied directory does not exist: {}".format(d))
# Build Requirement objects and remove directory based dict
req_dict = get_requirements_from_tests(directories, specs.split(), self.include_non_testable)
requirements = []
for values in req_dict.values():
requirements += values
# Populate the lists of tests for SQARequirementDiffReport
self.test_names = set()
for req in requirements:
self.test_names.add((req.filename, req.name, req.line))
# Get list of files to search
if self.working_dirs is None: self.working_dirs = [mooseutils.git_root_dir()]
file_list = SQAReport._getFiles(self.working_dirs)
# Check the requirements
logger = check_requirements(requirements, color_text=self.color_text, file_list=file_list, **kwargs)
return logger
def __init__(self, *args, **kwargs):
command.CommandExtension.__init__(self, *args, **kwargs)
# Build requirements sets
self.__requirements = dict()
self.__dependencies = dict()
for index, (category,
info) in enumerate(self.get('categories').iteritems(), 1): #pylint: disable=no-member
specs = info.get('specs', ['tests'])
directories = []
for d in info.get('directories'):
path = mooseutils.eval_path(d)
if not os.path.isdir(path):
path = os.path.join(MooseDocs.ROOT_DIR, d)
if not os.path.isdir(path):
msg = "Input directory does not exist: %s"
LOG.error(msg, path)
directories.append(path)
# Create requirement database
self.__requirements[category] = common.get_requirements(
directories, specs, 'F', index)
# Create dependency database
self.__dependencies[category] = info.get('dependencies', [])
# Storage for requirement matrix counting (see SQARequirementMatricCommand)
self.__counts = collections.defaultdict(int)
def __initApplicationSyntax(self):
"""Initialize the application syntax."""
start = time.time()
LOG.info("Reading MOOSE application syntax...")
exe = mooseutils.eval_path(self['executable'])
exe = mooseutils.find_moose_executable(exe, name=self['app_name'], show_error=False)
if exe is None:
LOG.error("Failed to locate a valid executable in %s.", self['executable'])
else:
try:
self._app_syntax = app_syntax(exe,
alias=self['alias'],
remove=self['remove'],
hide=self['hide'],
allow_test_objects=self['allow-test-objects'])
out = mooseutils.runExe(exe, ['--type'])
match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$', out, flags=re.MULTILINE)
if match:
self._app_type = match.group("type")
else:
msg = "Failed to determine application type by running the following:\n"
msg += " {} --type".format(exe)
LOG.error(msg)
except Exception as e:
msg = "Failed to load application executable from '%s', " \
"application syntax is being disabled:\n%s"
self.setActive(False)
LOG.error(msg, exe, e)
LOG.info("MOOSE application syntax complete [%s sec.]", time.time() - start)
def doc_tree(items):
"""
Create a tree of files for processing.
Inputs:
inputs: [list[dict(),...] A list of dict items, each dict entry must contain the 'root_dir'
and 'content' fields that are passed to the doc_import function.
"""
# Error checking
if not isinstance(items, list) or any(not isinstance(x, dict)
for x in items):
LOG.error(
'The supplied items must be a list of dict items, each with a "root_dir" and '
'optionally a "content" entry.')
return None
# Define a dict for storing nodes by path
nodes = dict()
# Create the root node
nodes[()] = page.DirectoryNode(source='')
# Create the file tree
for value in items:
if 'root_dir' not in value:
LOG.error(
'The supplied items must be a list of dict items, each with a "root_dir" and '
'optionally a "content" entry.')
root = mooseutils.eval_path(value['root_dir'])
if not os.path.isabs(root):
root = os.path.join(MooseDocs.ROOT_DIR, root)
# Update the project files
MooseDocs.PROJECT_FILES.update(
mooseutils.git_ls_files(mooseutils.git_root_dir(root)))
files = doc_import(root, content=value.get('content', None))
for filename in files:
key = tuple(filename.replace(root, '').strip('/').split('/'))
# Create directory nodes if they don't exist
for i in range(1, len(key)):
dir_key = key[:i]
if dir_key not in nodes:
nodes[dir_key] = page.DirectoryNode(nodes[key[:i - 1]],
name=key[i - 1],
source=os.path.join(
root, *dir_key))
# Create the file node, if it doesn't already exist. This enforces that the first
# item in the supplied content lists is the page that is rendered.
if key not in nodes:
nodes[key] = create_file_node(nodes[key[:-1]], key[-1],
filename)
return nodes[()]
def _getFiles(locations):
"""Get complete list of files for the given *locations*."""
file_list = list()
for working_dir in locations:
path = mooseutils.eval_path(working_dir)
if mooseutils.git_is_repo(path):
file_list += mooseutils.git_ls_files(path)
else:
file_list += glob.glob(os.path.join(path, '**', '*.*'),
recursive=True)
return file_list
def __init__(self, **kwargs):
self._documents = dict()
kwargs.setdefault('required_documents', INL_DOCUMENTS)
for name in kwargs.get('required_documents'):
doc = kwargs.pop(name, None)
if doc is not None:
doc = mooseutils.eval_path(doc)
self._documents[name] = doc
super().__init__(**kwargs)
if self.working_dirs is None:
self.working_dirs = [mooseutils.git_root_dir()]
def execute(self, **kwargs):
"""Determine the status"""
file_list = list()
for working_dir in self.working_dirs:
path = mooseutils.eval_path(working_dir)
if mooseutils.is_git_repo(path):
file_list += mooseutils.git_ls_files(path)
else:
file_list += glob.glob(os.path.join(path, '**', '*.*'),
recursive=True)
logger = check_documents(self.documents, file_list, **kwargs)
return logger
def __initApplicationSyntax(self):
"""Initialize the application syntax."""
start = time.time()
LOG.info("Reading MOOSE application syntax...")
exe = mooseutils.eval_path(self['executable'])
exe = mooseutils.find_moose_executable(exe,
name=self['app_name'],
show_error=False)
self._app_exe = exe
if exe is None:
LOG.error("Failed to locate a valid executable in %s.",
self['executable'])
else:
try:
self._app_syntax = moosesyntax.get_moose_syntax_tree(
exe,
remove=self['remove'],
alias=self['alias'],
unregister=self['unregister'])
out = mooseutils.runExe(exe, ['--type'])
match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$',
out,
flags=re.MULTILINE)
if match:
self._app_type = match.group("type")
else:
msg = "Failed to determine application type by running the following:\n"
msg += " {} --type".format(exe)
LOG.error(msg)
except Exception as e:
msg = "Failed to load application executable from '{}' with the following error; " \
"application syntax is being disabled.\n".format(exe)
msg += '\n{}\n'.format(
mooseutils.colorText(traceback.format_exc(), 'GREY'))
msg += "This typically indicates that the application is not producing JSON output " \
"correctly, try running the following:\n" \
" {} --json --allow-test-objects\n".format(exe)
self.setActive(False)
LOG.error(msg)
# Enable test objects by removing the test flag (i.e., don't consider them test objects)
if self['allow-test-objects'] and (self._app_syntax is not None):
for node in moosetree.iterate(self._app_syntax):
node.test = False
LOG.info("MOOSE application syntax complete [%s sec.]",
time.time() - start)
def main(options):
"""
Main function for the build command.
Inputs:
options[argparse options]: Complete options from argparse, see MooseDocs/main.py
"""
# Make sure "large_media" exists in MOOSE
_init_large_media()
# Create translator
translator, _ = common.load_config(options.config)
if options.destination:
translator.update(destination=mooseutils.eval_path(options.destination))
translator.init()
# Replace "home" with local server
if options.serve:
home = 'http://127.0.0.1:{}'.format(options.port)
translator.renderer.update(home=home)
elif options.home:
translator.renderer.update(home=options.home)
# Dump page tree
if options.dump:
print translator.root
# Clean when --files is NOT used or when --clean is used with --files.
if ((options.files == []) or (options.files != [] and options.clean)) \
and os.path.exists(translator['destination']):
log = logging.getLogger('MooseDocs.build')
log.info("Cleaning destination %s", translator['destination'])
shutil.rmtree(translator['destination'])
# Perform check
if options.check:
check(translator)
# Perform build
if options.files:
for filename in options.files:
node = translator.root.findall(filename)[0]
node.build()
else:
translator.execute(options.num_threads)
if options.serve:
watcher = MooseDocsWatcher(translator, options)
server = livereload.Server(watcher=watcher)
server.serve(root=translator['destination'], host=options.host, port=options.port)
def doc_tree(items):
"""
Create a tree of files for processing.
Inputs:
inputs: [list[dict(),...] A list of dict items, each dict entry must contain the 'root_dir'
and 'content' fields that are passed to the doc_import function.
"""
# Error checking
if not isinstance(items, list) or any(not isinstance(x, dict) for x in items):
LOG.error('The supplied items must be a list of dict items, each with a "root_dir" and '
'optionally a "content" entry.')
return None
# Define a dict for storing nodes by path
nodes = dict()
# Create the root node
nodes[()] = page.DirectoryNode(source='')
# Create the file tree
for value in items:
if 'root_dir' not in value:
LOG.error('The supplied items must be a list of dict items, each with a "root_dir" and '
'optionally a "content" entry.')
root = mooseutils.eval_path(value['root_dir'])
if not os.path.isabs(root):
root = os.path.join(MooseDocs.ROOT_DIR, root)
# Update the project files
MooseDocs.PROJECT_FILES.update(mooseutils.git_ls_files(mooseutils.git_root_dir(root)))
files = doc_import(root, content=value.get('content', None))
for filename in files:
key = tuple(filename.replace(root, '').strip('/').split('/'))
# Create directory nodes if they don't exist
for i in range(1, len(key)):
dir_key = key[:i]
if dir_key not in nodes:
nodes[dir_key] = page.DirectoryNode(nodes[key[:i-1]],
name=key[i-1],
source=os.path.join(root, *dir_key))
# Create the file node
nodes[key] = create_file_node(nodes[key[:-1]], key[-1], filename)
return nodes[()]
def _loadYamlFiles(self, filenames):
"""Load the removed/alias yml files"""
content = dict()
if filenames is not None:
for fname in filenames:
yml_file = mooseutils.eval_path(fname)
if not os.path.isdir(yml_file):
yml_file = os.path.join(self.working_dir, yml_file)
if not os.path.isfile(yml_file):
raise NotADirectoryError("YAML file is not a directory: {}".format(fname))
content.update({fname:yaml_load(yml_file)})
return content
def __init__(self, *args, **kwargs):
command.CommandExtension.__init__(self, *args, **kwargs)
self._app_type = None
self._app_syntax = None
if not self['disable'] and self['executable'] is not None:
LOG.info("Reading MOOSE application syntax.")
exe = mooseutils.eval_path(self['executable'])
exe = mooseutils.find_moose_executable(exe, show_error=False)
if exe is None:
LOG.error("Failed to locate a valid executable in %s.", self['executable'])
else:
try:
self._app_syntax = app_syntax(exe,
alias=self['alias'],
remove=self['remove'],
hide=self['hide'],
allow_test_objects=self['allow-test-objects'])
self._app_type = mooseutils.runExe(exe, ['--type']).strip(' \n')
except Exception as e: #pylint: disable=broad-except
msg = "Failed to load application executable from '%s', " \
"application syntax is being disabled:\n%s"
LOG.error(msg, self['executable'], e.message)
LOG.info("Building MOOSE class database.")
self._database = common.build_class_database(self['includes'], self['inputs'])
# Cache the syntax entries, search the tree is very slow
if self._app_syntax:
self._cache = dict()
self._object_cache = dict()
self._syntax_cache = dict()
for node in anytree.PreOrderIter(self._app_syntax):
if not node.removed:
self._cache[node.fullpath] = node
if node.alias:
self._cache[node.alias] = node
if isinstance(node, syntax.ObjectNode):
self._object_cache[node.fullpath] = node
if node.alias:
self._object_cache[node.alias] = node
elif isinstance(node, syntax.SyntaxNode):
self._syntax_cache[node.fullpath] = node
if node.alias:
self._syntax_cache[node.alias] = node
def __init__(self, *args, **kwargs):
command.CommandExtension.__init__(self, *args, **kwargs)
# Always include MOOSE and libMesh
self['repos'].update(dict(moose="https://github.com/idaholab/moose",
libmesh="https://github.com/libMesh/libmesh"))
# Build requirements sets
repos = self.get('repos')
self.__has_civet = False
self.__requirements = dict()
self.__dependencies = dict()
self.__remotes = dict()
for index, (category, info) in enumerate(self.get('categories').items(), 1):
specs = info.get('specs', ['tests'])
repo = info.get('repo', 'default')
directories = []
for d in info.get('directories'):
path = mooseutils.eval_path(d)
if not os.path.isdir(path):
path = os.path.join(MooseDocs.ROOT_DIR, d)
if not os.path.isdir(path):
msg = "Input directory does not exist: %s"
LOG.error(msg, path)
directories.append(path)
# Create requirement database
self.__requirements[category] = common.get_requirements(directories, specs, 'F', index)
# Create dependency database
self.__dependencies[category] = info.get('dependencies', [])
# Create remote repository database
self.__remotes[category] = repos.get(repo, None)
# Storage for requirement matrix counting (see SQARequirementMatricCommand)
self.__counts = collections.defaultdict(int)
# Deprecate 'url' and 'repo' config options
url = self.get('url')
repo = self.get('repo')
if repo is not None:
msg = "The 'url' and 'repo' config options for MooseDocs.extensions.sqa are deprecated,"\
" add the 'repos' option with a 'default' entry instead."
LOG.warning(msg)
self['repos'].update(dict(default="{}/{}".format(url, repo)))
def execute(self, **kwargs):
"""
Computes the percent complete, number missing items, and the pass/fail status
"""
# Extract configuration parameters
working_dir = self.working_dir or mooseutils.git_root_dir()
local_dirs = self.directories
specs = self.specs or 'tests'
# Get complete directory paths
if local_dirs:
directories = list()
for local_dir in local_dirs:
d = mooseutils.eval_path(local_dir)
if not os.path.isdir(d):
d = os.path.join(working_dir, d)
directories.append(d)
else:
directories = [working_dir]
# Check that directories exist
for d in directories:
if not os.path.isdir(d):
raise NotADirectoryError(
"Supplied directory does not exist: {}".format(d))
# Build Requirement objects and remove directory based dict
req_dict = get_requirements(directories, specs.split())
requirements = []
for values in req_dict.values():
requirements += values
# Populate the lists of tests for SQARequirementDiffReport
self.test_names = set()
for req in requirements:
self.test_names.add((req.filename, req.name, req.line))
# Check the requirements
logger = check_requirements(requirements,
color_text=self.color_text,
**kwargs)
return logger
def __init__(self, *args, **kwargs):
command.CommandExtension.__init__(self, *args, **kwargs)
self._app_type = None
self._app_syntax = None
if not self['disable'] and self['executable'] is not None:
LOG.info("Reading MOOSE application syntax.")
exe = mooseutils.eval_path(self['executable'])
exe = mooseutils.find_moose_executable(exe, show_error=False)
if exe is None:
LOG.error("Failed to locate a valid executable in %s.", self['executable'])
else:
try:
self._app_syntax = app_syntax(exe,
alias=self['alias'],
remove=self['remove'],
hide=self['hide'],
allow_test_objects=self['allow-test-objects'])
self._app_type = mooseutils.runExe(exe, ['--type']).strip(' \n')
except Exception as e: #pylint: disable=broad-except
msg = "Failed to load application executable from '%s', " \
"application syntax is being disabled:\n%s"
LOG.error(msg, self['executable'], e.message)
LOG.info("Building MOOSE class database.")
self._database = common.build_class_database(self['includes'], self['inputs'])
# Cache the syntax entries, search the tree is very slow
if self._app_syntax:
self._cache = dict()
self._object_cache = dict()
self._syntax_cache = dict()
for node in anytree.PreOrderIter(self._app_syntax):
if not node.removed:
self._cache[node.fullpath] = node
if isinstance(node, syntax.ObjectNode):
self._object_cache[node.fullpath] = node
elif isinstance(node, syntax.SyntaxNode):
self._syntax_cache[node.fullpath] = node
def get_files(items, in_ext, git_ls_files=True):
"""
Get a list of files to consider given the content configuration.
"""
filenames = []
for value in items:
if 'root_dir' not in value:
LOG.error('The supplied items must be a list of dict items, each with a "root_dir" and '
'optionally a "content" entry.')
root = mooseutils.eval_path(value['root_dir'])
if not os.path.isabs(root):
root = os.path.join(MooseDocs.ROOT_DIR, root)
for fname in _doc_import(root, content=value.get('content', None)):
filenames.append((root, fname))
return filenames
def get_files(items, in_ext, git_ls_files=True):
"""
Get a list of files to consider given the content configuration.
"""
filenames = []
for value in items:
if 'root_dir' not in value:
LOG.error('The supplied items must be a list of dict items, each with a "root_dir" and '
'optionally a "content" entry.')
root = mooseutils.eval_path(value['root_dir'])
if not os.path.isabs(root):
root = os.path.join(MooseDocs.ROOT_DIR, root)
for fname in _doc_import(root, content=value.get('content', None)):
filenames.append((root, fname))
return filenames
def main(options):
"""
Main function for the build command.
Inputs:
options[argparse options]: Complete options from argparse, see MooseDocs/main.py
"""
t = time.time()
# Infinite nested dict
tree = lambda: collections.defaultdict(tree)
kwargs = tree()
# Setup executioner
if options.executioner:
kwargs['Executioner']['type'] = options.executioner
# Disable extensions
if options.stable:
pass
elif options.fast:
options.disable += ['MooseDocs.extensions.appsyntax', 'MooseDocs.extensions.navigation',
'MooseDocs.extensions.sqa', 'MooseDocs.extensions.civet',
'MooseDocs.extensions.gitutils']
else:
options.disable += ['MooseDocs.extensions.sqa', 'MooseDocs.extensions.civet',
'MooseDocs.extensions.gitutils']
for name in options.disable:
kwargs['Extensions'][name] = dict(active=False)
if options.hide_source:
kwargs['Extensions']['MooseDocs.extensions.modal']['show_source'] = False
# Apply Translator settings
if options.destination:
kwargs['Translator']['destination'] = mooseutils.eval_path(options.destination)
if options.profile:
kwargs['Translator']['profile'] = True
# Apply '--args' and override anything already set
if options.args is not None:
mooseutils.recursive_update(kwargs, options.args)
# Create translators for the specified configuration files, provide kwargs to override them
config_files = options.config if isinstance(options.config, list) else [options.config]
subconfigs = len(config_files) > 1
LOG.info("Loading configuration file{}".format('s' if subconfigs else ''))
translators, contents, configurations = common.load_configs(config_files, **kwargs)
# Initialize the translator objects
for index, translator in enumerate(translators):
if subconfigs:
LOG.info('Initializing translator object loaded from %s', config_files[index])
translator.init(contents[index])
# init methods can add pages (e.g., civet.py), but don't add pages from another translator
contents[index] = [page for page in translator.getPages() if page.translator is translator]
# Identify the first translator in the list as the "primary" one for convenience
primary = translators[0]
pooled = sorted(primary.getPages(), key=(lambda p: p.local))
# Dump page tree from content pool and syntax list from all translators with AppSyntax
if options.dump:
for page in pooled:
print('{}: {}'.format(page.local, page.source))
for index, translator in enumerate(translators):
for extension in translator.extensions:
if isinstance(extension, MooseDocs.extensions.appsyntax.AppSyntaxExtension):
if subconfigs:
LOG.info('Building syntax list specified by %s', config_files[index])
extension.preExecute()
print(extension.syntax)
break
return 0
# TODO: See `navigation.postExecute`
# The navigation "home" should be a markdown file, when all the apps update to this we
# can remove this as well as the use of it by CIVET
home = options.home
if options.serve and (home is not None) and (not home.endswith('.md')):
home = 'http://127.0.0.1:{}'.format(options.port)
if home is not None:
for ext in primary.extensions:
if 'home' in ext:
ext.update(home=home)
# Set default for --clean: clean when --files is NOT used.
if options.clean is None:
options.clean = options.files == []
else:
options.clean = options.clean.lower() in ['true', 'yes', '1']
if options.clean and os.path.exists(primary.destination):
LOG.info("Cleaning destination %s", primary.destination)
shutil.rmtree(primary.destination)
# Update contents lists if only building certain files
if options.files:
for index, translator in enumerate(translators):
func = lambda p: (p in contents[index]
and any([p.local.startswith(f) for f in options.files]))
contents[index] = translator.findPages(func)
# Execute the read and tokenize methods on all translators
for index, translator in enumerate(translators):
if subconfigs:
LOG.info('Reading content specified by %s', config_files[index])
translator.execute(contents[index], num_threads=options.num_threads, render=False, write=False)
# Finally, execute the render and write methods
for index, translator in enumerate(translators):
if subconfigs:
LOG.info('Writing content specified by %s', config_files[index])
translator.execute(contents[index], num_threads=options.num_threads, read=False, tokenize=False)
LOG.info('Total Time [%s sec.]', time.time() - t)
# Run live server and watch for content changes
if options.serve:
watcher = MooseDocsWatcher(translators, pooled, configurations, options.num_threads)
server = livereload.Server(watcher=watcher)
server.serve(root=primary.destination, host=options.host, port=options.port)
return 0
def update(self, **kwargs):
"""Update configuration and handle destination."""
dest = kwargs.get('destination', None)
if dest is not None:
kwargs['destination'] = mooseutils.eval_path(dest)
mixins.ConfigObject.update(self, **kwargs)
def init(self):
"""(override) Generate test reports."""
# Test result database
self.__database = dict()
# Only populate the database if the specified branch and author match current repository
if mooseutils.git_is_branch(self.get('branch')) and \
mooseutils.git_is_config('user.name', self.get('author')):
start = time.time()
LOG.info("Collecting CIVET results...")
for name, category in self.get('remotes').items():
working_dir = mooseutils.eval_path(
category.get('location', MooseDocs.ROOT_DIR))
LOG.info("Gathering CIVET results for '%s' category in %s",
name, working_dir)
hashes = None
if category.get('download_test_results',
self.get('download_test_results', True)):
hashes = mooseutils.git_civet_hashes(
start=self.get('branch'),
author=self.get('author'),
working_dir=working_dir)
LOG.info(
"Downloading CIVET results for '%s' category in %s",
name, working_dir)
local = mooseutils.eval_path(
category.get('test_results_cache',
self.get('test_results_cache')))
site = (category['url'], category['repo'])
local_db = mooseutils.get_civet_results(
local=local,
hashes=hashes,
site=site,
cache=local,
possible=['OK', 'FAIL', 'DIFF', 'TIMEOUT'],
logger=LOG)
self.__database.update(local_db)
LOG.info("Collecting CIVET results complete [%s sec.]",
time.time() - start)
if not self.__database and self.get('generate_test_reports', True):
LOG.info(
"CIVET test result reports are being disabled, it requires results to exist and the specified branch ('%s') and author ('%s') to match the current repository.",
self.get('branch'), self.get('author'))
self.update(generate_test_reports=False)
if self.get('generate_test_reports', True):
self.__has_test_reports = True
start = time.time()
LOG.info("Creating CIVET result pages...")
report_root = self.get('test_reports_location')
if not self.translator.findPage(
report_root, exact=True, throw_on_zero=False):
self.translator.addPage(
pages.Directory(report_root, source=report_root))
src = pages.Source('{}/index.md'.format(report_root),
source='{}/index.md'.format(report_root),
read=False,
tokenize=False)
self.translator.addPage(src)
count = 0
for key, item in self.__database.items():
name = 'result_{}'.format(count)
self.__test_result_numbers[key] = name
count += 1
fullname = '{}/{}.md'.format(report_root, name)
src = pages.Source(fullname,
source=fullname,
read=False,
tokenize=False,
key=key)
self.translator.addPage(src)
LOG.info("Creating CIVET result pages complete [%s sec.]",
time.time() - start)
def main(options):
"""
Main function for the build command.
Inputs:
options[argparse options]: Complete options from argparse, see MooseDocs/main.py
"""
# Make sure "large_media" exists in MOOSE
_init_large_media()
# Setup executioner
kwargs = dict()
if options.executioner:
kwargs['Executioner'] = {'type': options.executioner}
# Create translator
translator, _ = common.load_config(options.config, **kwargs)
if options.destination:
translator.update(
destination=mooseutils.eval_path(options.destination))
if options.profile:
translator.executioner.update(profile=True)
translator.init()
# Disable slow extensions for --fast
if options.fast:
options.disable.append('appsyntax')
# options.disable.append('navigation')
# Disable extensions based on command line arguments
if options.disable:
for ext in translator.extensions:
if ext.name in options.disable: #pylint: disable=protected-access
ext.setActive(False)
# Replace "home" with local server
if options.serve:
for ext in translator.extensions:
if 'home' in ext:
ext.update(home='http://127.0.0.1:{}'.format(options.port),
set_initial=True)
# Dump page tree
if options.dump:
print translator.root
sys.exit()
# Clean when --files is NOT used or when --clean is used with --files.
if ((options.files == []) or (options.files != [] and options.clean)) \
and os.path.exists(translator['destination']):
log = logging.getLogger('MooseDocs.build')
log.info("Cleaning destination %s", translator['destination'])
shutil.rmtree(translator['destination'])
# Perform check
if options.check:
check(translator)
# Perform build
if options.files:
nodes = []
for filename in options.files:
nodes += translator.findPages(filename)
translator.execute(options.num_threads, nodes)
else:
translator.execute(options.num_threads)
if options.serve:
watcher = MooseDocsWatcher(translator, options)
server = livereload.Server(watcher=watcher)
server.serve(root=translator['destination'],
host=options.host,
port=options.port)
return 0
def main(options):
"""
Main function for the build command.
Inputs:
options[argparse options]: Complete options from argparse, see MooseDocs/main.py
"""
# Make sure "large_media" exists in MOOSE
_init_large_media()
# Setup executioner
kwargs = dict()
if options.executioner:
kwargs['Executioner'] = {'type':options.executioner}
# Create translator
translator, _ = common.load_config(options.config, **kwargs)
if options.destination:
translator.update(destination=mooseutils.eval_path(options.destination))
if options.profile:
translator.executioner.update(profile=True)
translator.init()
# Disable slow extensions for --fast
if options.fast:
options.disable.append('appsyntax')
options.disable.append('navigation')
# Disable extensions based on command line arguments
if options.disable:
for ext in translator.extensions:
if ext.name in options.disable: #pylint: disable=protected-access
ext.setActive(False)
# Replace "home" with local server
if options.serve:
for ext in translator.extensions:
if 'home' in ext:
ext.update(home='http://127.0.0.1:{}'.format(options.port), set_initial=True)
# Dump page tree
if options.dump:
for page in translator.content:
print '{}: {}'.format(page.local, page.source)
sys.exit()
# Clean when --files is NOT used or when --clean is used with --files.
if ((options.files == []) or (options.files != [] and options.clean)) \
and os.path.exists(translator['destination']):
log = logging.getLogger('MooseDocs.build')
log.info("Cleaning destination %s", translator['destination'])
shutil.rmtree(translator['destination'])
# Perform check
if options.check:
check(translator)
# Perform build
if options.files:
nodes = []
for filename in options.files:
nodes += translator.findPages(filename)
translator.execute(options.num_threads, nodes)
else:
translator.execute(options.num_threads)
if options.serve:
watcher = MooseDocsWatcher(translator, options)
server = livereload.Server(watcher=watcher)
server.serve(root=translator['destination'], host=options.host, port=options.port)
return 0
def execute(self, **kwargs):
"""Perform app syntax checking"""
# Check that the supplied content dir exists
content_dir = mooseutils.eval_path(self.content_directory)
if not os.path.isdir(content_dir):
content_dir = os.path.join(self.working_dir, content_dir)
if not os.path.isdir(content_dir):
raise NotADirectoryError(
"'content_directory' input is not a directory: {}".format(
content_dir))
# Populate the available list of files
file_cache = mooseutils.git_ls_files(content_dir)
# Check that the supplied exe dir exists
exe_dir = mooseutils.eval_path(self.exe_directory)
if not os.path.isdir(exe_dir):
exe_dir = os.path.join(self.working_dir, exe_dir)
if not os.path.isdir(exe_dir):
raise NotADirectoryError(
"'exe_directory' input is not a directory: {}".format(exe_dir))
# Locate the executable
exe = mooseutils.find_moose_executable(exe_dir,
name=self.exe_name,
show_error=False)
if exe is None:
raise OSError(
"An executable was not found in '{}' with a name '{}'.".format(
exe_dir, self.exe_name))
# Build syntax tree if not provided
if self.app_syntax is None:
# Get the hidden/removed/alias information
hide = self._loadYamlFiles(self.hidden)
remove = self._loadYamlFiles(self.remove)
alias = self._loadYamlFiles(self.alias)
unregister = self._loadYamlFiles(self.unregister)
# Build the complete syntax tree
self.app_syntax = moosesyntax.get_moose_syntax_tree(
exe,
hide=hide,
remove=remove,
alias=alias,
unregister=unregister,
allow_test_objects=self.allow_test_objects)
# Determine the application type (e.g., MooseTestApp)
if self.app_types is None:
out = mooseutils.run_executable(exe, ['--type'])
match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$',
out,
flags=re.MULTILINE)
if match:
self.app_types = [
match.group("type").replace('TestApp', 'App')
]
# Perform the checks
kwargs.setdefault('syntax_prefix',
mooseutils.eval_path(self.syntax_prefix))
kwargs.setdefault('object_prefix',
mooseutils.eval_path(self.object_prefix))
logger = check_syntax(self.app_syntax, self.app_types, file_cache,
**kwargs)
# Create stub pages
if self.generate_stubs:
func = lambda n: (not n.removed) \
and ('_md_file' in n) \
and ((n['_md_file'] is None) or n['_is_stub']) \
and ((n.group in self.app_types) \
or (n.groups() == set(self.app_types)))
for node in moosetree.iterate(self.app_syntax, func):
self._createStubPage(node)
# Dump
if self.dump_syntax:
print(self.app_syntax)
return logger
def update(self, **kwargs):
"""Update configuration and handle destination."""
dest = kwargs.get('destination', None)
if dest is not None:
kwargs['destination'] = mooseutils.eval_path(dest)
mixins.ConfigObject.update(self, **kwargs)
def preExecute(self):
"""Initialize requirement information"""
# Clear any existing counts
self.__counts.clear()
# Do not repopulate
if self.__requirements:
return
start = time.time()
LOG.info("Gathering SQA requirement information...")
repos = self.get('repos')
for index, (category, info) in enumerate(self.get('categories').items(), 1):
specs = info.get('specs', ['tests'])
repo = info.get('repo', 'default')
reports = info.get('reports', None)
directories = []
for d in info.get('directories'):
path = mooseutils.eval_path(d)
if not os.path.isdir(path):
path = os.path.join(MooseDocs.ROOT_DIR, d)
if not os.path.isdir(path):
msg = "Input directory does not exist: %s"
LOG.error(msg, path)
directories.append(path)
# Create requirement database
self.__requirements[category] = moosesqa.get_requirements(directories, specs, 'F', index)
# Create dependency database
self.__dependencies[category] = info.get('dependencies', [])
# Create remote repository database
self.__remotes[category] = repos.get(repo, None)
# Create reports from included SQA apps (e.g., moose/modulus/stochastic_tools)
if reports:
self.__reports[category] = moosesqa.get_sqa_reports(reports)
# Report for the entire app (e.g, moose/modules/doc)
general_reports = self.get('reports')
if general_reports is not None:
self.__reports['__empty__'] = moosesqa.get_sqa_reports(general_reports)
# The following attempts to save computation time by avoiding re-computing the application
# syntax for each report. This is done by extracting the syntax from the appsyntax extension
# and using it within the SQAMooseAppReport objects.
# Get the syntax tree and executable info from the appsyntax extension
app_syntax = None
exe_dir = None
exe_name = None
for ext in self.translator.extensions:
if isinstance(ext, appsyntax.AppSyntaxExtension):
app_syntax = ext.syntax
exe_dir, exe_name = os.path.split(ext.executable) if ext.executable else (None, None)
break
# Setup the SQAMooseAppReports to use the syntax from the running app
for reports in self.__reports.values():
for app_report in reports[2]:
app_report.app_syntax = app_syntax
app_report.exe_directory = exe_dir
app_report.exe_name = exe_name.split('-')[0] if exe_name else None
if app_syntax is None:
msg = 'Attempting to inject application syntax into SQAMooseAppReport, but the syntax does not exist.'
LOG.warning(msg)
LOG.info("Gathering SQA requirement information complete [%s sec.]", time.time() - start)
def main(options):
"""
Main function for the build command.
Inputs:
options[argparse options]: Complete options from argparse, see MooseDocs/main.py
"""
# Infinite nested dict
tree = lambda: collections.defaultdict(tree)
kwargs = tree()
# Setup executioner
if options.executioner:
kwargs['Executioner']['type'] = options.executioner
# Disable extensions
if options.fast:
options.disable += [
'MooseDocs.extensions.appsyntax',
'MooseDocs.extensions.navigation', 'MooseDocs.extensions.sqa',
'MooseDocs.extensions.civet'
]
for name in options.disable:
kwargs['Extensions'][name] = dict(active=False)
# Apply '--args' and override anything already set
if options.args is not None:
mooseutils.recursive_update(kwargs, options.args)
# Create translator, provide kwargs to override content of file
translator, _ = common.load_config(options.config, **kwargs)
if options.destination:
translator.update(
destination=mooseutils.eval_path(options.destination))
if options.profile:
translator.executioner.update(profile=True)
translator.init()
# Replace "home" with local server
home = options.home
if options.serve:
home = 'http://127.0.0.1:{}'.format(options.port)
if home is not None:
for ext in translator.extensions:
if 'home' in ext:
ext.update(home=home)
# Dump page tree
if options.dump:
for page in translator.getPages():
print('{}: {}'.format(page.local, page.source))
for ext in translator.extensions:
if isinstance(ext,
MooseDocs.extensions.appsyntax.AppSyntaxExtension):
ext.preExecute()
print(ext.syntax)
return 0
# Set default for --clean: clean when --files is NOT used.
if options.clean is None:
options.clean = options.files == []
else:
options.clean = options.clean.lower() in ['true', 'yes', '1']
if options.clean and os.path.exists(translator['destination']):
log = logging.getLogger('MooseDocs.build')
log.info("Cleaning destination %s", translator['destination'])
shutil.rmtree(translator['destination'])
# Perform build
if options.files:
nodes = []
for filename in options.files:
nodes += translator.findPages(filename)
translator.execute(nodes, options.num_threads)
else:
translator.execute(None, options.num_threads)
if options.serve:
watcher = MooseDocsWatcher(translator, options)
server = livereload.Server(watcher=watcher)
server.serve(root=translator['destination'],
host=options.host,
port=options.port)
return 0
def main(options):
"""
Main function for the build command.
Inputs:
options[argparse options]: Complete options from argparse, see MooseDocs/main.py
"""
# Make sure "large_media" exists in MOOSE
_init_large_media()
# Setup executioner
kwargs = dict()
if options.executioner:
kwargs['Executioner'] = {'type':options.executioner}
# Disable extensions
if options.fast:
options.disable += ['MooseDocs.extensions.appsyntax', 'MooseDocs.extensions.navigation',
'MooseDocs.extensions.sqa', 'MooseDocs.extensions.civet']
kwargs['Extensions'] = dict()
for name in options.disable:
kwargs['Extensions'][name] = dict(active=False)
# Create translator
translator, _ = common.load_config(options.config, **kwargs)
if options.destination:
translator.update(destination=mooseutils.eval_path(options.destination))
if options.profile:
translator.executioner.update(profile=True)
translator.init()
# Replace "home" with local server
home = options.home
if options.serve:
home = 'http://127.0.0.1:{}'.format(options.port)
if home is not None:
for ext in translator.extensions:
if 'home' in ext:
ext.update(home=home)
# Dump page tree
if options.dump:
for page in translator.getPages():
print('{}: {}'.format(page.local, page.source))
sys.exit()
# Set default for --clean: clean when --files is NOT used.
if options.clean is None:
options.clean = options.files == []
else:
options.clean = options.clean.lower() in ['true', 'yes', '1']
if options.clean and os.path.exists(translator['destination']):
log = logging.getLogger('MooseDocs.build')
log.info("Cleaning destination %s", translator['destination'])
shutil.rmtree(translator['destination'])
# Perform check
if options.check:
check(translator)
# Perform build
if options.files:
nodes = []
for filename in options.files:
nodes += translator.findPages(filename)
translator.execute(nodes, options.num_threads)
else:
translator.execute(None, options.num_threads)
if options.serve:
watcher = MooseDocsWatcher(translator, options)
server = livereload.Server(watcher=watcher)
server.serve(root=translator['destination'], host=options.host, port=options.port)
return 0
def init(self):
"""(override) Generate test reports."""
# Test result database
self.__database = dict()
self.__hashes = None
# Only populate the database if the specified branch and author match current repository
start = time.time()
LOG.info("Collecting CIVET results...")
branch = self.get('branch')
remotes = mooseutils.git_remotes()
for name, category in self.get('remotes').items():
LOG.info("Gathering CIVET results for '%s'.", name)
if category.get('download_test_results', self.get('download_test_results', True)):
repo_url = category.get('repo_url', 'https://github.com').rstrip('/')
repo_url += '/{}'.format(category.get('repo'))
remote = remotes.get(repo_url, None)
if remote is None:
remote = '__MooseDocs.extensions.civet__'
mooseutils.git_add_and_fetch_remote(repo_url, remote, branch)
else:
mooseutils.get_fetch_remote(remote, branch)
self.__hashes = mooseutils.get_civet_hashes(f'{remote}/{branch}')
LOG.info("Downloading CIVET results for '%s' category.", name)
local = mooseutils.eval_path(category.get('test_results_cache', self.get('test_results_cache')))
site = (category['url'], category['repo'])
local_db = mooseutils.get_civet_results(local=local,
hashes=self.__hashes,
site=site,
cache=local,
possible=['OK', 'FAIL', 'DIFF', 'TIMEOUT'],
logger=LOG)
self.__database.update(local_db)
LOG.info("Collecting CIVET results complete [%s sec.]", time.time() - start)
if not self.__database and self.get('generate_test_reports', True):
LOG.info("CIVET test result reports are being disabled, it requires results to exist and the specified branch ('%s') to match the current repository.", self.get('branch'))
self.update(generate_test_reports=False)
if self.get('generate_test_reports', True):
self.__has_test_reports = True
start = time.time()
LOG.info("Creating CIVET result pages...")
result_pages = list()
report_root = self.get('test_reports_location')
if not self.translator.findPage(report_root, exact=True, throw_on_zero=False):
result_pages.append(pages.Directory(report_root, source=report_root))
result_pages.append(pages.Source('{}/index.md'.format(report_root),
source='{}/index.md'.format(report_root),
read=False, tokenize=False))
count = 0
for key, item in self.__database.items():
name = 'result_{}'.format(count)
self.__test_result_numbers[key] = name
count += 1
fullname = '{}/{}.md'.format(report_root, name)
result_pages.append(pages.Source(fullname, source=fullname,
read=False, tokenize=False, key=key))
self.translator.addPages(result_pages)
self.translator.executioner.initPages(result_pages)
LOG.info("Creating CIVET result pages complete [%s sec.]", time.time() - start)
def preExecute(self):
"""Initialize requirement information"""
# Clear any existing counts
self.__counts.clear()
# Do not repopulate
if self.__requirements:
return
start = time.time()
LOG.info("Gathering SQA requirement information...")
for category, info in self.get('categories').items():
specs = info.get('specs', ['tests'])
repo = info.get('repo', 'default')
reports = info.get('reports', None)
# Build the requirements, dependencies, remotes, and reports data structures
directories = list()
for d in info.get('directories', []):
path = mooseutils.eval_path(d)
if not os.path.isdir(path):
path = os.path.join(MooseDocs.ROOT_DIR, d)
if not os.path.isdir(path):
msg = "Input directory does not exist: %s"
LOG.error(msg, path)
continue
directories.append(path)
# Create requirement database from tests
self.__requirements[
category] = moosesqa.get_requirements_from_tests(
directories, specs,
self.get('include_non_testable_requirements'))
# Create dependency database
self.__dependencies[category] = info.get('dependencies', [])
# Create remote repository database
repos = self.get('repos')
self.__remotes[category] = repos.get(repo, None)
# Create reports from included SQA apps (e.g., moose/modulus/stochastic_tools)
if reports:
self.__reports[category] = moosesqa.get_sqa_reports(reports)
# Number the requirements
for c, req_dict in enumerate(self.__requirements.values(), start=1):
moosesqa.number_requirements(req_dict, c)
# Report for the entire app (e.g, moose/modules/doc)
general_reports = self.get('reports')
if general_reports is not None:
self.__reports['_empty_'] = moosesqa.get_sqa_reports(
general_reports)
# The following attempts to save computation time by avoiding re-computing the application
# syntax for each report. This is done by extracting the syntax from the appsyntax extension
# and using it within the SQAMooseAppReport objects.
# Get the syntax tree and executable info from the appsyntax extension
app_syntax = None
exe_dir = None
exe_name = None
for ext in self.translator.extensions:
if isinstance(ext, appsyntax.AppSyntaxExtension):
app_syntax = ext.syntax
exe_dir, exe_name = os.path.split(
ext.executable) if ext.executable else (None, None)
break
# Setup the SQAMooseAppReports to use the syntax from the running app
for reports in self.__reports.values():
for app_report in reports[2]:
app_report.app_syntax = app_syntax
app_report.exe_directory = exe_dir
app_report.exe_name = exe_name.rsplit(
'-', maxsplit=1)[0] if exe_name else None
if app_syntax is None:
msg = 'Attempting to inject application syntax into SQAMooseAppReport, but the syntax does not exist.'
LOG.warning(msg)
# Set default collection and add RunException tests to FAILURE_ANALYSIS collection
d_type = self['default_collection']
for req_category in self.__requirements.values():
for requirements in req_category.values():
for req in requirements:
t_types = req.types
if (req.collections is
None) and (t_types is not None) and ('RunException'
in t_types):
req.collections = set([d_type, 'FAILURE_ANALYSIS'])
elif (req.collections is None):
req.collections = set([d_type])
LOG.info("Gathering SQA requirement information complete [%s sec.]",
time.time() - start)
def check(self, generate=False, groups=None, update=None, locations=None):
"""
Check that the expected documentation exists.
Return:
True, False, or None, where True indicates that the page exists, False indicates the
page does not exist or doesn't contain content, and None indicates that the page is
hidden.
"""
if self.is_root:
return
if self.removed:
LOG.debug("Skipping documentation check for %s, it is REMOVED.", self.fullpath)
return
groups = groups if groups is not None else self.groups
for group in groups:
# Skip if the node is not in the desired groups
if group not in self.groups:
continue
# Locate all the possible locations for the markdown
filenames = set()
for location in locations:
install = mooseutils.eval_path(os.path.join(location, 'documentation', 'systems'))
if not os.path.isabs(install):
filename = os.path.join(MooseDocs.ROOT_DIR, install, self.markdown())
else:
filename = os.path.join(install, self.markdown())
filenames.add((filename, os.path.isfile(filename)))
# Determine the number of files that exist
count = sum([x[1] for x in filenames])
# Class description
if isinstance(self, ObjectNode) and not self.description and not self.hidden:
msg = "The class description is missing for %s, it can be added using the " \
"'addClassDescription' method from within the objects validParams function."
LOG.error(msg, self.fullpath)
# Error if multiple files exist
if count > 1 and not self.hidden:
msg = "Multiple markdown files were located for the '{}' syntax:". \
format(self.fullpath)
for filename, exists in filenames:
if exists:
msg += '\n {}'.format(filename)
LOG.error(msg)
# Error if no files exist
elif count == 0:
if not self.hidden:
msg = "No documentation for %s, documentation for this object should be " \
"created in one of the following locations:"
for filename, _ in filenames:
msg += '\n {}'.format(filename)
msg += "\nIt is possible to generate stub pages for your documentation " \
"using the './moosedocs.py check --generate' command."
LOG.error(msg, self.fullpath)
if generate:
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
LOG.info('Creating stub page for %s %s', self.fullpath, filename)
with open(filename, 'w') as fid:
content = self._defaultContent()
if not isinstance(content, str):
raise TypeError("The _defaultContent method must return a str.")
fid.write(content)
else:
for name, exists in filenames:
if exists:
filename = name
with open(filename, 'r') as fid:
lines = fid.readlines()
if lines and self.STUB_HEADER in lines[0]:
if not self.hidden:
msg = "A MOOSE generated stub page for %s exists, but no content was " \
"added. Add documentation content to %s."
LOG.error(msg, self.fullpath, filename)
if update:
LOG.info("Updating stub page for %s in file %s.", self.fullpath, filename)
with open(filename, 'w') as fid:
content = self._defaultContent()
if not isinstance(content, str):
raise TypeError("The _defaultContent method must return a str.")
fid.write(content)
elif self.hidden and isinstance(self, ObjectNode) and self.description:
msg = "The MOOSE syntax %s is listed has hidden; however, a modified page " \
"exists for the object, please remove the syntax from the list of " \
"hidden list."
LOG.error(msg, self.fullpath)
def check(self, generate=False, groups=None, update=None):
"""
Check that the expected documentation exists.
Return:
True, False, or None, where True indicates that the page exists, False indicates the
page does not exist or doesn't contain content, and None indicates that the page is
hidden.
"""
if self.is_root:
return
if self.removed:
LOG.debug("Skipping documentation check for %s, it is REMOVED.", self.fullpath)
return
groups = groups if groups is not None else self.groups
for group in groups:
# Locate all the possible locations for the markdown
filenames = []
for group in self.groups:
install = SyntaxNodeBase.DESTINATIONS.get(group,
'doc/content/documentation/systems')
install = mooseutils.eval_path(install)
if not os.path.isabs(install):
filename = os.path.join(MooseDocs.ROOT_DIR, install, self.markdown())
else:
filename = os.path.join(install, self.markdown())
filenames.append((filename, os.path.isfile(filename)))
# Determine the number of files that exist
count = sum([x[1] for x in filenames])
# Class description
if isinstance(self, ObjectNode) and not self.description and not self.hidden:
msg = "The class description is missing for %s, it can be added using the " \
"'addClassDescription' method from within the objects validParams function."
LOG.error(msg, self.fullpath)
# Error if multiple files exist
if count > 1 and not self.hidden:
msg = "Multiple markdown files were located for the '{}' syntax:". \
format(self.fullpath)
for filename, exists in filenames:
if exists:
msg += '\n {}'.format(filename)
LOG.error(msg)
# Error if no files exist
elif count == 0:
if not self.hidden:
msg = "No documentation for %s, documentation for this object should be " \
"created in one of the following locations:"
for filename, _ in filenames:
msg += '\n {}'.format(filename)
msg += "\nIt is possible to generate stub pages for your documentation " \
"using the './moosedocs.py check --generate' command."
LOG.error(msg, self.fullpath)
if generate:
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
LOG.info('Creating stub page for %s %s', self.fullpath, filename)
with open(filename, 'w') as fid:
content = self._defaultContent()
if not isinstance(content, str):
raise TypeError("The _defaultContent method must return a str.")
fid.write(content)
else:
for name, exists in filenames:
if exists:
filename = name
with open(filename, 'r') as fid:
lines = fid.readlines()
if lines and self.STUB_HEADER in lines[0]:
if not self.hidden:
msg = "A MOOSE generated stub page for %s exists, but no content was " \
"added. Add documentation content to %s."
LOG.error(msg, self.fullpath, filename)
if update:
LOG.info("Updating stub page for %s in file %s.", self.fullpath, filename)
with open(filename, 'w') as fid:
content = self._defaultContent()
if not isinstance(content, str):
raise TypeError("The _defaultContent method must return a str.")
fid.write(content)
elif self.hidden and isinstance(self, ObjectNode) and self.description:
msg = "The MOOSE syntax %s is listed has hidden; however, a modified page " \
"exists for the object, please remove the syntax from the list of " \
"hidden list."
LOG.error(msg, self.fullpath)