def setUpClass(cls):
config = MooseDocs.load_config(
os.path.join(MooseDocs.MOOSE_DIR, 'docs', 'moosedocs.yml'))
options = config['markdown_extensions'][6][
'MooseDocs.extensions.MooseMarkdown']
cls.database = MooseDocs.MooseLinkDatabase(**options)
def presentation(config_file=None,
md_file=None,
serve=None,
port=None,
host=None,
template=None,
**template_args):
"""
MOOSE markdown presentation blaster.
"""
# Load the YAML configuration file
config = MooseDocs.load_config(config_file,
template=template,
template_args=template_args)
parser = MooseDocs.MooseMarkdown(extensions=config.keys(),
extension_configs=config)
site_dir, _ = os.path.splitext(md_file)
if not os.path.isdir(site_dir):
os.mkdir(site_dir)
root = PresentationBuilder(name='',
markdown=md_file,
parser=parser,
site_dir=site_dir)
root.build()
if serve:
server = livereload.Server()
server.watch(md_file, root.build)
server.serve(root=site_dir, host=host, port=port, restart_delay=0)
return None
def check(config_file=None, locations=None, generate=None):
"""
Performs checks and optionally generates stub pages for missing documentation.
"""
# Read the configuration
app_ext = 'MooseDocs.extensions.app_syntax'
config = MooseDocs.load_config(config_file)
if app_ext not in config:
mooseutils.MooseException("The 'check' utility requires the 'app_syntax' extension.")
ext_config = config[app_ext]
# Run the executable
exe = MooseDocs.abspath(ext_config['executable'])
if not os.path.exists(exe):
raise IOError('The executable does not exist: {}'.format(exe))
else:
LOG.debug("Executing %s to extract syntax.", exe)
raw = mooseutils.runExe(exe, '--yaml')
yaml = mooseutils.MooseYaml(raw)
# Populate the syntax
for loc in ext_config['locations']:
for key, value in loc.iteritems():
if (locations is None) or (key in locations):
value['group'] = key
syntax = common.MooseApplicationSyntax(yaml, generate=generate,
install=ext_config['install'], **value)
LOG.info("Checking documentation for '%s'.", key)
syntax.check()
return None
def parametersElement(self, node, styles):
"""
Return table(s) of input parameters.
Args:
node[dict]: YAML data node.
styles[dict]: Styles from markdown.
"""
# Create the tables (generate 'Required' and 'Optional' initially so that they come out in the proper order)
tables = collections.OrderedDict()
tables['Required'] = MooseDocs.MooseObjectParameterTable()
tables['Optional'] = MooseDocs.MooseObjectParameterTable()
# Loop through the parameters in yaml object
for param in node['parameters'] or []:
name = param['group_name']
if not name and param['required']:
name = 'Required'
elif not name and not param['required']:
name = 'Optional'
if name not in tables:
tables[name] = MooseDocs.MooseObjectParameterTable()
tables[name].addParam(param)
el = self.addStyle(etree.Element('div'), **styles)
title = etree.SubElement(el, 'h2')
title.text = 'Input Parameters'
for key, table in tables.iteritems():
subtitle = etree.SubElement(el, 'h3')
subtitle.text = '{} {}'.format(key, 'Parameters')
el.append(table.html())
return el
def extendMarkdown(self, md, md_globals):
"""
Builds the extensions for MOOSE flavored markdown.
"""
md.registerExtension(self)
# Create a config object
config = self.getConfigs()
# Extract YAML
exe_yaml = self.execute(config.pop('executable', None))
# Generate YAML data from application
# Populate the database for input file and children objects
log.info('Creating input file and source code use database.')
database = MooseDocs.MooseLinkDatabase(**config)
# Populate the syntax
self.syntax = dict()
for key, value in config['locations'].iteritems():
if 'hide' in value:
value['hide'] += config['hide']
else:
value['hide'] = config['hide']
self.syntax[key] = MooseDocs.MooseApplicationSyntax(exe_yaml, **value)
# Preprocessors
md.preprocessors.add('moose_bibtex', MooseBibtex(markdown_instance=md, **config), '_end')
if config['slides']:
md.preprocessors.add('moose_slides', MooseSlidePreprocessor(markdown_instance=md), '_end')
# Block processors
md.parser.blockprocessors.add('diagrams', MooseDiagram(md.parser, **config), '_begin')
md.parser.blockprocessors.add('slideshow', MooseSlider(md.parser, **config), '_begin')
md.parser.blockprocessors.add('css', MooseCSS(md.parser, **config), '_begin')
# Inline Patterns
object_markdown = MooseObjectSyntax(markdown_instance=md,
yaml=exe_yaml,
syntax=self.syntax,
database=database,
**config)
md.inlinePatterns.add('moose_object_syntax', object_markdown, '_begin')
system_markdown = MooseSystemSyntax(markdown_instance=md,
yaml=exe_yaml,
syntax=self.syntax,
**config)
md.inlinePatterns.add('moose_system_syntax', system_markdown, '_begin')
md.inlinePatterns.add('moose_input_block', MooseInputBlock(markdown_instance=md, **config), '<image_link')
md.inlinePatterns.add('moose_cpp_method', MooseCppMethod(markdown_instance=md, **config), '<image_link')
md.inlinePatterns.add('moose_text', MooseTextFile(markdown_instance=md, **config), '<image_link')
md.inlinePatterns.add('moose_image', MooseImageFile(markdown_instance=md, **config), '<image_link')
md.inlinePatterns.add('moose_build_status', MooseBuildStatus(markdown_instance=md, **config), '_begin')
if config['package']:
md.inlinePatterns.add('moose_package_parser', MoosePackageParser(markdown_instance=md, **config), '_end')
def arguments(self, template_args, text): #pylint: disable=no-self-use
"""
Method for modifying the template arguments to be applied to the jinja2 templates engine.
Args:
template_args[dict]: Template arguments to be applied to jinja2 template.
text[str]: Convert markdown to be applied via the 'content' template argument.
"""
template_args['content'] = text
if 'navigation' in template_args:
template_args['navigation'] = \
MooseDocs.yaml_load(MooseDocs.abspath(template_args['navigation']))
def arguments(self, template_args, text): #pylint: disable=no-self-use
"""
Method for modifying the template arguments to be applied to the jinja2 templates engine.
Args:
template_args[dict]: Template arguments to be applied to jinja2 template.
text[str]: Convert markdown to be applied via the 'content' template argument.
"""
template_args['content'] = text
if 'navigation' in template_args:
template_args['navigation'] = \
MooseDocs.yaml_load(MooseDocs.abspath(template_args['navigation']))
def latex(config_file=None, output=None, md_file=None, **kwargs):
"""
Command for converting markdown file to latex.
"""
# Load the YAML configuration file
config = MooseDocs.load_config(config_file, **kwargs)
parser = MooseDocs.MooseMarkdown(extensions=config.keys(), extension_configs=config)
site_dir, _ = os.path.splitext(md_file)
root = LatexBuilder(output, name='', markdown=md_file, parser=parser, site_dir=site_dir)
root.build()
return None
def handleMatch(self, match):
"""
Process the input file supplied.
"""
# Update the settings from regex match
settings = self.getSettings(match.group(3))
# Build the complete filename.
rel_filename = match.group(2)
filename = MooseDocs.abspath(rel_filename)
# Read the file and create element
if not os.path.exists(filename):
el = self.createErrorElement(
"The input file was not located: {}".format(rel_filename))
elif settings['block'] is None:
el = self.createErrorElement(
"Use of !input syntax while not providing a block=some_block. If you wish to include the entire file, use !text instead"
)
else:
parser = ParseGetPot(filename)
node = parser.root_node.getNode(settings['block'])
if node == None:
el = self.createErrorElement('Failed to find {} in {}.'.format(
settings['block'], rel_filename))
else:
content = node.createString()
label = match.group(2) if match.group(2) else rel_filename
el = self.createElement(label, content, filename, rel_filename,
settings)
return el
def setUpClass(cls):
"""
Create link database.
"""
config = MooseDocs.load_config(os.path.join(MooseDocs.MOOSE_DIR, 'docs', 'website.yml'))
options = config['MooseDocs.extensions.app_syntax']
cls.database = MooseLinkDatabase(repo=options['repo'], links=options['links'])
def setUpClass(cls):
"""
Create link database.
"""
config = MooseDocs.load_config(os.path.join(MooseDocs.MOOSE_DIR, 'docs', 'website.yml'))
options = config['MooseDocs.extensions.app_syntax']
cls.database = MooseLinkDatabase(repo=options['repo'], links=options['links'])
def generate(config_file='moosedocs.yml', pages='pages.yml', stubs=False, pages_stubs=False, **kwargs):
"""
Generates MOOSE system and object markdown files from the source code.
Args:
config_file[str]: (Default: 'moosedocs.yml') The MooseMkDocs project configuration file.
"""
# Configuration file
if not os.path.exists(config_file):
raise IOError("The supplied configuration file was not found: {}".format(config_file))
# Read the configuration
config = MooseDocs.yaml_load(config_file)
config = config['markdown_extensions'][-1]['MooseDocs.extensions.MooseMarkdown']
# Run the executable
exe = config['executable']
if not os.path.exists(exe):
log.error('The executable does not exist: {}'.format(exe))
else:
log.debug("Executing {} to extract syntax.".format(exe))
raw = utils.runExe(exe, '--yaml')
yaml = utils.MooseYaml(raw)
# Populate the syntax
for key, value in config['locations'].iteritems():
if 'hide' in value:
value['hide'] += config.get('hide', [])
else:
value['hide'] = config.get('hide', [])
syntax = MooseDocs.MooseApplicationSyntax(yaml, name=key, stubs=stubs, pages_stubs=pages_stubs, pages=pages, **value)
log.info("Checking documentation for '{}'.".format(key))
syntax.check()
def check(config_file=None,
generate=None,
update=None,
dump=None,
template=None,
groups=None,
**template_args):
"""
Performs checks and optionally generates stub pages for missing documentation.
"""
# Create the markdown parser and get the AppSyntaxExtension
config = MooseDocs.load_config(config_file,
template=template,
template_args=template_args)
parser = MooseMarkdown(config)
ext = parser.getExtension(AppSyntaxExtension)
syntax = ext.getMooseAppSyntax()
# Dump the complete syntax tree if desired
if dump:
print syntax
# Check all nodes for documentation
for node in syntax.findall():
node.check(ext.getConfig('install'),
generate=generate,
groups=groups,
update=update)
return 0
def handleMatch(self, match):
"""
Process the text file provided.
"""
# Update the settings from regex match
settings = self.getSettings(match.group(3))
# Read the file
rel_filename = match.group(2).lstrip('/')
filename = MooseDocs.abspath(rel_filename)
if not os.path.exists(filename):
return self.createErrorElement(
"Unable to locate file: {}".format(rel_filename))
if settings['line']:
content = self.extractLine(filename, settings["line"])
elif settings['start'] or settings['end']:
content = self.extractLineRange(filename, settings['start'],
settings['end'],
settings['include_end'])
else:
with open(filename) as fid:
content = fid.read()
if content == None:
return self.createErrorElement(
"Failed to extract content from {}.".format(filename))
# Return the Element object
el = self.createElement(match.group(2), content, filename,
rel_filename, settings)
return el
def presentation(config_file=None, md_file=None, serve=None, port=None, host=None, template=None,
**template_args):
"""
MOOSE markdown presentation blaster.
"""
# The markdown file is provided via the command line, thus it is provided relative to the
# current working directory. The internals of MooseDocs are setup to always work from the
# repository root directory (i.e., MooseDocs.ROOT_DIR), thus the path of this file must be
# converted to be relative to MooseDocs.ROOT_DIR.
md_file = os.path.relpath(os.path.abspath(md_file), MooseDocs.ROOT_DIR)
# Create the markdown parser
config = MooseDocs.load_config(config_file, template=template, template_args=template_args)
parser = MooseMarkdown(config)
# Build the html
builder = PresentationBuilder(md_file=md_file, parser=parser)
builder.init()
builder.build(num_threads=1)
if serve:
server = livereload.Server()
server.watch(os.path.join(MooseDocs.ROOT_DIR, md_file),
lambda: builder.build(num_threads=1))
server.serve(root=builder.rootDirectory(), host=host, port=port, restart_delay=0)
return 0
def generate_html(input, config_file):
"""
Generates html from Moose flavored markdown.
Args:
input[str]: The *.md file to convert.
config_file[str]: The *.yml configuration file.
"""
# Load the config to extract MooseMarkdown settings
#TODO: Make this more robust
config = MooseDocs.yaml_load(config_file)
md_config = config['markdown_extensions'][-1][
'MooseDocs.extensions.MooseMarkdown']
md_config['dot_ext'] = 'svg'
# Convert markdown
with open(input, 'r') as fid:
md = fid.read()
# Extract Jinja2 blocks
settings = dict()
def sub(match):
settings[match.group(1).strip()] = eval(match.group(2))
return ''
md = re.sub(r'@\+\s*set\s+(.*?)=(.*?)\+@', sub, md)
moose = MooseDocs.extensions.MooseMarkdown(**md_config)
parser = markdown.Markdown(extensions=[
moose, 'markdown_include.include', 'admonition', 'mdx_math', 'toc',
'extra'
])
return parser.convert(md), settings
def serve(config_file='moosedocs.yml',
host='127.0.0.1',
port='8000',
num_threads=multiprocessing.cpu_count()):
"""
Create live server
"""
# Location of serve site
tempdir = os.path.abspath(
os.path.join(os.getenv('HOME'), '.local', 'share', 'moose', 'site'))
# Clean the "temp" directory (if desired)
if os.path.exists(tempdir):
log.info('Cleaning build directory: {}'.format(tempdir))
shutil.rmtree(tempdir)
# Create the "temp" directory
if not os.path.exists(tempdir):
os.makedirs(tempdir)
# Perform the initial build
log.info("Building documentation...")
# Wrapper for building complete website
def build_complete():
return build.build_site(config_file=config_file,
site_dir=tempdir,
num_threads=num_threads)
config, parser, builder = build_complete()
# Start the live server
server = livereload.Server()
# Watch markdown files
for page in builder:
server.watch(page.source(), page.build)
# Watch support directories
server.watch(os.path.join(os.getcwd(), 'media'), builder.copyFiles)
server.watch(os.path.join(os.getcwd(), 'css'), builder.copyFiles)
server.watch(os.path.join(os.getcwd(), 'js'), builder.copyFiles)
server.watch(os.path.join(os.getcwd(), 'fonts'), builder.copyFiles)
# Watch the files and directories that require complete rebuild
moose_extension = MooseDocs.get_moose_markdown_extension(parser)
if moose_extension:
server.watch(
os.path.join(os.getcwd(), moose_extension.getConfig('executable')),
build_complete)
server.watch(config['navigation'], build_complete)
server.watch(config_file, build_complete)
server.watch('templates')
# Start the server
server.serve(root=config['site_dir'],
host=host,
port=port,
restart_delay=0)
def _updateSyntax(self, path, objects, actions):
"""
A helper for populating the syntax/filename/markdown databases. (private)
Args:
path[str]: A valid source directory to inspect.
"""
reg_re = r'(?<!\:)register(?!RecoverableData|edError)\w+?\((?P<key>\w+)\);'
reg_named_re = r'registerNamed\w+?\((?P<class>\w+),\s*"(?P<key>\w+)"\);'
action_re = r'(registerActionSyntax|registerSyntax|registerSyntaxTask)\("(?P<action>\w+)"' \
r'\s*,\s*"(?P<key>.*?)\"[,\);]'
# Walk the directory, looking for files with the supplied extension.
for root, _, files in os.walk(MooseDocs.abspath(path), topdown=False):
for filename in files:
fullfile = os.path.join(root, filename)
# Inspect source files
if filename.endswith('.C') or filename.endswith('.h'):
fid = open(fullfile, 'r')
content = fid.read()
fid.close()
# Update class to source definition map
if filename.endswith('.h'):
for match in re.finditer(
r'class\s*(?P<class>\w+)\b[^;]', content):
key = match.group('class')
self._filenames[key] = [fullfile]
src = fullfile.replace('/include/',
'/src/')[:-2] + '.C'
if os.path.exists(src) and (
src not in self._filenames[key]):
self._filenames[key].append(src)
# Map of registered objects
for match in re.finditer(reg_re, content):
key = match.group('key')
objects[key] = key
# Map of named registered objects
for match in re.finditer(reg_named_re, content):
name = match.group('class')
key = match.group('key')
objects[key] = name
# Action syntax map
for match in re.finditer(action_re, content):
key = match.group('key')
action = match.group('action')
actions[key].add(action)
for root, _, files in os.walk(path, topdown=False):
for filename in files:
fullfile = os.path.join(root, filename)
# Inspect source files
name, ext = os.path.splitext(filename)
if (ext == '.C') and (name in self._filenames):
self._filenames[name].append(fullfile)
def setUpClass(cls):
"""
Create the markdown parser using the configuration file.
"""
super(MarkdownTestCase, cls).setUpClass()
# Setup logging
cls._stream = StringIO.StringIO()
cls._formatter = init_logging(stream=cls._stream)
# Define the local directory
cls._path = os.path.abspath(os.path.dirname(inspect.getfile(cls)))
# Read the YAML configurations
config = MooseMarkdown.getDefaultExtensions()
config.update(MooseDocs.load_config(os.path.join(MooseDocs.MOOSE_DIR, 'docs', cls.CONFIG)))
# Update extension list
if cls.EXTENSIONS:
for key in config:
if key not in cls.EXTENSIONS:
config.pop(key)
cls.updateExtensions(config)
cls.parser = MooseMarkdown(config, default=False)
def presentation(config_file=None, md_file=None, serve=None, port=None, host=None, template=None,
**template_args):
"""
MOOSE markdown presentation blaster.
"""
# The markdown file is provided via the command line, thus it is provided relative to the
# current working directory. The internals of MooseDocs are setup to always work from the
# repository root directory (i.e., MooseDocs.ROOT_DIR), thus the path of this file must be
# converted to be relative to MooseDocs.ROOT_DIR.
md_file = os.path.relpath(os.path.abspath(md_file), MooseDocs.ROOT_DIR)
# Create the markdown parser
config = MooseDocs.load_config(config_file, template=template, template_args=template_args)
parser = MooseMarkdown(config)
# Build the html
builder = PresentationBuilder(md_file=md_file, parser=parser)
builder.init()
builder.build(num_threads=1)
if serve:
server = livereload.Server()
server.watch(os.path.join(MooseDocs.ROOT_DIR, md_file),
lambda: builder.build(num_threads=1))
server.serve(root=builder.rootDirectory(), host=host, port=port, restart_delay=0)
return 0
def subsystemsElement(self, group, syntax, styles):
"""
Create table of sub-systems.
"""
node = self._yaml.find(syntax)
if not node:
return createErrorElement(
message=
"The are not any sub-systems for the supplied syntax: {} You likely need to remove the '!subobjects' syntax."
.format(syntax))
table = MooseDocs.MarkdownTable('Name', 'Description')
if node['subblocks']:
for child in node['subblocks']:
name = child['name']
if self._syntax[group].hasSystem(name):
name = name.split('/')[-1].strip()
a = etree.Element('a')
a.set('href', '{}/Overview'.format(name))
a.text = name
table.addRow(a, child['description'])
if table.size() == 0:
return self.createErrorElement(
message=
"No sub-systems exists for the supplied syntax: {}. You likely need to remove the '!subsystems' markdown."
.format(syntax))
el = etree.Element('div', styles)
h2 = etree.SubElement(el, 'h2')
h2.text = 'Available Sub-Systems'
el.append(table.html())
return table.html()
def __init__(self, **kwargs):
super(ListingClangPattern, self).__init__(**kwargs)
# The make command to execute
self._make_dir = MooseDocs.abspath(kwargs.pop('make_dir'))
if not os.path.exists(os.path.join(self._make_dir, 'Makefile')):
LOG.error("Invalid path provided for make: %s", self._make_dir)
def code(self, repo_url):
"""
Return the GitHub/GitLab address for editing the markdown file.
Args:
repo_url[str]: Web address to use as the base for creating the edit link
"""
info = []
for key, syntax in self.__syntax.iteritems():
for obj in syntax.objects().itervalues():
if obj.name == self.name():
info.append(obj)
for obj in syntax.actions().itervalues():
if obj.name == self.name():
info.append(obj)
output = []
for obj in info:
for filename in obj.code:
rel_filename = MooseDocs.relpath(filename)
output.append((os.path.basename(rel_filename),
os.path.join(repo_url, 'blob', 'master',
rel_filename)))
return output
def __init__(self, **kwargs):
super(ListingClangPattern, self).__init__(**kwargs)
# The make command to execute
self._make_dir = MooseDocs.abspath(kwargs.pop('make_dir'))
if not os.path.exists(os.path.join(self._make_dir, 'Makefile')):
LOG.error("Invalid path provided for make: %s", self._make_dir)
def handleMatch(self, match):
"""
Process the text file provided.
"""
# Update the settings from g match
settings = self.getSettings(match.group(3))
# Read the file
rel_filename = match.group('filename').lstrip('/')
filename = MooseDocs.abspath(rel_filename)
if not os.path.exists(filename):
return self.createErrorElement("Unable to locate file: {}".format(rel_filename))
# Figure out file extensions
if settings['language'] is None:
_, ext = os.path.splitext(rel_filename)
if ext in ['.C', '.h', '.cpp', '.hpp']:
settings['language'] = 'cpp'
elif ext == '.py':
settings['language'] = 'python'
else:
settings['language'] = 'text'
# Extract the content from the file
content = self.extractContent(filename, settings)
if content is None:
return self.createErrorElement("Failed to extract content from {}.".format(filename))
# Apply additional settings to content
content = self.prepareContent(content, settings)
# Return the Element object
el = self.createElement(content, rel_filename, settings)
return el
def setUpClass(cls):
"""
Create the markdown parser using the configuration file.
"""
super(MarkdownTestCase, cls).setUpClass()
# Setup logging
cls._stream = StringIO.StringIO()
cls._formatter = init_logging(stream=cls._stream)
# Define the local directory
cls._path = os.path.abspath(os.path.dirname(inspect.getfile(cls)))
# Read the YAML configurations
config = MooseMarkdown.getDefaultExtensions()
config.update(
MooseDocs.load_config(
os.path.join(MooseDocs.MOOSE_DIR, 'docs', cls.CONFIG)))
# Update extension list
if cls.EXTENSIONS:
for key in config:
if key not in cls.EXTENSIONS:
config.pop(key)
cls.updateExtensions(config)
cls.parser = MooseMarkdown(config, default=False)
def replace(self, match):
"""
Substitution function for the re.sub function.
"""
filename = MooseDocs.abspath(match.group(1))
settings = self.getSettings(match.group(2))
if not os.path.exists(filename):
msg = "Failed to located filename in following command.\n{}"
el = self.createErrorElement(msg.format(match.group(0)), title="Unknown Markdown File")
return etree.tostring(el)
if settings['start'] or settings['end']:
content = ListingPattern.extractLineRange(filename, settings['start'],
settings['end'], settings['include-end'])
else:
with open(filename, 'r') as fid:
content = fid.read()
if settings['re']:
match = re.search(settings['re'], content, flags=re.MULTILINE|re.DOTALL)
if not match:
msg = "Failed to located regex in following command.\n{}"
el = self.createErrorElement(msg.format(settings['re']), title="Failed Regex")
return etree.tostring(el)
content = match.group(0)
self._found = True
return content
def latex(config_file=None,
output=None,
md_file=None,
template=None,
**template_args):
"""
Command for converting markdown file to latex.
"""
LOG.warning(
"The latex command is experimental and requires additional development to be "
"complete, please be patient as this develops.\nIf you would like to aid in "
"improving this feature please contact the MOOSE mailing list: "
"[email protected].")
# The markdown file is provided via the command line, thus it is provided relative to the
# current working directory. The internals of MooseDocs are setup to always work from the
# repository root directory (i.e., MooseDocs.ROOT_DIR), thus the path of this file must be
# converted to be relative to MooseDocs.ROOT_DIR.
md_file = os.path.relpath(os.path.abspath(md_file), MooseDocs.ROOT_DIR)
# Create the markdown parser
config = MooseDocs.load_config(config_file,
template=template,
template_args=template_args)
parser = MooseMarkdown(config)
# Build the html
builder = LatexBuilder(md_file=md_file, output=output, parser=parser)
builder.init()
builder.build(num_threads=1)
return 0
def generate_html(input, config_file):
"""
Generates html from Moose flavored markdown.
Args:
input[str]: The *.md file to convert.
config_file[str]: The *.yml configuration file.
"""
# Load the config to extract MooseMarkdown settings
#TODO: Make this more robust
config = MooseDocs.yaml_load(config_file)
md_config = config['markdown_extensions'][-1]['MooseDocs.extensions.MooseMarkdown']
md_config['dot_ext'] = 'svg'
# Convert markdown
with open(input, 'r') as fid:
md = fid.read()
# Extract Jinja2 blocks
settings = dict()
def sub(match):
settings[match.group(1).strip()] = eval(match.group(2))
return ''
md = re.sub(r'@\+\s*set\s+(.*?)=(.*?)\+@', sub, md)
moose = MooseDocs.extensions.MooseMarkdown(**md_config)
parser = markdown.Markdown(extensions=[moose, 'markdown_include.include', 'admonition', 'mdx_math', 'toc', 'extra'])
return parser.convert(md), settings
def _getSlideID(section):
"""
Helper for getting slide name
"""
match = re.search(r'#+\s*(.*?)\s*\n', section)
if match:
return MooseDocs.html_id(match.group(1))
return None
def __init__(self, **kwargs):
super(AppSyntaxExtension, self).__init__(**kwargs)
# Storage for the MooseLinkDatabase object
self.syntax = None
# Create the absolute path to the executable
self.setConfig('executable', MooseDocs.abspath(self.getConfig('executable')))
def generate(config_file='moosedocs.yml',
pages='pages.yml',
stubs=False,
pages_stubs=False,
**kwargs):
"""
Generates MOOSE system and object markdown files from the source code.
Args:
config_file[str]: (Default: 'moosedocs.yml') The MooseMkDocs project configuration file.
"""
# Configuration file
if not os.path.exists(config_file):
raise IOError(
"The supplied configuration file was not found: {}".format(
config_file))
# Read the configuration
config = MooseDocs.yaml_load(config_file)
config = config['markdown_extensions'][-1][
'MooseDocs.extensions.MooseMarkdown']
# Run the executable
exe = config['executable']
if not os.path.exists(exe):
log.error('The executable does not exist: {}'.format(exe))
else:
log.debug("Executing {} to extract syntax.".format(exe))
raw = utils.runExe(exe, '--yaml')
yaml = utils.MooseYaml(raw)
# Populate the syntax
for key, value in config['locations'].iteritems():
if 'hide' in value:
value['hide'] += config['hide']
else:
value['hide'] = config['hide']
syntax = MooseDocs.MooseApplicationSyntax(yaml,
name=key,
stubs=stubs,
pages_stubs=pages_stubs,
pages=pages,
**value)
log.info("Checking documentation for '{}'.".format(key))
syntax.check()
def _getSlideID(section):
"""
Helper for getting slide name
"""
match = re.search(r'#+\s*(.*?)\s*\n', section)
if match:
return MooseDocs.html_id(match.group(1))
return None
def __init__(self, markdown_instance=None, **kwargs):
MooseCommonExtension.__init__(self, **kwargs)
Pattern.__init__(self, self.RE, markdown_instance)
# Load the yaml data containing package information
self.package = MooseDocs.yaml_load("packages.yml")
# The default settings
self._settings = {'arch': None, 'return': None}
def _updateSyntax(self, path, objects, actions):
"""
A helper for populating the syntax/filename/markdown databases. (private)
Args:
path[str]: A valid source directory to inspect.
"""
reg_re = r'(?<!\:)register(?!RecoverableData|edError)\w+?\((?P<key>\w+)\);'
reg_named_re = r'registerNamed\w+?\((?P<class>\w+),\s*"(?P<key>\w+)"\);'
action_re = r'(registerActionSyntax|registerSyntax|registerSyntaxTask)\("(?P<action>\w+)"' \
r'\s*,\s*"(?P<key>.*?)\"[,\);]'
# Walk the directory, looking for files with the supplied extension.
for root, _, files in os.walk(MooseDocs.abspath(path), topdown=False):
for filename in files:
fullfile = os.path.join(root, filename)
# Inspect source files
if filename.endswith('.C') or filename.endswith('.h'):
fid = open(fullfile, 'r')
content = fid.read()
fid.close()
# Update class to source definition map
if filename.endswith('.h'):
for match in re.finditer(r'class\s*(?P<class>\w+)\b[^;]', content):
key = match.group('class')
self._filenames[key] = [fullfile]
src = fullfile.replace('/include/', '/src/')[:-2] + '.C'
if os.path.exists(src) and (src not in self._filenames[key]):
self._filenames[key].append(src)
# Map of registered objects
for match in re.finditer(reg_re, content):
key = match.group('key')
objects[key] = key
# Map of named registered objects
for match in re.finditer(reg_named_re, content):
name = match.group('class')
key = match.group('key')
objects[key] = name
# Action syntax map
for match in re.finditer(action_re, content):
key = match.group('key')
action = match.group('action')
actions[key].add(action)
for root, _, files in os.walk(path, topdown=False):
for filename in files:
fullfile = os.path.join(root, filename)
# Inspect source files
name, ext = os.path.splitext(filename)
if (ext == '.C') and (name in self._filenames):
self._filenames[name].append(fullfile)
def setUp(self):
# clear site directory
for root, _, files in os.walk('site'):
for filename in files:
if filename.endswith('.html'):
os.remove(os.path.join(root, filename))
# Markdown parser
self._parser = MooseDocs.MooseMarkdown()
def generate_html(md_file, config_file):
"""
Generates html from Moose flavored markdown.
Args:
md_file[str]: The *.md file or text to convert.
config_file[str]: The *.yml configuration file.
"""
# Load the YAML configuration file
config = MooseDocs.load_config(config_file)
# Create the markdown parser
extensions, extension_configs = MooseDocs.get_markdown_extensions(config)
parser = markdown.Markdown(extensions=extensions, extension_configs=extension_configs)
# Read and parse the markdown
md, settings = MooseDocs.read_markdown(md_file)
return parser.convert(md), settings
def build(config_file='mkdocs.yml', pages='pages.yml', **kwargs):
"""
Build the documentation using mkdocs build command.
Args:
config_file[str]: (Default: 'mkdocs.yml') The configure file to pass to mkdocs.
"""
pages = MooseDocs.yaml_load(pages)
config = mkdocs.config.load_config(config_file, pages=pages, **kwargs)
mkdocs.commands.build.build(config)
def __init__(self, markdown_instance=None, **kwargs):
MooseCommonExtension.__init__(self, **kwargs)
Pattern.__init__(self, self.RE, markdown_instance)
# Load the yaml data containing package information
self.package = MooseDocs.yaml_load("packages.yml")
# The default settings
self._settings = {'arch' : None,
'return' : None}
def __init__(self, markdown_instance=None, **kwargs):
MooseMarkdownCommon.__init__(self, **kwargs)
Preprocessor.__init__(self, markdown_instance)
# Import the directly defined globals
self._globals = kwargs.pop('globals', dict())
# Add the defined globals from the supplied file(s)
for filename in kwargs.pop('import'):
self._globals.update(MooseDocs.yaml_load(os.path.join(MooseDocs.ROOT_DIR, filename)))
def build(config_file='moosedocs.yml', disable_threads=False, **kwargs):
"""
The main build command.
"""
# Load the YAML configuration file
config = MooseDocs.yaml_load(config_file)
config.update(kwargs)
# Set the default arguments
config.setdefault('docs_dir', os.getcwd())
config.setdefault('site_dir', os.path.join('..', 'site'))
config.setdefault('pages', 'pages.yml')
config.setdefault('template', 'materialize.html')
config.setdefault('template_arguments', dict())
config.setdefault('extra_pages', [])
# Load the site map
sitemap = MooseDocs.yaml_load(config['pages'])
# Create the markdown parser
extensions, extension_configs = get_markdown_extensions(config)
parser = markdown.Markdown(extensions=extensions,
extension_configs=extension_configs)
# Create object for storing pages to be generated
builder = Builder(sitemap, parser, config['site_dir'], config['template'],
**config['template_arguments'])
# Build "extra" pages (i.e., the home page)
if 'extra_pages' in config:
for extra in config['extra_pages']:
for name, kwargs in extra.iteritems():
kwargs.setdefault('template_arguments', dict())
builder.add(kwargs.pop('filename'),
kwargs['template'],
name=name,
**kwargs['template_arguments'])
# Create the html
builder.build(use_threads=not disable_threads)
return config, parser, builder
def __init__(self, content=None, **kwargs):
super(WebsiteBuilder, self).__init__(**kwargs)
if (content is None) or (not os.path.isfile(content)):
LOG.info("Using default content directory configuration "
"(i.e., --content does not include a valid filename).")
content = dict(default=dict(base=os.path.join(os.getcwd(), 'content'),
include=[os.path.join(os.getcwd(), 'content', '*')]))
else:
content = MooseDocs.yaml_load(content)
self._content = content
def _insertFiles(filenames):
"""
Helper function for jinja2 to read css file and return as string.
"""
if isinstance(filenames, str):
filenames = [filenames]
out = []
for filename in filenames:
with open(MooseDocs.abspath(filename), 'r') as fid:
out += [fid.read().strip('\n')]
return '\n'.join(out)
def build(config_file='moosedocs.yml', live_server=False, pages='pages.yml', page_keys=[], clean_site_dir=False, **kwargs):
"""
Build the documentation using mkdocs build command.
Args:
config_file[str]: (Default: 'mkdocs.yml') The configure file to pass to mkdocs.
"""
pages = MooseDocs.load_pages(pages, keys=page_keys)
config = mkdocs.config.load_config(config_file, pages=pages, **kwargs)
update_extra()
mkdocs.commands.build.build(config)
mkdocs.utils.copy_media_files(config['docs_dir'], config['site_dir'])
return config
def setUpClass(cls):
"""
Create the markdown parser using the 'moosedocs.yml' configuration file.
"""
cwd = os.getcwd()
os.chdir(os.path.join(MooseDocs.MOOSE_DIR, "docs"))
config = MooseDocs.yaml_load("moosedocs.yml")
extensions, extension_configs = MooseDocs.commands.get_markdown_extensions(config)
cls.parser = markdown.Markdown(extensions=extensions, extension_configs=extension_configs)
os.chdir(cwd)
def parseFilenames(self, filenames_block):
"""
Parse a set of lines with filenames, image options, and optional captions. Filenames can
contain wildcards and glob will be used to expand them. Any CSS styles after the filename
(but before caption if it exists) will be applied to the image (image is set as a background
in slider). CSS styles listed after the caption will be applied to it.
Expected input is similar to:
images/1.png caption=My caption color=blue
images/2.png background-color=gray caption= Another caption color=red
images/other*.png
Input:
filenames_block[str]: String block to parse
Return:
list of list of dicts. The list has an entry for each image (including
one for each expanded image from glob), each entry contains:
1. dict of "path" which is the filename path
2. dict of attributes to be applied to the image
3. dict of attributes to be applied to the caption
Each image will default to fit the slideshow window with white background
and no caption if no options are specified.
"""
lines = filenames_block.split("\n")
files = []
regular_expression = re.compile(r'(.*?\s|.*?$)(.*?)(caption.*|$)')
for line in lines:
line = line.strip()
matches = regular_expression.search(line)
fname = matches.group(1).strip()
# Build separate dictionaries for the image and caption
img_settings = self.getSettings(matches.group(2).strip())
img_settings.setdefault('background-size', 'contain')
img_settings.setdefault('background-repeat', 'no-repeat')
img_settings.setdefault('background-color', 'white')
caption_settings = self.getSettings(matches.group(3).strip())
img_settings.pop('counter')
caption_settings.pop('counter')
new_files = sorted(glob.glob(MooseDocs.abspath(fname)))
if not new_files:
LOG.error('Parser unable to detect file(s) %s in media.py', fname)
return []
for f in new_files:
files.append(SliderBlockProcessor.ImageInfo(os.path.relpath(f), img_settings,
caption_settings))
return files
def parseBibtexFile(self, bibfile):
"""
Returns parsed bibtex file. If "macro_files" are supplied in the configuration
file, then a temporary file will be made that contains the supplied macros
above the original bib file. This temporary combined file can then be
parsed by pybtex.
"""
if self._macro_files:
t_bib_path = MooseDocs.abspath("tBib.bib")
with open(t_bib_path, "wb") as t_bib:
for t_file in self._macro_files:
with open(MooseDocs.abspath(t_file.strip()), "rb") as in_file:
shutil.copyfileobj(in_file, t_bib)
with open(bibfile, "rb") as in_file:
shutil.copyfileobj(in_file, t_bib)
data = parse_file(t_bib_path)
if os.path.isfile(t_bib_path):
os.remove(t_bib_path)
else:
data = parse_file(bibfile)
return data
def build(config_file='moosedocs.yml', disable_threads=False, **kwargs):
"""
The main build command.
"""
# Load the YAML configuration file
config = MooseDocs.yaml_load(config_file)
config.update(kwargs)
# Set the default arguments
config.setdefault('docs_dir', os.getcwd())
config.setdefault('site_dir', os.path.join('..', 'site'))
config.setdefault('pages', 'pages.yml')
config.setdefault('template', 'materialize.html')
config.setdefault('template_arguments', dict())
config.setdefault('extra_pages', [])
# Load the site map
sitemap = MooseDocs.yaml_load(config['pages'])
# Create the markdown parser
extensions, extension_configs = get_markdown_extensions(config)
parser = markdown.Markdown(extensions=extensions, extension_configs=extension_configs)
# Create object for storing pages to be generated
builder = Builder(sitemap, parser, config['site_dir'], config['template'], **config['template_arguments'])
# Build "extra" pages (i.e., the home page)
if 'extra_pages' in config:
for extra in config['extra_pages']:
for name, kwargs in extra.iteritems():
kwargs.setdefault('template_arguments', dict())
builder.add(kwargs.pop('filename'), kwargs['template'], name=name, **kwargs['template_arguments'])
# Create the html
builder.build(use_threads=not disable_threads)
return config, parser, builder
def _configure(self):
"""
Build the configuration options for included sub-directories.
"""
# Read the moosedocs yml configuration file
yml = MooseDocs.yaml_load(self._config_file)
# Default settings
defaults = yml.get('defaults', dict())
defaults.setdefault('details', os.path.join(os.getcwd(), 'details'))
defaults.setdefault('source', None)
defaults.setdefault('install', os.path.join(os.getcwd(), 'content'))
defaults.setdefault('repo', None)
defaults.setdefault('doxygen', None)
defaults.setdefault('hide', list())
defaults.setdefault('links', dict())
def update_config(config):
"""
Helper for updating/creating local configuration dict.
"""
# Apply defaults
for key, value in defaults.iteritems():
config.setdefault(key, value)
# Append the hide data
config['hide'] = set(config.get('hide', list()) + list(defaults['hide']))
return config
# Extract executable
if 'app' not in yml:
self._exe = utils.find_moose_executable(self._root)
else:
app = yml['app']
if os.path.isdir(app):
self._exe = utils.find_moose_executable(app)
else:
self._exe = app
configs = []
if 'include' in yml:
for key, value in yml['include'].iteritems():
log.debug('Configuring settings for including {}'.format(key))
configs.append(update_config(value))
return configs
def testContentFile(self):
config = MooseDocs.yaml_load(os.path.join(MooseDocs.ROOT_DIR, 'docs', 'content.yml'))
root = common.moose_docs_file_tree(config)
node = self.finder(root, 'media/gitlab-logo.png')[0]
self.assertEqual(node.filename, os.path.join(MooseDocs.ROOT_DIR, 'docs', 'content', 'media',
'gitlab-logo.png'))
node0 = MooseMarkdown.find(root, 'utilities/moose_docs/index.md')[0]
self.assertIsNotNone(node0)
node1 = MooseMarkdown.find(root, 'utilities/moose_docs/moose_markdown/index.md')
self.assertIsNotNone(node1)
node2 = MooseMarkdown.find(root,
'utilities/moose_docs/moose_markdown/extensions/include.md')
self.assertIsNotNone(node2)
def __init__(self, repo=None, links=None, **kwargs): #pylint: disable=unused-argument
self._repo = repo
self.inputs = collections.OrderedDict()
self.children = collections.OrderedDict()
for key, paths in links.iteritems():
self.inputs[key] = dict()
self.children[key] = dict()
for path in paths:
for base, _, files in os.walk(MooseDocs.abspath(path), topdown=False):
for filename in files:
full_name = os.path.join(base, filename)
if filename.endswith('.i'):
self.search(full_name, self.INPUT_RE, self.inputs[key])
elif filename.endswith('.h'):
self.search(full_name, self.HEADER_RE, self.children[key])
def check(config_file=None, generate=None, update=None, dump=None, template=None, groups=None,
**template_args):
"""
Performs checks and optionally generates stub pages for missing documentation.
"""
# Create the markdown parser and get the AppSyntaxExtension
config = MooseDocs.load_config(config_file, template=template, template_args=template_args)
parser = MooseMarkdown(config)
ext = parser.getExtension(AppSyntaxExtension)
syntax = ext.getMooseAppSyntax()
# Dump the complete syntax tree if desired
if dump:
print syntax
# Check all nodes for documentation
for node in syntax.findall():
node.check(ext.getConfig('install'), generate=generate, groups=groups, update=update)
return 0
def _generate(self):
"""
Generate the documentation.
"""
# Setup the location
log.info('Generating Documentation: {}'.format(self._config_file))
# Parse the configuration file for the desired paths
configs = self._configure()
# Locate and run the MOOSE executable
raw = utils.runExe(self._exe, '--yaml')
ydata = utils.MooseYaml(raw)
for config in configs:
generator = MooseSubApplicationDocGenerator(ydata, config)
generator.write()
# Create the mkdocs.yml file
# TODO: When mkdocs plugins API is up and running this should go away.
def dumptree(node, level=0):
for item in node:
for key, value in item.iteritems():
if isinstance(value, list):
yield '{}- {}:'.format(' '*4*level, key)
for f in dumptree(value, level+1):
yield f
else:
yield '{}- {}: {}'.format(' '*4*(level), key, value)
pages = MooseDocs.yaml_load('pages.yml')
output = ['pages:']
output += dumptree(pages, 1)
shutil.copyfile('mkdocs.template.yml', 'mkdocs.yml')
with open('mkdocs.yml', 'a') as fid:
fid.write('\n'.join(output))
def setUpClass(cls):
"""
Create the markdown parser using the configuration file.
"""
# Define the local directory
cls._path = os.path.abspath(os.path.dirname(inspect.getfile(cls)))
# Create the markdown object
os.chdir(os.path.join(MooseDocs.MOOSE_DIR, 'docs'))
# Read the YAML configurations
config = MooseDocs.load_config(cls.CONFIG)
# Update extension list
if cls.EXTENSIONS:
for key in config:
if key not in cls.EXTENSIONS:
config.pop(key)
cls.updateExtensions(config)
cls.parser = MooseDocs.MooseMarkdown(extensions=config.keys(), extension_configs=config)
os.chdir(cls.WORKING_DIR)
def latex(config_file=None, output=None, md_file=None, template=None, **template_args):
"""
Command for converting markdown file to latex.
"""
LOG.warning("The latex command is experimental and requires additional development to be "
"complete, please be patient as this develops.\nIf you would like to aid in "
"improving this feature please contact the MOOSE mailing list: "
"[email protected].")
# The markdown file is provided via the command line, thus it is provided relative to the
# current working directory. The internals of MooseDocs are setup to always work from the
# repository root directory (i.e., MooseDocs.ROOT_DIR), thus the path of this file must be
# converted to be relative to MooseDocs.ROOT_DIR.
md_file = os.path.relpath(os.path.abspath(md_file), MooseDocs.ROOT_DIR)
# Create the markdown parser
config = MooseDocs.load_config(config_file, template=template, template_args=template_args)
parser = MooseMarkdown(config)
# Build the html
builder = LatexBuilder(md_file=md_file, output=output, parser=parser)
builder.init()
builder.build(num_threads=1)
return 0