def findPages(self, arg):
"""
Locate all Page objects that operates on a string or uses a filter.
Usage:
nodes = self.findPages('name')
nodes = self.findPages(lambda p: p.name == 'foo')
The string version is equivalent to:
nodes = self.findPages(lambda p: p.local.endswith(arg))
Inputs:
name[str|unicode|lambda]: The partial name to search against or the function to use
to test for matches.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', arg, (str, unicode, types.FunctionType))
if isinstance(arg, (str, unicode)):
items = self.__page_cache.get(arg, None)
if items is None:
func = lambda p: p.local.endswith(arg)
items = [page for page in self.__content if func(page)]
#self.__page_cache[arg] = items
else:
items = [page for page in self.__content if arg(page)]
return items
def tokenize(self, root, content, page, group=None, line=1, report=True):
"""
Perform the parsing of the supplied content into an AST with the provided root node.
Inputs:
root[tokens.Token]: The root node for the AST.
content[str:tree.page.PageNodeBase]: The content to parse, either as a str
string or a page node object.
"""
# Type checking
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('root', root, tokens.Token)
common.check_type('content', content, str)
# Tokenize
self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group),
line)
# Report errors
if report:
for token in moosetree.iterate(root):
if token.name == 'ErrorToken':
msg = common.report_error(
token['message'], page.source,
token.info.line if token.info else None,
token.info[0] if token.info else token.text(),
token['traceback'], 'TOKENIZE ERROR')
LOG.error(msg)
def add(self, name, regex, function, location='_end'):
"""
Method for adding a Token definition to the Grammar object.
Inputs:
name[str]: The name of the grammar definition, this is utilized for
ordering of the definitions.
regex[re]: A compiled re object that defines what text the token should
be associated.
function[function]: A function that accepts a regex match object as input and
returns a token object.
location[int or str]: (Optional) In the case of an int type, this is an
index indicating the location in the list of definitions
to insert. In the case of a str type the following syntax
is support to insert definitions relative to other
definitions.
'_begin': Insert the new definition at the beginning
of the list of definitions, this is the same
as using an index of 0.
'_end': Append the new definition at the end of the list
of definitions (this is the default).
'<foo': Insert the new definition before the definition
named 'foo'.
'>foo': Insert the new definition after the definition
named 'foo'.
"""
# Add the supplied information to the storage.
common.check_type('location', location, (int, str))
self.__patterns.add(name, Pattern(name, regex, function), location)
def __init__(self,
content,
reader,
renderer,
extensions,
executioner=None,
**kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('content', content, pages.Page)
common.check_type('reader', reader, Reader)
common.check_type('renderer', renderer, Renderer)
common.check_type('extensions', extensions, list)
for ext in extensions:
common.check_type('extensions', ext, Extension)
self.__initialized = False
self.__content = content
self.__extensions = extensions
self.__reader = reader
self.__renderer = renderer
# Define an Executioner if not provided
self.__executioner = executioner
if executioner is None:
self.__executioner = ParallelBarrier()
# Caching for page searches (see findPages)
self.__page_cache = dict()
# Cache for looking up markdown files for levenshtein distance
self.__markdown_file_list = None
self.__levenshtein_cache = dict()
def parse(self, root, content, group=None):
"""
Perform the parsing of the supplied content into an AST with the provided root node.
Inputs:
root[tokens.Token]: The root node for the AST.
content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode
string or a page node object.
"""
# Type checking
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('root', root, tokens.Token)
common.check_type('content', content, unicode)
# Re-initialize
config = self.getConfig()
self.reinit()
# Pre-tokenize
self.translator.executeExtensionFunction('preTokenize', root, config)
# Tokenize
self.__lexer.tokenize(root, self.__lexer.grammar(group), content)
# Post-tokenize
self.translator.executeExtensionFunction('postTokenize', root, config)
# Report errors
for token in anytree.PreOrderIter(root):
if isinstance(token, tokens.ErrorToken):
LOG.error(token.report(self.translator.current))
def load_extensions(ext_list, ext_configs=None):
"""
Convert the supplied list into MooseDocs extension objects by calling the make_extension method.
Inputs:
ext_list[list]: List of extension modules or module names.
ext_configs[dict]: A dict() connecting configurations to the module, the key is the
complete module name.
"""
if ext_configs is None:
ext_configs = dict()
check_type('ext_list', ext_list, list)
check_type('ext_configs', ext_configs, dict)
extensions = []
for ext in ext_list:
name, mod = _get_module(ext)
if not hasattr(mod, 'make_extension'):
msg = "The supplied module {} does not contain the required 'make_extension' function."
raise exceptions.MooseDocsException(msg, name)
else:
obj = mod.make_extension(**ext_configs.get(name, dict()))
extensions.append(obj)
return extensions
def tokenize(self, root, content, page, group=None, line=1):
"""
Perform the parsing of the supplied content into an AST with the provided root node.
Inputs:
root[tokens.Token]: The root node for the AST.
content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode
string or a page node object.
"""
# Type checking
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('root', root, tokens.Token)
common.check_type('content', content, unicode)
# Tokenize
self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group),
line)
# Report errors
for token in anytree.PreOrderIter(root):
if token.name == 'ErrorToken':
msg = common.report_error(token['message'], page, token.info,
token['traceback'],
u'TOKENIZE ERROR')
with MooseDocs.base.translators.Translator.LOCK:
LOG.error(msg)
def findPages(self, arg, exact=False):
"""
Locate all Page objects that operates on a string or uses a filter.
Usage:
nodes = self.findPages('name')
nodes = self.findPages(lambda p: p.name == 'foo')
Inputs:
name[str|lambda]: The partial name to search against or the function to use
to test for matches.
exact[bool]: (False) When True an exact path match is required.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', arg, (str, types.FunctionType))
if isinstance(arg, str):
items = self.__page_cache.get(arg, None)
if items is None:
func = lambda p: (p.local == arg) or \
(not exact and p.local.endswith(os.sep + arg.lstrip(os.sep)))
items = [page for page in self.__content if func(page)]
self.__page_cache[arg] = items
else:
items = [page for page in self.__content if arg(page)]
return items
def setTranslator(self, translator):
"""
Method called by Translator to allow find methods to operate.
"""
check_type('translator', translator,
MooseDocs.base.translators.Translator)
self.__translator = translator
def __init__(self, content, reader, renderer, extensions, executioner=None, **kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('content', content, pages.Page)
common.check_type('reader', reader, Reader)
common.check_type('renderer', renderer, Renderer)
common.check_type('extensions', extensions, list)
for ext in extensions:
common.check_type('extensions', ext, Extension)
self.__initialized = False
self.__content = content
self.__extensions = extensions
self.__reader = reader
self.__renderer = renderer
self.__destination = None # assigned during init()
# Define an Executioner if not provided
self.__executioner = executioner
if executioner is None:
self.__executioner = ParallelBarrier()
# Caching for page searches (see findPages)
self.__page_cache = dict()
def __init__(self,
content,
reader,
renderer,
extensions,
executioner=None,
**kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('content', content, pages.Page)
common.check_type('reader', reader, Reader)
common.check_type('renderer', renderer, Renderer)
common.check_type('extensions', extensions, list)
for ext in extensions:
common.check_type('extensions', ext, Extension)
self.__initialized = False
self.__content = content
self.__extensions = extensions
self.__reader = reader
self.__renderer = renderer
self.__destination = None # assigned during init()
# Define an Executioner if not provided
self.__executioner = executioner
if executioner is None:
self.__executioner = ParallelBarrier()
# Caching for page searches (see findPages)
self.__page_cache = dict()
def findPages(self, arg):
"""
Locate all Page objects that operates on a string or uses a filter.
Usage:
nodes = self.findPages('name')
nodes = self.findPages(lambda p: p.name == 'foo')
The string version is equivalent to:
nodes = self.findPages(lambda p: p.local.endswith(arg))
Inputs:
name[str|unicode|lambda]: The partial name to search against or the function to use
to test for matches.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', arg, (str, unicode, types.FunctionType))
if isinstance(arg, (str, unicode)):
items = self.__page_cache.get(arg, None)
if items is None:
func = lambda p: p.local.endswith(arg)
items = [page for page in self.__content if func(page)]
#self.__page_cache[arg] = items
else:
items = [page for page in self.__content if arg(page)]
return items
def tokenize(self, root, content, page, group=None, line=1, report=True):
"""
Perform the parsing of the supplied content into an AST with the provided root node.
Inputs:
root[tokens.Token]: The root node for the AST.
content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode
string or a page node object.
"""
# Type checking
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('root', root, tokens.Token)
common.check_type('content', content, unicode)
# Tokenize
self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group), line)
# Report errors
if report:
for token in anytree.PreOrderIter(root):
if token.name == 'ErrorToken':
msg = common.report_error(token['message'],
page.source,
token.info.line,
token.info[0],
token['traceback'],
u'TOKENIZE ERROR')
LOG.error(msg)
def parse(self, root, content, group=None):
"""
Perform the parsing of the supplied content into an AST with the provided root node.
Inputs:
root[tokens.Token]: The root node for the AST.
content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode
string or a page node object.
"""
# Type checking
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('root', root, tokens.Token)
common.check_type('content', content, unicode)
# Re-initialize
config = self.getConfig()
self.reinit()
# Pre-tokenize
self.translator.executeExtensionFunction('preTokenize', root, config)
# Tokenize
self.__lexer.tokenize(root, self.__lexer.grammar(group), content)
# Post-tokenize
self.translator.executeExtensionFunction('postTokenize', root, config)
# Report errors
for token in anytree.PreOrderIter(root):
if isinstance(token, tokens.ErrorToken):
LOG.error(token.report(self.translator.current))
def init(self):
"""
Initialize the translator with the output destination for the converted content.
This method also initializes all the various items within the translator for performing
the conversion. It is required to allow the build command to modify configuration items
(i.e., the 'destination' option) prior to setting up the extensions.
Inputs:
destination[str]: The path to the output directory.
"""
if self.__initialized:
msg = "The {} object has already been initialized, this method should not " \
"be called twice."
raise MooseDocs.common.exceptions.MooseDocsException(
msg, type(self))
# Attach translator to Executioner
self.__executioner.setTranslator(self)
# Initialize the extension and call the extend method, then set the extension object
# on each of the extensions.
destination = self.get("destination")
for ext in self.__extensions:
common.check_type('extensions', ext,
MooseDocs.base.components.Extension)
ext.setTranslator(self)
ext.extend(self.__reader, self.__renderer)
for comp in self.__reader.components:
if comp.extension is None:
comp.setExtension(ext)
for comp in self.__renderer.components:
if comp.extension is None:
comp.setExtension(ext)
# Set the translator
for comp in self.__reader.components:
comp.setTranslator(self)
for comp in self.__renderer.components:
comp.setTranslator(self)
# Check that the extension requirements are met
for ext in self.__extensions:
self.__checkRequires(ext)
# Call Extension init() method
self.__initialized = True
LOG.info('Executing extension init() methods...')
t = time.time()
self.executeExtensionFunction('init', None)
LOG.info('Executing extension init() methods complete [%s sec.]',
time.time() - t)
# Initialize the Page objects
for node in self.__content:
node.base = destination
if isinstance(node, pages.Source):
node.output_extension = self.__renderer.EXTENSION
def _executeExtensionFunction(self, name, page, args=tuple()):
"""Helper to call pre/post functions for extensions, reader, and renderer."""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
check_type('name', name, str)
for ext in self.translator.extensions:
if ext.active:
self._callFunction(ext, name, page, args)
def buildObject(self, parent, pattern, info): #pylint: disable=no-self-use
"""
Return a token object for the given lexer information.
"""
obj = pattern.function(info, parent)
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('obj', obj, (tokens.Token, type(None)),
exc=exceptions.TokenizeException)
return obj
def tokenize(self, parent, text, page, grammar, line=1):
"""
Perform tokenization of the supplied text.
Inputs:
parent[tree.tokens]: The parent token to which the new token(s) should be attached.
grammar[Grammar]: Object containing the grammar (defined by regexs) to search.
text[str]: The text to tokenize.
line[int]: The line number to startwith, this allows for nested calls to begin with
the correct line.
NOTE: If the functions attached to the Grammar object raise an Exception it will
be caught by this object and converted into an Exception token. This allows for
the entire text to be tokenized and have the errors report upon completion.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('page', page, pages.Page)
common.check_type('line', line, int)
if not isinstance(text, str):
msg = "EXCEPTION: {}:{}\n{}".format(
page.source, line, "The supplied text must be str.")
raise TypeError(msg)
n = len(text)
pos = 0
while pos < n:
match = None
for pattern in grammar:
match = pattern.regex.match(text, pos)
if match:
info = LexerInformation(match, pattern, line)
try:
obj = self.buildToken(parent, pattern, info, page)
except Exception as e: #pylint: disable=broad-except
obj = tokens.ErrorToken(
parent,
message=str(e),
traceback=traceback.format_exc())
if obj is not None:
obj.info = info
line += match.group(0).count('\n')
pos = match.end()
break
else:
continue
if match is None:
break
# Produce Exception token if text remains that was not matched
if pos < n:
msg = 'Unprocessed text exists.'
tokens.ErrorToken(parent, message=msg)
def buildObject(self, parent, pattern, info): #pylint: disable=no-self-use
"""
Return a token object for the given lexer information.
"""
obj = pattern.function(info, parent)
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('obj',
obj, (tokens.Token, type(None)),
exc=exceptions.TokenizeException)
return obj
def tokenize(self, parent, grammer, text, line=1):
"""
Perform tokenization of the supplied text.
Inputs:
parent[tree.tokens]: The parent token to which the new token(s) should be attached.
grammer[Grammer]: Object containing the grammer (defined by regexs) to search.
text[unicode]: The text to tokenize.
line[int]: The line number to startwith, this allows for nested calls to begin with
the correct line.
NOTE: If the functions attached to the Grammer object raise a TokenizeException it will
be caught by this object and converted into an Exception token. This allows for
the entire text to be tokenized and have the errors report upon completion. The
TokenizeException also contains information about the error, via a LexerInformation
object to improve error reports.
"""
common.check_type('text',
text,
unicode,
exc=exceptions.TokenizeException)
n = len(text)
pos = 0
while pos < n:
match = None
for pattern in grammer:
match = pattern.regex.match(text, pos)
if match:
info = LexerInformation(match, pattern, line)
try:
obj = self.buildObject(parent, pattern, info)
except Exception as e: #pylint: disable=broad-except
obj = tokens.ExceptionToken(
parent,
info=info,
message=e.message,
traceback=traceback.format_exc())
if obj is not None:
obj.info = info #TODO: set ptype on base Token, change to info
line += match.group(0).count('\n')
pos = match.end()
break
else:
continue
if match is None:
break
# Produce Exception token if text remains that was not matched
if pos < n:
msg = u'Unprocessed text exists:\n{}'.format(
common.box(text[pos:], line=line))
LOG.error(msg)
def executeExtensionFunction(self, name, *args):
"""
Execute pre/post functions for extensions.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', name, str)
if name not in self.__extension_functions:
msg = "The supplied extension function name '{}' does not exist, the possible " \
"names include: {}."
raise exceptions.MooseDocsException(msg, name, self.__extension_functions.keys())
for func in self.__extension_functions[name]:
func(*args)
def add(self, name, component):
"""
Associate a RenderComponent object with a token type.
Inputs:
name[str]: The token name (e.g., "String") to associate with the supplied component.
compoment[RenderComponent]: The component to execute with the associated token type.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type("name", name, unicode)
common.check_type("component", component, MooseDocs.base.components.RenderComponent)
component.setRenderer(self)
self.addComponent(component)
self.__functions[name] = self._method(component)
def addCommand(self, reader, command):
# Type checking
common.check_type('reader', reader, Reader)
common.check_type('command', command, CommandComponent)
common.check_type('COMMAND', command.COMMAND, str)
common.check_type('SUBCOMMAND', command.SUBCOMMAND,
(type(None), str, tuple))
# Initialize the component
command.setReader(reader)
command.setExtension(self)
# Subcommands can be tuples
if not isinstance(command.SUBCOMMAND, tuple):
subcommands = tuple([command.SUBCOMMAND])
else:
subcommands = command.SUBCOMMAND
# Add the command and error if it exists
for sub in subcommands:
pair = (command.COMMAND, sub)
if pair in CommandExtension.EXTENSION_COMMANDS:
msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'."
raise common.exceptions.MooseDocsException(
msg, pair[0], pair[1])
CommandExtension.EXTENSION_COMMANDS[pair] = command
def init(self, translator):
"""
Called by Translator object, this allows the objects to be
created independently then passed into the translator, which then
calls this method to provide access to translator for when the actual
tokenize and render commands are called.
"""
if self.initialized():
msg = "The {} object has already been initialized, this method should not " \
"be called twice."
raise MooseDocs.common.exceptions.MooseDocsException(msg, type(self))
common.check_type('translator', translator, MooseDocs.base.translators.Translator)
self.__translator = translator
def executeExtensionFunction(self, name, *args):
"""
Execute pre/post functions for extensions.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', name, str)
if name not in self.__extension_functions:
msg = "The supplied extension function name '{}' does not exist, the possible " \
"names include: {}."
raise exceptions.MooseDocsException(
msg, name, self.__extension_functions.keys())
for func in self.__extension_functions[name]:
func(*args)
def addCommand(self, reader, command):
# Type checking
common.check_type('reader', reader, Reader)
common.check_type('command', command, CommandComponent)
common.check_type('COMMAND', command.COMMAND, str)
common.check_type('SUBCOMMAND', command.SUBCOMMAND, (type(None), str, tuple))
# Initialize the component
command.setReader(reader)
command.setExtension(self)
# Subcommands can be tuples
if not isinstance(command.SUBCOMMAND, tuple):
subcommands = tuple([command.SUBCOMMAND])
else:
subcommands = command.SUBCOMMAND
# Add the command and error if it exists
for sub in subcommands:
pair = (command.COMMAND, sub)
if pair in CommandExtension.EXTENSION_COMMANDS:
msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'."
raise common.exceptions.MooseDocsException(msg, pair[0], pair[1])
CommandExtension.EXTENSION_COMMANDS[pair] = command
def add(self, name, component):
"""
Associate a RenderComponent object with a token type.
Inputs:
name[str]: The token name (e.g., "String") to associate with the supplied component.
compoment[RenderComponent]: The component to execute with the associated token type.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type("name", name, unicode)
common.check_type("component", component, MooseDocs.base.components.RenderComponent)
component.setRenderer(self)
self.addComponent(component)
self.__functions[name] = self._method(component)
def add(self, token, component):
"""
Associate a RenderComponent object with a token type.
Inputs:
token[type]: The token type (not instance) to associate with the supplied component.
compoment[RenderComponent]: The component to execute with the associated token type.
"""
common.check_type("token", token, type)
common.check_type("component", component, MooseDocs.base.components.RenderComponent)
if self.initialized(): # allow use without Translator object
component.init(self.translator)
self.addComponent(component)
self.__functions[token] = self._method(component)
def init(self):
"""
Initialize the translator with the output destination for the converted content.
This method also initializes all the various items within the translator for performing
the conversion. It is required to allow the build command to modify configuration items
(i.e., the 'destination' option) prior to setting up the extensions.
Inputs:
destination[str]: The path to the output directory.
"""
if self.__initialized:
msg = "The {} object has already been initialized, this method should not " \
"be called twice."
raise MooseDocs.common.exceptions.MooseDocsException(msg, type(self))
# Attach translator to Executioner
self.__executioner.setTranslator(self)
# Initialize the extension and call the extend method, then set the extension object
# on each of the extensions.
destination = self.get("destination")
for ext in self.__extensions:
common.check_type('extensions', ext, MooseDocs.base.components.Extension)
ext.setTranslator(self)
ext.extend(self.__reader, self.__renderer)
for comp in self.__reader.components:
if comp.extension is None:
comp.setExtension(ext)
for comp in self.__renderer.components:
if comp.extension is None:
comp.setExtension(ext)
# Set the translator
for comp in self.__reader.components:
comp.setTranslator(self)
for comp in self.__renderer.components:
comp.setTranslator(self)
# Check that the extension requirements are met
for ext in self.__extensions:
self.__checkRequires(ext)
for node in self.__content:
node.base = destination
if isinstance(node, pages.Source):
node.output_extension = self.__renderer.EXTENSION
self.__initialized = True
def tokenize(self, parent, grammar, text, line=1):
"""
Perform tokenization of the supplied text.
Inputs:
parent[tree.tokens]: The parent token to which the new token(s) should be attached.
grammar[Grammar]: Object containing the grammar (defined by regexs) to search.
text[unicode]: The text to tokenize.
line[int]: The line number to startwith, this allows for nested calls to begin with
the correct line.
NOTE: If the functions attached to the Grammar object raise a TokenizeException it will
be caught by this object and converted into an Exception token. This allows for
the entire text to be tokenized and have the errors report upon completion. The
TokenizeException also contains information about the error, via a LexerInformation
object to improve error reports.
"""
common.check_type('text', text, unicode, exc=exceptions.TokenizeException)
n = len(text)
pos = 0
while pos < n:
match = None
for pattern in grammar:
match = pattern.regex.match(text, pos)
if match:
info = LexerInformation(match, pattern, line)
try:
obj = self.buildObject(parent, pattern, info)
except Exception as e: #pylint: disable=broad-except
obj = tokens.ExceptionToken(parent,
info=info,
message=unicode(e.message),
traceback=traceback.format_exc())
if obj is not None:
obj.info = info #TODO: set ptype on base Token, change to info
line += match.group(0).count('\n')
pos = match.end()
break
else:
continue
if match is None:
break
# Produce Exception token if text remains that was not matched
if pos < n:
msg = u'Unprocessed text exists.'
tokens.ErrorToken(parent, info=info, message=msg)
def buildToken(self, parent, pattern, info, page):
"""
Override the Lexer.buildToken method to recursively tokenize base on group names.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('parent', parent, tokens.Token)
common.check_type('info', info, LexerInformation)
obj = super(RecursiveLexer, self).buildToken(parent, pattern, info, page)
if (obj is not None) and (obj is not parent) and obj.get('recursive'):
for key, grammar in self._grammars.iteritems():
if key in info.keys():
text = info[key]
if text is not None:
self.tokenize(obj, text, page, grammar, info.line)
return obj
def init(self, destination):
"""
Initialize the translator with the output destination for the converted content.
This method also initializes all the various items within the translator for performing
the conversion.
Inputs:
destination[str]: The path to the output directory.
"""
if self.__initialized:
msg = "The {} object has already been initialized, this method should not " \
"be called twice."
raise MooseDocs.common.exceptions.MooseDocsException(
msg, type(self))
common.check_type("destination", destination, str)
self.__destination = destination
self.__reader.init(self)
self.__renderer.init(self)
for ext in self.__extensions:
common.check_type('extensions', ext,
MooseDocs.base.components.Extension)
ext.init(self)
ext.extend(self.__reader, self.__renderer)
for comp in self.__reader.components:
if comp.extension is None:
comp.extension = ext
for comp in self.__renderer.components:
if comp.extension is None:
comp.extension = ext
for func_name in self.__extension_functions:
if hasattr(ext, func_name):
self.__extension_functions[func_name].append(
getattr(ext, func_name))
for node in anytree.PreOrderIter(self.__root):
node.base = destination
node.init(self)
self.__initialized = True
def __init__(self, content, reader, renderer, extensions, **kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
common.check_type('content', content, page.PageNodeBase)
common.check_type('reader', reader, Reader)
common.check_type('renderer', renderer, Renderer)
common.check_type('extensions', extensions, list)
for ext in extensions:
common.check_type('extensions', ext, Extension)
self.__initialized = False
self.__current = None
self.__lock = multiprocessing.Lock()
self.__root = content
self.__extensions = extensions
self.__reader = reader
self.__renderer = renderer
self.__extension_functions = dict(preRender=list(),
postRender=list(),
preTokenize=list(),
postTokenize=list())
def addCommand(self, command):
# Type checking
common.check_type('command', command, CommandComponent)
common.check_type('COMMAND', command.COMMAND, str)
common.check_type('SUBCOMMAND', command.SUBCOMMAND,
(type(None), str, tuple))
# Initialize the component
command.init(self.translator)
command.extension = self
# Subcommands can be tuples
if not isinstance(command.SUBCOMMAND, tuple):
subcommands = tuple([command.SUBCOMMAND])
else:
subcommands = command.SUBCOMMAND
# Add the command and error if it exists
for sub in subcommands:
pair = (command.COMMAND, sub)
if pair in self.translator.__EXTENSION_COMMANDS__:
msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'."
raise common.exceptions.MooseDocsException(
msg, pair[0], pair[1])
self.translator.__EXTENSION_COMMANDS__[pair] = command
def add(self, group, component, location='_end'):
"""
Add a component to Extened the Reader by adding a TokenComponent.
Inputs:
group[str]: Name of the lexer group to append.
component[components.TokenComponent]: The tokenize component to add.
location[str|int]: The location to insert this component (see Grammar.py)
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type("group", group, str)
common.check_type("component", component, MooseDocs.base.components.TokenComponent)
common.check_type("location", location, (str, int))
# Define the name of the component being added (for sorting within Grammar)
name = component.__class__.__name__
# Store and init component, checking self.initialized() allows this object to be used
# without the Translator which is useful in some cases.
self.addComponent(component)
if self.initialized():
component.init(self.translator)
# Update the lexer
self.__lexer.add(group, name, component.RE, component, location)
def init(self, destination):
"""
Initialize the translator with the output destination for the converted content.
This method also initializes all the various items within the translator for performing
the conversion.
Inputs:
destination[str]: The path to the output directory.
"""
if self.__initialized:
msg = "The {} object has already been initialized, this method should not " \
"be called twice."
raise MooseDocs.common.exceptions.MooseDocsException(msg, type(self))
common.check_type("destination", destination, str)
self.__destination = destination
self.__reader.init(self)
self.__renderer.init(self)
for ext in self.__extensions:
common.check_type('extensions', ext, MooseDocs.base.components.Extension)
ext.init(self)
ext.extend(self.__reader, self.__renderer)
for comp in self.__reader.components:
if comp.extension is None:
comp.extension = ext
for comp in self.__renderer.components:
if comp.extension is None:
comp.extension = ext
for func_name in self.__extension_functions:
if hasattr(ext, func_name):
self.__extension_functions[func_name].append(getattr(ext, func_name))
for node in anytree.PreOrderIter(self.__root):
node.base = destination
node.init(self)
self.__initialized = True
def addCommand(self, command):
# Type checking
common.check_type('command', command, CommandComponent)
common.check_type('COMMAND', command.COMMAND, str)
common.check_type('SUBCOMMAND', command.SUBCOMMAND, (type(None), str, tuple))
# Initialize the component
command.init(self.translator)
command.extension = self
# Subcommands can be tuples
if not isinstance(command.SUBCOMMAND, tuple):
subcommands = tuple([command.SUBCOMMAND])
else:
subcommands = command.SUBCOMMAND
# Add the command and error if it exists
for sub in subcommands:
pair = (command.COMMAND, sub)
if pair in self.translator.__EXTENSION_COMMANDS__:
msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'."
raise common.exceptions.MooseDocsException(msg, pair[0], pair[1])
self.translator.__EXTENSION_COMMANDS__[pair] = command
def add(self, group, component, location='_end'):
"""
Add a component to Extened the Reader by adding a TokenComponent.
Inputs:
group[str]: Name of the lexer group to append.
component[components.TokenComponent]: The tokenize component to add.
location[str|int]: The location to insert this component (see Grammar.py)
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type("group", group, str)
common.check_type("component", component,
MooseDocs.base.components.TokenComponent)
common.check_type("location", location, (str, int))
# Define the name of the component being added (for sorting within Grammar)
name = component.__class__.__name__
# Store and init component, checking self.initialized() allows this object to be used
# without the Translator which is useful in some cases.
self.addComponent(component)
if self.initialized():
component.init(self.translator)
# Update the lexer
self.__lexer.add(group, name, component.RE, component, location)
def __init__(self, content, reader, renderer, extensions, **kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
common.check_type('content', content, page.PageNodeBase)
common.check_type('reader', reader, Reader)
common.check_type('renderer', renderer, Renderer)
common.check_type('extensions', extensions, list)
for ext in extensions:
common.check_type('extensions', ext, Extension)
self.__initialized = False
self.__current = None
self.__lock = multiprocessing.Lock()
self.__root = content
self.__extensions = extensions
self.__reader = reader
self.__renderer = renderer
self.__destination = None # assigned during init()
self.__extension_functions = dict(preRender=list(),
postRender=list(),
preTokenize=list(),
postTokenize=list())
def findall(self,
name,
maxcount=1,
mincount=1,
exc=exceptions.MooseDocsException): #pylint: disable=no-self-use
"""
Find method for locating pages.
Inputs:
name[str]: The name of the page to search.
maxcount[int]: The maximum number of items to find (default: 1).
mincount[int]: The minimum number of items to find (default: 1).
exc[Exception]: The type of exception to raise if min/max are not satisfied.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', name, (str, unicode))
common.check_type('mincount', mincount, (int))
common.check_type('maxcount', maxcount, (int))
common.check_type('exc', exc, (type, types.LambdaType, type(None)))
try:
return list(CACHE[name])
except KeyError:
pass
nodes = set()
for key in CACHE:
if key.endswith(name):
nodes.update(CACHE[key])
if (maxcount is not None) and exc and (len(nodes) > maxcount):
msg = "The 'maxcount' was set to {} but {} nodes were found for the name '{}'." \
.format(maxcount, len(nodes), name)
for node in nodes:
msg += '\n {} (source: {})'.format(node.local, node.source)
raise exc(msg)
elif (mincount is not None) and exc and (len(nodes) < mincount):
msg = "The 'mincount' was set to {} but {} nodes were found for the name '{}'." \
.format(mincount, len(nodes), name)
for node in nodes:
msg += '\n {} (source: {})'.format(node.local, node.source)
raise exc(msg)
CACHE[name] = nodes
return list(nodes)
def __init__(self, name, regex, function):
# Perform input type checking
common.check_type('name', name, str)
common.check_type('regex', regex, type(re.compile('')))
common.check_type('function', function, types.FunctionType)
self.name = name
self.regex = regex
self.function = function
def findall(self, name, maxcount=1, mincount=1, exc=exceptions.MooseDocsException): #pylint: disable=no-self-use
"""
Find method for locating pages.
Inputs:
name[str]: The name of the page to search.
maxcount[int]: The maximum number of items to find (default: 1).
mincount[int]: The minimum number of items to find (default: 1).
exc[Exception]: The type of exception to raise if min/max are not satisfied.
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('name', name, (str, unicode))
common.check_type('mincount', mincount, (int))
common.check_type('maxcount', maxcount, (int))
common.check_type('exc', exc, (type, types.LambdaType, type(None)))
try:
return list(CACHE[name])
except KeyError:
pass
nodes = set()
for key in CACHE:
if key.endswith(name):
nodes.update(CACHE[key])
if (maxcount is not None) and exc and (len(nodes) > maxcount):
msg = "The 'maxcount' was set to {} but {} nodes were found for the name '{}'." \
.format(maxcount, len(nodes), name)
for node in nodes:
msg += '\n {} (source: {})'.format(node.local, node.source)
raise exc(msg)
elif (mincount is not None) and exc and (len(nodes) < mincount):
msg = "The 'mincount' was set to {} but {} nodes were found for the name '{}'." \
.format(mincount, len(nodes), name)
for node in nodes:
msg += '\n {} (source: {})'.format(node.local, node.source)
raise exc(msg)
CACHE[name] = nodes
return list(nodes)
def testCallable(self):
func = lambda: True
common.check_type('foo', func, types.FunctionType)
with self.assertRaises(Exception) as e:
common.check_type('foo', 42, types.FunctionType)
self.assertEqual("The argument 'foo' must be callable but <type 'int'> was provided.",
e.exception.message)
with self.assertRaises(Exception) as e:
common.check_type('foo', 42, list)
gold = "The argument 'foo' must be of type <type 'list'> but <type 'int'> was provided."
self.assertIn(gold, e.exception.message)
def testCallable(self):
func = lambda: True
common.check_type('foo', func, types.FunctionType)
with self.assertRaises(Exception) as e:
common.check_type('foo', 42, types.FunctionType)
self.assertEqual(
"The argument 'foo' must be callable but <type 'int'> was provided.",
e.exception.message)
with self.assertRaises(Exception) as e:
common.check_type('foo', 42, list)
gold = "The argument 'foo' must be of type <type 'list'> but <type 'int'> was provided."
self.assertIn(gold, e.exception.message)
def add(self, group, component, location='_end'):
"""
Add a component to extend the Reader by adding a TokenComponent.
This method is called when adding ReaderComonents in the ::extend method.
Inputs:
group[str]: Name of the lexer group to append.
component[components.TokenComponent]: The tokenize component to add.
location[str|int]: The location to insert this component (see Grammar.py)
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type("group", group, str)
common.check_type("component", component, MooseDocs.base.components.TokenComponent)
common.check_type("location", location, (str, int))
component.setReader(self)
self.addComponent(component)
# Update the lexer
name = component.__class__.__name__
self.__lexer.add(group, name, component.RE, component, location)
def add(self, group, component, location='_end'):
"""
Add a component to extend the Reader by adding a TokenComponent.
This method is called when adding ReaderComonents in the ::extend method.
Inputs:
group[str]: Name of the lexer group to append.
component[components.TokenComponent]: The tokenize component to add.
location[str|int]: The location to insert this component (see Grammar.py)
"""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type("group", group, str)
common.check_type("component", component, MooseDocs.base.components.TokenComponent)
common.check_type("location", location, (str, int))
component.setReader(self)
self.addComponent(component)
# Update the lexer
name = component.__class__.__name__
self.__lexer.add(group, name, component.RE, component, location)
def execute(self, num_threads=1):
"""
Perform parallel build for all pages.
Inputs:
num_threads[int]: The number of threads to use (default: 1).
NOTICE:
A proper parallelization for MooseDocs would be three parallel steps, with minimal
communication.
1. Read all the markdown files (in parallel).
2. Perform the AST tokenization (in parallel), then communicate the completed
AST back to the main process.
3. Convert the AST to HTML (in parallel).
4. Write/copy (in parallel) the completed HTML and other files (images, js, etc.).
However, step two is problematic because python requires that the AST be pickled,
which is possible, for communication. In doing this I realized that the pickling was a
limiting factor and made the AST step very slow. I need to investigate this further to
make sure I was using a non-locking pool of workers, but this was taking too much
development time.
The current implementation performs all four steps together, which generally works just
fine, with one exception. The autolink extension actually interrogates the AST from other
pages. Hence, if the other page was generated off process the information is not available.
The current implementation will just compute the AST locally (i.e., I am performing repeated
calculations in favor of communication). This works well enough for now, but as more
autolinking is preformed and other similar extensions are created this could cause a slow
down.
Long term this should be looked into again, for now the current approach is working well.
This new system is already an order of 4 times faster than the previous implementation and
likely could be optimized further.
The multiprocessing.Manager() needs to be explored, it is working to pull the JSON index
information together.
"""
common.check_type('num_threads', num_threads, int)
self.__assertInitialize()
# Log start message and time
LOG.info("Building Pages...")
start = time.time()
manager = multiprocessing.Manager()
array = manager.list()
def target(nodes, lock):
"""Helper for building multiple nodes (i.e., a chunk for a process)."""
for node in nodes:
node.build()
if isinstance(node, page.MarkdownNode):
node.buildIndex(self.renderer.get('home', None))
with lock:
for entry in node.index:
array.append(entry)
# Complete list of nodes
nodes = [n for n in anytree.PreOrderIter(self.root)]
# Serial
if num_threads == 1:
target(nodes, self.lock)
# Multiprocessing
else:
jobs = []
for chunk in mooseutils.make_chunks(nodes, num_threads):
p = multiprocessing.Process(target=target, args=(chunk, self.lock))
p.start()
jobs.append(p)
for job in jobs:
job.join()
# Done
stop = time.time()
LOG.info("Build time %s sec.", stop - start)
iname = os.path.join(self.destination, 'js', 'search_index.js')
if not os.path.isdir(os.path.dirname(iname)):
os.makedirs(os.path.dirname(iname))
items = [v for v in array if v]
common.write(iname, 'var index_data = {};'.format(json.dumps(items)))
def __init__(self, lexer, **kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
mixins.ComponentObject.__init__(self)
mixins.TranslatorObject.__init__(self)
common.check_type('lexer', lexer, RecursiveLexer)
self.__lexer = lexer
def setTranslator(self, translator):
"""
Method called by Translator to allow find methods to operate.
"""
check_type('translator', translator, MooseDocs.base.translators.Translator)
self.__translator = translator
def addComponent(self, comp):
"""
Add a Component object.
"""
check_type("component", comp, MooseDocs.base.components.Component)
self.__components.append(comp)
def setRenderer(self, renderer):
"""Initialize the class with the Renderer object."""
check_type('renderer', renderer, MooseDocs.base.renderers.Renderer)
self.__renderer = renderer
def __init__(self, lexer, **kwargs):
mixins.ConfigObject.__init__(self, **kwargs)
mixins.ComponentObject.__init__(self)
mixins.TranslatorObject.__init__(self)
common.check_type('lexer', lexer, RecursiveLexer)
self.__lexer = lexer
def current(self, value):
"""Set the current page being translated, see page.MarkdownNode.build()."""
if MooseDocs.LOG_LEVEL == logging.DEBUG:
common.check_type('value', value, (type(None), page.PageNodeBase))
self.__current = value