def findPages(self, arg): """ Locate all Page objects that operates on a string or uses a filter. Usage: nodes = self.findPages('name') nodes = self.findPages(lambda p: p.name == 'foo') The string version is equivalent to: nodes = self.findPages(lambda p: p.local.endswith(arg)) Inputs: name[str|unicode|lambda]: The partial name to search against or the function to use to test for matches. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', arg, (str, unicode, types.FunctionType)) if isinstance(arg, (str, unicode)): items = self.__page_cache.get(arg, None) if items is None: func = lambda p: p.local.endswith(arg) items = [page for page in self.__content if func(page)] #self.__page_cache[arg] = items else: items = [page for page in self.__content if arg(page)] return items
def tokenize(self, root, content, page, group=None, line=1, report=True): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[str:tree.page.PageNodeBase]: The content to parse, either as a str string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, str) # Tokenize self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group), line) # Report errors if report: for token in moosetree.iterate(root): if token.name == 'ErrorToken': msg = common.report_error( token['message'], page.source, token.info.line if token.info else None, token.info[0] if token.info else token.text(), token['traceback'], 'TOKENIZE ERROR') LOG.error(msg)
def add(self, name, regex, function, location='_end'): """ Method for adding a Token definition to the Grammar object. Inputs: name[str]: The name of the grammar definition, this is utilized for ordering of the definitions. regex[re]: A compiled re object that defines what text the token should be associated. function[function]: A function that accepts a regex match object as input and returns a token object. location[int or str]: (Optional) In the case of an int type, this is an index indicating the location in the list of definitions to insert. In the case of a str type the following syntax is support to insert definitions relative to other definitions. '_begin': Insert the new definition at the beginning of the list of definitions, this is the same as using an index of 0. '_end': Append the new definition at the end of the list of definitions (this is the default). '<foo': Insert the new definition before the definition named 'foo'. '>foo': Insert the new definition after the definition named 'foo'. """ # Add the supplied information to the storage. common.check_type('location', location, (int, str)) self.__patterns.add(name, Pattern(name, regex, function), location)
def __init__(self, content, reader, renderer, extensions, executioner=None, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('content', content, pages.Page) common.check_type('reader', reader, Reader) common.check_type('renderer', renderer, Renderer) common.check_type('extensions', extensions, list) for ext in extensions: common.check_type('extensions', ext, Extension) self.__initialized = False self.__content = content self.__extensions = extensions self.__reader = reader self.__renderer = renderer # Define an Executioner if not provided self.__executioner = executioner if executioner is None: self.__executioner = ParallelBarrier() # Caching for page searches (see findPages) self.__page_cache = dict() # Cache for looking up markdown files for levenshtein distance self.__markdown_file_list = None self.__levenshtein_cache = dict()
def parse(self, root, content, group=None): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, unicode) # Re-initialize config = self.getConfig() self.reinit() # Pre-tokenize self.translator.executeExtensionFunction('preTokenize', root, config) # Tokenize self.__lexer.tokenize(root, self.__lexer.grammar(group), content) # Post-tokenize self.translator.executeExtensionFunction('postTokenize', root, config) # Report errors for token in anytree.PreOrderIter(root): if isinstance(token, tokens.ErrorToken): LOG.error(token.report(self.translator.current))
def load_extensions(ext_list, ext_configs=None): """ Convert the supplied list into MooseDocs extension objects by calling the make_extension method. Inputs: ext_list[list]: List of extension modules or module names. ext_configs[dict]: A dict() connecting configurations to the module, the key is the complete module name. """ if ext_configs is None: ext_configs = dict() check_type('ext_list', ext_list, list) check_type('ext_configs', ext_configs, dict) extensions = [] for ext in ext_list: name, mod = _get_module(ext) if not hasattr(mod, 'make_extension'): msg = "The supplied module {} does not contain the required 'make_extension' function." raise exceptions.MooseDocsException(msg, name) else: obj = mod.make_extension(**ext_configs.get(name, dict())) extensions.append(obj) return extensions
def tokenize(self, root, content, page, group=None, line=1): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, unicode) # Tokenize self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group), line) # Report errors for token in anytree.PreOrderIter(root): if token.name == 'ErrorToken': msg = common.report_error(token['message'], page, token.info, token['traceback'], u'TOKENIZE ERROR') with MooseDocs.base.translators.Translator.LOCK: LOG.error(msg)
def findPages(self, arg, exact=False): """ Locate all Page objects that operates on a string or uses a filter. Usage: nodes = self.findPages('name') nodes = self.findPages(lambda p: p.name == 'foo') Inputs: name[str|lambda]: The partial name to search against or the function to use to test for matches. exact[bool]: (False) When True an exact path match is required. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', arg, (str, types.FunctionType)) if isinstance(arg, str): items = self.__page_cache.get(arg, None) if items is None: func = lambda p: (p.local == arg) or \ (not exact and p.local.endswith(os.sep + arg.lstrip(os.sep))) items = [page for page in self.__content if func(page)] self.__page_cache[arg] = items else: items = [page for page in self.__content if arg(page)] return items
def setTranslator(self, translator): """ Method called by Translator to allow find methods to operate. """ check_type('translator', translator, MooseDocs.base.translators.Translator) self.__translator = translator
def __init__(self, content, reader, renderer, extensions, executioner=None, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('content', content, pages.Page) common.check_type('reader', reader, Reader) common.check_type('renderer', renderer, Renderer) common.check_type('extensions', extensions, list) for ext in extensions: common.check_type('extensions', ext, Extension) self.__initialized = False self.__content = content self.__extensions = extensions self.__reader = reader self.__renderer = renderer self.__destination = None # assigned during init() # Define an Executioner if not provided self.__executioner = executioner if executioner is None: self.__executioner = ParallelBarrier() # Caching for page searches (see findPages) self.__page_cache = dict()
def __init__(self, content, reader, renderer, extensions, executioner=None, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('content', content, pages.Page) common.check_type('reader', reader, Reader) common.check_type('renderer', renderer, Renderer) common.check_type('extensions', extensions, list) for ext in extensions: common.check_type('extensions', ext, Extension) self.__initialized = False self.__content = content self.__extensions = extensions self.__reader = reader self.__renderer = renderer self.__destination = None # assigned during init() # Define an Executioner if not provided self.__executioner = executioner if executioner is None: self.__executioner = ParallelBarrier() # Caching for page searches (see findPages) self.__page_cache = dict()
def findPages(self, arg): """ Locate all Page objects that operates on a string or uses a filter. Usage: nodes = self.findPages('name') nodes = self.findPages(lambda p: p.name == 'foo') The string version is equivalent to: nodes = self.findPages(lambda p: p.local.endswith(arg)) Inputs: name[str|unicode|lambda]: The partial name to search against or the function to use to test for matches. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', arg, (str, unicode, types.FunctionType)) if isinstance(arg, (str, unicode)): items = self.__page_cache.get(arg, None) if items is None: func = lambda p: p.local.endswith(arg) items = [page for page in self.__content if func(page)] #self.__page_cache[arg] = items else: items = [page for page in self.__content if arg(page)] return items
def tokenize(self, root, content, page, group=None, line=1, report=True): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, unicode) # Tokenize self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group), line) # Report errors if report: for token in anytree.PreOrderIter(root): if token.name == 'ErrorToken': msg = common.report_error(token['message'], page.source, token.info.line, token.info[0], token['traceback'], u'TOKENIZE ERROR') LOG.error(msg)
def parse(self, root, content, group=None): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, unicode) # Re-initialize config = self.getConfig() self.reinit() # Pre-tokenize self.translator.executeExtensionFunction('preTokenize', root, config) # Tokenize self.__lexer.tokenize(root, self.__lexer.grammar(group), content) # Post-tokenize self.translator.executeExtensionFunction('postTokenize', root, config) # Report errors for token in anytree.PreOrderIter(root): if isinstance(token, tokens.ErrorToken): LOG.error(token.report(self.translator.current))
def init(self): """ Initialize the translator with the output destination for the converted content. This method also initializes all the various items within the translator for performing the conversion. It is required to allow the build command to modify configuration items (i.e., the 'destination' option) prior to setting up the extensions. Inputs: destination[str]: The path to the output directory. """ if self.__initialized: msg = "The {} object has already been initialized, this method should not " \ "be called twice." raise MooseDocs.common.exceptions.MooseDocsException( msg, type(self)) # Attach translator to Executioner self.__executioner.setTranslator(self) # Initialize the extension and call the extend method, then set the extension object # on each of the extensions. destination = self.get("destination") for ext in self.__extensions: common.check_type('extensions', ext, MooseDocs.base.components.Extension) ext.setTranslator(self) ext.extend(self.__reader, self.__renderer) for comp in self.__reader.components: if comp.extension is None: comp.setExtension(ext) for comp in self.__renderer.components: if comp.extension is None: comp.setExtension(ext) # Set the translator for comp in self.__reader.components: comp.setTranslator(self) for comp in self.__renderer.components: comp.setTranslator(self) # Check that the extension requirements are met for ext in self.__extensions: self.__checkRequires(ext) # Call Extension init() method self.__initialized = True LOG.info('Executing extension init() methods...') t = time.time() self.executeExtensionFunction('init', None) LOG.info('Executing extension init() methods complete [%s sec.]', time.time() - t) # Initialize the Page objects for node in self.__content: node.base = destination if isinstance(node, pages.Source): node.output_extension = self.__renderer.EXTENSION
def _executeExtensionFunction(self, name, page, args=tuple()): """Helper to call pre/post functions for extensions, reader, and renderer.""" if MooseDocs.LOG_LEVEL == logging.DEBUG: check_type('name', name, str) for ext in self.translator.extensions: if ext.active: self._callFunction(ext, name, page, args)
def buildObject(self, parent, pattern, info): #pylint: disable=no-self-use """ Return a token object for the given lexer information. """ obj = pattern.function(info, parent) if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('obj', obj, (tokens.Token, type(None)), exc=exceptions.TokenizeException) return obj
def tokenize(self, parent, text, page, grammar, line=1): """ Perform tokenization of the supplied text. Inputs: parent[tree.tokens]: The parent token to which the new token(s) should be attached. grammar[Grammar]: Object containing the grammar (defined by regexs) to search. text[str]: The text to tokenize. line[int]: The line number to startwith, this allows for nested calls to begin with the correct line. NOTE: If the functions attached to the Grammar object raise an Exception it will be caught by this object and converted into an Exception token. This allows for the entire text to be tokenized and have the errors report upon completion. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('page', page, pages.Page) common.check_type('line', line, int) if not isinstance(text, str): msg = "EXCEPTION: {}:{}\n{}".format( page.source, line, "The supplied text must be str.") raise TypeError(msg) n = len(text) pos = 0 while pos < n: match = None for pattern in grammar: match = pattern.regex.match(text, pos) if match: info = LexerInformation(match, pattern, line) try: obj = self.buildToken(parent, pattern, info, page) except Exception as e: #pylint: disable=broad-except obj = tokens.ErrorToken( parent, message=str(e), traceback=traceback.format_exc()) if obj is not None: obj.info = info line += match.group(0).count('\n') pos = match.end() break else: continue if match is None: break # Produce Exception token if text remains that was not matched if pos < n: msg = 'Unprocessed text exists.' tokens.ErrorToken(parent, message=msg)
def buildObject(self, parent, pattern, info): #pylint: disable=no-self-use """ Return a token object for the given lexer information. """ obj = pattern.function(info, parent) if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('obj', obj, (tokens.Token, type(None)), exc=exceptions.TokenizeException) return obj
def tokenize(self, parent, grammer, text, line=1): """ Perform tokenization of the supplied text. Inputs: parent[tree.tokens]: The parent token to which the new token(s) should be attached. grammer[Grammer]: Object containing the grammer (defined by regexs) to search. text[unicode]: The text to tokenize. line[int]: The line number to startwith, this allows for nested calls to begin with the correct line. NOTE: If the functions attached to the Grammer object raise a TokenizeException it will be caught by this object and converted into an Exception token. This allows for the entire text to be tokenized and have the errors report upon completion. The TokenizeException also contains information about the error, via a LexerInformation object to improve error reports. """ common.check_type('text', text, unicode, exc=exceptions.TokenizeException) n = len(text) pos = 0 while pos < n: match = None for pattern in grammer: match = pattern.regex.match(text, pos) if match: info = LexerInformation(match, pattern, line) try: obj = self.buildObject(parent, pattern, info) except Exception as e: #pylint: disable=broad-except obj = tokens.ExceptionToken( parent, info=info, message=e.message, traceback=traceback.format_exc()) if obj is not None: obj.info = info #TODO: set ptype on base Token, change to info line += match.group(0).count('\n') pos = match.end() break else: continue if match is None: break # Produce Exception token if text remains that was not matched if pos < n: msg = u'Unprocessed text exists:\n{}'.format( common.box(text[pos:], line=line)) LOG.error(msg)
def executeExtensionFunction(self, name, *args): """ Execute pre/post functions for extensions. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', name, str) if name not in self.__extension_functions: msg = "The supplied extension function name '{}' does not exist, the possible " \ "names include: {}." raise exceptions.MooseDocsException(msg, name, self.__extension_functions.keys()) for func in self.__extension_functions[name]: func(*args)
def add(self, name, component): """ Associate a RenderComponent object with a token type. Inputs: name[str]: The token name (e.g., "String") to associate with the supplied component. compoment[RenderComponent]: The component to execute with the associated token type. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type("name", name, unicode) common.check_type("component", component, MooseDocs.base.components.RenderComponent) component.setRenderer(self) self.addComponent(component) self.__functions[name] = self._method(component)
def addCommand(self, reader, command): # Type checking common.check_type('reader', reader, Reader) common.check_type('command', command, CommandComponent) common.check_type('COMMAND', command.COMMAND, str) common.check_type('SUBCOMMAND', command.SUBCOMMAND, (type(None), str, tuple)) # Initialize the component command.setReader(reader) command.setExtension(self) # Subcommands can be tuples if not isinstance(command.SUBCOMMAND, tuple): subcommands = tuple([command.SUBCOMMAND]) else: subcommands = command.SUBCOMMAND # Add the command and error if it exists for sub in subcommands: pair = (command.COMMAND, sub) if pair in CommandExtension.EXTENSION_COMMANDS: msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'." raise common.exceptions.MooseDocsException( msg, pair[0], pair[1]) CommandExtension.EXTENSION_COMMANDS[pair] = command
def init(self, translator): """ Called by Translator object, this allows the objects to be created independently then passed into the translator, which then calls this method to provide access to translator for when the actual tokenize and render commands are called. """ if self.initialized(): msg = "The {} object has already been initialized, this method should not " \ "be called twice." raise MooseDocs.common.exceptions.MooseDocsException(msg, type(self)) common.check_type('translator', translator, MooseDocs.base.translators.Translator) self.__translator = translator
def executeExtensionFunction(self, name, *args): """ Execute pre/post functions for extensions. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', name, str) if name not in self.__extension_functions: msg = "The supplied extension function name '{}' does not exist, the possible " \ "names include: {}." raise exceptions.MooseDocsException( msg, name, self.__extension_functions.keys()) for func in self.__extension_functions[name]: func(*args)
def addCommand(self, reader, command): # Type checking common.check_type('reader', reader, Reader) common.check_type('command', command, CommandComponent) common.check_type('COMMAND', command.COMMAND, str) common.check_type('SUBCOMMAND', command.SUBCOMMAND, (type(None), str, tuple)) # Initialize the component command.setReader(reader) command.setExtension(self) # Subcommands can be tuples if not isinstance(command.SUBCOMMAND, tuple): subcommands = tuple([command.SUBCOMMAND]) else: subcommands = command.SUBCOMMAND # Add the command and error if it exists for sub in subcommands: pair = (command.COMMAND, sub) if pair in CommandExtension.EXTENSION_COMMANDS: msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'." raise common.exceptions.MooseDocsException(msg, pair[0], pair[1]) CommandExtension.EXTENSION_COMMANDS[pair] = command
def add(self, name, component): """ Associate a RenderComponent object with a token type. Inputs: name[str]: The token name (e.g., "String") to associate with the supplied component. compoment[RenderComponent]: The component to execute with the associated token type. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type("name", name, unicode) common.check_type("component", component, MooseDocs.base.components.RenderComponent) component.setRenderer(self) self.addComponent(component) self.__functions[name] = self._method(component)
def add(self, token, component): """ Associate a RenderComponent object with a token type. Inputs: token[type]: The token type (not instance) to associate with the supplied component. compoment[RenderComponent]: The component to execute with the associated token type. """ common.check_type("token", token, type) common.check_type("component", component, MooseDocs.base.components.RenderComponent) if self.initialized(): # allow use without Translator object component.init(self.translator) self.addComponent(component) self.__functions[token] = self._method(component)
def init(self): """ Initialize the translator with the output destination for the converted content. This method also initializes all the various items within the translator for performing the conversion. It is required to allow the build command to modify configuration items (i.e., the 'destination' option) prior to setting up the extensions. Inputs: destination[str]: The path to the output directory. """ if self.__initialized: msg = "The {} object has already been initialized, this method should not " \ "be called twice." raise MooseDocs.common.exceptions.MooseDocsException(msg, type(self)) # Attach translator to Executioner self.__executioner.setTranslator(self) # Initialize the extension and call the extend method, then set the extension object # on each of the extensions. destination = self.get("destination") for ext in self.__extensions: common.check_type('extensions', ext, MooseDocs.base.components.Extension) ext.setTranslator(self) ext.extend(self.__reader, self.__renderer) for comp in self.__reader.components: if comp.extension is None: comp.setExtension(ext) for comp in self.__renderer.components: if comp.extension is None: comp.setExtension(ext) # Set the translator for comp in self.__reader.components: comp.setTranslator(self) for comp in self.__renderer.components: comp.setTranslator(self) # Check that the extension requirements are met for ext in self.__extensions: self.__checkRequires(ext) for node in self.__content: node.base = destination if isinstance(node, pages.Source): node.output_extension = self.__renderer.EXTENSION self.__initialized = True
def tokenize(self, parent, grammar, text, line=1): """ Perform tokenization of the supplied text. Inputs: parent[tree.tokens]: The parent token to which the new token(s) should be attached. grammar[Grammar]: Object containing the grammar (defined by regexs) to search. text[unicode]: The text to tokenize. line[int]: The line number to startwith, this allows for nested calls to begin with the correct line. NOTE: If the functions attached to the Grammar object raise a TokenizeException it will be caught by this object and converted into an Exception token. This allows for the entire text to be tokenized and have the errors report upon completion. The TokenizeException also contains information about the error, via a LexerInformation object to improve error reports. """ common.check_type('text', text, unicode, exc=exceptions.TokenizeException) n = len(text) pos = 0 while pos < n: match = None for pattern in grammar: match = pattern.regex.match(text, pos) if match: info = LexerInformation(match, pattern, line) try: obj = self.buildObject(parent, pattern, info) except Exception as e: #pylint: disable=broad-except obj = tokens.ExceptionToken(parent, info=info, message=unicode(e.message), traceback=traceback.format_exc()) if obj is not None: obj.info = info #TODO: set ptype on base Token, change to info line += match.group(0).count('\n') pos = match.end() break else: continue if match is None: break # Produce Exception token if text remains that was not matched if pos < n: msg = u'Unprocessed text exists.' tokens.ErrorToken(parent, info=info, message=msg)
def buildToken(self, parent, pattern, info, page): """ Override the Lexer.buildToken method to recursively tokenize base on group names. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('parent', parent, tokens.Token) common.check_type('info', info, LexerInformation) obj = super(RecursiveLexer, self).buildToken(parent, pattern, info, page) if (obj is not None) and (obj is not parent) and obj.get('recursive'): for key, grammar in self._grammars.iteritems(): if key in info.keys(): text = info[key] if text is not None: self.tokenize(obj, text, page, grammar, info.line) return obj
def init(self, destination): """ Initialize the translator with the output destination for the converted content. This method also initializes all the various items within the translator for performing the conversion. Inputs: destination[str]: The path to the output directory. """ if self.__initialized: msg = "The {} object has already been initialized, this method should not " \ "be called twice." raise MooseDocs.common.exceptions.MooseDocsException( msg, type(self)) common.check_type("destination", destination, str) self.__destination = destination self.__reader.init(self) self.__renderer.init(self) for ext in self.__extensions: common.check_type('extensions', ext, MooseDocs.base.components.Extension) ext.init(self) ext.extend(self.__reader, self.__renderer) for comp in self.__reader.components: if comp.extension is None: comp.extension = ext for comp in self.__renderer.components: if comp.extension is None: comp.extension = ext for func_name in self.__extension_functions: if hasattr(ext, func_name): self.__extension_functions[func_name].append( getattr(ext, func_name)) for node in anytree.PreOrderIter(self.__root): node.base = destination node.init(self) self.__initialized = True
def __init__(self, content, reader, renderer, extensions, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) common.check_type('content', content, page.PageNodeBase) common.check_type('reader', reader, Reader) common.check_type('renderer', renderer, Renderer) common.check_type('extensions', extensions, list) for ext in extensions: common.check_type('extensions', ext, Extension) self.__initialized = False self.__current = None self.__lock = multiprocessing.Lock() self.__root = content self.__extensions = extensions self.__reader = reader self.__renderer = renderer self.__extension_functions = dict(preRender=list(), postRender=list(), preTokenize=list(), postTokenize=list())
def addCommand(self, command): # Type checking common.check_type('command', command, CommandComponent) common.check_type('COMMAND', command.COMMAND, str) common.check_type('SUBCOMMAND', command.SUBCOMMAND, (type(None), str, tuple)) # Initialize the component command.init(self.translator) command.extension = self # Subcommands can be tuples if not isinstance(command.SUBCOMMAND, tuple): subcommands = tuple([command.SUBCOMMAND]) else: subcommands = command.SUBCOMMAND # Add the command and error if it exists for sub in subcommands: pair = (command.COMMAND, sub) if pair in self.translator.__EXTENSION_COMMANDS__: msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'." raise common.exceptions.MooseDocsException( msg, pair[0], pair[1]) self.translator.__EXTENSION_COMMANDS__[pair] = command
def add(self, group, component, location='_end'): """ Add a component to Extened the Reader by adding a TokenComponent. Inputs: group[str]: Name of the lexer group to append. component[components.TokenComponent]: The tokenize component to add. location[str|int]: The location to insert this component (see Grammar.py) """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type("group", group, str) common.check_type("component", component, MooseDocs.base.components.TokenComponent) common.check_type("location", location, (str, int)) # Define the name of the component being added (for sorting within Grammar) name = component.__class__.__name__ # Store and init component, checking self.initialized() allows this object to be used # without the Translator which is useful in some cases. self.addComponent(component) if self.initialized(): component.init(self.translator) # Update the lexer self.__lexer.add(group, name, component.RE, component, location)
def init(self, destination): """ Initialize the translator with the output destination for the converted content. This method also initializes all the various items within the translator for performing the conversion. Inputs: destination[str]: The path to the output directory. """ if self.__initialized: msg = "The {} object has already been initialized, this method should not " \ "be called twice." raise MooseDocs.common.exceptions.MooseDocsException(msg, type(self)) common.check_type("destination", destination, str) self.__destination = destination self.__reader.init(self) self.__renderer.init(self) for ext in self.__extensions: common.check_type('extensions', ext, MooseDocs.base.components.Extension) ext.init(self) ext.extend(self.__reader, self.__renderer) for comp in self.__reader.components: if comp.extension is None: comp.extension = ext for comp in self.__renderer.components: if comp.extension is None: comp.extension = ext for func_name in self.__extension_functions: if hasattr(ext, func_name): self.__extension_functions[func_name].append(getattr(ext, func_name)) for node in anytree.PreOrderIter(self.__root): node.base = destination node.init(self) self.__initialized = True
def addCommand(self, command): # Type checking common.check_type('command', command, CommandComponent) common.check_type('COMMAND', command.COMMAND, str) common.check_type('SUBCOMMAND', command.SUBCOMMAND, (type(None), str, tuple)) # Initialize the component command.init(self.translator) command.extension = self # Subcommands can be tuples if not isinstance(command.SUBCOMMAND, tuple): subcommands = tuple([command.SUBCOMMAND]) else: subcommands = command.SUBCOMMAND # Add the command and error if it exists for sub in subcommands: pair = (command.COMMAND, sub) if pair in self.translator.__EXTENSION_COMMANDS__: msg = "A CommandComponent object exists with the command '{}' and subcommand '{}'." raise common.exceptions.MooseDocsException(msg, pair[0], pair[1]) self.translator.__EXTENSION_COMMANDS__[pair] = command
def add(self, group, component, location='_end'): """ Add a component to Extened the Reader by adding a TokenComponent. Inputs: group[str]: Name of the lexer group to append. component[components.TokenComponent]: The tokenize component to add. location[str|int]: The location to insert this component (see Grammar.py) """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type("group", group, str) common.check_type("component", component, MooseDocs.base.components.TokenComponent) common.check_type("location", location, (str, int)) # Define the name of the component being added (for sorting within Grammar) name = component.__class__.__name__ # Store and init component, checking self.initialized() allows this object to be used # without the Translator which is useful in some cases. self.addComponent(component) if self.initialized(): component.init(self.translator) # Update the lexer self.__lexer.add(group, name, component.RE, component, location)
def __init__(self, content, reader, renderer, extensions, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) common.check_type('content', content, page.PageNodeBase) common.check_type('reader', reader, Reader) common.check_type('renderer', renderer, Renderer) common.check_type('extensions', extensions, list) for ext in extensions: common.check_type('extensions', ext, Extension) self.__initialized = False self.__current = None self.__lock = multiprocessing.Lock() self.__root = content self.__extensions = extensions self.__reader = reader self.__renderer = renderer self.__destination = None # assigned during init() self.__extension_functions = dict(preRender=list(), postRender=list(), preTokenize=list(), postTokenize=list())
def findall(self, name, maxcount=1, mincount=1, exc=exceptions.MooseDocsException): #pylint: disable=no-self-use """ Find method for locating pages. Inputs: name[str]: The name of the page to search. maxcount[int]: The maximum number of items to find (default: 1). mincount[int]: The minimum number of items to find (default: 1). exc[Exception]: The type of exception to raise if min/max are not satisfied. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', name, (str, unicode)) common.check_type('mincount', mincount, (int)) common.check_type('maxcount', maxcount, (int)) common.check_type('exc', exc, (type, types.LambdaType, type(None))) try: return list(CACHE[name]) except KeyError: pass nodes = set() for key in CACHE: if key.endswith(name): nodes.update(CACHE[key]) if (maxcount is not None) and exc and (len(nodes) > maxcount): msg = "The 'maxcount' was set to {} but {} nodes were found for the name '{}'." \ .format(maxcount, len(nodes), name) for node in nodes: msg += '\n {} (source: {})'.format(node.local, node.source) raise exc(msg) elif (mincount is not None) and exc and (len(nodes) < mincount): msg = "The 'mincount' was set to {} but {} nodes were found for the name '{}'." \ .format(mincount, len(nodes), name) for node in nodes: msg += '\n {} (source: {})'.format(node.local, node.source) raise exc(msg) CACHE[name] = nodes return list(nodes)
def __init__(self, name, regex, function): # Perform input type checking common.check_type('name', name, str) common.check_type('regex', regex, type(re.compile(''))) common.check_type('function', function, types.FunctionType) self.name = name self.regex = regex self.function = function
def findall(self, name, maxcount=1, mincount=1, exc=exceptions.MooseDocsException): #pylint: disable=no-self-use """ Find method for locating pages. Inputs: name[str]: The name of the page to search. maxcount[int]: The maximum number of items to find (default: 1). mincount[int]: The minimum number of items to find (default: 1). exc[Exception]: The type of exception to raise if min/max are not satisfied. """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('name', name, (str, unicode)) common.check_type('mincount', mincount, (int)) common.check_type('maxcount', maxcount, (int)) common.check_type('exc', exc, (type, types.LambdaType, type(None))) try: return list(CACHE[name]) except KeyError: pass nodes = set() for key in CACHE: if key.endswith(name): nodes.update(CACHE[key]) if (maxcount is not None) and exc and (len(nodes) > maxcount): msg = "The 'maxcount' was set to {} but {} nodes were found for the name '{}'." \ .format(maxcount, len(nodes), name) for node in nodes: msg += '\n {} (source: {})'.format(node.local, node.source) raise exc(msg) elif (mincount is not None) and exc and (len(nodes) < mincount): msg = "The 'mincount' was set to {} but {} nodes were found for the name '{}'." \ .format(mincount, len(nodes), name) for node in nodes: msg += '\n {} (source: {})'.format(node.local, node.source) raise exc(msg) CACHE[name] = nodes return list(nodes)
def testCallable(self): func = lambda: True common.check_type('foo', func, types.FunctionType) with self.assertRaises(Exception) as e: common.check_type('foo', 42, types.FunctionType) self.assertEqual("The argument 'foo' must be callable but <type 'int'> was provided.", e.exception.message) with self.assertRaises(Exception) as e: common.check_type('foo', 42, list) gold = "The argument 'foo' must be of type <type 'list'> but <type 'int'> was provided." self.assertIn(gold, e.exception.message)
def testCallable(self): func = lambda: True common.check_type('foo', func, types.FunctionType) with self.assertRaises(Exception) as e: common.check_type('foo', 42, types.FunctionType) self.assertEqual( "The argument 'foo' must be callable but <type 'int'> was provided.", e.exception.message) with self.assertRaises(Exception) as e: common.check_type('foo', 42, list) gold = "The argument 'foo' must be of type <type 'list'> but <type 'int'> was provided." self.assertIn(gold, e.exception.message)
def add(self, group, component, location='_end'): """ Add a component to extend the Reader by adding a TokenComponent. This method is called when adding ReaderComonents in the ::extend method. Inputs: group[str]: Name of the lexer group to append. component[components.TokenComponent]: The tokenize component to add. location[str|int]: The location to insert this component (see Grammar.py) """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type("group", group, str) common.check_type("component", component, MooseDocs.base.components.TokenComponent) common.check_type("location", location, (str, int)) component.setReader(self) self.addComponent(component) # Update the lexer name = component.__class__.__name__ self.__lexer.add(group, name, component.RE, component, location)
def add(self, group, component, location='_end'): """ Add a component to extend the Reader by adding a TokenComponent. This method is called when adding ReaderComonents in the ::extend method. Inputs: group[str]: Name of the lexer group to append. component[components.TokenComponent]: The tokenize component to add. location[str|int]: The location to insert this component (see Grammar.py) """ if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type("group", group, str) common.check_type("component", component, MooseDocs.base.components.TokenComponent) common.check_type("location", location, (str, int)) component.setReader(self) self.addComponent(component) # Update the lexer name = component.__class__.__name__ self.__lexer.add(group, name, component.RE, component, location)
def execute(self, num_threads=1): """ Perform parallel build for all pages. Inputs: num_threads[int]: The number of threads to use (default: 1). NOTICE: A proper parallelization for MooseDocs would be three parallel steps, with minimal communication. 1. Read all the markdown files (in parallel). 2. Perform the AST tokenization (in parallel), then communicate the completed AST back to the main process. 3. Convert the AST to HTML (in parallel). 4. Write/copy (in parallel) the completed HTML and other files (images, js, etc.). However, step two is problematic because python requires that the AST be pickled, which is possible, for communication. In doing this I realized that the pickling was a limiting factor and made the AST step very slow. I need to investigate this further to make sure I was using a non-locking pool of workers, but this was taking too much development time. The current implementation performs all four steps together, which generally works just fine, with one exception. The autolink extension actually interrogates the AST from other pages. Hence, if the other page was generated off process the information is not available. The current implementation will just compute the AST locally (i.e., I am performing repeated calculations in favor of communication). This works well enough for now, but as more autolinking is preformed and other similar extensions are created this could cause a slow down. Long term this should be looked into again, for now the current approach is working well. This new system is already an order of 4 times faster than the previous implementation and likely could be optimized further. The multiprocessing.Manager() needs to be explored, it is working to pull the JSON index information together. """ common.check_type('num_threads', num_threads, int) self.__assertInitialize() # Log start message and time LOG.info("Building Pages...") start = time.time() manager = multiprocessing.Manager() array = manager.list() def target(nodes, lock): """Helper for building multiple nodes (i.e., a chunk for a process).""" for node in nodes: node.build() if isinstance(node, page.MarkdownNode): node.buildIndex(self.renderer.get('home', None)) with lock: for entry in node.index: array.append(entry) # Complete list of nodes nodes = [n for n in anytree.PreOrderIter(self.root)] # Serial if num_threads == 1: target(nodes, self.lock) # Multiprocessing else: jobs = [] for chunk in mooseutils.make_chunks(nodes, num_threads): p = multiprocessing.Process(target=target, args=(chunk, self.lock)) p.start() jobs.append(p) for job in jobs: job.join() # Done stop = time.time() LOG.info("Build time %s sec.", stop - start) iname = os.path.join(self.destination, 'js', 'search_index.js') if not os.path.isdir(os.path.dirname(iname)): os.makedirs(os.path.dirname(iname)) items = [v for v in array if v] common.write(iname, 'var index_data = {};'.format(json.dumps(items)))
def __init__(self, lexer, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) mixins.ComponentObject.__init__(self) mixins.TranslatorObject.__init__(self) common.check_type('lexer', lexer, RecursiveLexer) self.__lexer = lexer
def setTranslator(self, translator): """ Method called by Translator to allow find methods to operate. """ check_type('translator', translator, MooseDocs.base.translators.Translator) self.__translator = translator
def addComponent(self, comp): """ Add a Component object. """ check_type("component", comp, MooseDocs.base.components.Component) self.__components.append(comp)
def setRenderer(self, renderer): """Initialize the class with the Renderer object.""" check_type('renderer', renderer, MooseDocs.base.renderers.Renderer) self.__renderer = renderer
def __init__(self, lexer, **kwargs): mixins.ConfigObject.__init__(self, **kwargs) mixins.ComponentObject.__init__(self) mixins.TranslatorObject.__init__(self) common.check_type('lexer', lexer, RecursiveLexer) self.__lexer = lexer
def current(self, value): """Set the current page being translated, see page.MarkdownNode.build().""" if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('value', value, (type(None), page.PageNodeBase)) self.__current = value