def __initClassDatabase(self): """Initialize the class database for faster searching.""" # Do nothing if the syntax failed to build if self._app_syntax is None: return start = time.time() LOG.info("Building MOOSE class database...") self._database = common.build_class_database(self['includes'], self['inputs']) # Cache the syntax entries, search the tree is very slow self._cache = dict() self._object_cache = dict() self._syntax_cache = dict() for node in moosetree.iterate(self._app_syntax): if not node.removed: self._cache[node.fullpath] = node if node.alias: self._cache[node.alias] = node if isinstance(node, syntax.ObjectNode): self._object_cache[node.fullpath] = node if node.alias: self._object_cache[node.alias] = node elif isinstance(node, syntax.SyntaxNode): self._syntax_cache[node.fullpath] = node if node.alias: self._syntax_cache[node.alias] = node LOG.info("MOOSE class database complete [%s sec]", time.time() - start)
def testBreadthFirst(self): root = build_tree() nodes = list(moosetree.iterate(root)) self.assertEqual(len(nodes), 29) self.assertEqual(nodes[0].name, 'A') self.assertEqual(nodes[1].name, 'B') self.assertEqual(nodes[2].name, 'C') self.assertEqual(nodes[3].name, 'D') self.assertEqual(nodes[4].name, 'AA') self.assertEqual(nodes[5].name, 'AB') self.assertEqual(nodes[6].name, 'AC') self.assertEqual(nodes[7].name, 'BA') self.assertEqual(nodes[8].name, 'BB') self.assertEqual(nodes[9].name, 'BC') self.assertEqual(nodes[10].name, 'BD') self.assertEqual(nodes[11].name, 'CA') self.assertEqual(nodes[12].name, 'CB') self.assertEqual(nodes[13].name, 'CC') self.assertEqual(nodes[14].name, 'DA') self.assertEqual(nodes[15].name, 'DB') self.assertEqual(nodes[16].name, 'ABA') self.assertEqual(nodes[17].name, 'ABB') self.assertEqual(nodes[18].name, 'ABC') self.assertEqual(nodes[19].name, 'ABD') self.assertEqual(nodes[20].name, 'BAA') self.assertEqual(nodes[21].name, 'BAB') self.assertEqual(nodes[22].name, 'CCA') self.assertEqual(nodes[23].name, 'CCB') self.assertEqual(nodes[24].name, 'CCC') self.assertEqual(nodes[25].name, 'ABCA') self.assertEqual(nodes[26].name, 'ABCB') self.assertEqual(nodes[27].name, 'ABCAA') self.assertEqual(nodes[28].name, 'ABCAB')
def postTokenize(self, page, ast): labels = dict() count = 0 func = lambda n: (n.name == 'Equation') and (n['label'] is not None) for node in moosetree.iterate(ast, func): count += 1 node['number'] = count labels[node['label']] = (count, node['bookmark']) # TODO: When !eqref is used for references, this should be removed core.Shortcut(ast, key=node['label'], string='{} ({})'.format(self.get('prefix'), count), link='#{}'.format(node['bookmark'])) page['labels'] = labels renderer = self.translator.renderer if common.has_tokens(ast, 'Equation') and isinstance(renderer, renderers.HTMLRenderer): renderer.addCSS('katex', "contrib/katex/katex.min.css", page) renderer.addCSS('katex_moose', "css/katex_moose.css", page) renderer.addJavaScript('katex', "contrib/katex/katex.min.js", page, head=True) if self.get('macros', None): mc = ','.join('"{}":"{}"'.format(k, v) for k, v in self.get('macros').items()) self.setAttribute('macros', '{' + mc + '}')
def postTokenize(self, page, ast): """Set float number for each counter.""" counts = collections.defaultdict(int) floats = dict() for node in moosetree.iterate(ast, lambda n: n.name == 'FloatCaption'): prefix = node.get('prefix', None) if prefix is not None: counts[prefix] += 1 node['number'] = counts[prefix] key = node.get('key') if key: floats[key] = node.copy() shortcut = core.Shortcut(ast.root, key=key, link='#{}'.format(key)) # TODO: This is a bit of a hack to get Figure~\ref{} etc. working in general if isinstance(self.translator.renderer, LatexRenderer): shortcut['prefix'] = prefix.title() else: tokens.String(shortcut, content='{} {}'.format( prefix.title(), node['number'])) page['counts'] = counts page['floats'] = floats
def testPreOrder(self): root = build_tree() nodes = list(moosetree.iterate(root, method=moosetree.IterMethod.PRE_ORDER)) self.assertEqual(len(nodes), 29) self.assertEqual(nodes[0].name, 'A') self.assertEqual(nodes[1].name, 'AA') self.assertEqual(nodes[2].name, 'AB') self.assertEqual(nodes[3].name, 'ABA') self.assertEqual(nodes[4].name, 'ABB') self.assertEqual(nodes[5].name, 'ABC') self.assertEqual(nodes[6].name, 'ABCA') self.assertEqual(nodes[7].name, 'ABCAA') self.assertEqual(nodes[8].name, 'ABCAB') self.assertEqual(nodes[9].name, 'ABCB') self.assertEqual(nodes[10].name, 'ABD') self.assertEqual(nodes[11].name, 'AC') self.assertEqual(nodes[12].name, 'B') self.assertEqual(nodes[13].name, 'BA') self.assertEqual(nodes[14].name, 'BAA') self.assertEqual(nodes[15].name, 'BAB') self.assertEqual(nodes[16].name, 'BB') self.assertEqual(nodes[17].name, 'BC') self.assertEqual(nodes[18].name, 'BD') self.assertEqual(nodes[19].name, 'C') self.assertEqual(nodes[20].name, 'CA') self.assertEqual(nodes[21].name, 'CB') self.assertEqual(nodes[22].name, 'CC') self.assertEqual(nodes[23].name, 'CCA') self.assertEqual(nodes[24].name, 'CCB') self.assertEqual(nodes[25].name, 'CCC') self.assertEqual(nodes[26].name, 'D') self.assertEqual(nodes[27].name, 'DA') self.assertEqual(nodes[28].name, 'DB')
def processLatexOutput(output, nodes): """Convert the pdfLatex log into a tree structure to capture the warnings. Inputs: output[str]: The console output or the pdflatex log. content[list]: The list of page objects from the translator. """ # Items for capturing errors warn = collections.namedtuple('LatexWarning', 'content line') line_re = re.compile(r'line\s(?P<line>[0-9]+)') regex = re.compile(r'^(?P<filename>\.*[^\.]+\.\w+)(?P<content>.*)', flags=re.MULTILINE|re.DOTALL) # Convert output to a tree structure root = PDFExtension.parseOutput(output) # Loop through the result and capture filenames and warnings for n in moosetree.iterate(root): match = regex.search(n['content']) if match: n['content'] = match.group('content') n['filename'] = match.group('filename').replace('\n', '') n['content'] = [c.strip().replace('\n', '') for c in re.split(r'\n{2,}', n['content']) if c] for c in n['content']: if 'LaTeX Warning' in c: c = c.replace('LaTeX Warning:', '').strip() match = line_re.search(c) line = int(match.group('line')) if match else None n['warnings'].append(warn(content=c, line=line)) return root
def _addRequirement(self, parent, info, page, req): reqname = "{}:{}".format(req.path, req.name) if req.path != '.' else req.name item = SQARequirementMatrixItem(parent, label=req.label, reqname=reqname) self.reader.tokenize(item, req.text, page, MarkdownReader.INLINE, info.line, report=False) for token in moosetree.iterate(item): if token.name == 'ErrorToken': msg = common.report_error("Failed to tokenize SQA requirement.", req.filename, req.text_line, req.text, token['traceback'], 'SQA TOKENIZE ERROR') LOG.critical(msg) p = core.Paragraph(item) tokens.String(p, content='Specification: ') content = common.read(req.filename) floats.create_modal_link(p, string=reqname, content=core.Code(None, language='text', content=content), title=str(req.filename)) p = core.Paragraph(item) tokens.String(p, content='Documentation: ') filename = getattr(req, info['subcommand']) autolink.AutoLink(p, page=str(filename))
def _addSections(self, container, page): """ Group content into <section> tags based on the heading tag. Inputs: root[tree.html.Tag]: The root tree.html node tree to add sections.read collapsible[list]: A list with six entries indicating the sections to create as collapsible. """ collapsible = self.getConfig(page, 'collapsible-sections') if isinstance(collapsible, str): collapsible = eval(collapsible) section = container for child in section.children: if child.name in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'): level = int(child.name[-1]) current = section.get("data-section-level", 0) # get the current section level if level == current: section = html.Tag(section.parent, 'section') elif level > current: section = html.Tag(section, 'section') elif level < current: section = html.Tag(section.parent.parent, 'section') section['data-section-level'] = level section['data-section-text'] = child.text() section['id'] = uuid.uuid4() if 'data-details-open' in child: section['data-details-open'] = child['data-details-open'] child.parent = section for node in moosetree.iterate(container, lambda n: n.name == 'section'): if 'data-details-open' in node: status = node['data-details-open'] else: status = collapsible[node['data-section-level'] - 1] if status: summary = html.Tag(None, 'summary') node(0).parent = summary details = html.Tag(None, 'details', class_="moose-section-content") if status.lower() == 'open': details['open'] = 'open' details.children = node.children summary.parent = details details.parent = node icon = html.Tag(None, 'span', class_='moose-section-icon') summary(0).children = [icon] + list(summary(0).children)
def text(self, sep=' '): """ Convert String objects into a single string. """ strings = [] for node in moosetree.iterate(self): if node.name in ['Word', 'Number', 'String']: strings.append(node['content']) return sep.join(strings)
def postTokenize(self, page, ast): """ Adds a list of valid acronyms to the page attributes. """ func = lambda n: (n.name == 'AcronymToken') for node in moosetree.iterate(ast.root, func): acro = node.get('acronym') if acro in self.__acronyms.keys() and acro not in page['acronyms'].keys(): page['acronyms'][acro] = self.__acronyms.get(acro)
def text(self): """ Convert String objects into a single string. """ strings = [] for node in moosetree.iterate(self): if node.name == 'String': strings.append(node['content']) return re.sub(r' {2,}', ' ', ' '.join(strings))
def postTokenize(self, ast, page, meta, reader): data = dict() func = lambda n: (n.name == 'Heading') for node in moosetree.iterate(ast.root, func): id_ = node.get('id', '') if id_ not in data: data[id_] = node.copy() meta.setData('heading', data)
def _check_syntax_node(node, app_name, generate, update, prefix, log_type): """ Check that required pages for syntax exists (e.g., Adaptivity/index.md). """ # Tuple for storing filename and existence FileInfo = collections.namedtuple('FileInfo', 'name exists') # Build a set if information tuples to consider filenames = set() func = lambda n: isinstance(n, syntax.ActionNode) and not n.removed actions = moosetree.iterate(node, func) for action in actions: idx = action.source().find('/src/') name = os.path.join(action.source()[:idx], prefix, os.path.dirname(node.markdown()), 'index.md') filenames.add(FileInfo(name=name, exists=os.path.isfile(name))) # Case when no file exists not_exist = all(not f.exists for f in filenames) if not_exist and (not generate) and (not node.hidden): msg = "No documentation for {}.\n".format(node.fullpath) if len(filenames) == 1: msg += " - The page should be located at {}.\n".format( filenames.pop().name) msg += " - It is possible to generate stub pages using " \ "'./moosedocs.py check --generate'." else: msg += " - The page should be located at one of the following locations:\n" for info in filenames: msg += " {}\n".format(info.name) LOG.log(log_type, msg) # Case when when no file exists but --generate was provided elif not_exist and generate: if len(filenames) == 1: fname = filenames.pop().name LOG.info("Creating stub page for %s at %s.", node.fullpath, fname) if not os.path.exists(os.path.dirname(fname)): os.makedirs(os.path.dirname(fname)) with open(fname, 'w') as fid: content = _default_content(node) fid.write(content) else: msg = "A stub page for {} cannot be created because the syntax definition location " \ "exists in multiple applications, the file can be located at any of the " \ "following locations.\n".format(node.fullpath) for info in filenames: msg += " {}\n".format(info.name) LOG.log(log_type, msg) else: for info in filenames: if info.exists: _check_page_for_stub(node, app_name, info.name, update, log_type)
def postTokenize(self, page, ast): func = lambda n: (n.name == 'Heading') for node in moosetree.iterate(ast.root, func): id_ = node.get('id') or node.text('-').lower() node['id'] = id_ if page['title'] is None: page['title'] = node.copy() if id_ not in page['headings']: page['headings'][id_] = node.copy()
def _addRequirement(self, parent, info, page, req, requirements): reqname = "{}:{}".format(req.path, req.name) if req.path != '.' else req.name item = SQARequirementMatrixItem(parent, label=req.label, reqname=reqname, satisfied=req.satisfied) text = SQARequirementText(item) self.reader.tokenize(text, req.text, page, MooseDocs.INLINE, info.line, report=False) for token in moosetree.iterate(item): if token.name == 'ErrorToken': msg = common.report_error("Failed to tokenize SQA requirement.", req.filename, req.text_line, req.text, token['traceback'], 'SQA TOKENIZE ERROR') LOG.critical(msg) if req.details: details = SQARequirementDetails(item) for detail in req.details: ditem = SQARequirementDetailItem(details) text = SQARequirementText(ditem) self.reader.tokenize(text, detail.text, page, MooseDocs.INLINE, info.line, \ report=False) if self.settings['link']: if self.settings['link-spec']: p = SQARequirementSpecification(item, spec_path=req.path, spec_name=req.name) hit_root = mooseutils.hit_load(req.filename) h = hit_root.find(req.name) content = h.render() floats.create_modal_link(p, title=reqname, string=reqname, content=core.Code(None, language='text', content=content)) if self.settings['link-design'] and req.design: p = SQARequirementDesign(item, filename=req.filename, design=req.design, line=req.design_line) if self.settings['link-issues'] and req.issues: p = SQARequirementIssues(item, filename=req.filename, issues=req.issues, line=req.issues_line) if self.settings.get('link-prerequisites', False) and req.prerequisites: labels = [] for prereq in req.prerequisites: for other in requirements: if (other.name == prereq) and (other.path == req.path): labels.append((other.path, other.name, other.label)) for detail in other.details: if (detail.name == prereq) and (detail.path == req.path): labels.append((other.path, other.name, other.label)) SQARequirementPrequisites(item, specs=labels)
def _processPages(self, root): """ Build a main latex file that includes the others. """ main = base.NodeBase(None, None) latex.Command(main, 'documentclass', string='report', end='') for package, options in self.translator.renderer.getPackages().items(): args = [] if options[0] or options[1]: args = [self._getOptions(*options)] latex.Command(main, 'usepackage', args=args, string=package, start='\n', end='') latex.String(main, content='\\setlength{\\parindent}{0pt}', start='\n', escape=False) for preamble in self.translator.renderer.getPreamble(): latex.String(main, content='\n' + preamble, escape=False) # New Commands for cmd in self.translator.renderer.getNewCommands().values(): cmd.parent = main doc = latex.Environment(main, 'document', end='\n') for node in moosetree.iterate(root, lambda n: 'page' in n): page = node['page'] if self.translator.getMetaData(page, 'active'): cmd = latex.Command(doc, 'input', start='\n') latex.String(cmd, content=str(page.destination), escape=False) # BibTeX bib_files = [ n.source for n in self.translator.content if n.source.endswith('.bib') ] if bib_files: latex.Command(doc, 'bibliographystyle', start='\n', string='unsrtnat') latex.Command(doc, 'bibliography', string=','.join(bib_files), start='\n', escape=False) return main
def check(translator, dump=False, update=False, generate=False, object_prefix=os.path.join('doc', 'content', 'source'), syntax_prefix=os.path.join('doc', 'content', 'syntax')): """Helper to all both main and build.py:main to perform check.""" # Extract the syntax root node app_syntax = None extension = None for ext in translator.extensions: if ext.name == 'appsyntax': extension = ext extension.preExecute(translator.content) app_syntax = ext.syntax break if (extension is None) or (not extension.active): LOG.info("Syntax is disabled, skipping the check.") return 0 if app_syntax is None: msg = "Failed to locate AppSyntaxExtension for the given configuration." raise exceptions.MooseDocsException(msg) # Dump the complete syntax for the application if dump: print(app_syntax) # The default build for any application creates the test app (e.g., FrogTestApp), therefore # the actual application name must be captured from this name to properly generate stub pages. # This is because the --generate option will only create stub pages for objects registered # within the running application, otherwise pages for all MOOSE and modules objects would # be generated if they were not included. However, for a given object the registered application # from the register macros is used, thus the compiled application used for building # documentation (FrogTestApp) and the object registration name (FrogApp) are always out of sync. # The simple fix is to change the name of the running application here. app_name = extension.apptype.replace('TestApp', 'App') # Perform check for all the nodes for node in moosetree.iterate(app_syntax): if node.is_root or node.removed: continue elif isinstance(node, syntax.ObjectNode): _check_object_node(node, app_name, generate, update, object_prefix) elif isinstance(node, syntax.SyntaxNode): _check_syntax_node(node, app_name, generate, update, syntax_prefix) else: LOG.critical( "Unexpected object type of %s, only %s and %s based types are supported", type(node).__name__, syntax.ObjectNode.__name__, syntax.SyntaxNode.__name__) return 0
def postTokenize(self, page, ast): if page['citations']: has_bib = False for node in moosetree.iterate(ast): if node.name == 'BibtexBibliography': has_bib = True break if not has_bib: core.Heading(ast, level=2, string='References') BibtexBibliography(ast, bib_style='plain')
def preExecute(self): """Populate a list of registered applications.""" syntax = None for ext in self.translator.extensions: if isinstance(ext, appsyntax.AppSyntaxExtension): syntax = ext.syntax break if syntax is not None: for node in moosetree.iterate(syntax): self._registerd_apps.update(node.groups())
def getShortcut(self, token): key = token['key'] if key in self.__cache: return self.__cache[key] #TODO: error if more than one found for node in moosetree.iterate(token.root): if node.name == 'Shortcut' and node['key'] == key: self.__cache[key] = node return node msg = "The shortcut link key '{}' was not located in the list of shortcuts." raise exceptions.MooseDocsException(msg, key)
def postTokenize(self, ast, page, meta, reader): if self.__citations: meta.getData('citations').extend(self.__citations) del self.__citations[:] # TODO: In python 3 this should be a clear() has_bib = False for node in moosetree.iterate(ast): if node.name == 'BibtexBibliography': has_bib = True break if not has_bib: BibtexBibliography(ast)
def postTokenize(self, page, ast): if self.__citations: page['citations'].extend(self.__citations) self.__citations.clear() has_bib = False for node in moosetree.iterate(ast): if node.name == 'BibtexBibliography': has_bib = True break if not has_bib: BibtexBibliography(ast)
def __initApplicationSyntax(self): """Initialize the application syntax.""" start = time.time() LOG.info("Reading MOOSE application syntax...") exe = mooseutils.eval_path(self['executable']) exe = mooseutils.find_moose_executable(exe, name=self['app_name'], show_error=False) self._app_exe = exe if exe is None: LOG.error("Failed to locate a valid executable in %s.", self['executable']) else: try: self._app_syntax = moosesyntax.get_moose_syntax_tree( exe, remove=self['remove'], alias=self['alias'], unregister=self['unregister']) out = mooseutils.runExe(exe, ['--type']) match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$', out, flags=re.MULTILINE) if match: self._app_type = match.group("type") else: msg = "Failed to determine application type by running the following:\n" msg += " {} --type".format(exe) LOG.error(msg) except Exception as e: msg = "Failed to load application executable from '{}' with the following error; " \ "application syntax is being disabled.\n".format(exe) msg += '\n{}\n'.format( mooseutils.colorText(traceback.format_exc(), 'GREY')) msg += "This typically indicates that the application is not producing JSON output " \ "correctly, try running the following:\n" \ " {} --json --allow-test-objects\n".format(exe) self.setActive(False) LOG.error(msg) # Enable test objects by removing the test flag (i.e., don't consider them test objects) if self['allow-test-objects'] and (self._app_syntax is not None): for node in moosetree.iterate(self._app_syntax): node.test = False LOG.info("MOOSE application syntax complete [%s sec.]", time.time() - start)
def _getNodeIDs(root): """Return all 'ids' in a node tree.""" keys = [] id_ = root.get('id', None) if id_: keys.append(id_) for node in moosetree.iterate(root, method=moosetree.IterMethod.PRE_ORDER): id_ = node.get('id', None) if id_: keys.append(id_) return keys
def _reportLatexWarnings(self, lnode, content): """Helper to display latex warnings.""" # Locate the Page object where the error was producec. pnode = None for page in content: if lnode['filename'] in page.destination: pnode = page break # Get the rendered result tree and locate the start/end lines for each node result = None if pnode is not None: pass # TODO: The ability to get the ResultTree has been removed, this was the only function # calling it and the potential slow downs and abuse out pace the use case here #result = self.translator.getResultTree(pnode) #result['_start_line'] = 1 #self._lineCounter(result) # Report warning(s) for w in lnode['warnings']: # Locate the rendered node that that caused the error r_node = None if result: for r in moosetree.iterate(result): if w.line >= r.get('_start_line', float('Inf')): r_node = r # Build message msg = '\n' msg += mooseutils.colorText('pdfLaTeX Warning: {}\n'.format(w.content), 'LIGHT_YELLOW') if r_node: msg += box(r_node.write(), title='IN: {}:{}'.format(pnode.destination, w.line), width=100, color='GREY') msg += '\n' info = r_node.get('info', None) if info is not None: msg += box(info[0], title='FROM: {}:{}'.format(pnode.source, info.line), color='GREY') LOG.warning(msg)
def postTokenize(self, page, ast): items = set() fields = set() for node in moosetree.iterate(ast): if node.name == 'TemplateItem': items.add(node['key']) elif node.name == 'TemplateField': fields.add(node['key']) unknown_items = items.difference(fields) if unknown_items: msg = "Unknown template item(s): {}\n{}" raise exceptions.MooseDocsException(msg, ', '.join(unknown_items), page.source)
def checkDuplicates(root): """Check for duplicate blocks and/or parameters""" paths = set() for node in moosetree.iterate(root, method=moosetree.IterMethod.PRE_ORDER): if node.fullpath in paths: yield ('duplicate section "{}"'.format(node.fullpath), node) else: paths.add(node.fullpath) for key, _ in node.params(): fullparam = os.path.join(node.fullpath, key) if fullparam in paths: yield ('duplicate parameter "{}"'.format(fullparam), node, key) else: paths.add(fullparam)
def removeInputBlocks(hit, remove): """Remove input file block(s) and/or parameter(s)""" for r in remove.split(): for node in moosetree.iterate(hit): block, param = InputListingCommand.removeHelper(node, r) if block is not None: if param is None: node.remove() else: node.removeParam(param) break if block is None: msg = 'Unable to locate block or parameter with name: {}' raise exceptions.MooseDocsException(msg, r) return str(hit.render())
def __initApplicationSyntax(self): """Initialize the application syntax.""" start = time.time() LOG.info("Reading MOOSE application syntax...") exe = mooseutils.eval_path(self['executable']) exe = mooseutils.find_moose_executable(exe, name=self['app_name'], show_error=False) self._app_exe = exe if exe is None: LOG.error("Failed to locate a valid executable in %s.", self['executable']) else: try: self._app_syntax = moosesyntax.get_moose_syntax_tree( exe, remove=self['remove'], alias=self['alias'], unregister=self['unregister']) out = mooseutils.runExe(exe, ['--type']) match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$', out, flags=re.MULTILINE) if match: self._app_type = match.group("type") else: msg = "Failed to determine application type by running the following:\n" msg += " {} --type".format(exe) LOG.error(msg) except Exception as e: msg = "Failed to load application executable from '%s', " \ "application syntax is being disabled:\n%s" self.setActive(False) LOG.error(msg, exe, e) # Enable test objects by removing the test flag (i.e., don't consider them test objects) if self['allow-test-objects']: for node in moosetree.iterate(self._app_syntax): node.test = False LOG.info("MOOSE application syntax complete [%s sec.]", time.time() - start)
def postTokenize(self, ast, page, meta, reader): labels = set() count = 0 func = lambda n: (n.name == 'LatexBlockEquation') and (n['label'] is not None) for node in moosetree.iterate(ast, func): count += 1 node['number'] = count if node['label']: if isinstance(self.translator.renderer, renderers.LatexRenderer): labels.add(node['label']) else: core.Shortcut(ast, key=node['label'], string='{} ({})'.format( self.get('prefix'), count), link='#{}'.format(node['bookmark'])) meta.setData('labels', labels)