def find(self, name, attr=None, maxlevel=None): """ Search for a node, by name. """ if attr is None: for node in anytree.PreOrderIter(self, maxlevel=maxlevel): if node.name == name: return node else: for node in anytree.PreOrderIter(self, maxlevel=maxlevel): if (attr in node) and (name in node[attr]): return node
def main(options): """./moosedocs update""" # Load syntax translator, _ = common.load_config(options.config) root = None for ext in translator.extensions: if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension): root = ext.syntax break for node in anytree.PreOrderIter( root, filter_=lambda n: isinstance(n, syntax.ObjectNode)): idx = node.source().find('/src/') old = os.path.join(node.source()[:idx], 'doc', 'content', 'documentation', 'systems', node.fullpath.lstrip('/') + '.md') if not os.path.isfile(old): continue new = os.path.join(node.source()[:idx], 'doc', 'content', 'source', node.markdown()) loc = os.path.dirname(new) if not os.path.isdir(loc): os.makedirs(loc) cmd = ['git', 'mv', '--verbose', old, new] subprocess.call(cmd) func = lambda n: isinstance(n, syntax.SyntaxNode) and not n.is_root for node in anytree.PreOrderIter(root, filter_=func): action = anytree.search.findall( node, filter_=lambda n: isinstance(n, syntax.ActionNode))[0] idx = action.source().find('/src/') old = os.path.join(action.source()[:idx], 'doc', 'content', 'documentation', 'systems', os.path.dirname(node.markdown()), 'index.md') if not os.path.isfile(old): continue new = os.path.join(action.source()[:idx], 'doc', 'content', 'syntax', os.path.dirname(node.markdown()), 'index.md') loc = os.path.dirname(new) if not os.path.isdir(loc): os.makedirs(loc) cmd = ['git', 'mv', '--verbose', old, new] subprocess.call(cmd)
def make_flat_index(root, given=None): if given: classes_size = 0 for node in anytree.PreOrderIter(root): if len(node.children) == 0: node.flat_index = given.index(node.name) classes_size = given.index(node.name) + 1 if given.index(node.name) + 1 > classes_size else classes_size else: classes_size = 0 for node in anytree.PreOrderIter(root): if len(node.children) == 0: node.flat_index = classes_size classes_size += 1 return classes_size
def tree_to_tfexample(root, example=None): """Convert a tree of AnyNodes to a tf.Example. Args: root: AnyNode that represents the root of the tree. example: tf.Example to which to add the view hierarchy information. Returns: The tf.Example created. """ if example is None: example = tf.train.Example() else: # Delete existing fields. for field_name in get_all_attrs(N_FIELDS): example_utils.del_feat(example, getattr(N_FIELDS, field_name)) example_utils.delete_bboxes(example) if root is None: return example nodes = list(anytree.PreOrderIter(root)) for node in nodes: _add_node_to_tfexample(node, example, _find_parent_idx(node, nodes)) example_utils.add_bboxes(example, node.bbox) return example
def check(translator, config, groups=None, dump=False, update=False, generate=False): """Helper to all both main and build.py:main to perfor check.""" # Extract the syntax root node syntax = None extension = None for ext in translator.extensions: extension = ext if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension): syntax = ext.syntax break if syntax is None: msg = "Failed to locate AppSyntaxExtension for the given configuration." raise exceptions.MooseDocsException(msg) # Use config.yml "Check:groups" if groups not provided if (groups is None) and ('Check' in config) and ('groups' in config['Check']): groups = config['Check']['groups'] elif groups is None: groups = [extension.apptype] # Dump the complete syntax for the application if dump: print syntax # Perform check for all the nodes for node in anytree.PreOrderIter(syntax): node.check(generate=generate, update=update, groups=groups, locations=config['Content'])
def tokenize(self, root, content, page, group=None, line=1, report=True): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, unicode) # Tokenize self.__lexer.tokenize(root, content, page, self.__lexer.grammar(group), line) # Report errors if report: for token in anytree.PreOrderIter(root): if token.name == 'ErrorToken': msg = common.report_error(token['message'], page.source, token.info.line, token.info[0], token['traceback'], u'TOKENIZE ERROR') LOG.error(msg)
def createHTML(self, token, parent): try: style = find_plugin('pybtex.style.formatting', token.style) except PluginNotFound: msg = 'Unknown bibliography style "{}".' raise exceptions.RenderException(msg, token.style) citations = set() for tok in anytree.PreOrderIter(token.root): if isinstance(tok, BibtexCite): citations.update(tok.keys) formatted_bibliography = style().format_bibliography( self.extension.database, citations) html_backend = find_plugin('pybtex.backends', 'html') ol = html.Tag(parent, 'ol', class_='moose-bibliogrpahy') backend = html_backend(encoding='utf-8') for entry in formatted_bibliography: text = entry.text.render(backend) html.Tag(ol, 'li', id_=entry.key, string=text) return ol
def empty_bboxes(tree, cls, example, set_as_bg=False): """Erase content of boxes of class cls.""" image = example_utils.get_image_pil(example) orig_image = image.copy() w, h = image.size nodes = list(anytree.PreOrderIter(tree)) for i, node in enumerate(nodes): if node.bbox.ui_class != cls: continue box = node.bbox blank_img = Image.new('RGB', (int(w * box.width), int(h * box.height)), color=(255, 255, 255)) image.paste(blank_img, box=(int(box.left * w), int(box.top * h))) # Do not erase occluding elements (paste them back). for occ in nodes[i + 1:]: box = occ.bbox crop_box = (int(box.left * w), int(box.top * h), int(box.right * w), int(box.bottom * h)) crop = orig_image.crop(crop_box) image.paste(crop, box=(int(box.left * w), int(box.top * h))) # Finally delete the node of interest. delete_node(node, set_as_bg) img_bytes = image_utils.img_to_bytes(image) example_utils.del_feat(example, 'image/encoded') example_utils.add_feat(example, 'image/encoded', img_bytes) return image
def findfull(self, name, maxlevel=None): """ Search for a node, by full name. """ for node in anytree.PreOrderIter(self, maxlevel=maxlevel): if node.fullpath == name: return node
def postExecute(self, content): """ Combines all the LaTeX files into a single file. """ files = [] for page in content: if isinstance(page, pages.Source): files.append(page) root = self.buildTreeFromPageList(files) main = self._processPages(root) loc = self.translator['destination'] with open(os.path.join(loc, 'main.tex'), 'w+') as fid: fid.write(main.write()) main_tex = os.path.join(loc, 'main.tex') LOG.info("Building complete LaTeX document: %s", main_tex) commands = [['pdflatex', '-halt-on-error', 'main'], ['bibtex', 'main'], ['pdflatex', '-halt-on-error', 'main'], ['pdflatex', '-halt-on-error', 'main']] for cmd in commands: try: output = subprocess.check_output(cmd, cwd=loc, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: msg = 'Failed to run pdflatex command: {}\n\n{}' raise exceptions.MooseDocsException(msg, ' '.join(cmd), e.output) # Process output root = self.processLatexOutput(output, content) for node in anytree.PreOrderIter(root): if node.warnings: self._reportLatexWarnings(node, content)
def processLatexOutput(output, nodes): """Convert the pdfLatex log into a tree structure to capture the warnings. Inputs: output[str]: The console output or the pdflatex log. content[list]: The list of page objects from the translator. """ # Items for capturing errors warn = collections.namedtuple('LatexWarning', 'content line') line_re = re.compile(r'line\s(?P<line>[0-9]+)') regex = re.compile(r'^(?P<filename>\.*[^\.]+\.\w+)(?P<content>.*)', flags=re.MULTILINE|re.DOTALL) # Convert output to a tree structure root = PDFExtension.parseOutput(output) # Loop through the result and capture filenames and warnings for n in anytree.PreOrderIter(root): match = regex.search(n.content) if match: n.content = match.group('content') n.filename = match.group('filename').replace('\n', '') n.content = [c.strip().replace('\n', '') for c in re.split(r'\n{2,}', n.content) if c] for c in n.content: if 'LaTeX Warning' in c: c = c.replace('LaTeX Warning:', '').strip() match = line_re.search(c) line = int(match.group('line')) if match else None n.warnings.append(warn(content=c, line=line)) return root
def createHTML(self, parent, token, page): try: style = find_plugin('pybtex.style.formatting', token['bib_style']) except PluginNotFound: msg = 'Unknown bibliography style "{}".' raise exceptions.MooseDocsException(msg, token['bib_style']) citations = list() for tok in anytree.PreOrderIter(token.root): if tok.name == 'BibtexCite': citations.extend(tok['keys']) formatted_bibliography = style().format_bibliography( self.extension.database, citations) html_backend = find_plugin('pybtex.backends', 'html') ol = html.Tag(parent, 'ol', class_='moose-bibliogrpahy') backend = html_backend(encoding='utf-8') for entry in formatted_bibliography: text = entry.text.render(backend) html.Tag(ol, 'li', id_=entry.key, string=text) return ol
def main(options): """./moosedocs check""" translator = common.load_config(options.config) # Extract the syntax root node syntax = None for ext in translator.extensions: if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension): syntax = ext.syntax break if syntax is None: msg = "Failed to locate AppSyntaxExtension for the given configuration." raise exceptions.MooseDocsException(msg) # Dump the complete syntax for the application if options.dump: print syntax # Perform check for all the nodes for node in anytree.PreOrderIter(syntax): node.check(generate=options.generate, update=options.update, groups=options.groups)
def _add_layers_and_groups(project: qgc.QgsProject, layer_tree: LayerGroupNode) -> None: """Iterate through the layer tree and create the relevant Qgs objects.""" # `anytree.PreOrderIter` is necessary here so that the # `_create_and_add_layer` function receives the correct group. for node in anytree.PreOrderIter( layer_tree, filter_=lambda node: not node.is_root, ): if type(node) is LayerGroupNode: _get_or_create_and_configure_group( node=node, project=project, ) elif type(node) is LayerNode: # The parent group should already exist because `preOrderIter` will # return `LayerGroupNode`s first. parent_group = _get_group( project=project, group_path=node.group_name_path, ) _create_and_add_layer( node=node, project=project, group=parent_group, ) else: raise TypeError(f'Unexpected `node` type: {type(node)}') logger.debug('Done adding layers.')
def main(options): """./moosedocs update""" # Load syntax exe = os.path.abspath(os.path.join(os.getcwd(), options.executable)) if os.path.isdir(exe): exe = mooseutils.find_moose_executable(exe) LOG.info("Loading application syntax: %s", exe) root = app_syntax(exe) # Determine group name group_dir = os.path.basename(exe).split('-')[0] group = group_dir.replace('_', ' ').title().replace(' ', '') if options.move: for node in anytree.PreOrderIter(root): if isinstance(node, syntax.ObjectNode) and group in node.groups: _move(node, group_dir, options) if options.update: for root, _, files in os.walk(MooseDocs.ROOT_DIR): for name in files: if name.endswith('.md'): filename = os.path.join(root, name) print 'Update:', filename _update(filename)
def main(options): """./moosedocs check""" translator, config = common.load_config(options.config) # Extract the syntax root node syntax = None extension = None for ext in translator.extensions: extension = ext if isinstance(ext, MooseDocs.extensions.appsyntax.AppSyntaxExtension): syntax = ext.syntax break if syntax is None: msg = "Failed to locate AppSyntaxExtension for the given configuration." raise exceptions.MooseDocsException(msg) # Use config.yml "Check:groups" if groups not provided if (options.groups is None) and ('Check' in config) and ('groups' in config['Check']): options.groups = config['Check']['groups'] elif options.groups is None: options.groups = [extension.apptype] # Dump the complete syntax for the application if options.dump: print syntax # Perform check for all the nodes for node in anytree.PreOrderIter(syntax): node.check(generate=options.generate, update=options.update, groups=options.groups, locations=config['Content'])
def write_hierarchy(args, hierarchy_root): """ Write hierarchy in CSV format. Columns: *name*: feature name, must be unique across features *parent_name*: name of parent if it exists, else '' (root node) *description*: node description *static_indices*: [only required for leaf nodes] list of tab-separated indices corresponding to the indices of these features in the static data *temporal_indices*: [only required for leaf nodes] list of tab-separated indices corresponding to the indices of these features in the temporal data """ hierarchy_filename = "%s/%s" % (args.output_dir, "hierarchy.csv") with open(hierarchy_filename, "w", newline="") as hierarchy_file: writer = csv.writer(hierarchy_file, delimiter=",") writer.writerow([ constants.NODE_NAME, constants.PARENT_NAME, constants.DESCRIPTION, constants.STATIC_INDICES, constants.TEMPORAL_INDICES ]) for node in anytree.PreOrderIter(hierarchy_root): static_indices = node.static_indices if node.is_leaf else "" parent_name = node.parent.name if node.parent else "" writer.writerow( [node.name, parent_name, node.description, static_indices, ""]) return hierarchy_filename
def init(self, translator): command.CommandExtension.init(self, translator) bib_files = [] for node in anytree.PreOrderIter(self.translator.root): if node.source.endswith('.bib'): bib_files.append(node.source) for bfile in bib_files: try: db = parse_file(bfile) except UndefinedMacro as e: msg = "The BibTeX file %s has an undefined macro:\n%s" LOG.warning(msg, bfile, e.message) #TODO: https://bitbucket.org/pybtex-devs/pybtex/issues/93/ # databaseadd_entries-method-not-considering warn = self.get('duplicate_warning') for key in db.entries: if key in self.__database.entries: if warn: msg = "The BibTeX entry '%s' defined in %s already exists." LOG.warning(msg, key, bfile) else: self.__database.add_entry(key, db.entries[key])
def __init__(self, *args, **kwargs): command.CommandExtension.__init__(self, *args, **kwargs) self._app_type = None self._app_syntax = None if not self['disable'] and self['executable'] is not None: LOG.info("Reading MOOSE application syntax.") exe = mooseutils.eval_path(self['executable']) exe = mooseutils.find_moose_executable(exe, show_error=False) if exe is None: LOG.error("Failed to locate a valid executable in %s.", self['executable']) else: try: self._app_syntax = app_syntax( exe, alias=self['alias'], remove=self['remove'], hide=self['hide'], allow_test_objects=self['allow-test-objects']) out = mooseutils.runExe(exe, ['--type']) match = re.search(r'^MooseApp Type:\s+(?P<type>.*?)$', out, flags=re.MULTILINE) if match: self._app_type = match.group("type") else: msg = "Failed to determine application type by running the following:\n" msg += " {} --type".format(exe) LOG.error(msg) except Exception as e: #pylint: disable=broad-except msg = "Failed to load application executable from '%s', " \ "application syntax is being disabled:\n%s" LOG.error(msg, self['executable'], e.message) LOG.info("Building MOOSE class database.") self._database = common.build_class_database(self['includes'], self['inputs']) # Cache the syntax entries, search the tree is very slow if self._app_syntax: self._cache = dict() self._object_cache = dict() self._syntax_cache = dict() for node in anytree.PreOrderIter(self._app_syntax): if not node.removed: self._cache[node.fullpath] = node if node.alias: self._cache[node.alias] = node if isinstance(node, syntax.ObjectNode): self._object_cache[node.fullpath] = node if node.alias: self._object_cache[node.alias] = node elif isinstance(node, syntax.SyntaxNode): self._syntax_cache[node.fullpath] = node if node.alias: self._syntax_cache[node.alias] = node
def _tokenize(translator): func = lambda n: isinstance(n, page.MarkdownNode) for node in anytree.PreOrderIter(translator.root, filter_=func): translator.current = node translator.reinit() node.tokenize() translator.current = None
def explode(in_tree): leaves = [ node for node in anytree.PreOrderIter(in_tree, filter_=lambda n: n.name != '') ] for i in range(len(leaves)): l = leaves[i] if l.depth > 4: next_l = leaves[i + 1] if i > 0: leaves[i - 1].name += l.name zero_node = anytree.Node(name=0) else: l.name = 0 if i + 1 != len(leaves) - 1: leaves[i + 2].name += next_l.name else: next_l.name = 0 parent = l.parent zero_node = anytree.Node(name=0) l.parent.parent.children = [ zero_node if c == parent else c for c in l.parent.parent.children ] return True return False
def examine(self): """ Investigate directories for new files and add them to the tree if found. TODO: Remove nodes if page is deleted. TODO: Handle !include (see extensions.include.py for more information). """ for node in anytree.PreOrderIter(self._translator.root): # Only perform check on valid directories if not isinstance(node, page.DirectoryNode) or not os.path.exists(node.source): continue # Build map of child pages for the directory children = {child.name:child for child in node.children \ if isinstance(child, page.FileNode)} # Compare the list of files in the directory with those tracked by MooseDocs for filename in os.listdir(node.source): #pylint: disable=no-member if filename.endswith(MooseDocs.FILE_EXT) and not filename.startswith('.'): if filename not in children: source = os.path.join(node.source, filename) if filename.endswith('.md'): new = page.MarkdownNode(node, source=source) else: new = page.FileNode(node, source=source) #pylint: disable=redefined-variable-type new.base = self._options.destination self.watch(new.source, new.build, delay=2) #pylint: disable=no-member new.init(self._translator) new.build() return super(MooseDocsWatcher, self).examine()
def _addRequirement(self, parent, info, page, req): item = SQARequirementMatrixItem(parent, requirement=req) self.reader.tokenize(item, req.text, page, MooseDocs.INLINE, info.line, report=False) for token in anytree.PreOrderIter(item): if token.name == 'ErrorToken': msg = common.report_error( "Failed to tokenize SQA requirement.", req.filename, req.text_line, req.text, token['traceback'], u'SQA TOKENIZE ERROR') LOG.critical(msg) p = core.Paragraph(item) tokens.String(p, content=u'Specification: ') with codecs.open(req.filename, encoding='utf-8') as fid: content = fid.read() floats.create_modal_link(p, string=u"{}:{}".format( req.path, req.name), content=core.Code(None, language=u'text', content=content), title=unicode(req.filename)) p = core.Paragraph(item) tokens.String(p, content=u'Details: ') filename = u'{}/{}.md'.format(req.path, req.name) autolink.AutoLink(p, page=unicode(filename))
def _addRequirement(self, parent, info, page, req): reqname = "{}:{}".format(req.path, req.name) if req.path != '.' else req.name item = SQARequirementMatrixItem(parent, label=req.label, reqname=reqname) self.reader.tokenize(item, req.text, page, MooseDocs.INLINE, info.line, report=False) for token in anytree.PreOrderIter(item): if token.name == 'ErrorToken': msg = common.report_error("Failed to tokenize SQA requirement.", req.filename, req.text_line, req.text, token['traceback'], 'SQA TOKENIZE ERROR') LOG.critical(msg) p = core.Paragraph(item) tokens.String(p, content='Specification: ') content = common.read(req.filename) floats.create_modal_link(p, string=reqname, content=core.Code(None, language='text', content=content), title=str(req.filename)) p = core.Paragraph(item) tokens.String(p, content='Documentation: ') filename = getattr(req, info['subcommand']) autolink.AutoLink(p, page=str(filename))
def postTokenize(self, ast, page, meta, reader): """Set float number for each counter.""" for node in anytree.PreOrderIter(ast, filter_=lambda n: n.name == 'Caption'): prefix = node.get('prefix', None) if prefix: FloatExtension.COUNTS[prefix] += 1 node.set('number', FloatExtension.COUNTS[prefix])
def parse(self, root, content, group=None): """ Perform the parsing of the supplied content into an AST with the provided root node. Inputs: root[tokens.Token]: The root node for the AST. content[unicode:tree.page.PageNodeBase]: The content to parse, either as a unicode string or a page node object. """ # Type checking if MooseDocs.LOG_LEVEL == logging.DEBUG: common.check_type('root', root, tokens.Token) common.check_type('content', content, unicode) # Re-initialize config = self.getConfig() self.reinit() # Pre-tokenize self.translator.executeExtensionFunction('preTokenize', root, config) # Tokenize self.__lexer.tokenize(root, self.__lexer.grammar(group), content) # Post-tokenize self.translator.executeExtensionFunction('postTokenize', root, config) # Report errors for token in anytree.PreOrderIter(root): if isinstance(token, tokens.ErrorToken): LOG.error(token.report(self.translator.current))
def __initClassDatabase(self): """Initialize the class database for faster searching.""" # Do nothing if the syntax failed to build if self._app_syntax is None: return start = time.time() LOG.info("Building MOOSE class database...") self._database = common.build_class_database(self['includes'], self['inputs']) # Cache the syntax entries, search the tree is very slow self._cache = dict() self._object_cache = dict() self._syntax_cache = dict() for node in anytree.PreOrderIter(self._app_syntax): if not node.removed: self._cache[node.fullpath] = node if node.alias: self._cache[node.alias] = node if isinstance(node, syntax.ObjectNode): self._object_cache[node.fullpath] = node if node.alias: self._object_cache[node.alias] = node elif isinstance(node, syntax.SyntaxNode): self._syntax_cache[node.fullpath] = node if node.alias: self._syntax_cache[node.alias] = node LOG.info("MOOSE class database complete [%s sec]", time.time() - start)
def _addContents(self, toc, content, page): #pylint: disable=unused-argument,no-self-use """ Add the table of contents right-side bar that used scrollspy. Inputs: toc[html.Tag]: Table-of-contents <div> tag. content[html.Tag]: The complete html content that the TOC is going to reference. page[page.PageNodeBase]: The current page being converted. """ div = html.Tag(toc, 'div', class_='toc-wrapper pin-top') ul = html.Tag(div, 'ul', class_='section table-of-contents') for node in anytree.PreOrderIter(content): if node.get('data-section-level', None) == 2: node.addClass('scrollspy') li = html.Tag(ul, 'li') a = html.Tag(li, 'a', href='#{}'.format(node['id']), string=node.get('data-section-text', 'Unknown...'), class_='tooltipped') a['data-delay'] = '1000' a['data-position'] = 'left' a['data-tooltip'] = node['data-section-text']
def _addSections(self, config, container, root_page): #pylint: disable=unused-argument """ Group content into <section> tags based on the heading tag. Inputs: root[tree.html.Tag]: The root tree.html node tree to add sections.read collapsible[list]: A list with six entries indicating the sections to create as collapsible. """ if not self.get('sections', False): return collapsible = config.get('collapsible-sections', False) section = container for child in section.children: if child.name in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'): level = int(child.name[-1]) current = section.get("data-section-level", 0) # get the current section level if level == current: section = html.Tag(section.parent, 'section') elif level > current: section = html.Tag(section, 'section') elif level < current: section = html.Tag(section.parent.parent, 'section') section['data-section-level'] = level section['data-section-text'] = child.text() section['id'] = uuid.uuid4() if 'data-details-open' in child: section['data-details-open'] = child['data-details-open'] child.parent = section for node in anytree.PreOrderIter( container, filter_=lambda n: n.name == 'section'): if 'data-details-open' in node: status = node['data-details-open'] else: status = collapsible[node['data-section-level'] - 1] if status: summary = html.Tag(None, 'summary') node(0).parent = summary details = html.Tag(None, 'details', class_="moose-section-content") if status.lower() == 'open': details['open'] = 'open' details.children = node.children summary.parent = details details.parent = node icon = html.Tag(None, 'span', class_='moose-section-icon') summary(0).children = [icon] + list(summary(0).children)
def _getNodeIDs(root): """Return all 'ids' in a node tree.""" keys = [] for node in anytree.PreOrderIter(root): id_ = node.get('id', None) if id_: keys.append(id_) return keys