def test_multiline_comment(self): """Multi-line spans should close at the end of one line and reopen at the beginning of the next.""" c = Region('c') c2 = Region('c') l = LINE tags = [(0, True, c), (79, False, c), (80, False, l), (80, True, c2), (151, False, l), (222, False, l), (284, False, c2), (285, False, l), (286, False, l)] text = u"""/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ """ lines = split_content_lines(text) offsets = build_offset_map(lines) actual_lines = [ html_line(text_line.rstrip('\r\n'), e, offset) for text_line, e, offset in zip( lines, tags_per_line(balanced_tags(tags)), offsets) ] expected_lines = [ '<span class="c">/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */</span>', '<span class="c">/* This Source Code Form is subject to the terms of the Mozilla Public</span>', '<span class="c"> * License, v. 2.0. If a copy of the MPL was not distributed with this</span>', '<span class="c"> * file, You can obtain one at http://mozilla.org/MPL/2.0/. */</span>', '' ] eq_(actual_lines, expected_lines)
def test_simple(self): """Sanity-check the combination of finished_tags, es_lines and html_line, which constitutes an end-to-end run of the pipeline.""" eq_( text_to_html_lines('hello', regions=[(0, 3, Region('a')), (3, 5, Region('b'))]), [u'<span class="a">hel</span><span class="b">lo</span>'])
def test_empty_tag_boundaries(self): """Zero-length tags should be filtered out by ``tag_boundaries()``. If they are not, the start of a tag can sort after the end, crashing the tag balancer. """ text_to_html_lines('hello!', regions=[(3, 3, Region('a')), (3, 5, Region('b'))])
def test_simple_html_line(): """See if the offsets are right in simple HTML stitching.""" a = Region('a') b = Region('b') line = LINE text = 'hello' eq_( html_line( text, first( tags_per_line([(0, True, line), (0, True, a), (3, False, a), (3, True, b), (5, False, b), (5, False, line)])), 0), '<span class="a">hel</span><span class="b">lo</span>')
def _regions_for_contents(lexer, contents): """Yield regions for the tokens in text contents using given Pygments lexer.""" for index, token, text in lexer.get_tokens_unprocessed(contents): cls = token_classes.get(token) if cls: text = text.rstrip('\r\n') yield index, index + len(text), Region(cls)
def tags_from_text(text): """Return unsorted tags based on an ASCII art representation.""" for line in text.splitlines(): start = line.find('_') label, prespace, underscores = line[0], line[2:start], line[start:] ref = Region(label) yield len(prespace), True, ref yield len(prespace) + len(underscores) - 1, False, ref
def test_overlapping_regions(self): """Regions (as opposed to refs) are allowed to overlap and shouldn't be disturbed:: A _________ (2, 6) B (region) ____________ (5, 9) """ a = RefWithoutData('a') b = Region('b') tags = [(2, True, a), (5, True, b), (6, False, a), (9, False, b)] original_tags = tags[:] remove_overlapping_refs(tags) eq_(tags, original_tags)
def test_horrors(self): """Untangle a circus of interleaved tags, tags that start where others end, and other untold wretchedness.""" # This is a little brittle. All we really want to test is that each # span of text is within the right spans. We don't care what order the # span tags are in. eq_( text_to_html_lines('this&that', regions=[ (0, 9, Region('a')), (1, 8, Region('b')), (4, 7, Region('c')), (3, 4, Region('d')), (3, 5, Region('e')), (0, 4, Region('m')), (5, 9, Region('n')) ]), [ u'<span class="a"><span class="m">t<span class="b">hi<span class="d"><span class="e">s</span></span></span></span><span class="b"><span class="e"><span class="c">&</span></span><span class="c"><span class="n">th</span></span><span class="n">a</span></span><span class="n">t</span></span>' ])
def test_horrors(self): """Try a fairly horrific scenario:: A _______________ (0, 7) B _________ (2, 6) C ____________ (5, 9) D _______ (8, 11) E __ (10, 11) 0 2 5 6 7 8 9 A contains B. B closes while C's still going on. D and E end at the same time. There's even a Region in there. """ a = RefWithoutData('a') b = Region('b') c = RefWithoutData('c') d = RefWithoutData('d') e = RefWithoutData('e') tags = [(0, True, a), (2, True, b), (5, True, c), (6, False, b), (7, False, a), (8, True, d), (9, False, c), (10, True, e), (11, False, e), (11, False, d)] eq_( spaced_tags(balanced_tags(tags)), '<L>\n' '<a>\n' ' <b>\n' ' <c>\n' ' </c>\n' ' </b>\n' ' <c>\n' ' </c>\n' ' </a>\n' ' <c>\n' ' <d>\n' ' </d>\n' ' </c>\n' ' <d>\n' ' <e>\n' ' </e>\n' ' </d>\n' ' </L>')
def test_split_anchor_avoidance(self): """Don't split anchor tags when we can avoid it.""" eq_( text_to_html_lines('this that', [(0, 9, RefWithoutData([]))], [(0, 4, Region('k'))]), [u'<a data-menu="[]"><span class="k">this</span> that</a>'])
def test_tag_boundaries(): """Sanity-check ``tag_boundaries()``.""" eq_( str(list(tag_boundaries([(0, 3, Region('a')), (3, 5, Region('b'))]))), '[(0, True, Region("a")), (3, False, Region("a")), ' '(3, True, Region("b")), (5, False, Region("b"))]')
def _browse_file(tree, path, line_docs, file_doc, config, is_binary, date=None, contents=None, image_rev=None): """Return a rendered page displaying a source file. :arg string tree: name of tree on which file is found :arg string path: relative path from tree root of file :arg list line_docs: LINE documents as defined in the mapping of core.py, where the `content` field is dereferenced :arg file_doc: the FILE document as defined in core.py :arg config: TreeConfig object of this tree :arg is_binary: Whether file is binary or not :arg date: a formatted string representing the generated date, default to now :arg string contents: the contents of the source file, defaults to joining the `content` field of all line_docs :arg image_rev: revision number of a textual or binary image, for images displayed at a certain rev """ def process_link_templates(sections): """Look for {{line}} in the links of given sections, and duplicate them onto a 'template' field. """ for section in sections: for link in section['items']: if '{{line}}' in link['href']: link['template'] = link['href'] link['href'] = link['href'].replace('{{line}}', '') def sidebar_links(sections): """Return data structure to build nav sidebar from. :: [('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])] """ process_link_templates(sections) # Sort by order, resolving ties by section name: return sorted(sections, key=lambda section: (section['order'], section['heading'])) if not date: # Then assume that the file is generated now. Remark: we can't use this # as the default param because that is only evaluated once, so the same # time would always be used. date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") common = _build_common_file_template(tree, path, is_binary, date, config) links = file_doc.get('links', []) if is_binary_image(path): return render_template( 'image_file.html', **merge(common, { 'sections': sidebar_links(links), 'revision': image_rev})) elif is_binary: return render_template( 'text_file.html', **merge(common, { 'lines': [], 'is_binary': True, 'sections': sidebar_links(links)})) else: # We concretize the lines into a list because we iterate over it multiple times lines = [doc['content'] for doc in line_docs] if not contents: # If contents are not provided, we can reconstruct them by # stitching the lines together. contents = ''.join(lines) offsets = build_offset_map(lines) tree_config = config.trees[tree] if is_textual_image(path) and image_rev: # Add a link to view textual images on revs: links.extend(dictify_links([ (4, 'Image', [('svgview', 'View', url_for('.raw_rev', tree=tree_config.name, path=path, revision=image_rev))])])) # Construct skimmer objects for all enabled plugins that define a # file_to_skim class. skimmers = [plugin.file_to_skim(path, contents, plugin.name, tree_config, file_doc, line_docs) for plugin in tree_config.enabled_plugins if plugin.file_to_skim] skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs)) index_refs = (Ref.es_to_triple(ref, tree_config) for ref in chain.from_iterable(doc.get('refs', []) for doc in line_docs)) index_regions = (Region.es_to_triple(region) for region in chain.from_iterable(doc.get('regions', []) for doc in line_docs)) tags = finished_tags(lines, chain(chain.from_iterable(refses), index_refs), chain(chain.from_iterable(regionses), index_regions)) return render_template( 'text_file.html', **merge(common, { # Someday, it would be great to stream this and not concretize # the whole thing in RAM. The template will have to quit # looping through the whole thing 3 times. 'lines': [(html_line(doc['content'], tags_in_line, offset), doc.get('annotations', []) + skim_annotations) for doc, tags_in_line, offset, skim_annotations in izip(line_docs, tags_per_line(tags), offsets, annotationses)], 'sections': sidebar_links(links + skim_links), 'query': request.args.get('q', ''), 'bubble': request.args.get('redirect_type')}))
def _browse_file(tree, path, line_docs, file_doc, config, date=None, contents=None): """Return a rendered page displaying a source file. :arg string tree: name of tree on which file is found :arg string path: relative path from tree root of file :arg list line_docs: LINE documents as defined in the mapping of core.py, where the `content` field is dereferenced :arg file_doc: the FILE document as defined in core.py :arg config: TreeConfig object of this tree :arg date: a formatted string representing the generated date, default to now :arg string contents: the contents of the source file, defaults to joining the `content` field of all line_docs """ def sidebar_links(sections): """Return data structure to build nav sidebar from. :: [('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])] """ # Sort by order, resolving ties by section name: return sorted(sections, key=lambda section: (section['order'], section['heading'])) if not date: # Then assume that the file is generated now. Remark: we can't use this # as the default param because that is only evaluated once, so the same # time would always be used. date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") common = _build_common_file_template(tree, path, date, config) links = file_doc.get('links', []) if is_binary_image(path): return render_template('image_file.html', **common) else: # We don't allow browsing binary files, so this must be a text file. # We concretize the lines into a list because we iterate over it multiple times lines = [doc['content'] for doc in line_docs] if not contents: # If contents are not provided, we can reconstruct them by # stitching the lines together. contents = ''.join(lines) offsets = cumulative_sum(imap(len, lines)) tree_config = config.trees[tree] # Construct skimmer objects for all enabled plugins that define a # file_to_skim class. skimmers = [ plugin.file_to_skim(path, contents, plugin.name, tree_config, file_doc, line_docs) for plugin in tree_config.enabled_plugins if plugin.file_to_skim ] skim_links, refses, regionses, annotationses = skim_file( skimmers, len(line_docs)) index_refs = (Ref.es_to_triple(ref, tree_config) for ref in chain.from_iterable( doc.get('refs', []) for doc in line_docs)) index_regions = (Region.es_to_triple(region) for region in chain.from_iterable( doc.get('regions', []) for doc in line_docs)) tags = finished_tags( lines, chain(chain.from_iterable(refses), index_refs), chain(chain.from_iterable(regionses), index_regions)) return render_template( 'text_file.html', **merge( common, { # Someday, it would be great to stream this and not concretize # the whole thing in RAM. The template will have to quit # looping through the whole thing 3 times. 'lines': [(html_line(doc['content'], tags_in_line, offset), doc.get('annotations', []) + skim_annotations) for doc, tags_in_line, offset, skim_annotations in izip(line_docs, tags_per_line(tags), offsets, annotationses)], 'is_text': True, 'sections': sidebar_links(links + skim_links) }))
def _browse_file(tree, path, line_docs, file_doc, config, date=None, contents=None): """Return a rendered page displaying a source file. :arg string tree: name of tree on which file is found :arg string path: relative path from tree root of file :arg list line_docs: LINE documents as defined in the mapping of core.py, where the `content` field is dereferenced :arg file_doc: the FILE document as defined in core.py :arg config: TreeConfig object of this tree :arg date: a formatted string representing the generated date, default to now :arg string contents: the contents of the source file, defaults to joining the `content` field of all line_docs """ def sidebar_links(sections): """Return data structure to build nav sidebar from. :: [('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])] """ # Sort by order, resolving ties by section name: return sorted(sections, key=lambda section: (section['order'], section['heading'])) if not date: # Then assume that the file is generated now. Remark: we can't use this # as the default param because that is only evaluated once, so the same # time would always be used. date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") common = _build_common_file_template(tree, path, date, config) links = file_doc.get('links', []) if is_image(path): return render_template( 'image_file.html', **common) else: # We don't allow browsing binary files, so this must be a text file. # We concretize the lines into a list because we iterate over it multiple times lines = [doc['content'] for doc in line_docs] if not contents: # If contents are not provided, we can reconstruct them by # stitching the lines together. contents = ''.join(lines) offsets = cumulative_sum(imap(len, lines)) tree_config = config.trees[tree] # Construct skimmer objects for all enabled plugins that define a # file_to_skim class. skimmers = [plugin.file_to_skim(path, contents, plugin.name, tree_config, file_doc, line_docs) for plugin in tree_config.enabled_plugins if plugin.file_to_skim] skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs)) index_refs = (Ref.es_to_triple(ref, tree_config) for ref in chain.from_iterable(doc.get('refs', []) for doc in line_docs)) index_regions = (Region.es_to_triple(region) for region in chain.from_iterable(doc.get('regions', []) for doc in line_docs)) tags = finished_tags(lines, chain(chain.from_iterable(refses), index_refs), chain(chain.from_iterable(regionses), index_regions)) return render_template( 'text_file.html', **merge(common, { # Someday, it would be great to stream this and not concretize # the whole thing in RAM. The template will have to quit # looping through the whole thing 3 times. 'lines': [(html_line(doc['content'], tags_in_line, offset), doc.get('annotations', []) + skim_annotations) for doc, tags_in_line, offset, skim_annotations in izip(line_docs, tags_per_line(tags), offsets, annotationses)], 'is_text': True, 'sections': sidebar_links(links + skim_links)}))