def test_multiline_comment(self):
"""Multi-line spans should close at the end of one line and reopen at
the beginning of the next."""
c = Region('c')
c2 = Region('c')
l = LINE
tags = [(0, True, c), (79, False, c), (80, False, l), (80, True, c2),
(151, False, l), (222, False, l), (284, False, c2),
(285, False, l), (286, False, l)]
text = u"""/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"""
lines = split_content_lines(text)
offsets = build_offset_map(lines)
actual_lines = [
html_line(text_line.rstrip('\r\n'), e,
offset) for text_line, e, offset in zip(
lines, tags_per_line(balanced_tags(tags)), offsets)
]
expected_lines = [
'<span class="c">/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */</span>',
'<span class="c">/* This Source Code Form is subject to the terms of the Mozilla Public</span>',
'<span class="c"> * License, v. 2.0. If a copy of the MPL was not distributed with this</span>',
'<span class="c"> * file, You can obtain one at http://mozilla.org/MPL/2.0/. */</span>',
''
]
eq_(actual_lines, expected_lines)
def test_simple(self):
"""Sanity-check the combination of finished_tags, es_lines and
html_line, which constitutes an end-to-end run of the pipeline."""
eq_(
text_to_html_lines('hello',
regions=[(0, 3, Region('a')),
(3, 5, Region('b'))]),
[u'<span class="a">hel</span><span class="b">lo</span>'])
def test_empty_tag_boundaries(self):
"""Zero-length tags should be filtered out by ``tag_boundaries()``.
If they are not, the start of a tag can sort after the end, crashing
the tag balancer.
"""
text_to_html_lines('hello!',
regions=[(3, 3, Region('a')), (3, 5, Region('b'))])
def test_simple_html_line():
"""See if the offsets are right in simple HTML stitching."""
a = Region('a')
b = Region('b')
line = LINE
text = 'hello'
eq_(
html_line(
text,
first(
tags_per_line([(0, True, line), (0, True, a), (3, False, a),
(3, True, b), (5, False, b),
(5, False, line)])), 0),
'<span class="a">hel</span><span class="b">lo</span>')
def _regions_for_contents(lexer, contents):
"""Yield regions for the tokens in text contents using given Pygments lexer."""
for index, token, text in lexer.get_tokens_unprocessed(contents):
cls = token_classes.get(token)
if cls:
text = text.rstrip('\r\n')
yield index, index + len(text), Region(cls)
def tags_from_text(text):
"""Return unsorted tags based on an ASCII art representation."""
for line in text.splitlines():
start = line.find('_')
label, prespace, underscores = line[0], line[2:start], line[start:]
ref = Region(label)
yield len(prespace), True, ref
yield len(prespace) + len(underscores) - 1, False, ref
def test_overlapping_regions(self):
"""Regions (as opposed to refs) are allowed to overlap and shouldn't be
disturbed::
A _________ (2, 6)
B (region) ____________ (5, 9)
"""
a = RefWithoutData('a')
b = Region('b')
tags = [(2, True, a), (5, True, b), (6, False, a), (9, False, b)]
original_tags = tags[:]
remove_overlapping_refs(tags)
eq_(tags, original_tags)
def test_horrors(self):
"""Untangle a circus of interleaved tags, tags that start where others
end, and other untold wretchedness."""
# This is a little brittle. All we really want to test is that each
# span of text is within the right spans. We don't care what order the
# span tags are in.
eq_(
text_to_html_lines('this&that',
regions=[
(0, 9, Region('a')), (1, 8, Region('b')),
(4, 7, Region('c')), (3, 4, Region('d')),
(3, 5, Region('e')), (0, 4, Region('m')),
(5, 9, Region('n'))
]),
[
u'<span class="a"><span class="m">t<span class="b">hi<span class="d"><span class="e">s</span></span></span></span><span class="b"><span class="e"><span class="c">&</span></span><span class="c"><span class="n">th</span></span><span class="n">a</span></span><span class="n">t</span></span>'
])
def test_horrors(self):
"""Try a fairly horrific scenario::
A _______________ (0, 7)
B _________ (2, 6)
C ____________ (5, 9)
D _______ (8, 11)
E __ (10, 11)
0 2 5 6 7 8 9
A contains B. B closes while C's still going on. D and E end at the
same time. There's even a Region in there.
"""
a = RefWithoutData('a')
b = Region('b')
c = RefWithoutData('c')
d = RefWithoutData('d')
e = RefWithoutData('e')
tags = [(0, True, a), (2, True, b), (5, True, c), (6, False, b),
(7, False, a), (8, True, d), (9, False, c), (10, True, e),
(11, False, e), (11, False, d)]
eq_(
spaced_tags(balanced_tags(tags)), '<L>\n'
'<a>\n'
' <b>\n'
' <c>\n'
' </c>\n'
' </b>\n'
' <c>\n'
' </c>\n'
' </a>\n'
' <c>\n'
' <d>\n'
' </d>\n'
' </c>\n'
' <d>\n'
' <e>\n'
' </e>\n'
' </d>\n'
' </L>')
def test_split_anchor_avoidance(self):
"""Don't split anchor tags when we can avoid it."""
eq_(
text_to_html_lines('this that', [(0, 9, RefWithoutData([]))],
[(0, 4, Region('k'))]),
[u'<a data-menu="[]"><span class="k">this</span> that</a>'])
def test_tag_boundaries():
"""Sanity-check ``tag_boundaries()``."""
eq_(
str(list(tag_boundaries([(0, 3, Region('a')), (3, 5, Region('b'))]))),
'[(0, True, Region("a")), (3, False, Region("a")), '
'(3, True, Region("b")), (5, False, Region("b"))]')
def _browse_file(tree, path, line_docs, file_doc, config, is_binary,
date=None, contents=None, image_rev=None):
"""Return a rendered page displaying a source file.
:arg string tree: name of tree on which file is found
:arg string path: relative path from tree root of file
:arg list line_docs: LINE documents as defined in the mapping of core.py,
where the `content` field is dereferenced
:arg file_doc: the FILE document as defined in core.py
:arg config: TreeConfig object of this tree
:arg is_binary: Whether file is binary or not
:arg date: a formatted string representing the generated date, default to now
:arg string contents: the contents of the source file, defaults to joining
the `content` field of all line_docs
:arg image_rev: revision number of a textual or binary image, for images
displayed at a certain rev
"""
def process_link_templates(sections):
"""Look for {{line}} in the links of given sections, and duplicate them onto
a 'template' field.
"""
for section in sections:
for link in section['items']:
if '{{line}}' in link['href']:
link['template'] = link['href']
link['href'] = link['href'].replace('{{line}}', '')
def sidebar_links(sections):
"""Return data structure to build nav sidebar from. ::
[('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])]
"""
process_link_templates(sections)
# Sort by order, resolving ties by section name:
return sorted(sections, key=lambda section: (section['order'],
section['heading']))
if not date:
# Then assume that the file is generated now. Remark: we can't use this
# as the default param because that is only evaluated once, so the same
# time would always be used.
date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
common = _build_common_file_template(tree, path, is_binary, date, config)
links = file_doc.get('links', [])
if is_binary_image(path):
return render_template(
'image_file.html',
**merge(common, {
'sections': sidebar_links(links),
'revision': image_rev}))
elif is_binary:
return render_template(
'text_file.html',
**merge(common, {
'lines': [],
'is_binary': True,
'sections': sidebar_links(links)}))
else:
# We concretize the lines into a list because we iterate over it multiple times
lines = [doc['content'] for doc in line_docs]
if not contents:
# If contents are not provided, we can reconstruct them by
# stitching the lines together.
contents = ''.join(lines)
offsets = build_offset_map(lines)
tree_config = config.trees[tree]
if is_textual_image(path) and image_rev:
# Add a link to view textual images on revs:
links.extend(dictify_links([
(4,
'Image',
[('svgview', 'View', url_for('.raw_rev',
tree=tree_config.name,
path=path,
revision=image_rev))])]))
# Construct skimmer objects for all enabled plugins that define a
# file_to_skim class.
skimmers = [plugin.file_to_skim(path,
contents,
plugin.name,
tree_config,
file_doc,
line_docs)
for plugin in tree_config.enabled_plugins
if plugin.file_to_skim]
skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs))
index_refs = (Ref.es_to_triple(ref, tree_config) for ref in
chain.from_iterable(doc.get('refs', [])
for doc in line_docs))
index_regions = (Region.es_to_triple(region) for region in
chain.from_iterable(doc.get('regions', [])
for doc in line_docs))
tags = finished_tags(lines,
chain(chain.from_iterable(refses), index_refs),
chain(chain.from_iterable(regionses), index_regions))
return render_template(
'text_file.html',
**merge(common, {
# Someday, it would be great to stream this and not concretize
# the whole thing in RAM. The template will have to quit
# looping through the whole thing 3 times.
'lines': [(html_line(doc['content'], tags_in_line, offset),
doc.get('annotations', []) + skim_annotations)
for doc, tags_in_line, offset, skim_annotations
in izip(line_docs, tags_per_line(tags), offsets, annotationses)],
'sections': sidebar_links(links + skim_links),
'query': request.args.get('q', ''),
'bubble': request.args.get('redirect_type')}))
def _browse_file(tree,
path,
line_docs,
file_doc,
config,
date=None,
contents=None):
"""Return a rendered page displaying a source file.
:arg string tree: name of tree on which file is found
:arg string path: relative path from tree root of file
:arg list line_docs: LINE documents as defined in the mapping of core.py,
where the `content` field is dereferenced
:arg file_doc: the FILE document as defined in core.py
:arg config: TreeConfig object of this tree
:arg date: a formatted string representing the generated date, default to now
:arg string contents: the contents of the source file, defaults to joining
the `content` field of all line_docs
"""
def sidebar_links(sections):
"""Return data structure to build nav sidebar from. ::
[('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])]
"""
# Sort by order, resolving ties by section name:
return sorted(sections,
key=lambda section:
(section['order'], section['heading']))
if not date:
# Then assume that the file is generated now. Remark: we can't use this
# as the default param because that is only evaluated once, so the same
# time would always be used.
date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
common = _build_common_file_template(tree, path, date, config)
links = file_doc.get('links', [])
if is_binary_image(path):
return render_template('image_file.html', **common)
else: # We don't allow browsing binary files, so this must be a text file.
# We concretize the lines into a list because we iterate over it multiple times
lines = [doc['content'] for doc in line_docs]
if not contents:
# If contents are not provided, we can reconstruct them by
# stitching the lines together.
contents = ''.join(lines)
offsets = cumulative_sum(imap(len, lines))
tree_config = config.trees[tree]
# Construct skimmer objects for all enabled plugins that define a
# file_to_skim class.
skimmers = [
plugin.file_to_skim(path, contents, plugin.name, tree_config,
file_doc, line_docs)
for plugin in tree_config.enabled_plugins if plugin.file_to_skim
]
skim_links, refses, regionses, annotationses = skim_file(
skimmers, len(line_docs))
index_refs = (Ref.es_to_triple(ref, tree_config)
for ref in chain.from_iterable(
doc.get('refs', []) for doc in line_docs))
index_regions = (Region.es_to_triple(region)
for region in chain.from_iterable(
doc.get('regions', []) for doc in line_docs))
tags = finished_tags(
lines, chain(chain.from_iterable(refses), index_refs),
chain(chain.from_iterable(regionses), index_regions))
return render_template(
'text_file.html',
**merge(
common,
{
# Someday, it would be great to stream this and not concretize
# the whole thing in RAM. The template will have to quit
# looping through the whole thing 3 times.
'lines': [(html_line(doc['content'], tags_in_line, offset),
doc.get('annotations', []) + skim_annotations)
for doc, tags_in_line, offset, skim_annotations
in izip(line_docs, tags_per_line(tags), offsets,
annotationses)],
'is_text':
True,
'sections':
sidebar_links(links + skim_links)
}))
def _browse_file(tree, path, line_docs, file_doc, config, date=None, contents=None):
"""Return a rendered page displaying a source file.
:arg string tree: name of tree on which file is found
:arg string path: relative path from tree root of file
:arg list line_docs: LINE documents as defined in the mapping of core.py,
where the `content` field is dereferenced
:arg file_doc: the FILE document as defined in core.py
:arg config: TreeConfig object of this tree
:arg date: a formatted string representing the generated date, default to now
:arg string contents: the contents of the source file, defaults to joining
the `content` field of all line_docs
"""
def sidebar_links(sections):
"""Return data structure to build nav sidebar from. ::
[('Section Name', [{'icon': ..., 'title': ..., 'href': ...}])]
"""
# Sort by order, resolving ties by section name:
return sorted(sections, key=lambda section: (section['order'],
section['heading']))
if not date:
# Then assume that the file is generated now. Remark: we can't use this
# as the default param because that is only evaluated once, so the same
# time would always be used.
date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
common = _build_common_file_template(tree, path, date, config)
links = file_doc.get('links', [])
if is_image(path):
return render_template(
'image_file.html',
**common)
else: # We don't allow browsing binary files, so this must be a text file.
# We concretize the lines into a list because we iterate over it multiple times
lines = [doc['content'] for doc in line_docs]
if not contents:
# If contents are not provided, we can reconstruct them by
# stitching the lines together.
contents = ''.join(lines)
offsets = cumulative_sum(imap(len, lines))
tree_config = config.trees[tree]
# Construct skimmer objects for all enabled plugins that define a
# file_to_skim class.
skimmers = [plugin.file_to_skim(path,
contents,
plugin.name,
tree_config,
file_doc,
line_docs)
for plugin in tree_config.enabled_plugins
if plugin.file_to_skim]
skim_links, refses, regionses, annotationses = skim_file(skimmers, len(line_docs))
index_refs = (Ref.es_to_triple(ref, tree_config) for ref in
chain.from_iterable(doc.get('refs', [])
for doc in line_docs))
index_regions = (Region.es_to_triple(region) for region in
chain.from_iterable(doc.get('regions', [])
for doc in line_docs))
tags = finished_tags(lines,
chain(chain.from_iterable(refses), index_refs),
chain(chain.from_iterable(regionses), index_regions))
return render_template(
'text_file.html',
**merge(common, {
# Someday, it would be great to stream this and not concretize
# the whole thing in RAM. The template will have to quit
# looping through the whole thing 3 times.
'lines': [(html_line(doc['content'], tags_in_line, offset),
doc.get('annotations', []) + skim_annotations)
for doc, tags_in_line, offset, skim_annotations
in izip(line_docs, tags_per_line(tags), offsets, annotationses)],
'is_text': True,
'sections': sidebar_links(links + skim_links)}))
