def streamify(text,bozo):
""" add a .stream to a _detail textConstruct """
if text.type == 'text/plain':
text.stream = HTMLParser(StringIO(escape(text.value)))
elif text.type == 'text/html' or bozo != 'false':
text.stream = HTMLParser(StringIO(text.value))
else:
text.stream = XHTMLParser(text.value)
def _render_source(self, context, stream, annotations, marks=None):
from trac.web.chrome import add_warning
annotators, labels, titles = {}, {}, {}
for annotator in self.annotators:
atype, alabel, atitle = annotator.get_annotation_type()
if atype in annotations:
labels[atype] = alabel
titles[atype] = atitle
annotators[atype] = annotator
annotations = [a for a in annotations if a in annotators]
if isinstance(stream, list):
stream = HTMLParser(StringIO(u'\n'.join(stream)))
elif isinstance(stream, unicode):
text = stream
def linesplitter():
for line in text.splitlines(True):
yield TEXT, line, (None, -1, -1)
stream = linesplitter()
annotator_datas = []
for a in annotations:
annotator = annotators[a]
try:
data = (annotator, annotator.get_annotation_data(context))
except TracError, e:
self.log.warning("Can't use annotator '%s': %s", a, e.message)
add_warning(context.req, tag.strong(
tag_("Can't use %(annotator)s annotator: %(error)s",
annotator=tag.em(a), error=tag.pre(e.message))))
data = (None, None)
annotator_datas.append(data)
def test_hex_charref(self):
text = '<span>'</span>'
events = list(HTMLParser(StringIO(text)))
self.assertEqual(3, len(events))
self.assertEqual((Stream.START, ('span', ())), events[0][:2])
self.assertEqual((Stream.TEXT, "'"), events[1][:2])
self.assertEqual((Stream.END, 'span'), events[2][:2])
def test_processing_instruction_no_data_1(self):
text = u'<?foo ?>'
events = list(HTMLParser(StringIO(text)))
kind, (target, data), pos = events[0]
self.assertEqual(Stream.PI, kind)
self.assertEqual('foo', target)
self.assertEqual('', data)
def test_simplespan(self):
input = HTMLParser(StringIO(u"<span>test</span>"), encoding=None)
lines = list(_group_lines(input))
self.assertEqual(len(lines), 1)
self.assertIsInstance(lines[0], Stream)
for (a, b) in zip(lines[0], input):
self.assertEqual(a, b)
def test_processing_instruction_trailing_qmark(self):
text = '<?php echo "Foobar" ??>'
events = list(HTMLParser(StringIO(text)))
kind, (target, data), pos = events[0]
self.assertEqual(Stream.PI, kind)
self.assertEqual(u'php', target)
self.assertEqual(u'echo "Foobar" ?', data)
def test_text_node_pos_single_line(self):
text = '<elem>foo bar</elem>'
events = list(HTMLParser(StringIO(text)))
kind, data, pos = events[1]
self.assertEqual(Stream.TEXT, kind)
self.assertEqual('foo bar', data)
self.assertEqual((None, 1, 6), pos)
def test_simplespan(self):
input = HTMLParser(StringIO("<span>test</span>"))
lines = list(_group_lines(input))
self.assertEquals(len(lines), 1)
self.assertTrue(isinstance(lines[0], Stream))
for (a, b) in zip(lines[0], input):
self.assertEqual(a, b)
def test_html_entity_in_attribute(self):
text = '<p title=" "></p>'
events = list(HTMLParser(StringIO(text)))
kind, data, pos = events[0]
self.assertEqual(Stream.START, kind)
self.assertEqual(u'\xa0', data[1].get('title'))
kind, data, pos = events[1]
self.assertEqual(Stream.END, kind)
def _render_source(self, context, stream, annotations):
from trac.web.chrome import add_warning
annotators, labels, titles = {}, {}, {}
for annotator in self.annotators:
atype, alabel, atitle = annotator.get_annotation_type()
if atype in annotations:
labels[atype] = alabel
titles[atype] = atitle
annotators[atype] = annotator
annotations = [a for a in annotations if a in annotators]
if isinstance(stream, list):
stream = HTMLParser(StringIO(u'\n'.join(stream)))
elif isinstance(stream, unicode):
text = stream
def linesplitter():
for line in text.splitlines(True):
yield TEXT, line, (None, -1, -1)
stream = linesplitter()
annotator_datas = []
for a in annotations:
annotator = annotators[a]
try:
data = (annotator, annotator.get_annotation_data(context))
except TracError as e:
self.log.warning("Can't use annotator '%s': %s", a, e.message)
add_warning(
context.req,
tag.strong(
tag_("Can't use %(annotator)s annotator: %(error)s",
annotator=tag.em(a),
error=tag.pre(e.message))))
data = (None, None)
annotator_datas.append(data)
def _head_row():
return tag.tr([
tag.th(labels[a], class_=a, title=titles[a])
for a in annotations
] + [tag.th(u'\xa0', class_='content')])
def _body_rows():
for idx, line in enumerate(_group_lines(stream)):
row = tag.tr()
for annotator, data in annotator_datas:
if annotator:
annotator.annotate_row(context, row, idx + 1, line,
data)
else:
row.append(tag.td())
row.append(tag.td(line))
yield row
return tag.table(class_='code')(tag.thead(_head_row()),
tag.tbody(_body_rows()))
def test_out_of_order_tags3(self):
text = '<span><b>Foobar</i>'
events = list(HTMLParser(StringIO(text)))
self.assertEqual(5, len(events))
self.assertEqual((Stream.START, ('span', ())), events[0][:2])
self.assertEqual((Stream.START, ('b', ())), events[1][:2])
self.assertEqual((Stream.TEXT, 'Foobar'), events[2][:2])
self.assertEqual((Stream.END, 'b'), events[3][:2])
self.assertEqual((Stream.END, 'span'), events[4][:2])
def setUp(self):
env = EnvironmentStub(enable=[Chrome, PatchRenderer])
req = MockRequest(env)
self.context = web_context(req)
self.patch = Mimeview(env).renderers[0]
patch_html = open(
os.path.join(os.path.split(__file__)[0], 'patch.html'))
self.patch_html = Stream(list(HTMLParser(patch_html,
encoding='utf-8')))
def test_text_node_pos_multi_line(self):
text = '''<elem>foo
bar</elem>'''
events = list(HTMLParser(StringIO(text)))
kind, data, pos = events[1]
self.assertEqual(Stream.TEXT, kind)
self.assertEqual(u'foo\nbar', data)
if sys.version_info[:2] >= (2, 4):
self.assertEqual((None, 1, 6), pos)
def test_out_of_order_tags3(self):
text = u'<span><b>Foobar</i>'.encode('utf-8')
events = list(HTMLParser(BytesIO(text), encoding='utf-8'))
self.assertEqual(5, len(events))
self.assertEqual((Stream.START, ('span', ())), events[0][:2])
self.assertEqual((Stream.START, ('b', ())), events[1][:2])
self.assertEqual((Stream.TEXT, 'Foobar'), events[2][:2])
self.assertEqual((Stream.END, 'b'), events[3][:2])
self.assertEqual((Stream.END, 'span'), events[4][:2])
def setUp(self):
self.env = EnvironmentStub(enable=[Chrome, LineNumberAnnotator,
PygmentsRenderer])
self.pygments = Mimeview(self.env).renderers[0]
self.req = MockRequest(self.env)
self.context = web_context(self.req)
pygments_html = open(os.path.join(os.path.split(__file__)[0],
'pygments.html'))
self.pygments_html = Stream(list(HTMLParser(pygments_html, encoding='utf-8')))
def setUp(self):
self.env = EnvironmentStub(enable=[Chrome, PygmentsRenderer])
self.pygments = Mimeview(self.env).renderers[0]
self.req = Mock(base_path='', chrome={}, args={},
abs_href=Href('/'), href=Href('/'),
session={}, perm=None, authname=None, tz=None)
self.context = web_context(self.req)
pygments_html = open(os.path.join(os.path.split(__file__)[0],
'pygments.html'))
self.pygments_html = Stream(list(HTMLParser(pygments_html, encoding='utf-8')))
def test_processing_instruction_no_data_2(self):
text = u'<?experiment>...<?/experiment>'
events = list(HTMLParser(StringIO(text)))
kind, (target, data), pos = events[0]
self.assertEqual(Stream.PI, kind)
self.assertEqual('experiment', target)
self.assertEqual('', data)
kind, (target, data), pos = events[2]
self.assertEqual('/experiment', target)
self.assertEqual('', data)
def test_out_of_order_tags2(self):
text = '<span class="baz"><b><i>Foobar</span></b></i>'
events = list(HTMLParser(StringIO(text)))
self.assertEqual(7, len(events))
self.assertEqual((Stream.START, ('span', Attrs([('class', 'baz')]))),
events[0][:2])
self.assertEqual((Stream.START, ('b', ())), events[1][:2])
self.assertEqual((Stream.START, ('i', ())), events[2][:2])
self.assertEqual((Stream.TEXT, 'Foobar'), events[3][:2])
self.assertEqual((Stream.END, 'i'), events[4][:2])
self.assertEqual((Stream.END, 'b'), events[5][:2])
self.assertEqual((Stream.END, 'span'), events[6][:2])
def test_newline2(self):
"""
Same as test_newline above, but make sure it behaves properly wrt
the trailing \\n, especially given it's inside an element.
"""
input = HTMLParser(StringIO('<span class="c">a\nb\n</span>'))
expected = ['<span class="c">a</span>',
'<span class="c">b</span>',
]
lines = list(_group_lines(input))
self.assertEquals(len(lines), len(expected))
for a, b in zip(lines, expected):
self.assertEquals(a.render('html'), b)
def test_newline(self):
"""
If the text element does not end with a newline, it's not properly
closed.
"""
input = HTMLParser(StringIO('<span class="c">a\nb</span>'))
expected = ['<span class="c">a</span>',
'<span class="c">b</span>',
]
lines = list(_group_lines(input))
self.assertEquals(len(lines), len(expected))
for a, b in zip(lines, expected):
self.assertEquals(a.render('html'), b)
def _gen_ticket_entry(self, t, a_class=''):
id = str(t.get('id'))
status = t.get('status')
summary = to_unicode(t.get('summary'))
owner = to_unicode(t.get('owner'))
description = to_unicode(t.get('description')[:1024])
url = t.get('href')
if status == 'closed':
a_class = a_class + 'closed'
else:
a_class = a_class + 'open'
markup = format_to_html(self.env, self.ref.context, description)
# Escape, if requested
if self.sanitize is True:
try:
description = HTMLParser(StringIO(markup)
).parse() | HTMLSanitizer()
except ParseError:
description = escape(markup)
else:
description = markup
# Replace tags that destruct tooltips too much
desc = self.end_RE.sub(']', Markup(description))
desc = self.del_RE.sub('', desc)
# need 2nd run after purging newline in table cells in 1st run
desc = self.del_RE.sub('', desc)
desc = self.item_RE.sub('X', desc)
desc = self.tab_RE.sub('[|||]', desc)
description = self.open_RE.sub('[', desc)
tip = tag.span(Markup(description))
ticket = '#' + id
ticket = tag.a(ticket, href=url)
ticket(tip, class_='tip', target='_blank')
ticket = tag.div(ticket)
ticket(class_=a_class, align='left')
# fix stripping of regular leading space in IE
blank = ' '
ticket(Markup(blank), summary, ' (', owner, ')')
summary = tag(summary, ' (', owner, ')')
ticket_short = '#' + id
ticket_short = tag.a(ticket_short, href=url)
ticket_short(target='_blank', title_=summary)
ticket_short = tag.span(ticket_short)
ticket_short(class_=a_class)
return ticket,ticket_short
def test_multinewline(self):
"""
ditto.
"""
input = HTMLParser(StringIO('<span class="c">\n\n\na</span>'))
expected = ['<span class="c"></span>',
'<span class="c"></span>',
'<span class="c"></span>',
'<span class="c">a</span>',
]
lines = list(_group_lines(input))
self.assertEquals(len(lines), len(expected))
for a, b in zip(lines, expected):
self.assertEquals(a.render('html'), b)
def setUp(self):
env = EnvironmentStub(enable=[Chrome, PatchRenderer])
req = Mock(base_path='',
chrome={},
args={},
session={},
abs_href=Href('/'),
href=Href('/'),
locale='',
perm=MockPerm(),
authname=None,
tz=None)
self.context = Context.from_request(req)
self.patch = Mimeview(env).renderers[0]
patch_html = open(
os.path.join(os.path.split(__file__)[0], 'patch.html'))
self.patch_html = Stream(list(HTMLParser(patch_html)))
def expand_macro(self, formatter, macro, args):
args, kw = parse_args(args)
try:
source = args.pop(0).strip()
except NameError:
return system_message('%s: Missing HTML source argument.' % macro)
try:
stream = Stream(HTMLParser(StringIO(source)))
return (stream | TracHTMLSanitizer()).render('xhtml',
encoding=None)
except ParseError, e:
self.env.log.warn(e)
return system_message('%s: HTML parse error: %s.' %
(macro, escape(e.msg)))
def run(self):
from docutils.core import publish_cmdline
from docutils.nodes import raw
from docutils.parsers import rst
from genshi.input import HTMLParser
from genshi.template import TemplateLoader
docutils_conf = os.path.join(TOOLS_DIR, 'conf', 'docutils.ini')
epydoc_conf = os.path.join(TOOLS_DIR, 'conf', 'epydoc.ini')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
def code_block(name, arguments, options, content, lineno,
content_offset, block_text, state, state_machine):
lexer = get_lexer_by_name(arguments[0])
html = highlight('\n'.join(content), lexer, HtmlFormatter())
return [raw('', html, format='html')]
code_block.arguments = (1, 0, 0)
code_block.options = {'language': rst.directives.unchanged}
code_block.content = 1
rst.directives.register_directive('code-block', code_block)
except ImportError:
print 'Pygments not installed, syntax highlighting disabled'
loader = TemplateLoader(['doc', 'doc/common'],
variable_lookup='strict')
for source in glob('doc/*.txt'):
dest = os.path.splitext(source)[0] + '.html'
if self.force or not os.path.exists(dest) or \
os.path.getmtime(dest) < os.path.getmtime(source):
print 'building documentation file %s' % dest
publish_cmdline(
writer_name='html',
argv=['--config=%s' % docutils_conf, source, dest])
fileobj = open(dest)
try:
html = HTMLParser(fileobj)
template = loader.load('template.html')
output = template.generate(
html=html, project=self.distribution).render('html')
finally:
fileobj.close()
fileobj = open(dest, 'w')
try:
fileobj.write(output)
finally:
fileobj.close()
if not self.without_apidocs:
try:
from epydoc import cli
old_argv = sys.argv[1:]
sys.argv[1:] = [
'--config=%s' % epydoc_conf,
'--top=%s' % self.distribution.packages[0],
'--no-private', # epydoc bug, not read from config
'--simple-term',
'--verbose'
] + self.distribution.packages
cli.cli()
sys.argv[1:] = old_argv
except ImportError:
print 'epydoc not installed, skipping API documentation.'
res = re.search(r'FRAGMENT\(([^)]*)\)', line)
if res:
current_fragment_name = res.groups()[0]
else:
if current_fragment_name == fragment_name:
fragment.append(line)
out = '\n'.join(fragment)
# If we have a preview format, use it
if dest_format:
# We can trust the output and do not need to call the HTML sanitizer
# below. The HTML sanitization leads to whitespace being stripped.
safe_content = True
out = Mimeview(self.env).render(ctxt, dest_format, out, force_source=True)
# Escape if needed
if not safe_content and not self.config.getbool('wiki', 'render_unsafe_content', False):
try:
out = HTMLParser(StringIO(out)).parse() | HTMLSanitizer()
except ParseError:
out = escape(out)
return out
# IPermissionRequestor methods
def get_permission_actions(self):
yield 'INCLUDE_URL'
def test_html_entity_in_text(self):
text = '<p> </p>'
events = list(HTMLParser(StringIO(text)))
kind, data, pos = events[1]
self.assertEqual(Stream.TEXT, kind)
self.assertEqual(u'\xa0', data)
def test_input_encoding_text(self):
text = u'<div>\xf6</div>'.encode('iso-8859-1')
events = list(HTMLParser(StringIO(text), encoding='iso-8859-1'))
kind, data, pos = events[1]
self.assertEqual(Stream.TEXT, kind)
self.assertEqual(u'\xf6', data)
def test_unicode_input(self):
text = u'<div>\u2013</div>'
events = list(HTMLParser(StringIO(text)))
kind, data, pos = events[1]
self.assertEqual(Stream.TEXT, kind)
self.assertEqual(u'\u2013', data)
def test_input_encoding_attribute(self):
text = u'<div title="\xf6"></div>'.encode('iso-8859-1')
events = list(HTMLParser(StringIO(text), encoding='iso-8859-1'))
kind, (tag, attrib), pos = events[0]
self.assertEqual(Stream.START, kind)
self.assertEqual(u'\xf6', attrib.get('title'))
