def test_escaping(self):
# Test without special escaping. Only \ is escaped
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.assertEqual(
list(sfm.parser([
r"\marker text",
r"\escaped backslash\\character",
r"\test1 \test2 \\backslash \^hat \%\test3\\\^"])),
[elem('marker', Text('text')),
elem('escaped', Text(r'backslash\\character')),
elem('test1'),
elem('test2', Text(r'\\backslash ')),
elem('^hat'),
elem('%'),
elem('test3', Text(r'\\')),
elem('^')])
# Test with extended escaping rules.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.assertEqual(
list(sfm.parser([
"\\test1 \\test2 \\\\backslash \\^hat \\%\\test3\\\\\\^"],
tag_escapes="[^0-9a-zA-Z]")),
[elem('test1'),
elem('test2', Text('\\\\backslash \\^hat \\%')),
elem('test3', Text('\\\\\\^'))])
def test_transduction(self):
src = ['\\test\n',
'\\test text\n',
'\\sfm text\n',
'bare text\n',
'\\more-sfm more text\n',
'over a line break\\marker'
'\\le unix\n',
'\\le windows\r\n',
'\\le missing\n',
'\\test\\i1\\i2 deep text\\i1*\n',
'\\test\\i1\\i2 deep text\n',
# These forms do not transduce identically due to whitespace differences
'\\test \\inline text\\inline*\n',
'\\test \\i1\\i2 deep\\i2*\\i1*\n',
]
with warnings.catch_warnings(record=True) as ref_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
ref = list(sfm.parser(src))
trans = handler.handler()
trans_src = list(handler.transduce(sfm.parser, trans, src))
# Check known straight through transductions.
map(self.assertEqual, src[:10], trans_src[:10])
# Check the errors match
map(self.assertEqual, map(str,ref_parse_errors), map(str,trans.errors))
# Check the rest but do not ignore syntax warnings as there should be none
# produced by parse of the transduced output.
# Ignore the column posiions as whitespace differences will cause
# differences there however line numbers should remain stable
map(self.assertEqual,
((e.pos.line, getattr(e,'meta',None), getattr(e,'annotations',None)) for e in flatten(sfm.parser(trans_src))),
((e.pos.line,getattr(e,'meta',None), getattr(e,'annotations',None)) for e in flatten(ref)))
def test_pprint(self):
src = [
'\\test\n',
'\\test text\n',
'\\sfm text\n',
'bare text\n',
'\\more-sfm more text\n',
'over a line break\\marker'
'\\le unix\n',
'\\le windows\r\n',
'\\le missing\n',
'\\test\\i1\\i2 deep text\\i1*\n',
'\\test\\i1\\i2 deep text\n',
# These forms do not transduce identically due to whitespace differences
'\\test \\inline text\\inline*\n',
'\\test \\i1\\i2 deep\\i2*\\i1*\n'
]
with warnings.catch_warnings(record=True) as ref_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
ref_parse = list(sfm.parser(src))
trans_src = sfm.pprint(ref_parse).splitlines(True)
with warnings.catch_warnings(record=True) as trans_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
trans_parse = list(sfm.parser(trans_src))
# Check pretty printer output matches input, skip the last 2
map(self.assertEqual, src[:10], trans_src[:10])
# Check the parsed pretty printed doc matches the reference
self.assertEqual(ref_parse, trans_parse)
# Check the errors match
map(self.assertEqual, map(str, ref_parse_errors[:10]),
map(str, trans_parse_errors[:10]))
# Check the positions line up for the first 10 items
map(self.assertEqual, (e.pos for e in flatten(ref_parse[:16])),
(e.pos for e in flatten(trans_parse[:16])))
# Check all the line positions, meta data and annotations line up
map(self.assertEqual, ((e.pos.line, getattr(
e, 'meta', None), getattr(e, 'annotations', None))
for e in flatten(ref_parse)),
((e.pos.line, getattr(e, 'meta',
None), getattr(e, 'annotations', None))
for e in flatten(trans_parse)))
def test_position(self):
p=sfm.parser(['\\li1 text\n',
'\\l2\n',
'\\l3\n'])
self.assertEqual([tuple(e.pos) for e in flatten(p)],
[(1,1),(1,6), # \li1 text\n
(2,1),(2,4), # \l2\n
(3,1),(3,4)]) # \l3\n
def test_format(self):
src = ['\\test\n',
'\\test text\n',
'\\sfm text\n',
'bare text\n',
'\\more-sfm more text\n',
'over a line break\\marker'
'\\le unix\n',
'\\le windows\r\n',
'\\le missing\n',
'\\test\\i1\\i2 deep text\\i1*\n',
'\\test\\i1\\i2 deep text\n',
# These forms do not transduce identically due to whitespace
# differences
'\\test \\inline text\\inline*\n',
'\\test \\i1\\i2 deep\\i2*\\i1*\n']
with warnings.catch_warnings(record=True) as ref_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
ref_parse = list(sfm.parser(src))
trans_src = sfm.generate(ref_parse).splitlines(True)
with warnings.catch_warnings(record=True) as trans_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
trans_parse = list(sfm.parser(trans_src))
# Check the parsed pretty printed doc matches the reference
self.assertEqual(trans_parse, ref_parse)
# Check pretty printer output matches input, skip the last 2
self.assertEqual(trans_src[:10], src[:10])
# Check the errors match
for a, e in zip(trans_parse_errors[:31], ref_parse_errors):
with self.subTest(warning=str(e)):
self.assertEqual(a.message.args, e.message.args)
# Check all the line positions, meta data and annotations line up
for a, e in zip(flatten(trans_parse), flatten(ref_parse)):
with self.subTest():
self.assertEqual(a.pos.line, e.pos.line)
self.assertAlmostEqual(a.pos.col, e.pos.col, delta=1)
self.assertEqual(getattr(a, 'meta', None),
getattr(e, 'meta', None))
self.assertEqual(getattr(a, 'annotations', None),
getattr(e, 'annotations', None))
def test_position(self):
p = sfm.parser(['\\li1 text\n',
'\\l2\n',
'\\l3\n'])
self.assertEqual([tuple(e.pos) for e in flatten(p)],
[(1, 1), (1, 6), # \li1 text\n
(2, 1), (2, 4), # \l2\n
(3, 1), (3, 4)]) # \l3\n
def test_line_ends(self):
self.assertEqual(list(sfm.parser(['\\le unix\n',
'\\le windows\r\n',
'\\empty\n',
'\\le missing'])),
[elem('le',text('unix\n')),
elem('le',text('windows\r\n')),
elem('empty',text('\n')),
elem('le', text('missing'))])
def test_line_ends(self):
self.assertEqual(list(sfm.parser(['\\le unix\n',
'\\le windows\r\n',
'\\empty\n',
'\\le missing'])),
[elem('le', Text('unix\n')),
elem('le', Text('windows\r\n')),
elem('empty', Text('\n')),
elem('le', Text('missing'))])
def test_mixed(self):
self.assertEqual(list(sfm.parser(['\\sfm text\n',
'bare text\n',
'\\more-sfm more text\n',
'over a line break\\marker'])),
[elem('sfm',text('text\n'
'bare text\n')),
elem('more-sfm',text('more text\n'
'over a line break')),
elem('marker')])
def test_pprint(self):
src = ['\\test\n',
'\\test text\n',
'\\sfm text\n',
'bare text\n',
'\\more-sfm more text\n',
'over a line break\\marker'
'\\le unix\n',
'\\le windows\r\n',
'\\le missing\n',
'\\test\\i1\\i2 deep text\\i1*\n',
'\\test\\i1\\i2 deep text\n',
# These forms do not transduce identically due to whitespace differences
'\\test \\inline text\\inline*\n',
'\\test \\i1\\i2 deep\\i2*\\i1*\n']
with warnings.catch_warnings(record=True) as ref_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
ref_parse = list(sfm.parser(src))
trans_src = sfm.pprint(ref_parse).splitlines(True)
with warnings.catch_warnings(record=True) as trans_parse_errors:
warnings.resetwarnings()
warnings.simplefilter("always", SyntaxWarning)
trans_parse = list(sfm.parser(trans_src))
# Check pretty printer output matches input, skip the last 2
map(self.assertEqual, src[:10], trans_src[:10])
# Check the parsed pretty printed doc matches the reference
self.assertEqual(ref_parse, trans_parse)
# Check the errors match
map(self.assertEqual,
map(str,ref_parse_errors[:10]),
map(str,trans_parse_errors[:10]))
# Check the positions line up for the first 10 items
map(self.assertEqual,
(e.pos for e in flatten(ref_parse[:16])),
(e.pos for e in flatten(trans_parse[:16])))
# Check all the line positions, meta data and annotations line up
map(self.assertEqual,
((e.pos.line, getattr(e,'meta',None), getattr(e,'annotations',None)) for e in flatten(ref_parse)),
((e.pos.line, getattr(e,'meta',None), getattr(e,'annotations',None)) for e in flatten(trans_parse)))
def test_text(self):
self.assertEqual(list(sfm.parser([])),[])
self.assertEqual(list(sfm.parser([''])),[])
self.assertEqual(list(sfm.parser('text with no markers')),
[text('text with no markers')])