def test_generate_WikidPadHelp_selection():
test_fragments = [ # (page_name, text, node_name, formatted_text)
('pageName', '[.]', 'wikiWord', '[.]'), # 1
('PageName', '[.]', 'wikiWord', '[.]'),
('PageName', 'PageName', 'wikiWord', 'PageName'),
('PageName', '[PageName]', 'wikiWord', '[PageName]'),
('PageName', '[contact: "Carl [Home]"]', 'attribute',
'[contact: "Carl [Home]"]'),
('PageName', '[//OptionsDialog]', 'wikiWord', '[//OptionsDialog]'),
('PageName', '[//ebay/Circlet]', 'wikiWord', '[//ebay/Circlet]'),
('PageName', '[WikiWord| This is the title ]', 'wikiWord',
'[WikiWord|This is the title]'),
('PageName', '[:rel: parents]', 'insertion', '[:rel: parents]'),
('PageName', '[:rel: parents; aslist]', 'insertion',
'[:rel: parents; aslist]'),
('PageName', '[:rel: children; existingonly;columns 2]', 'insertion',
'[:rel: children; existingonly; columns 2]'),
('PageName', '[key: value]', 'attribute', '[key: value]'),
('PageName', '[:toc: ]', 'insertion', '[:toc:]'),
(
'ChangeLog2008',
'[test:foo; ]', # still legal?!
'attribute',
'[test: foo]'),
(
'TestPage',
'[test:foo;; ]', # still legal?!
'attribute',
'[test: foo]'),
('PageName', '[key: value with spaces]', 'attribute',
'[key: value with spaces]'),
('PageName', '[key: value; value2]', 'attribute',
'[key: value; value2]'),
('PageName', '[key: "value: with special char"]', 'attribute',
'[key: value: with special char]'),
('PageName', '[key: "value = special"]', 'attribute',
'[key: value = special]'),
('pageName', '[wikiword]#searchfragment', 'wikiWord',
'[wikiword#searchfragment]'),
('pageName', '[wikiword#searchfragment]', 'wikiWord',
'[wikiword#searchfragment]'),
('pageName', '[wikiword#search fragment]', 'wikiWord',
'[wikiword#search fragment]'),
('AutoCompletion', '[bookmarked=true]', 'attribute',
'[bookmarked: true]'),
('ChangeLog', '[ChangeLog2011]', 'wikiWord', '[ChangeLog2011]'),
('ChronViewWindow', '[OptionsDialog#+++ Chron. view]', 'wikiWord',
'[OptionsDialog#+++ Chron. view]'),
('ChronViewWindow', '[OptionsDialog#+++ Chronological]', 'wikiWord',
'[OptionsDialog#+++ Chronological]'),
('CommandLineSupport',
'[WikiMaintenance#++ Update ext. modif. wiki files]', 'wikiWord',
'[WikiMaintenance#++ Update ext. modif. wiki files]'),
('ExternalGraphicalApplications', '[:eqn:"a^2 + b^2 = c^2"]',
'insertion', '[:eqn: "a^2 + b^2 = c^2"]'),
('Icon airbrush', '[icon:airbrush]', 'attribute', '[icon: airbrush]'),
('Icon cd_audio', '[icon:cd_audio ]', 'attribute', '[icon: cd_audio]'),
('Insertions', '[:page: "IncrementalSearch"]', 'insertion',
'[:page: IncrementalSearch]'),
('Insertions', '[:page: "IncrementalSearch"]', 'insertion',
'[:page: IncrementalSearch]'),
('Insertions', '[:rel: children;existingonly;columns 2;coldir down]',
'insertion',
'[:rel: children; existingonly; columns 2; coldir down]'),
('Insertions', '[:search:"todo:todo"]', 'insertion',
'[:search: todo:todo]'),
('Insertions', '[:search:"todo:todo";showtext]', 'insertion',
'[:search: todo:todo; showtext]'),
('Insertions', '[:eval:"5+6"]', 'insertion', '[:eval: "5+6"]'),
('ExternalGraphicalApplications',
'[:dot:"\ndigraph {\na -> b\nb -> c\nb -> d\nd -> a\n}\n"; noerror]',
'insertion',
'[:dot: "\ndigraph {\na -> b\nb -> c\nb -> d\nd -> a\n}\n"; noerror]'
),
('ExternalGraphicalApplications',
('[:ploticus:"\n'
'#proc areadef\n'
' title: Annual Revenues, in thousands\n'
' rectangle: 1 1 5 2\n'
' xrange: 0 4\n'
' yrange: -5000 15000\n'
' yaxis.stubs: incremental 5000\n'
' yaxis.grid: color=pink\n'
' xaxis.stubs: text\n'
'ABC Corp\n'
'NetStuff\n'
'MicroMason\n'
'\n'
'#proc getdata\n'
' data: 6430 -780 13470\n'
'\n'
'#proc processdata\n'
' action: rotate\n'
'\n'
'#proc bars\n'
' lenfield: 1\n'
' color: dullyellow\n'
' labelword: $ @@N\n'
' crossover: 0\n'
'"]'), 'insertion',
('[:ploticus: "\n#proc areadef\n title: Annual Revenues, in '
'thousands\n rectangle: 1 1 5 2\n xrange: 0 4\n yrange: -5000 '
'15000\n yaxis.stubs: incremental 5000\n yaxis.grid: color=pink\n'
' xaxis.stubs: text\nABC Corp\nNetStuff\nMicroMason\n\n'
'#proc getdata\n data: 6430 -780 13470\n\n#proc processdata\n'
' action: rotate\n\n#proc bars\n lenfield: 1\n color: dullyellow\n'
' labelword: $ @@N\n crossover: 0\n"]')),
('ExternalGraphicalApplications', """[:gnuplot:"
set key right nobox
set samples 100
plot [-pi/2:pi] cos(x),-(sin(x) > sin(x+1) ? sin(x) : sin(x+1))
"]""", 'insertion', """[:gnuplot: "
set key right nobox
set samples 100
plot [-pi/2:pi] cos(x),-(sin(x) > sin(x+1) ? sin(x) : sin(x+1))
"]"""),
]
langHelper = getApp().createWikiLanguageHelper(LANGUAGE_NAME)
wikidoc = MockWikiDocument(None, LANGUAGE_NAME)
tests = enumerate(test_fragments, 1)
for nr, (pageName, text, node_name, formatted_text) in tests:
text_ = '\n%s\n\n' % text
try:
page = wikidoc.getWikiPage(pageName)
except WikiWordNotFoundException:
page = wikidoc.createWikiPage(pageName)
page.setContent(text_)
ast = page.getLivePageAst()
nf = NodeFinder(ast)
if node_name is not None:
assert nf.count(node_name) == 1
else:
assert nf.count('wikiWord') == 0
assert nf.count('attribute') == 0
assert nf.count('insertion') == 0
result = langHelper.generate_text(ast, page)[1:-2]
assert result == formatted_text, '%d: %r on %r -> %r != %r' % (
nr, text, pageName, result, formatted_text)
def test_generate_WikidPadHelp_selection():
test_fragments = [ # (page_name, text, node_name, formatted_text)
('pageName', '[.]',
'wikiWord', '[.]'), # 1
('PageName', '[.]',
'wikiWord', '[.]'),
('PageName', 'PageName',
'wikiWord', 'PageName'),
('PageName', '[PageName]',
'wikiWord', '[PageName]'),
('PageName', '[contact: "Carl [Home]"]',
'attribute', '[contact: "Carl [Home]"]'),
('PageName', '[//OptionsDialog]',
'wikiWord', '[//OptionsDialog]'),
('PageName', '[//ebay/Circlet]', 'wikiWord', '[//ebay/Circlet]'),
('PageName', '[WikiWord| This is the title ]',
'wikiWord', '[WikiWord|This is the title]'),
('PageName', '[:rel: parents]', 'insertion', '[:rel: parents]'),
('PageName', '[:rel: parents; aslist]',
'insertion', '[:rel: parents; aslist]'),
('PageName', '[:rel: children; existingonly;columns 2]',
'insertion', '[:rel: children; existingonly; columns 2]'),
('PageName', '[key: value]',
'attribute', '[key: value]'),
('PageName', '[:toc: ]',
'insertion', '[:toc:]'),
('ChangeLog2008', '[test:foo; ]', # still legal?!
'attribute', '[test: foo]'),
('TestPage', '[test:foo;; ]', # still legal?!
'attribute', '[test: foo]'),
('PageName', '[key: value with spaces]',
'attribute', '[key: value with spaces]'),
('PageName', '[key: value; value2]',
'attribute', '[key: value; value2]'),
('PageName', '[key: "value: with special char"]',
'attribute', '[key: value: with special char]'),
('PageName', '[key: "value = special"]',
'attribute', '[key: value = special]'),
('pageName', '[wikiword]#searchfragment',
'wikiWord', '[wikiword#searchfragment]'),
('pageName', '[wikiword#searchfragment]',
'wikiWord', '[wikiword#searchfragment]'),
('pageName', '[wikiword#search fragment]',
'wikiWord', '[wikiword#search fragment]'),
('AutoCompletion', '[bookmarked=true]',
'attribute', '[bookmarked: true]'),
('ChangeLog', '[ChangeLog2011]',
'wikiWord', '[ChangeLog2011]'),
('ChronViewWindow', '[OptionsDialog#+++ Chron. view]',
'wikiWord', '[OptionsDialog#+++ Chron. view]'),
('ChronViewWindow', '[OptionsDialog#+++ Chronological]',
'wikiWord', '[OptionsDialog#+++ Chronological]'),
('CommandLineSupport', '[WikiMaintenance#++ Update ext. modif. wiki files]',
'wikiWord', '[WikiMaintenance#++ Update ext. modif. wiki files]'),
('ExternalGraphicalApplications', '[:eqn:"a^2 + b^2 = c^2"]',
'insertion', '[:eqn: "a^2 + b^2 = c^2"]'),
('Icon airbrush', '[icon:airbrush]',
'attribute', '[icon: airbrush]'),
('Icon cd_audio', '[icon:cd_audio ]',
'attribute', '[icon: cd_audio]'),
('Insertions', '[:page: "IncrementalSearch"]',
'insertion', '[:page: IncrementalSearch]'),
('Insertions', '[:page: "IncrementalSearch"]',
'insertion', '[:page: IncrementalSearch]'),
('Insertions', '[:rel: children;existingonly;columns 2;coldir down]',
'insertion', '[:rel: children; existingonly; columns 2; coldir down]'),
('Insertions', '[:search:"todo:todo"]',
'insertion', '[:search: todo:todo]'),
('Insertions', '[:search:"todo:todo";showtext]',
'insertion', '[:search: todo:todo; showtext]'),
('Insertions', '[:eval:"5+6"]',
'insertion', '[:eval: "5+6"]'),
('ExternalGraphicalApplications',
'[:dot:"\ndigraph {\na -> b\nb -> c\nb -> d\nd -> a\n}\n"; noerror]',
'insertion',
'[:dot: "\ndigraph {\na -> b\nb -> c\nb -> d\nd -> a\n}\n"; noerror]'),
('ExternalGraphicalApplications',
('[:ploticus:"\n'
'#proc areadef\n'
' title: Annual Revenues, in thousands\n'
' rectangle: 1 1 5 2\n'
' xrange: 0 4\n'
' yrange: -5000 15000\n'
' yaxis.stubs: incremental 5000\n'
' yaxis.grid: color=pink\n'
' xaxis.stubs: text\n'
'ABC Corp\n'
'NetStuff\n'
'MicroMason\n'
'\n'
'#proc getdata\n'
' data: 6430 -780 13470\n'
'\n'
'#proc processdata\n'
' action: rotate\n'
'\n'
'#proc bars\n'
' lenfield: 1\n'
' color: dullyellow\n'
' labelword: $ @@N\n'
' crossover: 0\n'
'"]'),
'insertion',
('[:ploticus: "\n#proc areadef\n title: Annual Revenues, in '
'thousands\n rectangle: 1 1 5 2\n xrange: 0 4\n yrange: -5000 '
'15000\n yaxis.stubs: incremental 5000\n yaxis.grid: color=pink\n'
' xaxis.stubs: text\nABC Corp\nNetStuff\nMicroMason\n\n'
'#proc getdata\n data: 6430 -780 13470\n\n#proc processdata\n'
' action: rotate\n\n#proc bars\n lenfield: 1\n color: dullyellow\n'
' labelword: $ @@N\n crossover: 0\n"]')),
('ExternalGraphicalApplications',
"""[:gnuplot:"
set key right nobox
set samples 100
plot [-pi/2:pi] cos(x),-(sin(x) > sin(x+1) ? sin(x) : sin(x+1))
"]""",
'insertion',
"""[:gnuplot: "
set key right nobox
set samples 100
plot [-pi/2:pi] cos(x),-(sin(x) > sin(x+1) ? sin(x) : sin(x+1))
"]"""),
]
langHelper = getApp().createWikiLanguageHelper(LANGUAGE_NAME)
wikidoc = MockWikiDocument(None, LANGUAGE_NAME)
tests = enumerate(test_fragments, 1)
for nr, (pageName, text, node_name, formatted_text) in tests:
text_ = '\n%s\n\n' % text
try:
page = wikidoc.getWikiPage(pageName)
except WikiWordNotFoundException:
page = wikidoc.createWikiPage(pageName)
page.setContent(text_)
ast = page.getLivePageAst()
nf = NodeFinder(ast)
if node_name is not None:
assert nf.count(node_name) == 1
else:
assert nf.count('wikiWord') == 0
assert nf.count('attribute') == 0
assert nf.count('insertion') == 0
result = langHelper.generate_text(ast, page)[1:-2]
assert result == formatted_text, '%d: %r on %r -> %r != %r' % (
nr, text, pageName, result, formatted_text)
def test_generate_WikidPadHelp():
"""Run over *complete* WikidPadHelp wiki: parse each page, generate text
from AST using text generator, and check if generated text matches the
original text::
text -> |parser| -> AST -> |text generator| -> result
assert result == text
The *first time*, set `add_unknown_differences_to_annotation_file` to
True and annotate generated file with differences: put '!=' if different,
and '==' if equal (in semantics, maybe not in syntax, e.g.,
[key: value ] is equal to [key: value] (note the extra spaces)).
"""
# add_unknown_differences_to_annotation_file = True
add_unknown_differences_to_annotation_file = False
def load_annotations(path):
equivalents = defaultdict(dict)
known_differences = defaultdict(dict)
page_name, text, result = None, None, None
try:
with io.open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
if line.startswith('-- '):
text = line[3:]
elif line.startswith('== '):
result = line[3:]
equivalents[page_name][text] = result
elif line.startswith('!= '):
result = line[3:]
known_differences[page_name][text] = result
else:
page_name = line
except IOError:
pass # no file yet?!
return equivalents, known_differences
equivalents, known_differences = load_annotations(WIKIDPADHELP_ANNOTATIONS)
langHelper = getApp().createWikiLanguageHelper(LANGUAGE_NAME)
wikidoc = MockWikiDocument(None, LANGUAGE_NAME)
paths = glob.glob(os.path.join(WIKIDPADHELP_DATA_DIR, '*.wiki'))
skip = set()
# Page itself is in WikidPad syntax so it has to work as well
# skip = set(
# u'MediaWiki%2FTextFormatting', # Media Wiki syntax, not WikidPadParser
# )
nof_known_differences = 0
nof_unknown_differences = 0
for nr, path in enumerate(sorted(paths), 1):
pageName, _ = os.path.splitext(os.path.basename(path))
if pageName in skip:
continue
text = get_text(path)
try:
page = wikidoc.getWikiPage(pageName)
except WikiWordNotFoundException:
page = wikidoc.createWikiPage(pageName)
page.setContent(text)
ast = page.getLivePageAst()
result = langHelper.generate_text(ast, page)
# assert result == text
current_page_correct = True
with io.open(WIKIDPADHELP_ANNOTATIONS, 'a', encoding='utf-8') as f:
to_compare = list(
zip(result.splitlines(True), text.splitlines(True)))
for result_line, text_line in to_compare:
result_line = result_line.rstrip()
text_line = text_line.rstrip()
if result_line == text_line:
continue # ok, equal
try:
equivalent_line = equivalents[pageName][text_line]
except KeyError:
equivalent_line = None
if result_line == equivalent_line:
continue # ok, lines are considered equal
try:
known_difference = known_differences[pageName][text_line]
except KeyError:
known_difference = None
if result_line == known_difference:
nof_known_differences += 1
continue # ok, we know about this difference
# we have an unknown difference here
nof_unknown_differences += 1
if add_unknown_differences_to_annotation_file:
if current_page_correct: # first error for this page
current_page_correct = False
f.write(pageName + '\n')
f.write('-- ' + text_line + '\n')
f.write('!= ' + result_line + '\n')
msg = 'TOTAL: %d known differences, %d unknown differences'
msg %= (nof_known_differences, nof_unknown_differences)
assert not nof_unknown_differences, msg
def test_generate_WikidPadHelp():
"""Run over *complete* WikidPadHelp wiki: parse each page, generate text
from AST using text generator, and check if generated text matches the
original text::
text -> |parser| -> AST -> |text generator| -> result
assert result == text
The *first time*, set `add_unknown_differences_to_annotation_file` to
True and annotate generated file with differences: put '!=' if different,
and '==' if equal (in semantics, maybe not in syntax, e.g.,
[key: value ] is equal to [key: value] (note the extra spaces)).
"""
# add_unknown_differences_to_annotation_file = True
add_unknown_differences_to_annotation_file = False
def load_annotations(path):
equivalents = defaultdict(dict)
known_differences = defaultdict(dict)
page_name, text, result = None, None, None
try:
with io.open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
if line.startswith('-- '):
text = line[3:]
elif line.startswith('== '):
result = line[3:]
equivalents[page_name][text] = result
elif line.startswith('!= '):
result = line[3:]
known_differences[page_name][text] = result
else:
page_name = line
except IOError:
pass # no file yet?!
return equivalents, known_differences
equivalents, known_differences = load_annotations(WIKIDPADHELP_ANNOTATIONS)
langHelper = getApp().createWikiLanguageHelper(LANGUAGE_NAME)
wikidoc = MockWikiDocument(None, LANGUAGE_NAME)
paths = glob.glob(os.path.join(WIKIDPADHELP_DATA_DIR, '*.wiki'))
skip = set()
# Page itself is in WikidPad syntax so it has to work as well
# skip = set(
# u'MediaWiki%2FTextFormatting', # Media Wiki syntax, not WikidPadParser
# )
nof_known_differences = 0
nof_unknown_differences = 0
for nr, path in enumerate(sorted(paths), 1):
pageName, _ = os.path.splitext(os.path.basename(path))
if pageName in skip:
continue
text = get_text(path)
try:
page = wikidoc.getWikiPage(pageName)
except WikiWordNotFoundException:
page = wikidoc.createWikiPage(pageName)
page.setContent(text)
ast = page.getLivePageAst()
result = langHelper.generate_text(ast, page)
# assert result == text
current_page_correct = True
with io.open(WIKIDPADHELP_ANNOTATIONS, 'a', encoding='utf-8') as f:
to_compare = list(zip(result.splitlines(True), text.splitlines(True)))
for result_line, text_line in to_compare:
result_line = result_line.rstrip()
text_line = text_line.rstrip()
if result_line == text_line:
continue # ok, equal
try:
equivalent_line = equivalents[pageName][text_line]
except KeyError:
equivalent_line = None
if result_line == equivalent_line:
continue # ok, lines are considered equal
try:
known_difference = known_differences[pageName][text_line]
except KeyError:
known_difference = None
if result_line == known_difference:
nof_known_differences += 1
continue # ok, we know about this difference
# we have an unknown difference here
nof_unknown_differences += 1
if add_unknown_differences_to_annotation_file:
if current_page_correct: # first error for this page
current_page_correct = False
f.write(pageName + '\n')
f.write('-- ' + text_line + '\n')
f.write('!= ' + result_line + '\n')
msg = 'TOTAL: %d known differences, %d unknown differences'
msg %= (nof_known_differences, nof_unknown_differences)
assert not nof_unknown_differences, msg