def test_removes_unreachable_symbols(self):
grammar = r'''
start: <head>
<head> ::= <a> <b>
<a> ::= "-" | ε
<b> ::= <a> <d>
<d> ::= "1"
<c> ::= "Q"
'''
tree = Parser().parse(grammar)
Translator().translate(tree)
for node in tree.filter(lambda x: x.value == 'c'):
self.fail('Expected unused expressions to be removed.')
def test_external_filename_preserved_in_both_python_and_bnf(self):
external = ('from darglint.parse.identifiers import (\n'
' ArgumentIdentifier,\n'
' NoqaIdentifier,\n'
')')
grammar = f'''
{external}
<A> ::= "A"
'''
node = Parser().parse(grammar)
self.assertTrue(external in str(node))
self.assertTrue(external in node.to_python())
def test_with_name(self):
grammar = '\n'.join([
'import base.bnf', '', 'Grammar: MySpecialGrammar', '',
'<A> ::= "SPECIAL"'
])
tree = Parser().parse(grammar)
name = next(tree.filter(Node.is_name))
self.assertEqual(
name.value,
'MySpecialGrammar',
)
py_repr = tree.to_python()
self.assertTrue('class MySpecialGrammar' in py_repr)
def test_parse_rule_with_spaces(self):
"""Test conjunction in rules."""
node = Parser().parse('<heading> ::= <ident> ":"')
self.assertEqual(
str(node),
'<heading> ::= <ident> ":"',
)
def test_complete_conversion(self):
"""Test converting a complete example, and ensuring it's in CNF."""
grammar = r'''
<Expr> ::= <Term>
| <Expr> <AddOp> <Term>
| <AddOp> <Term>
<Term> ::= <Factor>
| <Term> <MulOp> <Factor>
<Factor> ::= <Primary>
| <Factor> "\^" <Primary>
<Primary> ::= <number>
| <variable>
| "\(" <Expr> "\)"
<AddOp> ::= "\+" | "\-"
<MulOp> ::= "\*" | "\/"
<number> ::= <digit> | <digit> <number>
<digit> ::= "0" | "1" | "2" | "3" | "4"
| "5" | "6" | "7" | "8" | "9"
<variable> ::= "a" | "b" | "c" | "d" | "e" | "f"
| "g" | "h" | "i" | "j" | "k" | "l"
| "m" | "n" | "o" | "p" | "q" | "r"
| "s" | "t" | "u" | "v" | "w" | "x"
| "y" | "z"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
self.assertTrue(Validator(raise_exception=True).validate(node))
class ValidateCnfTests(TestCase):
def setUp(self):
self.parser = Parser()
def _p(self, grammar):
return self.parser.parse(grammar)
def test_terminal_production(self):
"""Make sure we a terminal production passes."""
valid_productions = [
'<COLON> ::= ":"',
'<PARAM> ::= "Args"',
'<QUOTE> ::= "\\"\\"\\""',
# '<PERIOD> ::= "."',
]
for production in valid_productions:
self.assertTrue(Validator().validate(self._p(production)),
'"{}" should not be valid'.format(production))
def test_escaped_characters_okay(self):
"""Make sure that special characters are okay if escaped."""
for c in ', +*()[]|':
self.assertTrue(
Validator().validate(
self._p('<SOMETHING> ::= "A\\{}B"'.format(c))),
'Escaping "{}" should allow it.'.format(c),
)
def test_translate_retains_annotations_up(self):
"""Make sure we retain annotations when moving up."""
grammar = r'''
start: <phone-number>
<phone-number>
::= <confusion-number>
<confusion-number>
::= @ConfusionError <number-group> <dot> <confusion-number>
| @ConfusionError <number-group> <dash> <confusion-number>
| <number-group>
<number-group>
::= <number> <number-group>
| <number>
<number> ::= "PN\.NUMBER"
<dash> ::= "PN\.DASH"
<dot> ::= "PN\.DOT"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
encountered_annotation = False
for child in node.walk():
if child.node_type in [NodeType.ANNOTATION, NodeType.ANNOTATIONS]:
encountered_annotation = True
self.assertTrue(encountered_annotation, )
def test_no_unnamed_nodes(self):
"""Certain nodes, when translated, are missing names."""
grammar = r'''
Grammar: ArgumentsGrammar
start: <arguments-section>
<arguments-section>
::= <ahead> <line>
# Causes an missing production name.
| <ahead>
# Simplified the remaning grammar.
<line> ::= "TokenType\.LINE"
<ahead> ::= "TokenType\.AHEAD"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
python = node.to_python()
unacceptable_pattern = re.compile(r'P\([^"]')
self.assertEqual(
unacceptable_pattern.findall(python),
[],
'Found production which doesn\'t begin with a name:\n{}'.format(
python),
)
def test_remove_single_unit_production(self):
"""Make sure we can at least remove a single unit production."""
tree = Parser().parse('<A> ::= <B>\n' '<B> ::= "moch"')
node = Translator().translate(tree)
expected = ('<A> ::= "moch"\n' '<B> ::= "moch"')
self.assertEqual(
str(node), expected,
f'\n\nExpected:\n{expected}\n\nBut Got:\n{str(node)}\n\n')
def test_parse_simple_rule(self):
"""Make sure a simple, terminal rule works."""
node = Parser().parse('<args> ::= "Args"')
self.assertTrue(isinstance(node, Node))
self.assertEqual(
str(node),
'<args> ::= "Args"',
)
def test_eliminate_simplest_epsilon_form(self):
tree = Parser().parse('<E> ::= "e" | ε')
node = Translator().translate(tree)
expected = '<E> ::= "e"'
self.assertEqual(
str(node),
expected,
)
def test_removes_empty_productions_after_epsilon_elimination(self):
tree = Parser().parse('<A> ::= "a"\n' '<B> ::= ε')
node = Translator().translate(tree)
expected = '<A> ::= "a"'
self.assertEqual(
str(node),
expected,
)
def test_start_symbol_not_reassigned(self):
"""Make sure the start symbol points to the start terminal."""
tree = Parser().parse('start: <A>\n' '<A> ::= <B>\n' '<B> ::= "\\."')
node = Translator().translate(tree)
expected = 'start: <A>\n<A> ::= "\\."'
self.assertEqual(
str(node), expected,
f'\n\nExpected:\n{expected}\n\nBut Got:\n{str(node)}\n\n')
def test_translate_retains_start_annotation(self):
"""Make sure that an annotation on start is saved."""
grammar = r'''
@Q
<start> ::= <A> <B>
<B> ::= "b"
<A> ::= "a"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
expected = Parser().parse(r'''
@Q
<start> ::= <A> <B>
<B> ::= "b"
<A> ::= "a"
''')
self.assertTrue(node.equals(expected), f'{node}\n\n{expected}')
def test_nonsolitary_terminals_symbol_taken(self):
"""Make sure non-solitary teminals will have unique name."""
tree = Parser().parse('<arg-header> ::= <arg> ":"\n'
'<C> ::= "Another_value"')
node = Translator().translate(tree)
self.assertEqual(
str(node), '<arg-header> ::= <arg> <C0>\n'
'<C> ::= "Another_value"\n'
'<C0> ::= ":"')
def test_nodes_not_leading_to_terminals_removed(self):
grammar = r'''
start: <a>
<a> ::= "a" | <b>
<b> ::= <c> <d>
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
self.assertFalse(list(node.filter(lambda x: x.value == 'c')))
def test_translate_retains_annotations(self):
"""Make sure that annotations are retained with the grammar."""
grammar = r'''
<A> ::= <B> <C>
<B> ::= <C> | @Q <D>
<C> ::= "C"
<D> ::= "D"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
expected = Parser().parse(r'''
<A> ::= <B> <C>
<B> ::= "C" | @Q "D"
<C> ::= "C"
<D> ::= "D"
''')
self.assertTrue(node.equals(expected),
f'Expected:\n{expected}\n\nBut got:\n{node}')
def test_parse_from_import(self):
grammar = '''
from darglint.errors import (
ItemIndentationError,
)
<A> ::= "A"
'''
tree = Parser().parse(grammar)
self.assertTrue(tree)
def test_translate_works_with_annotation_on_nonterminal(self):
grammar = r'''
<A> ::= "b"
| @Q <B> <C>
<B> ::= "b"
<C> ::= <C>
'''
tree = Parser().parse(grammar)
Translator().translate(tree)
def test_parse_multiple_annotations(self):
"""Make sure a production can have multiple annotations."""
grammar = '@VERBLESS\n@PUNCT\n<sentence> ::= <period> <noun>'
tree = Parser().parse(grammar)
production = tree.children[0]
annotations = production.children[0]
self.assertEqual(
['VERBLESS', 'PUNCT'],
[x.value for x in annotations.children],
)
def test_failing_terminal_parse(self):
"""Make sure this particular instance, which failed, passes."""
value = ('<S> ::= <A> "a" | <B>\n'
'<A> ::= "b" | <B>\n'
'<B> ::= <A> | "a"')
node = Parser().parse(value)
self.assertEqual(
value,
str(node),
)
def test_mix_terminals_and_nonterminals(self):
"""Make sure we can mix terminals and non-terminals."""
values = [
'<S> ::= <A> "a" | <B>',
'<S> ::= "a" <A> | <B>',
'<S> ::= "a" | <B>',
]
for value in values:
node = Parser().parse(value)
self.assertEqual(str(node), value)
def test_eliminate_epsilon_forms(self):
"""Make sure we can eliminate all ε-rules."""
tree = Parser().parse('<S> ::= <A> <B>\n'
'<B> ::= "b"\n'
'<A> ::= "a" | ε')
node = Translator().translate(tree)
expected = ('<S> ::= <A> <B> | "b"\n' '<B> ::= "b"\n' '<A> ::= "a"')
self.assertEqual(
str(node),
expected,
)
def test_nonsolitary_terminals(self):
"""Make sure non-solitary terminals are factored out."""
tree = Parser().parse('<arg-header> ::= <arg> ":"')
node = Translator().translate(tree)
self.assertEqual(
str(node),
'\n'.join([
'<arg-header> ::= <arg> <C>',
'<C> ::= ":"',
]),
)
def test_parse_probability(self):
grammar = '''
<start>
::= 90 <A>
| 10 <B>
<A> ::= "A"
<B> ::= "B"
'''
tree = Parser().parse(grammar)
self.assertTrue(tree)
def test_translate_with_recursion(self):
grammar = r'''
<start> ::= <b>
<b> ::= <c> <b>
| <c>
<c> ::= "A"
'''
tree = Parser().parse(grammar)
Translator().translate(tree)
def test_with_imports(self):
grammar = ('import base.bnf\n'
'import utils.bnf\n'
'\n'
'start: <sentence>\n'
'<setence> ::= <verb> <noun>\n'
'<verb> ::= "TT\\.VERB"\n'
'<noun> ::= "TT\\.NOUN"')
node = Parser().parse(grammar)
self.assertEqual(grammar, str(node),
f'\nExpected:\n{grammar}\n\nBut got:\n{node}')
def test_remove_complex_unit_production(self):
tree = Parser().parse('<S> ::= <A> "a" | <B>\n'
'<A> ::= "b" | <B>\n'
'<B> ::= <A> | "a"')
node = Translator().translate(tree)
expected = ('<S> ::= <A> <a> | "a" | "b"\n'
'<A> ::= "b" | "a"\n'
'<B> ::= "a" | "b"\n'
'<a> ::= "a"')
self.assertEqual(
str(node), expected,
f'\n\nExpected:\n{expected}\n\nBut Got:\n{str(node)}\n\n')
def test_external_imports_transferred_verbatim(self):
grammar = r'''
from darglint.errors import (
ItemIndentationError,
)
<start> ::= @ItemIndentationError <A>
<A> ::= "A"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
self.assertTrue(node.equals(tree))
def test_translate_retains_probability(self):
"""Make sure that probabilities are retained in the grammar."""
grammar = r'''
start: <A>
<A> ::= 70 <B> <B> <C> | 20 <B> <B> | 10 <B>
<B> ::= "B"
<C> ::= "C"
'''
tree = Parser().parse(grammar)
node = Translator().translate(tree)
expected = Parser().parse(r'''
start: <A>
<A> ::= 70 <B> <A1> | 20 <B> <B> | 10 "B"
<B> ::= "B"
<C> ::= "C"
<A1> ::= <B> <C>
''')
self.assertTrue(
node.equals(expected),
f'Expected:\n{expected}\n\nBut got:\n{node}',
)