def test_removes_unreachable_symbols(self): grammar = r''' start: <head> <head> ::= <a> <b> <a> ::= "-" | ε <b> ::= <a> <d> <d> ::= "1" <c> ::= "Q" ''' tree = Parser().parse(grammar) Translator().translate(tree) for node in tree.filter(lambda x: x.value == 'c'): self.fail('Expected unused expressions to be removed.')
def test_external_filename_preserved_in_both_python_and_bnf(self): external = ('from darglint.parse.identifiers import (\n' ' ArgumentIdentifier,\n' ' NoqaIdentifier,\n' ')') grammar = f''' {external} <A> ::= "A" ''' node = Parser().parse(grammar) self.assertTrue(external in str(node)) self.assertTrue(external in node.to_python())
def test_with_name(self): grammar = '\n'.join([ 'import base.bnf', '', 'Grammar: MySpecialGrammar', '', '<A> ::= "SPECIAL"' ]) tree = Parser().parse(grammar) name = next(tree.filter(Node.is_name)) self.assertEqual( name.value, 'MySpecialGrammar', ) py_repr = tree.to_python() self.assertTrue('class MySpecialGrammar' in py_repr)
def test_parse_rule_with_spaces(self): """Test conjunction in rules.""" node = Parser().parse('<heading> ::= <ident> ":"') self.assertEqual( str(node), '<heading> ::= <ident> ":"', )
def test_complete_conversion(self): """Test converting a complete example, and ensuring it's in CNF.""" grammar = r''' <Expr> ::= <Term> | <Expr> <AddOp> <Term> | <AddOp> <Term> <Term> ::= <Factor> | <Term> <MulOp> <Factor> <Factor> ::= <Primary> | <Factor> "\^" <Primary> <Primary> ::= <number> | <variable> | "\(" <Expr> "\)" <AddOp> ::= "\+" | "\-" <MulOp> ::= "\*" | "\/" <number> ::= <digit> | <digit> <number> <digit> ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" <variable> ::= "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) self.assertTrue(Validator(raise_exception=True).validate(node))
class ValidateCnfTests(TestCase): def setUp(self): self.parser = Parser() def _p(self, grammar): return self.parser.parse(grammar) def test_terminal_production(self): """Make sure we a terminal production passes.""" valid_productions = [ '<COLON> ::= ":"', '<PARAM> ::= "Args"', '<QUOTE> ::= "\\"\\"\\""', # '<PERIOD> ::= "."', ] for production in valid_productions: self.assertTrue(Validator().validate(self._p(production)), '"{}" should not be valid'.format(production)) def test_escaped_characters_okay(self): """Make sure that special characters are okay if escaped.""" for c in ', +*()[]|': self.assertTrue( Validator().validate( self._p('<SOMETHING> ::= "A\\{}B"'.format(c))), 'Escaping "{}" should allow it.'.format(c), )
def test_translate_retains_annotations_up(self): """Make sure we retain annotations when moving up.""" grammar = r''' start: <phone-number> <phone-number> ::= <confusion-number> <confusion-number> ::= @ConfusionError <number-group> <dot> <confusion-number> | @ConfusionError <number-group> <dash> <confusion-number> | <number-group> <number-group> ::= <number> <number-group> | <number> <number> ::= "PN\.NUMBER" <dash> ::= "PN\.DASH" <dot> ::= "PN\.DOT" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) encountered_annotation = False for child in node.walk(): if child.node_type in [NodeType.ANNOTATION, NodeType.ANNOTATIONS]: encountered_annotation = True self.assertTrue(encountered_annotation, )
def test_no_unnamed_nodes(self): """Certain nodes, when translated, are missing names.""" grammar = r''' Grammar: ArgumentsGrammar start: <arguments-section> <arguments-section> ::= <ahead> <line> # Causes an missing production name. | <ahead> # Simplified the remaning grammar. <line> ::= "TokenType\.LINE" <ahead> ::= "TokenType\.AHEAD" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) python = node.to_python() unacceptable_pattern = re.compile(r'P\([^"]') self.assertEqual( unacceptable_pattern.findall(python), [], 'Found production which doesn\'t begin with a name:\n{}'.format( python), )
def test_remove_single_unit_production(self): """Make sure we can at least remove a single unit production.""" tree = Parser().parse('<A> ::= <B>\n' '<B> ::= "moch"') node = Translator().translate(tree) expected = ('<A> ::= "moch"\n' '<B> ::= "moch"') self.assertEqual( str(node), expected, f'\n\nExpected:\n{expected}\n\nBut Got:\n{str(node)}\n\n')
def test_parse_simple_rule(self): """Make sure a simple, terminal rule works.""" node = Parser().parse('<args> ::= "Args"') self.assertTrue(isinstance(node, Node)) self.assertEqual( str(node), '<args> ::= "Args"', )
def test_eliminate_simplest_epsilon_form(self): tree = Parser().parse('<E> ::= "e" | ε') node = Translator().translate(tree) expected = '<E> ::= "e"' self.assertEqual( str(node), expected, )
def test_removes_empty_productions_after_epsilon_elimination(self): tree = Parser().parse('<A> ::= "a"\n' '<B> ::= ε') node = Translator().translate(tree) expected = '<A> ::= "a"' self.assertEqual( str(node), expected, )
def test_start_symbol_not_reassigned(self): """Make sure the start symbol points to the start terminal.""" tree = Parser().parse('start: <A>\n' '<A> ::= <B>\n' '<B> ::= "\\."') node = Translator().translate(tree) expected = 'start: <A>\n<A> ::= "\\."' self.assertEqual( str(node), expected, f'\n\nExpected:\n{expected}\n\nBut Got:\n{str(node)}\n\n')
def test_translate_retains_start_annotation(self): """Make sure that an annotation on start is saved.""" grammar = r''' @Q <start> ::= <A> <B> <B> ::= "b" <A> ::= "a" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) expected = Parser().parse(r''' @Q <start> ::= <A> <B> <B> ::= "b" <A> ::= "a" ''') self.assertTrue(node.equals(expected), f'{node}\n\n{expected}')
def test_nonsolitary_terminals_symbol_taken(self): """Make sure non-solitary teminals will have unique name.""" tree = Parser().parse('<arg-header> ::= <arg> ":"\n' '<C> ::= "Another_value"') node = Translator().translate(tree) self.assertEqual( str(node), '<arg-header> ::= <arg> <C0>\n' '<C> ::= "Another_value"\n' '<C0> ::= ":"')
def test_nodes_not_leading_to_terminals_removed(self): grammar = r''' start: <a> <a> ::= "a" | <b> <b> ::= <c> <d> ''' tree = Parser().parse(grammar) node = Translator().translate(tree) self.assertFalse(list(node.filter(lambda x: x.value == 'c')))
def test_translate_retains_annotations(self): """Make sure that annotations are retained with the grammar.""" grammar = r''' <A> ::= <B> <C> <B> ::= <C> | @Q <D> <C> ::= "C" <D> ::= "D" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) expected = Parser().parse(r''' <A> ::= <B> <C> <B> ::= "C" | @Q "D" <C> ::= "C" <D> ::= "D" ''') self.assertTrue(node.equals(expected), f'Expected:\n{expected}\n\nBut got:\n{node}')
def test_parse_from_import(self): grammar = ''' from darglint.errors import ( ItemIndentationError, ) <A> ::= "A" ''' tree = Parser().parse(grammar) self.assertTrue(tree)
def test_translate_works_with_annotation_on_nonterminal(self): grammar = r''' <A> ::= "b" | @Q <B> <C> <B> ::= "b" <C> ::= <C> ''' tree = Parser().parse(grammar) Translator().translate(tree)
def test_parse_multiple_annotations(self): """Make sure a production can have multiple annotations.""" grammar = '@VERBLESS\n@PUNCT\n<sentence> ::= <period> <noun>' tree = Parser().parse(grammar) production = tree.children[0] annotations = production.children[0] self.assertEqual( ['VERBLESS', 'PUNCT'], [x.value for x in annotations.children], )
def test_failing_terminal_parse(self): """Make sure this particular instance, which failed, passes.""" value = ('<S> ::= <A> "a" | <B>\n' '<A> ::= "b" | <B>\n' '<B> ::= <A> | "a"') node = Parser().parse(value) self.assertEqual( value, str(node), )
def test_mix_terminals_and_nonterminals(self): """Make sure we can mix terminals and non-terminals.""" values = [ '<S> ::= <A> "a" | <B>', '<S> ::= "a" <A> | <B>', '<S> ::= "a" | <B>', ] for value in values: node = Parser().parse(value) self.assertEqual(str(node), value)
def test_eliminate_epsilon_forms(self): """Make sure we can eliminate all ε-rules.""" tree = Parser().parse('<S> ::= <A> <B>\n' '<B> ::= "b"\n' '<A> ::= "a" | ε') node = Translator().translate(tree) expected = ('<S> ::= <A> <B> | "b"\n' '<B> ::= "b"\n' '<A> ::= "a"') self.assertEqual( str(node), expected, )
def test_nonsolitary_terminals(self): """Make sure non-solitary terminals are factored out.""" tree = Parser().parse('<arg-header> ::= <arg> ":"') node = Translator().translate(tree) self.assertEqual( str(node), '\n'.join([ '<arg-header> ::= <arg> <C>', '<C> ::= ":"', ]), )
def test_parse_probability(self): grammar = ''' <start> ::= 90 <A> | 10 <B> <A> ::= "A" <B> ::= "B" ''' tree = Parser().parse(grammar) self.assertTrue(tree)
def test_translate_with_recursion(self): grammar = r''' <start> ::= <b> <b> ::= <c> <b> | <c> <c> ::= "A" ''' tree = Parser().parse(grammar) Translator().translate(tree)
def test_with_imports(self): grammar = ('import base.bnf\n' 'import utils.bnf\n' '\n' 'start: <sentence>\n' '<setence> ::= <verb> <noun>\n' '<verb> ::= "TT\\.VERB"\n' '<noun> ::= "TT\\.NOUN"') node = Parser().parse(grammar) self.assertEqual(grammar, str(node), f'\nExpected:\n{grammar}\n\nBut got:\n{node}')
def test_remove_complex_unit_production(self): tree = Parser().parse('<S> ::= <A> "a" | <B>\n' '<A> ::= "b" | <B>\n' '<B> ::= <A> | "a"') node = Translator().translate(tree) expected = ('<S> ::= <A> <a> | "a" | "b"\n' '<A> ::= "b" | "a"\n' '<B> ::= "a" | "b"\n' '<a> ::= "a"') self.assertEqual( str(node), expected, f'\n\nExpected:\n{expected}\n\nBut Got:\n{str(node)}\n\n')
def test_external_imports_transferred_verbatim(self): grammar = r''' from darglint.errors import ( ItemIndentationError, ) <start> ::= @ItemIndentationError <A> <A> ::= "A" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) self.assertTrue(node.equals(tree))
def test_translate_retains_probability(self): """Make sure that probabilities are retained in the grammar.""" grammar = r''' start: <A> <A> ::= 70 <B> <B> <C> | 20 <B> <B> | 10 <B> <B> ::= "B" <C> ::= "C" ''' tree = Parser().parse(grammar) node = Translator().translate(tree) expected = Parser().parse(r''' start: <A> <A> ::= 70 <B> <A1> | 20 <B> <B> | 10 "B" <B> ::= "B" <C> ::= "C" <A1> ::= <B> <C> ''') self.assertTrue( node.equals(expected), f'Expected:\n{expected}\n\nBut got:\n{node}', )