def test_iter_interface(self): ts = TokenStream(FakeStream( [FirstToken('Test'), WordToken('sentence'), PeriodToken()])) self.assertIs(ts, iter(ts)) self.assertEqual(FirstToken('Test'), next(ts)) self.assertEqual(WordToken('sentence'), next(ts)) self.assertEqual(PeriodToken(), next(ts)) with self.assertRaises(StopIteration): next(ts)
def test_equal_operator(self): left = Sentence([ FirstToken('Height'), WordToken('of'), WordToken('box'), PeriodToken() ]) right = Sentence([ FirstToken('Height'), WordToken('of'), WordToken('box'), PeriodToken() ]) self.assertTrue(left == right)
def test_parse_signature(self): # Define New value. to be Five. . parser = fake_parser([ FirstToken('Define'), FirstToken('New'), WordToken('value'), PeriodToken('.'), WordToken('to'), WordToken('be'), FirstToken('Five'), PeriodToken('.'), PeriodToken('.'), ]) sig = parser.parse_signature() self.assertIsInstance(sig, Sentence) self.assertIsInstance(sig._children[1], Sentence) self.assertIsInstance(sig._children[4], Sentence) self.assertEqual(sig._children[0].text, 'Define') self.assertIsInstance(sig._children[5], PeriodToken) self.assertEqual(sig._children[1]._children[0].text, 'New')
def test_empty(self): ts = TokenStream(FakeStream([PeriodToken()])) self.assertFalse(ts.is_empty()) self.assertTrue(ts.not_empty()) next(ts) self.assertTrue(ts.is_empty()) self.assertFalse(ts.not_empty())
def test_file_token_stream(self): with tempfile.NamedTemporaryFile() as file: file.write(b'''Hello world. This is a test.''') file.seek(0) tokens = list(file_token_stream(file.name)) self.assertEqual([ FirstToken('Hello'), WordToken('world'), PeriodToken('.'), FirstToken('This'), WordToken('is'), WordToken('a'), WordToken('test'), PeriodToken('.'), ], tokens)
def test_match_sentence(self): pattern = [FirstToken('Test'), WordToken('value')] to_match = Sentence(pattern + [PeriodToken()]) code = None test_def = Definition(pattern, code) scope = Scope(None) scope.add_definition(test_def) self.assertIs(test_def, scope.match_sentence(to_match))
def test_ends_with_dot_period(self): has_period = Sentence( [FirstToken('Short'), WordToken('sentence'), PeriodToken()]) self.assertTrue(has_period.ends_with_dot()) no_period = Sentence([FirstToken('Word')]) self.assertFalse(no_period.ends_with_dot()) super_has_period = Sentence([FirstToken('Run'), has_period]) self.assertTrue(super_has_period.ends_with_dot()) super_no_period = Sentence([FirstToken('Run'), no_period]) self.assertFalse(super_no_period.ends_with_dot())
def test_nested_signature(self): result = string_to_signature('Define Head. to be Body. .') self.assertEqual(FirstToken('Head'), result[1][0]) self.assertEqual(PeriodToken(), result[1][1]) self.assertEqual(WordToken('to'), result[2])
def test_flat_signature(self): result = string_to_signature('Frames per second.') self.assertEqual(FirstToken('Frames'), result[0]) self.assertEqual(WordToken('per'), result[1]) self.assertEqual(WordToken('second'), result[2]) self.assertEqual(PeriodToken(), result[3])
def test_push_back_overload(self): ts = TokenStream(FakeStream([])) ts.push_back(PeriodToken()) with self.assertRaises(ValueError): ts.push_back(PeriodToken())
def test_push_back(self): ts = TokenStream(FakeStream([PeriodToken()])) ts.push_back(FirstToken('Test')) self.assertEqual(FirstToken('Test'), next(ts)) self.assertEqual(PeriodToken(), next(ts))
def test_text_token_stream(self): tokens = list(text_token_stream(' Hello world. ')) self.assertEqual( [FirstToken('Hello'), WordToken('world'), PeriodToken('.')], tokens)
def test_token_repr(self): self.assertEqual("PeriodToken()", repr(PeriodToken())) self.assertEqual("FirstToken('First')", repr(FirstToken('First'))) self.assertEqual("WordToken('second')", repr(WordToken('second'))) self.assertEqual("DefineToken()", repr(DefineToken('Define'))) self.assertEqual("IntegerToken('101')", repr(IntegerToken('101')))