def test_iter_interface(self):
ts = TokenStream(FakeStream(
[FirstToken('Test'), WordToken('sentence'), PeriodToken()]))
self.assertIs(ts, iter(ts))
self.assertEqual(FirstToken('Test'), next(ts))
self.assertEqual(WordToken('sentence'), next(ts))
self.assertEqual(PeriodToken(), next(ts))
with self.assertRaises(StopIteration):
next(ts)
def test_ends_with_dot_period(self):
has_period = Sentence(
[FirstToken('Short'),
WordToken('sentence'),
PeriodToken()])
self.assertTrue(has_period.ends_with_dot())
no_period = Sentence([FirstToken('Word')])
self.assertFalse(no_period.ends_with_dot())
super_has_period = Sentence([FirstToken('Run'), has_period])
self.assertTrue(super_has_period.ends_with_dot())
super_no_period = Sentence([FirstToken('Run'), no_period])
self.assertFalse(super_no_period.ends_with_dot())
def test_equal_operator(self):
left = Sentence([
FirstToken('Height'),
WordToken('of'),
WordToken('box'),
PeriodToken()
])
right = Sentence([
FirstToken('Height'),
WordToken('of'),
WordToken('box'),
PeriodToken()
])
self.assertTrue(left == right)
def test_parse_signature(self):
# Define New value. to be Five. .
parser = fake_parser([
FirstToken('Define'), FirstToken('New'),
WordToken('value'), PeriodToken('.'), WordToken('to'),
WordToken('be'), FirstToken('Five'),
PeriodToken('.'), PeriodToken('.'),
])
sig = parser.parse_signature()
self.assertIsInstance(sig, Sentence)
self.assertIsInstance(sig._children[1], Sentence)
self.assertIsInstance(sig._children[4], Sentence)
self.assertEqual(sig._children[0].text, 'Define')
self.assertIsInstance(sig._children[5], PeriodToken)
self.assertEqual(sig._children[1]._children[0].text, 'New')
def test_file_token_stream(self):
with tempfile.NamedTemporaryFile() as file:
file.write(b'''Hello world.
This is a test.''')
file.seek(0)
tokens = list(file_token_stream(file.name))
self.assertEqual([
FirstToken('Hello'),
WordToken('world'),
PeriodToken('.'),
FirstToken('This'),
WordToken('is'),
WordToken('a'),
WordToken('test'),
PeriodToken('.'),
], tokens)
def test_match_sentence(self):
pattern = [FirstToken('Test'), WordToken('value')]
to_match = Sentence(pattern + [PeriodToken()])
code = None
test_def = Definition(pattern, code)
scope = Scope(None)
scope.add_definition(test_def)
self.assertIs(test_def, scope.match_sentence(to_match))
def test_has_define_function(self):
scope = create_built_in_scope()
ptr = scope.new_matcher()
ptr.next(FirstToken('Define'))
ptr.next()
ptr.next(WordToken('to'))
ptr.next(WordToken('be'))
ptr.next()
self.assertTrue(ptr.has_end())
self.assertIsInstance(
scope.match_sentence(
string_to_signature('Define Abc. to be Xyz. .')), Definition)
def test_nested_signature(self):
result = string_to_signature('Define Head. to be Body. .')
self.assertEqual(FirstToken('Head'), result[1][0])
self.assertEqual(PeriodToken(), result[1][1])
self.assertEqual(WordToken('to'), result[2])
def test_flat_signature(self):
result = string_to_signature('Frames per second.')
self.assertEqual(FirstToken('Frames'), result[0])
self.assertEqual(WordToken('per'), result[1])
self.assertEqual(WordToken('second'), result[2])
self.assertEqual(PeriodToken(), result[3])
def test_push_back(self):
ts = TokenStream(FakeStream([PeriodToken()]))
ts.push_back(FirstToken('Test'))
self.assertEqual(FirstToken('Test'), next(ts))
self.assertEqual(PeriodToken(), next(ts))
def test_text_token_stream(self):
tokens = list(text_token_stream(' Hello world. '))
self.assertEqual(
[FirstToken('Hello'),
WordToken('world'),
PeriodToken('.')], tokens)
def test_token_check(self):
with self.assertRaises(ValueError):
FirstToken('following')
with self.assertRaises(ValueError):
IntegerToken('101\n')
def test_token_repr(self):
self.assertEqual("PeriodToken()", repr(PeriodToken()))
self.assertEqual("FirstToken('First')", repr(FirstToken('First')))
self.assertEqual("WordToken('second')", repr(WordToken('second')))
self.assertEqual("DefineToken()", repr(DefineToken('Define')))
self.assertEqual("IntegerToken('101')", repr(IntegerToken('101')))