def test_token_first_pos(self):
for value in (TokenValue("test"), Item()):
create_token = token_factory()
token = create_token("OTHER", value, 0)
self.handle(token, "test")
self.assertEqual(token.value.head, "")
self.assertEqual(token.value.tail, "")
self.assertEqual(token.value.pos, 0)
self.assertEqual(token.value.size, 4)
def test_separator_first_in_head(self):
for value in (TokenValue("test"), Item()):
create_token = token_factory()
self.handle(create_token("SEPARATOR", "\t", 0), "\t")
token = create_token("OTHER", value, 1)
self.handle(token, "test")
self.assertEqual(token.value.head, "\t")
self.assertEqual(token.value.tail, "")
self.assertEqual(token.value.pos, 1)
self.assertEqual(token.value.size, 4)
def test_separator_last_elt_none(self):
# this is a robustness test
for value in (TokenValue("test"), Item()):
create_token = token_factory()
self.handle(create_token("SEPARATOR", "\t", 0), "\t")
self.handle(create_token("SEPARATOR", "\n", 1), "\n")
token = create_token("OTHER", value, 2)
self.handle(token, "test")
self.assertEqual(token.value.head, "\t")
self.assertEqual(token.value.tail, "")
self.assertEqual(token.value.pos, 2)
self.assertEqual(token.value.size, 4)
def test_simple_token_tail(self):
create_token = token_factory()
a = create_token("OTHER", TokenValue("a"), 0)
b = create_token("OTHER", Item(), 3)
c = create_token("OTHER", TokenValue("c"), 5)
self.handle(a, "a")
self.handle(create_token("SEPARATOR", " \t", 1), "\t")
self.handle(b, "")
self.handle(create_token("SEPARATOR", "\r", 4), "\r")
self.handle(c, "c")
self.assertEqual(a.value.head, "")
self.assertEqual(a.value.tail, " \t")
self.assertEqual(a.value.pos, 0)
self.assertEqual(a.value.size, 1)
self.assertEqual(b.value.head, "")
self.assertEqual(b.value.tail, "\r")
self.assertEqual(b.value.pos, 3)
self.assertEqual(b.value.size, 0)
self.assertEqual(c.value.head, "")
self.assertEqual(c.value.tail, "")
self.assertEqual(c.value.pos, 5)
self.assertEqual(c.value.size, 1)
def test_tail_at_end(self):
create_token = token_factory()
a = create_token("OTHER", TokenValue("a"), 0)
b = create_token("OTHER", Item(), 3)
self.handle(a, "a")
self.handle(create_token("SEPARATOR", " \t", 1), "\t")
self.handle(b, "")
# finish with a separator
self.handle(create_token("SEPARATOR", "\r", 4), "\r")
self.assertEqual(a.value.head, "")
self.assertEqual(a.value.tail, " \t")
self.assertEqual(a.value.pos, 0)
self.assertEqual(a.value.size, 1)
self.assertEqual(b.value.head, "")
self.assertEqual(b.value.tail, "\r")
self.assertEqual(b.value.pos, 3)
self.assertEqual(b.value.size, 0)
def test_tokenvalue_head_tail_pos(self):
t = TokenValue("foo")
self.assertEqual(t.pos, None)
self.assertEqual(t.head, "")
self.assertEqual(t.tail, "")
def test_tokenvalue_str(self):
self.assertEqual(repr(TokenValue("")), "TokenValue()")
self.assertEqual(str(TokenValue("")), "")
self.assertEqual(repr(TokenValue("foo")), "TokenValue(foo)")
self.assertEqual(str(TokenValue("foo")), "foo")