def test_syntax_error_exception_SHOULD_be_raised_WHEN_look_ahead_token_has_invalid_syntax(
self):
source = '"A'
# ACT #
ts = sut.TokenStream(source)
with self.assertRaises(TokenSyntaxError):
ts.consume()
def test_no_token_on_remaining_part_of_current_line(self):
test_cases = [
'',
' ',
]
for first_line in test_cases:
with self.subTest(msg=repr(first_line)):
ts = sut.TokenStream(first_line)
assert_is_null(self, ts)
def test_invalid_token(self):
test_cases = [
'\'missing end single quote',
]
for first_line in test_cases:
with self.subTest(msg=repr(first_line)):
ts = sut.TokenStream(first_line)
self.assertTrue(ts.is_null, 'is_null')
self.assertIs(LookAheadState.SYNTAX_ERROR, ts.look_ahead_state,
'look_ahead_state')
def test_remaining_part_of_current_line(self):
test_cases = [
('a', 'a'),
('b ', 'b '),
('c ', 'c '),
('d\n_', 'd'),
('d \n_', 'd '),
]
for source, expected in test_cases:
with self.subTest(msg=repr(source)):
ts = sut.TokenStream(source)
actual = ts.remaining_part_of_current_line
self.assertEqual(expected, actual)
def test_invalid_quoting(self):
test_cases = [
'\'missing end single quote',
'missing_start_single_quote\'',
'\"missing end double quote',
'missing_start_double_quote\"',
]
for first_line in test_cases:
with self.subTest(msg=repr(first_line)):
token_stream = sut.TokenStream(first_line)
self.assertIs(LookAheadState.SYNTAX_ERROR,
token_stream.look_ahead_state,
'look_ahead_state')
with self.assertRaises(TokenSyntaxError):
token_stream.consume()
def test_remaining_part_of_current_line_after_consumption_of_first_token(
self):
test_cases = [
('a', ''),
('b ', ''),
('c ', ' '),
('a\n_', ''),
('a b', 'b'),
('d b\n_', 'b'),
]
for source, expected in test_cases:
with self.subTest(msg=repr(source)):
ts = sut.TokenStream(source)
ts.consume()
actual = ts.remaining_part_of_current_line
self.assertEqual(expected, actual)
def test_syntax_error_of_look_ahead_token(self):
test_cases = [
('a "A', assert_plain('a'), '"A'),
]
for source, expected_token, remaining_source in test_cases:
with self.subTest(msg=repr(source)):
# ACT #
ts = sut.TokenStream(source)
consumed_token = ts.consume()
# ASSERT #
self.assertTrue(ts.is_null, 'is_null')
self.assertIs(LookAheadState.SYNTAX_ERROR, ts.look_ahead_state,
'look_ahead_state')
expected_token.apply_with_message(self, consumed_token,
'consumed token')
self.assertEqual(remaining_source, ts.remaining_source,
'remaining_source')
def test_single_token(self):
test_cases = [
('a', assert_plain('a'), ''),
('b ', assert_plain('b'), ''),
('c ', assert_plain('c'), ' '),
('x \n', assert_plain('x'), '\n'),
('x\n', assert_plain('x'), '\n'),
]
for source, expected_token, remaining_source in test_cases:
with self.subTest(msg=repr(source)):
ts = sut.TokenStream(source)
# ACT #
consumed_token = ts.consume()
# ASSERT #
expected_token.apply_with_message(self, consumed_token,
'token')
assert_is_null(self, ts)
self.assertEqual(remaining_source, ts.remaining_source,
'remaining_source')
def test_valid_token(self):
test_cases = [
(
'token',
assert_plain('token'),
),
(
'<<->> other_token',
assert_plain('<<->>'),
),
]
for first_line, token_assertion in test_cases:
with self.subTest(msg=repr(first_line)):
ts = sut.TokenStream(first_line)
token_assertion.apply_with_message(self, ts.head, 'token')
assert_is_not_null(self, ts)
actual_remaining_source = first_line[ts.position:]
self.assertEqual(
first_line, actual_remaining_source,
'remaining source according to the "position" attribute')
def test_multiple_tokens(self):
test_cases = [
('a A', assert_plain('a'), 'A', 'A'),
('b B', assert_plain('b'), 'B', ' B'),
('c C ', assert_plain('c'), 'C', ' C '),
('d D ', assert_plain('d'), 'D', ' D '),
('a A\n_', assert_plain('a'), 'A', 'A\n_'),
]
for source, expected_token, second_token, remaining_source in test_cases:
with self.subTest(msg=repr(source)):
# ACT #
ts = sut.TokenStream(source)
consumed_token = ts.consume()
# ASSERT #
assert_is_not_null(self, ts)
expected_token.apply_with_message(self, consumed_token,
'consumed token')
self.assertEqual(second_token, ts.head.string, 'second token')
self.assertEqual(remaining_source, ts.remaining_source,
'remaining_source')
def test_valid_token(self):
test_cases = [
(
'token',
assert_plain('token'),
),
(
' token_preceded_by_space',
assert_plain('token_preceded_by_space'),
),
(
'token_followed_by_space ',
assert_plain('token_followed_by_space'),
),
(
'token_followed_by_2_space ',
assert_plain('token_followed_by_2_space'),
),
(
'token_followed_by_other_token other_token',
assert_plain('token_followed_by_other_token'),
),
(
'token_followed_by_other_token_with_2_space_between other_token',
assert_plain(
'token_followed_by_other_token_with_2_space_between'),
),
(
'<<->> other_token',
assert_plain('<<->>'),
),
(
'\'single quoted\'',
assert_quoted('single quoted', '\'single quoted\''),
),
(
'\"double quoted\"',
assert_quoted('double quoted', '\"double quoted\"'),
),
(
' \'quoted preceded by space\'',
assert_quoted('quoted preceded by space',
'\'quoted preceded by space\''),
),
(
' \'quoted followed by space\' ',
assert_quoted('quoted followed by space',
'\'quoted followed by space\''),
),
(
' \'quoted token followed by other token\' \'other_token\'',
assert_quoted('quoted token followed by other token',
'\'quoted token followed by other token\''),
),
]
for first_line, token_assertion in test_cases:
with self.subTest(msg=repr(first_line)):
ts = sut.TokenStream(first_line)
token_assertion.apply_with_message(self, ts.head, 'token')
self.assertEqual(first_line, ts.remaining_source,
'remaining source')
assert_is_not_null(self, ts)
def test(self):
test_cases = [
# Single token
('a', 'a',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(1),
remaining_source=asrt.equals(''))),
('b ', 'b ',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(2),
remaining_source=asrt.equals(''))),
('x \n', 'x ',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(2),
remaining_source=asrt.equals('\n'))),
('x\ny', 'x',
assert_token_stream(head_token=assert_plain('y'),
look_ahead_state=asrt.is_not(
LookAheadState.NULL),
position=asrt.equals(1),
remaining_source=asrt.equals('\ny'))),
('x\n y', 'x',
assert_token_stream(head_token=assert_plain('y'),
look_ahead_state=asrt.is_not(
LookAheadState.NULL),
position=asrt.equals(1),
remaining_source=asrt.equals('\n y'))),
# Multiple tokens
('a A', 'a A',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(3),
remaining_source=asrt.equals(''))),
('a A\nb B', 'a A',
assert_token_stream(head_token=assert_plain('b'),
look_ahead_state=asrt.is_not(
LookAheadState.NULL),
position=asrt.equals(3),
remaining_source=asrt.equals('\nb B'))),
('a A\ninvalid_token"', 'a A',
assert_token_stream(
look_ahead_state=asrt.is_(LookAheadState.SYNTAX_ERROR),
is_null=asrt.is_true,
position=asrt.equals(3),
remaining_source=asrt.equals('\ninvalid_token"'))),
# No tokens
('', '',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(0),
remaining_source=asrt.equals(''))),
(' ', ' ',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(1),
remaining_source=asrt.equals(''))),
(' \n', ' ',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(1),
remaining_source=asrt.equals('\n'))),
(' \n ', ' ',
assert_token_stream(is_null=asrt.is_true,
look_ahead_state=asrt.is_(
LookAheadState.NULL),
position=asrt.equals(1),
remaining_source=asrt.equals('\n '))),
(' \n"invalid quoting', ' ',
assert_token_stream(
look_ahead_state=asrt.is_(LookAheadState.SYNTAX_ERROR),
position=asrt.equals(1),
remaining_source=asrt.equals('\n"invalid quoting'))),
]
for source, expected_consumed_string, token_stream_assertion in test_cases:
with self.subTest(msg=repr(source)):
ts = sut.TokenStream(source)
# ACT #
actual_consumed_string = ts.consume_remaining_part_of_current_line_as_string(
)
# ASSERT #
self.assertEqual(expected_consumed_string,
actual_consumed_string, 'consumed string')
token_stream_assertion.apply_with_message(
self, ts, 'token stream after parse')