def test_should_return_eoi(self):
with closing(io.StringIO('')) as input_stream:
tokenizer = Tokenizer(input_stream)
token = tokenizer.get_next_token()
self.assertEqual(token.id, Lexem.EOI)
token = tokenizer.get_next_token()
self.assertEqual(token.id, Lexem.EOI)
def test_should_return_number_tokens(self):
with closing(io.StringIO('<span>{{123}}{{12.3}}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>'), Token(Lexem.PRINT_OPEN), Token(Lexem.INT, 123),
Token(Lexem.PRINT_CLOSE), Token(Lexem.PRINT_OPEN),
Token(Lexem.NUMBER, 12.3), Token(Lexem.PRINT_CLOSE), Token(Lexem.HTML, '</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_string_tokens(self):
with closing(io.StringIO('<span>{{\'str1{{1+2}}b\'"str2\'/{#%"""\'\'}}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>'), Token(Lexem.PRINT_OPEN), Token(Lexem.STRING, 'str1{{1+2}}b'),
Token(Lexem.STRING, 'str2\'/{#%'), Token(Lexem.STRING, ''), Token(Lexem.STRING, ''),
Token(Lexem.PRINT_CLOSE), Token(Lexem.HTML, '</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_eoi(self):
with closing(io.StringIO('')) as input_stream:
tokenizer = Tokenizer(input_stream)
token = tokenizer.get_next_token()
self.assertEqual(token.id, Lexem.EOI)
token = tokenizer.get_next_token()
self.assertEqual(token.id, Lexem.EOI)
def test_should_return_template_token(self):
with closing(io.StringIO('{%%}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.STATEMENT_OPEN),
Token(Lexem.STATEMENT_CLOSE)
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_keyword_and_identifier_tokens(self):
with closing(io.StringIO('<span>{% if condition %}Hello World{% endif %}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>'), Token(Lexem.STATEMENT_OPEN), Token(Lexem.IF),
Token(Lexem.IDENTIFIER, 'condition'), Token(Lexem.STATEMENT_CLOSE), Token(Lexem.HTML, 'Hello World'),
Token(Lexem.STATEMENT_OPEN), Token(Lexem.ENDIF), Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, '</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_identifier_token(self):
with closing(io.StringIO('{%identifier')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.STATEMENT_OPEN),
Token(Lexem.IDENTIFIER, 'identifier')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_right_square_bracket_token(self):
with closing(io.StringIO('{{]')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.PRINT_OPEN),
Token(Lexem.RIGHT_SQUARE_BRACKET)
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_raise_incomplete_unrecognised_construction(self):
with closing(io.StringIO('<span>{{^2 + 1}}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
self.assertEqual(Token(Lexem.HTML, '<span>'),
tokenizer.get_next_token())
self.assertEqual(Token(Lexem.PRINT_OPEN),
tokenizer.get_next_token())
self.assertRaises(ParserSyntaxError, tokenizer.get_next_token)
def test_should_return_expression_tokens(self):
with closing(io.StringIO('<span>{% set var = (2+4%3.0) %}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>'), Token(Lexem.STATEMENT_OPEN), Token(Lexem.SET),
Token(Lexem.IDENTIFIER, 'var'), Token(Lexem.ASSIGN), Token(Lexem.LEFT_BRACKET),
Token(Lexem.INT, 2), Token(Lexem.PLUS), Token(Lexem.INT, 4), Token(Lexem.MOD),
Token(Lexem.NUMBER, 3), Token(Lexem.RIGHT_BRACKET), Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, '</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_string_token(self):
with closing(io.StringIO('{{"Hello World"}}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.PRINT_OPEN),
Token(Lexem.STRING, 'Hello World'),
Token(Lexem.PRINT_CLOSE)
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_omit_commented_text(self):
with closing(io.StringIO(
'<span>{#Comment {{12.3}}#}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.HTML, '<span>'),
Token(Lexem.HTML, '</span>')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_boolean_false(self):
with closing(io.StringIO('{{ False }}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.PRINT_OPEN),
Token(Lexem.FALSE),
Token(Lexem.PRINT_CLOSE)
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_statement_tokens(self):
with closing(io.StringIO('<span>{{}}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.HTML, '<span>'),
Token(Lexem.PRINT_OPEN),
Token(Lexem.PRINT_CLOSE),
Token(Lexem.HTML, '</span>')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_omit_whitespaces(self):
with closing(io.StringIO('<span>{% \n %}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.HTML, '<span>'),
Token(Lexem.STATEMENT_OPEN),
Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, '</span>')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_string_tokens(self):
with closing(
io.StringIO(
'<span>{{\'str1{{1+2}}b\'"str2\'/{#%"""\'\'}}</span>')
) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.HTML, '<span>'),
Token(Lexem.PRINT_OPEN),
Token(Lexem.STRING, 'str1{{1+2}}b'),
Token(Lexem.STRING, 'str2\'/{#%'),
Token(Lexem.STRING, ''),
Token(Lexem.STRING, ''),
Token(Lexem.PRINT_CLOSE),
Token(Lexem.HTML, '</span>')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_token_list_without_whitespaces(self):
with closing(
io.StringIO(
'<span>{% if condition %}Hello World{% endif %}</span>')
) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.HTML, '<span>'),
Token(Lexem.STATEMENT_OPEN),
Token(Lexem.IF),
Token(Lexem.IDENTIFIER, 'condition'),
Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, 'Hello World'),
Token(Lexem.STATEMENT_OPEN),
Token(Lexem.ENDIF),
Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, '</span>')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_expression_tokens(self):
with closing(io.StringIO(
'<span>{% set var = (2+4%3.0) %}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [
Token(Lexem.HTML, '<span>'),
Token(Lexem.STATEMENT_OPEN),
Token(Lexem.SET),
Token(Lexem.IDENTIFIER, 'var'),
Token(Lexem.ASSIGN),
Token(Lexem.LEFT_BRACKET),
Token(Lexem.INT, 2),
Token(Lexem.PLUS),
Token(Lexem.INT, 4),
Token(Lexem.MOD),
Token(Lexem.NUMBER, 3),
Token(Lexem.RIGHT_BRACKET),
Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, '</span>')
]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_update_position(self):
tokenizer = Tokenizer(self.input_stream)
i = 0
line_number = 1
position_number = 1
while tokenizer._source_controller.has_char():
char = tokenizer._source_controller.get_char()
msg = 'Got ' + char + ' instead of ' + self.FILE_CONTENT[i] + \
' on line: ' + str(line_number) + ' pos: ' + str(position_number)
self.assertEqual(tokenizer._source_controller.line_number,
line_number, msg)
self.assertEqual(tokenizer._source_controller.position_number,
position_number, msg)
if self.FILE_CONTENT[i] == '\n':
line_number += 1
position_number = 1
else:
position_number += 1
i += 1
def test_should_return_single_html_token(self):
with closing(io.StringIO('<span>Hello World</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>Hello World</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_greater_or_equal_token(self):
with closing(io.StringIO('{{>=')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.GE)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_left_bracket_token(self):
with closing(io.StringIO('{{(')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.LEFT_BRACKET)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_right_square_bracket_token(self):
with closing(io.StringIO('{{]')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.RIGHT_SQUARE_BRACKET)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_omit_comment(self):
with closing(io.StringIO('{##}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = []
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_string_token(self):
with closing(io.StringIO('{{"Hello World"}}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.STRING, 'Hello World'), Token(Lexem.PRINT_CLOSE)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_raise_incomplete_unrecognised_construction(self):
with closing(io.StringIO('<span>{{^2 + 1}}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
self.assertEqual(Token(Lexem.HTML, '<span>'), tokenizer.get_next_token())
self.assertEqual(Token(Lexem.PRINT_OPEN), tokenizer.get_next_token())
self.assertRaises(ParserSyntaxError, tokenizer.get_next_token)
def test_should_omit_comment(self):
with closing(io.StringIO('{##}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = []
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_omit_commented_text(self):
with closing(io.StringIO('<span>{#Comment {{12.3}}#}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>'), Token(Lexem.HTML, '</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_left_bracket_token(self):
with closing(io.StringIO('{{(')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.LEFT_BRACKET)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_greater_or_equal_token(self):
with closing(io.StringIO('{{>=')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.GE)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_omit_whitespaces(self):
with closing(io.StringIO('<span>{% \n %}</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>'), Token(Lexem.STATEMENT_OPEN), Token(Lexem.STATEMENT_CLOSE),
Token(Lexem.HTML, '</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_identifier_token(self):
with closing(io.StringIO('{%identifier')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.STATEMENT_OPEN), Token(Lexem.IDENTIFIER, 'identifier')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_boolean_false(self):
with closing(io.StringIO('{{ False }}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.FALSE), Token(Lexem.PRINT_CLOSE)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_template_token(self):
with closing(io.StringIO('{%%}')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.STATEMENT_OPEN), Token(Lexem.STATEMENT_CLOSE)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_div_token(self):
with closing(io.StringIO('{{/')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.SLASH)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_single_html_token(self):
with closing(io.StringIO('<span>Hello World</span>')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.HTML, '<span>Hello World</span>')]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_keyword_token(self):
with closing(io.StringIO('{%macro')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.STATEMENT_OPEN), Token(Lexem.MACRO)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_minus_token(self):
with closing(io.StringIO('{{-')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.PRINT_OPEN), Token(Lexem.MINUS)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))
def test_should_return_keyword_token(self):
with closing(io.StringIO('{%macro')) as input_stream:
tokenizer = Tokenizer(input_stream)
tokens = [Token(Lexem.STATEMENT_OPEN), Token(Lexem.MACRO)]
self.assertTokenListEqual(tokens, list(tokenizer.get_tokens()))