def test_3(self):
# Note that this will add a newline to the lexed output, since the
# `ensurenl <http://pygments.org/docs/lexers/>`_ option is True by
# default.
lexer = get_lexer_by_name('python')
token_iter = lex('', lexer)
# Capture both group and string for help in debugging.
token_group = list(_group_lexer_tokens(token_iter, True, False))
assert token_group == [(_GROUP.whitespace, '\n')]
def test_3(self):
# Note that this will add a newline to the lexed output, since the
# `ensurenl <http://pygments.org/docs/lexers/>`_ option is True by
# default.
lexer = get_lexer_by_name('python')
token_iter = lex('', lexer)
# Capture both group and string for help in debugging.
token_group = list(_group_lexer_tokens(token_iter, True, False))
assert token_group == [(_GROUP.whitespace, '\n')]
def test_10(self):
lexer = get_lexer_by_name('c')
token_iter = lex(self.test_c_code, lexer)
token_group = _group_lexer_tokens(token_iter, False, False)
gathered_group = _gather_groups_on_newlines(token_group, (2, 2, 2))
classified_group = list(_classify_groups(gathered_group, c_lexer))
assert classified_group == [(-1, '#include <stdio.h>\n'), (-1, '\n'),
(0, 'A multi-\n'), (0, 'line\n'),
(0, 'comment \n'), (-1, '\n'),
(-1, 'main(){\n'), (2, 'Empty.\n'),
(-1, '}\n')]
def test_10(self):
lexer = get_lexer_by_name('c')
token_iter = lex(self.test_c_code, lexer)
token_group = _group_lexer_tokens(token_iter, False, False)
gathered_group = _gather_groups_on_newlines(token_group, (2, 2, 2))
classified_group = list( _classify_groups(gathered_group, c_lexer) )
assert classified_group == [(-1, '#include <stdio.h>\n'),
(-1, '\n'),
( 0, 'A multi-\n'),
( 0, 'line\n'),
( 0, 'comment \n'),
(-1, '\n'),
(-1, 'main(){\n'),
( 2, 'Empty.\n'),
(-1, '}\n')]
def test_2(self):
lexer = get_lexer_by_name('c')
token_iter = lex(self.test_c_code, lexer)
# Capture both group and string for help in debugging.
token_group = list(_group_lexer_tokens(token_iter, False, False))
# But split the two into separate lists for unit tests.
group_list, string_list = list(zip(*token_group))
assert group_list == (
_GROUP.other, # The #include.
_GROUP.whitespace, # The space after #include.
_GROUP.other, # <stdio.h>\n
_GROUP.whitespace, # \n
_GROUP.block_comment, # The /* comment */.
_GROUP.whitespace, # Up to the code.
_GROUP.other, # main(){.
_GROUP.whitespace, # Up to the // comment.
_GROUP.inline_comment, # // commnet.
_GROUP.other, # Closing }.
_GROUP.whitespace, ) # Final \n.
def test_2(self):
lexer = get_lexer_by_name('c')
token_iter = lex(self.test_c_code, lexer)
# Capture both group and string for help in debugging.
token_group = list(_group_lexer_tokens(token_iter, False, False))
# But split the two into separate lists for unit tests.
group_list, string_list = list(zip(*token_group))
assert group_list == (
_GROUP.other, # The #include.
_GROUP.whitespace, # The space after #include.
_GROUP.other, # <stdio.h>\n
_GROUP.whitespace, # \n
_GROUP.block_comment, # The /* comment */.
_GROUP.whitespace, # Up to the code.
_GROUP.other, # main(){.
_GROUP.whitespace, # Up to the // comment.
_GROUP.inline_comment, # // commnet.
_GROUP.other, # Closing }.
_GROUP.whitespace,
) # Final \n.
def test_4(self):
lexer = get_lexer_by_name('c')
token_iter = lex(self.test_c_code, lexer)
token_group = _group_lexer_tokens(token_iter, False, False)
gathered_group = list(_gather_groups_on_newlines(token_group,
(1, 2, 2)))
expected_group = [
[(_GROUP.other, 0, '#include'),
(_GROUP.whitespace, 0, ' '),
(_GROUP.other, 0, '<stdio.h>\n')],
[(_GROUP.whitespace, 0, '\n')],
[(_GROUP.block_comment_start, 3, '/* A multi-\n')],
[(_GROUP.block_comment_body, 3, ' line\n')],
[(_GROUP.block_comment_end, 3, ' comment */'),
(_GROUP.whitespace, 0, '\n')],
[(_GROUP.whitespace, 0, '\n')],
[(_GROUP.other, 0, 'main(){'), (_GROUP.whitespace, 0, '\n')],
[(_GROUP.whitespace, 0, ' '),
(_GROUP.inline_comment, 0, '// Empty.\n')],
[(_GROUP.other, 0, '}'), (_GROUP.whitespace, 0, '\n')] ]
assert gathered_group == expected_group
def test_4(self):
lexer = get_lexer_by_name('c')
token_iter = lex(self.test_c_code, lexer)
token_group = _group_lexer_tokens(token_iter, False, False)
gathered_group = list(
_gather_groups_on_newlines(token_group, (1, 2, 2)))
expected_group = [[(_GROUP.other, 0, '#include'),
(_GROUP.whitespace, 0, ' '),
(_GROUP.other, 0, '<stdio.h>\n')],
[(_GROUP.whitespace, 0, '\n')],
[(_GROUP.block_comment_start, 3, '/* A multi-\n')],
[(_GROUP.block_comment_body, 3, ' line\n')],
[(_GROUP.block_comment_end, 3, ' comment */'),
(_GROUP.whitespace, 0, '\n')],
[(_GROUP.whitespace, 0, '\n')],
[(_GROUP.other, 0, 'main(){'),
(_GROUP.whitespace, 0, '\n')],
[(_GROUP.whitespace, 0, ' '),
(_GROUP.inline_comment, 0, '// Empty.\n')],
[(_GROUP.other, 0, '}'),
(_GROUP.whitespace, 0, '\n')]]
assert gathered_group == expected_group
