def testProcessedTokenMatcher_Bygroups(self):
matcher = (r'"f(oo)(bar)"', ('bygroups', (pygments.token.Comment,
pygments.token.String)))
processed = extract._ProcessedTokenMatcher(matcher, None)
self.assertEqual(
'TokenActions.byGroups(Token.COMMENT, Token.LITERAL_STRING)',
processed.token_action)
def testProcessedTokenMatcher(self):
matcher = (r'"fo\\o"', pygments.token.Comment)
processed = extract._ProcessedTokenMatcher(matcher, None)
self.assertEqual(r'\"fo\\\\o\"', processed.regex)
self.assertEqual('TokenActions.singleToken(Token.COMMENT)',
processed.token_action)
self.assertEqual('StateActions.NOOP', processed.state_action)
matcher = (r'foo', pygments.token.Comment, (-1,))
processed = extract._ProcessedTokenMatcher(matcher, None)
self.assertEqual('StateActions.pop(1)', processed.state_action)
matcher = (r'foo', pygments.token.Comment, ('root', 'import'))
processed = extract._ProcessedTokenMatcher(matcher, None)
self.assertEqual('StateActions.multiple(StateActions.push(State.ROOT), '
'StateActions.push(State.IMPORT))',
processed.state_action)
def testProcessedTokenMatcher_Using(self):
matcher = (r'"f(oo)(bar)"', ('bygroups', (pygments.token.Comment,
('using', (PythonLexer,)),
pygments.token.String)))
processed = extract._ProcessedTokenMatcher(matcher, None)
self.assertEqual(
'TokenActions.byGroups(TokenActions.singleToken(Token.COMMENT), '
'PythonSyntax.USING_THIS, '
'TokenActions.singleToken(Token.LITERAL_STRING))',
processed.token_action)
def testRegexConversion(self):
dummy = extract._ProcessedTokenMatcher(('', pygments.token.Comment), None)
def Check(transformed, initial):
self.assertEqual(transformed, dummy._ProcessRegex(initial))
Check('foo', 'foo')
Check(r'f\"o\"\\\\o', r'f"o"\\o')
# Various nasty character classes that contain punctuation.
Check(r'[\\]{}:(),;\\[]', '[]{}:(),;[]')
Check(r'\\[', r'[\[]')
Check(r'\\[', '[[]')
Check(r'[\\]\\[a]', '[][a]')
Check('(a)', '(a)')
Check(r'\\{foo.*\\}', '{foo.*}')
Check(r'\\{foo3foo\\}', '{foo3foo}')
Check('foo{1,2}bar{1,3}', r'foo{1,2}bar{1,3}')
Check('a{3}', 'a{3}')
Check('a{3,5}', 'a{3,5}')
Check(r'foobar\\{baz', 'foobar{baz')
Check(r'foobar\\]baz', 'foobar]baz')
Check(r'foo\\)barbaz', r'foo\)barbaz')
Check(r'\\(', r'\(')
Check(r'\\n', r'\n')
def testProcessedTokenMatcher_UsingThis(self):
matcher = (r'"f(oo)(bar)"', ('using', (pygments.lexer.this,)))
processed = extract._ProcessedTokenMatcher(matcher, RecordingPythonLexer())
self.assertEqual('USING_THIS', processed.token_action)
