class BashTest(unittest.TestCase): def setUp(self): self.lexer = BashLexer() self.maxDiff = None def testCurlyNoEscapeAndQuotes(self): fragment = u'echo "${a//["b"]/}"\n' tokens = [ (Token.Name.Builtin, u'echo'), (Token.Text, u' '), (Token.Literal.String.Double, u'"'), (Token.String.Interpol, u'${'), (Token.Name.Variable, u'a'), (Token.Punctuation, u'//['), (Token.Literal.String.Double, u'"b"'), (Token.Punctuation, u']/'), (Token.String.Interpol, u'}'), (Token.Literal.String.Double, u'"'), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testCurlyWithEscape(self): fragment = u'echo ${a//[\\"]/}\n' tokens = [ (Token.Name.Builtin, u'echo'), (Token.Text, u' '), (Token.String.Interpol, u'${'), (Token.Name.Variable, u'a'), (Token.Punctuation, u'//['), (Token.Literal.String.Escape, u'\\"'), (Token.Punctuation, u']/'), (Token.String.Interpol, u'}'), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testParsedSingle(self): fragment = u"a=$'abc\\''\n" tokens = [ (Token.Name.Variable, u'a'), (Token.Operator, u'='), (Token.Literal.String.Single, u"$'abc\\''"), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class BashTest(unittest.TestCase): def setUp(self): self.lexer = BashLexer() self.maxDiff = None def testCurlyNoEscapeAndQuotes(self): fragment = u'echo "${a//["b"]/}"\n' tokens = [ (Token.Name.Builtin, u'echo'), (Token.Text, u' '), (Token.Literal.String.Double, u'"'), (Token.String.Interpol, u'${'), (Token.Name.Variable, u'a'), (Token.Punctuation, u'//['), (Token.Literal.String.Double, u'"b"'), (Token.Punctuation, u']/'), (Token.String.Interpol, u'}'), (Token.Literal.String.Double, u'"'), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testCurlyWithEscape(self): fragment = u'echo ${a//[\\"]/}\n' tokens = [ (Token.Name.Builtin, u'echo'), (Token.Text, u' '), (Token.String.Interpol, u'${'), (Token.Name.Variable, u'a'), (Token.Punctuation, u'//['), (Token.Literal.String.Escape, u'\\"'), (Token.Punctuation, u']/'), (Token.String.Interpol, u'}'), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testParsedSingle(self): fragment = u"a=$'abc\\''\n" tokens = [ (Token.Name.Variable, u'a'), (Token.Operator, u'='), (Token.Literal.String.Single, u"$'abc\\''"), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testShortVariableNames(self): fragment = u'x="$"\ny="$_"\nz="$abc"\n' tokens = [ # single lone $ (Token.Name.Variable, u'x'), (Token.Operator, u'='), (Token.Literal.String.Double, u'"'), (Token.Text, u'$'), (Token.Literal.String.Double, u'"'), (Token.Text, u'\n'), # single letter shell var (Token.Name.Variable, u'y'), (Token.Operator, u'='), (Token.Literal.String.Double, u'"'), (Token.Name.Variable, u'$_'), (Token.Literal.String.Double, u'"'), (Token.Text, u'\n'), # multi-letter user var (Token.Name.Variable, u'z'), (Token.Operator, u'='), (Token.Literal.String.Double, u'"'), (Token.Name.Variable, u'$abc'), (Token.Literal.String.Double, u'"'), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testArrayNums(self): fragment = u'a=(1 2 3)\n' tokens = [ (Token.Name.Variable, u'a'), (Token.Operator, u'='), (Token.Operator, u'('), (Token.Literal.Number, u'1'), (Token.Text, u' '), (Token.Literal.Number, u'2'), (Token.Text, u' '), (Token.Literal.Number, u'3'), (Token.Operator, u')'), (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) def testEndOfLineNums(self): fragment = u'a=1\nb=2 # comment\n' tokens = [ (Token.Name.Variable, u'a'), (Token.Operator, u'='), (Token.Literal.Number, u'1'), (Token.Text, u'\n'), (Token.Name.Variable, u'b'), (Token.Operator, u'='), (Token.Literal.Number, u'2'), (Token.Text, u' '), (Token.Comment.Single, u'# comment\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class TestLexers(TestCase): """Collection of lexers tests""" def setUp(self): self.lexer = lexers.IPythonLexer() self.bash_lexer = BashLexer() def testIPythonLexer(self): fragment = '!echo $HOME\n' tokens = [ (Token.Operator, '!'), ] tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment_2 = '!' + fragment tokens_2 = [ (Token.Operator, '!!'), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = '\t %%!\n' + fragment[1:] tokens_2 = [ (Token.Text, '\t '), (Token.Operator, '%%!'), (Token.Text, '\n'), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x = ' + fragment tokens_2 = [ (Token.Name, 'x'), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), ] + tokens self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x, = ' + fragment tokens_2 = [ (Token.Name, 'x'), (Token.Punctuation, ','), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), ] + tokens self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x, = %sx ' + fragment[1:] tokens_2 = [ (Token.Name, 'x'), (Token.Punctuation, ','), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), (Token.Operator, '%'), (Token.Keyword, 'sx'), (Token.Text, ' '), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'f = %R function () {}\n' tokens_2 = [ (Token.Name, 'f'), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), (Token.Operator, '%'), (Token.Keyword, 'R'), (Token.Text, ' function () {}\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = '\t%%xyz\n$foo\n' tokens_2 = [ (Token.Text, '\t'), (Token.Operator, '%%'), (Token.Keyword, 'xyz'), (Token.Text, '\n$foo\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = '%system?\n' tokens_2 = [ (Token.Operator, '%'), (Token.Keyword, 'system'), (Token.Operator, '?'), (Token.Text, '\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x != y\n' tokens_2 = [ (Token.Name, 'x'), (Token.Text, ' '), (Token.Operator, '!='), (Token.Text, ' '), (Token.Name, 'y'), (Token.Text, '\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = ' ?math.sin\n' tokens_2 = [ (Token.Text, ' '), (Token.Operator, '?'), (Token.Text, 'math.sin'), (Token.Text, '\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment = ' *int*?\n' tokens = [ (Token.Text, ' *int*'), (Token.Operator, '?'), (Token.Text, '\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class TestLexers(TestCase): """Collection of lexers tests""" def setUp(self): self.lexer = lexers.IPythonLexer() self.bash_lexer = BashLexer() def testIPythonLexer(self): fragment = "!echo $HOME\n" tokens = [(Token.Operator, "!")] tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment_2 = "!" + fragment tokens_2 = [(Token.Operator, "!!")] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "\t %%!\n" + fragment[1:] tokens_2 = [ (Token.Text, "\t "), (Token.Operator, "%%!"), (Token.Text, "\n"), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "x = " + fragment tokens_2 = [ (Token.Name, "x"), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), ] + tokens self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "x, = " + fragment tokens_2 = [ (Token.Name, "x"), (Token.Punctuation, ","), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), ] + tokens self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "x, = %sx " + fragment[1:] tokens_2 = [ (Token.Name, "x"), (Token.Punctuation, ","), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), (Token.Operator, "%"), (Token.Keyword, "sx"), (Token.Text, " "), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "f = %R function () {}\n" tokens_2 = [ (Token.Name, "f"), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), (Token.Operator, "%"), (Token.Keyword, "R"), (Token.Text, " function () {}\n"), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "\t%%xyz\n$foo\n" tokens_2 = [ (Token.Text, "\t"), (Token.Operator, "%%"), (Token.Keyword, "xyz"), (Token.Text, "\n$foo\n"), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "%system?\n" tokens_2 = [ (Token.Operator, "%"), (Token.Keyword, "system"), (Token.Operator, "?"), (Token.Text, "\n"), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = "x != y\n" tokens_2 = [ (Token.Name, "x"), (Token.Text, " "), (Token.Operator, "!="), (Token.Text, " "), (Token.Name, "y"), (Token.Text, "\n"), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = " ?math.sin\n" tokens_2 = [ (Token.Text, " "), (Token.Operator, "?"), (Token.Text, "math.sin"), (Token.Text, "\n"), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment = " *int*?\n" tokens = [(Token.Text, " *int*"), (Token.Operator, "?"), (Token.Text, "\n")] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment = "%%writefile -a foo.py\nif a == b:\n pass" tokens = [ (Token.Operator, "%%writefile"), (Token.Text, " -a foo.py\n"), (Token.Keyword, "if"), (Token.Text, " "), (Token.Name, "a"), (Token.Text, " "), (Token.Operator, "=="), (Token.Text, " "), (Token.Name, "b"), (Token.Punctuation, ":"), (Token.Text, "\n"), (Token.Text, " "), (Token.Keyword, "pass"), (Token.Text, "\n"), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment = "%%timeit\nmath.sin(0)" tokens = [ (Token.Operator, "%%timeit\n"), (Token.Name, "math"), (Token.Operator, "."), (Token.Name, "sin"), (Token.Punctuation, "("), (Token.Literal.Number.Integer, "0"), (Token.Punctuation, ")"), (Token.Text, "\n"), ] fragment = "%%HTML\n<div>foo</div>" tokens = [ (Token.Operator, "%%HTML"), (Token.Text, "\n"), (Token.Punctuation, "<"), (Token.Name.Tag, "div"), (Token.Punctuation, ">"), (Token.Text, "foo"), (Token.Punctuation, "<"), (Token.Punctuation, "/"), (Token.Name.Tag, "div"), (Token.Punctuation, ">"), (Token.Text, "\n"), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class TestLexers(TestCase): """Collection of lexers tests""" def setUp(self): self.lexer = lexers.IPythonLexer() self.bash_lexer = BashLexer() def testIPythonLexer(self): fragment = '!echo $HOME\n' tokens = [ (Token.Operator, '!'), ] tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment_2 = '!' + fragment tokens_2 = [ (Token.Operator, '!!'), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = '\t %%!\n' + fragment[1:] tokens_2 = [ (Token.Text, '\t '), (Token.Operator, '%%!'), (Token.Text, '\n'), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x = ' + fragment tokens_2 = [ (Token.Name, 'x'), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), ] + tokens self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x, = ' + fragment tokens_2 = [ (Token.Name, 'x'), (Token.Punctuation, ','), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), ] + tokens self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x, = %sx ' + fragment[1:] tokens_2 = [ (Token.Name, 'x'), (Token.Punctuation, ','), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), (Token.Operator, '%'), (Token.Keyword, 'sx'), (Token.Text, ' '), ] + tokens[1:] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'f = %R function () {}\n' tokens_2 = [ (Token.Name, 'f'), (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), (Token.Operator, '%'), (Token.Keyword, 'R'), (Token.Text, ' function () {}\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = '\t%%xyz\n$foo\n' tokens_2 = [ (Token.Text, '\t'), (Token.Operator, '%%'), (Token.Keyword, 'xyz'), (Token.Text, '\n$foo\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = '%system?\n' tokens_2 = [ (Token.Operator, '%'), (Token.Keyword, 'system'), (Token.Operator, '?'), (Token.Text, '\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = 'x != y\n' tokens_2 = [ (Token.Name, 'x'), (Token.Text, ' '), (Token.Operator, '!='), (Token.Text, ' '), (Token.Name, 'y'), (Token.Text, '\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment_2 = ' ?math.sin\n' tokens_2 = [ (Token.Text, ' '), (Token.Operator, '?'), (Token.Text, 'math.sin'), (Token.Text, '\n'), ] self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) fragment = ' *int*?\n' tokens = [ (Token.Text, ' *int*'), (Token.Operator, '?'), (Token.Text, '\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment = '%%writefile -a foo.py\nif a == b:\n pass' tokens = [ (Token.Operator, '%%writefile'), (Token.Text, ' -a foo.py\n'), (Token.Keyword, 'if'), (Token.Text, ' '), (Token.Name, 'a'), (Token.Text, ' '), (Token.Operator, '=='), (Token.Text, ' '), (Token.Name, 'b'), (Token.Punctuation, ':'), (Token.Text, '\n'), (Token.Text, ' '), (Token.Keyword, 'pass'), (Token.Text, '\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) fragment = '%%timeit\nmath.sin(0)' tokens = [ (Token.Operator, '%%timeit\n'), (Token.Name, 'math'), (Token.Operator, '.'), (Token.Name, 'sin'), (Token.Punctuation, '('), (Token.Literal.Number.Integer, '0'), (Token.Punctuation, ')'), (Token.Text, '\n'), ] fragment = '%%HTML\n<div>foo</div>' tokens = [ (Token.Operator, '%%HTML'), (Token.Text, '\n'), (Token.Punctuation, '<'), (Token.Name.Tag, 'div'), (Token.Punctuation, '>'), (Token.Text, 'foo'), (Token.Punctuation, '<'), (Token.Punctuation, '/'), (Token.Name.Tag, 'div'), (Token.Punctuation, '>'), (Token.Text, '\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))