Esempio n. 1
0
def test_pprint():
    p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
                       '(select * from dual) q0 where 1=1 and 2=2')[0]
    output = StringIO()

    p._pprint_tree(f=output)
    pprint = '\n'.join([
        "|- 0 DML 'select'", "|- 1 Whitespace ' '",
        "|- 2 IdentifierList 'a0, b0...'", "|  |- 0 Identifier 'a0'",
        "|  |  `- 0 Name 'a0'", "|  |- 1 Punctuation ','",
        "|  |- 2 Whitespace ' '", "|  |- 3 Identifier 'b0'",
        "|  |  `- 0 Name 'b0'", "|  |- 4 Punctuation ','",
        "|  |- 5 Whitespace ' '", "|  |- 6 Identifier 'c0'",
        "|  |  `- 0 Name 'c0'", "|  |- 7 Punctuation ','",
        "|  |- 8 Whitespace ' '", "|  |- 9 Identifier 'd0'",
        "|  |  `- 0 Name 'd0'", "|  |- 10 Punctuation ','",
        "|  |- 11 Whitespace ' '", "|  `- 12 Float 'e0'",
        "|- 3 Whitespace ' '", "|- 4 Keyword 'from'", "|- 5 Whitespace ' '",
        "|- 6 Identifier '(selec...'", "|  |- 0 Parenthesis '(selec...'",
        "|  |  |- 0 Punctuation '('", "|  |  |- 1 DML 'select'",
        "|  |  |- 2 Whitespace ' '", "|  |  |- 3 Wildcard '*'",
        "|  |  |- 4 Whitespace ' '", "|  |  |- 5 Keyword 'from'",
        "|  |  |- 6 Whitespace ' '", "|  |  |- 7 Identifier 'dual'",
        "|  |  |  `- 0 Name 'dual'", "|  |  `- 8 Punctuation ')'",
        "|  |- 1 Whitespace ' '", "|  `- 2 Identifier 'q0'",
        "|     `- 0 Name 'q0'", "|- 7 Whitespace ' '",
        "`- 8 Where 'where ...'", "   |- 0 Keyword 'where'",
        "   |- 1 Whitespace ' '", "   |- 2 Comparison '1=1'",
        "   |  |- 0 Integer '1'", "   |  |- 1 Comparison '='",
        "   |  `- 2 Integer '1'", "   |- 3 Whitespace ' '",
        "   |- 4 Keyword 'and'", "   |- 5 Whitespace ' '",
        "   `- 6 Comparison '2=2'", "      |- 0 Integer '2'",
        "      |- 1 Comparison '='", "      `- 2 Integer '2'", ""
    ])
    assert output.getvalue() == pprint
Esempio n. 2
0
    def get_tokens(self, text, unfiltered=False):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
        """
        if isinstance(text, string_types):
            if self.stripall:
                text = text.strip()
            elif self.stripnl:
                text = text.strip('\n')

            if sys.version_info[0] < 3 and isinstance(text, text_type):
                text = StringIO(text.encode('utf-8'))
                self.encoding = 'utf-8'
            else:
                text = StringIO(text)

        def streamer():
            for i, t, v in self.get_tokens_unprocessed(text):
                yield t, v
        stream = streamer()
        if not unfiltered:
            stream = apply_filters(stream, self.filters, self)
        return stream
Esempio n. 3
0
    def get_tokens(self, text, unfiltered=False):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
        """
        if isinstance(text, string_types):
            if self.stripall:
                text = text.strip()
            elif self.stripnl:
                text = text.strip('\n')

            if sys.version_info[0] < 3 and isinstance(text, text_type):
                text = StringIO(text.encode('utf-8'))
                self.encoding = 'utf-8'
            else:
                text = StringIO(text)

        def streamer():
            for i, t, v in self.get_tokens_unprocessed(text):
                yield t, v
        stream = streamer()
        if not unfiltered:
            stream = apply_filters(stream, self.filters, self)
        return stream
Esempio n. 4
0
def test_pprint():
    p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
                       '(select * from dual) q0 where 1=1 and 2=2')[0]
    output = StringIO()

    p._pprint_tree(f=output)
    pprint = '\n'.join([
        " 0 DML 'select'", " 1 Whitespace ' '",
        " 2 IdentifierList 'a0, b0...'", " |  0 Identifier 'a0'",
        " |  |  0 Name 'a0'", " |  1 Punctuation ','", " |  2 Whitespace ' '",
        " |  3 Identifier 'b0'", " |  |  0 Name 'b0'", " |  4 Punctuation ','",
        " |  5 Whitespace ' '", " |  6 Identifier 'c0'", " |  |  0 Name 'c0'",
        " |  7 Punctuation ','", " |  8 Whitespace ' '",
        " |  9 Identifier 'd0'", " |  |  0 Name 'd0'", " | 10 Punctuation ','",
        " | 11 Whitespace ' '", " | 12 Float 'e0'", " 3 Whitespace ' '",
        " 4 Keyword 'from'", " 5 Whitespace ' '", " 6 Identifier '(selec...'",
        " |  0 Parenthesis '(selec...'", " |  |  0 Punctuation '('",
        " |  |  1 DML 'select'", " |  |  2 Whitespace ' '",
        " |  |  3 Wildcard '*'", " |  |  4 Whitespace ' '",
        " |  |  5 Keyword 'from'", " |  |  6 Whitespace ' '",
        " |  |  7 Identifier 'dual'", " |  |  |  0 Name 'dual'",
        " |  |  8 Punctuation ')'", " |  1 Whitespace ' '",
        " |  2 Identifier 'q0'", " |  |  0 Name 'q0'", " 7 Whitespace ' '",
        " 8 Where 'where ...'", " |  0 Keyword 'where'",
        " |  1 Whitespace ' '", " |  2 Comparison '1=1'",
        " |  |  0 Integer '1'", " |  |  1 Comparison '='",
        " |  |  2 Integer '1'", " |  3 Whitespace ' '", " |  4 Keyword 'and'",
        " |  5 Whitespace ' '", " |  6 Comparison '2=2'",
        " |  |  0 Integer '2'", " |  |  1 Comparison '='",
        " |  |  2 Integer '2'", ""
    ])
    assert output.getvalue() == pprint
Esempio n. 5
0
def test_stream_simple():
    stream = StringIO("SELECT 1; SELECT 2;")

    tokens = lexer.tokenize(stream)
    assert len(list(tokens)) == 9

    stream.seek(0)
    tokens = list(lexer.tokenize(stream))
    assert len(tokens) == 9

    stream.seek(0)
    tokens = list(lexer.tokenize(stream))
    assert len(tokens) == 9
Esempio n. 6
0
    def test_error(self):
        stream = StringIO("FOOBAR{")

        lex = lexer.Lexer()
        lex.bufsize = 4
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 2)
        self.assertEqual(tokens[1][0], T.Error)
Esempio n. 7
0
    def get_tokens(text, encoding=None):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.

        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        encoding = encoding or 'utf-8'

        if isinstance(text, string_types):
            text = StringIO(text)

        text = text.read()
        if not isinstance(text, text_type):
            try:
                text = text.decode(encoding)
            except UnicodeDecodeError:
                text = text.decode('unicode-escape')

        iterable = enumerate(text)
        for pos, char in iterable:
            for rexmatch, action in SQL_REGEX:
                m = rexmatch(text, pos)

                if not m:
                    continue
                elif isinstance(action, tokens._TokenType):
                    yield action, m.group()
                elif callable(action):
                    yield action(m.group())

                consume(iterable, m.end() - pos - 1)
                break
            else:
                yield tokens.Error, char
Esempio n. 8
0
    def get_tokens(text, encoding=None):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.

        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        encoding = encoding or 'utf-8'

        if isinstance(text, string_types):
            text = StringIO(text)

        text = text.read()
        if not isinstance(text, text_type):
            try:
                text = text.decode(encoding)
            except UnicodeDecodeError:
                text = text.decode('unicode-escape')

        iterable = enumerate(text)
        for pos, char in iterable:
            for rexmatch, action in SQL_REGEX:
                m = rexmatch(text, pos)

                if not m:
                    continue
                elif isinstance(action, tokens._TokenType):
                    yield action, m.group()
                elif callable(action):
                    yield action(m.group())

                consume(iterable, m.end() - pos - 1)
                break
            else:
                yield tokens.Error, char
Esempio n. 9
0
    def ndiffAssertEqual(self, first, second):
        """Like failUnlessEqual except use ndiff for readable output."""
        if first != second:
            # Using the built-in .splitlines() method here will cause incorrect
            # results when splitting statements that have quoted CR/CR+LF
            # characters.
            sfirst = split_unquoted_newlines(u(first))
            ssecond = split_unquoted_newlines(u(second))
            diff = difflib.ndiff(sfirst, ssecond)

            fp = StringIO()
            fp.write('\n')
            fp.write('\n'.join(diff))

            raise self.failureException(fp.getvalue())
Esempio n. 10
0
    def test_simple(self):
        stream = StringIO("SELECT 1; SELECT 2;")
        lex = lexer.Lexer()

        tokens = lex.get_tokens(stream)
        self.assertEqual(len(list(tokens)), 9)

        stream.seek(0)
        lex.bufsize = 4
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 9)

        stream.seek(0)
        lex.bufsize = len(stream.getvalue())
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 9)
Esempio n. 11
0
def test_stream_simple():
    stream = StringIO("SELECT 1; SELECT 2;")

    tokens = lexer.tokenize(stream)
    assert len(list(tokens)) == 9

    stream.seek(0)
    tokens = list(lexer.tokenize(stream))
    assert len(tokens) == 9

    stream.seek(0)
    tokens = list(lexer.tokenize(stream))
    assert len(tokens) == 9
Esempio n. 12
0
    def test_simple(self):
        stream = StringIO("SELECT 1; SELECT 2;")

        tokens = lexer.tokenize(stream)
        self.assertEqual(len(list(tokens)), 9)

        stream.seek(0)
        tokens = list(lexer.tokenize(stream))
        self.assertEqual(len(tokens), 9)

        stream.seek(0)
        tokens = list(lexer.tokenize(stream))
        self.assertEqual(len(tokens), 9)
Esempio n. 13
0
    def ndiffAssertEqual(self, first, second):
        """Like failUnlessEqual except use ndiff for readable output."""
        if first != second:
            # Using the built-in .splitlines() method here will cause incorrect
            # results when splitting statements that have quoted CR/CR+LF
            # characters.
            sfirst = split_unquoted_newlines(u(first))
            ssecond = split_unquoted_newlines(u(second))
            diff = difflib.ndiff(sfirst, ssecond)

            fp = StringIO()
            fp.write('\n')
            fp.write('\n'.join(diff))

            raise self.failureException(fp.getvalue())
Esempio n. 14
0
    def test_simple(self):
        stream = StringIO("SELECT 1; SELECT 2;")
        lex = lexer.Lexer()

        tokens = lex.get_tokens(stream)
        self.assertEqual(len(list(tokens)), 9)

        stream.seek(0)
        lex.bufsize = 4
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 9)

        stream.seek(0)
        lex.bufsize = len(stream.getvalue())
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 9)
Esempio n. 15
0
def test_stream_error():
    stream = StringIO("FOOBAR{")

    tokens = list(lexer.tokenize(stream))
    assert len(tokens) == 2
    assert tokens[1][0] == T.Error
Esempio n. 16
0
 def test_split_stream(self):
     stream = StringIO("SELECT 1; SELECT 2;")
     stmts = sqlparse.parsestream(stream)
     self.assertEqual(type(stmts), types.GeneratorType)
     self.assertEqual(len(list(stmts)), 2)
Esempio n. 17
0
def test_split_unicode_parsestream():
    stream = StringIO(u'SELECT ö')
    stmts = list(sqlparse.parsestream(stream))
    assert str(stmts[0]) == 'SELECT ö'
Esempio n. 18
0
def test_split_encoding_parsestream():
    stream = StringIO("SELECT 1; SELECT 2;")
    stmts = list(sqlparse.parsestream(stream))
    assert isinstance(stmts[0].tokens[0].value, text_type)
Esempio n. 19
0
def test_split_stream():
    stream = StringIO("SELECT 1; SELECT 2;")
    stmts = sqlparse.parsestream(stream)
    assert isinstance(stmts, types.GeneratorType)
    assert len(list(stmts)) == 2
Esempio n. 20
0
def test_pprint():
    p = sqlparse.parse("select a0, b0, c0, d0, e0 from " "(select * from dual) q0 where 1=1 and 2=2")[0]
    output = StringIO()

    p._pprint_tree(f=output)
    pprint = "\n".join(
        [
            " 0 DML 'select'",
            " 1 Whitespace ' '",
            " 2 IdentifierList 'a0, b0...'",
            " |  0 Identifier 'a0'",
            " |  |  0 Name 'a0'",
            " |  1 Punctuation ','",
            " |  2 Whitespace ' '",
            " |  3 Identifier 'b0'",
            " |  |  0 Name 'b0'",
            " |  4 Punctuation ','",
            " |  5 Whitespace ' '",
            " |  6 Identifier 'c0'",
            " |  |  0 Name 'c0'",
            " |  7 Punctuation ','",
            " |  8 Whitespace ' '",
            " |  9 Identifier 'd0'",
            " |  |  0 Name 'd0'",
            " | 10 Punctuation ','",
            " | 11 Whitespace ' '",
            " | 12 Float 'e0'",
            " 3 Whitespace ' '",
            " 4 Keyword 'from'",
            " 5 Whitespace ' '",
            " 6 Identifier '(selec...'",
            " |  0 Parenthesis '(selec...'",
            " |  |  0 Punctuation '('",
            " |  |  1 DML 'select'",
            " |  |  2 Whitespace ' '",
            " |  |  3 Wildcard '*'",
            " |  |  4 Whitespace ' '",
            " |  |  5 Keyword 'from'",
            " |  |  6 Whitespace ' '",
            " |  |  7 Identifier 'dual'",
            " |  |  |  0 Name 'dual'",
            " |  |  8 Punctuation ')'",
            " |  1 Whitespace ' '",
            " |  2 Identifier 'q0'",
            " |  |  0 Name 'q0'",
            " 7 Whitespace ' '",
            " 8 Where 'where ...'",
            " |  0 Keyword 'where'",
            " |  1 Whitespace ' '",
            " |  2 Comparison '1=1'",
            " |  |  0 Integer '1'",
            " |  |  1 Comparison '='",
            " |  |  2 Integer '1'",
            " |  3 Whitespace ' '",
            " |  4 Keyword 'and'",
            " |  5 Whitespace ' '",
            " |  6 Comparison '2=2'",
            " |  |  0 Integer '2'",
            " |  |  1 Comparison '='",
            " |  |  2 Integer '2'",
            "",
        ]
    )
    assert output.getvalue() == pprint
Esempio n. 21
0
 def test_encoding_parsestream(self):
     stream = StringIO("SELECT 1; SELECT 2;")
     stmts = list(sqlparse.parsestream(stream))
     self.assertEqual(type(stmts[0].tokens[0].value), text_type)
Esempio n. 22
0
    def test_error(self):
        stream = StringIO("FOOBAR{")

        tokens = list(lexer.tokenize(stream))
        self.assertEqual(len(tokens), 2)
        self.assertEqual(tokens[1][0], T.Error)
Esempio n. 23
0
def test_pprint():
    p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
                       '(select * from dual) q0 where 1=1 and 2=2')[0]
    output = StringIO()

    p._pprint_tree(f=output)
    pprint = '\n'.join([
        "|- 0 DML 'select'",
        "|- 1 Whitespace ' '",
        "|- 2 IdentifierList 'a0, b0...'",
        "|  |- 0 Identifier 'a0'",
        "|  |  `- 0 Name 'a0'",
        "|  |- 1 Punctuation ','",
        "|  |- 2 Whitespace ' '",
        "|  |- 3 Identifier 'b0'",
        "|  |  `- 0 Name 'b0'",
        "|  |- 4 Punctuation ','",
        "|  |- 5 Whitespace ' '",
        "|  |- 6 Identifier 'c0'",
        "|  |  `- 0 Name 'c0'",
        "|  |- 7 Punctuation ','",
        "|  |- 8 Whitespace ' '",
        "|  |- 9 Identifier 'd0'",
        "|  |  `- 0 Name 'd0'",
        "|  |- 10 Punctuation ','",
        "|  |- 11 Whitespace ' '",
        "|  `- 12 Float 'e0'",
        "|- 3 Whitespace ' '",
        "|- 4 Keyword 'from'",
        "|- 5 Whitespace ' '",
        "|- 6 Identifier '(selec...'",
        "|  |- 0 Parenthesis '(selec...'",
        "|  |  |- 0 Punctuation '('",
        "|  |  |- 1 DML 'select'",
        "|  |  |- 2 Whitespace ' '",
        "|  |  |- 3 Wildcard '*'",
        "|  |  |- 4 Whitespace ' '",
        "|  |  |- 5 Keyword 'from'",
        "|  |  |- 6 Whitespace ' '",
        "|  |  |- 7 Identifier 'dual'",
        "|  |  |  `- 0 Name 'dual'",
        "|  |  `- 8 Punctuation ')'",
        "|  |- 1 Whitespace ' '",
        "|  `- 2 Identifier 'q0'",
        "|     `- 0 Name 'q0'",
        "|- 7 Whitespace ' '",
        "`- 8 Where 'where ...'",
        "   |- 0 Keyword 'where'",
        "   |- 1 Whitespace ' '",
        "   |- 2 Comparison '1=1'",
        "   |  |- 0 Integer '1'",
        "   |  |- 1 Comparison '='",
        "   |  `- 2 Integer '1'",
        "   |- 3 Whitespace ' '",
        "   |- 4 Keyword 'and'",
        "   |- 5 Whitespace ' '",
        "   `- 6 Comparison '2=2'",
        "      |- 0 Integer '2'",
        "      |- 1 Comparison '='",
        "      `- 2 Integer '2'",
        ""])
    assert output.getvalue() == pprint