def test_parse_list_with_literals(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_two", 1), tokens.CommaToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_three", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=( SyntaxToken("view_name.field_one", 1), SyntaxToken("view_name.field_two", 1, prefix=" "), SyntaxToken("view_name.field_three", 1, prefix=" "), ), right_bracket=RightBracket(), )
def test_parse_list_with_only_commas(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.ListStartToken(1), tokens.CommaToken(1), tokens.CommaToken(1), tokens.CommaToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result is None
def test_parse_list_with_bad_token(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.LiteralToken("view_name.field_two", 1), tokens.CommaToken(1), tokens.ValueToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result is None
def test_parse_list_with_inner_comment(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.CommaToken(2), tokens.WhitespaceToken("\n ", 2), tokens.LiteralToken("view_name.field_two", 3), tokens.WhitespaceToken(" ", 3), tokens.CommentToken("# This is a comment", 3), tokens.WhitespaceToken("\n", 3), tokens.ListEndToken(4), tokens.StreamEndToken(4), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields"), left_bracket=LeftBracket(), items=( SyntaxToken("view_name.field_one", prefix="\n "), SyntaxToken( "view_name.field_two", prefix="\n ", suffix=" # This is a comment\n" ), ), right_bracket=RightBracket(), )
def test_parse_list_with_trailing_comma(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 1), ), trailing_comma=Comma(), right_bracket=RightBracket(), ) # Test when the list items are separated by newlines stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.CommaToken(2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 2, prefix="\n "), ), trailing_comma=Comma(), right_bracket=RightBracket(prefix="\n"), )
def test_parse_list_with_pairs(): stream = ( tokens.LiteralToken("sorts", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("orders.customer_id", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("asc", 1), tokens.CommaToken(1), tokens.LiteralToken("orders.order_id", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("desc", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("sorts"), left_bracket=LeftBracket(), items=( PairNode(SyntaxToken("orders.customer_id"), SyntaxToken("asc")), PairNode(SyntaxToken("orders.order_id"), SyntaxToken("desc")), ), right_bracket=RightBracket(), ) stream = ( tokens.LiteralToken("filters", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.ValueToken(2), tokens.WhitespaceToken(" ", 2), tokens.QuotedLiteralToken("-0,-1,-8,-9,-99,-NULL,-EMPTY", 2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("filters"), left_bracket=LeftBracket(), items=( PairNode( SyntaxToken("view_name.field_one", prefix="\n "), QuotedSyntaxToken("-0,-1,-8,-9,-99,-NULL,-EMPTY"), ), ), right_bracket=RightBracket(prefix="\n"), )
def test_parse_list_with_no_opening_bracket(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.LiteralToken("view_name.field_two", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result is None
def test_parse_list_with_trailing_comma(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == {"drill_fields": ["view_name.field_one"]}
def scan(self) -> Tuple[tokens.Token, ...]: self.tokens.append(tokens.StreamStartToken(self.line_number)) while True: self.scan_until_token() ch = self.peek() if ch == "\0": self.tokens.append(tokens.StreamEndToken(self.line_number)) break elif ch == "{": self.advance() self.tokens.append(tokens.BlockStartToken(self.line_number)) elif ch == "}": self.advance() self.tokens.append(tokens.BlockEndToken(self.line_number)) elif ch == "[": self.advance() self.tokens.append(tokens.ListStartToken(self.line_number)) elif ch == "]": self.advance() self.tokens.append(tokens.ListEndToken(self.line_number)) elif ch == ",": self.advance() self.tokens.append(tokens.CommaToken(self.line_number)) elif ch == ":": self.advance() self.tokens.append(tokens.ValueToken(self.line_number)) elif ch == ";": if self.peek_multiple(2) == ";;": self.advance(2) self.tokens.append(tokens.ExpressionBlockEndToken(self.line_number)) elif ch == '"': self.advance() self.tokens.append(self.scan_quoted_literal()) elif ( self.peek_multiple(3) == "sql" or self.peek_multiple(4) == "html" or self.peek_multiple(24) == "expression_custom_filter" ): self.tokens.append(self.scan_literal()) self.scan_until_token() self.advance() self.tokens.append(tokens.ValueToken(self.line_number)) self.scan_until_token() self.tokens.append(self.scan_expression_block()) else: # TODO: This should actually check for valid literals first # and throw an error if it doesn't match self.tokens.append(self.scan_literal()) return tuple(self.tokens)
def parser(): stream = ( tokens.StreamStartToken(1), tokens.LiteralToken("view", 1), tokens.ValueToken(1), tokens.LiteralToken("view_name", 1), tokens.BlockStartToken(1), tokens.LiteralToken("sql_table_name", 2), tokens.ValueToken(2), tokens.ExpressionBlockToken("schema.table_name", 2), tokens.ExpressionBlockEndToken(2), tokens.LiteralToken("drill_fields", 3), tokens.ValueToken(3), tokens.ListStartToken(3), tokens.LiteralToken("view_name.field_one", 3), tokens.CommaToken(3), tokens.LiteralToken("view_name.field_two", 3), tokens.ListEndToken(3), tokens.BlockEndToken(4), tokens.StreamEndToken(4), ) return lkml.parser.Parser(stream)
def test_scan_until_token_skips_comments(): text = "# This is a comment\nStart here" lexer = lkml.Lexer(text) lexer.scan_until_token() result = lexer.peek() assert result == "S" params = [ ("\0", tokens.StreamEndToken(1)), ("{", tokens.BlockStartToken(1)), ("}", tokens.BlockEndToken(1)), ("[", tokens.ListStartToken(1)), ("]", tokens.ListEndToken(1)), (",", tokens.CommaToken(1)), (":", tokens.ValueToken(1)), (";;", tokens.ExpressionBlockEndToken(1)), ] @pytest.mark.parametrize("text,expected", params) def test_scan_all_simple_tokens(text, expected): lexer = lkml.Lexer(text) result = lexer.scan() # Skip stream start token appended at the beginning assert result[1] == expected def test_scan_quoted_literal(): text = '"This is quoted text."'