def test_parse_list_with_literals(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_two", 1), tokens.CommaToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_three", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields"), left_bracket=LeftBracket(), items=( SyntaxToken("view_name.field_one"), SyntaxToken("view_name.field_two", prefix=" "), SyntaxToken("view_name.field_three", prefix=" "), ), right_bracket=RightBracket(), )
def test_parse_list_with_inner_comment(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.CommaToken(2), tokens.WhitespaceToken("\n ", 2), tokens.LiteralToken("view_name.field_two", 3), tokens.WhitespaceToken(" ", 3), tokens.CommentToken("# This is a comment", 3), tokens.WhitespaceToken("\n", 3), tokens.ListEndToken(4), tokens.StreamEndToken(4), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=( SyntaxToken("view_name.field_one", 2, prefix="\n "), SyntaxToken("view_name.field_two", 3, prefix="\n ", suffix=" # This is a comment\n"), ), right_bracket=RightBracket(), )
def test_scan_comment_with_surrounding_whitespace(): text = "\n# A comment\n " output = lkml.Lexer(text).scan() assert output == ( tokens.StreamStartToken(1), tokens.WhitespaceToken("\n", 1), tokens.CommentToken("# A comment", 2), tokens.WhitespaceToken("\n ", 2), tokens.StreamEndToken(3), )
def test_parse_block_without_closing_curly_brace(): stream = ( tokens.LiteralToken("view", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.BlockStartToken(1), tokens.WhitespaceToken("\n", 1), tokens.LiteralToken("hidden", 2), tokens.ValueToken(2), tokens.WhitespaceToken(" ", 2), tokens.LiteralToken("yes", 2), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_block() assert result is None
def test_parse_list_with_trailing_comma(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 1), ), trailing_comma=Comma(), right_bracket=RightBracket(), ) # Test when the list items are separated by newlines stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.CommaToken(2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 2, prefix="\n "), ), trailing_comma=Comma(), right_bracket=RightBracket(prefix="\n"), )
def test_parse_pair_without_value_token(): stream = ( tokens.LiteralToken("hidden", 1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("yes", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result is None
def test_parse_list_with_pairs(): stream = ( tokens.LiteralToken("sorts", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("orders.customer_id", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("asc", 1), tokens.CommaToken(1), tokens.LiteralToken("orders.order_id", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("desc", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("sorts"), left_bracket=LeftBracket(), items=( PairNode(SyntaxToken("orders.customer_id"), SyntaxToken("asc")), PairNode(SyntaxToken("orders.order_id"), SyntaxToken("desc")), ), right_bracket=RightBracket(), ) stream = ( tokens.LiteralToken("filters", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.ValueToken(2), tokens.WhitespaceToken(" ", 2), tokens.QuotedLiteralToken("-0,-1,-8,-9,-99,-NULL,-EMPTY", 2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("filters"), left_bracket=LeftBracket(), items=( PairNode( SyntaxToken("view_name.field_one", prefix="\n "), QuotedSyntaxToken("-0,-1,-8,-9,-99,-NULL,-EMPTY"), ), ), right_bracket=RightBracket(prefix="\n"), )
def test_parse_pair_with_literal(): stream = ( tokens.LiteralToken("hidden", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("yes", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result == PairNode(type=SyntaxToken("hidden"), value=SyntaxToken("yes"))
def test_scan_with_non_expression_block_starting_with_html(): text = "html_not_reserved_field: yes" output = lkml.Lexer(text).scan() assert output == ( tokens.StreamStartToken(1), tokens.LiteralToken("html_not_reserved_field", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("yes", 1), tokens.StreamEndToken(1), )
def test_parse_list_with_only_comment(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.CommentToken("# Put some fields here", 2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields"), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(prefix="\n # Put some fields here\n"), )
def test_parse_block_with_no_expression(): stream = ( tokens.LiteralToken("dimension", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("dimension_name", 1), tokens.WhitespaceToken(" ", 1), tokens.BlockStartToken(1), tokens.BlockEndToken(1), tokens.WhitespaceToken(" ", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_block() assert result == BlockNode( type=SyntaxToken("dimension"), name=SyntaxToken("dimension_name"), left_brace=LeftCurlyBrace(prefix=" "), container=ContainerNode(items=tuple()), right_brace=RightCurlyBrace(suffix=" "), )
def test_parse_list_with_no_opening_bracket(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.LiteralToken("view_name.field_two", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result is None
def test_parse_list_with_no_contents(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(), ) # Add whitespace between brackets stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(prefix=" "), )
def scan_whitespace(self) -> tokens.WhitespaceToken: r"""Returns a token from one or more whitespace characters. Example: >>> lexer = Lexer("\n\n\t Hello") >>> lexer.scan_whitespace() WhitespaceToken('\n\n\t ') """ chars = "" while self.peek() in "\n\t ": if self.peek() == "\n": self.line_number += 1 chars += self.consume() return tokens.WhitespaceToken(chars, self.line_number)
def test_parse_pair_with_quoted_literal(): stream = ( tokens.LiteralToken("view_label", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.QuotedLiteralToken("The View", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result == PairNode( type=SyntaxToken("view_label"), value=QuotedSyntaxToken("The View") ) with pytest.raises(AttributeError): result.prefix
def test_parse_pair_with_sql_block(): sql = " SELECT * FROM schema.table " stream = ( tokens.LiteralToken("sql", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ExpressionBlockToken(sql, 1), tokens.ExpressionBlockEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result == PairNode( type=SyntaxToken("sql"), value=ExpressionSyntaxToken(sql.strip()) )
def test_parse_list_with_leading_comma(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.CommaToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 1), ), right_bracket=RightBracket(), leading_comma=Comma(), )
def test_scan_whitespace(): text = "\n\t Hello World!" lexer = lkml.Lexer(text) token = lexer.scan_whitespace() assert token == tokens.WhitespaceToken("\n\t ", 1)