def test_parse_list_with_literals(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_two", 1), tokens.CommaToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("view_name.field_three", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields"), left_bracket=LeftBracket(), items=( SyntaxToken("view_name.field_one"), SyntaxToken("view_name.field_two", prefix=" "), SyntaxToken("view_name.field_three", prefix=" "), ), right_bracket=RightBracket(), )
def test_parse_list_with_inner_comment(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.CommaToken(2), tokens.WhitespaceToken("\n ", 2), tokens.LiteralToken("view_name.field_two", 3), tokens.WhitespaceToken(" ", 3), tokens.CommentToken("# This is a comment", 3), tokens.WhitespaceToken("\n", 3), tokens.ListEndToken(4), tokens.StreamEndToken(4), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=( SyntaxToken("view_name.field_one", 2, prefix="\n "), SyntaxToken("view_name.field_two", 3, prefix="\n ", suffix=" # This is a comment\n"), ), right_bracket=RightBracket(), )
def test_pair_node_str_should_return_formatted(): node = PairNode(type=SyntaxToken("foo"), value=SyntaxToken("bar")) assert str(node) == "foo: bar" # Add whitespace in an unconventional place node = PairNode(type=SyntaxToken("foo", suffix=" "), value=SyntaxToken("bar")) assert str(node) == "foo : bar"
def syntax_token_with_trivia_str_should_render(): token = SyntaxToken("foo", prefix="# Skip this\n ") assert str(token) == "# Skip this\n foo" token = SyntaxToken("foo", suffix="\n# Skip this\n ") assert str(token == "foo\n# Skip this\n ") token = SyntaxToken("foo", prefix="\n\t", suffix="\t\n") assert str(token) == "\n\tfoo\t\n"
def test_parse_pair_with_literal(): stream = ( tokens.LiteralToken("hidden", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("yes", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result == PairNode(type=SyntaxToken("hidden"), value=SyntaxToken("yes"))
def parse_token( key: str, value: str, force_quote: bool = False, prefix: str = "", suffix: str = "", ) -> SyntaxToken: """Parses a value into a token, quoting it if required by the key or forced. Args: key: A LookML field type (e.g. "hidden") value: The value string (e.g. "yes") force_quote: True if value should always be quoted Returns: A generator of serialized string chunks """ if force_quote or key in QUOTED_LITERAL_KEYS: return QuotedSyntaxToken(value, prefix=prefix, suffix=suffix) elif key in EXPR_BLOCK_KEYS: return ExpressionSyntaxToken(value.strip(), prefix=prefix, suffix=suffix) else: return SyntaxToken(value, prefix=prefix, suffix=suffix)
def test_block_node_str_should_return_formatted(): # Test a regular block node = BlockNode( type=SyntaxToken("set"), name=SyntaxToken("user_dimensions"), left_brace=LeftCurlyBrace(prefix=" ", suffix=" "), container=ContainerNode((ListNode( type=SyntaxToken("fields"), left_bracket=LeftBracket(), items=( SyntaxToken("user.user_id"), SyntaxToken("user.age", prefix=" "), ), right_bracket=RightBracket(), ), )), right_brace=RightCurlyBrace(prefix=" "), ) assert str( node) == "set: user_dimensions { fields: [user.user_id, user.age] }" # Test a block with no expression node = BlockNode( type=SyntaxToken("set"), name=SyntaxToken("foo"), left_brace=LeftCurlyBrace(prefix=" "), container=tuple(), right_brace=RightCurlyBrace(), ) assert str(node) == "set: foo {}"
def test_parse_list_with_trailing_comma(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.CommaToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 1), ), trailing_comma=Comma(), right_bracket=RightBracket(), ) # Test when the list items are separated by newlines stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.CommaToken(2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 2, prefix="\n "), ), trailing_comma=Comma(), right_bracket=RightBracket(prefix="\n"), )
def test_list_node_str_should_return_formatted(): # Test a node with PairNodes as items node = ListNode( type=SyntaxToken("filters"), left_bracket=LeftBracket(), items=( PairNode(SyntaxToken("created_date"), QuotedSyntaxToken("7 days")), PairNode(SyntaxToken("user.status", prefix=" "), QuotedSyntaxToken("-disabled")), ), right_bracket=RightBracket(), ) assert str( node) == 'filters: [created_date: "7 days", user.status: "-disabled"]' # Test a node with SyntaxTokens as items node = ListNode( type=SyntaxToken("fields"), left_bracket=LeftBracket(), items=( SyntaxToken("user.user_id", prefix="\n "), SyntaxToken("user.age", prefix="\n ", suffix="\n"), ), right_bracket=RightBracket(), ) assert str(node) == "fields: [\n user.user_id,\n user.age\n]" # Test a node with zero items node = ListNode( type=SyntaxToken("fields"), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(), ) assert str(node) == "fields: []"
def test_container_node_str_should_return_formatted(): node = ContainerNode(( PairNode(SyntaxToken("hidden"), SyntaxToken("true")), BlockNode( type=SyntaxToken("set", prefix=" "), name=SyntaxToken("foo"), left_brace=LeftCurlyBrace(prefix=" "), container=tuple(), right_brace=RightCurlyBrace(), ), ListNode( type=SyntaxToken("fields", prefix=" "), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(), ), )) assert str(node) == "hidden: true set: foo {} fields: []"
def test_parse_value_literal_with_sql_block(): literal = "SELECT * FROM tablename" stream = ( tokens.LiteralToken(literal, 1), tokens.ExpressionBlockEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_value() assert result == SyntaxToken(literal, 1)
def test_parse_list_with_pairs(): stream = ( tokens.LiteralToken("sorts", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.LiteralToken("orders.customer_id", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("asc", 1), tokens.CommaToken(1), tokens.LiteralToken("orders.order_id", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("desc", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("sorts"), left_bracket=LeftBracket(), items=( PairNode(SyntaxToken("orders.customer_id"), SyntaxToken("asc")), PairNode(SyntaxToken("orders.order_id"), SyntaxToken("desc")), ), right_bracket=RightBracket(), ) stream = ( tokens.LiteralToken("filters", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.LiteralToken("view_name.field_one", 2), tokens.ValueToken(2), tokens.WhitespaceToken(" ", 2), tokens.QuotedLiteralToken("-0,-1,-8,-9,-99,-NULL,-EMPTY", 2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("filters"), left_bracket=LeftBracket(), items=( PairNode( SyntaxToken("view_name.field_one", prefix="\n "), QuotedSyntaxToken("-0,-1,-8,-9,-99,-NULL,-EMPTY"), ), ), right_bracket=RightBracket(prefix="\n"), )
def parse_block( self, key: str, items: Dict[str, Any], name: Optional[str] = None ) -> BlockNode: """Serializes a dictionary to a LookML block. Args: key: A LookML field type (e.g. "dimension") fields: A dictionary to serialize (e.g. {"sql": "${TABLE}.order_id"}) name: An optional name of the block (e.g. "order_id") Returns: A generator of serialized string chunks """ prev_parent_key = self.parent_key self.parent_key = key latest_node_at_this_level = self.latest_node self.increase_level() nodes = [self.parse_any(key, value) for key, value in items.items()] self.decrease_level() self.latest_node = latest_node_at_this_level self.parent_key = prev_parent_key container = ContainerNode(items=tuple(flatten(nodes))) if self.latest_node and self.latest_node != DocumentNode: prefix = "\n" + self.newline_indent else: prefix = self.prefix node = BlockNode( type=SyntaxToken(key, prefix=prefix), left_brace=LeftCurlyBrace(prefix=" " if name else ""), right_brace=RightCurlyBrace( prefix=self.newline_indent if container.items else "" ), name=SyntaxToken(name) if name else None, container=container, ) self.latest_node = BlockNode return node
def test_parse_block_with_no_expression(): stream = ( tokens.LiteralToken("dimension", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.LiteralToken("dimension_name", 1), tokens.WhitespaceToken(" ", 1), tokens.BlockStartToken(1), tokens.BlockEndToken(1), tokens.WhitespaceToken(" ", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_block() assert result == BlockNode( type=SyntaxToken("dimension"), name=SyntaxToken("dimension_name"), left_brace=LeftCurlyBrace(prefix=" "), container=ContainerNode(items=tuple()), right_brace=RightCurlyBrace(suffix=" "), )
def test_parse_list_with_leading_comma(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.CommaToken(1), tokens.LiteralToken("view_name.field_one", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=(SyntaxToken("view_name.field_one", 1), ), right_bracket=RightBracket(), leading_comma=Comma(), )
def test_parse_list_with_no_contents(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(), ) # Add whitespace between brackets stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields", 1), colon=Colon(line_number=1, suffix=" "), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(prefix=" "), )
def test_parse_pair_with_quoted_literal(): stream = ( tokens.LiteralToken("view_label", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.QuotedLiteralToken("The View", 1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result == PairNode( type=SyntaxToken("view_label"), value=QuotedSyntaxToken("The View") ) with pytest.raises(AttributeError): result.prefix
def test_parse_pair_with_sql_block(): sql = " SELECT * FROM schema.table " stream = ( tokens.LiteralToken("sql", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ExpressionBlockToken(sql, 1), tokens.ExpressionBlockEndToken(1), tokens.StreamEndToken(1), ) parser = lkml.parser.Parser(stream) result = parser.parse_pair() assert result == PairNode( type=SyntaxToken("sql"), value=ExpressionSyntaxToken(sql.strip()) )
def parse_pair(self, key: str, value: str) -> PairNode: """Serializes a key and value to a LookML pair. Args: key: A LookML field type (e.g. "hidden") value: The value string (e.g. "yes") Returns: A generator of serialized string chunks """ force_quote = True if self.parent_key == "filters" and key != "field" else False value_syntax_token: SyntaxToken = self.parse_token(key, value, force_quote) node = PairNode( type=SyntaxToken(key, prefix=self.prefix), value=value_syntax_token ) self.latest_node = PairNode return node
def test_parse_list_with_only_comment(): stream = ( tokens.LiteralToken("drill_fields", 1), tokens.ValueToken(1), tokens.WhitespaceToken(" ", 1), tokens.ListStartToken(1), tokens.WhitespaceToken("\n ", 1), tokens.CommentToken("# Put some fields here", 2), tokens.WhitespaceToken("\n", 2), tokens.ListEndToken(3), tokens.StreamEndToken(3), ) parser = lkml.parser.Parser(stream) result = parser.parse_list() assert result == ListNode( type=SyntaxToken("drill_fields"), left_bracket=LeftBracket(), items=tuple(), right_bracket=RightBracket(prefix="\n # Put some fields here\n"), )
def test_parse_key_normal_returns_token_value(): stream = (tokens.LiteralToken("label", 1), tokens.ValueToken(1)) parser = lkml.parser.Parser(stream) result = parser.parse_key() assert result == (SyntaxToken("label", 1), Colon(line_number=1))
def test_lookml_visitor_should_visit_syntax_token_correctly(): visitor = LookMlVisitor() token = SyntaxToken("foo", suffix="\n") assert visitor.visit_token(token) == "foo\n"
def test_parse_value_literal(): literal = "This is an unquoted literal." stream = (tokens.LiteralToken(literal, 1), tokens.StreamEndToken(1)) parser = lkml.parser.Parser(stream) result = parser.parse_value() assert result == SyntaxToken(literal, 1)
def parse_list(self, key: str, values: Sequence[Union[str, Dict]]) -> ListNode: """Serializes a sequence to a LookML block. Args: key: A LookML field type (e.g. "fields") values: A sequence to serialize (e.g. ["orders.order_id", "orders.item"]) Returns: A generator of serialized string chunks """ # `suggestions` is only quoted when it's a list, so override the default force_quote = True if key == "suggestions" else False prev_parent_key = self.parent_key self.parent_key = key type_token = SyntaxToken(key, prefix=self.prefix) right_bracket = RightBracket() items: list = [] pair_mode = False # Check the first element to see if it's a single value or a pair if values and not isinstance(values[0], (str, int)): pair_mode = True # Coerce type depending on pair mode value if pair_mode: items = cast(List[PairNode], items) else: items = cast(List[SyntaxToken], items) # Choose newline delimiting or space delimiting based on contents if len(values) >= 5 or pair_mode: trailing_comma = True self.increase_level() for value in values: if pair_mode: value = cast(dict, value) # Extract key and value from dictionary with only one key [(key, val)] = value.items() pair: PairNode = self.parse_pair(key, val) items.append(pair) else: value = cast(str, value) token: SyntaxToken = self.parse_token( key, value, force_quote, prefix=self.newline_indent ) items.append(token) self.decrease_level() right_bracket = RightBracket(prefix=self.newline_indent) else: trailing_comma = False for i, value in enumerate(values): value = cast(str, value) if i == 0: token = self.parse_token(key, value, force_quote) else: token = self.parse_token(key, value, force_quote, prefix=" ") items.append(token) self.parent_key = prev_parent_key node = ListNode( type=type_token, left_bracket=LeftBracket(), items=tuple(items), right_bracket=right_bracket, trailing_comma=trailing_comma, ) self.latest_node = ListNode return node
def test_pair_node_should_not_have_children(): node = PairNode(type=SyntaxToken("foo"), value=SyntaxToken("bar")) assert node.children is None