def test_parse_list_with_literals():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("view_name.field_two", 1),
tokens.CommaToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("view_name.field_three", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields"),
left_bracket=LeftBracket(),
items=(
SyntaxToken("view_name.field_one"),
SyntaxToken("view_name.field_two", prefix=" "),
SyntaxToken("view_name.field_three", prefix=" "),
),
right_bracket=RightBracket(),
)
def test_parse_list_with_inner_comment():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.LiteralToken("view_name.field_one", 2),
tokens.CommaToken(2),
tokens.WhitespaceToken("\n ", 2),
tokens.LiteralToken("view_name.field_two", 3),
tokens.WhitespaceToken(" ", 3),
tokens.CommentToken("# This is a comment", 3),
tokens.WhitespaceToken("\n", 3),
tokens.ListEndToken(4),
tokens.StreamEndToken(4),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(
SyntaxToken("view_name.field_one", 2, prefix="\n "),
SyntaxToken("view_name.field_two",
3,
prefix="\n ",
suffix=" # This is a comment\n"),
),
right_bracket=RightBracket(),
)
def test_pair_node_str_should_return_formatted():
node = PairNode(type=SyntaxToken("foo"), value=SyntaxToken("bar"))
assert str(node) == "foo: bar"
# Add whitespace in an unconventional place
node = PairNode(type=SyntaxToken("foo", suffix=" "),
value=SyntaxToken("bar"))
assert str(node) == "foo : bar"
def syntax_token_with_trivia_str_should_render():
token = SyntaxToken("foo", prefix="# Skip this\n ")
assert str(token) == "# Skip this\n foo"
token = SyntaxToken("foo", suffix="\n# Skip this\n ")
assert str(token == "foo\n# Skip this\n ")
token = SyntaxToken("foo", prefix="\n\t", suffix="\t\n")
assert str(token) == "\n\tfoo\t\n"
def test_parse_pair_with_literal():
stream = (
tokens.LiteralToken("hidden", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("yes", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == PairNode(type=SyntaxToken("hidden"), value=SyntaxToken("yes"))
def parse_token(
key: str,
value: str,
force_quote: bool = False,
prefix: str = "",
suffix: str = "",
) -> SyntaxToken:
"""Parses a value into a token, quoting it if required by the key or forced.
Args:
key: A LookML field type (e.g. "hidden")
value: The value string (e.g. "yes")
force_quote: True if value should always be quoted
Returns:
A generator of serialized string chunks
"""
if force_quote or key in QUOTED_LITERAL_KEYS:
return QuotedSyntaxToken(value, prefix=prefix, suffix=suffix)
elif key in EXPR_BLOCK_KEYS:
return ExpressionSyntaxToken(value.strip(),
prefix=prefix,
suffix=suffix)
else:
return SyntaxToken(value, prefix=prefix, suffix=suffix)
def test_block_node_str_should_return_formatted():
# Test a regular block
node = BlockNode(
type=SyntaxToken("set"),
name=SyntaxToken("user_dimensions"),
left_brace=LeftCurlyBrace(prefix=" ", suffix=" "),
container=ContainerNode((ListNode(
type=SyntaxToken("fields"),
left_bracket=LeftBracket(),
items=(
SyntaxToken("user.user_id"),
SyntaxToken("user.age", prefix=" "),
),
right_bracket=RightBracket(),
), )),
right_brace=RightCurlyBrace(prefix=" "),
)
assert str(
node) == "set: user_dimensions { fields: [user.user_id, user.age] }"
# Test a block with no expression
node = BlockNode(
type=SyntaxToken("set"),
name=SyntaxToken("foo"),
left_brace=LeftCurlyBrace(prefix=" "),
container=tuple(),
right_brace=RightCurlyBrace(),
)
assert str(node) == "set: foo {}"
def test_parse_list_with_trailing_comma():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(SyntaxToken("view_name.field_one", 1), ),
trailing_comma=Comma(),
right_bracket=RightBracket(),
)
# Test when the list items are separated by newlines
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.LiteralToken("view_name.field_one", 2),
tokens.CommaToken(2),
tokens.WhitespaceToken("\n", 2),
tokens.ListEndToken(3),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(SyntaxToken("view_name.field_one", 2, prefix="\n "), ),
trailing_comma=Comma(),
right_bracket=RightBracket(prefix="\n"),
)
def test_list_node_str_should_return_formatted():
# Test a node with PairNodes as items
node = ListNode(
type=SyntaxToken("filters"),
left_bracket=LeftBracket(),
items=(
PairNode(SyntaxToken("created_date"), QuotedSyntaxToken("7 days")),
PairNode(SyntaxToken("user.status", prefix=" "),
QuotedSyntaxToken("-disabled")),
),
right_bracket=RightBracket(),
)
assert str(
node) == 'filters: [created_date: "7 days", user.status: "-disabled"]'
# Test a node with SyntaxTokens as items
node = ListNode(
type=SyntaxToken("fields"),
left_bracket=LeftBracket(),
items=(
SyntaxToken("user.user_id", prefix="\n "),
SyntaxToken("user.age", prefix="\n ", suffix="\n"),
),
right_bracket=RightBracket(),
)
assert str(node) == "fields: [\n user.user_id,\n user.age\n]"
# Test a node with zero items
node = ListNode(
type=SyntaxToken("fields"),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(),
)
assert str(node) == "fields: []"
def test_container_node_str_should_return_formatted():
node = ContainerNode((
PairNode(SyntaxToken("hidden"), SyntaxToken("true")),
BlockNode(
type=SyntaxToken("set", prefix=" "),
name=SyntaxToken("foo"),
left_brace=LeftCurlyBrace(prefix=" "),
container=tuple(),
right_brace=RightCurlyBrace(),
),
ListNode(
type=SyntaxToken("fields", prefix=" "),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(),
),
))
assert str(node) == "hidden: true set: foo {} fields: []"
def test_parse_value_literal_with_sql_block():
literal = "SELECT * FROM tablename"
stream = (
tokens.LiteralToken(literal, 1),
tokens.ExpressionBlockEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_value()
assert result == SyntaxToken(literal, 1)
def test_parse_list_with_pairs():
stream = (
tokens.LiteralToken("sorts", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.LiteralToken("orders.customer_id", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("asc", 1),
tokens.CommaToken(1),
tokens.LiteralToken("orders.order_id", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("desc", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("sorts"),
left_bracket=LeftBracket(),
items=(
PairNode(SyntaxToken("orders.customer_id"), SyntaxToken("asc")),
PairNode(SyntaxToken("orders.order_id"), SyntaxToken("desc")),
),
right_bracket=RightBracket(),
)
stream = (
tokens.LiteralToken("filters", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.LiteralToken("view_name.field_one", 2),
tokens.ValueToken(2),
tokens.WhitespaceToken(" ", 2),
tokens.QuotedLiteralToken("-0,-1,-8,-9,-99,-NULL,-EMPTY", 2),
tokens.WhitespaceToken("\n", 2),
tokens.ListEndToken(3),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("filters"),
left_bracket=LeftBracket(),
items=(
PairNode(
SyntaxToken("view_name.field_one", prefix="\n "),
QuotedSyntaxToken("-0,-1,-8,-9,-99,-NULL,-EMPTY"),
),
),
right_bracket=RightBracket(prefix="\n"),
)
def parse_block(
self, key: str, items: Dict[str, Any], name: Optional[str] = None
) -> BlockNode:
"""Serializes a dictionary to a LookML block.
Args:
key: A LookML field type (e.g. "dimension")
fields: A dictionary to serialize (e.g. {"sql": "${TABLE}.order_id"})
name: An optional name of the block (e.g. "order_id")
Returns:
A generator of serialized string chunks
"""
prev_parent_key = self.parent_key
self.parent_key = key
latest_node_at_this_level = self.latest_node
self.increase_level()
nodes = [self.parse_any(key, value) for key, value in items.items()]
self.decrease_level()
self.latest_node = latest_node_at_this_level
self.parent_key = prev_parent_key
container = ContainerNode(items=tuple(flatten(nodes)))
if self.latest_node and self.latest_node != DocumentNode:
prefix = "\n" + self.newline_indent
else:
prefix = self.prefix
node = BlockNode(
type=SyntaxToken(key, prefix=prefix),
left_brace=LeftCurlyBrace(prefix=" " if name else ""),
right_brace=RightCurlyBrace(
prefix=self.newline_indent if container.items else ""
),
name=SyntaxToken(name) if name else None,
container=container,
)
self.latest_node = BlockNode
return node
def test_parse_block_with_no_expression():
stream = (
tokens.LiteralToken("dimension", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("dimension_name", 1),
tokens.WhitespaceToken(" ", 1),
tokens.BlockStartToken(1),
tokens.BlockEndToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_block()
assert result == BlockNode(
type=SyntaxToken("dimension"),
name=SyntaxToken("dimension_name"),
left_brace=LeftCurlyBrace(prefix=" "),
container=ContainerNode(items=tuple()),
right_brace=RightCurlyBrace(suffix=" "),
)
def test_parse_list_with_leading_comma():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.CommaToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(SyntaxToken("view_name.field_one", 1), ),
right_bracket=RightBracket(),
leading_comma=Comma(),
)
def test_parse_list_with_no_contents():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(),
)
# Add whitespace between brackets
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(prefix=" "),
)
def test_parse_pair_with_quoted_literal():
stream = (
tokens.LiteralToken("view_label", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.QuotedLiteralToken("The View", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == PairNode(
type=SyntaxToken("view_label"), value=QuotedSyntaxToken("The View")
)
with pytest.raises(AttributeError):
result.prefix
def test_parse_pair_with_sql_block():
sql = " SELECT * FROM schema.table "
stream = (
tokens.LiteralToken("sql", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ExpressionBlockToken(sql, 1),
tokens.ExpressionBlockEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == PairNode(
type=SyntaxToken("sql"), value=ExpressionSyntaxToken(sql.strip())
)
def parse_pair(self, key: str, value: str) -> PairNode:
"""Serializes a key and value to a LookML pair.
Args:
key: A LookML field type (e.g. "hidden")
value: The value string (e.g. "yes")
Returns:
A generator of serialized string chunks
"""
force_quote = True if self.parent_key == "filters" and key != "field" else False
value_syntax_token: SyntaxToken = self.parse_token(key, value, force_quote)
node = PairNode(
type=SyntaxToken(key, prefix=self.prefix), value=value_syntax_token
)
self.latest_node = PairNode
return node
def test_parse_list_with_only_comment():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.CommentToken("# Put some fields here", 2),
tokens.WhitespaceToken("\n", 2),
tokens.ListEndToken(3),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields"),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(prefix="\n # Put some fields here\n"),
)
def test_parse_key_normal_returns_token_value():
stream = (tokens.LiteralToken("label", 1), tokens.ValueToken(1))
parser = lkml.parser.Parser(stream)
result = parser.parse_key()
assert result == (SyntaxToken("label", 1), Colon(line_number=1))
def test_lookml_visitor_should_visit_syntax_token_correctly():
visitor = LookMlVisitor()
token = SyntaxToken("foo", suffix="\n")
assert visitor.visit_token(token) == "foo\n"
def test_parse_value_literal():
literal = "This is an unquoted literal."
stream = (tokens.LiteralToken(literal, 1), tokens.StreamEndToken(1))
parser = lkml.parser.Parser(stream)
result = parser.parse_value()
assert result == SyntaxToken(literal, 1)
def parse_list(self, key: str, values: Sequence[Union[str, Dict]]) -> ListNode:
"""Serializes a sequence to a LookML block.
Args:
key: A LookML field type (e.g. "fields")
values: A sequence to serialize (e.g. ["orders.order_id", "orders.item"])
Returns:
A generator of serialized string chunks
"""
# `suggestions` is only quoted when it's a list, so override the default
force_quote = True if key == "suggestions" else False
prev_parent_key = self.parent_key
self.parent_key = key
type_token = SyntaxToken(key, prefix=self.prefix)
right_bracket = RightBracket()
items: list = []
pair_mode = False
# Check the first element to see if it's a single value or a pair
if values and not isinstance(values[0], (str, int)):
pair_mode = True
# Coerce type depending on pair mode value
if pair_mode:
items = cast(List[PairNode], items)
else:
items = cast(List[SyntaxToken], items)
# Choose newline delimiting or space delimiting based on contents
if len(values) >= 5 or pair_mode:
trailing_comma = True
self.increase_level()
for value in values:
if pair_mode:
value = cast(dict, value)
# Extract key and value from dictionary with only one key
[(key, val)] = value.items()
pair: PairNode = self.parse_pair(key, val)
items.append(pair)
else:
value = cast(str, value)
token: SyntaxToken = self.parse_token(
key, value, force_quote, prefix=self.newline_indent
)
items.append(token)
self.decrease_level()
right_bracket = RightBracket(prefix=self.newline_indent)
else:
trailing_comma = False
for i, value in enumerate(values):
value = cast(str, value)
if i == 0:
token = self.parse_token(key, value, force_quote)
else:
token = self.parse_token(key, value, force_quote, prefix=" ")
items.append(token)
self.parent_key = prev_parent_key
node = ListNode(
type=type_token,
left_bracket=LeftBracket(),
items=tuple(items),
right_bracket=right_bracket,
trailing_comma=trailing_comma,
)
self.latest_node = ListNode
return node
def test_pair_node_should_not_have_children():
node = PairNode(type=SyntaxToken("foo"), value=SyntaxToken("bar"))
assert node.children is None