def test_parse_list_with_pairs():
stream = (
tokens.LiteralToken("sorts", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.LiteralToken("orders.customer_id", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("asc", 1),
tokens.CommaToken(1),
tokens.LiteralToken("orders.order_id", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("desc", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("sorts"),
left_bracket=LeftBracket(),
items=(
PairNode(SyntaxToken("orders.customer_id"), SyntaxToken("asc")),
PairNode(SyntaxToken("orders.order_id"), SyntaxToken("desc")),
),
right_bracket=RightBracket(),
)
stream = (
tokens.LiteralToken("filters", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.LiteralToken("view_name.field_one", 2),
tokens.ValueToken(2),
tokens.WhitespaceToken(" ", 2),
tokens.QuotedLiteralToken("-0,-1,-8,-9,-99,-NULL,-EMPTY", 2),
tokens.WhitespaceToken("\n", 2),
tokens.ListEndToken(3),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("filters"),
left_bracket=LeftBracket(),
items=(
PairNode(
SyntaxToken("view_name.field_one", prefix="\n "),
QuotedSyntaxToken("-0,-1,-8,-9,-99,-NULL,-EMPTY"),
),
),
right_bracket=RightBracket(prefix="\n"),
)
def test_parse_block_without_closing_curly_brace():
stream = (
tokens.LiteralToken("view", 1),
tokens.ValueToken(1),
tokens.BlockStartToken(1),
tokens.LiteralToken("hidden", 2),
tokens.ValueToken(2),
tokens.LiteralToken("yes", 2),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_block()
assert result is None
def scan(self) -> Tuple[tokens.Token, ...]:
self.tokens.append(tokens.StreamStartToken(self.line_number))
while True:
self.scan_until_token()
ch = self.peek()
if ch == "\0":
self.tokens.append(tokens.StreamEndToken(self.line_number))
break
elif ch == "{":
self.advance()
self.tokens.append(tokens.BlockStartToken(self.line_number))
elif ch == "}":
self.advance()
self.tokens.append(tokens.BlockEndToken(self.line_number))
elif ch == "[":
self.advance()
self.tokens.append(tokens.ListStartToken(self.line_number))
elif ch == "]":
self.advance()
self.tokens.append(tokens.ListEndToken(self.line_number))
elif ch == ",":
self.advance()
self.tokens.append(tokens.CommaToken(self.line_number))
elif ch == ":":
self.advance()
self.tokens.append(tokens.ValueToken(self.line_number))
elif ch == ";":
if self.peek_multiple(2) == ";;":
self.advance(2)
self.tokens.append(tokens.ExpressionBlockEndToken(self.line_number))
elif ch == '"':
self.advance()
self.tokens.append(self.scan_quoted_literal())
elif (
self.peek_multiple(3) == "sql"
or self.peek_multiple(4) == "html"
or self.peek_multiple(24) == "expression_custom_filter"
):
self.tokens.append(self.scan_literal())
self.scan_until_token()
self.advance()
self.tokens.append(tokens.ValueToken(self.line_number))
self.scan_until_token()
self.tokens.append(self.scan_expression_block())
else:
# TODO: This should actually check for valid literals first
# and throw an error if it doesn't match
self.tokens.append(self.scan_literal())
return tuple(self.tokens)
def test_parse_list_with_literals():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("view_name.field_two", 1),
tokens.CommaToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("view_name.field_three", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields"),
left_bracket=LeftBracket(),
items=(
SyntaxToken("view_name.field_one"),
SyntaxToken("view_name.field_two", prefix=" "),
SyntaxToken("view_name.field_three", prefix=" "),
),
right_bracket=RightBracket(),
)
def scan(self) -> Tuple[tokens.Token, ...]:
self.tokens.append(tokens.StreamStartToken(self.line_number))
while True:
self.scan_until_token()
ch = self.peek()
if ch == "\0":
self.tokens.append(CH_MAPPING[ch](self.line_number))
break
elif ch == ";":
if self.peek_multiple(2) == ";;":
self.advance(2)
self.tokens.append(CH_MAPPING[ch](self.line_number))
elif ch == '"':
self.advance()
self.tokens.append(self.scan_quoted_literal())
elif ch in CH_MAPPING.keys():
self.advance()
self.tokens.append(CH_MAPPING[ch](self.line_number))
elif self.check_for_expression_block(self.peek_multiple(25)):
self.tokens.append(self.scan_literal())
self.scan_until_token()
self.advance()
self.tokens.append(tokens.ValueToken(self.line_number))
self.scan_until_token()
self.tokens.append(self.scan_expression_block())
else:
# TODO: This should actually check for valid literals first
# and throw an error if it doesn't match
self.tokens.append(self.scan_literal())
return tuple(self.tokens)
def test_parse_list_with_bad_token():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.ListStartToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.LiteralToken("view_name.field_two", 1),
tokens.CommaToken(1),
tokens.ValueToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result is None
def test_parse_list_with_inner_comment():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.LiteralToken("view_name.field_one", 2),
tokens.CommaToken(2),
tokens.WhitespaceToken("\n ", 2),
tokens.LiteralToken("view_name.field_two", 3),
tokens.WhitespaceToken(" ", 3),
tokens.CommentToken("# This is a comment", 3),
tokens.WhitespaceToken("\n", 3),
tokens.ListEndToken(4),
tokens.StreamEndToken(4),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(
SyntaxToken("view_name.field_one", 2, prefix="\n "),
SyntaxToken("view_name.field_two",
3,
prefix="\n ",
suffix=" # This is a comment\n"),
),
right_bracket=RightBracket(),
)
def test_parse_list_with_trailing_comma():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(SyntaxToken("view_name.field_one", 1), ),
trailing_comma=Comma(),
right_bracket=RightBracket(),
)
# Test when the list items are separated by newlines
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.LiteralToken("view_name.field_one", 2),
tokens.CommaToken(2),
tokens.WhitespaceToken("\n", 2),
tokens.ListEndToken(3),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(SyntaxToken("view_name.field_one", 2, prefix="\n "), ),
trailing_comma=Comma(),
right_bracket=RightBracket(prefix="\n"),
)
def test_parse_pair_with_literal():
stream = (
tokens.LiteralToken("hidden", 1),
tokens.ValueToken(1),
tokens.LiteralToken("yes", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == {"hidden": "yes"}
def test_scan_with_non_expression_block_starting_with_html():
text = "html_not_reserved_field: yes"
output = lkml.Lexer(text).scan()
assert output == (
tokens.StreamStartToken(1),
tokens.LiteralToken("html_not_reserved_field", 1),
tokens.ValueToken(1),
tokens.LiteralToken("yes", 1),
tokens.StreamEndToken(1),
)
def test_parse_pair_with_bad_key():
stream = (
tokens.QuotedLiteralToken("hidden", 1),
tokens.ValueToken(1),
tokens.LiteralToken("yes", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result is None
def test_parse_pair_with_quoted_literal():
stream = (
tokens.LiteralToken("view_label", 1),
tokens.ValueToken(1),
tokens.QuotedLiteralToken("View Label", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == {"view_label": "View Label"}
def test_parse_list_with_no_contents():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.ListStartToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == {"drill_fields": []}
def test_parse_pair_with_literal():
stream = (
tokens.LiteralToken("hidden", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("yes", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == PairNode(type=SyntaxToken("hidden"), value=SyntaxToken("yes"))
def test_parse_pair_with_sql_block():
sql = "SELECT * FROM schema.table"
stream = (
tokens.LiteralToken("sql", 1),
tokens.ValueToken(1),
tokens.LiteralToken(sql, 1),
tokens.ExpressionBlockEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == {"sql": sql}
def test_parse_block_with_no_expression():
stream = (
tokens.LiteralToken("dimension", 1),
tokens.ValueToken(1),
tokens.LiteralToken("dimension_name", 1),
tokens.BlockStartToken(1),
tokens.BlockEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_block()
assert result == {"dimension": {"name": "dimension_name"}}
def test_parse_list_with_no_opening_bracket():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.LiteralToken("view_name.field_two", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result is None
def test_parse_value_quoted_literal_with_leftovers():
quoted_literal = "This is a quoted literal."
literal = "Some other tokens following."
stream = (
tokens.QuotedLiteralToken(quoted_literal, 1),
tokens.LiteralToken(literal, 1),
tokens.ValueToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_value()
assert result == quoted_literal
assert parser.index == 1
def parser():
stream = (
tokens.StreamStartToken(1),
tokens.LiteralToken("view", 1),
tokens.ValueToken(1),
tokens.LiteralToken("view_name", 1),
tokens.BlockStartToken(1),
tokens.LiteralToken("sql_table_name", 2),
tokens.ValueToken(2),
tokens.ExpressionBlockToken("schema.table_name", 2),
tokens.ExpressionBlockEndToken(2),
tokens.LiteralToken("drill_fields", 3),
tokens.ValueToken(3),
tokens.ListStartToken(3),
tokens.LiteralToken("view_name.field_one", 3),
tokens.CommaToken(3),
tokens.LiteralToken("view_name.field_two", 3),
tokens.ListEndToken(3),
tokens.BlockEndToken(4),
tokens.StreamEndToken(4),
)
return lkml.parser.Parser(stream)
def test_parse_list_with_trailing_comma():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.ListStartToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.CommaToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == {"drill_fields": ["view_name.field_one"]}
def test_parse_list_with_no_contents():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(),
)
# Add whitespace between brackets
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(prefix=" "),
)
def test_scan_with_complex_sql_block():
text = ("sql_distinct_key: concat(${orders.order_id}, '|', "
"${orders__items.primary_key}) ;;")
output = lkml.Lexer(text).scan()
assert output == (
tokens.StreamStartToken(1),
tokens.LiteralToken("sql_distinct_key", 1),
tokens.ValueToken(1),
tokens.ExpressionBlockToken(
" concat(${orders.order_id}, '|', ${orders__items.primary_key}) ",
1),
tokens.ExpressionBlockEndToken(1),
tokens.StreamEndToken(1),
)
def test_parse_list_with_only_commas():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.ListStartToken(1),
tokens.CommaToken(1),
tokens.CommaToken(1),
tokens.CommaToken(1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result is None
def test_parse_pair_with_quoted_literal():
stream = (
tokens.LiteralToken("view_label", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.QuotedLiteralToken("The View", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == PairNode(
type=SyntaxToken("view_label"), value=QuotedSyntaxToken("The View")
)
with pytest.raises(AttributeError):
result.prefix
def test_parse_pair_with_sql_block():
sql = " SELECT * FROM schema.table "
stream = (
tokens.LiteralToken("sql", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ExpressionBlockToken(sql, 1),
tokens.ExpressionBlockEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_pair()
assert result == PairNode(
type=SyntaxToken("sql"), value=ExpressionSyntaxToken(sql.strip())
)
def test_parse_list_with_only_comment():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.WhitespaceToken("\n ", 1),
tokens.CommentToken("# Put some fields here", 2),
tokens.WhitespaceToken("\n", 2),
tokens.ListEndToken(3),
tokens.StreamEndToken(3),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields"),
left_bracket=LeftBracket(),
items=tuple(),
right_bracket=RightBracket(prefix="\n # Put some fields here\n"),
)
def scan(self) -> Tuple[tokens.Token, ...]:
"""Tokenizes LookML into a sequence of tokens.
This method skips through the text being lexed until it finds a character that
indicates the start of a new token. It consumes the relevant characters and adds
the tokens to a sequence until it reaches the end of the text.
"""
self.tokens.append(tokens.StreamStartToken(self.line_number))
while True:
ch = self.peek()
if ch == "\0":
self.tokens.append(CHARACTER_TO_TOKEN[ch](self.line_number))
break
elif ch in "\n\t ":
self.tokens.append(self.scan_whitespace())
elif ch == "#":
self.advance()
self.tokens.append(self.scan_comment())
elif ch == ";":
if self.peek_multiple(2) == ";;":
self.advance(2)
self.tokens.append(CHARACTER_TO_TOKEN[ch](
self.line_number))
elif ch == '"':
self.advance()
self.tokens.append(self.scan_quoted_literal())
elif ch in CHARACTER_TO_TOKEN.keys():
self.advance()
self.tokens.append(CHARACTER_TO_TOKEN[ch](self.line_number))
elif self.check_for_expression_block(self.peek_multiple(25)):
# TODO: Handle edges here with whitespace and comments
self.tokens.append(self.scan_literal())
self.advance()
self.tokens.append(tokens.ValueToken(self.line_number))
self.tokens.append(self.scan_expression_block())
else:
# TODO: This should actually check for valid literals first
# and throw an error if it doesn't match
self.tokens.append(self.scan_literal())
return tuple(self.tokens)
def test_parse_block_with_no_expression():
stream = (
tokens.LiteralToken("dimension", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.LiteralToken("dimension_name", 1),
tokens.WhitespaceToken(" ", 1),
tokens.BlockStartToken(1),
tokens.BlockEndToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_block()
assert result == BlockNode(
type=SyntaxToken("dimension"),
name=SyntaxToken("dimension_name"),
left_brace=LeftCurlyBrace(prefix=" "),
container=ContainerNode(items=tuple()),
right_brace=RightCurlyBrace(suffix=" "),
)
def test_parse_list_with_leading_comma():
stream = (
tokens.LiteralToken("drill_fields", 1),
tokens.ValueToken(1),
tokens.WhitespaceToken(" ", 1),
tokens.ListStartToken(1),
tokens.CommaToken(1),
tokens.LiteralToken("view_name.field_one", 1),
tokens.ListEndToken(1),
tokens.StreamEndToken(1),
)
parser = lkml.parser.Parser(stream)
result = parser.parse_list()
assert result == ListNode(
type=SyntaxToken("drill_fields", 1),
colon=Colon(line_number=1, suffix=" "),
left_bracket=LeftBracket(),
items=(SyntaxToken("view_name.field_one", 1), ),
right_bracket=RightBracket(),
leading_comma=Comma(),
)
def test_parse_key_without_literal_token():
stream = (tokens.ValueToken(1), tokens.StreamEndToken(1))
parser = lkml.parser.Parser(stream)
result = parser.parse_key()
assert result is None
