def _parse_es_api_call(self):
self._publish_event(ParserEventType.ES_METHOD)
method_token = self._consume_token(HttpMethod)
method_node = NameNode(method_token)
if method_token.value.upper() not in HTTP_METHODS:
raise PeekSyntaxError(
self.text,
method_token,
title='Invalid HTTP method',
message=
f'Expect HTTP method of value in {HTTP_METHODS!r}, got {method_token.value!r}'
)
if self._peek_token().ttype is Literal:
self._publish_event(ParserEventType.ES_URL)
path_node = TextNode(self._consume_token(Literal))
elif self._peek_token().ttype is ParenLeft:
self._publish_event(ParserEventType.BEFORE_ES_URL_EXPR)
path_node = self._parse_expr()
self._publish_event(ParserEventType.AFTER_ES_URL_EXPR)
else:
raise PeekSyntaxError(
self.text,
self._peek_token(),
message=
'HTTP path must be either text literal or an expression enclosed by parenthesis'
)
option_nodes = []
while self._peek_token().ttype is OptionName:
self._publish_event(ParserEventType.ES_OPTION_NAME)
n = NameNode(self._consume_token(OptionName))
self._consume_token(Assign)
self._publish_event(ParserEventType.BEFORE_ES_OPTION_VALUE)
option_nodes.append(KeyValueNode(n, self._parse_expr()))
self._publish_event(ParserEventType.AFTER_ES_OPTION_VALUE)
if self._peek_token().ttype is At:
self._publish_event(ParserEventType.ES_PAYLOAD_FILE_AT)
self._consume_token(At)
return EsApiCallFilePayloadNode(
method_node, path_node, DictNode(option_nodes),
TextNode(self._consume_token(Literal)))
else:
dict_nodes = []
while self._peek_token().ttype is not EOF:
if self._peek_token().ttype is CurlyLeft:
self._publish_event(
ParserEventType.BEFORE_ES_PAYLOAD_INLINE)
dict_nodes.append(self._parse_dict())
self._publish_event(
ParserEventType.AFTER_ES_PAYLOAD_INLINE)
else:
break
return EsApiCallInlinePayloadNode(method_node, path_node,
DictNode(option_nodes),
dict_nodes)
def _consume_token(self, ttype, value=None) -> PeekToken:
token = self._peek_token()
self.position += 1
if token.ttype is not ttype:
raise PeekSyntaxError(
self.text,
token,
message=f'Expect token of type {ttype!r}, got {token.ttype!r}')
if value and (token.value != value or token.value not in value):
raise PeekSyntaxError(
self.text,
token,
message=f'Expect token of value {value!r}, got {token.value!r}'
)
self._publish_event(ParserEventType.AFTER_TOKEN, token)
return token
def _parse_func_call_args(self, is_stmt=True):
self._publish_event(ParserEventType.BEFORE_FUNC_ARGS)
symbol_nodes = []
arg_nodes = []
kwarg_nodes = []
while self._peek_token().ttype is not EOF:
if self._peek_token().ttype is BlankLine:
self._consume_token(BlankLine)
if is_stmt:
break
elif self._peek_token().ttype is ParenRight:
if is_stmt:
raise PeekSyntaxError(
self.text,
self._peek_token(),
message=
'Found function expression while parsing for function stmt'
)
else:
break
elif self._peek_token().ttype is Name:
self._publish_event(ParserEventType.FUNC_OPTION_NAME_OR_ARG)
n = NameNode(self._consume_token(Name))
if self._peek_token().ttype is Assign:
self._consume_token(Assign)
self._publish_event(
ParserEventType.BEFORE_FUNC_OPTION_VALUE)
kwarg_nodes.append(KeyValueNode(n, self._parse_expr()))
self._publish_event(
ParserEventType.AFTER_FUNC_OPTION_VALUE)
elif self._peek_token(
).ttype is ParenLeft: # nested function expr
self._consume_token(ParenLeft)
sub_symbol_nodes, sub_arg_nodes, sub_kwarg_nodes = self._parse_func_call_args(
is_stmt=False)
arg_nodes.append(
FuncCallNode(n,
ArrayNode(sub_symbol_nodes),
ArrayNode(sub_arg_nodes),
DictNode(sub_kwarg_nodes),
is_stmt=False))
self._consume_token(ParenRight)
else:
arg_nodes.append(self._parse_expr_after_left_operand(n))
elif self._peek_token().ttype is At:
self._publish_event(ParserEventType.BEFORE_FUNC_SYMBOL_ARG)
self._consume_token(At)
symbol_nodes.append(SymbolNode(self._consume_token(Literal)))
self._publish_event(ParserEventType.AFTER_FUNC_SYMBOL_ARG)
else:
self._publish_event(ParserEventType.BEFORE_FUNC_REGULAR_ARG)
arg_nodes.append(self._parse_expr())
self._publish_event(ParserEventType.AFTER_FUNC_REGULAR_ARG)
self._publish_event(ParserEventType.AFTER_FUNC_ARGS)
return symbol_nodes, arg_nodes, kwarg_nodes
def get_tokens_unprocessed(self, text, stack=None) -> Iterable[PeekToken]:
"""
Convert DictKey to common string if it is part of an expression
"""
stack = stack or self.stack
stream = super().get_tokens_unprocessed(text, stack)
buffer = []
while True:
try:
pt = PeekToken(*next(stream))
if pt.ttype is DictKey:
assert 0 == len(buffer)
buffer.append(pt)
while True:
pt_next = PeekToken(*next(stream))
if pt_next.ttype in (DictKey, Whitespace, Comment.Single):
buffer.append(pt_next)
elif pt_next.ttype is Colon:
buffer.append(pt_next)
for t in buffer:
yield t
buffer = []
break
else:
buffer.append(pt_next)
first_token = buffer[0]
if first_token.value == "'":
actual_type = String.Single
elif first_token.value == '"':
actual_type = String.Double
elif first_token.value == "'''":
actual_type = String.TripleS
elif first_token.value == '"""':
actual_type = String.TripleD
else:
raise PeekSyntaxError(text, first_token, message='DictKey expected')
for t in buffer:
if t.ttype is DictKey:
t = PeekToken(t.index, actual_type, t.value)
yield t
buffer = []
break
else:
yield pt
except StopIteration:
if buffer:
for t in buffer:
yield t
break
def _parse_value(self):
token = self._peek_token()
if token.ttype is CurlyLeft:
return self._parse_dict()
elif token.ttype is BracketLeft:
return self._parse_array()
elif token.ttype in Number:
return NumberNode(self._consume_token(token.ttype))
elif token.ttype is Name.Builtin:
return TextNode(self._consume_token(token.ttype))
elif token.ttype in (String.Double, String.Single, TripleS, TripleD):
return StringNode(self._consume_token(token.ttype))
else:
raise PeekSyntaxError(
self.text,
token,
message=f'Unexpected token when parsing for value: {token!r}')
def _parse_stmt(self):
token = self._peek_token()
if token.ttype is FuncName:
return self._parse_func_call()
elif token.ttype is HttpMethod:
return self._parse_es_api_call()
elif token.ttype is Let:
return self._parse_let_stmt()
elif token.ttype is ShellOut:
return self._parse_shell_out()
elif token.ttype is For:
return self._parse_for_stmt()
else:
raise PeekSyntaxError(self.text,
token,
title='Invalid token',
message='Expect beginning of a statement')
def parse(self,
text,
payload_only=False,
fail_fast_on_error_token=True,
last_stmt_only=False,
log_level=None):
saved_log_level = _logger.getEffectiveLevel()
try:
if log_level is not None:
_logger.setLevel(log_level)
self.text = text
self.position = 0
self.tokens = []
stack = ('dict', ) if payload_only else ('root', )
self.tokens = process_tokens(
self.lexer.get_tokens_unprocessed(self.text, stack=stack))
if last_stmt_only:
idx_last_stmt_token = find_last_stmt_token(self.tokens)
if idx_last_stmt_token == -1:
return []
else:
self.position = idx_last_stmt_token
if fail_fast_on_error_token:
for token in self.tokens[self.position:]:
if token.ttype in Token.Error:
raise PeekSyntaxError(
self.text,
token,
message='Found error token and fail fast is enabled'
)
return self._do_parse_payload(
) if payload_only else self._do_parse()
finally:
if log_level is not None:
_logger.setLevel(saved_log_level)