def _parse_es_api_call(self): self._publish_event(ParserEventType.ES_METHOD) method_token = self._consume_token(HttpMethod) method_node = NameNode(method_token) if method_token.value.upper() not in HTTP_METHODS: raise PeekSyntaxError( self.text, method_token, title='Invalid HTTP method', message= f'Expect HTTP method of value in {HTTP_METHODS!r}, got {method_token.value!r}' ) if self._peek_token().ttype is Literal: self._publish_event(ParserEventType.ES_URL) path_node = TextNode(self._consume_token(Literal)) elif self._peek_token().ttype is ParenLeft: self._publish_event(ParserEventType.BEFORE_ES_URL_EXPR) path_node = self._parse_expr() self._publish_event(ParserEventType.AFTER_ES_URL_EXPR) else: raise PeekSyntaxError( self.text, self._peek_token(), message= 'HTTP path must be either text literal or an expression enclosed by parenthesis' ) option_nodes = [] while self._peek_token().ttype is OptionName: self._publish_event(ParserEventType.ES_OPTION_NAME) n = NameNode(self._consume_token(OptionName)) self._consume_token(Assign) self._publish_event(ParserEventType.BEFORE_ES_OPTION_VALUE) option_nodes.append(KeyValueNode(n, self._parse_expr())) self._publish_event(ParserEventType.AFTER_ES_OPTION_VALUE) if self._peek_token().ttype is At: self._publish_event(ParserEventType.ES_PAYLOAD_FILE_AT) self._consume_token(At) return EsApiCallFilePayloadNode( method_node, path_node, DictNode(option_nodes), TextNode(self._consume_token(Literal))) else: dict_nodes = [] while self._peek_token().ttype is not EOF: if self._peek_token().ttype is CurlyLeft: self._publish_event( ParserEventType.BEFORE_ES_PAYLOAD_INLINE) dict_nodes.append(self._parse_dict()) self._publish_event( ParserEventType.AFTER_ES_PAYLOAD_INLINE) else: break return EsApiCallInlinePayloadNode(method_node, path_node, DictNode(option_nodes), dict_nodes)
def _consume_token(self, ttype, value=None) -> PeekToken: token = self._peek_token() self.position += 1 if token.ttype is not ttype: raise PeekSyntaxError( self.text, token, message=f'Expect token of type {ttype!r}, got {token.ttype!r}') if value and (token.value != value or token.value not in value): raise PeekSyntaxError( self.text, token, message=f'Expect token of value {value!r}, got {token.value!r}' ) self._publish_event(ParserEventType.AFTER_TOKEN, token) return token
def _parse_func_call_args(self, is_stmt=True): self._publish_event(ParserEventType.BEFORE_FUNC_ARGS) symbol_nodes = [] arg_nodes = [] kwarg_nodes = [] while self._peek_token().ttype is not EOF: if self._peek_token().ttype is BlankLine: self._consume_token(BlankLine) if is_stmt: break elif self._peek_token().ttype is ParenRight: if is_stmt: raise PeekSyntaxError( self.text, self._peek_token(), message= 'Found function expression while parsing for function stmt' ) else: break elif self._peek_token().ttype is Name: self._publish_event(ParserEventType.FUNC_OPTION_NAME_OR_ARG) n = NameNode(self._consume_token(Name)) if self._peek_token().ttype is Assign: self._consume_token(Assign) self._publish_event( ParserEventType.BEFORE_FUNC_OPTION_VALUE) kwarg_nodes.append(KeyValueNode(n, self._parse_expr())) self._publish_event( ParserEventType.AFTER_FUNC_OPTION_VALUE) elif self._peek_token( ).ttype is ParenLeft: # nested function expr self._consume_token(ParenLeft) sub_symbol_nodes, sub_arg_nodes, sub_kwarg_nodes = self._parse_func_call_args( is_stmt=False) arg_nodes.append( FuncCallNode(n, ArrayNode(sub_symbol_nodes), ArrayNode(sub_arg_nodes), DictNode(sub_kwarg_nodes), is_stmt=False)) self._consume_token(ParenRight) else: arg_nodes.append(self._parse_expr_after_left_operand(n)) elif self._peek_token().ttype is At: self._publish_event(ParserEventType.BEFORE_FUNC_SYMBOL_ARG) self._consume_token(At) symbol_nodes.append(SymbolNode(self._consume_token(Literal))) self._publish_event(ParserEventType.AFTER_FUNC_SYMBOL_ARG) else: self._publish_event(ParserEventType.BEFORE_FUNC_REGULAR_ARG) arg_nodes.append(self._parse_expr()) self._publish_event(ParserEventType.AFTER_FUNC_REGULAR_ARG) self._publish_event(ParserEventType.AFTER_FUNC_ARGS) return symbol_nodes, arg_nodes, kwarg_nodes
def get_tokens_unprocessed(self, text, stack=None) -> Iterable[PeekToken]: """ Convert DictKey to common string if it is part of an expression """ stack = stack or self.stack stream = super().get_tokens_unprocessed(text, stack) buffer = [] while True: try: pt = PeekToken(*next(stream)) if pt.ttype is DictKey: assert 0 == len(buffer) buffer.append(pt) while True: pt_next = PeekToken(*next(stream)) if pt_next.ttype in (DictKey, Whitespace, Comment.Single): buffer.append(pt_next) elif pt_next.ttype is Colon: buffer.append(pt_next) for t in buffer: yield t buffer = [] break else: buffer.append(pt_next) first_token = buffer[0] if first_token.value == "'": actual_type = String.Single elif first_token.value == '"': actual_type = String.Double elif first_token.value == "'''": actual_type = String.TripleS elif first_token.value == '"""': actual_type = String.TripleD else: raise PeekSyntaxError(text, first_token, message='DictKey expected') for t in buffer: if t.ttype is DictKey: t = PeekToken(t.index, actual_type, t.value) yield t buffer = [] break else: yield pt except StopIteration: if buffer: for t in buffer: yield t break
def _parse_value(self): token = self._peek_token() if token.ttype is CurlyLeft: return self._parse_dict() elif token.ttype is BracketLeft: return self._parse_array() elif token.ttype in Number: return NumberNode(self._consume_token(token.ttype)) elif token.ttype is Name.Builtin: return TextNode(self._consume_token(token.ttype)) elif token.ttype in (String.Double, String.Single, TripleS, TripleD): return StringNode(self._consume_token(token.ttype)) else: raise PeekSyntaxError( self.text, token, message=f'Unexpected token when parsing for value: {token!r}')
def _parse_stmt(self): token = self._peek_token() if token.ttype is FuncName: return self._parse_func_call() elif token.ttype is HttpMethod: return self._parse_es_api_call() elif token.ttype is Let: return self._parse_let_stmt() elif token.ttype is ShellOut: return self._parse_shell_out() elif token.ttype is For: return self._parse_for_stmt() else: raise PeekSyntaxError(self.text, token, title='Invalid token', message='Expect beginning of a statement')
def parse(self, text, payload_only=False, fail_fast_on_error_token=True, last_stmt_only=False, log_level=None): saved_log_level = _logger.getEffectiveLevel() try: if log_level is not None: _logger.setLevel(log_level) self.text = text self.position = 0 self.tokens = [] stack = ('dict', ) if payload_only else ('root', ) self.tokens = process_tokens( self.lexer.get_tokens_unprocessed(self.text, stack=stack)) if last_stmt_only: idx_last_stmt_token = find_last_stmt_token(self.tokens) if idx_last_stmt_token == -1: return [] else: self.position = idx_last_stmt_token if fail_fast_on_error_token: for token in self.tokens[self.position:]: if token.ttype in Token.Error: raise PeekSyntaxError( self.text, token, message='Found error token and fail fast is enabled' ) return self._do_parse_payload( ) if payload_only else self._do_parse() finally: if log_level is not None: _logger.setLevel(saved_log_level)