def _bnf_symbol(stream: TokenStream, gram: Grammar) -> RHSItem: """A single identifier or literal, or a parenthesized group""" if stream.peek().kind == TokenCat.LPAREN: stream.take() subseq = _bnf_rhs(stream, gram) require(stream, TokenCat.RPAREN, consume=True) # log.debug(f"Subsequence group: {subseq}") return subseq token = stream.take() if token.kind == TokenCat.STRING or token.kind == TokenCat.CHAR: # log.debug("Forming literal") return gram.literal(token.value[1:-1]) # Clips quotes elif token.kind == TokenCat.IDENT: # log.debug("Forming symbol") return gram.symbol(token.value) else: raise InputError(f"Unexpected input token {token.value}")
def _statement(stream: TokenStream, gram: Grammar): """ _statement == production | merge (left-factored for lookahead) """ require(stream, TokenCat.IDENT, desc="Statement should begin with symbol") lhs_ident = stream.take().value prod_type = stream.take() if prod_type.kind == TokenCat.BNFPROD: lhs_sym = gram.symbol(lhs_ident) rhs = _bnf_rhs(stream, gram) gram.add_cfg_prod(lhs_sym, rhs) elif prod_type.kind == TokenCat.BNFMERGE: merge_list = _merge_symbols(stream) # Merges are symmetric, so order doesn't matter merge_list.append(lhs_ident) gram.merge_symbols(merge_list) require(stream, TokenCat.TERMINATOR, "Statement must end with terminator", consume=True)