def _handle_default_rule_body(self, node: Rule, rhs: Rhs,
result_type: str) -> None:
memoize = self._should_memoize(node)
with self.indent():
self.add_level()
self._check_for_errors()
self.print(f"{result_type} _res = NULL;")
if memoize:
self.print(
f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{"
)
with self.indent():
self.add_return("_res")
self.print("}")
self.print("int _mark = p->mark;")
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
rhs,
is_loop=False,
is_gather=node.is_gather(),
rulename=node.name,
)
if self.debug:
self.print(
f'D(fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark));'
)
self.print("_res = NULL;")
self.print(" done:")
with self.indent():
if memoize:
self.print(
f"_PyPegen_insert_memo(p, _mark, {node.name}_type, _res);")
self.add_return("_res")
def visit_Rule(self, node: Rule) -> None:
is_loop = node.is_loop()
is_gather = node.is_gather()
rhs = node.flatten()
if is_loop or is_gather:
result_type = "asdl_seq *"
elif node.type:
result_type = node.type
else:
result_type = "void *"
for line in str(node).splitlines():
self.print(f"// {line}")
if node.left_recursive and node.leader:
self.print(f"static {result_type} {node.name}_raw(Parser *);")
self.print(f"static {result_type}")
self.print(f"{node.name}_rule(Parser *p)")
if node.left_recursive and node.leader:
self._set_up_rule_memoization(node, result_type)
self.print("{")
if is_loop:
self._handle_loop_rule_body(node, rhs)
else:
self._handle_default_rule_body(node, rhs, result_type)
self.print("}")
def visit_Rule(self, node: Rule) -> None:
is_loop = node.is_loop()
is_gather = node.is_gather()
rhs = node.flatten()
if node.left_recursive:
if node.leader:
self.print("@memoize_left_rec")
else:
# Non-leader rules in a cycle are not memoized,
# but they must still be logged.
self.print("@logger")
else:
self.print("@memoize")
node_type = node.type or "Any"
self.print(f"def {node.name}(self) -> Optional[{node_type}]:")
with self.indent():
self.print(f"# {node.name}: {rhs}")
self.print("mark = self._mark()")
if self.alts_uses_locations(node.rhs.alts):
self.print("tok = self._tokenizer.peek()")
self.print("start_lineno, start_col_offset = tok.start")
if is_loop:
self.print("children = []")
self.visit(rhs, is_loop=is_loop, is_gather=is_gather)
if is_loop:
self.print("return children")
else:
self.print("return None")
def visit_Rule(self, node: Rule) -> None:
is_loop = node.is_loop()
is_gather = node.is_gather()
rhs = node.flatten()
if node.left_recursive:
if node.leader:
self.print("@memoize_left_rec")
else:
# Non-leader rules in a cycle are not memoized,
# but they must still be logged.
self.print("@logger")
else:
self.print("@memoize")
node_type = node.type or "Any"
self.print(f"def {node.name}(self) -> Optional[{node_type}]:")
with self.indent():
self.print(f"# {node.name}: {rhs}")
if node.nullable:
self.print(f"# nullable={node.nullable}")
self.print("mark = self.mark()")
if is_loop:
self.print("children = []")
self.visit(rhs, is_loop=is_loop, is_gather=is_gather)
if is_loop:
self.print("return children")
else:
self.print("return None")
def _handle_default_rule_body(self, node: Rule, rhs: Rhs,
result_type: str) -> None:
memoize = self._should_memoize(node)
with self.indent():
self.print(f"{result_type} res = NULL;")
if memoize:
self.print(f"if (is_memoized(p, {node.name}_type, &res))")
with self.indent():
self.print("return res;")
self.print("int mark = p->mark;")
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
rhs,
is_loop=False,
is_gather=node.is_gather(),
rulename=node.name if memoize else None,
)
if self.debug:
self.print(
f'fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark);')
self.print("res = NULL;")
self.print(" done:")
with self.indent():
if memoize:
self.print(f"insert_memo(p, mark, {node.name}_type, res);")
self.print("return res;")
def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
memoize = self._should_memoize(node)
is_repeat1 = node.name.startswith("_loop1")
with self.indent():
self._check_for_errors()
self.print("void *_res = NULL;")
if memoize:
self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res))")
with self.indent():
self.print("return _res;")
self.print("int _mark = p->mark;")
self.print("int _start_mark = p->mark;")
self.print("void **_children = PyMem_Malloc(sizeof(void *));")
self.out_of_memory_return(f"!_children")
self.print("ssize_t _children_capacity = 1;")
self.print("ssize_t _n = 0;")
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
rhs, is_loop=True, is_gather=node.is_gather(), rulename=node.name,
)
if is_repeat1:
self.print("if (_n == 0 || p->error_indicator) {")
with self.indent():
self.print("PyMem_Free(_children);")
self.print("return NULL;")
self.print("}")
self.print("asdl_seq *_seq = _Py_asdl_seq_new(_n, p->arena);")
self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);")
self.print("for (int i = 0; i < _n; i++) asdl_seq_SET(_seq, i, _children[i]);")
self.print("PyMem_Free(_children);")
if node.name:
self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);")
self.print("return _seq;")
def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
memoize = self._should_memoize(node)
is_repeat1 = node.name.startswith("_loop1")
with self.indent():
self.print("if (p->error_indicator) {")
with self.indent():
self.print("return NULL;")
self.print("}")
self.print(f"void *res = NULL;")
if memoize:
self.print(
f"if (_PyPegen_is_memoized(p, {node.name}_type, &res))")
with self.indent():
self.print("return res;")
self.print("int mark = p->mark;")
self.print("int start_mark = p->mark;")
self.print("void **children = PyMem_Malloc(sizeof(void *));")
self.out_of_memory_return(f"!children", "NULL")
self.print("ssize_t children_capacity = 1;")
self.print("ssize_t n = 0;")
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
rhs,
is_loop=True,
is_gather=node.is_gather(),
rulename=node.name if memoize else None,
)
if is_repeat1:
self.print("if (n == 0) {")
with self.indent():
self.print("PyMem_Free(children);")
self.print("return NULL;")
self.print("}")
self.print("asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);")
self.out_of_memory_return(
f"!seq",
"NULL",
message=f"asdl_seq_new {node.name}",
cleanup_code="PyMem_Free(children);",
)
self.print(
"for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);"
)
self.print("PyMem_Free(children);")
if node.name:
self.print(
f"_PyPegen_insert_memo(p, start_mark, {node.name}_type, seq);"
)
self.print("return seq;")
def visit_Rule(self, node: Rule) -> None:
is_loop = node.is_loop()
is_gather = node.is_gather()
rhs = node.flatten()
if is_loop or is_gather:
result_type = "asdl_seq *"
elif node.type:
result_type = node.type
else:
result_type = "void *"
for line in str(node).splitlines():
self.print(f"// {line}")
if node.left_recursive and node.leader:
self.print(f"static {result_type} {node.name}_raw(Parser *);")
self.print(f"static {result_type}")
self.print(f"{node.name}_rule(Parser *p)")
if node.left_recursive and node.leader:
self._set_up_rule_memoization(node, result_type)
self.print("{")
if node.name.endswith("without_invalid"):
with self.indent():
self.print("int _prev_call_invalid = p->call_invalid_rules;")
self.print("p->call_invalid_rules = 0;")
self.cleanup_statements.append(
"p->call_invalid_rules = _prev_call_invalid;")
if is_loop:
self._handle_loop_rule_body(node, rhs)
else:
self._handle_default_rule_body(node, rhs, result_type)
if node.name.endswith("without_invalid"):
self.cleanup_statements.pop()
self.print("}")
def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
memoize = not node.left_recursive
is_repeat1 = node.name.startswith("_loop1")
with self.indent():
self.print(f"void *res = NULL;")
if memoize:
self.print(f"if (is_memoized(p, {node.name}_type, &res))")
with self.indent():
self.print("return res;")
self.print("int mark = p->mark;")
self.print("void **children = PyMem_Malloc(sizeof(void *));")
self.out_of_memory_return(f"!children", "NULL")
self.print("ssize_t children_capacity = 1;")
self.print("ssize_t n = 0;")
self._set_up_token_start_metadata_extraction()
self.visit(
rhs,
is_loop=True,
is_gather=node.is_gather(),
rulename=node.name if memoize else None,
)
if is_repeat1:
self.print("if (n == 0) {")
with self.indent():
self.print("PyMem_Free(children);")
self.print("return NULL;")
self.print("}")
self.print("asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);")
self.out_of_memory_return(f"!seq",
"NULL",
message=f"asdl_seq_new {node.name}")
self.print(
"for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);"
)
self.print("PyMem_Free(children);")
if node.name:
self.print(f"insert_memo(p, mark, {node.name}_type, seq);")
self.print("return seq;")
