def get_node_type_id(self, node):
if isinstance(node, ASTNode):
type_repr = typename(node.type)
return self.node_type_to_id[type_repr]
else:
# assert isinstance(node, str)
# it is a type
type_repr = typename(node)
return self.node_type_to_id[type_repr]
def get_node_type_id(self, node):
from astnode import ASTNode
if isinstance(node, ASTNode):
type_repr = typename(node.type)
return self.node_type_to_id[type_repr]
else:
# assert isinstance(node, str)
# it is a type
type_repr = typename(node)
return self.node_type_to_id[type_repr]
def pretty_print_helper(self, sb, depth, new_line=False):
if new_line:
sb.write('\n')
for i in xrange(depth):
sb.write(' ')
sb.write('(')
sb.write(typename(self.type))
if self.label is not None:
sb.write('{%s}' % self.label)
if self.value is not None:
sb.write('{val=%s}' % self.value)
if len(self.children) == 0:
sb.write(')')
return
sb.write(' ')
new_line = True
for child in self.children:
child.pretty_print_helper(sb, depth + 2, new_line)
sb.write('\n')
for i in xrange(depth):
sb.write(' ')
sb.write(')')
def pretty_print_helper(self, sb, depth, new_line=False):
if new_line:
sb.write('\n')
for i in xrange(depth): sb.write(' ')
sb.write('(')
sb.write(typename(self.type))
if self.label is not None:
sb.write('{%s}' % self.label)
if self.value is not None:
sb.write('{val=%s}' % self.value)
if len(self.children) == 0:
sb.write(')')
return
sb.write(' ')
new_line = True
for child in self.children:
child.pretty_print_helper(sb, depth + 2, new_line)
sb.write('\n')
for i in xrange(depth): sb.write(' ')
sb.write(')')
def __repr__(self):
parent = typename(self.type)
if self.label is not None:
parent += '{%s}' % self.label
if self.value is not None:
parent += '{val=%s}' % self.value
return '%s -> %s' % (parent, ', '.join([repr(c) for c in self.children]))
def __repr__(self):
parent = typename(self.type)
if self.label is not None:
parent += '{%s}' % self.label
if self.value is not None:
parent += '{val=%s}' % self.value
return '%s -> %s' % (parent, ', '.join([repr(c) for c in self.children]))
def __init__(self, rules):
"""
instantiate a grammar with a set of production rules of type Rule
"""
self.rules = rules
self.rule_index = defaultdict(list)
self.rule_to_id = OrderedDict()
node_types = set()
lhs_nodes = set()
rhs_nodes = set()
for rule in self.rules:
self.rule_index[rule.parent].append(rule)
# we also store all unique node types
for node in rule.nodes:
node_types.add(typename(node.type))
lhs_nodes.add(rule.parent)
for child in rule.children:
rhs_nodes.add(child.as_type_node)
root_node = lhs_nodes - rhs_nodes
print(self.rules)
assert len(root_node) == 1
self.root_node = next(iter(root_node))
self.terminal_nodes = rhs_nodes - lhs_nodes
self.terminal_types = set([n.type for n in self.terminal_nodes])
self.node_type_to_id = OrderedDict()
for i, type in enumerate(node_types, start=0):
self.node_type_to_id[type] = i
for gid, rule in enumerate(rules, start=0):
self.rule_to_id[rule] = gid
self.id_to_rule = OrderedDict(
(v, k) for (k, v) in self.rule_to_id.items())
logging.info('num. rules: %d', len(self.rules))
logging.info('num. types: %d', len(self.node_type_to_id))
logging.info('root: %s', self.root_node)
logging.info('terminals: %s',
', '.join(repr(n) for n in self.terminal_nodes))
def __repr__(self):
repr_str = ''
# if not self.is_leaf:
repr_str += '('
repr_str += typename(self.type)
if self.label is not None:
repr_str += '{%s}' % self.label
if self.value is not None:
repr_str += '{val=%s}' % self.value
# if not self.is_leaf:
for child in self.children:
repr_str += ' ' + child.__repr__()
repr_str += ')'
return repr_str
def __repr__(self):
repr_str = ''
# if not self.is_leaf:
repr_str += '('
repr_str += typename(self.type)
if self.label is not None:
repr_str += '{%s}' % self.label
if self.value is not None:
repr_str += '{val=%s}' % self.value
# if not self.is_leaf:
for child in self.children:
repr_str += ' ' + child.__repr__()
repr_str += ')'
return repr_str
def __init__(self, rules):
"""
instantiate a grammar with a set of production rules of type Rule
"""
self.rules = rules
self.rule_index = defaultdict(list)
self.rule_to_id = OrderedDict()
node_types = set()
lhs_nodes = set()
rhs_nodes = set()
for rule in self.rules:
self.rule_index[rule.parent].append(rule)
# we also store all unique node types
for node in rule.nodes:
node_types.add(typename(node.type))
lhs_nodes.add(rule.parent)
for child in rule.children:
rhs_nodes.add(child.as_type_node)
root_node = lhs_nodes - rhs_nodes
assert len(root_node) == 1
self.root_node = next(iter(root_node))
self.terminal_nodes = rhs_nodes - lhs_nodes
self.terminal_types = set([n.type for n in self.terminal_nodes])
self.node_type_to_id = OrderedDict()
for i, type in enumerate(node_types, start=0):
self.node_type_to_id[type] = i
for gid, rule in enumerate(rules, start=0):
self.rule_to_id[rule] = gid
self.id_to_rule = OrderedDict((v, k) for (k, v) in self.rule_to_id.iteritems())
logging.info('num. rules: %d', len(self.rules))
logging.info('num. types: %d', len(self.node_type_to_id))
logging.info('root: %s', self.root_node)
logging.info('terminals: %s', ', '.join(repr(n) for n in self.terminal_nodes))
def ast_tree_to_seq2tree_repr(tree):
repr_str = ''
# node_name = typename(tree.type)
label_val = '' if tree.label is None else tree.label
value = '' if tree.value is None else tree.value
node_name = '%s{%s}{%s}' % (typename(tree.type), label_val, value)
repr_str += node_name
# wrap children with parentheses
if tree.children:
repr_str += ' ('
for child in tree.children:
child_repr = ast_tree_to_seq2tree_repr(child)
repr_str += ' ' + child_repr
repr_str += ' )'
return repr_str
def parse_tree_to_python_ast(tree):
node_type = tree.type
node_label = tree.label
# remove root
if node_type == 'root':
return parse_tree_to_python_ast(tree.children[0])
ast_node = node_type()
node_type_name = typename(node_type)
# if it's a compositional AST node, populate its children nodes,
# fill fields with empty(default) values otherwise
if node_type_name in PY_AST_NODE_FIELDS:
fields_info = PY_AST_NODE_FIELDS[node_type_name]
for child_node in tree.children:
# if it's a compositional leaf
if child_node.type == 'epsilon':
continue
field_type = child_node.type
field_label = child_node.label
field_entry = fields_info[field_label]
is_list = field_entry['is_list']
if is_list:
field_type = field_entry['type']
field_value = []
if field_type in {
ast.comprehension, ast.excepthandler, ast.arguments,
ast.keyword, ast.alias
}:
nodes_in_list = child_node.children
for sub_node in nodes_in_list:
sub_node_ast = parse_tree_to_python_ast(sub_node)
field_value.append(sub_node_ast)
else: # expr stuffs
inter_nodes = child_node.children
for inter_node in inter_nodes:
if inter_node.value is None:
assert len(inter_node.children) == 1
sub_node_ast = parse_tree_to_python_ast(
inter_node.children[0])
field_value.append(sub_node_ast)
else:
assert len(inter_node.children) == 0
field_value.append(inter_node.value)
else:
# this node either holds a value, or is an non-terminal
if child_node.value is None:
assert len(child_node.children) == 1
field_value = parse_tree_to_python_ast(
child_node.children[0])
else:
assert child_node.is_leaf
field_value = child_node.value
setattr(ast_node, field_label, field_value)
for field in ast_node._fields:
if not hasattr(ast_node, field) and not field in NODE_FIELD_BLACK_LIST:
if fields_info and fields_info[field][
'is_list'] and not fields_info[field]['is_optional']:
setattr(ast_node, field, list())
else:
setattr(ast_node, field, None)
return ast_node
def python_ast_to_parse_tree(node):
assert isinstance(node, ast.AST)
node_type = type(node)
tree = ASTNode(node_type)
# it's a leaf AST node, e.g., ADD, Break, etc.
if len(node._fields) == 0:
return tree
# if it's a compositional AST node with empty fields
if is_compositional_leaf(node):
epsilon = ASTNode('epsilon')
tree.add_child(epsilon)
return tree
fields_info = PY_AST_NODE_FIELDS[node_type.__name__]
for field_name, field_value in ast.iter_fields(node):
# remove ctx stuff
if field_name in NODE_FIELD_BLACK_LIST:
continue
# omit empty fields, including empty lists
if field_value is None or (isinstance(field_value, list)
and len(field_value) == 0):
continue
# now it's not empty!
field_type = fields_info[field_name]['type']
is_list_field = fields_info[field_name]['is_list']
if isinstance(field_value, ast.AST):
child = ASTNode(field_type, field_name)
child.add_child(python_ast_to_parse_tree(field_value))
elif type(field_value) is str or type(field_value) is int or \
type(field_value) is float or type(field_value) is object or \
type(field_value) is bool:
# if field_type != type(field_value):
# print 'expect [%s] type, got [%s]' % (field_type, type(field_value))
child = ASTNode(type(field_value), field_name, value=field_value)
elif is_list_field:
list_node_type = typename(field_type) + '*'
child = ASTNode(list_node_type, field_name)
for n in field_value:
if field_type in {
ast.comprehension, ast.excepthandler, ast.arguments,
ast.keyword, ast.alias
}:
child.add_child(python_ast_to_parse_tree(n))
else:
intermediate_node = ASTNode(field_type)
if field_type is str:
intermediate_node.value = n
else:
intermediate_node.add_child(
python_ast_to_parse_tree(n))
child.add_child(intermediate_node)
else:
raise RuntimeError('unknown AST node field!')
tree.add_child(child)
return tree
