def get_node_type_id(self, node): if isinstance(node, ASTNode): type_repr = typename(node.type) return self.node_type_to_id[type_repr] else: # assert isinstance(node, str) # it is a type type_repr = typename(node) return self.node_type_to_id[type_repr]
def get_node_type_id(self, node): from astnode import ASTNode if isinstance(node, ASTNode): type_repr = typename(node.type) return self.node_type_to_id[type_repr] else: # assert isinstance(node, str) # it is a type type_repr = typename(node) return self.node_type_to_id[type_repr]
def pretty_print_helper(self, sb, depth, new_line=False): if new_line: sb.write('\n') for i in xrange(depth): sb.write(' ') sb.write('(') sb.write(typename(self.type)) if self.label is not None: sb.write('{%s}' % self.label) if self.value is not None: sb.write('{val=%s}' % self.value) if len(self.children) == 0: sb.write(')') return sb.write(' ') new_line = True for child in self.children: child.pretty_print_helper(sb, depth + 2, new_line) sb.write('\n') for i in xrange(depth): sb.write(' ') sb.write(')')
def pretty_print_helper(self, sb, depth, new_line=False): if new_line: sb.write('\n') for i in xrange(depth): sb.write(' ') sb.write('(') sb.write(typename(self.type)) if self.label is not None: sb.write('{%s}' % self.label) if self.value is not None: sb.write('{val=%s}' % self.value) if len(self.children) == 0: sb.write(')') return sb.write(' ') new_line = True for child in self.children: child.pretty_print_helper(sb, depth + 2, new_line) sb.write('\n') for i in xrange(depth): sb.write(' ') sb.write(')')
def __repr__(self): parent = typename(self.type) if self.label is not None: parent += '{%s}' % self.label if self.value is not None: parent += '{val=%s}' % self.value return '%s -> %s' % (parent, ', '.join([repr(c) for c in self.children]))
def __repr__(self): parent = typename(self.type) if self.label is not None: parent += '{%s}' % self.label if self.value is not None: parent += '{val=%s}' % self.value return '%s -> %s' % (parent, ', '.join([repr(c) for c in self.children]))
def __init__(self, rules): """ instantiate a grammar with a set of production rules of type Rule """ self.rules = rules self.rule_index = defaultdict(list) self.rule_to_id = OrderedDict() node_types = set() lhs_nodes = set() rhs_nodes = set() for rule in self.rules: self.rule_index[rule.parent].append(rule) # we also store all unique node types for node in rule.nodes: node_types.add(typename(node.type)) lhs_nodes.add(rule.parent) for child in rule.children: rhs_nodes.add(child.as_type_node) root_node = lhs_nodes - rhs_nodes print(self.rules) assert len(root_node) == 1 self.root_node = next(iter(root_node)) self.terminal_nodes = rhs_nodes - lhs_nodes self.terminal_types = set([n.type for n in self.terminal_nodes]) self.node_type_to_id = OrderedDict() for i, type in enumerate(node_types, start=0): self.node_type_to_id[type] = i for gid, rule in enumerate(rules, start=0): self.rule_to_id[rule] = gid self.id_to_rule = OrderedDict( (v, k) for (k, v) in self.rule_to_id.items()) logging.info('num. rules: %d', len(self.rules)) logging.info('num. types: %d', len(self.node_type_to_id)) logging.info('root: %s', self.root_node) logging.info('terminals: %s', ', '.join(repr(n) for n in self.terminal_nodes))
def __repr__(self): repr_str = '' # if not self.is_leaf: repr_str += '(' repr_str += typename(self.type) if self.label is not None: repr_str += '{%s}' % self.label if self.value is not None: repr_str += '{val=%s}' % self.value # if not self.is_leaf: for child in self.children: repr_str += ' ' + child.__repr__() repr_str += ')' return repr_str
def __repr__(self): repr_str = '' # if not self.is_leaf: repr_str += '(' repr_str += typename(self.type) if self.label is not None: repr_str += '{%s}' % self.label if self.value is not None: repr_str += '{val=%s}' % self.value # if not self.is_leaf: for child in self.children: repr_str += ' ' + child.__repr__() repr_str += ')' return repr_str
def __init__(self, rules): """ instantiate a grammar with a set of production rules of type Rule """ self.rules = rules self.rule_index = defaultdict(list) self.rule_to_id = OrderedDict() node_types = set() lhs_nodes = set() rhs_nodes = set() for rule in self.rules: self.rule_index[rule.parent].append(rule) # we also store all unique node types for node in rule.nodes: node_types.add(typename(node.type)) lhs_nodes.add(rule.parent) for child in rule.children: rhs_nodes.add(child.as_type_node) root_node = lhs_nodes - rhs_nodes assert len(root_node) == 1 self.root_node = next(iter(root_node)) self.terminal_nodes = rhs_nodes - lhs_nodes self.terminal_types = set([n.type for n in self.terminal_nodes]) self.node_type_to_id = OrderedDict() for i, type in enumerate(node_types, start=0): self.node_type_to_id[type] = i for gid, rule in enumerate(rules, start=0): self.rule_to_id[rule] = gid self.id_to_rule = OrderedDict((v, k) for (k, v) in self.rule_to_id.iteritems()) logging.info('num. rules: %d', len(self.rules)) logging.info('num. types: %d', len(self.node_type_to_id)) logging.info('root: %s', self.root_node) logging.info('terminals: %s', ', '.join(repr(n) for n in self.terminal_nodes))
def ast_tree_to_seq2tree_repr(tree): repr_str = '' # node_name = typename(tree.type) label_val = '' if tree.label is None else tree.label value = '' if tree.value is None else tree.value node_name = '%s{%s}{%s}' % (typename(tree.type), label_val, value) repr_str += node_name # wrap children with parentheses if tree.children: repr_str += ' (' for child in tree.children: child_repr = ast_tree_to_seq2tree_repr(child) repr_str += ' ' + child_repr repr_str += ' )' return repr_str
def parse_tree_to_python_ast(tree): node_type = tree.type node_label = tree.label # remove root if node_type == 'root': return parse_tree_to_python_ast(tree.children[0]) ast_node = node_type() node_type_name = typename(node_type) # if it's a compositional AST node, populate its children nodes, # fill fields with empty(default) values otherwise if node_type_name in PY_AST_NODE_FIELDS: fields_info = PY_AST_NODE_FIELDS[node_type_name] for child_node in tree.children: # if it's a compositional leaf if child_node.type == 'epsilon': continue field_type = child_node.type field_label = child_node.label field_entry = fields_info[field_label] is_list = field_entry['is_list'] if is_list: field_type = field_entry['type'] field_value = [] if field_type in { ast.comprehension, ast.excepthandler, ast.arguments, ast.keyword, ast.alias }: nodes_in_list = child_node.children for sub_node in nodes_in_list: sub_node_ast = parse_tree_to_python_ast(sub_node) field_value.append(sub_node_ast) else: # expr stuffs inter_nodes = child_node.children for inter_node in inter_nodes: if inter_node.value is None: assert len(inter_node.children) == 1 sub_node_ast = parse_tree_to_python_ast( inter_node.children[0]) field_value.append(sub_node_ast) else: assert len(inter_node.children) == 0 field_value.append(inter_node.value) else: # this node either holds a value, or is an non-terminal if child_node.value is None: assert len(child_node.children) == 1 field_value = parse_tree_to_python_ast( child_node.children[0]) else: assert child_node.is_leaf field_value = child_node.value setattr(ast_node, field_label, field_value) for field in ast_node._fields: if not hasattr(ast_node, field) and not field in NODE_FIELD_BLACK_LIST: if fields_info and fields_info[field][ 'is_list'] and not fields_info[field]['is_optional']: setattr(ast_node, field, list()) else: setattr(ast_node, field, None) return ast_node
def python_ast_to_parse_tree(node): assert isinstance(node, ast.AST) node_type = type(node) tree = ASTNode(node_type) # it's a leaf AST node, e.g., ADD, Break, etc. if len(node._fields) == 0: return tree # if it's a compositional AST node with empty fields if is_compositional_leaf(node): epsilon = ASTNode('epsilon') tree.add_child(epsilon) return tree fields_info = PY_AST_NODE_FIELDS[node_type.__name__] for field_name, field_value in ast.iter_fields(node): # remove ctx stuff if field_name in NODE_FIELD_BLACK_LIST: continue # omit empty fields, including empty lists if field_value is None or (isinstance(field_value, list) and len(field_value) == 0): continue # now it's not empty! field_type = fields_info[field_name]['type'] is_list_field = fields_info[field_name]['is_list'] if isinstance(field_value, ast.AST): child = ASTNode(field_type, field_name) child.add_child(python_ast_to_parse_tree(field_value)) elif type(field_value) is str or type(field_value) is int or \ type(field_value) is float or type(field_value) is object or \ type(field_value) is bool: # if field_type != type(field_value): # print 'expect [%s] type, got [%s]' % (field_type, type(field_value)) child = ASTNode(type(field_value), field_name, value=field_value) elif is_list_field: list_node_type = typename(field_type) + '*' child = ASTNode(list_node_type, field_name) for n in field_value: if field_type in { ast.comprehension, ast.excepthandler, ast.arguments, ast.keyword, ast.alias }: child.add_child(python_ast_to_parse_tree(n)) else: intermediate_node = ASTNode(field_type) if field_type is str: intermediate_node.value = n else: intermediate_node.add_child( python_ast_to_parse_tree(n)) child.add_child(intermediate_node) else: raise RuntimeError('unknown AST node field!') tree.add_child(child) return tree