def javaMethods(self, type, clsname, ctorname, fields, depth): # The java ctors token = asdl.Field('Token', 'token') token.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [token] + fields]) self.emit("public %s(%s) {" % (ctorname, fpargs), depth) self.emit("super(token);", depth+1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) tree = asdl.Field('PythonTree', 'tree') tree.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [tree] + fields]) self.emit("public %s(%s) {" % (ctorname, fpargs), depth) self.emit("super(tree);", depth+1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) if fpargs: fpargs += ", " # The toString() method self.emit("public String toString() {", depth) self.emit('return "%s";' % clsname, depth+1) self.emit("}", depth) self.emit("", 0) # The pickle() method #self.emit("public void pickle(DataOutputStream ostream) throws IOException {", depth) #self.emit("pickleThis(%s, ostream);" % type.index, depth+1); #for f in fields: # self.emit("pickleThis(this.%s, ostream);" % f.name, depth+1) #self.emit("}", depth) #self.emit("", 0) # The accept() method self.emit("public <R> R accept(VisitorIF<R> visitor) throws Exception {", depth) if clsname == ctorname: self.emit('return visitor.visit%s(this);' % clsname, depth+1) else: self.emit('traverse(visitor);' % clsname, depth+1) self.emit('return null;' % clsname, depth+1) self.emit("}", depth) self.emit("", 0) # The visitChildren() method self.emit("public void traverse(VisitorIF visitor) throws Exception {", depth) for f in fields: if self.bltinnames.has_key(str(f.type)): continue if f.typedef.simple: continue if f.seq: self.emit('if (%s != null) {' % f.name, depth+1) self.emit('for (int i = 0; i < %s.length; i++) {' % f.name, depth+2) self.emit('if (%s[i] != null)' % f.name, depth+3) self.emit('%s[i].accept(visitor);' % f.name, depth+4) self.emit('}', depth+2) self.emit('}', depth+1) else: self.emit('if (%s != null)' % f.name, depth+1) self.emit('%s.accept(visitor);' % f.name, depth+2) self.emit('}', depth) self.emit("", 0)
def copy_field(field): return asdl.Field(field.type, field.name, field.seq, field.opt)
def javaConstructors(self, type, name, clsname, is_product, fields, depth): self.emit("public %s(PyType subType) {" % (clsname), depth) self.emit("super(subType);", depth + 1) self.emit("}", depth) if len(fields) > 0: self.emit("public %s() {" % (clsname), depth) self.emit("this(TYPE);", depth + 1) self.emit("}", depth) fnames = ['"%s"' % f.name for f in fields] else: fnames = [] if str(name) in ('stmt', 'expr', 'excepthandler'): fnames.extend(['"lineno"', '"col_offset"']) fpargs = ", ".join(fnames) self.emit("@ExposedNew", depth) self.emit("@ExposedMethod", depth) self.emit( "public void %s___init__(PyObject[] args, String[] keywords) {" % clsname, depth) self.emit( 'ArgParser ap = new ArgParser("%s", args, keywords, new String[]' % clsname, depth + 1) self.emit('{%s}, %s);' % (fpargs, len(fields)), depth + 2) i = 0 for f in fields: self.emit( "set%s(ap.getPyObject(%s));" % (self.processFieldName(f.name), str(i)), depth + 1) i += 1 if str(name) in ('stmt', 'expr', 'excepthandler'): self.emit("int lin = ap.getInt(%s, -1);" % str(i), depth + 1) self.emit("if (lin != -1) {", depth + 1) self.emit("setLineno(lin);", depth + 2) self.emit("}", depth + 1) self.emit("", 0) self.emit("int col = ap.getInt(%s, -1);" % str(i + 1), depth + 1) self.emit("if (col != -1) {", depth + 1) self.emit("setLineno(col);", depth + 2) self.emit("}", depth + 1) self.emit("", 0) self.emit("}", depth) self.emit("", 0) fpargs = ", ".join(["PyObject %s" % f.name for f in fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) for f in fields: self.emit("set%s(%s);" % (self.processFieldName(f.name), f.name), depth + 1) self.emit("}", depth) self.emit("", 0) token = asdl.Field('Token', 'token') token.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [token] + fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(token);", depth + 1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) ttype = asdl.Field('int', 'ttype') ttype.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [ttype, token] + fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(ttype, token);", depth + 1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) tree = asdl.Field('PythonTree', 'tree') tree.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [tree] + fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(tree);", depth + 1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0)
def extract_location(self, typename, depth): row = self.decode_field(asdl.Field('int', 'lineno'), typename) column = self.decode_field(asdl.Field('int', 'col_offset'), typename) self.emit(f"let _location = ast::Location::new({row}, {column});", depth)
def javaConstructors(self, type, name, clsname, is_product, fields, depth): if len(fields) > 0: self.emit("public %s() {" % (clsname), depth) self.emit("super(TYPE);", depth + 1) self.emit("}", depth) fnames = ['"%s"' % f.name for f in fields] else: fnames = [] if str(name) in ('stmt', 'expr', 'excepthandler'): fnames.extend(['"lineno"', '"col_offset"']) fpargs = ", ".join(fnames) self.emit("@ExposedNew", depth) self.emit("@ExposedSlot(SlotFunc.NEW)", depth) self.emit( "public static PyObject %s_new(PyNewWrapper _new, boolean init, PyType subtype, PyObject[] args, String[] keywords) {" % clsname, depth) self.emit("return new %s(subtype);" % clsname, depth + 1) self.emit("}", depth) self.emit('@ExposedMethod(names={"__init__"})', depth) self.emit( "public void %s___init__(PyObject[] args, String[] keywords) {" % clsname, depth) self.emit( 'ArgParser ap = new ArgParser("%s", args, keywords, new String[]' % clsname, depth + 1) self.emit('{%s}, %s, true);' % (fpargs, len(fields)), depth + 2) i = 0 for f in fields: self.emit( "set%s(ap.getPyObject(%s, Py.None));" % (self.processFieldName(f.name), str(i)), depth + 1) i += 1 if str(name) in ('stmt', 'expr', 'excepthandler'): self.emit("PyObject lin = ap.getOptionalArg(%s);" % str(i), depth + 1) self.emit("if (lin != null) {", depth + 1) self.emit("lineno = lin;", depth + 2) self.emit("}", depth + 1) self.emit("", 0) self.emit("PyObject col = ap.getOptionalArg(%s);" % str(i + 1), depth + 1) self.emit("if (col != null) {", depth + 1) self.emit("col_offset = col;", depth + 2) self.emit("}", depth + 1) self.emit("", 0) self.emit("}", depth) self.emit("", 0) fpargs = ", ".join(["PyObject %s" % f.name for f in fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(TYPE);", depth + 1) for f in fields: self.emit("set%s(%s);" % (self.processFieldName(f.name), f.name), depth + 1) self.emit("}", depth) self.emit("", 0) self.emit("// called from derived class", depth) self.emit("public %s(PyType subtype) {" % clsname, depth) self.emit("super(subtype);", depth + 1) self.emit("}", depth) self.emit("", 0) node = asdl.Field('Node', 'token') node.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [node] + fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(TYPE, token);", depth + 1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) token = asdl.Field('Token', 'token') token.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [token] + fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(TYPE, token);", depth + 1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) tree = asdl.Field('PythonTree', 'tree') tree.typedef = False fpargs = ", ".join([self.fieldDef(f) for f in [tree] + fields]) self.emit("public %s(%s) {" % (clsname, fpargs), depth) self.emit("super(TYPE, tree);", depth + 1) self.javaConstructorHelper(fields, depth) self.emit("}", depth) self.emit("", 0) token = asdl.Field('Token', 'getToken()') fpargs = ", ".join([f"this.{f.name}" for f in [token] + fields]) self.emit(f"public {clsname} copy() {{", depth) self.emit(f"return new {clsname}({fpargs});", depth + 1) self.emit("}", depth) self.emit("", 0)
def _template_to_constructor(self, template_dict, suffix, seq_field): hole_node_types = {} # Find where the holes occur stack = [(None, template_dict['idiom'], None)] while stack: parent, node, child_index = stack.pop() node_type, ref_symbols, hole_id, children = node if hole_id is not None: assert hole_id not in hole_node_types # node_type could be: # - name of field # => hole type is same as field's type # - name of type, if it only has one child # - binarizer hyphenated_node_type = None unhyphenated_node_type = None hole_type_str = template_dict['holes'][hole_id]['type'] if hole_type_str == 'AddChild': node_type_for_field_type = node_type elif hole_type_str == 'ReplaceSelf': # Two types of ReplaceSelf # 1. Node has a hyphen: should be a repeated field # 2. Node lacks a hyphen, and # 2a. node is same as parent: a repeated field # 2b. node is not the same as parent: an elem if '-' in node_type: node_type_for_field_type = node_type else: node_type_for_field_type = parent[0] #if '-' in parent_type: # hyphenated_node_type = parent_type #else: # unhyphenated_node_type = parent_type field_info = self._get_field_info_from_name( node_type_for_field_type) # Check for situations like # Call-args # | \ # List[0] Call-args[1] # # Tuple # | \ # Tuple[0] Tuple[1] # where hole 0 should not be a sequence. if field_info.seq and hole_type_str == 'ReplaceSelf' and '-' not in node_type: assert child_index in (0, 1) seq = child_index == 1 else: seq = field_info.seq hole_node_types[hole_id] = (field_info.type, seq, field_info.opt) stack += [(node, child, i) for i, child in enumerate(children)] # Create fields for the holes fields = [] for hole in template_dict['holes']: i = hole['id'] field_type, seq, opt = hole_node_types[i] field = asdl.Field(type=field_type, name='hole{}'.format(i), seq=seq, opt=opt) field.hole_type = HoleType[hole['type']] fields.append(field) constructor = asdl.Constructor( 'Template{}{}'.format(template_dict['id'], suffix), fields) constructor.template = self.convert_idiom_ast( template_dict['idiom'], template_id=template_dict['id'], seq_field=seq_field) return constructor
def run_iteration(self, trees): triple_occurrences, node_type_counts = self.count_triples(trees) self.pre_iteration_counts.append(node_type_counts) # Most frequent most_freq_triple, most_freq_occurrences = max( triple_occurrences.items(), key=lambda kv: len(kv[1])) if len(most_freq_occurrences) == 1: raise Exception('No more work to do!') existing_type_name, field_name, field_info = most_freq_triple tuple_name = field_name if isinstance(field_name, tuple) else (field_name, ) existing_type = self.type_infos[existing_type_name] existing_field = existing_type.unset_fields[field_name] promoted_fields = [] promoted_seq_elem_counts = collections.Counter() promoted_preset_fields = {} if isinstance(field_info, Primitive) or field_info is None: pass else: # Figure out which fields of type `field_info` should be promoted # Example: # most_freq_triple = ('Call', 'func', 'Name') # field_info = 'Name' # type_infos['Name'].unset_fields = {'id': Field(identifier, id)} for is_preset, (field_field_name, field_field) in itertools.chain( zip(itertools.repeat(False), self.type_infos[field_info].unset_fields.items()), zip(itertools.repeat(True), self.type_infos[field_info].preset_fields.items())): if isinstance(field_field_name, tuple): field_field_tuple_name = field_field_name else: field_field_tuple_name = (field_field_name, ) if existing_field.seq: suffix = (existing_type.preset_seq_elem_counts[tuple_name], ) + field_field_tuple_name else: suffix = field_field_tuple_name new_name = tuple_name + suffix if isinstance(field_field, asdl.Field): new_field = asdl.Field(type=field_field.type, name=new_name, seq=field_field.seq, opt=field_field.opt) else: new_field = field_field if is_preset: promoted_preset_fields[new_name] = new_field else: promoted_fields.append((field_field, new_field)) seq_elem_count = self.type_infos[ field_info].preset_seq_elem_counts[field_field_tuple_name] if seq_elem_count: promoted_seq_elem_counts[new_name] = seq_elem_count # Create a new type new_preset_fields = { **existing_type.preset_fields, **promoted_preset_fields } new_preset_seq_elem_counts = existing_type.preset_seq_elem_counts + promoted_seq_elem_counts if existing_field.seq and field_info is not None: new_preset_fields[tuple_name + ( new_preset_seq_elem_counts[tuple_name], )] = field_info new_preset_seq_elem_counts[tuple_name] += 1 else: new_preset_fields[tuple_name] = field_info new_unset_fields = { **{f.name: f for old_field, f in promoted_fields}, **existing_type.unset_fields } if field_info is None or not existing_field.seq: # Only unset if... # - field is not sequential # - field has been set to None, meaning the end of a sequence del new_unset_fields[field_name] new_type = TypeInfo(name='Type{:04d}_{}'.format( self.iterations_finished, existing_type.base_name), base_name=existing_type.base_name, predecessor_name=existing_type.name, predecessor_triple=most_freq_triple, unset_fields=new_unset_fields, preset_fields=new_preset_fields, preset_seq_elem_counts=new_preset_seq_elem_counts) self.type_infos[new_type.name] = new_type self.created_types.append(new_type) self.type_graph.add_edge(new_type.name, existing_type.name) self.iterations_finished += 1 # Tracks which occurrences have been removed due to promotion. discarded = IdentitySet() for occ in most_freq_occurrences: if occ in discarded: continue occ['_type'] = new_type.name def delete_obsoleted_field(): if existing_field.seq: # todo: change 0 if we can promote other elements del occ[field_name][0] if not occ[field_name]: del occ[field_name] else: del occ[field_name] if isinstance(field_info, Primitive): delete_obsoleted_field() elif field_info is None: pass else: if existing_field.seq: # todo: change 0 if we can promote other elements value_to_promote = occ[field_name][0] else: value_to_promote = occ[field_name] delete_obsoleted_field() discarded.add(value_to_promote) for old_field, new_field in promoted_fields: if old_field.name not in value_to_promote: assert old_field.opt or old_field.seq continue occ[new_field.name] = value_to_promote[old_field.name] assert occ[new_field.name]