def pushTok(self): if self.tok == "": return if self.tok in keywords: self.append(Token(self.tok, "keyword", self.line, self.column)) else: if self.tok == "\t": Error.errorAst("Found tab expecting only spaces", self.package, self.filename, Token(self.tok, "", self.line, self.column)) typ = "" match = None for (it, (group, regex)) in enumerate(tokenSpecification): regex = compiledSpecifications[it] match = regex.match(self.tok) if match: typ = group break if typ == "": Error.errorAst("Unexpected token " + self.tok, self.package, self.filename, Token(self.tok, "", self.line, self.column)) if not typ == "identifier": tok = self.tok.replace("_", "") else: tok = self.tok self.append(Token(tok, typ, self.line, self.column)) self.tok = ""
def pattern(name, names, parser, getName): if type(name) is Tree.Tuple: for i in name: pattern(i, names, parser, getName) return name elif type(name) is Tree.PlaceHolder: node = name.nodes[0] pattern(node, names, parser, getName) elif type(name) is Tree.InitStruct: for i in name: if type(i) is Tree.Assign: if not type(i.nodes[0]) is Tree.ReadVar: i.error("Expecting variable name") pattern(i.nodes[1], names, parser, getName) elif type(i) is Tree.ReadVar: pattern(i, names, parser, getName) else: i.error("Expecting variable name") return name elif type(name) is Tree.ReadVar: names.append(name.name) return getName(name.token) elif type(name) is Tree.Under: names.append(name.name) elif type(name) is Lexer.Token and name.type == "identifier": return getName(name) else: if type(name) is Lexer.Token: if name.type == "indent": Error.parseError(parser, "Expecting identifier") else: Error.parseError(parser, "Unexpected token " + name.token) else: name.error("Unexpected token")
def main(): err = False try: f = open("src/port.json") except: err = True if err: try: Error.error("cannot find port.json") except EOFError as e: print(e, file=sys.stderr) sys.exit() file = f.read() port = json.loads(file) if len(sys.argv) > 1: if sys.argv[1] == "install": installPackage() elif sys.argv[1] == "uninstall": uninstallPackage() elif sys.argv[1] == "deploy": deploy(port) elif sys.argv[1] == "doc": doc(port) else: print("Unknown command "+sys.argv[1], file=sys.stderr) else: print("Need command", file=sys.stderr)
def _packDec(parser, pack=False): name = parser.nextToken() if name.type != "identifier": Error.parseError(parser, "package name must be an identifier") name = name.token packDec(parser, name, pack)
def checkOther(self, parser, function, block, iter=0): if type(self) in [Tree.FuncStart, Tree.FuncBraceOpen]: return for (c, i) in enumerate(self.nodes): if type(self) in [Tree.While, Tree.For]: checkOther(i, parser, function, self, c) elif type(self) is Tree.FuncBody: checkOther(i, parser, self, block, c) else: checkOther(i, parser, function, block, c) if type(self) in [Tree.Continue, Tree.Break] and not type(block) in [Tree.While, Tree.For]: statement = "continue" if type(self) is Tree.Continue else "break" self.error(f"unexpected {statement}, outside of a loop") elif type(self) is Tree.Return: if not function: self.error(f"unexpected return statement, outside of a function") try: if len(self.nodes) > 0: actReturnType = self.nodes[0].type function.returnType.duckType(parser,actReturnType, self, self ,0) Tree.insertCast(self.nodes[0], actReturnType, function.returnType, 0) else: if not function.returnType.isType(Types.Null): self.error("Expecting return value, as function returns " + str(function.returnType)) except EOFError as e: Error.beforeError(e, "Return Type: ") else: self.validate(parser)
def elseExpr(parser): toplevel = Tree.Else(parser) try: inside = parser.currentNode.nodes[-1].nodes[-2] except IndexError: Error.parseError(parser, "unexpected else") if not type(inside) is Tree.IfCondition: Error.parseError(parser, "unexpected else") parser.currentNode.nodes[-1].addNode(toplevel) parser.currentNode = toplevel block = Tree.Block(parser) parser.currentNode.owner.addNode(block) parser.currentNode = block opening = None single = 0 while not Parser.isEnd(parser): token = parser.nextToken() Parser.callToken(parser) ExprParser.endExpr(parser) parser.currentNode = toplevel.owner.owner
def createParser(parser, name="", typ=None, check=True, imutable=True, attachTyp=False): # : creation if name == "": name = parser.lookBehind() if name.type != "identifier": Error.parseError( parser, "variable name must be of type identifier, not " + parser.lookBehind().type) name = name.token if name[0].lower() != name[0]: Error.parseError(parser, "variable name must be lower case") node = Tree.Create(name, Types.Null(), parser) node.package = parser.package node.imutable = imutable if attachTyp: node.attachTyp = attachTyp parser.currentNode.addNode(node) node.varType = typ if check and typ is None: parser.nextToken() typ = Types.parseType(parser) node.varType = typ
def index(parser, unary=False): if not unary: unary = ExprParser.isUnary(parser, parser.lookBehind()) if not unary and len(parser.currentNode.nodes) == 0: Error.parseError(parser, "unexpected .") field = parser.nextToken() if not field.type in ["identifier", "i32"]: Error.parseError(parser, "field name must be an identifier") acess = Tree.Field(0, Types.Null(), parser) acess.unary = unary acess.number = field.type == "i32" field = field.token acess.owner = parser.currentNode acess.field = field if not unary: acess.addNode(parser.currentNode.nodes[-1]) parser.currentNode.nodes[-1] = acess else: parser.currentNode.addNode(acess)
def initModule(moduleName): try: transforms[moduleName].init() except Exception as e: Error.error("Error " + str(e) + " happened when intializing syntax extension " + moduleName)
def generics(parser, fname): generic = coll.OrderedDict() while parser.thisToken().token != "]": name = parser.nextToken().token typ = Types.T(name, Types.All, parser.package+"."+fname) if parser.thisToken().type != "identifier": Error.parseError(parser, "type name must be an identifier") if not parser.nextToken().token in [":", ",", "]"]: Error.parseError(parser, "expecting ,") if parser.thisToken().token == ":": parser.nextToken() typ = Types.T(name, Types.parseType(parser), parser.package+"."+fname) if parser.lookInfront().token != "]": parser.nextToken() Scope.addVar(Tree.PlaceHolder(parser), parser, name, Scope.Type(False, typ)) generic[name] = typ if parser.lookInfront().token == "]": parser.nextToken() break parser.nextToken() return generic
def forExpr(parser): toplevel = Tree.For(parser) parser.currentNode.addNode(toplevel) parser.currentNode = toplevel while not (Parser.isEnd(parser) or parser.thisToken().token == "do"): token = parser.nextToken() if token.token == "do": break Parser.callToken(parser) ExprParser.endExpr(parser) parser.nodeBookmark.append(len(parser.currentNode.nodes)) if parser.thisToken().token != "do": Error.parseError(parser, "Expecting do") if len(toplevel.nodes) != 1: #or not type(toplevel.nodes[0]) is Tree.CreateAssign: Error.parseError(parser, "Expecting single node") count = 0 while not (Parser.isEnd(parser) and count > 0): #avoid breaking on do keyword count += 1 token = parser.nextToken() Parser.callToken(parser) parser.nodeBookmark.pop() parser.currentNode = toplevel.owner
def genericT(parser): parser.nextToken() if len(parser.currentNode.nodes) > 0: func = parser.currentNode.nodes.pop() else: Error.parseError(parser, "unexpected ::") generic = Tree.Generic(parser) parser.currentNode.addNode(generic) generic.addNode(func) generic.generic = [] if parser.thisToken().token != "[": Error.parseError(parser, "expecting [") parser.nextToken() while parser.thisToken().token != "]": if parser.thisToken().token == ",": parser.nextToken() parser.nodeBookmark[-1] = len(parser.currentNode.nodes) continue generic.generic.append(Types.parseType(parser)) t = parser.thisToken().token parser.nextToken()
def aliasParser(parser, name, decl, generic): parser.nextToken() typ = False while not Parser.isEnd(parser): if parser.thisToken().token != "\n" and parser.thisToken().type != "indent": if typ: Error.parseError(parser, "Unexpected token " + parser.thisToken().token) typ = Types.parseType(parser) parser.nextToken() if decl: alias = parser.interfaces[parser.package][name] tmp = Types.Alias(parser.package, name, typ, generic) alias.name = tmp.name alias.normalName = tmp.normalName alias.typ = tmp.typ alias.generic = tmp.generic alias.remainingGen = tmp.remainingGen alias.types = tmp.types alias.methods = tmp.methods if decl: parser.interfaces[parser.package][name] = alias Scope.decrScope(parser)
def createParser(parser, name= "", typ= None, check= True, imutable= True, attachTyp= False): # : creation if name == "": name = parser.lookBehind() if name.type != "identifier": Error.parseError(parser, "variable name must be of type identifier, not "+parser.lookBehind().type) name = name.token if name[0].lower() != name[0]: Error.parseError(parser, "variable name must be lower case") node = Tree.Create(name, Types.Null(), parser) node.package = parser.package node.imutable = imutable if attachTyp: node.attachTyp = attachTyp parser.currentNode.addNode(node) node.varType = typ if check and typ is None: parser.nextToken() typ = Types.parseType(parser) node.varType = typ
def _packDec(parser, pack= False): name = parser.nextToken() if name.type != "identifier": Error.parseError(parser, "package name must be an identifier") name = name.token packDec(parser, name, pack)
def condition_not_met(file, tags): f = open(file, "r") jsonLoads = json.loads(f.read()) for key in jsonLoads: if not key in tags: Error.error(file + ", unknown tag " + key) if jsonLoads[key] != tags[key]: return True return False
def addMethod(self, parser, name, method): package = parser.package if package in self.methods: if name in parser.structs[package][self.name].methods[package]: Error.parseError(parser, "method "+self.name+"."+name+" already exists") self.methods[package][name] = method else: self.methods[package] = {name: method}
def enumParser(parser, name, decl, generic): const = parser.interfaces[parser.package][name].const existing_generics = parser.interfaces[parser.package][name].generic existing_generics.update(generic) #const = coll.OrderedDict() enum = Types.Enum(parser.package, name, const, existing_generics) if decl: parser.interfaces[parser.package][name] = enum """if parser.lookInfront().token == "\n": parser.nextToken() Parser.callToken(parser) parser.nextToken()"" \ """ while not Parser.isEnd(parser): t = parser.nextToken() if t.token == "\n" or t.type == "indent": Parser.callToken(parser) continue if t.type != "identifier": Error.parseError(parser, "expecting identifier") varName = t.token if varName[0].upper() != varName[0]: Error.parseError(parser, "constructor type must be capitalized") args = [] nextT = parser.nextToken() #print(varName) #print(nextT) if nextT.token == "(": args = Types.parseType(parser).list const[varName] = args if decl: Scope.addVar(Tree.PlaceHolder(parser), parser, varName, Scope.Type( True, Types.FuncPointer(args, enum, generic=generic) if len(args) > 0 else enum), _global=True) t = parser.thisToken() if t.token == "\n" or t.type == "indent": Parser.callToken(parser) parser.currentNode.addNode(Tree.Enum(const, name, parser, generic)) Scope.decrScope(parser)
def f(parser): op = Tree.Operator(kind, parser) if kind != "|>" and len(parser.currentNode.nodes) != 0: if not unary and isUnary(parser, parser.lookBehind()): Error.parseError(parser, "unexpected "+kind) elif unary and not isUnary(parser, parser.lookBehind()): Error.parseError(parser, "unexpected "+kind) #parser.nodeBookmark.append(len(parser.currentNode.nodes)-1) Parser.Opcode(parser, kind, lambda: operatorPop(parser, op, takesIn, unary))
def compile(self, opt= 0): js = self.toJS() try: f = open("lib/"+self.filename.replace("/", ".") + ".js", mode="w") f.write(js) f.close() except: Error.error("Compilation failed")
def getName(name): if name.type != "identifier": Error.parseError(parser, "variable name must be of type identifier, not " + parser.lookBehind().type) name = name.token if name[0].lower() != name[0]: Error.parseError(parser, "variable name must be lower case") return name
def importModule(path): moduleName = os.path.basename(os.path.splitext(path)[0]) spec = importlib.util.spec_from_file_location(moduleName, path) if not spec: Error.error("Cannot load transform from path " + path) foo = importlib.util.module_from_spec(spec) spec.loader.exec_module(foo) transforms[moduleName] = foo
def loadRuntimeTypeData(): try: f = open(runtimeData, "rb") if os.stat(runtimeData).st_size == 0: Error.error("Runtime type data is empty, please recompile runtime") res = pickle.load(f) return res except FileNotFoundError: Error.error("Could not locate runtime")
def asOperator(parser): lastToken = parser.lookBehind() if not isUnary(parser, lastToken): op = Tree.Operator("as", parser) parser.nextToken() op.type = Types.parseType(parser) Parser.Opcode(parser, "as", lambda: operatorPop(parser, op, 1, unary=True)) else: Error.parseError(parser, "unexpected as operator ")
def castParser(parser): if len(parser.currentNode.nodes) == 0: Error.parseError(parser, "Unexpected cast") parser.nextToken() node = Tree.CastToType(parser) node.type = Types.parseType(parser) node.addNode(parser.currentNode.nodes[-1]) parser.currentNode.nodes[-1] = node node.owner = parser.currentNode
def elifExpr(parser): try: inside = parser.currentNode.nodes[-1].nodes[-2] except IndexError: Error.parseError(parser, "unexpected elif") if not type(inside) is Tree.IfCondition: Error.parseError(parser, "unexpected elif") parser.currentNode = parser.currentNode.nodes[-1] ifBody(parser)
def f(parser): op = Tree.Operator(kind, parser) actuallyUnary = isUnary(parser, parser.lookBehind()) if len(parser.currentNode.nodes) != 0: if not unary and actuallyUnary: Error.parseError(parser, "unexpected "+kind) elif unary and not actuallyUnary: Error.parseError(parser, "unexpected "+kind) #parser.nodeBookmark.append(len(parser.currentNode.nodes)-1) Parser.Opcode(parser, kind, lambda: operatorPop(parser, op, takesIn, actuallyUnary))
def elseExpr(parser, canHaveElse=False): toplevel = Tree.Else(parser) ifexpr = False if not canHaveElse: try: inside = parser.currentNode.nodes[-1].nodes[-2] except IndexError: Error.parseError(parser, "unexpected else") if not type(inside) is Tree.IfCondition: ifexpr = IfExpr.ifPatternMatch(parser) if not ifexpr: Error.parseError(parser, "unexpected else") if not ifexpr: parser.currentNode.nodes[-1].addNode(toplevel) parser.currentNode = toplevel block = Tree.Block(parser) parser.currentNode.owner.addNode(block) parser.currentNode = block else: parser.currentNode = parser.currentNode.nodes[-1].nodes[-1] while len(parser.currentNode.nodes) > 0: parser.currentNode = parser.currentNode.nodes[-1] add_block = len(parser.currentNode.nodes) > 0 if add_block: block = Tree.Block(parser) parser.currentNode.nodes[-1].addNode(toplevel) parser.currentNode.nodes[-1].addNode(block) parser.currentNode = parser.currentNode.nodes[-1] parser.currentNode = block opening = None single = 0 while not Parser.isEnd(parser): token = parser.nextToken() Parser.callToken(parser) ExprParser.endExpr(parser) if ifexpr: parser.currentNode = ifexpr else: parser.currentNode = toplevel.owner.owner
def newPack(name): if name[0].lower() != name[0]: Error.error("package name must be lowercase") try: os.mkdir("src/" + name) f = open("src/"+name+"/port.json", mode= "w") f.write(""" { "files": [] } """) except: Error.error("directory has no source folder")
def newPack(name): if name[0].lower() != name[0]: Error.error("package name must be lowercase") try: os.mkdir("src/" + name) f = open("src/" + name + "/port.json", mode="w") f.write(""" { "files": [] } """) except: Error.error("directory has no src folder")
def getCompilationFiles(target): try: proj = open("src/port.json", mode="r") proj.close() except: Error.error("missing port.json in source folder") def getCompFiles(dir=""): file = {} for root, dirs, files in os.walk(dir, topdown=False): package = root if package == "src/": continue package = package[len("src/"):] file[package] = {"client": [], "full": [], "node": []} try: port = open("src/" + package + "/port.json", mode="r") except: Error.error("missing file port.json in package " + package + "") files = [] try: j = json.loads(port.read()) files.append((target, j["files"])) except KeyError: pass except json.decoder.JSONDecodeError as e: Error.error("In file port.json, in directory " + package + ", " + str(e)) if "client-files" in j and (target in ["full", "client"]): files.append(("client", j["client-files"])) if "node-files" in j and (target in ["full", "node"]): files.append(("node", j["node-files"])) if len(files) == 0: Error.error("no compilation files are specified in package " + package + "/port.json") for f in files: for name in f[1]: file[package][f[0]].append((root, name + ".top")) if root[0].lower() != root[0]: Error.error("package name must be lowercase") return file return getCompFiles("src/")
def traitParser(parser, name, decl, generic): interface = parser.interfaces[parser.package][name] meth = {} while not Parser.isEnd(parser): parser.nextToken() t = parser.thisToken() if t.token == "def": currentNode = parser.currentNode p = PlaceHolder(parser) parser.currentNode = p (methodName, names, types, brace, returnType, do) = FuncParser.funcHead(parser, decl, dontAdd=True, interfaceMethod=True) Scope.decrScope(parser) if methodName in meth: Error.parseError(parser, "Method " + methodName + ", already defined") meth[methodName] = brace.ftype parser.currentNode = currentNode parser.nextToken() else: Parser.declareOnly(parser, noVar=True) if len(parser.currentNode.nodes) > 0 and type( parser.currentNode.nodes[0]) is Tree.Create: Error.parseError( parser, "Interfaces are abstract interfaces which is why only methods are supported" ) names = {i.name: i.varType for i in parser.currentNode} #args = [i.varType for i in parser.currentNode] #fields = parser.currentNode.nodes if decl: i = interface.fromObj( Types.Interface(False, names, generic, parser.package + "." + name if parser.package != "_global" else name, methods=meth)) parser.interfaces[parser.package][name] = i Scope.decrScope(parser)
def f(parser): op = Tree.Operator(kind, parser) if kind == "|>": print(kind) if not kind in ["|>"] and len(parser.currentNode.nodes) != 0: if not unary and isUnary(parser, parser.lookBehind()): Error.parseError(parser, "unexpected " + kind) elif unary and not isUnary(parser, parser.lookBehind()): Error.parseError(parser, "unexpected " + kind) #parser.nodeBookmark.append(len(parser.currentNode.nodes)-1) Parser.Opcode(parser, kind, lambda: operatorPop(parser, op, takesIn, unary))
def hasMethod(self, parser, name): packages = [] b = None for i in parser.imports+[parser.package]+["_global"]: if not i in self.methods: continue if name in self.methods[i]: b = self.methods[i][name] b.package = i packages.append(i) if len(packages) > 1: Error.parseError(parser, "ambiguous, multiple definitions of the method "+self.name+"."+name+" in packages: "+", ".join(packages[:-1])+" and "+packages[-1]) return b
def externVar(parser): target = parser.nextToken() if target.type != "identifier": Error.parseError(parser, "expecting target") target = target.token if not target in ["client", "full", "node"]: Error.parseError(parser, target + " is not a valid compilation target") VarParser.createAndAssignParser(parser) parser.currentNode.nodes[-1].extern = True parser.currentNode.nodes[-1].global_target = target parser.currentNode.nodes[-1].nodes[1].extern = True
def importParser(parser, decl= False): import os name = parser.nextToken() if name.type != "str": Error.parseError(parser, "expecting string") oname = name.token[1:-1] if not oname in parser.filenames: Error.parseError(parser, "package "+oname+" not found") name = os.path.basename(oname) if not decl: parser.externFuncs[parser.package] = [] if not parser.hotswap: sp = shouldParse(decl, oname, parser) else: sp = shouldCompile(decl, oname, parser) if sp: p = Parser.Parser(parser.lexed[oname], parser.filenames[oname]) ResolveSymbols.insert(parser, p) sc = shouldCompile(decl, oname, parser) p.sc = sc parser.compiled[name] = None parser.externFuncs[name] = [] parsed = p.parse() declar = parser.externFuncs[name] parser.compiled[name] = (sc, (parsed, declar)) ResolveSymbols.insert(p, parser) parser.currentNode.addNode(Tree.InitPack(name, parser)) else: if not name in parser.compiled: parser.compiled[name] = (False,) parser.currentNode.addNode(Tree.InitPack(name, parser)) parser.imports.append(oname)
def append(self, value): if value is None: raise Error.error( "expecting type string and got none, internal error") if self.target in [self.global_target, "full"]: if self.global_target == "full": if self.inAFunction: self.client_out_parts.append(value) self.node_out_parts.append(value) else: self.client_main_parts.append(value) self.node_main_parts.append(value) else: if self.inAFunction: self.out_parts.append(value) else: self.main_parts.append(value) elif self.target == "client": if self.inAFunction: self.client_out_parts.append(value) else: self.client_main_parts.append(value) elif self.target == "node": if self.inAFunction: self.node_out_parts.append(value) else: self.node_main_parts.append(value) elif self.target == "full": if self.inAFunction: self.out_parts.append(value) else: self.main_parts.append(value)
def linkWith(name): try: file = open("src/port.json", mode="r+") except: Error.error("missing port.json file in project") port = json.loads(file.read()) if name.endswith(".css"): port["linkCSS"].append(name) else: port["link"].append(name) file.write(json.dumps(port)) file.close()
def linkWith(name): try: file = open("src/port.json", mode="r+") except: Error.error("missing port.json file in project") port = json.loads(file.read()) file.write(""" { "name": \"""" + port["name"] + """\", "version": """+str(port["version"]) + """, "link": """+str(port["link"]+[name]) + """, } """) file.close()
def index(parser): if len(parser.currentNode.nodes) == 0: Error.parseError(parser, "unexpected .") field = parser.nextToken() if field.type != "identifier": Error.parseError(parser, "field name must be an identifer") field = field.token acess = Tree.Field(0, Types.Null(), parser) acess.addNode(parser.currentNode.nodes[-1]) acess.owner = parser.currentNode acess.field = field parser.currentNode.nodes[-1] = acess
def createAndAssignParser(parser, imutable= True): # let i assignment node = parser.currentNode parser.nextToken() #get current token to position of = checkIt = False attachTyp = False if parser.lookInfront().token == ".": attachTyp = Types.parseType(parser, attachTyp= True) parser.nextToken() if not imutable or not type(node) is Tree.Root: Error.parseError(parser, "expecting =, not .") parser.nextToken() name = parser.thisToken() typ = None if parser.nextToken().token == ":": checkIt = True parser.nextToken() typ = Types.parseType(parser) parser.nextToken() elif parser.thisToken().token != "=": Error.parseError(parser, "expecting =, not"+parser.thisToken().token) n = Tree.CreateAssign(parser) parser.currentNode.addNode(n) parser.currentNode = n createParser(parser, name= name, typ= typ, check= checkIt, imutable= imutable, attachTyp= attachTyp) if attachTyp: assignParser(parser, name=attachTyp.name+"_"+name.token, package= attachTyp.package, init=True) else: assignParser(parser, name= name.token, init= True) parser.currentNode = node
def importParser(parser, decl= False): import os name = parser.nextToken() if name.type != "str": Error.parseError(parser, "expecting string") oname = name.token[1:-1] if not oname in parser.filenames: Error.parseError(parser, "package "+oname+" not found") name = os.path.basename(oname) if not decl: parser.externFuncs[parser.package] = [] if shouldParse(decl, oname, parser): p = Parser.Parser(parser.lexed[oname], parser.filenames[oname]) ResolveSymbols.insert(parser, p) sc = shouldCompile(decl, oname, parser) parser.compiled[name] = None parser.externFuncs[name] = [] if sc: parsed = p.parse() else: parsed = None declar = parser.externFuncs[name] parser.compiled[name] = (sc, (parsed, declar)) ResolveSymbols.insert(p, parser) parser.currentNode.addNode(Tree.InitPack(name, parser)) else: if not name in parser.compiled: parser.compiled[name] = None parser.imports.append(oname)
def newProj(name): if name[0].upper() != name[0]: Error.error("project name must be uppercase") try: os.mkdir(name) os.mkdir(name + "/" + "src") file = open(name + "/src/port.json", mode="w") file.write(""" { "name": \"""" + name + """\", "version": 0.0, "link": [] } """) file.close() os.mkdir(name + "/" + "lib") os.mkdir(name + "/" + "bin") except: Error.error("project already created")
def getCompFiles(dir= ""): file = {} for root, dirs, files in os.walk(dir, topdown=False): package = root if package == "src/": continue package = package[len("src/"):] file[package] = [] try: port = open("src/"+package+"/port.json", mode= "r") except: Error.error("missing file port.json in package "+package+"") try: j = json.loads(port.read()) files = j["files"] except KeyError: Error.error("missing property files in file "+package+"port.json") for name in files: file[package].append((root, name+".top")) if root[0].lower() != root[0]: Error.error("package name must be lowercase") return file
def typeParser(parser, decl= False): name = parser.nextToken() Scope.incrScope(parser) if name.type != "identifier": Error.parseError(parser, "type name must be an identifier") name = name.token if name[0].lower() == name[0]: Error.parseError(parser, "struct name must be upper case") import collections as coll gen = coll.OrderedDict() if parser.nextToken().token == "[": gen = FuncParser.generics(parser, name) if parser.thisToken().token != "=": if parser.thisToken().token == "with": tmp = parser.currentNode parser.currentNode = Tree.PlaceHolder(parser) Interface.traitParser(parser, name, decl, gen) parser.currentNode = tmp return Error.parseError(parser, "expecting =") tmp = parser.currentNode typ = Tree.Type(parser.package, name, parser) typ.package = parser.package typ.normalName = name tmp.addNode(typ) parser.currentNode = typ while not Parser.isEnd(parser): parser.nextToken() Parser.declareOnly(parser, noVar=True) args = [i.varType for i in parser.currentNode] fields = parser.currentNode.nodes typ.fields = [i.name for i in typ] typ.nodes = [] parser.currentNode = tmp if decl: meth = parser.structs[parser.package][name].methods parser.structs[parser.package][name] = Struct(name, args, fields, gen) parser.structs[parser.package][name].methods = meth parser.structs[parser.package][name].package = parser.package Scope.decrScope(parser)
def initStruct(parser, package= ""): if ExprParser.isUnary(parser, parser.lookBehind()): Error.parseError(parser, "unexpected {") if package == "": package = parser.package if len(parser.currentNode.nodes) == 0: Error.parseError(parser, "unexpected {") if not type(parser.currentNode.nodes[-1]) in [Tree.ReadVar, Tree.Field]: Error.parseError(parser, "unexpected {") readVar = type(parser.currentNode.nodes[-1]) is Tree.ReadVar name = parser.currentNode.nodes[-1].name if readVar else parser.currentNode.nodes[-1].field init = Tree.InitStruct(parser) if not readVar: package = parser.currentNode.nodes[-1].nodes[0].name t = (parser.currentNode.nodes[-1].nodes[0]) if not package in parser.imports: t.error("no package called " + package) elif not type(t) is Tree.ReadVar: init.error("unexpected {") init.package = package del parser.currentNode.nodes[-1] s = parser.structs[package][name] init.paramNames = offsetsToList(parser.structs[package][name].offsets) init.s = s init.mutable = False parser.currentNode.addNode(init) parser.currentNode = init parser.nextToken() while parser.thisToken().token != "}": if parser.thisToken().token == ",": ExprParser.endExpr(parser) else: Parser.callToken(parser) parser.nextToken() t = parser.thisToken().token continue ExprParser.endExpr(parser) parser.currentNode = init.owner
def parserMethodGen(parser, gen, struct): sgen = struct.generic if len(gen) > len(sgen): Error.parseError(parser, str(len(gen)-len(sgen))+" generic arguments too many") elif len(gen) < len(sgen): Error.parseError(parser, str(len(gen) - len(sgen)) + " generic arguments too few") newGen = coll.OrderedDict() for a, b in zip(gen, sgen): if gen[a].type != Types.All: Error.parseError(parser, "unexpected :") newGen[a] = sgen[b] Scope.changeType(parser, a, sgen[b]) return newGen
def checkIfOperator(parser, attachTyp, name, func): operators = { "add": Types.FuncPointer([attachTyp, attachTyp], attachTyp), "sub": Types.FuncPointer([attachTyp, attachTyp], attachTyp), "mul": Types.FuncPointer([attachTyp, attachTyp], attachTyp), "div": Types.FuncPointer([attachTyp, attachTyp], attachTyp), "eq": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()), "ne": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()), "mod": Types.FuncPointer([attachTyp, attachTyp], attachTyp), "pow": Types.FuncPointer([attachTyp, attachTyp], attachTyp), "gt": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()), "lt": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()), } unary = { "add": Types.FuncPointer([attachTyp], attachTyp), "sub": Types.FuncPointer([attachTyp], attachTyp), "mul": Types.FuncPointer([attachTyp], attachTyp) } if name.startswith("operator_"): op = name[len("operator_"):] if not op in operators: Error.parseError(parser, "overload not found for operator_"+op) f = Types.FuncPointer(func.args, func.returnType) if f != operators[op]: Error.parseError(parser, "expecting function declaration "+str(operators[op])+", not "+str(f)) elif name.startswith("unary_"): op = name[len("unary_"):] if not op in unary: Error.parseError(parser, "overload not found for unary_"+op) f = Types.FuncPointer(func.args, func.returnType) if f != unary[op]: Error.parseError(parser, "expecting function declaration "+str(unary[op])+", not "+str(f))
parser.currentNode.addNode(n) parser.currentNode.addNode(cond) parser.currentNode = cond while not Parser.isEnd(parser): token = parser.nextToken() iter = parser.iter if token.token == "do" : ExprParser.endExpr(parser) block = Tree.WhileBlock(parser) cond.owner.addNode(block) parser.currentNode = block continue Parser.callToken(parser) ExprParser.endExpr(parser) parser.currentNode = n.owner.owner Parser.exprToken["while"] = whileExpr Parser.exprToken["break"] = lambda parser: parser.currentNode.addNode(Tree.Break(parser)) Parser.exprToken["continue"] = lambda parser: parser.currentNode.addNode(Tree.Continue(parser)) Parser.exprToken["then"] = lambda parser: Error.parseError(parser, "unexpected then keyword") Parser.exprToken["do"] = lambda parser: Error.parseError(parser, "unexpected do keyword")
def start(run= False, dev= False, init= False): try: opt = 0 skip = 0 outputFile = "" for (iter, i) in enumerate(sys.argv[2:]): if skip > 0: continue skip -= 1 if i == "-O3": opt = 3 elif i == "-O2": opt = 2 elif i == "-o": outputFile = sys.argv[iter + 3] skip = 1 elif i == "-O1": opt = 1 else: Error.error("unknown argument '" + i + "'.") files = getCompilationFiles() allfilenames = [] allsources = [] sources = {} filenames = {} for c in files: sources[c] = [] filenames[c] = [] for i in files[c]: try: file = open(os.path.join(i[0], i[1]), mode="r") r = file.read() allsources.append(r) sources[c].append(r) if i[1][0].upper() == i[1][0]: Error.error("file name must be lowercase") filenames[c].append((c, i[1][:-4])) allfilenames.append((c, i[1][:-4])) file.close() except FileNotFoundError: Error.error("file " + i[1] +", not found") if outputFile == "": port = open("src/port.json") data = port.read() outputFile = (json.loads(data)["name"]) port.close() if filenames == []: Error.error("no input files") """ import cProfile profile = cProfile.Profile() profile.enable() """ time1 = time() # print ("============= Compiling ==============\n") """ for i in lexed: print(i.token+"_"+i.type) """ lexed = Lexer.lex(sources, filenames) #print("lexed") declarations = Parser.Parser(lexed, filenames) declarations.files = files declarations.lexed = lexed declarations.filenames = filenames declarations.opt = opt declarations.compiled = {} declarations.externFuncs = {"main": []} ResolveSymbols.resolve(declarations) #print("declarations") if ImportParser.shouldCompile(False, "main", declarations): parser = Parser.Parser(lexed["main"], filenames["main"]) ResolveSymbols.insert(declarations, parser, only= True) parser.files = files parser.lexed = lexed parser.filenames = filenames parser.compiled = declarations.compiled parser.compiled["main"] = None parsed = parser.parse() parser.compiled["main"] = (True, (parsed, parser.externFuncs["main"])) import AST as Tree allCode = Tree.Root() if opt > 0: for d in parser.compiled: allCode.addNode(parser.compiled[d][1][0]) optimize(allCode, opt) #print("parsing") for i in parser.compiled: if parser.compiled[i][0]: CodeGen.CodeGen(i, parser.compiled[i][1][0], parser.compiled[i][1][1]).compile(opt=opt) l = CodeGen.link(parser.compiled, outputFile, run=run, opt= opt, dev= dev) print("Compilation took : "+str(time() - time1)) return (True, l) elif run: CodeGen.exec(outputFile) elif init: return (True, open("bin/"+outputFile+".js").read()) elif dev: return (False, "") print("Compilation took : "+str(time() - time1)) except EOFError as e: if dev: return (False, str(error)) else: print(e, file= sys.stderr)
generic = Tree.Generic(parser) parser.currentNode.addNode(generic) generic.addNode(func) generic.generic = [] if parser.thisToken().token != "[": Error.parseError(parser, "expecting [") parser.nextToken() while parser.thisToken().token != "]": if parser.thisToken().token == ",": parser.nextToken() parser.nodeBookmark[-1] = len(parser.currentNode.nodes) continue generic.generic.append(Types.parseType(parser)) t = parser.thisToken().token parser.nextToken() def under(parser): parser.currentNode.addNode(Tree.Under(parser)) Parser.stmts["def"] = func Parser.exprToken["none"] = lambda parser: Error.parseError(parser, "unexpected type none") Parser.exprToken[","] = lambda parser: Error.parseError(parser, "unexpected ,") Parser.exprToken["_"] = under Parser.exprToken["::"] = genericT Parser.exprToken["!"] = lambda parser: 0
def funcHead(parser, decl= False, dontAdd= False, method= False, attachTyp = False): Scope.incrScope(parser) if parser.tokens[parser.iter+2].token == ".": if attachTyp: Error.parseError(parser, "unexpected .") parser.nextToken() name = parser.thisToken().token parser.nextToken() try: attachTyp = Types.Struct(False, name, parser.structs[parser.package][name].types, parser.package, {}) except KeyError: Error.parseError(parser, "no attachable data structure found, called "+name) return funcHead(parser, decl, dontAdd, True, attachTyp) name = parser.nextToken() if name.type != "identifier": Error.parseError(parser, "function name must be of type identifier, not "+name.type) parser.nextToken() name = name.token g = {} if parser.thisToken().token != "(": if parser.thisToken().token == "[": g = generics(parser, (str(attachTyp)+"." if method else "")+name) if parser.thisToken().token == ".": if attachTyp: Error.parseError(parser, "unexpected .") if not Scope.varExists(parser, parser.package, name): Error.parseError(parser, "cannot attach method to unknown type main."+name) try: attachTyp = Types.Struct(False, name, parser.structs[parser.package][name].types, parser.package, parserMethodGen(parser, g, parser.structs[parser.package][name])) except KeyError: Error.parseError(parser, "no attachable data structure found, called " + name) return funcHead(parser, decl, dontAdd, True, attachTyp) if parser.thisToken().token != "(": Error.parseError(parser, "expecting (") header = Tree.FuncStart(name, Types.Null(), parser) header.package = parser.package parser.currentNode.addNode(header) brace = Tree.FuncBraceOpen(parser) brace.name = name brace.package = parser.package parser.currentNode.addNode(brace) parser.currentNode = brace if method: if not type(parser.currentNode.owner) is Tree.Root and not decl: Error.parseError(parser, "method extension must be in out-most scope") typ = attachTyp self = parser.nextToken() if self.type != "identifier": Error.parseError(parser, "binding name must be identifier not "+self.type) self = self.token selfNode = Tree.Create(self, typ, parser) selfNode.package = parser.package selfNode.imutable = True parser.currentNode.addNode(selfNode) if not parser.lookInfront().token in [")", ","]: Error.parseError(parser, "expecting comma not "+parser.thisToken().token) if name[0].lower() != name[0]: Error.parseError(parser, "function name must be lower case") returnType = Types.Null() parser.paren += 1 parser.nextToken() while parser.paren != parser.parenBookmark[-1] : b = parser.thisToken().token if b == ",": parser.nextToken() continue elif b == ")": parser.paren -= 1 parser.nextToken() continue elif b == "(": Error.parseError(parser, "unexpected (") Parser.declareOnly(parser) parser.nextToken() t = parser.thisToken() do = False if t.token != "=" and t.token != "do": returnType = Types.parseType(parser) t = parser.nextToken() if t.token != "=" and t.token != "do": Error.parseError(parser, "expecting = or do") if t.token == "do": do = True parser.currentNode = brace.owner names = [i.name for i in brace.nodes] types = [i.varType for i in brace.nodes] func = Types.FuncPointer( types, returnType, g, do ) if method: Scope.decrScope(parser) header.method = True header.types = types[1:] header.attachTyp = attachTyp header.normalName = name header.name = attachTyp.normalName+"_"+header.normalName MethodParser.checkIfOperator(parser, attachTyp, name, func) if decl: MethodParser.addMethod(brace, parser, attachTyp, name, func) return attachTyp.normalName+"_"+name, names, types, header, returnType, do parser.func[parser.package][name] = func header.ftype = Types.FuncPointer(types, returnType, g) if decl: if not dontAdd: Scope.addFunc(header, parser, name, Types.FuncPointer(types, returnType, g, do)) return name, names, types, header, returnType, do
from TopCompiler import topc from TopCompiler import Error import sys if __name__ == "__main__": if len(sys.argv) > 1: if sys.argv[1] == "new": if sys.argv[2] == "project": topc.newProj(sys.argv[3]) elif sys.argv[2] == "package": topc.newPack(sys.argv[3]) elif sys.argv[2] == "linkWith": topc.linkWith(sys.argv[3]) else: Error.error("invalid option to new"+sys.argv[2]) sys.exit() elif sys.argv[1] == "build": topc.start() elif sys.argv[1] == "run": topc.start(True) else: Error.error("invalid option "+sys.argv[1])
createParser(parser, name= name, typ= typ, check= checkIt, imutable= imutable, attachTyp= attachTyp) if attachTyp: assignParser(parser, name=attachTyp.name+"_"+name.token, package= attachTyp.package, init=True) else: assignParser(parser, name= name.token, init= True) parser.currentNode = node Parser.stmts["let"] = createAndAssignParser Parser.stmts["var"] = lambda parser: createAndAssignParser(parser, imutable= False) Parser.stmts["="] = assignParser Parser.stmts[":"] = createParser Parser.exprToken["i32"] = lambda parser: Error.parseError(parser, "unexpected type int") Parser.exprToken["|"] = lambda parser: Error.parseError(parser, "unexpected function declaration") Parser.exprToken["int"] = lambda parser: Error.parseError(parser, "unexpected type int") Parser.exprToken["float"] = lambda parser: Error.parseError(parser, "unexpected type float") Parser.exprToken["bool"] = lambda parser: Error.parseError(parser, "unexpected type bool") def read(parser, name, package= ""): if package == "": package = parser.package node = Tree.ReadVar(name, False, parser) node.package = package parser.currentNode.addNode(node) def equalAnd(parser, operator, package= ""): if package == "": package = parser.package
def error(self, message): Error.errorAst(message, self.selfpackage, self.filename, self.token)
def tokenize(s, filename, spos= 0, sline= 0, slinePos= 0): keywords = [ 'import', 'def', 'then', 'do', 'if', 'elif', 'else', 'while', 'int', 'float', 'none', 'bool', 'string', 'break', 'continue', 'true', 'false', 'let', 'ext', 'type', 'string', 'var', "not", "or", "and", ] special = ["bang", "arrow", "doublecolon", "line", "underscore", "assign", "assignPlus", "assignSub", "assignMul", "assignDiv", 'colon', 'dot', 'openC', 'openB', 'closeC', 'closeB', 'comma', 'closeS', 'openS', 'doubleDot', 'semi'] token_specification = [ ("comment", r"/\*([\s\S]*?)\*/"), ("indent", r'\n[ ]*'), ('commentLine', r'//.*'), ('newline', r'\n'), ('openB', '{'), ('closeB', '}'), ('openC', '\('), ('closeC', '\)'), ('f32', r'[\d_]+(\.[\d_]+|f)'), ('i32', r'[\d_]+'), ('arrow', r'->'), ('equal', r'=='), ('doublecolon', r'::'), ("colon", r":"), ("semi", r";"), ('ne', r'!='), ('assign', r'='), ('openS', r'\['), ('closeS', r'\]'), ('assignPlus', r'\+='), ('assignSub', r'\-='), ('assignMul', r'\*='), ('assignDiv', r'\/='), ('operator', r'[+*\/\-%><^]|(\|>)'), ('line', r'\|'), ('identifier', r'[A-Za-z0-9_]+'), ('underscore', '_'), ('skip', r'[ \t]'), ("str", r'"(?:\\.|({.*})|[^"\\])*"'), ('doubleDot', '\.\.'), ('dot', '\.'), ('tab', '\t'), ('comma', ','), ('bang', '!'), ] tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification) get_token = re.compile(tok_regex).match line = 1 pos = spos mo = get_token(s) lastIndent = 0 lastTyp = None linePos = slinePos line = sline array = [] while mo is not None: typ = mo.lastgroup next = get_token(s, mo.end()) if typ == "indent" or typ == "newline": val = mo.group(typ) array.append(Token("\n", "symbol", line, pos)) line += 1 linePos = mo.end() if next == None: array.append(Token(0,"indent", line, pos)) break if next.lastgroup == "indent": array.append(Token(lastIndent, "indent", line, pos )) else: array.append(Token(len(val)-1, "indent", line, pos)) lastIndent = len(val)-1 elif typ == "comment": val = mo.group(typ) c = mo.end() r = val.rfind("\n") linePos = c + r line += len(val) - len(val.replace("\n", "")) array.append(Token(val, "comment", line, pos )) elif typ in ["str"]: val = mo.group(typ) if typ == "str": def notBack(iter): if iter == 0: return True if val[iter-1] != "\\": return True return not notBack(iter-1) start = 0 inBrace = False tokens = [] val = val[1:-1] bcount = 0 shouldBe = 0 v = list(val) for iter in range(len(val)): i = val[iter] if notBack(iter) and i == "{" and not inBrace: tokens.append(Token('"'+val[start: iter]+'"', "str", line, pos)) inBrace = True start = iter+1 shouldBe = bcount if i == "{": bcount += 1 elif notBack(iter) and i == "}": bcount -= 1 if bcount == shouldBe and inBrace: tokens.append(Token("concat", "operator", line, pos+start)) tokens.append(Token("(", "symbol", line, pos+start)) tokens += tokenize(val[start: iter], filename, pos+start, line, linePos) tokens.append(Token(")", "symbol", line, pos+iter)) tokens.append(Token("concat", "operator", line, pos+iter)) start = iter + 1 inBrace = False tokens.append(Token('"'+val[start:]+'"', "str", line, pos)) array += tokens elif typ != 'skip': val = mo.group(typ) if typ == 'identifier' and val in keywords: if val in ["true", "false"]: typ = "bool" elif val in ["_"]: typ = "symbol" else: typ = "keyword" elif typ == "f32": val = val[:-1]+".0" if val[-1] == "f" else val if typ == "i32" or typ == "f32": val = val.replace("_", "") elif typ in special: typ = "symbol" elif typ == "equal" or typ == "mut" or typ == "ne": typ = "operator" if val != " ": array.append(Token(val, typ, line, pos)) elif val == "\t": Error.compileError(filename[1], line, "tabs are not allowed") lastTyp = typ pos = mo.end() - linePos mo = next if spos == 0 and sline == 0: array.append(Token("\n", "symbol", line-1, pos)) array.append(Token(0, "indent", line, pos)) return array