def importParser(parser, decl= False): import os name = parser.nextToken() if name.type != "str": Error.parseError(parser, "expecting string") oname = name.token[1:-1] if not oname in parser.filenames: Error.parseError(parser, "package "+oname+" not found") name = os.path.basename(oname) if not decl: parser.externFuncs[parser.package] = [] if not parser.hotswap: sp = shouldParse(decl, oname, parser) else: sp = shouldCompile(decl, oname, parser) if sp: p = Parser.Parser(parser.lexed[oname], parser.filenames[oname]) ResolveSymbols.insert(parser, p) sc = shouldCompile(decl, oname, parser) p.sc = sc parser.compiled[name] = None parser.externFuncs[name] = [] parsed = p.parse() declar = parser.externFuncs[name] parser.compiled[name] = (sc, (parsed, declar)) ResolveSymbols.insert(p, parser) parser.currentNode.addNode(Tree.InitPack(name, parser)) else: if not name in parser.compiled: parser.compiled[name] = (False,) parser.currentNode.addNode(Tree.InitPack(name, parser)) parser.imports.append(oname)
def importParser(parser, decl= False): import os name = parser.nextToken() if name.type != "str": Error.parseError(parser, "expecting string") oname = name.token[1:-1] if not oname in parser.filenames: Error.parseError(parser, "package "+oname+" not found") name = os.path.basename(oname) if not decl: parser.externFuncs[parser.package] = [] if shouldParse(decl, oname, parser): p = Parser.Parser(parser.lexed[oname], parser.filenames[oname]) ResolveSymbols.insert(parser, p) sc = shouldCompile(decl, oname, parser) parser.compiled[name] = None parser.externFuncs[name] = [] if sc: parsed = p.parse() else: parsed = None declar = parser.externFuncs[name] parser.compiled[name] = (sc, (parsed, declar)) ResolveSymbols.insert(p, parser) parser.currentNode.addNode(Tree.InitPack(name, parser)) else: if not name in parser.compiled: parser.compiled[name] = None parser.imports.append(oname)
def start(run= False, dev= False, init= False): try: opt = 0 skip = 0 outputFile = "" for (iter, i) in enumerate(sys.argv[2:]): if skip > 0: continue skip -= 1 if i == "-O3": opt = 3 elif i == "-O2": opt = 2 elif i == "-o": outputFile = sys.argv[iter + 3] skip = 1 elif i == "-O1": opt = 1 else: Error.error("unknown argument '" + i + "'.") files = getCompilationFiles() allfilenames = [] allsources = [] sources = {} filenames = {} for c in files: sources[c] = [] filenames[c] = [] for i in files[c]: try: file = open(os.path.join(i[0], i[1]), mode="r") r = file.read() allsources.append(r) sources[c].append(r) if i[1][0].upper() == i[1][0]: Error.error("file name must be lowercase") filenames[c].append((c, i[1][:-4])) allfilenames.append((c, i[1][:-4])) file.close() except FileNotFoundError: Error.error("file " + i[1] +", not found") if outputFile == "": port = open("src/port.json") data = port.read() outputFile = (json.loads(data)["name"]) port.close() if filenames == []: Error.error("no input files") """ import cProfile profile = cProfile.Profile() profile.enable() """ time1 = time() # print ("============= Compiling ==============\n") """ for i in lexed: print(i.token+"_"+i.type) """ lexed = Lexer.lex(sources, filenames) #print("lexed") declarations = Parser.Parser(lexed, filenames) declarations.files = files declarations.lexed = lexed declarations.filenames = filenames declarations.opt = opt declarations.compiled = {} declarations.externFuncs = {"main": []} ResolveSymbols.resolve(declarations) #print("declarations") if ImportParser.shouldCompile(False, "main", declarations): parser = Parser.Parser(lexed["main"], filenames["main"]) ResolveSymbols.insert(declarations, parser, only= True) parser.files = files parser.lexed = lexed parser.filenames = filenames parser.compiled = declarations.compiled parser.compiled["main"] = None parsed = parser.parse() parser.compiled["main"] = (True, (parsed, parser.externFuncs["main"])) import AST as Tree allCode = Tree.Root() if opt > 0: for d in parser.compiled: allCode.addNode(parser.compiled[d][1][0]) optimize(allCode, opt) #print("parsing") for i in parser.compiled: if parser.compiled[i][0]: CodeGen.CodeGen(i, parser.compiled[i][1][0], parser.compiled[i][1][1]).compile(opt=opt) l = CodeGen.link(parser.compiled, outputFile, run=run, opt= opt, dev= dev) print("Compilation took : "+str(time() - time1)) return (True, l) elif run: CodeGen.exec(outputFile) elif init: return (True, open("bin/"+outputFile+".js").read()) elif dev: return (False, "") print("Compilation took : "+str(time() - time1)) except EOFError as e: if dev: return (False, str(error)) else: print(e, file= sys.stderr)
def importParser(parser, decl=False): import os name = parser.nextToken() if name.type != "str": Error.parseError(parser, "expecting string") oname = name.token[1:-1] if not oname in parser.filenames and not oname in ignore: Error.parseError(parser, "package " + oname + " not found") name = os.path.basename(oname) if not decl: if not parser.hotswap: sp = shouldParse(decl, oname, parser) else: sp = shouldCompile(decl, oname, parser) outputfile = oname try: if parser.global_target == "full": t = max( os.path.getmtime("lib/" + outputfile.replace("/", ".") + "-client.js"), os.path.getmtime("lib/" + outputfile.replace("/", ".") + "-client.js")) else: t = os.path.getmtime("lib/" + outputfile.replace("/", ".") + "-" + parser.global_target + ".js") t = datetime.datetime.fromtimestamp(int(t)) except FileNotFoundError: sp = True print("Error ", outputfile) if sp: p = Parser.Parser(parser.lexed[oname], parser.filenames[oname]) p.package = oname ResolveSymbols.insert(parser, p) global_target = parser.global_target p.global_target = "full" sc = shouldCompile(decl, oname, parser) p.sc = sc parser.compiled[name] = None #parser.externFuncs[name] = [] parsed = p.parse() declar = [] #parser.externFuncs[name] parser.compiled[name] = (sc, (parsed, declar)) ResolveSymbols.insert(p, parser) parser.global_target = global_target parser.currentNode.addNode(Tree.InitPack(name, parser)) else: if not name in parser.compiled: parser.compiled[name] = (False, ) parser.currentNode.addNode(Tree.InitPack(name, parser)) parser.imports.append(oname)
def compile(target, sources, filenames, former=None): global global_parser global_parser = cache #print(cache.usedModules) lexed = Lexer.lex(target, sources, filenames, files, cache, cache.lexed if cache else {}, transforms) declarations = Parser.Parser(lexed, filenames) declarations.hotswap = False declarations.shouldCompile = {} declarations.atoms = 0 declarations.atomTyp = False declarations.outputFile = outputFile declarations.jsFiles = [ b for (a, b) in clientLinkWithFiles + linkWithFiles + linkCSSWithFiles + nodeLinkWithFiles ] declarations.cssFiles = linkCSSWithFiles declarations.transforms = transforms declarations.usedModules = {} global_parser = declarations if cache: declarations.scope = cache.scope declarations.interfaces = cache.interfaces declarations.structs = cache.structs declarations.hotswap = hotswap declarations.allImports = cache.allImports declarations.atomTyp = cache.atomTyp declarations.hotswap = True declarations.usedModules = cache.usedModules if former: #print("inserting", target) ResolveSymbols.insert(former, declarations, only=True, copy=True) #print(declarations.scope["_global"]) declarations.files = files declarations.lexed = lexed declarations.filenames = filenames declarations.opt = opt declarations.compiled = {} declarations.externFuncs = {"main": []} declarations.filenames_sources = filenames_sources declarations.global_target = target declarations.output_target = target declarations.didCompile = False if (dev and run): clearMain(declarations) ResolveSymbols.resolve(declarations) #print("declarations") #print(declarations.shouldCompile) if opt == 3 or doc or ImportParser.shouldCompile( False, "main", declarations): parser = Parser.Parser(lexed["main"], filenames["main"]) parser.package = "main" ResolveSymbols.insert(declarations, parser, only=True) parser.files = files parser.global_target = target parser.output_target = target parser.lexed = lexed parser.filenames = filenames parser.compiled = declarations.compiled parser.compiled["main"] = None parser.dev = dev parsed = parser.parse() parser.compiled["main"] = (True, (parsed, [])) global_parser = parser import AST as Tree allCode = Tree.Root() """ if opt > 0: for d in parser.compiled: allCode.addNode(parser.compiled[d][1][0]) optimize(allCode, opt) """ #print("parsing") if doc: return parser canStartWith = [''] order_of_modules = [] for i in parser.compiled: tmp = os.path.dirname(parser.filenames[i][0][0]) dir = tmp[tmp.find("packages") + len("packages") + 1:tmp.rfind("src") - 1] canStartWith.append(dir) if parser.compiled[i][0]: CodeGen.CodeGen(order_of_modules, i, parser.compiled[i][1][0], parser.compiled[i][1][1], target, opt).compile(opt=opt) order_of_modules.append("main") for i in parser.lexed: parser.usedModules[i] = datetime.datetime.now() _linkCSSWithFiles = [ i for (d, i) in linkCSSWithFiles if d in canStartWith ] _clientLinkWithFiles = [ i for (d, i) in clientLinkWithFiles if d in canStartWith ] _nodeLinkWithFiles = [ i for (d, i) in nodeLinkWithFiles if d in canStartWith ] _linkWithFiles = [ i for (d, i) in linkWithFiles if d in canStartWith ] compiled = order_of_modules #parser.compiled if not dev and not _raise: saveParser.save(parser) print("\n======== recompiling =========") print("Compilation took : " + str(time() - time1)) if target == "full": client_linkWithFiles = _linkWithFiles + _clientLinkWithFiles node_linkWithFiles = _linkWithFiles + _nodeLinkWithFiles if run: a = CodeGen.link(compiled, outputFile, hotswap=hotswap, run=False, debug=debug, opt=opt, dev=dev, linkWithCSS=_linkCSSWithFiles, linkWith=client_linkWithFiles, target="client") print("Open website, at", "http://127.0.0.1:3000/") l = CodeGen.link(compiled, outputFile, debug=debug, hotswap=hotswap, run=run, opt=opt, dev=dev, linkWithCSS=_linkCSSWithFiles, linkWith=node_linkWithFiles, target="node") else: l = CodeGen.link(compiled, outputFile, debug=debug, hotswap=hotswap, run=run, opt=opt, dev=dev, linkWithCSS=_linkCSSWithFiles, linkWith=node_linkWithFiles, target="node") a = CodeGen.link(compiled, outputFile, hotswap=hotswap, run=False, debug=debug, opt=opt, dev=dev, linkWithCSS=_linkCSSWithFiles, linkWith=client_linkWithFiles, target="client") else: _link_CSSWithFiles = [] if target != "client" else _linkCSSWithFiles _linkWithFiles = _linkWithFiles + _nodeLinkWithFiles if target == "node" else _linkWithFiles + _clientLinkWithFiles if target == "client" else [] l = CodeGen.link(compiled, outputFile, run=run, opt=opt, dev=dev, hotswap=hotswap, linkWithCSS=_link_CSSWithFiles, debug=debug, linkWith=_linkWithFiles, target=target) didCompile = True parser.didCompile = True return parser elif run: if target == "full": print("Open website, at", "http://127.0.0.1:3000/") CodeGen.execNode(outputFile, dev) else: if target == "node": CodeGen.execNode(outputFile, dev) else: CodeGen.exec(outputFile) return declarations
def compile(target, sources, filenames, former=None): global global_parser global_parser = cache #print(cache.usedModules) lexed = Lexer.lex(target, sources, filenames, files, cache, {}, tags) print("Lexed and parsed : " + str(Lexer.linesOfCode)) declarations = Parser.Parser(lexed, filenames) declarations.hotswap = False declarations.shouldCompile = {} declarations.atoms = 0 declarations.atomTyp = False declarations.outputFile = outputFile declarations.usedModules = {} declarations.path = os.path.abspath("") declarations.compilingRuntime = compileRuntime global_parser = declarations if cache: declarations.scope = cache.scope declarations.interfaces = cache.interfaces declarations.structs = cache.structs declarations.hotswap = hotswap declarations.allImports = cache.allImports declarations.atomTyp = cache.atomTyp declarations.hotswap = True declarations.usedModules = cache.usedModules declarations.specifications = cache.specifications declarations.includes = cache.includes Types.genericTypes = cache.generatedGenericTypes Types.inProjectTypes = { name: None for name in Types.genericTypes } from TopCompiler import Tree Tree.casted = cache.casted declarations.contextFields = cache.contextFields declarations.contextType = {} for package in declarations.contextFields: declarations.contextType.update( declarations.contextFields[package]) if former: #print("inserting", target) ResolveSymbols.insert(former, declarations, only=True, copy=False) #print(declarations.scope["_global"]) declarations.files = files declarations.lexed = lexed declarations.filenames = filenames declarations.opt = opt declarations.compiled = coll.OrderedDict() declarations.externFuncs = {"main": []} declarations.filenames_sources = filenames_sources declarations.global_target = target declarations.output_target = target declarations.didCompile = False declarations.linkWith = linkWith if (dev and run): clearMain(declarations) #print("declarations") #print(declarations.shouldCompile) declarations.setGlobalData(compileRuntime) ResolveSymbols.resolve(declarations) if opt == 3 or doc or ImportParser.shouldCompile( False, "main", declarations): print("Recompiling") parser = Parser.Parser(lexed["main"], filenames["main"]) parser.package = "main" ResolveSymbols.insert(declarations, parser, only=True) parser.files = files parser.global_target = target parser.output_target = target parser.lexed = lexed parser.filenames = filenames parser.compiled = declarations.compiled parser.compiled["main"] = None parser.contextFields["main"] = {} parser.dev = dev parser.sc = True parsed = parser.parse() parser.compiled["main"] = (True, (parsed, [])) global_parser = parser import AST as Tree allCode = Tree.Root() """ if opt > 0: for d in parser.compiled: allCode.addNode(parser.compiled[d][1][0]) optimize(allCode, opt) """ #print("parsing") if doc: return parser canStartWith = [''] order_of_modules = [] compiled = parser.order_of_modules # order_of_modules #parser.compiled typesInContext = [] for i in parser.compiled: parser.package = i sc = parser.compiled[i][0] if sc or not cache: SimplifyAst.resolveGeneric(parser, parser.compiled[i][1][0]) #generatedTypes = Types.genericTypes #Types.genericTypes = {} contextCCode = CodeGen.buildContext(parser) """ for i in parser.compiled: parser.package = i sc = parser.compiled[i][0] if not parser.compiled[i][0]: if cache and i in cache.generatedTypesPerPackage: parser.generatedTypesPerPackage[i] = cache.generatedTypesPerPackage[i] for typ in cache.generatedTypesPerPackage[i]: Types.genericTypes[typ] = None Types.inProjectTypes[typ] = None """ #print(Types.genericTypes) #generatedTypes.update(Types.genericTypes) #Types.genericTypes = generatedTypes #if not compileRuntime: # addTypes(removedTypes) # contextCCode = CodeGen.buildContext(parser.contextType) includes = [] for i in compiled: tmp = os.path.dirname(parser.filenames[i][0][0]) dir = tmp[tmp.find("packages") + len("packages") + 1:tmp.rfind("src") - 1] canStartWith.append(dir) sc = parser.compiled[i][0] if sc or not cache: inc = CodeGen.CodeGen(parser, order_of_modules, i, parser.compiled[i][1][0], parser.compiled[i][1][1], target, opt, debug=debug).compile(opt=opt) includes.extend(inc) parser.includes[i] = inc else: includes.extend(parser.includes[i]) order_of_modules.append("main") for i in parser.lexed: parser.usedModules[i] = datetime.datetime.now() _linkWith = [i for (d, i) in linkWith if d in canStartWith] _headerIncludePath = [ i for (d, i) in headerIncludePath if d in canStartWith ] parser.generatedGenericTypes = Types.genericTypes if compileRuntime: #not dev and not _raise: deleteQue = [] for c in parser.generatedGenericTypes: parser.generatedGenericTypes[c] = None if c in ["_global_Allocator", "_global_Type"]: deleteQue.append(c) for c in deleteQue: del parser.generatedGenericTypes[c] timeForCodeAnalysis = time() - beforeLoad print("Code Analysis : " + str(timeForCodeAnalysis)) if False: #not compileRuntime: c = [] for i in compiled: sc = parser.compiled[i][0] if sc: c.append(i) else: header = "lib/" + i + ".h" header = os.path.abspath(header) includes.append('#include "' + header + '"\n') print(includes) prelink = CodeGen.link( c, "recompiled_" + outputFile, to_obj=True, opt=opt, dev=dev, hotswap=hotswap, debug=debug, includes=includes, linkWith=[], headerIncludePath=_headerIncludePath, target=target, context=contextCCode, runtimeBuild=compileRuntime) l = CodeGen.link([], outputFile, to_obj=True, opt=opt, dev=dev, hotswap=hotswap, debug=debug, includes=includes, linkWith=[], headerIncludePath=_headerIncludePath, target=target, context=contextCCode, runtimeBuild=compileRuntime) if not to_obj: l = CodeGen.link([], outputFile, to_obj=to_obj, opt=opt, dev=dev, hotswap=hotswap, debug=debug, includes=includes, linkWith=_linkWith + [ outputFile + ".o", "recompiled_" + outputFile + ".o" ], headerIncludePath=_headerIncludePath, target=target, context=contextCCode, runtimeBuild=compileRuntime) else: l = CodeGen.link(compiled, outputFile, to_obj=to_obj, opt=opt, dev=dev, hotswap=hotswap, debug=debug, includes=includes, linkWith=_linkWith, headerIncludePath=_headerIncludePath, target=target, context=contextCCode, runtimeBuild=compileRuntime) if compileRuntime: saveParser.save(parser, compileRuntime) print("\n======== recompiling =========") print("Compilation took : " + str(time() - time1)) print("") if run: if to_obj: Error.error("Cannot run .obj") CodeGen.exec(outputFile) didCompile = True parser.didCompile = True return parser elif run: CodeGen.exec(outputFile) return declarations
def compile(target, sources, filenames, former=None): lexed = Lexer.lex(sources, filenames) declarations = Parser.Parser(lexed, filenames) declarations.hotswap = False declarations.shouldCompile = {} declarations.atoms = 0 declarations.atomTyp = False declarations.jsFiles = clientLinkWithFiles + linkWithFiles if cache: declarations.scope = cache.scope declarations.interfaces = cache.interfaces declarations.structs = cache.structs declarations.hotswap = hotswap declarations.allImports = cache.allImports declarations.atomTyp = cache.atomTyp if former: #print("inserting", target) ResolveSymbols.insert(former, declarations, only=True, copy=True) #print(declarations.scope["_global"]) declarations.files = files declarations.lexed = lexed declarations.filenames = filenames declarations.opt = opt declarations.compiled = {} declarations.externFuncs = {"main": []} declarations.filenames_sources = filenames_sources declarations.global_target = target ResolveSymbols.resolve(declarations) #print("declarations") #print(declarations.shouldCompile) if (dev and run) or ImportParser.shouldCompile( False, "main", declarations): parser = Parser.Parser(lexed["main"], filenames["main"]) ResolveSymbols.insert(declarations, parser, only=True) parser.files = files parser.global_target = target parser.lexed = lexed parser.filenames = filenames parser.compiled = declarations.compiled parser.compiled["main"] = None parsed = parser.parse() parser.compiled["main"] = (True, (parsed, parser.externFuncs["main"])) import AST as Tree allCode = Tree.Root() """ if opt > 0: for d in parser.compiled: allCode.addNode(parser.compiled[d][1][0]) optimize(allCode, opt) """ #print("parsing") for i in parser.compiled: if parser.compiled[i][0]: CodeGen.CodeGen(i, parser.compiled[i][1][0], parser.compiled[i][1][1], target).compile(opt=opt) if target == "full": _linkCSSWithFiles = linkCSSWithFiles client_linkWithFiles = linkWithFiles + clientLinkWithFiles node_linkWithFiles = linkWithFiles + nodeLinkWithFiles a = CodeGen.link(parser.compiled, outputFile, hotswap=hotswap, run=False, opt=opt, dev=dev, linkWithCSS=_linkCSSWithFiles, linkWith=client_linkWithFiles, target="client") if run: import webbrowser webbrowser.open("http://127.0.0.1:3000/") l = CodeGen.link(parser.compiled, outputFile, hotswap=hotswap, run=run, opt=opt, dev=dev, linkWithCSS=_linkCSSWithFiles, linkWith=node_linkWithFiles, target="node") else: _linkCSSWithFiles = [] if target != "client" else linkCSSWithFiles _linkWithFiles = linkWithFiles + nodeLinkWithFiles if target == "node" else linkWithFiles + clientLinkWithFiles if target == "client" else [] l = CodeGen.link(parser.compiled, outputFile, run=run, opt=opt, dev=dev, hotswap=hotswap, linkWithCSS=_linkCSSWithFiles, linkWith=_linkWithFiles, target=target) didCompile = True print("\n======== recompiling =========") return parser elif run: if target == "full": if run: import webbrowser webbrowser.open("http://127.0.0.1:3000/") CodeGen.execNode(outputFile, dev) else: CodeGen.exec(outputFile) return declarations
def importParser(parser, decl=False): import os name = parser.nextToken() if name.type != "str": Error.parseError(parser, "expecting string") oname = name.token[1:-1] if not oname in parser.filenames and not oname in ignore: Error.parseError(parser, "package " + oname + " not found") name = os.path.basename(oname) if not decl: #if not parser.hotswap: sp = shouldParse(decl, oname, parser) # print("should parse", sp) #else: sc = shouldCompile(decl, oname, parser) #outputfile = oname #print("Error ", outputfile) if sp: p = Parser.Parser(parser.lexed[oname], parser.filenames[oname]) p.package = oname sc = True ResolveSymbols.insert(parser, p) p.sc = sc global_target = parser.global_target p.global_target = "full" #sc = shouldCompile(decl, oname, parser) parser.compiled[name] = None #parser.externFuncs[name] = [] parsed = p.parse() declar = [] #parser.externFuncs[name] parser.compiled[name] = (sc, (parsed, declar)) ResolveSymbols.insert(p, parser) parser.global_target = global_target parser.currentNode.addNode(Tree.InitPack(name, parser)) else: if not name in parser.compiled: def loop(name): parser.compiled[name] = (False, ) parser.currentNode.addNode(Tree.InitPack(name, parser)) #print("not recompiling", name) #print(parser.allImports[name]) for imports in parser.allImports[name]: if not imports in parser.compiled: parser.compiled[imports] = ( False, ) #Assumption is correct as package has to be recompiled when it's dependencies change parser.currentNode.addNode( Tree.InitPack(imports, parser)) loop(imports) if not imports in parser.order_of_modules: parser.order_of_modules.append(imports) if not name in parser.order_of_modules: parser.order_of_modules.append(name) loop(name) parser.imports.append(oname)