def _resolve(self, tokens, filename, passN= 0 ):
if self.hotswap and not ImportParser.shouldCompile(False, self.package, self):
return
self.filename = filename
self.iter = 0
self.tokens = tokens
while self.iter < len(tokens) - 1 :
b = self.thisToken().token
if passN == 2:
if b == "import":
ImportParser.importParser(self, True)
elif b == "def" :
if self.indentLevel == 0:
nex = self.lookInfront()
Parser.addBookmark(self)
funcHead(self)
Parser.returnBookmark(self)
elif passN == 1:
if b == "import":
ImportParser.importParser(self, True)
elif b == "type":
Parser.addBookmark(self)
Struct.typeParser(self, decl= True)
Parser.returnBookmark(self)
elif passN == 0:
if b == "type":
Scope.addVar(Tree.Node(self), self, self.nextToken().token, Scope.Type(True, Types.StructInit(self.thisToken().token)))
#"""
self.structs[self.package][self.thisToken().token] = Struct.Struct(self.thisToken().token, [],[], {}, self)
self.structs[self.package][self.thisToken().token].methods = {}
#"""
if b == "\n":
Parser.addBookmark(self)
Parser.newLine(self)
Parser.returnBookmark(self)
self.nextToken()
for i in self.imports:
if not i in self.allImports[self.package]:
self.allImports[self.package].add(i)
self.imports = []
self.lineNumber = 1
self.normalIndent = 0
def resolve(self):
tokens = self.tokens
filenames = self.filename
for c in filenames:
self._filename = self.filenames[c]
self.package = c
self.opackage = c
if not self.hotswap or ImportParser.shouldCompile(False, c, self):
if c in self.contextFields:
for i in self.contextFields[c]:
del self.contextType[i]
PackageParser.packDec(self, c, pack=True)
if self.package != "_global":
self.scope[self.package] = [{}]
self.structs[self.package] = {}
self.interfaces[self.package] = {}
if not c in self.allImports:
self.allImports[c] = []
for i in range(len(tokens[c])):
_resolve(self, tokens[c][i], self._filename[i][1], passN=0)
for n in range(1, 3):
start = time()
for c in filenames:
self._filename = self.filenames[c]
if len(filenames[c]) == 0:
continue
self.package = c
self.opackage = c
for i in range(len(tokens[c])):
if self.package == "_global" and i == len(tokens[c]) - 1:
self.setArrayTypes()
_resolve(self, tokens[c][i], self._filename[i][1], passN=n)
self.rootAst = Tree.Root()
self.currentNode = self.rootAst
self.tokens = tokens
self.filename = filenames
return self
def resolve(self):
tokens = self.tokens
filenames = self.filename
for c in filenames:
self._filename = self.filenames[c]
PackageParser.packDec(self, c, pack=True)
if self.hotswap and ImportParser.shouldCompile(False, self.package,
self):
self.scope[self.package] = [{}]
self.structs[self.package] = {}
self.interfaces[self.package] = {}
if not c in self.allImports:
self.allImports[c] = []
for i in range(len(tokens[c])):
_resolve(self, tokens[c][i], self._filename[i][1], passN=0)
for n in range(1, 3):
for c in filenames:
self._filename = self.filenames[c]
if len(filenames[c]) == 0:
continue
self.package = c
self.opackage = c
for i in range(len(tokens[c])):
_resolve(self, tokens[c][i], self._filename[i][1], passN=n)
self.rootAst = Tree.Root()
self.currentNode = self.rootAst
self.tokens = tokens
self.filename = filenames
return self
def start(run= False, dev= False, init= False):
try:
opt = 0
skip = 0
outputFile = ""
for (iter, i) in enumerate(sys.argv[2:]):
if skip > 0:
continue
skip -= 1
if i == "-O3":
opt = 3
elif i == "-O2":
opt = 2
elif i == "-o":
outputFile = sys.argv[iter + 3]
skip = 1
elif i == "-O1":
opt = 1
else:
Error.error("unknown argument '" + i + "'.")
files = getCompilationFiles()
allfilenames = []
allsources = []
sources = {}
filenames = {}
for c in files:
sources[c] = []
filenames[c] = []
for i in files[c]:
try:
file = open(os.path.join(i[0], i[1]), mode="r")
r = file.read()
allsources.append(r)
sources[c].append(r)
if i[1][0].upper() == i[1][0]:
Error.error("file name must be lowercase")
filenames[c].append((c, i[1][:-4]))
allfilenames.append((c, i[1][:-4]))
file.close()
except FileNotFoundError:
Error.error("file " + i[1] +", not found")
if outputFile == "":
port = open("src/port.json")
data = port.read()
outputFile = (json.loads(data)["name"])
port.close()
if filenames == []:
Error.error("no input files")
"""
import cProfile
profile = cProfile.Profile()
profile.enable()
"""
time1 = time()
# print ("============= Compiling ==============\n")
"""
for i in lexed:
print(i.token+"_"+i.type)
"""
lexed = Lexer.lex(sources, filenames)
#print("lexed")
declarations = Parser.Parser(lexed, filenames)
declarations.files = files
declarations.lexed = lexed
declarations.filenames = filenames
declarations.opt = opt
declarations.compiled = {}
declarations.externFuncs = {"main": []}
ResolveSymbols.resolve(declarations)
#print("declarations")
if ImportParser.shouldCompile(False, "main", declarations):
parser = Parser.Parser(lexed["main"], filenames["main"])
ResolveSymbols.insert(declarations, parser, only= True)
parser.files = files
parser.lexed = lexed
parser.filenames = filenames
parser.compiled = declarations.compiled
parser.compiled["main"] = None
parsed = parser.parse()
parser.compiled["main"] = (True, (parsed, parser.externFuncs["main"]))
import AST as Tree
allCode = Tree.Root()
if opt > 0:
for d in parser.compiled:
allCode.addNode(parser.compiled[d][1][0])
optimize(allCode, opt)
#print("parsing")
for i in parser.compiled:
if parser.compiled[i][0]:
CodeGen.CodeGen(i, parser.compiled[i][1][0], parser.compiled[i][1][1]).compile(opt=opt)
l = CodeGen.link(parser.compiled, outputFile, run=run, opt= opt, dev= dev)
print("Compilation took : "+str(time() - time1))
return (True, l)
elif run:
CodeGen.exec(outputFile)
elif init:
return (True, open("bin/"+outputFile+".js").read())
elif dev:
return (False, "")
print("Compilation took : "+str(time() - time1))
except EOFError as e:
if dev:
return (False, str(error))
else:
print(e, file= sys.stderr)
def compile(target, sources, filenames, former=None):
global global_parser
global_parser = cache
#print(cache.usedModules)
lexed = Lexer.lex(target, sources, filenames, files, cache,
cache.lexed if cache else {}, transforms)
declarations = Parser.Parser(lexed, filenames)
declarations.hotswap = False
declarations.shouldCompile = {}
declarations.atoms = 0
declarations.atomTyp = False
declarations.outputFile = outputFile
declarations.jsFiles = [
b for (a, b) in clientLinkWithFiles + linkWithFiles +
linkCSSWithFiles + nodeLinkWithFiles
]
declarations.cssFiles = linkCSSWithFiles
declarations.transforms = transforms
declarations.usedModules = {}
global_parser = declarations
if cache:
declarations.scope = cache.scope
declarations.interfaces = cache.interfaces
declarations.structs = cache.structs
declarations.hotswap = hotswap
declarations.allImports = cache.allImports
declarations.atomTyp = cache.atomTyp
declarations.hotswap = True
declarations.usedModules = cache.usedModules
if former:
#print("inserting", target)
ResolveSymbols.insert(former,
declarations,
only=True,
copy=True)
#print(declarations.scope["_global"])
declarations.files = files
declarations.lexed = lexed
declarations.filenames = filenames
declarations.opt = opt
declarations.compiled = {}
declarations.externFuncs = {"main": []}
declarations.filenames_sources = filenames_sources
declarations.global_target = target
declarations.output_target = target
declarations.didCompile = False
if (dev and run):
clearMain(declarations)
ResolveSymbols.resolve(declarations)
#print("declarations")
#print(declarations.shouldCompile)
if opt == 3 or doc or ImportParser.shouldCompile(
False, "main", declarations):
parser = Parser.Parser(lexed["main"], filenames["main"])
parser.package = "main"
ResolveSymbols.insert(declarations, parser, only=True)
parser.files = files
parser.global_target = target
parser.output_target = target
parser.lexed = lexed
parser.filenames = filenames
parser.compiled = declarations.compiled
parser.compiled["main"] = None
parser.dev = dev
parsed = parser.parse()
parser.compiled["main"] = (True, (parsed, []))
global_parser = parser
import AST as Tree
allCode = Tree.Root()
"""
if opt > 0:
for d in parser.compiled:
allCode.addNode(parser.compiled[d][1][0])
optimize(allCode, opt)
"""
#print("parsing")
if doc:
return parser
canStartWith = ['']
order_of_modules = []
for i in parser.compiled:
tmp = os.path.dirname(parser.filenames[i][0][0])
dir = tmp[tmp.find("packages") + len("packages") +
1:tmp.rfind("src") - 1]
canStartWith.append(dir)
if parser.compiled[i][0]:
CodeGen.CodeGen(order_of_modules, i,
parser.compiled[i][1][0],
parser.compiled[i][1][1], target,
opt).compile(opt=opt)
order_of_modules.append("main")
for i in parser.lexed:
parser.usedModules[i] = datetime.datetime.now()
_linkCSSWithFiles = [
i for (d, i) in linkCSSWithFiles if d in canStartWith
]
_clientLinkWithFiles = [
i for (d, i) in clientLinkWithFiles if d in canStartWith
]
_nodeLinkWithFiles = [
i for (d, i) in nodeLinkWithFiles if d in canStartWith
]
_linkWithFiles = [
i for (d, i) in linkWithFiles if d in canStartWith
]
compiled = order_of_modules #parser.compiled
if not dev and not _raise:
saveParser.save(parser)
print("\n======== recompiling =========")
print("Compilation took : " + str(time() - time1))
if target == "full":
client_linkWithFiles = _linkWithFiles + _clientLinkWithFiles
node_linkWithFiles = _linkWithFiles + _nodeLinkWithFiles
if run:
a = CodeGen.link(compiled,
outputFile,
hotswap=hotswap,
run=False,
debug=debug,
opt=opt,
dev=dev,
linkWithCSS=_linkCSSWithFiles,
linkWith=client_linkWithFiles,
target="client")
print("Open website, at", "http://127.0.0.1:3000/")
l = CodeGen.link(compiled,
outputFile,
debug=debug,
hotswap=hotswap,
run=run,
opt=opt,
dev=dev,
linkWithCSS=_linkCSSWithFiles,
linkWith=node_linkWithFiles,
target="node")
else:
l = CodeGen.link(compiled,
outputFile,
debug=debug,
hotswap=hotswap,
run=run,
opt=opt,
dev=dev,
linkWithCSS=_linkCSSWithFiles,
linkWith=node_linkWithFiles,
target="node")
a = CodeGen.link(compiled,
outputFile,
hotswap=hotswap,
run=False,
debug=debug,
opt=opt,
dev=dev,
linkWithCSS=_linkCSSWithFiles,
linkWith=client_linkWithFiles,
target="client")
else:
_link_CSSWithFiles = [] if target != "client" else _linkCSSWithFiles
_linkWithFiles = _linkWithFiles + _nodeLinkWithFiles if target == "node" else _linkWithFiles + _clientLinkWithFiles if target == "client" else []
l = CodeGen.link(compiled,
outputFile,
run=run,
opt=opt,
dev=dev,
hotswap=hotswap,
linkWithCSS=_link_CSSWithFiles,
debug=debug,
linkWith=_linkWithFiles,
target=target)
didCompile = True
parser.didCompile = True
return parser
elif run:
if target == "full":
print("Open website, at", "http://127.0.0.1:3000/")
CodeGen.execNode(outputFile, dev)
else:
if target == "node":
CodeGen.execNode(outputFile, dev)
else:
CodeGen.exec(outputFile)
return declarations
def compile(target, sources, filenames, former=None):
global global_parser
global_parser = cache
#print(cache.usedModules)
lexed = Lexer.lex(target, sources, filenames, files, cache, {},
tags)
print("Lexed and parsed : " + str(Lexer.linesOfCode))
declarations = Parser.Parser(lexed, filenames)
declarations.hotswap = False
declarations.shouldCompile = {}
declarations.atoms = 0
declarations.atomTyp = False
declarations.outputFile = outputFile
declarations.usedModules = {}
declarations.path = os.path.abspath("")
declarations.compilingRuntime = compileRuntime
global_parser = declarations
if cache:
declarations.scope = cache.scope
declarations.interfaces = cache.interfaces
declarations.structs = cache.structs
declarations.hotswap = hotswap
declarations.allImports = cache.allImports
declarations.atomTyp = cache.atomTyp
declarations.hotswap = True
declarations.usedModules = cache.usedModules
declarations.specifications = cache.specifications
declarations.includes = cache.includes
Types.genericTypes = cache.generatedGenericTypes
Types.inProjectTypes = {
name: None
for name in Types.genericTypes
}
from TopCompiler import Tree
Tree.casted = cache.casted
declarations.contextFields = cache.contextFields
declarations.contextType = {}
for package in declarations.contextFields:
declarations.contextType.update(
declarations.contextFields[package])
if former:
#print("inserting", target)
ResolveSymbols.insert(former,
declarations,
only=True,
copy=False)
#print(declarations.scope["_global"])
declarations.files = files
declarations.lexed = lexed
declarations.filenames = filenames
declarations.opt = opt
declarations.compiled = coll.OrderedDict()
declarations.externFuncs = {"main": []}
declarations.filenames_sources = filenames_sources
declarations.global_target = target
declarations.output_target = target
declarations.didCompile = False
declarations.linkWith = linkWith
if (dev and run):
clearMain(declarations)
#print("declarations")
#print(declarations.shouldCompile)
declarations.setGlobalData(compileRuntime)
ResolveSymbols.resolve(declarations)
if opt == 3 or doc or ImportParser.shouldCompile(
False, "main", declarations):
print("Recompiling")
parser = Parser.Parser(lexed["main"], filenames["main"])
parser.package = "main"
ResolveSymbols.insert(declarations, parser, only=True)
parser.files = files
parser.global_target = target
parser.output_target = target
parser.lexed = lexed
parser.filenames = filenames
parser.compiled = declarations.compiled
parser.compiled["main"] = None
parser.contextFields["main"] = {}
parser.dev = dev
parser.sc = True
parsed = parser.parse()
parser.compiled["main"] = (True, (parsed, []))
global_parser = parser
import AST as Tree
allCode = Tree.Root()
"""
if opt > 0:
for d in parser.compiled:
allCode.addNode(parser.compiled[d][1][0])
optimize(allCode, opt)
"""
#print("parsing")
if doc:
return parser
canStartWith = ['']
order_of_modules = []
compiled = parser.order_of_modules # order_of_modules #parser.compiled
typesInContext = []
for i in parser.compiled:
parser.package = i
sc = parser.compiled[i][0]
if sc or not cache:
SimplifyAst.resolveGeneric(parser,
parser.compiled[i][1][0])
#generatedTypes = Types.genericTypes
#Types.genericTypes = {}
contextCCode = CodeGen.buildContext(parser)
"""
for i in parser.compiled:
parser.package = i
sc = parser.compiled[i][0]
if not parser.compiled[i][0]:
if cache and i in cache.generatedTypesPerPackage:
parser.generatedTypesPerPackage[i] = cache.generatedTypesPerPackage[i]
for typ in cache.generatedTypesPerPackage[i]:
Types.genericTypes[typ] = None
Types.inProjectTypes[typ] = None
"""
#print(Types.genericTypes)
#generatedTypes.update(Types.genericTypes)
#Types.genericTypes = generatedTypes
#if not compileRuntime:
# addTypes(removedTypes)
# contextCCode = CodeGen.buildContext(parser.contextType)
includes = []
for i in compiled:
tmp = os.path.dirname(parser.filenames[i][0][0])
dir = tmp[tmp.find("packages") + len("packages") +
1:tmp.rfind("src") - 1]
canStartWith.append(dir)
sc = parser.compiled[i][0]
if sc or not cache:
inc = CodeGen.CodeGen(parser,
order_of_modules,
i,
parser.compiled[i][1][0],
parser.compiled[i][1][1],
target,
opt,
debug=debug).compile(opt=opt)
includes.extend(inc)
parser.includes[i] = inc
else:
includes.extend(parser.includes[i])
order_of_modules.append("main")
for i in parser.lexed:
parser.usedModules[i] = datetime.datetime.now()
_linkWith = [i for (d, i) in linkWith if d in canStartWith]
_headerIncludePath = [
i for (d, i) in headerIncludePath if d in canStartWith
]
parser.generatedGenericTypes = Types.genericTypes
if compileRuntime: #not dev and not _raise:
deleteQue = []
for c in parser.generatedGenericTypes:
parser.generatedGenericTypes[c] = None
if c in ["_global_Allocator", "_global_Type"]:
deleteQue.append(c)
for c in deleteQue:
del parser.generatedGenericTypes[c]
timeForCodeAnalysis = time() - beforeLoad
print("Code Analysis : " + str(timeForCodeAnalysis))
if False: #not compileRuntime:
c = []
for i in compiled:
sc = parser.compiled[i][0]
if sc:
c.append(i)
else:
header = "lib/" + i + ".h"
header = os.path.abspath(header)
includes.append('#include "' + header + '"\n')
print(includes)
prelink = CodeGen.link(
c,
"recompiled_" + outputFile,
to_obj=True,
opt=opt,
dev=dev,
hotswap=hotswap,
debug=debug,
includes=includes,
linkWith=[],
headerIncludePath=_headerIncludePath,
target=target,
context=contextCCode,
runtimeBuild=compileRuntime)
l = CodeGen.link([],
outputFile,
to_obj=True,
opt=opt,
dev=dev,
hotswap=hotswap,
debug=debug,
includes=includes,
linkWith=[],
headerIncludePath=_headerIncludePath,
target=target,
context=contextCCode,
runtimeBuild=compileRuntime)
if not to_obj:
l = CodeGen.link([],
outputFile,
to_obj=to_obj,
opt=opt,
dev=dev,
hotswap=hotswap,
debug=debug,
includes=includes,
linkWith=_linkWith + [
outputFile + ".o",
"recompiled_" + outputFile + ".o"
],
headerIncludePath=_headerIncludePath,
target=target,
context=contextCCode,
runtimeBuild=compileRuntime)
else:
l = CodeGen.link(compiled,
outputFile,
to_obj=to_obj,
opt=opt,
dev=dev,
hotswap=hotswap,
debug=debug,
includes=includes,
linkWith=_linkWith,
headerIncludePath=_headerIncludePath,
target=target,
context=contextCCode,
runtimeBuild=compileRuntime)
if compileRuntime:
saveParser.save(parser, compileRuntime)
print("\n======== recompiling =========")
print("Compilation took : " + str(time() - time1))
print("")
if run:
if to_obj:
Error.error("Cannot run .obj")
CodeGen.exec(outputFile)
didCompile = True
parser.didCompile = True
return parser
elif run:
CodeGen.exec(outputFile)
return declarations
def _resolve(self, tokens, filename, passN=0):
target = self.global_target
if self.package != "main":
self.global_target = "full"
if self.hotswap and not ImportParser.shouldCompile(False, self.package,
self):
return
self.filename = filename
self.iter = 0
self.tokens = tokens
while self.iter < len(tokens) - 1:
b = self.thisToken().token
if passN == 2:
if b == "import":
ImportParser.importParser(self, True)
elif b == "from":
ImportParser.fromParser(self, True)
elif b == "def":
if self.indentLevel == 0:
nex = self.lookInfront()
Parser.addBookmark(self)
funcHead(self)
Parser.returnBookmark(self)
elif passN == 1:
if b == "import":
ImportParser.importParser(self, True)
elif b == "from":
ImportParser.fromParser(self, True)
elif b == "type":
Parser.addBookmark(self)
Struct.typeParser(self, decl=True)
Parser.returnBookmark(self)
elif passN == 0:
if b == "type":
name = self.nextToken().token
if name == "ext":
name = self.nextToken().token
ofType = None
gen = {}
if self.nextToken().token == "[":
Scope.incrScope(self)
gen = FuncParser.generics(self, name)
Scope.decrScope(self)
ofType = self.thisToken().token
else:
ofType = self.thisToken().token
Scope.addVar(Tree.Node(self), self, name,
Scope.Type(True, Types.StructInit(name)))
if ofType is None or ofType == "=":
#"""
self.structs[self.package][name] = Struct.Struct(
name, [], [], gen, self, self.package)
self.structs[self.package][name].methods = {}
#"""
elif ofType == "either":
self.interfaces[self.package][name] = Types.Enum(
self.package, name, coll.OrderedDict(), gen)
elif ofType == "with":
self.interfaces[self.package][name] = Types.Interface(
False, {}, name=self.package + "." + name)
elif ofType == "is":
self.interfaces[self.package][name] = Types.Alias(
self.package, name, Types.Null(), gen)
if b == "\n":
Parser.addBookmark(self)
Parser.newLine(self)
Parser.returnBookmark(self)
self.nextToken()
for i in self.imports:
if not i in self.allImports[self.package]:
self.allImports[self.package].append(i)
self.imports = []
self.lineNumber = 1
self.normalIndent = 0
self.global_target = target
def compile(target, sources, filenames, former=None):
lexed = Lexer.lex(sources, filenames)
declarations = Parser.Parser(lexed, filenames)
declarations.hotswap = False
declarations.shouldCompile = {}
declarations.atoms = 0
declarations.atomTyp = False
declarations.jsFiles = clientLinkWithFiles + linkWithFiles
if cache:
declarations.scope = cache.scope
declarations.interfaces = cache.interfaces
declarations.structs = cache.structs
declarations.hotswap = hotswap
declarations.allImports = cache.allImports
declarations.atomTyp = cache.atomTyp
if former:
#print("inserting", target)
ResolveSymbols.insert(former,
declarations,
only=True,
copy=True)
#print(declarations.scope["_global"])
declarations.files = files
declarations.lexed = lexed
declarations.filenames = filenames
declarations.opt = opt
declarations.compiled = {}
declarations.externFuncs = {"main": []}
declarations.filenames_sources = filenames_sources
declarations.global_target = target
ResolveSymbols.resolve(declarations)
#print("declarations")
#print(declarations.shouldCompile)
if (dev and run) or ImportParser.shouldCompile(
False, "main", declarations):
parser = Parser.Parser(lexed["main"], filenames["main"])
ResolveSymbols.insert(declarations, parser, only=True)
parser.files = files
parser.global_target = target
parser.lexed = lexed
parser.filenames = filenames
parser.compiled = declarations.compiled
parser.compiled["main"] = None
parsed = parser.parse()
parser.compiled["main"] = (True, (parsed,
parser.externFuncs["main"]))
import AST as Tree
allCode = Tree.Root()
"""
if opt > 0:
for d in parser.compiled:
allCode.addNode(parser.compiled[d][1][0])
optimize(allCode, opt)
"""
#print("parsing")
for i in parser.compiled:
if parser.compiled[i][0]:
CodeGen.CodeGen(i, parser.compiled[i][1][0],
parser.compiled[i][1][1],
target).compile(opt=opt)
if target == "full":
_linkCSSWithFiles = linkCSSWithFiles
client_linkWithFiles = linkWithFiles + clientLinkWithFiles
node_linkWithFiles = linkWithFiles + nodeLinkWithFiles
a = CodeGen.link(parser.compiled,
outputFile,
hotswap=hotswap,
run=False,
opt=opt,
dev=dev,
linkWithCSS=_linkCSSWithFiles,
linkWith=client_linkWithFiles,
target="client")
if run:
import webbrowser
webbrowser.open("http://127.0.0.1:3000/")
l = CodeGen.link(parser.compiled,
outputFile,
hotswap=hotswap,
run=run,
opt=opt,
dev=dev,
linkWithCSS=_linkCSSWithFiles,
linkWith=node_linkWithFiles,
target="node")
else:
_linkCSSWithFiles = [] if target != "client" else linkCSSWithFiles
_linkWithFiles = linkWithFiles + nodeLinkWithFiles if target == "node" else linkWithFiles + clientLinkWithFiles if target == "client" else []
l = CodeGen.link(parser.compiled,
outputFile,
run=run,
opt=opt,
dev=dev,
hotswap=hotswap,
linkWithCSS=_linkCSSWithFiles,
linkWith=_linkWithFiles,
target=target)
didCompile = True
print("\n======== recompiling =========")
return parser
elif run:
if target == "full":
if run:
import webbrowser
webbrowser.open("http://127.0.0.1:3000/")
CodeGen.execNode(outputFile, dev)
else:
CodeGen.exec(outputFile)
return declarations
