Esempio n. 1
0
    def pushTok(self):
        if self.tok == "": return
        if self.tok in keywords:
            self.append(Token(self.tok, "keyword", self.line, self.column))
        else:
            if self.tok == "\t": Error.errorAst("Found tab expecting only spaces", self.package, self.filename, Token(self.tok, "", self.line, self.column))
            typ = ""
            match = None
            for (it, (group, regex)) in enumerate(tokenSpecification):
                regex = compiledSpecifications[it]
                match = regex.match(self.tok)
                if match:
                    typ = group
                    break
            if typ == "":
                Error.errorAst("Unexpected token " + self.tok, self.package, self.filename, Token(self.tok, "", self.line, self.column))

            if not typ == "identifier":
                tok = self.tok.replace("_", "")
            else:
                tok = self.tok

            self.append(Token(tok, typ, self.line, self.column))

        self.tok = ""
Esempio n. 2
0
def pattern(name, names, parser, getName):
    if type(name) is Tree.Tuple:
        for i in name:
            pattern(i, names, parser, getName)
        return name
    elif type(name) is Tree.PlaceHolder:
        node = name.nodes[0]
        pattern(node, names, parser, getName)
    elif type(name) is Tree.InitStruct:
        for i in name:
            if type(i) is Tree.Assign:
                if not type(i.nodes[0]) is Tree.ReadVar:
                    i.error("Expecting variable name")
                pattern(i.nodes[1], names, parser, getName)
            elif type(i) is Tree.ReadVar:
                pattern(i, names, parser, getName)
            else:
                i.error("Expecting variable name")
        return name
    elif type(name) is Tree.ReadVar:
        names.append(name.name)
        return getName(name.token)
    elif type(name) is Tree.Under:
        names.append(name.name)
    elif type(name) is Lexer.Token and name.type == "identifier":
        return getName(name)
    else:
        if type(name) is Lexer.Token:
            if name.type == "indent":
                Error.parseError(parser, "Expecting identifier")
            else:
                Error.parseError(parser, "Unexpected token " + name.token)
        else:
            name.error("Unexpected token")
Esempio n. 3
0
def main():
    err = False
    try:
        f = open("src/port.json")
    except:
        err = True

    if err:
        try:
            Error.error("cannot find port.json")
        except EOFError as e:
            print(e, file=sys.stderr)
            sys.exit()

    file = f.read()
    port = json.loads(file)

    if len(sys.argv) > 1:
        if sys.argv[1] == "install":
            installPackage()
        elif sys.argv[1] == "uninstall":
            uninstallPackage()
        elif sys.argv[1] == "deploy":
            deploy(port)
        elif sys.argv[1] == "doc":
            doc(port)
        else:
            print("Unknown command "+sys.argv[1], file=sys.stderr)
    else:
        print("Need command", file=sys.stderr)
Esempio n. 4
0
def _packDec(parser, pack=False):
    name = parser.nextToken()
    if name.type != "identifier":
        Error.parseError(parser, "package name must be an identifier")
    name = name.token

    packDec(parser, name, pack)
Esempio n. 5
0
def checkOther(self, parser, function, block, iter=0):
    if type(self) in [Tree.FuncStart, Tree.FuncBraceOpen]: return

    for (c, i) in enumerate(self.nodes):
        if type(self) in [Tree.While, Tree.For]:
            checkOther(i, parser, function, self, c)
        elif type(self) is Tree.FuncBody:
            checkOther(i, parser, self, block, c)
        else:
            checkOther(i, parser, function, block, c)
    if type(self) in [Tree.Continue, Tree.Break] and not type(block) in [Tree.While, Tree.For]:
        statement  = "continue" if type(self) is Tree.Continue else "break"
        self.error(f"unexpected {statement}, outside of a loop")
    elif type(self) is Tree.Return:
        if not function:
            self.error(f"unexpected return statement, outside of a function")
        try:
            if len(self.nodes) > 0:
                actReturnType = self.nodes[0].type
                function.returnType.duckType(parser,actReturnType, self, self ,0)
                Tree.insertCast(self.nodes[0], actReturnType, function.returnType, 0)
            else:
                if not function.returnType.isType(Types.Null):
                    self.error("Expecting return value, as function returns " + str(function.returnType))
        except EOFError as e:
            Error.beforeError(e, "Return Type: ")
    else:
        self.validate(parser)
Esempio n. 6
0
def elseExpr(parser):
    toplevel = Tree.Else(parser)

    try:
        inside = parser.currentNode.nodes[-1].nodes[-2]
    except IndexError:
        Error.parseError(parser, "unexpected else")

    if not type(inside) is Tree.IfCondition:
        Error.parseError(parser, "unexpected else")

    parser.currentNode.nodes[-1].addNode(toplevel)
    parser.currentNode = toplevel

    block = Tree.Block(parser)
    parser.currentNode.owner.addNode(block)
    parser.currentNode = block

    opening = None
    single = 0

    while not Parser.isEnd(parser):
        token = parser.nextToken()

        Parser.callToken(parser)

    ExprParser.endExpr(parser)
    parser.currentNode = toplevel.owner.owner
Esempio n. 7
0
def createParser(parser,
                 name="",
                 typ=None,
                 check=True,
                 imutable=True,
                 attachTyp=False):  # : creation
    if name == "":
        name = parser.lookBehind()

    if name.type != "identifier":
        Error.parseError(
            parser, "variable name must be of type identifier, not " +
            parser.lookBehind().type)

    name = name.token

    if name[0].lower() != name[0]:
        Error.parseError(parser, "variable name must be lower case")

    node = Tree.Create(name, Types.Null(), parser)
    node.package = parser.package
    node.imutable = imutable

    if attachTyp:
        node.attachTyp = attachTyp

    parser.currentNode.addNode(node)

    node.varType = typ
    if check and typ is None:
        parser.nextToken()
        typ = Types.parseType(parser)

        node.varType = typ
Esempio n. 8
0
def index(parser, unary=False):

    if not unary:
        unary = ExprParser.isUnary(parser, parser.lookBehind())


    if not unary and len(parser.currentNode.nodes) == 0:
        Error.parseError(parser, "unexpected .")

    field = parser.nextToken()

    if not field.type in ["identifier", "i32"]:
        Error.parseError(parser, "field name must be an identifier")

    acess = Tree.Field(0, Types.Null(), parser)
    acess.unary = unary
    acess.number = field.type == "i32"

    field = field.token

    acess.owner = parser.currentNode
    acess.field = field

    if not unary:
        acess.addNode(parser.currentNode.nodes[-1])
        parser.currentNode.nodes[-1] = acess
    else:
        parser.currentNode.addNode(acess)
Esempio n. 9
0
def initModule(moduleName):
    try:
        transforms[moduleName].init()
    except Exception as e:
        Error.error("Error " + str(e) +
                    " happened when intializing syntax extension " +
                    moduleName)
Esempio n. 10
0
def generics(parser, fname):
    generic = coll.OrderedDict()

    while parser.thisToken().token != "]":
        name = parser.nextToken().token

        typ = Types.T(name, Types.All, parser.package+"."+fname)
        if parser.thisToken().type != "identifier":
            Error.parseError(parser, "type name must be an identifier")

        if not parser.nextToken().token in [":", ",", "]"]:
            Error.parseError(parser, "expecting ,")

        if parser.thisToken().token == ":":
            parser.nextToken()
            typ = Types.T(name, Types.parseType(parser), parser.package+"."+fname)

            if parser.lookInfront().token != "]":
                parser.nextToken()

        Scope.addVar(Tree.PlaceHolder(parser), parser, name, Scope.Type(False, typ))
        generic[name] = typ

        if parser.lookInfront().token == "]":
            parser.nextToken()
            break

    parser.nextToken()
    return generic
Esempio n. 11
0
def forExpr(parser):
    toplevel = Tree.For(parser)

    parser.currentNode.addNode(toplevel)
    parser.currentNode = toplevel

    while not (Parser.isEnd(parser) or parser.thisToken().token == "do"):
        token = parser.nextToken()
        if token.token == "do":
            break
        Parser.callToken(parser)

    ExprParser.endExpr(parser)

    parser.nodeBookmark.append(len(parser.currentNode.nodes))

    if parser.thisToken().token != "do":
        Error.parseError(parser, "Expecting do")

    if len(toplevel.nodes) != 1: #or not type(toplevel.nodes[0]) is Tree.CreateAssign:
        Error.parseError(parser, "Expecting single node")


    count = 0
    while not (Parser.isEnd(parser) and count > 0): #avoid breaking on do keyword
        count += 1
        token = parser.nextToken()
        Parser.callToken(parser)

    parser.nodeBookmark.pop()

    parser.currentNode = toplevel.owner
Esempio n. 12
0
def genericT(parser):
    parser.nextToken()
    if len(parser.currentNode.nodes) > 0:
        func = parser.currentNode.nodes.pop()
    else:
        Error.parseError(parser, "unexpected ::")

    generic = Tree.Generic(parser)
    parser.currentNode.addNode(generic)
    generic.addNode(func)

    generic.generic = []

    if parser.thisToken().token != "[":
        Error.parseError(parser, "expecting [")

    parser.nextToken()

    while parser.thisToken().token != "]":
        if parser.thisToken().token == ",":
            parser.nextToken()
            parser.nodeBookmark[-1] = len(parser.currentNode.nodes)
            continue

        generic.generic.append(Types.parseType(parser))
        t = parser.thisToken().token
        parser.nextToken()
Esempio n. 13
0
def aliasParser(parser, name, decl, generic):
    parser.nextToken()

    typ = False
    while not Parser.isEnd(parser):
        if parser.thisToken().token != "\n" and parser.thisToken().type != "indent":
            if typ:
                Error.parseError(parser, "Unexpected token " + parser.thisToken().token)
            typ = Types.parseType(parser)
        parser.nextToken()

    if decl:
        alias = parser.interfaces[parser.package][name]

        tmp = Types.Alias(parser.package, name, typ, generic)
        alias.name = tmp.name
        alias.normalName = tmp.normalName
        alias.typ = tmp.typ
        alias.generic = tmp.generic
        alias.remainingGen = tmp.remainingGen
        alias.types = tmp.types
        alias.methods = tmp.methods
    if decl:
        parser.interfaces[parser.package][name] = alias

    Scope.decrScope(parser)
Esempio n. 14
0
def genericT(parser):
    parser.nextToken()
    if len(parser.currentNode.nodes) > 0:
        func = parser.currentNode.nodes.pop()
    else:
        Error.parseError(parser, "unexpected ::")

    generic = Tree.Generic(parser)
    parser.currentNode.addNode(generic)
    generic.addNode(func)

    generic.generic = []

    if parser.thisToken().token != "[":
        Error.parseError(parser, "expecting [")

    parser.nextToken()

    while parser.thisToken().token != "]":
        if parser.thisToken().token == ",":
            parser.nextToken()
            parser.nodeBookmark[-1] = len(parser.currentNode.nodes)
            continue

        generic.generic.append(Types.parseType(parser))
        t = parser.thisToken().token
        parser.nextToken()
Esempio n. 15
0
def createParser(parser, name= "", typ= None, check= True, imutable= True, attachTyp= False): # : creation
    if name == "":
        name = parser.lookBehind()

    if name.type != "identifier":
        Error.parseError(parser, "variable name must be of type identifier, not "+parser.lookBehind().type)

    name = name.token

    if name[0].lower() != name[0]:
        Error.parseError(parser, "variable name must be lower case")

    node = Tree.Create(name, Types.Null(), parser)
    node.package = parser.package
    node.imutable = imutable

    if attachTyp:
        node.attachTyp = attachTyp

    parser.currentNode.addNode(node)

    node.varType = typ
    if check and typ is None:
        parser.nextToken()
        typ = Types.parseType(parser)

        node.varType = typ
Esempio n. 16
0
def _packDec(parser, pack= False):
    name = parser.nextToken()
    if name.type != "identifier":
        Error.parseError(parser, "package name must be an identifier")
    name = name.token

    packDec(parser, name, pack)
Esempio n. 17
0
def elseExpr(parser):
    toplevel = Tree.Else(parser)

    try:
        inside = parser.currentNode.nodes[-1].nodes[-2]
    except IndexError:
        Error.parseError(parser, "unexpected else")

    if not type(inside) is Tree.IfCondition:
        Error.parseError(parser, "unexpected else")

    parser.currentNode.nodes[-1].addNode(toplevel)
    parser.currentNode = toplevel

    block = Tree.Block(parser)
    parser.currentNode.owner.addNode(block)
    parser.currentNode = block

    opening = None
    single = 0

    while not Parser.isEnd(parser):
        token = parser.nextToken()

        Parser.callToken(parser)

    ExprParser.endExpr(parser)
    parser.currentNode = toplevel.owner.owner
Esempio n. 18
0
def condition_not_met(file, tags):
    f = open(file, "r")
    jsonLoads = json.loads(f.read())

    for key in jsonLoads:
        if not key in tags: Error.error(file + ", unknown tag " + key)
        if jsonLoads[key] != tags[key]:
            return True
    return False
Esempio n. 19
0
    def addMethod(self, parser, name, method):
        package = parser.package

        if package in self.methods:
            if name in parser.structs[package][self.name].methods[package]:
                Error.parseError(parser, "method "+self.name+"."+name+" already exists")
            self.methods[package][name] = method
        else:
            self.methods[package] = {name: method}
Esempio n. 20
0
def enumParser(parser, name, decl, generic):
    const = parser.interfaces[parser.package][name].const
    existing_generics = parser.interfaces[parser.package][name].generic
    existing_generics.update(generic)

    #const = coll.OrderedDict()
    enum = Types.Enum(parser.package, name, const, existing_generics)

    if decl:
        parser.interfaces[parser.package][name] = enum
    """if parser.lookInfront().token == "\n":
        parser.nextToken()
        Parser.callToken(parser)
        parser.nextToken()"" \
    """

    while not Parser.isEnd(parser):
        t = parser.nextToken()

        if t.token == "\n" or t.type == "indent":
            Parser.callToken(parser)
            continue

        if t.type != "identifier":
            Error.parseError(parser, "expecting identifier")

        varName = t.token

        if varName[0].upper() != varName[0]:
            Error.parseError(parser, "constructor type must be capitalized")

        args = []
        nextT = parser.nextToken()
        #print(varName)
        #print(nextT)
        if nextT.token == "(":
            args = Types.parseType(parser).list

        const[varName] = args

        if decl:
            Scope.addVar(Tree.PlaceHolder(parser),
                         parser,
                         varName,
                         Scope.Type(
                             True,
                             Types.FuncPointer(args, enum, generic=generic)
                             if len(args) > 0 else enum),
                         _global=True)

        t = parser.thisToken()
        if t.token == "\n" or t.type == "indent":
            Parser.callToken(parser)

    parser.currentNode.addNode(Tree.Enum(const, name, parser, generic))

    Scope.decrScope(parser)
Esempio n. 21
0
    def f(parser):
        op = Tree.Operator(kind, parser)
        if kind != "|>" and len(parser.currentNode.nodes) != 0:
            if not unary and isUnary(parser, parser.lookBehind()):
                Error.parseError(parser, "unexpected "+kind)
            elif unary and not isUnary(parser, parser.lookBehind()):
                Error.parseError(parser, "unexpected "+kind)

        #parser.nodeBookmark.append(len(parser.currentNode.nodes)-1)
        Parser.Opcode(parser, kind, lambda: operatorPop(parser, op, takesIn, unary))
Esempio n. 22
0
    def compile(self, opt= 0):
        js = self.toJS()

        try:

            f = open("lib/"+self.filename.replace("/", ".") + ".js", mode="w")
            f.write(js)
            f.close()
        except:
            Error.error("Compilation failed")
Esempio n. 23
0
    def getName(name):
        if name.type != "identifier":
            Error.parseError(parser, "variable name must be of type identifier, not " + parser.lookBehind().type)

        name = name.token

        if name[0].lower() != name[0]:
            Error.parseError(parser, "variable name must be lower case")

        return name
Esempio n. 24
0
def importModule(path):
    moduleName = os.path.basename(os.path.splitext(path)[0])
    spec = importlib.util.spec_from_file_location(moduleName, path)
    if not spec:
        Error.error("Cannot load transform from path " + path)

    foo = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(foo)

    transforms[moduleName] = foo
Esempio n. 25
0
def loadRuntimeTypeData():
    try:
        f = open(runtimeData, "rb")
        if os.stat(runtimeData).st_size == 0:
            Error.error("Runtime type data is empty, please recompile runtime")

        res = pickle.load(f)
        return res
    except FileNotFoundError:
        Error.error("Could not locate runtime")
Esempio n. 26
0
def asOperator(parser):
    lastToken = parser.lookBehind()

    if not isUnary(parser, lastToken):
        op = Tree.Operator("as", parser)
        parser.nextToken()
        op.type = Types.parseType(parser)
        Parser.Opcode(parser, "as", lambda: operatorPop(parser, op, 1, unary=True))
    else:
        Error.parseError(parser, "unexpected as operator ")
Esempio n. 27
0
def castParser(parser):
    if len(parser.currentNode.nodes) == 0:
        Error.parseError(parser, "Unexpected cast")

    parser.nextToken()

    node = Tree.CastToType(parser)
    node.type = Types.parseType(parser)
    node.addNode(parser.currentNode.nodes[-1])
    parser.currentNode.nodes[-1] = node
    node.owner = parser.currentNode
Esempio n. 28
0
def elifExpr(parser):
    try:
        inside = parser.currentNode.nodes[-1].nodes[-2]
    except IndexError:
        Error.parseError(parser, "unexpected elif")

    if not type(inside) is Tree.IfCondition:
        Error.parseError(parser, "unexpected elif")
    parser.currentNode = parser.currentNode.nodes[-1]

    ifBody(parser)
Esempio n. 29
0
def elifExpr(parser):
    try:
        inside = parser.currentNode.nodes[-1].nodes[-2]
    except IndexError:
        Error.parseError(parser, "unexpected elif")

    if not type(inside) is Tree.IfCondition:
        Error.parseError(parser, "unexpected elif")
    parser.currentNode = parser.currentNode.nodes[-1]

    ifBody(parser)
Esempio n. 30
0
    def f(parser):
        op = Tree.Operator(kind, parser)
        actuallyUnary = isUnary(parser, parser.lookBehind())

        if len(parser.currentNode.nodes) != 0:
            if not unary and actuallyUnary:
                Error.parseError(parser, "unexpected "+kind)
            elif unary and not actuallyUnary:
                Error.parseError(parser, "unexpected "+kind)

        #parser.nodeBookmark.append(len(parser.currentNode.nodes)-1)
        Parser.Opcode(parser, kind, lambda: operatorPop(parser, op, takesIn, actuallyUnary))
Esempio n. 31
0
def elseExpr(parser, canHaveElse=False):
    toplevel = Tree.Else(parser)
    ifexpr = False

    if not canHaveElse:
        try:
            inside = parser.currentNode.nodes[-1].nodes[-2]
        except IndexError:
            Error.parseError(parser, "unexpected else")

        if not type(inside) is Tree.IfCondition:
            ifexpr = IfExpr.ifPatternMatch(parser)

            if not ifexpr:
                Error.parseError(parser, "unexpected else")

    if not ifexpr:
        parser.currentNode.nodes[-1].addNode(toplevel)
        parser.currentNode = toplevel

        block = Tree.Block(parser)
        parser.currentNode.owner.addNode(block)
        parser.currentNode = block
    else:
        parser.currentNode = parser.currentNode.nodes[-1].nodes[-1]
        while len(parser.currentNode.nodes) > 0:
            parser.currentNode = parser.currentNode.nodes[-1]

        add_block = len(parser.currentNode.nodes) > 0
        if add_block:
            block = Tree.Block(parser)

            parser.currentNode.nodes[-1].addNode(toplevel)
            parser.currentNode.nodes[-1].addNode(block)
            parser.currentNode = parser.currentNode.nodes[-1]

            parser.currentNode = block

    opening = None
    single = 0

    while not Parser.isEnd(parser):
        token = parser.nextToken()

        Parser.callToken(parser)

    ExprParser.endExpr(parser)

    if ifexpr:
        parser.currentNode = ifexpr
    else:
        parser.currentNode = toplevel.owner.owner
Esempio n. 32
0
def newPack(name):
    if name[0].lower() != name[0]:
        Error.error("package name must be lowercase")
    try:
        os.mkdir("src/" + name)
        f = open("src/"+name+"/port.json", mode= "w")
        f.write("""
{
    "files": []
}
        """)
    except:
        Error.error("directory has no source folder")
Esempio n. 33
0
def newPack(name):
    if name[0].lower() != name[0]:
        Error.error("package name must be lowercase")
    try:
        os.mkdir("src/" + name)
        f = open("src/" + name + "/port.json", mode="w")
        f.write("""
{
    "files": []
}
        """)
    except:
        Error.error("directory has no src folder")
Esempio n. 34
0
def getCompilationFiles(target):
    try:
        proj = open("src/port.json", mode="r")
        proj.close()
    except:
        Error.error("missing port.json in source folder")

    def getCompFiles(dir=""):
        file = {}

        for root, dirs, files in os.walk(dir, topdown=False):
            package = root
            if package == "src/": continue
            package = package[len("src/"):]

            file[package] = {"client": [], "full": [], "node": []}

            try:
                port = open("src/" + package + "/port.json", mode="r")
            except:
                Error.error("missing file port.json in package " + package +
                            "")

            files = []
            try:
                j = json.loads(port.read())
                files.append((target, j["files"]))
            except KeyError:
                pass
            except json.decoder.JSONDecodeError as e:
                Error.error("In file port.json, in directory " + package +
                            ", " + str(e))

            if "client-files" in j and (target in ["full", "client"]):
                files.append(("client", j["client-files"]))
            if "node-files" in j and (target in ["full", "node"]):
                files.append(("node", j["node-files"]))

            if len(files) == 0:
                Error.error("no compilation files are specified in package " +
                            package + "/port.json")

            for f in files:
                for name in f[1]:
                    file[package][f[0]].append((root, name + ".top"))

            if root[0].lower() != root[0]:
                Error.error("package name must be lowercase")
        return file

    return getCompFiles("src/")
Esempio n. 35
0
def traitParser(parser, name, decl, generic):
    interface = parser.interfaces[parser.package][name]
    meth = {}

    while not Parser.isEnd(parser):
        parser.nextToken()

        t = parser.thisToken()

        if t.token == "def":
            currentNode = parser.currentNode
            p = PlaceHolder(parser)
            parser.currentNode = p

            (methodName, names, types, brace, returnType,
             do) = FuncParser.funcHead(parser,
                                       decl,
                                       dontAdd=True,
                                       interfaceMethod=True)
            Scope.decrScope(parser)
            if methodName in meth:
                Error.parseError(parser,
                                 "Method " + methodName + ", already defined")
            meth[methodName] = brace.ftype
            parser.currentNode = currentNode
            parser.nextToken()
        else:
            Parser.declareOnly(parser, noVar=True)
            if len(parser.currentNode.nodes) > 0 and type(
                    parser.currentNode.nodes[0]) is Tree.Create:
                Error.parseError(
                    parser,
                    "Interfaces are abstract interfaces which is why only methods are supported"
                )

    names = {i.name: i.varType for i in parser.currentNode}
    #args = [i.varType for i in parser.currentNode]
    #fields = parser.currentNode.nodes

    if decl:
        i = interface.fromObj(
            Types.Interface(False,
                            names,
                            generic,
                            parser.package + "." +
                            name if parser.package != "_global" else name,
                            methods=meth))
        parser.interfaces[parser.package][name] = i

    Scope.decrScope(parser)
Esempio n. 36
0
    def f(parser):
        op = Tree.Operator(kind, parser)

        if kind == "|>":
            print(kind)
        if not kind in ["|>"] and len(parser.currentNode.nodes) != 0:
            if not unary and isUnary(parser, parser.lookBehind()):
                Error.parseError(parser, "unexpected " + kind)
            elif unary and not isUnary(parser, parser.lookBehind()):
                Error.parseError(parser, "unexpected " + kind)

        #parser.nodeBookmark.append(len(parser.currentNode.nodes)-1)
        Parser.Opcode(parser, kind,
                      lambda: operatorPop(parser, op, takesIn, unary))
Esempio n. 37
0
    def hasMethod(self, parser, name):
        packages = []
        b = None
        for i in parser.imports+[parser.package]+["_global"]:
            if not i in self.methods: continue
            if name in self.methods[i]:
                b = self.methods[i][name]
                b.package = i
                packages.append(i)

        if len(packages) > 1:
            Error.parseError(parser, "ambiguous, multiple definitions of the method "+self.name+"."+name+" in packages: "+", ".join(packages[:-1])+" and "+packages[-1])

        return b
Esempio n. 38
0
def externVar(parser):
    target = parser.nextToken()

    if target.type != "identifier":
        Error.parseError(parser, "expecting target")

    target = target.token

    if not target in ["client", "full", "node"]:
        Error.parseError(parser, target + " is not a valid compilation target")

    VarParser.createAndAssignParser(parser)
    parser.currentNode.nodes[-1].extern = True
    parser.currentNode.nodes[-1].global_target = target
    parser.currentNode.nodes[-1].nodes[1].extern = True
Esempio n. 39
0
def importParser(parser, decl= False):
    import os
    name = parser.nextToken()
    if name.type != "str":
        Error.parseError(parser, "expecting string")

    oname = name.token[1:-1]

    if not oname in parser.filenames:
        Error.parseError(parser, "package "+oname+" not found")

    name = os.path.basename(oname)

    if not decl:
        parser.externFuncs[parser.package] = []

        if not parser.hotswap:
            sp = shouldParse(decl, oname, parser)
        else:
            sp = shouldCompile(decl, oname, parser)

        if sp:
            p = Parser.Parser(parser.lexed[oname], parser.filenames[oname])

            ResolveSymbols.insert(parser, p)

            sc = shouldCompile(decl, oname, parser)

            p.sc = sc

            parser.compiled[name] = None
            parser.externFuncs[name] = []

            parsed = p.parse()

            declar = parser.externFuncs[name]

            parser.compiled[name] = (sc, (parsed, declar))

            ResolveSymbols.insert(p, parser)

            parser.currentNode.addNode(Tree.InitPack(name, parser))
        else:
            if not name in parser.compiled:
                parser.compiled[name] = (False,)
                parser.currentNode.addNode(Tree.InitPack(name, parser))

    parser.imports.append(oname)
Esempio n. 40
0
    def append(self, value):
        if value is None:
            raise Error.error(
                "expecting type string and got none, internal error")

        if self.target in [self.global_target, "full"]:
            if self.global_target == "full":
                if self.inAFunction:
                    self.client_out_parts.append(value)
                    self.node_out_parts.append(value)
                else:
                    self.client_main_parts.append(value)
                    self.node_main_parts.append(value)
            else:
                if self.inAFunction:
                    self.out_parts.append(value)
                else:
                    self.main_parts.append(value)
        elif self.target == "client":
            if self.inAFunction:
                self.client_out_parts.append(value)
            else:
                self.client_main_parts.append(value)
        elif self.target == "node":
            if self.inAFunction:
                self.node_out_parts.append(value)
            else:
                self.node_main_parts.append(value)
        elif self.target == "full":
            if self.inAFunction:
                self.out_parts.append(value)
            else:
                self.main_parts.append(value)
Esempio n. 41
0
def linkWith(name):
    try:
        file = open("src/port.json", mode="r+")
    except:
        Error.error("missing port.json file in project")

    port = json.loads(file.read())

    if name.endswith(".css"):
        port["linkCSS"].append(name)
    else:
        port["link"].append(name)

        file.write(json.dumps(port))

    file.close()
Esempio n. 42
0
def linkWith(name):
    try:
        file = open("src/port.json", mode="r+")
    except:
        Error.error("missing port.json file in project")

    port = json.loads(file.read())

    file.write("""
{
    "name": \"""" + port["name"] + """\",
    "version": """+str(port["version"]) + """,
    "link": """+str(port["link"]+[name]) + """,
}
    """)

    file.close()
Esempio n. 43
0
def index(parser):
    if len(parser.currentNode.nodes) == 0:
        Error.parseError(parser, "unexpected .")

    field = parser.nextToken()

    if field.type != "identifier":
        Error.parseError(parser, "field name must be an identifer")
    field = field.token

    acess = Tree.Field(0, Types.Null(), parser)

    acess.addNode(parser.currentNode.nodes[-1])
    acess.owner = parser.currentNode

    acess.field = field

    parser.currentNode.nodes[-1] = acess
Esempio n. 44
0
def createAndAssignParser(parser, imutable= True): # let i assignment
    node = parser.currentNode

    parser.nextToken() #get current token to position of =

    checkIt = False
    attachTyp = False

    if parser.lookInfront().token == ".":

        attachTyp = Types.parseType(parser, attachTyp= True)
        parser.nextToken()
        if not imutable or not type(node) is Tree.Root:
            Error.parseError(parser, "expecting =, not .")
        parser.nextToken()


    name = parser.thisToken()

    typ = None

    if parser.nextToken().token == ":":
        checkIt = True

        parser.nextToken()
        typ = Types.parseType(parser)

        parser.nextToken()
    elif parser.thisToken().token != "=":
        Error.parseError(parser, "expecting =, not"+parser.thisToken().token)

    n = Tree.CreateAssign(parser)

    parser.currentNode.addNode(n)
    parser.currentNode = n

    createParser(parser, name= name, typ= typ, check= checkIt, imutable= imutable, attachTyp= attachTyp)

    if attachTyp:
        assignParser(parser, name=attachTyp.name+"_"+name.token, package= attachTyp.package, init=True)
    else:
        assignParser(parser, name= name.token, init= True)

    parser.currentNode = node
Esempio n. 45
0
def importParser(parser, decl= False):
    import os
    name = parser.nextToken()
    if name.type != "str":
        Error.parseError(parser, "expecting string")

    oname = name.token[1:-1]

    if not oname in parser.filenames:
        Error.parseError(parser, "package "+oname+" not found")

    name = os.path.basename(oname)

    if not decl:
        parser.externFuncs[parser.package] = []

        if shouldParse(decl, oname, parser):
            p = Parser.Parser(parser.lexed[oname], parser.filenames[oname])

            ResolveSymbols.insert(parser, p)

            sc = shouldCompile(decl, oname, parser)

            parser.compiled[name] = None
            parser.externFuncs[name] = []

            if sc:
                parsed = p.parse()
            else:
                parsed = None

            declar = parser.externFuncs[name]

            parser.compiled[name] = (sc, (parsed, declar))

            ResolveSymbols.insert(p, parser)

            parser.currentNode.addNode(Tree.InitPack(name, parser))
        else:
            if not name in parser.compiled:
                parser.compiled[name] = None

    parser.imports.append(oname)
Esempio n. 46
0
def newProj(name):
    if name[0].upper() != name[0]:
        Error.error("project name must be uppercase")
    try:
        os.mkdir(name)
        os.mkdir(name + "/" + "src")

        file = open(name + "/src/port.json", mode="w")
        file.write("""
{
    "name": \"""" + name + """\",
    "version": 0.0,
    "link": []
}
        """)
        file.close()

        os.mkdir(name + "/" + "lib")
        os.mkdir(name + "/" + "bin")
    except:
        Error.error("project already created")
Esempio n. 47
0
    def getCompFiles(dir= ""):
        file = {}

        for root, dirs, files in os.walk(dir, topdown=False):
            package = root
            if package == "src/": continue
            package = package[len("src/"):]

            file[package] = []

            try:
                port = open("src/"+package+"/port.json", mode= "r")
            except:
                Error.error("missing file port.json in package "+package+"")

            try:
                j = json.loads(port.read())
                files = j["files"]
            except KeyError:
                Error.error("missing property files in file "+package+"port.json")

            for name in files:
                file[package].append((root, name+".top"))

            if root[0].lower() != root[0]:
                Error.error("package name must be lowercase")
        return file
Esempio n. 48
0
def typeParser(parser, decl= False):
    name = parser.nextToken()

    Scope.incrScope(parser)


    if name.type != "identifier":
        Error.parseError(parser, "type name must be an identifier")
    name = name.token

    if name[0].lower() == name[0]:
        Error.parseError(parser, "struct name must be upper case")

    import collections as coll
    gen = coll.OrderedDict()

    if parser.nextToken().token == "[":
        gen = FuncParser.generics(parser, name)

    if parser.thisToken().token != "=":
        if parser.thisToken().token == "with":
            tmp = parser.currentNode
            parser.currentNode = Tree.PlaceHolder(parser)
            Interface.traitParser(parser, name, decl, gen)
            parser.currentNode = tmp
            return

        Error.parseError(parser, "expecting =")
    tmp = parser.currentNode

    typ = Tree.Type(parser.package, name, parser)
    typ.package = parser.package
    typ.normalName = name

    tmp.addNode(typ)

    parser.currentNode = typ

    while not Parser.isEnd(parser):
        parser.nextToken()
        Parser.declareOnly(parser, noVar=True)

    args = [i.varType for i in parser.currentNode]
    fields = parser.currentNode.nodes

    typ.fields = [i.name for i in typ]

    typ.nodes = []
    parser.currentNode = tmp

    if decl:
        meth = parser.structs[parser.package][name].methods
        parser.structs[parser.package][name] = Struct(name, args, fields, gen)
        parser.structs[parser.package][name].methods = meth
        parser.structs[parser.package][name].package = parser.package

    Scope.decrScope(parser)
Esempio n. 49
0
def initStruct(parser, package= ""):
    if ExprParser.isUnary(parser, parser.lookBehind()):
        Error.parseError(parser, "unexpected {")

    if package == "": package = parser.package

    if len(parser.currentNode.nodes) == 0:
        Error.parseError(parser, "unexpected {")
    if not type(parser.currentNode.nodes[-1]) in [Tree.ReadVar, Tree.Field]:
        Error.parseError(parser, "unexpected {")

    readVar = type(parser.currentNode.nodes[-1]) is Tree.ReadVar

    name = parser.currentNode.nodes[-1].name if readVar else parser.currentNode.nodes[-1].field

    init = Tree.InitStruct(parser)

    if not readVar:
        package = parser.currentNode.nodes[-1].nodes[0].name
        t = (parser.currentNode.nodes[-1].nodes[0])
        if not package in parser.imports:
            t.error("no package called " + package)
        elif not type(t) is Tree.ReadVar:
            init.error("unexpected {")

    init.package = package

    del parser.currentNode.nodes[-1]
    s = parser.structs[package][name]

    init.paramNames = offsetsToList(parser.structs[package][name].offsets)
    init.s = s
    init.mutable = False

    parser.currentNode.addNode(init)
    parser.currentNode = init

    parser.nextToken()

    while parser.thisToken().token != "}":
        if parser.thisToken().token == ",":
            ExprParser.endExpr(parser)
        else: Parser.callToken(parser)
        parser.nextToken()
        t = parser.thisToken().token
        continue

    ExprParser.endExpr(parser)

    parser.currentNode = init.owner
Esempio n. 50
0
def parserMethodGen(parser, gen, struct):
    sgen = struct.generic

    if len(gen) > len(sgen):
        Error.parseError(parser, str(len(gen)-len(sgen))+" generic arguments too many")
    elif len(gen) < len(sgen):
        Error.parseError(parser, str(len(gen) - len(sgen)) + " generic arguments too few")

    newGen = coll.OrderedDict()
    for a, b in zip(gen, sgen):
        if gen[a].type != Types.All: Error.parseError(parser, "unexpected :")
        newGen[a] = sgen[b]
        Scope.changeType(parser, a, sgen[b])

    return newGen
Esempio n. 51
0
def checkIfOperator(parser, attachTyp, name, func):
    operators = {
        "add": Types.FuncPointer([attachTyp, attachTyp], attachTyp),
        "sub": Types.FuncPointer([attachTyp, attachTyp], attachTyp),
        "mul": Types.FuncPointer([attachTyp, attachTyp], attachTyp),
        "div": Types.FuncPointer([attachTyp, attachTyp], attachTyp),
        "eq": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()),
        "ne": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()),
        "mod": Types.FuncPointer([attachTyp, attachTyp], attachTyp),
        "pow": Types.FuncPointer([attachTyp, attachTyp], attachTyp),
        "gt": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()),
        "lt": Types.FuncPointer([attachTyp, attachTyp], Types.Bool()),
    }

    unary = {
        "add": Types.FuncPointer([attachTyp], attachTyp),
        "sub": Types.FuncPointer([attachTyp], attachTyp),
        "mul": Types.FuncPointer([attachTyp], attachTyp)
    }

    if name.startswith("operator_"):
        op = name[len("operator_"):]
        if not op in operators:
            Error.parseError(parser, "overload not found for operator_"+op)

        f = Types.FuncPointer(func.args, func.returnType)

        if f != operators[op]:
            Error.parseError(parser, "expecting function declaration "+str(operators[op])+", not "+str(f))
    elif name.startswith("unary_"):
        op = name[len("unary_"):]
        if not op in unary:
            Error.parseError(parser, "overload not found for unary_"+op)

        f = Types.FuncPointer(func.args, func.returnType)

        if f != unary[op]:
            Error.parseError(parser, "expecting function declaration "+str(unary[op])+", not "+str(f))
Esempio n. 52
0
    parser.currentNode.addNode(n)
    parser.currentNode.addNode(cond)
    parser.currentNode = cond

    while not Parser.isEnd(parser):
        token = parser.nextToken()

        iter = parser.iter
        if token.token == "do" :
            ExprParser.endExpr(parser)

            block = Tree.WhileBlock(parser)
            cond.owner.addNode(block)
            parser.currentNode = block

            continue

        Parser.callToken(parser)

    ExprParser.endExpr(parser)

    parser.currentNode = n.owner.owner

Parser.exprToken["while"] = whileExpr

Parser.exprToken["break"] = lambda parser: parser.currentNode.addNode(Tree.Break(parser))
Parser.exprToken["continue"] = lambda parser: parser.currentNode.addNode(Tree.Continue(parser))

Parser.exprToken["then"] = lambda parser: Error.parseError(parser, "unexpected then keyword")
Parser.exprToken["do"] = lambda parser: Error.parseError(parser, "unexpected do keyword")
Esempio n. 53
0
def start(run= False, dev= False, init= False):
    try:
        opt = 0
        skip = 0

        outputFile = ""

        for (iter, i) in enumerate(sys.argv[2:]):
            if skip > 0:
                continue

            skip -= 1
            if i == "-O3":
                opt = 3
            elif i == "-O2":
                opt = 2
            elif i == "-o":
                outputFile = sys.argv[iter + 3]
                skip = 1
            elif i == "-O1":
                opt = 1
            else:
                Error.error("unknown argument '" + i + "'.")

        files = getCompilationFiles()
        allfilenames = []
        allsources = []

        sources = {}
        filenames = {}

        for c in files:
            sources[c] = []
            filenames[c] = []
            for i in files[c]:
                try:
                    file = open(os.path.join(i[0], i[1]), mode="r")
                    r = file.read()
                    allsources.append(r)
                    sources[c].append(r)

                    if i[1][0].upper() == i[1][0]:
                        Error.error("file name must be lowercase")

                    filenames[c].append((c, i[1][:-4]))
                    allfilenames.append((c, i[1][:-4]))

                    file.close()
                except FileNotFoundError:
                    Error.error("file " + i[1] +", not found")

        if outputFile == "":
            port = open("src/port.json")

            data = port.read()
            outputFile = (json.loads(data)["name"])
            port.close()

        if filenames == []:
            Error.error("no input files")

        """
        import cProfile

        profile = cProfile.Profile()
        profile.enable()
        """

        time1 = time()

        # print ("============= Compiling ==============\n")

        """
        for i in lexed:
            print(i.token+"_"+i.type)
        """

        lexed = Lexer.lex(sources, filenames)
        #print("lexed")

        declarations = Parser.Parser(lexed, filenames)
        declarations.files = files
        declarations.lexed = lexed
        declarations.filenames = filenames
        declarations.opt = opt
        declarations.compiled = {}
        declarations.externFuncs = {"main": []}

        ResolveSymbols.resolve(declarations)

        #print("declarations")

        if ImportParser.shouldCompile(False, "main", declarations):
            parser = Parser.Parser(lexed["main"], filenames["main"])
            ResolveSymbols.insert(declarations, parser, only= True)

            parser.files = files
            parser.lexed = lexed
            parser.filenames = filenames
            parser.compiled = declarations.compiled
            parser.compiled["main"] = None

            parsed = parser.parse()

            parser.compiled["main"] = (True, (parsed, parser.externFuncs["main"]))

            import AST as Tree
            allCode = Tree.Root()

            if opt > 0:
                for d in parser.compiled:
                    allCode.addNode(parser.compiled[d][1][0])
                optimize(allCode, opt)

            #print("parsing")

            for i in parser.compiled:
                if parser.compiled[i][0]:
                    CodeGen.CodeGen(i, parser.compiled[i][1][0], parser.compiled[i][1][1]).compile(opt=opt)

            l = CodeGen.link(parser.compiled, outputFile, run=run, opt= opt, dev= dev)
            print("Compilation took : "+str(time() - time1))
            return (True, l)
        elif run:
            CodeGen.exec(outputFile)
        elif init:
            return (True, open("bin/"+outputFile+".js").read())
        elif dev:
            return (False, "")

        print("Compilation took : "+str(time() - time1))
    except EOFError as e:
        if dev:
            return (False, str(error))
        else:
            print(e, file= sys.stderr)
Esempio n. 54
0
    generic = Tree.Generic(parser)
    parser.currentNode.addNode(generic)
    generic.addNode(func)

    generic.generic = []

    if parser.thisToken().token != "[":
        Error.parseError(parser, "expecting [")

    parser.nextToken()

    while parser.thisToken().token != "]":
        if parser.thisToken().token == ",":
            parser.nextToken()
            parser.nodeBookmark[-1] = len(parser.currentNode.nodes)
            continue

        generic.generic.append(Types.parseType(parser))
        t = parser.thisToken().token
        parser.nextToken()

def under(parser):
    parser.currentNode.addNode(Tree.Under(parser))

Parser.stmts["def"] = func
Parser.exprToken["none"] = lambda parser: Error.parseError(parser, "unexpected type none")
Parser.exprToken[","] = lambda  parser: Error.parseError(parser, "unexpected ,")
Parser.exprToken["_"] = under
Parser.exprToken["::"] = genericT
Parser.exprToken["!"] = lambda parser: 0
Esempio n. 55
0
def funcHead(parser, decl= False, dontAdd= False, method= False, attachTyp = False):
    Scope.incrScope(parser)

    if parser.tokens[parser.iter+2].token == ".":
        if attachTyp: Error.parseError(parser, "unexpected .")
        parser.nextToken()
        name = parser.thisToken().token
        parser.nextToken()

        try:
            attachTyp = Types.Struct(False, name, parser.structs[parser.package][name].types, parser.package, {})
        except KeyError:
            Error.parseError(parser, "no attachable data structure found, called "+name)
        return funcHead(parser, decl, dontAdd, True, attachTyp)
    name = parser.nextToken()

    if name.type != "identifier":
        Error.parseError(parser, "function name must be of type identifier, not "+name.type)
    parser.nextToken()

    name = name.token

    g = {}
    if parser.thisToken().token != "(":
        if parser.thisToken().token == "[":
            g = generics(parser, (str(attachTyp)+"." if method else "")+name)
            if parser.thisToken().token == ".":
                if attachTyp: Error.parseError(parser, "unexpected .")
                if not Scope.varExists(parser, parser.package, name): Error.parseError(parser,
                     "cannot attach method to unknown type main."+name)

                try:
                    attachTyp = Types.Struct(False, name, parser.structs[parser.package][name].types, parser.package,
                                         parserMethodGen(parser, g, parser.structs[parser.package][name]))
                except KeyError:
                    Error.parseError(parser, "no attachable data structure found, called " + name)

                return funcHead(parser, decl, dontAdd, True, attachTyp)


        if parser.thisToken().token != "(":
            Error.parseError(parser, "expecting (")

    header = Tree.FuncStart(name, Types.Null(), parser)
    header.package = parser.package
    parser.currentNode.addNode(header)

    brace = Tree.FuncBraceOpen(parser)
    brace.name = name
    brace.package = parser.package

    parser.currentNode.addNode(brace)

    parser.currentNode = brace

    if method:
        if not type(parser.currentNode.owner) is Tree.Root and not decl:
            Error.parseError(parser, "method extension must be in out-most scope")

        typ = attachTyp
        self = parser.nextToken()
        if self.type != "identifier": Error.parseError(parser, "binding name must be identifier not "+self.type)
        self = self.token

        selfNode = Tree.Create(self, typ, parser)
        selfNode.package = parser.package
        selfNode.imutable = True

        parser.currentNode.addNode(selfNode)

        if not parser.lookInfront().token in [")", ","]:
            Error.parseError(parser, "expecting comma not "+parser.thisToken().token)


    if name[0].lower() != name[0]:
        Error.parseError(parser, "function name must be lower case")

    returnType = Types.Null()

    parser.paren += 1
    parser.nextToken()

    while parser.paren != parser.parenBookmark[-1] :
        b = parser.thisToken().token
        if b == ",":
            parser.nextToken()
            continue
        elif b == ")":
            parser.paren -= 1
            parser.nextToken()
            continue
        elif b == "(":
            Error.parseError(parser, "unexpected (")
        Parser.declareOnly(parser)
        parser.nextToken()

    t = parser.thisToken()
    do = False

    if t.token != "=" and t.token != "do":
        returnType = Types.parseType(parser)

        t = parser.nextToken()
        if t.token != "=" and t.token != "do":
            Error.parseError(parser, "expecting = or do")

    if t.token == "do":
        do = True

    parser.currentNode = brace.owner

    names = [i.name for i in brace.nodes]
    types = [i.varType for i in brace.nodes]

    func = Types.FuncPointer(
        types,
        returnType,
        g,
        do
    )

    if method:
        Scope.decrScope(parser)

        header.method = True
        header.types = types[1:]
        header.attachTyp = attachTyp
        header.normalName = name
        header.name = attachTyp.normalName+"_"+header.normalName

        MethodParser.checkIfOperator(parser, attachTyp, name, func)

        if decl:
            MethodParser.addMethod(brace, parser, attachTyp, name, func)

        return attachTyp.normalName+"_"+name, names, types, header, returnType, do

    parser.func[parser.package][name] = func

    header.ftype = Types.FuncPointer(types, returnType, g)
    if decl:
        if not dontAdd:
            Scope.addFunc(header, parser, name, Types.FuncPointer(types, returnType, g, do))

    return name, names, types, header, returnType, do
Esempio n. 56
0
from TopCompiler import topc
from TopCompiler import Error
import sys

if __name__ == "__main__":
    if len(sys.argv) > 1:

        if sys.argv[1] == "new":
            if sys.argv[2] == "project":
                topc.newProj(sys.argv[3])
            elif sys.argv[2] == "package":
                topc.newPack(sys.argv[3])
            elif sys.argv[2] == "linkWith":
                topc.linkWith(sys.argv[3])
            else:
                Error.error("invalid option to new"+sys.argv[2])
            sys.exit()
        elif sys.argv[1] == "build":
            topc.start()
        elif sys.argv[1] == "run":
            topc.start(True)
        else:
            Error.error("invalid option "+sys.argv[1])
Esempio n. 57
0
    createParser(parser, name= name, typ= typ, check= checkIt, imutable= imutable, attachTyp= attachTyp)

    if attachTyp:
        assignParser(parser, name=attachTyp.name+"_"+name.token, package= attachTyp.package, init=True)
    else:
        assignParser(parser, name= name.token, init= True)

    parser.currentNode = node

Parser.stmts["let"] = createAndAssignParser
Parser.stmts["var"] = lambda parser: createAndAssignParser(parser, imutable= False)
Parser.stmts["="] = assignParser
Parser.stmts[":"] = createParser

Parser.exprToken["i32"] = lambda parser: Error.parseError(parser, "unexpected type int")
Parser.exprToken["|"] = lambda parser: Error.parseError(parser, "unexpected function declaration")
Parser.exprToken["int"] = lambda parser: Error.parseError(parser, "unexpected type int")
Parser.exprToken["float"] = lambda parser: Error.parseError(parser, "unexpected type float")
Parser.exprToken["bool"] = lambda parser: Error.parseError(parser, "unexpected type bool")

def read(parser, name, package= ""):
    if package == "": package = parser.package

    node = Tree.ReadVar(name,  False, parser)
    node.package = package

    parser.currentNode.addNode(node)

def equalAnd(parser, operator, package= ""):
    if package == "": package = parser.package
Esempio n. 58
0
 def error(self, message):
     Error.errorAst(message, self.selfpackage, self.filename, self.token)
Esempio n. 59
0
def tokenize(s, filename, spos= 0, sline= 0, slinePos= 0):
    keywords = [
        'import',
        'def',
        'then', 'do', 'if', 'elif', 'else', 'while',
        'int', 'float', 'none', 'bool', 'string',
        'break', 'continue',
        'true', 'false',
        'let',
        'ext',
        'type',
        'string',
        'var',
        "not", "or", "and",
    ]

    special = ["bang", "arrow", "doublecolon", "line", "underscore", "assign", "assignPlus", "assignSub", "assignMul", "assignDiv", 'colon', 'dot', 'openC', 'openB', 'closeC', 'closeB', 'comma', 'closeS', 'openS', 'doubleDot', 'semi']

    token_specification = [
        ("comment", r"/\*([\s\S]*?)\*/"),
        ("indent", r'\n[ ]*'),
        ('commentLine', r'//.*'),
        ('newline', r'\n'),
        ('openB', '{'),
        ('closeB', '}'),
        ('openC', '\('),
        ('closeC', '\)'),
        ('f32', r'[\d_]+(\.[\d_]+|f)'),
        ('i32', r'[\d_]+'),
        ('arrow', r'->'),
        ('equal',  r'=='),
        ('doublecolon', r'::'),
        ("colon", r":"),
        ("semi", r";"),
        ('ne', r'!='),
        ('assign',  r'='),
        ('openS', r'\['),
        ('closeS', r'\]'),
        ('assignPlus', r'\+='),
        ('assignSub', r'\-='),
        ('assignMul', r'\*='),
        ('assignDiv', r'\/='),
        ('operator',  r'[+*\/\-%><^]|(\|>)'),
        ('line', r'\|'),
        ('identifier', r'[A-Za-z0-9_]+'),
        ('underscore', '_'),
        ('skip', r'[ \t]'),
        ("str", r'"(?:\\.|({.*})|[^"\\])*"'),
        ('doubleDot', '\.\.'),
        ('dot', '\.'),
        ('tab', '\t'),
        ('comma', ','),
        ('bang', '!'),
    ]
    tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification)
    get_token = re.compile(tok_regex).match
    line = 1

    pos = spos
    mo = get_token(s)
    lastIndent = 0
    lastTyp = None
    linePos = slinePos

    line = sline

    array = []

    while mo is not None:
        typ = mo.lastgroup

        next = get_token(s, mo.end())

        if typ == "indent" or typ == "newline":
            val = mo.group(typ)
            array.append(Token("\n", "symbol", line, pos))
            line += 1

            linePos = mo.end()

            if next == None:
                array.append(Token(0,"indent", line, pos))
                break
            if next.lastgroup == "indent":
                array.append(Token(lastIndent, "indent", line, pos ))
            else:
                array.append(Token(len(val)-1, "indent", line, pos))
                lastIndent = len(val)-1
        elif typ == "comment":
            val = mo.group(typ)
            c = mo.end()
            r = val.rfind("\n")
            linePos = c + r
            line += len(val) - len(val.replace("\n", ""))
            array.append(Token(val, "comment", line, pos ))
        elif typ in ["str"]:
            val = mo.group(typ)
            if typ == "str":
                def notBack(iter):
                    if iter == 0: return True
                    if val[iter-1] != "\\": return True
                    return not notBack(iter-1)

                start = 0
                inBrace = False
                tokens = []
                val = val[1:-1]
                bcount = 0
                shouldBe = 0
                v = list(val)
                for iter in range(len(val)):
                    i = val[iter]
                    if notBack(iter) and i == "{" and not inBrace:
                        tokens.append(Token('"'+val[start: iter]+'"', "str", line, pos))
                        inBrace = True
                        start = iter+1
                        shouldBe = bcount

                    if i == "{":
                        bcount += 1
                    elif notBack(iter) and i == "}":
                        bcount -= 1
                        if bcount == shouldBe and inBrace:
                            tokens.append(Token("concat", "operator", line, pos+start))
                            tokens.append(Token("(", "symbol", line, pos+start))

                            tokens += tokenize(val[start: iter], filename, pos+start, line, linePos)
                            tokens.append(Token(")", "symbol", line, pos+iter))
                            tokens.append(Token("concat", "operator", line, pos+iter))
                            start = iter + 1
                            inBrace = False

                tokens.append(Token('"'+val[start:]+'"', "str", line, pos))
                array += tokens

        elif typ != 'skip':
            val = mo.group(typ)
            if typ == 'identifier' and val in keywords:
                if val in ["true", "false"]:
                    typ = "bool"
                elif val in ["_"]:
                    typ = "symbol"
                else:
                    typ = "keyword"
            elif typ == "f32":
                val = val[:-1]+".0" if val[-1] == "f" else val
            if typ == "i32" or typ == "f32":
                val = val.replace("_", "")
            elif typ in special:
                typ = "symbol"
            elif typ == "equal" or typ == "mut" or typ == "ne":
                typ = "operator"

            if val != " ": array.append(Token(val, typ, line, pos))
            elif val == "\t":
                Error.compileError(filename[1], line, "tabs are not allowed")

        lastTyp = typ

        pos = mo.end() - linePos
        mo = next

    if spos == 0 and sline == 0:
        array.append(Token("\n", "symbol", line-1, pos))
        array.append(Token(0, "indent", line, pos))

    return array