/
tokeniser.py
86 lines (76 loc) · 2.08 KB
/
tokeniser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from utils import readFile
tokenTypes = {
"": False,
"$": "CLASS_SYMBOL",
":": "CONNECTION",
"#": "INSTANCE_OF",
"*": "DEFINITION",
"[": "OPEN_BRACE",
"]": "CLOSE_BRACE",
";": "SEMICOLON",
"hi": "HIGH_CONSTANT",
"lo": "LOW_CONSTANT",
"im": "IMPEDANCE_CONSTANT",
"plu": "PULL_UP_RESISTOR",
"pld": "PULL_DOWN_RESISTOR",
"tri": "TRI_STATE_GATE",
"not": "NOT_GATE",
"and": "AND_GATE",
"or": "OR_GATE",
"xor": "XOR_GATE",
"nand": "NAND_GATE",
"nor": "NOR_GATE",
"xnor": "XNOR_GATE"
}
class Token:
def __init__(self, val, tkType, pos):
self.value = val
self.type = tkType
self.pos = pos
self.parent = None
if self.type == "IDENT" and len(self.value.split(".")) > 1:
self.parent = self.value.split(".")[:-1]
self.value = self.value.split(".")[-1]
def __repr__(self):
return f"Token({{value: '{self.value}', type: '{self.type}', pos: {self.pos}}}"
class TokenStream:
def __init__(self, tokens):
self.index = 0
self.tokens = tokens
def next(self):
item = self.tokens[self.index]
self.index += 1
return item
def peek(self):
return self.tokens[self.index]
def identifyToken(token):
tk = token.lower()
try:
return tokenTypes[tk]
except KeyError:
return "IDENT"
def tokenise(source):
program = readFile(source)
token = ""
tokens = []
posInSrc = [0, 1] # 0th char of 1st line
for char in program:
posInSrc[0] += 1
if char == "\n":
posInSrc[1] += 1 # increment line by one
posInSrc[0] = 1
if char.strip() == "" or char in list(tokenTypes.keys()):
tkType = identifyToken(token)
if tkType != False: tokens.append(Token(token, tkType, posInSrc.copy()))
token = ""
tkType = identifyToken(char.strip())
if tkType != False: tokens.append(Token(char, tkType, posInSrc.copy()))
else: token += char
return tokens
def nextToken(tokenStream):
try:
return tokenStream.next()
except IndexError:
with open(SOURCE_FILE, "r") as src:
lines = src.readlines()
throwErr(SOURCE_FILE, [len(lines[-1]), len(lines)], "Unexpected EOF", 2)