def parseClass(generator):
"""
Initial parse function. Each file holds a class, thus the `class` is the
main unit of compilation
"""
uniqueIfAndWhileIds.init()
backEnd.doesFunctionReturn.stackVarsInit()
token = next(generator)
if token.value == 'class':
tagOutput.startt('class')
tagOutput.outt(token)
token = next(generator)
if parseClassName(token) is False:
raise CompilerError(PErrorMsg.noClassName(token))
backEnd.setGlobals(currentClass=token)
parseLeftCurly(next(generator))
token = next(generator)
isCurly = parseRightCurly(token)
if isCurly is False:
# Here, we're parsing field & static vars + keeping a count
varTable.resetFieldStaticCount()
varTable.setInDeclaration(True)
moreClassVarDecd = parseClassVarDec(token, generator)
while moreClassVarDecd is True:
moreClassVarDecd = parseClassVarDec(next(generator), generator)
# Now we know the n fields that Class defines for its object type:
functionsInfo.setFieldN()
# Time to parse subroutines:
moreSubroutineDeclared = parseSubroutineDec(
moreClassVarDecd, generator)
while moreSubroutineDeclared is True:
moreSubroutineDeclared = parseSubroutineDec(None, generator)
rightCurly = parseRightCurly(moreSubroutineDeclared)
if rightCurly is False:
raise CompilerError(PErrorMsg.expectedRightCurly(token))
varTable.addToAvailableTypes()
tagOutput.endt('class')
try:
next(generator)
except:
pass
else:
raise CompilerError(PErrorMsg.expectedEndOfFile(token))
else:
raise CompilerError(PErrorMsg.noClassDeclr(token))
def parseWhileStatement(token, generator):
if token.value == 'while':
n = uniqueIfAndWhileIds.getWhileID()
backEnd.processCode.WhileStatement_1(n)
tagOutput.startt('whileStatement')
tagOutput.outt(token)
parseLeftParen(next(generator))
token = parseExpression(next(generator), generator)
parseRightParen(token)
backEnd.processCode.WhileStatement_2(n)
parseLeftCurly(next(generator))
token = parseStatements(next(generator), generator)
endcurly = parseRightCurly(token)
backEnd.processCode.WhileStatement_3(n)
if endcurly is False:
raise CompilerError(PErrorMsg.expectedEndOfStatement(token))
tagOutput.endt('whileStatement')
return True
else:
return False
def parseVarDec(token, generator):
# Parses varDec, returns next token, and adds variables to variable table
tagOutput.startt('varDec')
tagOutput.outt(token)
token = next(generator)
type_ = parseType(token)
varTable.checkTypeExistence(token)
token = next(generator)
parseVarName(token)
varTable.addVariable(token, type_, kind='var', scope='function')
token = next(generator)
comma = parseComma(token)
# kind=`var' means local variable. We're tallying the number of locals
while comma is True:
token = next(generator)
parseVarName(token)
varTable.addVariable(token, type_, kind='var', scope='function')
token = next(generator)
comma = parseComma(token)
end = parseSemicolon(token)
if end is False:
raise CompilerError(PErrorMsg.expectedSemicolon(token))
else:
token = next(generator)
tagOutput.endt('varDec')
return token
def SubroutineDeclaration(token):
if parseNum == 2:
currentFnContext = functionsInfo.getCurrentFnContext()
shouldReturnType = currentFnContext.returnsType
num = currentFnContext.nVars
# The function declaration, itself:
output.code('function %s.%s %s' % (currentClass, currentFn, num))
# If we're dealing with a constructor, we've got to allocate memory,
# and put the reference to our new object in the right memory location
if currentFnType == 'constructor':
if currentClass == shouldReturnType:
memToAlloc = functionsInfo.getFieldN()
output.code('push constant ' + str(memToAlloc))
output.code('call Memory.alloc 1')
# ^ Leaves the address of our allocated object @ top of stack
output.code('pop pointer 0')
# ^ Puts &NEW_OBJECT in pointer 0 (`@THIS', in Hack Assembly)
else:
raise CompilerError('A constructor\'s return type must be the '
'class type, Line %s, %s' %
(token.line, globalVars.inputFileName))
# If we're dealing with a method, we have to pop pointer to the object
# upon which our method operates
elif currentFnType == 'method':
# `argument 0` is the object called by the method. This pops it
# into pointer 0, so that `this n` VM commands work.
output.code('push argument 0')
output.code('pop pointer 0')
def lookupFn(self, tokenOrString):
"""
Returns information about the function specified by tokenOrString --
`function()`, in the case of a token; `Class.function()`, in the case
of a string.
Returns a 'functInfo' (named) tuple :: (kParams, nVars, fnType,
returnsType)
This is a utility function, and is used to evaluate semantics, tell us
about our current context (based on our position in the parse), and
give us info for code output.
The instance in which we do this where it /might/ not have a value is
`SubroutineCall_WithDot_B\', and it also crashes our exception handler.
That's good, because we want the exception handled /there/, and the
proper error message output.
"""
try:
if type(tokenOrString) is not str:
# `function()` -- either a function or internal method call
function = tokenOrString.value
return self.table[currentClass + '^' + function]
else:
# `Class.function()` or `object.function()`
return self.table[tokenOrString.replace('.', '^')]
except:
# tokenOrString won't have the attribute .line if tokenOrString was
# type str. That's okay. Our caller has an an exception handler for
# `AttributeError`s
raise CompilerError('`%s`: Function does not exist. Line %s, %s' %
(tokenOrString.value, tokenOrString.line,
globalVars.inputFileName))
def parseTerm(token, generator):
tagOutput.startt('term')
typ = token.typ
if typ in ['integerConstant', 'stringConstant'
] or parseKeywordConstant(token) is True:
# integerConstant | stringConstant | keywordConstant
if typ == 'integerConstant':
backEnd.processCode.TermINTEGER(token)
elif typ == 'stringConstant':
backEnd.processCode.TermSTRING(token)
else: #== KeywordConstant
backEnd.processCode.TermKEYWORD(token)
tagOutput.outt(token)
token = next(generator)
else:
lookahead = next(generator)
if lookahead.value == ('['):
# varName `[` expression `]`
parseVarName(token)
variableToken = token
tagOutput.outt(lookahead)
token = parseExpression(next(generator), generator)
backEnd.processCode.TermARRAY(variableToken)
if token.value != ']':
raise CompilerError(PErrorMsg.expectedClosingSquare(token))
else:
tagOutput.outt(token)
token = next(generator)
elif token.typ == 'identifier' and lookahead.value in ('(', '.'):
# subroutineCall
token = parseSubroutineCall(token,
lookahead,
generator,
callerExpectsReturnVal=True)
else:
if token.value in ('~', '-'):
# unaryOp term
op = token
tagOutput.outt(token)
token = parseTerm(lookahead, generator)
backEnd.processCode.TermUNARYOP(op)
else:
try:
# varName
parseVarName(token)
variableToken = token
backEnd.processCode.TermVARNAME(variableToken)
token = lookahead
except CompilerError:
# `(` expression `)`
parseLeftParen(token)
token = parseExpression(lookahead, generator)
parseRightParen(token)
token = next(generator)
tagOutput.endt('term')
return token
def parseClassVarDec(token, generator):
"""
Returns True if thing parsed is of the form classVarDec,
Returns token if it plainly isn't,
Raises CompilerError if things are badly-formed.
"""
if token.value in ('static', 'field'):
kind = token.value
tagOutput.startt('classVarDec')
tagOutput.outt(token)
token = next(generator)
type_ = parseType(token)
varTable.checkTypeExistence(token)
if not type_:
raise CompilerError(PErrorMsg.expectedType(token))
token = next(generator)
parseVarName(token)
varTable.addVariable(token, type_, kind, scope='class')
token = next(generator)
comma = parseComma(token)
while comma is True:
token = next(generator)
parseVarName(token)
varTable.addVariable(token, type_, kind, scope='class')
semi = parseSemicolon(token)
if semi is False:
token = next(generator)
comma = parseComma(token)
else:
raise CompilerError(PErrorMsg.expectedVarName(token))
if parseSemicolon(token) is True:
tagOutput.endt('classVarDec')
return True
else:
raise CompilerError(PErrorMsg.expectedSemiOrVariable(token))
else:
return token
def parseLeftCurly(token):
if token.value == '{':
# \/ Updates how far embedded in curly braces we are, for purposes of
# FunctionReturnStack,which determines whether fns are likely to return
backEnd.doesFunctionReturn.stackVarsIncr()
tagOutput.outt(token)
else:
raise CompilerError(PErrorMsg.expectedLeftCurly(token))
def checkTypeExistence(self, token, subroutineDeclaration=False):
"""
This checks whether a declared type actually exists in the files we're
compiling OR the Jack Standard Library
"""
if parseNum == 2:
type_ = token.value
if type_ == 'void':
if not subroutineDeclaration:
raise CompilerError(
'`void` only makes sense as a '
'subroutine return value. Line %s, %s' %
(token.line, globalVars.inputFileName))
elif type_ not in ('int', 'char', 'boolean', 'Array', 'String'):
if type_ not in self.listOfExtendedTypes:
raise CompilerError(
'Unknown type `%s`, line %s, %s' %
(token.value, token.line, globalVars.inputFileName))
def addVariable(self, token, type_, kind, scope):
"""
Adds a variable to either our class-level variable table (classScope)
or our function-level one (functScope). Included is information for
`Type' (int, char, boolean, void, and arbitrary types);
`kind` (field, static, local, argument); and the variable number,
which is used for relative addressing.
"""
if parseNum == 1:
variableName = token.value
classKey = currentClass + '^' + variableName
# Checks whether our local variable has already been declared as
# class-level variables or function parameters. Issues appropriate
# warning, if they have, then adds the variable and typing, scope,
# and position (variable number) to our function scope hash table.
if scope == 'function':
key = currentClass + '^' + currentFn + '^' + variableName
if classKey in self.classScope:
kind1 = self.classScope[classKey].kind
kind2 = kind
line = token.line
if kind2 == 'var': kind2 = 'local'
print('WARNING: Duplicate variable names. %s\'s scope is '
'overridden by %s\'s. Line %s, %s. Variable name: '
'`%s`' % (kind1, kind2, line,
globalVars.inputFileName, variableName),
file=sys.stderr)
if key in self.functScope:
line = token.line
print('WARNING: Local variable declaration overrides '
'argument -- Line %s, %s; variable name, `%s`' %
(line, globalVars.inputFileName, variableName),
file=sys.stderr)
self.functScope[key] = varInfo(type_, kind, self.localVarN)
self.localVarN += 1
else: # == 'class'
key = currentClass + '^' + variableName
if key in self.classScope:
raise CompilerError(
'Duplicate class-level variable name: '
'`%s\'. Line %s, %s' %
(variableName, line, globalVars.inputFileName))
if kind == 'field':
self.classScope[key] = varInfo(type_, kind, self.fieldVarN)
self.fieldVarN += 1
else: # kind == 'static':
self.classScope[key] = varInfo(type_, kind,
self.staticVarN)
self.staticVarN += 1
def parseStatement(token, generator):
if parseLetStatement(token, generator) is False:
boolORtoken = parseIfStatement(token, generator)
# == token, if it was an if statement.
# == False, if the parsed statement wasn't an if statement,
if boolORtoken:
return boolORtoken
else:
if parseWhileStatement(token, generator) is False:
if parseDoStatement(token, generator) is False:
if parseReturnStatement(token, generator) is False:
raise CompilerError(PErrorMsg.badStatement(token))
return None
def SubroutineCall_WithDot_B(subroutineToken, fn, nExprsInCall):
"""Second part of output logic for `class.subroutine()`"""
if parseNum == 2:
try:
expectedParams, *NULL = functionsInfo.lookupFn(fn)
if int(nExprsInCall) != int(expectedParams):
raise CompilerError(
'Function `%s` takes %s argument(s). '
'Number given: %s. Line %s, %s' %
(fn, expectedParams, nExprsInCall, subroutineToken.line,
globalVars.inputFileName))
except AttributeError:
raise CompilerError(
'Call to `%s`: Class/object does not exist or '
'subroutine doesn\'t (or both). Line %s, %s' %
(fn, subroutineToken.line, globalVars.inputFileName))
# if STRONGLINKING == True:
# ...
output.code('call %s %s' % (fn, nExprsInCall))
def checkReturn(token):
"""
Provides some limited error checking for return statements. Doesn't do much
type checking--simply makes sure that constructors, void functions, and
value-returning functions all do what they should.
Jack's very, very lax typing is kept intact. Type checking could be added,
but would require more effort when expressions are involved. Since the only
platform we're compiling to treats everything as a 16-bit word, there's at
least no impedance mismatch
"""
if parseNum == 2:
value = token.value
*NULL, functRole, returnsType = functionsInfo.getCurrentFnContext()
if functRole == 'constructor':
errorMiddle = ''
if value != 'this':
errorMiddle = '. Found: `%s`' % value
raise CompilerError('`%s` is a constructor and should return '
'`this`%s. Line %s, %s' %
(currentFn, errorMiddle,
token.line, globalVars.inputFileName))
elif returnsType == 'void':
if value != ';':
raise CompilerError('`%s` is a void function, and so mustn\`t'
'return a value. It does. Line %s, %s' %
(currentFn, token.line,
globalVars.inputFileName))
else:
if value == ';':
raise CompilerError('`%s` isn`t a void function, and so must '
'return a value. Line %s, %s' %
(currentFn, token.line,
globalVars.inputFileName))
def parseIfStatement(token, generator):
if token.value == 'if':
n = uniqueIfAndWhileIds.getIfID()
backEnd.doesFunctionReturn.stackAddIfStmnt()
tagOutput.startt('ifStatement')
tagOutput.outt(token)
parseLeftParen(next(generator))
token = parseExpression(next(generator), generator)
parseRightParen(token)
parseLeftCurly(next(generator))
backEnd.processCode.IfStatement_IF(n)
token = parseStatements(next(generator), generator)
endbracket = parseRightCurly(token)
if endbracket is False:
raise CompilerError(PErrorMsg.expectedRightCurly(token))
token = next(generator)
if token.value == 'else':
tagOutput.outt(token)
backEnd.doesFunctionReturn.stackAddElseStmnt()
parseLeftCurly(next(generator))
backEnd.processCode.IfStatement_ELSE_A(n)
token = parseStatements(next(generator), generator)
backEnd.processCode.IfStatement_ELSE_B(n)
endbracket = parseRightCurly(token)
if endbracket is False:
raise CompilerError(PErrorMsg.expectedRightCurly(token))
token = next(generator)
else:
backEnd.processCode.IfStatement_NOELSE(n)
tagOutput.endt('ifStatement')
return token
return False
def parseSubroutineBody(generator):
tagOutput.startt('subroutineBody')
parseLeftCurly(next(generator))
token = next(generator)
# a subroutine body is just variable declarations and statements
while token.value == 'var':
token = parseVarDec(token, generator)
token = parseStatements(token, generator)
end = parseRightCurly(token)
if end is False:
raise CompilerError(PErrorMsg.expectedRightCurly(token))
tagOutput.endt('subroutineBody')
def addFunction(self, returnsType, token):
if parseNum == 1:
key = currentClass + '^' + currentFn
totalLocalVars = varTable.localVarN
paramsVarsPair = functInfo(self.kParamsDeclrd, totalLocalVars,
currentFnType, returnsType)
if key in self.table:
raise CompilerError(
'Subroutine `%s` has already been declared'
'. Line %s, %s' %
(currentFn, token.line, globalVars.inputFileName))
self.table[key] = paramsVarsPair
def checkClassExistence(self, token):
"""
This checks whether a called class name exists, so that
Class.subroutine() actually calls code that exists
"""
if parseNum == 2:
class_ = token.value
if class_ not in ('Array', 'Keyboard', 'Math', 'Memory', 'Output',
'Screen', 'String', 'Sys'):
if class_ not in self.listOfExtendedTypes:
raise CompilerError(
'Class `%s` doesn\'t seem to exist. '
'Line %s, %s' %
(token.value, token.line, globalVars.inputFileName))
def parseLetStatement(token, generator):
if token.value == 'let':
tagOutput.startt('letStatement')
tagOutput.outt(token)
token = next(generator)
parseVarName(token)
variableToken = token
token = next(generator)
array = False
if token.value == '[':
array = True
tagOutput.outt(token)
token = parseExpression(next(generator), generator)
if token.value != ']':
raise CompilerError(PErrorMsg.badExpression(token))
backEnd.processCode.LetStatement_ARRAY_BASE(variableToken)
tagOutput.outt(token)
token = next(generator)
if token.value != '=':
raise CompilerError(PErrorMsg.expectedEquals(token))
else:
tagOutput.outt(token)
token = parseExpression(next(generator), generator)
backEnd.processCode.LetStatement(array, variableToken)
end = parseSemicolon(token)
if end is False:
raise CompilerError(PErrorMsg.expectedSemicolon(token))
tagOutput.endt('letStatement')
return True
else:
return False
def parseSubroutineCall(token, lookahead, generator, callerExpectsReturnVal):
if lookahead.value == '(':
subroutineToken = token
parseSubroutineName(token)
calledFunctRole = backEnd.returnSemantics.checkDotlessFunctionCall(
subroutineToken, callerExpectsReturnVal)
backEnd.processCode.SubroutineCall_NoDot_A(calledFunctRole)
parseLeftParen(lookahead)
token, numberOfParams = parseExpressionList(generator,
methodCall=False)
backEnd.processCode.SubroutineCall_NoDot_B(subroutineToken,
numberOfParams)
parseRightParen(token)
token = next(generator)
else:
worked = parseClassName(token) # parseVarName(token)
classOrObject = token
if worked is False:
return token
else:
token = lookahead
if token.value == '.':
tagOutput.outt(token)
subroutineToken = next(generator)
parseSubroutineName(subroutineToken)
methodCall, fn = backEnd.processCode.SubroutineCall_WithDot_A(
subroutineToken, classOrObject)
parseLeftParen(next(generator))
token, numberOfExprs = parseExpressionList(
generator, methodCall)
backEnd.processCode.SubroutineCall_WithDot_B(
subroutineToken, fn, numberOfExprs)
parseRightParen(token)
token = next(generator)
else:
raise CompilerError(PErrorMsg.expectedPeriod(token))
return token
def TermKEYWORD(token):
if parseNum == 2:
keyword = token.value
if keyword == 'true':
output.code("push constant 1")
output.code('neg')
elif keyword == 'false':
output.code("push constant 0")
elif keyword == 'null':
output.code("push constant 0")
elif keyword == 'this':
if functionsInfo.getCurrentFnContext().fnType != 'function':
output.code("push pointer 0")
else:
raise CompilerError(
'`this\' cannot be used in a function. Line %s, %s' %
(token.line, globalVars.inputFileName))
def parseParameterList(generator):
"""
Returns a token--either after evaluating `type varName` (',' `type varName`)
or after finding nothing of that form. Returned token should be a '(', else
calling function will grind to a halt
"""
tagOutput.startt('parameterList')
token = next(generator)
type_ = parseType(token)
if type_:
varTable.checkTypeExistence(token)
token = next(generator)
parseVarName(token)
varTable.addVariable(token, type_, kind='argument', scope='function')
token = next(generator)
functionsInfo.incrementKParams()
comma = parseComma(token)
# \/ Tallying the number of argument variables in our fn declarations:
while comma is True:
functionsInfo.incrementKParams()
token = next(generator)
type_ = parseType(token)
varTable.checkTypeExistence(token)
if type_:
token = next(generator)
parseVarName(token)
varTable.addVariable(token,
type_,
kind='argument',
scope='function')
token = next(generator)
comma = parseComma(token)
else:
raise CompilerError(PErrorMsg.expectedAddlArguments(token))
tagOutput.endt('parameterList')
return token
def parseSubroutineDec(token, generator):
if token is None: token = next(generator)
backEnd.setGlobals(currentFnType=token)
if token.value in ('constructor', 'function', 'method'):
backEnd.doesFunctionReturn.stackInit()
# \/ Sets an initial value for `k' parameters
functionsInfo.initKParams()
tagOutput.startt('subroutineDec')
tagOutput.outt(token)
token = next(generator)
if parseType(token):
varTable.checkTypeExistence(token, subroutineDeclaration=True)
returnTyp = token.value
else:
raise CompilerError(PErrorMsg.expectedVoidOrType(token))
subroutineToken = next(generator)
parseSubroutineName(subroutineToken)
backEnd.setGlobals(currentFn=subroutineToken)
backEnd.processCode.SubroutineDeclaration(token)
parseLeftParen(next(generator))
varTable.resetVarCounter() # Resets `localVarN' declared
token = parseParameterList(generator) # Figures out our `k' parameters
parseRightParen(token)
parseSubroutineBody(generator)
backEnd.doesFunctionReturn.codeCheck(subroutineToken)
# Checks whether the code in the this subroutine will actually return
functionsInfo.addFunction(returnTyp, subroutineToken)
# @ this point, variableTable.localVarN contains the total n of local
# variables declared in the current function
tagOutput.endt('subroutineDec')
return True
else:
return token
def SubroutineCall_NoDot_B(subroutineToken, numberOfParams):
"""
Second part of our logic/error-checking for method-less/Class-less
subroutine calls -- i.e., in the language of Jack, "function" call
"""
if parseNum == 2:
k, NULL, proceduretype, NULL = functionsInfo.lookupFn(subroutineToken)
if proceduretype == 'method': numberOfParams += 1
if numberOfParams != k:
raise CompilerError(
'Function `%s\' takes %s arguments. Number '
'given: %s. Line %s, %s' %
(subroutineToken.value, k, numberOfParams,
subroutineToken.line, globalVars.inputFileName))
else:
output.code('call %s.%s %s' %
(currentClass, subroutineToken.value, numberOfParams))
def fileOrPathParser(path):
"""
Returns a list containing either the single .jack file pointed to or the
.jack files in the specified directory
"""
import os, glob
try:
if path.endswith('.jack'):
files = [path]
else:
files = [] # enter'd nonsense
for inFile in glob.glob(os.path.join(path, '*.jack')):
files.append(inFile)
if files == []:
raise
except RuntimeError:
raise CompilerError("Badly formed file or path name, %s doesn't exist "
"or it doesn't point to .jack files" % path) \
from None
return files
def checkDotlessFunctionCall(subroutineToken, callerExpectsReturnval):
"""
This checks whether dotless function calls (i.e. ones of the form
`do function()` or `let thing = function()` return value when they should
(and don\'t when they should not) AND makes sure we don't issue meaningless
method calls.
"""
if parseNum != 2:
return
*NULL, calledFunct, calledReturns = functionsInfo.lookupFn(subroutineToken)
*NULL, callingfunct, NULL = functionsInfo.getCurrentFnContext()
if calledFunct == 'method' and callingfunct == 'function':
raise CompilerError('Argument-less method calls cannot be made from '
'within functions. Method calls must be of the '
'form `OBJECT.method(), or else we have no object '
'to work on`. Line %s, %s' %
(subroutineToken.line, globalVars.inputFileName))
if callerExpectsReturnval is True:
if calledReturns == 'void':
print('WARNING: Call to `%s` expects return value, but subroutine '
'is of type void. Line %s, %s' %
(subroutineToken.value, subroutineToken.line,
globalVars.inputFileName), file=sys.stderr)
else:
if calledReturns != 'void':
print('WARNING: Function `%s` is value-returning, but that value '
'is ignored. Line %s, %s' %
(subroutineToken.value, subroutineToken.line,
globalVars.inputFileName), file=sys.stderr)
return calledFunct
def lookupVariable(self, variableToken):
kind = i = None
if parseNum == 2:
variableName = variableToken.value
try:
key = currentClass + '^' + currentFn + '^' + variableName
if key in self.functScope:
base = self.functScope[key]
else:
key = currentClass + '^' + variableName
base = self.classScope[key]
kind = base.kind
if kind == 'field': kind = 'this'
elif kind == 'var': kind = 'local'
i = base.i
type_ = base.type_
except:
raise CompilerError('Variable `%s` not found. Line %s, %s' %
(variableName, variableToken.line,
globalVars.inputFileName))
return type_, kind, i
def lexer(filename):
"""
Modified version of the tokenizer in Python documentation
You can think of this as a data structure that yields tagged Jack tokens.
"""
import re
keywords = {'class', 'constructor', 'function', 'method', 'field',
'static', 'var', 'int', 'char', 'boolean', 'void', 'true',
'false', 'null', 'this', 'let', 'do', 'if', 'else', 'while',
'return'}
token_specification = [
('NEWLINE', r'\n'), # Line endings
('SKIP', r'[ \t]'), # Spaces and tabs(skip)
('COMMENT', r'//.*\n'), # Single-line comment
('WHOLECMNT', r'/\*.*\*/'), # SINGLELINESTARCOMMENT
('STRTCMNT', r'/\*'), # MULTILINECOMMENTstart
('ENDCMNT', r'.*\*/'), # MULTILINECOMMENTend
('BAD_ID', r'\d+[A-Za-z]+'), # 1dentifier
('integerConstant', r'\d+'), # Integer
('identifier', r'[A-Za-z_][\dA-Za-z_]*'), # Identifiers
('symbol', r'[{}\()\[\]\.,;\+\-\*/&\|<>=~]'), # Symbols
('stringConstant', r'"[^\n]*?"'), # String
('GARBAGE', r'.+?') # Everything else
]
tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification)
get_token = re.compile(tok_regex).match
ln = 1
opencomment = False
with open(filename) as inputvariable:
# `with` method assures file closes, even if exception is raised
for s in inputvariable:
pos = line_start = 0
mo = get_token(s)
while mo is not None:
# the loop continues until it's reg-exed the entire line
typ = mo.lastgroup
if typ == 'NEWLINE' or typ == 'COMMENT':
# increment line counter whether or not it falls in a comment
ln += 1
elif typ == 'ENDCMNT':
if opencomment == False:
# ENDCMNT tokens shouldn't happen unless 'opencomment' is True
raise CompilerError(TErrorMsg.cmnt_end() % (ln, filename))
opencomment = False
elif opencomment == False:
# tokenizing when we're not 'in' a comment:
if typ == 'STRTCMNT':
opencomment = True
startcommentline = ln
elif typ == 'GARBAGE':
# (s[pos], ln, filename))
raise CompilerError(TErrorMsg.bad_token() % \
(s[pos], ln, filename))
elif typ == 'BAD_ID':
err = mo.group(typ)
raise CompilerError(TErrorMsg.bad_id() % (err,ln,filename))
elif typ != 'SKIP' and typ not in ('ENDCMNT', 'WHOLECMNT'):
val = mo.group(typ)
if typ == 'identifier' and val in keywords:
typ = 'keyword'
elif typ == 'INTEGER':
if 0 > int(val) or 32767 < int(val):
raise CompilerError(TErrorMsg.int_overflow() % \
ln, filename)
elif typ == 'stringConstant':
val = val[1:-1]
yield Token(typ, val, ln)
pos = mo.end() # end() returns end char of current str
mo = get_token(s, pos)
inputvariable.close()
if opencomment == True:
# triggered if EOF is reached but a comment is still open
raise CompilerError(TErrorMsg.open_cmnt() % (startcommentline, filename))
def parseRightParen(token):
if token.value == ')':
tagOutput.outt(token)
else:
raise CompilerError(PErrorMsg.expectedRightParen(token))
def parseVarName(token):
if 'identifier' == token.typ:
tagOutput.outt(token)
else:
raise CompilerError(PErrorMsg.badIdentifier(token))
def parseSubroutineName(token):
if token.typ == 'identifier':
tagOutput.outt(token)
else:
raise CompilerError(PErrorMsg.badIdentifier(token))