def __processFile__(self, filePath):
''' processes a single file, first feeding the file to JackTokenizer to generate a list of tokens
(output as T.xml files for debugging use) and that token list is fed through
CompilationEngine to generate a final result list of XML tokens which is output into an .xml file. '''
#Phase 1 Tokenize/Analyze
tokenizer = JackTokenizer(filePath)
print(filePath)
xmlTokenList = ["<tokens>"]
taggedTokenList = [("listStart", "tokens", xmlTokenList[0])]
token = tokenizer.advance()
while token:
taggedToken = self.__wrapTokenInXML__(token)
taggedTokenList += [taggedToken]
xmlTokenList += [taggedToken[TT_XML]]
token = tokenizer.advance()
xmlTokenList += ["</tokens>"]
length = len(xmlTokenList)
taggedTokenList += [("listEnd", "tokens", xmlTokenList[length - 1])]
Tfilename = str(filePath.parent) + '/' + filePath.stem + "T.xml"
self.__output__(Tfilename, xmlTokenList)
#Phase 2 Compile/Translate
compiler = CompilationEngine(taggedTokenList)
compiledXMLList = compiler.compileTokens()
Cfilename = str(filePath.parent) + '/' + filePath.stem + ".xml"
self.__output__(Cfilename, compiledXMLList)
def __processFile__(self, filePath):
''' processes a single file, first feeding the file to JackTokenizer to generate a list of tokens
(output as T.xml files for debugging use) and that token list is fed through
CompilationEngine to generate a final result list of XML tokens which is output into an .xml file. '''
#TODO make it work
# create opening token tag for tokenizing lines of .jack
tokens = ["<tokens>"]
tokenizer = JackTokenizer(filePath)
line = tokenizer.advance()
# tokenize each line of .jack
while line:
tokens += [self.__wrapTokenInXML__(line)]
line = tokenizer.advance()
tokens += ["</tokens>"]
# 2. create a list for compiled tokens to go into, create compEngine instance
# compile the tokens
compiledTokens = []
compEngine = CompilationEngine(tokens)
compiledTokens += compEngine.compileTokens()
# create the filepath names for writing the tokens and full blown xml
xml_T_FilePath = Path(filePath.parent / (filePath.stem + 'T.xml'))
finalTokenPath = Path(filePath.parent / (filePath.stem + '.xml'))
# write out the raw tokens
self.__output__(xml_T_FilePath, tokens)
self.__output__(finalTokenPath, compiledTokens)
def compileParameterList(rfile, wfile):
wfile.write('<parameterList>\n')
token = JackTokenizer.advance(rfile)
while token != ')':
JackTokenizer.writeToken(wfile, token)
token = JackTokenizer.advance(rfile)
wfile.write('</parameterList>\n')
def compileDo(rfile, wfile):
wfile.write('<doStatement>\n')
wfile.write('<keyword> do </keyword>\n')
subroutinCall(rfile, wfile)
token = JackTokenizer.advance(rfile)
JackTokenizer.writeToken(wfile, token)
wfile.write('</doStatement>\n')
def compile_do_statement(tokenized):
global index
translated = jt.write_keyword(tokenized[index]) # do
index += 1
translated += compile_subroutine_call(tokenized) + jt.write_symbol(
tokenized[index])
index += 1
return translated
def analyze(filename):
"""
Given file/directory, analyze it
:param filename:
"""
tokenizer = JackTokenizer(filename)
tokenizer.tokenize()
engine = CompilationEngine(tokenizer)
engine.write(filename.replace('.jack', '.vm'))
def __init__(self, in_file, out_file):
"""
A compilation engine constructor
:param in_file: the file we are currently compiling
:param out_file: the file where we save the output
"""
self.tokenizer = JackTokenizer(in_file)
self.out_file = open(out_file, 'w')
self._indent_count = 0
def compileExpression(rfile, wfile):
wfile.write('<expression>\n')
compileTerm(rfile, wfile)
token = JackTokenizer.advance(rfile)
while not (token in (')', ']', ';', ',')):
JackTokenizer.writeToken(wfile, token)
compileTerm(rfile, wfile)
token = JackTokenizer.advance(rfile)
wfile.write('</expression>\n')
def __init__(self, input_file, output_file):
self.tokenizer = JackTokenizer(input_file)
self.xml_file = open(output_file, "w")
self.space_depth = 0
# starts the process
self.tokenizer.advance()
self.compile_class()
self.xml_file.close()
def __init__(self, inputFile, outputFile):
# self.outFile = outputFile
self.tokenizer = JackTokenizer(inputFile)
self.vmwriter = VMWriter(outputFile)
self.outFile = outputFile
self.currToken = ""
self.tabber = ""
self.argumentsCounter = 0
def main():
global fn
userInput = sys.argv[1]
if ".jack" in userInput:
filename = userInput
if "/" not in filename:
pos = filename.find(".jack")
fn = filename[: pos]
#newpath = fn
#if not os.path.isdir(newpath):
# os.makedirs(newpath)
outfilenameT = filename[:filename.index(".")] + "T.xml"
outfilename = filename[:filename.index(".")] + ".vm"
else:
last_pos = filename.rfind("/")
pos = filename.find(".jack")
fn = filename[last_pos + 1: pos]
savepath = filename[:last_pos]
#newpath = savepath + "/" + fn
#if not os.path.isdir(newpath):
# os.makedirs(newpath)
outfilenameT = filename[last_pos + 1:filename.rfind(".")] + "T.xml"
outfilename = filename[last_pos + 1:filename.rfind(".")] + ".vm"
ofT = open(outfilenameT, "w")
xmlOp = JackTokenizer.tokenizer(filename)
ofT.write(xmlOp)
ofT.close()
xmlOp = CompilationEngine.compilationEngine(outfilename, outfilenameT)
compilationEngine = CompilationEngine.compilationEngine(outfilenameT, outfilename)
else:
os.chdir(userInput)
if userInput.endswith("/"):
last_pos = userInput.rfind("/")
second_last_pos = userInput[: last_pos].rfind("/")
#newpath = userInput[second_last_pos + 1: last_pos]
else:
if "/" in userInput:
last_pos = userInput.rfind("/")
else:
last_pos = 0
#newpath = userInput[last_pos :]
#if not os.path.isdir(newpath):
# os.makedirs(newpath)
for filename in os.listdir(os.getcwd()):
if filename.endswith(".jack"):
pos = filename.find(".jack")
fn = filename[: pos]
outfilenameT = fn + "T.xml"
outfilename = fn + ".vm"
ofT = open(outfilenameT, "w")
xmlOp = JackTokenizer.tokenizer(filename)
ofT.write(xmlOp)
ofT.close()
print(outfilenameT)
compilationEngine = CompilationEngine.compilationEngine(outfilenameT, outfilename)
def compile_return_statement(tokenized):
global index
translated = jt.write_keyword(tokenized[index]) # return
index += 1
if tokenized[index] != ";": # eg return expression
translated += "<expression>\n" + compile_expression(
tokenized) + "</expression>\n"
translated += jt.write_symbol(tokenized[index]) # ;
index += 1
return translated
def compile_else(tokenized):
global index
translated = jt.write_keyword(tokenized[index]) # else
index += 1
translated += jt.write_symbol(tokenized[index]) # {
index += 1
statements = ""
while tokenized[index] != "}":
statements += compile_statement(tokenized)
return translated + "<statements>\n" + statements + "</statements>\n"
def translate_token(tokenized):
global index
translated = jt.write_keyword(tokenized[index]) # class
index += 1
translated += jt.write_identifier(tokenized[index]) # name
index += 1
translated += jt.write_symbol(tokenized[index]) # "{"
index += 1
return "<class>\n" + translated + inside_class(
tokenized) + jt.write_symbol(tokenized[index]) + "</class>"
def compile_params_list(tokenized):
global index
translated = check_type(tokenized[index])
index += 1
translated += jt.write_identifier(tokenized[index])
index += 1
if tokenized[index] == ",":
translated += jt.write_symbol(tokenized[index])
index += 1
return translated + compile_params_list(tokenized)
return translated
def compile_many_vars_dec(tokenized):
global index
translated = jt.write_symbol(tokenized[index]) # ","
index += 1
translated += jt.write_identifier(tokenized[index]) # var
index += 1
while tokenized[index] != ";":
translated += jt.write_symbol(tokenized[index]) # ","
index += 1
translated += jt.write_identifier(tokenized[index]) # var
index += 1
return translated
def __init__(self, file):
""" """
self.label_num = 0
self.tokenizer = JackTokenizer(file)
self.advance()
self.symbols = SymbolTable()
self.vm = VMWriter()
self.open_outfile(file)
self.compile_class()
self.close_outfile()
def __init__(self, input_path, output_path):
"""
creates a new compilation engine with the given input and output. the next routine called must be compileClass()
:param input_path: input stream/file
:param output_path: output stream/file
"""
self.labels = 0
self.jack_class = None
self.class_subroutine = None
self.tokenizer = JackTokenizer(input_path)
self._writer = VMWriter(output_path)
self.CompileClass()
def __init__(self, in_file, out_file):
"""
A compilation engine constructor
:param in_file: the file we are currently compiling
:param out_file: the file where we save the output
"""
self._tokenizer = JackTokenizer(in_file)
self._class_table = SymbolTable()
self._method_table = SymbolTable()
self._cur_class_name = ""
self._vm_writer = VMWriter(out_file)
self._label_count_while = 0
self._label_count_if = 0
def compileSubroutine(rfile, wfile, tokenAtt):
wfile.write('<subroutineDec>\n<keyword> ' + tokenAtt + ' </keyword>\n')
#(void|type) subroutineName (parameterList)
token = JackTokenizer.advance(rfile)
while token != '(':
JackTokenizer.writeToken(wfile, token)
token = JackTokenizer.advance(rfile)
wfile.write('<symbol> ( </symbol>\n')
compileParameterList(rfile, wfile)
wfile.write('<symbol> ) </symbol>\n')
#subroutinBody
compileSubroutineBody(rfile, wfile)
wfile.write('</subroutineDec>\n')
def compileExpressionList(rfile, wfile):
wfile.write('<expressionList>\n')
token = JackTokenizer.advance(rfile)
if token != ')':
lennum = -len(token)
rfile.seek(lennum, 1)
while token != ')':
compileExpression(rfile, wfile)
rfile.seek(-1, 1)
token = JackTokenizer.advance(rfile)
if token == ',':
wfile.write('<symbol> , </symbol>\n')
wfile.write('</expressionList>\n')
def compileReturn(rfile, wfile):
wfile.write('<returnStatement>\n')
wfile.write('<keyword> return </keyword>\n')
#expression?
token = JackTokenizer.advance(rfile)
if token == ';':
JackTokenizer.writeToken(wfile, token)
else:
lennum = -len(token)
rfile.seek(lennum, 1)
compileExpression(rfile, wfile)
wfile.write('<symbol> ; </symbol>\n')
wfile.write('</returnStatement>\n')
def __init__(self, input_file, output_file):
self.tokenizer = JackTokenizer(input_file)
self.symbol_table = SymbolTable()
self.vm_writer = VMWriter(output_file)
self.current_sub_name = None
self.class_name = None
self.func_counter = 0
self.while_counter = 0
self.if_counter = 0
# starts the process
self.tokenizer.advance()
self.compile_class()
self.vm_writer.close()
def compileSubroutineBody(rfile, wfile):
wfile.write('<subroutineBody>\n')
#{varDec* statements}
token = JackTokenizer.advance(rfile)
JackTokenizer.writeToken(wfile, token)
token = JackTokenizer.advance(rfile)
while token == 'var':
compileVarDec(rfile, wfile)
token = JackTokenizer.advance(rfile)
lennum = -len(token)
rfile.seek(lennum, 1)
compileStatements(rfile, wfile)
wfile.write('<symbol> } </symbol>\n')
wfile.write('</subroutineBody>\n')
def compile_subroutine_call_sec(tokenized):
global index
translated = ""
if tokenized[index] == ".":
translated += jt.write_symbol(tokenized[index]) # .
index += 1
translated += jt.write_identifier(tokenized[index]) # name
index += 1
translated += jt.write_symbol(tokenized[index]) # (
index += 1
translated += compile_expression_list(tokenized) + jt.write_symbol(
tokenized[index])
index += 1
return translated
def compile_var_dec(tokenized):
global index
translated = jt.write_keyword(tokenized[index])
index += 1
translated += check_type(tokenized[index])
index += 1
translated += jt.write_identifier(tokenized[index])
index += 1
if tokenized[index] == ",":
translated += compile_many_vars_dec(tokenized)
translated = "<varDec>\n" + translated + jt.write_symbol(
tokenized[index]) + "</varDec>\n"
index += 1
return translated
def __init__(self, inputFilepath, outputFilepath):
self.tokenizer = JackTokenizer.JackTokenizer(inputFilepath)
self.file = open(outputFilepath, "w+")
self.xmlIndentation = 0
self.types = ["int", "boolean", "char"]
self.compileClass()
def compile_while_statement(tokenized):
global index
translated = jt.write_keyword(tokenized[index]) # while
index += 1
translated += jt.write_symbol(tokenized[index]) # (
index += 1
translated += "<expression>\n" + compile_expression(tokenized) + \
"</expression>\n" + jt.write_symbol(tokenized[index])
index += 1
translated += jt.write_symbol(tokenized[index]) + "<statements>\n" # {
index += 1
while tokenized[index] != "}":
translated += compile_statement(tokenized)
translated += "</statements>\n" + jt.write_symbol(tokenized[index]) # }
index += 1
return translated
def CompileClass(classname,outputFile):
global pos
global fpw
pos = 0
tokens = JackTokenizer.tokens_all(classname) # get tokens of inputfile(a class file)
fpw = open(outputFile,'w')
fpw.write('<class>\n')
fpw.write('<keyword>'+tokens[pos]+'</keyword>\n') #class
pos += 1
fpw.write('<identifier>'+tokens[pos]+'</identifier>\n') # classname
pos += 1
fpw.write('<symbol>'+tokens[pos]+'</symbol>\n') #'{'
pos += 1
while (tokens[pos] in ['field','static'] ): # classVarDec
fpw.write('<ClassVarDec>\n')
CompileClassVarDec(tokens,pos)
fpw.write('</ClassVarDec>\n')
while (tokens[pos] in ['constructor','function','method']): # subroutineDec
fpw.write('<SubroutineDec>\n')
CompileSubroutine(tokens,pos)
fpw.write('</SubroutineDec>\n')
pos += 1
fpw.write('<symbol>'+tokens[pos]+'</symbol>\n') #'}'
fpw.write('</class>\n')
fpw.close()
def __init__(self, input_file, output_file):
"""
creates new compilation engine with given input and output
It gets its input from a JackTokenizer and emits its parsed structure into an output file/stream.
e.g. integerConstant 300 to push constant 1
The output is generated by a series of compilexxx() routines,
one for every syntactic element xxx of the Jack grammar.
The contract between these routines is that each compilexxx() routine should
read the syntactic construct xxx from the input, advance() the tokenizer
exactly beyond xxx, and output the parsing of xxx.
Thus, compilexxx()may only be called if indeed xxx is the next syntactic
element of the input.
:param input:
:param output:
"""
# init tokenizer with input, VMwriter with output, and symboltable no args
self.tokenizer = JackTokenizer.JackTokenizer(input_file)
self.vm_writer = VMwriter.VMWriter(output_file)
self.symbol_table = SymbolTable.SymbolTable()
# keep track of class name e.g. Main and subroutine name e.g. Main.main
self.class_name = ""
self.sub_name = ""
# initialise expression sets
self.binary_op = self.binary_op()
self.unary_op = self.unary_op()
self.keyword_constant = self.keyword_constant()
def compile_class_var_dec(tokenized):
global index
translated = jt.write_keyword(tokenized[index]) # field / static
index += 1
translated += check_type(tokenized[index]) # type
index += 1
translated += jt.write_identifier(tokenized[index]) # varName
index += 1
while tokenized[index] != ";":
translated += jt.write_symbol(tokenized[index]) # ,
index += 1
translated += jt.write_identifier(tokenized[index])
index += 1
translated += jt.write_symbol(tokenized[index]) # ;
index += 1
return "<classVarDec>\n" + translated + "</classVarDec>\n"
def process_file(jack_file, jack_directory=''):
if jack_directory == '':
x = ''
else:
x = '/'
print "Compiling file: %s" % (jack_directory + x + jack_file)
#print JackTokenizer.process_file(jack_file, jack_directory)
tokenized_file = JackTokenizer.process_file(jack_file, jack_directory)
tokenized_text = TokenizedText(tokenized_file)
process_tokens(tokenized_text, jack_file[:-5], jack_directory)
print "Wrote to: %s" % (jack_file[:-5] + '.vm')
return
def process_file(filename,pathname=''):
fullname = []
if pathname == '':
x = ''
else:
x = '/'
print "Compiling file: %s" % (pathname + x + filename)
fullname = pathname + x + filename
tokenized_file = JackTokenizer.begin(fullname)
tokenized_text = Vars(tokenized_file)
parser(tokenized_text, filename[:-5], pathname)
print "Waiting for ..."
print "Compilation Completed!"
return
def CompileClass(classname,outputFile):
global pos,fpw,className
outputFilename = classname[:classname.rfind('.')]+'.vm' # set output file name
fpw = open(outputFilename,'w')
pos = 0 # initialize to 0 at beginning each file
tokens = JackTokenizer.tokens_all(classname) # get tokens of inputfile(a class file)
SymbolTable.Constructor() # prepare to build up symbol tables
className = classname[:classname.find('.')] # get className
pos += 1
pos += 1 # arrive '{'
pos += 1 # begin class
while (tokens[pos] in ['field','static'] ): # classVarDec
CompileClassVarDec(tokens)
while (tokens[pos] in ['constructor','function','method']): # subroutineDec
CompileSubroutine(tokens)
# end of compilatione, close output file writer
fpw.close()
def CompileClassVarDec(tokens,pos_):
global fpw
global pos
pos = pos_
fpw.write('<keyword>'+tokens[pos]+'</keyword>\n') # static, field
pos += 1
if tokens[pos] in ['int','char','boolean']: # int, char, boolean
fpw.write('<keyword>'+tokens[pos]+'</keyword>\n')
pos += 1
elif JackTokenizer.tokenType(tokens[pos]) == 'IDENTIFIER': # className
fpw.write('<identifier>'+tokens[pos]+'</identifier>\n')
pos += 1
fpw.write('<identifier>'+tokens[pos]+'</identifier>\n') # varName
pos += 1
while (tokens[pos] == ','): # ',', many variable declaritions
fpw.write('<symbol>'+tokens[pos]+'</symbol>\n')
pos += 1
fpw.write('<identifier>'+tokens[pos]+'</identifier>\n')
pos += 1
if tokens[pos] == ';': # can delete if
fpw.write('<symbol>'+tokens[pos]+'</symbol>\n')
pos += 1
import JackTokenizer import CompilationEngine import os import VMWriter import SymbolTable rfile = r"C:\Users\Liu_100\Desktop\nand2tetris\nand2tetris\projects\11\Pong\Ball.jack" xml = os.path.splitext(rfile)[0] + 'MyVersion.xml' vm = os.path.splitext(rfile)[0] + 'MyVersion.vm' xml = open(xml,'w') vm = open(vm,'w') jackTokenizer = JackTokenizer.jacktokenizer(rfile, xml) vmWriter = VMWriter.VMWriter(vm) symbolTable = SymbolTable.SymbolTable() compiler = CompilationEngine.compilationengine(xml,symbolTable,vmWriter) compiler.compileClass(jackTokenizer) xml.close() vmWriter.close()
def main():
"""Drives the Jack-to-VM translation process"""
file_name = sys.argv[1]
tokenizers = []
output_files = []
abs_path = os.path.abspath(file_name)
if '.jack' in file_name and file_name[-5:] == '.jack':
tokenizer = JackTokenizer(abs_path)
tokenizers.append(tokenizer)
output_path = os.path.splitext(abs_path)[0] + '.xml'
output_files.append(output_path)
else:
for walk_obj in os.walk(abs_path):
for jack_file in walk_obj[2]:
if '.jack' in jack_file and jack_file[-5:] == '.jack':
tokenizer = JackTokenizer(abs_path + '/' + jack_file)
tokenizers.append(tokenizer)
output_path = abs_path + '/' + jack_file[:-5] + '.xml'
output_files.append(output_path)
for tokenizer in tokenizers:
while tokenizer.has_more_tokens():
tokenizer.advance()
token_type = tokenizer.token_type()
if token_type == 'KEYWORD':
keyword = tokenizer.keyword()
elif token_type == 'SYMBOL':
symbol = tokenizer.symbol()
elif token_type == 'IDENTIFIER':
identifier = tokenizer.identifier()
elif token_type == 'INT_CONST':
int_val = tokenizer.int_val()
elif token_type == 'STRING_CONST':
string_val = tokenizer.string_val()
import JackTokenizer import CompilationEngine import os rfile = r"C:\Users\Liu_100\Desktop\nand2tetris\nand2tetris\projects\10\Square\Square.jack" wfile = os.path.splitext(rfile)[0] + 'FinalVersion.xml' wfile = open(wfile,'w') jackTokenizer = JackTokenizer.jacktokenizer(rfile, wfile) compiler = CompilationEngine.compilationengine(wfile) compiler.compileClass(jackTokenizer) wfile.close()
