def __processFile__(self, filePath): ''' processes a single file, first feeding the file to JackTokenizer to generate a list of tokens (output as T.xml files for debugging use) and that token list is fed through CompilationEngine to generate a final result list of XML tokens which is output into an .xml file. ''' #Phase 1 Tokenize/Analyze tokenizer = JackTokenizer(filePath) print(filePath) xmlTokenList = ["<tokens>"] taggedTokenList = [("listStart", "tokens", xmlTokenList[0])] token = tokenizer.advance() while token: taggedToken = self.__wrapTokenInXML__(token) taggedTokenList += [taggedToken] xmlTokenList += [taggedToken[TT_XML]] token = tokenizer.advance() xmlTokenList += ["</tokens>"] length = len(xmlTokenList) taggedTokenList += [("listEnd", "tokens", xmlTokenList[length - 1])] Tfilename = str(filePath.parent) + '/' + filePath.stem + "T.xml" self.__output__(Tfilename, xmlTokenList) #Phase 2 Compile/Translate compiler = CompilationEngine(taggedTokenList) compiledXMLList = compiler.compileTokens() Cfilename = str(filePath.parent) + '/' + filePath.stem + ".xml" self.__output__(Cfilename, compiledXMLList)
def __processFile__(self, filePath): ''' processes a single file, first feeding the file to JackTokenizer to generate a list of tokens (output as T.xml files for debugging use) and that token list is fed through CompilationEngine to generate a final result list of XML tokens which is output into an .xml file. ''' #TODO make it work # create opening token tag for tokenizing lines of .jack tokens = ["<tokens>"] tokenizer = JackTokenizer(filePath) line = tokenizer.advance() # tokenize each line of .jack while line: tokens += [self.__wrapTokenInXML__(line)] line = tokenizer.advance() tokens += ["</tokens>"] # 2. create a list for compiled tokens to go into, create compEngine instance # compile the tokens compiledTokens = [] compEngine = CompilationEngine(tokens) compiledTokens += compEngine.compileTokens() # create the filepath names for writing the tokens and full blown xml xml_T_FilePath = Path(filePath.parent / (filePath.stem + 'T.xml')) finalTokenPath = Path(filePath.parent / (filePath.stem + '.xml')) # write out the raw tokens self.__output__(xml_T_FilePath, tokens) self.__output__(finalTokenPath, compiledTokens)
def compileParameterList(rfile, wfile): wfile.write('<parameterList>\n') token = JackTokenizer.advance(rfile) while token != ')': JackTokenizer.writeToken(wfile, token) token = JackTokenizer.advance(rfile) wfile.write('</parameterList>\n')
def compileDo(rfile, wfile): wfile.write('<doStatement>\n') wfile.write('<keyword> do </keyword>\n') subroutinCall(rfile, wfile) token = JackTokenizer.advance(rfile) JackTokenizer.writeToken(wfile, token) wfile.write('</doStatement>\n')
def compile_do_statement(tokenized): global index translated = jt.write_keyword(tokenized[index]) # do index += 1 translated += compile_subroutine_call(tokenized) + jt.write_symbol( tokenized[index]) index += 1 return translated
def analyze(filename): """ Given file/directory, analyze it :param filename: """ tokenizer = JackTokenizer(filename) tokenizer.tokenize() engine = CompilationEngine(tokenizer) engine.write(filename.replace('.jack', '.vm'))
def __init__(self, in_file, out_file): """ A compilation engine constructor :param in_file: the file we are currently compiling :param out_file: the file where we save the output """ self.tokenizer = JackTokenizer(in_file) self.out_file = open(out_file, 'w') self._indent_count = 0
def compileExpression(rfile, wfile): wfile.write('<expression>\n') compileTerm(rfile, wfile) token = JackTokenizer.advance(rfile) while not (token in (')', ']', ';', ',')): JackTokenizer.writeToken(wfile, token) compileTerm(rfile, wfile) token = JackTokenizer.advance(rfile) wfile.write('</expression>\n')
def __init__(self, input_file, output_file): self.tokenizer = JackTokenizer(input_file) self.xml_file = open(output_file, "w") self.space_depth = 0 # starts the process self.tokenizer.advance() self.compile_class() self.xml_file.close()
def __init__(self, inputFile, outputFile): # self.outFile = outputFile self.tokenizer = JackTokenizer(inputFile) self.vmwriter = VMWriter(outputFile) self.outFile = outputFile self.currToken = "" self.tabber = "" self.argumentsCounter = 0
def main(): global fn userInput = sys.argv[1] if ".jack" in userInput: filename = userInput if "/" not in filename: pos = filename.find(".jack") fn = filename[: pos] #newpath = fn #if not os.path.isdir(newpath): # os.makedirs(newpath) outfilenameT = filename[:filename.index(".")] + "T.xml" outfilename = filename[:filename.index(".")] + ".vm" else: last_pos = filename.rfind("/") pos = filename.find(".jack") fn = filename[last_pos + 1: pos] savepath = filename[:last_pos] #newpath = savepath + "/" + fn #if not os.path.isdir(newpath): # os.makedirs(newpath) outfilenameT = filename[last_pos + 1:filename.rfind(".")] + "T.xml" outfilename = filename[last_pos + 1:filename.rfind(".")] + ".vm" ofT = open(outfilenameT, "w") xmlOp = JackTokenizer.tokenizer(filename) ofT.write(xmlOp) ofT.close() xmlOp = CompilationEngine.compilationEngine(outfilename, outfilenameT) compilationEngine = CompilationEngine.compilationEngine(outfilenameT, outfilename) else: os.chdir(userInput) if userInput.endswith("/"): last_pos = userInput.rfind("/") second_last_pos = userInput[: last_pos].rfind("/") #newpath = userInput[second_last_pos + 1: last_pos] else: if "/" in userInput: last_pos = userInput.rfind("/") else: last_pos = 0 #newpath = userInput[last_pos :] #if not os.path.isdir(newpath): # os.makedirs(newpath) for filename in os.listdir(os.getcwd()): if filename.endswith(".jack"): pos = filename.find(".jack") fn = filename[: pos] outfilenameT = fn + "T.xml" outfilename = fn + ".vm" ofT = open(outfilenameT, "w") xmlOp = JackTokenizer.tokenizer(filename) ofT.write(xmlOp) ofT.close() print(outfilenameT) compilationEngine = CompilationEngine.compilationEngine(outfilenameT, outfilename)
def compile_return_statement(tokenized): global index translated = jt.write_keyword(tokenized[index]) # return index += 1 if tokenized[index] != ";": # eg return expression translated += "<expression>\n" + compile_expression( tokenized) + "</expression>\n" translated += jt.write_symbol(tokenized[index]) # ; index += 1 return translated
def compile_else(tokenized): global index translated = jt.write_keyword(tokenized[index]) # else index += 1 translated += jt.write_symbol(tokenized[index]) # { index += 1 statements = "" while tokenized[index] != "}": statements += compile_statement(tokenized) return translated + "<statements>\n" + statements + "</statements>\n"
def translate_token(tokenized): global index translated = jt.write_keyword(tokenized[index]) # class index += 1 translated += jt.write_identifier(tokenized[index]) # name index += 1 translated += jt.write_symbol(tokenized[index]) # "{" index += 1 return "<class>\n" + translated + inside_class( tokenized) + jt.write_symbol(tokenized[index]) + "</class>"
def compile_params_list(tokenized): global index translated = check_type(tokenized[index]) index += 1 translated += jt.write_identifier(tokenized[index]) index += 1 if tokenized[index] == ",": translated += jt.write_symbol(tokenized[index]) index += 1 return translated + compile_params_list(tokenized) return translated
def compile_many_vars_dec(tokenized): global index translated = jt.write_symbol(tokenized[index]) # "," index += 1 translated += jt.write_identifier(tokenized[index]) # var index += 1 while tokenized[index] != ";": translated += jt.write_symbol(tokenized[index]) # "," index += 1 translated += jt.write_identifier(tokenized[index]) # var index += 1 return translated
def __init__(self, file): """ """ self.label_num = 0 self.tokenizer = JackTokenizer(file) self.advance() self.symbols = SymbolTable() self.vm = VMWriter() self.open_outfile(file) self.compile_class() self.close_outfile()
def __init__(self, input_path, output_path): """ creates a new compilation engine with the given input and output. the next routine called must be compileClass() :param input_path: input stream/file :param output_path: output stream/file """ self.labels = 0 self.jack_class = None self.class_subroutine = None self.tokenizer = JackTokenizer(input_path) self._writer = VMWriter(output_path) self.CompileClass()
def __init__(self, in_file, out_file): """ A compilation engine constructor :param in_file: the file we are currently compiling :param out_file: the file where we save the output """ self._tokenizer = JackTokenizer(in_file) self._class_table = SymbolTable() self._method_table = SymbolTable() self._cur_class_name = "" self._vm_writer = VMWriter(out_file) self._label_count_while = 0 self._label_count_if = 0
def compileSubroutine(rfile, wfile, tokenAtt): wfile.write('<subroutineDec>\n<keyword> ' + tokenAtt + ' </keyword>\n') #(void|type) subroutineName (parameterList) token = JackTokenizer.advance(rfile) while token != '(': JackTokenizer.writeToken(wfile, token) token = JackTokenizer.advance(rfile) wfile.write('<symbol> ( </symbol>\n') compileParameterList(rfile, wfile) wfile.write('<symbol> ) </symbol>\n') #subroutinBody compileSubroutineBody(rfile, wfile) wfile.write('</subroutineDec>\n')
def compileExpressionList(rfile, wfile): wfile.write('<expressionList>\n') token = JackTokenizer.advance(rfile) if token != ')': lennum = -len(token) rfile.seek(lennum, 1) while token != ')': compileExpression(rfile, wfile) rfile.seek(-1, 1) token = JackTokenizer.advance(rfile) if token == ',': wfile.write('<symbol> , </symbol>\n') wfile.write('</expressionList>\n')
def compileReturn(rfile, wfile): wfile.write('<returnStatement>\n') wfile.write('<keyword> return </keyword>\n') #expression? token = JackTokenizer.advance(rfile) if token == ';': JackTokenizer.writeToken(wfile, token) else: lennum = -len(token) rfile.seek(lennum, 1) compileExpression(rfile, wfile) wfile.write('<symbol> ; </symbol>\n') wfile.write('</returnStatement>\n')
def __init__(self, input_file, output_file): self.tokenizer = JackTokenizer(input_file) self.symbol_table = SymbolTable() self.vm_writer = VMWriter(output_file) self.current_sub_name = None self.class_name = None self.func_counter = 0 self.while_counter = 0 self.if_counter = 0 # starts the process self.tokenizer.advance() self.compile_class() self.vm_writer.close()
def compileSubroutineBody(rfile, wfile): wfile.write('<subroutineBody>\n') #{varDec* statements} token = JackTokenizer.advance(rfile) JackTokenizer.writeToken(wfile, token) token = JackTokenizer.advance(rfile) while token == 'var': compileVarDec(rfile, wfile) token = JackTokenizer.advance(rfile) lennum = -len(token) rfile.seek(lennum, 1) compileStatements(rfile, wfile) wfile.write('<symbol> } </symbol>\n') wfile.write('</subroutineBody>\n')
def compile_subroutine_call_sec(tokenized): global index translated = "" if tokenized[index] == ".": translated += jt.write_symbol(tokenized[index]) # . index += 1 translated += jt.write_identifier(tokenized[index]) # name index += 1 translated += jt.write_symbol(tokenized[index]) # ( index += 1 translated += compile_expression_list(tokenized) + jt.write_symbol( tokenized[index]) index += 1 return translated
def compile_var_dec(tokenized): global index translated = jt.write_keyword(tokenized[index]) index += 1 translated += check_type(tokenized[index]) index += 1 translated += jt.write_identifier(tokenized[index]) index += 1 if tokenized[index] == ",": translated += compile_many_vars_dec(tokenized) translated = "<varDec>\n" + translated + jt.write_symbol( tokenized[index]) + "</varDec>\n" index += 1 return translated
def __init__(self, inputFilepath, outputFilepath): self.tokenizer = JackTokenizer.JackTokenizer(inputFilepath) self.file = open(outputFilepath, "w+") self.xmlIndentation = 0 self.types = ["int", "boolean", "char"] self.compileClass()
def compile_while_statement(tokenized): global index translated = jt.write_keyword(tokenized[index]) # while index += 1 translated += jt.write_symbol(tokenized[index]) # ( index += 1 translated += "<expression>\n" + compile_expression(tokenized) + \ "</expression>\n" + jt.write_symbol(tokenized[index]) index += 1 translated += jt.write_symbol(tokenized[index]) + "<statements>\n" # { index += 1 while tokenized[index] != "}": translated += compile_statement(tokenized) translated += "</statements>\n" + jt.write_symbol(tokenized[index]) # } index += 1 return translated
def CompileClass(classname,outputFile): global pos global fpw pos = 0 tokens = JackTokenizer.tokens_all(classname) # get tokens of inputfile(a class file) fpw = open(outputFile,'w') fpw.write('<class>\n') fpw.write('<keyword>'+tokens[pos]+'</keyword>\n') #class pos += 1 fpw.write('<identifier>'+tokens[pos]+'</identifier>\n') # classname pos += 1 fpw.write('<symbol>'+tokens[pos]+'</symbol>\n') #'{' pos += 1 while (tokens[pos] in ['field','static'] ): # classVarDec fpw.write('<ClassVarDec>\n') CompileClassVarDec(tokens,pos) fpw.write('</ClassVarDec>\n') while (tokens[pos] in ['constructor','function','method']): # subroutineDec fpw.write('<SubroutineDec>\n') CompileSubroutine(tokens,pos) fpw.write('</SubroutineDec>\n') pos += 1 fpw.write('<symbol>'+tokens[pos]+'</symbol>\n') #'}' fpw.write('</class>\n') fpw.close()
def __init__(self, input_file, output_file): """ creates new compilation engine with given input and output It gets its input from a JackTokenizer and emits its parsed structure into an output file/stream. e.g. integerConstant 300 to push constant 1 The output is generated by a series of compilexxx() routines, one for every syntactic element xxx of the Jack grammar. The contract between these routines is that each compilexxx() routine should read the syntactic construct xxx from the input, advance() the tokenizer exactly beyond xxx, and output the parsing of xxx. Thus, compilexxx()may only be called if indeed xxx is the next syntactic element of the input. :param input: :param output: """ # init tokenizer with input, VMwriter with output, and symboltable no args self.tokenizer = JackTokenizer.JackTokenizer(input_file) self.vm_writer = VMwriter.VMWriter(output_file) self.symbol_table = SymbolTable.SymbolTable() # keep track of class name e.g. Main and subroutine name e.g. Main.main self.class_name = "" self.sub_name = "" # initialise expression sets self.binary_op = self.binary_op() self.unary_op = self.unary_op() self.keyword_constant = self.keyword_constant()
def compile_class_var_dec(tokenized): global index translated = jt.write_keyword(tokenized[index]) # field / static index += 1 translated += check_type(tokenized[index]) # type index += 1 translated += jt.write_identifier(tokenized[index]) # varName index += 1 while tokenized[index] != ";": translated += jt.write_symbol(tokenized[index]) # , index += 1 translated += jt.write_identifier(tokenized[index]) index += 1 translated += jt.write_symbol(tokenized[index]) # ; index += 1 return "<classVarDec>\n" + translated + "</classVarDec>\n"
def process_file(jack_file, jack_directory=''): if jack_directory == '': x = '' else: x = '/' print "Compiling file: %s" % (jack_directory + x + jack_file) #print JackTokenizer.process_file(jack_file, jack_directory) tokenized_file = JackTokenizer.process_file(jack_file, jack_directory) tokenized_text = TokenizedText(tokenized_file) process_tokens(tokenized_text, jack_file[:-5], jack_directory) print "Wrote to: %s" % (jack_file[:-5] + '.vm') return
def process_file(filename,pathname=''): fullname = [] if pathname == '': x = '' else: x = '/' print "Compiling file: %s" % (pathname + x + filename) fullname = pathname + x + filename tokenized_file = JackTokenizer.begin(fullname) tokenized_text = Vars(tokenized_file) parser(tokenized_text, filename[:-5], pathname) print "Waiting for ..." print "Compilation Completed!" return
def CompileClass(classname,outputFile): global pos,fpw,className outputFilename = classname[:classname.rfind('.')]+'.vm' # set output file name fpw = open(outputFilename,'w') pos = 0 # initialize to 0 at beginning each file tokens = JackTokenizer.tokens_all(classname) # get tokens of inputfile(a class file) SymbolTable.Constructor() # prepare to build up symbol tables className = classname[:classname.find('.')] # get className pos += 1 pos += 1 # arrive '{' pos += 1 # begin class while (tokens[pos] in ['field','static'] ): # classVarDec CompileClassVarDec(tokens) while (tokens[pos] in ['constructor','function','method']): # subroutineDec CompileSubroutine(tokens) # end of compilatione, close output file writer fpw.close()
def CompileClassVarDec(tokens,pos_): global fpw global pos pos = pos_ fpw.write('<keyword>'+tokens[pos]+'</keyword>\n') # static, field pos += 1 if tokens[pos] in ['int','char','boolean']: # int, char, boolean fpw.write('<keyword>'+tokens[pos]+'</keyword>\n') pos += 1 elif JackTokenizer.tokenType(tokens[pos]) == 'IDENTIFIER': # className fpw.write('<identifier>'+tokens[pos]+'</identifier>\n') pos += 1 fpw.write('<identifier>'+tokens[pos]+'</identifier>\n') # varName pos += 1 while (tokens[pos] == ','): # ',', many variable declaritions fpw.write('<symbol>'+tokens[pos]+'</symbol>\n') pos += 1 fpw.write('<identifier>'+tokens[pos]+'</identifier>\n') pos += 1 if tokens[pos] == ';': # can delete if fpw.write('<symbol>'+tokens[pos]+'</symbol>\n') pos += 1
import JackTokenizer import CompilationEngine import os import VMWriter import SymbolTable rfile = r"C:\Users\Liu_100\Desktop\nand2tetris\nand2tetris\projects\11\Pong\Ball.jack" xml = os.path.splitext(rfile)[0] + 'MyVersion.xml' vm = os.path.splitext(rfile)[0] + 'MyVersion.vm' xml = open(xml,'w') vm = open(vm,'w') jackTokenizer = JackTokenizer.jacktokenizer(rfile, xml) vmWriter = VMWriter.VMWriter(vm) symbolTable = SymbolTable.SymbolTable() compiler = CompilationEngine.compilationengine(xml,symbolTable,vmWriter) compiler.compileClass(jackTokenizer) xml.close() vmWriter.close()
def main(): """Drives the Jack-to-VM translation process""" file_name = sys.argv[1] tokenizers = [] output_files = [] abs_path = os.path.abspath(file_name) if '.jack' in file_name and file_name[-5:] == '.jack': tokenizer = JackTokenizer(abs_path) tokenizers.append(tokenizer) output_path = os.path.splitext(abs_path)[0] + '.xml' output_files.append(output_path) else: for walk_obj in os.walk(abs_path): for jack_file in walk_obj[2]: if '.jack' in jack_file and jack_file[-5:] == '.jack': tokenizer = JackTokenizer(abs_path + '/' + jack_file) tokenizers.append(tokenizer) output_path = abs_path + '/' + jack_file[:-5] + '.xml' output_files.append(output_path) for tokenizer in tokenizers: while tokenizer.has_more_tokens(): tokenizer.advance() token_type = tokenizer.token_type() if token_type == 'KEYWORD': keyword = tokenizer.keyword() elif token_type == 'SYMBOL': symbol = tokenizer.symbol() elif token_type == 'IDENTIFIER': identifier = tokenizer.identifier() elif token_type == 'INT_CONST': int_val = tokenizer.int_val() elif token_type == 'STRING_CONST': string_val = tokenizer.string_val()
import JackTokenizer import CompilationEngine import os rfile = r"C:\Users\Liu_100\Desktop\nand2tetris\nand2tetris\projects\10\Square\Square.jack" wfile = os.path.splitext(rfile)[0] + 'FinalVersion.xml' wfile = open(wfile,'w') jackTokenizer = JackTokenizer.jacktokenizer(rfile, wfile) compiler = CompilationEngine.compilationengine(wfile) compiler.compileClass(jackTokenizer) wfile.close()