class CompilationEnginge(object): """ lalala """ def __init__(self, input_file, output_file): self.tokenizer = JackTokenizer(input_file) self.out = open(output_file, 'w') self.token = None self.class_name = None ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table = SymbolTable() self.vm_writer = VMWriter(output_file) ####################### def analyze(self): self.token = self.tokenizer.advance() self.compile_class() self.close() print('CLASS TABLE:') print(self.symbol_table.class_table) def close(self): if self.out: self.out.close() self.out = None def advance(self): self.token = self.tokenizer.advance() def write_to_out(self): pass def format_line(self, defined_or_used=''): token_type = self.tokenizer.token_type() running_index = '' if token_type == self.tokenizer.keyword_token: meat = self.tokenizer.keyword() defined_or_used='' elif token_type == self.tokenizer.symbol_token: meat = self.tokenizer.symbol() defined_or_used='' elif token_type == self.tokenizer.identifier_token: meat = self.tokenizer.identifier() ####################### ### PROJECT 11 CODE ### ####################### # Extending compilaiton engine to output <var/argument/static/field...> instead of <indentifier> name = self.tokenizer.token if self.symbol_table.kind_of(name): token_type = self.symbol_table.kind_of(name) running_index = str(self.symbol_table.index_of(name)) elif name[0].islower(): token_type = 'subroutine' else: token_type = 'class' ####################### elif token_type == self.tokenizer.int_const: meat = self.tokenizer.int_val() defined_or_used='' elif token_type == self.tokenizer.string_const: meat = self.tokenizer.string_val() defined_or_used='' else: raise ValueError('Something went wrong with token: {}'.format(self.token)) if defined_or_used != '': defined_or_used += ' ' if running_index != '': running_index = ' ' + running_index formated_line = '<{2}{0}{3}> {1} </{2}{0}{3}>\n'.format(token_type, meat, defined_or_used, running_index) return formated_line ######################### ### PROGARM STRUCTURE ### ######################### def compile_class(self): """ #################################################################### ### class: 'class' className '{' classVarDec* subroutineDec* '}' ### #################################################################### """ self.out.write('<class>\n') # 'class' keyword_line = self.format_line() self.out.write(keyword_line) # className self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.class_name = self.tokenizer.token ####################### identifier_line = self.format_line('defined') self.out.write(identifier_line) # '{' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) ### classVarDec* subroutineDec* ### self.advance() # classVarDec* while self.tokenizer.token_type() == self.tokenizer.keyword_token and self.tokenizer.keyword() in [self.tokenizer.key_field, self.tokenizer.key_static]: self.compile_class_var_dec() # subroutineDec* while self.tokenizer.token_type() == self.tokenizer.keyword_token and self.tokenizer.keyword() in [self.tokenizer.key_function, self.tokenizer.key_method, self.tokenizer.key_constructor]: self.compile_subroutine() # '}' if self.tokenizer.token_type() == self.tokenizer.symbol_token: # Class compilation is done symbol_line = self.format_line() self.out.write(symbol_line) else: raise ValueError('Something went wrong') # Closing with </class> self.out.write('</class>\n') is_sucessfull = not(self.advance()) if is_sucessfull: print('Compilation enginge succesfully finished') else: print('Something went wrong!') def compile_class_var_dec(self): """ ####################################################################### ### classVarDec: ('static'|'field') type varName (',' varName)* ';' ### ####################################################################### """ self.out.write('<classVarDec>\n') ####################### ### PROJECT 11 CODE ### ####################### # Extract field or static # field_or_static = re.match('<[a-z]*>', field_or_static_line)[0][1:-1] field_or_static = self.tokenizer.token ####################### # ('static' | 'field') field_or_static_line = self.format_line() self.out.write(field_or_static_line) # type self.advance() ####################### ### PROJECT 11 CODE ### ####################### # Extract token type type_ = self.tokenizer.token ####################### type_line = self.format_line() self.out.write(type_line) # varName self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table.define(name=self.tokenizer.token, type_=type_, kind=field_or_static) ####################### varname_line = self.format_line('defined') self.out.write(varname_line) # (',' varName)* self.advance() symbol = self.tokenizer.symbol() while symbol == ',': colon_line = self.format_line() self.out.write(colon_line) self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table.define(name=self.tokenizer.token, type_=type_, kind=field_or_static) ####################### varname_line = self.format_line('defined') self.out.write(varname_line) self.advance() symbol = self.tokenizer.symbol() # symbol == ';' semicolon_line = self.format_line() self.out.write(semicolon_line) self.advance() self.out.write('</classVarDec>\n') def compile_subroutine(self): """ ########################################################################### ### subroutineDec: ('constructor'|'function'|'method') ### ### ('void' | type) subroutineName '(' parameterList ')' ### ### subroutineBody ### ########################################################################### """ ####################### ### PROJECT 11 CODE ### ####################### print() print('SUBROUTINE TABLE:') print(self.symbol_table.subroutine_table) print() self.symbol_table.start_subroutine() self.symbol_table.define(name='this', type_=self.class_name, kind='argument') ####################### self.out.write('<subroutineDec>\n') # ('constructor'|'function'|'method') constructor_function_method_line = self.format_line() self.out.write(constructor_function_method_line) # ('void' | type) self.advance() void_or_type_line = self.format_line() self.out.write(void_or_type_line) # subroutineName self.advance() subroutine_name_line = self.format_line('defined') self.out.write(subroutine_name_line) # '(' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) # parameterList self.advance() self.compile_parameter_list() # ')' symbol_line = self.format_line() self.out.write(symbol_line) ################################################## ### subroutineBody: '{' varDec* statements '}' ### ################################################## self.out.write('<subroutineBody>\n') # '{' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) ############### ### varDec* ### ############### self.advance() while self.tokenizer.token == self.tokenizer.key_var: self.compile_var_dec() ################## ### statements ### ################## self.compile_statements() # '}' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() self.out.write('</subroutineBody>\n') self.out.write('</subroutineDec>\n') ####################### ### PROJECT 11 CODE ### ####################### print() print('SUBROUTINE TABLE:') print(self.symbol_table.subroutine_table) print() ####################### def compile_parameter_list(self): """ ############################################################ ### parameterList: ((type varName) (',' type varName)*)? ### ############################################################ """ self.out.write('<parameterList>\n') # If token type is symbol then we have empty parameter list # If we have symbol token then it means our parameter list is fully processed if self.tokenizer.token_type() != self.tokenizer.symbol_token: # type ####################### ### PROJECT 11 CODE ### ####################### type_ = self.tokenizer.token ####################### type_line = self.format_line() self.out.write(type_line) # varName self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table.define(name=self.tokenizer.token, type_=type_, kind='argument') ####################### var_name_line = self.format_line('defined') self.out.write(var_name_line) # If next token is ',' we have more then one parameter self.advance() while self.tokenizer.token_type() == self.tokenizer.symbol_token and self.tokenizer.symbol() == ',': # ',' comma_line = self.format_line() self.out.write(comma_line) # type self.advance() ####################### ### PROJECT 11 CODE ### ####################### type_ = self.tokenizer.token ####################### type_line = self.format_line() self.out.write(type_line) # varName self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table.define(name=self.tokenizer.token, type_=type_, kind='argument') # We are in new subroutine so add next nested scope # self.symbol_table.start_subroutine() ####################### var_name_line = self.format_line('defined') self.out.write(var_name_line) self.advance() self.out.write('</parameterList>\n') def compile_var_dec(self): """ ##################################################### ### varDec: 'var' type varName (',' varName)* ';' ### ##################################################### """ self.out.write('<varDec>\n') # var var_line = self.format_line() self.out.write(var_line) # type self.advance() ####################### ### PROJECT 11 CODE ### ####################### type_ = self.tokenizer.token ####################### type_line = self.format_line() self.out.write(type_line) # varName self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table.define(name=self.tokenizer.token, type_=type_, kind='local') ####################### var_name_line = self.format_line('defined') self.out.write(var_name_line) # (',' varName)* self.advance() while self.tokenizer.symbol() == ',': # ',' comma_line = self.format_line() self.out.write(comma_line) # varName self.advance() ####################### ### PROJECT 11 CODE ### ####################### self.symbol_table.define(name=self.tokenizer.token, type_=type_, kind='local') ####################### var_name_line = self.format_line('defined') self.out.write(var_name_line) self.advance() # ';' semicolon_line = self.format_line() self.out.write(semicolon_line) self.advance() self.out.write('</varDec>\n') ################## ### STATEMENTS ### ################## def compile_statements(self): """ ############################## ### statements: statement* ### ############################## """ self.out.write('<statements>\n') while self.tokenizer.token_type() != self.tokenizer.symbol_token: keyword = self.tokenizer.keyword() # letStatement if keyword == self.tokenizer.key_let: self.compile_let() # ifStatement elif keyword == self.tokenizer.key_if: self.compile_if() # whileStatement elif keyword == self.tokenizer.key_while: self.compile_while() # doStatement elif keyword == self.tokenizer.key_do: self.compile_do() # returnStatement elif keyword == self.tokenizer.key_return: self.compile_return() else: raise ValueError('Wrong statement: {}'.format(keyword)) self.out.write('</statements>\n') def compile_do(self): """ ############################################ ### doStatement: 'do' subroutineCall ';' ### ############################################ """ self.out.write('<doStatement>\n') # 'do' do_line = self.format_line() self.out.write(do_line) # subroutineCall self.advance() self.compile_subroutine_call() # ';' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() self.out.write('</doStatement>\n') def compile_let(self): """ ############################################################################ ### letStatement: 'let' varName ('[' expression ']')? '=' expression ';' ### ############################################################################ """ self.out.write('<letStatement>\n') # let let_line = self.format_line() self.out.write(let_line) # varName self.advance() var_name_line = self.format_line('used') self.out.write(var_name_line) # Check if '[' or '=' self.advance() if self.tokenizer.token == '[': # '[' symbol_line = self.format_line() self.out.write(symbol_line) # expression self.advance() self.compile_expression() # ']' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() # '=' symbol_line = self.format_line() self.out.write(symbol_line) # expression self.advance() self.compile_expression() # ';' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() self.out.write('</letStatement>\n') def compile_while(self): """ ##################################################################### ### whileStatement: 'while' '(' expression ')' '{' statements '}' ### ##################################################################### """ self.out.write('<whileStatement>\n') # 'while' while_line = self.format_line() self.out.write(while_line) # '(' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) # expression self.advance() self.compile_expression() # ')' symbol_line = self.format_line() self.out.write(symbol_line) # '{' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) # statements self.advance() self.compile_statements() # '}' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() self.out.write('</whileStatement>\n') def compile_return(self): """ ################################################ ### ReturnStatement 'return' expression? ';' ### ################################################ """ self.out.write('<returnStatement>\n') # 'return' return_line = self.format_line() self.out.write(return_line) # Ceck if expression self.advance() if self.tokenizer.token != ';': # 'expression' self.compile_expression() # ';' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() self.out.write('</returnStatement>\n') def compile_if(self): """ ############################################################### ### ifStatement: 'if' '(' expression ')' '{' statements '}' ### ### ('else' '{' statements '}')? ### ############################################################### """ self.out.write('<ifStatement>\n') # 'if' if_line = self.format_line() self.out.write(if_line) # '(' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) # expression self.advance() self.compile_expression() # ')' symbol_line = self.format_line() self.out.write(symbol_line) # '{' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) # statements self.advance() self.compile_statements() # '}' symbol_line = self.format_line() self.out.write(symbol_line) # Check if there is 'else' part of ifStatement self.advance() if self.tokenizer.token_type() == self.tokenizer.keyword_token and self.tokenizer.keyword() == 'else': # 'else' else_line = self.format_line() self.out.write(else_line) # '{' self.advance() symbol_line = self.format_line() self.out.write(symbol_line) # statements self.advance() self.compile_statements() # '}' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() self.out.write('</ifStatement>\n') ################### ### EXPRESSIONS ### ################### def compile_subroutine_call(self, skip_subroutine_name=False): """ ############################################################################ ### subroutineCall: subroutineName '(' expressionList ')' | (className | ### ### varName) '.' subroutineName '(' expressionList ')' ### ############################################################################ """ if not skip_subroutine_name: # subroutineName or className or varName subroutine_class_var_name_line = self.format_line('used') self.out.write(subroutine_class_var_name_line) self.advance() # Check '(' or '.' if self.tokenizer.token == '.': # '.' symbol_line = self.format_line() self.out.write(symbol_line) # subroutineName self.advance() subroutine_name_line = self.format_line('used') self.out.write(subroutine_name_line) self.advance() # '(' symbol_line = self.format_line() self.out.write(symbol_line) # expressionList self.advance() self.compile_expression_list() # ')' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() def compile_expression(self): """ ################################### ### expression: term (op term)* ### ################################### """ self.out.write('<expression>\n') ops = ['+', '-', '*', '/', '&', '|', '<', '>', '='] # 'term' self.compile_term() # Check if there is (op term)* part while self.tokenizer.token in ops: # op op_line = self.format_line() self.out.write(op_line) # term self.advance() self.compile_term() self.out.write('</expression>\n') def compile_term(self): """ ################################################################ ### integerConstant | stringConstant | keywordConstant | ### ### varName | varName '[' expression ']' | subroutineCall | ### ### '(' expression ')' | unaryOp term ### ################################################################ """ self.out.write('<term>\n') unary_ops = ['-', '~'] ############################################# ### constant, name, expression or unaryOp ### ############################################# # '(' expression ')' if self.tokenizer.token == '(': # '(' symbol_line = self.format_line() self.out.write(symbol_line) # expression self.advance() self.compile_expression() # ')' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() # unaryOp term elif self.tokenizer.token in unary_ops: # unaryOp unary_op_line = self.format_line() self.out.write(unary_op_line) # term self.advance() self.compile_term() # integerConstant | stringConstant | keywordConstant | # varName | varName '[' expression ']' | subroutineCall else: # constant or name constant_or_name = self.format_line('used') self.out.write(constant_or_name) # varName '[' expression ']' | subroutineCall or end of compile_term function # Check if expression: '[', subroutineCall: '(' with parameter skip_subroutine_name = True, # otherwise end of compile_term function self.advance() # '[' expression ']' if self.tokenizer.token == '[': # '[' symbol_line = self.format_line() self.out.write(symbol_line) # expression self.advance() self.compile_expression() # ']' symbol_line = self.format_line() self.out.write(symbol_line) self.advance() # subroutineCall with skip_subroutine_name=True elif self.tokenizer.token in ['(', '.']: self.compile_subroutine_call(skip_subroutine_name=True) self.out.write('</term>\n') def compile_expression_list(self): """ ######################################################## ### expressionList: (expression (',' expression)* )? ### ######################################################## """ self.out.write('<expressionList>\n') # Check if token is ')', if so we got empty expression list if self.tokenizer.token != ')': # 'expression' self.compile_expression() # Check if token is ',', if so we got more expressions while self.tokenizer.token == ',': # ',' comma_line = self.format_line() self.out.write(comma_line) # expression self.advance() self.compile_expression() self.out.write('</expressionList>\n')
class CompilationEngine(object): # the destination file for writing destination_file = None # the tokenizer for the input file tokenizer = None # symbol table symbol_table = None # vm writer vm_writer = None # the class name class_name = "" # indicies for if and while loops # start at -1 because we increment before use while_index = -1 if_index = -1 # the constructor for compiling a single class # the next method to be called after construction must be compile_class # source_filename must be a single file, not a directory def __init__(self, source_filename): # destination filename # if the original extension was .jack, then make the extension .vm # if the original extension was not .jack, then append .vm if source_filename.lower().endswith(".jack"): destination_filename = source_filename[:-5] + ".vm" else: destination_filename = source_filename + ".vm" # open the destination filename for writing self.destination_file = open(destination_filename, 'w') # create a tokenizer for the input file self.tokenizer = JackTokenizer(source_filename) # create the symbol table self.symbol_table = SymbolTable() # create the vm writer self.vm_writer = VMWriter(self.destination_file) # compiles a complete class and closes the output file def compile_class(self): # class keyword tt, t = self._token_next(True, "KEYWORD", "class") # name of class tt, t = self._token_next(True, "IDENTIFIER") self.class_name = t # open brace tt, t = self._token_next(True, "SYMBOL", "{") # one or more variable declarations self.tokenizer.advance() while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t in ["field", "static"]: self.compile_class_var_dec() else: # stop trying to process variable declarations break # one or more subroutine declarations while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t in ["constructor", "function", "method"]: self.compile_subroutine() else: # stop trying to process functions break # close brace # do not advance because we already advanced upon exiting the last loop tt, t = self._token_next(False, "SYMBOL", "}") # done with compilation; close the output file self.destination_file.close() # compiles a static declaration or field declaration def compile_class_var_dec(self): # compile the variable declaration # False means this is a class (not a subroutine) self.compile_var_dec(False) # compiles a complete method, function, or constructor def compile_subroutine(self): # start of subroutine self.symbol_table.start_subroutine() # constructor, function, or method keyword tt, type = self._token_next(False, "KEYWORD") # type of the return value # can be either keyword (void) or an identifier (any type) tt, t = self._token_next(True) # name of the method/function/constructor tt, name = self._token_next(True) name = self.class_name + "." + name # if the type is a method, "define" this as an argument, so the other # argument indexes work correctly if type == "method": self.symbol_table.define("this", self.class_name, SymbolTable.ARG) # opening parenthesis tt, t = self._token_next(True, "SYMBOL", "(") # arguments self.tokenizer.advance() self.compile_parameter_list() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") # opening brace tt, t = self._token_next(True, "SYMBOL", "{") # variable declarations self.tokenizer.advance() while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t == "var": self.compile_var_dec() else: # stop trying to process variable declarations break # write the function num_locals = self.symbol_table.var_count(self.symbol_table.VAR) self.vm_writer.write_function(name, num_locals) # write any special code at the top of the function if type == "constructor": # code to allocate memory and set "this" size = self.symbol_table.var_count(self.symbol_table.FIELD) self.vm_writer.write_push(self.vm_writer.CONST, size) self.vm_writer.write_call("Memory.alloc", 1) self.vm_writer.write_pop(self.vm_writer.POINTER, 0) elif type == "function": # nothing special pass elif type == "method": # put argument 0 into pointer 0 (this) self.vm_writer.write_push(self.vm_writer.ARG, 0) self.vm_writer.write_pop(self.vm_writer.POINTER, 0) else: print "WARNING: Expected constructor, function, or name; got", type # statements self.compile_statements() # closing brace tt, t = self._token_next(False, "SYMBOL", "}") self.tokenizer.advance() # compiles a (possibly empty) parameter list, not including the enclosing # parentheses def compile_parameter_list(self): # check for empty list tt, t = self._token_next(False) if tt == "SYMBOL" and t == ")": # the parameter list was empty; do not process any more pass else: # there are things in the parameter list while True: # keyword (variable type) tt, type = self._token_next(False) # identifier (variable name) tt, name = self._token_next(True) # the kind is always an arg, since these are all parameters to the # function kind = SymbolTable.ARG # define the variable in the symbol table self.symbol_table.define(name, type, kind) # possible comma tt, t = self._token_next(True) if tt != "SYMBOL" or t != ",": # not a comma; stop processing parameters break self.tokenizer.advance() # compiles a var declaration # if subroutine is true, only the var keyword can be used # if subroutine is false, only the static and field keywords can be used def compile_var_dec(self, subroutine=True): # the keyword to start the declaration tt, kind = self._token_next(False, "KEYWORD") # check for required types if subroutine: if kind == "var": kind = SymbolTable.VAR else: print "WARNING: expecting var, but received %s" % (str(kind)) else: if kind == "static": kind = SymbolTable.STATIC elif kind == "field": kind = SymbolTable.FIELD else: print "WARNING: expecting static or field, but received %s" % (str(kind)) # type of the declaration # could be an identifier or a keyword (int, etc) tt, type = self._token_next(True) # name of the declaration tt, name = self._token_next(True, "IDENTIFIER") # define the variable in the symbol table self.symbol_table.define(name, type, kind) # can support more than one identifier name, to declare more than one # variable, separated by commas; process the 2nd-infinite variables self.tokenizer.advance() while True: tt, t = self._token_next(False) if tt == "SYMBOL" and t == ",": # another variable name follows tt, name = self._token_next(True, "IDENTIFIER") # define the variable in the symbol table self.symbol_table.define(name, type, kind) self.tokenizer.advance() else: # no more variable names break # should be on the semicolon at the end of the line tt, t = self._token_next(False, "SYMBOL", ";") self.tokenizer.advance() # compiles a sequence of statements, not including the enclosing {} def compile_statements(self): while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t in ["do", "let", "while", "return", "if"]: # call compile_t, where t is the type of compilation we want token = getattr(self, "compile_" + t)() else: # not a statement; stop processing statements break # compiles a do statement def compile_do(self): # do keyword tt, t = self._token_next(False, "KEYWORD", "do") # subroutine call self.tokenizer.advance() self.compile_subroutine_call() # do statements do not have a return value, so eliminate the return # off of the stack self.vm_writer.write_pop(self.vm_writer.TEMP, 0) # semicolon tt, t = self._token_next(False, "SYMBOL", ";") self.tokenizer.advance() # compiles a let statement def compile_let(self): # let keyword tt, t = self._token_next(False, "KEYWORD", "let") # variable name tt, name = self._token_next(True, "IDENTIFIER") # possible brackets for array tt, t = self._token_next(True) if tt == "SYMBOL" and t == "[": # array - write operation array = True # compile the offset expression self.tokenizer.advance() self.compile_expression() # write the base address onto the stack segment, index = self._resolve_symbol(name) self.vm_writer.write_push(segment, index) # add base and offset self.vm_writer.write_arithmetic("add") # we cannot yet put the result into pointer 1, since the read # operation (which hasn't been parsed/computed yet) may use pointer 1 # to read from an arrya value # closing bracket tt, t = self._token_next(False, "SYMBOL", "]") # advance to the next token, since we are expected to be on the = for # the next line self.tokenizer.advance() else: array = False # equals sign tt, t = self._token_next(False, "SYMBOL", "=") # expression self.tokenizer.advance() self.compile_expression() if array: # our stack now looks like this: # TOP OF STACK # computed result to store # address in which value should be stored # ... previous stuff ... # pop the computed value to temp 0 self.vm_writer.write_pop(self.vm_writer.TEMP, 0) # pop the array address to pointer 1 (that) self.vm_writer.write_pop(self.vm_writer.POINTER, 1) # put the computed value back onto the stack self.vm_writer.write_push(self.vm_writer.TEMP, 0) # pop to the variable name or the array reference self.vm_writer.write_pop(self.vm_writer.THAT, 0) else: # not an array - pop the expression to the variable segment, index = self._resolve_symbol(name) self.vm_writer.write_pop(segment, index) # semicolon tt, t = self._token_next(False, "SYMBOL", ";") self.tokenizer.advance() # compiles a while statement def compile_while(self): # labels for this while loop self.while_index += 1 while_start = "WHILE_START_%d" % (self.while_index) while_end = "WHILE_END_%d" % (self.while_index) # while keyword tt, t = self._token_next(False, "KEYWORD", "while") # opening parenthesis tt, t = self._token_next(True, "SYMBOL", "(") # label for the start of the while statement self.vm_writer.write_label(while_start) # the expression that is the condition of the while statement self.tokenizer.advance() self.compile_expression() # the closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") # the result of the evaluation is now on the stack # if false, then goto to the end of the loop # to do this, negate and then call if-goto self.vm_writer.write_arithmetic("not") self.vm_writer.write_if(while_end) # the opening brace tt, t = self._token_next(True, "SYMBOL", "{") # the statments that is the body of the while loop self.tokenizer.advance() self.compile_statements() # the closing brace tt, t = self._token_next(False, "SYMBOL", "}") # after the last statement of the while loop # need to jump back up to the top of the loop to evaluate again self.vm_writer.write_goto(while_start) # label at the end of the loop self.vm_writer.write_label(while_end) self.tokenizer.advance() # compiles a return statement def compile_return(self): # return keyword tt, t = self._token_next(False, "KEYWORD", "return") # possible expression to return tt, t = self._token_next(True) if tt != "SYMBOL" and t != ";": self.compile_expression() else: # no return expression; return 0 self.vm_writer.write_push(self.vm_writer.CONST, 0) # ending semicolon tt, t = self._token_next(False, "SYMBOL", ";") self.vm_writer.write_return() self.tokenizer.advance() # compiles a if statement, including a possible trailing else clause def compile_if(self): # it is more efficient in an if-else case to have the else portion first # in the code when testing, but we use the less-efficient but # easier-to-write true-false pattern here # labels for this if statement self.if_index += 1 if_false = "IF_FALSE_%d" % (self.if_index) if_end = "IF_END_%d" % (self.if_index) # if keyword tt, t = self._token_next(False, "KEYWORD", "if") # opening parenthesis tt, t = self._token_next(True, "SYMBOL", "(") # expression of if statement self.tokenizer.advance() self.compile_expression() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") # the result of the evaluation is now on the stack # if false, then goto the false label # if true, fall through to executing code # if there is no else, then false and end are the same, but having two # labels does not increase code size self.vm_writer.write_arithmetic("not") self.vm_writer.write_if(if_false) # opening brace tt, t = self._token_next(True, "SYMBOL", "{") # statements for true portion self.tokenizer.advance() self.compile_statements() # closing brace tt, t = self._token_next(False, "SYMBOL", "}") tt, t = self._token_next(True) if tt == "KEYWORD" and t == "else": # else statement exists # goto the end of the if statement at the end of the true portion self.vm_writer.write_goto(if_end) # label for the start of the false portion self.vm_writer.write_label(if_false) # opening brace tt, t = self._token_next(True, "SYMBOL", "{") # statements self.tokenizer.advance() self.compile_statements() # closing brace tt, t = self._token_next(False, "SYMBOL", "}") # end label self.vm_writer.write_label(if_end) # advance tokenizer only if we are in the else, since otherwise the # token was advanced by the else check self.tokenizer.advance() else: # no else portion; only put in a label for false, since end is not # used self.vm_writer.write_label(if_false) # compiles an expression (one or more terms connected by operators) def compile_expression(self): # the first term self.compile_term() # finish any number of operators followed by terms while True: tt, t = self._token_next(False) if tt == "SYMBOL" and t in "+-*/&|<>=": # found an operator # postfix order - add the next term and then do the operator # the next term self.tokenizer.advance() self.compile_term() # the operator if t == "+": self.vm_writer.write_arithmetic("add") if t == "-": self.vm_writer.write_arithmetic("sub") if t == "=": self.vm_writer.write_arithmetic("eq") if t == ">": self.vm_writer.write_arithmetic("gt") if t == "<": self.vm_writer.write_arithmetic("lt") if t == "&": self.vm_writer.write_arithmetic("and") if t == "|": self.vm_writer.write_arithmetic("or") if t == "*": self.vm_writer.write_call("Math.multiply", 2) if t == "/": self.vm_writer.write_call("Math.divide", 2) else: # no term found; done parsing the expression break # compiles a term # this routine is faced with a slight difficulty when trying to decide # between some of the alternative parsing rules. specifically, if the # current token is an identifier, the routine must distinguish between a # variable, an array entry, and a subroutine call. a single lookahead token, # which may be one of [, (, or ., suffices to distinguish between the three # possibilities. any other token is not part of this term and should not # be advanced over. def compile_term(self): # a term: integer_constant | string_constant | keyword_constant | # varname | varname[expression] | subroutine_call | (expression) | # unary_op term tt, t = self._token_next(False) if tt == "INT_CONST": self.vm_writer.write_push(self.vm_writer.CONST, t) # advance for the next statement self.tokenizer.advance() elif tt == "STRING_CONST": # after this portion is run, a pointer to a string should be on the # stack # we create a new string of a certain size and then append characters # one by one; each append operation returns the pointer to the same # string # create the string # string is a len, data tuple; not null-terminated size = len(t) self.vm_writer.write_push(self.vm_writer.CONST, size) self.vm_writer.write_call("String.new", 1) # append each character for char in t: self.vm_writer.write_push(self.vm_writer.CONST, ord(char)) self.vm_writer.write_call("String.appendChar", 2) # advance for the next statement self.tokenizer.advance() elif tt == "KEYWORD": if t == "true": # true is -1, which is 0 negated self.vm_writer.write_push(self.vm_writer.CONST, 0) self.vm_writer.write_arithmetic("not") elif t == "false" or t == "null": self.vm_writer.write_push(self.vm_writer.CONST, 0) elif t == "this": self.vm_writer.write_push(self.vm_writer.POINTER, 0) # advance for the next statement self.tokenizer.advance() elif tt == "SYMBOL" and t == "(": # ( expression ) # parse the expression self.tokenizer.advance() self.compile_expression() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") # advance for the next statement self.tokenizer.advance() elif tt == "SYMBOL" and t in "-~": # unary_op term # postfix order - add the next term and then do the operator # parse the rest of the term self.tokenizer.advance() self.compile_term() # write the unary operation if t == "-": self.vm_writer.write_arithmetic("neg") elif t == "~": self.vm_writer.write_arithmetic("not") elif tt == "IDENTIFIER": # varname, varname[expression], subroutine_call # do not write the identifer yet # get the next bit of the expression # if it is a [, then array; if it is a ( or ., then subroutine call # if none of above, then pass over tt2, t2 = self._token_next(True) if tt2 == "SYMBOL" and t2 in "(.": # subroutine call # back up and then compile the subroutine call self.tokenizer.retreat() self.compile_subroutine_call() elif tt2 == "SYMBOL" and t2 == "[": # array - read operation # write the base address onto the stack segment, index = self._resolve_symbol(t) self.vm_writer.write_push(segment, index) # compile the offset expression self.tokenizer.advance() self.compile_expression() # add base and offset self.vm_writer.write_arithmetic("add") # put the resulting address into pointer 1 (that) self.vm_writer.write_pop(self.vm_writer.POINTER, 1) # read from that 0 onto the stack self.vm_writer.write_push(self.vm_writer.THAT, 0) # closing bracket tt, t = self._token_next(False, "SYMBOL", "]") # advance for the next statement self.tokenizer.advance() else: # none of above - just a single identifier segment, index = self._resolve_symbol(t) self.vm_writer.write_push(segment, index) else: # unknown print "WARNING: Unknown term expression object:", tt, t # compiles a (possible empty) comma-separated list of expressions def compile_expression_list(self): num_args = 0 # check for empty list tt, t = self._token_next(False) if tt == "SYMBOL" and t == ")": # the parameter list was empty; do not process any more pass else: # there are things in the parameter list while True: # expression to pass self.compile_expression() num_args += 1 # possible comma tt, t = self._token_next(False) if tt == "SYMBOL" and t == ",": self.tokenizer.advance() else: # not a comma; stop processing parameters break return num_args # compiles a subroutine call # two cases: # - subroutineName(expressionList) # - (class|var).subroutineName(expressionList) def compile_subroutine_call(self): # first part of name tt, name1 = self._token_next(False, "IDENTIFIER") # a dot and another name may exist, or it could be a parenthesis name2 = None tt, t = self._token_next(True) if tt == "SYMBOL" and t == ".": # the name after the dot tt, name2 = self._token_next(True, "IDENTIFIER") # advance so that we are on the parenthesis self.tokenizer.advance() # determine if this is a method call # three possibilities # - class.func() - function call # - var.func() - method call # - func() - method call on current object if self.symbol_table.contains(name1): method_call = True local_call = False elif name2 == None: method_call = True local_call = True else: method_call = False # if a method call, push variable name1 # this a method call if the symbol table contains name1 and name2 exists # OR name1 is a method in the current object if method_call and local_call: # push the current object onto the stack as a hidden argument self.vm_writer.write_push(self.vm_writer.POINTER, 0) elif method_call and not local_call: # push the variable onto the stack as a hidden argument segment, index = self._resolve_symbol(name1) self.vm_writer.write_push(segment, index) # opening parenthesis tt, t = self._token_next(False, "SYMBOL", "(") # expression list self.tokenizer.advance() num_args = self.compile_expression_list() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") # write the call if method_call and local_call: # methd + <blank> # get the name of the vm function to call classname = self.class_name vm_function_name = classname + "." + name1 # increase arguments by 1, since there is the hidden "this" num_args += 1 # make the call self.vm_writer.write_call(vm_function_name, num_args) elif method_call and not local_call: # variable name + method # get the name of the vm function to call classname = self.symbol_table.get(name1)[1] vm_function_name = classname + "." + name2 # increase arguments by 1, since there is the hidden "this" num_args += 1 # make the call self.vm_writer.write_call(vm_function_name, num_args) else: # get the name of the vm function to call vm_function_name = name1 + "." + name2 # make the call self.vm_writer.write_call(vm_function_name, num_args) self.tokenizer.advance() # returns the token_type and token of the next token after advancing the # tokenizer before reading if advance is True def _token_next(self, advance=False, expected_type=None, expected_value=None): # advance the tokenizer, if requested if advance: self.tokenizer.advance() # get the token type and the token itself token_type = self.tokenizer.token_type() token = str(getattr(self.tokenizer, token_type.lower())()) if expected_type and token_type != expected_type: print "WARNING: Type", token_type, "found; expected", expected_type import traceback, sys traceback.print_stack() sys.exit(1) if expected_value and token != expected_value: print "WARNING: Value", token, "found; expected", expected_value import traceback, sys traceback.print_stack() sys.exit(1) return token_type, token # convets a symbol table type into a segment type def _type_to_segment(self, type): if type == self.symbol_table.STATIC: return self.vm_writer.STATIC elif type == self.symbol_table.FIELD: return self.vm_writer.THIS elif type == self.symbol_table.ARG: return self.vm_writer.ARG elif type == self.symbol_table.VAR: return self.vm_writer.LOCAL else: print "ERROR: Bad type %s" % (str(type)) # resolves the symbol from the symbol table # the segment and index is returned as a 2-tuple def _resolve_symbol(self, name): kind, type, index = self.symbol_table.get(name) return self._type_to_segment(kind), index
class CompilationEngine: CLASS_VAR_DEC_KEYWORDS = ['static', 'field'] SUBROUTINE_DEC_KEYWORDS = ['constructor', 'function', 'method'] BINARY_OPERATOR_SYMBOLS = ['+', '-', '*', '/', '&', '|', '<', '>', '='] UNARY_OPERATOR_SYMBOLS = ['-', '~'] BINARY_OPERATORS_TO_COMMAND = { '+': 'add', '-': 'sub', '=': 'eq', '>': 'gt', '<': 'lt', '&': 'and', '|': 'or' } UNARY_OPERATORS_TO_COMMAND = {'-': 'neg', '~': 'not'} TYPE_TO_TAG = { 'STRING_CONST': 'stringConstant', 'INT_CONST': 'integerConstant', 'KEYWORD': 'keyword', 'IDENTIFIER': 'identifier', 'SYMBOL': 'symbol' } SYMBOLS_TO_XML_CONVENTION = { '<': '<', '>': '>', '&': '&', '"': '"' } def __init__(self, input_file_path, vm_writer: VMWriter): self.jack_tokenizer = JackTokenizer(input_file_path) self.symbol_table = SymbolTable() self.vm_writer = vm_writer if self.jack_tokenizer.has_more_tokens(): self.compile_class() def compile_class(self): self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), 'NONE', 'CLASS') self.jack_tokenizer.advance() self.jack_tokenizer.advance() if self.jack_tokenizer.key_word( ) in CompilationEngine.CLASS_VAR_DEC_KEYWORDS: self.compile_class_var_dec() if self.jack_tokenizer.key_word( ) in CompilationEngine.SUBROUTINE_DEC_KEYWORDS: self.compile_subroutine() self.jack_tokenizer.advance() self.vm_writer.close() def write_token(self, token_name): type_tag = CompilationEngine.TYPE_TO_TAG[ self.jack_tokenizer.token_type()] self.output_file.write('<{0}> {1} </{0}>\n'.format( type_tag, token_name)) self.jack_tokenizer.advance() def compile_class_var_dec(self): while self.jack_tokenizer.key_word( ) in CompilationEngine.CLASS_VAR_DEC_KEYWORDS: kind = '' if self.jack_tokenizer.key_word() == 'field': kind = 'FIELD' elif self.jack_tokenizer.key_word() == 'static': kind = 'STATIC' self.jack_tokenizer.advance() field_type = self.get_type() self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), field_type, kind) self.jack_tokenizer.advance() while self.jack_tokenizer.symbol() != ';': self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), field_type, kind) self.jack_tokenizer.advance() self.jack_tokenizer.advance() def write_type(self): if self.jack_tokenizer.token_type() == 'KEYWORD': self.write_token(self.jack_tokenizer.key_word()) elif self.jack_tokenizer.token_type() == 'IDENTIFIER': self.write_token(self.jack_tokenizer.identifier()) def compile_subroutine(self): self.vm_writer.zero_branching_indexes() while self.jack_tokenizer.key_word( ) in CompilationEngine.SUBROUTINE_DEC_KEYWORDS: self.symbol_table.start_subroutine() constructor = True if self.jack_tokenizer.key_word( ) == 'constructor' else False method = False if self.jack_tokenizer.key_word() == 'method': method = True self.symbol_table.define('this', self.symbol_table.get_class_name(), 'ARG') self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), 'NONE', 'SUBROUTINE') name = self.symbol_table.get_class_name( ) + '.' + self.jack_tokenizer.identifier() self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.compile_parameter_list() self.jack_tokenizer.advance() self.jack_tokenizer.advance() var_num = 0 while self.jack_tokenizer.key_word() == 'var': var_num += self.compile_var_dec() self.vm_writer.write_function(name, var_num) if method: self.vm_writer.write_push('ARG', 0) self.vm_writer.write_pop('POINTER', 0) elif constructor: field_count = self.symbol_table.var_count('FIELD') self.vm_writer.write_push('CONST', field_count) self.vm_writer.write_call('Memory.alloc', 1) self.vm_writer.write_pop('POINTER', 0) self.compile_statements() self.jack_tokenizer.advance() def compile_parameter_list(self): if self.jack_tokenizer.symbol() != ')': parameter_type = self.get_type() self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), parameter_type, 'ARG') self.jack_tokenizer.advance() while self.jack_tokenizer.symbol() == ",": self.jack_tokenizer.advance() parameter_type = self.get_type() self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), parameter_type, 'ARG') self.jack_tokenizer.advance() def get_type(self): if self.jack_tokenizer.token_type() == 'KEYWORD': parameter_type = self.jack_tokenizer.key_word() elif self.jack_tokenizer.token_type() == 'IDENTIFIER': parameter_type = self.jack_tokenizer.identifier() return parameter_type def compile_var_dec(self): var_num = 1 self.jack_tokenizer.advance() var_type = self.get_type() self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), var_type, 'VAR') self.jack_tokenizer.advance() while self.jack_tokenizer.symbol() == ",": var_num += 1 self.jack_tokenizer.advance() self.symbol_table.define(self.jack_tokenizer.identifier(), var_type, 'VAR') self.jack_tokenizer.advance() self.jack_tokenizer.advance() return var_num def compile_statements(self): while self.jack_tokenizer.token_type() == 'KEYWORD': if self.jack_tokenizer.key_word() == 'let': self.compile_let() elif self.jack_tokenizer.key_word() == 'if': self.compile_if() elif self.jack_tokenizer.key_word() == 'while': self.compile_while() elif self.jack_tokenizer.key_word() == 'do': self.compile_do() elif self.jack_tokenizer.key_word() == 'return': self.compile_return() def compile_do(self): self.jack_tokenizer.advance() name = self.jack_tokenizer.identifier() self.jack_tokenizer.advance() self.compile_subroutine_call(name) # must dispose of void function return value self.vm_writer.write_pop('TEMP', 0) self.jack_tokenizer.advance() def compile_subroutine_call(self, prefix_call=''): if self.jack_tokenizer.symbol() == '(': subroutine = False # If not in symbol table - then subroutine if not self.symbol_table.kind_of( prefix_call) or self.symbol_table.kind_of( prefix_call) == 'SUBROUTINE': subroutine = True self.jack_tokenizer.advance() args_count = 0 if subroutine: self.vm_writer.write_push('POINTER', 0) args_count += 1 args_count += self.compile_expression_list() if subroutine: self.vm_writer.write_call( self.symbol_table.get_class_name() + '.' + prefix_call, args_count) else: self.vm_writer.write_call(prefix_call, args_count) self.jack_tokenizer.advance() elif self.jack_tokenizer.symbol() == '.': variable = False self.jack_tokenizer.advance() if self.symbol_table.kind_of(prefix_call) in ['VAR', 'FIELD']: variable = True variable_name = prefix_call prefix_call = self.symbol_table.type_of(prefix_call) prefix_call += '.{0}'.format(self.jack_tokenizer.identifier()) self.jack_tokenizer.advance() self.jack_tokenizer.advance() args_count = 0 if variable: self.vm_writer.write_push( self.symbol_table.kind_of(variable_name), self.symbol_table.index_of(variable_name)) args_count += 1 args_count += self.compile_expression_list() self.vm_writer.write_call(prefix_call, args_count) self.jack_tokenizer.advance() def compile_let(self): self.jack_tokenizer.advance() var_name = self.jack_tokenizer.identifier() self.jack_tokenizer.advance() if self.jack_tokenizer.symbol() == '[': self.vm_writer.write_push(self.symbol_table.kind_of(var_name), self.symbol_table.index_of(var_name)) self.jack_tokenizer.advance() self.compile_expression() self.vm_writer.write_arithmetic("add") self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.compile_expression() self.vm_writer.write_pop('TEMP', 0) self.vm_writer.write_pop('POINTER', 1) self.vm_writer.write_push('TEMP', 0) self.vm_writer.write_pop('THAT', 0) else: self.jack_tokenizer.advance() self.compile_expression() self.vm_writer.write_pop(self.symbol_table.kind_of(var_name), self.symbol_table.index_of(var_name)) self.jack_tokenizer.advance() def compile_while(self): while_idx = self.vm_writer.get_next_label_index('while') if_label = 'WHILE_IF_{0}'.format(while_idx) end_label = 'WHILE_END_{0}'.format(while_idx) self.vm_writer.write_label(if_label) self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.compile_expression() self.vm_writer.write_arithmetic('not') self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.vm_writer.write_if(end_label) self.compile_statements() self.vm_writer.write_goto(if_label) self.jack_tokenizer.advance() self.vm_writer.write_label(end_label) def compile_return(self): self.jack_tokenizer.advance() if self.jack_tokenizer.symbol() != ';': self.compile_expression() else: self.vm_writer.write_push('CONST', 0) self.vm_writer.write_return() self.jack_tokenizer.advance() def compile_if(self): if_idx = self.vm_writer.get_next_label_index('if') else_label = 'IF_ELSE_{0}'.format(if_idx) end_label = 'IF_END_{0}'.format(if_idx) self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.compile_expression() self.vm_writer.write_arithmetic('not') self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.vm_writer.write_if(else_label) self.compile_statements() self.jack_tokenizer.advance() self.vm_writer.write_goto(end_label) self.vm_writer.write_label(else_label) if self.jack_tokenizer.key_word() == 'else': self.jack_tokenizer.advance() self.jack_tokenizer.advance() self.compile_statements() self.jack_tokenizer.advance() self.vm_writer.write_label(end_label) def compile_expression(self): self.compile_term() while self.jack_tokenizer.symbol( ) in CompilationEngine.BINARY_OPERATOR_SYMBOLS: symbol = self.jack_tokenizer.symbol() self.jack_tokenizer.advance() self.compile_term() if symbol in self.BINARY_OPERATORS_TO_COMMAND: self.vm_writer.write_arithmetic( self.BINARY_OPERATORS_TO_COMMAND[symbol]) elif symbol == '*': self.vm_writer.write_call('Math.multiply', 2) elif symbol == '/': self.vm_writer.write_call('Math.divide', 2) def compile_term(self): token_type = self.jack_tokenizer.token_type() if token_type == 'IDENTIFIER': name = self.jack_tokenizer.identifier() self.jack_tokenizer.advance() if self.jack_tokenizer.symbol( ) == '(' or self.jack_tokenizer.symbol() == '.': self.compile_subroutine_call(name) elif self.jack_tokenizer.symbol() == '[': self.vm_writer.write_push(self.symbol_table.kind_of(name), self.symbol_table.index_of(name)) self.jack_tokenizer.advance() self.compile_expression() self.jack_tokenizer.advance() self.vm_writer.write_arithmetic('add') self.vm_writer.write_pop('POINTER', 1) self.vm_writer.write_push('THAT', 0) else: kind = self.symbol_table.kind_of(name) idx = self.symbol_table.index_of(name) self.vm_writer.write_push(kind, idx) elif token_type == 'STRING_CONST': string_const = self.jack_tokenizer.string_val() self.vm_writer.write_push("CONST", len(string_const)) self.vm_writer.write_call("String.new", 1) for char in string_const: self.vm_writer.write_push('CONST', ord(char)) self.vm_writer.write_call("String.appendChar", 2) self.jack_tokenizer.advance() elif token_type == 'KEYWORD': keyword = self.jack_tokenizer.key_word() if keyword == 'true': self.vm_writer.write_push('CONST', 1) self.vm_writer.write_arithmetic('neg') elif keyword == 'false' or keyword == 'null': self.vm_writer.write_push('CONST', 0) elif keyword == 'this': self.vm_writer.write_push('POINTER', 0) self.jack_tokenizer.advance() elif token_type == 'SYMBOL': if self.jack_tokenizer.symbol() == '(': self.jack_tokenizer.advance() self.compile_expression() self.jack_tokenizer.advance() elif self.jack_tokenizer.symbol( ) in CompilationEngine.UNARY_OPERATOR_SYMBOLS: command = CompilationEngine.UNARY_OPERATORS_TO_COMMAND[ self.jack_tokenizer.symbol()] self.jack_tokenizer.advance() self.compile_term() self.vm_writer.write_arithmetic(command) elif token_type == 'INT_CONST': self.vm_writer.write_push('CONST', self.jack_tokenizer.int_val()) self.jack_tokenizer.advance() def compile_expression_list(self): expression_count = 0 if self.jack_tokenizer.symbol() != ')': self.compile_expression() expression_count += 1 while self.jack_tokenizer.symbol() == ',': self.jack_tokenizer.advance() self.compile_expression() expression_count += 1 return expression_count
class CompilationEngine(object): # the destination file for writing destination_file = None # the tokenizer for the input file tokenizer = None # current indentation level indent = 0 # the constructor for compiling a single class # the next method to be called after construction must be compile_class # source_filename must be a single file, not a directory def __init__(self, source_filename): # destination filename # if the original extension was .jack, then make the extension T.xml # if the original extension was not .jack, then append T.xml if source_filename.lower().endswith(".jack"): destination_filename = source_filename[:-5] + ".xml" else: destination_filename = source_filename + ".xml" # open the destination filename for writing self.destination_file = open(destination_filename, 'w') # create a tokenizer for the input file self.tokenizer = JackTokenizer(source_filename) # compiles a complete class and closes the output file def compile_class(self): # start the class self._start_block("class") # class tt, t = self._token_next(True, "KEYWORD", "class") self._write(tt, t) # name of class tt, t = self._token_next(True, "IDENTIFIER") self._write(tt, t) # open brace tt, t = self._token_next(True, "SYMBOL", "{") self._write(tt, t) # one or more variable declarations self.tokenizer.advance() while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t in ["field", "static"]: self.compile_class_var_dec() else: # stop trying to process variable declarations break # one or more subroutine declarations while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t in ["constructor", "function", "method"]: self.compile_subroutine() else: # stop trying to process functions break # close brace # do not advance because we already advanced upon exiting the last loop tt, t = self._token_next(False, "SYMBOL", "}") self._write(tt, t) # end the class self._end_block("class") # close the output file self.destination_file.close() # compiles a static declaration or field declaration def compile_class_var_dec(self): # start variable declaration self._start_block("classVarDec") # compile the variable declaration # False means don't print the tags self.compile_var_dec(False) # end variable declaration self._end_block("classVarDec") # compiles a complete method, function, or constructor def compile_subroutine(self): # start subroutine declaration self._start_block("subroutineDec") # constructor, function, or name keyword tt, t = self._token_next(False, "KEYWORD") self._write(tt, t) # type of the return value # can be either keyword (void) or an identifier (any type) tt, t = self._token_next(True) self._write(tt, t) # name of the method/function/constructor tt, t = self._token_next(True) self._write(tt, t) # opening parenthesis tt, t = self._token_next(True, "SYMBOL", "(") self._write(tt, t) # arguments self.tokenizer.advance() self.compile_parameter_list() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") self._write(tt, t) # start body of subroutine self._start_block("subroutineBody") # opening brace tt, t = self._token_next(True, "SYMBOL", "{") self._write(tt, t) # variable declarations self.tokenizer.advance() while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t == "var": self.compile_var_dec() else: # stop trying to process variable declarations break # statements self.compile_statements() # closing brace tt, t = self._token_next(False, "SYMBOL", "}") self._write(tt, t) # end body of subroutine self._end_block("subroutineBody") # finish subroutine declaration self._end_block("subroutineDec") self.tokenizer.advance() # compiles a (possibly empty) parameter list, not including the enclosing # parentheses def compile_parameter_list(self): self._start_block("parameterList") # check for empty list tt, t = self._token_next(False) if tt == "SYMBOL" and t == ")": # the parameter list was empty; do not process any more pass else: # there are things in the parameter list while True: # keyword (variable type) tt, t = self._token_next(False) self._write(tt, t) # identifier (variable name) tt, t = self._token_next(True) self._write(tt, t) # possible comma tt, t = self._token_next(True) if tt == "SYMBOL" and t == ",": self._write(tt, t) else: # not a comma; stop processing parameters break self.tokenizer.advance() self._end_block("parameterList") # compiles a var declaration def compile_var_dec(self, print_tags=True): if print_tags: self._start_block("varDec") # the keyword to start the declaration tt, t = self._token_next(False, "KEYWORD") self._write(tt, t) # type of the declaration # could be an identifier or a keyword (int, etc) tt, t = self._token_next(True) self._write(tt, t) # name of the declaration tt, t = self._token_next(True, "IDENTIFIER") self._write(tt, t) # can support more than one identifier name, to declare more than one # variable, separated by commas; process the 2nd-infinite variables self.tokenizer.advance() while True: tt, t = self._token_next(False) if tt == "SYMBOL" and t == ",": # write the comma self._write(tt, t) # another variable name follows tt, t = self._token_next(True, "IDENTIFIER") self._write(tt, t) self.tokenizer.advance() else: # no more variable names break # should be on the semicolon at the end of the line tt, t = self._token_next(False, "SYMBOL", ";") self._write(tt, t) self.tokenizer.advance() if print_tags: self._end_block("varDec") # compiles a sequence of statements, not including the enclosing {} def compile_statements(self): self._start_block("statements") while True: tt, t = self._token_next(False) if tt == "KEYWORD" and t in ["do", "let", "while", "return", "if"]: # call compile_t, where t is the type of compilation we want token = getattr(self, "compile_" + t)() else: # not a statement; stop processing statements break self._end_block("statements") # compiles a do statement def compile_do(self): self._start_block("doStatement") # do keyword tt, t = self._token_next(False, "KEYWORD", "do") self._write(tt, t) # subroutine call self.tokenizer.advance() self.compile_subroutine_call() # semicolon tt, t = self._token_next(False, "SYMBOL", ";") self._write(tt, t) self._end_block("doStatement") self.tokenizer.advance() # compiles a let statement def compile_let(self): self._start_block("letStatement") # let keyword tt, t = self._token_next(False, "KEYWORD", "let") self._write(tt, t) # variable name tt, t = self._token_next(True, "IDENTIFIER") self._write(tt, t) # possible brackets for array tt, t = self._token_next(True) if tt == "SYMBOL" and t == "[": # write bracket self._write(tt, t) # compile the expression self.tokenizer.advance() self.compile_expression() # closing bracket tt, t = self._token_next(False, "SYMBOL", "]") self._write(tt, t) # advance to the next token, since we are expected to be on the = for # the next line self.tokenizer.advance() # equals sign tt, t = self._token_next(False, "SYMBOL", "=") self._write(tt, t) # expression self.tokenizer.advance() self.compile_expression() # semicolon tt, t = self._token_next(False, "SYMBOL", ";") self._write(tt, t) self._end_block("letStatement") self.tokenizer.advance() # compiles a while statement def compile_while(self): self._start_block("whileStatement") # while keyword tt, t = self._token_next(False, "KEYWORD", "while") self._write(tt, t) # opening parenthesis tt, t = self._token_next(True, "SYMBOL", "(") self._write(tt, t) # the expression that is the condition of the while statement self.tokenizer.advance() self.compile_expression() # the closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") self._write(tt, t) # the opening brace tt, t = self._token_next(True, "SYMBOL", "{") self._write(tt, t) # the statments that is the body of the while loop self.tokenizer.advance() self.compile_statements() # the closing brace tt, t = self._token_next(False, "SYMBOL", "}") self._write(tt, t) self._end_block("whileStatement") self.tokenizer.advance() # compiles a return statement def compile_return(self): self._start_block("returnStatement") # return keyword tt, t = self._token_next(False, "KEYWORD", "return") self._write(tt, t) # possible expression to return tt, t = self._token_next(True) if tt != "SYMBOL" and t != ";": self.compile_expression() # ending semicolon tt, t = self._token_next(False, "SYMBOL", ";") self._write(tt, t) self._end_block("returnStatement") self.tokenizer.advance() # compiles a if statement, including a possible trailing else clause def compile_if(self): self._start_block("ifStatement") # if keyword tt, t = self._token_next(False, "KEYWORD", "if") self._write(tt, t) # opening parenthesis tt, t = self._token_next(True, "SYMBOL", "(") self._write(tt, t) # expression of if statement self.tokenizer.advance() self.compile_expression() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") self._write(tt, t) # opening brace tt, t = self._token_next(True, "SYMBOL", "{") self._write(tt, t) # statements self.tokenizer.advance() self.compile_statements() # closing brace tt, t = self._token_next(False, "SYMBOL", "}") self._write(tt, t) tt, t = self._token_next(True) if tt == "KEYWORD" and t == "else": # else statement exists # write else seld._write(tt, t) # opening brace tt, t = self._token_next(False, "SYMBOL", "{") self._write(tt, t) # statements self.compile_statements() # closing brace tt, t = self._token_next(False, "SYMBOL", "}") self._write(tt, t) # advance tokenizer only if we are in the else, since otherwise the # token was advanced by the else check self.tokenizer.advance() self._end_block("ifStatement") # compiles an expression (one or more terms connected by operators) def compile_expression(self): self._start_block("expression") # the first term self.compile_term() # finish any number of operators followed by terms while True: tt, t = self._token_next(False) if tt == "SYMBOL" and t in "+-*/&|<>=": # found an operator self._write(tt, t) # the next term self.tokenizer.advance() self.compile_term() else: # no term found; done parsing the expression break self._end_block("expression") # compiles a term # this routine is faced with a slight difficulty when trying to decide # between some of the alternative parsing rules. specifically, if the # current token is an identifier, the routine must distinguish between a # variable, an array entry, and a subroutine call. a single lookahead token, # which may be one of [, (, or ., suffices to distinguish between the three # possibilities. any other token is not part of this term and should not # be advanced over. def compile_term(self): self._start_block("term") # a term: integer_constant | string_constant | keyword_constant | # varname | varname[expression] | subroutine_call | (expression) | # unary_op term tt, t = self._token_next(False) if tt in ["INT_CONST", "STRING_CONST", "KEYWORD"]: self._write(tt, t) # advance for the next statement self.tokenizer.advance() elif tt == "SYMBOL" and t == "(": # ( expression ) # write the opening parenthesis self._write(tt, t) # parse the expression self.tokenizer.advance() self.compile_expression() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") self._write(tt, t) # advance for the next statement self.tokenizer.advance() elif tt == "SYMBOL" and t in "-~": # unary_op term # write the unary operation self._write(tt, t) # parse the rest of the term self.tokenizer.advance() self.compile_term() elif tt == "IDENTIFIER": # varname, varname[expression], subroutine_call # do not write the identiifer yet # get the next bit of the expression # if it is a [, then array; if it is a ( or ., then subroutine call # if none of above, then pass over tt2, t2 = self._token_next(True) if tt2 == "SYMBOL" and t2 in "(.": # subroutine call # back up and then compile the subroutine call self.tokenizer.retreat() self.compile_subroutine_call() elif tt2 == "SYMBOL" and t2 == "[": # array # write identifier self._write(tt, t) # write bracket self._write(tt2, t2) # compile the expression self.tokenizer.advance() self.compile_expression() # closing bracket tt, t = self._token_next(False, "SYMBOL", "]") self._write(tt, t) # advance for the next statement self.tokenizer.advance() else: # none of above - just a single identifier self._write(tt, t) else: # unknown print "WARNING: Unknown term expression object:", tt, t self._end_block("term") # compiles a (possible empty) comma-separated list of expressions def compile_expression_list(self): self._start_block("expressionList") # check for empty list tt, t = self._token_next(False) if tt == "SYMBOL" and t == ")": # the parameter list was empty; do not process any more pass else: # there are things in the parameter list while True: # expression to pass self.compile_expression() # possible comma tt, t = self._token_next(False) if tt == "SYMBOL" and t == ",": self._write(tt, t) self.tokenizer.advance() else: # not a comma; stop processing parameters break self._end_block("expressionList") # compiles a subroutine call # two cases: # - subroutineName(expressionList) # - (class|var).subroutineName(expressionList) def compile_subroutine_call(self): # first part of name tt, t = self._token_next(False, "IDENTIFIER") self._write(tt, t) # a dot and another name may exist, or it could be a parenthesis tt, t = self._token_next(True) if tt == "SYMBOL" and t == ".": self._write(tt, t) # the name after the dot tt, t = self._token_next(True, "IDENTIFIER") self._write(tt, t) # advance so that we are on the parenthesis self.tokenizer.advance() # opening parenthesis tt, t = self._token_next(False, "SYMBOL", "(") self._write(tt, t) # expression list self.tokenizer.advance() self.compile_expression_list() # closing parenthesis tt, t = self._token_next(False, "SYMBOL", ")") self._write(tt, t) self.tokenizer.advance() # returns the token_type and token of the next token after advancing the # tokenizer before reading if advance is True def _token_next(self, advance=False, expected_type=None, expected_value=None): # advance the tokenizer, if requested if advance: self.tokenizer.advance() # get the token type and the token itself token_type = self.tokenizer.token_type() token = str(getattr(self.tokenizer, token_type.lower())()) if expected_type and token_type != expected_type: print "WARNING: Type", token_type, "found; expected", expected_type import traceback, sys traceback.print_stack() sys.exit(1) if expected_value and token != expected_value: print "WARNING: Value", token, "found; expected", expected_value import traceback, sys traceback.print_stack() sys.exit(1) return token_type, token # writes the given token to the output file def _write(self, token_type, token): # lowercase for tag name token_type = token_type.lower() # special types token_type = token_type.replace("int_const", "integerConstant") token_type = token_type.replace("string_const", "stringConstant") # special values to replace for output s = {"<": "<", ">": ">", '"': """, "&": "&"} for s, r in s.iteritems(): token = token.replace(s, r) # print the token type and token to the file output = ['<', token_type, '>', ' ', token, ' ', '</', token_type, '>', '\n'] self.destination_file.write(self._indent("".join(output))) # starts an XML block def _start_block(self, block_name): self.destination_file.write(self._indent("<" + block_name + ">\n")) self.indent += 2 # ends an XML block def _end_block(self, block_name): self.indent -= 2 self.destination_file.write(self._indent("</" + block_name + ">\n")) # indents a single line of text at the current indentation level def _indent(self, text): return " " * self.indent + text
class CompilationEngine: CLASS_VAR_DEC_KEYWORDS = ['static', 'field'] SUBROUTINE_DEC_KEYWORDS = ['constructor', 'function', 'method'] BINARY_OPERATOR_SYMBOLS = ['+', '-', '*', '/', '&', '|', '<', '>', '='] UNARY_OPERATOR_SYMBOLS = ['-', '~'] TYPE_TO_TAG = {'STRING_CONST': 'stringConstant', 'INT_CONST': 'integerConstant', 'KEYWORD': 'keyword', 'IDENTIFIER': 'identifier', 'SYMBOL': 'symbol'} SYMBOLS_TO_XML_CONVENTION = {'<': '<', '>': '>', '&': '&', '"': '"'} def __init__(self, input_file_path, output_file_path): self.output_file = open(output_file_path, 'w') self.jack_tokenizer = JackTokenizer(input_file_path) if self.jack_tokenizer.has_more_tokens(): self.compile_class() def compile_class(self): self.output_file.write('<class>\n') # get first token self.jack_tokenizer.advance() self.write_token(self.jack_tokenizer.key_word()) self.write_token(self.jack_tokenizer.identifier()) self.write_token(self.jack_tokenizer.symbol()) if self.jack_tokenizer.key_word() in CompilationEngine.CLASS_VAR_DEC_KEYWORDS: self.compile_class_var_dec() if self.jack_tokenizer.key_word() in CompilationEngine.SUBROUTINE_DEC_KEYWORDS: self.compile_subroutine() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</class>') self.output_file.close() def write_token(self, token_name): type_tag = CompilationEngine.TYPE_TO_TAG[self.jack_tokenizer.token_type()] self.output_file.write('<{0}> {1} </{0}>\n'.format( type_tag, token_name)) self.jack_tokenizer.advance() def compile_class_var_dec(self): while self.jack_tokenizer.key_word() in CompilationEngine.CLASS_VAR_DEC_KEYWORDS: self.output_file.write('<classVarDec>\n') self.write_token(self.jack_tokenizer.key_word()) self.write_type() self.write_token(self.jack_tokenizer.identifier()) while self.jack_tokenizer.symbol() != ';': self.write_token(self.jack_tokenizer.symbol()) self.write_token(self.jack_tokenizer.identifier()) self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</classVarDec>\n') def write_type(self): if self.jack_tokenizer.token_type() == 'KEYWORD': self.write_token(self.jack_tokenizer.key_word()) elif self.jack_tokenizer.token_type() == 'IDENTIFIER': self.write_token(self.jack_tokenizer.identifier()) def compile_subroutine(self): while self.jack_tokenizer.key_word() in CompilationEngine.SUBROUTINE_DEC_KEYWORDS: self.output_file.write('<subroutineDec>\n') self.write_token(self.jack_tokenizer.key_word()) self.write_type() self.write_token(self.jack_tokenizer.identifier()) self.write_token(self.jack_tokenizer.symbol()) self.compile_parameter_list() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('<subroutineBody>\n') self.write_token(self.jack_tokenizer.symbol()) while self.jack_tokenizer.key_word() == 'var': self.compile_var_dec() self.compile_statements() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</subroutineBody>\n') self.output_file.write('</subroutineDec>\n') def compile_parameter_list(self): self.output_file.write('<parameterList>\n') if self.jack_tokenizer.symbol() != ')': self.write_type() self.write_token(self.jack_tokenizer.identifier()) while self.jack_tokenizer.symbol() == ",": self.write_token(self.jack_tokenizer.symbol()) self.write_type() self.write_token(self.jack_tokenizer.identifier()) self.output_file.write('</parameterList>\n') def compile_var_dec(self): self.output_file.write('<varDec>\n') self.write_token(self.jack_tokenizer.key_word()) self.write_type() self.write_token(self.jack_tokenizer.identifier()) while self.jack_tokenizer.symbol() == ",": self.write_token(self.jack_tokenizer.symbol()) self.write_token(self.jack_tokenizer.identifier()) self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</varDec>\n') def compile_statements(self): self.output_file.write('<statements>\n') while self.jack_tokenizer.token_type() == 'KEYWORD': if self.jack_tokenizer.key_word() == 'let': self.compile_let() elif self.jack_tokenizer.key_word() == 'if': self.compile_if() elif self.jack_tokenizer.key_word() == 'while': self.compile_while() elif self.jack_tokenizer.key_word() == 'do': self.compile_do() elif self.jack_tokenizer.key_word() == 'return': self.compile_return() self.output_file.write('</statements>\n') def compile_do(self): self.output_file.write('<doStatement>\n') self.write_token(self.jack_tokenizer.key_word()) self.write_token(self.jack_tokenizer.identifier()) self.compile_subroutine_call() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</doStatement>\n') def compile_subroutine_call(self): if self.jack_tokenizer.symbol() == '(': self.write_token(self.jack_tokenizer.symbol()) self.compile_expression_list() self.write_token(self.jack_tokenizer.symbol()) elif self.jack_tokenizer.symbol() == '.': self.write_token(self.jack_tokenizer.symbol()) self.write_token(self.jack_tokenizer.identifier()) self.write_token(self.jack_tokenizer.symbol()) self.compile_expression_list() self.write_token(self.jack_tokenizer.symbol()) def compile_let(self): self.output_file.write('<letStatement>\n') self.write_token(self.jack_tokenizer.key_word()) self.write_token(self.jack_tokenizer.identifier()) if self.jack_tokenizer.symbol() == '[': self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</letStatement>\n') def compile_while(self): self.output_file.write('<whileStatement>\n') self.write_token(self.jack_tokenizer.identifier()) self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) self.write_token(self.jack_tokenizer.symbol()) self.compile_statements() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</whileStatement>\n') def compile_return(self): self.output_file.write('<returnStatement>\n') self.write_token(self.jack_tokenizer.key_word()) if self.jack_tokenizer.symbol() != ';': self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</returnStatement>\n') def compile_if(self): self.output_file.write('<ifStatement>\n') self.write_token(self.jack_tokenizer.key_word()) self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) self.write_token(self.jack_tokenizer.symbol()) self.compile_statements() self.write_token(self.jack_tokenizer.symbol()) if self.jack_tokenizer.key_word() == 'else': self.write_token(self.jack_tokenizer.key_word()) self.write_token(self.jack_tokenizer.symbol()) self.compile_statements() self.write_token(self.jack_tokenizer.symbol()) self.output_file.write('</ifStatement>\n') def compile_expression(self): self.output_file.write('<expression>\n') self.compile_term() while self.jack_tokenizer.symbol() in CompilationEngine.BINARY_OPERATOR_SYMBOLS: symbol = self.jack_tokenizer.symbol() if symbol in CompilationEngine.SYMBOLS_TO_XML_CONVENTION: symbol = CompilationEngine.SYMBOLS_TO_XML_CONVENTION[symbol] self.write_token(symbol) self.compile_term() self.output_file.write('</expression>\n') def compile_term(self): self.output_file.write('<term>\n') token_type = self.jack_tokenizer.token_type() if token_type == 'IDENTIFIER': self.write_token(self.jack_tokenizer.identifier()) if self.jack_tokenizer.symbol() == '[': self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) elif self.jack_tokenizer.symbol() == '(' or self.jack_tokenizer.symbol() == '.': self.compile_subroutine_call() elif token_type == 'STRING_CONST': self.write_token(self.jack_tokenizer.string_val()) elif token_type == 'INT_CONST': self.write_token(self.jack_tokenizer.int_val()) elif token_type == 'KEYWORD': self.write_token(self.jack_tokenizer.key_word()) elif token_type == 'INT_CONST': self.write_token(self.jack_tokenizer.int_val()) elif token_type == 'SYMBOL': if self.jack_tokenizer.symbol() == '(': self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.write_token(self.jack_tokenizer.symbol()) elif self.jack_tokenizer.symbol() in CompilationEngine.UNARY_OPERATOR_SYMBOLS: self.write_token(self.jack_tokenizer.symbol()) self.compile_term() self.output_file.write('</term>\n') def compile_expression_list(self): self.output_file.write('<expressionList>\n') if self.jack_tokenizer.symbol() != ')': self.compile_expression() while self.jack_tokenizer.symbol() == ',': self.write_token(self.jack_tokenizer.symbol()) self.compile_expression() self.output_file.write('</expressionList>\n')