def is_assignment(statement): tokens = statement.split() if not len(tokens) == 4: return False if not tokens[-1] == ';': return False if not lexer(tokens[0]) == 'IDENTIFIER': return False if not tokens[1] == '=': return False if lexer(tokens[2]) not in ['IDENTIFIER', 'INTEGER', 'BOOLEAN']: return False return True
def instruction_assignment(statement): tokens = statement.split() VAL = tokens[2] ID = tokens[0] inst_type = 'PUSHM' if lexer(VAL) == 'IDENTIFIER' else 'PUSHI' instructions.append(create_instruction(instruction=inst_type, value=VAL)) instructions.append(create_instruction(instruction='POPM', value=ID))
def is_expression(statement): tokens = statement.split() if not tokens[-1] == ';': return False left = tokens[0] if not lexer(left) =='IDENTIFIER' and not lexer(left) == 'INTEGER': return False operator = tokens[1] if operator not in OPERATORS: return False right = tokens[2:] if len(right) > 2: return is_expression(' '.join(right)) else: return lexer(right[0]) == 'INTEGER' or lexer(right[0]) == 'IDENTIFIER'
def __init__(self, sourceText, outputFile, verbose=False): self.token = "" self.ast = "" self.indent = 0 self.lexer = lexer(sourceText, verbose) self.out = outputFile self.verbose = verbose # Set of executing instructions self.executing = set()
def Factor(ID): if lexer(TOKEN) == 'IDENTIFIER': MEM = get_mem_loc(TOKEN) if MEM == -1: raise Exception( "Identifier '{}' referenced before assignment!".format(TOKEN)) create_instruction('PUSHM', MEM) else: value = transform_value(TOKEN) create_instruction('PUSHI', value) next_token()
def add_symbol(TYPE): # Check if valid ID if not lexer(TOKEN) == 'IDENTIFIER': raise Exception("Invalid Identifier on TOKEN #{} ({})".format( ADDR, TOKEN)) # Check if already exists for symbol in SYMBOL_TABLE: if symbol.ID == TOKEN: raise Exception( "Identifier {} already previously declared.".format(TOKEN)) SYMBOL_TABLE.append(Symbol(TOKEN, BASE_MEMLOC + len(SYMBOL_TABLE), TYPE))
def __init__(self, file_name): self.file_name = file_name self.lex = lexer(self.file_name) self.lex.start() self.lex_tokens = self.lex.get_tokens() self.tokens = list(self.lex_tokens) self.current = -1 self.exprend = [')', '(', ';', '||', '&&', '<', '>', '!', '=='] self.declared = {} self.fill_declared() self.optimize() self.label_count = 0
def Assignment(): global ADDR global TOKEN if lexer(TOKEN) == 'IDENTIFIER': ID = TOKEN next_token() if TOKEN == '=': next_token() Expression(ID) create_instruction('POPM', ID) else: raise Exception("Expected IDENTIFIER on TOKEN #{} ({})".format( ADDR, TOKEN))
def transform_value(value): if value == 'true': return 1 if value == 'false': return 0 if lexer(value) == 'IDENTIFIER': MEM = get_mem_loc(value) if MEM == -1: raise Exception('Identifier {} referenced before assignment!'.format(value)) else: return symbol_table[MEM - BASE_MEMLOC].VAL try: return int(value) except: raise Exception('Invalid value {}. Expected integer, boolean, or previously assigned identifier!'.format(value))
def is_declaration(statement): token_tuples = [] for token in statement.split(): try: tup = token, lexer(token) except: return False token_tuples.append( tup ) if token_tuples[0][0] not in DATATYPES or not token_tuples[-1][0] == ';' or len(token_tuples) < 3: return False for idx, token_tuple in enumerate(token_tuples[1:-1]): if idx % 2 == 0: if not token_tuple[1] == 'IDENTIFIER': return False else: if not token_tuple[0] == ',': return False return True
from Lexer import lexer from Parser import parser from Generator import generator from Optimizer import optimizer rules = [(r'\b[X]{1,3}[V]?[I]{1,3}\b|\b[X]{1,3}[V]?\b|' + r'\b[X]{0,3}[I][XV]\b|\b[V][I]{0,3}\b|\b[I]{1,3}\b', 'ROMAN_DIGIT'), (r'\bwhile\b', 'LOOP'), (r'\bdone\b', 'ENDLOOP'), (r'[A-Za-z][A-Za-z0-9_]*', 'VAR'), (r'[0-9]*\.[0-9]*', 'FLOAT'), (r'[1-9][0-9]*|[0]', 'INT'), (r'\<', 'LESS'), (r'\=', 'EQ'), (r'\>', 'LARG'), (r'\*', 'MPY'), (r'\/', 'DIV'), (r'\+', 'ADD'), (r'\-', 'SUB'), (r':=', 'ASSIGN'), (r'[\(\)]', 'GROUP'), (r'\;', 'END_EXPR'), (r'[\^\&\%\:\#\@\!\~\`\'\"\$]*', 'UNKNOWN')] if __name__ == "__main__": with open('input.txt') as file: source_code = file.read() id_table, token_list = lexer(source_code, rules) print(id_table) try: ast = parser(token_list) print('Дерево разбора') print(ast) object_code = generator(ast, id_table) object_code = optimizer(object_code) with open('object_code.txt', 'w') as file: file.write('\n\n'.join( [''.join(command) for command in object_code])) except SyntaxError as error: print(error)
def leerCadena(src): tokens = lexer(src) tokens.append(('$', 'FIN DE CADENA')) return tokens
def create_instruction(instruction, value=None): if instruction == 'PUSHI': #Pushes the {Integer Value} onto the Top of the Stack (TOS) value = transform_value(value) STACK.append(value) elif instruction == 'PUSHM': # Pushes the value stored at {ML} onto TOS if lexer(value) == 'IDENTIFIER': value = get_mem_loc(value) VAL = SYMBOL_TABLE[value - BASE_MEMLOC].VAL if VAL is None: raise Exception( "Error: Identifier '{}' was never initialized!".format( get_ID(value))) STACK.append(VAL) elif instruction == 'POPM': # Pops the value from the top of the stack and stores it at {ML} if lexer(value) == 'IDENTIFIER': mem = get_mem_loc(value) if mem == -1: raise Exception( "Identifier '{}' referenced before assignment!".format( value)) else: mem = value set_mem_loc(mem, STACK.pop(-1)) value = mem elif instruction == 'STDOUT': STACK.pop(-1) value = '' elif instruction == 'STDIN': STACK.append(value) value = '' elif instruction == 'ADD': first = STACK.pop(-1) second = STACK.pop(-1) STACK.append(transform_value(first) + transform_value(second)) value = '' elif instruction == 'SUB': first = STACK.pop(-1) second = STACK.pop(-1) STACK.append(transform_value(second) - transform_value(first)) value = '' elif instruction == 'MUL': first = STACK.pop(-1) second = STACK.pop(-1) STACK.append(transform_value(second) * transform_value(first)) value = '' elif instruction == 'DIV': first = STACK.pop(-1) second = STACK.pop(-1) STACK.append(transform_value(second) / transform_value(first)) value = '' elif instruction == 'GRT': #Pops two items from the stack and pushes 1 onto TOS if second item is larger otherwise push 0 first = STACK.pop(-1) second = STACK.pop(-1) result = 1 if second > first else 0 STACK.append(result) elif instruction == 'LES': # Pops two items from the stack and pushes 1 onto TOS if the second item is smaller than first item otherwise push 0 first = STACK.pop(-1) second = STACK.pop(-1) result = 1 if second < first else 0 STACK.append(result) elif instruction == 'EQU': # Pops two items from the stack and pushes 1 onto TOS if they are equal otherwise push 0 first = STACK.pop(-1) second = STACK.pop(-1) result = 1 if first == second else 0 STACK.append(result) elif instruction == 'NEQ': # Pops two items from the stack and pushes 1 onto TOS if they are not equal otherwise push 0 first = STACK.pop(-1) second = STACK.pop(-1) result = 0 if first == second else 1 STACK.append(result) elif instruction == 'GEQ': # Pops two items from the stack and pushes 1 onto TOS if second item is larger or equal, otherwise push 0 first = STACK.pop(-1) second = STACK.pop(-1) result = 1 if second >= first else 0 STACK.append(result) elif instruction == 'LEQ': #Pops two items from the stack and pushes 1 onto TOS if second item is less or equal, otherwise push 0 first = STACK.pop(-1) second = STACK.pop(-1) result = 1 if second <= first else 0 STACK.append(result) elif instruction == 'JUMPZ': #{IL - Instruction Location} Pop the stack and if the value is 0 then jump to {IL} # TOS = STACK.pop(-1) # if not TOS == 0: # return JUMPSTACK.append(len(INSTRUCTIONS_TABLE)) elif instruction == 'LABEL': LABEL_QUEUE.append(len(INSTRUCTIONS_TABLE) + 1) value = '' elif instruction == 'JUMP': JUMPSTACK.append(len(INSTRUCTIONS_TABLE)) value = '' else: raise Exception("Instruction '{}' not recognized!".format(instruction)) num = str(len(INSTRUCTIONS_TABLE) + 1).ljust(4, ' ') instruction = str(instruction).ljust(8, ' ') instruction_row = Instruction(num, instruction, value) INSTRUCTIONS_TABLE.append(instruction_row)
def table_add(statement): tokens = statement.split() tuple_type = tokens[0] for ID in [token for token in tokens if lexer(token) == 'IDENTIFIER']: symbol = Symbol(ID, BASE_MEMLOC + len(symbol_table), tuple_type) symbol_table.append(symbol)
from Lexer import lexer, Token from Execute import ProgramActions, AST_to_actions from Enums import Errornr from Parser import parse import time if __name__ == '__main__': lexer_list, error = lexer("main.txt") if (error.nr == Errornr.NO_ERROR): tree, pv = parse(lexer_list) if (len(pv.error_list) > 0): print(pv.error_list[0]) elif (len(pv.unprocessedTokens) > 0): print("Error, the characters on line", pv.unprocessedTokens[0].linenr, " could not be processed") else: time.sleep(1) exec = ProgramActions() result, error = AST_to_actions(exec, tree) if (error.nr != Errornr.NO_ERROR): print(error) else: print("Whoops: ") print(error)
def getTokenTypes(): tokens = lexer(input) tokens.append(("#", "FinDeCadena")) return tokens