def prepare(l): for tp, val in tokenize(l): if tp == "comment": pass elif tp == "special": yield (val, val) else: yield (tp, val)
def parse(self, input_str): pointer = 0 scanner_errors = defaultdict(list) parser_errors = defaultdict(list) semantic_errors = defaultdict(list) number_of_failure = 0 while pointer < len(input_str): line_number = input_str[0:pointer].count('\n') try: token_type, token = tokenize(input_str, pointer) if token_type in ['W', 'COMMENT']: pointer += len(token) continue terminal = token if token_type in ['SYMBOL', 'KEYWORD'] \ else token_type try: while True: try: if self.diagram.move_forward(terminal, token): break except ParserExceptionWithoutSkip as e: parser_errors[line_number].append(e.message) number_of_failure = number_of_failure + 1 except SemanticException as e: semantic_errors[line_number].append(e.message) self.diagram.intermediate_code_generator.is_ok = False number_of_failure = 0 except ParserException as e: parser_errors[line_number].append(e.message) number_of_failure = number_of_failure + 1 except ScannerException as e: token = e.message scanner_errors[line_number].append(token) pointer += len(token) try: while not self.diagram.move_forward('$', '$'): pass except SemanticException as e: line_number = input_str.count('\n') semantic_errors[line_number].append(e.message) self.diagram.intermediate_code_generator.is_ok = False except ParserException: line_number = input_str.count('\n') if number_of_failure > 0: parser_errors[line_number].append( "Syntax Error! Unexpected EndOfFile") else: parser_errors[line_number].append( "Syntax Error! Malformed Input") return self.diagram.intermediate_code_generator.program_block, scanner_errors, parser_errors, semantic_errors
def run_scanner(code, output_file): err, tokens = scanner.tokenize(code) for t in tokens: if t.group == "OP_PUNCTUATION" or t.group == "KEYWORD": output_file.write(t.value + "\n") else: output_file.write(t.group + " " + t.value + "\n") if err is not None: output_file.write(err + "\n") output_file.close()
def main(argv): with open(argv[1]) as f: sexprs = parse(tokenize(f)) print sexprs
def compile(code): tokens = tokenize(code) parser = Parser() parse_tree = parser.parse(tokens) validate(parse_tree) return generate_code(parse_tree)
def main(argv): with open(argv[1]) as f: code = compile(parse(tokenize(f))) pprint(code.code)