示例#1
0
def fileToString(fileName, options=None):
    """
    takes a file and converts it into a string while replacing newline character with a space. Adds a EOF symbol at the end
    """
    file_content = ""
    with open(fileName, 'r') as file:
        for line in file:
            if scanner.isComment(line):
                continue
            else:
                file_content += line.replace('\n', ' ')
    file_content += EOF
    return file_content
示例#2
0
def parseFile(fileName):
    with open(fileName, 'r') as source_file:
        line_number = 1
        for line in source_file:
            lexeme_number = 1
            if scanner.isComment(line):
                comments.append(line)
                continue
            words = parseWords(line)
            try:
                for word in words:
                    t = (scanner.getToken(word))
                    if t is not None:
                        symbol_table.append(t)
                        lexeme_number += 1
                    else:
                        not_tokens.append(word)
                line_number += 1
            except ValueError as e:
                print ("error in line: " + str(line_number))
                print ("line content: " + line)
                print e
示例#3
0
def tokenize(files, options):
    for fileName in files:
        with open(fileName, 'r') as source_file:
            line_number = 1
            for line in source_file:
                lexeme_number = 1
                if scanner.isComment(line):
                    comments.append(line)
                    continue
                words = parseWords(line)
                try:
                    for word in words:
                        t = (scanner.getToken(word))
                        if t is not None:
                            if t.Word not in symbol_table:
                                symbol_table[t.Word] = t
                            lexeme_number += 1
                        else:
                            not_tokens.append(word)
                            raise TokenizerException(word, line, line_number)
                    line_number += 1
                except ValueError as e:
                    print ("Tokenizer error in line: " + str(line_number))
                    print ("line content: " + line)
                    print e

                except TokenizerException as e:
                    print e
                    if optionTokenizeAll not in options:
                        raise e

    if(optionPrintTokens in options):
        printTokens()
        printNotTokens()


    return symbol_table