def restore_tokens(self, file: _io.BytesIO): self.tokens.clear() while True: flag = int.from_bytes(file.read(1), "big") if flag == 0: self.tokens.append(stl.Token((stl.EOF, None))) break else: line = int(stl.read_string(file)) file_name = stl.read_string(file) lf = line, file_name if flag == 1: token: stl.NumToken = stl.NumToken(lf, stl.read_string(file)) elif flag == 2: token: stl.LiteralToken = stl.LiteralToken( lf, stl.read_string(file)) elif flag == 3: token: stl.IdToken = stl.IdToken(lf, stl.read_string(file)) elif flag == 4: token: stl.DocToken = stl.DocToken(lf, stl.read_string(file)) else: raise stl.ParseException("Unknown flag: {}".format(flag)) self.tokens.append(token)
def tokenize_text(self, lines): doc = "" in_doc = False for i in range(len(lines)): line_num = i + 1 tup = (line_num, self.file_name) line = lines[i] last_index = len(self.tokens) in_doc, doc = self.proceed_line(line, tup, in_doc, doc) self.find_import(last_index, len(self.tokens)) self.tokens.append(stl.Token((stl.EOF, None)))
def tokenize_file(self, file: _io.TextIOWrapper): """ :param file: :return: """ line = file.readline() line_num = 1 in_doc = False doc = "" while line: tup = (line_num, self.file_name) last_index = len(self.tokens) in_doc, doc = self.proceed_line(line, tup, in_doc, doc) self.find_include(last_index, len(self.tokens)) line = file.readline() line_num += 1 self.tokens.append(stl.Token((stl.EOF, None)))