def skip_white_lines(stream:CharacterStream, readtable:Readtable): while not stream.next_is_EOF(): seq, properties = readtable.probe(stream) seq_type = properties.type if seq_type == RT.NEWLINE: continue if seq_type != RT.WHITESPACE: stream.unread_seq(seq) return
def read_and_concatenate_constituent_sequences_ignore_isolation(stream:CharacterStream, readtable:Readtable): concatenation = "" while not stream.next_is_EOF(): seq, properties = readtable.probe(stream) if properties.type == RT.CONSTITUENT or properties.type == RT.ISOLATED_CONSTITUENT: concatenation += seq else: stream.unread_seq(seq) return concatenation return concatenation
def __init__(self, parent:CharacterStream): CharacterStream.__init__(self, parent.name) # The stream from which we obtain the (absolute) characters. self._parent = parent # Keeps the first StreamPosition_ of the various indentation levels self._stack = [parent.copy_position()] # Keeps the StreamPosition_ of the current character, relative to the position in the top of the stack self.current_relative_position = StreamPosition(self.name, 0, 1, 1, 1) # Remembers whether the substream has finished reading self.relative_eof = False
def probe(self, stream:CharacterStream): """ Reads a stream one character at a time, as long as the read sequence of characters is an entry of the readtable. :param stream: The stream to be probed. :return: A pair with the largest possible such sequence and the associated properties. """ node = self.root_node # read first char char = stream.read() if char not in node: return char, self.default_properties else: node, properties = node[char] # if char can be continued to something appearing in the readtable: seq = char ret_seq, ret_properties = char, properties if properties is not None else self.default_properties while not stream.next_is_EOF(): char = stream.read() if char in node: seq += char node, properties = node[char] if properties is not None: ret_seq, ret_properties = seq, properties else: stream.unread() break if len(ret_seq) < len(seq): stream.unread(len(seq) - len(ret_seq)) return ret_seq, ret_properties
def __init__(self, string, name="<string>"): CharacterStream.__init__(self, name) self.string = string self.current_stream_position = StreamPosition(self.name, 0, 1, 1, 1) self.stream_positions = [self.current_stream_position] self.eof = False