def probe(self, stream:CharacterStream):
"""
Reads a stream one character at a time, as long as the read sequence of characters is an entry of the readtable.
:param stream: The stream to be probed.
:return: A pair with the largest possible such sequence and the associated properties.
"""
node = self.root_node
# read first char
char = stream.read()
if char not in node:
return char, self.default_properties
else:
node, properties = node[char]
# if char can be continued to something appearing in the readtable:
seq = char
ret_seq, ret_properties = char, properties if properties is not None else self.default_properties
while not stream.next_is_EOF():
char = stream.read()
if char in node:
seq += char
node, properties = node[char]
if properties is not None:
ret_seq, ret_properties = seq, properties
else:
stream.unread()
break
if len(ret_seq) < len(seq):
stream.unread(len(seq) - len(ret_seq))
return ret_seq, ret_properties
def skip_white_lines(stream:CharacterStream, readtable:Readtable):
while not stream.next_is_EOF():
seq, properties = readtable.probe(stream)
seq_type = properties.type
if seq_type == RT.NEWLINE:
continue
if seq_type != RT.WHITESPACE:
stream.unread_seq(seq)
return
def read_and_concatenate_constituent_sequences_ignore_isolation(stream:CharacterStream, readtable:Readtable):
concatenation = ""
while not stream.next_is_EOF():
seq, properties = readtable.probe(stream)
if properties.type == RT.CONSTITUENT or properties.type == RT.ISOLATED_CONSTITUENT:
concatenation += seq
else:
stream.unread_seq(seq)
return concatenation
return concatenation