def _read_from(self:object, tr:TextReader, **options) -> ReadResult:
if self.mode == "first":
for th in self.thseq:
rslt = tr.read_thing(th, **options)
if rslt.is_fullmatch():
return rslt
return Nomatch([], self)
elif self.mode == "longer":
maxl = -1
mid = -1
i = 0
for th in self.thseq:
rslt = tr.read_thing(th, **options)
if rslt.is_fullmatch():
l = len(thing_as_string(rslt.readedlist))
if l > maxl:
maxl = l
mid = i
tr.push_back(rslt.readedlist)
i += 1
if mid < 0:
return Nomatch([], self)
else:
rslt = tr.read_thing(self.thseq[mid])
return rslt
def _read_from(self:object, tr : TextReader, **options) -> ReadResult:
escaped = False
while True:
for th in self.thseq:
rslt = tr.read_thing(th, **options)
if rslt.is_fullmatch():
tr.push_back(rslt.readedlist)
if not escaped:
return Nomatch()
rslt = tr.read_next(**options)
if rslt.is_fullmatch():
if escaped:
escaped = False
return rslt
else:
if self.allow_escaped and self.escape_char == rslt.readedlist[0]:
escaped = True
continue
else:
return rslt
else:
if escaped:
tr.push_back(self.escape_char)
rslt.readed_object = self
return rslt
return Nomatch([], self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult:
rslt = tr.read_thing(self.thing, **options)
if rslt.is_fullmatch():
tr.push_back(rslt.readedlist)
rslt.readedlist = []
rslt.readed_object = self
return rslt
def _read_from(self:object, tr:TextReader, **options) -> ReadResult:
acc = ""
for th in self.thseq:
rslt = tr.read_thing(th, **options)
if rslt.is_fullmatch():
acc += thing_as_string(rslt.readedlist)
else:
tr.push_back(acc)
return ReadResult(rslt.state, acc, self)
break
return Fullmatch(acc, self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult:
rslt = Fullmatch([], self)
acc = []
for th in self.thseq:
rslt = tr.read_thing(th, **options)
if rslt.is_fullmatch():
acc.extend(rslt.readedlist)
else:
tr.push_back(acc)
break
#print(repr(rslt))
if rslt.is_fullmatch():
return Fullmatch(acc, self)
else:
return ReadResult(rslt.state, acc, self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult:
rslt = Fullmatch([],self)
acc = []
n = 0
while True:
if self.max_num >= 0 and n >= self.max_num:
break
rslt = tr.read_thing(self.thing, **options)
if rslt.is_fullmatch():
acc.extend(rslt.readedlist)
else:
break
n += 1
if not rslt.is_fullmatch():
if n >= self.min_num:
return Fullmatch(acc, self)
else:
tr.push_back(acc)
return ReadResult(rslt.state, acc, self)
else:
return Fullmatch(acc, self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult:
n_chars = 0
acc = ""
if 'char' == self.mode:
while True:
if self.max_chars >= 0 and n_chars >= self.max_chars:
break
ch = tr.read_next(**options)
if not ch.is_fullmatch():
break
if None is not re.match(self.regexp, thing_as_string(ch.readedlist)):
acc += thing_as_string(ch.readedlist)
else:
tr.push_back(ch.readedlist)
break
n_chars += 1
elif 'string' == self.mode:
acc = ""
while True:
if self.max_chars >= 0 and n_chars >= self.max_chars:
break
ch = tr.read_next(**options)
if not ch.is_fullmatch():
break
if None is re.match(self.regexp, acc + thing_as_string(ch.readedlist)):
tr.push_back(ch.readedlist)
break
else:
acc += thing_as_string(ch.readedlist)
n_chars += 1
else:
return Nomatch([], self)
if n_chars >= self.min_chars:
return Fullmatch(thing_as_string(acc), self)
else:
tr.push_back(acc)
return Nomatch(thing_as_string(acc), self)
class TokenIterator(BufferedIterator):
def __init__(self, tr:TextReader=None, token_types=None, tokens_skip=None
, mode="first", n_prelook=25):
self.text_reader = tr
if isinstance(self.text_reader, BufferedIterator):
self.text_reader = TextReader(self.text_reader)
self.token_types = _or(token_types, [])
self.tokens_skip = _or(tokens_skip, [])
self.mode = mode # "first" or "longer"
self.n_prelook = n_prelook
self.buffer, self.backbuffer = deque([]), deque([])
self.at_end = False
def __iter__(self):
return self
def __next__(self):
if self.at_end:
raise StopIteration
ret = TOK_EOF
try:
ret = self.buffer.popleft()
except IndexError:
self.refillbuffer()
try:
ret = self.buffer.popleft()
except IndexError:
self.at_end = True
raise StopIteration
else:
self.backbuffer.append(ret)
else:
self.backbuffer.append(ret)
if ret == TOK_EOF:
self.at_end = True
if ret in self.tokens_skip:
ret = next(self)
return ret
def re_iter(self):
self.text_reader.inp_buf_iter.re_iter()
return self
def refillbuffer(self):
for n in range(self.n_prelook):
if self.mode == "first":
for tt in self.token_types:
rslt = self.text_reader.read_thing(tt)
if rslt.is_fullmatch():
self.push_forward(rslt.readedlist[0])
break
elif self.mode == "longer":
ml = -1
rt = None
for tt in self.token_types:
rslt = self.text_reader.read_thing(tt)
if rslt.is_fullmatch():
tok = rslt.readedlist[0]
tl = len(tok.string)
if tl > ml:
ml = tl
rt = tok
self.text_reader.push_back(rslt.readedlist)
if None is not rt:
rslt = self.text_reader.read_thing(rt)
if rslt.is_fullmatch():
self.push_forward(rslt.readedlist[0])
else:
break
return self
def push_forward(self, el=None):
if el is not None:
if isinstance(el, str):
tok = string_to_tok_by_type(el, self.token_types)
if None is tok:
tok = Token(el)
self.push_forward(tok)
elif isinstance(el, Iterable):
for e in el:
self.push_forward(e)
else:
self.at_end = False
self.buffer.append(el)
return self
def push_back(self, el=None):
if el is not None:
if isinstance(el, str):
tok = string_to_tok_by_type(el, self.token_types)
if None is tok:
tok = Token(el)
self.push_back(tok)
elif isinstance(el, Iterable):
seq = reversed(list(el))
for e in seq:
self.push_back(e)
else:
self.at_end = False
be = self.backbuffer.pop()
if el != be:
self.backbuffer.append(be)
self.buffer.appendleft(el)
return self
