def _read_from(self:object, tr:TextReader, **options) -> ReadResult: if self.mode == "first": for th in self.thseq: rslt = tr.read_thing(th, **options) if rslt.is_fullmatch(): return rslt return Nomatch([], self) elif self.mode == "longer": maxl = -1 mid = -1 i = 0 for th in self.thseq: rslt = tr.read_thing(th, **options) if rslt.is_fullmatch(): l = len(thing_as_string(rslt.readedlist)) if l > maxl: maxl = l mid = i tr.push_back(rslt.readedlist) i += 1 if mid < 0: return Nomatch([], self) else: rslt = tr.read_thing(self.thseq[mid]) return rslt
def string_to_tok_by_type(string=None, tok_types=None, mode="first" #or "longer" , only_fullmatch=True): string = _or(string, "") tok_types = _or(tok_types, []) if mode == "first": tr = TextReader(string) for tt in tok_types: rslt = tr.read_thing(tt) if rslt.is_fullmatch(): return rslt.readedlist[0] #tr.inp_buf_iter.re_iter() return None elif mode == "longer": ml = -1 rt = None tr = TextReader(string) for tt in tok_types: rslt = tr.read_thing(tt) if rslt.is_fullmatch(): tok = rslt.readedlist[0] tl = len(tok.string) if tl > ml: ml = tl rt = tok tr.inp_buf_iter.re_iter() return rt else: return None
def _read_from(self:object, tr : TextReader, **options) -> ReadResult: escaped = False while True: for th in self.thseq: rslt = tr.read_thing(th, **options) if rslt.is_fullmatch(): tr.push_back(rslt.readedlist) if not escaped: return Nomatch() rslt = tr.read_next(**options) if rslt.is_fullmatch(): if escaped: escaped = False return rslt else: if self.allow_escaped and self.escape_char == rslt.readedlist[0]: escaped = True continue else: return rslt else: if escaped: tr.push_back(self.escape_char) rslt.readed_object = self return rslt return Nomatch([], self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: rslt = tr.read_thing(self.thing, **options) if rslt.is_fullmatch(): tr.push_back(rslt.readedlist) rslt.readedlist = [] rslt.readed_object = self return rslt
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: acc = "" for th in self.thseq: rslt = tr.read_thing(th, **options) if rslt.is_fullmatch(): acc += thing_as_string(rslt.readedlist) else: tr.push_back(acc) return ReadResult(rslt.state, acc, self) break return Fullmatch(acc, self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: rslt = tr.read_thing(self.thing, **options) if rslt.is_fullmatch(): flat = _or(get_from_nested_dict(options, self.name + ".read_from", "flat") , self.flat) skip = _or(get_from_nested_dict(options, self.name + ".read_from", "skip") , self.skip) # set_to_nested_dict(options, None, "BIReadable.read_from", "flat") # set_to_nested_dict(options, None, "BIReadable.read_from", "skip") if not flat: rslt.readedlist = [ParseNode(self.name, rslt.readedlist, skip=skip, flat_eq_name=self.flat_eq_name , parent=None, Type=self.type , priority=self.priority)] return rslt
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: rslt = Fullmatch([], self) acc = [] for th in self.thseq: rslt = tr.read_thing(th, **options) if rslt.is_fullmatch(): acc.extend(rslt.readedlist) else: tr.push_back(acc) break #print(repr(rslt)) if rslt.is_fullmatch(): return Fullmatch(acc, self) else: return ReadResult(rslt.state, acc, self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: rslt = Fullmatch([],self) acc = [] n = 0 while True: if self.max_num >= 0 and n >= self.max_num: break rslt = tr.read_thing(self.thing, **options) if rslt.is_fullmatch(): acc.extend(rslt.readedlist) else: break n += 1 if not rslt.is_fullmatch(): if n >= self.min_num: return Fullmatch(acc, self) else: tr.push_back(acc) return ReadResult(rslt.state, acc, self) else: return Fullmatch(acc, self)
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: return tr.read_thing(self.unref(), **merge_nested_dicts(self.opts, options))
def _read_from(self:object, tr:TextReader, **options) -> ReadResult: return tr.read_thing(self.string, **options)
class TokenIterator(BufferedIterator): def __init__(self, tr:TextReader=None, token_types=None, tokens_skip=None , mode="first", n_prelook=25): self.text_reader = tr if isinstance(self.text_reader, BufferedIterator): self.text_reader = TextReader(self.text_reader) self.token_types = _or(token_types, []) self.tokens_skip = _or(tokens_skip, []) self.mode = mode # "first" or "longer" self.n_prelook = n_prelook self.buffer, self.backbuffer = deque([]), deque([]) self.at_end = False def __iter__(self): return self def __next__(self): if self.at_end: raise StopIteration ret = TOK_EOF try: ret = self.buffer.popleft() except IndexError: self.refillbuffer() try: ret = self.buffer.popleft() except IndexError: self.at_end = True raise StopIteration else: self.backbuffer.append(ret) else: self.backbuffer.append(ret) if ret == TOK_EOF: self.at_end = True if ret in self.tokens_skip: ret = next(self) return ret def re_iter(self): self.text_reader.inp_buf_iter.re_iter() return self def refillbuffer(self): for n in range(self.n_prelook): if self.mode == "first": for tt in self.token_types: rslt = self.text_reader.read_thing(tt) if rslt.is_fullmatch(): self.push_forward(rslt.readedlist[0]) break elif self.mode == "longer": ml = -1 rt = None for tt in self.token_types: rslt = self.text_reader.read_thing(tt) if rslt.is_fullmatch(): tok = rslt.readedlist[0] tl = len(tok.string) if tl > ml: ml = tl rt = tok self.text_reader.push_back(rslt.readedlist) if None is not rt: rslt = self.text_reader.read_thing(rt) if rslt.is_fullmatch(): self.push_forward(rslt.readedlist[0]) else: break return self def push_forward(self, el=None): if el is not None: if isinstance(el, str): tok = string_to_tok_by_type(el, self.token_types) if None is tok: tok = Token(el) self.push_forward(tok) elif isinstance(el, Iterable): for e in el: self.push_forward(e) else: self.at_end = False self.buffer.append(el) return self def push_back(self, el=None): if el is not None: if isinstance(el, str): tok = string_to_tok_by_type(el, self.token_types) if None is tok: tok = Token(el) self.push_back(tok) elif isinstance(el, Iterable): seq = reversed(list(el)) for e in seq: self.push_back(e) else: self.at_end = False be = self.backbuffer.pop() if el != be: self.backbuffer.append(be) self.buffer.appendleft(el) return self