def __getitem__(self, key): if isinstance(key, numbers.Integral): return sent.CompoundSentence(sents=[b[key] for b in self.batches]) else: assert isinstance(key, slice) sel_batches = [b[key] for b in self.batches] return CompoundBatch(sel_batches)
def read_sents(self, filename: Union[str,Sequence[str]], filter_ids: Sequence[numbers.Integral] = None) \ -> Iterator[sent.Sentence]: if isinstance(filename, str): filename = [filename] * len(self.readers) generators = [reader.read_sents(filename=cur_filename, filter_ids=filter_ids) for (reader, cur_filename) in zip(self.readers, filename)] while True: try: sub_sents = tuple([next(gen) for gen in generators]) yield sent.CompoundSentence(sents=sub_sents) except StopIteration: return
def __iter__(self): for i in range(self.batch_size()): yield sent.CompoundSentence(sents=[b[i] for b in self.batches])