def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get("as_dict", False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def get_data(self, request=None): if request is not None: raise ValueError data = next(self.child_epoch_iterator) return [d for d, s in izip(data, self.data_stream.sources) if s in self.sources]
def get_data_from_batch(self, request=None): if not self.data: data = next(self.child_epoch_iterator) self.data = izip(*data) try: return next(self.data) except StopIteration: self.data = None return self.get_data()
def get_data(self, request=None): if request is not None: raise ValueError if not self.data: data = next(self.child_epoch_iterator) self.data = izip(*data) try: return next(self.data) except StopIteration: self.data = None return self.get_data()
def transform_any(self, data): return [d for d, s in izip(data, self.data_stream.sources) if s in self.sources]
def open(self): iterators = [iter_(channel) for channel in self.iterables] return izip(*iterators)
def open(self): return chain.from_iterable(izip(*[chain.from_iterable( imap(open, repeat(f))) for f in self.files]))
def _get_src_sentences_with_references(stream): iterator = _get_iterator(stream) references = [] n = next(iterator) references.extend(izip(*n[2:])) return list(n[:2]), references
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable( izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) return partition(len(self.sources), chain.from_iterable(batches))
def transform_any(self, data): return [ d for d, s in izip(data, self.data_stream.sources) if s in self.sources ]