def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get("as_dict", False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable( izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get('as_dict', False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def get_request_iterator(self): request_iterator = self.iteration_scheme.get_request_iterator() return chain.from_iterable(imap(partial(repeat, times=self.times), request_iterator))
def open(self): return chain.from_iterable(izip(*[chain.from_iterable( imap(open, repeat(f))) for f in self.files]))
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable( izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) return partition(len(self.sources), chain.from_iterable(batches))