def get_request_iterator(self): indices = list(range(self.num_examples)) self.rng.shuffle(indices) return imap( list, imap(islice, repeat(_iter(indices), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): if self.times: return repeat(self.batch_size, self.times) if self.num_examples: d, r = divmod(self.num_examples, self.batch_size) return chain(repeat(self.batch_size, d), [r] if r else []) return repeat(self.batch_size)
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get("as_dict", False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable( izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get('as_dict', False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def test_num_examples(): assert_raises(ValueError, IterableDataset, {'features': range(10), 'targets': range(7)}) dataset = IterableDataset({'features': range(7), 'targets': range(7)}) assert dataset.num_examples == 7 dataset = IterableDataset(repeat(1)) assert numpy.isnan(dataset.num_examples) x = numpy.random.rand(5, 3) y = numpy.random.rand(5, 4) dataset = IndexableDataset({'features': x, 'targets': y}) assert dataset.num_examples == 5 assert_raises(ValueError, IndexableDataset, {'features': x, 'targets': y[:4]})
def open(self): return chain.from_iterable(izip(*[chain.from_iterable( imap(open, repeat(f))) for f in self.files]))
def get_request_iterator(self): indices = list(range(self.num_examples)) self.rng.shuffle(indices) return imap(list, imap( islice, repeat(iter_(indices), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): return imap(list, imap( islice, repeat(xrange(self.num_examples), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): return imap( list, imap(islice, repeat(_iter(xrange(self.num_examples)), self.num_batches), repeat(self.batch_size, self.num_batches)))