def get_request_iterator(self):
indices = list(range(self.num_examples))
self.rng.shuffle(indices)
return imap(
list,
imap(islice, repeat(_iter(indices), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
if self.times:
return repeat(self.batch_size, self.times)
if self.num_examples:
d, r = divmod(self.num_examples, self.batch_size)
return chain(repeat(self.batch_size, d), [r] if r else [])
return repeat(self.batch_size)
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get("as_dict", False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(
izip(*[data_stream.get_epoch_iterator()
for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get('as_dict', False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def test_num_examples():
assert_raises(ValueError, IterableDataset,
{'features': range(10), 'targets': range(7)})
dataset = IterableDataset({'features': range(7),
'targets': range(7)})
assert dataset.num_examples == 7
dataset = IterableDataset(repeat(1))
assert numpy.isnan(dataset.num_examples)
x = numpy.random.rand(5, 3)
y = numpy.random.rand(5, 4)
dataset = IndexableDataset({'features': x, 'targets': y})
assert dataset.num_examples == 5
assert_raises(ValueError, IndexableDataset,
{'features': x, 'targets': y[:4]})
def open(self):
return chain.from_iterable(izip(*[chain.from_iterable(
imap(open, repeat(f))) for f in self.files]))
def get_request_iterator(self):
indices = list(range(self.num_examples))
self.rng.shuffle(indices)
return imap(list, imap(
islice, repeat(iter_(indices), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
return imap(list, imap(
islice, repeat(xrange(self.num_examples), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
return imap(
list,
imap(islice,
repeat(_iter(xrange(self.num_examples)), self.num_batches),
repeat(self.batch_size, self.num_batches)))
