def batches(self, data, batch=None):
if batch is None:
batch = self.batch
data_type = type(data)
if data_type == list or data_type == tuple:
return fp.batches(data, pick=fp.pick_value, batch_size=batch)
elif data_type == dict:
return fp.batches(data, pick=self.picker, batch_size=batch)
elif data_type == types.GeneratorType:
return fp.batches(data, pick=fp.pick_value, batch_size=batch)
elif data_type == types.FunctionType:
return fp.batches(data, pick=fp.pick_value, batch_size=batch)
def test_batches(self):
elevens = list(range(11))
self.assertListEqual(
[[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10]],
list(p.batches(elevens, batch_size=1)))
self.assertListEqual([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]],
list(p.batches(elevens, batch_size=2)))
self.assertListEqual([[0, 1, 2], [3, 4, 5], [6, 7, 8]],
list(p.batches(elevens, batch_size=3)))
self.assertListEqual([[0, 1, 2, 3], [4, 5, 6, 7]],
list(p.batches(elevens, batch_size=4)))
self.assertListEqual([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]],
list(p.batches(elevens, batch_size=5)))
def test_rolled_batches(self):
elevens = list(range(11))
self.assertListEqual([[[0, 1, 2], [1, 2, 3], [2, 3, 4]],
[[3, 4, 5], [4, 5, 6], [5, 6, 7]],
[[6, 7, 8], [7, 8, 9], [8, 9, 10]]],
list(
p.batches(p.roll(elevens, window_size=3),
batch_size=3)))
def datagen():
elevens = list(range(11))
return p.batches(elevens, batch_size=3)
def test_batched_roll(self):
elevens = list(range(11))
self.assertListEqual([[[0, 1, 2], [3, 4, 5]], [[3, 4, 5], [6, 7, 8]]],
list(
p.roll(p.batches(elevens, batch_size=3),
window_size=2)))
def wrapped(*args, **kwargs):
for seg in fp.batches(f(*args, **kwargs), batch_size=segment_size):
yield Sequential(seg)