def batches(self, data, batch=None): if batch is None: batch = self.batch data_type = type(data) if data_type == list or data_type == tuple: return fp.batches(data, pick=fp.pick_value, batch_size=batch) elif data_type == dict: return fp.batches(data, pick=self.picker, batch_size=batch) elif data_type == types.GeneratorType: return fp.batches(data, pick=fp.pick_value, batch_size=batch) elif data_type == types.FunctionType: return fp.batches(data, pick=fp.pick_value, batch_size=batch)
def test_batches(self): elevens = list(range(11)) self.assertListEqual( [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10]], list(p.batches(elevens, batch_size=1))) self.assertListEqual([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]], list(p.batches(elevens, batch_size=2))) self.assertListEqual([[0, 1, 2], [3, 4, 5], [6, 7, 8]], list(p.batches(elevens, batch_size=3))) self.assertListEqual([[0, 1, 2, 3], [4, 5, 6, 7]], list(p.batches(elevens, batch_size=4))) self.assertListEqual([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]], list(p.batches(elevens, batch_size=5)))
def test_rolled_batches(self): elevens = list(range(11)) self.assertListEqual([[[0, 1, 2], [1, 2, 3], [2, 3, 4]], [[3, 4, 5], [4, 5, 6], [5, 6, 7]], [[6, 7, 8], [7, 8, 9], [8, 9, 10]]], list( p.batches(p.roll(elevens, window_size=3), batch_size=3)))
def datagen(): elevens = list(range(11)) return p.batches(elevens, batch_size=3)
def test_batched_roll(self): elevens = list(range(11)) self.assertListEqual([[[0, 1, 2], [3, 4, 5]], [[3, 4, 5], [6, 7, 8]]], list( p.roll(p.batches(elevens, batch_size=3), window_size=2)))
def wrapped(*args, **kwargs): for seg in fp.batches(f(*args, **kwargs), batch_size=segment_size): yield Sequential(seg)