def test_batch_iterable():
iterable = [1, 2, 3, 4, 5, 6, 7, 8]
res = batch_iterable(iterable, 2)
assert isinstance(res, GeneratorType)
assert sum(1 for i in res) == 4
res = batch_iterable(iterable, 4)
assert sum(1 for i in res) == 2
res = batch_iterable(iterable, 4)
first_half = next(res)
assert isinstance(first_half, list)
assert first_half == [1, 2, 3, 4]
second_half = next(res)
assert isinstance(second_half, list)
assert second_half == [5, 6, 7, 8]
def test_batch_iterable():
iterable = [1, 2, 3, 4, 5, 6, 7, 8]
res = batch_iterable(iterable, 2)
assert isinstance(res, GeneratorType)
assert sum(1 for i in res) == 4
res = batch_iterable(iterable, 4)
assert sum(1 for i in res) == 2
res = batch_iterable(iterable, 4)
first_half = next(res)
assert isinstance(first_half, list)
assert first_half == [1, 2, 3, 4]
second_half = next(res)
assert isinstance(second_half, list)
assert second_half == [5, 6, 7, 8]
def _iterate(self, stims, *args, **kwargs):
batches = batch_iterable(stims, self._batch_size)
results = []
for batch in progress_bar_wrapper(batches):
res = self._transform(batch, *args, **kwargs)
for i, stim in enumerate(batch):
res[i] = _log_transformation(stim, res[i], self)
results.extend(res)
return results
def _iterate(self, stims, validation='strict', *args, **kwargs):
batches = batch_iterable(stims, self._batch_size)
results = []
for batch in progress_bar_wrapper(batches):
use_cache = config.get_option('cache_transformers')
target_inds = {}
non_cached = []
for stim in batch:
key = hash((hash(self), hash(stim)))
# If using the cache, only transform stims that aren't in the
# cache and haven't already appeared in the batch
if not (use_cache and (key in _cache or key in target_inds)):
target_inds[key] = len(non_cached)
non_cached.append(stim)
# _transform will likely fail if given an empty list
if len(non_cached) > 0:
batch_results = self._transform(non_cached, *args, **kwargs)
else:
batch_results = []
for i, stim in enumerate(batch):
key = hash((hash(self), hash(stim)))
# Use the target index to get the result from batch_results
if key in target_inds:
result = batch_results[target_inds[key]]
result = _log_transformation(stim, result, self)
self._propagate_context(stim, result)
if use_cache:
if isgenerator(result):
result = list(result)
_cache[key] = result
results.append(result)
# Otherwise, the result should be in the cache
else:
results.append(_cache[key])
return results