def inner_scope():
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(
test_iterator, executor)
new_intermediate = test_object.map(lambda item: item * item)
def test_last_with_default_not_used():
test_iterable = [4, 3, 8, 5, 6]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
actual_last = test_object.last('Moof')
assert actual_last == test_iterable[-1]
def test_map():
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
new_intermediate = test_object.map(lambda item: item * item)
assert new_intermediate.list() == [item * item for item in test_iterable]
def test_first():
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
actual_first = test_object.first()
assert actual_first == test_iterable[0]
def test_count():
test_iterable = [4, 3, 8, 5, 6]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
actual_count = test_object.count()
assert actual_count == len(test_iterable)
def test_last_with_default():
test_iterable = []
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
test_default = 'Moof'
actual_last = test_object.last(test_default)
assert actual_last == test_default
def test_with_specified_chunksize_on_specific_method():
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
test_lambda = lambda item: item > 4
new_intermediate = test_object.filter(test_lambda, chunksize=2)
assert new_intermediate.list() == list(filter(test_lambda,
[4, 3, 8, 5, 1]))
def test_filter():
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
test_lambda = lambda item: item > 4
new_intermediate = test_object.filter(test_lambda)
assert new_intermediate.list() == list(filter(test_lambda,
[4, 3, 8, 5, 1]))
def from_iterable_parallel(iterable, chunksize=None):
"""
Starts the iterator chain with the supplied iterable. Chaining and terminating methods can now be called on the result. Certain chaining and terminating methods will occur in parallel. Parallel means separate processes to get around Python's GIL.
:param iterable: An iterable to be used in the iterator chain.
:param chunksize: How big of chunks to split the iterator up across the parallel execution units. If unspecified or None, the chunk size will start at 1 and send that many elements to each execution unit. The chunk size will then increment in powers of two and send that many items to each execution unit. This is repeated until the iterator is exhausted. This value is used as the default chunksize for all the following parallel based methods. A specific parallel based method's chunksize can be overrided by supplying the `chunksize` keyword to that method.
:return: An intermediate object that subsequent chaining and terminating methods can be called on.
"""
iterator = iter(iterable)
executor = ProcessPoolExecutor()
return _IntermediateParallelIteratorChain(iterator, executor, chunksize=chunksize)
def test_reverse():
test_iterable = [4, 3, 8, 5, 6]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
new_serial_intermediate = test_serial_object.reverse()
new_parallel_intermediate = test_parallel_object.reverse()
assert new_parallel_intermediate.list() == new_serial_intermediate.list()
def test_flatten():
test_iterable = [[4, 3], 'DogCow', 5, {'dogCow': 'Moof', 'meep': 'moop'}]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
new_serial_intermediate = test_serial_object.flatten()
new_parallel_intermediate = test_parallel_object.flatten()
assert new_parallel_intermediate.list() == new_serial_intermediate.list()
def test_distinct():
test_iterable = [4, 3, 4, 5, 1, 3, 4]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
new_serial_intermediate = test_serial_object.distinct()
new_parallel_intermediate = test_parallel_object.distinct()
assert new_parallel_intermediate.list() == new_serial_intermediate.list()
def test_all_match_false():
test_iterable = [4, 3, 27, 5, 1]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
serial_value = test_serial_object.all_match(lambda item: item < 26)
parallel_value = test_parallel_object.all_match(lambda item: item < 26)
assert parallel_value == serial_value
def test_sum():
test_iterable = [4, 3, 8, 5, 6]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
serial_value = test_serial_object.sum()
parallel_value = test_parallel_object.sum()
assert parallel_value == serial_value
def test_max_with_default():
test_iterable = []
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
test_default = 'Moof'
serial_value = test_serial_object.max(default=test_default)
parallel_value = test_parallel_object.max(default=test_default)
assert parallel_value == serial_value
def test_sum_default_not_used():
test_iterable = [4, 3, 8, 5, 6]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
test_default = 'Moof'
serial_value = test_serial_object.sum(default=test_default)
parallel_value = test_parallel_object.sum(default=test_default)
assert parallel_value == serial_value
def test_sum_with_default():
test_iterable = ['D', 'o', 'g', 'C', 'o', 'w']
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
test_default = 'Moof'
serial_value = test_serial_object.sum(default=test_default)
parallel_value = test_parallel_object.sum(default=test_default)
assert parallel_value == serial_value
def test_for_each():
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator,
SerialExecutor())
test_parallel_output = []
def function_test(item):
test_parallel_output.append(item)
test_object.for_each(function_test)
assert test_parallel_output == test_iterable
def test_reduce():
test_iterable = [4, 3, 8, 5, 6]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
serial_value = test_serial_object.reduce(
lambda first, second: first * second)
parallel_value = test_parallel_object.reduce(
lambda first, second: first * second)
assert parallel_value == serial_value
def test_none_match_true():
test_value = 26
test_iterable = [4, 3, 8, 5, 1]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
serial_value = test_serial_object.none_match(
lambda item: item == test_value)
parallel_value = test_parallel_object.none_match(
lambda item: item == test_value)
assert parallel_value == serial_value
def test_executor_shutdowns_when_exception_raised():
executor = SerialExecutor()
def exception_function(item):
raise Exception('kaboom')
test_iterable = [4, 3, 8, 5, 1]
test_iterator = iter(test_iterable)
test_object = _IntermediateParallelIteratorChain(test_iterator, executor)
try:
new_intermediate = test_object.map(exception_function).list()
except:
pass
assert executor.shutdown_called is True
def test_sort_with_cmp():
test_iterable = [{
'inner': 8
}, {
'inner': 2
}, {
'inner': 6
}, {
'inner': 3
}, {
'inner': 9
}]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
new_serial_intermediate = test_serial_object.sort(cmp=_test_cmp)
new_parallel_intermediate = test_parallel_object.sort(cmp=_test_cmp)
assert new_parallel_intermediate.list() == new_serial_intermediate.list()
def test_sort_with_key():
test_iterable = [{
'inner': 8
}, {
'inner': 2
}, {
'inner': 6
}, {
'inner': 3
}, {
'inner': 9
}]
test_serial_iterator = iter(test_iterable)
test_parallel_iterator = iter(test_iterable)
test_serial_object = _IntermediateIteratorChain(test_serial_iterator)
test_parallel_object = _IntermediateParallelIteratorChain(
test_parallel_iterator, SerialExecutor())
test_key = lambda item: item['inner']
new_serial_intermediate = test_serial_object.sort(key=test_key)
new_parallel_intermediate = test_parallel_object.sort(key=test_key)
assert new_parallel_intermediate.list() == new_serial_intermediate.list()
