def test_runner_attrs_and_exceptions():
# repr
r = tools.runner(_func, range(10), 2)
assert repr(r).startswith(r.__class__.__name__)
assert 'jobs=2' in repr(r)
assert 'iterable={}'.format(repr(range(10))) in repr(r)
# Bad values
with pytest.raises(ValueError):
tools.runner(None, None, -1)
def test_runner_next():
input = list(range(10))
expected = list(i + 1 for i in input)
r = tools.runner(_func, input, 1)
assert next(r) == _func(input[0])
# Multiple jobs - have to pretty much run the whole thing and sort to compare
results = []
with tools.runner(_func, input, 2) as proc:
for i in input:
results.append(next(proc))
assert sorted(results) == expected
def _stream2tempfiles(
stream, jobs=1, chunksize=CHUNKSIZE, writer=_PICKLE_IO, **kwargs):
"""
Sort a stream of data into temporary files. Caller is responsible for
deleting files. Tempfile paths are generated with `tempfile.mkstemp()`.
Parameters
----------
stream : iter
Input stream to sort.
jobs : int, optional
Sort data with a pool of N workers.
chunksize : int, optional
Process this many objects from the input stream in each job. Also
the maximum amount of objects per tempfile.
writer : None or tinysort.io.BaseSerializer, optional
Instance of the serializer for writing the stream to disk.
kwargs : **kwargs, optional
Keyword arguments for `sorted()`.
Returns
-------
list
Temporary file paths.
"""
tasks = ({
'data': data,
'writer': writer,
'sort_args': kwargs
} for data in tools.slicer(stream, chunksize))
with tools.runner(_mp_sort_into_tempfile, tasks, jobs) as run:
return list(run)
def test_runner_1job():
input = list(range(10))
expected = tuple(i + 1 for i in input)
j1 = tools.runner(_func, input, 1)
assert isinstance(j1, tools.runner)
assert isinstance(iter(j1), GeneratorType)
assert tuple(j1) == expected
def test_runner_2job():
input = list(range(10))
expected = tuple(i + 1 for i in input)
# Also tests context manager
with tools.runner(_func, input, 2) as j2:
assert not j2._closed
assert isinstance(j2, tools.runner)
assert isinstance(iter(j2), IMapUnorderedIterator)
assert tuple(sorted(j2)) == expected
assert j2._closed