def test_runner_attrs_and_exceptions(): # repr r = tools.runner(_func, range(10), 2) assert repr(r).startswith(r.__class__.__name__) assert 'jobs=2' in repr(r) assert 'iterable={}'.format(repr(range(10))) in repr(r) # Bad values with pytest.raises(ValueError): tools.runner(None, None, -1)
def test_runner_next(): input = list(range(10)) expected = list(i + 1 for i in input) r = tools.runner(_func, input, 1) assert next(r) == _func(input[0]) # Multiple jobs - have to pretty much run the whole thing and sort to compare results = [] with tools.runner(_func, input, 2) as proc: for i in input: results.append(next(proc)) assert sorted(results) == expected
def _stream2tempfiles( stream, jobs=1, chunksize=CHUNKSIZE, writer=_PICKLE_IO, **kwargs): """ Sort a stream of data into temporary files. Caller is responsible for deleting files. Tempfile paths are generated with `tempfile.mkstemp()`. Parameters ---------- stream : iter Input stream to sort. jobs : int, optional Sort data with a pool of N workers. chunksize : int, optional Process this many objects from the input stream in each job. Also the maximum amount of objects per tempfile. writer : None or tinysort.io.BaseSerializer, optional Instance of the serializer for writing the stream to disk. kwargs : **kwargs, optional Keyword arguments for `sorted()`. Returns ------- list Temporary file paths. """ tasks = ({ 'data': data, 'writer': writer, 'sort_args': kwargs } for data in tools.slicer(stream, chunksize)) with tools.runner(_mp_sort_into_tempfile, tasks, jobs) as run: return list(run)
def test_runner_1job(): input = list(range(10)) expected = tuple(i + 1 for i in input) j1 = tools.runner(_func, input, 1) assert isinstance(j1, tools.runner) assert isinstance(iter(j1), GeneratorType) assert tuple(j1) == expected
def test_runner_2job(): input = list(range(10)) expected = tuple(i + 1 for i in input) # Also tests context manager with tools.runner(_func, input, 2) as j2: assert not j2._closed assert isinstance(j2, tools.runner) assert isinstance(iter(j2), IMapUnorderedIterator) assert tuple(sorted(j2)) == expected assert j2._closed