def test_nested_threading(self) -> None:
# char -> char -> ord -> char -> ord
g1 = parallel_map(lambda e: e,
self.test_string,
ordered=True,
use_multiprocessing=False,
cores=2,
name='g1')
g2 = parallel_map(ord,
g1,
ordered=True,
use_multiprocessing=False,
cores=2,
name='g2')
g3 = parallel_map(chr,
g2,
ordered=True,
use_multiprocessing=False,
cores=2,
name='g3')
g4 = parallel_map(ord,
g3,
ordered=True,
use_multiprocessing=False,
cores=2,
name='g4')
expected = list(map(ord, self.test_string))
self.assertEqual(list(g4), expected)
def test_exception_handing_multiprocess(self) -> None:
def raise_ex(_: Any) -> None:
raise RuntimeError("Expected exception")
self.assertRaises(
RuntimeError, list,
parallel_map(raise_ex, [1], use_multiprocessing=True))
def test_simple_unordered_multiprocess(self) -> None:
r = list(
parallel_map(self.test_func,
self.test_string,
ordered=False,
use_multiprocessing=True))
self.assertEqual(set(r), set(self.expected))
def test_simple_ordered_threaded(self) -> None:
# Make sure results are still in order as requested
r = list(
parallel_map(self.test_func,
self.test_string,
ordered=True,
use_multiprocessing=False))
self.assertEqual(r, self.expected)
def test_multisequence(self) -> None:
def test_func(a: int, b: int, c: int) -> int:
return a + b + c
s1 = [1] * 10
s2 = [2] * 10
s3 = [3] * 10
r = list(parallel_map(test_func, s1, s2, s3,
use_multiprocessing=False))
expected = [6] * 10
self.assertEqual(r, expected)
def test_multisequence_short_cutoff(self) -> None:
def test_func(a: int, b: int, c: int) -> int:
return a + b + c
s1 = [1] * 10
s2 = [2] * 4
s3 = [3] * 10
r = list(
parallel_map(test_func,
s1,
s2,
s3,
use_multiprocessing=False,
ordered=True))
exp = [6] * 4
self.assertEqual(r, exp)
def test_multisequence_fill_void(self) -> None:
def test_func(a: int, b: int, c: int) -> int:
return a + b + c
s1 = [1] * 10
s2 = [2] * 4
s3 = [3] * 10
r = list(
parallel_map(test_func,
s1,
s2,
s3,
use_multiprocessing=False,
fill_void=10,
ordered=True))
expected = [6] * 4 + [14] * 6
self.assertEqual(r, expected)
def _get_many_vectors(
cls, descriptors: Iterable["DescriptorElement"]
) -> Generator[Tuple[Hashable, Optional[numpy.ndarray]], None, None]:
"""
Internal method to be overridden by subclasses to return many vectors
associated with given descriptors.
:note: Returned vectors are *not* guaranteed to be returned in the
order they are requested. Missing vectors may be returned as None
or omitted entirely from results. The wrapper function
`get_many_vectors` handles re-ordering as necessary and insertion
of None for missing values.
:param descriptors: Iterable of descriptors to query for.
:return: Iterator of tuples containing the descriptor uuid and the
vector associated with the given descriptors or None if the
descriptor has no associated vector
"""
for uuid_vector_pair in parallel_map(_uuid_and_vector_from_descriptor,
descriptors,
name='retrieve_vectors'):
yield uuid_vector_pair