def test_windows(self, mock_hasattr):
with self.assertRaises(CompositionError) as e:
concurrent(a + a)
self.assertEqual(
str(e.exception),
"ConcurrentMerge requires os.fork, and thus is only available on unix",
)
def test_multi_arg_map(self):
@node
def f(*args):
return args
m = concurrent(mmap(f))
with self.assertRaises(CompositionError):
# Because star returns a simple function, we can't upgrade it.
starmap = concurrent(star(mmap(f)))
# we have to wrap concurrent in star instead.
starmap = star(concurrent(mmap(f)))
mapstar = concurrent(mmap(star(f)))
self.assertEqual(m([1, 2, 3], [4, 5, 6]), ((1, 4), (2, 5), (3, 6)))
self.assertEqual(m([1, 2, 3]), ((1, ), (2, ), (3, )))
with self.assertRaises(TypeError):
starmap([1, 2, 3])
self.assertEqual(starmap([[1, 2, 3]]), m([1, 2, 3]))
cmp = ([1, 2, 3], [4, 5, 6]) | starmap
self.assertEqual(cmp(), ((1, 4), (2, 5), (3, 6)))
cmp = ([1, 2, 3], [4, 5, 6]) | mapstar
self.assertEqual(cmp(), ((1, 2, 3), (4, 5, 6)))
def test_basic_map(self):
# We can upgrade maps to run in parallel
banana = "bnn" | concurrent(mmap(a)) | "".join
str_concat = operators.concat | node("".join)
batman = concurrent(mmap(a, operator=str_concat))
self.assertEqual(banana(), "banana")
self.assertEqual(batman("nnnn"), "nananana")
def test_not_concurrent(self):
# can only upgrade FunctionMerges
with self.assertRaises(CompositionError):
concurrent(a)
with self.assertRaises(CompositionError):
concurrent(a | b)
def test_concurrent(self):
# Concurrent and star can work together, although this organization no longer makes sense
with self.assertRaises(exceptions.CompositionError):
aabbcc = a & b & c | concurrent(star(a & b & c))
# self.assertEqual(aabbcc('_'), '_aa_bb_cc')
aabbcc = (a & b & c) | star(concurrent(a & b & c))
self.assertEqual(aabbcc("_"), ("_aa", "_bb", "_cc"))
def test_call(self):
c = concurrent(a + b)
self.assertEqual(c("_"), "_a_b")
self.assertEqual(c("-", "_"), "-a_b")
with self.assertRaises(CallError):
c("_", "_", "_")
@node
def d():
return "d"
abd = concurrent(a & b & d)
self.assertEqual(abd("-", "_"), ("-a", "_b", "d"))
def test_call_state(self):
# Call state should be usable in concurrent chains
chain_a = a | b | store("ab")
chain_b = b | a | store("ba")
cmp = concurrent(chain_a & chain_b)
state = CallState()
self.assertEqual(cmp("_", call_state=state), ("_ab", "_ba"))
self.assertDictEqual(state.data, {"ab": "_ab", "ba": "_ba"})
# If call_state.data contains something that isn't pickleable, fail gracefully
bad = [lambda: None] | store("o")
cmp = concurrent(bad & bad)
with self.assertRaises(ConcurrentException):
cmp()
def test_str_repr(self):
cab = ConcurrentMerge(a + b)
cmap = concurrent(mmap(a))
self.assertEqual(
repr(cab),
"ConcurrentMerge({0}, ({1!r}, {2!r}))".format(operator.add, a, b))
self.assertEqual(str(cab), "concurrent(a + b)")
self.assertEqual(str(cmap), "concurrent(mmap(a))")
def test_unpicklable_return(self):
# Concurrent can't handle functions that return unpicklable objects. Raise a descriptive
# exception
@node
def f():
return lambda: None
cmp = concurrent(f & f)
with self.assertRaises(ConcurrentException):
cmp()
def test_unpicklable_exception(self):
# Don't let child processes crash, even if they do weird things like raise unpickleable
# exceptions
@node
def f():
class BadException(Exception):
pass
raise BadException()
cmp = concurrent(f + f)
with self.assertRaises(ConcurrentException):
cmp()
def test_concurrent(self):
c = concurrent(a + b)
self.assertIsInstance(c, ConcurrentMerge)
self.assertEqual(c("_"), "_a_b")