class StarmapTestCase(unittest.TestCase):
monitor = parallel.Monitor()
def test_apply(self):
res = parallel.Starmap.apply(get_length, (numpy.arange(10), ),
concurrent_tasks=3).reduce()
self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2]
# this case is non-trivial since there is a key, so two groups are
# generated even if everything is run in a single core
def test_apply_no_tasks(self):
res = parallel.Starmap.apply(get_length, ('aaabb', ),
concurrent_tasks=0,
key=lambda char: char)
# chunks [['a', 'a', 'a'], ['b', 'b']]
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [2, 3])
def test_apply_maxweight(self):
res = parallel.Starmap.apply(get_length, ('aaabb', ),
maxweight=2,
key=lambda char: char)
# chunks ['aa', 'ab', 'b']
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [1, 2, 2])
def test_spawn(self):
all_data = [('a', list(range(10))), ('b', list(range(20))),
('c', list(range(15)))]
res = {
key: parallel.Starmap(get_length, [(data, )])
for key, data in all_data
}
for key, val in res.items():
res[key] = val.reduce()
parallel.Starmap.restart()
self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})
def test_no_flush(self):
mon = parallel.Monitor('test')
res = parallel.safely_call(get_len, ('ab', mon))
self.assertIn(
'Monitor(\'test\').flush() must not be called'
' by get_len!', res[0])
self.assertEqual(res[1], RuntimeError)
self.assertEqual(res[2].operation, mon.operation)
if celery:
def test_received(self):
with mock.patch('os.environ', OQ_DISTRIBUTE='celery'):
res = parallel.Starmap.apply(
get_length, (numpy.arange(10), )).submit_all()
list(res) # iterate on the results
self.assertGreater(len(res.received), 0)
def test_operation(self):
res = SumPairs().run(range(5)).items()
# task 1: 0 + 1, task 2: 2 + 3, task 3: 4
self.assertEqual(sorted(res), [(1, 1), (2, 5), (3, 4)])
def test_no_flush(self):
mon = parallel.Monitor('test')
res = parallel.safely_call(get_len, ('ab', mon))
self.assertIn(
'Monitor(\'test\').flush() must not be called in a worker', res[0])
self.assertEqual(res[1], RuntimeError)
self.assertEqual(res[2].operation, mon.operation)
class StarmapTestCase(unittest.TestCase):
monitor = parallel.Monitor()
@classmethod
def setUpClass(cls):
parallel.Starmap.init() # initialize the pool
def test_apply(self):
res = parallel.Starmap.apply(get_length,
(numpy.arange(10), self.monitor),
concurrent_tasks=3).reduce()
self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2]
# this case is non-trivial since there is a key, so two groups are
# generated even if everything is run in a single core
def test_apply_no_tasks(self):
res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor),
concurrent_tasks=0,
key=lambda char: char)
# chunks [['a', 'a', 'a'], ['b', 'b']]
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [2, 3])
def test_apply_maxweight(self):
res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor),
maxweight=2,
key=lambda char: char)
# chunks ['aa', 'ab', 'b']
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [1, 2, 2])
def test_spawn(self):
all_data = [('a', list(range(10))), ('b', list(range(20))),
('c', list(range(15)))]
res = {}
for key, data in all_data:
res[key] = parallel.Starmap(get_length,
[(data, self.monitor)]).submit_all()
for key, val in res.items():
res[key] = val.reduce()
self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})
def test_gfunc(self):
mon = self.monitor
res = list(
parallel.Starmap(gfunc, [('xy', mon), ('z', mon)],
distribute='no'))
self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz'])
res = list(parallel.Starmap(gfunc, [('xy', mon), ('z', mon)]))
self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz'])
@classmethod
def tearDownClass(cls):
parallel.Starmap.shutdown()
def test(self):
monitor = parallel.Monitor()
with mock.patch.dict(os.environ, {'OQ_DISTRIBUTE': 'threadpool'}):
parallel.Starmap.init()
try:
res = parallel.Starmap.apply(get_length,
(numpy.arange(10), monitor),
concurrent_tasks=3).reduce()
self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2]
finally:
parallel.Starmap.shutdown()
class StarmapTestCase(unittest.TestCase):
monitor = parallel.Monitor()
def test_apply(self):
res = parallel.Starmap.apply(
get_length, (numpy.arange(10),), concurrent_tasks=3).reduce()
self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2]
# this case is non-trivial since there is a key, so two groups are
# generated even if everything is run in a single core
def test_apply_no_tasks(self):
res = parallel.Starmap.apply(
get_length, ('aaabb',), concurrent_tasks=0,
key=lambda char: char)
# chunks [['a', 'a', 'a'], ['b', 'b']]
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [2, 3])
def test_apply_maxweight(self):
res = parallel.Starmap.apply(
get_length, ('aaabb',), maxweight=2,
key=lambda char: char)
# chunks ['aa', 'ab', 'b']
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [1, 2, 2])
def test_spawn(self):
all_data = [
('a', list(range(10))), ('b', list(range(20))),
('c', list(range(15)))]
res = {key: parallel.Starmap(get_length, [(data,)])
for key, data in all_data}
for key, val in res.items():
res[key] = val.reduce()
parallel.Starmap.restart()
self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})
def test_no_flush(self):
mon = parallel.Monitor('test')
res = parallel.safely_call(get_len, ('ab', mon))
self.assertIn(
'Monitor(\'test\').flush() must not be called in a worker', res[0])
self.assertEqual(res[1], RuntimeError)
self.assertEqual(res[2].operation, mon.operation)
class StarmapTestCase(unittest.TestCase):
monitor = parallel.Monitor()
@classmethod
def setUpClass(cls):
parallel.Starmap.init() # initialize the pool
if parallel.oq_distribute() == 'zmq':
err = workerpool.check_status()
if err:
raise unittest.SkipTest(err)
def test_apply(self):
res = parallel.Starmap.apply(get_length,
(numpy.arange(10), self.monitor),
concurrent_tasks=3).reduce()
self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2]
# this case is non-trivial since there is a key, so two groups are
# generated even if everything is run in a single core
def test_apply_no_tasks(self):
res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor),
concurrent_tasks=0,
key=lambda char: char)
# chunks [['a', 'a', 'a'], ['b', 'b']]
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [2, 3])
def test_apply_maxweight(self):
res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor),
maxweight=2,
key=lambda char: char)
# chunks ['aa', 'ab', 'b']
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [1, 2, 2])
def test_spawn(self):
all_data = [('a', list(range(10))), ('b', list(range(20))),
('c', list(range(15)))]
res = {}
for key, data in all_data:
res[key] = parallel.Starmap(get_length, [(data, )]).submit_all()
for key, val in res.items():
res[key] = val.reduce()
self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})
def test_gfunc(self):
res = list(
parallel.Starmap(gfunc, [('xy', ), ('z', )], distribute='no'))
self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz'])
res = list(parallel.Starmap(gfunc, [('xy', ), ('z', )]))
self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz'])
def test_supertask(self):
# this test has 4 supertasks generating 4 + 5 + 3 + 5 = 17 subtasks
# and 5 real outputs (one from the yield {})
allargs = [('aaaaeeeeiii', ), ('uuuuaaaaeeeeiii', ),
('aaaaaaaaeeeeiii', ), ('aaaaeeeeiiiiiooooooo', )]
numchars = sum(len(arg) for arg, in allargs) # 61
tmpdir = tempfile.mkdtemp()
tmp = os.path.join(tmpdir, 'calc_1.hdf5')
hdf5.File(tmp, 'w').close() # the file must exist
smap = parallel.Starmap(supertask, allargs, hdf5path=tmp)
res = smap.reduce()
self.assertEqual(res, {'n': numchars})
# check that the correct information is stored in the hdf5 file
with hdf5.File(tmp, 'r') as h5:
num = general.countby(h5['performance_data'][()], 'operation')
self.assertEqual(num[b'waiting'], 4)
self.assertEqual(num[b'total supertask'], 5) # outputs
self.assertEqual(num[b'total get_length'], 17) # subtasks
self.assertGreater(len(h5['task_info']), 0)
shutil.rmtree(tmpdir)
def test_countletters(self):
data = [('hello', 'world'), ('ciao', 'mondo')]
smap = parallel.Starmap(countletters, data)
self.assertEqual(smap.reduce(), {'n': 19})
@classmethod
def tearDownClass(cls):
parallel.Starmap.shutdown()
class StarmapTestCase(unittest.TestCase):
monitor = parallel.Monitor()
@classmethod
def setUpClass(cls):
parallel.Starmap.init() # initialize the pool
def test_apply(self):
res = parallel.Starmap.apply(get_length,
(numpy.arange(10), self.monitor),
concurrent_tasks=3).reduce()
self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2]
# this case is non-trivial since there is a key, so two groups are
# generated even if everything is run in a single core
def test_apply_no_tasks(self):
res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor),
concurrent_tasks=0,
key=lambda char: char)
# chunks [['a', 'a', 'a'], ['b', 'b']]
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [2, 3])
def test_apply_maxweight(self):
res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor),
maxweight=2,
key=lambda char: char)
# chunks ['aa', 'ab', 'b']
partial_sums = sorted(dic['n'] for dic in res)
self.assertEqual(partial_sums, [1, 2, 2])
def test_spawn(self):
all_data = [('a', list(range(10))), ('b', list(range(20))),
('c', list(range(15)))]
res = {}
for key, data in all_data:
res[key] = parallel.Starmap(get_length, [(data, )]).submit_all()
for key, val in res.items():
res[key] = val.reduce()
self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}})
def test_gfunc(self):
res = list(
parallel.Starmap(gfunc, [('xy', ), ('z', )], distribute='no'))
self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz'])
res = list(parallel.Starmap(gfunc, [('xy', ), ('z', )]))
self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz'])
def test_supertask(self):
# this test has 4 supertasks generating 4 + 5 + 3 + 5 = 17 subtasks
# and 18 outputs (1 output does not produce a subtask)
allargs = [('aaaaeeeeiii', ), ('uuuuaaaaeeeeiii', ),
('aaaaaaaaeeeeiii', ), ('aaaaeeeeiiiiiooooooo', )]
numchars = sum(len(arg) for arg, in allargs) # 61
tmp = pathlib.Path(tempfile.mkdtemp(), 'calc_1.hdf5')
with hdf5.File(tmp) as h5:
monitor = performance.Monitor(hdf5=h5)
res = parallel.Starmap(supertask, allargs, monitor).reduce()
self.assertEqual(res, {'n': numchars})
# check that the correct information is stored in the hdf5 file
with hdf5.File(tmp) as h5:
num = general.countby(h5['performance_data'].value, 'operation')
self.assertEqual(num[b'waiting'], 4)
self.assertEqual(num[b'total supertask'], 18) # outputs
self.assertEqual(num[b'total get_length'], 17) # subtasks
self.assertGreater(len(h5['task_info/supertask']), 0)
shutil.rmtree(tmp.parent)
@classmethod
def tearDownClass(cls):
parallel.Starmap.shutdown()
