class StarmapTestCase(unittest.TestCase): monitor = parallel.Monitor() def test_apply(self): res = parallel.Starmap.apply(get_length, (numpy.arange(10), ), concurrent_tasks=3).reduce() self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2] # this case is non-trivial since there is a key, so two groups are # generated even if everything is run in a single core def test_apply_no_tasks(self): res = parallel.Starmap.apply(get_length, ('aaabb', ), concurrent_tasks=0, key=lambda char: char) # chunks [['a', 'a', 'a'], ['b', 'b']] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [2, 3]) def test_apply_maxweight(self): res = parallel.Starmap.apply(get_length, ('aaabb', ), maxweight=2, key=lambda char: char) # chunks ['aa', 'ab', 'b'] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [1, 2, 2]) def test_spawn(self): all_data = [('a', list(range(10))), ('b', list(range(20))), ('c', list(range(15)))] res = { key: parallel.Starmap(get_length, [(data, )]) for key, data in all_data } for key, val in res.items(): res[key] = val.reduce() parallel.Starmap.restart() self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}}) def test_no_flush(self): mon = parallel.Monitor('test') res = parallel.safely_call(get_len, ('ab', mon)) self.assertIn( 'Monitor(\'test\').flush() must not be called' ' by get_len!', res[0]) self.assertEqual(res[1], RuntimeError) self.assertEqual(res[2].operation, mon.operation) if celery: def test_received(self): with mock.patch('os.environ', OQ_DISTRIBUTE='celery'): res = parallel.Starmap.apply( get_length, (numpy.arange(10), )).submit_all() list(res) # iterate on the results self.assertGreater(len(res.received), 0) def test_operation(self): res = SumPairs().run(range(5)).items() # task 1: 0 + 1, task 2: 2 + 3, task 3: 4 self.assertEqual(sorted(res), [(1, 1), (2, 5), (3, 4)])
def test_no_flush(self): mon = parallel.Monitor('test') res = parallel.safely_call(get_len, ('ab', mon)) self.assertIn( 'Monitor(\'test\').flush() must not be called in a worker', res[0]) self.assertEqual(res[1], RuntimeError) self.assertEqual(res[2].operation, mon.operation)
class StarmapTestCase(unittest.TestCase): monitor = parallel.Monitor() @classmethod def setUpClass(cls): parallel.Starmap.init() # initialize the pool def test_apply(self): res = parallel.Starmap.apply(get_length, (numpy.arange(10), self.monitor), concurrent_tasks=3).reduce() self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2] # this case is non-trivial since there is a key, so two groups are # generated even if everything is run in a single core def test_apply_no_tasks(self): res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor), concurrent_tasks=0, key=lambda char: char) # chunks [['a', 'a', 'a'], ['b', 'b']] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [2, 3]) def test_apply_maxweight(self): res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor), maxweight=2, key=lambda char: char) # chunks ['aa', 'ab', 'b'] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [1, 2, 2]) def test_spawn(self): all_data = [('a', list(range(10))), ('b', list(range(20))), ('c', list(range(15)))] res = {} for key, data in all_data: res[key] = parallel.Starmap(get_length, [(data, self.monitor)]).submit_all() for key, val in res.items(): res[key] = val.reduce() self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}}) def test_gfunc(self): mon = self.monitor res = list( parallel.Starmap(gfunc, [('xy', mon), ('z', mon)], distribute='no')) self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz']) res = list(parallel.Starmap(gfunc, [('xy', mon), ('z', mon)])) self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz']) @classmethod def tearDownClass(cls): parallel.Starmap.shutdown()
def test(self): monitor = parallel.Monitor() with mock.patch.dict(os.environ, {'OQ_DISTRIBUTE': 'threadpool'}): parallel.Starmap.init() try: res = parallel.Starmap.apply(get_length, (numpy.arange(10), monitor), concurrent_tasks=3).reduce() self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2] finally: parallel.Starmap.shutdown()
class StarmapTestCase(unittest.TestCase): monitor = parallel.Monitor() def test_apply(self): res = parallel.Starmap.apply( get_length, (numpy.arange(10),), concurrent_tasks=3).reduce() self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2] # this case is non-trivial since there is a key, so two groups are # generated even if everything is run in a single core def test_apply_no_tasks(self): res = parallel.Starmap.apply( get_length, ('aaabb',), concurrent_tasks=0, key=lambda char: char) # chunks [['a', 'a', 'a'], ['b', 'b']] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [2, 3]) def test_apply_maxweight(self): res = parallel.Starmap.apply( get_length, ('aaabb',), maxweight=2, key=lambda char: char) # chunks ['aa', 'ab', 'b'] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [1, 2, 2]) def test_spawn(self): all_data = [ ('a', list(range(10))), ('b', list(range(20))), ('c', list(range(15)))] res = {key: parallel.Starmap(get_length, [(data,)]) for key, data in all_data} for key, val in res.items(): res[key] = val.reduce() parallel.Starmap.restart() self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}}) def test_no_flush(self): mon = parallel.Monitor('test') res = parallel.safely_call(get_len, ('ab', mon)) self.assertIn( 'Monitor(\'test\').flush() must not be called in a worker', res[0]) self.assertEqual(res[1], RuntimeError) self.assertEqual(res[2].operation, mon.operation)
class StarmapTestCase(unittest.TestCase): monitor = parallel.Monitor() @classmethod def setUpClass(cls): parallel.Starmap.init() # initialize the pool if parallel.oq_distribute() == 'zmq': err = workerpool.check_status() if err: raise unittest.SkipTest(err) def test_apply(self): res = parallel.Starmap.apply(get_length, (numpy.arange(10), self.monitor), concurrent_tasks=3).reduce() self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2] # this case is non-trivial since there is a key, so two groups are # generated even if everything is run in a single core def test_apply_no_tasks(self): res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor), concurrent_tasks=0, key=lambda char: char) # chunks [['a', 'a', 'a'], ['b', 'b']] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [2, 3]) def test_apply_maxweight(self): res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor), maxweight=2, key=lambda char: char) # chunks ['aa', 'ab', 'b'] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [1, 2, 2]) def test_spawn(self): all_data = [('a', list(range(10))), ('b', list(range(20))), ('c', list(range(15)))] res = {} for key, data in all_data: res[key] = parallel.Starmap(get_length, [(data, )]).submit_all() for key, val in res.items(): res[key] = val.reduce() self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}}) def test_gfunc(self): res = list( parallel.Starmap(gfunc, [('xy', ), ('z', )], distribute='no')) self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz']) res = list(parallel.Starmap(gfunc, [('xy', ), ('z', )])) self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz']) def test_supertask(self): # this test has 4 supertasks generating 4 + 5 + 3 + 5 = 17 subtasks # and 5 real outputs (one from the yield {}) allargs = [('aaaaeeeeiii', ), ('uuuuaaaaeeeeiii', ), ('aaaaaaaaeeeeiii', ), ('aaaaeeeeiiiiiooooooo', )] numchars = sum(len(arg) for arg, in allargs) # 61 tmpdir = tempfile.mkdtemp() tmp = os.path.join(tmpdir, 'calc_1.hdf5') hdf5.File(tmp, 'w').close() # the file must exist smap = parallel.Starmap(supertask, allargs, hdf5path=tmp) res = smap.reduce() self.assertEqual(res, {'n': numchars}) # check that the correct information is stored in the hdf5 file with hdf5.File(tmp, 'r') as h5: num = general.countby(h5['performance_data'][()], 'operation') self.assertEqual(num[b'waiting'], 4) self.assertEqual(num[b'total supertask'], 5) # outputs self.assertEqual(num[b'total get_length'], 17) # subtasks self.assertGreater(len(h5['task_info']), 0) shutil.rmtree(tmpdir) def test_countletters(self): data = [('hello', 'world'), ('ciao', 'mondo')] smap = parallel.Starmap(countletters, data) self.assertEqual(smap.reduce(), {'n': 19}) @classmethod def tearDownClass(cls): parallel.Starmap.shutdown()
class StarmapTestCase(unittest.TestCase): monitor = parallel.Monitor() @classmethod def setUpClass(cls): parallel.Starmap.init() # initialize the pool def test_apply(self): res = parallel.Starmap.apply(get_length, (numpy.arange(10), self.monitor), concurrent_tasks=3).reduce() self.assertEqual(res, {'n': 10}) # chunks [4, 4, 2] # this case is non-trivial since there is a key, so two groups are # generated even if everything is run in a single core def test_apply_no_tasks(self): res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor), concurrent_tasks=0, key=lambda char: char) # chunks [['a', 'a', 'a'], ['b', 'b']] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [2, 3]) def test_apply_maxweight(self): res = parallel.Starmap.apply(get_length, ('aaabb', self.monitor), maxweight=2, key=lambda char: char) # chunks ['aa', 'ab', 'b'] partial_sums = sorted(dic['n'] for dic in res) self.assertEqual(partial_sums, [1, 2, 2]) def test_spawn(self): all_data = [('a', list(range(10))), ('b', list(range(20))), ('c', list(range(15)))] res = {} for key, data in all_data: res[key] = parallel.Starmap(get_length, [(data, )]).submit_all() for key, val in res.items(): res[key] = val.reduce() self.assertEqual(res, {'a': {'n': 10}, 'c': {'n': 15}, 'b': {'n': 20}}) def test_gfunc(self): res = list( parallel.Starmap(gfunc, [('xy', ), ('z', )], distribute='no')) self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz']) res = list(parallel.Starmap(gfunc, [('xy', ), ('z', )])) self.assertEqual(sorted(res), ['xxx', 'yyy', 'zzz']) def test_supertask(self): # this test has 4 supertasks generating 4 + 5 + 3 + 5 = 17 subtasks # and 18 outputs (1 output does not produce a subtask) allargs = [('aaaaeeeeiii', ), ('uuuuaaaaeeeeiii', ), ('aaaaaaaaeeeeiii', ), ('aaaaeeeeiiiiiooooooo', )] numchars = sum(len(arg) for arg, in allargs) # 61 tmp = pathlib.Path(tempfile.mkdtemp(), 'calc_1.hdf5') with hdf5.File(tmp) as h5: monitor = performance.Monitor(hdf5=h5) res = parallel.Starmap(supertask, allargs, monitor).reduce() self.assertEqual(res, {'n': numchars}) # check that the correct information is stored in the hdf5 file with hdf5.File(tmp) as h5: num = general.countby(h5['performance_data'].value, 'operation') self.assertEqual(num[b'waiting'], 4) self.assertEqual(num[b'total supertask'], 18) # outputs self.assertEqual(num[b'total get_length'], 17) # subtasks self.assertGreater(len(h5['task_info/supertask']), 0) shutil.rmtree(tmp.parent) @classmethod def tearDownClass(cls): parallel.Starmap.shutdown()