def execute(partial_func: partial, num_operations: int, progress=None, msg: str = '', cores=None): """ Executes a function in parallel with shared memory between the processes. The array must have been created using parallel.utility.create_shared_array(shape, dtype). If the input array IS NOT a shared array, the data will NOT BE CHANGED! The reason for that is that the processes don't work on the data, but on a copy. When they process it and return the result, THE RESULT IS NOT ASSIGNED BACK TO REPLACE THE ORIGINAL, it is discarded. - imap_unordered gives the images back in random order - map and map_async do not improve speed performance - imap seems to be the best choice Using _ in the for _ enumerate is slightly faster, because the tuple from enumerate isn't unpacked, and thus some time is saved. From performance tests, the chunksize doesn't seem to make much of a difference, but having larger chunks usually led to slower performance: Shape: (50,512,512) 1 chunk 3.06s 2 chunks 3.05s 3 chunks 3.07s 4 chunks 3.06s 5 chunks 3.16s 6 chunks 3.06s 7 chunks 3.058s 8 chunks 3.25s 9 chunks 3.45s :param partial_func: A function constructed using create_partial :param num_operations: The expected number of operations - should match the number of images being processed Also used to set the number of progress steps :param cores: number of cores that the processing will use :param progress: Progress instance to use for progress reporting (optional) :param msg: Message to be shown on the progress bar :return: """ if not cores: cores = pu.get_cores() chunksize = pu.calculate_chunksize(cores) pu.execute_impl(num_operations, partial_func, cores, chunksize, progress, msg) global shared_list shared_list = []
def test_execute_impl_par(mock_pool): mock_partial = mock.Mock() mock_progress = mock.Mock() mock_pool_instance = mock.Mock() mock_pool_instance.imap.return_value = range(15) mock_pool.return_value.__enter__.return_value = mock_pool_instance execute_impl(15, mock_partial, 10, 1, mock_progress, "Test") mock_pool_instance.imap.assert_called_once() assert mock_progress.update.call_count == 15
def execute(data=None, partial_func=None, cores=None, chunksize=None, progress=None, msg: str = ''): """ Executes a function in parallel with shared memory between the processes. The array MUST HAVE BEEN created using parallel.utility.create_shared_array(shape, dtype). If the input array IS NOT a shared array, the data will NOT BE CHANGED! The reason for that is that the processes don't work on the data, but on a copy. When they process it and return the result, THE RESULT IS NOT ASSIGNED BACK TO REPLACE THE ORIGINAL, it is discarded. Function choice for iterating over the data: - imap_unordered gives the images back in random order! - map and map_async cannot replace the data in place and end up doubling the memory. They do not improve speed performance either - imap seems to be the best choice Using _ in the for _ enumerate is slightly faster, because the tuple from enumerate isn't unpacked, and thus some time is saved. From performance tests, the chunksize doesn't seem to make much of a difference, but having larger chunks usually led to slower performance: Shape: (50,512,512) 1 chunk 3.06s 2 chunks 3.05s 3 chunks 3.07s 4 chunks 3.06s 5 chunks 3.16s 6 chunks 3.06s 7 chunks 3.058s 8 chunks 3.25s 9 chunks 3.45s :param data: the data array that will be processed in parallel :param partial_func: a function constructed using partial to pass the correct arguments :param cores: number of cores that the processing will use :param chunksize: chunk of work per process(worker) :param name: name of the task used in progress reporting :param progress: Progress instance to use for progress reporting (optional) :return: reference to the input shared array """ if not cores: cores = pu.get_cores() if not chunksize: chunksize = pu.calculate_chunksize(cores) global shared_data # get reference to output data # if different shape it will get the reference to the new array shared_data = data img_num = shared_data.shape[0] pu.execute_impl(img_num, partial_func, cores, chunksize, progress, msg) # remove the global references to remove unused dangling handles to the # data, which might prevent it from being GCed temp_data_ref = shared_data del shared_data return temp_data_ref
def test_execute_impl_one_core(mock_pool): mock_partial = mock.Mock() mock_progress = mock.Mock() execute_impl(1, mock_partial, 1, 1, mock_progress, "Test") mock_partial.assert_called_once_with(0) mock_progress.update.assert_called_once_with(1, "Test")