def test_pool_with_memmap_array_view(tmpdir): """Check that subprocess can access and update shared memory array""" assert_array_equal = np.testing.assert_array_equal # Fork the subprocess before allocating the objects to be passed pool_temp_folder = tmpdir.mkdir('pool').strpath p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder) try: filename = tmpdir.join('test.mmap').strpath a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+') a.fill(1.0) # Create an ndarray view on the memmap instance a_view = np.asarray(a) assert not isinstance(a_view, np.memmap) assert has_shareable_memory(a_view) p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0]) for j in range(a.shape[1])]) # Both a and the a_view have been updated assert_array_equal(a, 2 * np.ones(a.shape)) assert_array_equal(a_view, 2 * np.ones(a.shape)) # Passing memmap array view to the pool should not trigger the # creation of new files on the FS assert os.listdir(pool_temp_folder) == [] finally: p.terminate() del p
def test_pool_with_memmap_array_view(): """Check that subprocess can access and update shared memory array""" assert_array_equal = np.testing.assert_array_equal # Fork the subprocess before allocating the objects to be passed pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool') os.makedirs(pool_temp_folder) p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder) try: filename = os.path.join(TEMP_FOLDER, 'test.mmap') a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+') a.fill(1.0) # Create an ndarray view on the memmap instance a_view = np.asarray(a) assert_false(isinstance(a_view, np.memmap)) assert_true(has_shareable_memory(a_view)) p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0]) for j in range(a.shape[1])]) # Both a and the a_view have been updated assert_array_equal(a, 2 * np.ones(a.shape)) assert_array_equal(a_view, 2 * np.ones(a.shape)) # Passing memmap array view to the pool should not trigger the # creation of new files on the FS assert_equal(os.listdir(pool_temp_folder), []) finally: p.terminate() del p
def test_high_dimension_memmap_array_reducing(): assert_array_equal = np.testing.assert_array_equal filename = os.path.join(TEMP_FOLDER, 'test.mmap') # Create a high dimensional memmap a = np.memmap(filename, dtype=np.float64, shape=(100, 15, 15, 3), mode='w+') a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape) # Create some slices/indices at various dimensions b = a[0:10] c = a[:, 5:10] d = a[:, :, :, 0] e = a[1:3:4] def reconstruct_memmap(x): cons, args = reduce_memmap(x) res = cons(*args) return res a_reconstructed = reconstruct_memmap(a) assert_true(has_shareable_memory(a_reconstructed)) assert_true(isinstance(a_reconstructed, np.memmap)) assert_array_equal(a_reconstructed, a) b_reconstructed = reconstruct_memmap(b) assert_true(has_shareable_memory(b_reconstructed)) assert_array_equal(b_reconstructed, b) c_reconstructed = reconstruct_memmap(c) assert_true(has_shareable_memory(c_reconstructed)) assert_array_equal(c_reconstructed, c) d_reconstructed = reconstruct_memmap(d) assert_true(has_shareable_memory(d_reconstructed)) assert_array_equal(d_reconstructed, d) e_reconstructed = reconstruct_memmap(e) assert_true(has_shareable_memory(e_reconstructed)) assert_array_equal(e_reconstructed, e)
def test_high_dimension_memmap_array_reducing(tmpdir): assert_array_equal = np.testing.assert_array_equal filename = tmpdir.join('test.mmap').strpath # Create a high dimensional memmap a = np.memmap(filename, dtype=np.float64, shape=(100, 15, 15, 3), mode='w+') a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape) # Create some slices/indices at various dimensions b = a[0:10] c = a[:, 5:10] d = a[:, :, :, 0] e = a[1:3:4] def reconstruct_memmap(x): cons, args = reduce_memmap(x) res = cons(*args) return res a_reconstructed = reconstruct_memmap(a) assert has_shareable_memory(a_reconstructed) assert isinstance(a_reconstructed, np.memmap) assert_array_equal(a_reconstructed, a) b_reconstructed = reconstruct_memmap(b) assert has_shareable_memory(b_reconstructed) assert_array_equal(b_reconstructed, b) c_reconstructed = reconstruct_memmap(c) assert has_shareable_memory(c_reconstructed) assert_array_equal(c_reconstructed, c) d_reconstructed = reconstruct_memmap(d) assert has_shareable_memory(d_reconstructed) assert_array_equal(d_reconstructed, d) e_reconstructed = reconstruct_memmap(e) assert has_shareable_memory(e_reconstructed) assert_array_equal(e_reconstructed, e)
def test_memmaping_pool_for_large_arrays_in_return(tmpdir): """Check that large arrays are not copied in memory in return""" assert_array_equal = np.testing.assert_array_equal # Build an array reducers that automaticaly dump large array content # but check that the returned datastructure are regular arrays to avoid # passing a memmap array pointing to a pool controlled temp folder that # might be confusing to the user # The MemmapingPool user can always return numpy.memmap object explicitly # to avoid memory copy p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath) try: res = p.apply_async(np.ones, args=(1000,)) large = res.get() assert not has_shareable_memory(large) assert_array_equal(large, np.ones(1000)) finally: p.terminate() del p
def test_memmaping_pool_for_large_arrays_in_return(): """Check that large arrays are not copied in memory in return""" assert_array_equal = np.testing.assert_array_equal # Build an array reducers that automaticaly dump large array content # but check that the returned datastructure are regular arrays to avoid # passing a memmap array pointing to a pool controlled temp folder that # might be confusing to the user # The MemmapingPool user can always return numpy.memmap object explicitly # to avoid memory copy p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER) try: res = p.apply_async(np.ones, args=(1000, )) large = res.get() assert_false(has_shareable_memory(large)) assert_array_equal(large, np.ones(1000)) finally: p.terminate() del p
def test_workaround_against_bad_memmap_with_copied_buffers(tmpdir): """Check that memmaps with a bad buffer are returned as regular arrays Unary operations and ufuncs on memmap instances return a new memmap instance with an in-memory buffer (probably a numpy bug). """ assert_array_equal = np.testing.assert_array_equal p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath) try: # Send a complex, large-ish view on a array that will be converted to # a memmap in the worker process a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order='F')[:, :1, :] # Call a non-inplace multiply operation on the worker and memmap and # send it back to the parent. b = p.apply_async(_worker_multiply, args=(a, 3)).get() assert not has_shareable_memory(b) assert_array_equal(b, 3 * a) finally: p.terminate() del p
def test_workaround_against_bad_memmap_with_copied_buffers(): """Check that memmaps with a bad buffer are returned as regular arrays Unary operations and ufuncs on memmap instances return a new memmap instance with an in-memory buffer (probably a numpy bug). """ assert_array_equal = np.testing.assert_array_equal p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER) try: # Send a complex, large-ish view on a array that will be converted to # a memmap in the worker process a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order='F')[:, :1, :] # Call a non-inplace multiply operation on the worker and memmap and # send it back to the parent. b = p.apply_async(_worker_multiply, args=(a, 3)).get() assert_false(has_shareable_memory(b)) assert_array_equal(b, 3 * a) finally: p.terminate() del p
def test_memmap_based_array_reducing(tmpdir): """Check that it is possible to reduce a memmap backed array""" assert_array_equal = np.testing.assert_array_equal filename = tmpdir.join('test.mmap').strpath # Create a file larger than what will be used by a buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+') # Fill the original buffer with negative markers to detect over of # underflow in case of test failures buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype) buffer.flush() # Memmap a 2D fortran array on a offseted subsection of the previous # buffer a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4), mode='r+', order='F', offset=4) a[:] = np.arange(60).reshape(a.shape) # Build various views that share the buffer with the original memmap # b is an memmap sliced view on an memmap instance b = a[1:-1, 2:-1, 2:4] # c and d are array views c = np.asarray(b) d = c.T # Array reducer with auto dumping disabled reducer = ArrayMemmapReducer(None, tmpdir.strpath, 'c') def reconstruct_array(x): cons, args = reducer(x) return cons(*args) def reconstruct_memmap(x): cons, args = reduce_memmap(x) return cons(*args) # Reconstruct original memmap a_reconstructed = reconstruct_memmap(a) assert has_shareable_memory(a_reconstructed) assert isinstance(a_reconstructed, np.memmap) assert_array_equal(a_reconstructed, a) # Reconstruct strided memmap view b_reconstructed = reconstruct_memmap(b) assert has_shareable_memory(b_reconstructed) assert_array_equal(b_reconstructed, b) # Reconstruct arrays views on memmap base c_reconstructed = reconstruct_array(c) assert not isinstance(c_reconstructed, np.memmap) assert has_shareable_memory(c_reconstructed) assert_array_equal(c_reconstructed, c) d_reconstructed = reconstruct_array(d) assert not isinstance(d_reconstructed, np.memmap) assert has_shareable_memory(d_reconstructed) assert_array_equal(d_reconstructed, d) # Test graceful degradation on fake memmap instances with in-memory # buffers a3 = a * 3 assert not has_shareable_memory(a3) a3_reconstructed = reconstruct_memmap(a3) assert not has_shareable_memory(a3_reconstructed) assert not isinstance(a3_reconstructed, np.memmap) assert_array_equal(a3_reconstructed, a * 3) # Test graceful degradation on arrays derived from fake memmap instances b3 = np.asarray(a3) assert not has_shareable_memory(b3) b3_reconstructed = reconstruct_array(b3) assert isinstance(b3_reconstructed, np.ndarray) assert not has_shareable_memory(b3_reconstructed) assert_array_equal(b3_reconstructed, b3)
def _worker_multiply(a, n_times): """Multiplication function to be executed by subprocess""" assert has_shareable_memory(a) return a * n_times
def test_memmap_based_array_reducing(): """Check that it is possible to reduce a memmap backed array""" assert_array_equal = np.testing.assert_array_equal filename = os.path.join(TEMP_FOLDER, 'test.mmap') # Create a file larger than what will be used by a buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+') # Fill the original buffer with negative markers to detect over of # underflow in case of test failures buffer[:] = -1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype) buffer.flush() # Memmap a 2D fortran array on a offseted subsection of the previous # buffer a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4), mode='r+', order='F', offset=4) a[:] = np.arange(60).reshape(a.shape) # Build various views that share the buffer with the original memmap # b is an memmap sliced view on an memmap instance b = a[1:-1, 2:-1, 2:4] # c and d are array views c = np.asarray(b) d = c.T # Array reducer with auto dumping disabled reducer = ArrayMemmapReducer(None, TEMP_FOLDER, 'c') def reconstruct_array(x): cons, args = reducer(x) return cons(*args) def reconstruct_memmap(x): cons, args = reduce_memmap(x) return cons(*args) # Reconstruct original memmap a_reconstructed = reconstruct_memmap(a) assert_true(has_shareable_memory(a_reconstructed)) assert_true(isinstance(a_reconstructed, np.memmap)) assert_array_equal(a_reconstructed, a) # Reconstruct strided memmap view b_reconstructed = reconstruct_memmap(b) assert_true(has_shareable_memory(b_reconstructed)) assert_array_equal(b_reconstructed, b) # Reconstruct arrays views on memmap base c_reconstructed = reconstruct_array(c) assert_false(isinstance(c_reconstructed, np.memmap)) assert_true(has_shareable_memory(c_reconstructed)) assert_array_equal(c_reconstructed, c) d_reconstructed = reconstruct_array(d) assert_false(isinstance(d_reconstructed, np.memmap)) assert_true(has_shareable_memory(d_reconstructed)) assert_array_equal(d_reconstructed, d) # Test graceful degradation on fake memmap instances with in-memory # buffers a3 = a * 3 assert_false(has_shareable_memory(a3)) a3_reconstructed = reconstruct_memmap(a3) assert_false(has_shareable_memory(a3_reconstructed)) assert_false(isinstance(a3_reconstructed, np.memmap)) assert_array_equal(a3_reconstructed, a * 3) # Test graceful degradation on arrays derived from fake memmap instances b3 = np.asarray(a3) assert_false(has_shareable_memory(b3)) b3_reconstructed = reconstruct_array(b3) assert_true(isinstance(b3_reconstructed, np.ndarray)) assert_false(has_shareable_memory(b3_reconstructed)) assert_array_equal(b3_reconstructed, b3)
def _worker_multiply(a, n_times): """Multiplication function to be executed by subprocess""" assert_true(has_shareable_memory(a)) return a * n_times