def test_hash_memmap():
""" Check that memmap and arrays hash identically if coerce_mmap is
True.
"""
filename = tempfile.mktemp(prefix='joblib_test_hash_memmap_')
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
for coerce_mmap in (False, True):
yield (assert_equal,
hash(a, coerce_mmap=coerce_mmap) ==
hash(m, coerce_mmap=coerce_mmap),
coerce_mmap)
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
try:
os.unlink(filename)
except OSError as e:
# Under windows, some files don't get erased.
if not os.name == 'nt':
raise e
def test_pool_with_memmap_array_view(tmpdir):
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = tmpdir.mkdir('pool').strpath
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = tmpdir.join('test.mmap').strpath
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert not isinstance(a_view, np.memmap)
assert has_shareable_memory(a_view)
p.map(inplace_double, [(a_view, (i, j), 1.0)
for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert os.listdir(pool_temp_folder) == []
finally:
p.terminate()
del p
def test_pool_with_memmap_array_view(factory, tmpdir):
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = tmpdir.mkdir('pool').strpath
p = factory(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = tmpdir.join('test.mmap').strpath
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert not isinstance(a_view, np.memmap)
assert has_shareable_memory(a_view)
p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert os.listdir(pool_temp_folder) == []
finally:
p.terminate()
del p
def test_hash_numpy_noncontiguous():
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order='F')[:, :1, :]
b = np.ascontiguousarray(a)
assert hash(a) != hash(b)
c = np.asfortranarray(a)
assert hash(a) != hash(c)
def test_hash_memmap():
""" Check that memmap and arrays hash identically if coerce_mmap is
True.
"""
filename = tempfile.mktemp(prefix='joblib_test_hash_memmap_')
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
for coerce_mmap in (False, True):
yield (nose.tools.assert_equal,
hash(a, coerce_mmap=coerce_mmap) == hash(
m, coerce_mmap=coerce_mmap), coerce_mmap)
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
try:
os.unlink(filename)
except OSError as e:
# Under windows, some files don't get erased.
if not os.name == 'nt':
raise e
def test_hash_numpy_noncontiguous():
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order='F')[:, :1, :]
b = np.ascontiguousarray(a)
nose.tools.assert_not_equal(hash(a), hash(b))
c = np.asfortranarray(a)
nose.tools.assert_not_equal(hash(a), hash(c))
def test_hash_numpy_noncontiguous():
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
b = np.ascontiguousarray(a)
assert hash(a) != hash(b)
c = np.asfortranarray(a)
assert hash(a) != hash(c)
def test_hash_numpy_noncontiguous():
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
b = np.ascontiguousarray(a)
nose.tools.assert_not_equal(hash(a), hash(b))
c = np.asfortranarray(a)
nose.tools.assert_not_equal(hash(a), hash(c))
def test_hash_memmap(tmpdir, coerce_mmap):
"""Check that memmap and arrays hash identically if coerce_mmap is True."""
filename = tmpdir.join('memmap_temp').strpath
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
are_hashes_equal = (hash(a, coerce_mmap=coerce_mmap) == hash(
m, coerce_mmap=coerce_mmap))
assert are_hashes_equal == coerce_mmap
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
def test_hash_memmap(tmpdir, coerce_mmap):
"""Check that memmap and arrays hash identically if coerce_mmap is True."""
filename = tmpdir.join('memmap_temp').strpath
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
are_hashes_equal = (hash(a, coerce_mmap=coerce_mmap) ==
hash(m, coerce_mmap=coerce_mmap))
assert are_hashes_equal == coerce_mmap
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
def test_workaround_against_bad_memmap_with_copied_buffers(tmpdir):
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert not has_shareable_memory(b)
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
def test_workaround_against_bad_memmap_with_copied_buffers(factory, tmpdir):
"""Check that memmaps with a bad buffer are returned as regular arrays
Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
"""
assert_array_equal = np.testing.assert_array_equal
p = factory(3, max_nbytes=10, temp_folder=tmpdir.strpath)
try:
# Send a complex, large-ish view on a array that will be converted to
# a memmap in the worker process
a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
order='F')[:, :1, :]
# Call a non-inplace multiply operation on the worker and memmap and
# send it back to the parent.
b = p.apply_async(_worker_multiply, args=(a, 3)).get()
assert not has_shareable_memory(b)
assert_array_equal(b, 3 * a)
finally:
p.terminate()
del p
def __reduce__(self):
return _load_sub_array, (np.asarray(self), )
def test_memmap_based_array_reducing(tmpdir):
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4),
mode='r+', order='F', offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapReducer(None, tmpdir.strpath, 'c')
def reconstruct_array(x):
cons, args = reducer(x)
return cons(*args)
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array(c)
assert not isinstance(c_reconstructed, np.memmap)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array(d)
assert not isinstance(d_reconstructed, np.memmap)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert not has_shareable_memory(a3)
a3_reconstructed = reconstruct_memmap(a3)
assert not has_shareable_memory(a3_reconstructed)
assert not isinstance(a3_reconstructed, np.memmap)
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert not has_shareable_memory(b3)
b3_reconstructed = reconstruct_array(b3)
assert isinstance(b3_reconstructed, np.ndarray)
assert not has_shareable_memory(b3_reconstructed)
assert_array_equal(b3_reconstructed, b3)
def test_memmap_based_array_reducing(tmpdir):
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = -1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename,
dtype=np.float64,
shape=(3, 5, 4),
mode='r+',
order='F',
offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapReducer(None, tmpdir.strpath, 'c')
def reconstruct_array(x):
cons, args = reducer(x)
return cons(*args)
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array(c)
assert not isinstance(c_reconstructed, np.memmap)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array(d)
assert not isinstance(d_reconstructed, np.memmap)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert not has_shareable_memory(a3)
a3_reconstructed = reconstruct_memmap(a3)
assert not has_shareable_memory(a3_reconstructed)
assert not isinstance(a3_reconstructed, np.memmap)
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert not has_shareable_memory(b3)
b3_reconstructed = reconstruct_array(b3)
assert isinstance(b3_reconstructed, np.ndarray)
assert not has_shareable_memory(b3_reconstructed)
assert_array_equal(b3_reconstructed, b3)
def __reduce__(self):
return _load_sub_array, (np.asarray(self), )
