def test_pool_with_memmap():
"""Check that subprocess can access and update shared memory memmap"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
os.makedirs(pool_temp_folder)
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
p.map(inplace_double, [(a, (i, j), 1.0) for i in range(a.shape[0])
for j in range(a.shape[1])])
assert_array_equal(a, 2 * np.ones(a.shape))
# Open a copy-on-write view on the previous data
b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c')
p.map(inplace_double, [(b, (i, j), 2.0) for i in range(b.shape[0])
for j in range(b.shape[1])])
# Passing memmap instances to the pool should not trigger the creation
# of new files on the FS
assert_equal(os.listdir(pool_temp_folder), [])
# the original data is untouched
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(b, 2 * np.ones(b.shape))
# readonly maps can be read but not updated
c = np.memmap(filename,
dtype=np.float32,
shape=(10, ),
mode='r',
offset=5 * 4)
assert_raises(AssertionError, p.map, check_array,
[(c, i, 3.0) for i in range(c.shape[0])])
# depending on the version of numpy one can either get a RuntimeError
# or a ValueError
assert_raises((RuntimeError, ValueError), p.map, inplace_double,
[(c, i, 2.0) for i in range(c.shape[0])])
finally:
# Clean all filehandlers held by the pool
p.terminate()
del p
def test_pool_with_memmap(tmpdir_path):
"""Check that subprocess can access and update shared memory memmap"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = os.path.join(tmpdir_path, 'pool')
os.makedirs(pool_temp_folder)
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = os.path.join(tmpdir_path, 'test.mmap')
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
p.map(inplace_double, [(a, (i, j), 1.0)
for i in range(a.shape[0])
for j in range(a.shape[1])])
assert_array_equal(a, 2 * np.ones(a.shape))
# Open a copy-on-write view on the previous data
b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c')
p.map(inplace_double, [(b, (i, j), 2.0)
for i in range(b.shape[0])
for j in range(b.shape[1])])
# Passing memmap instances to the pool should not trigger the creation
# of new files on the FS
assert os.listdir(pool_temp_folder) == []
# the original data is untouched
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(b, 2 * np.ones(b.shape))
# readonly maps can be read but not updated
c = np.memmap(filename, dtype=np.float32, shape=(10,), mode='r',
offset=5 * 4)
assert_raises(AssertionError, p.map, check_array,
[(c, i, 3.0) for i in range(c.shape[0])])
# depending on the version of numpy one can either get a RuntimeError
# or a ValueError
assert_raises((RuntimeError, ValueError), p.map, inplace_double,
[(c, i, 2.0) for i in range(c.shape[0])])
finally:
# Clean all filehandlers held by the pool
p.terminate()
del p
def test_hash_memmap():
""" Check that memmap and arrays hash identically if coerce_mmap is
True.
"""
filename = tempfile.mktemp(prefix='joblib_test_hash_memmap_')
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
for coerce_mmap in (False, True):
yield (assert_equal,
hash(a, coerce_mmap=coerce_mmap) ==
hash(m, coerce_mmap=coerce_mmap),
coerce_mmap)
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
try:
os.unlink(filename)
except OSError as e:
# Under windows, some files don't get erased.
if not os.name == 'nt':
raise e
def test_pool_with_memmap_array_view(tmpdir):
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = tmpdir.mkdir('pool').strpath
p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = tmpdir.join('test.mmap').strpath
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert not isinstance(a_view, np.memmap)
assert has_shareable_memory(a_view)
p.map(inplace_double, [(a_view, (i, j), 1.0)
for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert os.listdir(pool_temp_folder) == []
finally:
p.terminate()
del p
def test_hash_memmap():
""" Check that memmap and arrays hash identically if coerce_mmap is
True.
"""
filename = tempfile.mktemp(prefix='joblib_test_hash_memmap_')
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
for coerce_mmap in (False, True):
yield (nose.tools.assert_equal,
hash(a, coerce_mmap=coerce_mmap) == hash(
m, coerce_mmap=coerce_mmap), coerce_mmap)
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
try:
os.unlink(filename)
except OSError as e:
# Under windows, some files don't get erased.
if not os.name == 'nt':
raise e
def test_pool_with_memmap_array_view(factory, tmpdir):
"""Check that subprocess can access and update shared memory array"""
assert_array_equal = np.testing.assert_array_equal
# Fork the subprocess before allocating the objects to be passed
pool_temp_folder = tmpdir.mkdir('pool').strpath
p = factory(10, max_nbytes=2, temp_folder=pool_temp_folder)
try:
filename = tmpdir.join('test.mmap').strpath
a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
a.fill(1.0)
# Create an ndarray view on the memmap instance
a_view = np.asarray(a)
assert not isinstance(a_view, np.memmap)
assert has_shareable_memory(a_view)
p.map(inplace_double, [(a_view, (i, j), 1.0) for i in range(a.shape[0])
for j in range(a.shape[1])])
# Both a and the a_view have been updated
assert_array_equal(a, 2 * np.ones(a.shape))
assert_array_equal(a_view, 2 * np.ones(a.shape))
# Passing memmap array view to the pool should not trigger the
# creation of new files on the FS
assert os.listdir(pool_temp_folder) == []
finally:
p.terminate()
del p
def test__strided_from_memmap(tmpdir):
fname = tmpdir.join('test.mmap').strpath
size = 5 * mmap.ALLOCATIONGRANULARITY
offset = mmap.ALLOCATIONGRANULARITY + 1
# This line creates the mmap file that is reused later
memmap_obj = np.memmap(fname, mode='w+', shape=size + offset)
# filename, dtype, mode, offset, order, shape, strides, total_buffer_len
memmap_obj = _strided_from_memmap(fname,
dtype='uint8',
mode='r',
offset=offset,
order='C',
shape=size,
strides=None,
total_buffer_len=None)
assert isinstance(memmap_obj, np.memmap)
assert memmap_obj.offset == offset
memmap_backed_obj = _strided_from_memmap(fname,
dtype='uint8',
mode='r',
offset=offset,
order='C',
shape=(size // 2, ),
strides=(2, ),
total_buffer_len=size)
assert _get_backing_memmap(memmap_backed_obj).offset == offset
def test_numpy_persistence():
filename = env['filename']
rnd = np.random.RandomState(0)
a = rnd.random_sample((10, 2))
for compress in (False, True, 0, 3):
# We use 'a.T' to have a non C-contiguous array.
for index, obj in enumerate(((a,), (a.T,), (a, a), [a, a, a])):
# Change the file name to avoid side effects between tests
this_filename = filename + str(random.randint(0, 1000))
filenames = numpy_pickle.dump(obj, this_filename,
compress=compress)
# All is cached in one file
nose.tools.assert_equal(len(filenames), 1)
# Check that only one file was created
nose.tools.assert_equal(filenames[0], this_filename)
# Check that this file does exist
nose.tools.assert_true(
os.path.exists(os.path.join(env['dir'], filenames[0])))
# Unpickle the object
obj_ = numpy_pickle.load(this_filename)
# Check that the items are indeed arrays
for item in obj_:
nose.tools.assert_true(isinstance(item, np.ndarray))
# And finally, check that all the values are equal.
np.testing.assert_array_equal(np.array(obj), np.array(obj_))
# Now test with array subclasses
for obj in (np.matrix(np.zeros(10)),
np.memmap(filename + str(random.randint(0, 1000)) + 'mmap',
mode='w+', shape=4, dtype=np.float)):
this_filename = filename + str(random.randint(0, 1000))
filenames = numpy_pickle.dump(obj, this_filename,
compress=compress)
# All is cached in one file
nose.tools.assert_equal(len(filenames), 1)
obj_ = numpy_pickle.load(this_filename)
if (type(obj) is not np.memmap and
hasattr(obj, '__array_prepare__')):
# We don't reconstruct memmaps
nose.tools.assert_true(isinstance(obj_, type(obj)))
np.testing.assert_array_equal(obj_, obj)
# Test with an object containing multiple numpy arrays
obj = ComplexTestObject()
filenames = numpy_pickle.dump(obj, this_filename,
compress=compress)
# All is cached in one file
nose.tools.assert_equal(len(filenames), 1)
obj_loaded = numpy_pickle.load(this_filename)
nose.tools.assert_true(isinstance(obj_loaded, type(obj)))
np.testing.assert_array_equal(obj_loaded.array_float, obj.array_float)
np.testing.assert_array_equal(obj_loaded.array_int, obj.array_int)
np.testing.assert_array_equal(obj_loaded.array_obj, obj.array_obj)
def test_numpy_persistence(tmpdir, compress):
filename = tmpdir.join('test.pkl').strpath
rnd = np.random.RandomState(0)
a = rnd.random_sample((10, 2))
# We use 'a.T' to have a non C-contiguous array.
for index, obj in enumerate(((a, ), (a.T, ), (a, a), [a, a, a])):
filenames = numpy_pickle.dump(obj, filename, compress=compress)
# All is cached in one file
assert len(filenames) == 1
# Check that only one file was created
assert filenames[0] == filename
# Check that this file does exist
assert os.path.exists(filenames[0])
# Unpickle the object
obj_ = numpy_pickle.load(filename)
# Check that the items are indeed arrays
for item in obj_:
assert isinstance(item, np.ndarray)
# And finally, check that all the values are equal.
np.testing.assert_array_equal(np.array(obj), np.array(obj_))
# Now test with array subclasses
for obj in (np.matrix(np.zeros(10)),
np.memmap(filename + 'mmap',
mode='w+',
shape=4,
dtype=np.float)):
filenames = numpy_pickle.dump(obj, filename, compress=compress)
# All is cached in one file
assert len(filenames) == 1
obj_ = numpy_pickle.load(filename)
if (type(obj) is not np.memmap and hasattr(obj, '__array_prepare__')):
# We don't reconstruct memmaps
assert isinstance(obj_, type(obj))
np.testing.assert_array_equal(obj_, obj)
# Test with an object containing multiple numpy arrays
obj = ComplexTestObject()
filenames = numpy_pickle.dump(obj, filename, compress=compress)
# All is cached in one file
assert len(filenames) == 1
obj_loaded = numpy_pickle.load(filename)
assert isinstance(obj_loaded, type(obj))
np.testing.assert_array_equal(obj_loaded.array_float, obj.array_float)
np.testing.assert_array_equal(obj_loaded.array_int, obj.array_int)
np.testing.assert_array_equal(obj_loaded.array_obj, obj.array_obj)
def test_hash_memmap(tmpdir, coerce_mmap):
"""Check that memmap and arrays hash identically if coerce_mmap is True."""
filename = tmpdir.join('memmap_temp').strpath
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
are_hashes_equal = (hash(a, coerce_mmap=coerce_mmap) == hash(
m, coerce_mmap=coerce_mmap))
assert are_hashes_equal == coerce_mmap
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
def test_hash_memmap(tmpdir, coerce_mmap):
"""Check that memmap and arrays hash identically if coerce_mmap is True."""
filename = tmpdir.join('memmap_temp').strpath
try:
m = np.memmap(filename, shape=(10, 10), mode='w+')
a = np.asarray(m)
are_hashes_equal = (hash(a, coerce_mmap=coerce_mmap) ==
hash(m, coerce_mmap=coerce_mmap))
assert are_hashes_equal == coerce_mmap
finally:
if 'm' in locals():
del m
# Force a garbage-collection cycle, to be certain that the
# object is delete, and we don't run in a problem under
# Windows with a file handle still open.
gc.collect()
def test__strided_from_memmap(tmpdir):
fname = tmpdir.join('test.mmap').strpath
size = 5 * mmap.ALLOCATIONGRANULARITY
offset = mmap.ALLOCATIONGRANULARITY + 1
# This line creates the mmap file that is reused later
memmap_obj = np.memmap(fname, mode='w+', shape=size + offset)
# filename, dtype, mode, offset, order, shape, strides, total_buffer_len
memmap_obj = _strided_from_memmap(fname, dtype='uint8', mode='r',
offset=offset, order='C', shape=size,
strides=None, total_buffer_len=None)
assert isinstance(memmap_obj, np.memmap)
assert memmap_obj.offset == offset
memmap_backed_obj = _strided_from_memmap(fname, dtype='uint8', mode='r',
offset=offset, order='C',
shape=(size // 2,), strides=(2,),
total_buffer_len=size)
assert _get_backing_memmap(memmap_backed_obj).offset == offset
def test_high_dimension_memmap_array_reducing(tmpdir):
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a high dimensional memmap
a = np.memmap(filename,
dtype=np.float64,
shape=(100, 15, 15, 3),
mode='w+')
a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape)
# Create some slices/indices at various dimensions
b = a[0:10]
c = a[:, 5:10]
d = a[:, :, :, 0]
e = a[1:3:4]
# Array reducer with auto dumping disabled
reducer = ArrayMemmapForwardReducer(None, tmpdir.strpath, 'c', True)
def reconstruct_array_or_memmap(x):
cons, args = reducer(x)
return cons(*args)
a_reconstructed = reconstruct_array_or_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
b_reconstructed = reconstruct_array_or_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
c_reconstructed = reconstruct_array_or_memmap(c)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array_or_memmap(d)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
e_reconstructed = reconstruct_array_or_memmap(e)
assert has_shareable_memory(e_reconstructed)
assert_array_equal(e_reconstructed, e)
def test_high_dimension_memmap_array_reducing():
assert_array_equal = np.testing.assert_array_equal
filename = os.path.join(TEMP_FOLDER, 'test.mmap')
# Create a high dimensional memmap
a = np.memmap(filename,
dtype=np.float64,
shape=(100, 15, 15, 3),
mode='w+')
a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape)
# Create some slices/indices at various dimensions
b = a[0:10]
c = a[:, 5:10]
d = a[:, :, :, 0]
e = a[1:3:4]
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
res = cons(*args)
return res
a_reconstructed = reconstruct_memmap(a)
assert_true(has_shareable_memory(a_reconstructed))
assert_true(isinstance(a_reconstructed, np.memmap))
assert_array_equal(a_reconstructed, a)
b_reconstructed = reconstruct_memmap(b)
assert_true(has_shareable_memory(b_reconstructed))
assert_array_equal(b_reconstructed, b)
c_reconstructed = reconstruct_memmap(c)
assert_true(has_shareable_memory(c_reconstructed))
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_memmap(d)
assert_true(has_shareable_memory(d_reconstructed))
assert_array_equal(d_reconstructed, d)
e_reconstructed = reconstruct_memmap(e)
assert_true(has_shareable_memory(e_reconstructed))
assert_array_equal(e_reconstructed, e)
def test_high_dimension_memmap_array_reducing(tmpdir):
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a high dimensional memmap
a = np.memmap(filename, dtype=np.float64, shape=(100, 15, 15, 3),
mode='w+')
a[:] = np.arange(100 * 15 * 15 * 3).reshape(a.shape)
# Create some slices/indices at various dimensions
b = a[0:10]
c = a[:, 5:10]
d = a[:, :, :, 0]
e = a[1:3:4]
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
res = cons(*args)
return res
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
c_reconstructed = reconstruct_memmap(c)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_memmap(d)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
e_reconstructed = reconstruct_memmap(e)
assert has_shareable_memory(e_reconstructed)
assert_array_equal(e_reconstructed, e)
def test_numpy_persistence():
filename = env['filename']
rnd = np.random.RandomState(0)
a = rnd.random_sample((10, 2))
for compress in (False, True, 0, 3):
# We use 'a.T' to have a non C-contiguous array.
for index, obj in enumerate(((a, ), (a.T, ), (a, a), [a, a, a])):
# Change the file name to avoid side effects between tests
this_filename = filename + str(random.randint(0, 1000))
filenames = numpy_pickle.dump(obj,
this_filename,
compress=compress)
# All is cached in one file
nose.tools.assert_equal(len(filenames), 1)
# Check that only one file was created
nose.tools.assert_equal(filenames[0], this_filename)
# Check that this file does exist
nose.tools.assert_true(
os.path.exists(os.path.join(env['dir'], filenames[0])))
# Unpickle the object
obj_ = numpy_pickle.load(this_filename)
# Check that the items are indeed arrays
for item in obj_:
nose.tools.assert_true(isinstance(item, np.ndarray))
# And finally, check that all the values are equal.
np.testing.assert_array_equal(np.array(obj), np.array(obj_))
# Now test with array subclasses
for obj in (np.matrix(np.zeros(10)),
np.memmap(filename + str(random.randint(0, 1000)) + 'mmap',
mode='w+',
shape=4,
dtype=np.float)):
this_filename = filename + str(random.randint(0, 1000))
filenames = numpy_pickle.dump(obj,
this_filename,
compress=compress)
# All is cached in one file
nose.tools.assert_equal(len(filenames), 1)
obj_ = numpy_pickle.load(this_filename)
if (type(obj) is not np.memmap
and hasattr(obj, '__array_prepare__')):
# We don't reconstruct memmaps
nose.tools.assert_true(isinstance(obj_, type(obj)))
np.testing.assert_array_equal(obj_, obj)
# Test with an object containing multiple numpy arrays
obj = ComplexTestObject()
filenames = numpy_pickle.dump(obj, this_filename, compress=compress)
# All is cached in one file
nose.tools.assert_equal(len(filenames), 1)
obj_loaded = numpy_pickle.load(this_filename)
nose.tools.assert_true(isinstance(obj_loaded, type(obj)))
np.testing.assert_array_equal(obj_loaded.array_float, obj.array_float)
np.testing.assert_array_equal(obj_loaded.array_int, obj.array_int)
np.testing.assert_array_equal(obj_loaded.array_obj, obj.array_obj)
def test_memmap_based_array_reducing(tmpdir):
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4),
mode='r+', order='F', offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapReducer(None, tmpdir.strpath, 'c')
def reconstruct_array(x):
cons, args = reducer(x)
return cons(*args)
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array(c)
assert not isinstance(c_reconstructed, np.memmap)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array(d)
assert not isinstance(d_reconstructed, np.memmap)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert not has_shareable_memory(a3)
a3_reconstructed = reconstruct_memmap(a3)
assert not has_shareable_memory(a3_reconstructed)
assert not isinstance(a3_reconstructed, np.memmap)
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert not has_shareable_memory(b3)
b3_reconstructed = reconstruct_array(b3)
assert isinstance(b3_reconstructed, np.ndarray)
assert not has_shareable_memory(b3_reconstructed)
assert_array_equal(b3_reconstructed, b3)
def test_memmap_based_array_reducing(tmpdir):
"""Check that it is possible to reduce a memmap backed array"""
assert_array_equal = np.testing.assert_array_equal
filename = tmpdir.join('test.mmap').strpath
# Create a file larger than what will be used by a
buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')
# Fill the original buffer with negative markers to detect over of
# underflow in case of test failures
buffer[:] = -1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
buffer.flush()
# Memmap a 2D fortran array on a offseted subsection of the previous
# buffer
a = np.memmap(filename,
dtype=np.float64,
shape=(3, 5, 4),
mode='r+',
order='F',
offset=4)
a[:] = np.arange(60).reshape(a.shape)
# Build various views that share the buffer with the original memmap
# b is an memmap sliced view on an memmap instance
b = a[1:-1, 2:-1, 2:4]
# c and d are array views
c = np.asarray(b)
d = c.T
# Array reducer with auto dumping disabled
reducer = ArrayMemmapReducer(None, tmpdir.strpath, 'c')
def reconstruct_array(x):
cons, args = reducer(x)
return cons(*args)
def reconstruct_memmap(x):
cons, args = reduce_memmap(x)
return cons(*args)
# Reconstruct original memmap
a_reconstructed = reconstruct_memmap(a)
assert has_shareable_memory(a_reconstructed)
assert isinstance(a_reconstructed, np.memmap)
assert_array_equal(a_reconstructed, a)
# Reconstruct strided memmap view
b_reconstructed = reconstruct_memmap(b)
assert has_shareable_memory(b_reconstructed)
assert_array_equal(b_reconstructed, b)
# Reconstruct arrays views on memmap base
c_reconstructed = reconstruct_array(c)
assert not isinstance(c_reconstructed, np.memmap)
assert has_shareable_memory(c_reconstructed)
assert_array_equal(c_reconstructed, c)
d_reconstructed = reconstruct_array(d)
assert not isinstance(d_reconstructed, np.memmap)
assert has_shareable_memory(d_reconstructed)
assert_array_equal(d_reconstructed, d)
# Test graceful degradation on fake memmap instances with in-memory
# buffers
a3 = a * 3
assert not has_shareable_memory(a3)
a3_reconstructed = reconstruct_memmap(a3)
assert not has_shareable_memory(a3_reconstructed)
assert not isinstance(a3_reconstructed, np.memmap)
assert_array_equal(a3_reconstructed, a * 3)
# Test graceful degradation on arrays derived from fake memmap instances
b3 = np.asarray(a3)
assert not has_shareable_memory(b3)
b3_reconstructed = reconstruct_array(b3)
assert isinstance(b3_reconstructed, np.ndarray)
assert not has_shareable_memory(b3_reconstructed)
assert_array_equal(b3_reconstructed, b3)
