def test_writing_two_datasets(self):
h5py = import_or_skip('h5py')
datalen = 33
dac = Context(self.client)
da = dac.empty((datalen,), dist={0: 'b'})
for i in range(datalen):
da[i] = i
output_path = temp_filepath('.hdf5')
try:
# make a file, and write to dataset 'foo'
with h5py.File(output_path, 'w') as fp:
fp['foo'] = np.arange(10)
# try saving to a different dataset
dac.save_hdf5(output_path, da, key='bar', mode='a')
with h5py.File(output_path, 'r') as fp:
self.assertTrue("foo" in fp)
self.assertTrue("bar" in fp)
finally:
if os.path.exists(output_path):
os.remove(output_path)
def setUp(self):
self.h5py = import_or_skip('h5py')
self.dac = Context(targets=[0, 1])
self.output_path = temp_filepath('.hdf5')
self.expected = np.arange(20).reshape(2, 10)
with self.h5py.File(self.output_path, 'w') as fp:
fp["test"] = self.expected
def setUp(self):
self.dac = Context(targets=[0, 1])
# make a test file
self.output_path = temp_filepath('.npy')
self.expected = np.arange(20).reshape(2, 10)
np.save(self.output_path, self.expected)
def test_write_3d(self):
h5py = import_or_skip('h5py')
shape = (4, 5, 3)
source = np.random.random(shape)
dac = Context(self.client)
dist = {0: 'b', 1: 'c', 2: 'n'}
da = dac.empty(shape, dist=dist)
for i in range(shape[0]):
for j in range(shape[1]):
for k in range(shape[2]):
da[i, j, k] = source[i, j, k]
output_path = temp_filepath('.hdf5')
try:
dac.save_hdf5(output_path, da, mode='w')
self.assertTrue(os.path.exists(output_path))
with h5py.File(output_path, 'r') as fp:
self.assertTrue("buffer" in fp)
assert_allclose(source, fp["buffer"])
finally:
if os.path.exists(output_path):
os.remove(output_path)
def setUp(self):
self.rank = self.comm.Get_rank()
self.h5py = import_or_skip('h5py')
self.key = "data"
# set up a common file to work with
if self.rank == 0:
self.output_path = temp_filepath(extension='.hdf5')
else:
self.output_path = None
self.output_path = self.comm.bcast(self.output_path, root=0)
def setUp(self):
self.rank = self.comm.Get_rank()
self.h5py = import_or_skip('h5py')
self.key = "data"
# set up a common file to work with
if self.rank == 0:
self.output_path = temp_filepath(extension='.hdf5')
else:
self.output_path = None
self.output_path = self.comm.bcast(self.output_path, root=0)
def test_save_load_with_filenames(self):
dac = Context(self.client)
da = dac.empty((100,), dist={0: 'b'})
output_paths = [temp_filepath() for target in dac.targets]
try:
dac.save(output_paths, da)
db = dac.load(output_paths)
self.assertTrue(isinstance(db, DistArray))
self.assertEqual(da, db)
finally:
for filepath in output_paths:
if os.path.exists(filepath):
os.remove(filepath)
def test_save_load_with_prefix(self):
dac = Context(self.client)
da = dac.empty((100,), dist={0: 'b'})
output_path = temp_filepath()
try:
dac.save(output_path, da)
db = dac.load(output_path)
self.assertTrue(isinstance(db, DistArray))
self.assertEqual(da, db)
finally:
for rank in dac.targets:
filepath = output_path + "_" + str(rank) + ".dnpy"
if os.path.exists(filepath):
os.remove(filepath)
def setUp(self):
self.rank = self.comm.Get_rank()
self.h5py = import_or_skip('h5py')
self.key = "data"
self.expected = numpy.arange(20).reshape(2, 10)
# set up a common file to work with
if self.rank == 0:
self.output_path = temp_filepath(extension='.hdf5')
with self.h5py.File(self.output_path, 'w') as fp:
fp[self.key] = self.expected
else:
self.output_path = None
self.comm.Barrier() # wait until file exists
self.output_path = self.comm.bcast(self.output_path, root=0)
def setUp(self):
self.rank = self.comm.Get_rank()
self.h5py = import_or_skip('h5py')
self.key = "data"
self.expected = numpy.arange(20).reshape(2, 10)
# set up a common file to work with
if self.rank == 0:
self.output_path = temp_filepath(extension='.hdf5')
with self.h5py.File(self.output_path, 'w') as fp:
fp[self.key] = self.expected
else:
self.output_path = None
self.comm.Barrier() # wait until file exists
self.output_path = self.comm.bcast(self.output_path, root=0)
def setUp(self):
self.rank = self.comm.Get_rank()
# set up a common filename to work with
if self.rank == 0:
self.output_path = temp_filepath(extension='.npy')
else:
self.output_path = None
self.output_path = self.comm.bcast(self.output_path, root=0)
# save some data to that file
self.expected = numpy.arange(20).reshape(2, 10)
if self.rank == 0:
numpy.save(self.output_path, self.expected)
self.comm.Barrier()
def setUp(self):
self.rank = self.comm.Get_rank()
# set up a common filename to work with
if self.rank == 0:
self.output_path = temp_filepath(extension='.npy')
else:
self.output_path = None
self.output_path = self.comm.bcast(self.output_path, root=0)
# save some data to that file
self.expected = numpy.arange(20).reshape(2, 10)
if self.rank == 0:
numpy.save(self.output_path, self.expected)
self.comm.Barrier()
def test_write_block(self):
h5py = import_or_skip('h5py')
datalen = 33
dac = Context(self.client)
da = dac.empty((datalen,), dist={0: 'b'})
for i in range(datalen):
da[i] = i
output_path = temp_filepath('.hdf5')
try:
dac.save_hdf5(output_path, da, mode='w')
self.assertTrue(os.path.exists(output_path))
with h5py.File(output_path, 'r') as fp:
self.assertTrue("buffer" in fp)
expected = np.arange(datalen)
assert_equal(expected, fp["buffer"])
finally:
if os.path.exists(output_path):
os.remove(output_path)
def setUp(self):
self.larr0 = LocalArray((7,), comm=self.comm)
self.output_path = temp_filepath(extension='.dnpy')
def setUp(self):
d = Distribution.from_shape(comm=self.comm, shape=(7, ))
self.larr0 = LocalArray(d)
# a different file on every engine
self.output_path = temp_filepath(extension='.dnpy')
def setUp(self):
d = Distribution.from_shape((7,), comm=self.comm)
self.larr0 = LocalArray(d)
# a different file on every engine
self.output_path = temp_filepath(extension='.dnpy')
def engine_temp_path(extension=''):
from distarray.testing import temp_filepath
return temp_filepath(extension)
def save_test_file(data):
import numpy
from distarray.testing import temp_filepath
output_path = temp_filepath('.npy')
numpy.save(output_path, data)
return output_path
def setUp(self):
self.h5py = import_or_skip('h5py')
self.output_path = temp_filepath('.hdf5')
self.dac = Context()
