def test_writing_two_datasets(self): h5py = import_or_skip('h5py') datalen = 33 dac = Context(self.client) da = dac.empty((datalen,), dist={0: 'b'}) for i in range(datalen): da[i] = i output_path = temp_filepath('.hdf5') try: # make a file, and write to dataset 'foo' with h5py.File(output_path, 'w') as fp: fp['foo'] = np.arange(10) # try saving to a different dataset dac.save_hdf5(output_path, da, key='bar', mode='a') with h5py.File(output_path, 'r') as fp: self.assertTrue("foo" in fp) self.assertTrue("bar" in fp) finally: if os.path.exists(output_path): os.remove(output_path)
def setUp(self): self.h5py = import_or_skip('h5py') self.dac = Context(targets=[0, 1]) self.output_path = temp_filepath('.hdf5') self.expected = np.arange(20).reshape(2, 10) with self.h5py.File(self.output_path, 'w') as fp: fp["test"] = self.expected
def setUp(self): self.dac = Context(targets=[0, 1]) # make a test file self.output_path = temp_filepath('.npy') self.expected = np.arange(20).reshape(2, 10) np.save(self.output_path, self.expected)
def test_write_3d(self): h5py = import_or_skip('h5py') shape = (4, 5, 3) source = np.random.random(shape) dac = Context(self.client) dist = {0: 'b', 1: 'c', 2: 'n'} da = dac.empty(shape, dist=dist) for i in range(shape[0]): for j in range(shape[1]): for k in range(shape[2]): da[i, j, k] = source[i, j, k] output_path = temp_filepath('.hdf5') try: dac.save_hdf5(output_path, da, mode='w') self.assertTrue(os.path.exists(output_path)) with h5py.File(output_path, 'r') as fp: self.assertTrue("buffer" in fp) assert_allclose(source, fp["buffer"]) finally: if os.path.exists(output_path): os.remove(output_path)
def setUp(self): self.rank = self.comm.Get_rank() self.h5py = import_or_skip('h5py') self.key = "data" # set up a common file to work with if self.rank == 0: self.output_path = temp_filepath(extension='.hdf5') else: self.output_path = None self.output_path = self.comm.bcast(self.output_path, root=0)
def test_save_load_with_filenames(self): dac = Context(self.client) da = dac.empty((100,), dist={0: 'b'}) output_paths = [temp_filepath() for target in dac.targets] try: dac.save(output_paths, da) db = dac.load(output_paths) self.assertTrue(isinstance(db, DistArray)) self.assertEqual(da, db) finally: for filepath in output_paths: if os.path.exists(filepath): os.remove(filepath)
def test_save_load_with_prefix(self): dac = Context(self.client) da = dac.empty((100,), dist={0: 'b'}) output_path = temp_filepath() try: dac.save(output_path, da) db = dac.load(output_path) self.assertTrue(isinstance(db, DistArray)) self.assertEqual(da, db) finally: for rank in dac.targets: filepath = output_path + "_" + str(rank) + ".dnpy" if os.path.exists(filepath): os.remove(filepath)
def setUp(self): self.rank = self.comm.Get_rank() self.h5py = import_or_skip('h5py') self.key = "data" self.expected = numpy.arange(20).reshape(2, 10) # set up a common file to work with if self.rank == 0: self.output_path = temp_filepath(extension='.hdf5') with self.h5py.File(self.output_path, 'w') as fp: fp[self.key] = self.expected else: self.output_path = None self.comm.Barrier() # wait until file exists self.output_path = self.comm.bcast(self.output_path, root=0)
def setUp(self): self.rank = self.comm.Get_rank() # set up a common filename to work with if self.rank == 0: self.output_path = temp_filepath(extension='.npy') else: self.output_path = None self.output_path = self.comm.bcast(self.output_path, root=0) # save some data to that file self.expected = numpy.arange(20).reshape(2, 10) if self.rank == 0: numpy.save(self.output_path, self.expected) self.comm.Barrier()
def test_write_block(self): h5py = import_or_skip('h5py') datalen = 33 dac = Context(self.client) da = dac.empty((datalen,), dist={0: 'b'}) for i in range(datalen): da[i] = i output_path = temp_filepath('.hdf5') try: dac.save_hdf5(output_path, da, mode='w') self.assertTrue(os.path.exists(output_path)) with h5py.File(output_path, 'r') as fp: self.assertTrue("buffer" in fp) expected = np.arange(datalen) assert_equal(expected, fp["buffer"]) finally: if os.path.exists(output_path): os.remove(output_path)
def setUp(self): self.larr0 = LocalArray((7,), comm=self.comm) self.output_path = temp_filepath(extension='.dnpy')
def setUp(self): d = Distribution.from_shape(comm=self.comm, shape=(7, )) self.larr0 = LocalArray(d) # a different file on every engine self.output_path = temp_filepath(extension='.dnpy')
def setUp(self): d = Distribution.from_shape((7,), comm=self.comm) self.larr0 = LocalArray(d) # a different file on every engine self.output_path = temp_filepath(extension='.dnpy')
def engine_temp_path(extension=''): from distarray.testing import temp_filepath return temp_filepath(extension)
def save_test_file(data): import numpy from distarray.testing import temp_filepath output_path = temp_filepath('.npy') numpy.save(output_path, data) return output_path
def setUp(self): self.h5py = import_or_skip('h5py') self.output_path = temp_filepath('.hdf5') self.dac = Context()