def get_output_dataset():
dname = '{}/{}'.format(backend.name, mpi_size())
fname = join(OUT_PATH, 'result.h5')
if mpi_root():
shape = (len(DATA_SIZE), len(CHUNK_SIZE), 3, NTESTS)
with h5.File(fname, 'a') as g:
if dname in g:
if g[dname].shape != shape:
raise ValueError('Dataset exists with invalid shape')
else:
g.create_dataset(dname, shape=shape, dtype=np.float32)
g[dname].attrs['axis0_label'] = 'Data Size'
g[dname].attrs['axis1_label'] = 'Chunk Size'
g[dname].attrs['axis2_label'] = 'Tests'
g[dname].attrs['axis3_label'] = '# Tests'
g[dname].attrs['axis0_value'] = DATA_SIZE
g[dname].attrs['axis1_value'] = CHUNK_SIZE
g[dname].attrs['axis2_value'] = np.array(['DATA', '3D', '1D'], dtype='S4')
g[dname].attrs['axis3_value'] = list(range(NTESTS))
mpi_barrier()
h5out = h5.File(fname, 'a')
h5result = h5out[dname]
return h5out, h5result
def create_random_dataset(DS):
pprint('Populating random data on disk', rank=0)
if mpi_root():
f = tempfile.NamedTemporaryFile()
data = np.random.rand(DS, DS, DS).astype(np.float32)
fname = f.name + '.h5'
with h5.File(fname, 'w') as g:
g.create_dataset('data', data=data)
else:
fname = None
fname = mpi_comm().bcast(fname, root=0)
pprint(fname)
h5in = h5.File(fname, 'r')
h5data = h5in['data']
return h5in, h5data
import logging as log
log.getLogger().setLevel(log.INFO)
dn.use(engine='mpi', backend='hdf5')
###############################################################################
DS = 256
CS = 50
engine, backend = dn.status()
pprint('Loading data', rank=0)
if mpi_root():
f = tempfile.NamedTemporaryFile()
data = np.random.rand(DS, DS, DS).astype(np.float32)
with h5.File(f.name, 'w') as g:
g.create_dataset('data', data=data)
fname = f.name
else:
fname = None
fname = mpi_comm().bcast(fname, root=0)
h5in = h5.File(fname, 'r')
h5data = h5in['data']
###############################################################################
with MpiTimer('DosNa %s (%s)' % (engine.name, backend.name)):