def test_mpi_attr(comm):
if comm.rank == 0:
fname = tempfile.mkdtemp()
fname = comm.bcast(fname)
else:
fname = comm.bcast(None)
x = BigFileMPI(comm, fname, create=True)
with x.create('.', dtype=None) as b:
b.attrs['int'] = 128
b.attrs['float'] = [128.0, 3, 4]
b.attrs['string'] = 'abcdefg'
with x.open('.') as b:
assert_equal(b.attrs['int'], 128)
assert_equal(b.attrs['float'], [128.0, 3, 4])
assert_equal(b.attrs['string'], 'abcdefg')
b.attrs['int'] = 30
b.attrs['float'] = [3, 4]
b.attrs['string'] = 'defg'
with x.open('.') as b:
assert_equal(b.attrs['int'], 30)
assert_equal(b.attrs['float'], [3, 4])
assert_equal(b.attrs['string'], 'defg')
comm.barrier()
if comm.rank == 0:
shutil.rmtree(fname)
def test_mpi_attr(comm):
if comm.rank == 0:
fname = tempfile.mkdtemp()
fname = comm.bcast(fname)
else:
fname = comm.bcast(None)
x = BigFileMPI(comm, fname, create=True)
with x.create('.', dtype=None) as b:
b.attrs['int'] = 128
b.attrs['float'] = [128.0, 3, 4]
b.attrs['string'] = 'abcdefg'
with x.open('.') as b:
assert_equal(b.attrs['int'], 128)
assert_equal(b.attrs['float'], [128.0, 3, 4])
assert_equal(b.attrs['string'], 'abcdefg')
b.attrs['int'] = 30
b.attrs['float'] = [3, 4]
b.attrs['string'] = 'defg'
with x.open('.') as b:
assert_equal(b.attrs['int'], 30)
assert_equal(b.attrs['float'], [3, 4])
assert_equal(b.attrs['string'], 'defg')
comm.barrier()
if comm.rank == 0:
shutil.rmtree(fname)
def test_mpi_create(comm):
if comm.rank == 0:
fname = tempfile.mkdtemp()
fname = comm.bcast(fname)
else:
fname = comm.bcast(None)
x = BigFileMPI(comm, fname, create=True)
for d in dtypes:
d = numpy.dtype(d)
numpy.random.seed(1234)
# test creating
with x.create(d.str, Nfile=1, dtype=d, size=128) as b:
data = numpy.random.uniform(100000, size=128 * 128).view(
dtype=b.dtype.base).reshape([-1] + list(d.shape))[:b.size]
b.write(0, data)
with x[d.str] as b:
assert_equal(b[:], data.astype(d.base))
# test writing with an offset
with x[d.str] as b:
b.write(1, data[0:1])
assert_equal(b[1:2], data[0:1].astype(d.base))
# test writing beyond file length
with x[d.str] as b:
caught = False
try:
b.write(1, data)
except:
caught = True
assert caught
assert_equal(set(x.blocks), set([numpy.dtype(d).str for d in dtypes]))
for b in x.blocks:
assert b in x
for b in x:
assert b in x
bd = BigData(x)
assert set(bd.dtype.names) == set(x.blocks)
d = bd[:]
comm.barrier()
if comm.rank == 0:
shutil.rmtree(fname)
def test_mpi_create(comm):
if comm.rank == 0:
fname = tempfile.mkdtemp()
fname = comm.bcast(fname)
else:
fname = comm.bcast(None)
x = BigFileMPI(comm, fname, create=True)
for d in dtypes:
d = numpy.dtype(d)
numpy.random.seed(1234)
# test creating
with x.create(d.str, Nfile=1, dtype=d, size=128) as b:
data = numpy.random.uniform(100000, size=128*128).view(dtype=b.dtype.base).reshape([-1] + list(d.shape))[:b.size]
b.write(0, data)
with x[d.str] as b:
assert_equal(b[:], data.astype(d.base))
# test writing with an offset
with x[d.str] as b:
b.write(1, data[0:1])
assert_equal(b[1:2], data[0:1].astype(d.base))
# test writing beyond file length
with x[d.str] as b:
caught = False
try:
b.write(1, data)
except:
caught = True
assert caught
assert_equal(set(x.blocks), set([numpy.dtype(d).str for d in dtypes]))
for b in x.blocks:
assert b in x
for b in x:
assert b in x
bd = BigData(x)
assert set(bd.dtype.names) == set(x.blocks)
d = bd[:]
comm.barrier()
if comm.rank == 0:
shutil.rmtree(fname)
