def test_create(comm):
fname = tempfile.mkdtemp()
x = BigFile(fname, create=True)
x.create('.')
for name, d in dtypes:
d = numpy.dtype(d)
numpy.random.seed(1234)
# test creating
with x.create(name, Nfile=1, dtype=d, size=128) as b:
data = numpy.random.uniform(100000, size=128*128).view(dtype=b.dtype.base).reshape([-1] + list(d.shape))[:b.size]
b.write(0, data)
with x[name] as b:
assert_equal(b[:], data.astype(d.base))
assert_equal(b[:], b[...])
# test creating
data = numpy.random.uniform(100000, size=128*128).view(dtype=d.base).reshape([-1] + list(d.shape))[:128]
with x.create_from_array(name, data) as b:
pass
with x[name] as b:
assert_equal(b[:], data)
# test writing with an offset
with x[name] as b:
b.write(1, data[0:1])
assert_equal(b[1:2], data[0:1].astype(d.base))
# test writing beyond file length
with x[name] as b:
caught = False
try:
b.write(1, data)
except:
caught = True
assert caught
assert_equal(set(x.blocks), set([name for name, d in dtypes]))
import os
os.system("ls -r %s" % fname)
for b in x.blocks:
assert b in x
for b in x:
assert b in x
bd = BigData(x)
assert set(bd.dtype.names) == set(x.blocks)
d = bd[:]
shutil.rmtree(fname)
def test_mpi_create(comm):
if comm.rank == 0:
fname = tempfile.mkdtemp()
fname = comm.bcast(fname)
else:
fname = comm.bcast(None)
x = BigFileMPI(comm, fname, create=True)
for d in dtypes:
d = numpy.dtype(d)
numpy.random.seed(1234)
# test creating
with x.create(d.str, Nfile=1, dtype=d, size=128) as b:
data = numpy.random.uniform(100000, size=128 * 128).view(
dtype=b.dtype.base).reshape([-1] + list(d.shape))[:b.size]
b.write(0, data)
with x[d.str] as b:
assert_equal(b[:], data.astype(d.base))
# test writing with an offset
with x[d.str] as b:
b.write(1, data[0:1])
assert_equal(b[1:2], data[0:1].astype(d.base))
# test writing beyond file length
with x[d.str] as b:
caught = False
try:
b.write(1, data)
except:
caught = True
assert caught
assert_equal(set(x.blocks), set([numpy.dtype(d).str for d in dtypes]))
for b in x.blocks:
assert b in x
for b in x:
assert b in x
bd = BigData(x)
assert set(bd.dtype.names) == set(x.blocks)
d = bd[:]
comm.barrier()
if comm.rank == 0:
shutil.rmtree(fname)
def test_bigdata(comm):
fname = tempfile.mkdtemp()
x = BigFile(fname, create=True)
x.create('.')
for d in dtypes:
dt = numpy.dtype(d)
numpy.random.seed(1234)
# test creating
with x.create(str(d), Nfile=1, dtype=dt, size=128) as b:
data = numpy.random.uniform(100000, size=128 * 128).view(
dtype=b.dtype.base).reshape([-1] + list(dt.shape))[:b.size]
b.write(0, data)
bd = BigData(x)
assert set(bd.dtype.names) == set(x.blocks)
assert isinstance(bd[:], numpy.ndarray)
assert isinstance(bd['f8'], BigBlock)
assert_equal(len(bd['f8'].dtype), 0)
# tuple of one item is the same as non-tuple
assert isinstance(bd[('f8', )], BigBlock)
assert_equal(len(bd[('f8', )].dtype), 0)
assert isinstance(bd['f8', :10], numpy.ndarray)
assert_equal(len(bd['f8', :10]), 10)
assert_equal(len(bd['f8', :10].dtype), 0)
assert_equal(len(bd[[
'f8',
], :10].dtype), 1)
# tuple of one item is the same as non-tuple
assert_equal(len(bd[('f8', ), :10].dtype), 0)
assert isinstance(bd[:10, 'f8'], numpy.ndarray)
assert isinstance(bd['f8'], BigBlock)
assert isinstance(bd[['f8', 'f4'], ], BigData)
assert_equal(len(bd[['f8', 'f4'], ].dtype), 2)
assert isinstance(bd[['f8', 'f4'], :10], numpy.ndarray)
shutil.rmtree(fname)