def test_writer(self):
if io.h5py is None:
logger.warning(
"H5py is absent on the system, skip HDF5 writing test")
return
h5file = os.path.join(self.tmpdir, "junk.h5")
shape = 1024, 1024
n = 100
m = 10 # number of frames in memory
data = numpy.random.random(
(m, shape[0], shape[1])).astype(numpy.float32)
nmbytes = data.nbytes / 1e6 * n / m
t0 = time.perf_counter()
writer = io.HDF5Writer(filename=h5file, hpath="data")
writer.init({"nbpt_azim": shape[0], "nbpt_rad": shape[1]})
for i in range(n):
writer.write(data[i % m], i)
writer.close()
t = time.perf_counter() - t0
logger.info(
"Writing of HDF5 of %ix%s (%.3fMB) took %.3f (%.3fMByte/s)", n,
shape, nmbytes, t, nmbytes / t)
statinfo = os.stat(h5file)
self.assertTrue(
statinfo.st_size / 1e6 > nmbytes,
"file size (%s) is larger than dataset" % statinfo.st_size)
def bench_hdf5(n=1024,
shape=(1024, 1024),
dtype="float32",
dirname=None,
bsize=10):
"""
Actually performs the HDF5 writing benchmark
@param n: number of frames to be written
@param shape: 2-tuple of integer describing the shape of the image
@param bsize: number of frames in buffer
"""
tmp_dir = tempfile.mkdtemp(dir=dirname)
h5file = os.path.join(tmp_dir, "junk.h5")
logger.info("Writing large dataset %ix(%i,%i) of %s to %s." %
(n, shape[0], shape[1], dtype, h5file))
dtype = numpy.dtype(dtype)
if dtype.kind == "f":
data = numpy.random.random((bsize, shape[0], shape[1])).astype(dtype)
elif dtype.name.find("int") >= 0:
size = bsize * shape[0] * shape[1]
maxi = 2**(dtype.itemsize * 8 - 1) - 1
data = numpy.random.random_integers(0, maxi, size=size).astype(dtype)
data.shape = (bsize, shape[0], shape[1])
else:
raise RuntimeError("unhandled data type %s" % dtype)
size = n * shape[0] * shape[1]
nbytes = size * dtype.itemsize
nmbytes = nbytes / 1e6
t0 = time.time()
writer = io.HDF5Writer(filename=h5file, hpath="data")
writer.init({
"nbpt_azim": shape[0],
"nbpt_rad": shape[1],
"dtype": dtype.name
})
for i in range(n):
writer.write(data[i % bsize], i)
writer.close()
t = time.time() - t0
bps = nbytes / t
logger.info("Writing of %.3fMB in HDF5 took %.3fs (%.3f MByte/s)" %
(nmbytes, t, nmbytes / t))
statinfo = os.stat(h5file)
assert statinfo.st_size > nbytes
# Clean up
os.unlink(h5file)
os.removedirs(tmp_dir)
return bps
if os.path.isdir(hurl):
# write .dat or .edf files ...
if options.cake < 2:
writer = io.AsciiWriter(hurl)
# Else HDF5
else:
if hurl.startswith("hdf5:"):
hurl = hurl[5:]
if ":" in hurl:
hsplit = hurl.split(":")
hdfpath = hsplit[-1]
hdffile = ":".join(hsplit[:-1]) # special windows
else:
hdfpath = "test_LImA+pyFAI"
hdffile = hurl
writer = io.HDF5Writer(hdffile, hdfpath, options.scan)
elif len(args) > 1:
logger.error(
"Specify the HDF5 output file like hdf5:///home/user/filename.h5:/path/to/group"
)
sys.exit(1)
else:
writer = None
if options.verbose:
logger.info("setLevel: debug")
logger.setLevel(logging.DEBUG)
if options.lima:
sys.path.insert(0, options.lima)
try:
from Lima import Core, Basler
