def main():
ctx = api.Context(executor=InlineJobExecutor())
ds = RawFileDataSet(path="/home/clausen/Data/EMPAD/scan_11_x256_y256.raw",
scan_size=(256, 256),
detector_size_raw=(130, 128),
crop_detector_to=(128, 128),
tileshape=(1, 8, 128, 128),
dtype="float32")
ds.initialize()
job = ctx.create_mask_analysis(dataset=ds,
factories=[lambda: np.ones(ds.shape.sig)])
result = ctx.run(job)
def test_missing_frames(lt_ctx, raw_data_8x8x8x8_path, io_backend):
ds = RawFileDataSet(
path=raw_data_8x8x8x8_path,
nav_shape=(10, 8),
sig_shape=(8, 8),
dtype="float32",
io_backend=io_backend,
)
ds.set_num_cores(4)
ds = ds.initialize(lt_ctx.executor)
tileshape = Shape((4, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
for p in ds.get_partitions():
for t in p.get_tiles(tiling_scheme=tiling_scheme):
pass
assert p._start_frame == 60
assert p._num_frames == 20
assert p.slice.origin == (60, 0, 0)
assert p.slice.shape[0] == 20
assert t.tile_slice.origin == (60, 0, 0)
assert t.tile_slice.shape[0] == 4
def medium_raw_float32(medium_raw_file_float32):
filename, shape, dtype = medium_raw_file_float32
ds = RawFileDataSet(path=str(filename),
nav_shape=shape[:2],
dtype=dtype,
sig_shape=shape[2:],
io_backend=MMapBackend())
ds = ds.initialize(InlineJobExecutor())
yield ds
def large_raw(large_raw_file):
filename, shape, dtype = large_raw_file
ds = RawFileDataSet(
path=str(filename),
nav_shape=shape[:2],
dtype=dtype,
sig_shape=shape[2:],
)
ds = ds.initialize(InlineJobExecutor())
yield ds
def large_raw(large_raw_file):
filename, shape, dtype = large_raw_file
ds = RawFileDataSet(
path=str(filename),
scan_size=shape[:2],
dtype=dtype,
detector_size=shape[2:],
)
ds.set_num_cores(2)
ds = ds.initialize(InlineJobExecutor())
yield ds
def raw_dataset_8x8x8x8(lt_ctx, raw_data_8x8x8x8_path):
ds = RawFileDataSet(
path=raw_data_8x8x8x8_path,
nav_shape=(8, 8),
sig_shape=(8, 8),
dtype="float32",
)
ds.set_num_cores(4)
ds = ds.initialize(lt_ctx.executor)
return ds
def raw_same_dataset_4d(tmpdir_factory, hdf5_4d_data):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-same-data-4d'
hdf5_4d_data.tofile(str(filename))
ds = RawFileDataSet(
path=str(filename),
nav_shape=(2, 10),
dtype="float32",
sig_shape=(26, 26),
)
ds.set_num_cores(4)
ds = ds.initialize(InlineJobExecutor())
yield ds
def uint16_raw(tmpdir_factory):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-default'
data = utils._mk_random(size=(16, 16, 128, 128), dtype='uint16')
data.tofile(str(filename))
del data
ds = RawFileDataSet(
path=str(filename),
scan_size=(16, 16),
dtype="uint16",
detector_size=(128, 128),
)
ds = ds.initialize(InlineJobExecutor())
yield ds
def default_raw(tmpdir_factory, default_raw_data):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-default'
default_raw_data.tofile(str(filename))
del default_raw_data
ds = RawFileDataSet(
path=str(filename),
scan_size=(16, 16),
dtype="float32",
detector_size=(128, 128),
)
ds.set_num_cores(2)
ds = ds.initialize(InlineJobExecutor())
yield ds
def default_raw(tmpdir_factory):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-default'
data = _mk_random(size=(16, 16, 128, 128), dtype='float32')
data.tofile(str(filename))
del data
ds = RawFileDataSet(
path=str(filename),
scan_size=(16, 16),
dtype="float32",
detector_size=(128, 128),
)
ds = ds.initialize()
yield ds
def raw_with_zeros(tmpdir_factory):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-with-zeros'
data = np.zeros((16, 16, 128, 128), dtype='float32')
data.tofile(str(filename))
del data
ds = RawFileDataSet(
path=str(filename),
scan_size=(16, 16),
dtype="float32",
detector_size=(128, 128),
)
ds.set_num_cores(2)
ds = ds.initialize(InlineJobExecutor())
yield ds
def big_endian_raw(tmpdir_factory):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-default-big-endian'
data = utils._mk_random(size=(16, 16, 128, 128), dtype='>u2')
data.tofile(str(filename))
del data
ds = RawFileDataSet(
path=str(filename),
nav_shape=(16, 16),
dtype=">u2",
sig_shape=(128, 128),
)
ds.set_num_cores(2)
ds = ds.initialize(InlineJobExecutor())
yield ds
def test_negative_sync_offset(lt_ctx, raw_dataset_8x8x8x8,
raw_data_8x8x8x8_path, io_backend):
udf = SumSigUDF()
sync_offset = -2
ds_with_offset = RawFileDataSet(
path=raw_data_8x8x8x8_path,
nav_shape=(8, 8),
sig_shape=(8, 8),
dtype="float32",
enable_direct=False,
sync_offset=sync_offset,
io_backend=io_backend,
)
ds_with_offset.set_num_cores(4)
ds_with_offset = ds_with_offset.initialize(lt_ctx.executor)
ds_with_offset.check_valid()
p0 = next(ds_with_offset.get_partitions())
assert p0._start_frame == -2
assert p0.slice.origin == (0, 0, 0)
tileshape = Shape((4, ) + tuple(ds_with_offset.shape.sig),
sig_dims=ds_with_offset.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds_with_offset.shape,
)
t0 = next(p0.get_tiles(tiling_scheme))
assert tuple(t0.tile_slice.origin) == (2, 0, 0)
for p in ds_with_offset.get_partitions():
for t in p.get_tiles(tiling_scheme=tiling_scheme):
pass
assert p.slice.origin == (48, 0, 0)
assert p.slice.shape[0] == 16
result = lt_ctx.run_udf(dataset=raw_dataset_8x8x8x8, udf=udf)
result = result['intensity'].raw_data[:raw_dataset_8x8x8x8._meta.
image_count - abs(sync_offset)]
result_with_offset = lt_ctx.run_udf(dataset=ds_with_offset, udf=udf)
result_with_offset = result_with_offset['intensity'].raw_data[
abs(sync_offset):]
assert np.allclose(result, result_with_offset)
def test_positive_sync_offset_raw(lt_ctx, benchmark, raw_data_8x8x8x8_path):
ds = RawFileDataSet(path=raw_data_8x8x8x8_path,
nav_shape=(8, 8),
sig_shape=(8, 8),
dtype="float32",
enable_direct=False,
sync_offset=2)
ds.set_num_cores(4)
ds = ds.initialize(lt_ctx.executor)
tileshape = Shape((4, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
p0 = next(ds.get_partitions())
benchmark(get_first_tile, p0=p0, tiling_scheme=tiling_scheme)
def test_too_many_frames(lt_ctx, raw_data_8x8x8x8_path, io_backend):
ds = RawFileDataSet(
path=raw_data_8x8x8x8_path,
nav_shape=(6, 8),
sig_shape=(8, 8),
dtype="float32",
io_backend=io_backend,
)
ds.set_num_cores(4)
ds = ds.initialize(lt_ctx.executor)
tileshape = Shape((4, ) + tuple(ds.shape.sig), sig_dims=ds.shape.sig.dims)
tiling_scheme = TilingScheme.make_for_shape(
tileshape=tileshape,
dataset_shape=ds.shape,
)
for p in ds.get_partitions():
for t in p.get_tiles(tiling_scheme=tiling_scheme):
pass
def large_raw(tmpdir_factory):
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-large-sparse'
shape = (100, 100, 1216, 1216)
dtype = np.uint16
size = np.prod(np.int64(shape)) * np.dtype(dtype).itemsize
if platform.system() == "Windows":
os.system('FSUtil File CreateNew "%s" 0x%X' % (filename, size))
os.system('FSUtil Sparse SetFlag "%s"' % filename)
os.system('FSUtil Sparse SetRange "%s" 0 0x%X' % (filename, size))
else:
with open(filename, 'wb') as f:
f.truncate(size)
stat = os.stat(filename)
assert stat.st_blocks == 0
ds = RawFileDataSet(
path=str(filename),
scan_size=shape[:2],
dtype=dtype,
detector_size=shape[2:],
)
ds.set_num_cores(2)
ds = ds.initialize(InlineJobExecutor())
yield ds