def test_ssb():
ctx = lt.Context(executor=InlineJobExecutor())
dtype = np.float64
scaling = 4
shape = (29, 30, 189 // scaling, 197 // scaling)
# ? shape = np.random.uniform(1, 300, (4,1,))
# The acceleration voltage U in keV
U = 300
# STEM pixel size in m, here 50 STEM pixels on 0.5654 nm
dpix = 0.5654 / 50 * 1e-9
# STEM semiconvergence angle in radians
semiconv = 25e-3
# Diameter of the primary beam in the diffraction pattern in pixels
semiconv_pix = 78.6649 / scaling
cy = 93 // scaling
cx = 97 // scaling
input_data = np.random.uniform(0, 1, shape)
LG = np.linspace(1.0,
1000.0,
num=shape[0] * shape[1] * shape[2] * shape[3])
LG = LG.reshape(shape[0], shape[1], shape[2], shape[3])
input_data = input_data * LG
input_data = input_data.astype(np.float64)
udf = SSB_UDF(U=U,
dpix=dpix,
semiconv=semiconv,
semiconv_pix=semiconv_pix,
dtype=dtype,
cy=cy,
cx=cx)
dataset = MemoryDataSet(
data=input_data,
tileshape=(20, shape[2], shape[3]),
num_partitions=2,
sig_dims=2,
)
result = ctx.run_udf(udf=udf, dataset=dataset)
result_f, _, _ = reference_ssb(input_data,
U=U,
dpix=dpix,
semiconv=semiconv,
semiconv_pix=semiconv_pix,
cy=cy,
cx=cx)
# atol = np.max(np.abs(result_f))*0.009
# print(np.max(np.abs(np.abs(result['pixels']) - np.abs(result_f))))
assert np.allclose(np.abs(result['pixels']), np.abs(result_f))
def test_ssb_rotate():
ctx = lt.Context(executor=InlineJobExecutor())
dtype = np.float64
scaling = 4
det = 45
shape = (29, 30, det, det)
# ? shape = np.random.uniform(1, 300, (4,1,))
# The acceleration voltage U in keV
U = 300
# STEM pixel size in m, here 50 STEM pixels on 0.5654 nm
dpix = 0.5654 / 50 * 1e-9
# STEM semiconvergence angle in radians
semiconv = 25e-3
# Diameter of the primary beam in the diffraction pattern in pixels
semiconv_pix = 78.6649 / scaling
cy = det // 2
cx = det // 2
input_data = (np.random.uniform(0, 1, np.prod(shape)) *
np.linspace(1.0, 1000.0, num=np.prod(shape)))
input_data = input_data.astype(np.float64).reshape(shape)
data_90deg = np.zeros_like(input_data)
# Rotate 90 degrees clockwise
for y in range(det):
for x in range(det):
data_90deg[:, :, x, det - 1 - y] = input_data[:, :, y, x]
udf = SSB_UDF(U=U,
dpix=dpix,
semiconv=semiconv,
semiconv_pix=semiconv_pix,
dtype=dtype,
center=(cy, cx),
transformation=rotate_deg(-90.))
dataset = MemoryDataSet(
data=data_90deg,
tileshape=(20, shape[2], shape[3]),
num_partitions=2,
sig_dims=2,
)
result = ctx.run_udf(udf=udf, dataset=dataset)
result_f, _ = reference_ssb(input_data,
U=U,
dpix=dpix,
semiconv=semiconv,
semiconv_pix=semiconv_pix,
cy=cy,
cx=cx)
assert np.allclose(result['pixels'].data, result_f)
def test_ssb(dpix, backend, n_threads):
lt_ctx = lt.Context(InlineJobExecutor(debug=True, inline_threads=n_threads))
try:
if backend == 'cupy':
set_use_cuda(0)
dtype = np.float64
scaling = 4
shape = (29, 30, 189 // scaling, 197 // scaling)
# The acceleration voltage U in keV
U = 300
lamb = wavelength(U)
# STEM semiconvergence angle in radians
semiconv = 25e-3
# Diameter of the primary beam in the diffraction pattern in pixels
semiconv_pix = 78.6649 / scaling
cy = 93 // scaling
cx = 97 // scaling
input_data = (
np.random.uniform(0, 1, np.prod(shape))
* np.linspace(1.0, 1000.0, num=np.prod(shape))
)
input_data = input_data.astype(np.float64).reshape(shape)
udf = SSB_UDF(lamb=lamb, dpix=dpix, semiconv=semiconv, semiconv_pix=semiconv_pix,
dtype=dtype, cy=cy, cx=cx, method='subpix')
dataset = MemoryDataSet(
data=input_data, tileshape=(20, shape[2], shape[3]), num_partitions=2, sig_dims=2,
)
result = lt_ctx.run_udf(udf=udf, dataset=dataset)
result_f, reference_masks = reference_ssb(input_data, U=U, dpix=dpix, semiconv=semiconv,
semiconv_pix=semiconv_pix, cy=cy, cx=cx)
task_data = udf.get_task_data()
udf_masks = task_data['masks'].computed_masks
half_y = shape[0] // 2 + 1
# Use symmetry and reshape like generate_masks()
reference_masks = reference_masks[:half_y].reshape((half_y*shape[1], shape[2], shape[3]))
print(np.max(np.abs(udf_masks.todense() - reference_masks)))
print(np.max(np.abs(result['fourier'].data - result_f)))
assert np.allclose(result['fourier'].data, result_f)
backwards = result['amplitude'].data**2 * np.exp(1j*result['phase'].data)
assert np.allclose(result['fourier'].data, np.fft.fft2(backwards))
finally:
if backend == 'cupy':
set_use_cpu(0)
def test_start_local_cupyonly(hdf5_ds_1):
cudas = detect()['cudas']
# Make sure we have enough partitions
hdf5_ds_1.set_num_cores(len(cudas))
mask = _mk_random(size=(16, 16))
with hdf5_ds_1.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = _naive_mask_apply([mask], data)
spec = cluster_spec(cpus=(), cudas=cudas, has_cupy=True)
with DaskJobExecutor.make_local(spec=spec) as executor:
ctx = api.Context(executor=executor)
# Uses ApplyMasksUDF, which supports CuPy
analysis = ctx.create_mask_analysis(dataset=hdf5_ds_1,
factories=[lambda: mask])
results = ctx.run(analysis)
udf_res = ctx.run_udf(udf=DebugDeviceUDF(), dataset=hdf5_ds_1)
# No CPU compute resources
with pytest.raises(RuntimeError):
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('numpy', )),
dataset=hdf5_ds_1)
cuda_res = ctx.run_udf(udf=DebugDeviceUDF(backends=('cuda', )),
dataset=hdf5_ds_1)
assert np.allclose(results.mask_0.raw_data, expected)
found = {}
for val in udf_res['device_id'].data[0].values():
print(val)
# no CPU
assert val["cpu"] is None
# Register which GPUs got work
found[val["cuda"]] = True
for val in cuda_res['device_id'].data[0].values():
print(val)
# no CPU
assert val["cpu"] is None
# Register which GPUs got work
found[val["cuda"]] = True
for val in udf_res['backend'].data[0].values():
# use CuPy
print(val)
assert 'cupy' in val
for val in cuda_res['backend'].data[0].values():
# no CuPy, i.e. NumPy
print(val)
assert 'numpy' in val
# Test if each GPU got work. We have to see if this
# actually works always since this depends on the scheduler behavior
assert set(found.keys()) == set(cudas)
assert np.all(udf_res['device_class'].data == 'cuda')
assert np.allclose(udf_res['on_device'].data, data.sum(axis=(0, 1)))
def ipy_ctx():
import ipyparallel
client = ipyparallel.Client()
# wait for two engines: see also docker-compose.yml where the engines are started
client.wait_for_engines(2)
dask_client = client.become_dask()
executor = DaskJobExecutor(client=dask_client, is_local=False)
with lt.Context(executor=executor) as ctx:
yield ctx
def test_ssb_roi():
ctx = lt.Context(executor=InlineJobExecutor())
dtype = np.float64
scaling = 4
shape = (29, 30, 189 // scaling, 197 // scaling)
# ? shape = np.random.uniform(1, 300, (4,1,))
# The acceleration voltage U in keV
U = 300
lamb = wavelength(U)
# STEM pixel size in m, here 50 STEM pixels on 0.5654 nm
dpix = 0.5654 / 50 * 1e-9
# STEM semiconvergence angle in radians
semiconv = 25e-3
# Diameter of the primary beam in the diffraction pattern in pixels
semiconv_pix = 78.6649 / scaling
cy = 93 // scaling
cx = 97 // scaling
input_data = (np.random.uniform(0, 1, np.prod(shape)) *
np.linspace(1.0, 1000.0, num=np.prod(shape)))
input_data = input_data.astype(np.float64).reshape(shape)
udf = SSB_UDF(lamb=lamb,
dpix=dpix,
semiconv=semiconv,
semiconv_pix=semiconv_pix,
dtype=dtype,
cy=cy,
cx=cx)
dataset = MemoryDataSet(
data=input_data,
tileshape=(20, shape[2], shape[3]),
num_partitions=2,
sig_dims=2,
)
roi_1 = np.random.choice([True, False], shape[:2])
roi_2 = np.invert(roi_1)
result_1 = ctx.run_udf(udf=udf, dataset=dataset, roi=roi_1)
result_2 = ctx.run_udf(udf=udf, dataset=dataset, roi=roi_2)
result_f, _ = reference_ssb(input_data,
U=U,
dpix=dpix,
semiconv=semiconv,
semiconv_pix=semiconv_pix,
cy=cy,
cx=cx)
assert np.allclose(result_1['pixels'].data + result_2['pixels'].data,
result_f)
def test_avoid_calculating_masks_on_client_udf(hdf5_ds_1):
mask = _mk_random(size=(16, 16))
# We have to start a local cluster so that the masks are
# computed in a different process
with api.Context() as ctx:
analysis = ctx.create_mask_analysis(
dataset=hdf5_ds_1, factories=[lambda: mask], mask_count=1, mask_dtype=np.float32
)
udf = analysis.get_udf()
ctx.run_udf(udf=udf, dataset=hdf5_ds_1)
assert udf._mask_container is None
def test_avoid_calculating_masks_on_client(hdf5_ds_1):
mask = _mk_random(size=(16, 16))
with api.Context() as ctx:
analysis = ctx.create_mask_analysis(dataset=hdf5_ds_1,
factories=[lambda: mask],
mask_count=1,
mask_dtype=np.float32)
job = analysis.get_job()
ctx.run(job)
assert job.masks._computed_masks is None
def test_start_local(hdf5_ds_1):
mask = _mk_random(size=(16, 16))
with hdf5_ds_1.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = _naive_mask_apply([mask], data)
with api.Context() as ctx:
analysis = ctx.create_mask_analysis(dataset=hdf5_ds_1,
factories=[lambda: mask])
results = ctx.run(analysis)
assert np.allclose(results.mask_0.raw_data, expected)
def dist_ctx(scheduler_addr):
"""
This Context needs to have an external dask cluster running, with the following
assumptions:
- two workers: hostnames worker-1 and worker-2
- one scheduler node
- data availability TBD
"""
executor = DaskJobExecutor.connect(scheduler_addr)
with lt.Context(executor=executor) as ctx:
yield ctx
def ipy_ctx():
import ipyparallel
client = ipyparallel.Client()
retries = 10
while retries > 0:
retries -= 1
if len(client.ids) > 0:
break
time.sleep(1)
dask_client = client.become_dask()
executor = DaskJobExecutor(client=dask_client, is_local=False)
with lt.Context(executor=executor) as ctx:
yield ctx
def __init__(self, path, dtype, scan_size, detector_size, warmup_rounds,
roi, mask):
super().__init__(path, dtype, scan_size, detector_size, warmup_rounds,
roi, mask)
self.ctx = lt.Context()
self.ds = self.ctx.load(
'raw',
path=path,
dtype=dtype,
scan_size=scan_size,
detector_size_raw=detector_size,
crop_detector_to=detector_size,
)
self.boolean_mask = self.mask == 1
def dist_ctx():
"""
This Context needs to have an external dask cluster running, with the following
assumptions:
- two workers: hostnames worker-1 and worker-2
- one scheduler node
- data availability TBD
- the address of the dask scheduler is passed in as DASK_SCHEDULER_ADDRESS
"""
scheduler_addr = os.environ['DASK_SCHEDULER_ADDRESS']
executor = DaskJobExecutor.connect(scheduler_addr)
with lt.Context(executor=executor) as ctx:
yield ctx
def test_difftodect_com_flip_rot_scale(dim):
lt_ctx = lt.Context(InlineJobExecutor())
data_shape = (2, 2, dim, dim)
data = np.zeros(data_shape)
data[0, 0, 7, 7] = 1
data[0, 1, 7, 8] = 1
data[1, 1, 8, 8] = 1
data[1, 0, 8, 7] = 1
source_shape = data_shape[2:]
target_shape = data_shape[2:]
f = diffraction_to_detector(lamb=1,
diffraction_shape=target_shape,
pixel_size_real=1,
pixel_size_detector=1 /
(np.array(target_shape)) * 4,
cy=source_shape[0] / 2,
cx=source_shape[1] / 2,
flip_y=True,
scan_rotation=-90.)
m = image_transformation_matrix(
source_shape=source_shape,
target_shape=target_shape,
affine_transformation=f,
)
transformed_data = apply_matrix(data, m, target_shape)
ds = lt_ctx.load('memory', data=data, sig_dims=2)
transformed_ds = lt_ctx.load('memory', data=transformed_data, sig_dims=2)
com_a = lt_ctx.create_com_analysis(dataset=ds,
mask_radius=np.inf,
flip_y=True,
scan_rotation=-90.,
cy=target_shape[0] / 2,
cx=target_shape[1] / 2)
com_res = lt_ctx.run(com_a)
trans_com_a = lt_ctx.create_com_analysis(dataset=transformed_ds,
mask_radius=np.inf,
flip_y=False,
scan_rotation=0.,
cy=target_shape[0] / 2,
cx=target_shape[1] / 2)
trans_com_res = lt_ctx.run(trans_com_a)
print(com_res.field.raw_data)
print(trans_com_res.field.raw_data)
assert np.allclose(com_res.field.raw_data,
np.array(trans_com_res.field.raw_data) / 4)
def main():
path = r'C:\Users\lesnic\Nextcloud\Dieter\cGaN_sim_300kV\DPs\CBED_MSAP.raw'
shape = (50, 50, 189, 189)
ctx = lt.Context(executor=InlineJobExecutor())
data_s = ctx.load(
"raw", path=path, dtype="float32",
scan_size=shape[:2], detector_size=shape[-2:]
)
udf = SSB_UDF(dpix=0.5654/50*1e-9, semiconv=25e-3, semiconv_pix=78.6649, lamb=1.96e-12)
result = ctx.run_udf(udf=udf, dataset=data_s)
fig, axes = plt.subplots(1, 2)
axes[0].imshow(np.abs(result['pixelsum']), norm=LogNorm())
axes[1].imshow(np.angle(np.fft.ifft2(result['pixelsum'])))
input("press return to continue")
def main():
ctx = api.Context(executor=InlineJobExecutor())
ds = RawFileDataSet(path="/home/clausen/Data/EMPAD/scan_11_x256_y256.raw",
scan_size=(256, 256),
detector_size_raw=(130, 128),
crop_detector_to=(128, 128),
tileshape=(1, 8, 128, 128),
dtype="float32")
ds.initialize()
job = ctx.create_mask_analysis(dataset=ds,
factories=[lambda: np.ones(ds.shape.sig)])
result = ctx.run(job)
def default_raw_asymm(tmpdir_factory, default_raw_data):
lt_ctx = lt.Context(executor=InlineJobExecutor())
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-default'
default_raw_data.tofile(str(filename))
del default_raw_data
ds = lt_ctx.load(
"raw",
path=str(filename),
dtype="float32",
nav_shape=(14, 17),
sig_shape=(128, 128),
io_backend=MMapBackend(),
)
ds.set_num_cores(2)
yield ds
def buffered_raw(tmpdir_factory, default_raw_data):
lt_ctx = lt.Context(executor=InlineJobExecutor())
datadir = tmpdir_factory.mktemp('data')
filename = datadir + '/raw-test-buffered'
default_raw_data.tofile(str(filename))
del default_raw_data
ds = lt_ctx.load(
"raw",
path=str(filename),
dtype="float32",
nav_shape=(16, 16),
sig_shape=(128, 128),
io_backend=BufferedBackend(),
)
yield ds
def test_start_local_default(hdf5_ds_1):
mask = _mk_random(size=(16, 16))
d = detect()
cudas = d['cudas']
with hdf5_ds_1.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = _naive_mask_apply([mask], data)
with api.Context() as ctx:
analysis = ctx.create_mask_analysis(dataset=hdf5_ds_1,
factories=[lambda: mask])
# Based on ApplyMasksUDF, which is CuPy-enabled
hybrid = ctx.run(analysis)
_ = ctx.run_udf(udf=DebugDeviceUDF(), dataset=hdf5_ds_1)
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('cupy', 'numpy')),
dataset=hdf5_ds_1)
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('cuda', 'numpy')),
dataset=hdf5_ds_1)
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('cupy', 'cuda', 'numpy')),
dataset=hdf5_ds_1)
if cudas:
cuda_only = ctx.run_udf(udf=DebugDeviceUDF(backends=('cuda',
'numpy')),
dataset=hdf5_ds_1,
backends=('cuda', ))
if d['has_cupy']:
cupy_only = ctx.run_udf(udf=DebugDeviceUDF(backends=('cupy',
'numpy')),
dataset=hdf5_ds_1,
backends=('cupy', ))
else:
with pytest.raises(RuntimeError):
cupy_only = ctx.run_udf(
udf=DebugDeviceUDF(backends=('cupy', 'numpy')),
dataset=hdf5_ds_1,
backends=('cupy', ))
cupy_only = None
numpy_only = ctx.run_udf(udf=DebugDeviceUDF(backends=('numpy', )),
dataset=hdf5_ds_1)
assert np.allclose(hybrid.mask_0.raw_data, expected)
if cudas:
assert np.all(cuda_only['device_class'].data == 'cuda')
if cupy_only is not None:
assert np.all(cupy_only['device_class'].data == 'cuda')
assert np.all(numpy_only['device_class'].data == 'cpu')
def main():
ctx = api.Context(executor=InlineJobExecutor())
ds = RawFilesDataSet(
path="/home/clausen/Data/many_small_files/frame00016293.bin",
# path="/home/clausen/Data/many_medsize_files/frame00000001.bin",
nav_shape=(256, 256),
sig_shape=(128, 128),
file_shape=(16, 128, 128),
tileshape=(1, 8, 128, 128),
dtype="float32")
ds.initialize()
pprint.pprint(list(ds.get_partitions()))
job = ctx.create_mask_analysis(dataset=ds,
factories=[lambda: np.ones(ds.shape.sig)])
result = ctx.run(job)
def test_start_local_cpuonly(hdf5_ds_1):
# We don't use all since that might be too many
cpus = (0, 1)
hdf5_ds_1.set_num_cores(len(cpus))
mask = _mk_random(size=(16, 16))
with hdf5_ds_1.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = _naive_mask_apply([mask], data)
spec = cluster_spec(cpus=cpus, cudas=(), has_cupy=False)
with DaskJobExecutor.make_local(spec=spec) as executor:
ctx = api.Context(executor=executor)
analysis = ctx.create_mask_analysis(
dataset=hdf5_ds_1, factories=[lambda: mask]
)
results = ctx.run(analysis)
udf_res = ctx.run_udf(udf=DebugDeviceUDF(), dataset=hdf5_ds_1)
# No CuPy resources
with pytest.raises(RuntimeError):
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('cupy',)), dataset=hdf5_ds_1)
assert np.allclose(
results.mask_0.raw_data,
expected
)
found = {}
for val in udf_res['device_id'].data[0].values():
print(val)
# no CUDA
assert val["cuda"] is None
found[val["cpu"]] = True
for val in udf_res['backend'].data[0].values():
print(val)
# no CUDA
assert 'numpy' in val
# Each CPU got work. We have to see if this
# actually works always since this depends on the scheduler behavior
assert set(found.keys()) == set(cpus)
assert np.all(udf_res['device_class'].data == 'cpu')
assert np.allclose(udf_res['on_device'].data, data.sum(axis=(0, 1)))
def test_use_plain_dask(hdf5_ds_1):
# We deactivate the resource scheduling and run on a plain dask cluster
hdf5_ds_1.set_num_cores(2)
mask = _mk_random(size=(16, 16))
with hdf5_ds_1.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = _naive_mask_apply([mask], data)
with dd.LocalCluster(n_workers=2, threads_per_worker=1) as cluster:
client = dd.Client(cluster, set_as_default=False)
try:
executor = DaskJobExecutor(client=client)
ctx = api.Context(executor=executor)
analysis = ctx.create_mask_analysis(
dataset=hdf5_ds_1, factories=[lambda: mask]
)
results = ctx.run(analysis)
udf_res = ctx.run_udf(udf=DebugDeviceUDF(), dataset=hdf5_ds_1)
# Requesting CuPy, which is not available
with pytest.raises(RuntimeError):
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('cupy',)), dataset=hdf5_ds_1)
finally:
# to fix "distributed.client - ERROR - Failed to reconnect to scheduler after 10.00 seconds, closing client" # NOQA
client.close()
assert np.allclose(
results.mask_0.raw_data,
expected
)
for val in udf_res['device_id'].data[0].values():
print(val)
# no CUDA
assert val["cuda"] is None
# Default without worker setup
assert val["cpu"] == 0
for val in udf_res['backend'].data[0].values():
print(val)
# no CUDA
assert 'numpy' in val
assert np.all(udf_res['device_class'].data == 'cpu')
assert np.allclose(udf_res['on_device'].data, data.sum(axis=(0, 1)))
def test_preload(hdf5_ds_1):
# We don't use all since that might be too many
cpus = (0, 1)
hdf5_ds_1.set_num_cores(len(cpus))
class CheckEnvUDF(NoOpUDF):
def process_tile(self, tile):
assert os.environ['LT_TEST_1'] == 'hello'
assert os.environ['LT_TEST_2'] == 'world'
preloads = (
"import os; os.environ['LT_TEST_1'] = 'hello'",
"import os; os.environ['LT_TEST_2'] = 'world'",
)
spec = cluster_spec(cpus=cpus, cudas=(), has_cupy=False, preload=preloads)
with DaskJobExecutor.make_local(spec=spec) as executor:
ctx = api.Context(executor=executor)
ctx.run_udf(udf=CheckEnvUDF(), dataset=hdf5_ds_1)
def main():
with api.Context() as ctx:
ds = ctx.load("RAW",
path=r"C:\Users\Dieter\testfile-32-32-32-32-float32.raw",
nav_shape=(32, 32),
sig_shape=(32, 32),
dtype=np.float32)
sum_analysis = ctx.create_sum_analysis(dataset=ds)
sum_result = ctx.run(sum_analysis)
sum_image = DM.CreateImage(sum_result.intensity.raw_data.copy())
sum_image.ShowImage()
haadf_analysis = ctx.create_ring_analysis(dataset=ds)
haadf_result = ctx.run(haadf_analysis)
haadf_image = DM.CreateImage(haadf_result.intensity.raw_data.copy())
haadf_image.ShowImage()
def main():
with DaskJobExecutor.connect('tcp://localhost:8786') as executor:
ctx = api.Context(executor=executor)
ds = ctx.load("RAW",
path=r"C:\Users\Dieter\testfile-32-32-32-32-float32.raw",
nav_shape=(32, 32),
sig_shape=(32, 32),
dtype=np.float32)
sum_analysis = ctx.create_sum_analysis(dataset=ds)
sum_result = ctx.run(sum_analysis)
sum_image = DM.CreateImage(sum_result.intensity.raw_data.copy())
sum_image.ShowImage()
haadf_analysis = ctx.create_ring_analysis(dataset=ds)
haadf_result = ctx.run(haadf_analysis)
haadf_image = DM.CreateImage(haadf_result.intensity.raw_data.copy())
haadf_image.ShowImage()
def test_start_local_cudaonly(hdf5_ds_1):
cudas = detect()['cudas']
# Make sure we have enough partitions
hdf5_ds_1.set_num_cores(len(cudas))
with hdf5_ds_1.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
spec = cluster_spec(cpus=(), cudas=cudas, has_cupy=False)
with DaskJobExecutor.make_local(spec=spec) as executor:
ctx = api.Context(executor=executor)
udf_res = ctx.run_udf(udf=DebugDeviceUDF(backends=('cuda', )),
dataset=hdf5_ds_1)
# No CPU compute resources
with pytest.raises(RuntimeError):
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('numpy', )),
dataset=hdf5_ds_1)
# No ndarray (CuPy) resources
with pytest.raises(RuntimeError):
_ = ctx.run_udf(udf=DebugDeviceUDF(backends=('cupy', )),
dataset=hdf5_ds_1)
found = {}
for val in udf_res['device_id'].data[0].values():
print(val)
# no CPU
assert val["cpu"] is None
# Register which GPUs got work
found[val["cuda"]] = True
for val in udf_res['backend'].data[0].values():
print(val)
# CUDA, but no CuPy, i.e. use NumPy
assert 'numpy' in val
# Test if each GPU got work. We have to see if this
# actually works always since this depends on the scheduler behavior
assert set(found.keys()) == set(cudas)
assert np.all(udf_res['device_class'].data == 'cuda')
assert np.allclose(udf_res['on_device'].data, data.sum(axis=(0, 1)))
def auto_ctx(doctest_namespace):
ctx = lt.Context(executor=InlineJobExecutor())
doctest_namespace["ctx"] = ctx
def get_inline_context():
return api.Context(executor=InlineJobExecutor())
def get_context():
executor = Registry.get_component('libertem_executor')
return api.Context(executor=executor.ensure_sync())
import numpy as np
import matplotlib.pyplot as plt
from libertem import api
logging.basicConfig(level=logging.WARNING)
# Protect the entry point.
# LiberTEM uses dask, which uses multiprocessing to
# start worker processes.
# https://docs.python.org/3/library/multiprocessing.html
if __name__ == '__main__':
# api.Context() starts a new local cluster.
# The "with" clause makes sure we shut it down in the end.
with api.Context() as ctx:
try:
path = sys.argv[1]
except IndexError:
path = ('C:/Users/weber/Nextcloud/Projects/'
'Open Pixelated STEM framework/Data/EMPAD/'
'scan_11_x256_y256.emd')
ds = ctx.load('hdf5',
path=path,
ds_path='experimental/science_data/data',
tileshape=(1, 8, 128, 128))
(scan_y, scan_x, detector_y, detector_x) = ds.shape
mask_shape = (detector_y, detector_x)
# LiberTEM sends functions that create the masks
