def test_stddev(lt_ctx, delayed_ctx, use_roi):
"""
Test variance, standard deviation, sum of frames, and mean computation
implemented in udf/stddev.py
Parameters
----------
lt_ctx
Context class for loading dataset and creating jobs on them
"""
data = _mk_random(size=(30, 3, 516), dtype="float32")
dataset = MemoryDataSet(data=data,
tileshape=(3, 2, 257),
num_partitions=2,
sig_dims=2)
if use_roi:
roi = np.random.choice([True, False], size=dataset.shape.nav)
res = run_stddev(lt_ctx, dataset, roi=roi)
res_delayed = run_stddev(delayed_ctx, dataset, roi=roi)
else:
roi = np.ones(dataset.shape.nav, dtype=bool)
res = run_stddev(lt_ctx, dataset)
res_delayed = run_stddev(delayed_ctx, dataset)
assert 'sum' in res
assert 'num_frames' in res
assert 'var' in res
assert 'mean' in res
assert 'std' in res
N = np.count_nonzero(roi)
assert res['num_frames'] == N # check the total number of frames
assert res_delayed['num_frames'] == N
print(res['sum'])
print(np.sum(data[roi], axis=0))
print(res['sum'] - np.sum(data[roi], axis=0))
assert np.allclose(res['sum'], np.sum(data[roi],
axis=0)) # check sum of frames
assert np.allclose(res_delayed['sum'], np.sum(data[roi], axis=0))
assert np.allclose(res['mean'], np.mean(data[roi], axis=0)) # check mean
assert np.allclose(res_delayed['mean'], np.mean(data[roi], axis=0))
var = np.var(data[roi], axis=0)
assert np.allclose(var, res['var']) # check variance
assert np.allclose(var, res_delayed['var'])
std = np.std(data[roi], axis=0)
assert np.allclose(std, res['std']) # check standard deviation
assert np.allclose(std, res_delayed['std'])
def test_stddev(lt_ctx):
"""
Test variance, standard deviation, sum of frames, and mean computation
implemented in udf/stddev.py
Parameters
----------
lt_ctx
Context class for loading dataset and creating jobs on them
"""
data = _mk_random(size=(16, 16, 16, 16), dtype="float32")
dataset = MemoryDataSet(data=data, tileshape=(1, 16, 16),
num_partitions=2, sig_dims=2)
res = run_stddev(lt_ctx, dataset)
assert 'sum_frame' in res
assert 'num_frame' in res
assert 'var' in res
assert 'mean' in res
assert 'std' in res
N = data.shape[2] * data.shape[3]
assert res['num_frame'] == N # check the total number of frames
assert np.allclose(res['sum_frame'], np.sum(data, axis=(0, 1))) # check sum of frames
assert np.allclose(res['mean'], np.mean(data, axis=(0, 1))) # check mean
var = np.var(data, axis=(0, 1))
assert np.allclose(var, res['var']) # check variance
std = np.std(data, axis=(0, 1))
assert np.allclose(std, res['std']) # check standard deviation
def test_stddev(lt_ctx, use_roi):
"""
Test variance, standard deviation, sum of frames, and mean computation
implemented in udf/stddev.py
Parameters
----------
lt_ctx
Context class for loading dataset and creating jobs on them
"""
data = _mk_random(size=(16, 17, 18, 19), dtype="float32")
# FIXME the tiling in signal dimension can only be tested once MemoryDataSet
# actually supports it
dataset = MemoryDataSet(data=data,
tileshape=(3, 2, 16),
num_partitions=8,
sig_dims=2)
if use_roi:
roi = np.random.choice([True, False], size=dataset.shape.nav)
res = run_stddev(lt_ctx, dataset, roi=roi)
else:
roi = np.ones(dataset.shape.nav, dtype=bool)
res = run_stddev(lt_ctx, dataset)
assert 'sum_frame' in res
assert 'num_frame' in res
assert 'var' in res
assert 'mean' in res
assert 'std' in res
N = np.count_nonzero(roi)
assert res['num_frame'] == N # check the total number of frames
assert np.allclose(res['sum_frame'], np.sum(data[roi],
axis=0)) # check sum of frames
assert np.allclose(res['mean'], np.mean(data[roi], axis=0)) # check mean
var = np.var(data[roi], axis=0)
assert np.allclose(var, res['var']) # check variance
std = np.std(data[roi], axis=0)
assert np.allclose(std, res['std']) # check standard deviation
def make_feature_vec(ctx,
dataset,
delta,
n_peaks,
min_dist=None,
center=None,
rad_in=None,
rad_out=None,
roi=None):
"""
Creates a feature vector for each frame in ROI based on non-zero order diffraction peaks
positions
Parameters
----------
ctx : libertem.api.Context
dataset : libertem.io.dataset.DataSet
A dataset with 1- or 2-D scan dimensions and 2-D frame dimensions
num : int
Number of possible peak positions to detect (better put higher value,
the output is limited to the number of peaks the algorithm could find)
delta : float
Relative intensity difference between current frame and reference image for decision making
for feature vector value (delta = (x-ref)/ref, so, normally, value should be in range [0,1])
rad_in : int, optional
Inner radius in pixels of a ring to mask region of interest of SD image to delete outliers
for peak finding
rad_out : int, optional
Outer radius in pixels of a ring to mask region of interest of SD image to delete outliers
for peak finding
center : tuple, optional
(y,x) - pixels, coordinates of a ring to mask region of interest of SD image
to delete outliers for peak finding
roi : numpy.ndarray, optional
boolean array which limits the elements the UDF is working on.
Has a shape of dataset_shape.nav
Returns
-------
pass_results: dict
Returns a feature vector for each frame.
"1" - denotes presence of peak for current frame for given possible peak position,
"0" - absence of peak for current frame for given possible peak position,
To return 2-D array use pass_results['feature_vec'].data
coordinates: numpy array of int
Returns array of coordinates of possible peaks positions
"""
res_stat = run_stddev(ctx, dataset, roi)
savg = res_stat['mean']
sstd = res_stat['std']
sshape = sstd.shape
if not (center is None or rad_in is None or rad_out is None):
mask_out = 1 * _make_circular_mask(center[1], center[0], sshape[1],
sshape[0], rad_out)
mask_in = 1 * _make_circular_mask(center[1], center[0], sshape[1],
sshape[0], rad_in)
mask = mask_out - mask_in
masked_sstd = sstd * mask
else:
masked_sstd = sstd
if min_dist is None:
min_dist = 1
coordinates = peak_local_max(masked_sstd,
num_peaks=n_peaks,
min_distance=min_dist)
udf = FeatureVecMakerUDF(delta=delta, savg=savg, coordinates=coordinates)
pass_results = ctx.run_udf(dataset=dataset, udf=udf, roi=roi)
return (pass_results, coordinates)
def clustering(interactive: Interactive, api: API):
window = api.application.document_windows[0]
target_data_item = window.target_data_item
ctx = iface.get_context()
ds = iface.dataset_from_data_item(ctx, target_data_item)
fy, fx = tuple(ds.shape.sig)
y, x = tuple(ds.shape.nav)
# roi = np.random.choice([True, False], tuple(ds.shape.nav), p=[0.01, 0.99])
# We only sample 5 % of the frame for the std deviation map
# since the UDF still needs optimization
std_roi = np.random.choice([True, False],
tuple(ds.shape.nav),
p=[0.05, 0.95])
roi = np.ones((y, x), dtype=bool)
# roi = np.zeros((y, x), dtype=bool)
# roi[:, :50] = True
stddev_res = run_stddev(ctx=ctx, dataset=ds, roi=std_roi * roi)
ref_frame = stddev_res['std']
# sum_res = ctx.run_udf(udf=SumUDF(), dataset=ds)
# ref_frame = sum_res['intensity'].data
update_data(target_data_item, ref_frame)
peaks = peak_local_max(ref_frame, min_distance=3, num_peaks=500)
masks = sparse.COO(shape=(len(peaks), fy, fx),
coords=(range(len(peaks)), peaks[..., 0], peaks[...,
1]),
data=1)
feature_udf = ApplyMasksUDF(mask_factories=lambda: masks,
mask_dtype=np.uint8,
mask_count=len(peaks),
use_sparse=True)
feature_res = ctx.run_udf(udf=feature_udf, dataset=ds, roi=roi)
f = feature_res['intensity'].raw_data.astype(np.float32)
f = np.log(f - np.min(f) + 1)
feature_vector = f / np.abs(f).mean(axis=0)
# too slow
# nion_peaks = peaks / tuple(ds.shape.sig)
# with api.library.data_ref_for_data_item(target_data_item):
# for p in nion_peaks:
# target_data_item.add_ellipse_region(*p, 0.01, 0.01)
connectivity = scipy.sparse.csc_matrix(
grid_to_graph(
# Transposed!
n_x=y,
n_y=x,
))
roi_connectivity = connectivity[roi.flatten()][:, roi.flatten()]
threshold = interactive.get_float("Cluster distance threshold: ", 10)
clusterer = AgglomerativeClustering(
affinity='euclidean',
distance_threshold=threshold,
n_clusters=None,
linkage='ward',
connectivity=roi_connectivity,
)
clusterer.fit(feature_vector)
labels = np.zeros((y, x), dtype=np.int32)
labels[roi] = clusterer.labels_ + 1
new_data = api.library.create_data_item_from_data(labels)
window.display_data_item(new_data)