def getBoundariesOfimage(image):
"""
find edges by using erosion
"""
if np.ndim(image) == 2:
sElement = ndimage.generate_binary_structure(2, 1)
else:
sElement = ndimage.generate_binary_structure(3, 1)
erode_im = scipy.ndimage.morphology.binary_erosion(image, sElement)
b = image - erode_im
return b
def adjust_spot_positions(image, label_image, hp, debug=None):
"""Re-evaluate the spot positions based on the segmentation.
Parameters:
image: The original image (can be masked) that was sent to findspot3d
label_image: the label image containing two labels
hp: the original hotpoints
debug: set to true to write out an image debugimg.nii.gz with the stuff
"""
struct2 = generate_binary_structure(3, 2)
struct1 = generate_binary_structure(3, 1)
peak_points =[]
if debug is None:
temp_path = os.getenv("PYSBR_TEMP")
if temp_path is not None:
debug = os.path.join(temp_path, "debug-labels.nii.gz")
if debug is not None:
debimg = image.copy()
nlabels = label_image.max()
if nlabels!=len(hp):
raise RuntimeError( 'number of labels and hotspots should be the same' )
tins = []
for n in range(nlabels):
label = n+1
area = binary_closing(label_image == label, struct2)
thiniter = np.sum(area.reshape(-1)) / 1500 + 1
csbr.thinning3d(area, thiniter)
tins.append(area)
for n in range(nlabels):
label = n+1
#avoid that a single pixel breaks the evaluation by running a closing
area = label_image == label
#evaluate the boundary
dmask = binary_dilation(area, struct1)
border = np.bitwise_xor(dmask, area)
p = adjust_spot_position(image, border, image[tuple(hp[n])], tins[n], tins[(n + 1) % 2])
peak_points.append(p)
if debug is not None:
debimg[border>0] = 196
debimg[p] = 0
nib.save(nib.Nifti1Image(debimg, global_affine), debug)
peak_points = np.array( peak_points )
return peak_points
def remove_small_objects(arr, min_size, connectivity=8):
'''
Remove objects less than the given size.
Function is based on skimage.morphology.remove_small_objects
Parameters
----------
arr : numpy.ndarray
Binary array containing the mask.
min_size : int
Smallest allowed size.
connectivity : int, optional
Connectivity of the neighborhood.
'''
struct = nd.generate_binary_structure(arr.ndim, connectivity)
labels, num = nd.label(arr, struct)
sizes = nd.sum(arr, labels, range(1, num + 1))
for i, size in enumerate(sizes):
if size >= min_size:
continue
posns = np.where(labels == i + 1)
arr[posns] = 0
return arr
def moment_masking(cube, kernel_size, clip=5, dilations=1):
'''
'''
if not signal_id_flag:
raise ImportError("signal-id is not installed."
" This function is not available.")
smooth_data = convolve(cube.filled_data[:], gauss_kern(kernel_size))
fake_mask = LazyMask(np.isfinite, cube=cube)
smooth_cube = SpectralCube(data=smooth_data, wcs=cube.wcs, mask=fake_mask)
smooth_scale = Noise(smooth_cube).scale
mask = (smooth_cube > (clip * smooth_scale)).include()
# Now dilate the mask once
dilate_struct = nd.generate_binary_structure(3, 3)
mask = nd.binary_dilation(mask, structure=dilate_struct,
iterations=dilations)
return mask
def getPatchNamesFromMask(mask, x, y, root='mask'):
"""
Returns an array of patch names for each (x, y) pair
"""
import math
import scipy.ndimage as nd
import numpy as np
act_pixels = mask
rank = len(act_pixels.shape)
connectivity = nd.generate_binary_structure(rank, rank)
mask_labels, count = nd.label(act_pixels, connectivity)
patchNums = []
patchNames = []
for xs, ys in zip(x, y):
try:
patchNums.append(mask_labels[xs, ys])
except:
patchNums.append(0)
# Check if there is a patch with id = 0. If so, this means there were
# some Gaussians that fell outside of the regions in the patch
# mask file.
n = 0
for p in patchNums:
if p != 0:
in_patch = np.where(patchNums == p)
patchNames.append('{0}_patch_'.format(root)+str(p))
else:
patchNames.append('patch_'+str(n))
n += 1
return np.array(patchNames)
def art_qi2(img, airmask, ncoils=12, erodemask=True):
"""
Calculates **qi2**, the distance between the distribution
of noise voxel (non-artifact background voxels) intensities, and a
centered Chi distribution.
:param numpy.ndarray img: input data
:param numpy.ndarray airmask: input air mask without artifacts
"""
from matplotlib import rc
import seaborn as sn
import matplotlib.pyplot as plt
rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
# rc('text', usetex=True)
if erodemask:
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
airmask = nd.binary_erosion(airmask, structure=struc).astype(np.uint8)
# Artifact-free air region
data = img[airmask > 0]
data = data[data < np.percentile(data, 99.5)]
maxvalue = int(data.max())
nbins = maxvalue if maxvalue < 100 else 100
# Estimate data pdf
hist, bin_edges = np.histogram(data, density=True, bins=nbins)
bin_centers = [np.mean(bin_edges[i:i+1]) for i in range(len(bin_edges)-1)]
max_pos = np.argmax(hist)
# Fit central chi distribution
param = chi.fit(data, 2*ncoils, loc=bin_centers[max_pos])
pdf_fitted = chi.pdf(bin_centers, *param[:-2], loc=param[-2], scale=param[-1])
# Write out figure of the fitting
out_file = op.abspath('background_fit.png')
fig = plt.figure()
ax1 = fig.add_subplot(111)
sn.distplot(data, bins=nbins, norm_hist=True, kde=False, ax=ax1)
#_, bins, _ = ax1.hist(data, nbins, normed=True, color='gray', linewidth=0)
ax1.plot(bin_centers, pdf_fitted, 'k--', linewidth=1.2)
fig.suptitle('Noise distribution on the air mask, and fitted chi distribution')
ax1.set_xlabel('Intensity')
ax1.set_ylabel('Frequency')
fig.savefig(out_file, format='png', dpi=300)
plt.close()
# Find t2 (intensity at half width, right side)
ihw = 0.5 * hist[max_pos]
t2idx = 0
for i in range(max_pos + 1, len(bin_centers)):
if hist[i] < ihw:
t2idx = i
break
# Compute goodness-of-fit (gof)
return (float(np.abs(hist[t2idx:] - pdf_fitted[t2idx:]).sum() /
len(pdf_fitted[t2idx:])), out_file)
def compute_mask(aparc, labels=[0, 5000]):
import nibabel as nb
import numpy as np
import os.path as op
import scipy.ndimage as nd
segnii = nb.load(aparc)
seg = segnii.get_data()
mask = np.ones_like(seg, dtype=np.uint8)
for l in labels:
mask[seg == l] = 0
struct = nd.iterate_structure(nd.generate_binary_structure(3, 1), 4)
mask = nd.binary_dilation(mask, structure=struct).astype(np.uint8)
mask = nd.binary_closing(mask, structure=struct)
mask = nd.binary_fill_holes(mask, structure=struct).astype(np.uint8)
mask[mask > 0] = 1
mask[mask <= 0] = 0
hdr = segnii.get_header().copy()
hdr.set_data_dtype(np.uint8)
hdr.set_xyzt_units("mm", "sec")
out_file = op.abspath("nobstem_mask.nii.gz")
nii = nb.Nifti1Image(mask, segnii.get_affine(), hdr).to_filename(out_file)
return out_file
def gen_data(xsize, ysize, nstars=3, starradius=10, brightness=2000):
# 1) lots of stars, big
# 2) lots of tiny stars
# 3) few stars, big
# 4) few stars, tiny
footprint = ndimage.generate_binary_structure(2,1)
ret = numpy.zeros((xsize, ysize))
for star in xrange(nstars):
xcenter = random.randint(0, xsize-1)
ycenter = random.randint(0, ysize-1)
for x in xrange(xcenter-1, xcenter+2):
for y in xrange(ycenter-1, ycenter+2):
if x >= 0 and y >= 0 and x < xsize and y < ysize:
ret[x,y] = brightness / 3
ret[xcenter, ycenter] = brightness
for i in xrange(starradius):
ret = ndimage.grey_dilation(ret, footprint=footprint)
# add some cosmic rays (single points)
for i in xrange(30):
xcenter = random.randint(0, xsize-1)
ycenter = random.randint(0, ysize-1)
ret[xcenter, ycenter] = brightness
return ret
def ClusterizeImage(image,thresh=None,connectivity=3):
if thresh is None:
thresh = 0
image[np.where(image<=thresh)] = 0
s = generate_binary_structure(3,connectivity)
larray, nf = label(image,s)
return larray
def edge_matrix(labels, connectivity=1):
"""Generate a COO matrix containing the coordinates of edge pixels.
Parameters
----------
labels : array of int
An array of labeled pixels (or voxels).
connectivity : int in {1, ..., labels.ndim}
The square connectivity for considering neighborhood.
Returns
-------
edges : sparse.coo_matrix
A COO matrix where (i, j) indicate neighboring labels and the
corresponding data element is the linear index of the edge pixel
in the labels array.
"""
conn = ndi.generate_binary_structure(labels.ndim, connectivity)
eroded = ndi.grey_erosion(labels, footprint=conn).ravel()
dilated = ndi.grey_dilation(labels, footprint=conn).ravel()
labels = labels.ravel()
boundaries0 = np.flatnonzero(eroded != labels)
boundaries1 = np.flatnonzero(dilated != labels)
labels_small = np.concatenate((eroded[boundaries0], labels[boundaries1]))
labels_large = np.concatenate((labels[boundaries0], dilated[boundaries1]))
n = np.max(labels_large) + 1
data = np.concatenate((boundaries0, boundaries1))
sparse_graph = sparse.coo_matrix((data, (labels_small, labels_large)),
dtype=np.int_, shape=(n, n))
return sparse_graph
def objextract(Fg):
s = nd.generate_binary_structure(2,2)
labeled_array, num_features = nd.measurements.label(Fg, structure=s)
coor = []
cnt = []
if num_features == 0:
idx = []
else:
lth = 200 # label pixel number less than lth will be removed
Lth = 6500
for i in range(1,num_features+1):
coor.append(np.where(labeled_array==i))
cnt.append(len(np.where(labeled_array==i)[1]))
cnt = array(cnt)
idx = arange(num_features)
idx = idx[(cnt<Lth)&(cnt>lth)]
if len(idx)==0:
idx = []
elif len(idx)>1:
#idx = [idx[cnt[idx].argmax()]]
idx = sorted(range(len(cnt)),key=lambda x:cnt[x])[::-1][0:2]
return idx,labeled_array,coor,cnt
def find_local_max(img, d_rad, threshold=1e-15):
"""
This is effectively a replacement for pkfnd in the matlab/IDL code.
The output of this function is meant to be feed into :py:func:`~subpixel_centroid`
The magic of numpy means this should work for any dimension data.
:param img: an ndarray representing the data to find the local maxes
:param d_rad: the radius of the dilation, the smallest possible spacing between local maximum
:param threshold: optional, voxels < threshold are ignored.
:rtype: (d,N) array of the local maximums.
"""
d_rad = int(d_rad)
img = np.array(np.squeeze(img)) # knock out singleton dimensions
img[img < threshold] = -np.inf # mask out pixels below threshold
dim = img.ndim # get the dimension of data
# make structuring element
s = ndimage.generate_binary_structure(dim, 1)
# scale it up to the desired size
d_struct = ndimage.iterate_structure(s, int(d_rad))
dilated_img = ndimage.grey_dilation(img,
footprint=d_struct,
cval=0,
mode='constant') # do the dilation
# find the locations that are the local maximum
# TODO clean this up
local_max = np.where(np.exp(img - dilated_img) > (1 - 1e-15))
# the extra [::-1] is because matplotlib and ndimage disagree an xy vs yx
return np.vstack(local_max[::-1])
def thresholding(img, thresh, size=9):
"""
Segment using a thresholding algorithm
Input:
- img ndarray : Image array (ndim=2)
- thresh float : Threshold value for pixels selectino
- size int : Minimum size a group of pixels must have
Output:
- regions : Binary array for each segmented region
---
"""
logging.debug("Threshold: %.2f", thresh)
logging.debug("Objects min size: %d", size)
# Take the binary image thresholded
img_bin = img > thresh
# And use (MO) binary opening (erosion + dilation) for cleaning spurious Trues
strct = ndi.generate_binary_structure(2, 2)
img_bin = ndi.binary_opening(img_bin, strct)
# Label each group/region (value==True) of pixels
regions, nlbl = ndi.label(img_bin)
for i in xrange(1, nlbl + 1):
inds = np.where(regions == i)
if inds[0].size < size:
regions[inds] = 0
logging.debug("Threshold labels: %s", np.unique(regions))
return regions.astype(np.bool)
def remove_small_objects(ar, min_size=64, connectivity=1, in_place=False):
# Should use `issubdtype` for bool below, but there's a bug in numpy 1.7
if not (ar.dtype == bool or np.issubdtype(ar.dtype, np.integer)):
raise TypeError("Only bool or integer image types are supported. "
"Got %s." % ar.dtype)
if in_place:
out = ar
else:
out = ar.copy()
if min_size == 0: # shortcut for efficiency
return out
if out.dtype == bool:
selem = nd.generate_binary_structure(ar.ndim, connectivity)
ccs = np.zeros_like(ar, dtype=np.int32)
nd.label(ar, selem, output=ccs)
else:
ccs = out
try:
component_sizes = np.bincount(ccs.ravel())
except ValueError:
raise ValueError("Negative value labels are not supported. Try "
"relabeling the input with `scipy.ndimage.label` or "
"`skimage.morphology.label`.")
too_small = component_sizes < min_size
too_small_mask = too_small[ccs]
out[too_small_mask] = 0
return out
def artifact_mask(imdata, airdata, distance):
"""Computes a mask of artifacts found in the air region"""
import nibabel as nb
if not np.issubdtype(airdata.dtype, np.integer):
airdata[airdata < .95] = 0
airdata[airdata > 0.] = 1
bg_img = imdata * airdata
# Find the background threshold (the most frequently occurring value
# excluding 0)
# CHANGED - to the 75 percentile
bg_threshold = np.percentile(bg_img[airdata > 0], 75)
# Apply this threshold to the background voxels to identify voxels
# contributing artifacts.
qi1_img = np.zeros_like(bg_img)
qi1_img[bg_img > bg_threshold] = 1
qi1_img[distance < .10] = 0
# Create a structural element to be used in an opening operation.
struc = nd.generate_binary_structure(3, 1)
qi1_img = nd.binary_opening(qi1_img, struc).astype(np.uint8)
qi1_img[airdata <= 0] = 0
return qi1_img
def test_3d(self):
"""tests the detection of maxima in 3D."""
img = np.zeros((8, 8, 8), dtype=np.uint8)
local_maxima = np.zeros((8, 8, 8), dtype=np.uint8)
# first maximum: only one pixel
img[1, 1:3, 1:3] = 100
img[2, 2, 2] = 200
img[3, 1:3, 1:3] = 100
local_maxima[2, 2, 2] = 1
# second maximum: three pixels in z-direction
img[5:8, 1, 1] = 200
local_maxima[5:8, 1, 1] = 1
# third: two maxima in 0 and 3.
img[0, 5:8, 5:8] = 200
img[1, 6, 6] = 100
img[2, 5:7, 5:7] = 200
img[0:3, 5:8, 5:8] += 50
local_maxima[0, 5:8, 5:8] = 1
local_maxima[2, 5:7, 5:7] = 1
# four : one maximum in the corner of the square
img[6:8, 6:8, 6:8] = 200
img[7, 7, 7] = 255
local_maxima[7, 7, 7] = 1
se = ndi.generate_binary_structure(3, 1)
out = extrema.local_maxima(img, se)
error = diff(local_maxima, out)
assert error < eps
def artifact_mask(imdata, airdata):
"""Computes a mask of artifacts found in the air region"""
if not np.issubdtype(airdata.dtype, np.integer):
airdata[airdata < .95] = 0
airdata[airdata > 0.] = 1
bg_img = imdata * airdata
# Find the background threshold (the most frequently occurring value
# excluding 0)
hist, bin_edges = np.histogram(bg_img[bg_img > 0], bins=128)
bg_threshold = np.mean(bin_edges[np.argmax(hist)])
# Apply this threshold to the background voxels to identify voxels
# contributing artifacts.
qi1_img = np.zeros_like(bg_img)
qi1_img[bg_img > bg_threshold] = bg_img[bg_img > bg_threshold]
# Create a structural element to be used in an opening operation.
struc = nd.generate_binary_structure(3, 2)
# Perform an a grayscale erosion operation.
qi1_img = nd.grey_erosion(qi1_img, structure=struc).astype(np.float32)
# Binarize and binary dilation
qi1_img[qi1_img > 0.] = 1
qi1_img[qi1_img < 1.] = 0
qi1_img = nd.binary_dilation(qi1_img, structure=struc).astype(np.uint8)
return qi1_img
def test_mask():
vol = np.zeros((30, 30, 30))
vol[15, 15, 15] = 1
struct = generate_binary_structure(3, 1)
voln = binary_dilation(vol, structure=struct, iterations=4).astype('f4')
initial = np.sum(voln > 0)
mask = voln.copy()
thresh = otsu(mask)
mask = mask > thresh
initial_otsu = np.sum(mask > 0)
assert_equal(initial_otsu, initial)
mins, maxs = bounding_box(mask)
voln_crop = crop(mask, mins, maxs)
initial_crop = np.sum(voln_crop > 0)
assert_equal(initial_crop, initial)
applymask(voln, mask)
final = np.sum(voln > 0)
assert_equal(final, initial)
# Test multi_median.
median_test = np.arange(25).reshape(5, 5)
median_control = median_test.copy()
medianradius = 3
median_test = multi_median(median_test, medianradius, 3)
medarr = np.ones_like(median_control.shape) * ((medianradius * 2) + 1)
median_filter(median_control, medarr, output=median_control)
median_filter(median_control, medarr, output=median_control)
median_filter(median_control, medarr, output=median_control)
assert_equal(median_test, median_control)
def multi_label_edge_detection(data):
f = nd.generate_binary_structure(len(data.shape), 1)
bound = (nd.grey_erosion(data,footprint=f) != nd.grey_dilation(data,footprint=f)) - \
(nd.binary_dilation(data.astype(np.bool)) - data.astype(np.bool)) # the unwanted thick bounds
data=bound.astype(data.dtype)
return data
def test_labeling():
"Test cluster labeling"
shape = flat_shape = (4, 20)
pmap = np.empty(shape, np.float_)
struct = ndimage.generate_binary_structure(2, 1)
struct[::2] = False
conn = np.array([(0, 1), (0, 3), (1, 2), (2, 3)], np.uint32)
criteria = None
# some clusters
pmap[:] = [[ 3, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0],
[ 0, 1, 0, 0, 0, 0, 8, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 0],
[ 0, 3, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4],
[ 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0]]
cmap, cids = label_clusters(pmap, 2, 0, conn, criteria)
assert_equal(len(cids), 6)
assert_array_equal(cmap > 0, np.abs(pmap) > 2)
# some other clusters
pmap[:] = [[ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0],
[ 0, 4, 0, 0, 0, 0, 0, 4, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 4, 4, 0, 4, 4, 0, 4, 0, 0, 0, 4, 4, 1, 0, 4, 4, 0, 0],
[ 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0]]
cmap, cids = label_clusters(pmap, 2, 0, conn, criteria)
assert_equal(len(cids), 6)
assert_array_equal(cmap > 0, np.abs(pmap) > 2)
def _run_interface(self, runtime):
in_file = nb.load(self.inputs.in_file)
wm_mask = nb.load(self.inputs.wm_mask).get_data()
wm_mask[wm_mask < 0.9] = 0
wm_mask[wm_mask > 0] = 1
wm_mask = wm_mask.astype(np.uint8)
if self.inputs.erodemsk:
# Create a structural element to be used in an opening operation.
struc = nd.generate_binary_structure(3, 2)
# Perform an opening operation on the background data.
wm_mask = nd.binary_erosion(wm_mask, structure=struc).astype(np.uint8)
data = in_file.get_data()
data *= 1000.0 / np.median(data[wm_mask > 0])
out_file = fname_presuffix(self.inputs.in_file,
suffix='_harmonized', newpath='.')
in_file.__class__(data, in_file.affine, in_file.header).to_filename(
out_file)
self._results['out_file'] = out_file
return runtime
def artifact_mask(imdata, airdata, distance, zscore=10.):
"""Computes a mask of artifacts found in the air region"""
from statsmodels.robust.scale import mad
if not np.issubdtype(airdata.dtype, np.integer):
airdata[airdata < .95] = 0
airdata[airdata > 0.] = 1
bg_img = imdata * airdata
if np.sum((bg_img > 0).astype(np.uint8)) < 100:
return np.zeros_like(airdata)
# Find the background threshold (the most frequently occurring value
# excluding 0)
bg_location = np.median(bg_img[bg_img > 0])
bg_spread = mad(bg_img[bg_img > 0])
bg_img[bg_img > 0] -= bg_location
bg_img[bg_img > 0] /= bg_spread
# Apply this threshold to the background voxels to identify voxels
# contributing artifacts.
qi1_img = np.zeros_like(bg_img)
qi1_img[bg_img > zscore] = 1
qi1_img[distance < .10] = 0
# Create a structural element to be used in an opening operation.
struc = nd.generate_binary_structure(3, 1)
qi1_img = nd.binary_opening(qi1_img, struc).astype(np.uint8)
qi1_img[airdata <= 0] = 0
return qi1_img
def split_exclusions(image, labels, exclusions, dilation=0, connectivity=1,
standard_seeds=False):
"""Ensure that no segment in 'labels' overlaps more than one exclusion."""
labels = labels.copy()
cur_label = labels.max()
dilated_exclusions = exclusions.copy()
foot = generate_binary_structure(exclusions.ndim, connectivity)
for i in range(dilation):
dilated_exclusions = grey_dilation(exclusions, footprint=foot)
hashed = labels * (exclusions.max() + 1) + exclusions
hashed[exclusions == 0] = 0
violations = bincount(hashed.ravel()) > 1
violations[0] = False
if sum(violations) != 0:
offending_labels = labels[violations[hashed]]
mask = zeros(labels.shape, dtype=bool)
for offlabel in offending_labels:
mask += labels == offlabel
if standard_seeds:
seeds = label(mask * (image == 0))[0]
else:
seeds = label(mask * dilated_exclusions)[0]
seeds[seeds > 0] += cur_label
labels[mask] = watershed(image, seeds, connectivity, mask)[mask]
return labels
def manual_split(probs, seg, body, seeds, connectivity=1, boundary_seeds=None):
"""Manually split a body from a segmentation using seeded watershed.
Input:
- probs: the probability of boundary in the volume given.
- seg: the current segmentation.
- body: the label to be split.
- seeds: the seeds for the splitting (should be just two labels).
[-connectivity: the connectivity to use for watershed.]
[-boundary_seeds: if not None, these locations become inf in probs.]
Value:
- the segmentation with the selected body split.
"""
struct = generate_binary_structure(seg.ndim, connectivity)
body_pixels = seg == body
bbox = find_objects(body_pixels)[0]
body_pixels = body_pixels[bbox]
body_boundary = binary_dilation(body_pixels, struct) - body_pixels
non_body_pixels = True - body_pixels - body_boundary
probs = probs.copy()[bbox]
probs[non_body_pixels] = probs.min()-1
if boundary_seeds is not None:
probs[boundary_seeds[bbox]] = probs.max()+1
probs[body_boundary] = probs.max()+1
seeds = label(seeds.astype(bool)[bbox], struct)[0]
outer_seed = seeds.max()+1 # should be 3
seeds[non_body_pixels] = outer_seed
seg_new = watershed(probs, seeds,
dams=(seg==0).any(), connectivity=connectivity, show_progress=True)
seg = seg.copy()
new_seeds = unique(seeds)[:-1]
for new_seed, new_label in zip(new_seeds, [0, body, seg.max()+1]):
seg[bbox][seg_new == new_seed] = new_label
return seg
def get_largest_two_component(img, prt = False, threshold = None):
s = ndimage.generate_binary_structure(3,2) # iterate structure
labeled_array, numpatches = ndimage.label(img,s) # labeling
sizes = ndimage.sum(img,labeled_array,range(1,numpatches+1))
sizes_list = [sizes[i] for i in range(len(sizes))]
sizes_list.sort()
#if(prt):
# print('component size', sizes_list, flush = True)
if(len(sizes) == 1):
return img
else:
if(threshold):
out_img = np.zeros_like(img)
for temp_size in sizes_list:
if(temp_size > threshold):
temp_lab = np.where(sizes == temp_size)[0] + 1
temp_cmp = labeled_array == temp_lab
out_img = (out_img + temp_cmp) > 0
return out_img
else:
max_size1 = sizes_list[-1]
max_size2 = sizes_list[-2]
max_label1 = np.where(sizes == max_size1)[0] + 1
max_label2 = np.where(sizes == max_size2)[0] + 1
component1 = labeled_array == max_label1
component2 = labeled_array == max_label2
#if(prt):
# print(max_size2, max_size1, max_size2/max_size1, flush = True)
if(max_size2*10 > max_size1):
component1 = (component1 + component2) > 0
return component1
def __init__(self, label_image=None, connectivity=1, data=None, **attr):
super(RAG, self).__init__(data, **attr)
if self.number_of_nodes() == 0:
self.max_id = 0
else:
self.max_id = max(self.nodes_iter())
if label_image is not None:
fp = ndi.generate_binary_structure(label_image.ndim, connectivity)
# In the next ``ndi.generic_filter`` function, the kwarg
# ``output`` is used to provide a strided array with a single
# 64-bit floating point number, to which the function repeatedly
# writes. This is done because even if we don't care about the
# output, without this, a float array of the same shape as the
# input image will be created and that could be expensive in
# memory consumption.
ndi.generic_filter(
label_image,
function=_add_edge_filter,
footprint=fp,
mode='nearest',
output=as_strided(np.empty((1,), dtype=np.float_),
shape=label_image.shape,
strides=((0,) * label_image.ndim)),
extra_arguments=(self,))
def test_labeling():
"Test cluster labeling"
shape = flat_shape = (4, 20)
pmap = np.empty(shape, np.float_)
out = np.empty(shape, np.uint32)
bin_buff = np.empty(shape, np.bool_)
int_buff = np.empty(shape, np.uint32)
struct = ndimage.generate_binary_structure(2, 1)
struct[::2] = False
conn = np.array([(0, 1), (0, 3), (1, 2), (2, 3)], np.uint32)
criteria = None
# some clusters
pmap[:] = [[ 3, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0],
[ 0, 1, 0, 0, 0, 0, 8, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 0],
[ 0, 3, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4],
[ 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 0]]
cids = _label_clusters(pmap, out, bin_buff, int_buff, 2, 0, struct, False,
flat_shape, conn, criteria)
assert_equal(len(cids), 6)
# some other clusters
pmap[:] = [[ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0],
[ 0, 4, 0, 0, 0, 0, 0, 4, 0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 4, 4, 0, 4, 4, 0, 4, 0, 0, 0, 4, 4, 1, 0, 4, 4, 0, 0],
[ 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0]]
cids = _label_clusters(pmap, out, bin_buff, int_buff, 2, 0, struct, False,
flat_shape, conn, criteria)
assert_equal(len(cids), 6)
def f_returnInternalEdge(self,cl_array):
# Internal edge: Count of neighboring non-zero cell
kernel = ndimage.generate_binary_structure(2, 1) # Make a kernel
kernel[1, 1] = 0
b = ndimage.convolve(cl_array, kernel, mode="constant")
n_interior = b[cl_array != 0].sum() # Number of interiror edges
return n_interior
def InDecPatch(self,which,amount):
s = ndimage.generate_binary_structure(2,1) # taxi-cab struct
if which == 0:
ras = ndimage.binary_dilation(self.cl_array,s,iterations=amount,border_value=0)
else:
ras = ndimage.binary_erosion(self.cl_array,s,iterations=amount,border_value=0)
return(ras)
def compute_sparsity(im):
l_x = len(im)
X, Y = np.ogrid[:l_x, :l_x]
mask = ((X - l_x/2)**2 + (Y - l_x/2)**2 <= (l_x/2)**2)
grad1 = ndimage.morphological_gradient(im, footprint=np.ones((3, 3)))
grad2 = ndimage.morphological_gradient(im, footprint=ndimage.generate_binary_structure(2, 1))
return (grad1[mask] > 0).mean(), (grad2[mask] > 0).mean()
def noise_cube(data,
mask=None,
nThresh=30,
iterations=1,
do_map=True,
do_spec=True,
box=None,
spec_box=None,
bandpass_smooth_window=None,
bandpass_smooth_order=3,
oversample_boundary=False):
"""
Makes an empirical estimate of the noise in a cube assuming that
it is normally distributed about zero. Treats the spatial and
spectral dimensions as separable.
Parameters:
-----------
data : np.array
Array of data (floats)
Keywords:
---------
mask : np.bool
Boolean array with False indicating where data can be
used in the noise estimate. (i.e., True is signal)
do_map : np.bool
Estimate spatial variations in the noise. Default is True. If
set to False, all locations in a plane have the same noise
estimate.
do_spec : np.bool
Estimate spectral variations in the noise. Default is True. If
set to False, all channels in a spectrum have the same noise
estimate.
box : int
Spatial size of the box over which noise is calculated in
pixels. Default: no box, every pixel gets its own noise
estimte.
spec_box : int
Spectral size of the box overwhich the noise is calculated.
Default: no box, each channel gets its own noise estimate.
nThresh : int
Minimum number of data to be used in an individual noise estimate.
iterations : int
Number of times to iterate the noise solution to force Gaussian
statistics. Default: no iterations.
bandpass_smooth_window : int
Number of channels used in bandpass smoothing kernel. Defaults to
nChan / 4 where nChan number of channels. Set to zero to suppress
smoothing. Uses Savitzky-Golay smoothing
bandpass_smooth_order : int
Polynomial order used in smoothing kernel. Defaults to 3.
"""
# TBD: add error checking
# Create a mask that identifies voxels to be fitting the noise
noisemask = np.isfinite(data)
if mask is not None:
noisemask[mask] = False
# Default the spatial step size to individual pixels
step = 1
halfbox = step // 2
# If the user has supplied a spatial box size, recast this into a
# step size that critically samples the box and a halfbox size
# used for convenience.
if box is not None:
step = np.floor(box / 2.5).astype(np.int)
halfbox = int(box // 2)
# Include all pixels adjacent to the spatial
# boundary of the data as set by NaNs
boundary = np.all(np.isnan(data), axis=0)
if oversample_boundary:
struct = nd.generate_binary_structure(2, 1)
struct = nd.iterate_structure(struct, halfbox)
rind = np.logical_xor(nd.binary_dilation(boundary, struct), boundary)
extray, extrax = np.where(rind)
else:
extray, extrax = None, None
# If the user has supplied a spectral box size, use this to
# calculate a spectral step size.
if spec_box is not None:
spec_step = np.floor(spec_box / 2).astype(np.int)
boxv = int(spec_box // 2)
else:
boxv = 0
# Default the bandpass smoothing window
if bandpass_smooth_window is None:
bandpass_smooth_window = 2 * (data.shape[0] // 8) + 1
# Initialize output to be used in the case of iterative
# estimation.
noise_cube_out = np.ones_like(data)
# Iterate
for ii in np.arange(iterations):
if not do_map:
# If spatial variations are turned off then estimate a
# single value and fill the noise map with this value.
noise_value = mad_zero_centered(data, mask=noisemask)
noise_map = np.zeros(data.shape[1:]) + noise_value
else:
# Initialize map to be full of not-a-numbers
noise_map = np.zeros(data.shape[1:]) + np.nan
# Make a noise map
xx = np.arange(data.shape[2])
yy = np.arange(data.shape[1])
# Sample starting at halfbox and stepping by step. In the
# individual pixel limit this just samples every spectrum.
xsamps = xx[halfbox::step]
ysamps = yy[halfbox::step]
xsampsf = (xsamps[np.newaxis, :] *
(np.ones_like(ysamps))[:, np.newaxis]).flatten()
ysampsf = (ysamps[:, np.newaxis] * np.ones_like(xsamps)).flatten()
for x, y in zip(xsampsf, ysampsf):
# Extract a minicube and associated mask from the cube
minicube = data[:, (y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
minicube_mask = noisemask[:, (y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
# If we have enough data, fit a noise value for this entry
if np.sum(minicube_mask) > nThresh:
noise_map[y, x] = mad_zero_centered(minicube,
mask=minicube_mask)
if extrax is not None and extray is not None:
for x, y in zip(extrax, extray):
minicube = data[:, (y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
minicube_mask = noisemask[:,
(y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
if np.sum(minicube_mask) > nThresh:
noise_map[y, x] = mad_zero_centered(minicube,
mask=minicube_mask)
noise_map[boundary] = np.nan
# If we are using a box size greater than an individual pixel
# interpolate to fill in the noise map.
if halfbox > 0:
# Note the location of data, this is the location
# where we want to fill in noise values.
data_footprint = np.any(np.isfinite(data), axis=0)
# Generate a smoothing kernel based on the box size.
kernel = Gaussian2DKernel(box / np.sqrt(8 * np.log(2)))
# Make a weight map to be used in the convolution, in
# this weight map locations with measured values have
# unity weight. This without measured values have zero
# weight.
# wt_map = np.isfinite(noise_map).astype(np.float)
# wt_map[boundary] = np.nan
# Take an average weighted by the kernel at each
# location.
# noise_map[np.isnan(noise_map)] = 0.0
# y, x = np.where(np.isfinite(noise_map))
# import scipy.interpolate as interp
# func = interp.interp2d(x, y, noise_map[y, x], kind='cubic')
noise_map = convolve(noise_map, kernel, boundary='extend')
# yy, xx = np.indices(noise_map.shape)
# noise_map_beta = interp.griddata((y, x), noise_map[y,x],
# (yy, xx), method='cubic')
# noise_map_beta = func(yy, xx)
# noise_map_beta[boundary] = np.nan
# noise_map = noise_map_beta
# wt_map = convolve(wt_map, kernel, boundary='extend')
# noise_map /= wt_map
# Set the noise map to not-a-number outside the data
# footprint.
noise_map[~data_footprint] = np.nan
# Initialize spectrum
noise_spec = np.zeros(data.shape[0]) + np.nan
if not do_spec:
# If spectral variations are turned off then assume that
# the noise_map describes all channels of the cube.
pass
else:
# Loop over channels
zz = np.arange(data.shape[0])
for z in zz:
# Idententify the range of channels to be considered
# in this estimate.
lowz = np.clip(z - boxv, 0, data.shape[0])
hiz = np.clip(z + boxv + 1, 0, data.shape[0])
# Extract a slab from the cube and normalize it by the
# noise map. Now any measured noise variations are
# relative to those in the noise map.
slab = data[lowz:hiz, :, :] / noise_map[np.newaxis, :, :]
slab_mask = noisemask[lowz:hiz, :, :]
noise_spec[z] = mad_zero_centered(slab, mask=slab_mask)
# Smooth the spectral variations in the noise.
if bandpass_smooth_window > 0:
# Initialize a Savitzky-Golay filter then run it over
# the noise spectrum.
kernel = savgol_coeffs(int(bandpass_smooth_window),
int(bandpass_smooth_order))
baddata = np.isnan(noise_spec)
noise_spec = convolve(noise_spec,
kernel,
nan_treatment='interpolate',
boundary='extend')
noise_spec[baddata] = np.nan
# Make sure that the noise spectrum is normalized by
# setting the median to one.
noise_spec /= np.nanmedian(noise_spec)
# Combine the spatial and spectral variations into a
# three-dimensional noise estimate.
noise_cube = np.ones_like(data)
noise_cube *= (noise_map[np.newaxis, :] *
noise_spec[:, np.newaxis, np.newaxis])
if iterations == 1:
return (noise_cube)
else:
# If iterating, normalize the data by the current noise
# estimate and scale the current noise cube by the new
# estimate.
data = data / noise_cube
noise_cube_out *= noise_cube
# If iterating return the iterated noise cube.
return (noise_cube_out)
def recipe_phangs_noise(incube=None,
outfile=None,
mask=None,
noise_kwargs=None,
return_spectral_cube=False,
overwrite=False):
"""
Wrap noise_cube with a set of preferred parameters for the
PHANGS-ALMA CO work.
Parameters:
-----------
cube : np.array
Array of data (floats)
Keywords:
---------
mask : np.bool
Boolean array with False indicating where data can be used in
the noise estimate. (i.e., True is signal).
"""
# &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Error checking and work out inputs
# &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
if type(incube) is SpectralCube:
cube = incube
elif type(incube) == str:
cube = SpectralCube.read(incube)
else:
logger.error("Input must be a SpectralCube object or a filename.")
# Initialize an empty kwargs dictionary
if noise_kwargs is None:
noise_kwargs = {}
# If no box is specified, default to one about two beams across
if 'box' not in noise_kwargs:
pixels_per_beam = cube.pixels_per_beam
box = np.ceil(2.5 * pixels_per_beam**0.5)
noise_kwargs['box'] = box
# Default to an odd bandpass smothing window
if 'bandpass_smooth_window' not in noise_kwargs:
spectral_smooth = np.ceil(cube.shape[0] / 5) // 2 * 2 + 1
noise_kwargs['bandpass_smooth_window'] = spectral_smooth
if 'spec_box' not in noise_kwargs:
noise_kwargs['spec_box'] = 5
if 'iterations' not in noise_kwargs:
noise_kwargs['iterations'] = 4
# Require a valid cube input as a
if mask is not None:
if type(mask) is SpectralCube:
noise_kwargs['mask'] = mask
elif type(mask) == type("hello"):
noise_kwargs['mask'] = SpectralCube.read(mask)
else:
logger.error(
"Mask must be a SpectralCube object or a filename or None.")
# Fill in the mask if it hasn't already been filled in.
if 'mask' not in noise_kwargs:
# Check if a non-trivial signal mask is attached to the cube
if (np.sum(cube.mask.include()) < np.sum(
np.isfinite(cube.filled_data[:].value))):
noise_kwargs['mask'] = cube.mask.include()
else:
noise_kwargs['mask'] = None
# &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Run the noise estimate
# &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
data = cube.filled_data[:].value
badmask = np.isnan(data)
badmask = nd.binary_dilation(badmask,
structure=nd.generate_binary_structure(3, 2))
data[badmask] = np.nan
rms = noise_cube(data, **noise_kwargs)
# &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
# Write or return as requested
# &%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%
# In this case can avoid a recast
if not return_spectral_cube and (outfile is None):
return (rms)
# Recast from numpy array to spectral cube
header = cube.header
header['DATAMIN'] = np.nanmin(rms)
header['DATAMAX'] = np.nanmax(rms)
header['COMMENT'] = 'Produced with PHANGS-ALMA pipeline version ' + version
if tableversion:
header[
'COMMENT'] = 'Galaxy properties from PHANGS sample table version ' + tableversion
rms = SpectralCube(rms,
wcs=cube.wcs,
header=header,
meta={'BUNIT': cube.header['BUNIT']})
# Write to disk, if desired
if outfile is not None:
rms.write(outfile, overwrite=overwrite)
if return_spectral_cube:
return (rms)
else:
return (rms.filled_data[:].value)
def process(self, input=None, output_folder=None, progress_callback=None, filter=None,
correction_factor=2,
cutoff_cell_fusion=None,
restore_safe_cells=False,
_DEBUG=False,
_VISUAL_DEBUG=False, **kwargs):
start = timer()
# filename0 = path
# filename0_without_path = os.path.basename(filename0)
# filename0_without_ext = os.path.splitext(filename0_without_path)[0]
# parent_dir_of_filename0 = os.path.dirname(filename0)
# TA_output_filename = os.path.join(parent_dir_of_filename0, filename0_without_ext,
# 'handCorrection.tif') # TODO allow custom names here to allow ensemble methods
# non_TA_final_output_name = os.path.join(output_folder, filename0_without_ext + '.tif')
#
# filename_to_use_to_save = non_TA_final_output_name
# if TA_mode:
# filename_to_use_to_save = TA_output_filename
#
# if TA_mode:
# # try also to change path input name
# if os.path.exists(
# os.path.join(parent_dir_of_filename0, filename0_without_ext, 'raw_epyseg_output.tif')):
# path = os.path.join(parent_dir_of_filename0, filename0_without_ext, 'raw_epyseg_output.tif')
# img_orig = Img(path)
# print('analyzing', path, self.stop_now)
# try:
# if self.progress_callback is not None:
# self.progress_callback.emit((iii / len(list_of_files)) * 100)
# else:
# logger.info(str((iii / len(list_of_files)) * 100) + '%')
# except:
# traceback.print_exc()
# pass
# DO A DILATION OF SEEDS THEN AN EROSION TO JOIN CLOSE BY SEEDS
img_orig = input
img_has_seeds = True
# mask with several channels
if img_orig.has_c():
if restore_safe_cells:
img_seg = img_orig[..., 0].copy()
seeds_1 = img_orig[..., img_orig.shape[-1] - 1]
seeds_1 = Img.invert(seeds_1)
# seeds_1[seeds_1 >= 0.5] = 255
# seeds_1[seeds_1 < 0.5] = 0
seeds_1[seeds_1 >= 0.2] = 255 # TODO maybe be more stringent here
seeds_1[seeds_1 < 0.2] = 0
s = ndimage.generate_binary_structure(2, 1)
seeds_1 = ndimage.grey_dilation(seeds_1, footprint=s)
seeds_1 = ndimage.grey_dilation(seeds_1, footprint=s)
seeds_1 = ndimage.grey_dilation(seeds_1, footprint=s)
seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s)
seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s)
# seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s)
# seeds_1 = ndimage.grey_erosion(seeds_1, footprint=s)
# for debug
if _DEBUG:
Img(seeds_1, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'wshed_seeds.tif')) # not bad
lab_seeds = label(seeds_1.astype(np.uint8), connectivity=2, background=0)
#
for region in regionprops(lab_seeds):
if region.area < 10:
for coordinates in region.coords:
lab_seeds[coordinates[0], coordinates[1]] = 0
if _DEBUG:
Img(seeds_1, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'wshed_seeds_deblobed.tif'))
img_orig[..., 3] = Img.invert(img_orig[..., 3])
img_orig[..., 4] = Img.invert(img_orig[..., 4])
# seems to work --> now need to do the projection
for c in range(1, img_orig.shape[-1] - 2):
img_orig[..., 0] += img_orig[..., 1]
img_orig[..., 0] /= img_orig.shape[-1] - 2
img_orig = img_orig[..., 0]
else:
# mask with single channel
img_has_seeds = False
if restore_safe_cells:
img_seg = img_orig.copy()
if restore_safe_cells:
if _DEBUG:
print(os.path.join(output_folder, 'extras', 'img_seg.tif'))
Img(img_seg, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'img_seg.tif'))
# for debug
if _DEBUG:
Img(img_orig, dimensions='hw').save(os.path.join(output_folder, 'extras', 'avg.tif'))
img_saturated = img_orig.copy()
if img_has_seeds:
img_saturated[img_saturated >= 0.5] = 255
img_saturated[img_saturated < 0.5] = 0
if restore_safe_cells:
# TODO maybe do a safe image
img_seg[img_seg >= 0.3] = 255
img_seg[img_seg < 0.3] = 0
secure_mask = img_seg
else:
img_saturated[img_saturated >= 0.3] = 255
img_saturated[img_saturated < 0.3] = 0
if restore_safe_cells:
img_seg[img_seg >= 0.95] = 255
img_seg[img_seg < 0.95] = 0
secure_mask = img_seg
# convert it to seeds and make sure they are all present in there
# if pixel is not labeled then read it
if restore_safe_cells:
labels_n_area_rescue_seeds = {}
rescue_seeds = label(Img.invert(secure_mask), connectivity=1, background=0)
for region in regionprops(rescue_seeds):
labels_n_area_rescue_seeds[region.label] = region.area
if _DEBUG:
Img(secure_mask, dimensions='hw').save(os.path.join(output_folder, 'extras', 'secure_mask.tif'))
# loop over those seeds to rescue
# for debug
if _DEBUG:
Img(img_saturated, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'handCorrection.tif'))
deblob = True
if deblob:
image_thresh = label(img_saturated, connectivity=2, background=0)
# for debug
if _DEBUG:
Img(image_thresh, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'before_deblobed.tif'))
# deblob
min_size = 200
for region in regionprops(image_thresh):
# take regions with large enough areas
if region.area < min_size:
for coordinates in region.coords:
image_thresh[coordinates[0], coordinates[1]] = 0
image_thresh[image_thresh > 0] = 255
img_saturated = image_thresh
# for debug
if _DEBUG:
Img(img_saturated, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'deblobed.tif'))
del image_thresh
# for debug
if _DEBUG:
Img(img_saturated, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'deblobed_out.tif'))
extra_dilations = True
if extra_dilations:
# do a dilation of 2 to close bonds
s = ndimage.generate_binary_structure(2, 1)
dilated = ndimage.grey_dilation(img_saturated, footprint=s)
dilated = ndimage.grey_dilation(dilated, footprint=s)
# Img(dilated, dimensions='hw').save(os.path.join(os.path.splitext(path)[0], 'filled_one_px_holes.tif'))
# other_seeds = label(invert(np.grey_dilation(dilated, footprint=s).astype(np.uint8)), connectivity=1, background=0)
labs = label(Img.invert(img_saturated.astype(np.uint8)), connectivity=1, background=0)
for region in regionprops(labs):
seeds = []
# exclude tiny cells form dilation because they may end up completely closed
if region.area >= 10 and region.area < 350:
for coordinates in region.coords:
dilated[coordinates[0], coordinates[1]] = 0
continue
else:
# pb when big cells around cause connections are not done
# preserve cells at edges because they have to e naturally smaller because they are cut
# put a size criterion too
if region.area < 100 and (
region.bbox[0] <= 1 or region.bbox[1] <= 1 or region.bbox[2] >= labs.shape[-2] - 2 or
region.bbox[
3] >= \
labs.shape[-1] - 2):
# edge cell detected --> removing dilation
for coordinates in region.coords:
dilated[coordinates[0], coordinates[1]] = 0
continue
img_saturated = dilated
# for debug
if _DEBUG:
Img(img_saturated, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'dilated_further.tif'))
del dilated
list_of_cells_to_dilate = []
labs = label(Img.invert(img_saturated.astype(np.uint8)), connectivity=1, background=0)
# c'est cette correction qui fixe bcp de choses mais recree aussi des choses qui n'existent pas... --> voir à quoi sont dus ces lignes blobs
# faudrait redeblober
if img_has_seeds:
for region in regionprops(labs, intensity_image=img_orig):
seeds = []
if not extra_dilations and region.area < 10:
continue
# if small and no associated seeds --> remove it ??? maybe or not
for coordinates in region.coords:
id = lab_seeds[coordinates[0], coordinates[1]]
if id != 0:
seeds.append(id)
seeds = set(seeds)
if len(seeds) >= 2:
# we may have found an undersegmented cell --> try segment it better
list_of_cells_to_dilate.append(region.label)
if len(list_of_cells_to_dilate) != 0:
props = regionprops(labs, intensity_image=img_orig)
for run in range(10):
something_changed = False # early stop
for region in props:
if region.label not in list_of_cells_to_dilate:
continue
# TODO recheck those values and wether it makes sense
threshold_values = [80 / 255, 60 / 255, 40 / 255, 30 / 255,
20 / 255,
10 / 255] # 160 / 255, 140 / 255, 120 / 255, 100 / 255, 1 / 255 , 2 / 255, , 5 / 255
try:
for threshold in threshold_values:
mask = region.image.copy()
image = region.image.copy()
image[region.intensity_image > threshold] = True
image[region.intensity_image <= threshold] = False
final = Img.invert(image.astype(np.uint8))
final[final < 255] = 0
final[mask == False] = 0
new_seeds = label(final, connectivity=1, background=0)
props2 = regionprops(new_seeds)
if len(props2) > 1: # cell was resplitted into smaller
for r in props2:
if r.area < 20:
raise Exception
region.image[mask == False] = False
region.image[mask == True] = True
region.image[new_seeds > 0] = False
something_changed = True
for coordinates in region.coords:
img_saturated[coordinates[0], coordinates[1]] = 255
region.image[mask == False] = False
region.image[mask == True] = True
del final
del new_seeds
except:
traceback.print_exc()
pass
if not something_changed:
# print('no more changes anymore --> quitting')
break
# for debug
if _DEBUG:
Img(img_saturated, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'saturated_mask4.tif'))
final_seeds = label(Img.invert(img_saturated), connectivity=1,
background=0) # keep like that otherwise creates tiny cells with erroneous wshed
# for debug
if _DEBUG:
Img(final_seeds, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'final_seeds_before.tif'))
final_seeds = label(Img.invert(img_saturated), connectivity=2, background=0) # is that needed ???
# for debug
if _DEBUG:
Img(final_seeds, dimensions='hw').save(
os.path.join(output_folder, 'extras', 'final_seeds_before2.tif'))
final_seeds[img_saturated == 255] = 0
final_wshed = watershed(img_orig, markers=final_seeds,
watershed_line=True)
final_wshed[final_wshed != 0] = 1 # remove all seeds
final_wshed[final_wshed == 0] = 255 # set wshed values to 255
final_wshed[final_wshed == 1] = 0 # set all other cell content to
# filename0 = os.path.basename(path)
# parent_path = os.path.dirname(os.path.dirname(path))
if filter is None or filter == 0:
# TODO maybe offer the choice between saving wshed on predict or on orig
# Img(final_wshed, dimensions='hw').save(os.path.join(output_folder, os.path.splitext(filename0)[
# 0]) + '.tif') # need put original name here TODO put image default name here
# print('saving', filename_to_use_to_save)
# Img(final_wshed.astype(np.uint8), dimensions='hw').save(filename_to_use_to_save)
return final_wshed.astype(np.uint8)
else:
if isinstance(filter, int):
filter_by_size = filter
else:
filter_by_size = None
avg_area = 0
count = 0
if _DEBUG:
Img(final_wshed, dimensions='hw').save(os.path.join(output_folder, 'extras', 'test_size_cells.tif'))
final_seeds = Img.invert(final_wshed)
final_seeds = label(final_seeds, connectivity=1, background=0)
if _VISUAL_DEBUG:
plt.imshow(final_seeds)
plt.show()
removed_seeds = []
keep_seeds = []
labels_n_bbox = {}
labels_n_area = {}
border_cells = []
ids_n_local_median = {}
correspondance_between_cur_seeds_and_safe_ones = {}
if isinstance(filter, str) and 'local' in filter:
rps = regionprops(final_seeds)
for region in rps:
labels_n_bbox[region.label] = region.bbox
labels_n_area[region.label] = region.area
if (region.bbox[0] <= 3 or region.bbox[1] <= 3 or region.bbox[2] >= final_seeds.shape[-2] - 5 or
region.bbox[
3] >= \
final_seeds.shape[-1] - 5):
border_cells.append(region.label)
if restore_safe_cells:
for coordinates in region.coords:
if rescue_seeds[coordinates[0], coordinates[1]] != 0: # do r
correspondance_between_cur_seeds_and_safe_ones[region.label] = rescue_seeds[
coordinates[0], coordinates[1]]
break
break
_, tiles = Img.get_2D_tiles_with_overlap(final_seeds, overlap=64, dimension_h=-2, dimension_w=-1)
for r in tiles:
for tile in r:
rps2 = regionprops(tile)
for region in rps2:
if self.stop_now:
return
if region.label in border_cells:
continue
if (region.bbox[0] <= 3 or region.bbox[1] <= 3 or region.bbox[2] >= final_seeds.shape[
-2] - 5 or
region.bbox[
3] >= \
final_seeds.shape[-1] - 5):
continue
area_of_neighboring_cells = []
for region2 in rps2:
if region2.label == region.label:
continue
# find all cells with
if self.rect_distance(region.bbox, region2.bbox) <= 1:
area_of_neighboring_cells.append(labels_n_area[region2.label])
if area_of_neighboring_cells:
median = statistics.median_low(area_of_neighboring_cells)
ids_n_local_median[
region.label] = median / correction_factor
if region.area <= median / correction_factor:
removed_seeds.append(region.label)
else:
keep_seeds.append(region.label)
removed_seeds = [x for x in removed_seeds if x not in keep_seeds]
# TODO offer the things below as an option --> prevent removal of sure seeds or something like that
if restore_safe_cells:
removed_seeds_to_restore = []
for region in regionprops(final_seeds):
if region.label in removed_seeds:
first = True
for coordinates in region.coords:
if first and rescue_seeds[coordinates[0], coordinates[1]] != 0:
percent_diff = min(labels_n_area[region.label], labels_n_area_rescue_seeds[
rescue_seeds[coordinates[0], coordinates[1]]]) / max(
labels_n_area[region.label], labels_n_area_rescue_seeds[
rescue_seeds[coordinates[0], coordinates[1]]])
if (percent_diff >= 0.7 and percent_diff < 1.0) or (
labels_n_area[region.label] <= 200 and (
percent_diff >= 0.3 and percent_diff < 1.0)):
if _DEBUG:
print('0 finally not removing seed, safe seed', region.label,
percent_diff,
labels_n_area[region.label],
labels_n_area_rescue_seeds[
rescue_seeds[coordinates[0], coordinates[1]]],
labels_n_area[region.label] / labels_n_area_rescue_seeds[
rescue_seeds[coordinates[0], coordinates[1]]],
region.centroid)
removed_seeds_to_restore.append(region.label)
break
break
removed_seeds = [x for x in removed_seeds if x not in removed_seeds_to_restore]
else:
areas = []
for region in regionprops(final_seeds):
if (region.bbox[0] <= 3 or region.bbox[1] <= 3 or region.bbox[2] >= final_seeds.shape[-2] - 5 or
region.bbox[3] >= final_seeds.shape[-1] - 5):
continue
avg_area += region.area
count += 1
areas.append(region.area)
avg_area /= count
median = statistics.median_low(areas)
if isinstance(filter, int):
filter_by_size = filter
elif 'avg' in filter:
filter_by_size = avg_area / correction_factor
elif 'median' in filter:
filter_by_size = median / correction_factor
# TODO maybe use stdev or alike to see if cell should really be removed
if _DEBUG:
print('filter cells below=', filter_by_size, 'avg cell area=', avg_area, 'median=',
median) # , 'median', median
if filter_by_size is not None and filter_by_size != 0:
if _VISUAL_DEBUG:
plt.imshow(final_seeds)
plt.show()
for region in regionprops(final_seeds):
labels_n_bbox[region.label] = region.bbox
labels_n_area[region.label] = region.area
if region.area < filter_by_size:
if (region.bbox[0] <= 2 or region.bbox[1] <= 2 or region.bbox[2] >= labs.shape[
-2] - 3 or
region.bbox[
3] >= \
labs.shape[
-1] - 3):
continue
removed_seeds.append(region.label)
if cutoff_cell_fusion is not None and cutoff_cell_fusion > 1:
cells_to_fuse = []
for idx, removed_seed in enumerate(removed_seeds):
current_cells_to_fuse = set()
closest_pair = None
smallest_distance = None
for idx2 in range(idx + 1, len(removed_seeds)):
removed_seed2 = removed_seeds[idx2]
if closest_pair is None:
if self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) <= 1:
closest_pair = removed_seed2
smallest_distance = self.rect_distance(labels_n_bbox[removed_seed],
labels_n_bbox[removed_seed2])
elif self.rect_distance(labels_n_bbox[removed_seed],
labels_n_bbox[removed_seed2]) <= smallest_distance:
closest_pair = removed_seed2
smallest_distance = self.rect_distance(labels_n_bbox[removed_seed],
labels_n_bbox[removed_seed2])
if self.rect_distance(labels_n_bbox[removed_seed], labels_n_bbox[removed_seed2]) <= 1:
current_cells_to_fuse.add(removed_seed)
current_cells_to_fuse.add(removed_seed2)
if current_cells_to_fuse:
cells_to_fuse.append(current_cells_to_fuse)
cells_to_fuse = [frozenset(i) for i in cells_to_fuse]
cells_to_fuse = list(dict.fromkeys(cells_to_fuse))
cells_to_keep = []
if cutoff_cell_fusion is not None and cutoff_cell_fusion > 0:
superfuse = []
copy_of_cells_to_fuse = cells_to_fuse.copy()
for idx, fuse in enumerate(copy_of_cells_to_fuse):
current_fusion = set(fuse.copy())
changed = True
while changed:
changed = False
for idx2 in range(len(copy_of_cells_to_fuse) - 1, idx, -1):
fuse2 = copy_of_cells_to_fuse[idx2]
if idx2 == idx:
continue
if fuse2.intersection(current_fusion):
current_fusion.update(fuse2)
del copy_of_cells_to_fuse[idx2]
changed = True
superfuse.append(current_fusion)
for sf in superfuse:
if len(sf) > cutoff_cell_fusion:
for val in sf:
cells_to_keep.append(val)
seeds_to_fuse = []
cells_to_fuse = sorted(cells_to_fuse, key=len)
for fuse in cells_to_fuse:
cumulative_area = 0
for _id in fuse:
if _id in cells_to_keep:
if _id in removed_seeds:
removed_seeds.remove(_id)
continue
cumulative_area += labels_n_area[_id]
if filter_by_size is not None:
if cumulative_area >= filter_by_size: #: #1200: #filter_by_size: # need hack this to get local area
seeds_to_fuse.append(fuse)
for _id in fuse:
if _id in removed_seeds:
removed_seeds.remove(_id)
else:
if cumulative_area >= ids_n_local_median[_id]:
seeds_to_fuse.append(fuse)
for _id in fuse:
if _id in removed_seeds:
removed_seeds.remove(_id)
# need recolor all the seeds in there with the new seed stuff
for fuse in seeds_to_fuse:
for _id in fuse:
break
for region in regionprops(final_seeds):
if region.label in fuse:
for coordinates in region.coords:
final_seeds[coordinates[0], coordinates[1]] = _id
if _VISUAL_DEBUG:
plt.imshow(final_seeds)
plt.show()
for region in regionprops(final_seeds):
if region.label in removed_seeds:
for coordinates in region.coords:
final_seeds[coordinates[0], coordinates[1]] = 0
if _VISUAL_DEBUG:
plt.imshow(final_seeds)
plt.show()
if _VISUAL_DEBUG:
plt.imshow(final_seeds)
plt.show()
final_wshed = watershed(img_orig, markers=final_seeds, watershed_line=True)
final_wshed[final_wshed != 0] = 1 # remove all seeds
final_wshed[final_wshed == 0] = 255 # set wshed values to 255
final_wshed[final_wshed == 1] = 0 # set all other cell content to
if _VISUAL_DEBUG:
plt.imshow(final_wshed)
plt.show()
# print('saving', filename_to_use_to_save)
# Img(final_wshed.astype(np.uint8), dimensions='hw').save(filename_to_use_to_save)
duration = timer() - start
if _DEBUG:
print('final duration wshed in secs', duration)
return final_wshed.astype(np.uint8) # is indeed a 2D image
def _deblend_source(data,
segment_img,
npixels,
nlevels=32,
contrast=0.001,
mode='exponential',
connectivity=8):
"""
Deblend a single labeled source.
Parameters
----------
data : array_like
The 2D array of the image. The should be a cutout for a single
source. ``data`` should already be smoothed by the same filter
used in :func:`~photutils.detect_sources`, if applicable.
segment_img : `~photutils.segmentation.SegmentationImage`
A cutout `~photutils.segmentation.SegmentationImage` object with
the same shape as ``data``. ``segment_img`` should contain only
*one* source label.
npixels : int
The number of connected pixels, each greater than ``threshold``,
that an object must have to be detected. ``npixels`` must be a
positive integer.
nlevels : int, optional
The number of multi-thresholding levels to use. Each source
will be re-thresholded at ``nlevels``, spaced exponentially or
linearly (see the ``mode`` keyword), between its minimum and
maximum values within the source segment.
contrast : float, optional
The fraction of the total (blended) source flux that a local
peak must have to be considered as a separate object.
``contrast`` must be between 0 and 1, inclusive. If ``contrast
= 0`` then every local peak will be made a separate object
(maximum deblending). If ``contrast = 1`` then no deblending
will occur. The default is 0.001, which will deblend sources with
a magnitude differences of about 7.5.
mode : {'exponential', 'linear'}, optional
The mode used in defining the spacing between the
multi-thresholding levels (see the ``nlevels`` keyword).
connectivity : {4, 8}, optional
The type of pixel connectivity used in determining how pixels
are grouped into a detected source. The options are 4 or 8
(default). 4-connected pixels touch along their edges.
8-connected pixels touch along their edges or corners. For
reference, SExtractor uses 8-connected pixels.
Returns
-------
segment_image : `~photutils.segmentation.SegmentationImage`
A 2D segmentation image, with the same shape as ``data``, where
sources are marked by different positive integer values. A
value of zero is reserved for the background.
"""
from scipy import ndimage
from skimage.morphology import watershed
if nlevels < 1:
raise ValueError('nlevels must be >= 1, got "{0}"'.format(nlevels))
if contrast < 0 or contrast > 1:
raise ValueError('contrast must be >= 0 or <= 1, got '
'"{0}"'.format(contrast))
if connectivity == 4:
selem = ndimage.generate_binary_structure(2, 1)
elif connectivity == 8:
selem = ndimage.generate_binary_structure(2, 2)
else:
raise ValueError('Invalid connectivity={0}. '
'Options are 4 or 8'.format(connectivity))
segm_mask = (segment_img.data > 0)
source_values = data[segm_mask]
source_min = np.min(source_values)
source_max = np.max(source_values)
if source_min == source_max:
return segment_img # no deblending
if source_min < 0:
warnings.warn(
'Source "{0}" contains negative values, setting '
'deblending mode to "linear"'.format(segment_img.labels[0]),
AstropyUserWarning)
mode = 'linear'
source_sum = float(np.sum(source_values))
steps = np.arange(1., nlevels + 1)
if mode == 'exponential':
if source_min == 0:
source_min = source_max * 0.01
thresholds = source_min * ((source_max / source_min)**(steps /
(nlevels + 1)))
elif mode == 'linear':
thresholds = source_min + ((source_max - source_min) /
(nlevels + 1)) * steps
else:
raise ValueError('"{0}" is an invalid mode; mode must be '
'"exponential" or "linear"')
# create top-down tree of local peaks
segm_tree = []
for level in thresholds[::-1]:
segm_tmp = detect_sources(data,
level,
npixels=npixels,
connectivity=connectivity)
if segm_tmp.nlabels >= 2:
fluxes = []
for i in segm_tmp.labels:
fluxes.append(np.sum(data[segm_tmp == i]))
idx = np.where((np.array(fluxes) / source_sum) >= contrast)[0]
if len(idx >= 2):
segm_tree.append(segm_tmp)
nbranch = len(segm_tree)
if nbranch == 0:
return segment_img
else:
for j in np.arange(nbranch - 1, 0, -1):
intersect_mask = (segm_tree[j].data *
segm_tree[j - 1].data).astype(bool)
intersect_labels = np.unique(segm_tree[j].data[intersect_mask])
if segm_tree[j - 1].nlabels <= len(intersect_labels):
segm_tree[j - 1] = segm_tree[j]
else:
# If a higher tree level has more peaks than in the
# intersected label(s) with the level below, then remove
# the intersected label(s) in the lower level, add the
# higher level, and relabel.
segm_tree[j].remove_labels(intersect_labels)
new_segments = segm_tree[j].data + segm_tree[j - 1].data
new_segm, nsegm = ndimage.label(new_segments)
segm_tree[j - 1] = SegmentationImage(new_segm)
return SegmentationImage(
watershed(-data,
segm_tree[0].data,
mask=segment_img.data,
connectivity=selem))
def count_cells(self, rawim, probim, maskim):
raw = rawim.copy()
# binarize probability
# THIS ASSUMES THAT probim HAS VALUE RANGE OF [0,255]!
# This normalization is handled by Reader process
binary = (probim > self.prob_threshold * 255)
# apply mask
binary[maskim] = 0
# define connectivity
struct = ndi.generate_binary_structure(3, 1)
# label objects
lbl, _ = ndi.label(binary, struct)
# unique ids and counts
unique, counts = np.unique(lbl, return_counts=True)
# if no object was found, continue
if unique.max() == 0:
return []
# filter objects by its volume
small, medium, large = [], [], []
for uq, ct in zip(unique, counts):
if uq == 0:
continue # skip zero!
if ct <= self.min_volume:
small.append([
uq, ct
]) # if object is smaller than mimimum size, it gets excluded
elif self.min_volume < ct <= self.max_volume:
medium.append([uq, ct])
else:
large.append([uq, ct])
# list to store detected objects
detected_obj = []
# take care of medium objects
obj_ids = [e[0] for e in medium]
volumes = [e[1] for e in medium]
if obj_ids: # skip if empty
coms = ndi.center_of_mass(binary, lbl, obj_ids)
coms = np.array(coms).round().astype(np.int) # convert to integer
for i, com in enumerate(coms):
this_idx = obj_ids[i]
deltaI, bg = self._get_intensity(raw, lbl, this_idx, com,
self.intensity_mode,
self.local_max_rad,
self.local_min_rad,
self.local_min_percentile)
vol = volumes[i]
obj = [com[2], com[1], com[0], deltaI, bg,
vol] # X, Y, Z, intensity, volume
detected_obj.append(obj)
# take care of large objects
obj_ids = [e[0] for e in large]
if obj_ids: # skip if empty
coms = self._com_large_obj(raw, lbl,
obj_ids) # centers of mass and its ID
for com in coms:
this_idx = com[3]
xyz = com[0:3]
deltaI, bg = self._get_intensity(
raw,
lbl,
this_idx,
xyz,
mode="local_mean",
max_rad=2,
min_rad=self.local_min_rad,
min_percentile=self.local_min_percentile)
vol = self.max_volume
obj = [com[2], com[1], com[0], deltaI, bg,
vol] # X, Y, Z, intensity, volume
detected_obj.append(obj)
return detected_obj
dice = []
for f in files:
gt_file = 'gt/' + f
img_gt = nib.load(gt_file).get_data() > 0
pred_file_1 = 'pred_whole_tumor_axial/' + f[:-12] + '_niftynet_out.nii.gz'
pred_file_2 = 'pred_whole_tumor_coronal/' + f[:-12] + '_niftynet_out.nii.gz'
pred_file_3 = 'pred_whole_tumor_sagittal/' + f[:-12] + '_niftynet_out.nii.gz'
img_pred_1 = nib.load(pred_file_1).get_data()[..., 0, 1]
img_pred_2 = nib.load(pred_file_2).get_data()[..., 0, 1]
img_pred_3 = nib.load(pred_file_3).get_data()[..., 0, 1]
img_pred = (img_pred_1 + img_pred_2 + img_pred_3) / 3.0
#img_pred = img_pred_1
#img_pred = img_pred_2
#img_pred = img_pred_3
img_pred = img_pred > 0.5
struct = ndimage.generate_binary_structure(3, 2)
img_pred = ndimage.morphology.binary_closing(img_pred, structure=struct)
img_pred = get_largest_two_component(img_pred, False, 2000)
true_pos = np.float(np.sum(img_gt * img_pred))
union = np.float(np.sum(img_gt) + np.sum(img_pred))
d = true_pos * 2.0 / union
print('%s: %s' % (f[:-12], d))
dice.append(d)
print('%s images mean %s std %s' % (len(dice), np.mean(dice), np.std(dice)))
def _binary_significant_features(radar,
binary_field,
size_bins=75,
size_limits=(0, 300),
structure=None,
debug=False,
verbose=False):
"""
Objectively determine the minimum echo feature size (in radar gates) and
remove features smaller than this from the specified radar field. This
function can be used to objectively remove salt and pepper noise from
binary (mask) radar fields. Unexpected results may occur if the specified
radar field is not a binary field.
Parameters
----------
radar : Radar
Radar object containing the specified binary field.
binary_field : str
The binary radar field that will have insignificant echo features
removed.
size_bins : int, optional
Number of bins used to bin echo feature sizes and thus define its
distribution.
size_limits : list or tuple, optional
Limits of the echo feature size distribution. The upper limit needs to
be large enough to include the minimum feature size. The size bin width
is defined by both the size_bins and size_limits parameters.
structure : array_like, optional
Binary structuring element used to define connected features. The
default structuring element has a squared connectivity equal to one.
debug, verbose : bool, optional
True to print debugging and progress information, respectively, False
to suppress.
"""
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
# Parse feature size arrays
size_data = np.zeros_like(radar.fields[binary_field]['data'],
dtype=np.int32)
feature_sizes = []
for i, sweep in enumerate(radar.iter_slice()):
# Parse radar sweep data
# Non-zero elements of the array form the subset to be dilated
data = radar.fields[binary_field]['data'][sweep]
is_valid_gate = np.ma.filled(data, 0)
# Label the connected features in the sweep data and create index
# array which defines each unique labeled feature
labels, nlabels = ndimage.label(is_valid_gate,
structure=structure,
output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Unique features in sweep {}: {}'.format(i, nlabels)
# Compute the size in radar gates of each labeled feature
sweep_sizes = ndimage.labeled_comprehension(is_valid_gate, labels,
index, np.count_nonzero,
np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each labeled feature to its total size in radar gates
for label, size in zip(index, sweep_sizes):
size_data[sweep][labels == label] = size
feature_sizes = np.hstack(feature_sizes)
# Bin and count occurrences of labeled feature sizes
# Compute bin centers and bin width
counts, bin_edges = np.histogram(feature_sizes,
bins=size_bins,
range=size_limits,
normed=False,
weights=None,
density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Feature size bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the labeled feature size distribution
# We expect the peak of this distribution to be close to 1 radar gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) of a feature
# size bin reaches 0 in the right side of the feature size distribution
# This will define the minimum feature size
is_zero_size = np.logical_and(bin_centers > peak_size,
np.isclose(counts, 0, atol=1.0e-5))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Remove insignificant features from the binary radar field
radar.fields[binary_field]['data'][size_data < min_size] = 0
return
def _raveled_offsets_and_distances(
image_shape,
*,
footprint=None,
connectivity=1,
center=None,
spacing=None,
order='C',
):
"""Compute offsets to neighboring pixels in raveled coordinate space.
This function also returns the corresponding distances from the center
pixel given a spacing (assumed to be 1 along each axis by default).
Parameters
----------
image_shape : tuple of int
The shape of the image for which the offsets are being computed.
footprint : array of bool
The footprint of the neighborhood, expressed as an n-dimensional array
of 1s and 0s. If provided, the connectivity argument is ignored.
connectivity : {1, ..., ndim}
The square connectivity of the neighborhood: the number of orthogonal
steps allowed to consider a pixel a neighbor. See
`scipy.ndimage.generate_binary_structure`. Ignored if footprint is
provided.
center : tuple of int
Tuple of indices to the center of the footprint. If not provided, it
is assumed to be the center of the footprint, either provided or
generated by the connectivity argument.
spacing : tuple of float
The spacing between pixels/voxels along each axis.
order : 'C' or 'F'
The ordering of the array, either C or Fortran ordering.
Returns
-------
raveled_offsets : ndarray
Linear offsets to a samples neighbors in the raveled image, sorted by
their distance from the center.
distances : ndarray
The pixel distances correspoding to each offset.
Notes
-----
This function will return values even if `image_shape` contains a dimension
length that is smaller than `footprint`.
Examples
--------
>>> off, d = _raveled_offsets_and_distances(
... (4, 5), footprint=np.ones((4, 3)), center=(1, 1)
... )
>>> off
array([-5, -1, 1, 5, -6, -4, 4, 6, 10, 9, 11])
>>> d[0]
1.0
>>> d[-1] # distance from (1, 1) to (3, 2)
2.236...
"""
ndim = len(image_shape)
if footprint is None:
footprint = ndi.generate_binary_structure(rank=ndim,
connectivity=connectivity)
if center is None:
center = tuple(s // 2 for s in footprint.shape)
if not footprint.ndim == ndim == len(center):
raise ValueError(
"number of dimensions in image shape, footprint and its"
"center index does not match")
offsets = np.stack([(idx - c)
for idx, c in zip(np.nonzero(footprint), center)],
axis=-1)
if order == 'F':
offsets = offsets[:, ::-1]
image_shape = image_shape[::-1]
elif order != 'C':
raise ValueError("order must be 'C' or 'F'")
# Scale offsets in each dimension and sum
ravel_factors = image_shape[1:] + (1, )
ravel_factors = np.cumprod(ravel_factors[::-1])[::-1]
raveled_offsets = (offsets * ravel_factors).sum(axis=1)
# Sort by distance
if spacing is None:
spacing = np.ones(ndim)
weighted_offsets = offsets * spacing
distances = np.sqrt(np.sum(weighted_offsets**2, axis=1))
sorted_raveled_offsets = raveled_offsets[np.argsort(distances)]
sorted_distances = np.sort(distances)
# If any dimension in image_shape is smaller than footprint.shape
# duplicates might occur, remove them
if any(x < y for x, y in zip(image_shape, footprint.shape)):
# np.unique reorders, which we don't want
_, indices = np.unique(sorted_raveled_offsets, return_index=True)
sorted_raveled_offsets = sorted_raveled_offsets[np.sort(indices)]
sorted_distances = sorted_distances[np.sort(indices)]
# Remove "offset to center"
sorted_raveled_offsets = sorted_raveled_offsets[1:]
sorted_distances = sorted_distances[1:]
return sorted_raveled_offsets, sorted_distances
lambda i, j: (i, -j), lambda i, j: (j, i)
]
def rotate(piece_type, pattern):
return np.array(
[rotate_func[pattern](i, j) for (i, j) in pieces[piece_type]],
dtype=np.int8)
def get_blocks(piece_type, pattern, location):
x, y = location
return rotate(piece_type, pattern) + [x, y]
struct4 = ndimage.generate_binary_structure(2, 1)
struct8 = ndimage.generate_binary_structure(2, 2)
class Blokus(BaseState):
init_pieces_sets = [set(), set(), set(), set()]
init_player = 0
num_players = 4
def __init__(self,
pieces_sets=init_pieces_sets,
current_player=init_player):
self.pieces_sets = pieces_sets
self.current_player = current_player
self.legal = self.get_legal()
def _resolve_neighborhood(footprint,
connectivity,
ndim,
enforce_adjacency=True):
"""Validate or create a footprint (structuring element).
Depending on the values of `connectivity` and `footprint` this function
either creates a new footprint (`footprint` is None) using `connectivity`
or validates the given footprint (`footprint` is not None).
Parameters
----------
footprint : ndarray
The footprint (structuring) element used to determine the neighborhood
of each evaluated pixel (``True`` denotes a connected pixel). It must
be a boolean array and have the same number of dimensions as `image`.
If neither `footprint` nor `connectivity` are given, all adjacent
pixels are considered as part of the neighborhood.
connectivity : int
A number used to determine the neighborhood of each evaluated pixel.
Adjacent pixels whose squared distance from the center is less than or
equal to `connectivity` are considered neighbors. Ignored if
`footprint` is not None.
ndim : int
Number of dimensions `footprint` ought to have.
enforce_adjacency : bool
A boolean that determines whether footprint must only specify direct
neighbors.
Returns
-------
footprint : ndarray
Validated or new footprint specifying the neighborhood.
Examples
--------
>>> _resolve_neighborhood(None, 1, 2)
array([[False, True, False],
[ True, True, True],
[False, True, False]])
>>> _resolve_neighborhood(None, None, 3).shape
(3, 3, 3)
"""
if footprint is None:
if connectivity is None:
connectivity = ndim
footprint = ndi.generate_binary_structure(ndim, connectivity)
else:
# Validate custom structured element
footprint = np.asarray(footprint, dtype=bool)
# Must specify neighbors for all dimensions
if footprint.ndim != ndim:
raise ValueError(
"number of dimensions in image and footprint do not"
"match")
# Must only specify direct neighbors
if enforce_adjacency and any(s != 3 for s in footprint.shape):
raise ValueError("dimension size in footprint is not 3")
elif any((s % 2 != 1) for s in footprint.shape):
raise ValueError("footprint size must be odd along all dimensions")
return footprint
def test_morphology():
"""
测试图像的数学形态学。
:return:
"""
show_content = False
show_images = False
el = ndimage.generate_binary_structure(2, 1)
el_int = el.astype(np.int)
if show_content:
print(el)
print(el_int)
# 腐蚀：最小化滤镜
a = np.zeros((7, 7), dtype=np.int)
a[1:6, 2:5] = 1
a_erosion1 = ndimage.binary_erosion(a).astype(a.dtype)
a_erosion2 = ndimage.binary_erosion(a, structure=np.ones(
(5, 5))).astype(a.dtype)
if show_content:
print(a)
print(a_erosion1)
print(a_erosion2)
if show_images:
image_arrays = [a, a_erosion1, a_erosion2]
image_arrays = bin2gray(image_arrays)
images = ndarray_2_image(image_arrays)
pil_image_demo.plt_images(images, 3)
# 膨胀：最大化滤镜
a = np.zeros((5, 5))
a[2, 2] = 1
a_dilation = ndimage.binary_dilation(a).astype(a.dtype)
if show_content:
print(a)
print(a_dilation)
if show_images:
image_arrays = [a, a_dilation]
image_arrays = bin2gray(image_arrays)
images = ndarray_2_image(image_arrays)
pil_image_demo.plt_images(images, 3)
# 开操作：腐蚀 + 膨胀
square = np.zeros((32, 32))
square[10:-10, 10:-10] = 1
np.random.seed(2)
x, y = (32 * np.random.random((2, 20))).astype(np.int)
square[x, y] = 1
# 图像开操作
open_square = ndimage.binary_opening(square)
# 先腐蚀后膨胀
eroded_square = ndimage.binary_erosion(square)
reconstruction = ndimage.binary_propagation(eroded_square, mask=square)
if show_images:
image_arrays = [square, open_square, eroded_square, reconstruction]
image_arrays = bin2gray(image_arrays)
images = ndarray_2_image(image_arrays)
pil_image_demo.plt_images(images, 2)
def S2_PSF_optimization(self):
self.h, self.v = mtile_cal(self.lat, self.lon)
m = mgrs.MGRS()
mg_coor = m.toMGRS(self.lat, self.lon, MGRSPrecision=4)
self.place = mg_coor[:5]
#self.Hfiles = glob.glob(directory +'l_data/LC8%03d%03d%d*LGN00_sr_band1.tif'%(self.path, self.row, self.year))
self.Hfile = os.getcwd() + '/s_data/%s/%s/%s/%d/%d/%d/0/' % (
mg_coor[:2], mg_coor[2], mg_coor[3:5], self.year, self.month,
self.day)
#Lfile = glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf'%(year,doy,h,v))[0]
self.doy = datetime.datetime(self.year, self.month,
self.day).timetuple().tm_yday
self.Lfiles = [
glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf' %
(self.year, i, self.h, self.v))[0]
for i in range(self.doy - 8, self.doy + 9)
]
if glob.glob(self.Hfile + 'cloud.tif') == []:
cl = classification(fhead=self.Hfile,
bands=(2, 3, 4, 8, 11, 12, 13),
bounds=None)
cl.Get_cm_p()
self.cloud = cl.cm.copy()
tifffile.imsave(self.Hfile + 'cloud.tif', self.cloud.astype(int))
self.H_data = np.repeat(np.repeat(cl.b12, 2, axis=1), 2, axis=0)
del cl
else:
b12 = gdal.Open(self.Hfile + 'B12.jp2').ReadAsArray() * 0.0001
self.H_data = np.repeat(np.repeat(b12, 2, axis=1), 2, axis=0)
self.cloud = tifffile.imread(self.Hfile + 'cloud.tif').astype(bool)
cloud_cover = 1. * self.cloud.sum() / self.cloud.size
cloud_cover = 1. * self.cloud.sum() / self.cloud.size
if cloud_cover > 0.2:
print 'Too much cloud, cloud proportion: %.03f !!' % cloud_cover
else:
mete = readxml('%smetadata.xml' % self.Hfile)
self.sza = np.zeros(7)
self.sza[:] = mete['mSz']
self.saa = self.sza.copy()
self.saa[:] = mete['mSa']
self.vza = (mete['mVz'])[[1, 2, 3, 7, 8, 11, 12], ]
self.vaa = (mete['mVa'])[[1, 2, 3, 7, 8, 11, 12], ]
self.L_inds, self.H_inds = get_coords(self.lat, self.lon)
self.Lx, self.Ly = self.L_inds
self.Hx, self.Hy = self.H_inds
angles = (self.sza[-1], self.vza[-1], (self.vaa - self.saa)[-1])
if glob.glob(self.Lfiles[8][:-3] + 'L8.16days.pkl') == []:
self.BRDF_16_days = np.array([get_brdf_six(Lfile,angles,bands=(7,), \
flag=None, Linds= self.L_inds) for Lfile in self.Lfiles]).squeeze()
valid_range = (self.BRDF_16_days[:, 0, :] >=
0) & (self.BRDF_16_days[:, 0, :] <= 1)
magic = 0.618034
test = self.BRDF_16_days[:, 0, :].copy()
test[~valid_range] = np.nan
W = magic**self.BRDF_16_days[:, 1, :]
W[self.BRDF_16_days[:, 1, :] > 1] = 0
#smothed = smoothn(test, axis=0, isrobust=1, W=W, s=1)[0]
pkl.dump(self.BRDF_16_days,
open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'w'))
#pkl.dump(smothed, open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'w'))
else:
self.BRDF_16_days = pkl.load(
open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'r'))
#smothed = pkl.load(open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'r'))
struct = ndimage.generate_binary_structure(2, 2)
dia_cloud = ndimage.binary_dilation(self.cloud,
structure=struct,
iterations=60).astype(
self.cloud.dtype)
mask = ~(self.H_data <= 0).astype('bool')
small_mask = ndimage.binary_erosion(mask,
structure=struct,
iterations=60).astype(
mask.dtype)
self.val_mask = (~dia_cloud) & small_mask
self.L_data = np.zeros(self.BRDF_16_days[8, 0, :].shape[0])
self.L_data[:] = np.nan
self.L_data[self.BRDF_16_days[8, 1, :] == 0] = self.BRDF_16_days[
8, 0, :][self.BRDF_16_days[8, 1, :] == 0]
#args = s, self.L_data,
avker = np.ones((120, 120))
navker = avker / avker.sum()
self.s = signal.fftconvolve(self.H_data, navker, mode='same')
self.s[~self.val_mask] = np.nan
min_val = [-100, -100]
max_val = [100, 100]
ps, distributions = create_training_set(['xs', 'ys'],
min_val,
max_val,
n_train=50)
solved = parmap(self.op1, ps, nprocs=10)
paras, costs = np.array([i[0] for i in solved
]), np.array([i[1] for i in solved])
xs, ys = paras[costs == costs.min()][0]
if costs.min() < 0.1:
min_val = [5, 5, -15, xs - 5, ys - 5]
max_val = [100, 100, 15, xs + 5, ys + 5]
self.bounds = [5,
100], [5,
100], [-15,
15], [xs - 5,
xs + 5], [ys - 5, ys + 5]
ps, distributions = create_training_set(
['xstd', 'ystd', 'ang', 'xs', 'ys'],
min_val,
max_val,
n_train=50)
#ps = zip(xstd.ravel(), ystd.ravel())
print 'Start solving...'
self.solved = parmap(self.op, ps, nprocs=10)
#costs = np.array([i[1] for i in self.solved])
#paras = np.array([i[0] for i in self.solved])
#min_mask = costs==costs.min()
#min_costs = costs.min()
#para = paras[min_mask]
print self.solved
return self.solved, self.BRDF_16_days
else:
print 'Cost is too large, plese check!', xs, ys, costs.min()
return [[xs, ys, costs.min()], self.BRDF_16_days]
'''
# Granulometry of synthetic image
# read binary simulated image, normalize it
I = imageio.imread("simulation.png") / 255
I = I[:, :, 2] > .5
plt.figure()
plt.imshow(I)
plt.show()
granulometry(I, 35)
# Granulometry of real image
I = imageio.imread("poudre.bmp")
# segmentation
BW = I > 74
BW = ndimage.morphology.binary_fill_holes(BW)
# suppress small objects
se = ndimage.generate_binary_structure(2, 1)
m = ndimage.morphology.binary_opening(BW)
# opening by reconstruction
BW = ndimage.morphology.binary_propagation(m, mask=BW)
imageio.imwrite("segmentation_granulo.python.png", BW.astype('int'))
plt.imshow(BW)
plt.show()
granulometry(BW, 20, filename="poudre")
def _permuted_ols_on_chunk(
scores_original_data,
tested_vars,
target_vars,
thread_id,
threshold=None,
confounding_vars=None,
masker=None,
n_perm=10000,
n_perm_chunk=10000,
intercept_test=True,
two_sided_test=True,
random_state=None,
verbose=0,
):
"""Perform massively univariate analysis with permuted OLS on a data chunk.
To be used in a parallel computing context.
Parameters
----------
scores_original_data : array-like, shape=(n_descriptors, n_regressors)
t-scores obtained for the original (non-permuted) data.
tested_vars : array-like, shape=(n_samples, n_regressors)
Explanatory variates.
target_vars : array-like, shape=(n_samples, n_targets)
fMRI data. F-ordered for efficient computations.
thread_id : int
process id, used for display.
threshold : :obj:`float`
Cluster-forming threshold in t-scale.
This is only used for cluster-level inference.
If ``masker`` is set to None, it will be ignored.
.. versionadded:: 0.9.2.dev
confounding_vars : array-like, shape=(n_samples, n_covars), optional
Clinical data (covariates).
masker : None or :class:`~nilearn.maskers.NiftiMasker` or \
:class:`nilearn.maskers.MultiNiftiMasker`, optional
A mask to be used on the data.
This is only used for cluster-level inference.
If None, cluster-level inference will not be performed.
.. versionadded:: 0.9.2.dev
n_perm : int, optional
Total number of permutations to perform, only used for
display in this function. Default=10000.
n_perm_chunk : int, optional
Number of permutations to be performed. Default=10000.
intercept_test : boolean, optional
Change the permutation scheme (swap signs for intercept,
switch labels otherwise). See :footcite:`Fisher1935`.
Default=True.
two_sided_test : boolean, optional
If True, performs an unsigned t-test. Both positive and negative
effects are considered; the null hypothesis is that the effect is zero.
If False, only positive effects are considered as relevant. The null
hypothesis is that the effect is zero or negative.
Default=True
random_state : int or None, optional
Seed for random number generator, to have the same permutations
in each computing units.
verbose : int, optional
Defines the verbosity level. Default=0.
Returns
-------
scores_as_ranks_part : array-like, shape=(n_regressors, n_descriptors)
The ranks of the original scores in ``h0_fmax_part``.
When ``n_descriptors`` or ``n_perm`` are large, it can be quite long to
find the rank of the original scores into the whole H0 distribution.
Here, it is performed in parallel by the workers involved in the
permutation computation.
h0_fmax_part : array-like, shape=(n_perm_chunk, n_regressors)
Distribution of the (max) t-statistic under the null hypothesis
(limited to this permutation chunk).
h0_csfwe_part, h0_cmfwe_part : array-like, \
shape=(n_perm_chunk, n_regressors)
Distribution of max cluster sizes/masses under the null hypothesis.
Only calculated if ``masker`` is not None.
Otherwise, these will both be None.
.. versionadded:: 0.9.2.dev
References
----------
.. footbibliography::
"""
# initialize the seed of the random generator
rng = check_random_state(random_state)
n_samples, n_regressors = tested_vars.shape
n_descriptors = target_vars.shape[1]
# run the permutations
t0 = time.time()
h0_vfwe_part = np.empty((n_regressors, n_perm_chunk))
vfwe_scores_as_ranks_part = np.zeros((n_regressors, n_descriptors))
if threshold is not None:
h0_csfwe_part = np.empty((n_regressors, n_perm_chunk))
h0_cmfwe_part = np.empty((n_regressors, n_perm_chunk))
bin_struct = ndimage.generate_binary_structure(3, 1)
else:
h0_csfwe_part, h0_cmfwe_part = None, None
for i_perm in range(n_perm_chunk):
if intercept_test:
# sign swap (random multiplication by 1 or -1)
target_vars = (
target_vars * (rng.randint(2, size=(n_samples, 1)) * 2 - 1)
)
else:
# shuffle data
# Regarding computation costs, we choose to shuffle testvars
# and covars rather than fmri_signal.
# Also, it is important to shuffle tested_vars and covars
# jointly to simplify t-scores computation (null dot product).
shuffle_idx = rng.permutation(n_samples)
tested_vars = tested_vars[shuffle_idx]
if confounding_vars is not None:
confounding_vars = confounding_vars[shuffle_idx]
# OLS regression on randomized data
perm_scores = np.asfortranarray(
_t_score_with_covars_and_normalized_design(
tested_vars, target_vars, confounding_vars
)
)
if threshold is not None:
arr4d = masker.inverse_transform(perm_scores.T).get_fdata()
(
h0_csfwe_part[:, i_perm],
h0_cmfwe_part[:, i_perm],
) = _calculate_cluster_measures(
arr4d,
threshold,
bin_struct,
two_sided_test=two_sided_test,
)
# find the rank of the original scores in h0_vfwe_part
# (when n_descriptors or n_perm are large, it can be quite long to
# find the rank of the original scores into the whole H0 distribution.
# Here, it is performed in parallel by the workers involved in the
# permutation computation)
# NOTE: This is not done for the cluster-level methods.
if two_sided_test:
# Get maximum absolute value for voxel-level FWE
h0_vfwe_part[:, i_perm] = np.nanmax(np.fabs(perm_scores), axis=0)
vfwe_scores_as_ranks_part += (
h0_vfwe_part[:, i_perm].reshape((-1, 1))
< np.fabs(scores_original_data).T
)
else:
# Get maximum value for voxel-level FWE
h0_vfwe_part[:, i_perm] = np.nanmax(perm_scores, axis=0)
vfwe_scores_as_ranks_part += (
h0_vfwe_part[:, i_perm].reshape((-1, 1))
< scores_original_data.T
)
if verbose > 0:
step = 11 - min(verbose, 10)
if i_perm % step == 0:
# If there is only one job, progress information is fixed
if n_perm == n_perm_chunk:
crlf = '\r'
else:
crlf = '\n'
percent = float(i_perm) / n_perm_chunk
percent = round(percent * 100, 2)
dt = time.time() - t0
remaining = (100. - percent) / max(0.01, percent) * dt
sys.stderr.write(
f'Job #{thread_id}, processed {i_perm}/{n_perm_chunk} '
f'permutations ({percent:0.2f}%, {remaining} seconds '
f'remaining){crlf}'
)
return (
vfwe_scores_as_ranks_part,
h0_vfwe_part,
h0_csfwe_part,
h0_cmfwe_part,
)
def count_lesions(netseg, target, thresh):
"""
Comparing segmentations volumetrically
Connected component analysis of between prediction `netseg` and ground truth `target` across lesion bin sizes.
:param netseg: network output on range [0,1], shape=(NxMxO)
:type netseg: float16, float32, float64
:param target: ground truth labels, shape=(NxMxO)
:type target: int16
:param thresh: threshold to binarize prediction `h`
:type thresh: float16, float32, float64
:return: dict
**************** Courtesy of Tanya Nair ********************
"""
netseg[netseg >= thresh] = 1
netseg[netseg < thresh] = 0
netseg = netseg[0]
target = target[0]
mask_target = np.zeros((target.shape[1], target.shape[2], target.shape[3]))
for lesion in range(target.shape[0]):
mask_target += target[lesion]
# To Test netseg = gt_mask (should get ROC as tpr = 1 and fdr = 0 everywhere)
target, _ = utils.remove_tiny_les(mask_target, nvox=2)
netseg0 = netseg.copy()
netseg = ndimage.binary_dilation(netseg, structure=ndimage.generate_binary_structure(3, 2))
labels = {}
nles = {}
labels['target'], nles['target'] = ndimage.label(target)
labels['netseg'], nles['netseg'] = ndimage.label(netseg)
found_h = np.ones(nles['netseg'], np.int16)
ntp = {'all': 0, 'small': 0, 'med': 0, 'large': 0}
nfp = {'all': 0, 'small': 0, 'med': 0, 'large': 0}
nfn = {'all': 0, 'small': 0, 'med': 0, 'large': 0}
nb_les = {'all': 0, 'small': 0, 'med': 0, 'large': 0}
nles_gt = {'all': nles['target'], 'small': 0, 'med': 0, 'large': 0}
# Go through ground truth segmentation masks and count true positives/false negatives
for i in range(1, nles['target'] + 1):
gt_lesion_size = np.sum(target[labels['target'] == i])
nles_gt[utils.get_lesion_bin(gt_lesion_size)] += 1
# List of detected lesions in this area
h_lesions = np.unique(labels['netseg'][labels['target'] == i])
# All the voxels in this area contribute to detecting the lesion
nb_overlap = netseg[labels['target'] == i].sum()
if nb_overlap >= 3 or nb_overlap >= 0.5 * gt_lesion_size:
nb_les[utils.get_lesion_bin(gt_lesion_size)] += 1
ntp[utils.get_lesion_bin(gt_lesion_size)] += 1
for h_lesion in h_lesions:
if h_lesion != 0:
found_h[h_lesion - 1] = 0
else:
nfn[utils.get_lesion_bin(gt_lesion_size)] += 1
for i in range(1, nles['netseg'] + 1):
nb_vox = np.sum(netseg0[labels['netseg'] == i])
if found_h[i - 1] == 1:
nfp[utils.get_lesion_bin(nb_vox)] += 1
nb_les['all'] = nb_les['small'] + nb_les['med'] + nb_les['large']
ntp['all'] = ntp['small'] + ntp['med'] + ntp['large']
nfp['all'] = nfp['small'] + nfp['med'] + nfp['large']
nfn['all'] = nfn['small'] + nfn['med'] + nfn['large']
tpr = {}
fdr = {}
for s in ntp.keys():
# tpr (sensitivity)
if nb_les[s] != 0:
tpr[s] = ntp[s] / nb_les[s]
elif nb_les[s] == 0 and ntp[s] == 0:
tpr[s] = 1
else:
tpr[s] = 0
# ppv (1-fdr)
if ntp[s] + nfp[s] != 0:
ppv = ntp[s] / (ntp[s] + nfp[s])
elif ntp[s] == 0:
ppv = 1
else:
ppv = 0
fdr[s] = 1 - ppv
return {'ntp': ntp, 'nfp': nfp, 'nfn': nfn, 'fdr': fdr, 'tpr': tpr, 'nles': nb_les, 'nles_gt': nles_gt}
def permuted_ols(
tested_vars,
target_vars,
confounding_vars=None,
model_intercept=True,
n_perm=10000,
two_sided_test=True,
random_state=None,
n_jobs=1,
verbose=0,
masker=None,
threshold=None,
output_type='legacy',
):
"""Massively univariate group analysis with permuted OLS.
Tested variates are independently fitted to target variates descriptors
(e.g. brain imaging signal) according to a linear model solved with an
Ordinary Least Squares criterion.
Confounding variates may be included in the model.
Permutation testing is used to assess the significance of the relationship
between the tested variates and the target variates
:footcite:`Anderson2001`, :footcite:`Winkler2014`.
A max-type procedure is used to obtain family-wise corrected p-values
based on t-statistics (voxel-level FWE), cluster sizes, and cluster masses.
The specific permutation scheme implemented here is the one of
:footcite:`Freedman1983`. Its has been demonstrated in
:footcite:`Anderson2001` that this scheme conveys more sensitivity than
alternative schemes. This holds for neuroimaging applications, as
discussed in details in :footcite:`Winkler2014`.
Permutations are performed on parallel computing units.
Each of them performs a fraction of permutations on the whole dataset.
Thus, the max t-score amongst data descriptors can be computed directly,
which avoids storing all the computed t-scores.
The variates should be given C-contiguous.
``target_vars`` are fortran-ordered automatically to speed-up computations.
Parameters
----------
tested_vars : array-like, shape=(n_samples, n_regressors)
Explanatory variates, fitted and tested independently from each others.
target_vars : array-like, shape=(n_samples, n_descriptors)
fMRI data to analyze according to the explanatory and confounding
variates.
In a group-level analysis, the samples will typically be voxels
(for volumetric data) or vertices (for surface data), while the
descriptors will generally be images, such as run-wise z-statistic
maps.
confounding_vars : array-like, shape=(n_samples, n_covars), optional
Confounding variates (covariates), fitted but not tested.
If None, no confounding variate is added to the model
(except maybe a constant column according to the value of
``model_intercept``).
model_intercept : :obj:`bool`, optional
If True, a constant column is added to the confounding variates
unless the tested variate is already the intercept.
Default=True.
n_perm : :obj:`int`, optional
Number of permutations to perform.
Permutations are costly but the more are performed, the more precision
one gets in the p-values estimation.
If ``n_perm`` is set to 0, then no p-values will be estimated.
Default=10000.
two_sided_test : :obj:`bool`, optional
If True, performs an unsigned t-test. Both positive and negative
effects are considered; the null hypothesis is that the effect is zero.
If False, only positive effects are considered as relevant. The null
hypothesis is that the effect is zero or negative. Default=True.
random_state : :obj:`int` or None, optional
Seed for random number generator, to have the same permutations
in each computing units.
n_jobs : :obj:`int`, optional
Number of parallel workers.
If -1 is provided, all CPUs are used.
A negative number indicates that all the CPUs except (abs(n_jobs) - 1)
ones will be used. Default=1.
verbose : :obj:`int`, optional
verbosity level (0 means no message). Default=0.
masker : None or :class:`~nilearn.maskers.NiftiMasker` or \
:class:`~nilearn.maskers.MultiNiftiMasker`, optional
A mask to be used on the data.
This is required for cluster-level inference, so it must be provided
if ``threshold`` is not None.
.. versionadded:: 0.9.2.dev
threshold : None or :obj:`float`, optional
Cluster-forming threshold in p-scale.
This is only used for cluster-level inference.
If None, cluster-level inference will not be performed.
Default=None.
.. warning::
Performing cluster-level inference will increase the computation
time of the permutation procedure.
.. versionadded:: 0.9.2.dev
output_type : {'legacy', 'dict'}, optional
Determines how outputs should be returned.
The two options are:
- 'legacy': return a pvals, score_orig_data, and h0_fmax.
This option is the default, but it is deprecated until 0.13,
when the default will be changed to 'dict'.
It will be removed in 0.15.
- 'dict': return a dictionary containing output arrays.
This option will be made the default in 0.13.
.. deprecated:: 0.9.2.dev
The default value for this parameter will change from 'legacy' to
'dict' in 0.13, and the parameter will be removed completely in
0.15.
.. versionadded:: 0.9.2.dev
Returns
-------
pvals : array-like, shape=(n_regressors, n_descriptors)
Negative log10 p-values associated with the significance test of the
n_regressors explanatory variates against the n_descriptors target
variates. Family-wise corrected p-values.
.. note::
This is returned if ``output_type`` == 'legacy'.
.. deprecated:: 0.9.2.dev
The 'legacy' option for ``output_type`` is deprecated.
The default value will change to 'dict' in 0.13,
and the ``output_type`` parameter will be removed in 0.15.
score_orig_data : numpy.ndarray, shape=(n_regressors, n_descriptors)
t-statistic associated with the significance test of the n_regressors
explanatory variates against the n_descriptors target variates.
The ranks of the scores into the h0 distribution correspond to the
p-values.
.. note::
This is returned if ``output_type`` == 'legacy'.
.. deprecated:: 0.9.2.dev
The 'legacy' option for ``output_type`` is deprecated.
The default value will change to 'dict' in 0.13,
and the ``output_type`` parameter will be removed in 0.15.
h0_fmax : array-like, shape=(n_regressors, n_perm)
Distribution of the (max) t-statistic under the null hypothesis
(obtained from the permutations). Array is sorted.
.. note::
This is returned if ``output_type`` == 'legacy'.
.. deprecated:: 0.9.2.dev
The 'legacy' option for ``output_type`` is deprecated.
The default value will change to 'dict' in 0.13,
and the ``output_type`` parameter will be removed in 0.15.
.. versionchanged:: 0.9.2.dev
Return H0 for all regressors, instead of only the first one.
outputs : :obj:`dict`
Output arrays, organized in a dictionary.
.. note::
This is returned if ``output_type`` == 'dict'.
This will be the default output starting in version 0.13.
.. versionadded:: 0.9.2.dev
Here are the keys:
============= ============== ==========================================
key shape description
============= ============== ==========================================
t (n_regressors, t-statistic associated with the
n_descriptors) significance test of the n_regressors
explanatory variates against the
n_descriptors target variates.
The ranks of the scores into the h0
distribution correspond to the p-values.
logp_max_t (n_regressors, Negative log10 p-values associated with
n_descriptors) the significance test of the n_regressors
explanatory variates against the
n_descriptors target variates.
Family-wise corrected p-values, based on
``h0_max_t``.
h0_max_t (n_regressors, Distribution of the max t-statistic under
n_perm) the null hypothesis (obtained from the
permutations). Array is sorted.
logp_max_size (n_regressors, Negative log10 p-values associated with
n_descriptors) the cluster-level significance test of the
n_regressors explanatory variates against
the n_descriptors target variates.
Family-wise corrected, cluster-level
p-values, based on ``h0_max_size``.
Returned only if ``masker`` is not None.
h0_max_size (n_regressors, Distribution of the max cluster size value
n_perm) under the null hypothesis (obtained from
the permutations). Array is sorted.
Returned only if ``masker`` is not None.
logp_max_mass (n_regressors, Negative log10 p-values associated with
n_descriptors) the cluster-level significance test of the
n_regressors explanatory variates against
the n_descriptors target variates.
Family-wise corrected, cluster-level
p-values, based on ``h0_max_mass``.
Returned only if ``masker`` is not None.
h0_max_mass (n_regressors, Distribution of the max cluster mass value
n_perm) under the null hypothesis (obtained from
the permutations). Array is sorted.
Returned only if ``masker`` is not None.
============= ============== ==========================================
References
----------
.. footbibliography::
"""
# initialize the seed of the random generator
rng = check_random_state(random_state)
# check n_jobs (number of CPUs)
if n_jobs == 0: # invalid according to joblib's conventions
raise ValueError(
"'n_jobs == 0' is not a valid choice. "
"Please provide a positive number of CPUs, or -1 for all CPUs, "
"or a negative number (-i) for 'all but (i-1)' CPUs "
"(joblib conventions)."
)
elif n_jobs < 0:
n_jobs = max(1, joblib.cpu_count() - int(n_jobs) + 1)
else:
n_jobs = min(n_jobs, joblib.cpu_count())
if (threshold is not None) and (masker is None):
raise ValueError(
'If "threshold" is not None, masker must be defined as well.'
)
elif (threshold is None) and (masker is not None):
warnings.warn(
'If "threshold" is not set, then masker will be ignored.'
)
if (threshold is not None) and (output_type == 'legacy'):
warnings.warn(
'If "threshold" is not None, "output_type" must be set to "dict". '
'Overriding.'
)
output_type = 'dict'
elif output_type == 'legacy':
warnings.warn(
category=DeprecationWarning,
message=(
'The "legacy" output structure for "permuted_ols" is '
'deprecated. '
'The default output structure will be changed to "dict" '
'in version 0.13.'
),
stacklevel=3,
)
# make target_vars F-ordered to speed-up computation
if target_vars.ndim != 2:
raise ValueError(
"'target_vars' should be a 2D array. "
f"An array with {target_vars.ndim} dimension(s) was passed."
)
target_vars = np.asfortranarray(target_vars) # efficient for chunking
n_descriptors = target_vars.shape[1]
if np.any(np.all(target_vars == 0, axis=0)):
warnings.warn(
"Some descriptors in 'target_vars' have zeros across all samples. "
"These descriptors will be ignored during null distribution "
"generation."
)
# check explanatory variates' dimensions
if tested_vars.ndim == 1:
tested_vars = np.atleast_2d(tested_vars).T
n_samples, n_regressors = tested_vars.shape
# check if explanatory variates is intercept (constant) or not
if (n_regressors == 1 and np.unique(tested_vars).size == 1):
intercept_test = True
else:
intercept_test = False
# optionally add intercept
if model_intercept and not intercept_test:
if confounding_vars is not None:
confounding_vars = np.hstack(
(confounding_vars, np.ones((n_samples, 1))))
else:
confounding_vars = np.ones((n_samples, 1))
# OLS regression on original data
if confounding_vars is not None:
# step 1: extract effect of covars from target vars
covars_orthonormalized = _orthonormalize_matrix(confounding_vars)
if not covars_orthonormalized.flags['C_CONTIGUOUS']:
# useful to developer
warnings.warn('Confounding variates not C_CONTIGUOUS.')
covars_orthonormalized = np.ascontiguousarray(
covars_orthonormalized)
targetvars_normalized = _normalize_matrix_on_axis(
target_vars).T # faster with F-ordered target_vars_chunk
if not targetvars_normalized.flags['C_CONTIGUOUS']:
# useful to developer
warnings.warn('Target variates not C_CONTIGUOUS.')
targetvars_normalized = np.ascontiguousarray(targetvars_normalized)
beta_targetvars_covars = np.dot(
targetvars_normalized, covars_orthonormalized
)
targetvars_resid_covars = targetvars_normalized - np.dot(
beta_targetvars_covars, covars_orthonormalized.T)
targetvars_resid_covars = _normalize_matrix_on_axis(
targetvars_resid_covars, axis=1
)
# step 2: extract effect of covars from tested vars
testedvars_normalized = _normalize_matrix_on_axis(
tested_vars.T, axis=1
)
beta_testedvars_covars = np.dot(
testedvars_normalized, covars_orthonormalized
)
testedvars_resid_covars = testedvars_normalized - np.dot(
beta_testedvars_covars, covars_orthonormalized.T)
testedvars_resid_covars = _normalize_matrix_on_axis(
testedvars_resid_covars, axis=1
).T.copy()
n_covars = confounding_vars.shape[1]
else:
targetvars_resid_covars = _normalize_matrix_on_axis(target_vars).T
testedvars_resid_covars = _normalize_matrix_on_axis(tested_vars).copy()
covars_orthonormalized = None
n_covars = 0
# check arrays contiguousity (for the sake of code efficiency)
if not targetvars_resid_covars.flags['C_CONTIGUOUS']:
# useful to developer
warnings.warn('Target variates not C_CONTIGUOUS.')
targetvars_resid_covars = np.ascontiguousarray(targetvars_resid_covars)
if not testedvars_resid_covars.flags['C_CONTIGUOUS']:
# useful to developer
warnings.warn('Tested variates not C_CONTIGUOUS.')
testedvars_resid_covars = np.ascontiguousarray(testedvars_resid_covars)
# step 3: original regression (= regression on residuals + adjust t-score)
# compute t score map of each tested var for original data
# scores_original_data is in samples-by-regressors shape
scores_original_data = _t_score_with_covars_and_normalized_design(
testedvars_resid_covars,
targetvars_resid_covars.T,
covars_orthonormalized,
)
if threshold is not None:
# determine t-statistic threshold
dof = n_samples - (n_regressors + n_covars)
if two_sided_test:
threshold_t = stats.t.isf(threshold / 2, df=dof)
else:
threshold_t = stats.t.isf(threshold, df=dof)
else:
threshold_t = None
# Permutations
# parallel computing units perform a reduced number of permutations each
if n_perm > n_jobs:
n_perm_chunks = np.asarray([n_perm / n_jobs] * n_jobs, dtype=int)
n_perm_chunks[-1] += n_perm % n_jobs
elif n_perm > 0:
warnings.warn(
f'The specified number of permutations is {n_perm} and the number '
f'of jobs to be performed in parallel has set to {n_jobs}. '
f'This is incompatible so only {n_perm} jobs will be running. '
'You may want to perform more permutations in order to take the '
'most of the available computing resources.'
)
n_perm_chunks = np.ones(n_perm, dtype=int)
# 0 or negative number of permutations => original data scores only
elif output_type == 'legacy':
return np.asarray([]), scores_original_data.T, np.asarray([])
else:
return {'t': scores_original_data.T}
# actual permutations, seeded from a random integer between 0 and maximum
# value represented by np.int32 (to have a large entropy).
ret = joblib.Parallel(n_jobs=n_jobs, verbose=verbose)(
joblib.delayed(_permuted_ols_on_chunk)(
scores_original_data,
testedvars_resid_covars,
targetvars_resid_covars.T,
thread_id=thread_id + 1,
threshold=threshold_t,
confounding_vars=covars_orthonormalized,
masker=masker,
n_perm=n_perm,
n_perm_chunk=n_perm_chunk,
intercept_test=intercept_test,
two_sided_test=two_sided_test,
random_state=rng.randint(1, np.iinfo(np.int32).max - 1),
verbose=verbose,
)
for thread_id, n_perm_chunk in enumerate(n_perm_chunks))
# reduce results
(
vfwe_scores_as_ranks_parts,
vfwe_h0_parts, # maximum t-value null distribution
csfwe_h0_parts, # cluster size null distribution
cmfwe_h0_parts, # cluster mass null distribution
) = zip(*ret)
# Voxel-level FWE
vfwe_h0 = np.hstack((vfwe_h0_parts))
vfwe_scores_as_ranks = np.zeros((n_regressors, n_descriptors))
for vfwe_scores_as_ranks_part in vfwe_scores_as_ranks_parts:
vfwe_scores_as_ranks += vfwe_scores_as_ranks_part
vfwe_pvals = (n_perm + 1 - vfwe_scores_as_ranks) / float(1 + n_perm)
if threshold is not None:
# Cluster-size and cluster-mass FWE
cluster_dict = {} # a dictionary to collect mass/size measures
cluster_dict['size_h0'] = np.hstack((csfwe_h0_parts))
cluster_dict['mass_h0'] = np.hstack((cmfwe_h0_parts))
cluster_dict['size'] = np.zeros_like(vfwe_pvals).astype(int)
cluster_dict['mass'] = np.zeros_like(vfwe_pvals)
cluster_dict['size_pvals'] = np.zeros_like(vfwe_pvals)
cluster_dict['mass_pvals'] = np.zeros_like(vfwe_pvals)
scores_original_data_4d = masker.inverse_transform(
scores_original_data.T
).get_fdata()
bin_struct = ndimage.generate_binary_structure(3, 1)
for i_regressor in range(n_regressors):
scores_original_data_3d = scores_original_data_4d[..., i_regressor]
# Label the clusters for both cluster mass and size inference
labeled_arr3d, _ = ndimage.measurements.label(
scores_original_data_3d > threshold_t,
bin_struct,
)
if two_sided_test:
# Label positive and negative clusters separately
n_positive_clusters = np.max(labeled_arr3d)
temp_labeled_arr3d, _ = ndimage.measurements.label(
scores_original_data_3d < -threshold_t,
bin_struct,
)
temp_labeled_arr3d[
temp_labeled_arr3d > threshold_t
] += n_positive_clusters
labeled_arr3d = labeled_arr3d + temp_labeled_arr3d
del temp_labeled_arr3d
cluster_labels, idx, cluster_dict['size_regressor'] = np.unique(
labeled_arr3d,
return_inverse=True,
return_counts=True,
)
assert cluster_labels[0] == 0 # the background
# Replace background's "cluster size" w zeros
cluster_dict['size_regressor'][0] = 0
# Calculate mass for each cluster
cluster_dict['mass_regressor'] = np.zeros(cluster_labels.shape)
for j_val in cluster_labels[1:]: # skip background
cluster_mass = np.sum(
np.fabs(scores_original_data_3d[labeled_arr3d == j_val])
- threshold_t
)
cluster_dict['mass_regressor'][j_val] = cluster_mass
# Calculate p-values from size/mass values and associated h0s
for metric in ['mass', 'size']:
p_vals = _null_to_p(
cluster_dict[f'{metric}_regressor'],
cluster_dict[f'{metric}_h0'][i_regressor, :],
'larger',
)
p_map = p_vals[np.reshape(idx, labeled_arr3d.shape)]
metric_map = cluster_dict[f'{metric}_regressor'][
np.reshape(idx, labeled_arr3d.shape)
]
# Convert 3D to image, then to 1D
# There is a problem if the masker performs preprocessing,
# so we use apply_mask here.
cluster_dict[f'{metric}_pvals'][i_regressor, :] = np.squeeze(
apply_mask(
nib.Nifti1Image(
p_map,
masker.mask_img_.affine,
masker.mask_img_.header,
),
masker.mask_img_,
)
)
cluster_dict[metric][i_regressor, :] = np.squeeze(
apply_mask(
nib.Nifti1Image(
metric_map,
masker.mask_img_.affine,
masker.mask_img_.header,
),
masker.mask_img_,
)
)
if output_type == 'legacy':
outputs = (-np.log10(vfwe_pvals), scores_original_data.T, vfwe_h0)
else:
outputs = {
't': scores_original_data.T,
'logp_max_t': -np.log10(vfwe_pvals),
'h0_max_t': vfwe_h0,
}
if threshold is not None:
outputs['size'] = cluster_dict['size']
outputs['logp_max_size'] = -np.log10(cluster_dict['size_pvals'])
outputs['h0_max_size'] = cluster_dict['size_h0']
outputs['mass'] = cluster_dict['mass']
outputs['logp_max_mass'] = -np.log10(cluster_dict['mass_pvals'])
outputs['h0_max_mass'] = cluster_dict['mass_h0']
return outputs
# Scipy Measurements Label with boundary correction
import scipy.ndimage as sn
import numpy as n
default_struct = sn.generate_binary_structure(3, 3)
def clean_tuples(tuples):
return sorted(set([(min(pair), max(pair)) for pair in tuples]))
def merge_tuples_unionfind(tuples):
# use classic algorithms union find with path compression
# https://en.wikipedia.org/wiki/Disjoint-set_data_structure
parent_dict = {}
def subfind(x):
# update roots while visiting parents
if parent_dict[x] != x:
parent_dict[x] = subfind(parent_dict[x])
return parent_dict[x]
def find(x):
if x not in parent_dict:
# x forms new set and becomes a root
parent_dict[x] = x
return x
if parent_dict[x] != x:
# follow chain of parents of parents to find root
parent_dict[x] = subfind(parent_dict[x])
3)_struct_zeros: structure = np.zeros((3,3,3)),other default
4)_blocks_two: no_of_blocks=2,other default
5)_blocks_ten: no_of_blocks=10,other default
6)_fakeghost_one: fakeghost=1 ,other default : will turn to 2 because fakeghost need to be >=2
7)_fakeghost_four: fakeghost=4 ,other default
'''
from scipy.sparse import random
import time as t
import numpy as np
from scipy import ndimage
import os, sys
import unittest
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath("voxel.py"))))
import voxel as vc
structure = ndimage.generate_binary_structure(3, 1) #np.ones((3,3,3))
try:
input_dvar = np.load("dense_array.npy", mmap_mode="r")
except:
print("creating dense input array will take time...")
input_dvar = random(400, 160000, density=0.7, dtype="float64")
input_dvar = input_dvar.todense()
input_dvar = np.array(input_dvar)
input_dvar = np.reshape(input_dvar, (400, 400, 400))
np.save("dense_array.npy", input_dvar)
# creating sparse input array
try:
input_svar = np.load("sparse_array.npy", mmap_mode="r")
except:
print("creating sparse input array will take time...")
def _raw_to_displayed(self, raw):
"""Determine displayed image from a saved raw image and a saved seed.
This function ensures that the 0 label gets mapped to the 0 displayed
pixel.
Parameters
----------
raw : array or int
Raw integer input image.
Returns
-------
image : array
Image mapped between 0 and 1 to be displayed.
"""
if self._color_lookup_func is None:
max_val = np.max(raw)
self._color_lookup_func = self._get_color_lookup_func(raw, max_val)
if (not self.show_selected_label
and self._color_mode == LabelColorMode.DIRECT):
u, inv = np.unique(raw, return_inverse=True)
image = np.array([
self._label_color_index[x] if x in self._label_color_index else
self._label_color_index[None] for x in u
])[inv].reshape(raw.shape)
elif (not self.show_selected_label
and self._color_mode == LabelColorMode.AUTO):
image = self._color_lookup_func(raw)
elif (self.show_selected_label
and self._color_mode == LabelColorMode.AUTO):
image = self._color_lookup_func(raw, self._selected_label)
elif (self.show_selected_label
and self._color_mode == LabelColorMode.DIRECT):
selected = self._selected_label
if selected not in self._label_color_index:
selected = None
index = self._label_color_index
image = np.where(
raw == selected,
index[selected],
np.where(
raw != self._background_label,
index[None],
index[self._background_label],
),
)
else:
raise ValueError("Unsupported Color Mode")
if self.contour > 0 and raw.ndim == 2:
image = np.zeros_like(raw)
struct_elem = ndi.generate_binary_structure(raw.ndim, 1)
thickness = self.contour
thick_struct_elem = ndi.iterate_structure(struct_elem,
thickness).astype(bool)
boundaries = ndi.grey_dilation(
raw, footprint=struct_elem) != ndi.grey_erosion(
raw, footprint=thick_struct_elem)
image[boundaries] = raw[boundaries]
image = self._all_vals[image]
elif self.contour > 0 and raw.ndim > 2:
warnings.warn(
trans._(
"Contours are not displayed during 3D rendering",
deferred=True,
))
return image
def threshold_map(img,
min_cluster_size,
threshold=None,
mask=None,
binarize=True,
sided='two'):
"""
Cluster-extent threshold and binarize image.
Parameters
----------
img : img_like or array_like
Image object or 3D array to be clustered
min_cluster_size : int
Minimum cluster size (in voxels)
threshold : float or None, optional
Cluster-defining threshold for img. If None (default), assume img is
already thresholded.
mask : (S,) array_like or None, optional
Boolean array for masking resultant data array. Default is None.
binarize : bool, optional
Default is True.
sided : {'two', 'one', 'bi'}, optional
How to apply thresholding. One-sided thresholds on the positive side.
Two-sided thresholds positive and negative values together. Bi-sided
thresholds positive and negative values separately. Default is 'two'.
"""
if not isinstance(img, np.ndarray):
arr = img.get_data()
else:
arr = img.copy()
if mask is not None:
mask = mask.astype(bool)
arr *= mask.reshape(arr.shape)
clust_thresholded = np.zeros(arr.shape, int)
if sided == 'two':
test_arr = np.abs(arr)
else:
test_arr = arr.copy()
# Positive values (or absolute values) first
if threshold is not None:
thresh_arr = test_arr >= threshold
else:
thresh_arr = test_arr > 0
# 6 connectivity
struc = ndimage.generate_binary_structure(3, 1)
labeled, _ = ndimage.label(thresh_arr, struc)
unique, counts = np.unique(labeled, return_counts=True)
clust_sizes = dict(zip(unique, counts))
clust_sizes = {
k: v
for k, v in clust_sizes.items() if v >= min_cluster_size
}
for i_clust in clust_sizes.keys():
if np.all(thresh_arr[labeled == i_clust] == 1):
if binarize:
clust_thresholded[labeled == i_clust] = 1
else:
clust_thresholded[labeled == i_clust] = arr[labeled == i_clust]
# Now negative values *if bi-sided*
if sided == 'bi':
if threshold is not None:
thresh_arr = test_arr <= (-1 * threshold)
else:
thresh_arr = test_arr < 0
labeled, _ = ndimage.label(thresh_arr, struc)
unique, counts = np.unique(labeled, return_counts=True)
clust_sizes = dict(zip(unique, counts))
clust_sizes = {
k: v
for k, v in clust_sizes.items() if v >= min_cluster_size
}
for i_clust in clust_sizes.keys():
if np.all(thresh_arr[labeled == i_clust] == 1):
if binarize:
clust_thresholded[labeled == i_clust] = 1
else:
clust_thresholded[labeled == i_clust] = arr[labeled ==
i_clust]
# reshape to (S,)
clust_thresholded = clust_thresholded.ravel()
# if mask provided, mask output
if mask is not None:
clust_thresholded = clust_thresholded[mask]
return clust_thresholded
def run(self, ips, imgs, para=None):
inten = ImageManager.get(para['inten'])
if not para['slice']:
imgs = [inten.img]
msks = [ips.img]
else:
msks = ips.imgs
imgs = inten.imgs
if len(msks) == 1:
msks *= len(imgs)
buf = imgs[0].astype(np.uint16)
strc = ndimage.generate_binary_structure(
2, 1 if para['con'] == '4-connect' else 2)
idct = ['Max', 'Min', 'Mean', 'Variance', 'Standard', 'Sum']
key = {
'Max': 'max',
'Min': 'min',
'Mean': 'mean',
'Variance': 'var',
'Standard': 'std',
'Sum': 'sum'
}
idct = [i for i in idct if para[key[i]]]
titles = ['Slice', 'ID'][0 if para['slice'] else 1:]
if para['center']: titles.extend(['Center-X', 'Center-Y'])
if para['extent']: titles.extend(['Min-Y', 'Min-X', 'Max-Y', 'Max-X'])
titles.extend(idct)
k = ips.unit[0]
data, mark = [], {'type': 'layers', 'body': {}}
# data,mark=[],[]
for i in range(len(imgs)):
n = ndimage.label(msks[i], strc, output=buf)
index = range(1, n + 1)
dt = []
if para['slice']: dt.append([i] * n)
dt.append(range(n))
xy = ndimage.center_of_mass(imgs[i], buf, index)
xy = np.array(xy).round(2).T
if para['center']: dt.extend([xy[1] * k, xy[0] * k])
boxs = [None] * n
if para['extent']:
boxs = ndimage.find_objects(buf)
boxs = [(i[1].start + (i[1].stop - i[1].start) / 2,
i[0].start + (i[0].stop - i[0].start) / 2,
i[1].stop - i[1].start, i[0].stop - i[0].start)
for i in boxs]
for j in (0, 1, 2, 3):
dt.append([i[j] * k for i in boxs])
if para['max']:
dt.append(ndimage.maximum(imgs[i], buf, index).round(2))
if para['min']:
dt.append(ndimage.minimum(imgs[i], buf, index).round(2))
if para['mean']:
dt.append(ndimage.mean(imgs[i], buf, index).round(2))
if para['var']:
dt.append(ndimage.variance(imgs[i], buf, index).round(2))
if para['std']:
dt.append(
ndimage.standard_deviation(imgs[i], buf, index).round(2))
if para['sum']:
dt.append(ndimage.sum(imgs[i], buf, index).round(2))
layer = {'type': 'layer', 'body': []}
xy = np.int0(xy).T
texts = [(i[1], i[0]) + ('id=%d' % n, )
for i, n in zip(xy, range(len(xy)))]
layer['body'].append({'type': 'texts', 'body': texts})
if para['extent']:
layer['body'].append({'type': 'rectangles', 'body': boxs})
mark['body'][i] = layer
data.extend(list(zip(*dt)))
IPy.show_table(pd.DataFrame(data, columns=titles),
inten.title + '-region statistic')
inten.mark = GeometryMark(mark)
inten.update = True
def clusterization(images: torch.Tensor, size=256, padding=70):
imgs = images.cpu().numpy().squeeze()
pattern = generate_binary_structure(2, 2)
coord_result, prob_result = [], []
# print("img sum:", images.sum(dim=[1,2,3]).max())
# t1 = time.time()
# for sample in range(imgs.shape[0]):
def compute(sample):
x, y = np.where((imgs[sample] > 1e-6))
measure_mask = np.zeros((2, size, size))
measure_mask[0, x, y] = 1
measure_mask[1, x, y] = imgs[sample, x, y]
labeled_array, num_features = label(measure_mask[0], structure=pattern)
# if num_features > 75:
# print(num_features)
x_coords, y_coords, prob_value = [], [], []
sample_centroids_coords, sample_probs_value = [], []
for i in range(1, num_features + 1):
x_clust, y_clust = np.where(labeled_array == i)
x_coords.append(np.average(x_clust) / size)
y_coords.append(np.average(y_clust) / size)
prob_value.append(np.sum(measure_mask[1, x_clust, y_clust]))
assert (measure_mask[1, x_clust, y_clust].all() != 0)
# print("PROB_VALUE ", prob_value)
[x_coords.append(0) for i in range(padding - len(x_coords))]
[y_coords.append(0) for i in range(padding - len(y_coords))]
[prob_value.append(0) for i in range(padding - len(prob_value))]
sample_centroids_coords.append([x_coords, y_coords])
sample_probs_value.append(prob_value)
sample_centroids_coords = np.transpose(
np.array(sample_centroids_coords), axes=(0, 2, 1))
sample_probs_value = np.array(sample_probs_value)
# coord_result.append(sample_centroids_coords)
# assert(sample_probs_value.sum() != 0)
# assert(sample_probs_value.all() / sample_probs_value.sum() >= 0)
# prob_result.append(sample_probs_value / sample_probs_value.sum())
return x_coords, y_coords, sample_probs_value / (
sample_probs_value.sum() + 1e-8)
#return sample_centroids_coords, sample_probs_value / (sample_probs_value.sum() + 1e-8)
processed_list = Parallel(n_jobs=16)(delayed(compute)(i)
for i in range(imgs.shape[0]))
for x, y, p in processed_list:
coord_result.append(
torch.cat((torch.tensor(y)[:, None], torch.tensor(x)[:, None]),
dim=1)[None, ...])
prob_result.append(p)
# print(time.time() - t1)
return ProbabilityMeasure(
torch.tensor(np.concatenate(prob_result, axis=0)).type(torch.float32),
torch.cat(coord_result).type(torch.float32)).cuda()
def getLength(
self, segments, boundaries, contacts, ids=None, distance='b2b',
structElConn=1, line='straight', position=False):
"""
Calculates lengts of segments specified by (args) segments and ids. The
segments can contact exactly one or two boundaries.
In the one boundary case, the length is calculated as the maximal
distance between segment points and the contact region, that is using
the 'straight' line mode. (The distance between a point and a contact
region is the distance between the point and its closest contact point.)
In the two boundary case, there are two possibilities. If the line mode
is 'straight', the length is calculated as a smallest straight
(Euclidean) distance between points on the two contact regions.
Otherwise, in the 'mid' or 'mid-seg' line modes, the length is
calculated as a smallest sum of distances between a 'central' and two
contact points. A central point has to belong to the intersection of
the segment and a central layer formed exactly in the middle between
the two boundaries. In other words, the sum of distances is minimized
over all contact and mid points.
The difference between the 'mid' and the 'mid-seg' modes is that in the
'mid-seg' mode the layers between the boundaries are formed on the
segment alone, while in the 'mid' mode they are formed on both the
segment and the neighboring inter-boundary region. Consequently, the
layers formed using the 'mid-seg' mode and the distance calculated,
might be a bit more precise.
If argument distance is 'b2b' (two boundaries) or 'b-max'
(one boundary), contact points are elements of boundaries that contact
a segment, so the length is calculated between the boundaries. If it is
'c2c' (two boundaries) or 'c-max' (one boundary), contact points are
elements of segments that contact a boundary. Consequently, the lengths
calculated in the 'b2b' or 'b-max' modes are expected to be up to two
pixels longer than those calculated in the 'c2c' or 'c-max' modes.
In the case of two boundaries, the length is calculated between
the contact points on the boundary (first end) and the segment
(second end) dor arg distance 'b2c' and the other way round for 'c2b'.
Arguments line and distance are saved as attributes lengthLine and
contactMode, respectivly.
If arg position is True, the positions of contact points (contact and
end point for one boundary) are also calculated, generally increasing
the run-time.
Segments and boundaries objects have to have the same positioning
(attributes offset and inset).
Arguments:
- segments: (Segment) object containing segments whose langths are
calculated
- bondaries: (Segment) object defining boundaries
- contacts: (Contact) object containing the info about the contacts
between the segments and the boundaries
- ids: segment ids
- distance: for two boundaries: 'b2b' (or 'boundary') for
distance between contact points on boundaries, 'c2c' (or 'contact')
between contact points on segments, 'b2c' between boundary and
segment contacts and 'c2b' between segment and boundary contacts.
For one boundary: 'b-max' and 'c-max'.
- structElConn: (square) connectivity of the structuring element used
to detect contacts (can be 1 to ndim).
- line: The type of the line used to calculate the distance in the
2-boundary cases: 'straight', 'mid' or 'mid-seg'
- position: flag indicating if the positions of the contact points
used for the length measurment are calculated and returned, used only
Return:
- length: if pos is False
- length, position_contact, position_other: in the one boundary case
- length, position_1, position_2: in the two boundary case
ToDo: make something like an average of b2b and c2c
"""
# set segments and ids
self.setSegments(segments=segments, ids=ids)
ids = self.ids
# set structuring element
struct_el = ndimage.generate_binary_structure(
rank=self.ndim, connectivity=structElConn)
# figure out expected number of boundaries
if (distance == 'b-max') or (distance == 'c-max'):
n_bound = 1
elif ((distance == 'b2b') or (distance == 'boundary')
or (distance == 'c2c') or (distance == 'contact')
or (distance == 'b2c') or (distance == 'c2b')):
n_bound = 2
else:
n_bound = -1
raise ValueError(
"Argument distance: " + str(distance) + " was not understood."
+ "Defined values are 'b2b', 'boundary', 'c2c', 'contact', "
+ "'b2c' and 'c2c', 'b-max' and 'c-max'.")
# save original insets
seg_data, seg_inset = segments.getDataInset()
seg_inset = copy(seg_inset)
bound_data, bound_inset = boundaries.getDataInset()
bound_inset = copy(bound_inset)
# find length for each segment
length = numpy.zeros(self.maxId+1) - 1
if position:
position_1 = numpy.zeros((self.maxId+1, self.ndim), dtype='int') - 1
position_2 = numpy.zeros((self.maxId+1, self.ndim), dtype='int') - 1
for seg_id in ids:
# find boundaries
b_ids = contacts.findBoundaries(segmentIds=seg_id, nSegment=1)
# use appropriate method to get the length of this segment
if len(b_ids) == n_bound:
# use smaller arrays for calculations
segments.makeInset(ids=[seg_id], extend=1, expand=True)
boundaries.useInset(
inset=segments.inset, mode='abs', expand=True)
# calculate
if n_bound == 1:
res = self._getSingleLength1Bound(
segments=segments, boundaries=boundaries,
boundaryIds=b_ids, id_=seg_id, distance=distance,
structEl=struct_el, line=line, position=position)
elif n_bound == 2:
res = self._getSingleLength2Bound(
segments=segments, boundaries=boundaries,
boundaryIds=b_ids, id_=seg_id, distance=distance,
structEl=struct_el, line=line, position=position)
# parse result
if position:
length[seg_id] = res[0]
position_1[seg_id] = [pos + ins.start for pos, ins \
in zip(res[1], segments.inset)]
position_2[seg_id] = [pos + ins.start for pos, ins \
in zip(res[2], segments.inset)]
else:
length[seg_id] = res
# recover full data
segments.setDataInset(data=seg_data, inset=seg_inset)
boundaries.setDataInset(data=bound_data, inset=bound_inset)
# assign attributes
self.length = length
self.lengthLine = line
self.contactMode = distance
if position:
self.end1 = position_1
self.end2 = position_2
# return
if position:
return length, position_1, position_2
else:
return length
def get_largest_component(img):
s = ndimage.generate_binary_structure(3, 1) # iterate structure
labeled_array, numpatches = ndimage.label(img, s) # labeling
sizes = ndimage.sum(img, labeled_array, range(1, numpatches + 1))
max_label = np.where(sizes == sizes.max())[0] + 1
return labeled_array == max_label
def rag_boundary(labels, edge_map, connectivity=2):
""" Comouter RAG based on region boundaries
Given an image's initial segmentation and its edge map this method
constructs the corresponding Region Adjacency Graph (RAG). Each node in the
RAG represents a set of pixels within the image with the same label in
`labels`. The weight between two adjacent regions is the average value
in `edge_map` along their boundary.
labels : ndarray
The labelled image.
edge_map : ndarray
This should have the same shape as that of `labels`. For all pixels
along the boundary between 2 adjacent regions, the average value of the
corresponding pixels in `edge_map` is the edge weight between them.
connectivity : int, optional
Pixels with a squared distance less than `connectivity` from each other
are considered adjacent. It can range from 1 to `labels.ndim`. Its
behavior is the same as `connectivity` parameter in
`scipy.ndimage.filters.generate_binary_structure`.
Examples
--------
>>> from skimage import data, segmentation, filters, color
>>> from skimage.future import graph
>>> img = data.chelsea()
>>> labels = segmentation.slic(img)
>>> edge_map = filters.sobel(color.rgb2gray(img))
>>> rag = graph.rag_boundary(labels, edge_map)
"""
conn = ndi.generate_binary_structure(labels.ndim, connectivity)
eroded = ndi.grey_erosion(labels, footprint=conn)
dilated = ndi.grey_dilation(labels, footprint=conn)
boundaries0 = (eroded != labels)
boundaries1 = (dilated != labels)
labels_small = np.concatenate((eroded[boundaries0], labels[boundaries1]))
labels_large = np.concatenate((labels[boundaries0], dilated[boundaries1]))
n = np.max(labels_large) + 1
# use a dummy broadcast array as data for RAG
ones = as_strided(np.ones((1,), dtype=np.float), shape=labels_small.shape,
strides=(0,))
count_matrix = sparse.coo_matrix((ones, (labels_small, labels_large)),
dtype=np.int_, shape=(n, n)).tocsr()
data = np.concatenate((edge_map[boundaries0], edge_map[boundaries1]))
data_coo = sparse.coo_matrix((data, (labels_small, labels_large)))
graph_matrix = data_coo.tocsr()
graph_matrix.data /= count_matrix.data
rag = RAG()
rag.add_weighted_edges_from(_edge_generator_from_csr(graph_matrix),
weight='weight')
rag.add_weighted_edges_from(_edge_generator_from_csr(count_matrix),
weight='count')
for n in rag.nodes():
rag.node[n].update({'labels': [n]})
return rag
codes = np.take(lut, neighbours)
code_mask = (codes == 2)
if np.any(code_mask):
pixel_removed = True
skeleton[code_mask] = 0
code_mask = (codes == 3)
if np.any(code_mask):
pixel_removed = True
skeleton[code_mask] = 0
return skeleton.astype(bool)
# --------- Skeletonization by medial axis transform --------
_eight_connect = ndi.generate_binary_structure(2, 2)
def medial_axis(image, mask=None, return_distance=False):
"""
Compute the medial axis transform of a binary image
Parameters
----------
image : binary ndarray, shape (M, N)
The image of the shape to be skeletonized.
mask : binary ndarray, shape (M, N), optional
If a mask is given, only those elements in `image` with a true
value in `mask` are used for computing the medial axis.
return_distance : bool, optional
If true, the distance transform is returned as well as the skeleton.
def run_paintGrid(pd,
omeEta,
seed_hkl_ids,
threshold,
fiber_ndiv,
omeTol=None,
etaTol=None,
omeRange=None,
etaRange=None,
omePeriod=(-np.pi, np.pi),
qTol=1e-7,
doMultiProc=True,
nCPUs=multiprocessing.cpu_count(),
useGrid=None):
"""
wrapper for indexer.paintGrid
"""
del_ome = omeEta.omegas[1] - omeEta.omegas[0]
del_eta = omeEta.etas[1] - omeEta.etas[0]
# tolerances in degrees... I know, pathological
if omeTol is None:
omeTol = 360. / float(fiber_ndiv)
if etaTol is None:
etaTol = 360. / float(fiber_ndiv)
# must be consistent
pd_hkl_ids = omeEta.iHKLList[seed_hkl_ids]
tTh = pd.getTTh()
bMat = pd.latVecOps['B']
csym = pd.getLaueGroup()
qsym = pd.getQSym()
if useGrid is not None:
try:
print "loading quaternion grid file: %s" % (useGrid)
qfib = np.loadtxt(useGrid).T
except:
raise RuntimeError, "unable to load quaternion grid file"
else:
structureNDI_label = ndimage.generate_binary_structure(2, 2)
qfib = []
ii = 0
jj = fiber_ndiv
print "labeling maps..."
labels = []
numSpots = []
coms = []
for i in seed_hkl_ids:
labels_t, numSpots_t = ndimage.label(
omeEta.dataStore[i] > threshold, structureNDI_label)
coms_t = np.atleast_2d(
ndimage.center_of_mass(omeEta.dataStore[i],
labels=labels_t,
index=np.arange(1,
np.amax(labels_t) + 1)))
labels.append(labels_t)
numSpots.append(numSpots_t)
coms.append(coms_t)
pass
# second pass for generation
print "generating quaternions..."
qfib_tmp = np.empty((4, fiber_ndiv * sum(numSpots)))
for i in range(len(pd_hkl_ids)):
for ispot in range(numSpots[i]):
if not np.isnan(coms[i][ispot][0]):
ome_c = omeEta.omeEdges[0] + (0.5 +
coms[i][ispot][0]) * del_ome
eta_c = omeEta.etaEdges[0] + (0.5 +
coms[i][ispot][1]) * del_eta
gVec_s = xrdutil.makeMeasuredScatteringVectors(
tTh[pd_hkl_ids[i]], eta_c, ome_c)
qfib_tmp[:, ii:jj] = rot.discreteFiber(
pd.hkls[:, pd_hkl_ids[i]].reshape(3, 1),
gVec_s,
B=bMat,
ndiv=fiber_ndiv,
invert=False,
csym=csym)[0]
ii = jj
jj += fiber_ndiv
pass
pass
qfib.append(mutil.uniqueVectors(qfib_tmp))
pass
qfib = np.hstack(qfib)
print "Running paintGrid on %d orientations" % (qfib.shape[1])
complPG = idx.paintGrid(qfib,
omeEta,
omegaRange=omeRange,
etaRange=etaRange,
omeTol=d2r * omeTol,
etaTol=d2r * etaTol,
omePeriod=omePeriod,
threshold=threshold,
doMultiProc=doMultiProc,
nCPUs=nCPUs)
return complPG, qfib
def run(config_file):
# construct graph
config = parse_config(config_file)
config_data = config['data']
config_net1 = config.get('network1', None)
config_net2 = config.get('network2', None)
config_net3 = config.get('network3', None)
config_test = config['testing']
batch_size = config_test.get('batch_size', 5)
if(config_net1):
net_type1 = config_net1['net_type']
net_name1 = config_net1['net_name']
data_shape1 = config_net1['data_shape']
label_shape1 = config_net1['label_shape']
data_channel1= config_net1['data_channel']
class_num1 = config_net1['class_num']
# construct graph for 1st network
full_data_shape1 = [batch_size] + data_shape1 + [data_channel1]
x1 = tf.placeholder(tf.float32, shape = full_data_shape1)
net_class1 = NetFactory.create(net_type1)
net1 = net_class1(num_classes = class_num1,w_regularizer = None,
b_regularizer = None, name = net_name1)
net1.set_params(config_net1)
predicty1 = net1(x1, is_training = True)
proby1 = tf.nn.softmax(predicty1)
else:
config_net1ax = config['network1ax']
config_net1sg = config['network1sg']
config_net1cr = config['network1cr']
# construct graph for 1st network axial
net_type1ax = config_net1ax['net_type']
net_name1ax = config_net1ax['net_name']
data_shape1ax = config_net1ax['data_shape']
label_shape1ax = config_net1ax['label_shape']
data_channel1ax= config_net1ax['data_channel']
class_num1ax = config_net1ax['class_num']
full_data_shape1ax = [batch_size] + data_shape1ax + [data_channel1ax]
x1ax = tf.placeholder(tf.float32, shape = full_data_shape1ax)
net_class1ax = NetFactory.create(net_type1ax)
net1ax = net_class1ax(num_classes = class_num1ax,w_regularizer = None,
b_regularizer = None, name = net_name1ax)
net1ax.set_params(config_net1ax)
predicty1ax = net1ax(x1ax, is_training = True)
proby1ax = tf.nn.softmax(predicty1ax)
# construct graph for 1st network sagittal
net_type1sg = config_net1sg['net_type']
net_name1sg = config_net1sg['net_name']
data_shape1sg = config_net1sg['data_shape']
label_shape1sg = config_net1sg['label_shape']
data_channel1sg= config_net1sg['data_channel']
class_num1sg = config_net1sg['class_num']
# construct graph for 1st network
full_data_shape1sg = [batch_size] + data_shape1sg + [data_channel1sg]
x1sg = tf.placeholder(tf.float32, shape = full_data_shape1sg)
net_class1sg = NetFactory.create(net_type1sg)
net1sg = net_class1sg(num_classes = class_num1sg,w_regularizer = None,
b_regularizer = None, name = net_name1sg)
net1sg.set_params(config_net1sg)
predicty1sg = net1sg(x1sg, is_training = True)
proby1sg = tf.nn.softmax(predicty1sg)
# construct graph for 1st network corogal
net_type1cr = config_net1cr['net_type']
net_name1cr = config_net1cr['net_name']
data_shape1cr = config_net1cr['data_shape']
label_shape1cr = config_net1cr['label_shape']
data_channel1cr= config_net1cr['data_channel']
class_num1cr = config_net1cr['class_num']
# construct graph for 1st network
full_data_shape1cr = [batch_size] + data_shape1cr + [data_channel1cr]
x1cr = tf.placeholder(tf.float32, shape = full_data_shape1cr)
net_class1cr = NetFactory.create(net_type1cr)
net1cr = net_class1cr(num_classes = class_num1cr,w_regularizer = None,
b_regularizer = None, name = net_name1cr)
net1cr.set_params(config_net1cr)
predicty1cr = net1cr(x1cr, is_training = True)
proby1cr = tf.nn.softmax(predicty1cr)
# networks for tumor core
if(config_net2):
net_type2 = config_net2['net_type']
net_name2 = config_net2['net_name']
data_shape2 = config_net2['data_shape']
label_shape2 = config_net2['label_shape']
data_channel2= config_net2['data_channel']
class_num2 = config_net2['class_num']
# construct graph for 2st network
full_data_shape2 = [batch_size] + data_shape2 + [data_channel2]
x2 = tf.placeholder(tf.float32, shape = full_data_shape2)
net_class2 = NetFactory.create(net_type2)
net2 = net_class2(num_classes = class_num2,w_regularizer = None,
b_regularizer = None, name = net_name2)
net2.set_params(config_net2)
predicty2 = net2(x2, is_training = True)
proby2 = tf.nn.softmax(predicty2)
else:
config_net2ax = config['network2ax']
config_net2sg = config['network2sg']
config_net2cr = config['network2cr']
# construct graph for 2st network axial
net_type2ax = config_net2ax['net_type']
net_name2ax = config_net2ax['net_name']
data_shape2ax = config_net2ax['data_shape']
label_shape2ax = config_net2ax['label_shape']
data_channel2ax= config_net2ax['data_channel']
class_num2ax = config_net2ax['class_num']
full_data_shape2ax = [batch_size] + data_shape2ax + [data_channel2ax]
x2ax = tf.placeholder(tf.float32, shape = full_data_shape2ax)
net_class2ax = NetFactory.create(net_type2ax)
net2ax = net_class2ax(num_classes = class_num2ax,w_regularizer = None,
b_regularizer = None, name = net_name2ax)
net2ax.set_params(config_net2ax)
predicty2ax = net2ax(x2ax, is_training = True)
proby2ax = tf.nn.softmax(predicty2ax)
# construct graph for 2st network sagittal
net_type2sg = config_net2sg['net_type']
net_name2sg = config_net2sg['net_name']
data_shape2sg = config_net2sg['data_shape']
label_shape2sg = config_net2sg['label_shape']
data_channel2sg= config_net2sg['data_channel']
class_num2sg = config_net2sg['class_num']
# construct graph for 2st network
full_data_shape2sg = [batch_size] + data_shape2sg + [data_channel2sg]
x2sg = tf.placeholder(tf.float32, shape = full_data_shape2sg)
net_class2sg = NetFactory.create(net_type2sg)
net2sg = net_class2sg(num_classes = class_num2sg,w_regularizer = None,
b_regularizer = None, name = net_name2sg)
net2sg.set_params(config_net2sg)
predicty2sg = net2sg(x2sg, is_training = True)
proby2sg = tf.nn.softmax(predicty2sg)
# construct graph for 2st network corogal
net_type2cr = config_net2cr['net_type']
net_name2cr = config_net2cr['net_name']
data_shape2cr = config_net2cr['data_shape']
label_shape2cr = config_net2cr['label_shape']
data_channel2cr= config_net2cr['data_channel']
class_num2cr = config_net2cr['class_num']
# construct graph for 2st network
full_data_shape2cr = [batch_size] + data_shape2cr + [data_channel2cr]
x2cr = tf.placeholder(tf.float32, shape = full_data_shape2cr)
net_class2cr = NetFactory.create(net_type2cr)
net2cr = net_class2cr(num_classes = class_num2cr,w_regularizer = None,
b_regularizer = None, name = net_name2cr)
net2cr.set_params(config_net2cr)
predicty2cr = net2cr(x2cr, is_training = True)
proby2cr = tf.nn.softmax(predicty2cr)
# for enhanced tumor
if(config_net3):
net_type3 = config_net3['net_type']
net_name3 = config_net3['net_name']
data_shape3 = config_net3['data_shape']
label_shape3 = config_net3['label_shape']
data_channel3= config_net3['data_channel']
class_num3 = config_net3['class_num']
# construct graph for 3st network
full_data_shape3 = [batch_size] + data_shape3 + [data_channel3]
x3 = tf.placeholder(tf.float32, shape = full_data_shape3)
net_class3 = NetFactory.create(net_type3)
net3 = net_class3(num_classes = class_num3,w_regularizer = None,
b_regularizer = None, name = net_name3)
net3.set_params(config_net3)
predicty3 = net3(x3, is_training = True)
proby3 = tf.nn.softmax(predicty3)
else:
config_net3ax = config['network3ax']
config_net3sg = config['network3sg']
config_net3cr = config['network3cr']
# construct graph for 3st network axial
net_type3ax = config_net3ax['net_type']
net_name3ax = config_net3ax['net_name']
data_shape3ax = config_net3ax['data_shape']
label_shape3ax = config_net3ax['label_shape']
data_channel3ax= config_net3ax['data_channel']
class_num3ax = config_net3ax['class_num']
full_data_shape3ax = [batch_size] + data_shape3ax + [data_channel3ax]
x3ax = tf.placeholder(tf.float32, shape = full_data_shape3ax)
net_class3ax = NetFactory.create(net_type3ax)
net3ax = net_class3ax(num_classes = class_num3ax,w_regularizer = None,
b_regularizer = None, name = net_name3ax)
net3ax.set_params(config_net3ax)
predicty3ax = net3ax(x3ax, is_training = True)
proby3ax = tf.nn.softmax(predicty3ax)
# construct graph for 3st network sagittal
net_type3sg = config_net3sg['net_type']
net_name3sg = config_net3sg['net_name']
data_shape3sg = config_net3sg['data_shape']
label_shape3sg = config_net3sg['label_shape']
data_channel3sg= config_net3sg['data_channel']
class_num3sg = config_net3sg['class_num']
# construct graph for 3st network
full_data_shape3sg = [batch_size] + data_shape3sg + [data_channel3sg]
x3sg = tf.placeholder(tf.float32, shape = full_data_shape3sg)
net_class3sg = NetFactory.create(net_type3sg)
net3sg = net_class3sg(num_classes = class_num3sg,w_regularizer = None,
b_regularizer = None, name = net_name3sg)
net3sg.set_params(config_net3sg)
predicty3sg = net3sg(x3sg, is_training = True)
proby3sg = tf.nn.softmax(predicty3sg)
# construct graph for 3st network corogal
net_type3cr = config_net3cr['net_type']
net_name3cr = config_net3cr['net_name']
data_shape3cr = config_net3cr['data_shape']
label_shape3cr = config_net3cr['label_shape']
data_channel3cr= config_net3cr['data_channel']
class_num3cr = config_net3cr['class_num']
# construct graph for 3st network
full_data_shape3cr = [batch_size] + data_shape3cr + [data_channel3cr]
x3cr = tf.placeholder(tf.float32, shape = full_data_shape3cr)
net_class3cr = NetFactory.create(net_type3cr)
net3cr = net_class3cr(num_classes = class_num3cr,w_regularizer = None,
b_regularizer = None, name = net_name3cr)
net3cr.set_params(config_net3cr)
predicty3cr = net3cr(x3cr, is_training = True)
proby3cr = tf.nn.softmax(predicty3cr)
all_vars = tf.global_variables()
print('all vars', len(all_vars))
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
if(config_net1):
net1_vars = [x for x in all_vars if x.name[0:len(net_name1) + 1]==net_name1 + '/']
saver1 = tf.train.Saver(net1_vars)
saver1.restore(sess, config_net1['model_file'])
else:
# net1ax_vars = [x for x in all_vars if x.name[0:len(net_name1ax)+1]==net_name1ax + '/']
# saver1ax = tf.train.Saver(net1ax_vars)
#
# saver1ax.restore(sess, config_net1sg['model_file'])
# net1sg_vars = [x for x in all_vars if x.name[0:len(net_name1sg)+1]==net_name1sg + '/']
# for i in range(len(net1sg_vars)):
# copy_value = tf.assign(net1sg_vars[i], net1ax_vars[i])
# copy_value.eval()
# print('net1sg loaded')
# saver1sg = tf.train.Saver(net1sg_vars)
# saver1sg.save(sess, "model/msnet_wt32sg_20000cp.ckpt")
# print('netsg saved')
#
# saver1ax.restore(sess, config_net1cr['model_file'])
# net1cr_vars = [x for x in all_vars if x.name[0:len(net_name1cr)+1]==net_name1cr + '/']
# for i in range(len(net1cr_vars)):
# copy_value = tf.assign(net1cr_vars[i], net1ax_vars[i])
# copy_value.eval()
# saver1cr = tf.train.Saver(net1cr_vars)
# saver1cr.save(sess, "model/msnet_wt32cr_20000cp.ckpt")
# print('net1cr saved')
#
# saver1ax.restore(sess, config_net1ax['model_file'])
net1ax_vars = [x for x in all_vars if x.name[0:len(net_name1ax) + 1]==net_name1ax + '/']
saver1ax = tf.train.Saver(net1ax_vars)
print('net1ax', len(net1ax_vars))
saver1ax.restore(sess, config_net1ax['model_file'])
net1sg_vars = [x for x in all_vars if x.name[0:len(net_name1sg) + 1]==net_name1sg + '/']
print('net1sg', len(net1sg_vars))
saver1sg = tf.train.Saver(net1sg_vars)
saver1sg.restore(sess, config_net1sg['model_file'])
net1cr_vars = [x for x in all_vars if x.name[0:len(net_name1cr) + 1]==net_name1cr + '/']
saver1cr = tf.train.Saver(net1cr_vars)
print('net1cr', len(net1cr_vars))
saver1cr.restore(sess, config_net1cr['model_file'])
if(config_net2):
net2_vars = [x for x in all_vars if x.name[0:len(net_name2) + 1]==net_name2 + '/']
saver2 = tf.train.Saver(net2_vars)
saver2.restore(sess, config_net2['model_file'])
else:
# net2ax_vars = [x for x in all_vars if x.name[0:len(net_name2ax)+1]==net_name2ax + '/']
# saver2ax = tf.train.Saver(net2ax_vars)
# saver2ax.restore(sess, config_net2sg['model_file'])
# net2sg_vars = [x for x in all_vars if x.name[0:len(net_name2sg)+1]==net_name2sg + '/']
# for i in range(len(net2sg_vars)):
# copy_value = tf.assign(net2sg_vars[i], net2ax_vars[i])
# copy_value.eval()
# print('net2sg loaded')
# saver2sg = tf.train.Saver(net2sg_vars)
# saver2sg.save(sess, "model/msnet_tc32sg_15000cp.ckpt")
# print('net2sg saved')
#
# saver2ax.restore(sess, config_net2cr['model_file'])
# net2cr_vars = [x for x in all_vars if x.name[0:len(net_name2cr)+1]==net_name2cr + '/']
# for i in range(len(net2cr_vars)):
# copy_value = tf.assign(net2cr_vars[i], net2ax_vars[i])
# copy_value.eval()
# saver2cr = tf.train.Saver(net2cr_vars)
# saver2cr.save(sess, "model/msnet_tc32cr_10000cp.ckpt")
# print('net2cr saved')
# return
# saver2ax.restore(sess, config_net2ax['model_file'])
net2ax_vars = [x for x in all_vars if x.name[0:len(net_name2ax)+1]==net_name2ax + '/']
saver2ax = tf.train.Saver(net2ax_vars)
saver2ax.restore(sess, config_net2ax['model_file'])
net2sg_vars = [x for x in all_vars if x.name[0:len(net_name2sg)+1]==net_name2sg + '/']
saver2sg = tf.train.Saver(net2sg_vars)
saver2sg.restore(sess, config_net2sg['model_file'])
net2cr_vars = [x for x in all_vars if x.name[0:len(net_name2cr)+1]==net_name2cr + '/']
saver2cr = tf.train.Saver(net2cr_vars)
saver2cr.restore(sess, config_net2cr['model_file'])
if(config_net3):
net3_vars = [x for x in all_vars if x.name[0:len(net_name3) + 1]==net_name3 + '/']
saver3 = tf.train.Saver(net3_vars)
saver3.restore(sess, config_net3['model_file'])
else:
# net3ax_vars = [x for x in all_vars if x.name[0:len(net_name3ax)+1]==net_name3ax + '/']
# saver3ax = tf.train.Saver(net3ax_vars)
# saver3ax.restore(sess, config_net3sg['model_file'])
# net3sg_vars = [x for x in all_vars if x.name[0:len(net_name3sg)+1]==net_name3sg + '/']
# for i in range(len(net3sg_vars)):
# copy_value = tf.assign(net3sg_vars[i], net3ax_vars[i])
# copy_value.eval()
# print('net3sg loaded')
# saver3sg = tf.train.Saver(net3sg_vars)
# saver3sg.save(sess, "model/msnet_en32sg_20000cp.ckpt")
# print('net3sg saved')
#
# saver3ax.restore(sess, config_net3cr['model_file'])
# net3cr_vars = [x for x in all_vars if x.name[0:len(net_name3cr)+1]==net_name3cr + '/']
# for i in range(len(net3cr_vars)):
# copy_value = tf.assign(net3cr_vars[i], net3ax_vars[i])
# copy_value.eval()
# saver3cr = tf.train.Saver(net3cr_vars)
# saver3cr.save(sess, "model/msnet_en32cr_20000cp.ckpt")
# print('net3cr saved')
#
# saver3ax.restore(sess, config_net3ax['model_file'])
net3ax_vars = [x for x in all_vars if x.name[0:len(net_name3ax) + 1]==net_name3ax+ '/']
saver3ax = tf.train.Saver(net3ax_vars)
saver3ax.restore(sess, config_net3ax['model_file'])
net3sg_vars = [x for x in all_vars if x.name[0:len(net_name3sg) + 1]==net_name3sg+ '/']
saver3sg = tf.train.Saver(net3sg_vars)
saver3sg.restore(sess, config_net3sg['model_file'])
net3cr_vars = [x for x in all_vars if x.name[0:len(net_name3cr) + 1]==net_name3cr+ '/']
saver3cr = tf.train.Saver(net3cr_vars)
saver3cr.restore(sess, config_net3cr['model_file'])
loader = DataLoader()
loader.set_params(config_data)
loader.load_data()
# start to test
image_num = loader.get_total_image_number()
test_slice_direction = config_test.get('test_slice_direction', 'all')
save_folder = config_test['save_folder']
test_time = []
struct = ndimage.generate_binary_structure(3, 2)
margin = config_test.get('roi_patch_margin', 5)
for i in range(image_num):
# test of 1st network
t0 = time.time()
[imgs, weight, temp_name] = loader.get_image_data_with_name(i)
groi = get_roi(weight > 0, margin)
temp_imgs = [x[np.ix_(range(groi[0], groi[1]), range(groi[2], groi[3]), range(groi[4], groi[5]))] \
for x in imgs]
temp_weight = weight[np.ix_(range(groi[0], groi[1]), range(groi[2], groi[3]), range(groi[4], groi[5]))]
if(config_net1):
data_shapes = [data_shape1, data_shape1, data_shape1]
label_shapes = [label_shape1, label_shape1, label_shape1]
nets = [net1, net1, net1]
outputs = [proby1, proby1, proby1]
inputs = [x1, x1, x1]
data_channel = data_channel1
class_num = class_num1
else:
data_shapes = [data_shape1ax, data_shape1sg, data_shape1cr]
label_shapes = [label_shape1ax, label_shape1sg, label_shape1cr]
nets = [net1ax, net1sg, net1cr]
outputs = [proby1ax, proby1sg, proby1cr]
inputs = [x1ax, x1sg, x1cr]
data_channel = data_channel1ax
class_num = class_num1ax
prob1 = test_one_image_three_nets_adaptive_shape(temp_imgs, data_shapes, label_shapes, data_channel, class_num,
batch_size, sess, nets, outputs, inputs, shape_mode = 0)
pred1 = np.asarray(np.argmax(prob1, axis = 3), np.uint16)
pred1 = pred1 * temp_weight
# out_label = pred1 * temp_weight
# label_convert_source = config_test.get('label_convert_source', None)
# label_convert_target = config_test.get('label_convert_target', None)
# if(label_convert_source and label_convert_target):
# assert(len(label_convert_source) == len(label_convert_target))
# out_label = convert_label(out_label, label_convert_source, label_convert_target)
# test of 2nd network
wt_threshold = 2000
if(pred1.sum() == 0):
print('net1 output is null', temp_name)
roi2 = get_roi(temp_imgs[0] > 0, margin)
else:
pred1_lc = ndimage.morphology.binary_closing(pred1, structure = struct)
pred1_lc = get_largest_two_component(pred1_lc, True, wt_threshold)
roi2 = get_roi(pred1_lc, margin)
sub_imgs = [x[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] \
for x in temp_imgs]
sub_weight = temp_weight[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))]
if(config_net2):
data_shapes = [data_shape2, data_shape2, data_shape2]
label_shapes = [label_shape2, label_shape2, label_shape2]
nets = [net2, net2, net2]
outputs = [proby2, proby2, proby2]
inputs = [x2, x2, x2]
data_channel = data_channel2
class_num = class_num2
else:
data_shapes = [data_shape2ax, data_shape2sg, data_shape2cr]
label_shapes = [label_shape2ax, label_shape2sg, label_shape2cr]
nets = [net2ax, net2sg, net2cr]
outputs = [proby2ax, proby2sg, proby2cr]
inputs = [x2ax, x2sg, x2cr]
data_channel = data_channel2ax
class_num = class_num2ax
prob2 = test_one_image_three_nets_adaptive_3dshape(sub_imgs, data_shapes, label_shapes, data_channel, class_num, sess, nets, outputs, inputs)
# prob2 = test_one_image_three_nets_adaptive_shape(sub_imgs, data_shapes, label_shapes, data_channel, class_num, batch_size, sess, nets, outputs, inputs, shape_mode = 1)
pred2 = np.asarray(np.argmax(prob2, axis = 3), np.uint16)
pred2 = pred2 * sub_weight
# test of 3rd network
if(pred2.sum() == 0):
[roid, roih, roiw] = sub_imgs[0].shape
roi3 = [0, roid, 0, roih, 0, roiw]
subsub_imgs = sub_imgs
subsub_weight = sub_weight
else:
pred2_lc = ndimage.morphology.binary_closing(pred2, structure = struct)
pred2_lc = get_largest_two_component(pred2_lc)
roi3 = get_roi(pred2_lc, margin)
subsub_imgs = [x[np.ix_(range(roi3[0], roi3[1]), range(roi3[2], roi3[3]), range(roi3[4], roi3[5]))] \
for x in sub_imgs]
subsub_weight = sub_weight[np.ix_(range(roi3[0], roi3[1]), range(roi3[2], roi3[3]), range(roi3[4], roi3[5]))]
if(config_net3):
data_shapes = [data_shape3, data_shape3, data_shape3]
label_shapes = [label_shape3, label_shape3, label_shape3]
nets = [net3, net3, net3]
outputs = [proby3, proby3, proby3]
inputs = [x3, x3, x3]
data_channel = data_channel3
class_num = class_num3
else:
data_shapes = [data_shape3ax, data_shape3sg, data_shape3cr]
label_shapes = [label_shape3ax, label_shape3sg, label_shape3cr]
nets = [net3ax, net3sg, net3cr]
outputs = [proby3ax, proby3sg, proby3cr]
inputs = [x3ax, x3sg, x3cr]
data_channel = data_channel3ax
class_num = class_num3ax
prob3 = test_one_image_three_nets_adaptive_shape(subsub_imgs, data_shapes, label_shapes, data_channel, class_num,
batch_size, sess, nets, outputs, inputs, shape_mode = 1)
pred3 = np.asarray(np.argmax(prob3, axis = 3), np.uint16)
pred3 = pred3 * subsub_weight
# fuse results at 3 levels
# convert subsub_label to full size (non-enhanced)
label3_roi = np.zeros_like(pred2)
label3_roi[np.ix_(range(roi3[0], roi3[1]), range(roi3[2], roi3[3]), range(roi3[4], roi3[5]))] = pred3
label3 = np.zeros_like(pred1)
label3[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] = label3_roi
# convert sub_label to full size (tumor core)
label2 = np.zeros_like(pred1)
label2[np.ix_(range(roi2[0], roi2[1]), range(roi2[2], roi2[3]), range(roi2[4], roi2[5]))] = pred2
# fuse the results
label1_mask = (pred1 + label2 + label3) > 0
label1_mask = ndimage.morphology.binary_closing(label1_mask, structure = struct)
label1_mask = get_largest_two_component(label1_mask, False, wt_threshold)
label1 = pred1 * label1_mask
label2_3_mask = (label2 + label3) > 0
label2_3_mask = label2_3_mask * label1_mask
label2_3_mask = ndimage.morphology.binary_closing(label2_3_mask, structure = struct)
label2_3_mask = remove_external_core(label1, label2_3_mask)
if(label2_3_mask.sum() > 0):
label2_3_mask = get_largest_two_component(label2_3_mask)
label1 = (label1 + label2_3_mask) > 0
label2 = label2_3_mask
label3 = label2 * label3
vox_3 = label3.sum()
if(0 < vox_3 and vox_3 < 30):
print('ignored voxel number ', vox_3, flush = True)
label3 = np.zeros_like(label2)
out_label = label1 * 2
out_label[label2>0] = 1
out_label[label3>0] = 4
out_label = np.asarray(out_label, np.int16)
test_time.append(time.time() - t0)
final_label = np.zeros_like(weight, np.int16)
final_label[np.ix_(range(groi[0], groi[1]), range(groi[2], groi[3]), range(groi[4], groi[5]))] = out_label
save_array_as_nifty_volume(final_label, save_folder+"/{0:}.nii.gz".format(temp_name))
print(temp_name, flush = True)
test_time = np.asarray(test_time)
print('test time', test_time.mean(), flush= True)
np.savetxt(save_folder + '/test_time.txt', test_time)
sess.close()
def L8_PSF_optimization(self):
self.h, self.v = mtile_cal(self.lat, self.lon)
pr = get_wrs(self.lat, self.lon)
self.path, self.row = pr[0]['path'], pr[0]['row']
#self.Hfiles = glob.glob(directory +'l_data/LC8%03d%03d%d*LGN00_sr_band1.tif'%(self.path, self.row, self.year))
self.Hfile = directory + 'l_data/LC8%03d%03d%d%03dLGN00_toa_' % (
self.path, self.row, self.year, self.doy)
#Lfile = glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf'%(year,doy,h,v))[0]
self.Lfiles = [
glob.glob('m_data/MCD43A1.A%d%03d.h%02dv%02d.006.*.hdf' %
(self.year, i, self.h, self.v))[0]
for i in range(self.doy - 8, self.doy + 9)
]
if self.read_meta(self.Hfile, self.path, self.row) == None:
print 'Too much cloud!!'
else:
self.sza, self.saa, self.vza, self.vaa, self.dic, self.corners = self.read_meta(
self.Hfile, self.path, self.row)
self.L_inds, self.H_inds = ML_geo_trans(self.lat, self.lon,
self.dic, self.corners)
self.Lx, self.Ly = self.L_inds
self.Hx, self.Hy = self.H_inds
tems = np.zeros((3, 6))
tems[0, :] = self.sza
tems[1, :] = self.vza
tems[2, :] = self.vaa - self.saa
angles = (tems[0][-1], tems[1][-1], tems[2][-1])
#self.BRDF_16_days, self.composite_brdf, self.base_mask = self.compositing(angles, thre=0.005)
if glob.glob(self.Lfiles[8][:-3] + 'L8.16days.pkl') == []:
self.BRDF_16_days = np.array([get_brdf_six(Lfile,angles,bands=(7,), \
flag=None, Linds= self.L_inds) for Lfile in self.Lfiles]).squeeze()
valid_range = (self.BRDF_16_days[:, 0, :] >=
0) & (self.BRDF_16_days[:, 0, :] <= 1)
magic = 0.618034
test = self.BRDF_16_days[:, 0, :].copy()
test[~valid_range] = np.nan
W = magic**self.BRDF_16_days[:, 1, :]
W[self.BRDF_16_days[:, 1, :] > 1] = 0
#smothed = smoothn(test, axis=0, isrobust=1, W =W, s=1)[0]
pkl.dump(self.BRDF_16_days,
open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'w'))
#pkl.dump(smothed, open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'w'))
else:
self.BRDF_16_days = pkl.load(
open(self.Lfiles[8][:-3] + 'L8.16days.pkl', 'r'))
#smothed = pkl.load(open(self.Lfiles[8][:-3]+'L8.16days.smoothed.pkl', 'r'))
cloud = gdal.Open(self.Hfile[:-5] + '_cfmask.tif').ReadAsArray()
cl_mask = cloud == 4 # cloud pixels; strictest way is to set the clear pixels with cloud==0
struct = ndimage.generate_binary_structure(2, 2)
dia_cloud = ndimage.binary_dilation(cl_mask,
structure=struct,
iterations=20).astype(
cl_mask.dtype)
self.H_data = gdal.Open(self.Hfile +
'band%d.tif' % 7).ReadAsArray() * 0.0001
mask = ~(self.H_data < 0).astype('bool')
small_mask = ndimage.binary_erosion(mask,
structure=struct,
iterations=20).astype(
mask.dtype)
self.val_mask = (~dia_cloud) & small_mask
self.L_data = np.zeros(self.BRDF_16_days[8, 0, :].shape[0])
self.L_data[:] = np.nan
self.L_data[self.BRDF_16_days[8, 1, :] == 0] = self.BRDF_16_days[
8, 0, :][self.BRDF_16_days[8, 1, :] == 0]
#args = s, self.L_data,
avker = np.ones((40, 40))
navker = avker / avker.sum()
self.s = signal.fftconvolve(self.H_data, navker, mode='same')
self.s[~self.val_mask] = np.nan
min_val = [-40, -40]
max_val = [40, 40]
ps, distributions = create_training_set(['xs', 'ys'],
min_val,
max_val,
n_train=50)
solved = parmap(self.op1, ps, nprocs=10)
paras, costs = np.array([i[0] for i in solved
]), np.array([i[1] for i in solved])
xs, ys = paras[costs == costs.min()][0]
if costs.min() < 0.1:
min_val = [5, 5, -15, xs - 5, ys - 5]
max_val = [100, 100, 15, xs + 5, ys + 5]
self.bounds = [5,
100], [5,
100], [-15,
15], [xs - 5,
xs + 5], [ys - 5, ys + 5]
ps, distributions = create_training_set(
['xstd', 'ystd', 'ang', 'xs', 'ys'],
min_val,
max_val,
n_train=50)
#ps = zip(xstd.ravel(), ystd.ravel())
print 'Start solving...'
self.solved = parmap(self.op, ps, nprocs=10)
#costs = np.array([i[1] for i in self.solved])
#paras = np.array([i[0] for i in self.solved])
#min_mask = costs==costs.min()
#min_costs = costs.min()
#para = paras[min_mask]
print self.solved
return self.solved, self.BRDF_16_days
else:
print 'Cost is too large, plese check!', xs, ys, costs.min()
return [[xs, ys, costs.min()], self.BRDF_16_days]
'''
def diamond_se(radius, dimension):
se = generate_binary_structure(dimension, 1)
return iterate_structure(se, radius)
