def finalProcessingSpur(s, params):
logging.info(f"{s['filename']} - \tfinalProcessingSpur")
disk_radius = int(params.get("disk_radius", "25"))
selem = disk(disk_radius)
mask = s["img_mask_use"]
mask_opened = binary_opening(mask, selem)
mask_spur = ~mask_opened & mask
io.imsave(s["outdir"] + os.sep + s["filename"] + "_spur.png",
img_as_ubyte(mask_spur))
prev_mask = s["img_mask_use"]
s["img_mask_use"] = mask_opened
s.addToPrintList(
"spur_pixels",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After BasicModule.finalProcessingSpur NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After BasicModule.finalProcessingSpur NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
def finalProcessingArea(s, params):
logging.info(f"{s['filename']} - \tfinalProcessingArea")
area_thresh = int(params.get("area_threshold", "1000"))
mask = s["img_mask_use"]
mask_opened = remove_small_objects(mask, min_size=area_thresh)
mask_removed_area = ~mask_opened & mask
io.imsave(s["outdir"] + os.sep + s["filename"] + "_areathresh.png",
img_as_ubyte(mask_removed_area))
prev_mask = s["img_mask_use"]
s["img_mask_use"] = mask_opened > 0
s.addToPrintList(
"areaThresh",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After BasicModule.finalProcessingArea NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After BasicModule.finalProcessingArea NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
def identifyBlurryRegions(s, params):
logging.info(f"{s['filename']} - \tidentifyBlurryRegions")
blur_radius = int(params.get("blur_radius", 7))
blur_threshold = float(params.get("blur_threshold", .1))
img = s.getImgThumb(params.get("image_work_size", "2.5x"))
img = rgb2gray(img)
img_laplace = np.abs(skimage.filters.laplace(img))
mask = skimage.filters.gaussian(img_laplace,
sigma=blur_radius) <= blur_threshold
mask = skimage.transform.resize(
mask, s.getImgThumb(s["image_work_size"]).shape, order=0
)[:, :, 1] # for some reason resize takes a grayscale and produces a 3chan
mask = s["img_mask_use"] & (mask > 0)
io.imsave(s["outdir"] + os.sep + s["filename"] + "_blurry.png",
img_as_ubyte(mask))
s["img_mask_blurry"] = (mask * 255) > 0
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_blurry"]
rps = measure.regionprops(morphology.label(mask))
if rps:
areas = np.asarray([rp.area for rp in rps])
nobj = len(rps)
area_max = areas.max()
area_mean = areas.mean()
else:
nobj = area_max = area_mean = 0
s.addToPrintList("blurry_removed_num_regions", str(nobj))
s.addToPrintList("blurry_removed_mean_area", str(area_mean))
s.addToPrintList("blurry_removed_max_area", str(area_max))
s.addToPrintList(
"blurry_removed_percent",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After BlurDetectionModule.identifyBlurryRegions NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After BlurDetectionModule.identifyBlurryRegions NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
return
def pixelWise(s, params):
name = params.get("name", "classTask")
logging.info(f"{s['filename']} - \tpixelWise:\t", name)
thresh = float(params.get("threshold", .5))
fname = params.get("tsv_file", "")
if fname == "":
logging.error(
f"{s['filename']} - tsv_file not set in ClassificationModule.pixelWise for ",
name)
sys.exit(1)
return
model_vals = np.loadtxt(fname, delimiter="\t", skiprows=1)
img = s.getImgThumb(s["image_work_size"])
gnb = GaussianNB()
gnb.fit(model_vals[:, 1:], model_vals[:, 0])
cal = gnb.predict_proba(img.reshape(-1, 3))
cal = cal.reshape(img.shape[0], img.shape[1], 2)
mask = cal[:, :, 1] > thresh
mask = s["img_mask_use"] & (mask > 0)
s.addToPrintList(name, str(mask.mean()))
io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png",
img_as_ubyte(mask))
s["img_mask_" + name] = (mask * 255) > 0
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_" + name]
s.addToPrintList(
name,
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After ClassificationModule.pixelWise:{name} NO tissue "
f"remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After ClassificationModule.pixelWise:{name} NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
return
def getIntensityThresholdPercent(s, params):
name = params.get("name", "classTask")
logging.info(
f"{s['filename']} - \tLightDarkModule.getIntensityThresholdPercent:\t {name}"
)
lower_thresh = float(params.get("lower_threshold", -float("inf")))
upper_thresh = float(params.get("upper_threshold", float("inf")))
lower_var = float(params.get("lower_variance", -float("inf")))
upper_var = float(params.get("upper_variance", float("inf")))
img = s.getImgThumb(s["image_work_size"])
img_var = img.std(axis=2)
map_var = np.bitwise_and(img_var > lower_var, img_var < upper_var)
img = color.rgb2gray(img)
map = np.bitwise_and(img > lower_thresh, img < upper_thresh)
map = np.bitwise_and(map, map_var)
s["img_mask_" + name] = map > 0
if strtobool(params.get("invert", "False")):
s["img_mask_" + name] = ~s["img_mask_" + name]
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & s["img_mask_" + name]
io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png",
img_as_ubyte(prev_mask & ~s["img_mask_" + name]))
s.addToPrintList(
name,
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After LightDarkModule.getIntensityThresholdPercent:{name} NO tissue "
f"remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After LightDarkModule.getIntensityThresholdPercent:{name} NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
return
def getIntensityThresholdOtsu(s, params):
logging.info(
f"{s['filename']} - \tLightDarkModule.getIntensityThresholdOtsu")
name = "otsu"
local = strtobool(params.get("local", "False"))
radius = float(params.get("radius", 15))
selem = disk(radius)
img = s.getImgThumb(s["image_work_size"])
img = color.rgb2gray(img)
if local:
thresh = rank.otsu(img, selem)
name += "local"
else:
thresh = threshold_otsu(img)
map = img < thresh
s["img_mask_" + name] = map > 0
if strtobool(params.get("invert", "False")):
s["img_mask_" + name] = ~s["img_mask_" + name]
io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png",
img_as_ubyte(s["img_mask_" + name]))
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & s["img_mask_" + name]
s.addToPrintList(
name,
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After LightDarkModule.getIntensityThresholdOtsu:{name} NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
s["warnings"].append(
f"After LightDarkModule.getIntensityThresholdOtsu:{name} NO tissue remains detectable! "
f"Downstream modules likely to be incorrect/fail")
return
def removeBrightestPixels(s, params):
logging.info(f"{s['filename']} - \tLightDarkModule.removeBrightestPixels")
# lower_thresh = float(params.get("lower_threshold", -float("inf")))
# upper_thresh = float(params.get("upper_threshold", float("inf")))
#
# lower_var = float(params.get("lower_variance", -float("inf")))
# upper_var = float(params.get("upper_variance", float("inf")))
img = s.getImgThumb(s["image_work_size"])
img = color.rgb2gray(img)
kmeans = KMeans(n_clusters=3, n_init=1).fit(img.reshape([-1, 1]))
brightest_cluster = np.argmax(kmeans.cluster_centers_)
darkest_point_in_brightest_cluster = (img.reshape(
[-1, 1])[kmeans.labels_ == brightest_cluster]).min()
s["img_mask_bright"] = img > darkest_point_in_brightest_cluster
if strtobool(params.get("invert", "False")):
s["img_mask_bright"] = ~s["img_mask_bright"]
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & s["img_mask_bright"]
io.imsave(s["outdir"] + os.sep + s["filename"] + "_bright.png",
img_as_ubyte(prev_mask & ~s["img_mask_bright"]))
s.addToPrintList(
"brightestPixels",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After LightDarkModule.removeBrightestPixels NO tissue "
f"remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After LightDarkModule.removeBrightestPixels NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
return
def geoJSONMask(s, params):
logging.info(f"{s['filename']} - \tgeoJSONMask")
mask = s["img_mask_use"]
geojson_basepath = params.get("geojson_filepath", None)
geojson_suffix = params.get("geojson_suffix", "")
if not geojson_basepath:
geojson_basepath = s["dir"]
fname = geojson_basepath + os.sep + PurePosixPath(
s['filename']).stem + geojson_suffix + '.json'
if not Path(fname).is_file():
msg = f"Annotation file {fname} does not exist. Skipping."
logging.warning(f"{s['filename']} - {msg}")
s["warnings"].append(msg)
return
logging.info(f"{s['filename']} - \tusing {fname}")
point_sets = get_points_from_geojson(s, fname)
annotationMask = mask_out_annotation(s, point_sets) > 0
io.imsave(s["outdir"] + os.sep + s["filename"] + "_geoJSONMask.png",
img_as_ubyte(annotationMask))
prev_mask = s["img_mask_use"]
s["img_mask_use"] = prev_mask & annotationMask
s.addToPrintList(
"geoJSONMask",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After AnnotationModule.geoJSONMask NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After AnnotationModule.geoJSONMask NO tissue remains detectable! Downstream modules likely to be incorrect/fail"
)
return
def minimumPixelIntensityNeighborhoodFiltering(s, params):
logging.info(
f"{s['filename']} - \tLightDarkModule.minimumPixelNeighborhoodFiltering"
)
disk_size = int(params.get("disk_size", 10000))
threshold = int(params.get("upper_threshold", 200))
img = s.getImgThumb(s["image_work_size"])
img = color.rgb2gray(img)
img = (img * 255).astype(np.uint8)
selem = disk(disk_size)
imgfilt = rank.minimum(img, selem)
s["img_mask_bright"] = imgfilt > threshold
if strtobool(params.get("invert", "True")):
s["img_mask_bright"] = ~s["img_mask_bright"]
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & s["img_mask_bright"]
io.imsave(s["outdir"] + os.sep + s["filename"] + "_bright.png",
img_as_ubyte(prev_mask & ~s["img_mask_bright"]))
s.addToPrintList(
"brightestPixels",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After LightDarkModule.minimumPixelNeighborhoodFiltering NO tissue "
f"remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After LightDarkModule.minimumPixelNeighborhoodFiltering NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
return
def detectSmoothness(s, params):
logging.info(f"{s['filename']} - \tBubbleRegionByRegion.detectSmoothness")
thresh = float(params.get("threshold", ".01"))
kernel_size = int(params.get("kernel_size", "10"))
min_object_size = int(params.get("min_object_size", "100"))
img = s.getImgThumb(s["image_work_size"])
img = color.rgb2gray(img)
avg = np.ones((kernel_size, kernel_size)) / (kernel_size**2)
imf = scipy.signal.convolve2d(img, avg, mode="same")
mask_flat = abs(imf - img) < thresh
mask_flat = remove_small_objects(mask_flat, min_size=min_object_size)
mask_flat = ~remove_small_objects(~mask_flat, min_size=min_object_size)
prev_mask = s["img_mask_use"]
s["img_mask_flat"] = mask_flat
io.imsave(s["outdir"] + os.sep + s["filename"] + "_flat.png",
img_as_ubyte(mask_flat & prev_mask))
s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_flat"]
s.addToPrintList(
"flat_areas",
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After BubbleRegionByRegion.detectSmoothness: NO tissue "
f"remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After BubbleRegionByRegion.detectSmoothness: NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
return
def byExampleWithFeatures(s, params):
name = params.get("name", "classTask")
logging.info(
f"{s['filename']} - \tClassificationModule.byExample:\t{name}")
thresh = float(params.get("threshold", .5))
nsamples_per_example = float(params.get("nsamples_per_example", -1))
examples = params.get("examples", "")
if examples == "":
logging.error(
f"{s['filename']} - No examples provided in ClassificationModule.byExample for {name} !!"
)
sys.exit(1)
return
if params.get("features", "") == "":
logging.error(
f"{s['filename']} - No features provided in ClassificationModule.byExample for {name} !!"
)
sys.exit(1)
return
with params[
"lock"]: # this lock is shared across all threads such that only one thread needs to train the model
# then it is shared with all other modules
if not params["shared_dict"].get("model_" + name, False):
logging.info(
f"{s['filename']} - Training model ClassificationModule.byExample:{name}"
)
model_vals = []
model_labels = np.empty([0, 1])
for ex in params["examples"].splitlines():
ex = re.split(
r'(?<!\W[A-Za-z]):(?!\\)',
ex) # workaround for windows: don't split on i.e. C:\
img = io.imread(ex[0])
eximg = compute_features(img, params)
eximg = eximg.reshape(-1, eximg.shape[2])
mask = io.imread(ex[1], as_gray=True).reshape(-1, 1)
if nsamples_per_example != -1: #sub sambling required
nitems = nsamples_per_example if nsamples_per_example > 1 else int(
mask.shape[0] * nsamples_per_example)
idxkeep = np.random.choice(mask.shape[0], size=int(nitems))
eximg = eximg[idxkeep, :]
mask = mask[idxkeep]
model_vals.append(eximg)
model_labels = np.vstack((model_labels, mask))
# do stuff here with model_vals
model_vals = np.vstack(model_vals)
clf = RandomForestClassifier(n_jobs=-1)
clf.fit(model_vals, model_labels.ravel())
params["shared_dict"]["model_" + name] = clf
logging.info(
f"{s['filename']} - Training model ClassificationModule.byExample:{name}....done"
)
clf = params["shared_dict"]["model_" + name]
img = s.getImgThumb(s["image_work_size"])
feats = compute_features(img, params)
cal = clf.predict_proba(feats.reshape(-1, feats.shape[2]))
cal = cal.reshape(img.shape[0], img.shape[1], 2)
mask = cal[:, :, 1] > thresh
area_thresh = int(params.get("area_threshold", "5"))
if area_thresh > 0:
mask = remove_small_objects(mask, min_size=area_thresh, in_place=True)
dilate_kernel_size = int(params.get("dilate_kernel_size", "0"))
if dilate_kernel_size > 0:
mask = dilation(mask,
selem=np.ones(
(dilate_kernel_size, dilate_kernel_size)))
mask = s["img_mask_use"] & (mask > 0)
io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png",
img_as_ubyte(mask))
s["img_mask_" + name] = (mask * 255) > 0
prev_mask = s["img_mask_use"]
s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_" + name]
s.addToPrintList(
name,
printMaskHelper(params.get("mask_statistics", s["mask_statistics"]),
prev_mask, s["img_mask_use"]))
if len(s["img_mask_use"].nonzero()
[0]) == 0: # add warning in case the final tissue is empty
logging.warning(
f"{s['filename']} - After ClassificationModule.byExampleWithFeatures:{name} NO tissue "
f"remains detectable! Downstream modules likely to be incorrect/fail"
)
s["warnings"].append(
f"After ClassificationModule.byExampleWithFeatures:{name} NO tissue remains "
f"detectable! Downstream modules likely to be incorrect/fail")
s["img_mask_force"].append("img_mask_" + name)
s["completed"].append(f"byExampleWithFeatures:{name}")
return