def finalProcessingSpur(s, params): logging.info(f"{s['filename']} - \tfinalProcessingSpur") disk_radius = int(params.get("disk_radius", "25")) selem = disk(disk_radius) mask = s["img_mask_use"] mask_opened = binary_opening(mask, selem) mask_spur = ~mask_opened & mask io.imsave(s["outdir"] + os.sep + s["filename"] + "_spur.png", img_as_ubyte(mask_spur)) prev_mask = s["img_mask_use"] s["img_mask_use"] = mask_opened s.addToPrintList( "spur_pixels", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After BasicModule.finalProcessingSpur NO tissue remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After BasicModule.finalProcessingSpur NO tissue remains detectable! Downstream modules likely to be incorrect/fail" )
def finalProcessingArea(s, params): logging.info(f"{s['filename']} - \tfinalProcessingArea") area_thresh = int(params.get("area_threshold", "1000")) mask = s["img_mask_use"] mask_opened = remove_small_objects(mask, min_size=area_thresh) mask_removed_area = ~mask_opened & mask io.imsave(s["outdir"] + os.sep + s["filename"] + "_areathresh.png", img_as_ubyte(mask_removed_area)) prev_mask = s["img_mask_use"] s["img_mask_use"] = mask_opened > 0 s.addToPrintList( "areaThresh", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After BasicModule.finalProcessingArea NO tissue remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After BasicModule.finalProcessingArea NO tissue remains detectable! Downstream modules likely to be incorrect/fail" )
def identifyBlurryRegions(s, params): logging.info(f"{s['filename']} - \tidentifyBlurryRegions") blur_radius = int(params.get("blur_radius", 7)) blur_threshold = float(params.get("blur_threshold", .1)) img = s.getImgThumb(params.get("image_work_size", "2.5x")) img = rgb2gray(img) img_laplace = np.abs(skimage.filters.laplace(img)) mask = skimage.filters.gaussian(img_laplace, sigma=blur_radius) <= blur_threshold mask = skimage.transform.resize( mask, s.getImgThumb(s["image_work_size"]).shape, order=0 )[:, :, 1] # for some reason resize takes a grayscale and produces a 3chan mask = s["img_mask_use"] & (mask > 0) io.imsave(s["outdir"] + os.sep + s["filename"] + "_blurry.png", img_as_ubyte(mask)) s["img_mask_blurry"] = (mask * 255) > 0 prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_blurry"] rps = measure.regionprops(morphology.label(mask)) if rps: areas = np.asarray([rp.area for rp in rps]) nobj = len(rps) area_max = areas.max() area_mean = areas.mean() else: nobj = area_max = area_mean = 0 s.addToPrintList("blurry_removed_num_regions", str(nobj)) s.addToPrintList("blurry_removed_mean_area", str(area_mean)) s.addToPrintList("blurry_removed_max_area", str(area_max)) s.addToPrintList( "blurry_removed_percent", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After BlurDetectionModule.identifyBlurryRegions NO tissue remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After BlurDetectionModule.identifyBlurryRegions NO tissue remains detectable! Downstream modules likely to be incorrect/fail" ) return
def pixelWise(s, params): name = params.get("name", "classTask") logging.info(f"{s['filename']} - \tpixelWise:\t", name) thresh = float(params.get("threshold", .5)) fname = params.get("tsv_file", "") if fname == "": logging.error( f"{s['filename']} - tsv_file not set in ClassificationModule.pixelWise for ", name) sys.exit(1) return model_vals = np.loadtxt(fname, delimiter="\t", skiprows=1) img = s.getImgThumb(s["image_work_size"]) gnb = GaussianNB() gnb.fit(model_vals[:, 1:], model_vals[:, 0]) cal = gnb.predict_proba(img.reshape(-1, 3)) cal = cal.reshape(img.shape[0], img.shape[1], 2) mask = cal[:, :, 1] > thresh mask = s["img_mask_use"] & (mask > 0) s.addToPrintList(name, str(mask.mean())) io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png", img_as_ubyte(mask)) s["img_mask_" + name] = (mask * 255) > 0 prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_" + name] s.addToPrintList( name, printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After ClassificationModule.pixelWise:{name} NO tissue " f"remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After ClassificationModule.pixelWise:{name} NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") return
def getIntensityThresholdPercent(s, params): name = params.get("name", "classTask") logging.info( f"{s['filename']} - \tLightDarkModule.getIntensityThresholdPercent:\t {name}" ) lower_thresh = float(params.get("lower_threshold", -float("inf"))) upper_thresh = float(params.get("upper_threshold", float("inf"))) lower_var = float(params.get("lower_variance", -float("inf"))) upper_var = float(params.get("upper_variance", float("inf"))) img = s.getImgThumb(s["image_work_size"]) img_var = img.std(axis=2) map_var = np.bitwise_and(img_var > lower_var, img_var < upper_var) img = color.rgb2gray(img) map = np.bitwise_and(img > lower_thresh, img < upper_thresh) map = np.bitwise_and(map, map_var) s["img_mask_" + name] = map > 0 if strtobool(params.get("invert", "False")): s["img_mask_" + name] = ~s["img_mask_" + name] prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & s["img_mask_" + name] io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png", img_as_ubyte(prev_mask & ~s["img_mask_" + name])) s.addToPrintList( name, printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After LightDarkModule.getIntensityThresholdPercent:{name} NO tissue " f"remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After LightDarkModule.getIntensityThresholdPercent:{name} NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") return
def getIntensityThresholdOtsu(s, params): logging.info( f"{s['filename']} - \tLightDarkModule.getIntensityThresholdOtsu") name = "otsu" local = strtobool(params.get("local", "False")) radius = float(params.get("radius", 15)) selem = disk(radius) img = s.getImgThumb(s["image_work_size"]) img = color.rgb2gray(img) if local: thresh = rank.otsu(img, selem) name += "local" else: thresh = threshold_otsu(img) map = img < thresh s["img_mask_" + name] = map > 0 if strtobool(params.get("invert", "False")): s["img_mask_" + name] = ~s["img_mask_" + name] io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png", img_as_ubyte(s["img_mask_" + name])) prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & s["img_mask_" + name] s.addToPrintList( name, printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After LightDarkModule.getIntensityThresholdOtsu:{name} NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") s["warnings"].append( f"After LightDarkModule.getIntensityThresholdOtsu:{name} NO tissue remains detectable! " f"Downstream modules likely to be incorrect/fail") return
def removeBrightestPixels(s, params): logging.info(f"{s['filename']} - \tLightDarkModule.removeBrightestPixels") # lower_thresh = float(params.get("lower_threshold", -float("inf"))) # upper_thresh = float(params.get("upper_threshold", float("inf"))) # # lower_var = float(params.get("lower_variance", -float("inf"))) # upper_var = float(params.get("upper_variance", float("inf"))) img = s.getImgThumb(s["image_work_size"]) img = color.rgb2gray(img) kmeans = KMeans(n_clusters=3, n_init=1).fit(img.reshape([-1, 1])) brightest_cluster = np.argmax(kmeans.cluster_centers_) darkest_point_in_brightest_cluster = (img.reshape( [-1, 1])[kmeans.labels_ == brightest_cluster]).min() s["img_mask_bright"] = img > darkest_point_in_brightest_cluster if strtobool(params.get("invert", "False")): s["img_mask_bright"] = ~s["img_mask_bright"] prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & s["img_mask_bright"] io.imsave(s["outdir"] + os.sep + s["filename"] + "_bright.png", img_as_ubyte(prev_mask & ~s["img_mask_bright"])) s.addToPrintList( "brightestPixels", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After LightDarkModule.removeBrightestPixels NO tissue " f"remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After LightDarkModule.removeBrightestPixels NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") return
def geoJSONMask(s, params): logging.info(f"{s['filename']} - \tgeoJSONMask") mask = s["img_mask_use"] geojson_basepath = params.get("geojson_filepath", None) geojson_suffix = params.get("geojson_suffix", "") if not geojson_basepath: geojson_basepath = s["dir"] fname = geojson_basepath + os.sep + PurePosixPath( s['filename']).stem + geojson_suffix + '.json' if not Path(fname).is_file(): msg = f"Annotation file {fname} does not exist. Skipping." logging.warning(f"{s['filename']} - {msg}") s["warnings"].append(msg) return logging.info(f"{s['filename']} - \tusing {fname}") point_sets = get_points_from_geojson(s, fname) annotationMask = mask_out_annotation(s, point_sets) > 0 io.imsave(s["outdir"] + os.sep + s["filename"] + "_geoJSONMask.png", img_as_ubyte(annotationMask)) prev_mask = s["img_mask_use"] s["img_mask_use"] = prev_mask & annotationMask s.addToPrintList( "geoJSONMask", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After AnnotationModule.geoJSONMask NO tissue remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After AnnotationModule.geoJSONMask NO tissue remains detectable! Downstream modules likely to be incorrect/fail" ) return
def minimumPixelIntensityNeighborhoodFiltering(s, params): logging.info( f"{s['filename']} - \tLightDarkModule.minimumPixelNeighborhoodFiltering" ) disk_size = int(params.get("disk_size", 10000)) threshold = int(params.get("upper_threshold", 200)) img = s.getImgThumb(s["image_work_size"]) img = color.rgb2gray(img) img = (img * 255).astype(np.uint8) selem = disk(disk_size) imgfilt = rank.minimum(img, selem) s["img_mask_bright"] = imgfilt > threshold if strtobool(params.get("invert", "True")): s["img_mask_bright"] = ~s["img_mask_bright"] prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & s["img_mask_bright"] io.imsave(s["outdir"] + os.sep + s["filename"] + "_bright.png", img_as_ubyte(prev_mask & ~s["img_mask_bright"])) s.addToPrintList( "brightestPixels", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After LightDarkModule.minimumPixelNeighborhoodFiltering NO tissue " f"remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After LightDarkModule.minimumPixelNeighborhoodFiltering NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") return
def detectSmoothness(s, params): logging.info(f"{s['filename']} - \tBubbleRegionByRegion.detectSmoothness") thresh = float(params.get("threshold", ".01")) kernel_size = int(params.get("kernel_size", "10")) min_object_size = int(params.get("min_object_size", "100")) img = s.getImgThumb(s["image_work_size"]) img = color.rgb2gray(img) avg = np.ones((kernel_size, kernel_size)) / (kernel_size**2) imf = scipy.signal.convolve2d(img, avg, mode="same") mask_flat = abs(imf - img) < thresh mask_flat = remove_small_objects(mask_flat, min_size=min_object_size) mask_flat = ~remove_small_objects(~mask_flat, min_size=min_object_size) prev_mask = s["img_mask_use"] s["img_mask_flat"] = mask_flat io.imsave(s["outdir"] + os.sep + s["filename"] + "_flat.png", img_as_ubyte(mask_flat & prev_mask)) s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_flat"] s.addToPrintList( "flat_areas", printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After BubbleRegionByRegion.detectSmoothness: NO tissue " f"remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After BubbleRegionByRegion.detectSmoothness: NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") return
def byExampleWithFeatures(s, params): name = params.get("name", "classTask") logging.info( f"{s['filename']} - \tClassificationModule.byExample:\t{name}") thresh = float(params.get("threshold", .5)) nsamples_per_example = float(params.get("nsamples_per_example", -1)) examples = params.get("examples", "") if examples == "": logging.error( f"{s['filename']} - No examples provided in ClassificationModule.byExample for {name} !!" ) sys.exit(1) return if params.get("features", "") == "": logging.error( f"{s['filename']} - No features provided in ClassificationModule.byExample for {name} !!" ) sys.exit(1) return with params[ "lock"]: # this lock is shared across all threads such that only one thread needs to train the model # then it is shared with all other modules if not params["shared_dict"].get("model_" + name, False): logging.info( f"{s['filename']} - Training model ClassificationModule.byExample:{name}" ) model_vals = [] model_labels = np.empty([0, 1]) for ex in params["examples"].splitlines(): ex = re.split( r'(?<!\W[A-Za-z]):(?!\\)', ex) # workaround for windows: don't split on i.e. C:\ img = io.imread(ex[0]) eximg = compute_features(img, params) eximg = eximg.reshape(-1, eximg.shape[2]) mask = io.imread(ex[1], as_gray=True).reshape(-1, 1) if nsamples_per_example != -1: #sub sambling required nitems = nsamples_per_example if nsamples_per_example > 1 else int( mask.shape[0] * nsamples_per_example) idxkeep = np.random.choice(mask.shape[0], size=int(nitems)) eximg = eximg[idxkeep, :] mask = mask[idxkeep] model_vals.append(eximg) model_labels = np.vstack((model_labels, mask)) # do stuff here with model_vals model_vals = np.vstack(model_vals) clf = RandomForestClassifier(n_jobs=-1) clf.fit(model_vals, model_labels.ravel()) params["shared_dict"]["model_" + name] = clf logging.info( f"{s['filename']} - Training model ClassificationModule.byExample:{name}....done" ) clf = params["shared_dict"]["model_" + name] img = s.getImgThumb(s["image_work_size"]) feats = compute_features(img, params) cal = clf.predict_proba(feats.reshape(-1, feats.shape[2])) cal = cal.reshape(img.shape[0], img.shape[1], 2) mask = cal[:, :, 1] > thresh area_thresh = int(params.get("area_threshold", "5")) if area_thresh > 0: mask = remove_small_objects(mask, min_size=area_thresh, in_place=True) dilate_kernel_size = int(params.get("dilate_kernel_size", "0")) if dilate_kernel_size > 0: mask = dilation(mask, selem=np.ones( (dilate_kernel_size, dilate_kernel_size))) mask = s["img_mask_use"] & (mask > 0) io.imsave(s["outdir"] + os.sep + s["filename"] + "_" + name + ".png", img_as_ubyte(mask)) s["img_mask_" + name] = (mask * 255) > 0 prev_mask = s["img_mask_use"] s["img_mask_use"] = s["img_mask_use"] & ~s["img_mask_" + name] s.addToPrintList( name, printMaskHelper(params.get("mask_statistics", s["mask_statistics"]), prev_mask, s["img_mask_use"])) if len(s["img_mask_use"].nonzero() [0]) == 0: # add warning in case the final tissue is empty logging.warning( f"{s['filename']} - After ClassificationModule.byExampleWithFeatures:{name} NO tissue " f"remains detectable! Downstream modules likely to be incorrect/fail" ) s["warnings"].append( f"After ClassificationModule.byExampleWithFeatures:{name} NO tissue remains " f"detectable! Downstream modules likely to be incorrect/fail") s["img_mask_force"].append("img_mask_" + name) s["completed"].append(f"byExampleWithFeatures:{name}") return