def efficiency_analysis(features):
    im = features['im_no_fingers']
    im_peaks = features['im_norm'][features['_peak_row_nums'], :]
    bbs = features['_busbar_cols']

    # make sure no zero values
    im_peaks = im_peaks.copy()
    pixel_ops.ApplyThresholdLT_F32(im_peaks, im_peaks, 0.01, 0.01)
    im = im.copy()
    pixel_ops.ApplyThresholdLT_F32(im, im, 0.01, 0.01)

    # IDEA:
    # two defect mask:
    # 1. existing one (highlights anything dark)
    # 2. do a dark line (local mins). mask out & interpolate.
    #    this one won't pick up grain boundaries
    # - slider blends between them

    #################
    # DEFECT MASK 1 #
    #################
    # - very sensitive to all dark areas

    # need to interpolate values along edges/busbars so defects in these regions can be found
    xs = np.arange(im_peaks.shape[0])
    cols = [features['cell_edge_left'], features['cell_edge_right']]
    cols += list(np.where(features['mask_busbar_edges'][0, :])[0])
    for c in cols:
        ys = im_peaks[:, c].copy()
        ys[ys >
           features['_bright_area_thresh']] = features['_bright_area_thresh']
        params_op = optimize.fmin_powell(line_error, [0.0, ys.mean()],
                                         disp=0,
                                         args=(xs, ys),
                                         xtol=0.05,
                                         ftol=0.05)
        vals = xs * params_op[0] + params_op[1]

        im_peaks[:, c] = vals

        if False:
            print features['_bright_area_thresh']
            mask = np.zeros_like(im, np.uint8)
            mask[:, c] = 1
            ImageViewer(ip.overlay_mask(im, mask))
            plt.figure()
            plt.plot(ys)
            plt.plot(vals)
            plt.show()

    # max of actual val and: interpolate vertical line at:
    # - left & right edge
    # - left & right of each BB

    # make monotonic (don't care about local mins)
    bb_mono = between_bb_mono(im_peaks, bbs)
    background1 = bb_mono
    background1 = cv2.GaussianBlur(background1,
                                   ksize=(0, 0),
                                   sigmaX=3,
                                   borderType=cv2.BORDER_REPLICATE)

    # relative difference
    foreground1 = background1 / im_peaks
    foreground1 -= 1
    pixel_ops.ClipImage(foreground1, 0, 3.0)
    foreground1[:, features['mask_busbar_filled'][0, :]] = 0

    # expand to full size
    full_size = np.zeros(im.shape, np.float32)
    pixel_ops.ExpandFingers(full_size, foreground1, features['_peak_row_nums'])
    foreground1 = full_size
    background_full = np.zeros(im.shape, np.float32)
    pixel_ops.ExpandFingers(background_full, background1,
                            features['_peak_row_nums'])

    if False:
        view = ImageViewer(im)
        ImageViewer(foreground1)
        ImageViewer(background1)
        view.show()

    #################
    # DEFECT MASK 2 #
    #################
    # - less likely to include dark grains
    # - mask out local mins and areas of high gradient, and the interpolate background
    flatten_cols = np.r_[im.mean(axis=0)]
    im_flattened = im - flatten_cols
    flatten_rows = np.c_[im_flattened.mean(axis=1)]
    im_flattened -= flatten_rows
    im_flattened[features['mask_busbar_filled']] = 0
    im_smoothed = cv2.GaussianBlur(im_flattened, ksize=(0, 0), sigmaX=2)

    # local mins
    dark_lines = np.zeros_like(im_flattened, np.uint8)
    pixel_ops.LocalMins(im_smoothed, dark_lines)
    s = ndimage.generate_binary_structure(2, 1)
    dark_lines = ndimage.binary_dilation(dark_lines, structure=s)

    # high gradient
    edgesH = cv2.Sobel(im_smoothed, cv2.CV_32F, 1, 0)
    edgesV = cv2.Sobel(im_smoothed, cv2.CV_32F, 0, 1)
    edges = cv2.magnitude(edgesH, edgesV)

    # combine
    defect_candidates = ((edges > 0.2) | dark_lines)

    if False:
        view = ImageViewer(im_flattened)
        ImageViewer(ip.overlay_mask(im_flattened, dark_lines))
        ImageViewer(ip.overlay_mask(im_flattened, edges > 0.2))
        ImageViewer(ip.overlay_mask(im_flattened, defect_candidates))
        view.show()
        sys.exit()

    background2 = interpolate_background(im_flattened,
                                         defect_candidates.astype(np.uint8))
    background2 += flatten_rows
    background2 += flatten_cols

    # relative difference
    foreground2 = background2 / im
    foreground2 -= 1
    pixel_ops.ClipImage(foreground2, 0, 3.0)
    foreground2[features['mask_busbar_filled']] = 0

    if False:
        view = ImageViewer(im)
        ImageViewer(background2)
        ImageViewer(foreground2)
        view.show()
        sys.exit()

    if False:
        features['_defect1'] = foreground1
        features['_defect2'] = foreground2

    # discrimination function
    x1, y1 = 0.0, 0.5
    x2, y2 = 0.4, 0.0
    foreground = (-1 * ((y2 - y1) * foreground2 -
                        (x2 - x1) * foreground1 + x2 * y1 - y2 * x1) /
                  np.sqrt((y2 - y1)**2 + (x2 - x1)**2))

    foreground += 0.1 + parameters.CELL_DISLOCATION_SENSITIVITY
    foreground *= 0.25
    pixel_ops.ClipImage(foreground, 0, 1)

    features['ov_dislocations_u8'] = (foreground * 255).astype(np.uint8)

    if False:
        view = ImageViewer(im)
        ImageViewer(foreground1)
        ImageViewer(foreground2)
        ImageViewer(foreground)
        #ImageViewer(background_full)
        # ImageViewer(foreground > 0.2)
        view.show()
        sys.exit()

    ################
    # IMPURE AREAS #
    ################

    def FindImpure(imp_profile, imp_profile_r, defects, debug=False):
        if False:
            ImageViewer(im)
            plt.figure()
            plt.plot(imp_profile)
            plt.plot(defects)
            plt.show()

        imp_ratio = (imp_profile / imp_profile_r).astype(np.float32)
        imp_ratio[imp_ratio > 0.9] = 0.9
        imp_ratio /= 0.9
        global_min = np.argmin(imp_ratio)
        local_mins = np.where((imp_ratio < np.roll(imp_ratio, 1))
                              & (imp_ratio < np.roll(imp_ratio, -1)))[0]

        # make monotonic from global minimum
        imp_mono = imp_ratio.copy()
        flip = False
        if global_min > len(imp_mono) // 2:
            flip = True
            imp_mono = imp_mono[::-1]
            global_min = np.argmin(imp_mono)
        rest = np.ascontiguousarray(imp_mono[global_min:])
        rest_mono = np.empty_like(rest)
        pixel_ops.MakeMonotonic(rest, rest_mono)
        imp_mono[global_min:] = rest_mono
        # if edge_dist < 0.075:
        #     imp_mono[:global_min] = imp_ratio[global_min]
        if flip:
            imp_mono = imp_mono[::-1]
            global_min = np.argmin(imp_mono)

        # for a real impure area, the global minimum should be close to an edge
        edge_dist = min(global_min,
                        len(imp_ratio) - global_min) / float(len(imp_ratio))
        edge_spot = int(0.03 * len(imp_mono))
        imp_at_edge = min(imp_mono[edge_spot], imp_mono[-edge_spot])
        imp_1d = np.ones_like(imp_mono)

        # check the defect content in the "impure" area
        if (imp_mono < 1.0).sum() == 0:
            defect_strength = 0
        else:
            # defect_strength = defects[imp_mono < 1.0].mean()
            defect_strength = np.median(defects[imp_mono < 1.0])

        if (edge_dist < 0.075 and  # lowest point close to edge
                imp_at_edge < 0.8 and  # edge is dark
                defect_strength < 0.1 and  # not too many defects
                # (global_min == local_mins[0] or global_min == local_mins[-1]) and # no local mins closer to edge
            (imp_mono - imp_ratio).mean() < 0.035):  # no large non-impure dips
            imp_1d = imp_mono.copy()
            imp_1d -= 0.3
            if global_min < len(imp_profile) // 2:
                imp_1d[:global_min] = imp_1d[global_min]
            else:
                imp_1d[global_min:] = imp_1d[global_min]

        if debug:
            print edge_dist, edge_dist < 0.075
            print imp_at_edge, imp_at_edge < 0.8
            print defect_strength, defect_strength < 0.1
            # print global_min, local_mins[0], (global_min == local_mins[0] or global_min == local_mins[-1])
            print(imp_mono -
                  imp_ratio).mean(), (imp_mono - imp_ratio).mean() < 0.035
            ImageViewer(im)
            plt.figure()
            plt.plot(imp_profile, 'r')
            plt.plot(imp_mono, 'g')
            plt.plot(imp_ratio, 'b')
            # plt.plot((0, len(signal_var)), (0.1, 0.1))
            plt.plot(defects, 'c')
            plt.plot(imp_1d, 'c')

            plt.show()

        # find impure edge width
        THRESH = 0.5
        if imp_1d[0] < THRESH:
            edge_width = np.where(imp_1d < THRESH)[0][-1] / float(
                len(imp_profile))
            left = True
        elif imp_1d[-1] < THRESH:
            edge_width = (len(imp_profile) -
                          np.where(imp_1d < THRESH)[0][0]) / float(
                              len(imp_profile))
            left = False
        else:
            edge_width = 0
            left = True

        return imp_1d, edge_width, left

    if False:
        # ignore defect areas
        defect_mask = foreground > 0.2
        mx = ma.masked_array(im, mask=defect_mask)

        if False:
            view = ImageViewer(im)
            ImageViewer(defect_mask)
            view.show()
    '''
    # left/right edges
    cols = np.apply_along_axis(stats.scoreatpercentile, 0, im, per=75)
    cols[cols > 1.0] = 1.0
    imp_profile = ndimage.gaussian_filter1d(cols, sigma=3, mode="nearest")
    imp_profile_r = imp_profile[::-1]
    if len(bbs) >= 2:
        mid = (bbs[0] + bbs[-1]) // 2
    else:
        mid = im.shape[1] // 2
    imp_profile_r = np.roll(imp_profile_r, mid - (len(imp_profile) // 2))
    col_defect = foreground.mean(axis=0)
    imp_h, ew_h, e_left = FindImpure(imp_profile, imp_profile_r, col_defect, debug=False)
    '''

    # top/bottom edges
    rows = np.apply_along_axis(stats.scoreatpercentile, 1, im, per=75)
    rows[rows > 1.0] = 1.0
    imp_profile = ndimage.gaussian_filter1d(rows, sigma=3, mode="nearest")
    imp_profile_r = imp_profile[::-1]
    row_defect = foreground.mean(axis=1)
    imp_v, ew_v, e_top = FindImpure(imp_profile,
                                    imp_profile_r,
                                    row_defect,
                                    debug=False)

    features['impure_edge_width'] = ew_v
    if e_top:
        features['impure_edge_side'] = 0
    else:
        features['impure_edge_side'] = 2

    impure = np.ones_like(im)
    impure[:, :] = np.c_[imp_v]

    if False:
        print features['impure_edge_width'], features['impure_edge_side']
        view = ImageViewer(impure)
        view.show()
        sys.exit()

    imp_cutoff = 0.55
    pixel_ops.ApplyThresholdGT_F32(impure, impure, imp_cutoff, imp_cutoff)
    impure /= imp_cutoff

    impure_overlay = np.log10(2 - impure)

    if False:
        plt.figure()
        plt.plot(impure[:, 0], 'r')
        plt.plot(np.log(impure[:, 0] + 1), 'g')
        plt.plot(np.log10(impure[:, 0] + 1), 'b')
        plt.show()

        view = ImageViewer(impure)
        view.show()

    pixel_ops.ClipImage(impure_overlay, 0, 1)
    features['ov_impure2_u8'] = (impure_overlay * 255).astype(np.uint8)

    ###########
    # METRICS #
    ###########
    num_pixels = im.shape[0] * im.shape[1]

    # impure
    impure_thresh = 0.9  # 0.8
    dislocation_thresh = 0.1
    num_impure_pixels = pixel_ops.CountThresholdLT_F32(impure, impure_thresh)
    features['impure_area_fraction'] = (num_impure_pixels / float(num_pixels))

    if features['impure_area_fraction'] > 0.001:
        pure_mask = (impure > impure_thresh) & (foreground <
                                                dislocation_thresh)
        features['impure_strength'] = (im[pure_mask].mean() /
                                       im[impure < impure_thresh].mean()) - 1
    else:
        features['impure_strength'] = 0
        features['impure_area_fraction'] = 0

    features['impure_strength2'] = features['impure_strength'] * features[
        'impure_area_fraction'] * 100

    # dislocations
    num_dislocations = pixel_ops.CountThresholdGT_F32(foreground,
                                                      dislocation_thresh)
    features['dislocation_area_fraction'] = (num_dislocations /
                                             float(num_pixels))
    features['_foreground'] = foreground
    features['_dislocation_thresh'] = dislocation_thresh
    features['_impure'] = impure
    features['_impure_thresh'] = impure_thresh

    # find the average distance between dislocation pixels
    ys, xs = np.where(foreground > dislocation_thresh)
    points = np.c_[ys, xs]
    num_samples = 1500
    if len(ys) > 0:
        points = points[::max(1, points.shape[0] // num_samples), :]
        pixel_dists = distance.pdist(points)
        avg_dist = np.mean(pixel_dists)
        avg_dist = (avg_dist - 0.3 * im.shape[0]) / (0.5 * im.shape[0])
    else:
        avg_dist = 0
    features['dislocation_density'] = 1.0 - avg_dist

    if len(ys) > 0:
        dis_vals = foreground[ys, xs]
        dislocation_strength = math.sqrt((dis_vals**2).mean())
        features['dislocation_severity_A'] = dislocation_strength

        # create a second defect strength.
        ratio2 = background_full / im
        ratio2 = np.clip(ratio2, 1, 5)
        ratio2[:, features['mask_busbar_filled'][0, :]] = 1
        features['dislocation_severity_B'] = ratio2[ys, xs].mean()

        if False:
            view = ImageViewer(background_full)
            ImageViewer(im)
            ImageViewer(ratio2)
            view.show()
    else:
        features['dislocation_severity_A'] = 0
        features['dislocation_severity_B'] = 0

    if False:
        view = ImageViewer(im)
        foreground[foreground < 0] = 0
        ImageViewer(foreground)
        ImageViewer(background_full)
        view.show()
        sys.exit()

    # dislocation histogram features
    foreground_bins = np.zeros((5), np.float32)
    pixel_ops.BackgrounHistogram(foreground, foreground_bins)
    foreground_bins = (foreground_bins / num_pixels) * 100
    features['dislocation_hist_01'] = foreground_bins[0]
    features['dislocation_hist_02'] = foreground_bins[1]
    features['dislocation_hist_03'] = foreground_bins[2]
    features['dislocation_hist_04'] = foreground_bins[3]
    features['dislocation_hist_05'] = foreground_bins[4]
    features['dislocation_strength'] = features[
        'dislocation_area_fraction'] * features['dislocation_severity_A']

    if False:
        # print features['foreground_hist_01'], features['foreground_hist_02'], features['foreground_hist_03'],
        # print features['foreground_hist_04'], features['foreground_hist_05']
        view = ImageViewer(im)
        ImageViewer(foreground)
        ImageViewer(impure)
        ImageViewer(create_overlay(features))
        view.show()
def firing_defects(features):
    im = features['im_norm']
    finger_rows = features['_finger_row_nums']
    # im_fingers = features['_finger_im']
    im_smooth = cv2.GaussianBlur(im, ksize=(0, 0), sigmaX=1)
    im_fingers = im_smooth[finger_rows, :]

    if False:
        view = ImageViewer(im)
        # view = ImageViewer(im_smooth)
        ImageViewer(im_fingers)
        view.show()
        sys.exit()

    # find depth of local minimums
    S = 5
    dips = np.minimum(np.roll(im_fingers, S, axis=1), np.roll(im_fingers, -S, axis=1)) - im_fingers
    pixel_ops.ApplyThresholdLT_F32(dips, dips, 0, 0)
    dips[:, features['mask_busbar_filled'][0, :]] = 0
    dips[:, :features['cell_edge_left']] = 0
    dips[:, features['cell_edge_right']:] = 0

    # TODO: add upper bound as well
    locs = ((im_fingers < np.roll(im_fingers, 1, axis=1)) &
            (im_fingers < np.roll(im_fingers, -1, axis=1)) &
            (dips > 0.02) & (dips < 0.05)).astype(np.float32)

    # count num dips in a region. ignore areas with only 1
    w = im.shape[1] // 10
    weights = np.ones(w, np.int32)
    local_count = ndimage.convolve1d(locs, weights, axis=1, mode="constant")
    dips_filtered = dips.copy()
    dips_filtered[local_count <= 1] = 0

    if False:
        view = ImageViewer(im)
        ImageViewer(dips)
        # ImageViewer(locs)
        # ImageViewer(local_count)
        # ImageViewer(dips_filtered)
        view.show()

    im_dots = cv2.GaussianBlur(dips_filtered, ksize=(0, 0), sigmaX=10, sigmaY=2)
    # im_dots -= 0.001

    if False:
        view = ImageViewer(dips_filtered)
        view = ImageViewer(im_dots)
        view.show()

    splotch = np.empty_like(im)
    pixel_ops.ExpandFingers(splotch, im_dots, finger_rows)
    splotch = ip.fast_smooth(splotch, sigma=10)
    features["_firing"] = splotch.copy()
    splotch *= 100.0 * (1.0 + parameters.FIRING_SENSITIVITY)
    pixel_ops.ClipImage(splotch, 0.0, 1.0)
    pixel_ops.ApplyThresholdGT_F32(features['im_center_dist_im'],
                                   splotch, features['wafer_radius'], 0)
    features["ov_splotches_u8"] = (splotch * 255).astype(np.uint8)

    # metrics
    if splotch.max() >= 0.2:
        features['firing_area_strength'] = splotch.mean() * 100
        mask_firing = (splotch > 0.2).astype(np.uint8)
        im_pl = features['_cropped_f32']
        firingPL = pixel_ops.MaskMean_F32(im_pl, mask_firing, 1)
        goodPL = pixel_ops.MaskMean_F32(im_pl, mask_firing, 0)
        features['firing_area_mean_PL'] = firingPL
        features['firing_area_fraction'] = mask_firing.mean()
        features['firing_area_PL_intensity_ratio'] = firingPL / max(1, goodPL)
    else:
        features['firing_area_strength'] = 0.0
        features['firing_area_mean_PL'] = 0.0
        features['firing_area_fraction'] = 0.0
        features['firing_area_PL_intensity_ratio'] = 0.0

    if False:
        print features['firing_area_strength']
        view = ImageViewer(im)
        ImageViewer(splotch, vmin=0, vmax=1)
        ImageViewer(mask_firing)
        f2 = {'im_cropped_u8': features['im_cropped_u8'],
              'ov_splotches_u8': features['ov_splotches_u8']}
        ImageViewer(create_overlay(f2))
        view.show()
        sys.exit()
def finger_shape(features):
    if 'DEBUG' in features:
        DEBUG = features['DEBUG']
    else:
        DEBUG = False

    # use an image that has been normalised to [0, 1]
    im = features['_cropped_f32'] / features['hist_percentile_99.9']

    if parameters.CELL_BB_MID_POINTS:
        locs = np.round((features['_busbar_cols'][:-1] + features['_busbar_cols'][1:]) / 2.0).astype(np.int32)
        pixel_ops.InterpolateBBs(im, locs, 3)

    im_finger = im[features['_peak_row_nums']]
    # firing = np.zeros_like(im_finger)

    if False:
        view = ImageViewer(im)
        ImageViewer(im_finger)
        plt.figure()
        plt.plot(im_finger.mean(axis=0))
        view.show()
        sys.exit()

    TRAINING_MODE = False
    if TRAINING_MODE:
        import os
        fn = "finger_shape.csv"
        # bb_locs = features['_busbar_cols']
        bb_locs = np.r_[0, features['_busbar_cols'], im.shape[1] - 1]
        with open(fn, 'a') as f:
            def on_click(event):
                tb = plt.get_current_fig_manager().toolbar
                if event.xdata is None: return
                if tb.mode != '':
                    print 'Not in click mode - turn of pan or zoom'
                    return

                if event.button == 1:
                    classification = "good"
                elif event.button == 3:
                    classification = "bad"
                else:
                    return

                y = round(event.ydata)
                x = int(round(event.xdata))
                i = np.searchsorted(bb_locs, x)

                left, right = bb_locs[i - 1], bb_locs[i]
                assert left < x < right
                vals = im_finger[y, left:right]

                if x < im_finger.shape[1] // 2:
                    vals = vals[::-1]

                plt.figure()
                plt.plot(vals)
                plt.show()

                valstr = ','.join(["%0.02f" % (v) for v in vals])

                f.write("%s,%s\n" % (classification, valstr))

            fig = plt.figure()
            fig.canvas.mpl_connect('button_press_event', on_click)
            plt.imshow(im_finger, cmap=plt.cm.gray, interpolation='nearest')
            plt.show()
        return

    if len(features['_busbar_cols']) > 1:
        bb_locs = features['_busbar_cols']
    else:
        # only 1 busbar, so add left and right edges
        bb_locs = np.r_[0, features['_busbar_cols'], im.shape[1] - 1]

    S = 15
    # analyse each finger independently
    peak_broken = []
    peak_fine = []
    finger_rs = []
    finger_maes = []

    if False:
        locs = []
        for bb in range(len(bb_locs) - 1):
            locs.append((bb_locs[bb] + bb_locs[bb + 1]) // 2)
        im_finger2 = im_finger.copy()
        pixel_ops.InterpolateBBs(im_finger2, np.array(locs, np.int32), 4)
        view = ImageViewer(im_finger)
        ImageViewer(im_finger2)
        view.show()
        im_finger = im_finger2

    for bb in range(len(bb_locs) - 1):
        segment = im_finger[:, bb_locs[bb] + S:bb_locs[bb + 1] - S]
        if segment.shape[1] == 0:
            continue
        xs = np.linspace(-1.0, 1.0, num=segment.shape[1])
        for y in range(5, segment.shape[0] - 5):
            # fit a quadratic curve
            bbb = segment[y, :]
            params = np.polyfit(xs, bbb, 2)

            f = np.poly1d(params)
            ys = f(xs)

            # calculate the mean absolute error between the actual pixel values and the fitted parabola
            mae = np.abs(ys - bbb).mean() / bbb.mean()
            sigmoid = expit((mae - 0.02) / 0.001)

            # save curevature & goodness of fit
            finger_rs.append(params[0] * -1)
            finger_maes.append(mae)

            if sigmoid > 0.7:
                peak_broken.append(bbb.max())
            elif sigmoid < 0.3:
                peak_fine.append(bbb.max())

            # if True and bb == 0 and y == 5:
            if False and sigmoid > 0.7:
                print mae, sigmoid
                print bb, y
                im_fin = im_finger.copy()
                im_fin[y - 2, bb_locs[bb] + S:bb_locs[bb + 1] - S] = 0
                im_fin[y + 2, bb_locs[bb] + S:bb_locs[bb + 1] - S] = 0
                ImageViewer(im_fin)
                plt.figure()
                plt.plot(xs, bbb, label="x-profile")
                plt.plot(xs, ys, label="Fitted parabola")
                plt.legend()
                plt.show()

    if len(finger_rs) > 0:
        features['resistance_finger'] = np.median(finger_rs)
        features['resistance_finger_error'] = np.median(finger_maes)
    else:
        features['resistance_finger'] = 0
        features['resistance_finger_error'] = 0

    if len(peak_broken) > 0 or len(peak_fine) > 0:
        range_min = np.array(peak_broken + peak_fine).min()
        range_max = np.array(peak_broken + peak_fine).max()
        range_vals = np.linspace(range_min, range_max, num=100)

        peak_broken = np.array(peak_broken)
        peak_fine = np.array(peak_fine)
        broken_percent = peak_broken.shape[0] * 100 / float(peak_broken.shape[0] + peak_fine.shape[0])
        features['fingers_non_para'] = broken_percent

        # compute distribution of peak vals of bad fingers
        bad_fingers_dist = None
        bad_mode = None
        if len(peak_broken) > 5:
            f_bad_fingers = stats.gaussian_kde(peak_broken)
            bad_fingers_dist = f_bad_fingers(range_vals)
            bad_maxs = np.where((bad_fingers_dist > np.roll(bad_fingers_dist, 1)) &
                                (bad_fingers_dist > np.roll(bad_fingers_dist, -1)) &
                                (bad_fingers_dist > 2.0))[0]
            if len(bad_maxs) > 0:
                bad_mode = range_vals[bad_maxs[np.argmax(bad_fingers_dist[bad_maxs])]]

        # compute distribution of peak vals of good fingers
        good_fingers_dist = None
        good_maxs = []
        good_mode, good_mode_i = None, None
        if len(peak_fine) > 5:
            f_good_fingers = stats.gaussian_kde(peak_fine)
            good_fingers_dist = f_good_fingers(range_vals)
            good_maxs = np.where((good_fingers_dist > np.roll(good_fingers_dist, 1)) &
                                 (good_fingers_dist > np.roll(good_fingers_dist, -1)) &
                                 (good_fingers_dist > 2))[0]

            if len(good_maxs) > 0:
                good_mode_i = good_maxs[np.argmax(good_fingers_dist[good_maxs])]
                good_mode = range_vals[good_mode_i]

        if False:
            ImageViewer(im_finger)
            plt.figure()
            if good_fingers_dist is not None:
                plt.plot(range_vals, good_fingers_dist, 'g')
            if bad_fingers_dist is not None:
                plt.plot(range_vals, bad_fingers_dist, 'b')
            plt.show()
            sys.exit()

        if broken_percent >= 70:
            if DEBUG:
                print "0"
            # lots of broken, use fixed threshold
            threshold = 0.7
        elif broken_percent >= 33 and (good_mode is None or bad_mode < good_mode):
            if DEBUG:
                print "1"
            # significant broken and "non-broken" fingers are brighter than broken ones
            threshold = 0.7
        elif len(good_maxs) >= 2:
            # 2+ peaks in good dist.
            # - perhaps some good fingers are being classified as bad
            if broken_percent < 30 and (bad_mode is None or good_mode < bad_mode):
                if DEBUG:
                    print "2"
                # mostly good and broken fingers brighter
                # - use the highest peak
                i = good_mode_i
                while True:
                    if (good_fingers_dist[i] < 0.5 or i == good_fingers_dist.shape[0] - 1 or
                            (good_fingers_dist[i] < good_fingers_dist[i + 1] and
                                     good_fingers_dist[i] < 3)): break
                    i += 1
                threshold = range_vals[i]
            elif broken_percent < 20 and bad_mode is not None and bad_mode < good_mode:
                if DEBUG:
                    print "2B"
                # mostly good but broken fingers darker
                # - use the highest peak
                threshold = (range_vals[good_maxs[-2]] + range_vals[good_maxs[-1]]) / 2.0
            elif bad_mode is not None:
                if DEBUG:
                    print "3"
                # quite a few bad, or broken fingers darker
                # - use first peak
                i = good_maxs[0]
                while True:
                    if (good_fingers_dist[i] < 0.5 or i == good_fingers_dist.shape[0] - 1 or
                            (good_fingers_dist[i] < good_fingers_dist[i + 1] and
                                     good_fingers_dist[i] < 3)): break
                    i += 1
                threshold = min(range_vals[i], bad_mode - 0.1)
            else:
                threshold = good_mode
        elif (broken_percent <= 33 >= 1 and (bad_mode is None or good_mode < bad_mode) or
                      broken_percent < 10) and len(good_maxs):
            # - majority good fingers, single peak & broken fingers (if they exist) are brighter
            # - base the treshold on the distribution of good finger peaks
            if DEBUG:
                print "4"

            # find main mode in good dist
            i = good_maxs[0]
            while True:
                if (good_fingers_dist[i] < 0.5 or
                            i == good_fingers_dist.shape[0] - 1 or
                        (good_fingers_dist[i] < good_fingers_dist[i + 1] and
                                 good_fingers_dist[i] < 3)):
                    break
                i += 1

            threshold = min(range_vals[good_maxs[-1]] + 0.2, range_vals[i])
        else:
            if broken_percent > 33:
                # roughly 50/50. broken fingers are brighter
                # - use middle between good & bad
                if DEBUG:
                    print "5"
                threshold = (good_mode + bad_mode) / 2.0
            else:
                if DEBUG:
                    print "6"
                # majority good. assume the "bad" ones aren't actually bad
                if good_mode_i is not None:
                    i = good_mode_i
                else:
                    i = np.argmax(good_fingers_dist)
                while True:
                    if (good_fingers_dist[i] < 0.5 or
                                i == good_fingers_dist.shape[0] - 1 or
                            (good_fingers_dist[i] < good_fingers_dist[i + 1] and
                                     good_fingers_dist[i] < 3)):
                        break
                    i += 1

                threshold = range_vals[i]
    else:
        threshold = 1.0

    threshold -= parameters.BRIGHT_AREA_SENSITIVITY

    if False:
        print "Percent broken: ", broken_percent
        print "Threshold:", threshold

        view = ImageViewer(im)
        plt.figure()
        m1, m2 = 0, 0
        if good_fingers_dist is not None:
            plt.plot(range_vals, good_fingers_dist, 'g', label="Not broken")
            m1 = good_fingers_dist.max()
        if bad_fingers_dist is not None:
            plt.plot(range_vals, bad_fingers_dist, 'r', label="Broken")
            m2 = bad_fingers_dist.max()
        plt.vlines(threshold, 0, max(m1, m2))
        plt.legend()
        view.show()

    # highlight areas brighter than threshold
    # features['bright_line_threshold'] = threshold
    if threshold < 1.0:
        bright_lines = (im[features['_peak_row_nums']] - threshold) / (1.0 - threshold)
    else:
        bright_lines = np.zeros_like(im_finger)
    bright_lines *= 0.5
    pixel_ops.ClipImage(bright_lines, 0, 1)

    # don't want to highlight single lines, so apply vertical median filter
    rows_filtered = np.zeros_like(bright_lines)
    pixel_ops.FilterV(bright_lines, rows_filtered)
    rows_filtered[:2, :] = bright_lines[:2, :]
    rows_filtered[-2:, :] = bright_lines[-2:, :]

    if False:
        view = ImageViewer(im_finger)
        ImageViewer(bright_lines)
        ImageViewer(rows_filtered)
        view.show()
        sys.exit()

    bright_lines = rows_filtered

    # create a full size image of bright areas
    bright_area_full = np.zeros_like(im)
    pixel_ops.ExpandFingers(bright_area_full, bright_lines, features['_peak_row_nums'])
    bright_area_full = cv2.GaussianBlur(bright_area_full, sigmaX=3, ksize=(0, 0))
    bright_u8 = (bright_area_full * 255).astype(np.uint8)
    if 'ov_bright_area_u8' in features:
        assert False
        # features['ov_bright_area_u8'] = np.maximum(bright_u8, features['ov_bright_area_u8'])
    else:
        features['ov_bright_area_u8'] = bright_u8
    features['bright_area_strength'] = bright_lines.mean() * 100

    # bright area metrics
    mask_bright = (bright_area_full > 0.1).astype(np.uint8)
    im_pl = features['_cropped_f32']
    brightPL = pixel_ops.MaskMean_F32(im_pl, mask_bright, 1)
    darkPL = pixel_ops.MaskMean_F32(im_pl, mask_bright, 0)
    features['bright_area_mean_PL'] = brightPL
    features['bright_area_fraction'] = mask_bright.mean()
    features['bright_area_PL_intensity_ratio'] = brightPL / max(1, darkPL)

    # firing problems
    # firing_full = np.zeros_like(im)
    # pixel_ops.ExpandFingers(firing_full, firing, features['_peak_row_nums'])


    if False:
        view = ImageViewer(im_finger)
        ImageViewer(bright_area_full)
        # view = ImageViewer(firing_full)
        ImageViewer(features['ov_bright_area_u8'])
        view.show()
        sys.exit()
def dark_areas(features):
    # dark areas (not dark spots)
    im = features['im_norm']
    h, w = im.shape
    row_nums = features['_peak_row_nums']
    cell_mask = features['bl_cropped_u8'][row_nums, :]
    bb_locs = features['_busbar_cols']

    # fill edges & corners
    foreground = im[row_nums].copy()
    edge = features['cell_edge_tb']
    rr, cc = draw.circle_perimeter(features['wafer_middle_y'], features['wafer_middle_x'],
                                   int(round(features['wafer_radius'])) - edge)
    mask = (cc >= 0) & (cc < w) & np.in1d(rr, row_nums)
    lut = np.zeros(h, np.int32)
    lut[row_nums] = np.arange(len(row_nums))
    rr = rr[mask]
    cc = cc[mask]
    rr = np.take(lut, rr)
    pixel_ops.FillCorners(foreground, rr.astype(np.int32), cc.astype(np.int32))
    foreground[:, :edge] = np.c_[foreground[:, edge]]
    foreground[:, -edge:] = np.c_[foreground[:, -edge]]

    # create background
    # 1. make monotonic between edges & BBs
    mono_lr = np.empty_like(foreground)
    mono_rl = np.empty_like(foreground)
    pixel_ops.MakeMonotonicBBs(foreground, mono_lr, bb_locs)
    pixel_ops.MakeMonotonicBBs(np.ascontiguousarray(foreground[:, ::-1]), mono_rl,
                               np.ascontiguousarray(w - bb_locs[::-1]))
    mono_rl = mono_rl[:, ::-1]
    mono = np.minimum(mono_lr, mono_rl)
    background = mono
    da1 = background - foreground

    # fill BBs and flatten
    background2 = background.copy()
    pixel_ops.InterpolateBBs(background2, features['_busbar_cols'], features['busbar_width'])
    cols = background2.mean(axis=0)
    background2 -= np.r_[cols]
    rows = background2.mean(axis=1)
    background2 -= np.c_[rows]

    da2 = -1 * background2
    pixel_ops.ApplyThresholdLT_F32(da2, da2, 0.0, 0.0)

    if False:
        view = ImageViewer(foreground)
        ImageViewer(background)
        ImageViewer(background2)
        ImageViewer(da1)
        ImageViewer(da2)
        ImageViewer(da1 + da2)
        # plt.figure()
        # for x in range(0, background.shape[1], 50):
        #    plt.plot(background[:, x], label="Col %d"%(x))
        # plt.legend()
        view.show()
        sys.exit()

    dark_areas = da1 + da2
    dark_areas -= (0.1 - parameters.DARK_AREA_SENSITIVITY)
    dark_areas[(cell_mask == 1) | (cell_mask == 8)] = 0  # corners
    pixel_ops.ClipImage(dark_areas, 0.0, 1.0)
    dark_areas_full = np.empty_like(im)
    pixel_ops.ExpandFingers(dark_areas_full, dark_areas, features['_peak_row_nums'])

    if False:
        pixel_ops.ApplyThresholdGT_F32(features['im_center_dist_im'], dark_areas_full,
                                       features['wafer_radius'], 0)
        print features['wafer_radius']
        view = ImageViewer(dark_areas)
        ImageViewer(dark_areas_full)
        ImageViewer(features['im_center_dist_im'])
        view.show()

    dark_areas_full = cv2.GaussianBlur(dark_areas_full, ksize=(0, 0), sigmaX=1)

    # metrics
    mask_dark = (dark_areas_full > 0.2).astype(np.uint8)
    features['dark_area_strength'] = dark_areas_full.mean() * 100
    im_pl = features['_cropped_f32']
    darkPL = pixel_ops.MaskMean_F32(im_pl, mask_dark, 1)
    brightPL = pixel_ops.MaskMean_F32(im_pl, mask_dark, 0)
    features['dark_area_mean_PL'] = darkPL
    features['dark_area_fraction'] = mask_dark.mean()
    features['dark_area_PL_intensity_ratio'] = brightPL / max(1, darkPL)

    features['ov_dark_areas_u8'] = (dark_areas_full * 255).astype(np.uint8)

    if False:
        print features['dark_area_fraction'], features['dark_area_strength']
        # plt.figure()
        # plt.plot(cols)
        # plt.plot(cols_mono)
        view = ImageViewer(im)
        ImageViewer(foreground)
        ImageViewer(dark_areas)
        ImageViewer(mask_dark)
        ImageViewer(dark_areas_full)
        view.show()