Exemple #1
0
def predict_patient(input_dir, patient_id, image_dims, model, output_dir):
    logger.info('>>> Predict patient_id ' + patient_id)
    logger.info('Loading pre-processed images for patient')

    #patient pre-processed image cache
    dataset_file = utils.dataset_path(output_dir, 'cache-predict', image_dims)    
    patient_pixels = None
    with h5py.File(dataset_file, 'a') as h5f:
        try:
            patient_pixels = h5f[patient_id]
            logger.debug('Patient image found in cache. Using it.')
            #disconnect from HDF5
            patient_pixels = np.array(patient_pixels)
            
        except KeyError:
            logger.debug('Patient image not found in cache')
            t = Timer('Preparing patient scan image volume. patient_id=' + patient_id)
            patient_pixels = lungprepare.process_patient_images(input_dir + patient_id, image_dims)
            if(patient_pixels is None):
                logger.warning('Patient lung not found. Skipping.')
            logger.debug('Storing patient image in cache')
            h5f[patient_id] = patient_pixels
            t.stop()
    
    t = Timer('Predicting result on CNN (forward)')
    y = model.predict(np.expand_dims(patient_pixels, axis=0))
    logger.info('PATIENT '+ patient_id +' PREDICT=' + str(y))
    utils.show_slices(patient_pixels, patient_id)
    t.stop()
    
    return y
Exemple #2
0
 def process_shard(self, shard_id, process_shard_function, ramp_delay):
     try:
         sleep(randint(ramp_delay[0], ramp_delay[1]))
         return process_shard_function(self, shard_id)
     except BaseException as e:
         details = traceback.format_exc()
         if (details == None):
             details = 'None'
         logger.warning('Exception while processing shard ' +
                        str(shard_id) + ': ' + str(e) + '; details=' +
                        details)
         return 'shard ' + str(shard_id) + ' exception: ' + str(e)
Exemple #3
0
def discover_lung_rotation(lung_mask):
    bbox = bounding_box(lung_mask)
    if (bbox == None): return 0
    slicen = int((bbox[1][2] - bbox[0][2]) / 2)
    half = int(bbox[0][0] + (bbox[1][0] - bbox[0][0]) / 2)

    l1 = find_minmax_halfx(lung_mask,
                           half,
                           bottom2up=True,
                           left2right=True,
                           slicen=slicen)
    r1 = find_minmax_halfx(lung_mask,
                           half,
                           bottom2up=True,
                           left2right=False,
                           slicen=slicen)
    l2 = find_minmax_halfx(lung_mask,
                           half,
                           bottom2up=False,
                           left2right=True,
                           slicen=slicen)
    r2 = find_minmax_halfx(lung_mask,
                           half,
                           bottom2up=False,
                           left2right=False,
                           slicen=slicen)

    r = (l1, r1, l2, r2)
    xs, ys = zip(*r)

    #verify points sanity
    if (not value_between(xs[1] - xs[0], 50, 200)
            or not value_between(xs[3] - xs[2], 50, 200)
            or not value_between(ys[0] - ys[2], 100, 250)
            or not value_between(ys[1] - ys[3], 100, 250)):
        logger.warning('Strange rotation detected. returning 0 degrees')
        return 0

    angle1 = calculate_angle(l1, r1)
    angle2 = calculate_angle(l2, r2)

    a = ((angle1 + angle2) / 2)
    return min(max(a, -10), 10)
Exemple #4
0
def validate_xy_dataset(dataset_file, save_dir=None):
    ok = True
    logger.info('VALIDATING DATASET ' + dataset_file)

    with h5py.File(dataset_file, 'r') as h5f:
        x_ds = h5f['X']
        y_ds = h5f['Y']

        if (len(x_ds) != len(y_ds)):
            logger.warning(
                'VALIDATION ERROR: x and y datasets with different lengths')
            ok = False

        for px in range(len(x_ds)):
            arr = np.array(x_ds[px])
            if (not np.any(arr)):
                logger.warning('VALIDATION ERROR: Image not found at index=' +
                               str(px))
                ok = False

        label_total = np.array([[0, 0]])
        for py in range(len(y_ds)):
            arr = np.array(y_ds[py])
            label_total = arr + label_total
            if (not np.any(arr) or np.all(arr) or arr[0] == arr[1]):
                logger.warning(
                    'VALIDATION ERROR: Invalid label found at index=' +
                    str(py) + ' label=' + str(arr))
                ok = False

        label0_ratio = label_total[0][0] / len(y_ds)
        label1_ratio = label_total[0][1] / len(y_ds)

        logger.info('Summary')
        logger.info('X shape=' + str(x_ds.shape))
        logger.info('Y shape=' + str(y_ds.shape))
        logger.info('Y: total: ' + str(len(y_ds)))
        logger.info('Y: label 0: ' + str(label_total[0][0]) + ' ' +
                    str(100 * label0_ratio) + '%')
        logger.info('Y: label 1: ' + str(label_total[0][1]) + ' ' +
                    str(100 * label1_ratio) + '%')

        logger.info('Recording sample data')
        size = len(x_ds)
        qtty = min(3, size)
        f = size / qtty
        for i in range(qtty):
            pi = round(i * f)
            logger.info('image_index ' + str(pi))
            logger.info('x=')
            if (save_dir != None):
                mkdirs(save_dir)
                show_slices(x_ds[pi],
                            str(i) + str(y_ds[pi]),
                            output_dir=save_dir)
                logger.info('y=' + str(y_ds[pi]))
    return ok
Exemple #5
0
def export_lions(image_raw,
                 image_dotted,
                 target_x_ds,
                 target_y_ds,
                 image_dims,
                 debug=False,
                 min_distance_others=50,
                 non_lion_distance=150,
                 export_non_lion=True):

    NR_CLASSES = 6

    #BLACKOUT PORTIONS OF IMAGE IN RAW PICTURE
    image_dotted_bw = cv2.cvtColor(image_dotted, cv2.COLOR_BGR2GRAY)
    #utils.show_image(image_dotted_bw, size=8)

    mask = cv2.threshold(image_dotted_bw, 5, 255, cv2.THRESH_BINARY)[1]
    #utils.show_image(mask, size=8)

    image_raw_bw = cv2.cvtColor(image_raw, cv2.COLOR_BGR2GRAY)
    image_raw = cv2.bitwise_and(image_raw, image_raw, mask=mask)
    #utils.show_image(image_raw, size=8, is_bgr=True)

    #ISOLATE HUMAN MARKS ON DOTTED PICTURE
    diff_color = cv2.absdiff(image_dotted, image_raw)
    diff = cv2.cvtColor(diff_color, cv2.COLOR_BGR2GRAY)

    kernel = np.ones((2, 2), np.uint8)
    diff = cv2.morphologyEx(diff, cv2.MORPH_OPEN, kernel)
    ret, diff = cv2.threshold(diff, 10, 255, cv2.THRESH_TOZERO)
    ret, diff = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY)

    #debug data
    debug_image = image_dotted.copy()
    images = []

    #find all dotted sea lions
    count1 = 0
    count_class = np.zeros(NR_CLASSES)
    lion_positions = []
    lion_classes = []
    im2, contours, hierarchy = cv2.findContours(diff, cv2.RETR_LIST,
                                                cv2.CHAIN_APPROX_SIMPLE)
    for c in contours:
        x, y, w, h = cv2.boundingRect(c)
        if (w > 4 and h > 4):
            count1 = count1 + 1
            center = (x + round(w / 3), y + round(h / 3))
            clazz = find_class(image_dotted, center)

            if (clazz == -1):
                logger.warning('could not detect sea lion class at ' +
                               str(center))
                continue

            lion_positions.append(center)

            count_class[clazz] = count_class[clazz] + 1
            lion_classes.append(clazz)

            if (debug):
                cv2.circle(debug_image, center, round(w / 2), (255, 0, 0), 1)

    count_class_added = np.zeros(NR_CLASSES)

    #add found sea lions to training dataset
    #filter out lions that are too near each other to minimize noise on training set
    count2 = 0
    for i, lion_pos in enumerate(lion_positions):

        lion_class = lion_classes[i]

        is_far = True
        if (min_distance_others > 0):
            is_far = utils.is_far_from_others(lion_pos, lion_positions,
                                              min_distance_others)

        if (is_far):
            #export patch to train dataset
            count2 = count2 + 1
            pw = round(image_dims[1] / 2)
            ph = image_dims[1] - pw
            #trainX = image_raw[lion_pos[1]-pw:lion_pos[1]+ph,lion_pos[0]-pw:lion_pos[0]+ph]
            trainX = utils.crop_image_fill(
                image_raw, (lion_pos[1] - pw, lion_pos[0] - pw),
                (lion_pos[1] + ph, lion_pos[0] + ph))

            m = np.mean(trainX)

            if (m > 30 and m < 225 and m != 127):

                if (debug):
                    images.append(trainX)
                    cv2.circle(debug_image, lion_pos, round(w / 2),
                               (0, 0, 255), 2)
                    font = cv2.FONT_HERSHEY_SIMPLEX
                    cv2.putText(debug_image, str(lion_class), lion_pos, font,
                                1.1, (255, 255, 255), 2, cv2.LINE_AA)

                #normalize between 0-1
                #trainX = trainX/255
                trainY = keras.utils.np_utils.to_categorical([lion_class],
                                                             NR_CLASSES)[0]
                if (target_x_ds != None and target_y_ds != None):
                    utils.add_sample_to_dataset(target_x_ds, target_y_ds,
                                                trainX, trainY)
                count_class_added[
                    lion_class] = count_class_added[lion_class] + 1

    #identify non sea lion patches
    count3 = 0
    if (export_non_lion):
        s = np.shape(image_raw)
        for i in range(int(count2 * 1.1)):
            patch_pos = (random.randint(image_dims[1] * 2,
                                        s[1] - image_dims[1] * 2),
                         random.randint(image_dims[0] * 2,
                                        s[0] - image_dims[0] * 2))
            is_far = utils.is_far_from_others(patch_pos, lion_positions,
                                              non_lion_distance)

            if (is_far):
                #export patch to train dataset
                pw = round(image_dims[1] / 2)
                ph = image_dims[1] - pw
                #trainX = image_raw[lion_pos[1]-pw:lion_pos[1]+ph,lion_pos[0]-pw:lion_pos[0]+ph]
                trainX = utils.crop_image_fill(
                    image_raw, (patch_pos[1] - pw, patch_pos[0] - pw),
                    (patch_pos[1] + ph, patch_pos[0] + ph))

                m = np.mean(trainX)

                if (m > 50 and m < 200):
                    count3 = count3 + 1
                    if (debug):
                        images.append(trainX)
                        cv2.circle(debug_image, patch_pos, round(w / 2),
                                   (0, 255, 0), 3)

                    #normalize between 0-1
                    #trainX = trainX/255
                    trainY = keras.utils.np_utils.to_categorical([5],
                                                                 NR_CLASSES)[0]
                    if (target_x_ds != None and target_y_ds != None):
                        utils.add_sample_to_dataset(target_x_ds, target_y_ds,
                                                    trainX, trainY)
                    count_class[5] = count_class[5] + 1
                    count_class_added[5] = count_class_added[5] + 1

    logger.info('sea lions found: ' + str(count1))
    logger.info('sea lions added to dataset: ' + str(count2))
    logger.info('non sea lions added to dataset: ' + str(count3))
    if (target_x_ds != None and target_y_ds != None):
        logger.info('dataset size: ' + str(len(target_x_ds)))

    if (debug):
        utils.show_image(debug_image, size=40, is_bgr=True)
        utils.show_images(images, cols=10, is_bgr=True, size=1.5)

    return count_class, count_class_added, lion_positions, lion_classes