def extract_patches_from_heatmap_false_region_tumor(wsi_ops, patch_extractor, patch_index, augmentation=False):
tumor_heatmap_prob_paths = glob.glob(os.path.join(utils.HEAT_MAP_DIR, '*umor*prob.png'))
tumor_heatmap_prob_paths.sort()
wsi_paths = glob.glob(os.path.join(utils.TUMOR_WSI_PATH, '*.tif'))
wsi_paths.sort()
mask_paths = glob.glob(os.path.join(utils.TUMOR_MASK_PATH, '*.tif'))
mask_paths.sort()
assert len(tumor_heatmap_prob_paths) == len(wsi_paths), 'Some heatmaps are missing!'
image_mask_heatmap_tuple = zip(wsi_paths, mask_paths, tumor_heatmap_prob_paths)
image_mask_heatmap_tuple = list(image_mask_heatmap_tuple)
# image_mask_heatmap_tuple = image_mask_heatmap_tuple[32:]
# delete Tumor slides with mirror(duplicate regions) and incomplete annotation: Tumor_018, Tumor_046, Tumor_054
delete_index = [17, 45, 53]
for i in range(len(delete_index)):
print('deleting: %s' % utils.get_filename_from_path(image_mask_heatmap_tuple[delete_index[i] - i][0]))
del image_mask_heatmap_tuple[delete_index[i] - i]
patch_save_dir_pos = utils.PATCHES_TRAIN_AUG_EXCLUDE_MIRROR_WSI_POSITIVE_PATH if augmentation else utils.PATCHES_TRAIN_POSITIVE_PATH
patch_prefix_pos = utils.PATCH_AUG_TUMOR_PREFIX if augmentation else utils.PATCH_TUMOR_PREFIX
patch_save_dir_neg = utils.PATCHES_TRAIN_AUG_EXCLUDE_MIRROR_WSI_NEGATIVE_PATH if augmentation else utils.PATCHES_TRAIN_NEGATIVE_PATH
patch_prefix_neg = utils.PATCH_AUG_NORMAL_PREFIX if augmentation else utils.PATCH_NORMAL_PREFIX
not_0_255_cnt = 0
for image_path, mask_path, heatmap_prob_path in image_mask_heatmap_tuple:
print('extract_patches_from_heatmap_false_region_normal(): %s, %s, %s' %
(utils.get_filename_from_path(image_path), utils.get_filename_from_path(mask_path),
utils.get_filename_from_path(heatmap_prob_path)))
wsi_image, rgb_image, wsi_mask, tumor_gt_mask, level_used = wsi_ops.read_wsi_tumor(image_path, mask_path)
assert wsi_image is not None, 'Failed to read Whole Slide Image %s.' % image_path
# tumor_gt_mask = cv2.cvtColor(tumor_gt_mask, cv2.COLOR_BGR2GRAY)
# not_0_255_cnt += (tumor_gt_mask[tumor_gt_mask != 255].shape[0]-tumor_gt_mask[tumor_gt_mask == 0].shape[0])
# print(tumor_gt_mask[tumor_gt_mask != 255].shape[0], tumor_gt_mask[tumor_gt_mask == 0].shape[0], not_0_255_cnt)
bounding_boxes, image_open = wsi_ops.find_roi_bbox(np.array(rgb_image))
heatmap_prob = cv2.imread(heatmap_prob_path)
heatmap_prob = heatmap_prob[:, :, :1]
heatmap_prob = np.reshape(heatmap_prob, (heatmap_prob.shape[0], heatmap_prob.shape[1]))
heatmap_prob = np.array(heatmap_prob, dtype=np.float32)
heatmap_prob /= 255
patch_index = patch_extractor.extract_patches_from_heatmap_false_region_tumor(wsi_image, wsi_mask,
tumor_gt_mask,
image_open,
heatmap_prob,
level_used, bounding_boxes,
patch_save_dir_pos,
patch_save_dir_neg,
patch_prefix_pos,
patch_prefix_neg,
patch_index)
print('patch count: %d' % (patch_index - utils.PATCH_INDEX_NEGATIVE))
wsi_image.close()
wsi_mask.close()
# print('not_0_255_cnt: %d' % not_0_255_cnt)
return patch_index
def extract_negative_patches_from_normal_wsi(wsi_ops, patch_extractor, patch_index, augmentation=False):
"""
Extracted up to Normal_060.
:param wsi_ops:
:param patch_extractor:
:param patch_index:
:param augmentation:
:return:
"""
wsi_paths = glob.glob(os.path.join(utils.NORMAL_WSI_PATH, '*.tif'))
wsi_paths.sort()
wsi_paths = wsi_paths[61:]
patch_save_dir = utils.PATCHES_VALIDATION_AUG_NEGATIVE_PATH if augmentation \
else utils.PATCHES_VALIDATION_NEGATIVE_PATH
patch_prefix = utils.PATCH_AUG_NORMAL_PREFIX if augmentation else utils.PATCH_NORMAL_PREFIX
for image_path in wsi_paths:
print('extract_negative_patches_from_normal_wsi(): %s' % utils.get_filename_from_path(image_path))
wsi_image, rgb_image, level_used = wsi_ops.read_wsi_normal(image_path)
assert wsi_image is not None, 'Failed to read Whole Slide Image %s.' % image_path
bounding_boxes, image_open = wsi_ops.find_roi_bbox(np.array(rgb_image))
patch_index = patch_extractor.extract_negative_patches_from_normal_wsi(wsi_image, image_open,
level_used,
bounding_boxes,
patch_save_dir, patch_prefix,
patch_index)
print('Negative patches count: %d' % (patch_index - utils.PATCH_INDEX_NEGATIVE))
wsi_image.close()
return patch_index
def extract_negative_patches_from_tumor_wsi(wsi_ops,
patch_extractor,
patch_index,
augmentation=False):
wsi_paths = glob.glob(os.path.join(utils.TUMOR_WSI_PATH, '*.tif'))
wsi_paths.sort()
mask_paths = glob.glob(os.path.join(utils.TUMOR_MASK_PATH, '*.tif'))
mask_paths.sort()
image_mask_pair = zip(wsi_paths, mask_paths)
image_mask_pair = list(image_mask_pair)
# image_mask_pair = image_mask_pair[67:68]
patch_save_dir = utils.PATCHES_TRAIN_AUG_NEGATIVE_PATH if augmentation else utils.PATCHES_TRAIN_NEGATIVE_PATH
patch_prefix = utils.PATCH_AUG_NORMAL_PREFIX if augmentation else utils.PATCH_NORMAL_PREFIX
for image_path, mask_path in image_mask_pair:
print('extract_negative_patches_from_tumor_wsi(): %s' %
utils.get_filename_from_path(image_path))
wsi_image, rgb_image, tumor_gt_mask, level_used = wsi_ops.read_wsi_tumor(
image_path, mask_path)
assert wsi_image is not None, 'Failed to read Whole Slide Image %s.' % image_path
bounding_boxes, image_open = wsi_ops.find_roi_bbox_tumor(
np.array(rgb_image))
patch_index = patch_extractor.extract_negative_patches_from_tumor_wsi(
wsi_image, np.array(tumor_gt_mask), image_open, level_used,
bounding_boxes, patch_save_dir, patch_prefix, patch_index)
print('Negative patches count: %d' %
(patch_index - utils.PATCH_INDEX_NEGATIVE))
wsi_image.close()
return patch_index
def extract_features_test(heatmap_prob_name_postfix_first_model, heatmap_prob_name_postfix_second_model, f_test):
print('************************** extract_features_test() ***************************')
print('heatmap_prob_name_postfix_first_model: %s' % heatmap_prob_name_postfix_first_model)
print('heatmap_prob_name_postfix_second_model: %s' % heatmap_prob_name_postfix_second_model)
print('f_test: %s' % f_test)
test_wsi_paths = glob.glob(os.path.join(utils.TEST_WSI_PATH, '*.tif'))
test_wsi_paths.sort()
features_file_test = open(f_test, 'w')
wr_test = csv.writer(features_file_test, quoting=csv.QUOTE_NONNUMERIC)
wr_test.writerow(utils.heatmap_feature_names[:len(utils.heatmap_feature_names) - 1])
for wsi_path in test_wsi_paths:
wsi_name = utils.get_filename_from_path(wsi_path)
print('extracting features for: %s' % wsi_name)
heatmap_prob_path = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, heatmap_prob_name_postfix_first_model)))
# print(heatmap_prob_path)
image_open = wsi_ops.get_image_open(wsi_path)
heatmap_prob = cv2.imread(heatmap_prob_path[0])
if heatmap_prob_name_postfix_second_model is not None:
heatmap_prob_path_second_model = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, heatmap_prob_name_postfix_second_model)))
heatmap_prob_second_model = cv2.imread(heatmap_prob_path_second_model[0])
for row in range(heatmap_prob.shape[0]):
for col in range(heatmap_prob.shape[1]):
if heatmap_prob[row, col, 0] >= 0.90 * 255 and heatmap_prob_second_model[row, col, 0] < 0.50 * 255:
heatmap_prob[row, col, :] = heatmap_prob_second_model[row, col, :]
features = extract_features(heatmap_prob, image_open)
print(features)
wr_test.writerow(features)
def move_files(from_dir, to_dir, name_pattern=None, n_sample=None):
print('move_files() - From: %s' % from_dir)
print('move_files() - To: %s' % to_dir)
file_names = os.listdir(from_dir)
file_names = sorted(file_names)
if n_sample is None:
move_all(from_dir, to_dir, file_names, name_pattern)
else:
assert len(file_names) >= n_sample, "Not enough files to move. available: %d, requested: %d" % \
(len(file_names), n_sample)
if name_pattern is None:
shuffled_index = list(range(len(file_names)))
random.seed(12345)
random.shuffle(shuffled_index)
# print(sorted(shuffled_index[:n_sample]))
for index in range(n_sample):
os.rename(from_dir + file_names[shuffled_index[index]], to_dir + file_names[shuffled_index[index]])
if not index % 1000:
print('moved %d of %d files.' % (index, n_sample))
sys.stdout.flush()
else:
print('moving files with pattern: %s' % name_pattern)
file_paths = glob.glob(os.path.join(from_dir, name_pattern))
total_file_count = len(file_paths)
print('Files found: %d' % total_file_count)
assert len(file_paths) >= n_sample, "Not enough files to move with requested pattern. available: %d, " \
"requested: %d" % (total_file_count, n_sample)
file_paths = file_paths[:n_sample]
for index in range(n_sample):
os.rename(file_paths[index], to_dir + utils.get_filename_from_path(file_paths[index]))
if not index % 1000:
print('moved %d of %d files.' % (index, n_sample))
sys.stdout.flush()
def extract_patches_normal():
wsi_image_names = glob.glob(os.path.join(utils.NORMAL_WSI_PATH, '*.tif'))
wsi_image_names.sort()
# wsi_image_names = wsi_image_names[1:2]
for image_path in wsi_image_names:
extract_patches(image_path, utils.get_filename_from_path(image_path))
def extract_patches_test():
wsi_image_names = glob.glob(os.path.join(utils.TEST_WSI_PATH, '*.tif'))
wsi_image_names.sort()
print(wsi_image_names)
# wsi_image_names = wsi_image_names[1:2]
for image_path in wsi_image_names:
extract_patches(image_path, utils.get_filename_from_path(image_path))
def extract_patches_normal():
wsi_image_names = glob.glob(os.path.join(utils.NORMAL_WSI_PATH, '*.tif'))
wsi_image_names.sort()
# wsi_image_names = wsi_image_names[1:2]
for image_path in wsi_image_names:
extract_patches(image_path, utils.get_filename_from_path(image_path))
def extract_patches_from_heatmap_false_region_normal(wsi_ops,
patch_extractor,
patch_index,
augmentation=False):
normal_heatmap_prob_paths = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, 'Normal*prob.png'))
normal_heatmap_prob_paths.sort()
wsi_paths = glob.glob(os.path.join(utils.NORMAL_WSI_PATH, '*.tif'))
wsi_paths.sort()
assert len(normal_heatmap_prob_paths) == len(
wsi_paths), 'Some heatmaps are missing!'
image_heatmap_tuple = zip(wsi_paths, normal_heatmap_prob_paths)
image_heatmap_tuple = list(image_heatmap_tuple)
# image_mask_pair = image_mask_pair[67:68]
patch_save_dir_neg = utils.PATCHES_TRAIN_AUG_NEGATIVE_PATH if augmentation else utils.PATCHES_TRAIN_NEGATIVE_PATH
patch_prefix_neg = utils.PATCH_AUG_NORMAL_PREFIX if augmentation else utils.PATCH_NORMAL_PREFIX
for image_path, heatmap_prob_path in image_heatmap_tuple:
print('extract_patches_from_heatmap_false_region_normal(): %s, %s' %
(utils.get_filename_from_path(image_path),
utils.get_filename_from_path(heatmap_prob_path)))
wsi_image, rgb_image, level_used = wsi_ops.read_wsi_normal(image_path)
assert wsi_image is not None, 'Failed to read Whole Slide Image %s.' % image_path
bounding_boxes, image_open = wsi_ops.find_roi_bbox(np.array(rgb_image))
heatmap_prob = cv2.imread(heatmap_prob_path)
heatmap_prob = heatmap_prob[:, :, :1]
heatmap_prob = np.reshape(
heatmap_prob, (heatmap_prob.shape[0], heatmap_prob.shape[1]))
heatmap_prob = np.array(heatmap_prob, dtype=np.float32)
heatmap_prob /= 255
patch_index = patch_extractor.extract_patches_from_heatmap_false_region_normal(
wsi_image, image_open, heatmap_prob, level_used, bounding_boxes,
patch_save_dir_neg, patch_prefix_neg, patch_index)
print('patch count: %d' % (patch_index - utils.PATCH_INDEX_NEGATIVE))
wsi_image.close()
return patch_index
def extract_patches_tumor():
wsi_image_names = glob.glob(os.path.join(utils.TUMOR_WSI_PATH, '*.tif'))
wsi_image_names.sort()
wsi_mask_names = glob.glob(os.path.join(utils.TUMOR_MASK_PATH, '*.tif'))
wsi_mask_names.sort()
image_mask_pair = zip(wsi_image_names, wsi_mask_names)
image_mask_pair = list(image_mask_pair)
# image_mask_pair = image_mask_pair[1:2]
for image_path, mask_path in image_mask_pair:
extract_patches(image_path, utils.get_filename_from_path(image_path), mask_path)
def move_all(from_dir, to_dir, file_names, name_pattern):
if name_pattern is not None:
print('moving files with pattern: %s' % name_pattern)
file_paths = glob.glob(os.path.join(from_dir, name_pattern))
total_file_count = len(file_paths)
print('Files found: %d' % total_file_count)
for index in range(total_file_count):
os.rename(file_paths[index], to_dir + utils.get_filename_from_path(file_paths[index]))
if not index % 1000:
print('moved %d of %d files.' % (index, total_file_count))
sys.stdout.flush()
else:
total_file_count = len(file_names)
print('moving all files: %d' % total_file_count)
for index in range(total_file_count):
os.rename(from_dir + file_names[index], to_dir + file_names[index])
if not index % 1000:
print('moved %d of %d files.' % (index, total_file_count))
sys.stdout.flush()
def extract_features_train_all(heatmap_prob_name_postfix_first_model, heatmap_prob_name_postfix_second_model, f_train):
print('********************** extract_features_train_all() *************************')
print('heatmap_prob_name_postfix_first_model: %s' % heatmap_prob_name_postfix_first_model)
print('heatmap_prob_name_postfix_second_model: %s' % heatmap_prob_name_postfix_second_model)
print('f_train: %s' % f_train)
tumor_wsi_paths = glob.glob(os.path.join(utils.TUMOR_WSI_PATH, '*.tif'))
tumor_wsi_paths.sort()
normal_wsi_paths = glob.glob(os.path.join(utils.NORMAL_WSI_PATH, '*.tif'))
normal_wsi_paths.sort()
wsi_paths = tumor_wsi_paths + normal_wsi_paths
features_file_train_all = open(f_train, 'w')
wr_train = csv.writer(features_file_train_all, quoting=csv.QUOTE_NONNUMERIC)
wr_train.writerow(utils.heatmap_feature_names)
for wsi_path in wsi_paths:
wsi_name = utils.get_filename_from_path(wsi_path)
# print('extracting features for: %s' % wsi_name)
heatmap_prob_path = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, heatmap_prob_name_postfix_first_model)))
# print(heatmap_prob_path)
image_open = wsi_ops.get_image_open(wsi_path)
heatmap_prob = cv2.imread(heatmap_prob_path[0])
if heatmap_prob_name_postfix_second_model is not None:
heatmap_prob_path_second_model = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, heatmap_prob_name_postfix_second_model)))
heatmap_prob_second_model = cv2.imread(heatmap_prob_path_second_model[0])
for row in range(heatmap_prob.shape[0]):
for col in range(heatmap_prob.shape[1]):
if heatmap_prob[row, col, 0] >= 0.90 * 255 and heatmap_prob_second_model[row, col, 0] < 0.50 * 255:
heatmap_prob[row, col, :] = heatmap_prob_second_model[row, col, :]
features = extract_features(heatmap_prob, image_open)
if 'umor' in wsi_name:
features += [1]
else:
features += [0]
print(features)
wr_train.writerow(features)
def find_roi_tumor(self):
# self.mask = cv2.cvtColor(self.mask, cv2.CV_32SC1)
hsv = cv2.cvtColor(self.rgb_image, cv2.COLOR_BGR2HSV)
lower_red = np.array([20, 20, 20])
upper_red = np.array([200, 200, 200])
mask = cv2.inRange(hsv, lower_red, upper_red)
res = cv2.bitwise_and(self.rgb_image, self.rgb_image, mask=mask)
# (50, 50)
close_kernel = np.ones((20, 20), dtype=np.uint8)
close_kernel_tmp = np.ones((50, 50), dtype=np.uint8)
image_close = Image.fromarray(cv2.morphologyEx(np.array(mask), cv2.MORPH_CLOSE, close_kernel))
image_close_tmp = Image.fromarray(cv2.morphologyEx(np.array(mask), cv2.MORPH_CLOSE, close_kernel_tmp))
# (30, 30)
open_kernel = np.ones((5, 5), dtype=np.uint8)
open_kernel_tmp = np.ones((30, 30), dtype=np.uint8)
image_open = Image.fromarray(cv2.morphologyEx(np.array(image_close), cv2.MORPH_OPEN, open_kernel))
image_open_tmp = Image.fromarray(cv2.morphologyEx(np.array(image_close_tmp), cv2.MORPH_OPEN, open_kernel_tmp))
contour_rgb, contour_mask, bounding_boxes, contour_rgb_tmp, contour_mask_tmp = self.get_tumor_image_contours(
np.array(image_open), self.rgb_image,
self.mask, np.array(image_open_tmp))
wsi_name = utils.get_filename_from_path(self.wsi_path)
# cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + '_hsv_mask.png', mask)
# cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + '_mask.png', self.mask)
# cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + '_image_close.png', np.array(image_close))
# cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + '_image_open.png', np.array(image_open))
# rgb_bbox = self.rgb_image.copy()
# for i, bounding_box in enumerate(bounding_boxes):
# x = int(bounding_box[0])
# y = int(bounding_box[1])
# cv2.rectangle(rgb_bbox, (x, y), (x + bounding_box[2], y + bounding_box[3]), color=(0, 0, 255),
# thickness=2)
# cv2.imshow('contour_rgb', contour_rgb)
# cv2.imshow('contour_bbox', rgb_bbox)
# cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + 'contour.png', contour_rgb)
# cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + '_bbox.png', rgb_bbox)
cv2.imwrite(os.path.join(utils.THESIS_FIGURE_DIR, wsi_name) + '_hsv.png', hsv)
cv2.imshow('hsv', hsv)
cv2.waitKey(0)
coord = tf.train.Coordinator()
threads = []
for thread_index in range(len(bounding_boxes)):
args = (thread_index, bounding_boxes[thread_index], wsi_image, image_open, level_used, heatmap_patch_dir)
t = threading.Thread(target=extract_patch_from_bb, args=args)
t.start()
threads.append(t)
# Wait for all the threads to terminate.
coord.join(threads)
wsi_image.close()
sys.stdout.flush()
if __name__ == '__main__':
# dataset = Dataset(DATA_SET_NAME, data_subset[1])
# evaluate(dataset)
wsi_ops = WSIOps()
wsi_image_names = glob.glob(os.path.join(utils.TUMOR_WSI_PATH, '*.tif'))
wsi_image_names.sort()
wsi_mask_names = glob.glob(os.path.join(utils.TUMOR_MASK_PATH, '*.tif'))
wsi_mask_names.sort()
image_mask_pair = zip(wsi_image_names, wsi_mask_names)
image_mask_pair = list(image_mask_pair)
# image_mask_pair = image_mask_pair[67:68]
for image_path, mask_path in image_mask_pair:
extract_patches(image_path, mask_path, utils.get_filename_from_path(image_path))
h_flip_prob = cv2.flip(prob_2d, 0)
plt.imshow(h_flip_prob, cmap='jet', interpolation='nearest')
plt.colorbar()
plt.clim(0.00, 1.00)
plt.axis([0, h_flip_prob.shape[1], 0, h_flip_prob.shape[0]])
plt.savefig(heatmap_path)
plt.clf()
if __name__ == '__main__':
heatmap_prob_paths_first_model = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*' % '*umor*prob.png'))
heatmap_prob_paths_first_model.sort()
for heatmap_prob_path_first_model in heatmap_prob_paths_first_model:
wsi_name = utils.get_filename_from_path(heatmap_prob_path_first_model)
wsi_name_tokens = wsi_name.split('_')
wsi_name = wsi_name_tokens[0] + '_' + wsi_name_tokens[1]
print('processing: %s' % wsi_name)
heatmap_prob_path_second_model = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, '_prob_%s.png' % utils.SECOND_HEATMAP_MODEL)))
heatmap_prob_second_model = cv2.imread(heatmap_prob_path_second_model[0])
heatmap_prob_first_model = cv2.imread(heatmap_prob_path_first_model)
for row in range(heatmap_prob_first_model.shape[0]):
for col in range(heatmap_prob_first_model.shape[1]):
if heatmap_prob_first_model[row, col, 0] >= 0.70 * 255 and \
heatmap_prob_second_model[row, col, 0] < 0.25 * 255:
heatmap_prob_first_model[row, col, :] = heatmap_prob_second_model[row, col, :]
heatmap_path = heatmap_prob_path_first_model.replace('prob', 'heatmap_ensemble')
def extract_features_train_validation(heatmap_prob_name_postfix_first_model, heatmap_prob_name_postfix_second_model,
f_train, f_validation):
print('********************** extract_features_train_validation() ********************************')
print('heatmap_prob_name_postfix_first_model: %s' % heatmap_prob_name_postfix_first_model)
print('heatmap_prob_name_postfix_second_model: %s' % heatmap_prob_name_postfix_second_model)
print('f_train: %s' % f_train)
print('f_validation: %s' % f_validation)
tumor_wsi_paths = glob.glob(os.path.join(utils.TUMOR_WSI_PATH, '*.tif'))
tumor_wsi_paths.sort()
normal_wsi_paths = glob.glob(os.path.join(utils.NORMAL_WSI_PATH, '*.tif'))
normal_wsi_paths.sort()
tumor_shuffled_index = list(range(len(tumor_wsi_paths)))
random.seed(12345)
random.shuffle(tumor_shuffled_index)
normal_shuffled_index = list(range(len(tumor_wsi_paths), len(tumor_wsi_paths) + len(normal_wsi_paths)))
random.seed(12345)
random.shuffle(normal_shuffled_index)
tumor_shuffled_index = tumor_shuffled_index[:20]
normal_shuffled_index = normal_shuffled_index[:30]
validation_index = tumor_shuffled_index + normal_shuffled_index
print('number of validation samples: %d' % len(validation_index))
wsi_paths = tumor_wsi_paths + normal_wsi_paths
print(len(wsi_paths))
features_file_train = open(f_train, 'w')
features_file_validation = open(f_validation, 'w')
wr_train = csv.writer(features_file_train, quoting=csv.QUOTE_NONNUMERIC)
wr_validation = csv.writer(features_file_validation, quoting=csv.QUOTE_NONNUMERIC)
wr_train.writerow(utils.heatmap_feature_names)
wr_validation.writerow(utils.heatmap_feature_names)
index = 0
for wsi_path in wsi_paths:
wsi_name = utils.get_filename_from_path(wsi_path)
# print('extracting features for: %s' % wsi_name)
heatmap_prob_path = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, heatmap_prob_name_postfix_first_model)))
# print(heatmap_prob_path)
image_open = wsi_ops.get_image_open(wsi_path)
heatmap_prob = cv2.imread(heatmap_prob_path[0])
if heatmap_prob_name_postfix_second_model is not None:
heatmap_prob_path_second_model = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*%s' % (wsi_name, heatmap_prob_name_postfix_second_model)))
heatmap_prob_second_model = cv2.imread(heatmap_prob_path_second_model[0])
for row in range(heatmap_prob.shape[0]):
for col in range(heatmap_prob.shape[1]):
if heatmap_prob[row, col, 0] >= 0.90 * 255 and heatmap_prob_second_model[row, col, 0] < 0.20 * 255:
heatmap_prob[row, col, :] = heatmap_prob_second_model[row, col, :]
features = extract_features(heatmap_prob, image_open)
if 'umor' in wsi_name:
features += [1]
else:
features += [0]
print(features)
if index in validation_index:
wr_validation.writerow(features)
else:
wr_train.writerow(features)
index += 1
h_flip_prob = cv2.flip(prob_2d, 0)
plt.imshow(h_flip_prob, cmap='jet', interpolation='nearest')
plt.colorbar()
plt.clim(0.00, 1.00)
plt.axis([0, h_flip_prob.shape[1], 0, h_flip_prob.shape[0]])
plt.savefig(heatmap_path)
plt.clf()
if __name__ == '__main__':
heatmap_prob_paths_first_model = glob.glob(
os.path.join(utils.HEAT_MAP_DIR, '*%s*' % '*umor*prob.png'))
heatmap_prob_paths_first_model.sort()
for heatmap_prob_path_first_model in heatmap_prob_paths_first_model:
wsi_name = utils.get_filename_from_path(heatmap_prob_path_first_model)
wsi_name_tokens = wsi_name.split('_')
wsi_name = wsi_name_tokens[0] + '_' + wsi_name_tokens[1]
print('processing: %s' % wsi_name)
heatmap_prob_path_second_model = glob.glob(
os.path.join(
utils.HEAT_MAP_DIR, '*%s*%s' %
(wsi_name, '_prob_%s.png' % utils.SECOND_HEATMAP_MODEL)))
heatmap_prob_second_model = cv2.imread(
heatmap_prob_path_second_model[0])
heatmap_prob_first_model = cv2.imread(heatmap_prob_path_first_model)
for row in range(heatmap_prob_first_model.shape[0]):
for col in range(heatmap_prob_first_model.shape[1]):
if heatmap_prob_first_model[row, col, 0] >= 0.70 * 255 and \
heatmap_prob_second_model[row, col, 0] < 0.25 * 255:
