def main(args):
config = MitosisInferenceConfig()
if args.reorganize_folder_structure:
print("1. Reorganize the data structure for Mask_RCNN")
reorganize_mitosis_images(config.mitosis_input_dir,
config.mitosis_reorganized_dir)
if args.split_big_images_to_small_ones:
print("2. Crop big images into small ones")
mitosis_files = \
[str(f) for f in Path(config.mitosis_reorganized_dir)
.glob('**/**/*.tif')]
crop_image(mitosis_files,
config.inference_input_dir,
size=config.crop_image_size,
overlap=config.crop_image_overlap)
if args.run_nucleus_detection:
print("3. Run the nucleus segmentation inference")
nucleus_mitosis.inference(config.inference_input_dir, config.weights,
config.maskrcnn_inference_result_dir)
print("Inference result dir: ", config.maskrcnn_inference_result_dir)
if args.combine_small_inference_results:
print("4. Combine small inference images and csvs to big ones")
combine_images(config.maskrcnn_inference_result_dir,
config.maskrcnn_inference_combined_result,
size=config.crop_image_size)
combine_csvs(config.maskrcnn_inference_result_dir,
config.maskrcnn_inference_combined_result,
hasProb=config.hasProb,
clean_output_dir=False)
if args.cluster_nucleus_detection_results:
print("4.1 Cluster the nucleus detection results")
cluster_prediction_result(config.maskrcnn_inference_combined_result,
eps=32,
min_samples=1,
hasHeader=True,
isWeightedAvg=False,
prob_threshold=0.8)
for file_name in [
"11-01", "11-02", "11-03", "12-01", "12-02", "12-03"
]:
add_mark(os.path.join(config.maskrcnn_inference_combined_result,
"{}.png".format(file_name)),
os.path.join(
config.maskrcnn_inference_combined_clusterd_result,
"{}.csv".format(file_name)),
hasHeader=True,
shape=Shape.SQUARE,
mark_color=(255, 100, 100, 200))
if args.visualize_the_ground_truth:
print("5. Visualize the ground truth masks")
add_groundtruth_mark(config.maskrcnn_inference_combined_result,
config.ground_truth_dir,
hasHeader=False,
shape=Shape.CIRCLE)
# add_mark('/Users/fei/Documents/Github/Mask_RCNN/samples/nucleus/datasets/stage1_combine_test/01-01.png',
# '/Users/fei/Documents/Github/Mask_RCNN/samples/nucleus/datasets/stage1_combine_test/01-01.csv',
# hasHeader=True, shape=Shape.CIRCLE, mark_color=(255,0,0,50))
if args.evaluate_nucleus_inference:
print("6. Evaluate the nucleus inference result")
check_nucleus_inference(config.maskrcnn_inference_combined_result,
config.ground_truth_dir)
if args.extract_nucleus_patch:
print("7. Extract patches according to the inference result")
extract_patches(config.mitosis_input_dir,
config.maskrcnn_inference_combined_result,
config.extracted_nucleus_dir,
patch_size=config.mitosis_patch_size)
if args.run_mitosis_classification:
print("8. Run mitosis classification inference")
batch_size = 128
num_parallel_calls = 8
prob_thres = 0.5,
eps = 64
min_samples = 1
is_weighted_avg = False
run_mitosis_classification_in_batch(
batch_size=batch_size,
input_dir_basepath=config.extracted_nucleus_dir,
output_dir_basepath=config.mitosis_classification_result_dir,
model_file=config.mitosis_classification_model_file,
augmentation_number=config.augmentation_number,
mitosis_tile_size=config.mitosis_tile_size,
num_parallel_calls=config.
mitosis_classification_num_parallel_calls,
prefectch=config.mitosis_classification_prefetch,
prob_thres=prob_thres,
eps=eps,
min_samples=min_samples,
isWeightedAvg=is_weighted_avg)
if args.compute_f1:
print("9. Compute F1 score")
f1, precision, recall, over_detected, non_detected, FP, TP, FN = \
evaluate_global_f1(config.mitosis_classification_result_dir,
config.ground_truth_dir,
threshold=30,
prob_threshold=config.f1_prob_threshhold)
print("F1: {} \n"
"Precision: {} \n"
"Recall: {} \n"
"Over_detected: {} \n"
"Non_detected: {} \n"
"FP: {} \n"
"TP: {} \n"
"FN: {} \n".format(f1, precision, recall, len(over_detected),
len(non_detected), len(FP), len(TP), len(FN)))
def main(args):
config = MitosisTrainConfig()
if args.reorganize_folder_structure:
print("1. Reorganize the data structure for Mask_RCNN")
reorganize_mitosis_images(
config.mitosis_input_dir, config.mitosis_reorganized_dir)
if args.split_big_images_to_small_ones:
print("2. Crop big images into small ones")
mitosis_files = \
[str(f) for f in Path(config.mitosis_reorganized_dir)
.glob('**/**/*.tif')]
crop_image(mitosis_files, config.inference_input_dir,
size=config.crop_image_size,
overlap=config.crop_image_overlap)
if args.run_nucleus_detection:
print("3. Run the nucleus segmentation inference")
nucleus_mitosis.inference(config.inference_input_dir, config.weights,
config.maskrcnn_inference_result_dir)
print("Inference result dir: ", config.maskrcnn_inference_result_dir)
if args.combine_small_inference_results:
print("4. Combine small inference images and csvs to big ones")
combine_images(config.maskrcnn_inference_result_dir,
config.maskrcnn_inference_combined_result,
size=config.crop_image_size)
combine_csvs(config.maskrcnn_inference_result_dir,
config.maskrcnn_inference_combined_result,
hasProb=config.hasProb, clean_output_dir=False)
if args.cluster_nucleus_detection_results:
print("4.1 Cluster the nucleus detection results")
cluster_prediction_result(config.maskrcnn_inference_combined_result,
eps=32, min_samples=1, hasHeader=True,
isWeightedAvg=False, prob_threshold=0.8)
# for file_name in ["11-01", "11-02", "11-03", "12-01", "12-02", "12-03"]:
# add_mark(os.path.join(config.maskrcnn_inference_combined_result,
# "{}.png".format(file_name)),
# os.path.join(config.maskrcnn_inference_combined_clusterd_result,
# "{}.csv".format(file_name)),
# hasHeader=True, shape=Shape.SQUARE,
# mark_color=(255, 100, 100, 200))
if args.visualize_the_ground_truth:
print("5. Visualize the ground truth masks")
add_groundtruth_mark(config.maskrcnn_inference_combined_result,
config.ground_truth_dir, hasHeader=False,
shape=Shape.CIRCLE)
# add_mark('/Users/fei/Documents/Github/Mask_RCNN/samples/nucleus/datasets/stage1_combine_test/01-01.png',
# '/Users/fei/Documents/Github/Mask_RCNN/samples/nucleus/datasets/stage1_combine_test/01-01.csv',
# hasHeader=True, shape=Shape.CIRCLE, mark_color=(255,0,0,50))
if args.evaluate_nucleus_inference:
print("6. Evaluate the nucleus inference result")
check_nucleus_inference(config.maskrcnn_inference_combined_result,
config.ground_truth_dir)
if args.label_detected_nucleus:
print("7. Label the detected nucleus as mitosis or not based on the "
"ground truth")
label_detected_nucleus(config.maskrcnn_inference_combined_result,
config.ground_truth_dir)
if args.gen_mitosis_normal_tile:
print("8. Generate the mitosis and normal tiles based on the label csv")
gen_mitosis_normal_tiles(config.mitosis_input_dir,
config.maskrcnn_inference_combined_result,
config.extracted_normal_mitosis_patch_dir,
tile_size=config.mitosis_tile_size)
if args.split_train_val_datasets:
print("9. Split the datasets into train and validation folders")
split_train_val_datasets(config.extracted_normal_mitosis_patch_dir,
config.val_cases)
if args.train_model:
print("10. Train the classification model")
train_args = []
train_args.append("--patches_path={}".format(config.extracted_normal_mitosis_patch_dir))
train_args.append("--exp_parent_path={}".format(config.exp_parent_path))
train_args.append("--model={}".format(config.model))
train_args.append("--model_weights={}".format(config.mitosis_classification_model_file))
train_args.append("--train_batch_size={}".format(config.train_batch_size))
train_args.append("--val_batch_size={}".format(config.val_batch_size))
train_args.append("--clf_epochs={}".format(config.clf_epochs))
train_args.append("--finetune_epochs={}".format(config.finetune_epochs))
train_args.append("--clf_lr={}".format(config.clf_lr))
train_args.append("--finetune_lr={}".format(config.finetune_lr))
train_args.append("--finetune_momentum={}".format(config.finetune_momentum))
train_args.append("--finetune_layers={}".format(config.finetune_layers))
train_args.append("--l2={}".format(config.l2))
if config.reg_biases:
train_args.append("--reg_biases")
if config.augment:
train_args.append("--augment")
if config.oversample:
train_args.append("--oversample")
train_args.append("--num_gpus={}".format(config.num_gpus))
train_args.append("--threads={}".format(config.threads))
train_args.append("--prefetch_batches={}".format(config.prefetch_batches))
train_args.append("--log_interval={}".format(config.log_interval))
try:
train_mitoses.main(train_args)
except tf.errors.InvalidArgumentError as e: # if values become nan or inf
print(e)
print("Experiment failed!")
threshold = 30
prob_thresh = 0.65 #prob_thresh
f1, ppv, sens, over_detected, non_detected, fp_list, tp_list, fn_list = evaluate_global_f1(
detected_pred_dir, ground_true_dir, threshold, prob_thresh)
print(f"F1: {f1}; Precision: {ppv}; Recall: {sens}; FP: {len(fp_list)}; TP: {len(tp_list)}; FN: "\
f"{len(fn_list)}")
# ---
# compare to clustering on the original probs
# NOTE: if this is re-run, be sure to remove the existing directory prior to running
#if os.path.exists(clustered_pred_dir)
#shutil.rmtree(clustered_pred_dir)
#`rm -rf pred/result/2018/new_orig_v3_fp_v3_8_icpr/scratch/resnet_custom/resnet_custom_stride_16_marg_model_180307_175241_clustered/`
eps = 60 # radius from center is 7.5 µm = 30 pixels, so diameter = 60 pixels
prob_thresh = 0.4
det.cluster_prediction_result(pred_dir,
eps=eps,
min_samples=2,
hasHeader=True,
isWeightedAvg=False,
prob_threshold=prob_thresh)
print("clustered raw predictions:")
clustered_pred_dir = pred_dir + "_clustered"
threshold = 30
prob_thresh = 0.59
f1, ppv, sens, over_detected, non_detected, fp_list, tp_list, fn_list = evaluate_global_f1(
clustered_pred_dir, ground_true_dir, threshold, prob_thresh)
print(f"F1: {f1}; Precision: {ppv}; Recall: {sens}; FP: {len(fp_list)}; TP: {len(tp_list)}; FN: "\
f"{len(fn_list)}")