def create_results_dict(): results = {} for test_set in ALL_VOLUME_IDS: if test_set not in results: results[test_set] = {} gt_path = get_paths("gala-evaluate", "train", SIZE, test_set, "XX", "XX", "XX", "XX")["groundtruth"] gt = io.read_image_stack(gt_path) for train_set in ALL_VOLUME_IDS: if train_set not in results[test_set]: results[test_set][train_set] = {} for feature in ALL_FEATURES: if feature not in results[test_set][train_set]: results[test_set][train_set][feature] = {} for cue in ALL_CUES: seg_path = get_paths("gala-evaluate", "train", SIZE, test_set, cue, feature, "", train_set)["segmentation"] try: seg = io.read_image_stack(seg_path) err = rand_error(seg,gt) print "test %s, train %s, %s, %s: %f" % (test_set, train_set, feature, cue, err) results[test_set][train_set][feature][cue] = err except: print " -- missing: test %s, train %s, %s, %s: %s" % ( test_set, train_set, feature, cue, seg_path) return results
def run_gala_evaluate(traintest, size, volume_id, cues_id, features_id, exec_id, classifier_volume_id): """ Parameters ---------- traintest: {"train", "test"} whether to evaluate the output of the segmentation of the training or testing data size: the size of the dataset being evaluated and the groundtruth cues_id: a '+' separated list of cues used in training and segmentation features_id: the id of the features used in training and segmentation """ if classifier_volume_id == "all": for new_cl_volume_id in ALL_VOLUME_IDS: run_gala_evaluate(traintest, size, volume_id, cues_id, features_id, exec_id, new_cl_volume_id) return if volume_id == "all": for new_volume_id in ALL_VOLUME_IDS: run_gala_evaluate(traintest, size, new_volume_id, cues_id, features_id, exec_id, classifier_volume_id) return paths = get_paths("gala-evaluate", traintest, size, volume_id, cues_id, features_id, exec_id, classifier_volume_id) for file_use in ["segmentation", "groundtruth"]: if not os.path.isfile(paths[file_use]): print "Unable to find %s file for %s %s %s %s %s classifier tested on %s" % ( file_use, classifier_volume_id, size, cues_id, features_id, exec_id, volume_id) return seg = read_image_stack(paths["segmentation"]) gt = read_image_stack(paths["groundtruth"]) error = rand_error(seg, gt) print "For classifier trained on %s %s %s %s %s and tested on %s %s" % ( classifier_volume_id, size, cues_id, features_id, exec_id, volume_id, size) print "\tfound Rand error: %f" % (error)