def __init__(self, gt, predict, threshold = 0.5):
     self.gt                 = gt
     self.adjusted_predict   = adjust_predict(predict)
     self.relevant_data      = exclude_ignore_label(gt, self.adjusted_predict)
     self.nlables            = get_number_of_labels(gt)
     self.apr                = apr(self.relevant_data[0], self.relevant_data[1])
     self.auc_score          = roc_auc(self.relevant_data[0], self.relevant_data[1])
     self.quality            = np.array((self.apr[0], self.apr[1], self.apr[2], self.auc_score))
     self.roc_curve          = draw_roc_curve(self.relevant_data[0], self.relevant_data[1])
def redo_quality2(block_path, dense_gt_path, minMemb, minSeg, sigMin, sigWeights, sigSmooth, edgeLengths=None,
                      nodeFeatures=None, nodeSizes=None, nodeLabels=None, nodeNumStop=None, beta=0, metric='l1',
                      wardness=0.2, out=None):
    folder_list = [f for f in os.listdir(block_path) if not os.path.isfile(os.path.join(block_path,f))
                   and ("figure" not in f) and "redone" not in f]
    print "folder_list", folder_list
    outpath = block_path + "/redone_quality"
    if not os.path.exists(outpath):
        os.mkdir(outpath)

    dense_gt_data = read_h5(dense_gt_path)
    nodes_in_dgt = len(np.unique(dense_gt_data))
    for f in folder_list:
        folder_path = os.path.join(block_path, f)
        prob_file = [h for h in os.listdir(folder_path) if "probs" in h]
        prob_file_path = os.path.join(folder_path,prob_file[0])
        print
        print
        print prob_file_path
        predict_data = read_h5(prob_file_path)
        adjusted_predict_data = adjust_predict(predict_data)
        print "#nodes in dense gt", nodes_in_dgt
        pmin = 0.5

        seg, sup, wsDt_data, agglCl_data = get_segmentation(adjusted_predict_data, pmin, minMemb, minSeg, sigMin, sigWeights,
                                                            sigSmooth,True,False, edgeLengths,nodeFeatures, nodeSizes,
                                                            nodeLabels, nodeNumStop,beta, metric, wardness, out)
        nodes_in_seg = len(np.unique(seg))
        nodes_in_sup = len(np.unique(sup))

        ri, voi = rand_index_variation_of_information(seg, dense_gt_data)

        new_folder_path = outpath + "/" + f
        if not os.path.exists(new_folder_path):
            os.mkdir(new_folder_path)

        key = "pmin_"+ str(pmin) + "_minMemb_" + str(minMemb)+ "_minSeg_"+ str(minSeg) +"_sigMin_" + str(sigMin) + \
              "_sigWeights_" + str(sigWeights) + "_sigSmooth_" + str(sigSmooth) + "_nodeNumStop_" + str(nodeNumStop)
        save_h5(seg, new_folder_path + "/segmentation.h5", key)
        save_h5(sup, new_folder_path + "/super_pixels.h5", key)
        seg_data = np.zeros((2,3))
        seg_data[0,0] = ri
        seg_data[0,1] = voi
        seg_data[1,0] = nodes_in_dgt
        seg_data[1,1] = nodes_in_sup
        seg_data[1,2] = nodes_in_seg
        save_h5(seg_data, new_folder_path + "/seg_data", key, None)
def test_wsDt_agglCl_configs(predict_path, dense_gt_path, pmin=0.5, minMemb=10, minSeg=10, sigMin=2, sigWeights=2, sigSmooth=0.1):
    """
    :param predict_path:
    :param dense_gt_path:
    :param pmin:
    :param minMemb:
    :param minSeg:
    :param sigMin:
    :param sigWeights:
    :param sigSmooth:
    :return:
    """

    #get rand index and variation of information
    predict_data = read_h5(predict_path)
    adjusted_predict_data = adjust_predict(predict_data)
    dense_gt_data = read_h5(dense_gt_path)
    seg, sup = get_segmentation(adjusted_predict_data,pmin,minMemb,minSeg,sigMin,sigWeights,sigSmooth)
    ri, voi = rand_index_variation_of_information(seg, dense_gt_data)
    print "ri, voi", ri, voi

    config = predict_path.split("/")[-2]
    filename = config + "_" + predict_path.split("/")[-1]
    outpath_folder = "/home/stamylew/test_folder/q_data/wsDt_agglCl_tests/"
    outpath = outpath_folder + filename

    key = "sigMin_" + str(sigMin) + "_sigWeights_" + str(sigWeights) + "_sigSmooth_" + str(sigSmooth)

    if not os.path.exists(outpath):
        print "Output h5 file did not exist."
        data = np.zeros((1,2))
        data[0,0] = ri
        data[0,1] = voi
        save_h5(data, outpath, key, None)
    else:
        old_data = read_h5(outpath, key)
        data = np.zeros((1,2))
        data[0,0] = ri
        data[0,1] = voi
        new_data = np.vstack((old_data, data))
        save_h5(new_data, outpath, key, None)
Example #4
0
def test(ilp, files, gt_path, dense_gt_path, labels="", loops=3, weights="", repeats=1, outpath= "",
         t_cache = "", p_cache = ""):
    """ Run test
    :param: ilp             : path to ilp project to use for training
    :param: files           : path to file to do batch prediction on
    :param: gt_path         : path to trimap groundtruth
    :param: dense_gt_path   : path to dense groundtruth
    :param: labels          : amount of labeled pixels to use in training
    :param: loops           : amount of autocontext loops
    :param: weights         : weighting of the labels over the autocontext loops
    :param: repeats         : amount of repeat runs of the test
    :param: outpath         : outpath for test outputs
    :param: t_cache         : path to training data cache
    :param: p_cache         : path to prediction data cache
    """

    hostname = socket.gethostname()
    print "test information:"
    print
    print "hostname:", hostname
    print
    print "ilp_file:", ilp
    print
    print "files to predict:", files
    print
    print "groundtruth:", gt_path
    print
    print "dense groundtruth:", dense_gt_path
    print
    print "labels:", labels
    print
    print "loops:", loops
    print
    print "weights:", weights
    print
    print "repeats:", repeats

    # Collect the test specifications
    ilp_split = ilp.split(".")[-2]
    training_file = "training file: "+ilp_split.split("/")[-1]

    gt_split = gt_path.split(".")[-2]
    trimap_file = "trimap file: " + gt_split.split("/")[-1]


    # Assign paths
    filesplit = files.split(".")[-2]
    filename = filesplit.split("/")[-1]
    prediction_file = "prediction file: "+ filename

    test_folder_path = assign_path(hostname)[5]

    if t_cache == "":
        t_cache = test_folder_path + "/t_cache"

    if p_cache == "":
        p_cache = test_folder_path + "/p_cache/" + filename
        if not os.path.exists(p_cache):
            os.mkdir(p_cache)

    if outpath == "":
        output = test_folder_path + "/q_data"
    else:
        output = outpath

    print
    print "outpath:", outpath
    print
    print "t_cache:", t_cache
    print
    print "p_cache:", p_cache

    # Create file tags
    if labels == "":
        label_tag = "all"
    else:
        # assert labels == check_ilp_labels(ilp)
        true_labels = check_ilp_labels(ilp)
        label_tag = true_labels
    if weights == "":
        weight_tag = "none"
    else:
        weight_tag = str(weights)

    # Make folder for quality data

    if "manual" in ilp:
        filename += "_manual"
    if "hand_drawn" in ilp:
        filename += "_hand_drawn"
    if "clever" in ilp:
        filename += "_clever"
    if "less_feat" in ilp:
        filename += "_less_feat"


    #change labels for every test
    # if labels != "":
    #     print
    #     print "reducing labels to " + str(labels)
    #     ilp = reduce_labels_in_ilp(ilp, labels)

    file_dir = output + "/" + filename
    if not os.path.exists(file_dir):
        os.mkdir(file_dir)

    # Overwrite folder directory
    # file_dir = "/mnt/CLAWS1/stamilev/delme"

    # Check if file directory exists, if not make such directory
    if not os.path.exists(file_dir):
        print
        print "Output folder did not exist."
        os.mkdir(file_dir)
        print
        print "New one named " + file_dir + " was created."

    q_outpath = file_dir + "/n_" + str(loops) + "_l_" + str(label_tag) + "_w_" + weight_tag
    prob_folder = q_outpath + "/prob_files"
    q_data_outpath = q_outpath + "/n_" + str(loops) + "_l_" + str(label_tag) + "_w_" + weight_tag + ".h5"


    # Check if test directory exists, if not make such directory
    if not os.path.exists(q_outpath):
        print
        print "Output h5 file did not exist"
        os.mkdir(q_outpath)
        os.mkdir(prob_folder)
        print
        print "New one named " + q_outpath + " was created."
    if not os.path.exists(prob_folder):
        os.mkdir(prob_folder)

    # Run the test
    for i in range(repeats):
        print
        print "round of repeats %d of %d" % (i+1, repeats)

        #train on ilp project
        ac_train(ilp, labels, loops, weights, t_cache, outpath)
        print
        print "training completed"

        #batch predict files
        ac_batch_predict(files, t_cache, p_cache, overwrite = "")
        print
        print "batch prediction completed"

        #save prob files and quality data
        prob_file = [x for x in os.listdir(p_cache) if ("probs" in x)]
        predict_path = p_cache + "/" + prob_file[0]
        predict_data = adjust_predict(read_h5(predict_path))
        prob_path = prob_folder + "/prob_"+ str(i+1)+ ".h5"
        # if os.path.exists(prob_path):
        #     # TODO:not working yet
        #     prob_path = prob_path.split("/")[-2] + "/new_" + prob_path.split("/")[-1]
        #     print prob_path
        save_h5(predict_data, prob_path, "data")
        save_quality_values(predict_path, gt_path, dense_gt_path, q_data_outpath, (0,49,99))

        #save test specification data
        save_h5([training_file, prediction_file, trimap_file], q_data_outpath, "used files", None)
        save_h5([labels],q_data_outpath, "autocontext_parameters/#labels")
        save_h5([loops], q_data_outpath, "autocontext_parameters/#loops")
        save_h5([str(weight_tag)], q_data_outpath, "autocontext_parameters/weights")
        print
        print "quality data saved"

    #save configuration data
    # call(["cp", predict_path, q_outpath])
    save_h5(["pmin", "minMemb", "minSeg", "sigMin", "sigWeights", "sigSmooth", "cleanCloseSeeds", "returnSeedsOnly"],
            q_data_outpath, "segmentation/wsDt parameters", None)
    save_h5(["edge_weights", "edgeLengths", "nodeFeatures", "nodeSizes", "nodeLabels", "nodeNumStop", "beta", "metric",
             "wardness", "out"], q_data_outpath, "segmentation/agglCl parameters", None)
    print
    print "quality data saved"
 #save_h5(q, "/home/stamylew/volumes/training_data/50cube3_bp.h5", "all_labels/n3/w_1_2_3")
 #q = adjust_predict_file(q)
 #q = exclude_ignore_label(g, q)
 #
 #
 #
 #c = config(p[0], p[1])
 #c.show_quality()
 #
 #d = config(q[0], q[1])
 #d.show_quality()
 
 I = read_h5("/home/stamylew/volumes/trimaps/50cube2_tri.h5", "50cube2_tri")
 
 II = read_h5("/home/stamylew/src/autocontext/prediction/cache/smallcubes_probs.h5", "exported_data")
 II = adjust_predict(II)
 
 
 q = exclude_ignore_label(I,II)
 c = predict_class(q[0], q[1])
 c.show_quality()
 print c.return_quality()
 
 #III = read_h5("/home/stamylew/volumes/training_data/50cube3_bp.h5", "all_labels/n3/w_3_2_1")
 #III = adjust_predict_file(III)
 
 #plt.figure()
 #plt.imshow(I[11])
 #
 #plt.figure()
 #plt.imshow(II[11])