Example #1
0
def test_storing_loading():
    """Test store_preds and load_preds"""

    # Create confusion matrices for random classifiers
    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred1 = np.random.randint(3, size=12)

    cm1 = ConfusionMatrix(yactual, ypred1, "cls_1")

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred2 = np.random.randint(3, size=12)

    cm2 = ConfusionMatrix(yactual, ypred2, "cls_2")

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred3 = np.random.randint(3, size=12)

    cm3 = ConfusionMatrix(yactual, ypred3, "cls_3")

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred4 = np.random.randint(3, size=12)

    cm4 = ConfusionMatrix(yactual, ypred4, "cls_4")

    preds = [ypred1, ypred2, ypred3, ypred4]

    print("Preds before saving", preds)

    store_preds(preds, yactual, 1)
    new_preds, actual = load_preds(1)

    print("Preds after saving", new_preds, "Actual after saving", actual)
Example #2
0
def two_years_analysis(two_years_df
                       , first_metric
                       , second_metric
                       , key):
    print()
    print("Co-change"
          , first_metric
          , second_metric)
    g = two_years_df.groupby([first_metric, second_metric]
                             , as_index=False).agg({key : 'count'})

    print(g)

    cm = ConfusionMatrix(g_df=g
                             , classifier=first_metric
                             , concept=second_metric, count=key)

    print(cm.summarize())
    print()
    print("Samples", cm.samples())
    print("Both metrics increment match", cm.accuracy())
    print(second_metric
            , " improvement given "
            , first_metric
            , " improvement", cm.precision(), "lift", cm.precision_lift())
    print(first_metric
            , " improvement given "
            , second_metric
            , "improvement",  cm.recall(), "lift", ifnull(safe_divide(ifnull(cm.recall()),cm.hit_rate())) - 1)
    print()
    def fit_predict(self,
                    Gtr,
                    Ytr,
                    Gt,
                    Yt,
                    grid_search,
                    acc_param="F1Mean",
                    RS=0,
                    VERBOSE=False):
        """Learn the model on training dataset and predict on the testing dataset.
        
        Input:
            - Gtr (array): training subset
            - Ytr (array): true labels of training subset
            - Gt (array): testing subset
            - Yt (array): true labels of testing subset
            - grid_search (dict): grid search for the CV
            - acc (str): accuracy parameter for the cross-validation (default "F1Mean", otherwise it will be the OA)
            - RS (int) : random seed used for the stratified k-fold CV
            - VERBOSE (bool): verbose (default: False)
        
        Return:
            - confMatrix (object ConfusionMatrix): confusion matrix issued from the classification
            - yp (array): vector of predicted labels of the testing subset
            - grid.best_params_ (dict): combination of parameters that gave the best results during the CV
        
        TODO: implement own scoring parameter
        """

        if acc_param == "F1Mean":
            score = 'f1_macro'
        else:
            score = 'accuracy'

        ## Initialization of the stratifield K-CV
        cv = StratifiedKFold(Ytr, n_folds=self.n_folds, random_state=RS)

        #Implementation of a fit and a predict methods with parameters from grid search
        grid = GridSearchCV(self.pipe,
                            param_grid=grid_search,
                            scoring=score,
                            verbose=VERBOSE,
                            cv=cv,
                            n_jobs=3)
        grid.fit(Gtr, Ytr)

        model = grid.best_estimator_
        if VERBOSE:
            print grid.best_score_

        #Learn model
        model.fit(Gtr, Ytr)  #could use refit in version 0.19 of sklearn
        #Predict
        yp = model.predict(Gt)

        #Compute confusion matrix
        confMatrix = ConfusionMatrix()
        confMatrix.compute_confusion_matrix(yp, Yt)

        return confMatrix, yp, grid.best_params_
Example #4
0
def evaluate_sense(gold_list, predicted_list):
	"""Evaluate sense classifier

	The label 'no' is for the relations that are missed by the system
	because the arguments don't match any of the gold relations.
	"""
	sense_alphabet = Alphabet()
	for relation in gold_list:
		sense_alphabet.add(relation['Sense'][0])
	sense_alphabet.add('no')
	sense_cm = ConfusionMatrix(sense_alphabet)
	gold_to_predicted_map, predicted_to_gold_map = \
			_link_gold_predicted(gold_list, predicted_list, spans_exact_matching)

	for i, gold_relation in enumerate(gold_list):
		if i in gold_to_predicted_map:
			predicted_sense = gold_to_predicted_map[i]['Sense'][0]
			if predicted_sense in gold_relation['Sense']:
				sense_cm.add(predicted_sense, predicted_sense)
			else:
				if not sense_cm.alphabet.has_label(predicted_sense):
					predicted_sense = 'no'
				sense_cm.add(predicted_sense, gold_relation['Sense'][0])
		else:
			sense_cm.add('no', gold_relation['Sense'][0])

	for i, predicted_relation in enumerate(predicted_list):
		if i not in predicted_to_gold_map:
			predicted_sense = predicted_relation['Sense'][0]
			if not sense_cm.alphabet.has_label(predicted_sense):
				predicted_sense = 'no'
			sense_cm.add(predicted_sense, 'no')
	return sense_cm
Example #5
0
def compute_binary_eval_metric(gold_list, predicted_list, matching_fn):
    """Compute binary evaluation metric

    """
    binary_alphabet = Alphabet()
    binary_alphabet.add('yes')
    binary_alphabet.add('no')
    cm = ConfusionMatrix(binary_alphabet)
    matched_predicted = [False for x in predicted_list]
    for gold_span in gold_list:
        found_match = False
        for i, predicted_span in enumerate(predicted_list):
            if matching_fn(gold_span,
                           predicted_span) and not matched_predicted[i]:
                cm.add('yes', 'yes')
                matched_predicted[i] = True
                found_match = True
                break
        if not found_match:
            cm.add('no', 'yes')
    # Predicted span that does not match with any
    for matched in matched_predicted:
        if not matched:
            cm.add('yes', 'no')
    return cm
def train(model, iterator, optimizer, criterion, reg_ratio):
    model.train()

    epoch_loss = 0
    conf_mat = ConfusionMatrix(np.zeros((NUM_CLASSES, NUM_CLASSES)))

    for batch, labels in iterator():
        optimizer.zero_grad()

        predictions = model(batch).squeeze(1)

        loss = criterion(predictions, labels)

        reg_loss = 0
        for param in model.parameters():
            reg_loss += param.norm(2)

        total_loss = loss + reg_ratio*reg_loss
        total_loss.backward()

        optimizer.step()

        epoch_loss += loss.item()

        conf_mat += ConfusionMatrix.from_predictions(predictions, labels)

    average_loss = epoch_loss / len(iterator)

    return average_loss, conf_mat
def train_ensemble(model, acoustic_iterator, linguistic_iterator, optimizer, criterion, reg_ratio):
    model.train()

    epoch_loss = 0
    conf_mat = ConfusionMatrix(np.zeros((NUM_CLASSES, NUM_CLASSES)))

    assert len(acoustic_iterator) == len(linguistic_iterator)

    for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(), linguistic_iterator()):
        acoustic_batch = acoustic_tuple[0]
        labels = acoustic_tuple[1]
        linguistic_batch = linguistic_tuple[0]
        optimizer.zero_grad()

        predictions = model(acoustic_batch, linguistic_batch).squeeze(1)

        loss = criterion(predictions, labels)

        reg_loss = 0
        for param in model.parameters():
            reg_loss += param.norm(2)

        total_loss = loss + reg_ratio*reg_loss
        total_loss.backward()

        optimizer.step()

        epoch_loss += loss.item()

        conf_mat += ConfusionMatrix.from_predictions(predictions, labels)

    average_loss = epoch_loss / len(acoustic_iterator)

    return average_loss, conf_mat
    def predict(
        self, dataset: Dataset
    ) -> dict[str, Union[str, list[str], float, ConfusionMatrix]]:
        """predicts the class labels of the given test set, based on a previously fitted model.

        :param dataset: dataset consisting of
        :return: list of predicted values
        """

        prediction_params: dict[str, Union[str, list[str], float,
                                           ConfusionMatrix]] = {
                                               "branches": self.__branches
                                           }
        predicted_values: list[str] = []
        for example, label in dataset:
            example: dict[str, str]
            label: str

            # predict the example
            predicted_values.append(self.__label_example(example, self.__root))

        prediction_params["predictions"]: list[str] = predicted_values
        prediction_params["accuracy"] = accuracy(dataset.label_sample,
                                                 predicted_values)
        prediction_params[
            "confusion_matrix"]: ConfusionMatrix = ConfusionMatrix(
                dataset.label_space, dataset.label_sample, predicted_values)

        return prediction_params
Example #9
0
 def setUp(self):
     label_names = ['banana', 'apple', 'orange']
     labels = np.array(
         [
             [0, 1, 0],
             [1, 0, 0],
             [1, 0, 0],
             [0, 1, 0],
             [0, 0, 1],
             [0, 0, 1],
             [1, 0, 0],
             [0, 1, 0],
             [1, 0, 0],
             [0, 0, 1],
             [0, 0, 1]
         ]
     )
     predictions = np.array(
         [
             [0.9, 0.1, 0.0],
             [0.7, 0.1, 0.2],
             [0.4, 0.3, 0.3],
             [0.2, 0.6, 0.2],
             [0.5, 0.2, 0.3],
             [0.0, 0.0, 1.0],
             [0.2, 0.1, 0.7],
             [0.1, 0.8, 0.1],
             [0.3, 0.5, 0.2],
             [0.1, 0.0, 0.9],
             [0.0, 1.0, 0.0]
         ]
     )
     self.confusion_matrix = ConfusionMatrix(predictions, labels, class_names=label_names)
Example #10
0
def bootstrap_diff(df, ccp_estimator, rounds, sample_size):
    bootstrap_results = []
    for i in range(rounds):
        # Get first model parameters
        s1 = df.sample(sample_size, replace=True)
        bug_g = s1.groupby([classifier, concept],
                           as_index=False).agg({count: 'count'})
        bug_cm = ConfusionMatrix(g_df=bug_g,
                                 classifier=classifier,
                                 concept=concept,
                                 count=count)

        positive_rate = bug_cm.positive_rate()
        hit_rate = bug_cm.hit_rate()
        ccp = ccp_estimator.estimate_positives(hit_rate)
        ccp_diff = ccp - positive_rate

        # Find difference in given points
        bootstrap_results.append([positive_rate, hit_rate, ccp, ccp_diff])

        if (i % 100 == 0):
            print("finished " + str(i), datetime.datetime.now())

    results_df = pd.DataFrame(
        bootstrap_results,
        columns=['positive_rate', 'hit_rate', 'ccp', 'ccp_diff'])
    return results_df
Example #11
0
def evaluate_sense(relation_pairs, valid_senses):
    sense_alphabet = Alphabet()
    #for g_relation, _ in relation_pairs:
    #if g_relation is not None:
    #sense = g_relation['Sense'][0]
    #if sense in valid_senses:
    #sense_alphabet.add(sense)
    for sense in valid_senses:
        sense_alphabet.add(sense)

    sense_alphabet.add(ConfusionMatrix.NEGATIVE_CLASS)
    sense_alphabet.growing = False

    sense_cm = ConfusionMatrix(sense_alphabet)
    for g_relation, p_relation in relation_pairs:
        assert g_relation is not None or p_relation is not None
        if g_relation is None:
            predicted_sense = p_relation['Sense'][0]
            sense_cm.add(predicted_sense, ConfusionMatrix.NEGATIVE_CLASS)
        elif p_relation is None:
            gold_sense = g_relation['Sense'][0]
            if gold_sense in valid_senses:
                sense_cm.add(ConfusionMatrix.NEGATIVE_CLASS, gold_sense)
        else:
            predicted_sense = p_relation['Sense'][0]
            gold_sense = g_relation['Sense'][0]
            if gold_sense in valid_senses:
                sense_cm.add(predicted_sense, gold_sense)
    return sense_cm
Example #12
0
def test_confusion_matrix():
    """Test ConfusionMatrix class"""

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]

    cm = ConfusionMatrix(yactual, ypred, "Sample_classifier")

    number_cm = cm.get_number_cm()
    normalized_cm = cm.get_normalized_cm()
    metrics = {
        "TP": cm.get_true_pos(),
        "TN": cm.get_true_neg(),
        "FP": cm.get_false_pos(),
        "FN": cm.get_false_neg()
    }

    print("Confusion matrix:")
    print(number_cm)

    print("Normalized confusion matrix:")
    print(normalized_cm)

    print("Precision per label is:", cm.get_precision())
    print("Metrics:", metrics)

    print("MCC is", cm.get_mcc())

    return cm.get_precision()
Example #13
0
def make_confusion_matrix(results):
    true_positives = [r for r in results if r.tp]
    true_negatives = [r for r in results if r.tn]
    false_positives = [r for r in results if r.fp]
    false_negatives = [r for r in results if r.fn]

    return ConfusionMatrix(len(true_positives), len(false_positives),
                           len(true_negatives), len(false_negatives))
Example #14
0
 def evaluate(self, test_set):
     x = test_set.x
     x = self.preprocess_x(x)
     y = test_set.y
     y = np_utils.to_categorical(y, test_set.num_classes)
     y_pred = self.model.predict_proba(x, verbose=0)
     cf = ConfusionMatrix(y_pred, y)
     return cf
Example #15
0
def make_pr_graph(entropies, correct, graph_name, title, mpl_figure=None):
    """
    Plot entropy as a PR curve predicting whether examples are correct
    or incorrect.
    """
    if mpl_figure is None:
        mpl_figure = mpl.figure()

    axes = mpl_figure.gca()
    assert len(entropies) == len(correct), (len(entropies), len(correct))
    pairs = zip(entropies, correct)
    pairs.sort()

    max_entropy = float(pairs[-1][0])
    min_entropy = float(pairs[0][0])
    num_segments = 20.0
    segment_size = (max_entropy - min_entropy)/num_segments
    if segment_size == 0:
        segment_size = 0.01
    thresholds = na.arange(min_entropy, max_entropy + 1, segment_size)
    X_recall = []
    Y_precision = []

    for threshold in thresholds:
        predicted_correct = [(entropy, correct)
                             for (entropy, correct) in pairs
                             if entropy < threshold]
        predicted_incorrect = [(entropy, correct)
                             for (entropy, correct) in pairs
                               if entropy >= threshold]
        tp = len([(entropy, correct) for entropy, correct in predicted_incorrect
                 if not correct])
        fp = len([(entropy, correct) for entropy, correct in predicted_incorrect
                 if correct])

        tn = len([(entropy, correct) for entropy, correct in predicted_correct
                 if correct])
        fn = len([(entropy, correct) for entropy, correct in predicted_correct
                 if not correct])
        if tp == 0:
            continue
        cm = ConfusionMatrix(tp, fp, tn, fn)
        X_recall.append(cm.recall)
        Y_precision.append(cm.precision)

    global marker_i
    graph_name_paper = ENTROPY_METRIC_PAPER_NAMES[graph_name]
    axes.plot(X_recall, Y_precision, label=graph_name_paper,
              marker=markers[marker_i])
    marker_i = (marker_i + 1) % len(markers)
    axes.legend(loc='upper right')
    axes.set_xlabel("Recall")
    axes.set_ylabel("Precision")
    axes.set_ylim(0, 1.1)
    axes.set_xlim(0, 1.1)
    axes.set_title("Precision vs Recall")
    mpl_figure.savefig(title.replace(" ", "_") + ".eps")
    mpl.show()
def evaluate_performance(df, classification_column, concept_column):
    g = df.groupby([classification_column, concept_column],
                   as_index=False).agg({'commit': 'count'})
    cm = ConfusionMatrix(g_df=g,
                         classifier=classification_column,
                         concept=concept_column,
                         count='commit')

    return cm.summarize()
    def train_and_test(self, train_set, test_set):
        self.classifier = self.classifier_class.train(train_set)

        predicted_polarities = [
            self.classify(document) for (document, polarity) in test_set
        ]
        actual_polarities = [polarity for (document, polarity) in test_set]

        return ConfusionMatrix(predicted_polarities, actual_polarities)
Example #18
0
def evaluate_objects(model, corpus_fname, state_type):
    corpus = annotationIo.load(corpus_fname)
    state_cls = state_type_from_name(state_type)
    from g3.inference import nodeSearch
    taskPlanner = nodeSearch.BeamSearch(model)
    predictions = []
    done = False
    phrases = set()
    for i, annotation in enumerate(corpus):
        start_state = state_cls.from_context(annotation.context)

        for esdc in annotation.esdcs:
            #if esdc.text != "the pallet of boxes":
            #    continue
            #if esdc.text in phrases:
            #    continue
            isCorrect = annotation.isGroundingCorrect(esdc)
            if isCorrect != None:
                ggg = ggg_from_esdc(esdc)
                groundings = annotation.getGroundings(esdc)
                assert len(groundings) == 1
                grounding = groundings[0]
                #if "generator" not in grounding.tags:
                #    continue
                prob = evaluate_ggg(ggg, grounding, start_state, taskPlanner)
                if prob > 0.7:
                    predicted_class = True
                else:
                    predicted_class = False
                predictions.append((predicted_class, isCorrect))
                #print "Query: Is object", " ".join(grounding.tags),
                #print "'" + esdc.text + "'?"
                #print "System: ",
                #if predicted_class:
                #    print "Yes."
                #else:
                #    print "No."
                #done = True
                phrases.add(esdc.text)
            if done:
                break
        if done:
            break

    tp = len([(p, l) for p, l in predictions if p and p == l])
    fp = len([(p, l) for p, l in predictions if p and p != l])
    tn = len([(p, l) for p, l in predictions if not p and p == l])
    fn = len([(p, l) for p, l in predictions if not p and p != l])
    cm = ConfusionMatrix(tp, fp, tn, fn)
    #cm.print_all()

    #if len(phrases) > 20:
    #    phrases = random.sample(phrases, 20)
    #for phrase in sorted(phrases):
    #    print phrase
    return cm
Example #19
0
def Evaluation_all(gold_label, predict_label):
    binary_alphabet = Alphabet()
    for i in range(20):
        binary_alphabet.add(DICT_INDEX_TO_LABEL[i])

    cm = ConfusionMatrix(binary_alphabet)
    cm.add_list(predict_label, gold_label)
    macro_p, macro_r, macro_f1 = cm.get_average_prf()
    overall_accuracy = cm.get_accuracy()
    return overall_accuracy, macro_p, macro_r, macro_f1
Example #20
0
def run_multiple_voting():
    """Run tests on multiple weighting systems given stored predictions of the classifiers in main()
       To be used in conjunction with alternative_main to determine which weighting method performs better
   """

    # Load predictions from all classifiers and actual labels for test_set_1
    preds1, actual1 = load_preds(1)

    # Load predictions from all classifiers and actual labels for test_set_2
    preds2, actual2 = load_preds(2)

    # Create confusion matrices for each classifier
    b_cm = ConfusionMatrix(actual1, preds1[0], "Bayes")
    p_cm = ConfusionMatrix(actual1, preds1[1], "Proximity")
    v_cm = ConfusionMatrix(actual1, preds1[2], "Voting")
    # r_cm = ConfusionMatrix(actual1, preds1[2], "LSTM")

    confusionMatrices = [b_cm, p_cm, v_cm]
    # confusionMatices = [p_cm, v_cm, b_cm, r_cm]

    # Save individual confusion matrices to files
    for cm in confusionMatrices:
        cm.store_cm()

    print("Individual confusion matrices created and stored!")

    # Weight second set of results, using confusion matrices from first set
    weightingInput = [
        [confusionMatrices[0], preds2[0]], [confusionMatrices[1], preds2[1]]
        # [confusionMatrices[2] ,b.batchTest(test_set2)],
        # [confusionMatrices[3], r.batchTest(test_set2)],
    ]

    # Get the weighted voting results
    votes_p = voting(weightingInput, "Precision")
    votes_CEN_p = voting(weightingInput, "CEN_Precision")
    votes_CEN = voting(weightingInput, "CEN")
    votes_eq = voting(weightingInput, "Equal_Vote")

    # Check metrics
    print(classification_report(actual2, votes_p))
    print(classification_report(actual2, votes_CEN_p))
    print(classification_report(actual2, votes_CEN))
    print(classification_report(actual2, votes_eq))

    # Create final confusion matrices depending on votes
    p_cm = ConfusionMatrix(actual2, votes_p, "Precision")
    p_CEN_cm = ConfusionMatrix(actual2, votes_CEN_p, "CEN_Precision")
    CEN_cm = ConfusionMatrix(actual2, votes_CEN, "CEN")
    eq_cm = ConfusionMatrix(actual2, votes_eq, "Equal")

    # Store confusion matrices
    p_cm.store_cm()
    p_CEN_cm.store_cm()
    CEN_cm.store_cm()
    eq_cm.store_cm()

    return votes_p, votes_CEN_p, votes_CEN, votes_eq
Example #21
0
def applyClassifiers(scst_pkl_path: str,
                     keras_pkl_path: str,
                     data_path: Optional[str] = None,
                     truth_path: Optional[str] = None,
                     start_second: Optional[float] = None,
                     end_second: Optional[float] = None) -> None:
    '''
    Apply both a ``SCSTClassifier`` and a ``TransientKerasClassifier`` to the data sequence
    specified by the inputs, and plot the results.

    :param scst_pkl_path: Full path to a pickle file containing a saved ``SCSTClassifier`` object.
    :param keras_pkl_path: Full path to a pickle file containing a saved
        ``TransientKerasClassifier`` object.
    :param data_path, truth_path, start_second, end_second: See ``data_utils.getPlotDataTuple``.
    '''

    # Define SCST classification parameters
    SIG_THRESH = 100
    NOISE_THRESH = 100

    # Load data and classifiers
    test_matrix, truth_array, start_second, title = getPlotDataTuple(
        truth_path, data_path, start_second, end_second)
    scst_classifier = SCSTClassifier.load(scst_pkl_path)
    keras_classifier = TransientKerasClassifier.load(keras_pkl_path)

    # Apply classifiers
    num_obs = test_matrix.shape[1]
    for test_idx in range(num_obs):
        scst_classifier.classify(test_matrix[:, test_idx], SIG_THRESH,
                                 NOISE_THRESH)
        keras_classifier.classify(test_matrix[:, test_idx])
        _printProgress(test_idx / float(num_obs), title)

    _printProgress(1.0, title)
    sys.stdout.write("\n")

    # Display results
    id_array_list = [
        scst_classifier.class_labels, keras_classifier.class_labels
    ]
    id_tag_list = ["SCST IDs", "Keras IDs"]

    if truth_array is not None:
        for classifier, name in zip([scst_classifier, keras_classifier],
                                    ["SCST", "Keras"]):
            conf_matrix = ConfusionMatrix(classifier.class_labels, truth_array,
                                          True, name)
            conf_matrix.display()

        id_array_list.append(truth_array)
        id_tag_list.append("Truth IDs")

    plotSequence(test_matrix, start_second, title, id_array_list, id_tag_list)
    plt.show()
Example #22
0
def test_weighting():
    """Test weighting.py"""

    # Create confusion matrices for random classifiers
    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred1 = np.random.randint(3, size=12)

    cm1 = ConfusionMatrix(yactual, ypred1, "cls_1")

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred2 = np.random.randint(3, size=12)

    cm2 = ConfusionMatrix(yactual, ypred2, "cls_2")

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred3 = np.random.randint(3, size=12)

    cm3 = ConfusionMatrix(yactual, ypred3, "cls_3")

    yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
    ypred4 = np.random.randint(3, size=12)

    cm4 = ConfusionMatrix(yactual, ypred4, "cls_4")

    weight_pairs = [[cm1, ypred1], [cm2, ypred2], [cm3, ypred3], [cm4, ypred4]]

    # Check that CEN score is being calculated
    #print(cm1.get_CEN_score(), cm2.get_CEN_score(), cm3.get_CEN_score(), cm4.get_CEN_score())

    # Get final votes based on pairs
    votes_p = voting(weight_pairs, "Precision")
    votes_CEN_p = voting(weight_pairs, "CEN_Precision")
    votes_CEN = voting(weight_pairs, "CEN")
    votes_eq = voting(weight_pairs, "Equal_Vote")

    # Check metrics
    print(classification_report(yactual, votes_p))
    print(classification_report(yactual, votes_CEN_p))
    print(classification_report(yactual, votes_CEN))
    print(classification_report(yactual, votes_eq))

    # Create final confusion matrices depending on votes
    p_cm = ConfusionMatrix(yactual, votes_p, "Precision_Voting")
    p_CEN_cm = ConfusionMatrix(yactual, votes_CEN_p, "CEN_Precision_Voting")
    CEN_cm = ConfusionMatrix(yactual, votes_CEN, "CEN_Voting")
    eq_cm = ConfusionMatrix(yactual, votes_eq, "Equal_Voting")

    # Store confusion matrices
    p_cm.store_cm()
    p_CEN_cm.store_cm()
    CEN_cm.store_cm()
    eq_cm.store_cm()
Example #23
0
def adaptive_performance(df):
    adaptive_g = df.groupby(['adaptive_pred', 'Is_Adaptive'],
                            as_index=False).agg({'commit': 'count'})
    adaptive_cm = ConfusionMatrix(g_df=adaptive_g,
                                  classifier='adaptive_pred',
                                  concept='Is_Adaptive',
                                  count='commit')
    print("adaptive commit performance")
    print(adaptive_cm.summarize())

    return adaptive_cm
Example #24
0
def refactor_performance(df):
    refactor_g = df.groupby(['is_refactor_pred', 'Is_Refactor'],
                            as_index=False).agg({'commit': 'count'})
    refactor_cm = ConfusionMatrix(g_df=refactor_g,
                                  classifier='is_refactor_pred',
                                  concept='Is_Refactor',
                                  count='commit')
    print("refactor commit performance")
    print(refactor_cm.summarize())

    return refactor_cm
Example #25
0
def corrective_performance(df):
    bug_g = df.groupby(['corrective_pred', 'Is_Corrective'],
                       as_index=False).agg({'commit': 'count'})
    bug_cm = ConfusionMatrix(g_df=bug_g,
                             classifier='corrective_pred',
                             concept='Is_Corrective',
                             count='commit')
    print("corrective commit performance")
    print(bug_cm.summarize())

    return bug_cm
def test_independent_prob(classifier
                  , concept
                  , count
                  , g_df
                  , expected):
    cm = ConfusionMatrix(classifier
                         , concept
                         , count
                         , g_df)
    actual = cm.independent_prob()
    assert expected == actual
def test_summrize(classifier
                         , concept
                         , count
                         , g_df
                          , expected):
    cm = ConfusionMatrix(classifier
                         , concept
                         , count
                         , g_df)
    actual = cm.summarize()
    assert expected == actual
Example #28
0
    def compute_all_data(self):

        # 生成数据
        for thr in self.thresholds:
            cm = ConfusionMatrix(self.mu_true, self.sigma_true, self.mu_false,
                                 self.sigma_false, thr)
            self.Ps.append(cm.P)
            self.Rs.append(cm.R)
            self.TPRs.append(cm.TPR)
            self.FPRs.append(cm.FPR)
            self.F1s.append(cm.F1)
Example #29
0
    def evaluate_combined(self, test_sets, models):
        x_test = [None] * len(test_sets)
        for i, test_set in enumerate(test_sets):
            x_test[i] = test_set.x
            x_test[i] = models[i].preprocess_x(x_test[i])

        y = test_sets[0].y
        y = np_utils.to_categorical(y, test_sets[0].num_classes)
        y_pred = self.model.predict_proba(x_test, verbose=0)
        cf = ConfusionMatrix(y_pred, y)
        return cf
def perfective_performance(df):
    perfective_g = df.groupby(
        ['perfective_pred', 'Is_Perfective'], as_index=False).agg({'commit' : 'count'})
    perfective_cm = ConfusionMatrix(g_df=perfective_g
                                  , classifier='perfective_pred'
                                  , concept='Is_Perfective'
                                  , count='commit')
    print("perfective commit performance")
    print(perfective_cm.summarize())

    return perfective_cm