def test_storing_loading(): """Test store_preds and load_preds""" # Create confusion matrices for random classifiers yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred1 = np.random.randint(3, size=12) cm1 = ConfusionMatrix(yactual, ypred1, "cls_1") yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred2 = np.random.randint(3, size=12) cm2 = ConfusionMatrix(yactual, ypred2, "cls_2") yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred3 = np.random.randint(3, size=12) cm3 = ConfusionMatrix(yactual, ypred3, "cls_3") yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred4 = np.random.randint(3, size=12) cm4 = ConfusionMatrix(yactual, ypred4, "cls_4") preds = [ypred1, ypred2, ypred3, ypred4] print("Preds before saving", preds) store_preds(preds, yactual, 1) new_preds, actual = load_preds(1) print("Preds after saving", new_preds, "Actual after saving", actual)
def two_years_analysis(two_years_df , first_metric , second_metric , key): print() print("Co-change" , first_metric , second_metric) g = two_years_df.groupby([first_metric, second_metric] , as_index=False).agg({key : 'count'}) print(g) cm = ConfusionMatrix(g_df=g , classifier=first_metric , concept=second_metric, count=key) print(cm.summarize()) print() print("Samples", cm.samples()) print("Both metrics increment match", cm.accuracy()) print(second_metric , " improvement given " , first_metric , " improvement", cm.precision(), "lift", cm.precision_lift()) print(first_metric , " improvement given " , second_metric , "improvement", cm.recall(), "lift", ifnull(safe_divide(ifnull(cm.recall()),cm.hit_rate())) - 1) print()
def fit_predict(self, Gtr, Ytr, Gt, Yt, grid_search, acc_param="F1Mean", RS=0, VERBOSE=False): """Learn the model on training dataset and predict on the testing dataset. Input: - Gtr (array): training subset - Ytr (array): true labels of training subset - Gt (array): testing subset - Yt (array): true labels of testing subset - grid_search (dict): grid search for the CV - acc (str): accuracy parameter for the cross-validation (default "F1Mean", otherwise it will be the OA) - RS (int) : random seed used for the stratified k-fold CV - VERBOSE (bool): verbose (default: False) Return: - confMatrix (object ConfusionMatrix): confusion matrix issued from the classification - yp (array): vector of predicted labels of the testing subset - grid.best_params_ (dict): combination of parameters that gave the best results during the CV TODO: implement own scoring parameter """ if acc_param == "F1Mean": score = 'f1_macro' else: score = 'accuracy' ## Initialization of the stratifield K-CV cv = StratifiedKFold(Ytr, n_folds=self.n_folds, random_state=RS) #Implementation of a fit and a predict methods with parameters from grid search grid = GridSearchCV(self.pipe, param_grid=grid_search, scoring=score, verbose=VERBOSE, cv=cv, n_jobs=3) grid.fit(Gtr, Ytr) model = grid.best_estimator_ if VERBOSE: print grid.best_score_ #Learn model model.fit(Gtr, Ytr) #could use refit in version 0.19 of sklearn #Predict yp = model.predict(Gt) #Compute confusion matrix confMatrix = ConfusionMatrix() confMatrix.compute_confusion_matrix(yp, Yt) return confMatrix, yp, grid.best_params_
def evaluate_sense(gold_list, predicted_list): """Evaluate sense classifier The label 'no' is for the relations that are missed by the system because the arguments don't match any of the gold relations. """ sense_alphabet = Alphabet() for relation in gold_list: sense_alphabet.add(relation['Sense'][0]) sense_alphabet.add('no') sense_cm = ConfusionMatrix(sense_alphabet) gold_to_predicted_map, predicted_to_gold_map = \ _link_gold_predicted(gold_list, predicted_list, spans_exact_matching) for i, gold_relation in enumerate(gold_list): if i in gold_to_predicted_map: predicted_sense = gold_to_predicted_map[i]['Sense'][0] if predicted_sense in gold_relation['Sense']: sense_cm.add(predicted_sense, predicted_sense) else: if not sense_cm.alphabet.has_label(predicted_sense): predicted_sense = 'no' sense_cm.add(predicted_sense, gold_relation['Sense'][0]) else: sense_cm.add('no', gold_relation['Sense'][0]) for i, predicted_relation in enumerate(predicted_list): if i not in predicted_to_gold_map: predicted_sense = predicted_relation['Sense'][0] if not sense_cm.alphabet.has_label(predicted_sense): predicted_sense = 'no' sense_cm.add(predicted_sense, 'no') return sense_cm
def compute_binary_eval_metric(gold_list, predicted_list, matching_fn): """Compute binary evaluation metric """ binary_alphabet = Alphabet() binary_alphabet.add('yes') binary_alphabet.add('no') cm = ConfusionMatrix(binary_alphabet) matched_predicted = [False for x in predicted_list] for gold_span in gold_list: found_match = False for i, predicted_span in enumerate(predicted_list): if matching_fn(gold_span, predicted_span) and not matched_predicted[i]: cm.add('yes', 'yes') matched_predicted[i] = True found_match = True break if not found_match: cm.add('no', 'yes') # Predicted span that does not match with any for matched in matched_predicted: if not matched: cm.add('yes', 'no') return cm
def train(model, iterator, optimizer, criterion, reg_ratio): model.train() epoch_loss = 0 conf_mat = ConfusionMatrix(np.zeros((NUM_CLASSES, NUM_CLASSES))) for batch, labels in iterator(): optimizer.zero_grad() predictions = model(batch).squeeze(1) loss = criterion(predictions, labels) reg_loss = 0 for param in model.parameters(): reg_loss += param.norm(2) total_loss = loss + reg_ratio*reg_loss total_loss.backward() optimizer.step() epoch_loss += loss.item() conf_mat += ConfusionMatrix.from_predictions(predictions, labels) average_loss = epoch_loss / len(iterator) return average_loss, conf_mat
def train_ensemble(model, acoustic_iterator, linguistic_iterator, optimizer, criterion, reg_ratio): model.train() epoch_loss = 0 conf_mat = ConfusionMatrix(np.zeros((NUM_CLASSES, NUM_CLASSES))) assert len(acoustic_iterator) == len(linguistic_iterator) for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(), linguistic_iterator()): acoustic_batch = acoustic_tuple[0] labels = acoustic_tuple[1] linguistic_batch = linguistic_tuple[0] optimizer.zero_grad() predictions = model(acoustic_batch, linguistic_batch).squeeze(1) loss = criterion(predictions, labels) reg_loss = 0 for param in model.parameters(): reg_loss += param.norm(2) total_loss = loss + reg_ratio*reg_loss total_loss.backward() optimizer.step() epoch_loss += loss.item() conf_mat += ConfusionMatrix.from_predictions(predictions, labels) average_loss = epoch_loss / len(acoustic_iterator) return average_loss, conf_mat
def predict( self, dataset: Dataset ) -> dict[str, Union[str, list[str], float, ConfusionMatrix]]: """predicts the class labels of the given test set, based on a previously fitted model. :param dataset: dataset consisting of :return: list of predicted values """ prediction_params: dict[str, Union[str, list[str], float, ConfusionMatrix]] = { "branches": self.__branches } predicted_values: list[str] = [] for example, label in dataset: example: dict[str, str] label: str # predict the example predicted_values.append(self.__label_example(example, self.__root)) prediction_params["predictions"]: list[str] = predicted_values prediction_params["accuracy"] = accuracy(dataset.label_sample, predicted_values) prediction_params[ "confusion_matrix"]: ConfusionMatrix = ConfusionMatrix( dataset.label_space, dataset.label_sample, predicted_values) return prediction_params
def setUp(self): label_names = ['banana', 'apple', 'orange'] labels = np.array( [ [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1] ] ) predictions = np.array( [ [0.9, 0.1, 0.0], [0.7, 0.1, 0.2], [0.4, 0.3, 0.3], [0.2, 0.6, 0.2], [0.5, 0.2, 0.3], [0.0, 0.0, 1.0], [0.2, 0.1, 0.7], [0.1, 0.8, 0.1], [0.3, 0.5, 0.2], [0.1, 0.0, 0.9], [0.0, 1.0, 0.0] ] ) self.confusion_matrix = ConfusionMatrix(predictions, labels, class_names=label_names)
def bootstrap_diff(df, ccp_estimator, rounds, sample_size): bootstrap_results = [] for i in range(rounds): # Get first model parameters s1 = df.sample(sample_size, replace=True) bug_g = s1.groupby([classifier, concept], as_index=False).agg({count: 'count'}) bug_cm = ConfusionMatrix(g_df=bug_g, classifier=classifier, concept=concept, count=count) positive_rate = bug_cm.positive_rate() hit_rate = bug_cm.hit_rate() ccp = ccp_estimator.estimate_positives(hit_rate) ccp_diff = ccp - positive_rate # Find difference in given points bootstrap_results.append([positive_rate, hit_rate, ccp, ccp_diff]) if (i % 100 == 0): print("finished " + str(i), datetime.datetime.now()) results_df = pd.DataFrame( bootstrap_results, columns=['positive_rate', 'hit_rate', 'ccp', 'ccp_diff']) return results_df
def evaluate_sense(relation_pairs, valid_senses): sense_alphabet = Alphabet() #for g_relation, _ in relation_pairs: #if g_relation is not None: #sense = g_relation['Sense'][0] #if sense in valid_senses: #sense_alphabet.add(sense) for sense in valid_senses: sense_alphabet.add(sense) sense_alphabet.add(ConfusionMatrix.NEGATIVE_CLASS) sense_alphabet.growing = False sense_cm = ConfusionMatrix(sense_alphabet) for g_relation, p_relation in relation_pairs: assert g_relation is not None or p_relation is not None if g_relation is None: predicted_sense = p_relation['Sense'][0] sense_cm.add(predicted_sense, ConfusionMatrix.NEGATIVE_CLASS) elif p_relation is None: gold_sense = g_relation['Sense'][0] if gold_sense in valid_senses: sense_cm.add(ConfusionMatrix.NEGATIVE_CLASS, gold_sense) else: predicted_sense = p_relation['Sense'][0] gold_sense = g_relation['Sense'][0] if gold_sense in valid_senses: sense_cm.add(predicted_sense, gold_sense) return sense_cm
def test_confusion_matrix(): """Test ConfusionMatrix class""" yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2] cm = ConfusionMatrix(yactual, ypred, "Sample_classifier") number_cm = cm.get_number_cm() normalized_cm = cm.get_normalized_cm() metrics = { "TP": cm.get_true_pos(), "TN": cm.get_true_neg(), "FP": cm.get_false_pos(), "FN": cm.get_false_neg() } print("Confusion matrix:") print(number_cm) print("Normalized confusion matrix:") print(normalized_cm) print("Precision per label is:", cm.get_precision()) print("Metrics:", metrics) print("MCC is", cm.get_mcc()) return cm.get_precision()
def make_confusion_matrix(results): true_positives = [r for r in results if r.tp] true_negatives = [r for r in results if r.tn] false_positives = [r for r in results if r.fp] false_negatives = [r for r in results if r.fn] return ConfusionMatrix(len(true_positives), len(false_positives), len(true_negatives), len(false_negatives))
def evaluate(self, test_set): x = test_set.x x = self.preprocess_x(x) y = test_set.y y = np_utils.to_categorical(y, test_set.num_classes) y_pred = self.model.predict_proba(x, verbose=0) cf = ConfusionMatrix(y_pred, y) return cf
def make_pr_graph(entropies, correct, graph_name, title, mpl_figure=None): """ Plot entropy as a PR curve predicting whether examples are correct or incorrect. """ if mpl_figure is None: mpl_figure = mpl.figure() axes = mpl_figure.gca() assert len(entropies) == len(correct), (len(entropies), len(correct)) pairs = zip(entropies, correct) pairs.sort() max_entropy = float(pairs[-1][0]) min_entropy = float(pairs[0][0]) num_segments = 20.0 segment_size = (max_entropy - min_entropy)/num_segments if segment_size == 0: segment_size = 0.01 thresholds = na.arange(min_entropy, max_entropy + 1, segment_size) X_recall = [] Y_precision = [] for threshold in thresholds: predicted_correct = [(entropy, correct) for (entropy, correct) in pairs if entropy < threshold] predicted_incorrect = [(entropy, correct) for (entropy, correct) in pairs if entropy >= threshold] tp = len([(entropy, correct) for entropy, correct in predicted_incorrect if not correct]) fp = len([(entropy, correct) for entropy, correct in predicted_incorrect if correct]) tn = len([(entropy, correct) for entropy, correct in predicted_correct if correct]) fn = len([(entropy, correct) for entropy, correct in predicted_correct if not correct]) if tp == 0: continue cm = ConfusionMatrix(tp, fp, tn, fn) X_recall.append(cm.recall) Y_precision.append(cm.precision) global marker_i graph_name_paper = ENTROPY_METRIC_PAPER_NAMES[graph_name] axes.plot(X_recall, Y_precision, label=graph_name_paper, marker=markers[marker_i]) marker_i = (marker_i + 1) % len(markers) axes.legend(loc='upper right') axes.set_xlabel("Recall") axes.set_ylabel("Precision") axes.set_ylim(0, 1.1) axes.set_xlim(0, 1.1) axes.set_title("Precision vs Recall") mpl_figure.savefig(title.replace(" ", "_") + ".eps") mpl.show()
def evaluate_performance(df, classification_column, concept_column): g = df.groupby([classification_column, concept_column], as_index=False).agg({'commit': 'count'}) cm = ConfusionMatrix(g_df=g, classifier=classification_column, concept=concept_column, count='commit') return cm.summarize()
def train_and_test(self, train_set, test_set): self.classifier = self.classifier_class.train(train_set) predicted_polarities = [ self.classify(document) for (document, polarity) in test_set ] actual_polarities = [polarity for (document, polarity) in test_set] return ConfusionMatrix(predicted_polarities, actual_polarities)
def evaluate_objects(model, corpus_fname, state_type): corpus = annotationIo.load(corpus_fname) state_cls = state_type_from_name(state_type) from g3.inference import nodeSearch taskPlanner = nodeSearch.BeamSearch(model) predictions = [] done = False phrases = set() for i, annotation in enumerate(corpus): start_state = state_cls.from_context(annotation.context) for esdc in annotation.esdcs: #if esdc.text != "the pallet of boxes": # continue #if esdc.text in phrases: # continue isCorrect = annotation.isGroundingCorrect(esdc) if isCorrect != None: ggg = ggg_from_esdc(esdc) groundings = annotation.getGroundings(esdc) assert len(groundings) == 1 grounding = groundings[0] #if "generator" not in grounding.tags: # continue prob = evaluate_ggg(ggg, grounding, start_state, taskPlanner) if prob > 0.7: predicted_class = True else: predicted_class = False predictions.append((predicted_class, isCorrect)) #print "Query: Is object", " ".join(grounding.tags), #print "'" + esdc.text + "'?" #print "System: ", #if predicted_class: # print "Yes." #else: # print "No." #done = True phrases.add(esdc.text) if done: break if done: break tp = len([(p, l) for p, l in predictions if p and p == l]) fp = len([(p, l) for p, l in predictions if p and p != l]) tn = len([(p, l) for p, l in predictions if not p and p == l]) fn = len([(p, l) for p, l in predictions if not p and p != l]) cm = ConfusionMatrix(tp, fp, tn, fn) #cm.print_all() #if len(phrases) > 20: # phrases = random.sample(phrases, 20) #for phrase in sorted(phrases): # print phrase return cm
def Evaluation_all(gold_label, predict_label): binary_alphabet = Alphabet() for i in range(20): binary_alphabet.add(DICT_INDEX_TO_LABEL[i]) cm = ConfusionMatrix(binary_alphabet) cm.add_list(predict_label, gold_label) macro_p, macro_r, macro_f1 = cm.get_average_prf() overall_accuracy = cm.get_accuracy() return overall_accuracy, macro_p, macro_r, macro_f1
def run_multiple_voting(): """Run tests on multiple weighting systems given stored predictions of the classifiers in main() To be used in conjunction with alternative_main to determine which weighting method performs better """ # Load predictions from all classifiers and actual labels for test_set_1 preds1, actual1 = load_preds(1) # Load predictions from all classifiers and actual labels for test_set_2 preds2, actual2 = load_preds(2) # Create confusion matrices for each classifier b_cm = ConfusionMatrix(actual1, preds1[0], "Bayes") p_cm = ConfusionMatrix(actual1, preds1[1], "Proximity") v_cm = ConfusionMatrix(actual1, preds1[2], "Voting") # r_cm = ConfusionMatrix(actual1, preds1[2], "LSTM") confusionMatrices = [b_cm, p_cm, v_cm] # confusionMatices = [p_cm, v_cm, b_cm, r_cm] # Save individual confusion matrices to files for cm in confusionMatrices: cm.store_cm() print("Individual confusion matrices created and stored!") # Weight second set of results, using confusion matrices from first set weightingInput = [ [confusionMatrices[0], preds2[0]], [confusionMatrices[1], preds2[1]] # [confusionMatrices[2] ,b.batchTest(test_set2)], # [confusionMatrices[3], r.batchTest(test_set2)], ] # Get the weighted voting results votes_p = voting(weightingInput, "Precision") votes_CEN_p = voting(weightingInput, "CEN_Precision") votes_CEN = voting(weightingInput, "CEN") votes_eq = voting(weightingInput, "Equal_Vote") # Check metrics print(classification_report(actual2, votes_p)) print(classification_report(actual2, votes_CEN_p)) print(classification_report(actual2, votes_CEN)) print(classification_report(actual2, votes_eq)) # Create final confusion matrices depending on votes p_cm = ConfusionMatrix(actual2, votes_p, "Precision") p_CEN_cm = ConfusionMatrix(actual2, votes_CEN_p, "CEN_Precision") CEN_cm = ConfusionMatrix(actual2, votes_CEN, "CEN") eq_cm = ConfusionMatrix(actual2, votes_eq, "Equal") # Store confusion matrices p_cm.store_cm() p_CEN_cm.store_cm() CEN_cm.store_cm() eq_cm.store_cm() return votes_p, votes_CEN_p, votes_CEN, votes_eq
def applyClassifiers(scst_pkl_path: str, keras_pkl_path: str, data_path: Optional[str] = None, truth_path: Optional[str] = None, start_second: Optional[float] = None, end_second: Optional[float] = None) -> None: ''' Apply both a ``SCSTClassifier`` and a ``TransientKerasClassifier`` to the data sequence specified by the inputs, and plot the results. :param scst_pkl_path: Full path to a pickle file containing a saved ``SCSTClassifier`` object. :param keras_pkl_path: Full path to a pickle file containing a saved ``TransientKerasClassifier`` object. :param data_path, truth_path, start_second, end_second: See ``data_utils.getPlotDataTuple``. ''' # Define SCST classification parameters SIG_THRESH = 100 NOISE_THRESH = 100 # Load data and classifiers test_matrix, truth_array, start_second, title = getPlotDataTuple( truth_path, data_path, start_second, end_second) scst_classifier = SCSTClassifier.load(scst_pkl_path) keras_classifier = TransientKerasClassifier.load(keras_pkl_path) # Apply classifiers num_obs = test_matrix.shape[1] for test_idx in range(num_obs): scst_classifier.classify(test_matrix[:, test_idx], SIG_THRESH, NOISE_THRESH) keras_classifier.classify(test_matrix[:, test_idx]) _printProgress(test_idx / float(num_obs), title) _printProgress(1.0, title) sys.stdout.write("\n") # Display results id_array_list = [ scst_classifier.class_labels, keras_classifier.class_labels ] id_tag_list = ["SCST IDs", "Keras IDs"] if truth_array is not None: for classifier, name in zip([scst_classifier, keras_classifier], ["SCST", "Keras"]): conf_matrix = ConfusionMatrix(classifier.class_labels, truth_array, True, name) conf_matrix.display() id_array_list.append(truth_array) id_tag_list.append("Truth IDs") plotSequence(test_matrix, start_second, title, id_array_list, id_tag_list) plt.show()
def test_weighting(): """Test weighting.py""" # Create confusion matrices for random classifiers yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred1 = np.random.randint(3, size=12) cm1 = ConfusionMatrix(yactual, ypred1, "cls_1") yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred2 = np.random.randint(3, size=12) cm2 = ConfusionMatrix(yactual, ypred2, "cls_2") yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred3 = np.random.randint(3, size=12) cm3 = ConfusionMatrix(yactual, ypred3, "cls_3") yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] ypred4 = np.random.randint(3, size=12) cm4 = ConfusionMatrix(yactual, ypred4, "cls_4") weight_pairs = [[cm1, ypred1], [cm2, ypred2], [cm3, ypred3], [cm4, ypred4]] # Check that CEN score is being calculated #print(cm1.get_CEN_score(), cm2.get_CEN_score(), cm3.get_CEN_score(), cm4.get_CEN_score()) # Get final votes based on pairs votes_p = voting(weight_pairs, "Precision") votes_CEN_p = voting(weight_pairs, "CEN_Precision") votes_CEN = voting(weight_pairs, "CEN") votes_eq = voting(weight_pairs, "Equal_Vote") # Check metrics print(classification_report(yactual, votes_p)) print(classification_report(yactual, votes_CEN_p)) print(classification_report(yactual, votes_CEN)) print(classification_report(yactual, votes_eq)) # Create final confusion matrices depending on votes p_cm = ConfusionMatrix(yactual, votes_p, "Precision_Voting") p_CEN_cm = ConfusionMatrix(yactual, votes_CEN_p, "CEN_Precision_Voting") CEN_cm = ConfusionMatrix(yactual, votes_CEN, "CEN_Voting") eq_cm = ConfusionMatrix(yactual, votes_eq, "Equal_Voting") # Store confusion matrices p_cm.store_cm() p_CEN_cm.store_cm() CEN_cm.store_cm() eq_cm.store_cm()
def adaptive_performance(df): adaptive_g = df.groupby(['adaptive_pred', 'Is_Adaptive'], as_index=False).agg({'commit': 'count'}) adaptive_cm = ConfusionMatrix(g_df=adaptive_g, classifier='adaptive_pred', concept='Is_Adaptive', count='commit') print("adaptive commit performance") print(adaptive_cm.summarize()) return adaptive_cm
def refactor_performance(df): refactor_g = df.groupby(['is_refactor_pred', 'Is_Refactor'], as_index=False).agg({'commit': 'count'}) refactor_cm = ConfusionMatrix(g_df=refactor_g, classifier='is_refactor_pred', concept='Is_Refactor', count='commit') print("refactor commit performance") print(refactor_cm.summarize()) return refactor_cm
def corrective_performance(df): bug_g = df.groupby(['corrective_pred', 'Is_Corrective'], as_index=False).agg({'commit': 'count'}) bug_cm = ConfusionMatrix(g_df=bug_g, classifier='corrective_pred', concept='Is_Corrective', count='commit') print("corrective commit performance") print(bug_cm.summarize()) return bug_cm
def test_independent_prob(classifier , concept , count , g_df , expected): cm = ConfusionMatrix(classifier , concept , count , g_df) actual = cm.independent_prob() assert expected == actual
def test_summrize(classifier , concept , count , g_df , expected): cm = ConfusionMatrix(classifier , concept , count , g_df) actual = cm.summarize() assert expected == actual
def compute_all_data(self): # 生成数据 for thr in self.thresholds: cm = ConfusionMatrix(self.mu_true, self.sigma_true, self.mu_false, self.sigma_false, thr) self.Ps.append(cm.P) self.Rs.append(cm.R) self.TPRs.append(cm.TPR) self.FPRs.append(cm.FPR) self.F1s.append(cm.F1)
def evaluate_combined(self, test_sets, models): x_test = [None] * len(test_sets) for i, test_set in enumerate(test_sets): x_test[i] = test_set.x x_test[i] = models[i].preprocess_x(x_test[i]) y = test_sets[0].y y = np_utils.to_categorical(y, test_sets[0].num_classes) y_pred = self.model.predict_proba(x_test, verbose=0) cf = ConfusionMatrix(y_pred, y) return cf
def perfective_performance(df): perfective_g = df.groupby( ['perfective_pred', 'Is_Perfective'], as_index=False).agg({'commit' : 'count'}) perfective_cm = ConfusionMatrix(g_df=perfective_g , classifier='perfective_pred' , concept='Is_Perfective' , count='commit') print("perfective commit performance") print(perfective_cm.summarize()) return perfective_cm