def test_storing_loading():
"""Test store_preds and load_preds"""
# Create confusion matrices for random classifiers
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred1 = np.random.randint(3, size=12)
cm1 = ConfusionMatrix(yactual, ypred1, "cls_1")
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred2 = np.random.randint(3, size=12)
cm2 = ConfusionMatrix(yactual, ypred2, "cls_2")
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred3 = np.random.randint(3, size=12)
cm3 = ConfusionMatrix(yactual, ypred3, "cls_3")
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred4 = np.random.randint(3, size=12)
cm4 = ConfusionMatrix(yactual, ypred4, "cls_4")
preds = [ypred1, ypred2, ypred3, ypred4]
print("Preds before saving", preds)
store_preds(preds, yactual, 1)
new_preds, actual = load_preds(1)
print("Preds after saving", new_preds, "Actual after saving", actual)
def two_years_analysis(two_years_df
, first_metric
, second_metric
, key):
print()
print("Co-change"
, first_metric
, second_metric)
g = two_years_df.groupby([first_metric, second_metric]
, as_index=False).agg({key : 'count'})
print(g)
cm = ConfusionMatrix(g_df=g
, classifier=first_metric
, concept=second_metric, count=key)
print(cm.summarize())
print()
print("Samples", cm.samples())
print("Both metrics increment match", cm.accuracy())
print(second_metric
, " improvement given "
, first_metric
, " improvement", cm.precision(), "lift", cm.precision_lift())
print(first_metric
, " improvement given "
, second_metric
, "improvement", cm.recall(), "lift", ifnull(safe_divide(ifnull(cm.recall()),cm.hit_rate())) - 1)
print()
def fit_predict(self,
Gtr,
Ytr,
Gt,
Yt,
grid_search,
acc_param="F1Mean",
RS=0,
VERBOSE=False):
"""Learn the model on training dataset and predict on the testing dataset.
Input:
- Gtr (array): training subset
- Ytr (array): true labels of training subset
- Gt (array): testing subset
- Yt (array): true labels of testing subset
- grid_search (dict): grid search for the CV
- acc (str): accuracy parameter for the cross-validation (default "F1Mean", otherwise it will be the OA)
- RS (int) : random seed used for the stratified k-fold CV
- VERBOSE (bool): verbose (default: False)
Return:
- confMatrix (object ConfusionMatrix): confusion matrix issued from the classification
- yp (array): vector of predicted labels of the testing subset
- grid.best_params_ (dict): combination of parameters that gave the best results during the CV
TODO: implement own scoring parameter
"""
if acc_param == "F1Mean":
score = 'f1_macro'
else:
score = 'accuracy'
## Initialization of the stratifield K-CV
cv = StratifiedKFold(Ytr, n_folds=self.n_folds, random_state=RS)
#Implementation of a fit and a predict methods with parameters from grid search
grid = GridSearchCV(self.pipe,
param_grid=grid_search,
scoring=score,
verbose=VERBOSE,
cv=cv,
n_jobs=3)
grid.fit(Gtr, Ytr)
model = grid.best_estimator_
if VERBOSE:
print grid.best_score_
#Learn model
model.fit(Gtr, Ytr) #could use refit in version 0.19 of sklearn
#Predict
yp = model.predict(Gt)
#Compute confusion matrix
confMatrix = ConfusionMatrix()
confMatrix.compute_confusion_matrix(yp, Yt)
return confMatrix, yp, grid.best_params_
def evaluate_sense(gold_list, predicted_list):
"""Evaluate sense classifier
The label 'no' is for the relations that are missed by the system
because the arguments don't match any of the gold relations.
"""
sense_alphabet = Alphabet()
for relation in gold_list:
sense_alphabet.add(relation['Sense'][0])
sense_alphabet.add('no')
sense_cm = ConfusionMatrix(sense_alphabet)
gold_to_predicted_map, predicted_to_gold_map = \
_link_gold_predicted(gold_list, predicted_list, spans_exact_matching)
for i, gold_relation in enumerate(gold_list):
if i in gold_to_predicted_map:
predicted_sense = gold_to_predicted_map[i]['Sense'][0]
if predicted_sense in gold_relation['Sense']:
sense_cm.add(predicted_sense, predicted_sense)
else:
if not sense_cm.alphabet.has_label(predicted_sense):
predicted_sense = 'no'
sense_cm.add(predicted_sense, gold_relation['Sense'][0])
else:
sense_cm.add('no', gold_relation['Sense'][0])
for i, predicted_relation in enumerate(predicted_list):
if i not in predicted_to_gold_map:
predicted_sense = predicted_relation['Sense'][0]
if not sense_cm.alphabet.has_label(predicted_sense):
predicted_sense = 'no'
sense_cm.add(predicted_sense, 'no')
return sense_cm
def compute_binary_eval_metric(gold_list, predicted_list, matching_fn):
"""Compute binary evaluation metric
"""
binary_alphabet = Alphabet()
binary_alphabet.add('yes')
binary_alphabet.add('no')
cm = ConfusionMatrix(binary_alphabet)
matched_predicted = [False for x in predicted_list]
for gold_span in gold_list:
found_match = False
for i, predicted_span in enumerate(predicted_list):
if matching_fn(gold_span,
predicted_span) and not matched_predicted[i]:
cm.add('yes', 'yes')
matched_predicted[i] = True
found_match = True
break
if not found_match:
cm.add('no', 'yes')
# Predicted span that does not match with any
for matched in matched_predicted:
if not matched:
cm.add('yes', 'no')
return cm
def train(model, iterator, optimizer, criterion, reg_ratio):
model.train()
epoch_loss = 0
conf_mat = ConfusionMatrix(np.zeros((NUM_CLASSES, NUM_CLASSES)))
for batch, labels in iterator():
optimizer.zero_grad()
predictions = model(batch).squeeze(1)
loss = criterion(predictions, labels)
reg_loss = 0
for param in model.parameters():
reg_loss += param.norm(2)
total_loss = loss + reg_ratio*reg_loss
total_loss.backward()
optimizer.step()
epoch_loss += loss.item()
conf_mat += ConfusionMatrix.from_predictions(predictions, labels)
average_loss = epoch_loss / len(iterator)
return average_loss, conf_mat
def train_ensemble(model, acoustic_iterator, linguistic_iterator, optimizer, criterion, reg_ratio):
model.train()
epoch_loss = 0
conf_mat = ConfusionMatrix(np.zeros((NUM_CLASSES, NUM_CLASSES)))
assert len(acoustic_iterator) == len(linguistic_iterator)
for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(), linguistic_iterator()):
acoustic_batch = acoustic_tuple[0]
labels = acoustic_tuple[1]
linguistic_batch = linguistic_tuple[0]
optimizer.zero_grad()
predictions = model(acoustic_batch, linguistic_batch).squeeze(1)
loss = criterion(predictions, labels)
reg_loss = 0
for param in model.parameters():
reg_loss += param.norm(2)
total_loss = loss + reg_ratio*reg_loss
total_loss.backward()
optimizer.step()
epoch_loss += loss.item()
conf_mat += ConfusionMatrix.from_predictions(predictions, labels)
average_loss = epoch_loss / len(acoustic_iterator)
return average_loss, conf_mat
def predict(
self, dataset: Dataset
) -> dict[str, Union[str, list[str], float, ConfusionMatrix]]:
"""predicts the class labels of the given test set, based on a previously fitted model.
:param dataset: dataset consisting of
:return: list of predicted values
"""
prediction_params: dict[str, Union[str, list[str], float,
ConfusionMatrix]] = {
"branches": self.__branches
}
predicted_values: list[str] = []
for example, label in dataset:
example: dict[str, str]
label: str
# predict the example
predicted_values.append(self.__label_example(example, self.__root))
prediction_params["predictions"]: list[str] = predicted_values
prediction_params["accuracy"] = accuracy(dataset.label_sample,
predicted_values)
prediction_params[
"confusion_matrix"]: ConfusionMatrix = ConfusionMatrix(
dataset.label_space, dataset.label_sample, predicted_values)
return prediction_params
def setUp(self):
label_names = ['banana', 'apple', 'orange']
labels = np.array(
[
[0, 1, 0],
[1, 0, 0],
[1, 0, 0],
[0, 1, 0],
[0, 0, 1],
[0, 0, 1],
[1, 0, 0],
[0, 1, 0],
[1, 0, 0],
[0, 0, 1],
[0, 0, 1]
]
)
predictions = np.array(
[
[0.9, 0.1, 0.0],
[0.7, 0.1, 0.2],
[0.4, 0.3, 0.3],
[0.2, 0.6, 0.2],
[0.5, 0.2, 0.3],
[0.0, 0.0, 1.0],
[0.2, 0.1, 0.7],
[0.1, 0.8, 0.1],
[0.3, 0.5, 0.2],
[0.1, 0.0, 0.9],
[0.0, 1.0, 0.0]
]
)
self.confusion_matrix = ConfusionMatrix(predictions, labels, class_names=label_names)
def bootstrap_diff(df, ccp_estimator, rounds, sample_size):
bootstrap_results = []
for i in range(rounds):
# Get first model parameters
s1 = df.sample(sample_size, replace=True)
bug_g = s1.groupby([classifier, concept],
as_index=False).agg({count: 'count'})
bug_cm = ConfusionMatrix(g_df=bug_g,
classifier=classifier,
concept=concept,
count=count)
positive_rate = bug_cm.positive_rate()
hit_rate = bug_cm.hit_rate()
ccp = ccp_estimator.estimate_positives(hit_rate)
ccp_diff = ccp - positive_rate
# Find difference in given points
bootstrap_results.append([positive_rate, hit_rate, ccp, ccp_diff])
if (i % 100 == 0):
print("finished " + str(i), datetime.datetime.now())
results_df = pd.DataFrame(
bootstrap_results,
columns=['positive_rate', 'hit_rate', 'ccp', 'ccp_diff'])
return results_df
def evaluate_sense(relation_pairs, valid_senses):
sense_alphabet = Alphabet()
#for g_relation, _ in relation_pairs:
#if g_relation is not None:
#sense = g_relation['Sense'][0]
#if sense in valid_senses:
#sense_alphabet.add(sense)
for sense in valid_senses:
sense_alphabet.add(sense)
sense_alphabet.add(ConfusionMatrix.NEGATIVE_CLASS)
sense_alphabet.growing = False
sense_cm = ConfusionMatrix(sense_alphabet)
for g_relation, p_relation in relation_pairs:
assert g_relation is not None or p_relation is not None
if g_relation is None:
predicted_sense = p_relation['Sense'][0]
sense_cm.add(predicted_sense, ConfusionMatrix.NEGATIVE_CLASS)
elif p_relation is None:
gold_sense = g_relation['Sense'][0]
if gold_sense in valid_senses:
sense_cm.add(ConfusionMatrix.NEGATIVE_CLASS, gold_sense)
else:
predicted_sense = p_relation['Sense'][0]
gold_sense = g_relation['Sense'][0]
if gold_sense in valid_senses:
sense_cm.add(predicted_sense, gold_sense)
return sense_cm
def test_confusion_matrix():
"""Test ConfusionMatrix class"""
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]
cm = ConfusionMatrix(yactual, ypred, "Sample_classifier")
number_cm = cm.get_number_cm()
normalized_cm = cm.get_normalized_cm()
metrics = {
"TP": cm.get_true_pos(),
"TN": cm.get_true_neg(),
"FP": cm.get_false_pos(),
"FN": cm.get_false_neg()
}
print("Confusion matrix:")
print(number_cm)
print("Normalized confusion matrix:")
print(normalized_cm)
print("Precision per label is:", cm.get_precision())
print("Metrics:", metrics)
print("MCC is", cm.get_mcc())
return cm.get_precision()
def make_confusion_matrix(results):
true_positives = [r for r in results if r.tp]
true_negatives = [r for r in results if r.tn]
false_positives = [r for r in results if r.fp]
false_negatives = [r for r in results if r.fn]
return ConfusionMatrix(len(true_positives), len(false_positives),
len(true_negatives), len(false_negatives))
def evaluate(self, test_set):
x = test_set.x
x = self.preprocess_x(x)
y = test_set.y
y = np_utils.to_categorical(y, test_set.num_classes)
y_pred = self.model.predict_proba(x, verbose=0)
cf = ConfusionMatrix(y_pred, y)
return cf
def make_pr_graph(entropies, correct, graph_name, title, mpl_figure=None):
"""
Plot entropy as a PR curve predicting whether examples are correct
or incorrect.
"""
if mpl_figure is None:
mpl_figure = mpl.figure()
axes = mpl_figure.gca()
assert len(entropies) == len(correct), (len(entropies), len(correct))
pairs = zip(entropies, correct)
pairs.sort()
max_entropy = float(pairs[-1][0])
min_entropy = float(pairs[0][0])
num_segments = 20.0
segment_size = (max_entropy - min_entropy)/num_segments
if segment_size == 0:
segment_size = 0.01
thresholds = na.arange(min_entropy, max_entropy + 1, segment_size)
X_recall = []
Y_precision = []
for threshold in thresholds:
predicted_correct = [(entropy, correct)
for (entropy, correct) in pairs
if entropy < threshold]
predicted_incorrect = [(entropy, correct)
for (entropy, correct) in pairs
if entropy >= threshold]
tp = len([(entropy, correct) for entropy, correct in predicted_incorrect
if not correct])
fp = len([(entropy, correct) for entropy, correct in predicted_incorrect
if correct])
tn = len([(entropy, correct) for entropy, correct in predicted_correct
if correct])
fn = len([(entropy, correct) for entropy, correct in predicted_correct
if not correct])
if tp == 0:
continue
cm = ConfusionMatrix(tp, fp, tn, fn)
X_recall.append(cm.recall)
Y_precision.append(cm.precision)
global marker_i
graph_name_paper = ENTROPY_METRIC_PAPER_NAMES[graph_name]
axes.plot(X_recall, Y_precision, label=graph_name_paper,
marker=markers[marker_i])
marker_i = (marker_i + 1) % len(markers)
axes.legend(loc='upper right')
axes.set_xlabel("Recall")
axes.set_ylabel("Precision")
axes.set_ylim(0, 1.1)
axes.set_xlim(0, 1.1)
axes.set_title("Precision vs Recall")
mpl_figure.savefig(title.replace(" ", "_") + ".eps")
mpl.show()
def evaluate_performance(df, classification_column, concept_column):
g = df.groupby([classification_column, concept_column],
as_index=False).agg({'commit': 'count'})
cm = ConfusionMatrix(g_df=g,
classifier=classification_column,
concept=concept_column,
count='commit')
return cm.summarize()
def train_and_test(self, train_set, test_set):
self.classifier = self.classifier_class.train(train_set)
predicted_polarities = [
self.classify(document) for (document, polarity) in test_set
]
actual_polarities = [polarity for (document, polarity) in test_set]
return ConfusionMatrix(predicted_polarities, actual_polarities)
def evaluate_objects(model, corpus_fname, state_type):
corpus = annotationIo.load(corpus_fname)
state_cls = state_type_from_name(state_type)
from g3.inference import nodeSearch
taskPlanner = nodeSearch.BeamSearch(model)
predictions = []
done = False
phrases = set()
for i, annotation in enumerate(corpus):
start_state = state_cls.from_context(annotation.context)
for esdc in annotation.esdcs:
#if esdc.text != "the pallet of boxes":
# continue
#if esdc.text in phrases:
# continue
isCorrect = annotation.isGroundingCorrect(esdc)
if isCorrect != None:
ggg = ggg_from_esdc(esdc)
groundings = annotation.getGroundings(esdc)
assert len(groundings) == 1
grounding = groundings[0]
#if "generator" not in grounding.tags:
# continue
prob = evaluate_ggg(ggg, grounding, start_state, taskPlanner)
if prob > 0.7:
predicted_class = True
else:
predicted_class = False
predictions.append((predicted_class, isCorrect))
#print "Query: Is object", " ".join(grounding.tags),
#print "'" + esdc.text + "'?"
#print "System: ",
#if predicted_class:
# print "Yes."
#else:
# print "No."
#done = True
phrases.add(esdc.text)
if done:
break
if done:
break
tp = len([(p, l) for p, l in predictions if p and p == l])
fp = len([(p, l) for p, l in predictions if p and p != l])
tn = len([(p, l) for p, l in predictions if not p and p == l])
fn = len([(p, l) for p, l in predictions if not p and p != l])
cm = ConfusionMatrix(tp, fp, tn, fn)
#cm.print_all()
#if len(phrases) > 20:
# phrases = random.sample(phrases, 20)
#for phrase in sorted(phrases):
# print phrase
return cm
def Evaluation_all(gold_label, predict_label):
binary_alphabet = Alphabet()
for i in range(20):
binary_alphabet.add(DICT_INDEX_TO_LABEL[i])
cm = ConfusionMatrix(binary_alphabet)
cm.add_list(predict_label, gold_label)
macro_p, macro_r, macro_f1 = cm.get_average_prf()
overall_accuracy = cm.get_accuracy()
return overall_accuracy, macro_p, macro_r, macro_f1
def run_multiple_voting():
"""Run tests on multiple weighting systems given stored predictions of the classifiers in main()
To be used in conjunction with alternative_main to determine which weighting method performs better
"""
# Load predictions from all classifiers and actual labels for test_set_1
preds1, actual1 = load_preds(1)
# Load predictions from all classifiers and actual labels for test_set_2
preds2, actual2 = load_preds(2)
# Create confusion matrices for each classifier
b_cm = ConfusionMatrix(actual1, preds1[0], "Bayes")
p_cm = ConfusionMatrix(actual1, preds1[1], "Proximity")
v_cm = ConfusionMatrix(actual1, preds1[2], "Voting")
# r_cm = ConfusionMatrix(actual1, preds1[2], "LSTM")
confusionMatrices = [b_cm, p_cm, v_cm]
# confusionMatices = [p_cm, v_cm, b_cm, r_cm]
# Save individual confusion matrices to files
for cm in confusionMatrices:
cm.store_cm()
print("Individual confusion matrices created and stored!")
# Weight second set of results, using confusion matrices from first set
weightingInput = [
[confusionMatrices[0], preds2[0]], [confusionMatrices[1], preds2[1]]
# [confusionMatrices[2] ,b.batchTest(test_set2)],
# [confusionMatrices[3], r.batchTest(test_set2)],
]
# Get the weighted voting results
votes_p = voting(weightingInput, "Precision")
votes_CEN_p = voting(weightingInput, "CEN_Precision")
votes_CEN = voting(weightingInput, "CEN")
votes_eq = voting(weightingInput, "Equal_Vote")
# Check metrics
print(classification_report(actual2, votes_p))
print(classification_report(actual2, votes_CEN_p))
print(classification_report(actual2, votes_CEN))
print(classification_report(actual2, votes_eq))
# Create final confusion matrices depending on votes
p_cm = ConfusionMatrix(actual2, votes_p, "Precision")
p_CEN_cm = ConfusionMatrix(actual2, votes_CEN_p, "CEN_Precision")
CEN_cm = ConfusionMatrix(actual2, votes_CEN, "CEN")
eq_cm = ConfusionMatrix(actual2, votes_eq, "Equal")
# Store confusion matrices
p_cm.store_cm()
p_CEN_cm.store_cm()
CEN_cm.store_cm()
eq_cm.store_cm()
return votes_p, votes_CEN_p, votes_CEN, votes_eq
def applyClassifiers(scst_pkl_path: str,
keras_pkl_path: str,
data_path: Optional[str] = None,
truth_path: Optional[str] = None,
start_second: Optional[float] = None,
end_second: Optional[float] = None) -> None:
'''
Apply both a ``SCSTClassifier`` and a ``TransientKerasClassifier`` to the data sequence
specified by the inputs, and plot the results.
:param scst_pkl_path: Full path to a pickle file containing a saved ``SCSTClassifier`` object.
:param keras_pkl_path: Full path to a pickle file containing a saved
``TransientKerasClassifier`` object.
:param data_path, truth_path, start_second, end_second: See ``data_utils.getPlotDataTuple``.
'''
# Define SCST classification parameters
SIG_THRESH = 100
NOISE_THRESH = 100
# Load data and classifiers
test_matrix, truth_array, start_second, title = getPlotDataTuple(
truth_path, data_path, start_second, end_second)
scst_classifier = SCSTClassifier.load(scst_pkl_path)
keras_classifier = TransientKerasClassifier.load(keras_pkl_path)
# Apply classifiers
num_obs = test_matrix.shape[1]
for test_idx in range(num_obs):
scst_classifier.classify(test_matrix[:, test_idx], SIG_THRESH,
NOISE_THRESH)
keras_classifier.classify(test_matrix[:, test_idx])
_printProgress(test_idx / float(num_obs), title)
_printProgress(1.0, title)
sys.stdout.write("\n")
# Display results
id_array_list = [
scst_classifier.class_labels, keras_classifier.class_labels
]
id_tag_list = ["SCST IDs", "Keras IDs"]
if truth_array is not None:
for classifier, name in zip([scst_classifier, keras_classifier],
["SCST", "Keras"]):
conf_matrix = ConfusionMatrix(classifier.class_labels, truth_array,
True, name)
conf_matrix.display()
id_array_list.append(truth_array)
id_tag_list.append("Truth IDs")
plotSequence(test_matrix, start_second, title, id_array_list, id_tag_list)
plt.show()
def test_weighting():
"""Test weighting.py"""
# Create confusion matrices for random classifiers
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred1 = np.random.randint(3, size=12)
cm1 = ConfusionMatrix(yactual, ypred1, "cls_1")
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred2 = np.random.randint(3, size=12)
cm2 = ConfusionMatrix(yactual, ypred2, "cls_2")
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred3 = np.random.randint(3, size=12)
cm3 = ConfusionMatrix(yactual, ypred3, "cls_3")
yactual = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
ypred4 = np.random.randint(3, size=12)
cm4 = ConfusionMatrix(yactual, ypred4, "cls_4")
weight_pairs = [[cm1, ypred1], [cm2, ypred2], [cm3, ypred3], [cm4, ypred4]]
# Check that CEN score is being calculated
#print(cm1.get_CEN_score(), cm2.get_CEN_score(), cm3.get_CEN_score(), cm4.get_CEN_score())
# Get final votes based on pairs
votes_p = voting(weight_pairs, "Precision")
votes_CEN_p = voting(weight_pairs, "CEN_Precision")
votes_CEN = voting(weight_pairs, "CEN")
votes_eq = voting(weight_pairs, "Equal_Vote")
# Check metrics
print(classification_report(yactual, votes_p))
print(classification_report(yactual, votes_CEN_p))
print(classification_report(yactual, votes_CEN))
print(classification_report(yactual, votes_eq))
# Create final confusion matrices depending on votes
p_cm = ConfusionMatrix(yactual, votes_p, "Precision_Voting")
p_CEN_cm = ConfusionMatrix(yactual, votes_CEN_p, "CEN_Precision_Voting")
CEN_cm = ConfusionMatrix(yactual, votes_CEN, "CEN_Voting")
eq_cm = ConfusionMatrix(yactual, votes_eq, "Equal_Voting")
# Store confusion matrices
p_cm.store_cm()
p_CEN_cm.store_cm()
CEN_cm.store_cm()
eq_cm.store_cm()
def adaptive_performance(df):
adaptive_g = df.groupby(['adaptive_pred', 'Is_Adaptive'],
as_index=False).agg({'commit': 'count'})
adaptive_cm = ConfusionMatrix(g_df=adaptive_g,
classifier='adaptive_pred',
concept='Is_Adaptive',
count='commit')
print("adaptive commit performance")
print(adaptive_cm.summarize())
return adaptive_cm
def refactor_performance(df):
refactor_g = df.groupby(['is_refactor_pred', 'Is_Refactor'],
as_index=False).agg({'commit': 'count'})
refactor_cm = ConfusionMatrix(g_df=refactor_g,
classifier='is_refactor_pred',
concept='Is_Refactor',
count='commit')
print("refactor commit performance")
print(refactor_cm.summarize())
return refactor_cm
def corrective_performance(df):
bug_g = df.groupby(['corrective_pred', 'Is_Corrective'],
as_index=False).agg({'commit': 'count'})
bug_cm = ConfusionMatrix(g_df=bug_g,
classifier='corrective_pred',
concept='Is_Corrective',
count='commit')
print("corrective commit performance")
print(bug_cm.summarize())
return bug_cm
def test_independent_prob(classifier
, concept
, count
, g_df
, expected):
cm = ConfusionMatrix(classifier
, concept
, count
, g_df)
actual = cm.independent_prob()
assert expected == actual
def test_summrize(classifier
, concept
, count
, g_df
, expected):
cm = ConfusionMatrix(classifier
, concept
, count
, g_df)
actual = cm.summarize()
assert expected == actual
def compute_all_data(self):
# 生成数据
for thr in self.thresholds:
cm = ConfusionMatrix(self.mu_true, self.sigma_true, self.mu_false,
self.sigma_false, thr)
self.Ps.append(cm.P)
self.Rs.append(cm.R)
self.TPRs.append(cm.TPR)
self.FPRs.append(cm.FPR)
self.F1s.append(cm.F1)
def evaluate_combined(self, test_sets, models):
x_test = [None] * len(test_sets)
for i, test_set in enumerate(test_sets):
x_test[i] = test_set.x
x_test[i] = models[i].preprocess_x(x_test[i])
y = test_sets[0].y
y = np_utils.to_categorical(y, test_sets[0].num_classes)
y_pred = self.model.predict_proba(x_test, verbose=0)
cf = ConfusionMatrix(y_pred, y)
return cf
def perfective_performance(df):
perfective_g = df.groupby(
['perfective_pred', 'Is_Perfective'], as_index=False).agg({'commit' : 'count'})
perfective_cm = ConfusionMatrix(g_df=perfective_g
, classifier='perfective_pred'
, concept='Is_Perfective'
, count='commit')
print("perfective commit performance")
print(perfective_cm.summarize())
return perfective_cm