def show_classification_metrics(pred,
label,
use_matric=None,
label_index=None,
matrics_list=None,
display=True):
"""calc metrics for classification model using PYCM
Args:
pred (numpy.array) : pred label for each batch (batch_size X number of pred classes)
label (numpy.array) : label for each batch (batch_size X number of classes)
label_index (dict) : class name (default=None)
matrics_list (list) : add matrics name (refer to the pycm metrics list) (defailt=None)
display (bool) : Whether to print the overall result (default=True)
Returns:
metrics (dict) : contain the 2-level result (overall_stat, class_stat)
"""
# pred = pred.reshape(-1)
# label = label.reshape(-1)
cm = ConfusionMatrix(pred, label)
if label_index is not None:
cm.relabel(mapping=label_index)
default_matrics_list = cm.recommended_list
if matrics_list is not None:
default_matrics_list.extend(matrics_list)
if display:
cm.stat(summary=True)
print("[Matrix]")
cm.print_matrix()
print("[Normalized Matrix]")
cm.print_normalized_matrix()
overall_stat = cm.overall_stat
class_stat = cm.class_stat
filter_overall_stat = {
k: v
for k, v in overall_stat.items() if k in default_matrics_list
}
filter_class_stat = {
k: v
for k, v in class_stat.items() if k in default_matrics_list
}
output = dict()
output["overall_stat"] = filter_overall_stat
output["class_stat"] = filter_class_stat
return output
features = ['CATEGORY', 'TITLE']
train_df = pd.read_table('./data/train.txt', header=None, names=features)
test_df = pd.read_table('./data/test.txt', header=None, names=features)
clf = pickle.load(open('./models/52lr.pickle', mode='rb'))
vectorizer = pickle.load(open('./models/51vectorizer.pickle', mode='rb'))
le = pickle.load(open('./models/52le.pickle', mode='rb'))
sc = pickle.load(open('./models/52sc.pickle', mode='rb'))
X_train_sparse = load_npz("./feature/train.feature.npz")
X_valid_sparse = load_npz("./feature/valid.feature.npz")
X_test_sparse = load_npz("./feature/test.feature.npz")
X_train, X_valid, X_test = X_train_sparse.toarray(), \
X_valid_sparse.toarray(), X_test_sparse.toarray()
#訓練データでの混同行列
labels_train_pred = clf.predict(sc.transform(X_train))
labels_train_true = le.transform(train_df['CATEGORY'].values)
cm_train = ConfusionMatrix(le.inverse_transform(labels_train_true)\
, le.inverse_transform(labels_train_pred))
print('By training data\n')
cm_train.print_matrix()
#評価データでの混同行列
labels_test_pred = clf.predict(sc.transform(X_test))
labels_test_true = le.transform(test_df['CATEGORY'].values)
cm_test = ConfusionMatrix(le.inverse_transform(labels_test_true)\
, le.inverse_transform(labels_test_pred))
print('By test data\n')
cm_test.print_matrix()
rf_clf = deepcopy(clf.best_estimator_)
print("RF params:", len(rf_clf.estimators_))
###################
# Error Evaluations
###################
print("RF Ntrain %s RnSeed %s Train Score %s" %(ntrain, rnseed, rf_clf.score(X_train, y_train)))
pred = rf_clf.predict(X_train)
print('Unweighted Accuracy',accuracy(y_pred=pred,y_true=y_train))
print('Weighted Accuracy',bal_score(y_pred=pred,y_true=y_train))
cm = ConfusionMatrix(actual_vector=y_train, predict_vector=pred)
cm.print_matrix()
print("RF Ntrain %s RnSeed %s Test Score %s" %(ntrain, rnseed, rf_clf.score(X_test, y_test)))
pred = rf_clf.predict(X_test)
print('Unweighted Accuracy',accuracy(y_pred=pred,y_true=y_test))
print('Weighted Accuracy',bal_score(y_pred=pred,y_true=y_test))
cm = ConfusionMatrix(actual_vector=y_test, predict_vector=pred)
cm.print_matrix()
print(cm)
sys.stdout.flush()
def showStats(true_labels,
predictions,
labels_dir,
showExtraStats,
writeToFile=''):
"""
Prints various statistics about model preformance.
Arguments:
true_labels: true labels of the predictions
predictions: predicted labels
labels_dir: directory to text file of labels to integers
Returns:
N/A
"""
#Print basic info
print("Labels:")
label_dict = fileToLabelDict(labels_dir)
print(label_dict)
print("\nPredictions:")
print(predictions)
print("\nActual:")
print(true_labels)
print("\n")
#Builds confusion matrix and additional stats
my_inverted_dict = {}
for elem in label_dict.keys():
my_inverted_dict[elem] = int(label_dict[elem])
my_inverted_dict = dict(
zip(my_inverted_dict.values(), my_inverted_dict.keys()))
cm = ConfusionMatrix(actual_vector=true_labels, predict_vector=predictions)
cm.relabel(mapping=my_inverted_dict)
cm.print_matrix()
if showExtraStats:
#print("Micro F1 Score: ",cm.overall_stat['F1 Micro'])
#print("Macro F1 Score: ",cm.overall_stat['F1 Macro'])
#print("Cross Entropy: ",cm.overall_stat['Cross Entropy'])
#print("95% CI: ",cm.overall_stat['95% CI'])
print("AUC: ", cm.AUC)
print("AUC quality:", cm.AUCI)
#Outputs to output.txt
if writeToFile == '':
pass
else:
with open(writeToFile, 'w') as f:
f.write("Labels:\n\n")
f.write(str(label_dict))
f.write("\n\nPredictions::\n\n")
f.write(str(predictions))
f.write("\n\nActual:\n\n")
f.write(str(true_labels))
f.write("\n\n")
f.write(dictToString(cm.matrix))
f.write("\n\n")
f.write("AUC: \n\n")
f.write(str(cm.AUC))
f.write("AUC Quality: \n\n")
f.write(str(cm.AUCI))
f.write("\n\n")