def confusion_matrix(pred: list,
label: list,
label_index: dict,
file_name: str = None,
output_dir='results'):
cm = ConfusionMatrix(pred, label)
cm.relabel(mapping=label_index)
cm_matrix = cm.matrix
cm_normalized_matrix = cm.normalized_matrix
if file_name is None:
file_name = 'confusion_matrix.json'
normalized_file_name = file_name.replace('.', '_normalized.')
if output_dir is not None:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(os.path.join(output_dir, file_name), 'w') as fp:
json.dump(cm_matrix, fp, indent=4)
# with open(os.path.join(output_dir, normalized_file_name), 'w') as fp:
# json.dump(cm_normalized_matrix, fp, indent=4)
return cm_matrix
def classification_report(self, external=False, intercv=False, file=None):
from sklearn.metrics import classification_report, cohen_kappa_score, matthews_corrcoef
if self.mode == 'classification':
if external:
y_true = self.y_test
y_pred = self.clf.predict(self.X_test)
info = 'external-validation'
else:
y_true = self.y_train
if intercv:
info = 'internal-validation'
y_pred = self.crossValPredict
else:
info = 'training-validation'
y_pred = self.clf.predict(self.X_train)
acc = cohen_kappa_score(y_true, y_pred)
cm = confusion_matrix(y_true, y_pred)
matt = matthews_corrcoef(y_true, y_pred)
cr = classification_report(y_true, y_pred, digits=3)
print(f'\nConfusion matrix for {info}:\n{cm}')
print(f'\nCohen kappa score for {info}: {np.round(acc, 3)}')
print(
f'Matthews correlation coef for {info}: {np.round(matt, 3)}\n')
print(cr)
if file != None:
self.CM = ConfusionMatrix(actual_vector=y_true,
predict_vector=y_pred)
self.CM.save_html(f'{str(file)}-{info}')
def save_html(confusion_matrix_folder, ground_truths, predictions, model):
html_folder = join(confusion_matrix_folder, 'html')
makedirs(html_folder, exist_ok=True)
cm = ConfusionMatrix(actual_vector=ground_truths,
predict_vector=predictions)
filename = f'{model}.html'
cm.save_html(join(html_folder, filename), summary=True)
def show_classification_metrics(pred,
label,
use_matric=None,
label_index=None,
matrics_list=None,
display=True):
"""calc metrics for classification model using PYCM
Args:
pred (numpy.array) : pred label for each batch (batch_size X number of pred classes)
label (numpy.array) : label for each batch (batch_size X number of classes)
label_index (dict) : class name (default=None)
matrics_list (list) : add matrics name (refer to the pycm metrics list) (defailt=None)
display (bool) : Whether to print the overall result (default=True)
Returns:
metrics (dict) : contain the 2-level result (overall_stat, class_stat)
"""
# pred = pred.reshape(-1)
# label = label.reshape(-1)
cm = ConfusionMatrix(pred, label)
if label_index is not None:
cm.relabel(mapping=label_index)
default_matrics_list = cm.recommended_list
if matrics_list is not None:
default_matrics_list.extend(matrics_list)
if display:
cm.stat(summary=True)
print("[Matrix]")
cm.print_matrix()
print("[Normalized Matrix]")
cm.print_normalized_matrix()
overall_stat = cm.overall_stat
class_stat = cm.class_stat
filter_overall_stat = {
k: v
for k, v in overall_stat.items() if k in default_matrics_list
}
filter_class_stat = {
k: v
for k, v in class_stat.items() if k in default_matrics_list
}
output = dict()
output["overall_stat"] = filter_overall_stat
output["class_stat"] = filter_class_stat
return output
def __call__(self):
confusion_matrix = ConfusionMatrix(
matrix=deepcopy(self._confusion_matrix))
confusion_matrix.relabel(
mapping={value - 1: key
for key, value in self._label_map.items()})
metrics = {
metric_name: getattr(confusion_matrix, metric_name)
for metric_name in self._metric_names
}
self.reset()
return metrics
def experiment(model, X, Y, alg_title, files, report = True, n = 1):
x = []
f = []
e = []
t = []
p = []
for i in range(n):
t_start = perf_counter()
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 0)
model.fit(X_train, Y_train)
Y_pred = model.predict(X_test)
t_stop = perf_counter()
elapsed_time = t_stop - t_start
cm = ConfusionMatrix(actual_vector = Y_test, predict_vector = Y_pred)
x.append(alg_title)
f.append(files)
e.append(elapsed_time)
t.append(Y_test)
p.append(Y_pred)
sq = { 'alg': x, 'file': f, 'elapsed_time': e, 'test': t, 'pred': p }
df = pd.DataFrame(sq)
file_name = re.sub('[\s+]', '', files + '_' + alg_title + '_cm.csv')
df.to_csv(file_name, index = False)
if report and n == 1:
print(alg_title)
print(classification_report(Y_test, Y_pred))
cm.plot(cmap = plt.cm.Greens, number_label = True, title = alg_title, plot_lib = 'matplotlib')
file_name = re.sub('[\s+]', '', files + '_' + alg_title + '_cm.png')
plt.savefig(file_name)
plt.show()
if n == 1:
return {
'elapsed_time': elapsed_time,
'test': Y_test,
'pred': Y_pred,
'cm': cm,
'title': alg_title,
}
def objective(args):
# Define model
model = RankedNetworkCNNModule(args['learning_rate'],
dataset.get_embeddings(),
hidden_dim=args['hidden'],
output_labels=2)
# Evaluation on held-out test-set
with torch.no_grad():
model.eval()
results = pd.DataFrame(columns=['labels', 'predictions'])
for batch_idx, batch in enumerate(test_loader):
y_hat = model(batch['a'], batch['b'])
results: pd.DataFrame = results.append(pd.DataFrame({
'labels':
batch['label'].flatten(),
'predictions':
y_hat.detach().argmax(axis=1)
}),
ignore_index=True)
results.to_csv()
# With a nice confusion matrix
confusion_matrix(y_pred=results['predictions'].values,
y_true=results['labels'].values,
classes=[0, 1])
cm = ConfusionMatrix(actual_vector=results['labels'].values,
predict_vector=results['predictions'].values)
output_test_results = "cm.txt"
cm.save_stat(output_test_results)
output_test_predictions_file = "test_predictions.txt"
np.savetxt(output_test_predictions_file,
results['predictions'].values,
delimiter=",")
kiwi.log_metric(key="test_acc", value=cm.Overall_ACC)
kiwi.log_metric(key="test_f1_micro", value=cm.F1_Micro)
kiwi.log_metric(key="test_f1_macro", value=cm.F1_Macro)
kiwi.log_metric(key="test_ci_pm",
value=cm.CI95[1] - cm.Overall_ACC)
kiwi.log_metric(key="test_ci_pm",
value=cm.CI95[1] - cm.Overall_ACC)
kiwi.log_artifact(output_test_predictions_file)
kiwi.log_artifact(output_test_results + ".pycm")
return cm.Overall_ACC
def get_metrics_table(self, doc_markup, ai_markup, norm=255.0):
"""Получение таблицы (pandas.DataFrame) метрик на основе сегментированных данных ИНС и эксперта
Arguments:
doc_markup (str): путь до PNG, размеченного ИНС
ai_markup (str): путь до PNG, размеченного экспертом
norm (float): значение нормировки изображений (Изображение / norm), default: 255.0
Returns:
metrics (pandas.DataFrame): таблица со значениями рассчитаных метрик
"""
img_pred_data = [cv2.imread(ai_markup) / norm]
img_true_data = [cv2.imread(doc_markup) / norm]
metrics = pd.DataFrame()
for i in range(1):
temp_overal = ConfusionMatrix(
actual_vector=img_true_data[i].ravel(),
predict_vector=img_pred_data[i].ravel())
temp_metrics = {
"name":
doc_markup,
"PA":
pixel_accuracy(img_pred_data[i].ravel(),
img_true_data[i].ravel()),
"Jaccard":
Jaccard(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Sorensen":
Sorensen(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Kulchinski":
Kulchinski(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Simpson":
Simpson(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Braun_Blanke":
Braun(img_pred_data[i].ravel(), img_true_data[i].ravel())
}
temp_overal = temp_overal.overall_stat
for j in temp_overal.keys():
temp_metrics[j] = temp_overal[j]
metrics = metrics.append(temp_metrics, ignore_index=True)
metrics = metrics[[
"name", "ACC Macro", "Bangdiwala B", "Bennett S",
"Conditional Entropy", "Cross Entropy", "F1 Micro", "FNR Micro",
"FPR Micro", "Gwet AC1", "Hamming Loss", "Joint Entropy",
"Kappa No Prevalence", "Mutual Information", "NIR", "Overall ACC",
"Overall RACC", "Overall RACCU", "PPV Micro", "Reference Entropy",
"Response Entropy", "Standard Error", "TNR Micro", "TPR Micro"
]]
metrics = metrics.set_index("name")
return metrics
def get_metrics(results, args, threshold, fraction):
"Create the metrics from an output df."
# Calculate biases after training
dem_parity = abs(
bm(results).P(pred=lambda x: x > threshold).given(race=0) -
bm(results).P(pred=lambda x: x > threshold).given(race=1))
eq_op = abs(
bm(results).P(
pred=lambda x: x > threshold).given(race=0, compas=True) -
bm(results).P(pred=lambda x: x > threshold).given(race=1, compas=True))
dem_parity_ratio = abs(
bm(results).P(pred=lambda x: x > threshold).given(race=0) /
bm(results).P(pred=lambda x: x > threshold).given(race=1))
cm = ConfusionMatrix(actual_vector=(results['true'] == True).values,
predict_vector=(results['pred'] > threshold).values)
if args.dataset == 'compas':
cm_high_risk = ConfusionMatrix(
actual_vector=(results['compas'] > 8).values,
predict_vector=(results['pred'] > 8).values)
result = {
"DP": dem_parity,
"EO": eq_op,
"DP ratio": dem_parity_ratio,
"acc": cm.Overall_ACC,
"acc_ci_min": cm.CI95[0],
"acc_ci_max": cm.CI95[1],
"f1": cm.F1_Macro,
"acc_high_risk": cm_high_risk.Overall_ACC,
"acc_ci_min_high_risk": cm_high_risk.CI95[0],
"acc_ci_max_high_risk": cm_high_risk.CI95[1],
"f1_high_risk": cm_high_risk.F1_Macro,
"adversarial_fraction": fraction
}
else:
result = {
"DP": dem_parity,
"EO": eq_op,
"DP ratio": dem_parity_ratio,
"acc": cm.Overall_ACC,
"acc_ci_min": cm.CI95[0],
"acc_ci_max": cm.CI95[1],
"f1": cm.F1_Macro,
"adversarial_fraction": fraction
}
return result
def plotcm(self, external=False, intercv=False, **options):
from nanolib.utils import plot_confusion_matrix
a = options.get('adj_left', 0.1)
b = options.get('adj_bottom', 0.2)
ps = options.get('print_stats', False)
title = options.get('title', False)
figsize = options.get('figsize', [5, 5])
axes_size = options.get('axes_size', 22)
if self.mode == 'classification':
if external:
y_true = self.y_test
y_pred = self.clf.predict(self.X_test)
else:
y_true = self.y_train
if intercv:
y_pred = self.crossValPredict
else:
y_pred = self.clf.predict(self.X_train)
cm_ = confusion_matrix(y_true, y_pred)
np.set_printoptions(precision=2)
class_names = np.unique(y_true)
self.CM = ConfusionMatrix(actual_vector=y_true,
predict_vector=y_pred)
if ps:
print(self.CM)
fig, _ = customplot(adj_bottom=b,
adj_left=a,
figsize=figsize,
axes_size=axes_size)
if title:
if external:
plt.title('External-Validation')
else:
if intercv:
plt.title('Internal-Validation')
else:
plt.title('Training Results')
plot_confusion_matrix(cm_, classes=class_names)
plt.show()
return fig
else:
print('Just for classification')
def plot_conf_mtx(Y_true,
Y_pred,
target_names,
file_name='files/output/Confusion matrix.png'):
print("Regular/Normalized confusion matrix")
count = len(Y_true)
ones = np.count_nonzero(Y_true)
zero = count - ones
cm = confusion_matrix(Y_true, Y_pred).ravel()
#tn, fp, fn, tp = cm.ravel()
cm = ConfusionMatrix(actual_vector=Y_true, predict_vector=Y_pred)
#cm.print_matrix()
#cm.print_normalized_matrix()
cnf_matrix = confusion_matrix(Y_true, Y_pred)
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
plt.clf()
plt.figure()
plt.subplot(1, 2, 1)
title = 'not normalized'
plot_confusion_matrix(cnf_matrix, classes=target_names, title=title)
plt.subplot(1, 2, 2)
# plt.savefig('files/output/'+title+'.png')
# Plot normalized confusion matrix
# plt.figure()
title = 'normalized'
plot_confusion_matrix(cnf_matrix,
classes=target_names,
normalize=True,
title=title)
plt.savefig(file_name)
def show_results(train_labels, test_labels, categories, abbr_categories,
predicted_categories):
"""
shows the results
:param train_image_paths:
:param test_image_paths:
:param train_labels:
:param test_labels:
:param categories:
:param abbr_categories:
:param predicted_categories:
:return:
"""
cat2idx = {cat: idx for idx, cat in enumerate(categories)}
# confusion matrix
y_true = [cat2idx[cat] for cat in test_labels]
y_pred = [cat2idx[cat] for cat in predicted_categories]
cm = ConfusionMatrix(y_true, y_pred)
plt.figure()
plt_cm = []
for i in cm.classes:
row = []
for j in cm.classes:
row.append(cm.table[i][j])
plt_cm.append(row)
plt_cm = np.array(plt_cm)
acc = np.mean(np.diag(plt_cm))
plt_cm = plt_cm.astype('float') / plt_cm.sum(axis=1)[:, np.newaxis]
plt.imshow(plt_cm, interpolation='nearest', cmap=plt.cm.get_cmap('jet'))
plt.title('Confusion matrix. Mean of diagonal = {:4.2f}%'.format(acc))
tick_marks = np.arange(len(categories))
plt.tight_layout()
plt.xticks(tick_marks, abbr_categories, rotation=45)
plt.yticks(tick_marks, categories)
def print_confusion_scores(y_test, y_pred):
cm = ConfusionMatrix(actual_vector=y_test, predict_vector=y_pred)
Log.info("TPR (True positive): %s", to_str(cm.TPR))
Log.info("TNR (True negative): %s", to_str(cm.TNR))
Log.info("PPV (Positive predictive value): %s", to_str(cm.PPV))
Log.info("NPV (Negative predictive value): %s", to_str(cm.NPV))
Log.info("ACC (Accuracy): %s", to_str(cm.ACC))
Log.info("AUC (Area under the ROC curve): %s", to_str(cm.AUC))
def calculateMCC(self, labels, predictions):
cm = ConfusionMatrix(labels, predictions, digit=5)
#print("\n Kappa-AC1: ", cm.Kappa, cm.AC1)
#print("\nMCC:")
#print(cm.MCC)
return cm.MCC
def ClassificationReport(ytrue, ypred, plotcm=False, file=None, **options):
from sklearn.metrics import classification_report, cohen_kappa_score, matthews_corrcoef, confusion_matrix
a = options.get('adj_left', 0.1)
b = options.get('adj_bottom', 0.2)
figsize = options.get('figsize', [4, 4])
axes_size = options.get('axes_size', 22)
# noinspection PyUnresolvedReferences
cmap = options.get('cmap', plt.cm.RdPu)
fontsize = options.get('fontsize', 26)
xrot = options.get('xrot', 0)
show = options.get('show', True)
acc = cohen_kappa_score(ytrue, ypred)
cm = confusion_matrix(ytrue, ypred)
matt = matthews_corrcoef(ytrue, ypred)
cr = classification_report(ytrue, ypred)
print(f'\nConfusion matrix:\n{cm}')
print(f'\nCohen kappa score : {np.round(acc, 3)}')
print(f'Matthews correlation coef: {np.round(matt, 3)}\n')
print(cr)
if file is not None:
from pycm import ConfusionMatrix
path_ = os.path.dirname(file)
if not os.path.exists(path_):
os.makedirs(path_)
CM = ConfusionMatrix(ytrue, ypred)
CM.save_html(file)
if plotcm:
fig, _ = customplot(adj_bottom=b,
adj_left=a,
figsize=figsize,
axes_size=axes_size)
plot_confusion_matrix(cm,
classes=np.unique(ytrue),
cmap=cmap,
fontsize=fontsize,
xrot=xrot)
if show:
plt.show()
return fig
def evaluate_accurancy(self, filename=''):
"""
Evaluate the Result of the emotion analyze
"""
from pycm import ConfusionMatrix
#actual_vector = self._data[CorporaProperties.EMOTION.value]
actual_vector = self._data.loc[:,
CorporaProperties.EMOTION.value].values
#print(actual_vector)
#predict_vector = self._data[CorporaProperties.CALCULATED_EMOTION.value]
predict_vector = self._data.loc[:, CorporaProperties.
CALCULATED_EMOTION.value].values
#print(predict_vector)
cm = ConfusionMatrix(actual_vector=actual_vector,
predict_vector=predict_vector)
#print(cm)
cm.save_csv(filename)
def get_phase_acc(self, true_label, pred):
pred = torch.FloatTensor(pred)
pred_phase = torch.softmax(pred, dim=1)
labels_pred = torch.argmax(pred_phase, dim=1).cpu().numpy()
cm = ConfusionMatrix(
actual_vector=true_label,
predict_vector=labels_pred,
)
return cm.Overall_ACC, cm.PPV, cm.TPR, cm.classes, cm.F1_Macro
def get_metrics_table(path_to_preds, path_to_true, norm=255.0):
"""Получение таблицы (pandas.DataFrame) метрик на основе сегментированных данных ИНС и эксперта
Arguments:
path_to_preds (str): путь до папки с изображениями, размеченными ИНС (без знака / в конце строки, "/home/data")
path_to_true (str): путь до папки с изображениями, размеченными экспертом (без знака / в конце строки, "/home/data"
norm (float): значение нормировки изображений (Изображение / norm), default: 255.0
Returns:
metrics (pandas.DataFrame): таблица со значениями рассчитаных метрик
"""
img_pred_data = []
names_pred = sorted(list(os.listdir(path_to_preds + "/")))
for img_pred_name in names_pred:
img_pred_data.append(cv2.imread(f"{path_to_preds}/{img_pred_name}") / norm)
img_true_data = []
names_true = sorted(list(os.listdir(path_to_true + "/")))
for img_true_name in names_true:
img_true_data.append(cv2.imread(f"{path_to_true}/{img_true_name}") / norm)
metrics = pd.DataFrame()
for i in range(len(img_pred_data)):
temp_overal = ConfusionMatrix(actual_vector=img_true_data[i].ravel(), predict_vector=img_pred_data[i].ravel())
temp_metrics = { "name": names_pred[i],
"PA": pixel_accuracy(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Jaccard": Jaccard(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Sorensen": Sorensen(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Kulchinski": Kulchinski(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Simpson": Simpson(img_pred_data[i].ravel(), img_true_data[i].ravel()),
"Braun_Blanke": Braun(img_pred_data[i].ravel(), img_true_data[i].ravel())
}
temp_overal = temp_overal.overall_stat
for j in temp_overal.keys():
temp_metrics[j] = temp_overal[j]
metrics = metrics.append(temp_metrics, ignore_index=True)
metrics = metrics[["name", "ACC Macro", "Bangdiwala B", "Bennett S", "Conditional Entropy", "Cross Entropy", "F1 Micro", "FNR Micro", "FPR Micro", "Gwet AC1","Hamming Loss", "Joint Entropy", "Kappa No Prevalence", "Mutual Information", "NIR", "Overall ACC", "Overall RACC", "Overall RACCU", "PPV Micro", "Reference Entropy", "Response Entropy", "Standard Error", "TNR Micro", "TPR Micro"]]
metrics = metrics.set_index("name")
return metrics
def eval(model, data_iter, is_predict=False, writer=None, step=None):
if len(data_iter) == 0:
return
model.to(device)
model.eval()
loss_total = 0
predict_all = np.array([], dtype=int)
labels_all = np.array([], dtype=int)
predict_prob = np.array([], dtype=int)
with no_grad():
for batch_data in data_iter:
labels_all = np.append(
labels_all,
batch_data.batch_label_list.detach().numpy())
batch_token_text_a = batch_data.batch_token_text_a.to(device)
batch_token_text_b = batch_data.batch_token_text_b.to(device)
batch_ids = batch_data.batch_input_ids.to(device)
batch_token_type = None if batch_data.batch_token_type_ids is None else batch_data.batch_token_type_ids.to(
device)
batch_mask = batch_data.batch_attention_mask.to(device)
batch_label = None if is_predict is True else batch_data.batch_label_list.to(
device)
ret = model(input_ids=batch_ids,
token_type_ids=batch_token_type,
attention_mask=batch_mask,
labels=batch_label)
loss = ret.loss
logits = ret.logits
if not is_predict:
loss_total += loss.item()
predict_prob = np.append(
predict_prob,
[logit[1] for logit in softmax(logits, dim=1).cpu().numpy()])
predict = argmax(logits, 1).cpu().numpy()
predict_all = np.append(predict_all, predict)
if is_predict:
np.savetxt('result.tsv', predict_prob, fmt='%.03f', delimiter='\r\n')
os.system('zip result.zip {}'.format('result.tsv'))
return None
else:
matrix = ConfusionMatrix(labels_all.flatten(), predict_all.flatten())
print('Test loss: {}'.format(loss_total / len(data_iter)))
print(matrix)
if writer:
writer.add_scalar("Test/Overall ACC",
matrix.overall_stat['Overall ACC'], step)
writer.add_scalar("Test/loss", (loss_total / len(data_iter)), step)
return round(loss_total / len(data_iter), 4)
def update(self, y, predictions, sample_weight=None, **kwargs):
n_classes = y.shape[-1]
actual_vector = np.argmax(y.reshape(-1, n_classes), -1)
predict_vector = np.argmax(predictions.reshape(-1, n_classes), -1)
if sample_weight is not None:
sample_weight = sample_weight.reshape(-1)
confusion_matrix = ConfusionMatrix(
actual_vector=actual_vector,
predict_vector=predict_vector,
sample_weight=sample_weight,
)
for key, value in confusion_matrix.table.items():
self._confusion_matrix[key].update(value)
def accuracy(pred, label):
"""calculate accuracy using PYCM
Args:
pred (numpy.array) : pred label for each batch (batch_size X number of pred classes)
label (numpy.array) : label for each batch (batch_size X number of classes)
Returns:
accuracy (float) : overall accuracy value
"""
assert type(pred) == type(label), "each type should be same"
pred = pred.reshape(-1)
label = label.reshape(-1)
cm = ConfusionMatrix(pred, label)
return cm.Overall_ACC
def confusion_matrix(self):
y_preds, ys = [], []
# same code as validate
self.model.eval()
for X, y in self.validate_loader:
with torch.no_grad():
X, y = X.to(self.device), y.to(self.device)
output = self.model(X)
y_pred = F.log_softmax(output, dim=1)
y_pred = y_pred.max(1)[1]
y_preds.append(y_pred.cpu().numpy())
ys.append(y.cpu().numpy())
y_preds = np.array(y_preds).flatten()
ys = np.array(ys).flatten()
return ConfusionMatrix(actual_vector=ys, predict_vector=y_preds)
def draw_conf_matrix(confusion_matrix: ConfusionMatrix, algo_name: str,
fold_iter: int):
classification_classes = confusion_matrix.classes
confusion_matrix_df = DataFrame(confusion_matrix.to_array())
row_sums = confusion_matrix_df.sum(axis=1).to_list()
x_labels = [
"{} ({})".format(label, row_sums[index])
for index, label in enumerate(classification_classes)
]
temp_df = confusion_matrix_df.T
shape = temp_df.shape
annotations = [[
f'''{classification_classes[row]}\n{classification_classes[col]}\n{temp_df[row][col]}'''
for col in range(0, shape[0])
] for row in range(0, shape[1])]
confusion_matrix_heatmap = sns.heatmap(
confusion_matrix_df.T,
annot=DataFrame(annotations).T,
xticklabels=x_labels,
yticklabels=classification_classes,
cmap="OrRd",
linewidths=0.1,
linecolor="black",
fmt='',
cbar=False,
)
fig = confusion_matrix_heatmap.get_figure()
fig.savefig("./test_result_data/{}-fold-{}.svg".format(
algo_name, fold_iter),
bbox_inches='tight')
fig.savefig("./test_result_data/{}-fold-{}.png".format(
algo_name, fold_iter),
bbox_inches='tight')
plt.clf()
pass
def sentiment_metrics(tweets_corpus, method):
prediction_list = list()
y_list = list()
feature_list = set()
for tweet, label in tweets_corpus:
result = classify_tweet(tweet, method)
pred, score, pos_sent, neg_sent, features = result
if features:
for f in features:
feature_list.add(f)
print('predicted:', pred)
print('actual label:', label)
print('label == predicted?', label == pred)
print('============================\n\n')
prediction_list.append(pred)
y_list.append(label)
print('features mentioned in the corpus:')
for f in feature_list:
print(f)
# label_names = ['negative', 'positive', 'neutral']
# label_names = ['negative', 'positive', 'neutral']
label_names = ['negative', 'positive']
pred_labels = np.asarray([label_names.index(p) for p in prediction_list])
true_labels = np.asarray([label_names.index(y) for y in y_list])
cm = ConfusionMatrix(actual_vector=true_labels,
predict_vector=pred_labels) # Create CM From Data
print('----------- summary results -----------------')
print('classes:\n', cm.classes)
print('classes names: ', *label_names)
print('ACC(Accuracy)', cm.class_stat.get('ACC'))
print('F1 score', cm.class_stat.get('F1'))
print(
'Accuracy AVG',
sum(cm.class_stat.get('ACC').values()) / len(cm.class_stat.get('ACC')))
print('F1 AVG',
sum(cm.class_stat.get('F1').values()) / len(cm.class_stat.get('F1')))
print('----------------------------------------------')
def numpy_confusion_matrix_to_pycm(confusion_matrix_numpy, labels=None):
"""Create a pycm confusion matrix from a NumPy confusion matrix
Creates a confusion matrix object with the pycm library based on a confusion matrix as 2D NumPy array (such as
the one generated by the sklearn confusion matrix function).
See more about pycm confusion matrices at `pycm`_, and see more
about sklearn confusion matrices at `sklearn confusion matrix`_.
Args:
confusion_matrix_numpy (np.array((num_classes, num_classes)) :
labels (list) :
Returns:
confusion_matrix_pycm (pycm.ConfusionMatrix) :
.. _`pycm`: https://github.com/sepandhaghighi/pycm
.. _`sklearn confusion matrix`:
https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html
"""
# Create empty dict to be used as input for pycm (see https://github.com/sepandhaghighi/pycm#direct-cm)
confusion_matrix_dict = {}
# Find number and classes and check labels
num_classes = np.shape(confusion_matrix_numpy)[0]
if not labels: # If no labels are provided just use [0, 1, ..., num_classes]
labels = range(num_classes)
elif len(labels) != num_classes:
raise AttributeError(
"Number of provided labels does not match number of classes.")
# Fill the dict in the format required by pycm with values from the sklearn confusion matrix
for row in range(num_classes):
row_dict = {}
for col in range(num_classes):
row_dict[str(labels[col])] = int(confusion_matrix_numpy[row, col])
confusion_matrix_dict[str(labels[row])] = row_dict
# Instantiate the pycm confusion matrix from the dict
confusion_matrix_pycm = ConfusionMatrix(matrix=confusion_matrix_dict)
return confusion_matrix_pycm
test_file = pandas.read_csv(get_test_file_name(fold))
data_test = test_file[DATA_KEY]
target_test = test_file[TARGET_KEY]
tfidf_vectorizer = joblib.load(get_vectorizer_file_name(fold))
processed_data_test = tfidf_vectorizer.transform(data_test).toarray()
predicted_data_test = model.predict(processed_data_test)
precision, recall, f_score, support = precision_recall_fscore_support(
target_test, predicted_data_test, average='macro')
report_confusion_matrix = ConfusionMatrix(
actual_vector=target_test.to_list(),
predict_vector=predicted_data_test)
draw_conf_matrix(
report_confusion_matrix,
ALGORITHM_LABELS[algorithm_id],
fold_iteration,
)
positions = report_confusion_matrix.position()
precisions = []
recalls = []
f1_scores = []
accuracies = []
report_text = ""
plt.ylim(0.8, 1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr, tpr, color='blue', lw=lw, label='RAW (area = {:.2f})'.format(roc_auc))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve (zoomed in at top left)')
plt.legend(loc='best')
plt.show()
from sklearn import metrics
print('Accuracy = ',metrics.accuracy_score(labels,y_cal))
from pycm import ConfusionMatrix
cm = ConfusionMatrix(actual_vector=labels, predict_vector=y_cal)
print(cm)
print(__doc__)
import itertools
import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
def train(args, train_dataset, model, tokenizer, evaluate_fn=None):
""" Train the model """
model.train()
if args.local_rank in [-1, 0]:
tb_writer = SummaryWriter()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
args.train_batch_size = args.per_gpu_train_batch_size * max(
1, args.n_gpu)
train_sampler = RandomSampler(
train_dataset) if args.local_rank == -1 else DistributedSampler(
train_dataset)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=args.train_batch_size)
if args.max_steps > 0:
t_total = args.max_steps
args.num_train_epochs = args.max_steps // (
len(train_dataloader) // args.gradient_accumulation_steps) + 1
else:
t_total = len(
train_dataloader
) // args.gradient_accumulation_steps * args.num_train_epochs
# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
args.weight_decay,
},
{
"params": [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0
},
]
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=args.warmup_steps,
num_training_steps=t_total)
# Check if saved optimizer or scheduler states exist
if os.path.isfile(os.path.join(
args.model_name_or_path, "optimizer.pt")) and os.path.isfile(
os.path.join(args.model_name_or_path, "scheduler.pt")):
# Load in optimizer and scheduler states
optimizer.load_state_dict(
torch.load(
os.path.join(args.model_name_or_path, "optimizer.pt")))
scheduler.load_state_dict(
torch.load(
os.path.join(args.model_name_or_path, "scheduler.pt")))
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
# multi-gpu training (should be after apex fp16 initialization)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
# Distributed training (should be after apex fp16 initialization)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model,
device_ids=[args.local_rank],
output_device=args.local_rank,
find_unused_parameters=True,
)
if args.local_rank == 0:
torch.distributed.barrier(
) # Make sure only the first process in distributed training will download model & vocab
model.to(args.device)
# Train!
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_dataset))
logger.info(" Num Epochs = %d", args.num_train_epochs)
logger.info(" Instantaneous batch size per GPU = %d",
args.per_gpu_train_batch_size)
logger.info(
" Total train batch size (w. parallel, distributed & accumulation) = %d",
args.train_batch_size * args.gradient_accumulation_steps *
(torch.distributed.get_world_size()
if args.local_rank != -1 else 1),
)
logger.info(" Gradient Accumulation steps = %d",
args.gradient_accumulation_steps)
logger.info(" Total optimization steps = %d", t_total)
global_step = 0
epochs_trained = 0
steps_trained_in_current_epoch = 0
# Check if continuing training from a checkpoint
if os.path.exists(args.model_name_or_path):
# set global_step to gobal_step of last saved checkpoint from model path
global_step = int(
args.model_name_or_path.split("-")[-1].split("/")[0])
epochs_trained = global_step // (len(train_dataloader) //
args.gradient_accumulation_steps)
steps_trained_in_current_epoch = global_step % (
len(train_dataloader) // args.gradient_accumulation_steps)
logger.info(
" Continuing training from checkpoint, will skip to saved global_step"
)
logger.info(" Continuing training from epoch %d", epochs_trained)
logger.info(" Continuing training from global step %d",
global_step)
logger.info(" Will skip the first %d steps in the first epoch",
steps_trained_in_current_epoch)
tr_loss, logging_loss = 0.0, 0.0
model.zero_grad()
train_iterator = trange(
epochs_trained,
int(args.num_train_epochs),
desc="Epoch",
disable=args.local_rank not in [-1, 0],
)
#set_seed(args) # Added here for reproductibility
for _ in train_iterator:
epoch_iterator = tqdm(train_dataloader,
desc="Iteration",
disable=args.local_rank not in [-1, 0],
position=0,
leave=True)
for step, batch in enumerate(epoch_iterator):
# Skip past any already trained steps if resuming training
if steps_trained_in_current_epoch > 0:
steps_trained_in_current_epoch -= 1
continue
model.train()
batch = tuple(t.to(args.device) for t in batch)
inputs = {
"input_ids": batch[0],
"attention_mask": batch[1],
"labels": batch[3]
}
if args.model_type != "distilbert":
inputs["token_type_ids"] = (
batch[2] if args.model_type
in ["bert", "xlnet", "albert"] else None
) # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids
outputs = model(**inputs)
loss = outputs[
0] # model outputs are always tuple in transformers (see doc)
if args.n_gpu > 1:
loss = loss.mean(
) # mean() to average on multi-gpu parallel training
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
tr_loss += loss.item()
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
torch.nn.utils.clip_grad_norm_(
amp.master_params(optimizer), args.max_grad_norm)
else:
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
scheduler.step() # Update learning rate schedule
model.zero_grad()
global_step += 1
if args.local_rank in [
-1, 0
] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
logs = {}
if (
args.local_rank == -1
and args.evaluate_during_training
): # Only evaluate when single GPU otherwise metrics may not average well
results = evaluate(args, model, tokenizer)
for key, value in results.items():
eval_key = "eval_{}".format(key)
logs[eval_key] = value
loss_scalar = (tr_loss -
logging_loss) / args.logging_steps
learning_rate_scalar = scheduler.get_lr()[0]
logs["learning_rate"] = learning_rate_scalar
logs["loss"] = loss_scalar
logging_loss = tr_loss
for key, value in logs.items():
tb_writer.add_scalar(key, value, global_step)
epoch_iterator.set_postfix({
**logs,
**{
"step": global_step
}
})
if args.local_rank in [
-1, 0
] and args.save_steps > 0 and global_step % args.save_steps == 0:
# Save model checkpoint
output_dir = os.path.join(
args.output_dir,
"checkpoint-{}".format(global_step))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
model_to_save = (
model.module if hasattr(model, "module") else model
) # Take care of distributed/parallel training
model_to_save.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)
torch.save(
args, os.path.join(output_dir,
"training_args.bin"))
logger.info("Saving model checkpoint to %s",
output_dir)
torch.save(optimizer.state_dict(),
os.path.join(output_dir, "optimizer.pt"))
torch.save(scheduler.state_dict(),
os.path.join(output_dir, "scheduler.pt"))
logger.info(
"Saving optimizer and scheduler states to %s",
output_dir)
if args.max_steps > 0 and global_step > args.max_steps:
epoch_iterator.close()
break
if args.max_steps > 0 and global_step > args.max_steps:
train_iterator.close()
break
if evaluate_fn is not None:
results = pd.DataFrame(
evaluate_fn(args.evaluate_dataset, model))
cm = ConfusionMatrix(
actual_vector=results['true'].values,
predict_vector=results['predicted'].values)
logs = {}
logs["eval_f1_macro"] = cm.F1_Macro
logs["eval_acc_macro"] = cm.ACC_Macro
logs["eval_acc_overall"] = cm.Overall_ACC
logger.info("Results on eval: {}".format(logs))
if args.local_rank in [-1, 0]:
tb_writer.close()
return global_step, tr_loss / global_step
class Train:
"""
==========================================================
Training procedure for classification and regression model
==========================================================
Train(random_state, cv, mode, colors, scaler, multiclassroc, verbose)
Methods:
- fit(x, y)
- evaluate(x, y)
- predict(x)
- classification_report(external=bool, intercv=bool)
- scores_()
- estimator_()
- plotcm(external, intercv, print_stats, adj_left, adj_bottom, fromatplot, title)
- save_stats(file)
- pointplot_(adj_left, adj_bottom, cross_val_predict)
- results()
- plotregression_(adj_left, adj_bottom_cross_val_predict)
"""
def __init__(self, estimator, **options):
from sklearn.model_selection import StratifiedKFold
self.X_train = None
self.X_test = None
self.y_train = None
self.y_test = None
self.intercv = None
self.estimator = estimator
self.random_state = options.get('random_state', 99)
self.cv = options.get(
'cv',
StratifiedKFold(n_splits=10,
shuffle=True,
random_state=self.random_state))
self.mode = options.get('mode', 'classification')
self.colors = options.get('colors', None)
self.scaler = options.get('scaler', StandardScaler())
self.multiroc = options.get('multiclassroc', True)
self.verbose = options.get('verbose', 2)
self.compact = options.get('compact', True)
self.internal = list()
self.external = list()
self.training = list()
self.crossValPredict = list()
self.clf = list()
self.CM = list()
self.predictCVDF = pd.DataFrame()
self.predictTrainDF = pd.DataFrame()
self.predictTestDF = pd.DataFrame()
def __repr__(self):
if self.compact:
return f'Scaling : {self.scaler!r}\nEstimator: {self.estimator!r}\n'
else:
return f'Estimator: {self.estimator!r}\n'
def fit(self, *arrays):
from sklearn.pipeline import Pipeline
from nanolib.utils import plotroc
if len(arrays) == 2:
self.X_train = arrays[0]
self.y_train = arrays[1]
self.intercv = True
else:
self.X_train = arrays[0]
self.X_test = arrays[1]
self.y_train = arrays[2]
self.y_test = arrays[3]
self.intercv = False
# classification using accuracy metric
if self.mode == 'classification':
if self.compact:
pipe = Pipeline([('scaler', self.scaler),
('clf', self.estimator)])
else:
pipe = self.estimator
self.internal = cross_val_score(pipe,
self.X_train,
self.y_train,
n_jobs=-1,
cv=self.cv,
scoring=None)
self.crossValPredict = cross_val_predict(pipe,
self.X_train,
self.y_train,
cv=self.cv,
n_jobs=-1)
self.clf = pipe.fit(self.X_train, self.y_train)
self.training = self.clf.predict(self.X_train)
predTrain = self.clf.predict(self.X_train)
self.predictTrainDF = pd.concat([
pd.DataFrame(data=self.y_train, columns=['Actual']),
pd.DataFrame(data=predTrain, columns=['Prediction']),
],
axis=1)
self.predictCVDF = pd.concat([
pd.DataFrame(data=self.y_train, columns=['Actual']),
pd.DataFrame(data=self.crossValPredict, columns=['Prediction'])
],
axis=1)
print('Metric - accuracy_score:')
internalcv = cross_val_score(pipe,
self.X_train,
self.y_train,
n_jobs=-1,
cv=self.cv,
scoring=None)
print(
f'Mean of Internal-Validation : {round(np.mean(internalcv), 3)}'
)
print(
f'Stdev of Internal-Validation : {round(np.std(internalcv), 3)}'
)
print(
f'Training score : {round(accuracy_score(self.y_train, self.training), 3)}\n'
)
print('Metric - F1-Score (macro):')
scoring = make_scorer(f1_score, average='macro')
internalcv = cross_val_score(pipe,
self.X_train,
self.y_train,
n_jobs=-1,
cv=self.cv,
scoring=scoring)
print(
f'Mean of Internal-Validation : {round(np.mean(internalcv), 3)}'
)
print(
f'Stdev of Internal-Validation : {round(np.std(internalcv), 3)}'
)
print(
f'Training score : {round(f1_score(self.y_train, self.training, average="macro"), 3)}\n'
)
if not self.intercv:
self.external = self.clf.predict(self.X_test)
self.predictTestDF = pd.concat([
pd.DataFrame(data=self.y_test, columns=['Actual']),
pd.DataFrame(data=self.external, columns=['Prediction'])
],
axis=1)
print(
f'External-validation accuracy score : {round(accuracy_score(self.y_test, self.external), 3)}\n'
)
print(
f'External-validation F1-score (macro): {round(f1_score(self.y_test, self.external, average="macro"), 3)}\n'
)
# regression using rsquared metric
elif self.mode == 'regression':
if self.compact:
pipe = Pipeline([('scaler', self.scaler),
('clf', self.estimator)])
else:
pipe = self.estimator
self.crossValPredict = cross_val_predict(pipe,
self.X_train,
self.y_train,
cv=self.cv,
n_jobs=-1)
self.predictCVDF = pd.concat([
pd.DataFrame(data=self.y_train, columns=['Actual']),
pd.DataFrame(data=self.crossValPredict, columns=['Prediction'])
],
axis=1)
self.internal = cross_val_score(pipe,
self.X_train,
self.y_train,
n_jobs=-1,
cv=self.cv,
scoring='r2')
print('Internal-Validation Score')
print(f'Mean of R2 score : {np.mean(self.internal)}')
print(f'Stdev of R2 score : {np.std(self.internal)}')
print(
f'Mean of adj-R2 score : {np.mean(self.adj_r2_squared(self.internal, self.X_train, self.y_train))}'
)
print(
f'Stdev of adj-R2 score : {np.std(self.adj_r2_squared(self.internal, self.X_train, self.y_train))}'
)
self.internal = cross_val_score(
pipe,
self.X_train,
self.y_train,
n_jobs=-1,
cv=self.cv,
scoring='neg_root_mean_squared_error')
print(f'Mean of RMSE score : {np.mean(self.internal)}')
print(f'Stdev of RMSE score : {np.std(self.internal)}\n')
self.internal = cross_val_score(pipe,
self.X_train,
self.y_train,
n_jobs=-1,
cv=self.cv,
scoring='neg_mean_absolute_error')
print(f'Mean of MAE score : {np.mean(self.internal)}')
print(f'Stdev of MAE score : {np.std(self.internal)}\n')
self.clf = pipe.fit(self.X_train, self.y_train)
self.training = self.clf.predict(self.X_train)
pred_training = pd.DataFrame(data=self.training,
columns=['Prediction'])
self.predictTrainDF = pd.concat([
pd.DataFrame(data=self.y_train, columns=['Actual']),
pred_training
],
axis=1)
print('Training Score ')
print(
f'R2 score : {r2_score(self.y_train, self.training)}'
)
print(
f'adj-R2 score : {self.adj_r2_squared(r2_score(self.y_train, self.training), self.X_train, self.y_train)}'
)
print(
f'RMSE score : {mean_squared_error(self.y_train, self.training)}'
)
print(
f'MAE score : {mean_absolute_error(self.y_train, self.training)}\n'
)
if not self.intercv:
self.external = self.clf.predict(self.X_test)
pred_external = pd.DataFrame(data=self.external,
columns=['Prediction'])
self.predictTestDF = pd.concat([
pd.DataFrame(data=self.y_test, columns=['Actual']),
pred_external
],
axis=1)
print('External-Validation Score')
print(
f'R2 score : {r2_score(self.y_test, self.external)}'
)
print(
f'adj-R2 score : {self.adj_r2_squared(r2_score(self.y_test, self.external), self.X_test, self.y_test)}'
)
print(
f'RMSE score : {mean_squared_error(self.y_test, self.external)}'
)
print(
f'MAE score : {mean_absolute_error(self.y_test, self.external)}\n'
)
elif self.mode == 'roc':
from sklearn.multiclass import OneVsRestClassifier
clf = OneVsRestClassifier(self.estimator)
if self.compact:
pipe = Pipeline([('scaler', self.scaler), ('clf', clf)])
else:
pipe = clf
y_score = clf.fit(self.X_train,
self.y_train).decision_function(self.X_test)
fig, res = plotroc(self.y_test,
y_score,
colors=self.colors,
multiclass=self.multiroc)
return fig, res
else:
return print(
f'Your input scoring is {self.scoring} that not fit with one of accuracy, rsquared, and roc'
)
def predict(self, *array):
return self.clf.predict(array[0])
def evaluate(self, *arrays):
self.X_test = arrays[0]
self.y_test = arrays[1]
self.external = self.clf.predict(self.X_test)
if self.mode == 'classification':
self.predictTestDF = pd.concat([
pd.DataFrame(data=self.y_test, columns=['Actual']),
pd.DataFrame(data=self.external, columns=['Prediction'])
],
axis=1)
print(
f'External-validation accuracy score : {accuracy_score(self.y_test, self.external)}\n'
)
# print(f'External-validation F1-score (micro): {f1_score(self.y_test, self.external, average="micro")}\n')
print(
f'External-validation F1-score (macro): {f1_score(self.y_test, self.external, average="macro")}\n'
)
elif self.mode == 'regression':
pred_external = pd.DataFrame(data=self.external,
columns=['Prediction'])
self.predictTestDF = pd.concat([
pd.DataFrame(data=self.y_test, columns=['Actual']),
pred_external
],
axis=1)
print('External-Validation Score')
print(
f'R2 score : {r2_score(self.y_test, self.external)}'
)
print(
f'adj-R2 score : {self.adj_r2_squared(r2_score(self.y_test, self.external), self.X_test, self.y_test)}'
)
print(
f'RMSE score : {mean_squared_error(self.y_test, self.external)}'
)
print(
f'MAE score : {mean_absolute_error(self.y_test, self.external)}\n'
)
def scores_(self):
if self.intercv:
return self.internal, self.training, self.crossValPredict
else:
return self.internal, self.training, self.crossValPredict, self.external
def estimator_(self):
return self.clf
def classification_report(self, external=False, intercv=False, file=None):
from sklearn.metrics import classification_report, cohen_kappa_score, matthews_corrcoef
if self.mode == 'classification':
if external:
y_true = self.y_test
y_pred = self.clf.predict(self.X_test)
info = 'external-validation'
else:
y_true = self.y_train
if intercv:
info = 'internal-validation'
y_pred = self.crossValPredict
else:
info = 'training-validation'
y_pred = self.clf.predict(self.X_train)
acc = cohen_kappa_score(y_true, y_pred)
cm = confusion_matrix(y_true, y_pred)
matt = matthews_corrcoef(y_true, y_pred)
cr = classification_report(y_true, y_pred, digits=3)
print(f'\nConfusion matrix for {info}:\n{cm}')
print(f'\nCohen kappa score for {info}: {np.round(acc, 3)}')
print(
f'Matthews correlation coef for {info}: {np.round(matt, 3)}\n')
print(cr)
if file != None:
self.CM = ConfusionMatrix(actual_vector=y_true,
predict_vector=y_pred)
self.CM.save_html(f'{str(file)}-{info}')
def plotcm(self, external=False, intercv=False, **options):
from nanolib.utils import plot_confusion_matrix
a = options.get('adj_left', 0.1)
b = options.get('adj_bottom', 0.2)
ps = options.get('print_stats', False)
title = options.get('title', False)
figsize = options.get('figsize', [5, 5])
axes_size = options.get('axes_size', 22)
if self.mode == 'classification':
if external:
y_true = self.y_test
y_pred = self.clf.predict(self.X_test)
else:
y_true = self.y_train
if intercv:
y_pred = self.crossValPredict
else:
y_pred = self.clf.predict(self.X_train)
cm_ = confusion_matrix(y_true, y_pred)
np.set_printoptions(precision=2)
class_names = np.unique(y_true)
self.CM = ConfusionMatrix(actual_vector=y_true,
predict_vector=y_pred)
if ps:
print(self.CM)
fig, _ = customplot(adj_bottom=b,
adj_left=a,
figsize=figsize,
axes_size=axes_size)
if title:
if external:
plt.title('External-Validation')
else:
if intercv:
plt.title('Internal-Validation')
else:
plt.title('Training Results')
plot_confusion_matrix(cm_, classes=class_names)
plt.show()
return fig
else:
print('Just for classification')
# noinspection PyUnresolvedReferences
def save_stats(self, **kwargs):
fullpathname = kwargs.get('file', 'cm')
self.CM.save_html(fullpathname)
def pointplot_(self, **options):
import math
yint = range(min(self.y_train), math.ceil(max(self.y_train)) + 1)
a = options.get('adj_left', 0.12)
b = options.get('adj_bottom', 0.12)
mode = options.get('cross_val_predict', False)
if mode:
trainDF = self.predictCVDF
else:
trainDF = self.predictTrainDF
trainDF = trainDF.sort_values(by='Actual')
if self.intercv:
fig, ax = customplot(adj_left=a, adj_bottom=b)
plt.plot(trainDF.Actual.values, '--r')
plt.plot(trainDF.Prediction.values, '-bo')
plt.xlabel('Samples')
plt.ylabel('Class prediction')
plt.yticks(yint)
return fig
else:
testDF = self.predictTestDF
testDF = testDF.sort_values(by='Actual')
fig, ax = customplot(adj_bottom=b, adj_left=a)
plt.subplot(2, 1, 1)
plt.plot(trainDF.Actual.values, '--r')
plt.plot(trainDF.Prediction.values, '-bo')
plt.ylabel('Train prediction')
plt.yticks(yint)
plt.subplot(2, 1, 2)
plt.plot(testDF.Actual.values, '--r')
plt.plot(testDF.Prediction.values, '-bo')
plt.ylabel('Test prediction')
plt.xlabel('Samples')
plt.yticks(yint)
return fig
def results(self):
if self.intercv:
return self.predictTrainDF, self.predictCVDF
else:
return self.predictTrainDF, self.predictCVDF, self.predictTestDF
@staticmethod
def abline(slope, intercept):
axes = plt.gca()
x_vals = np.array(axes.get_xlim())
y_vals = intercept + slope * x_vals
plt.plot(x_vals, y_vals, '--')
@staticmethod
def adj_r2_squared(r2, X, y):
return 1 - (1 - r2) * (len(y) - 1) / (len(y) - X.shape[1] - 1)
def plotregression_(self, **options):
a = options.get('adj_left', 0.12)
b = options.get('adj_bottom', 0.12)
mode = options.get('cross_val_predict', False)
fig, ax = customplot(adj_left=a, adj_bottom=b)
if mode:
ax.scatter(self.predictCVDF.Actual,
self.predictCVDF.Prediction,
s=70,
c='b',
marker='o',
label='Training')
else:
ax.scatter(self.predictTrainDF.Actual,
self.predictTrainDF.Prediction,
s=70,
c='b',
marker='o',
label='Training')
if not self.intercv:
ax.scatter(self.predictTestDF.Actual,
self.predictTestDF.Prediction,
s=70,
c='r',
marker='v',
label='Testing')
ax.legend()
self.abline(1, 0)
plt.xlabel('Actual')
plt.ylabel('Prediction')
return fig
features = ['CATEGORY', 'TITLE']
train_df = pd.read_table('./data/train.txt', header=None, names=features)
test_df = pd.read_table('./data/test.txt', header=None, names=features)
clf = pickle.load(open('./models/52lr.pickle', mode='rb'))
vectorizer = pickle.load(open('./models/51vectorizer.pickle', mode='rb'))
le = pickle.load(open('./models/52le.pickle', mode='rb'))
sc = pickle.load(open('./models/52sc.pickle', mode='rb'))
X_train_sparse = load_npz("./feature/train.feature.npz")
X_valid_sparse = load_npz("./feature/valid.feature.npz")
X_test_sparse = load_npz("./feature/test.feature.npz")
X_train, X_valid, X_test = X_train_sparse.toarray(), \
X_valid_sparse.toarray(), X_test_sparse.toarray()
#訓練データでの混同行列
labels_train_pred = clf.predict(sc.transform(X_train))
labels_train_true = le.transform(train_df['CATEGORY'].values)
cm_train = ConfusionMatrix(le.inverse_transform(labels_train_true)\
, le.inverse_transform(labels_train_pred))
print('By training data\n')
cm_train.print_matrix()
#評価データでの混同行列
labels_test_pred = clf.predict(sc.transform(X_test))
labels_test_true = le.transform(test_df['CATEGORY'].values)
cm_test = ConfusionMatrix(le.inverse_transform(labels_test_true)\
, le.inverse_transform(labels_test_pred))
print('By test data\n')
cm_test.print_matrix()