if model_name == 'ensemble':
rad_preds = np.load(
os.path.join(cv_dir, str(i + 1), 'rad1', 'preds.npy'))
desc_preds = np.load(
os.path.join(cv_dir, str(i + 1), 'desc', 'preds.npy'))
preds = rad_preds + desc_preds
labels = np.load(
os.path.join(cv_dir, str(i + 1), 'rad1', 'labels.npy'))
else:
preds = np.load(
os.path.join(cv_dir, str(i + 1), model_name, 'preds.npy'))
labels = np.load(
os.path.join(cv_dir, str(i + 1), model_name, 'labels.npy'))
result = ddie_compute_metrics('ddie',
np.argmax(preds, axis=1),
labels,
every_type=True)
#for label_type in label_list:
for label_i, label_type in enumerate(label_list):
key = model_list[model_i] + ' ' + label_type
#print(i, key, result[label_type])
#print('& {:.2f}'.format(result[label_type] * 100), end=' ')
if key in macro_result_dict:
macro_result_dict[key].append(result[label_type])
else:
macro_result_dict[key] = [result[label_type]]
result_table[model_i, label_i] = result[label_type]
print('Fold {} '.format(i + 1), end='')
print_result(result_table)
sentence_fscores = [[] for i in range(N)]
for i in range(k):
cnn_preds_path = os.path.join(cv_dir, str(i+1), 'cnn', 'preds.npy')
rad_preds_path = os.path.join(cv_dir, str(i+1), 'rad1', 'preds.npy')
desc_preds_path = os.path.join(cv_dir, str(i+1), 'desc', 'preds.npy')
cnn_labels_path = os.path.join(cv_dir, str(i+1), 'cnn', 'labels.npy')
cnn_preds = np.load(cnn_preds_path)
rad_preds = np.load(rad_preds_path)
desc_preds = np.load(desc_preds_path)
labels = np.load(cnn_labels_path)
cnn_result = ddie_compute_metrics('ddie', np.argmax(cnn_preds, axis=1), labels, every_type=False)
print(cnn_result)
ensembled_preds = rad_preds + desc_preds
ensembled_result = ddie_compute_metrics('ddie', np.argmax(ensembled_preds, axis=1), labels, every_type=False)
print(ensembled_result)
fscore = ensembled_result['microF']
ensembled_fscores.append(fscore)
for j in range(N):
if len(indices[i][j]) == 0:
print('---')
continue
#div_preds = cnn_preds[np.array(indices[i][j])]
rad_preds_path = os.path.join(cv_dir, str(i + 1), 'rad1',
'preds.npy')
desc_preds_path = os.path.join(cv_dir, str(i + 1), 'desc',
'preds.npy')
rad_preds = np.load(rad_preds_path)
desc_preds = np.load(desc_preds_path)
preds = rad_preds + desc_preds
labels_path = os.path.join(cv_dir, str(i + 1), 'cnn',
'labels.npy')
labels = np.load(labels_path)
div_preds = preds[np.array(indices[i][j])]
div_labels = labels[np.array(indices[i][j])]
div_result = ddie_compute_metrics('ddie',
np.argmax(div_preds, axis=1),
div_labels,
every_type=False)
div_fscore = div_result['microF']
#print(j, div_fscore, len(indices[i][j]))
sentence_fscores[j].append(div_fscore)
if micro_preds is None:
micro_preds = div_preds
micro_labels = div_labels
else:
micro_preds = np.concatenate((micro_preds, div_preds), axis=0)
micro_labels = np.concatenate((micro_labels, div_labels),
axis=0)
micro_result = ddie_compute_metrics('ddie',
np.argmax(micro_preds, axis=1),
micro_labels,
import numpy as np
import random
from metrics_ddie import ddie_compute_metrics
from statsmodels.stats import contingency_tables
_, output_dir1, output_dir2 = sys.argv
preds1 = np.load(os.path.join(output_dir1, 'preds.npy'))
preds2 = np.load(os.path.join(output_dir2, 'preds.npy'))
labels = np.load(os.path.join(output_dir1, 'labels.npy'))
preds1 = np.argmax(preds1, axis=1)
preds2 = np.argmax(preds2, axis=1)
result1 = ddie_compute_metrics('ddie', preds1, labels, every_type=True)
result2 = ddie_compute_metrics('ddie', preds2, labels, every_type=True)
print(result1, result2)
apbp, apbn, anbp, anbn = 0, 0, 0, 0
for p1, p2, label in zip(preds1, preds2, labels):
if p1 == label:
if p2 == label:
apbp += 1
else:
apbn += 1
else:
if p2 == label:
anbp += 1
else:
anbn += 1