def calc_hubness(indices, K=[3, 5, 7, 11, 17], verbose=False, label=''):
if verbose:
plt.figure()
h = np.zeros(len(K))
for i in range(len(K)):
h_ = hubness(indices, K[i])
h[i] = h_[0]
if verbose:
#plt.subplot(1,len(K), i+1)
plt.plot(smooth(h_[1], window_length=1))
h_mean, h_std = np.mean(h), np.std(h)
if verbose:
plt.legend(['K={}: {:.2f}'.format(k, ind) for k, ind in zip(K, h)])
plt.title('{} (Hubness = {:.2f}))'.format(label, h_mean))
plt.xlabel('Nk')
plt.ylabel('P(Nk)')
return h_mean, h_std
plt.figure(experiment_name + ' Correlation: ' + dset)
plt_ = [None] * len(rating_metrics) * n_cells
for i in range(n_cells):
plt_[i] = plt.subplot(n_rows, n_cols, i + 1)
#plt_[i].grid(which='major', axis='y')
legend = []
for n in run_names:
legend += [n]
legend += ['']
legend += ['']
for valid_epochs, idx_malig_pearson, idx_malig_kendall, idx_rating_pearson, idx_rating_kendall \
in zip(Valid_epochs, Idx_malig_pearson, Idx_malig_kendall, Idx_rating_pearson, Idx_rating_kendall):
# Malignancy Pearson Correlation
q = plt_[2 * m + 0].plot(valid_epochs, smooth(idx_malig_pearson[0]))
plt_[2 * m + 0].plot(valid_epochs,
smooth(idx_malig_pearson[0] +
idx_malig_pearson[1]),
color=q[0].get_color(),
ls='--',
alpha=alpha)
plt_[2 * m + 0].plot(valid_epochs,
smooth(idx_malig_pearson[0] -
idx_malig_pearson[1]),
color=q[0].get_color(),
ls='--',
alpha=alpha)
plt_[2 * m + 0].grid(which='major', axis='y')
# Malignancy Kendall Correlation
plt_[idx(1, 0)].plot(epochs, prec + prec_std, color=q[0].get_color(), ls='--', alpha=alpha)
plt_[idx(1, 0)].plot(epochs, prec - prec_std, color=q[0].get_color(), ls='--', alpha=alpha)
Axes.set_ylim(plt_[idx(0, 0)].axes, .75, .85)
# Precision Index
q = plt_[idx(2, 0)].plot(epochs, index, '-*')
plt_[idx(2, 0)].plot(epochs, index + index_std, color=q[0].get_color(), ls='--', alpha=alpha)
plt_[idx(2, 0)].plot(epochs, index - index_std, color=q[0].get_color(), ls='--', alpha=alpha)
Axes.set_ylim(plt_[idx(0, 0)].axes, .75, .85)
'''
# Smoothed
row = 0
# Accuracy
q = plt_[idx(0, row)].plot(epochs, smooth(acc), '-')
plt_[idx(0, row)].plot(epochs,
smooth(acc + acc_std),
color=q[0].get_color(),
ls='--',
alpha=alpha)
plt_[idx(0, row)].plot(epochs,
smooth(acc - acc_std),
color=q[0].get_color(),
ls='--',
alpha=alpha)
Axes.set_ylim(plt_[idx(0, row)].axes, .6, .9)
# Precision
q = plt_[idx(1, row)].plot(epochs, smooth(prec), '-')
plt_[idx(1, row)].plot(epochs,
Idx_concentration += [idx_concentration]
Idx_contrast += [idx_contrast]
Idx_kummar += [idx_kummar]
Idx_featCorr += [idx_featCorr]
Idx_sampCorr += [idx_sampCorr]
# plot
for epochs, idx_hubness, idx_symmetry, idx_concentration, idx_contrast, idx_kummar, idx_featCorr, idx_sampCorr \
in zip(Epochs, Idx_hubness, Idx_symmetry, Idx_concentration, Idx_contrast, Idx_kummar, Idx_featCorr, Idx_sampCorr):
# hubness
next_plot = 0
if 'Hubness' in indexes:
q = p[M * m + next_plot].plot(epochs,
smooth(idx_hubness[0]),
marker='x') #
p[M * m + next_plot].plot(epochs,
smooth(idx_hubness[0] + idx_hubness[1]),
color=q[0].get_color(),
ls='--',
alpha=alpha)
p[M * m + next_plot].plot(epochs,
smooth(idx_hubness[0] - idx_hubness[1]),
color=q[0].get_color(),
ls='--',
alpha=alpha)
next_plot += 1
# symmetry
if 'Symmetry' in indexes:
def dir_rating_correlate(run,
post,
epochs,
rating_norm='none',
clustered_rating_distance=True,
n_groups=5):
pear_corr = [[] for i in range(n_groups)]
kend_corr = [[] for i in range(n_groups)]
plot_data_filename = './Plots/Data/rating_correlation_{}{}.p'.format(
'dirR', run)
try:
print('SKIPING')
assert False
pear_corr, kend_corr = pickle.load(open(plot_data_filename, 'br'))
print("Loaded results for {}".format(run))
except:
print("Evaluating Rating Correlation for {}".format(run))
for c, run_config in enumerate(
[run + 'c{}'.format(config) for config in range(n_groups)]):
PredFile = FileManager.Pred(type='rating', pre='dirR')
Reg = RatingCorrelator(PredFile(run=run_config, dset=post),
multi_epoch=True)
for e in epochs:
Reg.evaluate_embed_distance_matrix(
method='euclidean',
epoch=e,
round=(rating_norm == 'Round'))
Reg.evaluate_rating_space(norm=rating_norm)
Reg.evaluate_rating_distance_matrix(
method='euclidean',
clustered_rating_distance=clustered_rating_distance)
Reg.linear_regression()
# Reg.scatter('embed', 'rating', xMethod="euclidean", yMethod='euclidean', sub=False)
p, s, k = Reg.correlate_retrieval(
'embed',
'rating',
round=(rating_norm == 'Round'),
verbose=False)
pear_corr[c].append(p)
kend_corr[c].append(k)
pear_corr[c] = np.array(pear_corr[c])
kend_corr[c] = np.array(kend_corr[c])
pear_corr = np.mean(pear_corr, axis=0)
kend_corr = np.mean(kend_corr, axis=0)
print('NO DUMP')
#pickle.dump((pear_corr, kend_corr), open(plot_data_filename, 'bw'))
pear_corr = smooth(pear_corr[:, 0]), smooth(pear_corr[:, 1])
kend_corr = smooth(kend_corr[:, 0]), smooth(kend_corr[:, 1])
epochs = np.array(epochs)
plt.figure('Rating2Rating:' + run + '-' + post)
q = plt.plot(epochs, pear_corr[0])
plt.plot(epochs,
pear_corr[0] + pear_corr[1],
color=q[0].get_color(),
ls='--',
alpha=alpha)
plt.plot(epochs,
pear_corr[0] - pear_corr[1],
color=q[0].get_color(),
ls='--',
alpha=alpha)
q = plt.plot(epochs, kend_corr[0])
plt.plot(epochs,
kend_corr[0] + kend_corr[1],
color=q[0].get_color(),
ls='--',
alpha=alpha)
plt.plot(epochs,
kend_corr[0] - kend_corr[1],
color=q[0].get_color(),
ls='--',
alpha=alpha)
plt.grid(which='major', axis='y')
plt.title('rating_' + run + '_' + post)
plt.xlabel('epochs')
plt.ylabel('correlation')
plt.legend(['pearson', '', '', 'kendall', '', ''])
def dir_size_rmse(run,
post,
epochs,
net_type,
dist='RMSE',
weighted=False,
n_groups=5):
plot_data_filename = './Plots/Data/size{}_{}{}.p'.format(
dist, net_type, run)
try:
assert False
R = pickle.load(open(plot_data_filename, 'br'))
print("Loaded results for {}".format(run))
except:
print("Evaluating Size RMSE for {}".format(run))
PredFile = FileManager.Pred(type='size', pre=net_type)
R = np.zeros([len(epochs), n_groups])
for c, run_config in enumerate(
[run + 'c{}'.format(config) for config in range(n_groups)]):
predict, valid_epochs, images, meta_data, classes, labels, masks = PredFile.load(
run=run_config, dset=post)
labels = np.array(labels)
for i, e in enumerate(epochs):
print(" Epoch {}:".format(e))
try:
idx = int(np.argwhere(valid_epochs == e))
except:
print('skip epoch {}'.format(e))
continue
pred = predict[idx]
'''
W = np.ones(labels.shape[0])
if weighted:
assert False
w = np.histogram(labels[:, r], bins=np.array(range(64))+0.5)[0]
w = 1 - w / np.sum(w)
pred_w = np.minimum(np.maximum(pred[:, r], 1.0), max_val)
W = w[np.round(pred_w - 1).astype('int')]
if dist=='RMSE':
err = W.dot((pred - labels)**2)
err = np.sqrt(err/np.sum(W))
elif dist=='ABS':
err = W.dot(np.abs(pred - labels)) / np.sum(W)
else:
print('{} unrecognized distance'.format(dist))
assert False
'''
rmse = np.sqrt(np.mean(np.sum((pred - labels)**2, axis=1)))
R[i, c] = rmse
R = np.mean(R, axis=1)
pickle.dump(R, open(plot_data_filename, 'bw'))
# smooth
R = smooth(R)
plt.figure(dist + ' ' + net_type + run + '-' + post)
plt.title('Size ' + dist)
plt.plot(epochs, R)
#plt.legend(rating_property+['Overall'])
plt.grid(True, axis='y')
return R
def dir_rating_params_correlate(run,
post,
epochs,
net_type,
rating_norm='none',
configurations=list(range(5)),
USE_CACHE=True,
DUMP=True):
reference = [0.7567, 0.5945, 0.7394, 0.5777, 0.6155, 0.7445,
0.6481] # 0, 0,
rating_property = [
'Subtlety', 'Sphericity', 'Margin', 'Lobulation', 'Spiculation',
'Texture', 'Malignancy'
] # 'Internalstructure', 'Calcification',
mask = [True, False, False, True, True, True, True, True, True]
pear_corr = [[] for i in configurations]
plot_data_filename = './Plots/Data/rating_params_correlation_{}{}.p'.format(
net_type, run)
try:
if USE_CACHE is False:
print('SKIPPING')
assert False
pear_corr = pickle.load(open(plot_data_filename, 'br'))
print("Loaded results for {}".format(run))
except:
print("Evaluating Rating Correlation for {}".format(run))
for c, run_config in enumerate(
[run + 'c{}'.format(config) for config in configurations]):
PredFile = FileManager.Pred(type='rating', pre=net_type)
Reg = RatingCorrelator(PredFile(run=run_config, dset=post),
multi_epoch=True,
conf=c)
Reg.evaluate_rating_space(norm=rating_norm)
#valid_epochs = []
for e in epochs:
p = Reg.correlate_to_ratings(epoch=e,
round=(rating_norm == 'Round'))
if not np.all(np.isfinite(p[mask])):
print('nan at: conf={}, epoch={}'.format(c, e))
pear_corr[c].append(p[mask])
#valid_epochs.append(e)
pear_corr[c] = np.array(pear_corr[c])
pear_corr = np.mean(pear_corr, axis=0)
if DUMP:
pickle.dump(pear_corr, open(plot_data_filename, 'bw'))
else:
print('NO DUMP')
for i, e in enumerate(epochs):
print("=" * 20)
print(" Epoch {}:".format(e))
print("-" * 20)
for p, property in enumerate(rating_property):
print("\t{}: \t{:.2f}".format(property, pear_corr[i, p]))
#print("\t" + ("-" * 10))
#print("\toverall: \t{:.2f}".format(R[i, 9]))
for p in range(pear_corr.shape[1]):
pear_corr[:, p] = smooth(pear_corr[:, p], window_length=5, polyorder=2)
epochs = np.array(epochs)
plt.figure('RatingParams2Rating:' + run + '-' + post)
q = plt.plot(epochs, pear_corr, linewidth=2.5)
for line, ref in zip(q, reference):
plt.plot(epochs,
ref * np.ones_like(epochs),
color=line.get_color(),
ls='--',
linewidth=4,
alpha=0.6)
plt.grid(which='major', axis='y')
plt.title('rating_' + run + '_' + post)
plt.xlabel('epochs')
plt.ylabel('correlation')
plt.legend(rating_property)
def dir_rating_rmse(run,
post,
epochs,
net_type,
dist='RMSE',
weighted=False,
configurations=list(range(5)),
USE_CACHE=True,
DUMP=True):
#images, predict, meta_data, labels, masks = pred_loader.load(run, epochs[-1], post)
rating_property = [
'Subtlety', 'Internalstructure', 'Calcification', 'Sphericity',
'Margin', 'Lobulation', 'Spiculation', 'Texture', 'Malignancy'
]
plot_data_filename = './Plots/Data/{}_{}{}.p'.format(dist, net_type, run)
try:
if USE_CACHE is False:
print("skipping...")
assert False
R = pickle.load(open(plot_data_filename, 'br'))
print("Loaded results for {}".format(run))
except:
print("Evaluating RMSE for {}".format(run))
PredFile = FileManager.Pred(type='rating', pre=net_type)
R = np.zeros([len(epochs), 10, len(configurations)])
for c, run_config in enumerate(
[run + 'c{}'.format(config) for config in configurations]):
predict, valid_epochs, images, meta_data, classes, labels, masks, conf, rating_weights, z = PredFile.load(
run=run_config, dset=post)
labels = np.array([np.mean(l, axis=0) for l in labels])
for i, e in enumerate(epochs):
#print("=" * 20)
#print(" Epoch {}:".format(e))
#print("-" * 20)
try:
idx = int(np.argwhere(valid_epochs == e))
except:
print('skip epoch {}'.format(e))
continue
pred = predict[idx]
for r, max_val in zip(range(9), [5, 5, 6, 5, 5, 5, 5, 5, 5]):
#print("{}:".format(rating_property[r]))
W = np.ones(labels.shape[0])
if weighted:
w = np.histogram(labels[:, r],
bins=np.array(range(max_val + 1)) +
0.5)[0]
#print("\tcounts - {}".format(w))
w = 1 - w / np.sum(w)
w /= (len(w) - 1)
assert np.abs(w.sum() - 1) < 1e-6
#print("\tweighted by {}".format(w))
#pred_w = np.minimum(np.maximum(pred[:, r], 1.0), max_val)
W = w[np.round(labels[:, r] - 1).astype('int')]
if dist == 'RMSE':
err = W.dot((pred[:, r] - labels[:, r])**2)
err = np.sqrt(err / np.sum(W))
elif dist == 'ABS':
err = W.dot(
np.abs(pred[:, r] - labels[:, r])) / np.sum(W)
else:
print('{} unrecognized distance'.format(dist))
assert False
#print("rmse: \t{:.2f}".format(err))
R[i, r, c] = err
rmse = np.sqrt(np.mean(np.sum((pred - labels)**2, axis=1)))
#print("=" * 20)
#print("overall: \t{:.2f}".format(rmse))
R[i, 9, c] = rmse
R = np.mean(R, axis=2)
for i, e in enumerate(epochs):
print("=" * 20)
print(" Epoch {}:".format(e))
print("-" * 20)
for p, property in enumerate(rating_property):
print("\t{}: \t{:.2f}".format(property, R[i, p]))
print("\t" + ("-" * 10))
print("\toverall: \t{:.2f}".format(R[i, 9]))
if DUMP:
pickle.dump(R, open(plot_data_filename, 'bw'))
else:
print("No Dump")
# smooth
for r in range(9):
R[:, r] = smooth(R[:, r])
plt.figure(dist + ' ' + run + '-' + post)
plt.title('Rating ' + dist)
plt.plot(epochs, R)
plt.legend(rating_property + ['Overall'])
plt.grid(True, axis='y')
return R
Idx_symmetry += [idx_symmetry]
Idx_concentration += [idx_concentration]
Idx_contrast += [idx_contrast]
Idx_kummar += [idx_kummar]
Idx_featCorr += [idx_featCorr]
Idx_sampCorr += [idx_sampCorr]
# plot
for epochs, idx_hubness, idx_symmetry, idx_concentration, idx_contrast, idx_kummar, idx_featCorr, idx_sampCorr \
in zip(Epochs, Idx_hubness, Idx_symmetry, Idx_concentration, Idx_contrast, Idx_kummar, Idx_featCorr, Idx_sampCorr):
# hubness
next_plot = 0
if 'Hubness' in indexes:
q = p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0]))
p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0] + idx_hubness[1]), color=q[0].get_color(), ls='--', alpha=alpha)
p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0] - idx_hubness[1]), color=q[0].get_color(), ls='--', alpha=alpha)
next_plot += 1
# symmetry
if 'Symmetry' in indexes:
q = p[M * m + next_plot].plot(epochs, smooth(idx_symmetry[0]))
p[M * m + next_plot].plot(epochs, smooth(idx_symmetry[0] + idx_symmetry[1]), color=q[0].get_color(), ls='--', alpha=alpha)
p[M * m + next_plot].plot(epochs, smooth(idx_symmetry[0] - idx_symmetry[1]), color=q[0].get_color(), ls='--', alpha=alpha)
next_plot += 1
# contrast
if 'Contrast' in indexes:
q = p[M * m + next_plot].plot(epochs, smooth(idx_contrast[0]))
p[M * m + next_plot].plot(epochs, smooth(idx_contrast[0] + idx_contrast[1]), color=q[0].get_color(), ls='--', alpha=alpha)