def calc_hubness(indices, K=[3, 5, 7, 11, 17], verbose=False, label=''): if verbose: plt.figure() h = np.zeros(len(K)) for i in range(len(K)): h_ = hubness(indices, K[i]) h[i] = h_[0] if verbose: #plt.subplot(1,len(K), i+1) plt.plot(smooth(h_[1], window_length=1)) h_mean, h_std = np.mean(h), np.std(h) if verbose: plt.legend(['K={}: {:.2f}'.format(k, ind) for k, ind in zip(K, h)]) plt.title('{} (Hubness = {:.2f}))'.format(label, h_mean)) plt.xlabel('Nk') plt.ylabel('P(Nk)') return h_mean, h_std
plt.figure(experiment_name + ' Correlation: ' + dset) plt_ = [None] * len(rating_metrics) * n_cells for i in range(n_cells): plt_[i] = plt.subplot(n_rows, n_cols, i + 1) #plt_[i].grid(which='major', axis='y') legend = [] for n in run_names: legend += [n] legend += [''] legend += [''] for valid_epochs, idx_malig_pearson, idx_malig_kendall, idx_rating_pearson, idx_rating_kendall \ in zip(Valid_epochs, Idx_malig_pearson, Idx_malig_kendall, Idx_rating_pearson, Idx_rating_kendall): # Malignancy Pearson Correlation q = plt_[2 * m + 0].plot(valid_epochs, smooth(idx_malig_pearson[0])) plt_[2 * m + 0].plot(valid_epochs, smooth(idx_malig_pearson[0] + idx_malig_pearson[1]), color=q[0].get_color(), ls='--', alpha=alpha) plt_[2 * m + 0].plot(valid_epochs, smooth(idx_malig_pearson[0] - idx_malig_pearson[1]), color=q[0].get_color(), ls='--', alpha=alpha) plt_[2 * m + 0].grid(which='major', axis='y') # Malignancy Kendall Correlation
plt_[idx(1, 0)].plot(epochs, prec + prec_std, color=q[0].get_color(), ls='--', alpha=alpha) plt_[idx(1, 0)].plot(epochs, prec - prec_std, color=q[0].get_color(), ls='--', alpha=alpha) Axes.set_ylim(plt_[idx(0, 0)].axes, .75, .85) # Precision Index q = plt_[idx(2, 0)].plot(epochs, index, '-*') plt_[idx(2, 0)].plot(epochs, index + index_std, color=q[0].get_color(), ls='--', alpha=alpha) plt_[idx(2, 0)].plot(epochs, index - index_std, color=q[0].get_color(), ls='--', alpha=alpha) Axes.set_ylim(plt_[idx(0, 0)].axes, .75, .85) ''' # Smoothed row = 0 # Accuracy q = plt_[idx(0, row)].plot(epochs, smooth(acc), '-') plt_[idx(0, row)].plot(epochs, smooth(acc + acc_std), color=q[0].get_color(), ls='--', alpha=alpha) plt_[idx(0, row)].plot(epochs, smooth(acc - acc_std), color=q[0].get_color(), ls='--', alpha=alpha) Axes.set_ylim(plt_[idx(0, row)].axes, .6, .9) # Precision q = plt_[idx(1, row)].plot(epochs, smooth(prec), '-') plt_[idx(1, row)].plot(epochs,
Idx_concentration += [idx_concentration] Idx_contrast += [idx_contrast] Idx_kummar += [idx_kummar] Idx_featCorr += [idx_featCorr] Idx_sampCorr += [idx_sampCorr] # plot for epochs, idx_hubness, idx_symmetry, idx_concentration, idx_contrast, idx_kummar, idx_featCorr, idx_sampCorr \ in zip(Epochs, Idx_hubness, Idx_symmetry, Idx_concentration, Idx_contrast, Idx_kummar, Idx_featCorr, Idx_sampCorr): # hubness next_plot = 0 if 'Hubness' in indexes: q = p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0]), marker='x') # p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0] + idx_hubness[1]), color=q[0].get_color(), ls='--', alpha=alpha) p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0] - idx_hubness[1]), color=q[0].get_color(), ls='--', alpha=alpha) next_plot += 1 # symmetry if 'Symmetry' in indexes:
def dir_rating_correlate(run, post, epochs, rating_norm='none', clustered_rating_distance=True, n_groups=5): pear_corr = [[] for i in range(n_groups)] kend_corr = [[] for i in range(n_groups)] plot_data_filename = './Plots/Data/rating_correlation_{}{}.p'.format( 'dirR', run) try: print('SKIPING') assert False pear_corr, kend_corr = pickle.load(open(plot_data_filename, 'br')) print("Loaded results for {}".format(run)) except: print("Evaluating Rating Correlation for {}".format(run)) for c, run_config in enumerate( [run + 'c{}'.format(config) for config in range(n_groups)]): PredFile = FileManager.Pred(type='rating', pre='dirR') Reg = RatingCorrelator(PredFile(run=run_config, dset=post), multi_epoch=True) for e in epochs: Reg.evaluate_embed_distance_matrix( method='euclidean', epoch=e, round=(rating_norm == 'Round')) Reg.evaluate_rating_space(norm=rating_norm) Reg.evaluate_rating_distance_matrix( method='euclidean', clustered_rating_distance=clustered_rating_distance) Reg.linear_regression() # Reg.scatter('embed', 'rating', xMethod="euclidean", yMethod='euclidean', sub=False) p, s, k = Reg.correlate_retrieval( 'embed', 'rating', round=(rating_norm == 'Round'), verbose=False) pear_corr[c].append(p) kend_corr[c].append(k) pear_corr[c] = np.array(pear_corr[c]) kend_corr[c] = np.array(kend_corr[c]) pear_corr = np.mean(pear_corr, axis=0) kend_corr = np.mean(kend_corr, axis=0) print('NO DUMP') #pickle.dump((pear_corr, kend_corr), open(plot_data_filename, 'bw')) pear_corr = smooth(pear_corr[:, 0]), smooth(pear_corr[:, 1]) kend_corr = smooth(kend_corr[:, 0]), smooth(kend_corr[:, 1]) epochs = np.array(epochs) plt.figure('Rating2Rating:' + run + '-' + post) q = plt.plot(epochs, pear_corr[0]) plt.plot(epochs, pear_corr[0] + pear_corr[1], color=q[0].get_color(), ls='--', alpha=alpha) plt.plot(epochs, pear_corr[0] - pear_corr[1], color=q[0].get_color(), ls='--', alpha=alpha) q = plt.plot(epochs, kend_corr[0]) plt.plot(epochs, kend_corr[0] + kend_corr[1], color=q[0].get_color(), ls='--', alpha=alpha) plt.plot(epochs, kend_corr[0] - kend_corr[1], color=q[0].get_color(), ls='--', alpha=alpha) plt.grid(which='major', axis='y') plt.title('rating_' + run + '_' + post) plt.xlabel('epochs') plt.ylabel('correlation') plt.legend(['pearson', '', '', 'kendall', '', ''])
def dir_size_rmse(run, post, epochs, net_type, dist='RMSE', weighted=False, n_groups=5): plot_data_filename = './Plots/Data/size{}_{}{}.p'.format( dist, net_type, run) try: assert False R = pickle.load(open(plot_data_filename, 'br')) print("Loaded results for {}".format(run)) except: print("Evaluating Size RMSE for {}".format(run)) PredFile = FileManager.Pred(type='size', pre=net_type) R = np.zeros([len(epochs), n_groups]) for c, run_config in enumerate( [run + 'c{}'.format(config) for config in range(n_groups)]): predict, valid_epochs, images, meta_data, classes, labels, masks = PredFile.load( run=run_config, dset=post) labels = np.array(labels) for i, e in enumerate(epochs): print(" Epoch {}:".format(e)) try: idx = int(np.argwhere(valid_epochs == e)) except: print('skip epoch {}'.format(e)) continue pred = predict[idx] ''' W = np.ones(labels.shape[0]) if weighted: assert False w = np.histogram(labels[:, r], bins=np.array(range(64))+0.5)[0] w = 1 - w / np.sum(w) pred_w = np.minimum(np.maximum(pred[:, r], 1.0), max_val) W = w[np.round(pred_w - 1).astype('int')] if dist=='RMSE': err = W.dot((pred - labels)**2) err = np.sqrt(err/np.sum(W)) elif dist=='ABS': err = W.dot(np.abs(pred - labels)) / np.sum(W) else: print('{} unrecognized distance'.format(dist)) assert False ''' rmse = np.sqrt(np.mean(np.sum((pred - labels)**2, axis=1))) R[i, c] = rmse R = np.mean(R, axis=1) pickle.dump(R, open(plot_data_filename, 'bw')) # smooth R = smooth(R) plt.figure(dist + ' ' + net_type + run + '-' + post) plt.title('Size ' + dist) plt.plot(epochs, R) #plt.legend(rating_property+['Overall']) plt.grid(True, axis='y') return R
def dir_rating_params_correlate(run, post, epochs, net_type, rating_norm='none', configurations=list(range(5)), USE_CACHE=True, DUMP=True): reference = [0.7567, 0.5945, 0.7394, 0.5777, 0.6155, 0.7445, 0.6481] # 0, 0, rating_property = [ 'Subtlety', 'Sphericity', 'Margin', 'Lobulation', 'Spiculation', 'Texture', 'Malignancy' ] # 'Internalstructure', 'Calcification', mask = [True, False, False, True, True, True, True, True, True] pear_corr = [[] for i in configurations] plot_data_filename = './Plots/Data/rating_params_correlation_{}{}.p'.format( net_type, run) try: if USE_CACHE is False: print('SKIPPING') assert False pear_corr = pickle.load(open(plot_data_filename, 'br')) print("Loaded results for {}".format(run)) except: print("Evaluating Rating Correlation for {}".format(run)) for c, run_config in enumerate( [run + 'c{}'.format(config) for config in configurations]): PredFile = FileManager.Pred(type='rating', pre=net_type) Reg = RatingCorrelator(PredFile(run=run_config, dset=post), multi_epoch=True, conf=c) Reg.evaluate_rating_space(norm=rating_norm) #valid_epochs = [] for e in epochs: p = Reg.correlate_to_ratings(epoch=e, round=(rating_norm == 'Round')) if not np.all(np.isfinite(p[mask])): print('nan at: conf={}, epoch={}'.format(c, e)) pear_corr[c].append(p[mask]) #valid_epochs.append(e) pear_corr[c] = np.array(pear_corr[c]) pear_corr = np.mean(pear_corr, axis=0) if DUMP: pickle.dump(pear_corr, open(plot_data_filename, 'bw')) else: print('NO DUMP') for i, e in enumerate(epochs): print("=" * 20) print(" Epoch {}:".format(e)) print("-" * 20) for p, property in enumerate(rating_property): print("\t{}: \t{:.2f}".format(property, pear_corr[i, p])) #print("\t" + ("-" * 10)) #print("\toverall: \t{:.2f}".format(R[i, 9])) for p in range(pear_corr.shape[1]): pear_corr[:, p] = smooth(pear_corr[:, p], window_length=5, polyorder=2) epochs = np.array(epochs) plt.figure('RatingParams2Rating:' + run + '-' + post) q = plt.plot(epochs, pear_corr, linewidth=2.5) for line, ref in zip(q, reference): plt.plot(epochs, ref * np.ones_like(epochs), color=line.get_color(), ls='--', linewidth=4, alpha=0.6) plt.grid(which='major', axis='y') plt.title('rating_' + run + '_' + post) plt.xlabel('epochs') plt.ylabel('correlation') plt.legend(rating_property)
def dir_rating_rmse(run, post, epochs, net_type, dist='RMSE', weighted=False, configurations=list(range(5)), USE_CACHE=True, DUMP=True): #images, predict, meta_data, labels, masks = pred_loader.load(run, epochs[-1], post) rating_property = [ 'Subtlety', 'Internalstructure', 'Calcification', 'Sphericity', 'Margin', 'Lobulation', 'Spiculation', 'Texture', 'Malignancy' ] plot_data_filename = './Plots/Data/{}_{}{}.p'.format(dist, net_type, run) try: if USE_CACHE is False: print("skipping...") assert False R = pickle.load(open(plot_data_filename, 'br')) print("Loaded results for {}".format(run)) except: print("Evaluating RMSE for {}".format(run)) PredFile = FileManager.Pred(type='rating', pre=net_type) R = np.zeros([len(epochs), 10, len(configurations)]) for c, run_config in enumerate( [run + 'c{}'.format(config) for config in configurations]): predict, valid_epochs, images, meta_data, classes, labels, masks, conf, rating_weights, z = PredFile.load( run=run_config, dset=post) labels = np.array([np.mean(l, axis=0) for l in labels]) for i, e in enumerate(epochs): #print("=" * 20) #print(" Epoch {}:".format(e)) #print("-" * 20) try: idx = int(np.argwhere(valid_epochs == e)) except: print('skip epoch {}'.format(e)) continue pred = predict[idx] for r, max_val in zip(range(9), [5, 5, 6, 5, 5, 5, 5, 5, 5]): #print("{}:".format(rating_property[r])) W = np.ones(labels.shape[0]) if weighted: w = np.histogram(labels[:, r], bins=np.array(range(max_val + 1)) + 0.5)[0] #print("\tcounts - {}".format(w)) w = 1 - w / np.sum(w) w /= (len(w) - 1) assert np.abs(w.sum() - 1) < 1e-6 #print("\tweighted by {}".format(w)) #pred_w = np.minimum(np.maximum(pred[:, r], 1.0), max_val) W = w[np.round(labels[:, r] - 1).astype('int')] if dist == 'RMSE': err = W.dot((pred[:, r] - labels[:, r])**2) err = np.sqrt(err / np.sum(W)) elif dist == 'ABS': err = W.dot( np.abs(pred[:, r] - labels[:, r])) / np.sum(W) else: print('{} unrecognized distance'.format(dist)) assert False #print("rmse: \t{:.2f}".format(err)) R[i, r, c] = err rmse = np.sqrt(np.mean(np.sum((pred - labels)**2, axis=1))) #print("=" * 20) #print("overall: \t{:.2f}".format(rmse)) R[i, 9, c] = rmse R = np.mean(R, axis=2) for i, e in enumerate(epochs): print("=" * 20) print(" Epoch {}:".format(e)) print("-" * 20) for p, property in enumerate(rating_property): print("\t{}: \t{:.2f}".format(property, R[i, p])) print("\t" + ("-" * 10)) print("\toverall: \t{:.2f}".format(R[i, 9])) if DUMP: pickle.dump(R, open(plot_data_filename, 'bw')) else: print("No Dump") # smooth for r in range(9): R[:, r] = smooth(R[:, r]) plt.figure(dist + ' ' + run + '-' + post) plt.title('Rating ' + dist) plt.plot(epochs, R) plt.legend(rating_property + ['Overall']) plt.grid(True, axis='y') return R
Idx_symmetry += [idx_symmetry] Idx_concentration += [idx_concentration] Idx_contrast += [idx_contrast] Idx_kummar += [idx_kummar] Idx_featCorr += [idx_featCorr] Idx_sampCorr += [idx_sampCorr] # plot for epochs, idx_hubness, idx_symmetry, idx_concentration, idx_contrast, idx_kummar, idx_featCorr, idx_sampCorr \ in zip(Epochs, Idx_hubness, Idx_symmetry, Idx_concentration, Idx_contrast, Idx_kummar, Idx_featCorr, Idx_sampCorr): # hubness next_plot = 0 if 'Hubness' in indexes: q = p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0])) p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0] + idx_hubness[1]), color=q[0].get_color(), ls='--', alpha=alpha) p[M * m + next_plot].plot(epochs, smooth(idx_hubness[0] - idx_hubness[1]), color=q[0].get_color(), ls='--', alpha=alpha) next_plot += 1 # symmetry if 'Symmetry' in indexes: q = p[M * m + next_plot].plot(epochs, smooth(idx_symmetry[0])) p[M * m + next_plot].plot(epochs, smooth(idx_symmetry[0] + idx_symmetry[1]), color=q[0].get_color(), ls='--', alpha=alpha) p[M * m + next_plot].plot(epochs, smooth(idx_symmetry[0] - idx_symmetry[1]), color=q[0].get_color(), ls='--', alpha=alpha) next_plot += 1 # contrast if 'Contrast' in indexes: q = p[M * m + next_plot].plot(epochs, smooth(idx_contrast[0])) p[M * m + next_plot].plot(epochs, smooth(idx_contrast[0] + idx_contrast[1]), color=q[0].get_color(), ls='--', alpha=alpha)