def plot_scatter(ax, stats, content_ids=None): assert len(stats['ys_label']) == len(stats['ys_label_pred']) if content_ids is None: ax.scatter(stats['ys_label'], stats['ys_label_pred']) else: assert len(stats['ys_label']) == len(content_ids) unique_content_ids = list(set(content_ids)) import matplotlib.pyplot as plt cmap = plt.get_cmap() colors = [ cmap(i) for i in np.linspace(0, 1, len(unique_content_ids)) ] for idx, curr_content_id in enumerate(unique_content_ids): curr_idxs = indices(content_ids, lambda cid: cid == curr_content_id) curr_ys_label = np.array(stats['ys_label'])[curr_idxs] curr_ys_label_pred = np.array( stats['ys_label_pred'])[curr_idxs] ax.scatter(curr_ys_label, curr_ys_label_pred, label=curr_content_id, color=colors[idx % len(colors)])
def construct_kfold_list(assets, contentid_groups): # construct cross validation kfold input list content_ids = map(lambda asset: asset.content_id, assets) kfold = [] for curr_content_group in contentid_groups: curr_indices = indices(content_ids, lambda x: x in curr_content_group) kfold.append(curr_indices) return kfold
def _get_ref_mos(dataset_reader, mos): ref_mos = [] for dis_video in dataset_reader.dataset.dis_videos: # get the dis video's ref video's mos curr_content_id = dis_video['content_id'] ref_indices = indices( zip(dataset_reader.content_id_of_dis_videos, dataset_reader.disvideo_is_refvideo), lambda (content_id, is_refvideo): is_refvideo and content_id == curr_content_id ) assert len(ref_indices) == 1, \ 'Should have only and one ref video for a dis video, ' \ 'but got {}'.format(len(ref_indices)) ref_idx = ref_indices[0] ref_mos.append(mos[ref_idx]) return np.array(ref_mos)
def plot_scatter(ax, stats, content_ids=None): assert len(stats['ys_label']) == len(stats['ys_label_pred']) if content_ids is None: ax.scatter(stats['ys_label'], stats['ys_label_pred']) else: assert len(stats['ys_label']) == len(content_ids) unique_content_ids = list(set(content_ids)) import matplotlib.pyplot as plt cmap = plt.get_cmap() colors = [cmap(i) for i in np.linspace(0, 1, len(unique_content_ids))] for idx, curr_content_id in enumerate(unique_content_ids): curr_idxs = indices(content_ids, lambda cid: cid==curr_content_id) curr_ys_label = np.array(stats['ys_label'])[curr_idxs] curr_ys_label_pred = np.array(stats['ys_label_pred'])[curr_idxs] ax.scatter(curr_ys_label, curr_ys_label_pred, label=curr_content_id, color=colors[idx % len(colors)])
def test_disvideo_is_refvideo(self): l = self.dataset_reader.disvideo_is_refvideo self.assertItemsEqual(indices(l, lambda e: e is True), range(9))
def _metrics_performance(objScoDif, signif): """ mirroring matlab function: %[results] = Metrics_performance(objScoDif, signif, doPlot) % INPUT: objScoDif : differences of objective scores [M,N] % M : number of metrics % N : number of pairs % signif : statistical outcome of paired comparison [1,N] % 0 : no difference % -1 : first stimulus is worse % 1 : first stimulus is better % doPlot : boolean indicating if graphs should be plotted % % OUTPUT: results - structure with following fields % % AUC_DS : Area Under the Curve for Different/Similar ROC % analysis % pDS_DL : p-values for AUC_DS from DeLong test % pDS_HM : p-values for AUC_DS from Hanley and McNeil test % AUC_BW : Area Under the Curve for Better/Worse ROC % analysis % pBW_DL : p-values for AUC_BW from DeLong test % pBW_HM : p-values for AUC_BW from Hanley and McNeil test % CC_0 : Correct Classification @ DeltaOM = 0 for % Better/Worse ROC analysis % pCC0_b : p-values for CC_0 from binomial test % pCC0_F : p-values for CC_0 from Fisher's exact test % THR : threshold for 95% probability that the stimuli % are different """ # M = size(objScoDif,1); # D = abs(objScoDif(:,signif ~= 0)); # S = abs(objScoDif(:,signif == 0)); # samples.spsizes = [size(D,2),size(S,2)]; # samples.ratings = [D,S]; M = objScoDif.shape[0] D = np.abs(objScoDif[:, indices(signif[0], lambda x: x != 0)]) S = np.abs(objScoDif[:, indices(signif[0], lambda x: x == 0)]) samples = empty_object() samples.spsizes = [D.shape[1], S.shape[1]] samples.ratings = np.hstack([D, S]) # % calculate AUCs # [AUC_DS,C] = fastDeLong(samples); AUC_DS, C, _, _ = fastDeLong(samples) # % significance calculation # pDS_DL = ones(M); # for i=1:M-1 # for j=i+1:M # pDS_DL(i,j) = calpvalue(AUC_DS([i,j]), C([i,j],[i,j])); # pDS_DL(j,i) = pDS_DL(i,j); # end # end pDS_DL = np.ones([M, M]) for i in range(1, M): for j in range(i + 1, M + 1): # http://stackoverflow.com/questions/4257394/slicing-of-a-numpy-2d-array-or-how-do-i-extract-an-mxm-submatrix-from-an-nxn-ar pDS_DL[i - 1, j - 1] = calpvalue( AUC_DS[[i - 1, j - 1]], C[[[i - 1], [j - 1]], [i - 1, j - 1]]) pDS_DL[j - 1, i - 1] = pDS_DL[i - 1, j - 1] # [pDS_HM,CI_DS] = significanceHM(S, D, AUC_DS); pDS_HM, CI_DS = significanceHM(S, D, AUC_DS) # THR = prctile(D',95); THR = np.percentile(D, 95, axis=1) # %%%%%%%%%%%%%%%%%%%%%%% Better / Worse %%%%%%%%%%%%%%%%%%%%%%%%%%% # B = [objScoDif(:,signif == 1),-objScoDif(:,signif == -1)]; # W = -B; # samples.ratings = [B,W]; # samples.spsizes = [size(B,2),size(W,2)]; B1 = objScoDif[:, indices(signif[0], lambda x: x == 1)] B2 = objScoDif[:, indices(signif[0], lambda x: x == -1)] B = np.hstack([B1, -B2]) W = -B samples = empty_object() samples.ratings = np.hstack([B, W]) samples.spsizes = [B.shape[1], W.shape[1]] # % calculate AUCs # [AUC_BW,C] = fastDeLong(samples); AUC_BW, C, _, _ = fastDeLong(samples) # % calculate correct classification for DeltaOM = 0 # L = size(B,2) + size(W,2); # CC_0 = zeros(M,1); # for m=1:M # CC_0(m) = (sum(B(m,:)>0) + sum(W(m,:)<0)) / L; # end L = B.shape[1] + W.shape[1] CC_0 = np.zeros(M) for m in range(M): CC_0[m] = float(np.sum(B[m, :] > 0) + np.sum(W[m, :] < 0)) / L # % significance calculation # pBW_DL = ones(M); # pCC0_b = ones(M); # pCC0_F = ones(M); # for i=1:M-1 # for j=i+1:M # pBW_DL(i,j) = calpvalue(AUC_BW([i,j]), C([i,j],[i,j])); # pBW_DL(j,i) = pBW_DL(i,j); # # pCC0_b(i,j) = significanceBinomial(CC_0(i), CC_0(j), L); # pCC0_b(j,i) = pCC0_b(i,j); # # pCC0_F(i,j) = fexact(CC_0(i)*L, 2*L, CC_0(i)*L + CC_0(j)*L, L, 'tail', 'b')/2; # pCC0_F(j,i) = pCC0_F(i,j); # end # end pBW_DL = np.ones([M, M]) pCC0_b = np.ones([M, M]) # pCC0_F = np.ones([M, M]) for i in range(1, M): for j in range(i + 1, M + 1): pBW_DL[i - 1, j - 1] = calpvalue( AUC_BW[[i - 1, j - 1]], C[[[i - 1], [j - 1]], [i - 1, j - 1]]) pBW_DL[j - 1, i - 1] = pBW_DL[i - 1, j - 1] pCC0_b[i - 1, j - 1] = significanceBinomial(CC_0[i - 1], CC_0[j - 1], L) pCC0_b[j - 1, i - 1] = pCC0_b[i - 1, j - 1] # pCC0_F[i-1, j-1] = fexact(CC_0[i-1]*L, 2*L, CC_0[i-1]*L + CC_0[j-1]*L, L, 'tail', 'b') / 2.0 # pCC0_F[j-1, i-1] = pCC0_F[i-1,j] # [pBW_HM,CI_BW] = significanceHM(B, W, AUC_BW); pBW_HM, CI_BW = significanceHM(B, W, AUC_BW) # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # % Adding outputs to the structure # results.AUC_DS = AUC_DS; # results.pDS_DL = pDS_DL; # results.pDS_HM = pDS_HM; # results.AUC_BW = AUC_BW; # results.pBW_DL = pBW_DL; # results.pBW_HM = pBW_HM; # results.CC_0 = CC_0; # results.pCC0_b = pCC0_b; # results.pCC0_F = pCC0_F; # results.THR = THR; result = { 'AUC_DS': AUC_DS, 'pDS_DL': pDS_DL, 'pDS_HM': pDS_HM, 'AUC_BW': AUC_BW, 'pBW_DL': pBW_DL, 'pBW_HM': pBW_HM, 'CC_0': CC_0, 'pCC0_b': pCC0_b, # 'pCC0_F': pCC0_F, 'THR': THR, } # %%%%%%%%%%%%%%%%%%%%%%%% Plot Results %%%%%%%%%%%%%%%%%%%%%%%%%%% # # if(doPlot == 1) # # % Using Benjamini-Hochberg procedure for multiple comparisons in plots # % (note: correlation between groups has to be positive) # # plot_auc(results.pDS_HM,results.AUC_DS, CI_DS, 'AUC (-)','Different/Similar') # plot_cc(results.pCC0_F,results.CC_0,'C_0 (%)','Better/Worse') # plot_auc(results.pBW_HM,results.AUC_BW, CI_BW, 'AUC (-)','Better/Worse') # # end return result
def _metrics_performance(objScoDif, signif): """ mirroring matlab function: %[results] = Metrics_performance(objScoDif, signif, doPlot) % INPUT: objScoDif : differences of objective scores [M,N] % M : number of metrics % N : number of pairs % signif : statistical outcome of paired comparison [1,N] % 0 : no difference % -1 : first stimulus is worse % 1 : first stimulus is better % doPlot : boolean indicating if graphs should be plotted % % OUTPUT: results - structure with following fields % % AUC_DS : Area Under the Curve for Different/Similar ROC % analysis % pDS_DL : p-values for AUC_DS from DeLong test % pDS_HM : p-values for AUC_DS from Hanley and McNeil test % AUC_BW : Area Under the Curve for Better/Worse ROC % analysis % pBW_DL : p-values for AUC_BW from DeLong test % pBW_HM : p-values for AUC_BW from Hanley and McNeil test % CC_0 : Correct Classification @ DeltaOM = 0 for % Better/Worse ROC analysis % pCC0_b : p-values for CC_0 from binomial test % pCC0_F : p-values for CC_0 from Fisher's exact test % THR : threshold for 95% probability that the stimuli % are different """ # M = size(objScoDif,1); # D = abs(objScoDif(:,signif ~= 0)); # S = abs(objScoDif(:,signif == 0)); # samples.spsizes = [size(D,2),size(S,2)]; # samples.ratings = [D,S]; M = objScoDif.shape[0] D = np.abs(objScoDif[:, indices(signif[0], lambda x: x!=0)]) S = np.abs(objScoDif[:, indices(signif[0], lambda x: x==0)]) samples = empty_object() samples.spsizes = [D.shape[1], S.shape[1]] samples.ratings = np.hstack([D, S]) # % calculate AUCs # [AUC_DS,C] = fastDeLong(samples); AUC_DS, C, _, _ = fastDeLong(samples) # % significance calculation # pDS_DL = ones(M); # for i=1:M-1 # for j=i+1:M # pDS_DL(i,j) = calpvalue(AUC_DS([i,j]), C([i,j],[i,j])); # pDS_DL(j,i) = pDS_DL(i,j); # end # end pDS_DL = np.ones([M, M]) for i in range(1, M): for j in range(i+1, M+1): # http://stackoverflow.com/questions/4257394/slicing-of-a-numpy-2d-array-or-how-do-i-extract-an-mxm-submatrix-from-an-nxn-ar pDS_DL[i-1, j-1] = calpvalue(AUC_DS[[i-1, j-1]], C[[[i-1],[j-1]],[i-1, j-1]]) pDS_DL[j-1, i-1] = pDS_DL[i-1, j-1] # [pDS_HM,CI_DS] = significanceHM(S, D, AUC_DS); pDS_HM, CI_DS = significanceHM(S, D, AUC_DS) # THR = prctile(D',95); THR = np.percentile(D, 95, axis=1) # %%%%%%%%%%%%%%%%%%%%%%% Better / Worse %%%%%%%%%%%%%%%%%%%%%%%%%%% # B = [objScoDif(:,signif == 1),-objScoDif(:,signif == -1)]; # W = -B; # samples.ratings = [B,W]; # samples.spsizes = [size(B,2),size(W,2)]; B1 = objScoDif[:, indices(signif[0], lambda x: x== 1)] B2 = objScoDif[:, indices(signif[0], lambda x: x==-1)] B = np.hstack([B1, -B2]) W = -B samples = empty_object() samples.ratings = np.hstack([B, W]) samples.spsizes = [B.shape[1], W.shape[1]] # % calculate AUCs # [AUC_BW,C] = fastDeLong(samples); AUC_BW, C, _, _ = fastDeLong(samples) # % calculate correct classification for DeltaOM = 0 # L = size(B,2) + size(W,2); # CC_0 = zeros(M,1); # for m=1:M # CC_0(m) = (sum(B(m,:)>0) + sum(W(m,:)<0)) / L; # end L = B.shape[1] + W.shape[1] CC_0 = np.zeros(M) for m in range(M): CC_0[m] = float(np.sum(B[m,:]>0) + np.sum(W[m,:]<0)) / L # % significance calculation # pBW_DL = ones(M); # pCC0_b = ones(M); # pCC0_F = ones(M); # for i=1:M-1 # for j=i+1:M # pBW_DL(i,j) = calpvalue(AUC_BW([i,j]), C([i,j],[i,j])); # pBW_DL(j,i) = pBW_DL(i,j); # # pCC0_b(i,j) = significanceBinomial(CC_0(i), CC_0(j), L); # pCC0_b(j,i) = pCC0_b(i,j); # # pCC0_F(i,j) = fexact(CC_0(i)*L, 2*L, CC_0(i)*L + CC_0(j)*L, L, 'tail', 'b')/2; # pCC0_F(j,i) = pCC0_F(i,j); # end # end pBW_DL = np.ones([M, M]) pCC0_b = np.ones([M, M]) # pCC0_F = np.ones([M, M]) for i in range(1, M): for j in range(i+1, M+1): pBW_DL[i-1, j-1] = calpvalue(AUC_BW[[i-1, j-1]], C[[[i-1],[j-1]],[i-1, j-1]]) pBW_DL[j-1, i-1] = pBW_DL[i-1, j-1] pCC0_b[i-1, j-1] = significanceBinomial(CC_0[i-1], CC_0[j-1], L) pCC0_b[j-1, i-1] = pCC0_b[i-1, j-1] # pCC0_F[i-1, j-1] = fexact(CC_0[i-1]*L, 2*L, CC_0[i-1]*L + CC_0[j-1]*L, L, 'tail', 'b') / 2.0 # pCC0_F[j-1, i-1] = pCC0_F[i-1,j] # [pBW_HM,CI_BW] = significanceHM(B, W, AUC_BW); pBW_HM,CI_BW = significanceHM(B, W, AUC_BW) # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # % Adding outputs to the structure # results.AUC_DS = AUC_DS; # results.pDS_DL = pDS_DL; # results.pDS_HM = pDS_HM; # results.AUC_BW = AUC_BW; # results.pBW_DL = pBW_DL; # results.pBW_HM = pBW_HM; # results.CC_0 = CC_0; # results.pCC0_b = pCC0_b; # results.pCC0_F = pCC0_F; # results.THR = THR; result = { 'AUC_DS': AUC_DS, 'pDS_DL': pDS_DL, 'pDS_HM': pDS_HM, 'AUC_BW': AUC_BW, 'pBW_DL': pBW_DL, 'pBW_HM': pBW_HM, 'CC_0': CC_0, 'pCC0_b': pCC0_b, # 'pCC0_F': pCC0_F, 'THR': THR, } # %%%%%%%%%%%%%%%%%%%%%%%% Plot Results %%%%%%%%%%%%%%%%%%%%%%%%%%% # # if(doPlot == 1) # # % Using Benjamini-Hochberg procedure for multiple comparisons in plots # % (note: correlation between groups has to be positive) # # plot_auc(results.pDS_HM,results.AUC_DS, CI_DS, 'AUC (-)','Different/Similar') # plot_cc(results.pCC0_F,results.CC_0,'C_0 (%)','Better/Worse') # plot_auc(results.pBW_HM,results.AUC_BW, CI_BW, 'AUC (-)','Better/Worse') # # end return result