def plot_scatter(ax, stats, content_ids=None):
assert len(stats['ys_label']) == len(stats['ys_label_pred'])
if content_ids is None:
ax.scatter(stats['ys_label'], stats['ys_label_pred'])
else:
assert len(stats['ys_label']) == len(content_ids)
unique_content_ids = list(set(content_ids))
import matplotlib.pyplot as plt
cmap = plt.get_cmap()
colors = [
cmap(i) for i in np.linspace(0, 1, len(unique_content_ids))
]
for idx, curr_content_id in enumerate(unique_content_ids):
curr_idxs = indices(content_ids,
lambda cid: cid == curr_content_id)
curr_ys_label = np.array(stats['ys_label'])[curr_idxs]
curr_ys_label_pred = np.array(
stats['ys_label_pred'])[curr_idxs]
ax.scatter(curr_ys_label,
curr_ys_label_pred,
label=curr_content_id,
color=colors[idx % len(colors)])
def construct_kfold_list(assets, contentid_groups):
# construct cross validation kfold input list
content_ids = map(lambda asset: asset.content_id, assets)
kfold = []
for curr_content_group in contentid_groups:
curr_indices = indices(content_ids, lambda x: x in curr_content_group)
kfold.append(curr_indices)
return kfold
def construct_kfold_list(assets, contentid_groups):
# construct cross validation kfold input list
content_ids = map(lambda asset: asset.content_id, assets)
kfold = []
for curr_content_group in contentid_groups:
curr_indices = indices(content_ids, lambda x: x in curr_content_group)
kfold.append(curr_indices)
return kfold
def _get_ref_mos(dataset_reader, mos):
ref_mos = []
for dis_video in dataset_reader.dataset.dis_videos:
# get the dis video's ref video's mos
curr_content_id = dis_video['content_id']
ref_indices = indices(
zip(dataset_reader.content_id_of_dis_videos,
dataset_reader.disvideo_is_refvideo),
lambda (content_id, is_refvideo):
is_refvideo and content_id == curr_content_id
)
assert len(ref_indices) == 1, \
'Should have only and one ref video for a dis video, ' \
'but got {}'.format(len(ref_indices))
ref_idx = ref_indices[0]
ref_mos.append(mos[ref_idx])
return np.array(ref_mos)
def plot_scatter(ax, stats, content_ids=None):
assert len(stats['ys_label']) == len(stats['ys_label_pred'])
if content_ids is None:
ax.scatter(stats['ys_label'], stats['ys_label_pred'])
else:
assert len(stats['ys_label']) == len(content_ids)
unique_content_ids = list(set(content_ids))
import matplotlib.pyplot as plt
cmap = plt.get_cmap()
colors = [cmap(i) for i in np.linspace(0, 1, len(unique_content_ids))]
for idx, curr_content_id in enumerate(unique_content_ids):
curr_idxs = indices(content_ids, lambda cid: cid==curr_content_id)
curr_ys_label = np.array(stats['ys_label'])[curr_idxs]
curr_ys_label_pred = np.array(stats['ys_label_pred'])[curr_idxs]
ax.scatter(curr_ys_label, curr_ys_label_pred,
label=curr_content_id, color=colors[idx % len(colors)])
def test_disvideo_is_refvideo(self):
l = self.dataset_reader.disvideo_is_refvideo
self.assertItemsEqual(indices(l, lambda e: e is True), range(9))
def _metrics_performance(objScoDif, signif):
"""
mirroring matlab function:
%[results] = Metrics_performance(objScoDif, signif, doPlot)
% INPUT: objScoDif : differences of objective scores [M,N]
% M : number of metrics
% N : number of pairs
% signif : statistical outcome of paired comparison [1,N]
% 0 : no difference
% -1 : first stimulus is worse
% 1 : first stimulus is better
% doPlot : boolean indicating if graphs should be plotted
%
% OUTPUT: results - structure with following fields
%
% AUC_DS : Area Under the Curve for Different/Similar ROC
% analysis
% pDS_DL : p-values for AUC_DS from DeLong test
% pDS_HM : p-values for AUC_DS from Hanley and McNeil test
% AUC_BW : Area Under the Curve for Better/Worse ROC
% analysis
% pBW_DL : p-values for AUC_BW from DeLong test
% pBW_HM : p-values for AUC_BW from Hanley and McNeil test
% CC_0 : Correct Classification @ DeltaOM = 0 for
% Better/Worse ROC analysis
% pCC0_b : p-values for CC_0 from binomial test
% pCC0_F : p-values for CC_0 from Fisher's exact test
% THR : threshold for 95% probability that the stimuli
% are different
"""
# M = size(objScoDif,1);
# D = abs(objScoDif(:,signif ~= 0));
# S = abs(objScoDif(:,signif == 0));
# samples.spsizes = [size(D,2),size(S,2)];
# samples.ratings = [D,S];
M = objScoDif.shape[0]
D = np.abs(objScoDif[:, indices(signif[0], lambda x: x != 0)])
S = np.abs(objScoDif[:, indices(signif[0], lambda x: x == 0)])
samples = empty_object()
samples.spsizes = [D.shape[1], S.shape[1]]
samples.ratings = np.hstack([D, S])
# % calculate AUCs
# [AUC_DS,C] = fastDeLong(samples);
AUC_DS, C, _, _ = fastDeLong(samples)
# % significance calculation
# pDS_DL = ones(M);
# for i=1:M-1
# for j=i+1:M
# pDS_DL(i,j) = calpvalue(AUC_DS([i,j]), C([i,j],[i,j]));
# pDS_DL(j,i) = pDS_DL(i,j);
# end
# end
pDS_DL = np.ones([M, M])
for i in range(1, M):
for j in range(i + 1, M + 1):
# http://stackoverflow.com/questions/4257394/slicing-of-a-numpy-2d-array-or-how-do-i-extract-an-mxm-submatrix-from-an-nxn-ar
pDS_DL[i - 1, j - 1] = calpvalue(
AUC_DS[[i - 1, j - 1]], C[[[i - 1], [j - 1]],
[i - 1, j - 1]])
pDS_DL[j - 1, i - 1] = pDS_DL[i - 1, j - 1]
# [pDS_HM,CI_DS] = significanceHM(S, D, AUC_DS);
pDS_HM, CI_DS = significanceHM(S, D, AUC_DS)
# THR = prctile(D',95);
THR = np.percentile(D, 95, axis=1)
# %%%%%%%%%%%%%%%%%%%%%%% Better / Worse %%%%%%%%%%%%%%%%%%%%%%%%%%%
# B = [objScoDif(:,signif == 1),-objScoDif(:,signif == -1)];
# W = -B;
# samples.ratings = [B,W];
# samples.spsizes = [size(B,2),size(W,2)];
B1 = objScoDif[:, indices(signif[0], lambda x: x == 1)]
B2 = objScoDif[:, indices(signif[0], lambda x: x == -1)]
B = np.hstack([B1, -B2])
W = -B
samples = empty_object()
samples.ratings = np.hstack([B, W])
samples.spsizes = [B.shape[1], W.shape[1]]
# % calculate AUCs
# [AUC_BW,C] = fastDeLong(samples);
AUC_BW, C, _, _ = fastDeLong(samples)
# % calculate correct classification for DeltaOM = 0
# L = size(B,2) + size(W,2);
# CC_0 = zeros(M,1);
# for m=1:M
# CC_0(m) = (sum(B(m,:)>0) + sum(W(m,:)<0)) / L;
# end
L = B.shape[1] + W.shape[1]
CC_0 = np.zeros(M)
for m in range(M):
CC_0[m] = float(np.sum(B[m, :] > 0) + np.sum(W[m, :] < 0)) / L
# % significance calculation
# pBW_DL = ones(M);
# pCC0_b = ones(M);
# pCC0_F = ones(M);
# for i=1:M-1
# for j=i+1:M
# pBW_DL(i,j) = calpvalue(AUC_BW([i,j]), C([i,j],[i,j]));
# pBW_DL(j,i) = pBW_DL(i,j);
#
# pCC0_b(i,j) = significanceBinomial(CC_0(i), CC_0(j), L);
# pCC0_b(j,i) = pCC0_b(i,j);
#
# pCC0_F(i,j) = fexact(CC_0(i)*L, 2*L, CC_0(i)*L + CC_0(j)*L, L, 'tail', 'b')/2;
# pCC0_F(j,i) = pCC0_F(i,j);
# end
# end
pBW_DL = np.ones([M, M])
pCC0_b = np.ones([M, M])
# pCC0_F = np.ones([M, M])
for i in range(1, M):
for j in range(i + 1, M + 1):
pBW_DL[i - 1, j - 1] = calpvalue(
AUC_BW[[i - 1, j - 1]], C[[[i - 1], [j - 1]],
[i - 1, j - 1]])
pBW_DL[j - 1, i - 1] = pBW_DL[i - 1, j - 1]
pCC0_b[i - 1,
j - 1] = significanceBinomial(CC_0[i - 1], CC_0[j - 1],
L)
pCC0_b[j - 1, i - 1] = pCC0_b[i - 1, j - 1]
# pCC0_F[i-1, j-1] = fexact(CC_0[i-1]*L, 2*L, CC_0[i-1]*L + CC_0[j-1]*L, L, 'tail', 'b') / 2.0
# pCC0_F[j-1, i-1] = pCC0_F[i-1,j]
# [pBW_HM,CI_BW] = significanceHM(B, W, AUC_BW);
pBW_HM, CI_BW = significanceHM(B, W, AUC_BW)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# % Adding outputs to the structure
# results.AUC_DS = AUC_DS;
# results.pDS_DL = pDS_DL;
# results.pDS_HM = pDS_HM;
# results.AUC_BW = AUC_BW;
# results.pBW_DL = pBW_DL;
# results.pBW_HM = pBW_HM;
# results.CC_0 = CC_0;
# results.pCC0_b = pCC0_b;
# results.pCC0_F = pCC0_F;
# results.THR = THR;
result = {
'AUC_DS': AUC_DS,
'pDS_DL': pDS_DL,
'pDS_HM': pDS_HM,
'AUC_BW': AUC_BW,
'pBW_DL': pBW_DL,
'pBW_HM': pBW_HM,
'CC_0': CC_0,
'pCC0_b': pCC0_b,
# 'pCC0_F': pCC0_F,
'THR': THR,
}
# %%%%%%%%%%%%%%%%%%%%%%%% Plot Results %%%%%%%%%%%%%%%%%%%%%%%%%%%
#
# if(doPlot == 1)
#
# % Using Benjamini-Hochberg procedure for multiple comparisons in plots
# % (note: correlation between groups has to be positive)
#
# plot_auc(results.pDS_HM,results.AUC_DS, CI_DS, 'AUC (-)','Different/Similar')
# plot_cc(results.pCC0_F,results.CC_0,'C_0 (%)','Better/Worse')
# plot_auc(results.pBW_HM,results.AUC_BW, CI_BW, 'AUC (-)','Better/Worse')
#
# end
return result
def _metrics_performance(objScoDif, signif):
"""
mirroring matlab function:
%[results] = Metrics_performance(objScoDif, signif, doPlot)
% INPUT: objScoDif : differences of objective scores [M,N]
% M : number of metrics
% N : number of pairs
% signif : statistical outcome of paired comparison [1,N]
% 0 : no difference
% -1 : first stimulus is worse
% 1 : first stimulus is better
% doPlot : boolean indicating if graphs should be plotted
%
% OUTPUT: results - structure with following fields
%
% AUC_DS : Area Under the Curve for Different/Similar ROC
% analysis
% pDS_DL : p-values for AUC_DS from DeLong test
% pDS_HM : p-values for AUC_DS from Hanley and McNeil test
% AUC_BW : Area Under the Curve for Better/Worse ROC
% analysis
% pBW_DL : p-values for AUC_BW from DeLong test
% pBW_HM : p-values for AUC_BW from Hanley and McNeil test
% CC_0 : Correct Classification @ DeltaOM = 0 for
% Better/Worse ROC analysis
% pCC0_b : p-values for CC_0 from binomial test
% pCC0_F : p-values for CC_0 from Fisher's exact test
% THR : threshold for 95% probability that the stimuli
% are different
"""
# M = size(objScoDif,1);
# D = abs(objScoDif(:,signif ~= 0));
# S = abs(objScoDif(:,signif == 0));
# samples.spsizes = [size(D,2),size(S,2)];
# samples.ratings = [D,S];
M = objScoDif.shape[0]
D = np.abs(objScoDif[:, indices(signif[0], lambda x: x!=0)])
S = np.abs(objScoDif[:, indices(signif[0], lambda x: x==0)])
samples = empty_object()
samples.spsizes = [D.shape[1], S.shape[1]]
samples.ratings = np.hstack([D, S])
# % calculate AUCs
# [AUC_DS,C] = fastDeLong(samples);
AUC_DS, C, _, _ = fastDeLong(samples)
# % significance calculation
# pDS_DL = ones(M);
# for i=1:M-1
# for j=i+1:M
# pDS_DL(i,j) = calpvalue(AUC_DS([i,j]), C([i,j],[i,j]));
# pDS_DL(j,i) = pDS_DL(i,j);
# end
# end
pDS_DL = np.ones([M, M])
for i in range(1, M):
for j in range(i+1, M+1):
# http://stackoverflow.com/questions/4257394/slicing-of-a-numpy-2d-array-or-how-do-i-extract-an-mxm-submatrix-from-an-nxn-ar
pDS_DL[i-1, j-1] = calpvalue(AUC_DS[[i-1, j-1]], C[[[i-1],[j-1]],[i-1, j-1]])
pDS_DL[j-1, i-1] = pDS_DL[i-1, j-1]
# [pDS_HM,CI_DS] = significanceHM(S, D, AUC_DS);
pDS_HM, CI_DS = significanceHM(S, D, AUC_DS)
# THR = prctile(D',95);
THR = np.percentile(D, 95, axis=1)
# %%%%%%%%%%%%%%%%%%%%%%% Better / Worse %%%%%%%%%%%%%%%%%%%%%%%%%%%
# B = [objScoDif(:,signif == 1),-objScoDif(:,signif == -1)];
# W = -B;
# samples.ratings = [B,W];
# samples.spsizes = [size(B,2),size(W,2)];
B1 = objScoDif[:, indices(signif[0], lambda x: x== 1)]
B2 = objScoDif[:, indices(signif[0], lambda x: x==-1)]
B = np.hstack([B1, -B2])
W = -B
samples = empty_object()
samples.ratings = np.hstack([B, W])
samples.spsizes = [B.shape[1], W.shape[1]]
# % calculate AUCs
# [AUC_BW,C] = fastDeLong(samples);
AUC_BW, C, _, _ = fastDeLong(samples)
# % calculate correct classification for DeltaOM = 0
# L = size(B,2) + size(W,2);
# CC_0 = zeros(M,1);
# for m=1:M
# CC_0(m) = (sum(B(m,:)>0) + sum(W(m,:)<0)) / L;
# end
L = B.shape[1] + W.shape[1]
CC_0 = np.zeros(M)
for m in range(M):
CC_0[m] = float(np.sum(B[m,:]>0) + np.sum(W[m,:]<0)) / L
# % significance calculation
# pBW_DL = ones(M);
# pCC0_b = ones(M);
# pCC0_F = ones(M);
# for i=1:M-1
# for j=i+1:M
# pBW_DL(i,j) = calpvalue(AUC_BW([i,j]), C([i,j],[i,j]));
# pBW_DL(j,i) = pBW_DL(i,j);
#
# pCC0_b(i,j) = significanceBinomial(CC_0(i), CC_0(j), L);
# pCC0_b(j,i) = pCC0_b(i,j);
#
# pCC0_F(i,j) = fexact(CC_0(i)*L, 2*L, CC_0(i)*L + CC_0(j)*L, L, 'tail', 'b')/2;
# pCC0_F(j,i) = pCC0_F(i,j);
# end
# end
pBW_DL = np.ones([M, M])
pCC0_b = np.ones([M, M])
# pCC0_F = np.ones([M, M])
for i in range(1, M):
for j in range(i+1, M+1):
pBW_DL[i-1, j-1] = calpvalue(AUC_BW[[i-1, j-1]], C[[[i-1],[j-1]],[i-1, j-1]])
pBW_DL[j-1, i-1] = pBW_DL[i-1, j-1]
pCC0_b[i-1, j-1] = significanceBinomial(CC_0[i-1], CC_0[j-1], L)
pCC0_b[j-1, i-1] = pCC0_b[i-1, j-1]
# pCC0_F[i-1, j-1] = fexact(CC_0[i-1]*L, 2*L, CC_0[i-1]*L + CC_0[j-1]*L, L, 'tail', 'b') / 2.0
# pCC0_F[j-1, i-1] = pCC0_F[i-1,j]
# [pBW_HM,CI_BW] = significanceHM(B, W, AUC_BW);
pBW_HM,CI_BW = significanceHM(B, W, AUC_BW)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# % Adding outputs to the structure
# results.AUC_DS = AUC_DS;
# results.pDS_DL = pDS_DL;
# results.pDS_HM = pDS_HM;
# results.AUC_BW = AUC_BW;
# results.pBW_DL = pBW_DL;
# results.pBW_HM = pBW_HM;
# results.CC_0 = CC_0;
# results.pCC0_b = pCC0_b;
# results.pCC0_F = pCC0_F;
# results.THR = THR;
result = {
'AUC_DS': AUC_DS,
'pDS_DL': pDS_DL,
'pDS_HM': pDS_HM,
'AUC_BW': AUC_BW,
'pBW_DL': pBW_DL,
'pBW_HM': pBW_HM,
'CC_0': CC_0,
'pCC0_b': pCC0_b,
# 'pCC0_F': pCC0_F,
'THR': THR,
}
# %%%%%%%%%%%%%%%%%%%%%%%% Plot Results %%%%%%%%%%%%%%%%%%%%%%%%%%%
#
# if(doPlot == 1)
#
# % Using Benjamini-Hochberg procedure for multiple comparisons in plots
# % (note: correlation between groups has to be positive)
#
# plot_auc(results.pDS_HM,results.AUC_DS, CI_DS, 'AUC (-)','Different/Similar')
# plot_cc(results.pCC0_F,results.CC_0,'C_0 (%)','Better/Worse')
# plot_auc(results.pBW_HM,results.AUC_BW, CI_BW, 'AUC (-)','Better/Worse')
#
# end
return result
