示例#1
0
    def to_persubject_dataset(self, quality_scores, **kwargs):

        newone = empty_object()
        newone.dataset_name = self.dataset.dataset_name
        newone.yuv_fmt = self.dataset.yuv_fmt
        newone.width = self.dataset.width
        newone.height = self.dataset.height

        if 'quality_width' in kwargs and kwargs['quality_width'] is not None:
            newone.quality_width = kwargs['quality_width']
        elif hasattr(self.dataset, 'quality_width'):
            newone.quality_width = self.dataset.quality_width

        if 'quality_height' in kwargs and kwargs['quality_height'] is not None:
            newone.quality_height = kwargs['quality_height']
        elif hasattr(self.dataset, 'quality_height'):
            newone.quality_height = self.dataset.quality_height

        if 'resampling_type' in kwargs and kwargs['resampling_type'] is not None:
            newone.resampling_type = kwargs['resampling_type']
        elif hasattr(self.dataset, 'resampling_type'):
            newone.resampling_type = self.dataset.resampling_type

        # ref_videos: deepcopy
        newone.ref_videos = copy.deepcopy(self.dataset.ref_videos)

        # dis_videos: use input aggregate scores
        dis_videos = []
        for dis_video, quality_score in zip(self.dataset.dis_videos, quality_scores):
            assert 'os' in dis_video

            # quality_score should be a 1-D array with (processed) per-subject scores
            assert hasattr(quality_score, '__len__')
            assert len(dis_video['os']) == len(quality_score)

            for persubject_score in quality_score:
                dis_video2 = copy.deepcopy(dis_video)
                if 'os' in dis_video2: # remove 'os' - opinion score
                    del dis_video2['os']
                dis_video2['groundtruth'] = persubject_score
                dis_videos.append(dis_video2)
        newone.dis_videos = dis_videos

        return newone
示例#2
0
    def to_dataset(self):
        """
        Override DatasetReader.to_dataset(). Need to overwrite dis_video['os']
        """

        newone = empty_object()
        newone.__dict__.update(self.dataset.__dict__)

        # deep copy ref_videos and dis_videos
        newone.ref_videos = copy.deepcopy(self.dataset.ref_videos)
        newone.dis_videos = copy.deepcopy(self.dataset.dis_videos)

        # overwrite dis_video['os']
        score_mtx = self.opinion_score_2darray
        num_videos, num_subjects = score_mtx.shape
        assert num_videos == len(newone.dis_videos)
        for scores, dis_video in zip(score_mtx, newone.dis_videos):
            dis_video['os'] = list(scores)

        return newone
示例#3
0
    def to_dataset(self):
        """
        Override DatasetReader.to_dataset(). Need to overwrite dis_video['os']
        """

        newone = empty_object()
        newone.__dict__.update(self.dataset.__dict__)

        # deep copy ref_videos and dis_videos
        newone.ref_videos = copy.deepcopy(self.dataset.ref_videos)
        newone.dis_videos = copy.deepcopy(self.dataset.dis_videos)

        # overwrite dis_video['os']
        score_mtx = self.opinion_score_2darray
        num_videos, num_subjects = score_mtx.shape
        assert num_videos == len(newone.dis_videos)
        for scores, dis_video in zip(score_mtx, newone.dis_videos):
            dis_video['os'] = list(scores)

        return newone
示例#4
0
    def to_aggregated_dataset(self, aggregate_scores, **kwargs):

        newone = empty_object()
        newone.dataset_name = self.dataset.dataset_name
        newone.yuv_fmt = self.dataset.yuv_fmt
        newone.width = self.dataset.width
        newone.height = self.dataset.height

        if 'quality_width' in kwargs and kwargs['quality_width'] is not None:
            newone.quality_width = kwargs['quality_width']
        elif hasattr(self.dataset, 'quality_width'):
            newone.quality_width = self.dataset.quality_width

        if 'quality_height' in kwargs and kwargs['quality_height'] is not None:
            newone.quality_height = kwargs['quality_height']
        elif hasattr(self.dataset, 'quality_height'):
            newone.quality_height = self.dataset.quality_height

        if 'resampling_type' in kwargs and kwargs[
                'resampling_type'] is not None:
            newone.resampling_type = kwargs['resampling_type']
        elif hasattr(self.dataset, 'resampling_type'):
            newone.resampling_type = self.dataset.resampling_type

        # ref_videos: deepcopy
        newone.ref_videos = copy.deepcopy(self.dataset.ref_videos)

        # dis_videos: use input aggregate scores
        dis_videos = []
        assert len(self.dataset.dis_videos) == len(aggregate_scores)
        for dis_video, score in zip(self.dataset.dis_videos, aggregate_scores):
            dis_video2 = copy.deepcopy(dis_video)
            if 'os' in dis_video2:  # remove 'os' - opinion score
                del dis_video2['os']
            dis_video2['groundtruth'] = score
            dis_videos.append(dis_video2)
        newone.dis_videos = dis_videos

        return newone
示例#5
0
    def to_aggregated_dataset(self, aggregate_scores, **kwargs):

        newone = empty_object()
        newone.dataset_name = self.dataset.dataset_name
        newone.yuv_fmt = self.dataset.yuv_fmt
        newone.width = self.dataset.width
        newone.height = self.dataset.height

        if 'quality_width' in kwargs and kwargs['quality_width'] is not None:
            newone.quality_width = kwargs['quality_width']
        elif hasattr(self.dataset, 'quality_width'):
            newone.quality_width = self.dataset.quality_width

        if 'quality_height' in kwargs and kwargs['quality_height'] is not None:
            newone.quality_height = kwargs['quality_height']
        elif hasattr(self.dataset, 'quality_height'):
            newone.quality_height = self.dataset.quality_height

        if 'resampling_type' in kwargs and kwargs['resampling_type'] is not None:
            newone.resampling_type = kwargs['resampling_type']
        elif hasattr(self.dataset, 'resampling_type'):
            newone.resampling_type = self.dataset.resampling_type

        # ref_videos: deepcopy
        newone.ref_videos = copy.deepcopy(self.dataset.ref_videos)

        # dis_videos: use input aggregate scores
        dis_videos = []
        assert len(self.dataset.dis_videos) == len(aggregate_scores)
        for dis_video, score in zip(self.dataset.dis_videos, aggregate_scores):
            dis_video2 = copy.deepcopy(dis_video)
            if 'os' in dis_video2: # remove 'os' - opinion score
                del dis_video2['os']
            dis_video2['groundtruth'] = score
            dis_videos.append(dis_video2)
        newone.dis_videos = dis_videos

        return newone
示例#6
0
    def _metrics_performance(objScoDif, signif):
        """
        mirroring matlab function:
        %[results] = Metrics_performance(objScoDif, signif, doPlot)
        % INPUT:    objScoDif   : differences of objective scores [M,N]
        %                         M : number of metrics
        %                         N : number of pairs
        %           signif      : statistical outcome of paired comparison [1,N]
        %                          0 : no difference
        %                         -1 : first stimulus is worse
        %                          1 : first stimulus is better
        %           doPlot      : boolean indicating if graphs should be plotted
        %
        % OUTPUT:   results - structure with following fields
        %
        %           AUC_DS      : Area Under the Curve for Different/Similar ROC
        %                         analysis
        %           pDS_DL      : p-values for AUC_DS from DeLong test
        %           pDS_HM      : p-values for AUC_DS from Hanley and McNeil test
        %           AUC_BW      : Area Under the Curve for Better/Worse ROC
        %                         analysis
        %           pBW_DL      : p-values for AUC_BW from DeLong test
        %           pBW_HM      : p-values for AUC_BW from Hanley and McNeil test
        %           CC_0        : Correct Classification @ DeltaOM = 0 for
        %                         Better/Worse ROC analysis
        %           pCC0_b      : p-values for CC_0 from binomial test
        %           pCC0_F      : p-values for CC_0 from Fisher's exact test
        %           THR         : threshold for 95% probability that the stimuli
        %                         are different
        """

        # M = size(objScoDif,1);
        # D = abs(objScoDif(:,signif ~= 0));
        # S = abs(objScoDif(:,signif == 0));
        # samples.spsizes = [size(D,2),size(S,2)];
        # samples.ratings = [D,S];

        M = objScoDif.shape[0]
        D = np.abs(objScoDif[:, indices(signif[0], lambda x: x != 0)])
        S = np.abs(objScoDif[:, indices(signif[0], lambda x: x == 0)])
        samples = empty_object()
        samples.spsizes = [D.shape[1], S.shape[1]]
        samples.ratings = np.hstack([D, S])

        # % calculate AUCs

        # [AUC_DS,C] = fastDeLong(samples);
        AUC_DS, C, _, _ = fastDeLong(samples)

        # % significance calculation

        # pDS_DL = ones(M);
        # for i=1:M-1
        #     for j=i+1:M
        #         pDS_DL(i,j) = calpvalue(AUC_DS([i,j]), C([i,j],[i,j]));
        #         pDS_DL(j,i) = pDS_DL(i,j);
        #     end
        # end
        pDS_DL = np.ones([M, M])
        for i in range(1, M):
            for j in range(i + 1, M + 1):
                # http://stackoverflow.com/questions/4257394/slicing-of-a-numpy-2d-array-or-how-do-i-extract-an-mxm-submatrix-from-an-nxn-ar
                pDS_DL[i - 1, j - 1] = calpvalue(
                    AUC_DS[[i - 1, j - 1]], C[[[i - 1], [j - 1]],
                                              [i - 1, j - 1]])
                pDS_DL[j - 1, i - 1] = pDS_DL[i - 1, j - 1]

        # [pDS_HM,CI_DS] = significanceHM(S, D, AUC_DS);
        pDS_HM, CI_DS = significanceHM(S, D, AUC_DS)

        # THR = prctile(D',95);
        THR = np.percentile(D, 95, axis=1)

        # %%%%%%%%%%%%%%%%%%%%%%% Better / Worse %%%%%%%%%%%%%%%%%%%%%%%%%%%

        # B = [objScoDif(:,signif == 1),-objScoDif(:,signif == -1)];
        # W = -B;
        # samples.ratings = [B,W];
        # samples.spsizes = [size(B,2),size(W,2)];
        B1 = objScoDif[:, indices(signif[0], lambda x: x == 1)]
        B2 = objScoDif[:, indices(signif[0], lambda x: x == -1)]
        B = np.hstack([B1, -B2])
        W = -B
        samples = empty_object()
        samples.ratings = np.hstack([B, W])
        samples.spsizes = [B.shape[1], W.shape[1]]

        # % calculate AUCs

        # [AUC_BW,C] = fastDeLong(samples);
        AUC_BW, C, _, _ = fastDeLong(samples)

        # % calculate correct classification for DeltaOM = 0

        # L = size(B,2) + size(W,2);
        # CC_0 = zeros(M,1);
        # for m=1:M
        #     CC_0(m) = (sum(B(m,:)>0) + sum(W(m,:)<0)) / L;
        # end
        L = B.shape[1] + W.shape[1]
        CC_0 = np.zeros(M)
        for m in range(M):
            CC_0[m] = float(np.sum(B[m, :] > 0) + np.sum(W[m, :] < 0)) / L

        # % significance calculation

        # pBW_DL = ones(M);
        # pCC0_b = ones(M);
        # pCC0_F = ones(M);
        # for i=1:M-1
        #     for j=i+1:M
        #         pBW_DL(i,j) = calpvalue(AUC_BW([i,j]), C([i,j],[i,j]));
        #         pBW_DL(j,i) = pBW_DL(i,j);
        #
        #         pCC0_b(i,j) = significanceBinomial(CC_0(i), CC_0(j), L);
        #         pCC0_b(j,i) = pCC0_b(i,j);
        #
        #         pCC0_F(i,j) = fexact(CC_0(i)*L, 2*L, CC_0(i)*L + CC_0(j)*L, L, 'tail', 'b')/2;
        #         pCC0_F(j,i) = pCC0_F(i,j);
        #     end
        # end
        pBW_DL = np.ones([M, M])
        pCC0_b = np.ones([M, M])
        # pCC0_F = np.ones([M, M])
        for i in range(1, M):
            for j in range(i + 1, M + 1):
                pBW_DL[i - 1, j - 1] = calpvalue(
                    AUC_BW[[i - 1, j - 1]], C[[[i - 1], [j - 1]],
                                              [i - 1, j - 1]])
                pBW_DL[j - 1, i - 1] = pBW_DL[i - 1, j - 1]

                pCC0_b[i - 1,
                       j - 1] = significanceBinomial(CC_0[i - 1], CC_0[j - 1],
                                                     L)
                pCC0_b[j - 1, i - 1] = pCC0_b[i - 1, j - 1]

                # pCC0_F[i-1, j-1] = fexact(CC_0[i-1]*L, 2*L, CC_0[i-1]*L + CC_0[j-1]*L, L, 'tail', 'b') / 2.0
                # pCC0_F[j-1, i-1] = pCC0_F[i-1,j]

        # [pBW_HM,CI_BW] = significanceHM(B, W, AUC_BW);
        pBW_HM, CI_BW = significanceHM(B, W, AUC_BW)

        # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

        # % Adding outputs to the structure

        # results.AUC_DS = AUC_DS;
        # results.pDS_DL = pDS_DL;
        # results.pDS_HM = pDS_HM;
        # results.AUC_BW = AUC_BW;
        # results.pBW_DL = pBW_DL;
        # results.pBW_HM = pBW_HM;
        # results.CC_0 = CC_0;
        # results.pCC0_b = pCC0_b;
        # results.pCC0_F = pCC0_F;
        # results.THR = THR;
        result = {
            'AUC_DS': AUC_DS,
            'pDS_DL': pDS_DL,
            'pDS_HM': pDS_HM,
            'AUC_BW': AUC_BW,
            'pBW_DL': pBW_DL,
            'pBW_HM': pBW_HM,
            'CC_0': CC_0,
            'pCC0_b': pCC0_b,
            # 'pCC0_F': pCC0_F,
            'THR': THR,
        }

        # %%%%%%%%%%%%%%%%%%%%%%%% Plot Results %%%%%%%%%%%%%%%%%%%%%%%%%%%
        #
        # if(doPlot == 1)
        #
        # % Using Benjamini-Hochberg procedure for multiple comparisons in plots
        # % (note: correlation between groups has to be positive)
        #
        # plot_auc(results.pDS_HM,results.AUC_DS, CI_DS, 'AUC (-)','Different/Similar')
        # plot_cc(results.pCC0_F,results.CC_0,'C_0 (%)','Better/Worse')
        # plot_auc(results.pBW_HM,results.AUC_BW, CI_BW, 'AUC (-)','Better/Worse')
        #
        # end

        return result
示例#7
0
    def _metrics_performance(objScoDif, signif):
        """
        mirroring matlab function:
        %[results] = Metrics_performance(objScoDif, signif, doPlot)
        % INPUT:    objScoDif   : differences of objective scores [M,N]
        %                         M : number of metrics
        %                         N : number of pairs
        %           signif      : statistical outcome of paired comparison [1,N]
        %                          0 : no difference
        %                         -1 : first stimulus is worse
        %                          1 : first stimulus is better
        %           doPlot      : boolean indicating if graphs should be plotted
        %
        % OUTPUT:   results - structure with following fields
        %
        %           AUC_DS      : Area Under the Curve for Different/Similar ROC
        %                         analysis
        %           pDS_DL      : p-values for AUC_DS from DeLong test
        %           pDS_HM      : p-values for AUC_DS from Hanley and McNeil test
        %           AUC_BW      : Area Under the Curve for Better/Worse ROC
        %                         analysis
        %           pBW_DL      : p-values for AUC_BW from DeLong test
        %           pBW_HM      : p-values for AUC_BW from Hanley and McNeil test
        %           CC_0        : Correct Classification @ DeltaOM = 0 for
        %                         Better/Worse ROC analysis
        %           pCC0_b      : p-values for CC_0 from binomial test
        %           pCC0_F      : p-values for CC_0 from Fisher's exact test
        %           THR         : threshold for 95% probability that the stimuli
        %                         are different
        """

        # M = size(objScoDif,1);
        # D = abs(objScoDif(:,signif ~= 0));
        # S = abs(objScoDif(:,signif == 0));
        # samples.spsizes = [size(D,2),size(S,2)];
        # samples.ratings = [D,S];

        M = objScoDif.shape[0]
        D = np.abs(objScoDif[:, indices(signif[0], lambda x: x!=0)])
        S = np.abs(objScoDif[:, indices(signif[0], lambda x: x==0)])
        samples = empty_object()
        samples.spsizes = [D.shape[1], S.shape[1]]
        samples.ratings = np.hstack([D, S])

        # % calculate AUCs

        # [AUC_DS,C] = fastDeLong(samples);
        AUC_DS, C, _, _ = fastDeLong(samples)

        # % significance calculation

        # pDS_DL = ones(M);
        # for i=1:M-1
        #     for j=i+1:M
        #         pDS_DL(i,j) = calpvalue(AUC_DS([i,j]), C([i,j],[i,j]));
        #         pDS_DL(j,i) = pDS_DL(i,j);
        #     end
        # end
        pDS_DL = np.ones([M, M])
        for i in range(1, M):
            for j in range(i+1, M+1):
                # http://stackoverflow.com/questions/4257394/slicing-of-a-numpy-2d-array-or-how-do-i-extract-an-mxm-submatrix-from-an-nxn-ar
                pDS_DL[i-1, j-1] = calpvalue(AUC_DS[[i-1, j-1]], C[[[i-1],[j-1]],[i-1, j-1]])
                pDS_DL[j-1, i-1] = pDS_DL[i-1, j-1]

        # [pDS_HM,CI_DS] = significanceHM(S, D, AUC_DS);
        pDS_HM, CI_DS = significanceHM(S, D, AUC_DS)

        # THR = prctile(D',95);
        THR = np.percentile(D, 95, axis=1)

        # %%%%%%%%%%%%%%%%%%%%%%% Better / Worse %%%%%%%%%%%%%%%%%%%%%%%%%%%

        # B = [objScoDif(:,signif == 1),-objScoDif(:,signif == -1)];
        # W = -B;
        # samples.ratings = [B,W];
        # samples.spsizes = [size(B,2),size(W,2)];
        B1 = objScoDif[:, indices(signif[0], lambda x: x== 1)]
        B2 = objScoDif[:, indices(signif[0], lambda x: x==-1)]
        B = np.hstack([B1, -B2])
        W = -B
        samples = empty_object()
        samples.ratings = np.hstack([B, W])
        samples.spsizes = [B.shape[1], W.shape[1]]

        # % calculate AUCs

        # [AUC_BW,C] = fastDeLong(samples);
        AUC_BW, C, _, _ = fastDeLong(samples)

        # % calculate correct classification for DeltaOM = 0

        # L = size(B,2) + size(W,2);
        # CC_0 = zeros(M,1);
        # for m=1:M
        #     CC_0(m) = (sum(B(m,:)>0) + sum(W(m,:)<0)) / L;
        # end
        L = B.shape[1] + W.shape[1]
        CC_0 = np.zeros(M)
        for m in range(M):
            CC_0[m] = float(np.sum(B[m,:]>0) + np.sum(W[m,:]<0)) / L

        # % significance calculation

        # pBW_DL = ones(M);
        # pCC0_b = ones(M);
        # pCC0_F = ones(M);
        # for i=1:M-1
        #     for j=i+1:M
        #         pBW_DL(i,j) = calpvalue(AUC_BW([i,j]), C([i,j],[i,j]));
        #         pBW_DL(j,i) = pBW_DL(i,j);
        #
        #         pCC0_b(i,j) = significanceBinomial(CC_0(i), CC_0(j), L);
        #         pCC0_b(j,i) = pCC0_b(i,j);
        #
        #         pCC0_F(i,j) = fexact(CC_0(i)*L, 2*L, CC_0(i)*L + CC_0(j)*L, L, 'tail', 'b')/2;
        #         pCC0_F(j,i) = pCC0_F(i,j);
        #     end
        # end
        pBW_DL = np.ones([M, M])
        pCC0_b = np.ones([M, M])
        # pCC0_F = np.ones([M, M])
        for i in range(1, M):
            for j in range(i+1, M+1):
                pBW_DL[i-1, j-1] = calpvalue(AUC_BW[[i-1, j-1]], C[[[i-1],[j-1]],[i-1, j-1]])
                pBW_DL[j-1, i-1] = pBW_DL[i-1, j-1]

                pCC0_b[i-1, j-1] = significanceBinomial(CC_0[i-1], CC_0[j-1], L)
                pCC0_b[j-1, i-1] = pCC0_b[i-1, j-1]

                # pCC0_F[i-1, j-1] = fexact(CC_0[i-1]*L, 2*L, CC_0[i-1]*L + CC_0[j-1]*L, L, 'tail', 'b') / 2.0
                # pCC0_F[j-1, i-1] = pCC0_F[i-1,j]

        # [pBW_HM,CI_BW] = significanceHM(B, W, AUC_BW);
        pBW_HM,CI_BW = significanceHM(B, W, AUC_BW)

        # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

        # % Adding outputs to the structure

        # results.AUC_DS = AUC_DS;
        # results.pDS_DL = pDS_DL;
        # results.pDS_HM = pDS_HM;
        # results.AUC_BW = AUC_BW;
        # results.pBW_DL = pBW_DL;
        # results.pBW_HM = pBW_HM;
        # results.CC_0 = CC_0;
        # results.pCC0_b = pCC0_b;
        # results.pCC0_F = pCC0_F;
        # results.THR = THR;
        result = {
            'AUC_DS': AUC_DS,
            'pDS_DL': pDS_DL,
            'pDS_HM': pDS_HM,
            'AUC_BW': AUC_BW,
            'pBW_DL': pBW_DL,
            'pBW_HM': pBW_HM,
            'CC_0': CC_0,
            'pCC0_b': pCC0_b,
            # 'pCC0_F': pCC0_F,
            'THR': THR,
        }

        # %%%%%%%%%%%%%%%%%%%%%%%% Plot Results %%%%%%%%%%%%%%%%%%%%%%%%%%%
        #
        # if(doPlot == 1)
        #
        # % Using Benjamini-Hochberg procedure for multiple comparisons in plots
        # % (note: correlation between groups has to be positive)
        #
        # plot_auc(results.pDS_HM,results.AUC_DS, CI_DS, 'AUC (-)','Different/Similar')
        # plot_cc(results.pCC0_F,results.CC_0,'C_0 (%)','Better/Worse')
        # plot_auc(results.pBW_HM,results.AUC_BW, CI_BW, 'AUC (-)','Better/Worse')
        #
        # end

        return result