def test_chi_square(self, exp): """Test chi-square distribution""" # test equal distribution tbl = np.array([[5, 5], [5, 5]]) chi, p = chisquare(tbl, exp=exp) self.failUnless( chi == 0.0 ) self.failUnless( p == 1.0 ) # test perfect "generalization" tbl = np.array([[4, 0], [0, 4]]) chi, p = chisquare(tbl, exp=exp) self.failUnless(chi == 8.0) self.failUnless(p < 0.05)
def testChiSquare(self): """Test chi-square distribution""" # test equal distribution tbl = N.array([[5, 5], [5, 5]]) chi, p = chisquare(tbl) self.failUnless( chi == 0.0 ) self.failUnless( p == 1.0 ) # test non-equal distribution tbl = N.array([[4, 0], [0, 4]]) chi, p = chisquare(tbl) self.failUnless(chi == 8.0) self.failUnless(p < 0.05)
def test_chi_square_disbalanced(self): # test perfect "generalization" tbl = np.array([[1, 100], [1, 100]]) chi, p = chisquare(tbl, exp='indep_rows') self.failUnless(chi == 0) self.failUnless(p == 1) chi, p = chisquare(tbl, exp='uniform') self.failUnless(chi > 194) self.failUnless(p < 1e-10) # by default lets do uniform chi_, p_ = chisquare(tbl) self.failUnless(chi == chi_) self.failUnless(p == p_)
def test_chi_square_disbalanced(self): # test perfect "generalization" tbl = np.array([[1,100], [1,100]]) chi, p = chisquare(tbl, exp='indep_rows') self.failUnless(chi == 0) self.failUnless(p == 1) chi, p = chisquare(tbl, exp='uniform') self.failUnless(chi > 194) self.failUnless(p < 1e-10) # by default lets do uniform chi_, p_ = chisquare(tbl) self.failUnless(chi == chi_) self.failUnless(p == p_)
def _compute(self): """Actually compute the confusion matrix based on all the sets""" super(ConfusionMatrix, self)._compute() if __debug__: if not self.__matrix is None: debug("LAZY", "Have to recompute %s#%s" \ % (self.__class__.__name__, id(self))) # TODO: BinaryClassifier might spit out a list of predictions for each # value need to handle it... for now just keep original labels try: # figure out what labels we have labels = \ list(reduce(lambda x, y: x.union(set(y[0]).union(set(y[1]))), self.sets, set(self.__labels))) except: labels = self.__labels # Check labels_map if it was provided if it covers all the labels labels_map = self.__labels_map if labels_map is not None: labels_set = set(labels) map_labels_set = set(labels_map.values()) if not map_labels_set.issuperset(labels_set): warning("Provided labels_map %s is not coherent with labels " "provided to ConfusionMatrix. No reverse mapping " "will be provided" % labels_map) labels_map = None # Create reverse map labels_map_rev = None if labels_map is not None: labels_map_rev = {} for k,v in labels_map.iteritems(): v_mapping = labels_map_rev.get(v, []) v_mapping.append(k) labels_map_rev[v] = v_mapping self.__labels_map_rev = labels_map_rev labels.sort() self.__labels = labels # store the recomputed labels Nlabels, Nsets = len(labels), len(self.sets) if __debug__: debug("CM", "Got labels %s" % labels) # Create a matrix for all votes mat_all = np.zeros( (Nsets, Nlabels, Nlabels), dtype=int ) # create total number of samples of each label counts # just for convinience I guess since it can always be # computed from mat_all counts_all = np.zeros( (Nsets, Nlabels) ) # reverse mapping from label into index in the list of labels rev_map = dict([ (x[1], x[0]) for x in enumerate(labels)]) for iset, set_ in enumerate(self.sets): for t,p in zip(*set_[:2]): mat_all[iset, rev_map[p], rev_map[t]] += 1 # for now simply compute a sum of votes across different sets # we might do something more sophisticated later on, and this setup # should easily allow it self.__matrix = np.sum(mat_all, axis=0) self.__Nsamples = np.sum(self.__matrix, axis=0) self.__Ncorrect = sum(np.diag(self.__matrix)) TP = np.diag(self.__matrix) offdiag = self.__matrix - np.diag(TP) stats = { '# of labels' : Nlabels, 'TP' : TP, 'FP' : np.sum(offdiag, axis=1), 'FN' : np.sum(offdiag, axis=0)} stats['CORR'] = np.sum(TP) stats['TN'] = stats['CORR'] - stats['TP'] stats['P'] = stats['TP'] + stats['FN'] stats['N'] = np.sum(stats['P']) - stats['P'] stats["P'"] = stats['TP'] + stats['FP'] stats["N'"] = stats['TN'] + stats['FN'] stats['TPR'] = stats['TP'] / (1.0*stats['P']) # reset nans in TPRs to 0s whenever there is no entries # for those labels among the targets stats['TPR'][stats['P'] == 0] = 0 stats['PPV'] = stats['TP'] / (1.0*stats["P'"]) stats['NPV'] = stats['TN'] / (1.0*stats["N'"]) stats['FDR'] = stats['FP'] / (1.0*stats["P'"]) stats['SPC'] = (stats['TN']) / (1.0*stats['FP'] + stats['TN']) MCC_denom = np.sqrt(1.0*stats['P']*stats['N']*stats["P'"]*stats["N'"]) nz = MCC_denom!=0.0 stats['MCC'] = np.zeros(stats['TP'].shape) stats['MCC'][nz] = \ (stats['TP'] * stats['TN'] - stats['FP'] * stats['FN'])[nz] \ / MCC_denom[nz] stats['ACC'] = np.sum(TP)/(1.0*np.sum(stats['P'])) # TODO: STD of accuracy and corrected one according to # Nadeau and Bengio [50] stats['ACC%'] = stats['ACC'] * 100.0 if chisquare: # indep_rows to assure reasonable handling of disbalanced # cases stats['CHI^2'] = chisquare(self.__matrix, exp='indep_rows') # # ROC computation if available ROC = ROCCurve(labels=labels, sets=self.sets) aucs = ROC.aucs if len(aucs)>0: stats['AUC'] = aucs if len(aucs) != Nlabels: raise RuntimeError, \ "We must got a AUC per label. Got %d instead of %d" % \ (len(aucs), Nlabels) self.ROC = ROC else: # we don't want to provide ROC if it is bogus stats['AUC'] = [np.nan] * Nlabels self.ROC = None # compute mean stats for k,v in stats.items(): stats['mean(%s)' % k] = np.mean(v) self._stats.update(stats)
def getconfusion(data): cv(data) return chisquare(cv.ca.stats.matrix)[0]
def getconfusion(data): cv(data) return chisquare(cv.ca.confusion.matrix)[0]