def test_chi_square(self, exp):
"""Test chi-square distribution"""
# test equal distribution
tbl = np.array([[5, 5], [5, 5]])
chi, p = chisquare(tbl, exp=exp)
self.failUnless( chi == 0.0 )
self.failUnless( p == 1.0 )
# test perfect "generalization"
tbl = np.array([[4, 0], [0, 4]])
chi, p = chisquare(tbl, exp=exp)
self.failUnless(chi == 8.0)
self.failUnless(p < 0.05)
def test_chi_square(self, exp):
"""Test chi-square distribution"""
# test equal distribution
tbl = np.array([[5, 5], [5, 5]])
chi, p = chisquare(tbl, exp=exp)
self.failUnless( chi == 0.0 )
self.failUnless( p == 1.0 )
# test perfect "generalization"
tbl = np.array([[4, 0], [0, 4]])
chi, p = chisquare(tbl, exp=exp)
self.failUnless(chi == 8.0)
self.failUnless(p < 0.05)
def testChiSquare(self):
"""Test chi-square distribution"""
# test equal distribution
tbl = N.array([[5, 5], [5, 5]])
chi, p = chisquare(tbl)
self.failUnless( chi == 0.0 )
self.failUnless( p == 1.0 )
# test non-equal distribution
tbl = N.array([[4, 0], [0, 4]])
chi, p = chisquare(tbl)
self.failUnless(chi == 8.0)
self.failUnless(p < 0.05)
def test_chi_square_disbalanced(self):
# test perfect "generalization"
tbl = np.array([[1, 100], [1, 100]])
chi, p = chisquare(tbl, exp='indep_rows')
self.failUnless(chi == 0)
self.failUnless(p == 1)
chi, p = chisquare(tbl, exp='uniform')
self.failUnless(chi > 194)
self.failUnless(p < 1e-10)
# by default lets do uniform
chi_, p_ = chisquare(tbl)
self.failUnless(chi == chi_)
self.failUnless(p == p_)
def test_chi_square_disbalanced(self):
# test perfect "generalization"
tbl = np.array([[1,100], [1,100]])
chi, p = chisquare(tbl, exp='indep_rows')
self.failUnless(chi == 0)
self.failUnless(p == 1)
chi, p = chisquare(tbl, exp='uniform')
self.failUnless(chi > 194)
self.failUnless(p < 1e-10)
# by default lets do uniform
chi_, p_ = chisquare(tbl)
self.failUnless(chi == chi_)
self.failUnless(p == p_)
def _compute(self):
"""Actually compute the confusion matrix based on all the sets"""
super(ConfusionMatrix, self)._compute()
if __debug__:
if not self.__matrix is None:
debug("LAZY",
"Have to recompute %s#%s" \
% (self.__class__.__name__, id(self)))
# TODO: BinaryClassifier might spit out a list of predictions for each
# value need to handle it... for now just keep original labels
try:
# figure out what labels we have
labels = \
list(reduce(lambda x, y: x.union(set(y[0]).union(set(y[1]))),
self.sets,
set(self.__labels)))
except:
labels = self.__labels
# Check labels_map if it was provided if it covers all the labels
labels_map = self.__labels_map
if labels_map is not None:
labels_set = set(labels)
map_labels_set = set(labels_map.values())
if not map_labels_set.issuperset(labels_set):
warning("Provided labels_map %s is not coherent with labels "
"provided to ConfusionMatrix. No reverse mapping "
"will be provided" % labels_map)
labels_map = None
# Create reverse map
labels_map_rev = None
if labels_map is not None:
labels_map_rev = {}
for k,v in labels_map.iteritems():
v_mapping = labels_map_rev.get(v, [])
v_mapping.append(k)
labels_map_rev[v] = v_mapping
self.__labels_map_rev = labels_map_rev
labels.sort()
self.__labels = labels # store the recomputed labels
Nlabels, Nsets = len(labels), len(self.sets)
if __debug__:
debug("CM", "Got labels %s" % labels)
# Create a matrix for all votes
mat_all = np.zeros( (Nsets, Nlabels, Nlabels), dtype=int )
# create total number of samples of each label counts
# just for convinience I guess since it can always be
# computed from mat_all
counts_all = np.zeros( (Nsets, Nlabels) )
# reverse mapping from label into index in the list of labels
rev_map = dict([ (x[1], x[0]) for x in enumerate(labels)])
for iset, set_ in enumerate(self.sets):
for t,p in zip(*set_[:2]):
mat_all[iset, rev_map[p], rev_map[t]] += 1
# for now simply compute a sum of votes across different sets
# we might do something more sophisticated later on, and this setup
# should easily allow it
self.__matrix = np.sum(mat_all, axis=0)
self.__Nsamples = np.sum(self.__matrix, axis=0)
self.__Ncorrect = sum(np.diag(self.__matrix))
TP = np.diag(self.__matrix)
offdiag = self.__matrix - np.diag(TP)
stats = {
'# of labels' : Nlabels,
'TP' : TP,
'FP' : np.sum(offdiag, axis=1),
'FN' : np.sum(offdiag, axis=0)}
stats['CORR'] = np.sum(TP)
stats['TN'] = stats['CORR'] - stats['TP']
stats['P'] = stats['TP'] + stats['FN']
stats['N'] = np.sum(stats['P']) - stats['P']
stats["P'"] = stats['TP'] + stats['FP']
stats["N'"] = stats['TN'] + stats['FN']
stats['TPR'] = stats['TP'] / (1.0*stats['P'])
# reset nans in TPRs to 0s whenever there is no entries
# for those labels among the targets
stats['TPR'][stats['P'] == 0] = 0
stats['PPV'] = stats['TP'] / (1.0*stats["P'"])
stats['NPV'] = stats['TN'] / (1.0*stats["N'"])
stats['FDR'] = stats['FP'] / (1.0*stats["P'"])
stats['SPC'] = (stats['TN']) / (1.0*stats['FP'] + stats['TN'])
MCC_denom = np.sqrt(1.0*stats['P']*stats['N']*stats["P'"]*stats["N'"])
nz = MCC_denom!=0.0
stats['MCC'] = np.zeros(stats['TP'].shape)
stats['MCC'][nz] = \
(stats['TP'] * stats['TN'] - stats['FP'] * stats['FN'])[nz] \
/ MCC_denom[nz]
stats['ACC'] = np.sum(TP)/(1.0*np.sum(stats['P']))
# TODO: STD of accuracy and corrected one according to
# Nadeau and Bengio [50]
stats['ACC%'] = stats['ACC'] * 100.0
if chisquare:
# indep_rows to assure reasonable handling of disbalanced
# cases
stats['CHI^2'] = chisquare(self.__matrix, exp='indep_rows')
#
# ROC computation if available
ROC = ROCCurve(labels=labels, sets=self.sets)
aucs = ROC.aucs
if len(aucs)>0:
stats['AUC'] = aucs
if len(aucs) != Nlabels:
raise RuntimeError, \
"We must got a AUC per label. Got %d instead of %d" % \
(len(aucs), Nlabels)
self.ROC = ROC
else:
# we don't want to provide ROC if it is bogus
stats['AUC'] = [np.nan] * Nlabels
self.ROC = None
# compute mean stats
for k,v in stats.items():
stats['mean(%s)' % k] = np.mean(v)
self._stats.update(stats)
def getconfusion(data):
cv(data)
return chisquare(cv.ca.stats.matrix)[0]
def getconfusion(data):
cv(data)
return chisquare(cv.ca.confusion.matrix)[0]
