def test_unpartitioned_cv(self): data = get_mv_pattern(10) # only one big chunk data.sa.chunks[:] = 1 cv = CrossValidation(sample_clf_nl, NFoldPartitioner()) # need to fail, because it can't be split into training and testing assert_raises(ValueError, cv, data)
def test_noise_classification(self): # get a dataset with a very high SNR data = get_mv_pattern(10) # do crossval with default errorfx and 'mean' combiner cv = CrossValidation(sample_clf_nl, NFoldPartitioner()) # must return a scalar value result = cv(data) # must be perfect self.assertTrue((result.samples < 0.05).all()) # do crossval with permuted regressors cv = CrossValidation( sample_clf_nl, ChainNode( [NFoldPartitioner(), AttributePermutator('targets', count=10)], space='partitions')) results = cv(data) # results must not be the same self.assertTrue(len(np.unique(results.samples)) > 1) # must be at chance level pmean = np.array(results).mean() self.assertTrue(pmean < 0.58 and pmean > 0.42)
def test_noise_classification(self): # get a dataset with a very high SNR data = get_mv_pattern(10) # do crossval with default errorfx and 'mean' combiner cv = CrossValidation(sample_clf_nl, NFoldPartitioner()) # must return a scalar value result = cv(data) # must be perfect self.assertTrue((result.samples < 0.05).all()) # do crossval with permuted regressors cv = CrossValidation(sample_clf_nl, ChainNode([NFoldPartitioner(), AttributePermutator('targets', count=10)], space='partitions')) results = cv(data) # results must not be the same self.assertTrue(len(np.unique(results.samples))>1) # must be at chance level pmean = np.array(results).mean() self.assertTrue( pmean < 0.58 and pmean > 0.42 )
def test_simple_n_minus_one_cv(self): data = get_mv_pattern(3) data.init_origids("samples") self.assertTrue(data.nsamples == 120) self.assertTrue(data.nfeatures == 2) self.assertTrue((data.sa.targets == [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0] * 6).all()) self.assertTrue((data.sa.chunks == [k for k in range(1, 7) for i in range(20)]).all()) assert_equal(len(np.unique(data.sa.origids)), data.nsamples) cv = CrossValidation(sample_clf_nl, NFoldPartitioner(), enable_ca=["stats", "training_stats"]) # 'samples_error']) results = cv(data) self.assertTrue((results.samples < 0.2).all() and (results.samples >= 0.0).all())
def test_simple_n_minus_one_cv(self): data = get_mv_pattern(3) data.init_origids('samples') self.assertTrue(data.nsamples == 120) self.assertTrue(data.nfeatures == 2) self.assertTrue( (data.sa.targets == \ [0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0] * 6).all()) self.assertTrue( (data.sa.chunks == \ [k for k in range(1, 7) for i in range(20)]).all()) assert_equal(len(np.unique(data.sa.origids)), data.nsamples) cv = CrossValidation(sample_clf_nl, NFoldPartitioner(), enable_ca=['stats', 'training_stats']) # 'samples_error']) results = cv(data) self.assertTrue((results.samples < 0.2).all() and (results.samples >= 0.0).all())