def test_PerSampleStatisticsWITHPredictedValue(self):
"""DISCRETE PerSampleStatistics with numeric predicted value"""
fs_kwargs = {}
fs_kwargs['name'] = "DISCRETE PerSampleStatistics WITH Pred Values"
fs_kwargs['n_samples'] = n_samples = 40
fs_kwargs['n_classes'] = 2
fs_kwargs[
'num_features_per_signal_type'] = 10 # small on purpose, to make test fast
fs_kwargs['noise_gradient'] = 50
fs_kwargs['initial_noise_sigma'] = 75
fs_kwargs['n_samples_per_group'] = 1
fs_kwargs['random_state'] = 42
fs_kwargs['interpolatable'] = True
fs_kwargs['singularity'] = False
fs_kwargs['clip'] = False
fs = CreateArtificialFeatureSpace_Discrete(**fs_kwargs)
# Use case 1: Straight, classic WND-CHARM train/test splits
ss_kwargs = {}
ss_kwargs[
'name'] = "Discrete PerSampleStatistics ShuffleSplit WITH Pred Values"
ss_kwargs['quiet'] = True
ss_kwargs['n_iter'] = n_iter = 10
ss_kwargs['train_size'] = train_size = 8 # per-class
ss_kwargs['test_size'] = test_size = 2 # per-class
ss_kwargs['random_state'] = 42
exp = FeatureSpaceClassificationExperiment.NewShuffleSplit(
fs, **ss_kwargs)
# Use case 2: Put LDA in pipeline (no fisher feature space prefilter, by default)
ss_kwargs['lda'] = True
exp = FeatureSpaceClassificationExperiment.NewShuffleSplit(
fs, **ss_kwargs)
## Use case 3: LDA AND Fisher feature space prefilter
#ss_kwargs['pre_lda_feature_filter'] = True
#exp = FeatureSpaceClassificationExperiment.NewShuffleSplit( fs, **ss_kwargs )
## Use case 4: LDA AND Fisher feature space prefilter, AND post-LDA dimension reduction
#ss_kwargs['lda_features_size'] = 0.5
#exp = FeatureSpaceClassificationExperiment.NewShuffleSplit( fs, **ss_kwargs )
#Print calls self.GenereateStats()
#from os import devnull
exp.Print() #output_stream=devnull )
exp.PerSampleStatistics() #output_stream=devnull )
self.assertTrue(True)
def test_FromDiscreteClassificationExperimentResults(self):
"""Rank Ordered Predicted values graph from an experiment result (multiple splits)"""
testfilename = 'test_graph_rank_ordered_experiment.npy'
# Make a smaller featureset to do multiple splits
fs_kwargs = {}
fs_kwargs['name'] = "DiscreteArtificialFS RANK ORDERED SHUFFLE SPLIT"
fs_kwargs['n_samples'] = 100 # smaller
fs_kwargs['n_classes'] = 5 # smaller, 20 samples per class
fs_kwargs['num_features_per_signal_type'] = 10 # smaller
fs_kwargs['initial_noise_sigma'] = 50
fs_kwargs['noise_gradient'] = 20
fs_kwargs['n_samples_per_group'] = 1
fs_kwargs['interpolatable'] = True
fs_kwargs['random_state'] = 42
fs_kwargs['singularity'] = False
fs_kwargs['clip'] = False
small_fs = CreateArtificialFeatureSpace_Discrete(**fs_kwargs)
ss_kwargs = {}
ss_kwargs['quiet'] = True
ss_kwargs['n_iter'] = n_iter = 10
ss_kwargs['train_size'] = train_size = 18 # per-class
ss_kwargs['test_size'] = test_size = 2 # per-class
ss_kwargs['random_state'] = 42
exp = FeatureSpaceClassificationExperiment.NewShuffleSplit(
small_fs, **ss_kwargs)
graph = PredictedValuesGraph(exp, use_averaged_results=False)
graph.RankOrderedPredictedValuesGraph()
self.CompareGraphs(graph, testfilename)
def test_PerSampleStatisticsWITHPredictedValue(self):
"""DISCRETE PerSampleStatistics with numeric predicted value"""
fs_kwargs = {}
fs_kwargs['name'] = "DISCRETE PerSampleStatistics WITH Pred Values"
fs_kwargs['n_samples'] = n_samples = 40
fs_kwargs['n_classes'] = 2
fs_kwargs[
'num_features_per_signal_type'] = 10 # small on purpose, to make test fast
fs_kwargs['noise_gradient'] = 50
fs_kwargs['initial_noise_sigma'] = 75
fs_kwargs['n_samples_per_group'] = 1
fs_kwargs['random_state'] = 42
fs_kwargs['interpolatable'] = True
fs_kwargs['singularity'] = False
fs_kwargs['clip'] = False
fs = CreateArtificialFeatureSpace_Discrete(**fs_kwargs)
ss_kwargs = {}
ss_kwargs[
'name'] = "Discrete PerSampleStatistics ShuffleSplit WITH Pred Values"
ss_kwargs['quiet'] = True
ss_kwargs['n_iter'] = n_iter = 10
ss_kwargs['train_size'] = train_size = 8 # per-class
ss_kwargs['test_size'] = test_size = 2 # per-class
ss_kwargs['random_state'] = 42
exp = FeatureSpaceClassificationExperiment.NewShuffleSplit(
fs, **ss_kwargs)
#Print calls self.GenereateStats()
#from os import devnull
exp.Print() #output_stream=devnull )
exp.PerSampleStatistics() #output_stream=devnull )
self.assertTrue(True)
def test_PerSampleStatisticsWITHOUTPredictedValue(self):
"""DISCRETE ShuffleSplit/PerSampleStatistics w/ no predicted value"""
# CAN'T USE THIS, SINCE THE CLASS NAMES ARE INTERPOLATABLE
# 2-class, 10 samples per class
#fs = FeatureSet_Discrete.NewFromFitFile( '../wndchrm_tests/test-l.fit' )
fs_kwargs = {}
fs_kwargs['name'] = "DISCRETE PerSampleStatistics No Pred Values"
fs_kwargs['n_samples'] = n_samples = 20
fs_kwargs['n_classes'] = 2
fs_kwargs[
'num_features_per_signal_type'] = 10 # small on purpose, to make test fast
fs_kwargs['noise_gradient'] = 50
fs_kwargs['initial_noise_sigma'] = 75
fs_kwargs['n_samples_per_group'] = 1
fs_kwargs['random_state'] = 42
fs_kwargs['interpolatable'] = False
fs_kwargs['singularity'] = False
fs_kwargs['clip'] = False
fs = CreateArtificialFeatureSpace_Discrete(**fs_kwargs)
ss_kwargs = {}
ss_kwargs[
'name'] = "Discrete PerSampleStatistics ShuffleSplit No Pred Values"
ss_kwargs['quiet'] = True
ss_kwargs['n_iter'] = n_iter = 1
ss_kwargs['train_size'] = train_size = 8 # per-class
ss_kwargs['test_size'] = test_size = 2 # per-class
ss_kwargs['random_state'] = 42
exp = FeatureSpaceClassificationExperiment.NewShuffleSplit(
fs, **ss_kwargs)
ss_kwargs['lda'] = True
exp = FeatureSpaceClassificationExperiment.NewShuffleSplit(
fs, **ss_kwargs)
#Print calls self.GenereateStats()
#from os import devnull
exp.Print() #output_stream=devnull )
exp.PerSampleStatistics() #output_stream=devnull )
self.assertTrue(True)