def test_stratified_dataset_shuffle_split():
"""Test StratifiedDatasetShuffleSplit."""
skip_if_no_sklearn()
mapping = {'dataset_iterator': 'StratifiedDatasetShuffleSplit'}
test_yaml = test_yaml_dataset_iterator % mapping
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
def test_stratified_dataset_k_fold():
"""Test StratifiedDatasetKFold."""
skip_if_no_sklearn()
mapping = {'dataset_iterator': 'StratifiedDatasetKFold'}
test_yaml = test_yaml_dataset_iterator % mapping
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
def test_which_set():
"""Test which_set selector."""
skip_if_no_sklearn()
# one label
this_yaml = test_yaml_which_set % {'which_set': 'train'}
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
# multiple labels
this_yaml = test_yaml_which_set % {'which_set': ['train', 'test']}
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
# improper label (iterator only returns 'train' and 'test' subsets)
this_yaml = test_yaml_which_set % {'which_set': 'valid'}
try:
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
raise AssertionError
except ValueError:
pass
# bogus label (not in approved list)
this_yaml = test_yaml_which_set % {'which_set': 'bogus'}
try:
yaml_parse.load(this_yaml)
raise AssertionError
except ValueError:
pass
def test_dataset_validation_shuffle_split():
"""Test DatasetValidShuffleSplit."""
skip_if_no_sklearn()
mapping = {'dataset_iterator': 'DatasetValidationShuffleSplit'}
test_yaml = test_yaml_dataset_iterator % mapping
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
def test_dataset_validation_k_fold():
"""Test DatasetValidKFold."""
skip_if_no_sklearn()
mapping = {'dataset_iterator': 'DatasetValidationKFold'}
test_yaml = test_yaml_dataset_iterator % mapping
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
def test_monitor_based_save_best_cv():
"""Test MonitorBasedSaveBestCV."""
handle, filename = tempfile.mkstemp()
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml_monitor_based_save_best_cv %
{'save_path': filename})
trainer.main_loop()
# clean up
os.remove(filename)
def test_monitor_based_save_best_cv():
"""Test MonitorBasedSaveBestCV."""
handle, filename = tempfile.mkstemp()
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml_monitor_based_save_best_cv %
{'save_path': filename})
trainer.main_loop()
# clean up
os.remove(filename)
def setUp(self):
"""
Set up test for DenseMulticlassSVM.
Imports DenseMulticlassSVM if available, skips the test otherwise.
"""
global DenseMulticlassSVM
skip_if_no_sklearn()
skip_if_no_data()
import pylearn2.models.svm
DenseMulticlassSVM = pylearn2.models.svm.DenseMulticlassSVM
def setUp(self):
"""
Set up test for DenseMulticlassSVM.
Imports DenseMulticlassSVM if available, skips the test otherwise.
"""
global DenseMulticlassSVM
skip_if_no_sklearn()
skip_if_no_data()
import pylearn2.models.svm
DenseMulticlassSVM = pylearn2.models.svm.DenseMulticlassSVM
def test_validation_k_fold():
"""Test ValidationKFold."""
skip_if_no_sklearn()
from pylearn2.cross_validation.subset_iterators import ValidationKFold
n = 30
# test with indices
cv = ValidationKFold(n)
for train, valid, test in cv:
assert np.unique(np.concatenate((train, valid, test))).size == n
assert valid.size == n / cv.n_folds
assert test.size == n / cv.n_folds
def test_validation_shuffle_split():
"""Test ValidationShuffleSplit."""
skip_if_no_sklearn()
from pylearn2.cross_validation.subset_iterators import (
ValidationShuffleSplit)
n = 30
# test with indices
cv = ValidationShuffleSplit(n)
for train, valid, test in cv:
assert np.unique(np.concatenate((train, valid, test))).size == n
assert valid.size == n * cv.test_size
assert test.size == n * cv.test_size
def test_stratified_validation_shuffle_split():
"""Test StratifiedValidationShuffleSplit."""
skip_if_no_sklearn()
from pylearn2.cross_validation.subset_iterators import (
StratifiedValidationShuffleSplit)
n = 60
y = np.concatenate((np.zeros(n / 2, dtype=int), np.ones(n / 2, dtype=int)))
# test with indices
cv = StratifiedValidationShuffleSplit(y)
for train, valid, test in cv:
assert np.unique(np.concatenate((train, valid, test))).size == n
assert valid.size == n * cv.test_size
assert test.size == n * cv.test_size
assert np.count_nonzero(y[valid]) == (n / 2) * cv.test_size
assert np.count_nonzero(y[test]) == (n / 2) * cv.test_size
def test_stratified_validation_k_fold():
"""Test StratifiedValidationKFold."""
skip_if_no_sklearn()
from pylearn2.cross_validation.subset_iterators import (
StratifiedValidationKFold)
n = 30
y = np.concatenate((np.zeros(n / 2, dtype=int), np.ones(n / 2, dtype=int)))
# test with indices
cv = StratifiedValidationKFold(y)
for train, valid, test in cv:
assert np.unique(np.concatenate((train, valid, test))).size == n
assert valid.size == n / cv.n_folds
assert test.size == n / cv.n_folds
assert np.count_nonzero(y[valid]) == (n / 2) * (1. / cv.n_folds)
assert np.count_nonzero(y[test]) == (n / 2) * (1. / cv.n_folds)
def test_print_monitor_cv():
"""Test print_monitor_cv.py."""
skip_if_no_sklearn()
handle, filename = tempfile.mkstemp()
trainer = yaml_parse.load(test_print_monitor_cv_yaml %
{'filename': filename})
trainer.main_loop()
# run print_monitor_cv.py main
print_monitor_cv.main(filename)
# run print_monitor_cv.py main with all=True
print_monitor_cv.main(filename, all=True)
# cleanup
os.remove(filename)
def test_print_monitor_cv():
"""Test print_monitor_cv.py."""
skip_if_no_sklearn()
handle, filename = tempfile.mkstemp()
trainer = yaml_parse.load(test_print_monitor_cv_yaml %
{'filename': filename})
trainer.main_loop()
# run print_monitor_cv.py main
print_monitor_cv.main(filename)
# run print_monitor_cv.py main with all=True
print_monitor_cv.main(filename, all=True)
# cleanup
os.remove(filename)
def test_train_cv():
"""Test TrainCV class."""
skip_if_no_sklearn()
handle, layer0_filename = tempfile.mkstemp()
handle, layer1_filename = tempfile.mkstemp()
handle, layer2_filename = tempfile.mkstemp()
# train the first hidden layer (unsupervised)
# (test for TrainCV)
trainer = yaml_parse.load(test_yaml_layer0 %
{'layer0_filename': layer0_filename})
trainer.main_loop()
# train the second hidden layer (unsupervised)
# (test for TransformerDatasetCV)
trainer = yaml_parse.load(test_yaml_layer1 % {
'layer0_filename': layer0_filename,
'layer1_filename': layer1_filename
})
trainer.main_loop()
# train the third hidden layer (unsupervised)
# (test for StackedBlocksCV)
trainer = yaml_parse.load(
test_yaml_layer2 % {
'layer0_filename': layer0_filename,
'layer1_filename': layer1_filename,
'layer2_filename': layer2_filename
})
trainer.main_loop()
# train the full model (supervised)
# (test for PretrainedLayerCV)
trainer = yaml_parse.load(
test_yaml_layer3 % {
'layer0_filename': layer0_filename,
'layer1_filename': layer1_filename,
'layer2_filename': layer2_filename
})
trainer.main_loop()
# clean up
os.remove(layer0_filename)
os.remove(layer1_filename)
def test_which_set():
"""Test which_set selector."""
skip_if_no_sklearn()
# one label
# must be 'train' or Train object won't have any data to train on
this_yaml = test_yaml_which_set % {'which_set': 'train'}
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
# one label, not 'train'
# this tests any internal stuff in the dataset iterator that might
# require training data, such as preprocessing
try:
this_yaml = test_yaml_which_set % {'which_set': 'test'}
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
except KeyError:
pass
# multiple labels
this_yaml = test_yaml_which_set % {'which_set': ['train', 'test']}
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
# improper label (this iterator only returns 'train' and 'test' subsets)
this_yaml = test_yaml_which_set % {'which_set': 'valid'}
try:
trainer = yaml_parse.load(this_yaml)
trainer.main_loop()
raise AssertionError
except ValueError:
pass
# bogus label (not in approved list)
this_yaml = test_yaml_which_set % {'which_set': 'bogus'}
try:
yaml_parse.load(this_yaml)
raise AssertionError
except ValueError:
pass
def test_train_cv():
"""Test TrainCV class."""
skip_if_no_sklearn()
handle, layer0_filename = tempfile.mkstemp()
handle, layer1_filename = tempfile.mkstemp()
handle, layer2_filename = tempfile.mkstemp()
# train the first hidden layer (unsupervised)
# (test for TrainCV)
trainer = yaml_parse.load(test_yaml_layer0 %
{'layer0_filename': layer0_filename})
trainer.main_loop()
# train the second hidden layer (unsupervised)
# (test for TransformerDatasetCV)
trainer = yaml_parse.load(test_yaml_layer1 %
{'layer0_filename': layer0_filename,
'layer1_filename': layer1_filename})
trainer.main_loop()
# train the third hidden layer (unsupervised)
# (test for StackedBlocksCV)
trainer = yaml_parse.load(test_yaml_layer2 %
{'layer0_filename': layer0_filename,
'layer1_filename': layer1_filename,
'layer2_filename': layer2_filename})
trainer.main_loop()
# train the full model (supervised)
# (test for PretrainedLayerCV)
trainer = yaml_parse.load(test_yaml_layer3 %
{'layer0_filename': layer0_filename,
'layer1_filename': layer1_filename,
'layer2_filename': layer2_filename})
trainer.main_loop()
# clean up
os.remove(layer0_filename)
os.remove(layer1_filename)
def test_wmape():
"""Test WMapeChannel."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
X = trainer.model.get_input_space().make_theano_batch()
Y = trainer.model.fprop(X)
f = function([X], Y, allow_input_downcast=True)
y_hat = f(trainer.dataset.X)
wmape_num_exp = abs(trainer.dataset.y - y_hat).sum()
wmape_den_exp = abs(trainer.dataset.y).sum()
exp_array = np.asarray([wmape_num_exp, wmape_den_exp])
wmape_num_real = trainer.model.monitor.channels['train_wmape_num'].\
val_record
wmape_den_real = trainer.model.monitor.channels['train_wmape_den'].\
val_record
real_array = np.asarray([wmape_num_real[-1], wmape_den_real[-1]])
assert_allclose(exp_array, real_array)
def test_roc_auc():
"""Test RocAucChannel."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
def setUp(self):
global DenseMulticlassSVM
skip_if_no_sklearn()
skip_if_no_data()
import pylearn2.models.svm
DenseMulticlassSVM = pylearn2.models.svm.DenseMulticlassSVM
def setUp(self):
global DenseMulticlassSVM
skip_if_no_sklearn()
import pylearn2.models.svm
DenseMulticlassSVM = pylearn2.models.svm.DenseMulticlassSVM
def test_roc_auc():
"""Test RocAucChannel."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml)
trainer.main_loop()
def test_roc_auc_one_vs_one():
"""Test one vs. rest RocAucChannel."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml_ovo)
trainer.main_loop()
def test_roc_auc_one_vs_one():
"""Test one vs. rest RocAucChannel."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml_ovo)
trainer.main_loop()
def test_preprocessing():
"""Test cross-validation with preprocessing."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml_preprocessing)
trainer.main_loop()
def test_no_targets():
"""Test cross-validation without targets."""
skip_if_no_sklearn()
trainer = yaml_parse.load(test_yaml_no_targets)
trainer.main_loop()
