def test_edge_cases_model_wrapper_evaluate(self):
"""`modelwrapper.ModelWrapper.evaluate`: Edge Case Validator.
Tests the behavior of `ModelWrapper.evaluate` with edge cases.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
Y = _random_matrix(self.data_shape)
"""np.matrix: Random-valued observation set."""
for name, ModelWrapper in self.wrappers.iteritems():
# Model string should indicate that no parameters are set at this
# point.
self._validate_model_getter(ModelWrapper, name, self._params_unset)
ModelWrapper.model = dict(X=X)
# Model string should indicate that all parameters are set at this
# point.
self._validate_model_getter(ModelWrapper, name, self._params_set)
with self.assertRaises(_InvalidFeatureSetError):
# Empty feature set.
ModelWrapper.evaluate(_np.matrix([[]]), Y)
with self.assertRaises(_InvalidObservationSetError):
# Empty observation set.
ModelWrapper.evaluate(X, _np.matrix([[]]))
def test_edge_cases_model_wrapper_train(self):
"""`modelwrapper.ModelWrapper.train`: Edge Case Validator.
Tests the behavior of `ModelWrapper.train` with edge cases.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
Y = _random_matrix((self.data_shape[0], 1))
"""np.matrix: Random-valued observation set."""
for name, ModelWrapper in self.learner:
with self.assertRaises(_InvalidFeatureSetError):
# Empty feature set.
ModelWrapper.train(_np.matrix([[]]), Y)
with self.assertRaises(_InvalidObservationSetError):
# Empty observation set.
ModelWrapper.train(X, _np.matrix([[]]))
with self.assertRaises(ValueError):
# Incompatible `k` value.
ModelWrapper.train(X, Y, exact=True, k=self.data_shape[0])
def test_invalid_args_model_wrapper_train(self):
"""`modelwrapper.ModelWrapper.train`: Argument Validator.
Tests the behavior of `ModelWrapper.train` with invalid argument counts
and values.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
Y = _random_matrix((self.data_shape[0], 1))
"""np.matrix: Random-valued observation set."""
for name, ModelWrapper in self.learner:
with self.assertRaises(TypeError):
# No arguments.
ModelWrapper.train()
with self.assertRaises(TypeError):
# Too many arguments.
ModelWrapper.train(X, Y, X)
with self.assertRaises(TypeError):
# Invalid kwarg.
ModelWrapper.train(X, Y, key="value")
with self.assertRaises(_InvalidFeatureSetError):
# `None` instead of feature set `X`.
ModelWrapper.train(None, Y)
with self.assertRaises(_InvalidFeatureSetError):
# ndarray instead of feature set `X`.
ModelWrapper.train(_np.zeros(self.data_shape), Y)
with self.assertRaises(_InvalidObservationSetError):
# `None` instead of observation set `Y`.
ModelWrapper.train(X, None)
with self.assertRaises(_InvalidObservationSetError):
# ndarray instead of observation set `Y`.
ModelWrapper.train(X, _np.zeros((self.data_shape[0], 1)))
with self.assertRaises(TypeError):
# None instead of int `k`.
ModelWrapper.train(X, Y, k=None)
with self.assertRaises(TypeError):
# Float instead of int `k`.
ModelWrapper.train(X, Y, k=0.5)
with self.assertRaises(ValueError):
# Negative integer instead of positive integer `k`.
ModelWrapper.train(X, Y, k=-10)
with self.assertRaises(ValueError):
# Zero of positive integer `k`.
print ModelWrapper.train(X, Y, k=0)
def test_invalid_args_model_wrapper_evaluate(self):
"""`modelwrapper.ModelWrapper.evaluate`: Argument Validator.
Tests the behavior of `ModelWrapper.evaluate` with invalid argument
counts and values.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
Y = _random_matrix((self.data_shape[0], 1))
"""np.matrix: Random-valued feature set."""
for name, ModelWrapper in self.wrappers.iteritems():
# Model string should indicate that no parameters are set at this
# point.
self._validate_model_getter(ModelWrapper, name, self._params_unset)
with self.assertRaises(_IncompleteModelError):
# Unset parameters.
ModelWrapper.evaluate(X, Y)
ModelWrapper.model = dict(X=X)
# Model string should indicate that all parameters are set at this
# point.
self._validate_model_getter(ModelWrapper, name, self._params_set)
with self.assertRaises(TypeError):
# No arguments.
ModelWrapper.evaluate()
with self.assertRaises(TypeError):
# Too many arguments.
ModelWrapper.evaluate(X, Y, X)
with self.assertRaises(TypeError):
# Invalid kwarg.
ModelWrapper.evaluate(X, Y, key="value")
with self.assertRaises(_InvalidFeatureSetError):
# `None` instead of feature set `X`.
ModelWrapper.evaluate(None, Y)
with self.assertRaises(_InvalidFeatureSetError):
# ndarray instead of feature set `X`.
ModelWrapper.evaluate(_np.zeros(self.data_shape), Y)
with self.assertRaises(_InvalidObservationSetError):
# `None` instead of observation set `Y`.
ModelWrapper.evaluate(X, None)
with self.assertRaises(_InvalidObservationSetError):
# ndarray instead of observation set `Y`.
ModelWrapper.evaluate(X, _np.zeros((self.data_shape[0], 1)))
def test_random_model_wrapper_train(self):
"""`modelwrapper.ModelWrapper.train`: Randomized Validator.
Tests the behavior of `ModelWrapper.train` by feeding it randomly
generated arguments.
Raises:
AssertionError: If `ModelWrapper.train` needs debugging.
"""
n = self.data_shape[0]
"""int: Total number of data points."""
for i in range(self.n_tests):
for name, ModelWrapper in self.learner:
for percent in range(10, 100, 18):
f = percent / 100.0
"""float: Training to testing ratio."""
min_k = min(10, _compose(int, _np.floor)(f / 2.0 * n))
"""int: Minimum number of data points per batch."""
max_k = _compose(int, _np.floor)(f * n)
"""int: Minimum number of data points per batch."""
for k in range(
min_k, max_k,
_compose(int, _np.floor)((max_k - min_k) / 5)):
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
Y = _random_matrix((n, 1))
"""np.matrix: Random-valued observation set."""
train_X, train_Y, test_X, test_Y = _partition_data(
X, Y, f)
"""(np.matrix, np.matrix, np.matrix, np.matrix): Train-
ing and testing feature and observation sets."""
ModelWrapper.model = dict(X=train_X)
init_err = ModelWrapper.evaluate(test_X, test_Y)
"""float: Loss before training."""
ModelWrapper.train(train_X, train_Y, exact=True, k=k)
a_err = ModelWrapper.evaluate(test_X, test_Y)
"""float: Loss after analytical training."""
# Analytical error should be a float.
self.assertIsInstance(a_err, float)
if f >= 0.5:
# Analytical training should provide a global
# minimum.
self.assertLess(a_err, init_err)
def setUp(self):
"""Model Wrapper Testing Configuration.
Sets up the necessary information to begin testing.
"""
self.data_shape = 100, 20
self.label = '`learner.Learner`'
self.n_tests = 50
self.name = __name__
self.shapes = {}
self.wrappers = {
n: ModelWrapperClass(m)
for n, m in _active_models.iteritems()
}
p_to_shape = lambda p: p.shape
"""callable: Maps parameters to their matrix dimensions."""
for name, ModelWrapper in self.wrappers.iteritems():
ModelWrapper.model = dict(X=_random_matrix(self.data_shape))
self.shapes[name] = _compose(tuple,
map)(p_to_shape,
ModelWrapper._model.params)
# Model string should indicate that all parameters are set at this
# point.
self._validate_model_getter(ModelWrapper, name, self._params_set)
del ModelWrapper._model.params
# Model string should indicate unset parameters at this point.
self._validate_model_getter(ModelWrapper, name, self._params_unset)
def setUp(self):
"""Learner Testing Configuration.
Sets up the necessary information to begin testing.
"""
self.data_shape = 100, 20
self.label = '`learner.Learner`'
self.learner = Learner()
self.n_tests = 50
self.name = __name__
self.shapes = {}
p_to_shape = lambda p: p.shape
"""callable: Maps parameters to their matrix dimensions."""
for name, ModelWrapper in self.learner:
ModelWrapper.model = dict(X=_random_matrix(self.data_shape))
self.shapes[name] = _compose(tuple,
map)(p_to_shape,
ModelWrapper._model.params)
# Model string should indicate that all parameters are set at this
# point.
self.assertIsNotNone(ModelWrapper._model.params)
del ModelWrapper._model.params
# Model string should indicate unset parameters at this point.
self.assertIsNone(ModelWrapper._model.params)
def test_random_model_wrapper_model(self):
"""`modelwrapper.ModelWrapper.model`: Randomized Validator.
Tests the behavior of `ModelWrapper.model` by feeding it randomly
generated arguments.
Raises:
AssertionError: If `ModelWrapper.model` needs debugging.
"""
shape_regex = _re.compile(r"^\([0-9]+, [0-9]+\)$")
"""SRE_Pattern: Identifies stringified shapes."""
for i in range(self.n_tests):
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued matrix."""
for name, ModelWrapper in self.wrappers.iteritems():
# Model string should indicate unset parameters at this point.
self._validate_model_getter(ModelWrapper, name,
self._params_unset)
params = _compose(tuple, map)(_random_matrix,
self.shapes[name])
"""tuple of np.matrix: Random-valued parameters."""
ModelWrapper.model = dict(params=params)
# Model string should indicate that all parameters are set at
# this point.
self._validate_model_getter(ModelWrapper, name,
self._params_set)
curr_params = ModelWrapper._model.params
"""tuple of np.matrix: Currently set model parameters."""
# Number of parameters in output should match input number.
self.assertEqual(len(curr_params), len(params))
# Matrix norm sums should match for input and output.
self.assertEqual(*map(_np.linalg.norm, [curr_params, params]))
del ModelWrapper._model.params
# Model string should indicate unset parameters again.
self._validate_model_getter(ModelWrapper, name,
self._params_unset)
ModelWrapper.model = dict(X=X)
# Model string should indicate that all parameters are set once
# more.
self._validate_model_getter(ModelWrapper, name,
self._params_set)
del ModelWrapper._model.params
def test_random_model_wrapper_evaluate(self):
"""`modelwrapper.ModelWrapper.evaluate`: Randomized Validator.
Tests the behavior of `ModelWrapper.evaluate` by feeding it randomly
generated arguments.
Raises:
AssertionError: If `ModelWrapper.evaluate` needs debugging.
"""
for i in range(self.n_tests):
for name, ModelWrapper in self.wrappers.iteritems():
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
Y = _random_matrix((self.data_shape[0], 1))
"""np.matrix: Random-valued observation set."""
# Model string should indicate that no parameters are set at
# this point.
self._validate_model_getter(ModelWrapper, name,
self._params_unset)
ModelWrapper.model = dict(X=X)
err = ModelWrapper.evaluate(X, Y)
"""np.matrix: Test input 1."""
# Loss should be a float.
self.assertEqual(type(err), _np.float64)
# Loss should be a positive integer.
self.assertGreaterEqual(err, 0.0)
# Model string should indicate that all parameters are set at
# this point.
self._validate_model_getter(ModelWrapper, name,
self._params_set)
del ModelWrapper._model.params
def test_invalid_args_model_wrapper_model(self):
"""`modelwrapper.ModelWrapper.model`: Argument Validator.
Tests the behavior of `ModelWrapper.model` with invalid argument counts
and values.
Raises:
Exception: If at least one `Exception` raised is not of the expected
kind.
"""
for name, ModelWrapper in self.wrappers.iteritems():
params = _compose(tuple, map)(_random_matrix, self.shapes[name])
"""tuple of np.matrix: Random-valued parameters."""
with self.assertRaises(AttributeError):
# Too many parameters.
ModelWrapper.model = dict(params=params), dict(params=params)
with self.assertRaises(AttributeError):
# No parameters set.
ModelWrapper.model = {}
with self.assertRaises(AttributeError):
# Both parameters set.
ModelWrapper.model = dict(params=params,
X=_random_matrix(self.data_shape))
with self.assertRaises(_InvalidModelParametersError):
# `None` instead of parameters.
ModelWrapper.model = dict(params=None)
with self.assertRaises(_InvalidModelParametersError):
# List instead of parameter tuple.
ModelWrapper.model = dict(params=list(params))
with self.assertRaises(_InvalidModelParametersError):
ndarray_parms = dict(
params=_compose(tuple, map)(_np.zeros, self.shapes[name]))
# Tuple of ndarray instead of matrix tuple.
ModelWrapper.model = ndarray_parms
with self.assertRaises(_InvalidFeatureSetError):
# `None` instead of feature set `X`.
ModelWrapper.model = dict(X=None)
with self.assertRaises(_InvalidFeatureSetError):
# ndarray instead of matrix `X`.
ModelWrapper.model = dict(X=_np.zeros(self.data_shape))
def test_random_model_wrapper_predict(self):
"""`modelwrapper.ModelWrapper.predict`: Randomized Validator.
Tests the behavior of `ModelWrapper.predict` by feeding it randomly
generated arguments.
Raises:
AssertionError: If `ModelWrapper.predict` needs debugging.
"""
for i in range(self.n_tests):
for name, ModelWrapper in self.wrappers.iteritems():
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued feature set."""
# Model string should indicate that no parameters are set at
# this point.
self._validate_model_getter(ModelWrapper, name,
self._params_unset)
ModelWrapper.model = dict(X=X)
# First, test `params` as a method argument.
Y_hat = ModelWrapper.predict(X)
"""np.matrix: Test input 1."""
# Gradients should be a tuple.
self.assertIsInstance(Y_hat, _np.matrix)
# All params should have a gradient.
self.assertEqual(Y_hat.shape, (X.shape[0], 1))
# Model string should indicate that all parameters are set at
# this point.
self._validate_model_getter(ModelWrapper, name,
self._params_set)
del ModelWrapper._model.params
def test_random_model_augment(self):
"""`LinearModel.augment`: Randomized Validator.
Tests the behavior of `augment` by feeding it randomly generated
arguments.
Raises:
AssertionError: If `augment` needs debugging.
"""
# Run `Model`-wide `augment` unit tests.
super(_Test, self).test_random_model_augment()
for i in range(self.n_tests):
X = _random_matrix(self.data_shape)
"""np.matrix: Random-valued matrix."""
n = X.shape[0]
"""int: Number of data points."""
new_X = self.model.augment(X)
"""np.matrix: Test input."""
# Augmentation should also be a matrix.
self.assertIsInstance(X, _np.matrix)
# Unit-valued vector should have been appended to the left of `X`.
self.assertEqual(new_X.shape[1], X.shape[1] + 1)
# Yet the number of rows should remain the same.
self.assertEqual(new_X.shape[0], X.shape[0])
# Yet the number of rows should remain the same.
self.assertEqual(new_X.shape[0], X.shape[0])
# The norm of the leftmost column vector of `new_X` should be
# computable accordin to the following formula.
self.assertAlmostEqual(_np.linalg.norm(new_X[:, 0]), _math.sqrt(n))