def test2_illegal_name_value(self):
"""
This function will make sure that only valid parameter names and values should be included in the
hyper-parameter dict for grid search. It will randomly go into the hyper_parameters that we have
specified, either change the hyper_parameter name or insert an illegal value into the hyper_parameter
list, check to make sure that the test failed with error messages.
The following error conditions will be created depending on the error_number generated:
error_number = 0: randomly alter the name of a hyper-parameter name;
error_number = 1: randomly choose a hyper-parameter and remove all elements in its list
error_number = 2: add randomly generated new hyper-parameter names with random list
error_number other: randomly choose a hyper-parameter and insert an illegal type into it
:return: None
"""
print("*******************************************************************************************")
print("test2_illegal_name_value for GLM " + self.family)
h2o.cluster_info()
error_number = np.random.random_integers(0, 3, 1) # randomly choose an error
error_hyper_params = \
pyunit_utils.insert_error_grid_search(self.hyper_params, self.gridable_parameters, self.gridable_types,
error_number[0])
print("test2_illegal_name_value: the bad hyper-parameters are: ")
print(error_hyper_params)
# copied from Eric to catch execution run errors and not quit
try:
grid_model = H2OGridSearch(H2OGeneralizedLinearEstimator(family=self.family, nfolds=self.nfolds),
hyper_params=error_hyper_params)
grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data)
if error_number[0] > 2:
# grid search should not failed in this case and build same number of models as test1.
if not (len(grid_model) == self.correct_model_number):
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed. Number of model generated is "
"incorrect.")
else:
print("test2_illegal_name_value passed.")
else: # other errors should cause exceptions being thrown and if not, something is wrong.
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed: exception should have been thrown for illegal"
"parameter name or empty hyper-parameter parameter list but did not!")
except:
print("test2_illegal_name_value passed: exception is thrown for illegal parameter name or empty"
"hyper-parameter parameter list.")
self.test_num += 1
def test2_illegal_name_value(self):
"""
test2_illegal_name_value: test for condition 1 and 2. Randomly go into the hyper_parameters that we
have specified, either
a. randomly alter the name of a hyper-parameter name (fatal, exception will be thrown)
b. randomly choose a hyper-parameter and remove all elements in its list (fatal)
c. add randomly generated new hyper-parameter names with random list (fatal)
d: randomly choose a hyper-parameter and insert an illegal type into it (non fatal, model built with
legal hyper-parameters settings only and error messages printed out for illegal hyper-parameters
settings)
The following error conditions will be created depending on the error_number generated:
error_number = 0: randomly alter the name of a hyper-parameter name;
error_number = 1: randomly choose a hyper-parameter and remove all elements in its list
error_number = 2: add randomly generated new hyper-parameter names with random list
error_number = 3: randomly choose a hyper-parameter and insert an illegal type into it
:return: None
"""
print("*******************************************************************************************")
print("test2_illegal_name_value for GLM " + self.family)
h2o.cluster_info()
error_number = np.random.random_integers(0, 3, 1) # randomly choose an error
error_hyper_params = \
pyunit_utils.insert_error_grid_search(self.hyper_params, self.gridable_parameters, self.gridable_types,
error_number[0])
print("test2_illegal_name_value: the bad hyper-parameters are: ")
print(error_hyper_params)
# copied from Eric to catch execution run errors and not quit
try:
grid_model = H2OGridSearch(H2OGeneralizedLinearEstimator(family=self.family, nfolds=self.nfolds),
hyper_params=error_hyper_params)
grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data)
if error_number[0] > 2:
# grid search should not failed in this case and check number of models built.
if not (len(grid_model) == self.true_correct_model_number):
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed. Number of model generated is "
"incorrect.")
else:
print("test2_illegal_name_value passed.")
else: # other errors should cause exceptions being thrown and if not, something is wrong.
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed: exception should have been thrown for illegal"
"parameter name or empty hyper-parameter parameter list but did not!")
except:
if (error_number[0] <= 2) and (error_number[0] >= 0):
print("test2_illegal_name_value passed: exception is thrown for illegal parameter name or empty"
"hyper-parameter parameter list.")
else:
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed: exception should not have been thrown but did!")
self.test_num += 1
def test2_illegal_name_value(self):
"""
test2_illegal_name_value: test for condition 1 and 2. Randomly go into the hyper_parameters that we
have specified, either
a. randomly alter the name of a hyper-parameter name (fatal, exception will be thrown)
b. randomly choose a hyper-parameter and remove all elements in its list (fatal)
c. add randomly generated new hyper-parameter names with random list (fatal)
d: randomly choose a hyper-parameter and insert an illegal type into it (non fatal, model built with
legal hyper-parameters settings only and error messages printed out for illegal hyper-parameters
settings)
The following error conditions will be created depending on the error_number generated:
error_number = 0: randomly alter the name of a hyper-parameter name;
error_number = 1: randomly choose a hyper-parameter and remove all elements in its list
error_number = 2: add randomly generated new hyper-parameter names with random list
error_number = 3: randomly choose a hyper-parameter and insert an illegal type into it
:return: None
"""
print("*******************************************************************************************")
print("test2_illegal_name_value for GLM " + self.family)
h2o.cluster_info()
error_number = np.random.random_integers(0, 3, 1) # randomly choose an error
print("Test 2 error_number is {0}".format(error_number[0]))
error_hyper_params = \
pyunit_utils.insert_error_grid_search(self.hyper_params, self.gridable_parameters, self.gridable_types,
error_number[0])
print("test2_illegal_name_value: the bad hyper-parameters are: ")
print(error_hyper_params)
# copied from Eric to catch execution run errors and not quit
try:
grid_model = H2OGridSearch(H2OGeneralizedLinearEstimator(family=self.family, nfolds=self.nfolds),
hyper_params=error_hyper_params)
grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data)
if error_number[0] > 2:
# grid search should not failed in this case and check number of models built.
if not (len(grid_model) == self.true_correct_model_number):
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed. Number of model generated is "
"incorrect.")
else:
print("test2_illegal_name_value passed.")
else: # other errors should cause exceptions being thrown and if not, something is wrong.
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed: exception should have been thrown for illegal"
"parameter name or empty hyper-parameter parameter list but did not!")
except:
if (error_number[0] <= 2) and (error_number[0] >= 0):
print("test2_illegal_name_value passed: exception is thrown for illegal parameter name or empty"
"hyper-parameter parameter list.")
else:
self.test_failed += 1
self.test_failed_array[self.test_num] = 1
print("test2_illegal_name_value failed: exception should not have been thrown but did!")
self.test_num += 1