def test2_illegal_name_value(self): """ This function will make sure that only valid parameter names and values should be included in the hyper-parameter dict for grid search. It will randomly go into the hyper_parameters that we have specified, either change the hyper_parameter name or insert an illegal value into the hyper_parameter list, check to make sure that the test failed with error messages. The following error conditions will be created depending on the error_number generated: error_number = 0: randomly alter the name of a hyper-parameter name; error_number = 1: randomly choose a hyper-parameter and remove all elements in its list error_number = 2: add randomly generated new hyper-parameter names with random list error_number other: randomly choose a hyper-parameter and insert an illegal type into it :return: None """ print("*******************************************************************************************") print("test2_illegal_name_value for GLM " + self.family) h2o.cluster_info() error_number = np.random.random_integers(0, 3, 1) # randomly choose an error error_hyper_params = \ pyunit_utils.insert_error_grid_search(self.hyper_params, self.gridable_parameters, self.gridable_types, error_number[0]) print("test2_illegal_name_value: the bad hyper-parameters are: ") print(error_hyper_params) # copied from Eric to catch execution run errors and not quit try: grid_model = H2OGridSearch(H2OGeneralizedLinearEstimator(family=self.family, nfolds=self.nfolds), hyper_params=error_hyper_params) grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) if error_number[0] > 2: # grid search should not failed in this case and build same number of models as test1. if not (len(grid_model) == self.correct_model_number): self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed. Number of model generated is " "incorrect.") else: print("test2_illegal_name_value passed.") else: # other errors should cause exceptions being thrown and if not, something is wrong. self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed: exception should have been thrown for illegal" "parameter name or empty hyper-parameter parameter list but did not!") except: print("test2_illegal_name_value passed: exception is thrown for illegal parameter name or empty" "hyper-parameter parameter list.") self.test_num += 1
def test2_illegal_name_value(self): """ test2_illegal_name_value: test for condition 1 and 2. Randomly go into the hyper_parameters that we have specified, either a. randomly alter the name of a hyper-parameter name (fatal, exception will be thrown) b. randomly choose a hyper-parameter and remove all elements in its list (fatal) c. add randomly generated new hyper-parameter names with random list (fatal) d: randomly choose a hyper-parameter and insert an illegal type into it (non fatal, model built with legal hyper-parameters settings only and error messages printed out for illegal hyper-parameters settings) The following error conditions will be created depending on the error_number generated: error_number = 0: randomly alter the name of a hyper-parameter name; error_number = 1: randomly choose a hyper-parameter and remove all elements in its list error_number = 2: add randomly generated new hyper-parameter names with random list error_number = 3: randomly choose a hyper-parameter and insert an illegal type into it :return: None """ print("*******************************************************************************************") print("test2_illegal_name_value for GLM " + self.family) h2o.cluster_info() error_number = np.random.random_integers(0, 3, 1) # randomly choose an error error_hyper_params = \ pyunit_utils.insert_error_grid_search(self.hyper_params, self.gridable_parameters, self.gridable_types, error_number[0]) print("test2_illegal_name_value: the bad hyper-parameters are: ") print(error_hyper_params) # copied from Eric to catch execution run errors and not quit try: grid_model = H2OGridSearch(H2OGeneralizedLinearEstimator(family=self.family, nfolds=self.nfolds), hyper_params=error_hyper_params) grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) if error_number[0] > 2: # grid search should not failed in this case and check number of models built. if not (len(grid_model) == self.true_correct_model_number): self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed. Number of model generated is " "incorrect.") else: print("test2_illegal_name_value passed.") else: # other errors should cause exceptions being thrown and if not, something is wrong. self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed: exception should have been thrown for illegal" "parameter name or empty hyper-parameter parameter list but did not!") except: if (error_number[0] <= 2) and (error_number[0] >= 0): print("test2_illegal_name_value passed: exception is thrown for illegal parameter name or empty" "hyper-parameter parameter list.") else: self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed: exception should not have been thrown but did!") self.test_num += 1
def test2_illegal_name_value(self): """ test2_illegal_name_value: test for condition 1 and 2. Randomly go into the hyper_parameters that we have specified, either a. randomly alter the name of a hyper-parameter name (fatal, exception will be thrown) b. randomly choose a hyper-parameter and remove all elements in its list (fatal) c. add randomly generated new hyper-parameter names with random list (fatal) d: randomly choose a hyper-parameter and insert an illegal type into it (non fatal, model built with legal hyper-parameters settings only and error messages printed out for illegal hyper-parameters settings) The following error conditions will be created depending on the error_number generated: error_number = 0: randomly alter the name of a hyper-parameter name; error_number = 1: randomly choose a hyper-parameter and remove all elements in its list error_number = 2: add randomly generated new hyper-parameter names with random list error_number = 3: randomly choose a hyper-parameter and insert an illegal type into it :return: None """ print("*******************************************************************************************") print("test2_illegal_name_value for GLM " + self.family) h2o.cluster_info() error_number = np.random.random_integers(0, 3, 1) # randomly choose an error print("Test 2 error_number is {0}".format(error_number[0])) error_hyper_params = \ pyunit_utils.insert_error_grid_search(self.hyper_params, self.gridable_parameters, self.gridable_types, error_number[0]) print("test2_illegal_name_value: the bad hyper-parameters are: ") print(error_hyper_params) # copied from Eric to catch execution run errors and not quit try: grid_model = H2OGridSearch(H2OGeneralizedLinearEstimator(family=self.family, nfolds=self.nfolds), hyper_params=error_hyper_params) grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) if error_number[0] > 2: # grid search should not failed in this case and check number of models built. if not (len(grid_model) == self.true_correct_model_number): self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed. Number of model generated is " "incorrect.") else: print("test2_illegal_name_value passed.") else: # other errors should cause exceptions being thrown and if not, something is wrong. self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed: exception should have been thrown for illegal" "parameter name or empty hyper-parameter parameter list but did not!") except: if (error_number[0] <= 2) and (error_number[0] >= 0): print("test2_illegal_name_value passed: exception is thrown for illegal parameter name or empty" "hyper-parameter parameter list.") else: self.test_failed += 1 self.test_failed_array[self.test_num] = 1 print("test2_illegal_name_value failed: exception should not have been thrown but did!") self.test_num += 1