def setup_data(self):
"""
This function performs all initializations necessary:
1. generates all the random values for our dynamic tests like the Gaussian
noise std, column count and row count for training/test data sets.
2. generate the appropriate data sets.
"""
# clean out the sandbox directory first
self.sandbox_dir = pyunit_utils.make_Rsandbox_dir(self.current_dir, self.test_name, True)
# randomly set Gaussian noise standard deviation as a fraction of actual predictor standard deviation
self.noise_std = random.uniform(0, math.sqrt(pow((self.max_p_value - self.min_p_value), 2) / 12))
self.noise_var = self.noise_std*self.noise_std
# randomly determine data set size in terms of column and row counts
self.train_col_count = random.randint(1, self.max_col_count)
self.train_row_count = round(self.train_col_count * random.uniform(self.min_col_count_ratio,
self.max_col_count_ratio))
# DEBUGGING setup_data, remember to comment them out once done.
# self.train_col_count = 3
# self.train_row_count = 200
# self.max_real_number = 1
# self.max_int_number = 1
# end DEBUGGING
if 'gaussian' in self.family: # increase data range
self.max_p_value *= 50
self.min_p_value *= 50
self.max_w_value *= 50
self.min_w_value *= 50
# generate real value weight vector and training/validation/test data sets for GLM
pyunit_utils.write_syn_floating_point_dataset_glm(self.training1_data_file, "",
"", self.weight_data_file,
self.train_row_count, self.train_col_count, self.data_type,
self.max_p_value, self.min_p_value, self.max_w_value,
self.min_w_value, self.noise_std, self.family,
self.train_row_count, self.train_row_count,
class_number=self.class_number,
class_method=[self.class_method, self.class_method,
self.test_class_method],
class_margin=[self.margin, self.margin,
self.test_class_margin])
# preload data sets
self.training1_data = h2o.import_file(pyunit_utils.locate(self.training1_data_file))
# set data set indices for predictors and response
self.y_index = self.training1_data.ncol-1
self.x_indices = list(range(self.y_index))
# set response to be categorical for classification tasks
if 'binomial' in self.family:
self.training1_data[self.y_index] = self.training1_data[self.y_index].round().asfactor()
# check to make sure all response classes are represented, otherwise, quit
if self.training1_data[self.y_index].nlevels()[0] < self.class_number:
print("Response classes are not represented in training dataset.")
sys.exit(0)
# save the training data files just in case the code crashed.
pyunit_utils.remove_csv_files(self.current_dir, ".csv", action='copy', new_dir_path=self.sandbox_dir)
def setup_data(self):
"""
This function performs all initializations necessary:
1. generates all the random parameter values for our dynamic tests like the Gaussian
noise std, column count and row count for training/test data sets.
2. randomly choose the distribution family (gaussian, binomial, multinomial)
to test.
3. with the chosen distribution family, generate the appropriate data sets
4. load the data sets and set the training set indices and response column index
"""
# create and clean out the sandbox directory first
self.sandbox_dir = pyunit_utils.make_Rsandbox_dir(self.current_dir, self.test_name, True)
# randomly set Gaussian noise standard deviation as a fraction of actual predictor standard deviation
self.noise_std = random.uniform(0, math.sqrt(pow((self.max_p_value - self.min_p_value), 2) / 12))
self.noise_var = self.noise_std*self.noise_std
# randomly determine data set size in terms of column and row counts
self.train_col_count = random.randint(1, self.max_col_count)
self.train_row_count = round(self.train_col_count * random.uniform(self.min_col_count_ratio,
self.max_col_count_ratio))
# DEBUGGING setup_data, remember to comment them out once done.
self.train_col_count = 3
self.train_row_count = 200
# self.max_real_number = 1
# self.max_int_number = 1
# end DEBUGGING
#### This is used to generate dataset for regression or classification. Nothing to do
#### with setting the distribution family in this case
# randomly choose which family of GLM algo to use
self.family = self.families[random.randint(0, len(self.families)-1)]
# set class number for classification
if 'multinomial' in self.family:
self.class_number = random.randint(2, self.max_class_number) # randomly set number of classes K
# generate real value weight vector and training/validation/test data sets for GLM
pyunit_utils.write_syn_floating_point_dataset_glm(self.training1_data_file, self.training2_data_file,
self.training3_data_file, self.weight_data_file,
self.train_row_count, self.train_col_count, 2,
self.max_p_value, self.min_p_value, self.max_w_value,
self.min_w_value, self.noise_std, self.family,
self.train_row_count, self.train_row_count,
class_number=self.class_number,
class_method=['probability', 'probability',
'probability'])
# preload data sets
self.training1_data = h2o.import_file(pyunit_utils.locate(self.training1_data_file))
self.training2_data = h2o.import_file(pyunit_utils.locate(self.training2_data_file))
self.training3_data = h2o.import_file(pyunit_utils.locate(self.training3_data_file))
# set data set indices for predictors and response
self.y_index = self.training1_data.ncol-1
self.x_indices = list(range(self.y_index))
# set response to be categorical for classification tasks
if 'multinomial' in self.family:
self.training1_data[self.y_index] = self.training1_data[self.y_index].round().asfactor()
# check to make sure all response classes are represented, otherwise, quit
if self.training1_data[self.y_index].nlevels()[0] < self.class_number:
print("Response classes are not represented in training dataset.")
sys.exit(0)
self.training2_data[self.y_index] = self.training2_data[self.y_index].round().asfactor()
self.training3_data[self.y_index] = self.training2_data[self.y_index].round().asfactor()
# self.hyper_params["validation_frame"] = [self.training1_data.frame_id, self.training2_data.frame_id,
# self.training3_data.frame_id]
# save the training data files just in case the code crashed.
pyunit_utils.remove_csv_files(self.current_dir, ".csv", action='copy', new_dir_path=self.sandbox_dir)
def setup_data(self):
"""
This function performs all initializations necessary:
1. generates all the random parameter values for our dynamic tests like the Gaussian
noise std, column count and row count for training/test data sets.
2. randomly choose the distribution family (gaussian, binomial, multinomial)
to test.
3. with the chosen distribution family, generate the appropriate data sets
4. load the data sets and set the training set indices and response column index
"""
# create and clean out the sandbox directory first
self.sandbox_dir = pyunit_utils.make_Rsandbox_dir(
self.current_dir, self.test_name, True)
# randomly set Gaussian noise standard deviation as a fraction of actual predictor standard deviation
self.noise_std = random.uniform(
0, math.sqrt(pow((self.max_p_value - self.min_p_value), 2) / 12))
self.noise_var = self.noise_std * self.noise_std
# randomly determine data set size in terms of column and row counts
self.train_col_count = random.randint(1, self.max_col_count)
self.train_row_count = round(
self.train_col_count *
random.uniform(self.min_col_count_ratio, self.max_col_count_ratio))
# DEBUGGING setup_data, remember to comment them out once done.
self.train_col_count = 3
self.train_row_count = 200
# self.max_real_number = 1
# self.max_int_number = 1
# end DEBUGGING
#### This is used to generate dataset for regression or classification. Nothing to do
#### with setting the distribution family in this case
# randomly choose which family of GLM algo to use
self.family = self.families[random.randint(0, len(self.families) - 1)]
# set class number for classification
if 'multinomial' in self.family:
self.class_number = random.randint(
2, self.max_class_number) # randomly set number of classes K
# generate real value weight vector and training/validation/test data sets for GLM
pyunit_utils.write_syn_floating_point_dataset_glm(
self.training1_data_file,
self.training2_data_file,
self.training3_data_file,
self.weight_data_file,
self.train_row_count,
self.train_col_count,
2,
self.max_p_value,
self.min_p_value,
self.max_w_value,
self.min_w_value,
self.noise_std,
self.family,
self.train_row_count,
self.train_row_count,
class_number=self.class_number,
class_method=['probability', 'probability', 'probability'])
# preload data sets
self.training1_data = h2o.import_file(
pyunit_utils.locate(self.training1_data_file))
self.training2_data = h2o.import_file(
pyunit_utils.locate(self.training2_data_file))
self.training3_data = h2o.import_file(
pyunit_utils.locate(self.training3_data_file))
# set data set indices for predictors and response
self.y_index = self.training1_data.ncol - 1
self.x_indices = list(range(self.y_index))
# set response to be categorical for classification tasks
if 'multinomial' in self.family:
self.training1_data[self.y_index] = self.training1_data[
self.y_index].round().asfactor()
# check to make sure all response classes are represented, otherwise, quit
if self.training1_data[
self.y_index].nlevels()[0] < self.class_number:
print(
"Response classes are not represented in training dataset."
)
sys.exit(0)
self.training2_data[self.y_index] = self.training2_data[
self.y_index].round().asfactor()
self.training3_data[self.y_index] = self.training2_data[
self.y_index].round().asfactor()
# self.hyper_params["validation_frame"] = [self.training1_data.frame_id, self.training2_data.frame_id,
# self.training3_data.frame_id]
# save the training data files just in case the code crashed.
pyunit_utils.remove_csv_files(self.current_dir,
".csv",
action='copy',
new_dir_path=self.sandbox_dir)
def setup_data(self):
"""
This function performs all initializations necessary:
1. generates all the random values for our dynamic tests like the Gaussian
noise std, column count and row count for training/test data sets.
2. generate the appropriate data sets.
"""
# clean out the sandbox directory first
self.sandbox_dir = pyunit_utils.make_Rsandbox_dir(
self.current_dir, self.test_name, True)
# randomly set Gaussian noise standard deviation as a fraction of actual predictor standard deviation
self.noise_std = random.uniform(
0, math.sqrt(pow((self.max_p_value - self.min_p_value), 2) / 12))
self.noise_var = self.noise_std * self.noise_std
# randomly determine data set size in terms of column and row counts
self.train_col_count = random.randint(1, self.max_col_count)
self.train_row_count = round(
self.train_col_count *
random.uniform(self.min_col_count_ratio, self.max_col_count_ratio))
# DEBUGGING setup_data, remember to comment them out once done.
# self.train_col_count = 3
# self.train_row_count = 200
# self.max_real_number = 5
# self.max_int_number = 5
# end DEBUGGING
if 'gaussian' in self.family: # increase data range
self.max_p_value *= 50
self.min_p_value *= 50
self.max_w_value *= 50
self.min_w_value *= 50
# generate real value weight vector and training/validation/test data sets for GLM
pyunit_utils.write_syn_floating_point_dataset_glm(
self.training1_data_file,
"",
"",
self.weight_data_file,
self.train_row_count,
self.train_col_count,
self.data_type,
self.max_p_value,
self.min_p_value,
self.max_w_value,
self.min_w_value,
self.noise_std,
self.family,
self.train_row_count,
self.train_row_count,
class_number=self.class_number,
class_method=[
self.class_method, self.class_method, self.test_class_method
],
class_margin=[self.margin, self.margin, self.test_class_margin])
# preload data sets
self.training1_data = h2o.import_file(
pyunit_utils.locate(self.training1_data_file))
# set data set indices for predictors and response
self.y_index = self.training1_data.ncol - 1
self.x_indices = list(range(self.y_index))
# set response to be categorical for classification tasks
if ('binomial' in self.family):
self.training1_data[self.y_index] = self.training1_data[
self.y_index].round().asfactor()
# check to make sure all response classes are represented, otherwise, quit
if self.training1_data[
self.y_index].nlevels()[0] < self.class_number:
print(
"Response classes are not represented in training dataset."
)
sys.exit(0)
# save the training data files just in case the code crashed.
pyunit_utils.remove_csv_files(self.current_dir,
".csv",
action='copy',
new_dir_path=self.sandbox_dir)