def do_algo(self, input):
control_params = input.algo_control.control_params
if not self.check_input_params(self.get_input_params_definition(),
control_params):
log.error("Check input params type error.")
return None
mode = input.algo_control.mode
data = input.algo_data.data
if mode == 'training':
try:
model = cluster.KMeans(n_clusters=control_params["n_clusters"],
n_init=control_params["n_init"],
max_iter=control_params["max_iter"])
model.fit(data)
algo_output = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': ''
},
algo_data={
'data': data,
'label': None
},
algo_model={
'model_params':
model.get_params(),
'model_instance': model
})
except Exception as e:
log.error(str(e))
algo_output = None
else:
algo_output = None
return algo_output
def do_algo(self, input_params):
control_params = input_params.algo_control.control_params
if not self.check_input_params(self.get_input_params_definition(),
control_params):
log.error("Check input params type error.")
return None
data = input_params.algo_data.data
label = input_params.algo_data.label
mode = input_params.algo_control.mode
if mode == 'training':
try:
model = tree.DecisionTreeClassifier(
max_depth=control_params["max_depth"])
model.fit(X=data, y=label)
algo_output = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': ''
},
algo_data={
'data': data,
'label': label
},
algo_model={
'model_params':
model.get_params(),
'model_instance': model
})
except Exception as e:
log.error(str(e))
algo_output = None
else:
algo_output = None
return algo_output
def main():
arg_dict = {"n_neighbors": 5, "weights": "uniform"}
data = load_iris()
iris_data = pd.DataFrame(data.data, columns=data.feature_names)
iris_label = data.target
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': iris_data,
'label': iris_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('knn')
algo_output = in_algo.do_algo(algo_input)
model = algo_output.algo_model.model_instance
predict_result = model.predict(iris_data)
x_axis_name = data.feature_names[0]
y_axis_name = data.feature_names[1]
knn_model_preview(iris_data, predict_result, x_axis_name, y_axis_name)
def test_correct_knn_parameter_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {"n_neighbors": 5, "weights": "uniform"}
algo_name = 'knn'
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': self.iris_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory(algo_name)
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is True)
self.assertEqual(Algorithm.get_project_type(algo_name),
"classification")
def test_error_dl_nn_classifier_parameter_neuron_1_string_type(self):
# given: error input parameter "layer_1_neuron" needs to be type(int)
arg_dict = {
"layer_1_neuron": "relu",
"layer_1_activation": "relu",
"layer_2_neuron": 16,
"layer_2_activation": "tanh",
"layer_3_neuron": 10,
"layer_3_activation": "softmax",
"optimizer": "Adam",
"loss": "mean_squared_error",
"epochs": 2,
"batch_size": 32,
}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.boston_data,
'label': self.boston_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('DL_NN_Classifier_r07525035')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def do_algo(self, input):
control_params = input.algo_control.control_params
if not self.check_input_params(self.get_input_params_definition(),
control_params):
log.error("Check input params type error.")
return None
mode = input.algo_control.mode
data = input.algo_data.data
label = input.algo_data.label
if mode == 'training':
try:
model = neighbors.KNeighborsClassifier(
n_neighbors=control_params["n_neighbors"],
weights=control_params["weights"])
model.fit(X=data, y=label)
algo_output = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': ''
},
algo_data={
'data': data,
'label': label
},
algo_model={
'model_params':
model.get_params(),
'model_instance': model
})
except Exception as e:
log.error(str(e))
algo_output = None
else:
algo_output = None
return algo_output
def test_correct_dl_nn_classifier_do_algo_mnist_dataset(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"layer_1_neuron": 32,
"layer_1_activation": "relu",
"layer_2_neuron": 16,
"layer_2_activation": "tanh",
"layer_3_neuron": 10,
"layer_3_activation": "softmax",
"optimizer": "RMSprop",
"loss": "categorical_crossentropy",
"epochs": 3,
"batch_size": 32
}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.X_train,
'label': self.y_train
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('DL_NN_Classifier_r07525035')
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is not None)
self.assertTrue(algo_output.algo_model.model_instance is not None)
def test_correct_one_class_svm_parameter_gamma_float_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {"gamma": 0.1, "nu": 0.5, "kernel": "rbf", "degree": 3}
algo_name = 'one-class_SVM'
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': None
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory(algo_name)
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is True)
self.assertEqual(Algorithm.get_project_type(algo_name),
"abnormal-detection")
def test_correct_linear_regression_parameter_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"fit_intercept": True,
"normalize": False,
"copy_X": True,
"n_jobs": 1
}
algo_name = 'linear-regression'
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.boston_data,
'label': self.boston_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory(algo_name)
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is True)
self.assertEqual(Algorithm.get_project_type(algo_name), "regression")
def test_correct_linear_regression_do_algo(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"fit_intercept": True,
"normalize": False,
"copy_X": True,
"n_jobs": 1
}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.boston_data,
'label': self.boston_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('linear-regression')
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is not None)
self.assertTrue(algo_output.algo_model.model_instance is not None)
def test_error_linear_regression_parameter_n_jobs_float_type(self):
# given: error input parameter "n_jobs" needs to be type(int)
arg_dict = {
"fit_intercept": True,
"normalize": False,
"copy_X": True,
"n_jobs": 1.0
}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.boston_data,
'label': self.boston_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('linear-regression')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def do_algo(self, input):
control_params = input.algo_control.control_params
if not self.check_input_params(self.get_input_params_definition(),
control_params):
log.error("Check input params type error.")
return None
mode = input.algo_control.mode
data = input.algo_data.data
if mode == 'training':
try:
model = svm.OneClassSVM(nu=control_params["nu"],
kernel=control_params["kernel"],
gamma=control_params["gamma"],
degree=control_params["degree"])
model.fit(data)
algo_output = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': ''
},
algo_data={
'data': data,
'label': None
},
algo_model={
'model_params':
model.get_params(),
'model_instance': model
})
except Exception as e:
log.error(str(e))
algo_output = None
else:
algo_output = None
return algo_output
def test_error_decision_tree_parameter_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {"max_depth": 10.0}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': self.iris_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('decision-tree')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
# given: collect input parameter, create algorithm object
arg_dict = {"max_depth": 'a_string'}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': self.iris_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('decision-tree')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def test_error_kmeans_parameter_n_init_string_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"n_clusters": 8,
"n_init": "string",
"max_iter": 300
}
algo_input = alc.AlgoParam(algo_control={'mode': 'training', 'control_params': arg_dict},
algo_data={'data': self.iris_data, 'label': None},
algo_model={'model_params': None, 'model_instance': None})
in_algo = AlgoUtils.algo_factory('k-means')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def test_error_ap_damping_out_of_range(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"damping": 1.5,
"preference": "-50",
"convergence_iter": 15,
"max_iter": 200
}
algo_name = 'affinity_propagation'
algo_input = alc.AlgoParam(algo_control={'mode': 'training', 'control_params': arg_dict},
algo_data={'data': self.iris_data, 'label': None},
algo_model={'model_params': None, 'model_instance': None})
in_algo = AlgoUtils.algo_factory(algo_name)
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def model_predict(arg_dict, data_train, label_train, data_test, label_test,
class_names, fname):
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': data_train,
'label': label_train
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('DL_NN_Classifier_r07525035')
algo_output = in_algo.do_algo(algo_input)
model = algo_output.algo_model.model_instance
predict_result = model.predict(data_test)
clsf_report = classification_report(label_test, predict_result)
print("Classification report for classifier %s:\n%s\n" %
(model, clsf_report))
classification_report_csv(clsf_report, fname)
# Compute confusion matrix
cnf_matrix = confusion_matrix(label_test, predict_result)
np.set_printoptions(precision=2, suppress=True)
# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,
classes=class_names,
fname=fname,
title='Confusion matrix, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,
classes=class_names,
normalize=True,
fname=fname,
title='Normalized confusion matrix')
plt.show()
def test_correct_kmeans_do_algo(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"n_clusters": 8,
"n_init": 10,
"max_iter": 300
}
algo_input = alc.AlgoParam(algo_control={'mode': 'training', 'control_params': arg_dict},
algo_data={'data': self.iris_data, 'label': None},
algo_model={'model_params': None, 'model_instance': None})
in_algo = AlgoUtils.algo_factory('k-means')
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is not None)
self.assertTrue(algo_output.algo_model.model_instance is not None)
def test_correct_affinitypropagation_do_algo(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"damping": 0.5,
"preference": "-50",
"convergence_iter": 15,
"max_iter": 200
}
algo_name = 'affinity_propagation'
algo_input = alc.AlgoParam(algo_control={'mode': 'training', 'control_params': arg_dict},
algo_data={'data': self.iris_data, 'label': None},
algo_model={'model_params': None, 'model_instance': None})
in_algo = AlgoUtils.algo_factory(algo_name)
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is not None)
self.assertTrue(algo_output.algo_model.model_instance is not None)
def do_algo(self, input):
control_params = input.algo_control.control_params
if not self.check_input_params(self.get_input_params_definition(),
control_params):
log.error("Check input params type error.")
return None
mode = input.algo_control.mode
data = input.algo_data.data
label = input.algo_data.label
if mode == 'training':
try:
regr = linear_model.LinearRegression(
fit_intercept=control_params["fit_intercept"],
normalize=control_params["normalize"],
copy_X=control_params["copy_X"],
n_jobs=control_params["n_jobs"])
degree = control_params["degree"]
model = make_pipeline(PolynomialFeatures(degree), regr)
model.fit(X=data, y=label)
algo_output = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': ''
},
algo_data={
'data': data,
'label': label
},
algo_model={
'model_params':
model.get_params(),
'model_instance': model
})
except Exception as e:
log.error(str(e))
algo_output = None
else:
algo_output = None
return algo_output
def test_error_one_class_svm_parameter_gamma_string_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {"gamma": "string", "nu": 0.5, "kernel": "rbf", "degree": 3}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': None
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('one-class_SVM')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def test_error_knn_parameter_weights_int_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {"n_neighbors": 5, "weights": 1}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': self.iris_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('knn')
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is False)
def test_error_decision_tree_do_algo(self):
# given: collect input parameter, create algorithm object
arg_dict = {"max_depth": 10}
# wrong data input
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.boston_data,
'label': self.boston_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('decision-tree')
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is None)
def test_correct_one_class_svm_do_algo(self):
# given: collect input parameter, create algorithm object
arg_dict = {"gamma": 0.5, "nu": 0.5, "kernel": "rbf", "degree": 3}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': None
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('one-class_SVM')
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is not None)
self.assertTrue(algo_output.algo_model.model_instance is not None)
def test_correct_dl_nn_classifier_parameter_type(self):
# given: collect input parameter, create algorithm object
arg_dict = {
"layer_1_neuron": 32,
"layer_1_activation": "relu",
"layer_2_neuron": 16,
"layer_2_activation": "tanh",
"layer_3_neuron": 10,
"layer_3_activation": "softmax",
"optimizer": "RMSprop",
"loss": "categorical_crossentropy",
"epochs": 3,
"batch_size": 32
}
algo_name = 'DL_NN_Classifier_r07525035'
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.X_train,
'label': self.y_train
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory(algo_name)
input_params_definition = in_algo.get_input_params_definition()
# when: checkout input type
check_result = in_algo.check_input_params(
input_params_definition, algo_input.algo_control.control_params)
# then: type match
self.assertTrue(check_result is True)
self.assertEqual(Algorithm.get_project_type(algo_name),
"classification")
def test_correct_knn_do_algo(self):
# given: collect input parameter, create algorithm object
arg_dict = {"n_neighbors": 5, "weights": "uniform"}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.iris_data,
'label': self.iris_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('knn')
log.debug(algo_input)
# when: do decision tree algorithm
algo_output = in_algo.do_algo(algo_input)
# then:
self.assertTrue(algo_output is not None)
self.assertTrue(algo_output.algo_model.model_instance is not None)
def test_error_dl_nn_classifier_do_algo_wrong_dataset(self):
# given: error database input, create algorithm object
arg_dict = {
"layer_1_neuron": 32,
"layer_1_activation": "relu",
"layer_2_neuron": 16,
"layer_2_activation": "relu",
"layer_3_neuron": 10,
"layer_3_activation": "relu",
"optimizer": "Adam",
"loss": "mean_squared_error",
"epochs": 3,
"batch_size": 32
}
algo_input = alc.AlgoParam(algo_control={
'mode': 'training',
'control_params': arg_dict
},
algo_data={
'data': self.boston_data,
'label': self.boston_label
},
algo_model={
'model_params': None,
'model_instance': None
})
in_algo = AlgoUtils.algo_factory('DL_NN_Classifier_r07525035')
log.debug(algo_input)
# when: do decision tree algorithm
try:
algo_output = in_algo.do_algo(algo_input)
except:
algo_output = None
# then:
self.assertTrue(algo_output is None)