def Train(self):
st_global = time.time()
CommonUtils.create_update_and_save_progress_message(
self._dataframe_context,
self._scriptWeightDict,
self._scriptStages,
self._slug,
"initialization",
"info",
display=True,
emptyBin=False,
customMsg=None,
weightKey="total")
algosToRun = self._dataframe_context.get_algorithms_to_run()
algoSetting = [
x for x in algosToRun if x.get_algorithm_slug() == self._slug
][0]
categorical_columns = self._dataframe_helper.get_string_columns()
uid_col = self._dataframe_context.get_uid_column()
if self._metaParser.check_column_isin_ignored_suggestion(uid_col):
categorical_columns = list(set(categorical_columns) - {uid_col})
allDateCols = self._dataframe_context.get_date_columns()
categorical_columns = list(set(categorical_columns) - set(allDateCols))
print(categorical_columns)
numerical_columns = self._dataframe_helper.get_numeric_columns()
result_column = self._dataframe_context.get_result_column()
model_path = self._dataframe_context.get_model_path()
if model_path.startswith("file"):
model_path = model_path[7:]
validationDict = self._dataframe_context.get_validation_dict()
print("model_path", model_path)
pipeline_filepath = "file://" + str(model_path) + "/" + str(
self._slug) + "/pipeline/"
model_filepath = "file://" + str(model_path) + "/" + str(
self._slug) + "/model"
pmml_filepath = "file://" + str(model_path) + "/" + str(
self._slug) + "/modelPmml"
df = self._data_frame
if self._mlEnv == "spark":
pass
elif self._mlEnv == "sklearn":
model_filepath = model_path + "/" + self._slug + "/model.pkl"
pmml_filepath = str(model_path) + "/" + str(
self._slug) + "/traindeModel.pmml"
x_train, x_test, y_train, y_test = self._dataframe_helper.get_train_test_data(
)
x_train = MLUtils.create_dummy_columns(
x_train,
[x for x in categorical_columns if x != result_column])
x_test = MLUtils.create_dummy_columns(
x_test, [x for x in categorical_columns if x != result_column])
x_test = MLUtils.fill_missing_columns(x_test, x_train.columns,
result_column)
CommonUtils.create_update_and_save_progress_message(
self._dataframe_context,
self._scriptWeightDict,
self._scriptStages,
self._slug,
"training",
"info",
display=True,
emptyBin=False,
customMsg=None,
weightKey="total")
st = time.time()
levels = df[result_column].unique()
clf = SVC(kernel='linear', probability=True)
labelEncoder = preprocessing.LabelEncoder()
labelEncoder.fit(np.concatenate([y_train, y_test]))
y_train = pd.Series(labelEncoder.transform(y_train))
y_test = labelEncoder.transform(y_test)
classes = labelEncoder.classes_
transformed = labelEncoder.transform(classes)
labelMapping = dict(list(zip(transformed, classes)))
inverseLabelMapping = dict(list(zip(classes, transformed)))
posLabel = inverseLabelMapping[self._targetLevel]
appType = self._dataframe_context.get_app_type()
print(appType, labelMapping, inverseLabelMapping, posLabel,
self._targetLevel)
if algoSetting.is_hyperparameter_tuning_enabled():
hyperParamInitParam = algoSetting.get_hyperparameter_params()
evaluationMetricDict = {
"name": hyperParamInitParam["evaluationMetric"]
}
evaluationMetricDict[
"displayName"] = GLOBALSETTINGS.SKLEARN_EVAL_METRIC_NAME_DISPLAY_MAP[
evaluationMetricDict["name"]]
hyperParamAlgoName = algoSetting.get_hyperparameter_algo_name()
params_grid = algoSetting.get_params_dict_hyperparameter()
params_grid = {
k: v
for k, v in list(params_grid.items())
if k in clf.get_params()
}
print(params_grid)
if hyperParamAlgoName == "gridsearchcv":
clfGrid = GridSearchCV(clf, params_grid)
gridParams = clfGrid.get_params()
hyperParamInitParam = {
k: v
for k, v in list(hyperParamInitParam.items())
if k in gridParams
}
clfGrid.set_params(**hyperParamInitParam)
#clfGrid.fit(x_train,y_train)
grid_param = {}
grid_param['params'] = ParameterGrid(params_grid)
#bestEstimator = clfGrid.best_estimator_
modelFilepath = "/".join(model_filepath.split("/")[:-1])
sklearnHyperParameterResultObj = SklearnGridSearchResult(
grid_param, clf, x_train, x_test, y_train, y_test,
appType, modelFilepath, levels, posLabel,
evaluationMetricDict)
resultArray = sklearnHyperParameterResultObj.train_and_save_models(
)
self._result_setter.set_hyper_parameter_results(
self._slug, resultArray)
self._result_setter.set_metadata_parallel_coordinates(
self._slug, {
"ignoreList":
sklearnHyperParameterResultObj.get_ignore_list(),
"hideColumns":
sklearnHyperParameterResultObj.get_hide_columns(),
"metricColName":
sklearnHyperParameterResultObj.
get_comparison_metric_colname(),
"columnOrder":
sklearnHyperParameterResultObj.get_keep_columns()
})
elif hyperParamAlgoName == "randomsearchcv":
clfRand = RandomizedSearchCV(clf, params_grid)
clfRand.set_params(**hyperParamInitParam)
bestEstimator = None
else:
evaluationMetricDict = {
"name":
GLOBALSETTINGS.CLASSIFICATION_MODEL_EVALUATION_METRIC
}
evaluationMetricDict[
"displayName"] = GLOBALSETTINGS.SKLEARN_EVAL_METRIC_NAME_DISPLAY_MAP[
evaluationMetricDict["name"]]
self._result_setter.set_hyper_parameter_results(
self._slug, None)
algoParams = algoSetting.get_params_dict()
algoParams = {
k: v
for k, v in list(algoParams.items())
if k in list(clf.get_params().keys())
}
clf.set_params(**algoParams)
print("!" * 50)
print(clf.get_params())
print("!" * 50)
if validationDict["name"] == "kFold":
defaultSplit = GLOBALSETTINGS.DEFAULT_VALIDATION_OBJECT[
"value"]
numFold = int(validationDict["value"])
if numFold == 0:
numFold = 3
kFoldClass = SkleanrKFoldResult(
numFold,
clf,
x_train,
x_test,
y_train,
y_test,
appType,
levels,
posLabel,
evaluationMetricDict=evaluationMetricDict)
kFoldClass.train_and_save_result()
kFoldOutput = kFoldClass.get_kfold_result()
bestEstimator = kFoldClass.get_best_estimator()
elif validationDict["name"] == "trainAndtest":
clf.fit(x_train, y_train)
bestEstimator = clf
# clf.fit(x_train, y_train)
# bestEstimator = clf
trainingTime = time.time() - st
y_score = bestEstimator.predict(x_test)
try:
y_prob = bestEstimator.predict_proba(x_test)
except:
y_prob = [0] * len(y_score)
# overall_precision_recall = MLUtils.calculate_overall_precision_recall(y_test,y_score,targetLevel = self._targetLevel)
# print overall_precision_recall
accuracy = metrics.accuracy_score(y_test, y_score)
if len(levels) <= 2:
precision = metrics.precision_score(y_test,
y_score,
pos_label=posLabel,
average="binary")
recall = metrics.recall_score(y_test,
y_score,
pos_label=posLabel,
average="binary")
auc = metrics.roc_auc_score(y_test, y_score)
elif len(levels) > 2:
precision = metrics.precision_score(y_test,
y_score,
pos_label=posLabel,
average="macro")
recall = metrics.recall_score(y_test,
y_score,
pos_label=posLabel,
average="macro")
# auc = metrics.roc_auc_score(y_test,y_score,average="weighted")
auc = None
y_score = labelEncoder.inverse_transform(y_score)
y_test = labelEncoder.inverse_transform(y_test)
featureImportance = {}
feature_importance = dict(
sorted(zip(x_train.columns,
bestEstimator.feature_importances_),
key=lambda x: x[1],
reverse=True))
for k, v in feature_importance.items():
feature_importance[k] = CommonUtils.round_sig(v)
objs = {
"trained_model": bestEstimator,
"actual": y_test,
"predicted": y_score,
"probability": y_prob,
"feature_importance": feature_importance,
"featureList": list(x_train.columns),
"labelMapping": labelMapping
}
if not algoSetting.is_hyperparameter_tuning_enabled():
modelName = "M" + "0" * (GLOBALSETTINGS.MODEL_NAME_MAX_LENGTH -
1) + "1"
modelFilepathArr = model_filepath.split("/")[:-1]
modelFilepathArr.append(modelName + ".pkl")
joblib.dump(objs["trained_model"], "/".join(modelFilepathArr))
runtime = round((time.time() - st_global), 2)
try:
modelPmmlPipeline = PMMLPipeline([("pretrained-estimator",
objs["trained_model"])])
modelPmmlPipeline.target_field = result_column
modelPmmlPipeline.active_fields = np.array(
[col for col in x_train.columns if col != result_column])
sklearn2pmml(modelPmmlPipeline, pmml_filepath, with_repr=True)
pmmlfile = open(pmml_filepath, "r")
pmmlText = pmmlfile.read()
pmmlfile.close()
self._result_setter.update_pmml_object({self._slug: pmmlText})
except:
pass
cat_cols = list(set(categorical_columns) - {result_column})
overall_precision_recall = MLUtils.calculate_overall_precision_recall(
objs["actual"], objs["predicted"], targetLevel=self._targetLevel)
self._model_summary = MLModelSummary()
self._model_summary.set_algorithm_name("Svm")
self._model_summary.set_algorithm_display_name(
"Support Vector Machine")
self._model_summary.set_slug(self._slug)
self._model_summary.set_training_time(runtime)
self._model_summary.set_confusion_matrix(
MLUtils.calculate_confusion_matrix(objs["actual"],
objs["predicted"]))
self._model_summary.set_feature_importance(objs["feature_importance"])
self._model_summary.set_feature_list(objs["featureList"])
self._model_summary.set_model_accuracy(
round(metrics.accuracy_score(objs["actual"], objs["predicted"]),
2))
self._model_summary.set_training_time(round((time.time() - st), 2))
self._model_summary.set_precision_recall_stats(
overall_precision_recall["classwise_stats"])
self._model_summary.set_model_precision(
overall_precision_recall["precision"])
self._model_summary.set_model_recall(
overall_precision_recall["recall"])
self._model_summary.set_target_variable(result_column)
self._model_summary.set_prediction_split(
overall_precision_recall["prediction_split"])
self._model_summary.set_validation_method("Train and Test")
self._model_summary.set_level_map_dict(objs["labelMapping"])
# self._model_summary.set_model_features(list(set(x_train.columns)-set([result_column])))
self._model_summary.set_model_features(
[col for col in x_train.columns if col != result_column])
self._model_summary.set_level_counts(
self._metaParser.get_unique_level_dict(
list(set(categorical_columns))))
self._model_summary.set_num_trees(100)
self._model_summary.set_num_rules(300)
if not algoSetting.is_hyperparameter_tuning_enabled():
modelDropDownObj = {
"name": self._model_summary.get_algorithm_name(),
"evaluationMetricValue":
self._model_summary.get_model_accuracy(),
"evaluationMetricName": "accuracy",
"slug": self._model_summary.get_slug(),
"Model Id": modelName
}
modelSummaryJson = {
"dropdown": modelDropDownObj,
"levelcount": self._model_summary.get_level_counts(),
"modelFeatureList": self._model_summary.get_feature_list(),
"levelMapping": self._model_summary.get_level_map_dict(),
"slug": self._model_summary.get_slug(),
"name": self._model_summary.get_algorithm_name()
}
else:
modelDropDownObj = {
"name": self._model_summary.get_algorithm_name(),
"evaluationMetricValue": resultArray[0]["Accuracy"],
"evaluationMetricName": "accuracy",
"slug": self._model_summary.get_slug(),
"Model Id": resultArray[0]["Model Id"]
}
modelSummaryJson = {
"dropdown": modelDropDownObj,
"levelcount": self._model_summary.get_level_counts(),
"modelFeatureList": self._model_summary.get_feature_list(),
"levelMapping": self._model_summary.get_level_map_dict(),
"slug": self._model_summary.get_slug(),
"name": self._model_summary.get_algorithm_name()
}
svmCards = [
json.loads(CommonUtils.convert_python_object_to_json(cardObj)) for
cardObj in MLUtils.create_model_summary_cards(self._model_summary)
]
for card in svmCards:
self._prediction_narrative.add_a_card(card)
self._result_setter.set_model_summary({
"svm":
json.loads(
CommonUtils.convert_python_object_to_json(self._model_summary))
})
self._result_setter.set_svm_model_summary(modelSummaryJson)
self._result_setter.set_rf_cards(svmCards)
CommonUtils.create_update_and_save_progress_message(
self._dataframe_context,
self._scriptWeightDict,
self._scriptStages,
self._slug,
"completion",
"info",
display=True,
emptyBin=False,
customMsg=None,
weightKey="total")
def Train(self):
st = time.time()
categorical_columns = self._dataframe_helper.get_string_columns()
numerical_columns = self._dataframe_helper.get_numeric_columns()
result_column = self._dataframe_context.get_result_column()
categorical_columns = [
x for x in categorical_columns if x != result_column
]
model_path = self._dataframe_context.get_model_path()
pipeline_filepath = model_path + "/LogisticRegression/TrainedModels/pipeline"
model_filepath = model_path + "/LogisticRegression/TrainedModels/model"
summary_filepath = model_path + "/LogisticRegression/ModelSummary/summary.json"
df = self._data_frame
pipeline = MLUtils.create_pyspark_ml_pipeline(numerical_columns,
categorical_columns,
result_column)
pipelineModel = pipeline.fit(df)
indexed = pipelineModel.transform(df)
MLUtils.save_pipeline_or_model(pipelineModel, pipeline_filepath)
trainingData, validationData = MLUtils.get_training_and_validation_data(
indexed, result_column, 0.8)
OriginalTargetconverter = IndexToString(
inputCol="label", outputCol="originalTargetColumn")
levels = trainingData.select("label").distinct().collect()
if self._classifier == "lr":
if len(levels) == 2:
lr = LogisticRegression(maxIter=10,
regParam=0.3,
elasticNetParam=0.8)
elif len(levels) > 2:
lr = LogisticRegression(maxIter=10,
regParam=0.3,
elasticNetParam=0.8,
family="multinomial")
fit = lr.fit(trainingData)
elif self._classifier == "OneVsRest":
lr = LogisticRegression()
ovr = OneVsRest(classifier=lr)
fit = ovr.fit(trainingData)
transformed = fit.transform(validationData)
MLUtils.save_pipeline_or_model(fit, model_filepath)
print fit.coefficientMatrix
print fit.interceptVector
# feature_importance = MLUtils.calculate_sparkml_feature_importance(indexed,fit,categorical_columns,numerical_columns)
label_classes = transformed.select("label").distinct().collect()
results = transformed.select(["prediction", "label"])
if len(label_classes) > 2:
evaluator = MulticlassClassificationEvaluator(
predictionCol="prediction")
evaluator.evaluate(results)
self._model_summary["model_accuracy"] = evaluator.evaluate(
results,
{evaluator.metricName: "accuracy"}) # accuracy of the model
else:
evaluator = BinaryClassificationEvaluator(
rawPredictionCol="prediction")
evaluator.evaluate(results)
# print evaluator.evaluate(results,{evaluator.metricName: "areaUnderROC"})
# print evaluator.evaluate(results,{evaluator.metricName: "areaUnderPR"})
self._model_summary["model_accuracy"] = evaluator.evaluate(
results,
{evaluator.metricName: "areaUnderPR"}) # accuracy of the model
# self._model_summary["feature_importance"] = MLUtils.transform_feature_importance(feature_importance)
self._model_summary["runtime_in_seconds"] = round((time.time() - st),
2)
transformed = OriginalTargetconverter.transform(transformed)
label_indexer_dict = [
dict(enumerate(field.metadata["ml_attr"]["vals"]))
for field in transformed.schema.fields if field.name == "label"
][0]
prediction_to_levels = udf(lambda x: label_indexer_dict[x],
StringType())
transformed = transformed.withColumn(
"predictedClass", prediction_to_levels(transformed.prediction))
prediction_df = transformed.select(
["originalTargetColumn", "predictedClass"]).toPandas()
objs = {
"actual": prediction_df["originalTargetColumn"],
"predicted": prediction_df["predictedClass"]
}
self._model_summary[
"confusion_matrix"] = MLUtils.calculate_confusion_matrix(
objs["actual"], objs["predicted"])
overall_precision_recall = MLUtils.calculate_overall_precision_recall(
objs["actual"], objs["predicted"])
self._model_summary[
"precision_recall_stats"] = overall_precision_recall[
"classwise_stats"]
self._model_summary["model_precision"] = overall_precision_recall[
"precision"]
self._model_summary["model_recall"] = overall_precision_recall[
"recall"]
self._model_summary["target_variable"] = result_column
self._model_summary[
"test_sample_prediction"] = overall_precision_recall[
"prediction_split"]
self._model_summary["algorithm_name"] = "Random Forest"
self._model_summary["validation_method"] = "Train and Test"
self._model_summary["independent_variables"] = len(
categorical_columns) + len(numerical_columns)
self._model_summary["level_counts"] = CommonUtils.get_level_count_dict(
trainingData,
categorical_columns,
self._dataframe_context.get_column_separator(),
dataType="spark")
# print json.dumps(self._model_summary,indent=2)
self._model_summary["total_trees"] = 100
self._model_summary["total_rules"] = 300
CommonUtils.write_to_file(
summary_filepath, json.dumps({"modelSummary":
self._model_summary}))