def train_model(self, train_model_params):
split_params = {}
try:
# update status in mongo
train_model_params["creationDate"] = str(datetime.utcnow())
train_model_params["status"] = "InProgress"
mp.ModelProcessor().save_model_mongo(train_model_params)
# Fetch Dataset
dataset = DCF.DatasetConnectorFactory().dataset(
train_model_params["dbconnect"])
# data preprocess & Data wrangling
model_dict, dataset, feature_list, target_list = dp.DataPreProcessor(
).data_preprocess(train_model_params, dataset)
# train test split
features_train, features_test, target_train, target_test = \
train_test_split(dataset[feature_list], dataset[target_list], train_size=train_model_params["trainTest"])
split_params["features_train"] = features_train
split_params["features_test"] = features_test
split_params["target_train"] = target_train
split_params["target_test"] = target_test
# train model
selected_algorithm = model_dict["algorithm"]["name"]
model_dict = AF.AlgorithmFactory().algorithm(
selected_algorithm).train(split_params, model_dict)
# update status in mongo
model_dict["status"] = "Done"
model_dict["completionDate"] = str(datetime.utcnow())
mp.ModelProcessor().update_model_mongo(model_dict)
print("thread executed")
except CE.DatasetConnectionFailed as e:
ErrorLogger.exception(
'EXCEPTION %s: Bad Dataset connection Params "%s"',
str(e.errors), str(e))
train_model_params["status"] = "Error"
print(str(e))
train_model_params["ErrorMessage"] = str(e)
train_model_params["completionDate"] = str(datetime.utcnow())
mp.ModelProcessor().update_model_mongo(train_model_params)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up "%s"',
500, e.args)
train_model_params["status"] = "Error"
train_model_params["ErrorMessage"] = str(e.args[0])
train_model_params["completionDate"] = str(datetime.utcnow())
mp.ModelProcessor().update_model_mongo(train_model_params)
def unload_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
status = dict()
response_code = 200
try:
if not LMS.LoadModelService().check_if_model_load(json_data):
raise CE.ModelNotInMemory("No such model loaded", 403)
response_json["success"] = LMS.LoadModelService().unload_model(
json_data)
status["loaded"] = False
mp.ModelProcessor().update_key_mongo(json_data["appId"],
json_data["modelName"], status)
except CE.ModelNotInMemory as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def delete_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_code = 200
try:
if mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) == 0:
raise CE.ModelDoesNotExist("Model name does not exist", 403)
GMS.ModelService().delete_model(json_data)
except CE.ModelDoesNotExist as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def load_model(self, load_model_params):
status = dict()
try:
# model file name
Directory = ConfigManager.ROOT_DIR + "/TrainedModelsDirectory"
filename = str(load_model_params["modelName"]) + "_" + str(load_model_params["algorithm"]["name"]) + "_" \
+ str(load_model_params["appId"]) + ".sav"
filepath = os.path.join(Directory, filename)
# load the model into memory from disk
if "Neural Network" in load_model_params["algorithm"]["name"]:
K.clear_session()
filename = str(load_model_params["modelName"]) + "_" + str(load_model_params["algorithm"]["name"]) + "_" \
+ str(load_model_params["appId"]) + ".hdf5"
filepath = os.path.join(Directory, filename)
sessName = str(load_model_params["modelName"]) + "_" + str(load_model_params["algorithm"]["name"]) + "_" \
+ str(load_model_params["appId"])
sessPath = os.path.join(Directory, sessName)
LoadModelService.load_model_dict[
filename] = keras.models.load_model(filepath)
saver = tf.train.Saver()
sess = keras.backend.get_session()
saver.restore(sess, sessPath)
else:
LoadModelService.load_model_dict[filename] = pickle.load(
open(filepath, 'rb'))
# update status in mongo
status["loaded"] = True
status["loadDate"] = str(datetime.utcnow())
mp.ModelProcessor().update_key_mongo(
load_model_params["appId"], load_model_params["modelName"],
status)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
status["loaded"] = False
status["loadDate"] = str(datetime.utcnow())
status["loadErrorMessage"] = str(e)
mp.ModelProcessor().update_key_mongo(
load_model_params["appId"], load_model_params["modelName"],
status)
def train_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_json["message"] = "InProgress"
response_code = 200
try:
if ConfigManager.algo_list.get(json_data["algorithm"]["name"]) is None:
raise CE.BadAlgorithmParams("Algorithm does not exist", 402)
elif mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) > 0:
raise CE.BadAlgorithmParams(
"Model name already exist for this app id", 403)
elif not dcf.DatasetConnectorFactory().check_connection(
json_data["dbconnect"]):
raise CE.DatasetConnectionFailed(
"Failed to establish connection with dataset connector: %s" %
(json_data["dbconnect"]["name"]), 403)
thread = Thread(target=TR.TrainService().train_model,
args=(json_data, ))
thread.start()
InfoLogger.info(
"Step1: New thread started to do model training for Appid: %s and Modelname: %s",
str(json_data["appId"]), str(json_data["modelName"]))
except CE.BadAlgorithmParams as e:
ErrorLogger.exception('EXCEPTION %s: Bad Algorithm Params "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.DatasetConnectionFailed as e:
ErrorLogger.exception(
'EXCEPTION %s: Bad Dataset connection Params "%s"', str(e.errors),
str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def batchPredict():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_code = 200
try:
if mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) == 0:
raise CE.ModelDoesNotExist("Model name does not exist", 403)
elif not LMS.LoadModelService().check_if_model_load(json_data):
raise CE.ModelNotInMemory(
"Model not deployed, Please deploy model", 403)
response_json["prediction"] = str(
PR.PredictService().predict_batch_service(json_data))
except CE.ModelDoesNotExist as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.ModelNotInMemory as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.InvalidPredictionParams as e:
ErrorLogger.exception('EXCEPTION %s:Invalid Prediction Params "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def get_all_model_details(self, model_params):
all_models = list()
try:
collection_data = mp.ModelProcessor().get_model_mongo_byAppId(
model_params)
print('\n All data from model Database by appId \n')
for model in collection_data:
model_values = {}
if 'appId' in model:
if model['appId'] is not None:
model_values['appId'] = model['appId']
if 'modelName' in model:
if model['modelName'] is not None:
model_values['modelName'] = model['modelName']
if 'creationDate' in model:
if model['creationDate'] is not None:
model_values['creationDate'] = model['creationDate']
if 'algorithm' in model:
if model['algorithm'] is not None:
model_values['algorithm'] = model['algorithm']
if 'accuracy' in model:
if model['accuracy'] is not None:
model_values['accuracy'] = model['accuracy']
if 'completionDate' in model:
if model['completionDate'] is not None:
model_values['completionDate'] = model[
'completionDate']
if 'status' in model:
if model['status'] is not None:
model_values['status'] = model['status']
if 'ErrorMessage' in model:
if model['ErrorMessage'] is not None:
model_values['ErrorMessage'] = model['ErrorMessage']
if 'loaded' in model:
if model['loaded'] is not None:
model_values['deployed'] = model['loaded']
if 'loadDate' in model:
if model['loadDate'] is not None:
model_values['loadDate'] = model['loadDate']
all_models.append(model_values)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
raise Exception("Failed to get all model details: %s" % e.message,
500)
return all_models
def load_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_json["message"] = "InProgress"
response_code = 200
try:
if mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) == 0:
raise CE.ModelDoesNotExist("Model name does not exist", 403)
elif LMS.LoadModelService().check_if_model_load(json_data):
raise CE.ModelAlreadyInMemory("Model already deployed", 403)
thread = Thread(target=LMS.LoadModelService().load_model,
args=(json_data, ))
thread.start()
InfoLogger.info(
"New thread started to load model for Appid: %s and Modelname: %s",
str(json_data["appId"]), str(json_data["modelName"]))
except CE.ModelDoesNotExist as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.ModelAlreadyInMemory as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def get_model_details(self, model_params):
model_values = {}
try:
model = mp.ModelProcessor().get_model_mongo_byAppIdandmodelName(
model_params)
print('\n model Database by appId and model name\n')
if 'appId' in model:
if model['appId'] is not None:
model_values['appId'] = model['appId']
if 'modelName' in model:
if model['modelName'] is not None:
model_values['modelName'] = model['modelName']
if 'creationDate' in model:
if model['creationDate'] is not None:
model_values['creationDate'] = model['creationDate']
if 'algorithm' in model:
if model['algorithm'] is not None:
model_values['algorithm'] = model['algorithm']
if 'accuracy' in model:
if model['accuracy'] is not None:
model_values['accuracy'] = model['accuracy']
if 'completionDate' in model:
if model['completionDate'] is not None:
model_values['completionDate'] = model['completionDate']
if 'status' in model:
if model['status'] is not None:
model_values['status'] = model['status']
if 'ErrorMessage' in model:
if model['ErrorMessage'] is not None:
model_values['ErrorMessage'] = model['ErrorMessage']
if 'loaded' in model:
if model['loaded'] is not None:
model_values['deployed'] = model['loaded']
if 'loadDate' in model:
if model['loadDate'] is not None:
model_values['deployDate'] = model['loadDate']
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
raise Exception("Failed to get all model details: %s" % str(e),
500)
return model_values
def delete_model(self, model_params):
status = False
Directory = ConfigManager.ROOT_DIR + "/TrainedModelsDirectory"
filename = str(model_params["modelName"]) + "_" + str(model_params["algorithm"]["name"]) + "_" \
+ str(model_params["appId"]) + ".sav"
filepath = os.path.join(Directory, filename)
try:
result = mp.ModelProcessor().delete_model_mongo(model_params)
if result["n"] >= 1:
status = True
if LMS.LoadModelService().check_if_model_load(model_params):
status = LMS.LoadModelService().unload_model(model_params)
# remove trained model file
if os.path.isfile(filepath):
os.remove(filepath)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
raise Exception("Failed to delete model: %s" % e.message, 500)
return status
def predict_service(self, model_params):
feature_dict = dict()
feature_list = list()
prediction_dict = dict()
model_collection_data = mp.ModelProcessor(
).get_model_mongo_byAppIdandmodelName(model_params)
if not len(model_params['features']) == len(
model_collection_data["featureList"]):
raise CE.InvalidPredictionParams(
"Invalid number of features, expected: " +
str(len(model_collection_data["featureList"])) + " got: " +
str(len(model_params['features'])), 403)
# Add feature value to list and check if any categorical feature
for feature in model_params["features"]:
feature = json.loads(feature)
if feature["value"]:
if feature["columnName"] in model_collection_data[
"featureList"]:
if 'categoricalData' in model_collection_data:
if feature["columnName"] in model_collection_data[
"categoricalData"]:
if feature["value"] in model_collection_data[
"categoricalData"][
feature["columnName"]]["encoding"]:
feature_dict[str(
feature["columnName"]
)] = model_collection_data["categoricalData"][
feature["columnName"]]["encoding"][
feature["value"]]
else:
raise CE.InvalidPredictionParams(
"Model not trained for feature:" +
feature["columnName"] + " with value:" +
str(feature["value"]), 403)
else:
if type(feature["value"]) == int or type(
feature["value"]) == float:
feature_dict[str(
feature["columnName"])] = feature["value"]
else:
raise CE.InvalidPredictionParams(
"Model not trained for this feature:" +
feature["columnName"] +
" to be categorical. Enter Int value", 403)
else:
if type(feature["value"]) == int or type(
feature["value"]) == float:
feature_dict[str(
feature["columnName"])] = feature["value"]
else:
raise CE.InvalidPredictionParams(
"Model not trained for this feature:" +
feature["columnName"] +
" to be categorical. Enter Int value", 403)
else:
raise CE.InvalidPredictionParams(
"Model not trained with feature: " +
feature["columnName"], 403)
else:
raise CE.InvalidPredictionParams(
"Value can not be null for feature: " +
feature["columnName"], 403)
for feature in model_collection_data["featureList"]:
feature_list.append(feature_dict[str(feature)])
# scale data
Directory = ConfigManager.ROOT_DIR + "/TrainedModelsDirectory"
filename = str(model_params["modelName"]) + "_" + "minmaxScaler_" \
+ str(model_params["appId"]) + ".sav"
filepath = os.path.join(Directory, filename)
min_max_scaler = pickle.load(open(filepath, 'rb'))
feature_list = min_max_scaler.transform([feature_list])[0]
# model file name
if "Neural Network" in model_params["algorithm"]["name"]:
filename = str(model_params["modelName"]) + "_" + str(model_params["algorithm"]["name"]) \
+ "_" + str(model_params["appId"]) + ".hdf5"
else:
filename = str(model_params["modelName"]) + "_" + str(model_params["algorithm"]["name"]) \
+ "_" + str(model_params["appId"]) + ".sav"
# load the model from memory
loaded_model = LMS.LoadModelService().load_model_dict[filename]
# Make prediction
if model_params["algorithm"]["name"] == 'Neural Network':
feature_list = np.array([feature_list])
from keras import backend
with backend.get_session().graph.as_default() as g:
prediction_list = loaded_model.predict_classes(feature_list)
prediction_list = prediction_list[0]
else:
print("Something else")
prediction_list = loaded_model.predict([feature_list])
target_counter = 0
for target in model_collection_data["targetList"]:
if 'categoricalData' in model_collection_data:
if target in model_collection_data["categoricalData"]:
cat_dict = dict(model_collection_data["categoricalData"]
[target]["encoding"])
cat_decoded_value = list(cat_dict.keys())[list(
cat_dict.values()).index(
prediction_list[target_counter])]
prediction_dict[str(target)] = str(cat_decoded_value)
target_counter = target_counter + 1
else:
prediction_dict[str(
target)] = prediction_list[target_counter]
target_counter = target_counter + 1
else:
prediction_dict[str(target)] = prediction_list[target_counter]
target_counter = target_counter + 1
return prediction_dict