def train_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_json["message"] = "InProgress"
response_code = 200
try:
if ConfigManager.algo_list.get(json_data["algorithm"]["name"]) is None:
raise CE.BadAlgorithmParams("Algorithm does not exist", 402)
elif mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) > 0:
raise CE.BadAlgorithmParams(
"Model name already exist for this app id", 403)
elif not dcf.DatasetConnectorFactory().check_connection(
json_data["dbconnect"]):
raise CE.DatasetConnectionFailed(
"Failed to establish connection with dataset connector: %s" %
(json_data["dbconnect"]["name"]), 403)
thread = Thread(target=TR.TrainService().train_model,
args=(json_data, ))
thread.start()
InfoLogger.info(
"Step1: New thread started to do model training for Appid: %s and Modelname: %s",
str(json_data["appId"]), str(json_data["modelName"]))
except CE.BadAlgorithmParams as e:
ErrorLogger.exception('EXCEPTION %s: Bad Algorithm Params "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.DatasetConnectionFailed as e:
ErrorLogger.exception(
'EXCEPTION %s: Bad Dataset connection Params "%s"', str(e.errors),
str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def batchPredict():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_code = 200
try:
if mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) == 0:
raise CE.ModelDoesNotExist("Model name does not exist", 403)
elif not LMS.LoadModelService().check_if_model_load(json_data):
raise CE.ModelNotInMemory(
"Model not deployed, Please deploy model", 403)
response_json["prediction"] = str(
PR.PredictService().predict_batch_service(json_data))
except CE.ModelDoesNotExist as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.ModelNotInMemory as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.InvalidPredictionParams as e:
ErrorLogger.exception('EXCEPTION %s:Invalid Prediction Params "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def unload_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
status = dict()
response_code = 200
try:
if not LMS.LoadModelService().check_if_model_load(json_data):
raise CE.ModelNotInMemory("No such model loaded", 403)
response_json["success"] = LMS.LoadModelService().unload_model(
json_data)
status["loaded"] = False
mp.ModelProcessor().update_key_mongo(json_data["appId"],
json_data["modelName"], status)
except CE.ModelNotInMemory as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def delete_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_code = 200
try:
if mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) == 0:
raise CE.ModelDoesNotExist("Model name does not exist", 403)
GMS.ModelService().delete_model(json_data)
except CE.ModelDoesNotExist as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def best_feature_selection():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_code = 200
try:
if not dcf.DatasetConnectorFactory().check_connection(
json_data["dbconnect"]):
raise CE.DatasetConnectionFailed(
"Failed to establish connection with dataset connector: %s" %
(json_data["dbconnect"]["name"]), 403)
response_json["bestFeaturesScore"] = DPP.DataPreProcessor(
).best_feature_selection(json_data)
response_json["success"] = True
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def load_model():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_json["message"] = "InProgress"
response_code = 200
try:
if mp.ModelProcessor().get_model_mongo_byAppIdandmodelName_count(
json_data) == 0:
raise CE.ModelDoesNotExist("Model name does not exist", 403)
elif LMS.LoadModelService().check_if_model_load(json_data):
raise CE.ModelAlreadyInMemory("Model already deployed", 403)
thread = Thread(target=LMS.LoadModelService().load_model,
args=(json_data, ))
thread.start()
InfoLogger.info(
"New thread started to load model for Appid: %s and Modelname: %s",
str(json_data["appId"]), str(json_data["modelName"]))
except CE.ModelDoesNotExist as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except CE.ModelAlreadyInMemory as e:
ErrorLogger.exception('EXCEPTION %s: NO such model "%s"',
str(e.errors), str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def connect(self, host, username, password, db):
try:
MySQL.connection = mysql.connector.connect(host=host,
database=db,
user=username,
password=password)
except mysql.connector.Error as err:
raise CE.DatasetConnectionFailed(
"Failed to establish connection with dataset connector: %s" %
"SQL", 403)
def parse_csv_and_get_columns(filename):
logger.info("Enter in Execution")
if os.path.isfile(filename):
csvFile = open(filename, 'r')
lines = csvFile.readlines()
else:
raise CustomException('File is not availble')
for line in lines[1:]:
val = line.split(",")
try:
test_zero_div = (int(val[0]) / int(val[11]))
print(test_zero_div)
except TypeError:
logger.info("Exception Occured TypeError")
raise CustomException('Invalid Types')
except ZeroDivisionError:
logger.info("Exception Occured ZeroDivisionError")
except ValueError:
logger.info("Exception Occured ValueError")
def analyzeMood(self, message=None):
try:
if message == None:
message = self.message
if message.__contains__("sad"):
return "SAD"
else:
return "HAPPY"
except Exception as exception:
raise CustomException("please enter proper message")
def parse_csv_and_get_columns(filename):
if os.path.isfile(filename):
csvFile = open(filename, 'r')
lines = csvFile.readlines()
else:
raise CustomException('Given file isn\'t available')
for line in lines[1:]:
val = line.split(',')
try:
test_zero_div = (int(val[0]) / int(val[11]))
print(test_zero_div)
except TypeError:
logger.error('Exception Occured TypeError')
raise CustomException('Invalid Types')
except ZeroDivisionError:
logger.error('Exception Occured ZeroDivisionError')
except ValueError:
logger.error('Exception Occured ValueError')
def _SetField(instance, field, value, strict):
'''Use reflection to set a field value or property value on an object'''
# Get Field
instanceType = instance.GetType()
fieldInfo = instanceType.GetField(field) # try to get as field
if fieldInfo is None:
fieldInfo = instanceType.GetProperty(field) # fallback on property
if fieldInfo is None and strict:
typeName = type(instance).__name__
raise CustomException("Could not get fieldInfo for instance of type '{0}' with field '{1}'".format(typeName, field))
elif fieldInfo is None:
return
# Set field
try:
fieldInfo.SetValue(instance, value)
except ValueError as e:
typeName = type(instance).__name__
raise CustomException("Could not set field '{0}' for instance of type '{1}' - innerexception: {2} ".format(field, typeName, e.message))
return
def GetDatasetColumnList():
if not request.json:
abort(ConfigManager.error_code_400)
json_data = request.json
response_json = dict()
response_json["success"] = True
response_code = 200
try:
if not dcf.DatasetConnectorFactory().check_connection(
json_data["dbconnect"]):
raise CE.DatasetConnectionFailed(
"Failed to establish connection with dataset connector: %s" %
(json_data["dbconnect"]["name"]), 403)
column_list = dcf.DatasetConnectorFactory().get_column_list(
json_data["dbconnect"])
if column_list:
response_json["columnList"] = column_list
else:
response_json["success"] = False
except CE.DatasetConnectionFailed as e:
ErrorLogger.exception(
'EXCEPTION %s: Bad Dataset connection Params "%s"', str(e.errors),
str(e))
response_json["success"] = False
response_json["message"] = str(e)
response_code = int(e.errors)
except Exception as e:
ErrorLogger.exception('EXCEPTION %s: Damm! Something Blew up', 500)
response_json["success"] = False
response_json["message"] = "EXCEPTION 500: Damm! Something Blew up"
response_code = 500
return jsonify(response_json), response_code
def New(service, typeName, **kwargs):
''' Create new instance using reflection, **this is necessary**(!) because of namespace collisions in generated C4C web service assemblies'''
if "->" in typeName:
typeName = typeName.split("->")[1].strip()
# create instance
assembly = clr.GetClrType(type(service)).Assembly
try:
instanceType = assembly.GetTypes().First(lambda x: x.Name == typeName)
except SystemError:
raise CustomException("Type '{0}' does not exist in this assembly".format(typeName))
from System import Activator
instance = Activator.CreateInstance(instanceType)
# set provided kwargs as fields on the newly created object
if kwargs is not None:
for key, value in kwargs.iteritems():
field = instanceType.GetField(key)
field.SetValue(instance, value)
return instance
def setObject(self, theObject):
if not isinstance(theObject, Option):
raise CustomException()
self._option = theObject
def _checkACont(self, aCont):
if not isinstance(aCont, AbstractController):
raise CustomException("No AbstractController stated")
def predict_service(self, model_params):
feature_dict = dict()
feature_list = list()
prediction_dict = dict()
model_collection_data = mp.ModelProcessor(
).get_model_mongo_byAppIdandmodelName(model_params)
if not len(model_params['features']) == len(
model_collection_data["featureList"]):
raise CE.InvalidPredictionParams(
"Invalid number of features, expected: " +
str(len(model_collection_data["featureList"])) + " got: " +
str(len(model_params['features'])), 403)
# Add feature value to list and check if any categorical feature
for feature in model_params["features"]:
feature = json.loads(feature)
if feature["value"]:
if feature["columnName"] in model_collection_data[
"featureList"]:
if 'categoricalData' in model_collection_data:
if feature["columnName"] in model_collection_data[
"categoricalData"]:
if feature["value"] in model_collection_data[
"categoricalData"][
feature["columnName"]]["encoding"]:
feature_dict[str(
feature["columnName"]
)] = model_collection_data["categoricalData"][
feature["columnName"]]["encoding"][
feature["value"]]
else:
raise CE.InvalidPredictionParams(
"Model not trained for feature:" +
feature["columnName"] + " with value:" +
str(feature["value"]), 403)
else:
if type(feature["value"]) == int or type(
feature["value"]) == float:
feature_dict[str(
feature["columnName"])] = feature["value"]
else:
raise CE.InvalidPredictionParams(
"Model not trained for this feature:" +
feature["columnName"] +
" to be categorical. Enter Int value", 403)
else:
if type(feature["value"]) == int or type(
feature["value"]) == float:
feature_dict[str(
feature["columnName"])] = feature["value"]
else:
raise CE.InvalidPredictionParams(
"Model not trained for this feature:" +
feature["columnName"] +
" to be categorical. Enter Int value", 403)
else:
raise CE.InvalidPredictionParams(
"Model not trained with feature: " +
feature["columnName"], 403)
else:
raise CE.InvalidPredictionParams(
"Value can not be null for feature: " +
feature["columnName"], 403)
for feature in model_collection_data["featureList"]:
feature_list.append(feature_dict[str(feature)])
# scale data
Directory = ConfigManager.ROOT_DIR + "/TrainedModelsDirectory"
filename = str(model_params["modelName"]) + "_" + "minmaxScaler_" \
+ str(model_params["appId"]) + ".sav"
filepath = os.path.join(Directory, filename)
min_max_scaler = pickle.load(open(filepath, 'rb'))
feature_list = min_max_scaler.transform([feature_list])[0]
# model file name
if "Neural Network" in model_params["algorithm"]["name"]:
filename = str(model_params["modelName"]) + "_" + str(model_params["algorithm"]["name"]) \
+ "_" + str(model_params["appId"]) + ".hdf5"
else:
filename = str(model_params["modelName"]) + "_" + str(model_params["algorithm"]["name"]) \
+ "_" + str(model_params["appId"]) + ".sav"
# load the model from memory
loaded_model = LMS.LoadModelService().load_model_dict[filename]
# Make prediction
if model_params["algorithm"]["name"] == 'Neural Network':
feature_list = np.array([feature_list])
from keras import backend
with backend.get_session().graph.as_default() as g:
prediction_list = loaded_model.predict_classes(feature_list)
prediction_list = prediction_list[0]
else:
print("Something else")
prediction_list = loaded_model.predict([feature_list])
target_counter = 0
for target in model_collection_data["targetList"]:
if 'categoricalData' in model_collection_data:
if target in model_collection_data["categoricalData"]:
cat_dict = dict(model_collection_data["categoricalData"]
[target]["encoding"])
cat_decoded_value = list(cat_dict.keys())[list(
cat_dict.values()).index(
prediction_list[target_counter])]
prediction_dict[str(target)] = str(cat_decoded_value)
target_counter = target_counter + 1
else:
prediction_dict[str(
target)] = prediction_list[target_counter]
target_counter = target_counter + 1
else:
prediction_dict[str(target)] = prediction_list[target_counter]
target_counter = target_counter + 1
return prediction_dict
def callObject(self):
if self._record is None:
raise CustomException()
def callObject(self):
if self._option is None:
raise CustomException()
def __init__(self, newRecord, functionDesc):
if not isinstance(newRecord, Record):
raise CustomException("Not a Record object")
self._record = newRecord
self._funcStr = functionDesc
def setObject(self, theObject):
if not isinstance(theObject, Record):
raise CustomException()
self._record = theObject