def train(modelPathName,dataFolder,statusfileLocation,idforData): self.statusFile=statusfileLocation dataset_train = CustomDataset() dataset_train.load_dataset(dataFolder, "train") dataset_train.prepare() dataset_val = CustomDataset() dataset_val.load_dataset(dataFolder, "validation") dataset_val.prepare() kerasUtilities.updateStatusOfTraining(statusfileLocation,'Data Loaded') MODEL_DIR='./logs/'+idforData+'_mrcnnModel'+'/' kerasUtilities.checkCreatePath(MODEL_DIR) class CustomConfig(config.Config): NAME = "customWork" IMAGES_PER_GPU = 1 NUM_CLASSES = len(dataset_train.class_names) STEPS_PER_EPOCH = 10 DETECTION_MIN_CONFIDENCE = 0.9 configure = CustomConfig() # configure.NUM_CLASSES=len(dataset_train.class_names) model = modellib.MaskRCNN(mode="training", config=configure, model_dir=MODEL_DIR) kerasUtilities.updateStatusOfTraining(statusfileLocation,'Model Object Created') model.load_weights(model.get_imagenet_weights(), by_name=True) kerasUtilities.updateStatusOfTraining(statusfileLocation,'Training Started') try: import tensorflow as tf with tf.device(gpuCPUSelect(selDev)): model.train(dataset_train, dataset_val, learning_rate=configure.LEARNING_RATE, epochs=2, layers='heads') kerasUtilities.updateStatusOfTraining(self.statusFile,'Training Completed') # model.fit_generator(tGen,steps_per_epoch=stepsPerEpoch,validation_steps=10,epochs=epoch,validation_data=vGen,callbacks=[tensor_board]) except Exception as e: data_details=self.upDateStatus() data_details['status']='Training Failed' data_details['errorMessage']='Please check the error >> '+ str(e) data_details['errorTraceback']=traceback.format_exc() with open(self.statusFile,'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return from nyokaBase.mrcnn import maskrcnn_to_pmml pmmlObj=maskrcnn_to_pmml.MaskrcnnToPMML(model,classes=dataset_train.class_names) pmmlObj.export(modelPathName) kerasUtilities.updateStatusOfTraining(statusfileLocation,'PMML file Successfully Saved') return ('Done')
def main(*argv): sys.argv = argv[:] # # def main(*argv): # # sys.argv = argv[:] args = sys.argv[1:] # print(args) # testCounter(*args) justTest(*args) from trainModel import kerasUtilities kerasUtilities=kerasUtilities.KerasUtilities() kerasUtilities.updateStatusOfTraining('./logs/UXEWJBAGBGDX/status.txt','Code Execution Completed')
def trainNeuralNetworkModels(requests): def getValueFromReq(keyVal, bodyVal): # print ('requests',requests.body) try: # print (requests.POST.get(keyVal)) return bodyVal[keyVal] except: return '' # pmmlFile=requests.POST.get('filePath') bodyVal = json.loads(requests.body) # print (bodyVal) # print ('came heer 2nd',bodyVal) pmmlFile = getValueFromReq('filePath', bodyVal) tensorboardUrl = getValueFromReq('tensorboardUrl', bodyVal) tensorboardLogFolder = getValueFromReq('tensorboardLogFolder', bodyVal) hyperParaUser = {} hyperParaUser['batchSize'] = getValueFromReq('batchSize', bodyVal) hyperParaUser['optimizer'] = getValueFromReq('optimizer', bodyVal) hyperParaUser['loss'] = getValueFromReq('loss', bodyVal) hyperParaUser['metrics'] = getValueFromReq('metrics', bodyVal) hyperParaUser['epoch'] = getValueFromReq('epoch', bodyVal) hyperParaUser['problemType'] = getValueFromReq('problemType', bodyVal) hyperParaUser['testSize'] = getValueFromReq('testSize', bodyVal) hyperParaUser['learningRate'] = getValueFromReq( 'learningRate', bodyVal) # hyperParaUser['']=getValueFromReq('',requests) # hyperParaUser['']=getValueFromReq('',requests) # print ('>>>>>>>>PPPPPPPPPPPPPPPP ',pmmlFile,tensorboardUrl,tensorboardLogFolder,hyperParaUser) idforData = int(time.time()) idforData = str(idforData) + '_NN' saveStatus = logFolder + idforData + '/' kerasUtilities.checkCreatePath(saveStatus) statusfileLocation = saveStatus + 'status.txt' data_details = {} data_details['tensorboardUrl'] = tensorboardUrl data_details['idforData'] = idforData data_details['status'] = 'In Progress' fObjScrpt = pathlib.Path(pmmlFile) print('fObjScrpt.name.replace(fObjScrpt.suffix,' ')', fObjScrpt.name.replace(fObjScrpt.suffix, '')) data_details['taskName'] = fObjScrpt.name.replace(fObjScrpt.suffix, '') data_details['createdOn'] = str(datetime.datetime.now()) data_details['type'] = 'NNProject' data_details['problem_type'] = hyperParaUser['problemType'] data_details["newPMMLFileName"] = pmmlFile nntrainer = mergeTrainingNN.NeuralNetworkModelTrainer() pID = nntrainer.train(idforData, pmmlFile, tensorboardLogFolder, hyperParaUser, pmmlFile) data_details['pID'] = str(pID) saveStatus = logFolder + idforData + '/' kerasUtilities.checkCreatePath(saveStatus) # statusfileLocation=saveStatus+'status.txt' with open(statusfileLocation, 'w') as filetosave: json.dump(data_details, filetosave) if pID == -1: # data_details['status']='In Progress' kerasUtilities.updateStatusOfTraining(statusfileLocation, 'Training Failed') else: pass runTemp = [i['idforData'] for i in RUNNING_TASK_MEMORY] if data_details['idforData'] not in runTemp: # print ('PPPPPPPPPPPPPPPPPPPP Saved to runningTask') tempRunMemory = data_details RUNNING_TASK_MEMORY.append(tempRunMemory) else: pass print('P' * 200) print('data_details', data_details) return JsonResponse(data_details, status=202)
def trainMRCNN(userInput): # userInput=requests.body # userInput=json.loads(userInput) # print (userInput) pmmlFile = userInput['filePath'] try: dataFolder = userInput['dataFolder'] except: print('Get Data folder') try: tensorboardLogFolder = userInput['tensorboardLogFolder'] except: tensorboardLogFolder = target_path = './logs/' + ''.join( choice(ascii_uppercase) for i in range(12)) + '/' # print ('tensorboardLogFolder',tensorboardLogFolder) kerasUtilities.checkCreatePath(tensorboardLogFolder) # batchSize=userInput['batchSize'] epoch = userInput['epoch'] stepsPerEpoch = userInput['stepPerEpoch'] # learningRate=userInput['learningRate'] try: tensorboardUrl = userInput['tensorboardUrl'] except: tensorboardUrl = '' # idforData=pmmlFile.split('/')[-1].replace('.pmml','') idforData = os.path.basename(pmmlFile).replace('.pmml', '') + '_MRCNN' saveStatus = logFolder + idforData + '/' kerasUtilities.checkCreatePath(saveStatus) statusfileLocation = saveStatus + 'status.txt' # print("status file generated") data_details = {} data_details['pmmlFile'] = idforData data_details['dataFolder'] = dataFolder data_details['fileName'] = pmmlFile data_details['tensorboardLogFolder'] = tensorboardLogFolder data_details['tensorboardUrl'] = tensorboardUrl # data_details['batchSize']=batchSize data_details['epoch'] = epoch data_details['stepsPerEpoch'] = stepsPerEpoch # data_details['learningRate']=learningRate data_details['idforData'] = idforData data_details['status'] = 'Building Architecture' with open(statusfileLocation, 'w') as filetosave: json.dump(data_details, filetosave) objtrainer = trainMaskRCNN.ObjectDetetctionModels() prc = Process(target=objtrainer.train, args=(pmmlFile,dataFolder,statusfileLocation,idforData,epoch,\ tensorboardLogFolder,stepsPerEpoch)) prc.start() pID = prc.ident data_details['pID'] = str(pID) if pID == -1: kerasUtilities.updateStatusOfTraining(statusfileLocation, 'Training Failed') else: with open(statusfileLocation, 'w') as filetosave: json.dump(data_details, filetosave) runTemp = [i['idforData'] for i in RUNNING_TASK_MEMORY] if data_details['idforData'] not in runTemp: # print ('PPPPPPPPPPPPPPPPPPPP Saved to runningTask') tempRunMemory = { 'idforData': idforData, 'status': 'Training Failed' if pID == -1 else 'In Progress', 'createdOn': str(datetime.datetime.now()), 'type': 'ObjectDetectionProject', 'pid': pID, 'newPMMLFileName': idforData + '.pmml' } tempRunMemory['taskName'] = tempRunMemory['newPMMLFileName'] RUNNING_TASK_MEMORY.append(tempRunMemory) else: pass return JsonResponse(data_details, status=202)
def trainSimpleDNNObj(self,dataObj,fileName,tensorboardLogFolder,lossType,listOfMetrics,\ problemType,optimizerName,learningRate,datHyperPara,testSize,scriptObj): print(">>>>>>>>>>>>>>SimpleDNN") print('pathofdata>>>>>', self.pathOfData) predictedClass = None targetColumnName = 'target' df = dataObj indevar = list(df.columns) indevar.remove('target') targetCol = df[targetColumnName] if problemType == 'classification': lb = preprocessing.LabelBinarizer() y = lb.fit_transform(targetCol) predictedClass = list(targetCol.unique()) else: y = df[targetColumnName] predictedClass = None ##### Split data into test and validation set for training################################# trainDataX, testDataX, trainDataY, testDataY = model_selection.train_test_split( df[indevar], y, test_size=datHyperPara['testSize']) stepsPerEpochT = int(len(trainDataX) / datHyperPara['batchSize']) stepsPerEpochV = int(len(testDataX) / datHyperPara['batchSize']) kerasUtilities.updateStatusOfTraining( self.statusFile, 'Data split in Train validation part') modelObj = self.generateAndCompileModel(lossType, optimizerName, learningRate, listOfMetrics) if modelObj.__class__.__name__ == 'dict': return model = modelObj.model tensor_board = self.startTensorBoard(tensorboardLogFolder) ##### Train model################################# kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Started') try: import tensorflow as tf with tf.device(gpuCPUSelect(selDev)): model.fit(x=trainDataX, y=trainDataY, epochs=datHyperPara['epoch'], callbacks=[tensor_board],\ validation_data=(testDataX, testDataY), steps_per_epoch=stepsPerEpochT, validation_steps=stepsPerEpochV) except Exception as e: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'Error while fitting data to Keras Model >> ' + str(e), traceback.format_exc(), self.statusFile) return kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Completed') try: toExportDict = { 'model1': { 'data': self.pathOfData, 'hyperparameters': datHyperPara, 'preProcessingScript': scriptObj, 'pipelineObj': None, 'modelObj': model, 'featuresUsed': indevar, 'targetName': 'target', 'postProcessingScript': None, 'taskType': 'trainAndscore', 'predictedClasses': predictedClass, 'dataSet': None } } from nyoka.skl.skl_to_pmml import model_to_pmml model_to_pmml(toExportDict, PMMLFileName=fileName) kerasUtilities.updateStatusOfTraining( self.statusFile, 'PMML file Successfully Saved') return 'Success' except Exception as e: data_details = self.upDateStatus() self.updateStatusWithError(data_details, 'Saving File Failed', ' ' + str(e), traceback.format_exc(), self.statusFile) return -1
def trainCustomNN(self,pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,\ batchSize,epoch,idforData,problemType,scriptOutput,optimizerName,learningRate,datHyperPara,testSize,scrDictObj): print('>>>>>> Step ', 5) self.trainFolder = dataFolder + '/' + 'train/' self.validationFolder = dataFolder + '/' + 'validation/' if os.path.isdir(dataFolder): print('Image Classifier 2nd time') # target=self.trainImageClassifierNN elif pathlib.Path(self.pathOfData).suffix == '.csv': print('Simple DNN') sData = pd.read_csv(dataFolder) print(scrDictObj) nData = scrDictObj['scripts'][0](sData) pp=self.trainSimpleDNNObj(nData,fileName,tensorboardLogFolder,lossType,listOfMetrics,\ problemType,optimizerName,learningRate,datHyperPara,testSize,scrDictObj) return pp ##### Split data into test and validation set for training################################# trainDataX, testDataX, trainDataY, testDataY = model_selection.train_test_split( data[XVar], data['target'], test_size=testSize) trainDataX, trainDataY, testDataX, testDataY = np.array( trainDataX), np.array(trainDataY), np.array(testDataX), np.array( testDataY) kerasUtilities.updateStatusOfTraining( self.statusFile, 'Data split in Train validation part') ##### Get script from the PMML file and save and load back################################# for num, sc in enumerate(self.pmmlObj['script']): useFor = sc.for_ fnaMe = self.scriptFilepath + 'scriptFilepathname_' + useFor + '.py' scripCode = sc.get_valueOf_() code = scripCode.lstrip('\n') lines = [] code = scripCode.lstrip('\n') leading_spaces = len(code) - len(code.lstrip(' ')) for line in code.split('\n'): lines.append(line[leading_spaces:]) code = '\n'.join(lines) with open(fnaMe, 'w') as k: k.write(code) if useFor == 'TRAIN': import importlib.util spec = importlib.util.spec_from_file_location( 'preProcessing', fnaMe) preProcessing = spec.loader.load_module() # print ('>>>>>> Step ',7) from preProcessing import preProcessing def createImages(trainDataX, trainDataY, trainFolder): for num, (raw, label) in enumerate(zip(trainDataX, trainDataY)): # print (num,label) labelFolder = trainFolder + label + '/' try: os.mkdir(labelFolder) except: pass namofFile = labelFolder + 'id_' + str(num).zfill(8) + '.png' imgFile = preProcessing(raw) imgFile.save(namofFile) # plt.clf() # plt.close() return 'done' kerasUtilities.updateStatusOfTraining(self.statusFile, 'Script saved and loaded') modelObj = self.generateAndCompileModel(lossType, optimizerName, learningRate, listOfMetrics) if modelObj.__class__.__name__ == 'dict': return model = modelObj.model ##### Crate images and set them for training################################# try: createImages(trainDataX, trainDataY, self.trainFolder) createImages(testDataX, testDataY, self.validationFolder) except Exception as e: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'Error while generating the images >> ' + str(e), traceback.format_exc(), self.statusFile) return os.remove(fnaMe) kerasUtilities.updateStatusOfTraining(self.statusFile, 'Image writing completed') img_height, img_width = modelObj.image_input.shape.as_list()[1:3] try: tGen, vGen, nClass = self.kerasDataPrep(dataFolder, batchSize, img_height, img_width) except Exception as e: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'Error while generating data for Keras >> ' + str(e), traceback.format_exc(), self.statusFile) sys.exit() tensor_board = self.startTensorBoard(tensorboardLogFolder) kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Started') try: import tensorflow as tf with tf.device(gpuCPUSelect(selDev)): model.fit_generator(tGen, steps_per_epoch=stepsPerEpoch, epochs=epoch, validation_data=vGen, callbacks=[tensor_board]) except Exception as e: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'There is a problem with training parameters >> ' + str(e), traceback.format_exc(), self.statusFile) return kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Completed') predictedClass = list(tGen.class_indices.keys()) returnVal = self.writePMML(model, predictedClass, fileName, 'image') if returnVal == -1: return kerasUtilities.updateStatusOfTraining(self.statusFile, 'PMML file Successfully Saved') return 'Success'
def generateAndCompileModel(self, lossType, optimizerName, learningRate, listOfMetrics, compileTestOnly=False): def f1(y_true, y_pred): def recall(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) recall = true_positives / (possible_positives + K.epsilon()) return recall def precision(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) return precision precision = precision(y_true, y_pred) recall = recall(y_true, y_pred) return 2 * ((precision * recall) / (precision + recall + K.epsilon())) try: optiMi = self.setOptimizer(optimizerName, learningRate) except Exception as e: if not compileTestOnly: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'Unable to get the optimizer ' + optimizerName + ' >> ' + str(e), traceback.format_exc(), self.statusFile) else: data_details = {} self.updateStatusWithError( data_details, 'Model Compilation Failed', 'Error while compiling the Model >> ' + str(e), traceback.format_exc(), self.statusFile) return data_details try: from nyoka.keras.pmml_to_keras_model import GenerateKerasModel modelObj = GenerateKerasModel(self.pmmlfileObj) except Exception as e: if not compileTestOnly: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'Error while generating Keras Model >> ' + str(e), traceback.format_exc(), self.statusFile) else: data_details = {} self.updateStatusWithError( data_details, 'Model Compilation Failed', 'Error while compiling Model >> ' + str(e), traceback.format_exc(), self.statusFile) return data_details model = modelObj.model try: if 'f1' in listOfMetrics: listOfMetrics.remove('f1') model.compile(optimizer=optiMi, loss=lossType, metrics=listOfMetrics + [f1]) else: model.compile(optimizer=optiMi, loss=lossType, metrics=listOfMetrics) except Exception as e: if not compileTestOnly: data_details = self.upDateStatus() self.updateStatusWithError( data_details, 'Training Failed', 'Error while compiling Model >> ' + str(e), traceback.format_exc(), self.statusFile) else: data_details = {} self.updateStatusWithError( data_details, 'Model Compilation Failed', 'Error while compiling Model >> ' + str(e), traceback.format_exc(), self.statusFile) return data_details if not compileTestOnly: kerasUtilities.updateStatusOfTraining( self.statusFile, 'Model Successfully compiled') modelObj.model = model return modelObj
def trainNeuralNetworkModels(userInput): # userInput=requests.body # userInput=json.loads(userInput) # print (userInput) pmmlFile = userInput['filePath'] try: dataFolder = userInput['dataFolder'] except: dataFolder = './logs/' + ''.join( choice(ascii_uppercase) for i in range(12)) + '/' kerasUtilities.checkCreatePath(dataFolder) try: fileName = userInput['filePath'] except: fileName = userInput['filePath'] try: tensorboardLogFolder = userInput['tensorboardLogFolder'] print('Log folder came correct') except: print('Log folder has some issue') tensorboardLogFolder = target_path = './logs/' + ''.join( choice(ascii_uppercase) for i in range(12)) + '/' # print ('tensorboardLogFolder',tensorboardLogFolder) kerasUtilities.checkCreatePath(tensorboardLogFolder) lossType = userInput['loss'] listOfMetrics = userInput['metrics'] batchSize = userInput['batchSize'] epoch = userInput['epoch'] stepsPerEpoch = userInput['stepPerEpoch'] problemType = userInput['problemType'] testSize = userInput['testSize'] scriptOutput = userInput['scriptOutput'] optimizerName = userInput['optimizer'] learningRate = userInput['learningRate'] try: tensorboardUrl = userInput['tensorboardUrl'] except: tensorboardUrl = '' # idforData=pmmlFile.split('/')[-1].replace('.pmml','') idforData = os.path.basename(pmmlFile).replace('.pmml', '') saveStatus = logFolder + idforData + '/' kerasUtilities.checkCreatePath(saveStatus) statusfileLocation = saveStatus + 'status.txt' # print("status file generated") data_details = {} data_details['pmmlFile'] = idforData data_details['dataFolder'] = dataFolder data_details['fileName'] = fileName data_details['tensorboardLogFolder'] = tensorboardLogFolder data_details['tensorboardUrl'] = tensorboardUrl data_details['lossType'] = lossType data_details['listOfMetrics'] = listOfMetrics data_details['batchSize'] = batchSize data_details['epoch'] = epoch data_details['stepsPerEpoch'] = stepsPerEpoch data_details['problemType'] = problemType data_details['testSize'] = testSize data_details['scriptOutput'] = scriptOutput data_details['optimizerName'] = optimizerName data_details['learningRate'] = learningRate data_details['idforData'] = idforData data_details['status'] = 'In Progress' fObjScrpt = pathlib.Path(fileName) data_details['taskName'] = fObjScrpt.name with open(statusfileLocation, 'w') as filetosave: json.dump(data_details, filetosave) nntrainer = mergeTrainingNN.NeuralNetworkModelTrainer() pID = nntrainer.train(pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,batchSize,\ epoch,stepsPerEpoch,idforData,testSize,problemType,scriptOutput,optimizerName,learningRate) data_details['pID'] = str(pID) saveStatus = logFolder + idforData + '/' kerasUtilities.checkCreatePath(saveStatus) statusfileLocation = saveStatus + 'status.txt' if pID == -1: kerasUtilities.updateStatusOfTraining(statusfileLocation, 'Training Failed') else: with open(statusfileLocation, 'w') as filetosave: json.dump(data_details, filetosave) runTemp = [i['idforData'] for i in RUNNING_TASK_MEMORY] if data_details['idforData'] not in runTemp: # print ('PPPPPPPPPPPPPPPPPPPP Saved to runningTask') tempRunMemory = { 'idforData': idforData, 'status': 'Training Failed' if pID == -1 else 'In Progress', 'createdOn': str(datetime.datetime.now()), 'type': 'NNProject', 'pid': pID, 'newPMMLFileName': fileName.split('/')[-1] } tempRunMemory['taskName'] = data_details['taskName'] RUNNING_TASK_MEMORY.append(tempRunMemory) else: pass return JsonResponse(data_details, status=202)
def trainSimpleDNN(self,pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,\ batchSize,epoch,stepsPerEpoch,idforData,testSize,problemType,scriptOutput,optimizerName,learningRate): print(">>>>>>>>>>>>>>SimpleDNN") print('pathofdata>>>>>', self.pathOfData) predictedClass = None if self.pathOfData is not None: targetColumnName = 'target' try: df = pd.read_csv(self.pathOfData) X = df.drop([targetColumnName], axis=1) except Exception as e: data_details = self.upDateStatus() data_details['status'] = 'Training Failed' data_details[ 'errorMessage'] = 'Error while reading the csv file >> ' + str( e) data_details['errorTraceback'] = traceback.format_exc() with open(self.statusFile, 'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return targetCol = df[targetColumnName] if problemType == 'classification': lb = preprocessing.LabelBinarizer() y = lb.fit_transform(targetCol) predictedClass = list(targetCol.unique()) else: y = targetCol.values else: print('Data not found') ##### Split data into test and validation set for training################################# trainDataX, testDataX, trainDataY, testDataY = model_selection.train_test_split( X.values, y, test_size=testSize) # trainDataX,trainDataY,testDataX,testDataY=np.array(trainDataX),np.array(trainDataY),np.array(testDataX),np.array(testDataY) kerasUtilities.updateStatusOfTraining( self.statusFile, 'Data split in Train validation part') if self.pmmlObj['script'] != []: for num, sc in enumerate(self.pmmlObj['script']): useFor = sc.for_ fnaMe = self.scriptFilepath + 'scriptFilepathname_' + useFor + '.py' scripCode = sc.get_valueOf_() code = scripCode.lstrip('\n') lines = [] code = scripCode.lstrip('\n') leading_spaces = len(code) - len(code.lstrip(' ')) for line in code.split('\n'): lines.append(line[leading_spaces:]) code = '\n'.join(lines) with open(fnaMe, 'w') as k: k.write(code) if useFor == 'train': import importlib.util kerasUtilities.updateStatusOfTraining( self.statusFile, 'Saving Image files from script') spec = importlib.util.spec_from_file_location( 'preProcessing', fnaMe) preProcessing = spec.loader.load_module() from preProcessing import preProcessing kerasUtilities.updateStatusOfTraining(self.statusFile, 'Script saved and loaded') modelObj = self.generateAndCompileModel(lossType, optimizerName, learningRate, listOfMetrics) if modelObj.__class__.__name__ == 'dict': return model = modelObj.model tensor_board = self.startTensorBoard(tensorboardLogFolder) ##### Train model################################# kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Started') try: import tensorflow as tf with tf.device(gpuCPUSelect(selDev)): model.fit(x=trainDataX, y=trainDataY, epochs=epoch, callbacks=[tensor_board],\ validation_data=(testDataX, testDataY), steps_per_epoch=stepsPerEpoch, validation_steps=stepsPerEpoch) except Exception as e: data_details = self.upDateStatus() data_details['status'] = 'Training Failed' data_details[ 'errorMessage'] = 'Error while fitting data to Keras Model >> ' + str( e) data_details['errorTraceback'] = traceback.format_exc() with open(self.statusFile, 'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Completed') returnVal = self.writePMML(model, predictedClass, fileName, 'dataSet') if returnVal == -1: return kerasUtilities.updateStatusOfTraining(self.statusFile, 'PMML file Successfully Saved') return 'Success'
def trainImageClassifierNN(self,pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,\ batchSize,epoch,stepsPerEpoch,idforData,testSize,problemType,scriptOutput,optimizerName,learningRate): # print ('Classification data folder at',dataFolder) try: self.trainFolder = dataFolder + '/' + 'train/' self.validationFolder = dataFolder + '/' + 'validation/' # print (self.trainFolder) # print (self.validationFolder) kerasUtilities.checkCreatePath(self.trainFolder) kerasUtilities.checkCreatePath(self.validationFolder) except Exception as e: # print ('Exception Occured') data_details = self.upDateStatus() data_details['status'] = 'Training Failed' data_details[ 'errorMessage'] = 'Unable to find train and validation folder >> ' + str( e) data_details['errorTraceback'] = traceback.format_exc() with open(self.statusFile, 'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return # print(">>>>>>>>>>>>>>ImageClassifier") modelObj = self.generateAndCompileModel(lossType, optimizerName, learningRate, listOfMetrics) if modelObj.__class__.__name__ == 'dict': return model = modelObj.model try: img_height, img_width = modelObj.image_input.shape.as_list()[1:3] except Exception as e: data_details = self.upDateStatus() data_details['status'] = 'Training Failed' data_details[ 'errorMessage'] = 'Model input_shape is invalid >> ' + str(e) data_details['errorTraceback'] = traceback.format_exc() with open(self.statusFile, 'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return try: tGen, vGen, nClass = self.kerasDataPrep(dataFolder, batchSize, img_height, img_width) except Exception as e: data_details = self.upDateStatus() data_details['status'] = 'Training Failed' data_details[ 'errorMessage'] = 'Error while generating data for Keras >> ' + str( e) data_details['errorTraceback'] = traceback.format_exc() with open(self.statusFile, 'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return tensor_board = self.startTensorBoard(tensorboardLogFolder) ##### Train model################################# kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Started') try: import tensorflow as tf with tf.device(gpuCPUSelect(selDev)): model.fit_generator(tGen, steps_per_epoch=stepsPerEpoch, validation_steps=10, epochs=epoch, validation_data=vGen, callbacks=[tensor_board]) except Exception as e: data_details = self.upDateStatus() data_details['status'] = 'Training Failed' data_details[ 'errorMessage'] = 'There is a problem with training parameters >> ' + str( e) data_details['errorTraceback'] = traceback.format_exc() with open(self.statusFile, 'w') as filetosave: json.dump(data_details, filetosave) # sys.exit() return kerasUtilities.updateStatusOfTraining(self.statusFile, 'Training Completed') predictedClass = list(tGen.class_indices.keys()) returnVal = self.writePMML(model, predictedClass, fileName, 'image') if returnVal == -1: return kerasUtilities.updateStatusOfTraining(self.statusFile, 'PMML file Successfully Saved') return 'Success'