def train(modelPathName,dataFolder,statusfileLocation,idforData):
self.statusFile=statusfileLocation
dataset_train = CustomDataset()
dataset_train.load_dataset(dataFolder, "train")
dataset_train.prepare()
dataset_val = CustomDataset()
dataset_val.load_dataset(dataFolder, "validation")
dataset_val.prepare()
kerasUtilities.updateStatusOfTraining(statusfileLocation,'Data Loaded')
MODEL_DIR='./logs/'+idforData+'_mrcnnModel'+'/'
kerasUtilities.checkCreatePath(MODEL_DIR)
class CustomConfig(config.Config):
NAME = "customWork"
IMAGES_PER_GPU = 1
NUM_CLASSES = len(dataset_train.class_names)
STEPS_PER_EPOCH = 10
DETECTION_MIN_CONFIDENCE = 0.9
configure = CustomConfig()
# configure.NUM_CLASSES=len(dataset_train.class_names)
model = modellib.MaskRCNN(mode="training", config=configure,
model_dir=MODEL_DIR)
kerasUtilities.updateStatusOfTraining(statusfileLocation,'Model Object Created')
model.load_weights(model.get_imagenet_weights(), by_name=True)
kerasUtilities.updateStatusOfTraining(statusfileLocation,'Training Started')
try:
import tensorflow as tf
with tf.device(gpuCPUSelect(selDev)):
model.train(dataset_train, dataset_val, learning_rate=configure.LEARNING_RATE, epochs=2, layers='heads')
kerasUtilities.updateStatusOfTraining(self.statusFile,'Training Completed')
# model.fit_generator(tGen,steps_per_epoch=stepsPerEpoch,validation_steps=10,epochs=epoch,validation_data=vGen,callbacks=[tensor_board])
except Exception as e:
data_details=self.upDateStatus()
data_details['status']='Training Failed'
data_details['errorMessage']='Please check the error >> '+ str(e)
data_details['errorTraceback']=traceback.format_exc()
with open(self.statusFile,'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
from nyokaBase.mrcnn import maskrcnn_to_pmml
pmmlObj=maskrcnn_to_pmml.MaskrcnnToPMML(model,classes=dataset_train.class_names)
pmmlObj.export(modelPathName)
kerasUtilities.updateStatusOfTraining(statusfileLocation,'PMML file Successfully Saved')
return ('Done')
def main(*argv):
sys.argv = argv[:]
# # def main(*argv):
# # sys.argv = argv[:]
args = sys.argv[1:]
# print(args)
# testCounter(*args)
justTest(*args)
from trainModel import kerasUtilities
kerasUtilities=kerasUtilities.KerasUtilities()
kerasUtilities.updateStatusOfTraining('./logs/UXEWJBAGBGDX/status.txt','Code Execution Completed')
def trainNeuralNetworkModels(requests):
def getValueFromReq(keyVal, bodyVal):
# print ('requests',requests.body)
try:
# print (requests.POST.get(keyVal))
return bodyVal[keyVal]
except:
return ''
# pmmlFile=requests.POST.get('filePath')
bodyVal = json.loads(requests.body)
# print (bodyVal)
# print ('came heer 2nd',bodyVal)
pmmlFile = getValueFromReq('filePath', bodyVal)
tensorboardUrl = getValueFromReq('tensorboardUrl', bodyVal)
tensorboardLogFolder = getValueFromReq('tensorboardLogFolder', bodyVal)
hyperParaUser = {}
hyperParaUser['batchSize'] = getValueFromReq('batchSize', bodyVal)
hyperParaUser['optimizer'] = getValueFromReq('optimizer', bodyVal)
hyperParaUser['loss'] = getValueFromReq('loss', bodyVal)
hyperParaUser['metrics'] = getValueFromReq('metrics', bodyVal)
hyperParaUser['epoch'] = getValueFromReq('epoch', bodyVal)
hyperParaUser['problemType'] = getValueFromReq('problemType', bodyVal)
hyperParaUser['testSize'] = getValueFromReq('testSize', bodyVal)
hyperParaUser['learningRate'] = getValueFromReq(
'learningRate', bodyVal)
# hyperParaUser['']=getValueFromReq('',requests)
# hyperParaUser['']=getValueFromReq('',requests)
# print ('>>>>>>>>PPPPPPPPPPPPPPPP ',pmmlFile,tensorboardUrl,tensorboardLogFolder,hyperParaUser)
idforData = int(time.time())
idforData = str(idforData) + '_NN'
saveStatus = logFolder + idforData + '/'
kerasUtilities.checkCreatePath(saveStatus)
statusfileLocation = saveStatus + 'status.txt'
data_details = {}
data_details['tensorboardUrl'] = tensorboardUrl
data_details['idforData'] = idforData
data_details['status'] = 'In Progress'
fObjScrpt = pathlib.Path(pmmlFile)
print('fObjScrpt.name.replace(fObjScrpt.suffix,'
')', fObjScrpt.name.replace(fObjScrpt.suffix, ''))
data_details['taskName'] = fObjScrpt.name.replace(fObjScrpt.suffix, '')
data_details['createdOn'] = str(datetime.datetime.now())
data_details['type'] = 'NNProject'
data_details['problem_type'] = hyperParaUser['problemType']
data_details["newPMMLFileName"] = pmmlFile
nntrainer = mergeTrainingNN.NeuralNetworkModelTrainer()
pID = nntrainer.train(idforData, pmmlFile, tensorboardLogFolder,
hyperParaUser, pmmlFile)
data_details['pID'] = str(pID)
saveStatus = logFolder + idforData + '/'
kerasUtilities.checkCreatePath(saveStatus)
# statusfileLocation=saveStatus+'status.txt'
with open(statusfileLocation, 'w') as filetosave:
json.dump(data_details, filetosave)
if pID == -1:
# data_details['status']='In Progress'
kerasUtilities.updateStatusOfTraining(statusfileLocation,
'Training Failed')
else:
pass
runTemp = [i['idforData'] for i in RUNNING_TASK_MEMORY]
if data_details['idforData'] not in runTemp:
# print ('PPPPPPPPPPPPPPPPPPPP Saved to runningTask')
tempRunMemory = data_details
RUNNING_TASK_MEMORY.append(tempRunMemory)
else:
pass
print('P' * 200)
print('data_details', data_details)
return JsonResponse(data_details, status=202)
def trainMRCNN(userInput):
# userInput=requests.body
# userInput=json.loads(userInput)
# print (userInput)
pmmlFile = userInput['filePath']
try:
dataFolder = userInput['dataFolder']
except:
print('Get Data folder')
try:
tensorboardLogFolder = userInput['tensorboardLogFolder']
except:
tensorboardLogFolder = target_path = './logs/' + ''.join(
choice(ascii_uppercase) for i in range(12)) + '/'
# print ('tensorboardLogFolder',tensorboardLogFolder)
kerasUtilities.checkCreatePath(tensorboardLogFolder)
# batchSize=userInput['batchSize']
epoch = userInput['epoch']
stepsPerEpoch = userInput['stepPerEpoch']
# learningRate=userInput['learningRate']
try:
tensorboardUrl = userInput['tensorboardUrl']
except:
tensorboardUrl = ''
# idforData=pmmlFile.split('/')[-1].replace('.pmml','')
idforData = os.path.basename(pmmlFile).replace('.pmml', '') + '_MRCNN'
saveStatus = logFolder + idforData + '/'
kerasUtilities.checkCreatePath(saveStatus)
statusfileLocation = saveStatus + 'status.txt'
# print("status file generated")
data_details = {}
data_details['pmmlFile'] = idforData
data_details['dataFolder'] = dataFolder
data_details['fileName'] = pmmlFile
data_details['tensorboardLogFolder'] = tensorboardLogFolder
data_details['tensorboardUrl'] = tensorboardUrl
# data_details['batchSize']=batchSize
data_details['epoch'] = epoch
data_details['stepsPerEpoch'] = stepsPerEpoch
# data_details['learningRate']=learningRate
data_details['idforData'] = idforData
data_details['status'] = 'Building Architecture'
with open(statusfileLocation, 'w') as filetosave:
json.dump(data_details, filetosave)
objtrainer = trainMaskRCNN.ObjectDetetctionModels()
prc = Process(target=objtrainer.train, args=(pmmlFile,dataFolder,statusfileLocation,idforData,epoch,\
tensorboardLogFolder,stepsPerEpoch))
prc.start()
pID = prc.ident
data_details['pID'] = str(pID)
if pID == -1:
kerasUtilities.updateStatusOfTraining(statusfileLocation,
'Training Failed')
else:
with open(statusfileLocation, 'w') as filetosave:
json.dump(data_details, filetosave)
runTemp = [i['idforData'] for i in RUNNING_TASK_MEMORY]
if data_details['idforData'] not in runTemp:
# print ('PPPPPPPPPPPPPPPPPPPP Saved to runningTask')
tempRunMemory = {
'idforData': idforData,
'status': 'Training Failed' if pID == -1 else 'In Progress',
'createdOn': str(datetime.datetime.now()),
'type': 'ObjectDetectionProject',
'pid': pID,
'newPMMLFileName': idforData + '.pmml'
}
tempRunMemory['taskName'] = tempRunMemory['newPMMLFileName']
RUNNING_TASK_MEMORY.append(tempRunMemory)
else:
pass
return JsonResponse(data_details, status=202)
def trainSimpleDNNObj(self,dataObj,fileName,tensorboardLogFolder,lossType,listOfMetrics,\
problemType,optimizerName,learningRate,datHyperPara,testSize,scriptObj):
print(">>>>>>>>>>>>>>SimpleDNN")
print('pathofdata>>>>>', self.pathOfData)
predictedClass = None
targetColumnName = 'target'
df = dataObj
indevar = list(df.columns)
indevar.remove('target')
targetCol = df[targetColumnName]
if problemType == 'classification':
lb = preprocessing.LabelBinarizer()
y = lb.fit_transform(targetCol)
predictedClass = list(targetCol.unique())
else:
y = df[targetColumnName]
predictedClass = None
##### Split data into test and validation set for training#################################
trainDataX, testDataX, trainDataY, testDataY = model_selection.train_test_split(
df[indevar], y, test_size=datHyperPara['testSize'])
stepsPerEpochT = int(len(trainDataX) / datHyperPara['batchSize'])
stepsPerEpochV = int(len(testDataX) / datHyperPara['batchSize'])
kerasUtilities.updateStatusOfTraining(
self.statusFile, 'Data split in Train validation part')
modelObj = self.generateAndCompileModel(lossType, optimizerName,
learningRate, listOfMetrics)
if modelObj.__class__.__name__ == 'dict':
return
model = modelObj.model
tensor_board = self.startTensorBoard(tensorboardLogFolder)
##### Train model#################################
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Started')
try:
import tensorflow as tf
with tf.device(gpuCPUSelect(selDev)):
model.fit(x=trainDataX, y=trainDataY, epochs=datHyperPara['epoch'], callbacks=[tensor_board],\
validation_data=(testDataX, testDataY), steps_per_epoch=stepsPerEpochT, validation_steps=stepsPerEpochV)
except Exception as e:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'Error while fitting data to Keras Model >> ' + str(e),
traceback.format_exc(), self.statusFile)
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Completed')
try:
toExportDict = {
'model1': {
'data': self.pathOfData,
'hyperparameters': datHyperPara,
'preProcessingScript': scriptObj,
'pipelineObj': None,
'modelObj': model,
'featuresUsed': indevar,
'targetName': 'target',
'postProcessingScript': None,
'taskType': 'trainAndscore',
'predictedClasses': predictedClass,
'dataSet': None
}
}
from nyoka.skl.skl_to_pmml import model_to_pmml
model_to_pmml(toExportDict, PMMLFileName=fileName)
kerasUtilities.updateStatusOfTraining(
self.statusFile, 'PMML file Successfully Saved')
return 'Success'
except Exception as e:
data_details = self.upDateStatus()
self.updateStatusWithError(data_details,
'Saving File Failed', ' ' + str(e),
traceback.format_exc(), self.statusFile)
return -1
def trainCustomNN(self,pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,\
batchSize,epoch,idforData,problemType,scriptOutput,optimizerName,learningRate,datHyperPara,testSize,scrDictObj):
print('>>>>>> Step ', 5)
self.trainFolder = dataFolder + '/' + 'train/'
self.validationFolder = dataFolder + '/' + 'validation/'
if os.path.isdir(dataFolder):
print('Image Classifier 2nd time')
# target=self.trainImageClassifierNN
elif pathlib.Path(self.pathOfData).suffix == '.csv':
print('Simple DNN')
sData = pd.read_csv(dataFolder)
print(scrDictObj)
nData = scrDictObj['scripts'][0](sData)
pp=self.trainSimpleDNNObj(nData,fileName,tensorboardLogFolder,lossType,listOfMetrics,\
problemType,optimizerName,learningRate,datHyperPara,testSize,scrDictObj)
return pp
##### Split data into test and validation set for training#################################
trainDataX, testDataX, trainDataY, testDataY = model_selection.train_test_split(
data[XVar], data['target'], test_size=testSize)
trainDataX, trainDataY, testDataX, testDataY = np.array(
trainDataX), np.array(trainDataY), np.array(testDataX), np.array(
testDataY)
kerasUtilities.updateStatusOfTraining(
self.statusFile, 'Data split in Train validation part')
##### Get script from the PMML file and save and load back#################################
for num, sc in enumerate(self.pmmlObj['script']):
useFor = sc.for_
fnaMe = self.scriptFilepath + 'scriptFilepathname_' + useFor + '.py'
scripCode = sc.get_valueOf_()
code = scripCode.lstrip('\n')
lines = []
code = scripCode.lstrip('\n')
leading_spaces = len(code) - len(code.lstrip(' '))
for line in code.split('\n'):
lines.append(line[leading_spaces:])
code = '\n'.join(lines)
with open(fnaMe, 'w') as k:
k.write(code)
if useFor == 'TRAIN':
import importlib.util
spec = importlib.util.spec_from_file_location(
'preProcessing', fnaMe)
preProcessing = spec.loader.load_module()
# print ('>>>>>> Step ',7)
from preProcessing import preProcessing
def createImages(trainDataX, trainDataY, trainFolder):
for num, (raw, label) in enumerate(zip(trainDataX, trainDataY)):
# print (num,label)
labelFolder = trainFolder + label + '/'
try:
os.mkdir(labelFolder)
except:
pass
namofFile = labelFolder + 'id_' + str(num).zfill(8) + '.png'
imgFile = preProcessing(raw)
imgFile.save(namofFile)
# plt.clf()
# plt.close()
return 'done'
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Script saved and loaded')
modelObj = self.generateAndCompileModel(lossType, optimizerName,
learningRate, listOfMetrics)
if modelObj.__class__.__name__ == 'dict':
return
model = modelObj.model
##### Crate images and set them for training#################################
try:
createImages(trainDataX, trainDataY, self.trainFolder)
createImages(testDataX, testDataY, self.validationFolder)
except Exception as e:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'Error while generating the images >> ' + str(e),
traceback.format_exc(), self.statusFile)
return
os.remove(fnaMe)
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Image writing completed')
img_height, img_width = modelObj.image_input.shape.as_list()[1:3]
try:
tGen, vGen, nClass = self.kerasDataPrep(dataFolder, batchSize,
img_height, img_width)
except Exception as e:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'Error while generating data for Keras >> ' + str(e),
traceback.format_exc(), self.statusFile)
sys.exit()
tensor_board = self.startTensorBoard(tensorboardLogFolder)
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Started')
try:
import tensorflow as tf
with tf.device(gpuCPUSelect(selDev)):
model.fit_generator(tGen,
steps_per_epoch=stepsPerEpoch,
epochs=epoch,
validation_data=vGen,
callbacks=[tensor_board])
except Exception as e:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'There is a problem with training parameters >> ' + str(e),
traceback.format_exc(), self.statusFile)
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Completed')
predictedClass = list(tGen.class_indices.keys())
returnVal = self.writePMML(model, predictedClass, fileName, 'image')
if returnVal == -1:
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'PMML file Successfully Saved')
return 'Success'
def generateAndCompileModel(self,
lossType,
optimizerName,
learningRate,
listOfMetrics,
compileTestOnly=False):
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives +
K.epsilon())
return precision
precision = precision(y_true, y_pred)
recall = recall(y_true, y_pred)
return 2 * ((precision * recall) /
(precision + recall + K.epsilon()))
try:
optiMi = self.setOptimizer(optimizerName, learningRate)
except Exception as e:
if not compileTestOnly:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'Unable to get the optimizer ' + optimizerName + ' >> ' +
str(e), traceback.format_exc(), self.statusFile)
else:
data_details = {}
self.updateStatusWithError(
data_details, 'Model Compilation Failed',
'Error while compiling the Model >> ' + str(e),
traceback.format_exc(), self.statusFile)
return data_details
try:
from nyoka.keras.pmml_to_keras_model import GenerateKerasModel
modelObj = GenerateKerasModel(self.pmmlfileObj)
except Exception as e:
if not compileTestOnly:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'Error while generating Keras Model >> ' + str(e),
traceback.format_exc(), self.statusFile)
else:
data_details = {}
self.updateStatusWithError(
data_details, 'Model Compilation Failed',
'Error while compiling Model >> ' + str(e),
traceback.format_exc(), self.statusFile)
return data_details
model = modelObj.model
try:
if 'f1' in listOfMetrics:
listOfMetrics.remove('f1')
model.compile(optimizer=optiMi,
loss=lossType,
metrics=listOfMetrics + [f1])
else:
model.compile(optimizer=optiMi,
loss=lossType,
metrics=listOfMetrics)
except Exception as e:
if not compileTestOnly:
data_details = self.upDateStatus()
self.updateStatusWithError(
data_details, 'Training Failed',
'Error while compiling Model >> ' + str(e),
traceback.format_exc(), self.statusFile)
else:
data_details = {}
self.updateStatusWithError(
data_details, 'Model Compilation Failed',
'Error while compiling Model >> ' + str(e),
traceback.format_exc(), self.statusFile)
return data_details
if not compileTestOnly:
kerasUtilities.updateStatusOfTraining(
self.statusFile, 'Model Successfully compiled')
modelObj.model = model
return modelObj
def trainNeuralNetworkModels(userInput):
# userInput=requests.body
# userInput=json.loads(userInput)
# print (userInput)
pmmlFile = userInput['filePath']
try:
dataFolder = userInput['dataFolder']
except:
dataFolder = './logs/' + ''.join(
choice(ascii_uppercase) for i in range(12)) + '/'
kerasUtilities.checkCreatePath(dataFolder)
try:
fileName = userInput['filePath']
except:
fileName = userInput['filePath']
try:
tensorboardLogFolder = userInput['tensorboardLogFolder']
print('Log folder came correct')
except:
print('Log folder has some issue')
tensorboardLogFolder = target_path = './logs/' + ''.join(
choice(ascii_uppercase) for i in range(12)) + '/'
# print ('tensorboardLogFolder',tensorboardLogFolder)
kerasUtilities.checkCreatePath(tensorboardLogFolder)
lossType = userInput['loss']
listOfMetrics = userInput['metrics']
batchSize = userInput['batchSize']
epoch = userInput['epoch']
stepsPerEpoch = userInput['stepPerEpoch']
problemType = userInput['problemType']
testSize = userInput['testSize']
scriptOutput = userInput['scriptOutput']
optimizerName = userInput['optimizer']
learningRate = userInput['learningRate']
try:
tensorboardUrl = userInput['tensorboardUrl']
except:
tensorboardUrl = ''
# idforData=pmmlFile.split('/')[-1].replace('.pmml','')
idforData = os.path.basename(pmmlFile).replace('.pmml', '')
saveStatus = logFolder + idforData + '/'
kerasUtilities.checkCreatePath(saveStatus)
statusfileLocation = saveStatus + 'status.txt'
# print("status file generated")
data_details = {}
data_details['pmmlFile'] = idforData
data_details['dataFolder'] = dataFolder
data_details['fileName'] = fileName
data_details['tensorboardLogFolder'] = tensorboardLogFolder
data_details['tensorboardUrl'] = tensorboardUrl
data_details['lossType'] = lossType
data_details['listOfMetrics'] = listOfMetrics
data_details['batchSize'] = batchSize
data_details['epoch'] = epoch
data_details['stepsPerEpoch'] = stepsPerEpoch
data_details['problemType'] = problemType
data_details['testSize'] = testSize
data_details['scriptOutput'] = scriptOutput
data_details['optimizerName'] = optimizerName
data_details['learningRate'] = learningRate
data_details['idforData'] = idforData
data_details['status'] = 'In Progress'
fObjScrpt = pathlib.Path(fileName)
data_details['taskName'] = fObjScrpt.name
with open(statusfileLocation, 'w') as filetosave:
json.dump(data_details, filetosave)
nntrainer = mergeTrainingNN.NeuralNetworkModelTrainer()
pID = nntrainer.train(pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,batchSize,\
epoch,stepsPerEpoch,idforData,testSize,problemType,scriptOutput,optimizerName,learningRate)
data_details['pID'] = str(pID)
saveStatus = logFolder + idforData + '/'
kerasUtilities.checkCreatePath(saveStatus)
statusfileLocation = saveStatus + 'status.txt'
if pID == -1:
kerasUtilities.updateStatusOfTraining(statusfileLocation,
'Training Failed')
else:
with open(statusfileLocation, 'w') as filetosave:
json.dump(data_details, filetosave)
runTemp = [i['idforData'] for i in RUNNING_TASK_MEMORY]
if data_details['idforData'] not in runTemp:
# print ('PPPPPPPPPPPPPPPPPPPP Saved to runningTask')
tempRunMemory = {
'idforData': idforData,
'status': 'Training Failed' if pID == -1 else 'In Progress',
'createdOn': str(datetime.datetime.now()),
'type': 'NNProject',
'pid': pID,
'newPMMLFileName': fileName.split('/')[-1]
}
tempRunMemory['taskName'] = data_details['taskName']
RUNNING_TASK_MEMORY.append(tempRunMemory)
else:
pass
return JsonResponse(data_details, status=202)
def trainSimpleDNN(self,pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,\
batchSize,epoch,stepsPerEpoch,idforData,testSize,problemType,scriptOutput,optimizerName,learningRate):
print(">>>>>>>>>>>>>>SimpleDNN")
print('pathofdata>>>>>', self.pathOfData)
predictedClass = None
if self.pathOfData is not None:
targetColumnName = 'target'
try:
df = pd.read_csv(self.pathOfData)
X = df.drop([targetColumnName], axis=1)
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Error while reading the csv file >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
targetCol = df[targetColumnName]
if problemType == 'classification':
lb = preprocessing.LabelBinarizer()
y = lb.fit_transform(targetCol)
predictedClass = list(targetCol.unique())
else:
y = targetCol.values
else:
print('Data not found')
##### Split data into test and validation set for training#################################
trainDataX, testDataX, trainDataY, testDataY = model_selection.train_test_split(
X.values, y, test_size=testSize)
# trainDataX,trainDataY,testDataX,testDataY=np.array(trainDataX),np.array(trainDataY),np.array(testDataX),np.array(testDataY)
kerasUtilities.updateStatusOfTraining(
self.statusFile, 'Data split in Train validation part')
if self.pmmlObj['script'] != []:
for num, sc in enumerate(self.pmmlObj['script']):
useFor = sc.for_
fnaMe = self.scriptFilepath + 'scriptFilepathname_' + useFor + '.py'
scripCode = sc.get_valueOf_()
code = scripCode.lstrip('\n')
lines = []
code = scripCode.lstrip('\n')
leading_spaces = len(code) - len(code.lstrip(' '))
for line in code.split('\n'):
lines.append(line[leading_spaces:])
code = '\n'.join(lines)
with open(fnaMe, 'w') as k:
k.write(code)
if useFor == 'train':
import importlib.util
kerasUtilities.updateStatusOfTraining(
self.statusFile, 'Saving Image files from script')
spec = importlib.util.spec_from_file_location(
'preProcessing', fnaMe)
preProcessing = spec.loader.load_module()
from preProcessing import preProcessing
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Script saved and loaded')
modelObj = self.generateAndCompileModel(lossType, optimizerName,
learningRate, listOfMetrics)
if modelObj.__class__.__name__ == 'dict':
return
model = modelObj.model
tensor_board = self.startTensorBoard(tensorboardLogFolder)
##### Train model#################################
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Started')
try:
import tensorflow as tf
with tf.device(gpuCPUSelect(selDev)):
model.fit(x=trainDataX, y=trainDataY, epochs=epoch, callbacks=[tensor_board],\
validation_data=(testDataX, testDataY), steps_per_epoch=stepsPerEpoch, validation_steps=stepsPerEpoch)
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Error while fitting data to Keras Model >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Completed')
returnVal = self.writePMML(model, predictedClass, fileName, 'dataSet')
if returnVal == -1:
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'PMML file Successfully Saved')
return 'Success'
def trainImageClassifierNN(self,pmmlFile,dataFolder,fileName,tensorboardLogFolder,lossType,listOfMetrics,\
batchSize,epoch,stepsPerEpoch,idforData,testSize,problemType,scriptOutput,optimizerName,learningRate):
# print ('Classification data folder at',dataFolder)
try:
self.trainFolder = dataFolder + '/' + 'train/'
self.validationFolder = dataFolder + '/' + 'validation/'
# print (self.trainFolder)
# print (self.validationFolder)
kerasUtilities.checkCreatePath(self.trainFolder)
kerasUtilities.checkCreatePath(self.validationFolder)
except Exception as e:
# print ('Exception Occured')
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Unable to find train and validation folder >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
# print(">>>>>>>>>>>>>>ImageClassifier")
modelObj = self.generateAndCompileModel(lossType, optimizerName,
learningRate, listOfMetrics)
if modelObj.__class__.__name__ == 'dict':
return
model = modelObj.model
try:
img_height, img_width = modelObj.image_input.shape.as_list()[1:3]
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Model input_shape is invalid >> ' + str(e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
try:
tGen, vGen, nClass = self.kerasDataPrep(dataFolder, batchSize,
img_height, img_width)
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Error while generating data for Keras >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
tensor_board = self.startTensorBoard(tensorboardLogFolder)
##### Train model#################################
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Started')
try:
import tensorflow as tf
with tf.device(gpuCPUSelect(selDev)):
model.fit_generator(tGen,
steps_per_epoch=stepsPerEpoch,
validation_steps=10,
epochs=epoch,
validation_data=vGen,
callbacks=[tensor_board])
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'There is a problem with training parameters >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'Training Completed')
predictedClass = list(tGen.class_indices.keys())
returnVal = self.writePMML(model, predictedClass, fileName, 'image')
if returnVal == -1:
return
kerasUtilities.updateStatusOfTraining(self.statusFile,
'PMML file Successfully Saved')
return 'Success'