def buildModelTrainTaskDir(cfgModel):
#
if not isinstance(cfgModel, dict):
with open(cfgModel, 'r') as f:
cfgModel = json.load(f)
#
modelParser = DLSDesignerFlowsParser(cfgModel)
modelTrainer, solverConfig = modelParser.buildKerasTrainer()
#
taskId = dlsutils.getUniqueTaskId(PREFIX_TASKS_DIR)
dirWithModels = dlsutils.getPathForModelsDir()
dirWithDatasets = dlsutils.getPathForDatasetDir()
dirTaskOut = os.path.join(dirWithModels, taskId)
#
datasetId = solverConfig['dataset-id']
dirDataset = os.path.join(dirWithDatasets, datasetId)
dlsutils.makeDirIfNotExists(dirTaskOut)
#
# modelAdjusted = modelTrainer.adjustModelInputOutput2DBData(modelTrainer.model, dirDataset)
modelAdjusted = modelTrainer.model
foutConfigModel = os.path.join(dirTaskOut, CFG_MODEL_TRAIN)
foutConfigNetwork = os.path.join(dirTaskOut, CFG_MODEL_NETWORK)
foutConfigSolver = os.path.join(dirTaskOut, CFG_SOLVER)
foutConfig = os.path.join(dirTaskOut, CFG_MODEL)
with open(foutConfigNetwork, 'w') as f:
f.write(json.dumps(cfgModel, indent=4))
with open(foutConfigModel, 'w') as f:
f.write(
modelAdjusted.to_json(sort_keys=True,
indent=4,
separators=(',', ': ')))
with open(foutConfigSolver, 'w') as f:
f.write(json.dumps(solverConfig, indent=4))
# prepare basic model config
tdateTime = getDateTimeForConfig()
if datasetId in dbapi.datasetWatcher.dictDbInfo.keys():
dbName = dbapi.datasetWatcher.dictDbInfo[datasetId].cfg.getDBName()
else:
dbName = 'Unknown DB-Name'
modelConfig = {
'id': taskId,
'dataset-id': datasetId,
'dataset-name': dbName,
'date': tdateTime['date'],
'time': tdateTime['time'],
'type': 'image2d-classification',
'name': cfgModel['name'],
'network': cfgModel['name'],
'description': cfgModel['description']
}
with open(foutConfig, 'w') as f:
f.write(json.dumps(modelConfig, indent=4))
return (taskId, dirTaskOut)
def __init__(self, configJson):
# (1) Task-constructor:
Task.__init__(self)
# (2) prepare db-directory with temporary saved config in Json format
tdirDbId = dlsutils.getUniqueTaskId(self.prefixDataset)
pathDatasets = dlsutils.getPathForDatasetDir()
pathDirOut = os.path.abspath(os.path.join(pathDatasets, tdirDbId))
dlsutils.makeDirIfNotExists(pathDirOut)
pathCfgInp = os.path.join(pathDirOut, 'cfg-inp.json')
with open(pathCfgInp, 'w') as f:
f.write(json.dumps(configJson, indent=4))
# (3) DBImage2DBuilder-constructor
DBImage2DBuilder.__init__(self, pathCfgInp, pathDirOut)
# self.initializeInfo()
self.type = 'db-image2d-cls'
self.basetype = 'dataset'
self.icon = "/frontend/assets/icon/img/img-dataset1.png"
def convertKeras2DLS(dataJson,
isDebug=False,
graphvizLayout='dot',
isHorizontal=False):
if isinstance(dataJson, str) or isinstance(dataJson, unicode):
outModelName = os.path.splitext(os.path.basename(dataJson))[0]
with open(dataJson, 'r') as f:
dataJson = f.read()
else:
outModelName = dlsutils.getUniqueTaskId('Imported_Model')
kerasModel = keras.models.model_from_json(dataJson,
custom_objects=dictExtraLayers)
tmpDictLayers = {}
for kk, layer in enumerate(kerasModel.layers):
layerClassName = type(layer).__name__
layerId, layerParams = convetLayersParamsKeras2DLS(layer)
tmpDictLayers[layerId] = {'id': layerId, 'cfg': layerParams}
if isDebug:
print('[%d] %s --> %s' %
(kk, layerClassName, layerParams['wires']))
if isDebug:
pprint(tmpDictLayers)
# theGraphPos = generateCoordsForLayersNX(tmpDictLayers, graphvizLayout=graphvizLayout, isHorizontal=isHorizontal)
theGraphPos = generateCoordsForLayersPG(tmpDictLayers,
graphvizLayout=graphvizLayout,
isHorizontal=isHorizontal)
#
theFinalLayers = []
for kk, vv in tmpDictLayers.items():
tcfg = vv['cfg']
tpos = theGraphPos[kk]
tcfg['pos'] = {'x': tpos[0], 'y': tpos[1]}
theFinalLayers.append(tcfg)
#
theFinalDLSModel = {
'name': outModelName,
'description': '',
'layers': theFinalLayers
}
return theFinalDLSModel
for modelInfo in modelsWatcher.dictModelsInfo.values():
# dbInfo = datasetWatcher.dictDbInfo.values()[0]
# modelInfo = modelsWatcher.dictModelsInfo.values()[0]
print('Selected Dataset: [%s]' % dbInfo)
print('Selected Model: [%s]' % modelInfo)
print('------------------')
# (1) Load and initialise trained model
modelProcessor = ModelProcessor()
modelProcessor.loadModelFromTrainingStateInDir(modelInfo.dirModel)
numLabelsDb = len(dbInfo.labels)
numLabelsModel = len(modelProcessor.batcherLMDB.lbl)
if numLabelsDb != numLabelsModel:
continue
# raise Exception('The number of model labels (%d) must be equal number of dataset labels (%d)' % (numLabelsDb, numLabelsModel))
# (2) Prepare directory with output
evalId = dlsutils.getUniqueTaskId(PREFIX_EVAL_ROC_DIR)
dirEvalROC = os.path.join(modelInfo.dirModel, evalId)
dlsutils.makeDirIfNotExists(dirEvalROC)
# (4) Iterate over dataset-type (train and val)
dbTypes = dbInfo.dbIndex.keys()
plt.figure()
lstLegend = []
rocResult = {}
for dbi, dbType in enumerate(dbTypes):
tdataIndex = dbInfo.dbIndex[dbType]
tnumImages = len(tdataIndex['keys'])
tnumLabels = len(dbInfo.labels)
arrProb = np.zeros((tnumImages, tnumLabels))
for imgId, imgStrId in enumerate(tdataIndex['keys']):
timg = dbInfo.getRawImageFromDB(dbType,
imgId,
import sys
import glob
import numpy as np
import matplotlib.pyplot as plt
import skimage.io as skio
import app.backend.core.utils as dlsutils
from app.backend.core.datasets.dbbuilder import DBImage2DBuilder
#################################################
pathToDirWithJson="../../../data-test"
pathDirOutRoot="../../../data/datasets"
#################################################
if __name__ == '__main__':
lstConfigFn=[os.path.abspath(xx) for xx in glob.glob('%s/dbconfig_*.json' % pathToDirWithJson)]
prefixDataset='dbset'
for ii,pp in enumerate(lstConfigFn):
print ('[%d/%d] : %s' % (ii, len(lstConfigFn), pp))
pathToJson = pp
tdirDbId = dlsutils.getUniqueTaskId(prefixDataset)
pathDirOut = os.path.abspath(os.path.join(pathDirOutRoot, tdirDbId))
dlsutils.makeDirIfNotExists(pathDirOut)
dbBuilder2DImage = DBImage2DBuilder(pathCfgInp=pathToJson,
pathDirOut=pathDirOut)
dbBuilder2DImage.initializeInfo()
print (dbBuilder2DImage)
dbBuilder2DImage.buildDataset(parProgressor=None)
print ('\n\n')
def perform(self):
self.progress=0
# (1) Check input parameters
self.logger.info('Check input parameters model=[%s] db=[%s]' % (self.modelId, self.datasetId))
if self.modelId not in modelsWatcher.dictModelsInfo.keys():
self.logger.error('Unknown model ID [%s]' % self.modelId)
self.state = 'error'
self.alive = False
return
if self.datasetId not in datasetWatcher.dictDbInfo.keys():
self.logger.error('Unknown dataset ID [%s]' % self.datasetId)
self.state = 'error'
self.alive = False
return
modelInfo = modelsWatcher.dictModelsInfo[self.modelId]
dbInfo = datasetWatcher.dictDbInfo[self.datasetId]
# (2) Load and initialise trained model
self.logger.info('Load and initialise trained model [%s]' % self.modelId)
modelProcessor = ModelProcessor()
modelProcessor.loadModelFromTrainingStateInDir(modelInfo.dirModel)
# (3) Check #Classes in Dataset and Model
self.logger.info('Check #Classes in Model and Dataset')
numLabelsDb = len(dbInfo.labels)
numLabelsModel = len(modelProcessor.batcherLMDB.lbl)
if numLabelsDb != numLabelsModel:
self.logger.error('The number of classes in Model [%d] and Dataset [%d] must been equals!' % (numLabelsModel, numLabelsDb))
self.state = 'error'
self.alive = False
return
# (4) Prepare directory for ROC-Analysis results
evalId = dlsutils.getUniqueTaskId(PREFIX_EVAL_ROC_DIR)
self.logger.info('Prepare directory for ROC-Analysis results id=[%s]' % evalId)
dirEvalROC = os.path.join(modelInfo.dirModel, evalId)
dlsutils.makeDirIfNotExists(dirEvalROC)
# (5) Initialyze counter for progress calculation
numTotalIter = dbInfo.getInfoStat()['info']['numTotal']
if numTotalIter < 1:
numTotalIter = 1
counter = 0
# (6) Iterate over dataset-type (train and val)
dbTypes = dbInfo.dbIndex.keys()
rocResult = {}
for dbi, dbType in enumerate(dbTypes):
self.logger.info('Calc ROC for sub-set: [%s]' % dbType)
tdataIndex = dbInfo.dbIndex[dbType]
tnumImages = len(tdataIndex['keys'])
tnumLabels = len(dbInfo.labels)
arrProb = np.zeros((tnumImages, tnumLabels))
for imgId, imgStrId in enumerate(tdataIndex['keys']):
timg = dbInfo.getRawImageFromDB(dbType, imgId, isNumpyArray=True)
tret = modelProcessor.inferOneImageU8(timg)
arrProb[imgId, :] = tret['prob'][0]
counter+=1
self.progress = int((100*counter)/numTotalIter)
#FIXME: check this code
if not self.alive:
if os.path.isdir(dirEvalROC):
shutil.rmtree(dirEvalROC)
return
#
csvDataLabels = pd.DataFrame(tdataIndex['lblid'], columns=['labelid'])
csvDataProb = pd.DataFrame(arrProb, columns=dbInfo.labels)
csvData = csvDataLabels.join(csvDataProb)
foutTableCSV = os.path.join(dirEvalROC, '%s_%s.csv' % (PREFIX_EVAL_ROC_TABLE, dbType))
csvData.to_csv(foutTableCSV, index=None)
#
tmpListROCs = []
for clsId, clsName in enumerate(dbInfo.labels):
fpr, tpr, thresholds = sklearn.metrics.roc_curve(tdataIndex['lblid'], arrProb[:, clsId], pos_label=clsId)
rocScore = roc_auc_score(tdataIndex['lblid'] == clsId, arrProb[:, clsId])
self.logger.info('[%s] AUC score for class [%s] is %0.3f' % (dbType, clsName, rocScore))
tmpClsROC = {
'name': clsName,
'auc': rocScore,
'rocPoints': [{'x': xx, 'y': yy} for xx, yy in zip(fpr, tpr)]
}
tmpListROCs.append(tmpClsROC)
rocResult[dbType] = tmpListROCs
foutCfg = os.path.join(dirEvalROC, CFG_EVAL_ROC)
tmpCfg = {
'id': evalId,
'model-id': modelInfo.getId(),
'model-name': modelInfo.getName(),
'dataset-id': dbInfo.getId(),
'dataset-name': dbInfo.getName(),
'dbtypes': dbTypes,
'date': datetime.now().strftime('%Y.%m.%d-%H:%M:%S'),
'roc': rocResult
}
with open(foutCfg, 'w') as f:
f.write(json.dumps(tmpCfg, indent=4))
#
self.logger.info('Refresh info about models in ModelsWatcher')
modelsWatcher.refreshModelsInfo()
self.progress=100
self.state = 'finished'
self.alive=False