def getDMVersionList(self, queryData):
dataList = detectModelTrainVersion.objects(dmid=queryData['dmid'], state=ConstantUtils.DATA_STATUS_ACTIVE) \
.exclude("state", "create_date").order_by('-create_date')
datasetIdNames = datasetsBean.objects(
userId=session.get("userId"),
state=ConstantUtils.DATA_STATUS_ACTIVE).only("dsId", "dsName")
datasetMap = {}
for item in datasetIdNames:
datasetMap[item['dsId']] = item['dsName']
modelVersionJsonList = json.loads(dataList.to_json().replace(
"_id", "dmtvid"))
for versionItem in modelVersionJsonList:
versionItem[
'trainState'] = ConstantUtils.getModelVersionTrainState(
versionItem['trainState'])
versionItem[
'inferencePlatformValue'] = ConstantUtils.getModelPlatform(
versionItem['inferencePlatform'])
versionItem['dmPrecisionValue'] = ConstantUtils.getModelPrisision(
versionItem['dmPrecision'])
datasetNames = []
for item in versionItem['ds_dl_list']:
datasetNames.append(datasetMap[item['dsId']])
versionItem['datasetNames'] = datasetNames
return resultPackerUtils.packJsonListResults(modelVersionJsonList)
def getDMVersionDetail(self, queryData):
dataItem = detectModelTrainVersion.objects(
dmtvid=queryData['dmtvid'],
state=ConstantUtils.DATA_STATUS_ACTIVE).exclude(
"state", "create_date")
return resultPackerUtils.packDataItemResults(dataItem.to_json(),
"dmtvid")
def modelTrainFinish(self):
dmtvid = self.modelConfigBean['dmtvid']
versionItem = detectModelTrainVersion.objects(
dmtvid=dmtvid, state=ConstantUtils.DATA_STATUS_ACTIVE)
#查询出最后一次训练统计
trainStatistics = detectModelTrainStatistics.objects(
dmtvid=dmtvid,
state=ConstantUtils.DATA_STATUS_ACTIVE).order_by('-create_date')[0]
versionItem.update(
trainState=ConstantUtils.model_version_train_state_success,
trainEndDateTime=datetime.now,
metrics_mAP=trainStatistics["metrics_mAP"],
metrics_precision=trainStatistics["metrics_precision"],
metrics_recall=trainStatistics['metrics_recall'])
def getDownloadModelUrl(self, dmtvid):
# 查询该版本
modelVersion = detectModelTrainVersion.objects(
dmtvid=int(dmtvid),
state=ConstantUtils.DATA_STATUS_ACTIVE).first()
if modelVersion[
"inferencePlatform"] == ConstantUtils.MODEL_PLATFORM_SERVER:
return modelVersion.ckptModelSavePath
elif modelVersion[
"inferencePlatform"] == ConstantUtils.MODEL_PLATFORM_LITE:
# 获取该项目的weights路径
cfg_path = r"data/nanodet-self.yml"
model_path = str(modelVersion["ckptModelSavePath"])
load_config(cfg, cfg_path)
return liteConveter.convertToNCNN_Android_model(cfg, model_path)
def updateDetectService(self, jsonData):
detectServiceIns = detectServiceBean.objects(
dtsid=jsonData['dtsid'], state=ConstantUtils.DATA_STATUS_ACTIVE)
updateMap = jsonData['updateClolumn']
updateMap["dmBean"] = detectModelBean.objects(
dmId=jsonData["updateClolumn"]['dmId'],
state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
updateMap['dmtvBean'] = detectModelTrainVersion.objects(
dmtvid=jsonData["updateClolumn"]["dmtvId"],
state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
detectServiceIns.update(**jsonData['updateClolumn'])
#加载或关闭模型
self.changeDtsSwitch(jsonData)
return resultPackerUtils.update_success()
def addDetectService(self, jsonData):
dtsServiceKey = randomUtils.getRandomStr()
serviceSecret = randomUtils.getRandomStr()
jsonData["dtsServiceKey"] = dtsServiceKey
jsonData["dtsSecret"] = serviceSecret
jsonData["dmBean"] = detectModelBean.objects(
dmId=jsonData["dmId"], state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
jsonData['dmtvBean'] = detectModelTrainVersion.objects(
dmtvid=jsonData["dmtvId"],
state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
detectService = detectServiceBean.convertToBean(jsonData, session)
detectService.save()
#开启服务
if jsonData['dtsSwitch'] == ConstantUtils.SERVICE_SWITCH_ON:
yoloDetectServiceImpl.launchYoloDetectService(
sessionId=dtsServiceKey, dmtvid=jsonData["dmtvId"])
return resultPackerUtils.save_success()
def getDetectModelsPages(self, pageItem, dmName):
selfQuery = {}
if dmName != None:
selfQuery['dmName'] = {'$regex': dmName}
totalCount = detectModelBean.objects(
__raw__=selfQuery,
state=ConstantUtils.DATA_STATUS_ACTIVE,
userId=session['userId']).count()
modelList = detectModelBean.objects(__raw__=selfQuery, state=ConstantUtils.DATA_STATUS_ACTIVE,
userId=session['userId']) \
.order_by('-create_date').exclude("create_date","state").skip(pageItem.skipIndex).limit(pageItem.pageSize)
modelJsonList = json.loads(modelList.to_json().replace("_id", "dmId"))
# 查询模型的最新版本
modelVersonIdList = []
for item in modelList:
if item['latestVersionId'] is not None:
modelVersonIdList.append(item['latestVersionId'])
if len(modelVersonIdList) > 0:
trainVersionList = detectModelTrainVersion.objects(
dmtvid__in=modelVersonIdList,
state=ConstantUtils.DATA_STATUS_ACTIVE).exclude(
"create_date", "state")
modelVersionJsonList = json.loads(
trainVersionList.to_json().replace("_id", "dmtvid"))
#找出所有的数据集id
allDataSetId = []
for versionItem in modelVersionJsonList:
for item in versionItem['ds_dl_list']:
if not allDataSetId.__contains__(item['dsId']):
allDataSetId.append(item['dsId'])
datasetIdNames = datasetsBean.objects(
dsId__in=allDataSetId,
state=ConstantUtils.DATA_STATUS_ACTIVE).only("dsId", "dsName")
datasetMap = {}
for item in datasetIdNames:
datasetMap[item['dsId']] = item['dsName']
for modelItem in modelJsonList:
if modelItem.keys().__contains__('cvTaskType'):
modelItem[
'cvTaskTypeName'] = ConstantUtils.getCVTaskTypaName(
modelItem['cvTaskType'])
for versionItem in modelVersionJsonList:
if modelItem.keys().__contains__("latestVersionId"):
if modelItem['latestVersionId'] == versionItem[
'dmtvid']:
versionItem[
'trainState'] = ConstantUtils.getModelVersionTrainState(
versionItem['trainState'])
versionItem[
'inferencePlatformValue'] = ConstantUtils.getModelPlatform(
versionItem['inferencePlatform'])
versionItem[
'dmPrecisionValue'] = ConstantUtils.getModelPrisision(
versionItem['dmPrecision'])
datasetNames = []
for item in versionItem['ds_dl_list']:
if datasetMap.keys().__contains__(
"item['dsId']"):
datasetNames.append(
datasetMap[item['dsId']])
versionItem['datasetNames'] = datasetNames
modelItem["latestVersionItem"] = [versionItem]
break
pageItem.set_totalCount(totalCount)
pageItem.set_numpy_dataList(modelJsonList)
return resultPackerUtils.packPageResult(pageItem)
def loadTrainData(self,dmtvid,ds_dl_list):
imagePathList = []
LabelsList = []
imageShapeList = []
dl_id_index_map = {}
dlOrderedList = []
dlIndex = 0
for dsItem in ds_dl_list:
if dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG:
datImageList = dataImageItem.objects(dsId=dsItem["dsId"], state=1)
else:
datImageList = dataImageItem.objects(dsId=dsItem["dsId"], labelIdList__in=dsItem["dlidList"], state=1)
for imageItem in datImageList:
reclabelList = imageItem['recLabelList']
# 如果图片有标注数据，才参与训练
if len(reclabelList) > 0:
itemLabelList = []
for item in reclabelList:
if (dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG or
(dsItem['isSelectAll'] == ConstantUtils.FALSE_TAG and dsItem["dlidList"].__contains__(item['dlid']))):
if not dl_id_index_map.keys().__contains__(item['dlid']):
dl_id_index_map[item['dlid']]=dlIndex
dlOrderedList.append(item['dlid'])
dlIndex+=1
itemLabelList.append([dl_id_index_map[item['dlid']], item['rec_yolo_x'], item['rec_yolo_y'], item['rec_w'], item['rec_h']])
imagePathList.append(fileUtils.getABSPath(imageItem['ditFilePath']))
imageShapeList.append([imageItem['ditWidth'], imageItem['ditHeight']])
LabelsList.append(np.array(itemLabelList))
print("***************dlid_dlIndex_map**************")
print(str(dl_id_index_map))
#将dlid和index的关系保存到trainVersion中
detectModelTrainVersion.objects(dmtvid=dmtvid,state=ConstantUtils.DATA_STATUS_ACTIVE).update(dl_id_index_map=str(dl_id_index_map))
labelMap, nameList = labelService.getLabelsBylids(dsItem["dsId"])
#对nameList进行排序
newnameList=[labelMap[item] for item in dlOrderedList]
loggerUtils.info("labelMap:" + str(labelMap))
index = 0
for i in range(imagePathList.__len__()):
print("-----------------****" + str(index) + "******-----------------")
print(imagePathList[i])
print(LabelsList[i])
print(imageShapeList[i])
index += 1
trainDataDict = {
"imagePathList": imagePathList,
"LabelsList": np.array(LabelsList),
"imageShapeList": np.array(imageShapeList),
"nc": newnameList.__len__(),
"names": newnameList
}
valDataDict = {
"imagePathList": imagePathList,
"LabelsList": np.array(LabelsList),
"imageShapeList": np.array(imageShapeList),
"nc": newnameList.__len__(),
"names": newnameList
}
return trainDataDict, valDataDict
def delDMVersion(self, queryData):
modelVersion = detectModelTrainVersion.objects(
dmtvid=queryData['dmtvid'])
modelVersion.update(state=ConstantUtils.DATA_STATUS_DELETED)
return resultPackerUtils.update_success()
def getDMVersionBean(self, dmtvid):
dataItem = detectModelTrainVersion.objects(
dmtvid=dmtvid, state=ConstantUtils.DATA_STATUS_ACTIVE).exclude(
"state", "create_date")
return dataItem
def getDetectModelVersionNameList(self, queryData):
dataList = detectModelTrainVersion.objects(
dmid=queryData['dmid'],
state=ConstantUtils.DATA_STATUS_ACTIVE).only("dmtvid", "dmtvName")
return resultPackerUtils.packDataListResults(dataList.to_json(),
"dmtvid")
