def getDataSetPages(self, pageItem, dsName):
userSearchDict = {}
if dsName != None:
userSearchDict['dsName'] = {'$regex': dsName}
totalCount = datasetsBean.objects(__raw__=userSearchDict,state=1,userId=session['userId']).count()
#获取数据集
datasetList = datasetsBean.objects(__raw__=userSearchDict,state=1,userId=session['userId']).order_by('-create_date').skip(
pageItem.skipIndex).exclude("state","userId").limit(pageItem.pageSize)
# only("dsName","dsType")
#获取数据集下的所有标签
dsIdList=[]
for item in datasetList:
dsIdList.append(item['dsId'])
labelList=dataLabelBean.objects(dsId__in=dsIdList)
labelMap={}
for labelItem in labelList:
dsId=labelItem['dsId']
if labelMap.keys().__contains__(dsId):
labelMap[dsId].append(labelItem['dlName'])
else:
labelMap[dsId]=[labelItem['dlName']]
datasetsList=[]
for item in datasetList:
if labelMap.keys().__contains__(item['dsId']):
labelList=labelMap[item['dsId']]
else:
labelList=[]
datasetItem={
"dsId":item['dsId'],
"dsName":item['dsName'],
"dsType":item['dsType'],
"dsImageCount":item['dsImageCount'],
"dsImgTagSP":item['dsImgTagSP'],
"labelCount":len(labelList),
"labelList":labelList,
"cvTaskType":item['cvTaskType'],
"cvTaskName":ConstantUtils.getCVTaskTypaName(item['cvTaskType'])
}
datasetsList.append(datasetItem)
pageItem.set_totalCount(totalCount)
pageItem.set_numpy_dataList(datasetsList)
return resultPackerUtils.packPageResult(pageItem);
def updateDataSet(self,data):
datasetIns = datasetsBean.objects(dsId=data['dsId'])
print(data['updateClolumn'])
datasetIns.update(**data['updateClolumn'])
return resultPackerUtils.update_success()
def getDMVersionList(self, queryData):
dataList = detectModelTrainVersion.objects(dmid=queryData['dmid'], state=ConstantUtils.DATA_STATUS_ACTIVE) \
.exclude("state", "create_date").order_by('-create_date')
datasetIdNames = datasetsBean.objects(
userId=session.get("userId"),
state=ConstantUtils.DATA_STATUS_ACTIVE).only("dsId", "dsName")
datasetMap = {}
for item in datasetIdNames:
datasetMap[item['dsId']] = item['dsName']
modelVersionJsonList = json.loads(dataList.to_json().replace(
"_id", "dmtvid"))
for versionItem in modelVersionJsonList:
versionItem[
'trainState'] = ConstantUtils.getModelVersionTrainState(
versionItem['trainState'])
versionItem[
'inferencePlatformValue'] = ConstantUtils.getModelPlatform(
versionItem['inferencePlatform'])
versionItem['dmPrecisionValue'] = ConstantUtils.getModelPrisision(
versionItem['dmPrecision'])
datasetNames = []
for item in versionItem['ds_dl_list']:
datasetNames.append(datasetMap[item['dsId']])
versionItem['datasetNames'] = datasetNames
return resultPackerUtils.packJsonListResults(modelVersionJsonList)
def updateDataSetStatisData(self,dsId):
dsItem = datasetsBean.objects(dsId=dsId, state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
#更新总数量
totalCount = dataImageItem.objects(dsId=dsId,state=1).count()
labledCount = dataImageItem.objects(dsId=dsId,isLabeled=1, state=1).count()
dsItem.update(dsImageCount=totalCount,dsImgTagSP=labledCount)
def upImageData(self, dsId, fileType, compreImgPack, imageslist):
#获取dataset detail
datasetItem = datasetsBean.objects(dsId=dsId, state=ConstantUtils.DATA_STATUS_ACTIVE).first()
imageFiles = None
folderName = str(uuid.uuid4())
desFolderBasePath = ConstantUtils.dataBasePath + folderName
imageNameList,imagePathList=None,None
if fileType == ConstantUtils.UP_FILE_TYPE_COMPRESSFILE:
imageNameList, imagePathList = self.saveCompressedFile(desFolderBasePath, compreImgPack)
elif fileType == ConstantUtils.UP_FILE_TYPE_IMAGEFILE:
imageNameList, imagePathList = self.saveMultiImages(desFolderBasePath, imageslist)
# 保存到数据库
#如果是图像分类数据集
if datasetItem["cvTaskType"]==ConstantUtils.CV_TASK_TYPE_CLASSIFY:
return self.dealClassifyDatasets(dsId,desFolderBasePath,folderName,imageNameList, imagePathList)
saveImageItemList = []
for i in range(imageNameList.__len__()):
fileName = imageNameList[i]
filePath = imagePathList[i]
# 获取图片信息
imageSize = imageUtils.getImageSize(desFolderBasePath + "/" + filePath)
saveImageItemList.append(dataImageItem(
ditFileName=fileName,
ditFilePath=folderName + "/" + filePath,
dsId=dsId,
ditWidth=imageSize[0],
ditHeight=imageSize[1],
isLabeled=ConstantUtils.IMAGE_UNLABEL
))
dataImageItem.objects.insert(saveImageItemList, load_bulk=False)
#更新数据集数量
self.updateDataSetStatisData(dsId)
return resultPackerUtils.save_success()
def getDetectModelsPages(self, pageItem, dmName):
selfQuery = {}
if dmName != None:
selfQuery['dmName'] = {'$regex': dmName}
totalCount = detectModelBean.objects(
__raw__=selfQuery,
state=ConstantUtils.DATA_STATUS_ACTIVE,
userId=session['userId']).count()
modelList = detectModelBean.objects(__raw__=selfQuery, state=ConstantUtils.DATA_STATUS_ACTIVE,
userId=session['userId']) \
.order_by('-create_date').exclude("create_date","state").skip(pageItem.skipIndex).limit(pageItem.pageSize)
modelJsonList = json.loads(modelList.to_json().replace("_id", "dmId"))
# 查询模型的最新版本
modelVersonIdList = []
for item in modelList:
if item['latestVersionId'] is not None:
modelVersonIdList.append(item['latestVersionId'])
if len(modelVersonIdList) > 0:
trainVersionList = detectModelTrainVersion.objects(
dmtvid__in=modelVersonIdList,
state=ConstantUtils.DATA_STATUS_ACTIVE).exclude(
"create_date", "state")
modelVersionJsonList = json.loads(
trainVersionList.to_json().replace("_id", "dmtvid"))
#找出所有的数据集id
allDataSetId = []
for versionItem in modelVersionJsonList:
for item in versionItem['ds_dl_list']:
if not allDataSetId.__contains__(item['dsId']):
allDataSetId.append(item['dsId'])
datasetIdNames = datasetsBean.objects(
dsId__in=allDataSetId,
state=ConstantUtils.DATA_STATUS_ACTIVE).only("dsId", "dsName")
datasetMap = {}
for item in datasetIdNames:
datasetMap[item['dsId']] = item['dsName']
for modelItem in modelJsonList:
if modelItem.keys().__contains__('cvTaskType'):
modelItem[
'cvTaskTypeName'] = ConstantUtils.getCVTaskTypaName(
modelItem['cvTaskType'])
for versionItem in modelVersionJsonList:
if modelItem.keys().__contains__("latestVersionId"):
if modelItem['latestVersionId'] == versionItem[
'dmtvid']:
versionItem[
'trainState'] = ConstantUtils.getModelVersionTrainState(
versionItem['trainState'])
versionItem[
'inferencePlatformValue'] = ConstantUtils.getModelPlatform(
versionItem['inferencePlatform'])
versionItem[
'dmPrecisionValue'] = ConstantUtils.getModelPrisision(
versionItem['dmPrecision'])
datasetNames = []
for item in versionItem['ds_dl_list']:
if datasetMap.keys().__contains__(
"item['dsId']"):
datasetNames.append(
datasetMap[item['dsId']])
versionItem['datasetNames'] = datasetNames
modelItem["latestVersionItem"] = [versionItem]
break
pageItem.set_totalCount(totalCount)
pageItem.set_numpy_dataList(modelJsonList)
return resultPackerUtils.packPageResult(pageItem)
def getDataSetDetail(self,dsId):
result = datasetsBean.objects(dsId=dsId, state=1).exclude("state", "userId").to_json()
return resultPackerUtils.packDataItemResults(result)
def getAllDSNamesList(self,jsonData):
datasetsList = datasetsBean.objects(state=1,
userId=session['userId'],
cvTaskType=jsonData['cvTaskType']).only("dsId","dsName")
return resultPackerUtils.packDataListResults(datasetsList.to_json(),"dsId")
def delDataSet(self,dsId):
detectsetItem = datasetsBean.objects(dsId=dsId)
detectsetItem.update(state=ConstantUtils.DATA_STATUS_DELETED)
return resultPackerUtils.update_success()