def getImageItemList(self, pageItem, data):
totalCount = dataImageItem.objects(__raw__={'dsId': data['dsId']},state=ConstantUtils.DATA_STATUS_ACTIVE).count()
dataList = dataImageItem.objects(__raw__={'dsId': data['dsId']},state=ConstantUtils.DATA_STATUS_ACTIVE).skip(
pageItem.skipIndex).limit(pageItem.pageSize)
pageItem.set_totalCount(totalCount)
#获取该数据集下所有的标签
labelMap,nameList=labelService.getLabelsBylids(data['dsId'])
dataArray = json.loads(dataList.to_json())
for item in dataArray:
item['ditFilePath'] = ConstantUtils.imageItemPrefix + item['ditFilePath'].replace("/","_")
if item.keys().__contains__("recLabelList"):
recLabelList=item['recLabelList']
newList=[]
for labelItem in recLabelList:
if labelMap.keys().__contains__(labelItem['dlid']):
labelItem['dlName']=labelMap[labelItem['dlid']]
newList.append(labelItem)
item['recLabelList']=newList
pageItem.set_numpy_dataList(dataArray)
return resultPackerUtils.packPageResult(pageItem);
def updateDataSetStatisData(self,dsId):
dsItem = datasetsBean.objects(dsId=dsId, state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
#更新总数量
totalCount = dataImageItem.objects(dsId=dsId,state=1).count()
labledCount = dataImageItem.objects(dsId=dsId,isLabeled=1, state=1).count()
dsItem.update(dsImageCount=totalCount,dsImgTagSP=labledCount)
def delImageItem(self,ditId):
imageItem = dataImageItem.objects(ditId=ditId)
imageItem.update(state=ConstantUtils.DATA_STATUS_DELETED)
#更新数据集图片数量和标注进度
print(imageItem[0]["dsId"])
self.updateDataSetStatisData(imageItem[0]["dsId"])
return resultPackerUtils.update_success()
def getImageItemDetail(self,queryData):
imageItem= dataImageItem.objects(ditId=queryData['ditId'],state=ConstantUtils.DATA_STATUS_ACTIVE)[0]
labelMap,nameList=labelService.getLabelsBylids(imageItem['dsId'])
imageItem['ditFilePath'] = ConstantUtils.imageItemPrefix + imageItem['ditFilePath'].replace("/", "_")
obj=json.loads(imageItem.to_json())
recLabelList=obj['recLabelList']
newList = []
for labelItem in recLabelList:
if labelMap.keys().__contains__(labelItem['dlid']):
labelItem['dlName'] = labelMap[labelItem['dlid']]
newList.append(labelItem)
imageItem['recLabelList'] = newList
return resultPackerUtils.packDataListResults(imageItem.to_json())
def upImageItemRecLabels(self, data):
dataImage = dataImageItem.objects(ditId=data["ditId"]).first()
jsons = data["recLabelList"]
jsons=imageUtils.addYOLOCorrdinate(jsons)
labelIdList = []
recLabelList = []
for item in jsons:
dlid = item['dlid']
if labelIdList.__contains__(dlid) == False:
labelIdList.append(dlid)
recLabelList.append(rectangleLabelBean.convertToBean(item))
dataImage.update(recLabelList=recLabelList, labelIdList=labelIdList,isLabeled=1)
self.updateDataSetStatisData(data['dsId'])
return resultPackerUtils.update_success()
def dataPreprocess(self):
# 查寻数据
resultList = []
dl_id_index_map = {}
dlIndex = 0
dlOrderedList = []
for dsItem in self.ds_dl_list:
if dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG:
dataImageList = dataImageItem.objects(
dsId=dsItem["dsId"],
state=ConstantUtils.DATA_STATUS_ACTIVE)
else:
dataImageList = dataImageItem.objects(
dsId=dsItem["dsId"],
labelIdList__in=dsItem["dlidList"],
state=ConstantUtils.DATA_STATUS_ACTIVE)
for dataItem in dataImageList:
img_info = {
'file_name': dataItem['ditFileName'],
'height': dataItem['ditHeight'],
'width': dataItem['ditWidth'],
'id': dataItem['ditId']
}
image_path = os.path.join(self.imageBasePath,
dataItem["ditFilePath"])
img = cv2.imread(image_path)
gt_bboxes = []
gt_labels = []
for boxItem in dataItem['recLabelList']:
if (dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG or
(dsItem['isSelectAll'] == ConstantUtils.FALSE_TAG and
dsItem["dlidList"].__contains__(boxItem['dlid']))):
x = self.getRecValue(dataItem['ditWidth'],
boxItem['rec_lt_x'])
y = self.getRecValue(dataItem['ditHeight'],
boxItem['rec_lt_y'])
w = self.getRecValue(dataItem['ditWidth'],
boxItem['rec_w'])
h = self.getRecValue(dataItem['ditHeight'],
boxItem['rec_h'])
boxItemObj = [x, y, x + w, y + h]
gt_bboxes.append(boxItemObj)
if not dl_id_index_map.keys().__contains__(
boxItem['dlid']):
dl_id_index_map[boxItem['dlid']] = dlIndex
dlOrderedList.append(boxItem['dlid'])
dlIndex += 1
gt_labels.append(dl_id_index_map[boxItem['dlid']])
if img is None:
print('image {} read failed.'.format(image_path))
raise FileNotFoundError(
'Cant load image! Please check image path!')
meta = dict(img=img,
img_info=img_info,
gt_bboxes=np.array(gt_bboxes),
gt_labels=np.array(gt_labels))
meta = self.pipeline(meta, self.input_size)
meta['img'] = torch.from_numpy(meta['img'].transpose(2, 0, 1))
print("---------------------data---------------------")
print(meta)
resultList.append(meta)
print("dl_id_index_map:")
print(dl_id_index_map)
print(dlOrderedList)
return resultList
def loadTrainData(self,dmtvid,ds_dl_list):
imagePathList = []
LabelsList = []
imageShapeList = []
dl_id_index_map = {}
dlOrderedList = []
dlIndex = 0
for dsItem in ds_dl_list:
if dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG:
datImageList = dataImageItem.objects(dsId=dsItem["dsId"], state=1)
else:
datImageList = dataImageItem.objects(dsId=dsItem["dsId"], labelIdList__in=dsItem["dlidList"], state=1)
for imageItem in datImageList:
reclabelList = imageItem['recLabelList']
# 如果图片有标注数据，才参与训练
if len(reclabelList) > 0:
itemLabelList = []
for item in reclabelList:
if (dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG or
(dsItem['isSelectAll'] == ConstantUtils.FALSE_TAG and dsItem["dlidList"].__contains__(item['dlid']))):
if not dl_id_index_map.keys().__contains__(item['dlid']):
dl_id_index_map[item['dlid']]=dlIndex
dlOrderedList.append(item['dlid'])
dlIndex+=1
itemLabelList.append([dl_id_index_map[item['dlid']], item['rec_yolo_x'], item['rec_yolo_y'], item['rec_w'], item['rec_h']])
imagePathList.append(fileUtils.getABSPath(imageItem['ditFilePath']))
imageShapeList.append([imageItem['ditWidth'], imageItem['ditHeight']])
LabelsList.append(np.array(itemLabelList))
print("***************dlid_dlIndex_map**************")
print(str(dl_id_index_map))
#将dlid和index的关系保存到trainVersion中
detectModelTrainVersion.objects(dmtvid=dmtvid,state=ConstantUtils.DATA_STATUS_ACTIVE).update(dl_id_index_map=str(dl_id_index_map))
labelMap, nameList = labelService.getLabelsBylids(dsItem["dsId"])
#对nameList进行排序
newnameList=[labelMap[item] for item in dlOrderedList]
loggerUtils.info("labelMap:" + str(labelMap))
index = 0
for i in range(imagePathList.__len__()):
print("-----------------****" + str(index) + "******-----------------")
print(imagePathList[i])
print(LabelsList[i])
print(imageShapeList[i])
index += 1
trainDataDict = {
"imagePathList": imagePathList,
"LabelsList": np.array(LabelsList),
"imageShapeList": np.array(imageShapeList),
"nc": newnameList.__len__(),
"names": newnameList
}
valDataDict = {
"imagePathList": imagePathList,
"LabelsList": np.array(LabelsList),
"imageShapeList": np.array(imageShapeList),
"nc": newnameList.__len__(),
"names": newnameList
}
return trainDataDict, valDataDict