def getImageItemList(self, pageItem, data): totalCount = dataImageItem.objects(__raw__={'dsId': data['dsId']},state=ConstantUtils.DATA_STATUS_ACTIVE).count() dataList = dataImageItem.objects(__raw__={'dsId': data['dsId']},state=ConstantUtils.DATA_STATUS_ACTIVE).skip( pageItem.skipIndex).limit(pageItem.pageSize) pageItem.set_totalCount(totalCount) #获取该数据集下所有的标签 labelMap,nameList=labelService.getLabelsBylids(data['dsId']) dataArray = json.loads(dataList.to_json()) for item in dataArray: item['ditFilePath'] = ConstantUtils.imageItemPrefix + item['ditFilePath'].replace("/","_") if item.keys().__contains__("recLabelList"): recLabelList=item['recLabelList'] newList=[] for labelItem in recLabelList: if labelMap.keys().__contains__(labelItem['dlid']): labelItem['dlName']=labelMap[labelItem['dlid']] newList.append(labelItem) item['recLabelList']=newList pageItem.set_numpy_dataList(dataArray) return resultPackerUtils.packPageResult(pageItem);
def updateDataSetStatisData(self,dsId): dsItem = datasetsBean.objects(dsId=dsId, state=ConstantUtils.DATA_STATUS_ACTIVE)[0] #更新总数量 totalCount = dataImageItem.objects(dsId=dsId,state=1).count() labledCount = dataImageItem.objects(dsId=dsId,isLabeled=1, state=1).count() dsItem.update(dsImageCount=totalCount,dsImgTagSP=labledCount)
def delImageItem(self,ditId): imageItem = dataImageItem.objects(ditId=ditId) imageItem.update(state=ConstantUtils.DATA_STATUS_DELETED) #更新数据集图片数量和标注进度 print(imageItem[0]["dsId"]) self.updateDataSetStatisData(imageItem[0]["dsId"]) return resultPackerUtils.update_success()
def getImageItemDetail(self,queryData): imageItem= dataImageItem.objects(ditId=queryData['ditId'],state=ConstantUtils.DATA_STATUS_ACTIVE)[0] labelMap,nameList=labelService.getLabelsBylids(imageItem['dsId']) imageItem['ditFilePath'] = ConstantUtils.imageItemPrefix + imageItem['ditFilePath'].replace("/", "_") obj=json.loads(imageItem.to_json()) recLabelList=obj['recLabelList'] newList = [] for labelItem in recLabelList: if labelMap.keys().__contains__(labelItem['dlid']): labelItem['dlName'] = labelMap[labelItem['dlid']] newList.append(labelItem) imageItem['recLabelList'] = newList return resultPackerUtils.packDataListResults(imageItem.to_json())
def upImageItemRecLabels(self, data): dataImage = dataImageItem.objects(ditId=data["ditId"]).first() jsons = data["recLabelList"] jsons=imageUtils.addYOLOCorrdinate(jsons) labelIdList = [] recLabelList = [] for item in jsons: dlid = item['dlid'] if labelIdList.__contains__(dlid) == False: labelIdList.append(dlid) recLabelList.append(rectangleLabelBean.convertToBean(item)) dataImage.update(recLabelList=recLabelList, labelIdList=labelIdList,isLabeled=1) self.updateDataSetStatisData(data['dsId']) return resultPackerUtils.update_success()
def dataPreprocess(self): # 查寻数据 resultList = [] dl_id_index_map = {} dlIndex = 0 dlOrderedList = [] for dsItem in self.ds_dl_list: if dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG: dataImageList = dataImageItem.objects( dsId=dsItem["dsId"], state=ConstantUtils.DATA_STATUS_ACTIVE) else: dataImageList = dataImageItem.objects( dsId=dsItem["dsId"], labelIdList__in=dsItem["dlidList"], state=ConstantUtils.DATA_STATUS_ACTIVE) for dataItem in dataImageList: img_info = { 'file_name': dataItem['ditFileName'], 'height': dataItem['ditHeight'], 'width': dataItem['ditWidth'], 'id': dataItem['ditId'] } image_path = os.path.join(self.imageBasePath, dataItem["ditFilePath"]) img = cv2.imread(image_path) gt_bboxes = [] gt_labels = [] for boxItem in dataItem['recLabelList']: if (dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG or (dsItem['isSelectAll'] == ConstantUtils.FALSE_TAG and dsItem["dlidList"].__contains__(boxItem['dlid']))): x = self.getRecValue(dataItem['ditWidth'], boxItem['rec_lt_x']) y = self.getRecValue(dataItem['ditHeight'], boxItem['rec_lt_y']) w = self.getRecValue(dataItem['ditWidth'], boxItem['rec_w']) h = self.getRecValue(dataItem['ditHeight'], boxItem['rec_h']) boxItemObj = [x, y, x + w, y + h] gt_bboxes.append(boxItemObj) if not dl_id_index_map.keys().__contains__( boxItem['dlid']): dl_id_index_map[boxItem['dlid']] = dlIndex dlOrderedList.append(boxItem['dlid']) dlIndex += 1 gt_labels.append(dl_id_index_map[boxItem['dlid']]) if img is None: print('image {} read failed.'.format(image_path)) raise FileNotFoundError( 'Cant load image! Please check image path!') meta = dict(img=img, img_info=img_info, gt_bboxes=np.array(gt_bboxes), gt_labels=np.array(gt_labels)) meta = self.pipeline(meta, self.input_size) meta['img'] = torch.from_numpy(meta['img'].transpose(2, 0, 1)) print("---------------------data---------------------") print(meta) resultList.append(meta) print("dl_id_index_map:") print(dl_id_index_map) print(dlOrderedList) return resultList
def loadTrainData(self,dmtvid,ds_dl_list): imagePathList = [] LabelsList = [] imageShapeList = [] dl_id_index_map = {} dlOrderedList = [] dlIndex = 0 for dsItem in ds_dl_list: if dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG: datImageList = dataImageItem.objects(dsId=dsItem["dsId"], state=1) else: datImageList = dataImageItem.objects(dsId=dsItem["dsId"], labelIdList__in=dsItem["dlidList"], state=1) for imageItem in datImageList: reclabelList = imageItem['recLabelList'] # 如果图片有标注数据,才参与训练 if len(reclabelList) > 0: itemLabelList = [] for item in reclabelList: if (dsItem['isSelectAll'] == ConstantUtils.TRUE_TAG or (dsItem['isSelectAll'] == ConstantUtils.FALSE_TAG and dsItem["dlidList"].__contains__(item['dlid']))): if not dl_id_index_map.keys().__contains__(item['dlid']): dl_id_index_map[item['dlid']]=dlIndex dlOrderedList.append(item['dlid']) dlIndex+=1 itemLabelList.append([dl_id_index_map[item['dlid']], item['rec_yolo_x'], item['rec_yolo_y'], item['rec_w'], item['rec_h']]) imagePathList.append(fileUtils.getABSPath(imageItem['ditFilePath'])) imageShapeList.append([imageItem['ditWidth'], imageItem['ditHeight']]) LabelsList.append(np.array(itemLabelList)) print("***************dlid_dlIndex_map**************") print(str(dl_id_index_map)) #将dlid和index的关系保存到trainVersion中 detectModelTrainVersion.objects(dmtvid=dmtvid,state=ConstantUtils.DATA_STATUS_ACTIVE).update(dl_id_index_map=str(dl_id_index_map)) labelMap, nameList = labelService.getLabelsBylids(dsItem["dsId"]) #对nameList进行排序 newnameList=[labelMap[item] for item in dlOrderedList] loggerUtils.info("labelMap:" + str(labelMap)) index = 0 for i in range(imagePathList.__len__()): print("-----------------****" + str(index) + "******-----------------") print(imagePathList[i]) print(LabelsList[i]) print(imageShapeList[i]) index += 1 trainDataDict = { "imagePathList": imagePathList, "LabelsList": np.array(LabelsList), "imageShapeList": np.array(imageShapeList), "nc": newnameList.__len__(), "names": newnameList } valDataDict = { "imagePathList": imagePathList, "LabelsList": np.array(LabelsList), "imageShapeList": np.array(imageShapeList), "nc": newnameList.__len__(), "names": newnameList } return trainDataDict, valDataDict