def detect(self, frame): """ Detect the hand :param frame: image frame :return: cropped image, transformation, center """ hd = HandDetector(frame, self.sync['config']['fx'], self.sync['config']['fy'], importer=self.importer, refineNet=self.comrefNet) doHS = (self.state.value == self.STATE_INIT) if self.tracking.value and not numpy.allclose(self.lastcom, 0): loc, handsz = hd.track(self.lastcom, self.sync['config']['cube'], doHandSize=doHS) else: loc, handsz = hd.detect(size=self.sync['config']['cube'], doHandSize=doHS) self.lastcom = loc if self.state.value == self.STATE_INIT: self.handsizes.append(handsz) if self.verbose is True: print(numpy.median(numpy.asarray(self.handsizes), axis=0)) else: self.handsizes = [] if self.state.value == self.STATE_INIT and len( self.handsizes) >= self.numinitframes: cfg = self.sync['config'] cfg['cube'] = tuple( numpy.median(numpy.asarray(self.handsizes), axis=0).astype('int')) self.sync.update(config=cfg) self.state.value = self.STATE_RUN self.handsizes = [] if numpy.allclose(loc, 0): return numpy.zeros( (self.poseNet.input_images.get_shape()[1].value, self.poseNet.input_images.get_shape()[2].value), dtype='float32'), numpy.eye(3), loc else: crop, M, com = hd.cropArea3D( com=loc, size=self.sync['config']['cube'], dsize=(self.poseNet.input_images.get_shape()[1].value, self.poseNet.input_images.get_shape()[2].value)) com3D = self.importer.jointImgTo3D(com) sc = (self.sync['config']['cube'][2] / 2.) crop[crop == 0] = com3D[2] + sc crop.clip(com3D[2] - sc, com3D[2] + sc) crop -= com3D[2] crop /= sc return crop, M, com3D
def detect(self, frame): """ Detect the hand :param frame: image frame :return: cropped image, transformation, center """ hd = HandDetector(frame, self.config['fx'], self.config['fy'], importer=self.importer, refineNet=self.comrefNet) doHS = (self.state == self.STATE_INIT) if self.tracking and not numpy.allclose(self.lastcom, 0): loc, handsz = hd.track(self.lastcom, self.config['cube'], doHandSize=doHS) else: loc, handsz = hd.detect(size=self.config['cube'], doHandSize=doHS) self.lastcom = loc if self.state == self.STATE_INIT: self.handsizes.append(handsz) print numpy.median(numpy.asarray(self.handsizes), axis=0) else: self.handsizes = [] if self.state == self.STATE_INIT and len( self.handsizes) >= self.numinitframes: self.config['cube'] = tuple( numpy.median(numpy.asarray(self.handsizes), axis=0).astype('int')) self.state = self.STATE_RUN self.handsizes = [] if numpy.allclose(loc, 0): return numpy.zeros((self.poseNet.cfgParams.inputDim[2], self.poseNet.cfgParams.inputDim[3]), dtype='float32'), numpy.eye(3), loc else: crop, M, com = hd.cropArea3D( loc, size=self.config['cube'], dsize=(self.poseNet.layers[0].cfgParams.inputDim[2], self.poseNet.layers[0].cfgParams.inputDim[3])) com3D = self.importer.jointImgTo3D(com) crop[crop == 0] = com3D[2] + (self.config['cube'][2] / 2.) crop[crop >= com3D[2] + (self.config['cube'][2] / 2.)] = com3D[2] + ( self.config['cube'][2] / 2.) crop[crop <= com3D[2] - (self.config['cube'][2] / 2.)] = com3D[2] - ( self.config['cube'][2] / 2.) crop -= com3D[2] crop /= (self.config['cube'][2] / 2.) return crop, M, com3D
def pointcloud(self): dm = self.importer.loadDepthMap(self.path_prefix + self._seq.data[self.curFrame].fileName) cur3D = self.importer.jointImgTo3D(self.curData[0]).reshape((1, 3)) hd = HandDetector(dm, self.importer.fx, self.importer.fy, importer=self.importer) dpt, M, com = hd.cropArea3D(self.curData[0].reshape((3, )), size=self._seq.config['cube'], docom=False) self.hpe.plotResult3D(dpt, M, cur3D, cur3D, showGT=False, niceColors=False)
def loadSequence(self, seqName, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. train :param Nmax: maximum number of samples to load :return: returns named image sequence """ if cube is None: config = {'cube': self.default_cubes[seqName]} else: assert isinstance(cube, tuple) assert len(cube) == 3 config = {'cube': cube} pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, self.allJoints, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) if self.useCache: if os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) self.loadRefineNetLazy(self.refineNet) # Load the dataset objdir = '{}/{}/'.format(self.basepath, seqName) trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName) mat = scipy.io.loadmat(trainlabels) names = mat['joint_names'][0] joints3D = mat['joint_xyz'][0] joints2D = mat['joint_uvd'][0] if self.allJoints: eval_idxs = np.arange(36) else: eval_idxs = self.restrictedJointsEval self.numJoints = len(eval_idxs) txt = 'Loading {}'.format(seqName) pbar = pb.ProgressBar(maxval=joints3D.shape[0], widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() data = [] i=0 for line in range(joints3D.shape[0]): dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line+1) if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) i += 1 continue dpt = self.loadDepthMap(dptFileName) if hand is not None: raise NotImplementedError() # joints in image coordinates gtorig = np.zeros((self.numJoints, 3), np.float32) jt = 0 for ii in range(joints2D.shape[1]): if ii not in eval_idxs: continue gtorig[jt, 0] = joints2D[line, ii, 0] gtorig[jt, 1] = joints2D[line, ii, 1] gtorig[jt, 2] = joints2D[line, ii, 2] jt += 1 # normalized joints in 3D coordinates gt3Dorig = np.zeros((self.numJoints, 3), np.float32) jt = 0 for jj in range(joints3D.shape[1]): if jj not in eval_idxs: continue gt3Dorig[jt, 0] = joints3D[line, jj, 0] gt3Dorig[jt, 1] = joints3D[line, jj, 1] gt3Dorig[jt, 2] = joints3D[line, jj, 2] jt += 1 # print gt3D # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,'','')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self) if not hd.checkImage(1): print("Skipping image {}, no content".format(dptFileName)) i += 1 continue try: dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom) except UserWarning: print("Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = transformPoints2D(gtorig, M) # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','')) data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, '', self.sides[seqName], {})) pbar.update(i) i += 1 # early stop if len(data) >= Nmax: break pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. subject1 :param Nmax: maximum number of samples to load :return: returns named image sequence """ if (subSeq is not None) and (not isinstance(subSeq, list)): raise TypeError("subSeq must be None or list") if cube is None: config = {'cube': self.default_cubes[seqName]} else: assert isinstance(cube, tuple) assert len(cube) == 3 config = {'cube': cube} if subSeq is None: pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) else: pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) if self.useCache & os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) self.loadRefineNetLazy(self.refineNet) # Load the dataset objdir = '{}/{}/'.format(self.basepath, seqName) subdirs = sorted([name for name in os.listdir(objdir) if os.path.isdir(os.path.join(objdir, name))]) txt = 'Loading {}'.format(seqName) nImgs = sum([len(files) for r, d, files in os.walk(objdir)]) // 2 pbar = pb.ProgressBar(maxval=nImgs, widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() data = [] pi = 0 for subdir in subdirs: # check for subsequences and skip them if necessary subSeqName = '' if subSeq is not None: if subdir not in subSeq: continue subSeqName = subdir # iterate all subdirectories trainlabels = '{}/{}/joint.txt'.format(objdir, subdir) inputfile = open(trainlabels) # read number of samples nImgs = int(inputfile.readline()) for i in range(nImgs): # early stop if len(data) >= Nmax: break line = inputfile.readline() part = line.split(' ') dptFileName = '{}/{}/{}_depth.bin'.format(objdir, subdir, str(i).zfill(6)) if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) continue dpt = self.loadDepthMap(dptFileName) if hand is not None: raise NotImplementedError() # joints in image coordinates gt3Dorig = np.zeros((self.numJoints, 3), np.float32) for joint in range(gt3Dorig.shape[0]): for xyz in range(0, 3): gt3Dorig[joint, xyz] = part[joint*3+xyz] # invert axis # gt3Dorig[:, 0] *= (-1.) # gt3Dorig[:, 1] *= (-1.) gt3Dorig[:, 2] *= (-1.) # normalized joints in 3D coordinates gtorig = self.joints3DToImg(gt3Dorig) # print gt3D # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self) if not hd.checkImage(1.): print("Skipping image {}, no content".format(dptFileName)) continue try: dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom) except UserWarning: print("Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = transformPoints2D(gtorig, M) # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','',{})) data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, subSeqName, self.sides[seqName], {})) pbar.update(pi) pi += 1 inputfile.close() pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. train :param subSeq: list of subsequence names, e.g. 0, 45, 122-5 :param Nmax: maximum number of samples to load :return: returns named image sequence """ if (subSeq is not None) and (not isinstance(subSeq, list)): raise TypeError("subSeq must be None or list") if cube is None: config = {'cube': self.default_cubes[seqName]} else: assert isinstance(cube, tuple) assert len(cube) == 3 config = {'cube': cube} if subSeq is None: pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) else: pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) if self.useCache: if os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) # check for multiple subsequences if subSeq is not None: if len(subSeq) > 1: missing = False for i in range(len(subSeq)): if not os.path.isfile('{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None))): missing = True print("missing: {}".format(subSeq[i])) break if not missing: # load first data pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[0], HandDetector.detectionModeToString(docom, self.refineNet is not None)) print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, fullData, config) = cPickle.load(f) f.close() # load rest of data for i in range(1, len(subSeq)): pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None)) print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) fullData.extend(data) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(fullData) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, fullData[0:Nmax], config) else: return NamedImgSequence(seqName, fullData, config) self.loadRefineNetLazy(self.refineNet) # Load the dataset objdir = '{}/Depth/'.format(self.basepath) trainlabels = '{}/{}.txt'.format(self.basepath, seqName) inputfile = open(trainlabels) txt = 'Loading {}'.format(seqName) pbar = pb.ProgressBar(maxval=len(inputfile.readlines()), widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() inputfile.seek(0) data = [] i = 0 for line in inputfile: # early stop if len(data) >= Nmax: break part = line.split(' ') # check for subsequences and skip them if necessary subSeqName = '' if subSeq is not None: p = part[0].split('/') # handle original data (unrotated '0') separately if ('0' in subSeq) and len(p[0]) > 6: pass elif not('0' in subSeq) and len(p[0]) > 6: i += 1 continue elif (p[0] in subSeq) and len(p[0]) <= 6: pass elif not(p[0] in subSeq) and len(p[0]) <= 6: i += 1 continue if len(p[0]) <= 6: subSeqName = p[0] else: subSeqName = '0' dptFileName = '{}/{}'.format(objdir, part[0]) if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) i += 1 continue dpt = self.loadDepthMap(dptFileName) if hand is not None: raise NotImplementedError() # joints in image coordinates gtorig = np.zeros((self.numJoints, 3), np.float32) for joint in range(self.numJoints): for xyz in range(0, 3): gtorig[joint, xyz] = part[joint*3+xyz+1] # normalized joints in 3D coordinates gt3Dorig = self.jointsImgTo3D(gtorig) # print gt3D # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,0,dptFileName,subSeqName,'')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self) if not hd.checkImage(1): print("Skipping image {}, no content".format(dptFileName)) i += 1 continue try: dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom) except UserWarning: print("Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = transformPoints2D(gtorig, M) # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,subSeqName,'')) data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, subSeqName, 'left', {})) pbar.update(i) i += 1 inputfile.close() pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, camera=None, Nmax=float('inf'), shuffle=False, rng=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. subject1 :param Nmax: maximum number of samples to load :return: returns named image sequence """ config = {'cube': (220, 220, 220)} pickleCache = '{}/{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, camera) if self.useCache & os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not (np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) # Load the dataset objdir = os.path.join(self.basepath, seqName) if camera is not None: dflt = "c{0:02d}_*.npy".format(camera) else: dflt = '*.npy' dptFiles = sorted([ os.path.join(dirpath, f) for dirpath, dirnames, files in os.walk(objdir) for f in fnmatch.filter(files, dflt) ]) # dptFiles = sorted(glob.glob(os.path.join(objdir, '*exr'))) if camera is not None: lflt = "c{0:02d}_*anno_blender.txt".format(camera) else: lflt = '*anno_blender.txt' labelFiles = sorted([ os.path.join(dirpath, f) for dirpath, dirnames, files in os.walk(objdir) for f in fnmatch.filter(files, lflt) ]) #labelFiles = sorted(glob.glob(os.path.join(objdir, '*txt'))) # sync lists newdpt = [] newlbl = [] if camera is not None: number_idx = 1 else: number_idx = 0 labelIdx = [ ''.join(os.path.basename(lbl).split('_')[number_idx]) for lbl in labelFiles ] for dpt in dptFiles: if ''.join( os.path.basename(dpt).split('_')[number_idx]) in labelIdx: newdpt.append(dpt) newlbl.append(labelFiles[labelIdx.index(''.join( os.path.basename(dpt).split('_')[number_idx]))]) dptFiles = newdpt labelFiles = newlbl assert len(dptFiles) == len(labelFiles) txt = 'Loading {}'.format(seqName) nImgs = len(dptFiles) pbar = pb.ProgressBar(maxval=nImgs, widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() data = [] for i in xrange(nImgs): labelFileName = labelFiles[i] dptFileName = dptFiles[i] if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) continue dpt = self.loadDepthMap(dptFileName) if not os.path.isfile(labelFileName): print("File {} does not exist!".format(labelFileName)) continue # joints in image coordinates gtorig = np.genfromtxt(labelFileName) gtorig = gtorig.reshape((gtorig.shape[0] / 3, 3)) gtorig[:, 2] *= 1000. # normalized joints in 3D coordinates gt3Dorig = self.jointsImgTo3D(gtorig) ### Shorting the tip bone, BUGFIX tips = [3, 8, 13, 18, 23] shrink_factor = 0.85 for joint_idx in tips: t = gt3Dorig[joint_idx, :] - gt3Dorig[joint_idx - 1, :] gt3Dorig[joint_idx, :] = gt3Dorig[joint_idx - 1, :] + shrink_factor * t gtorig = self.joints3DToImg(gt3Dorig) ### # print gtorig, gt3Dorig # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,0,0,dptFileName,'','',{})) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, importer=self) if not hd.checkImage(1.): print("Skipping image {}, no content".format(dptFileName)) continue try: dpt, M, com = hd.cropArea3D(gtorig[10], size=config['cube']) except UserWarning: print( "Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = np.zeros((gt3Dorig.shape[0], 3), np.float32) for joint in range(gtcrop.shape[0]): t = transformPoint2D(gtorig[joint], M) gtcrop[joint, 0] = t[0] gtcrop[joint, 1] = t[1] gtcrop[joint, 2] = gtorig[joint, 2] # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','')) data.append( DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, '', self.sides[seqName], {})) pbar.update(i) # early stop if len(data) >= Nmax: break pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, allJoints=False, Nmax=float('inf'), shuffle=False, rng=None, docom=False, rotation=False, dsize=(128, 128)): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. train :param subSeq: list of subsequence names, e.g. 0, 45, 122-5 :param Nmax: maximum number of samples to load :return: returns named image sequence """ config = {'cube': (300, 300, 300)} config['cube'] = [s * self.scales[seqName] for s in config['cube']] refineNet = None pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format( self.cacheDir, self.__class__.__name__, seqName, allJoints, docom) if self.useCache: if os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not (np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) # Load the dataset objdir = '{}/{}/'.format(self.basepath, seqName) trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName) mat = scipy.io.loadmat(trainlabels) names = mat['joint_names'][0] joints3D = mat['joint_xyz'][0] joints2D = mat['joint_uvd'][0] if allJoints: eval_idxs = np.arange(36) else: eval_idxs = self.restrictedJointsEval self.numJoints = len(eval_idxs) txt = 'Loading {}'.format(seqName) pbar = pb.ProgressBar(maxval=joints3D.shape[0], widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() data = [] if rotation: data_rotation = [] i = 0 for line in range(joints3D.shape[0]): dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line + 1) print dptFileName # augment the training dataset if rotation: assert seqName == 'train', 'we only rotate the training data' # 90', 180', or 270' rotation degree rotation_degree = np.random.randint(1, 4) * 90 if not os.path.isfile(dptFileName): if seqName == 'test_1' and line <= 2439 and line >= 0: print("File {} does not exist!".format(dptFileName)) i += 1 continue elif seqName == 'test_2' and line >= 2440 and line <= 8251: print("File {} does not exist!".format(dptFileName)) i += 1 continue else: i += 1 continue dpt = self.loadDepthMap(dptFileName) h, w = dpt.shape if rotation: import math def rotate_about_center(src, angle, scale=1.): w = src.shape[1] h = src.shape[0] rangle = np.deg2rad(angle) # angle in radians # now calculate new image width and height nw = (abs(np.sin(rangle) * h) + abs(np.cos(rangle) * w)) * scale nh = (abs(np.cos(rangle) * h) + abs(np.sin(rangle) * w)) * scale # ask OpenCV for the rotation matrix rot_mat = cv2.getRotationMatrix2D((nw * 0.5, nh * 0.5), angle, scale) # calculate the move from the old center to the new center combined # with the rotation rot_move = np.dot( rot_mat, np.array([(nw - w) * 0.5, (nh - h) * 0.5, 0])) # the move only affects the translation, so update the translation # part of the transform rot_mat[0, 2] += rot_move[0] rot_mat[1, 2] += rot_move[1] return cv2.warpAffine( src, rot_mat, (int(math.ceil(nw)), int(math.ceil(nh))), flags=cv2.INTER_LANCZOS4), rot_mat # get the rotation matrix for warpAffine dpt_rotation, rotMat = rotate_about_center( dpt, rotation_degree) # joints in image coordinates gtorig = np.zeros((self.numJoints, 3), np.float32) jt = 0 for ii in range(joints2D.shape[1]): if ii not in eval_idxs: continue gtorig[jt, 0] = joints2D[line, ii, 0] gtorig[jt, 1] = joints2D[line, ii, 1] gtorig[jt, 2] = joints2D[line, ii, 2] jt += 1 if rotation: gtorig_rotation = np.zeros((self.numJoints, 3), np.float32) for i, joint in enumerate(gtorig): m11 = rotMat[0, 0] m12 = rotMat[0, 1] m13 = rotMat[0, 2] m21 = rotMat[1, 0] m22 = rotMat[1, 1] m23 = rotMat[1, 2] gtorig_rotation[ i, 0] = gtorig[i, 0] * m11 + gtorig[i, 1] * m12 + m13 gtorig_rotation[ i, 1] = gtorig[i, 0] * m21 + gtorig[i, 1] * m22 + m23 gtorig_rotation[i, 2] = gtorig[i, 2] # normalized joints in 3D coordinates gt3Dorig = np.zeros((self.numJoints, 3), np.float32) jt = 0 for jj in range(joints3D.shape[1]): if jj not in eval_idxs: continue gt3Dorig[jt, 0] = joints3D[line, jj, 0] gt3Dorig[jt, 1] = joints3D[line, jj, 1] gt3Dorig[jt, 2] = joints3D[line, jj, 2] jt += 1 # transform from gtorign gt3Dorig_rotation = np.zeros((self.numJoints, 3), np.float32) if rotation: gt3Dorig_rotation = self.jointsImgTo3D(gtorig_rotation) #print gt3D #print("{}".format(gtorig)) #self.showAnnotatedDepth(ICVLFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,'')) #print("{}".format(gtorig_rotation)) #self.showAnnotatedDepth(ICVLFrame(dpt_rotation,gtorig_rotation,gtorig_rotation,0,gt3Dorig_rotation,gt3Dorig_rotation,0,dptFileName,'')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=refineNet, importer=self) if not hd.checkImage(1): print("Skipping image {}, no content".format(dptFileName)) i += 1 continue try: if allJoints: dpt, M, com = hd.cropArea3D(gtorig[34], size=config['cube'], docom=docom, dsize=dsize) else: dpt, M, com = hd.cropArea3D(gtorig[13], size=config['cube'], docom=docom, dsize=dsize) except UserWarning: print( "Skipping image {}, no hand detected".format(dptFileName)) continue if rotation: # Detect hand hd_rotation = HandDetector(dpt_rotation, self.fx, self.fy, refineNet=refineNet, importer=self) try: if allJoints: dpt_rotation, M_rotation, com_rotation = hd_rotation.cropArea3D( gtorig_rotation[34], size=config['cube'], docom=docom, dsize=dsize) else: dpt_rotation, M_rotation, com_rotation = hd_rotation.cropArea3D( gtorig_rotation[13], size=config['cube'], docom=docom, dsize=dsize) except UserWarning: print("Skipping image {}, no hand detected".format( dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = np.zeros((gtorig.shape[0], 3), np.float32) for joint in range(gtorig.shape[0]): t = transformPoint2D(gtorig[joint], M) gtcrop[joint, 0] = t[0] gtcrop[joint, 1] = t[1] gtcrop[joint, 2] = gtorig[joint, 2] # create 3D voxel dpt3D = np.zeros((8, dpt.shape[0], dpt.shape[1]), dtype=np.uint8) sorted_dpt = np.sort(dpt, axis=None) iii = np.where(sorted_dpt != 0)[0][0] min_d = sorted_dpt[iii] max_d = np.max(dpt) slice_range = [] slice_step = (max_d - min_d) / 8 for i in range(9): slice_range.append(min_d + slice_step * i) slice_range = np.array(slice_range) lh, lw = np.where(dpt != 0) for ii in xrange(lh.shape[0]): ih, iw = lh[ii], lw[ii] dptValue = dpt[ih, iw] slice_layer = np.where(dptValue >= slice_range)[0][-1] if slice_layer == 8: slice_layer = 7 dpt3D[slice_layer, ih, iw] = 1 if rotation: com3D_rotation = self.jointImgTo3D(com_rotation) gt3Dcrop_rotation = gt3Dorig_rotation - com3D_rotation # normalize to com gtcrop_rotation = np.zeros((gtorig_rotation.shape[0], 3), np.float32) for joint in range(gtorig_rotation.shape[0]): t = transformPoint2D(gtorig_rotation[joint], M_rotation) gtcrop_rotation[joint, 0] = t[0] gtcrop_rotation[joint, 1] = t[1] gtcrop_rotation[joint, 2] = gtorig_rotation[joint, 2] dpt3D_rotation = np.zeros((8, dpt.shape[0], dpt.shape[1]), dtype=np.uint8) sorted_dpt_rotation = np.sort(dpt_rotation, axis=None) iii = np.where(sorted_dpt_rotation != 0)[0][0] min_d = sorted_dpt_rotation[iii] max_d = np.max(dpt_rotation) slice_range = [] slice_step = (max_d - min_d) / 8 for i in range(9): slice_range.append(min_d + slice_step * i) slice_range = np.array(slice_range) lh, lw = np.where(dpt_rotation != 0) for ii in xrange(lh.shape[0]): ih, iw = lh[ii], lw[ii] dptValue = dpt_rotation[ih, iw] slice_layer = np.where(dptValue >= slice_range)[0][-1] if slice_layer == 8: slice_layer = 7 dpt3D_rotation[slice_layer, ih, iw] = 1 # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(ICVLFrame(dpt,dpt3D,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'')) # if rotation: # print("{}".format(gt3Dorig_rotation)) # self.showAnnotatedDepth(ICVLFrame(dpt_rotation,dpt3D_rotation,gtorig_rotation,gtcrop_rotation,M_rotation,gt3Dorig_rotation,gt3Dcrop_rotation,com3D_rotation,dptFileName,'')) # visualize 3D # from mpl_toolkits.mplot3d import Axes3D # fig = plt.figure() # ax = fig.add_subplot(111,projection='3d') # d,x,y = np.where(dpt3D == 1) # ax.scatter(x,y,8 - d) # # ax.view_init(0,0) # ax.set_xlabel('x') # ax.set_ylabel('y') # ax.set_zlabel('d') # plt.show() data.append( ICVLFrame(dpt.astype(np.float32), dpt3D, gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, '')) if rotation: data_rotation.append( ICVLFrame(dpt_rotation.astype(np.float32), dpt3D_rotation, gtorig_rotation, gtcrop_rotation, M_rotation, gt3Dorig_rotation, gt3Dcrop_rotation, com3D_rotation, dptFileName, '')) pbar.update(i) i += 1 # early stop if len(data) >= Nmax: break pbar.finish() # augmentation if rotation: data.extend(data_rotation) print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)