def loadSequence(self, seqName, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. train :param Nmax: maximum number of samples to load :return: returns named image sequence """ if cube is None: config = {'cube': self.default_cubes[seqName]} else: assert isinstance(cube, tuple) assert len(cube) == 3 config = {'cube': cube} pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, self.allJoints, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) if self.useCache: if os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) self.loadRefineNetLazy(self.refineNet) # Load the dataset objdir = '{}/{}/'.format(self.basepath, seqName) trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName) mat = scipy.io.loadmat(trainlabels) names = mat['joint_names'][0] joints3D = mat['joint_xyz'][0] joints2D = mat['joint_uvd'][0] if self.allJoints: eval_idxs = np.arange(36) else: eval_idxs = self.restrictedJointsEval self.numJoints = len(eval_idxs) txt = 'Loading {}'.format(seqName) pbar = pb.ProgressBar(maxval=joints3D.shape[0], widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() data = [] i=0 for line in range(joints3D.shape[0]): dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line+1) if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) i += 1 continue dpt = self.loadDepthMap(dptFileName) if hand is not None: raise NotImplementedError() # joints in image coordinates gtorig = np.zeros((self.numJoints, 3), np.float32) jt = 0 for ii in range(joints2D.shape[1]): if ii not in eval_idxs: continue gtorig[jt, 0] = joints2D[line, ii, 0] gtorig[jt, 1] = joints2D[line, ii, 1] gtorig[jt, 2] = joints2D[line, ii, 2] jt += 1 # normalized joints in 3D coordinates gt3Dorig = np.zeros((self.numJoints, 3), np.float32) jt = 0 for jj in range(joints3D.shape[1]): if jj not in eval_idxs: continue gt3Dorig[jt, 0] = joints3D[line, jj, 0] gt3Dorig[jt, 1] = joints3D[line, jj, 1] gt3Dorig[jt, 2] = joints3D[line, jj, 2] jt += 1 # print gt3D # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,'','')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self) if not hd.checkImage(1): print("Skipping image {}, no content".format(dptFileName)) i += 1 continue try: dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom) except UserWarning: print("Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = transformPoints2D(gtorig, M) # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','')) data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, '', self.sides[seqName], {})) pbar.update(i) i += 1 # early stop if len(data) >= Nmax: break pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
test_label[it] = gt3Dcrops_test[it] / (cubes_test[it][0] / 2.) test_data = np.expand_dims(test_data, 3) test_label = np.reshape(test_label, (-1, 21 * 3)) hd_edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]] visual = False visual_aug = False if visual == True: import matplotlib.pyplot as plt for i in range(0, test_num, 10): plt.imshow(imgs_test[i], cmap='gray') gt3D = gt3Dcrops_test[i] + coms_test[i] jtI = transformPoints2D(di.joints3DToImg(gt3D), Ms_test[i]) plt.scatter(jtI[:, 0], jtI[:, 1]) for edge in hd_edges: plt.plot(jtI[:, 0][edge], jtI[:, 1][edge], c='r') plt.pause(0.001) plt.cla() print cubes_test[i] hd = HandDetector(imgs_train[0].copy(), abs(di.fx), abs(di.fy), importer=di, refineNet=None) inputs = tf.placeholder(dtype=tf.float32, shape=(None, 96, 96, 1)) label = tf.placeholder(dtype=tf.float32, shape=(None, 21 * 3)) gt_ht = tf.placeholder(dtype=tf.float32, shape=(None, 24, 24, 21))
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. subject1 :param Nmax: maximum number of samples to load :return: returns named image sequence """ if (subSeq is not None) and (not isinstance(subSeq, list)): raise TypeError("subSeq must be None or list") if cube is None: config = {'cube': self.default_cubes[seqName]} else: assert isinstance(cube, tuple) assert len(cube) == 3 config = {'cube': cube} if subSeq is None: pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) else: pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) if self.useCache & os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) self.loadRefineNetLazy(self.refineNet) # Load the dataset objdir = '{}/{}/'.format(self.basepath, seqName) subdirs = sorted([name for name in os.listdir(objdir) if os.path.isdir(os.path.join(objdir, name))]) txt = 'Loading {}'.format(seqName) nImgs = sum([len(files) for r, d, files in os.walk(objdir)]) // 2 pbar = pb.ProgressBar(maxval=nImgs, widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() data = [] pi = 0 for subdir in subdirs: # check for subsequences and skip them if necessary subSeqName = '' if subSeq is not None: if subdir not in subSeq: continue subSeqName = subdir # iterate all subdirectories trainlabels = '{}/{}/joint.txt'.format(objdir, subdir) inputfile = open(trainlabels) # read number of samples nImgs = int(inputfile.readline()) for i in range(nImgs): # early stop if len(data) >= Nmax: break line = inputfile.readline() part = line.split(' ') dptFileName = '{}/{}/{}_depth.bin'.format(objdir, subdir, str(i).zfill(6)) if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) continue dpt = self.loadDepthMap(dptFileName) if hand is not None: raise NotImplementedError() # joints in image coordinates gt3Dorig = np.zeros((self.numJoints, 3), np.float32) for joint in range(gt3Dorig.shape[0]): for xyz in range(0, 3): gt3Dorig[joint, xyz] = part[joint*3+xyz] # invert axis # gt3Dorig[:, 0] *= (-1.) # gt3Dorig[:, 1] *= (-1.) gt3Dorig[:, 2] *= (-1.) # normalized joints in 3D coordinates gtorig = self.joints3DToImg(gt3Dorig) # print gt3D # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self) if not hd.checkImage(1.): print("Skipping image {}, no content".format(dptFileName)) continue try: dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom) except UserWarning: print("Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = transformPoints2D(gtorig, M) # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','',{})) data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, subSeqName, self.sides[seqName], {})) pbar.update(pi) pi += 1 inputfile.close() pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None): """ Load an image sequence from the dataset :param seqName: sequence name, e.g. train :param subSeq: list of subsequence names, e.g. 0, 45, 122-5 :param Nmax: maximum number of samples to load :return: returns named image sequence """ if (subSeq is not None) and (not isinstance(subSeq, list)): raise TypeError("subSeq must be None or list") if cube is None: config = {'cube': self.default_cubes[seqName]} else: assert isinstance(cube, tuple) assert len(cube) == 3 config = {'cube': cube} if subSeq is None: pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) else: pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0]) if self.useCache: if os.path.isfile(pickleCache): print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, data[0:Nmax], config) else: return NamedImgSequence(seqName, data, config) # check for multiple subsequences if subSeq is not None: if len(subSeq) > 1: missing = False for i in range(len(subSeq)): if not os.path.isfile('{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None))): missing = True print("missing: {}".format(subSeq[i])) break if not missing: # load first data pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[0], HandDetector.detectionModeToString(docom, self.refineNet is not None)) print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, fullData, config) = cPickle.load(f) f.close() # load rest of data for i in range(1, len(subSeq)): pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None)) print("Loading cache data from {}".format(pickleCache)) f = open(pickleCache, 'rb') (seqName, data, config) = cPickle.load(f) fullData.extend(data) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(fullData) if not(np.isinf(Nmax)): return NamedImgSequence(seqName, fullData[0:Nmax], config) else: return NamedImgSequence(seqName, fullData, config) self.loadRefineNetLazy(self.refineNet) # Load the dataset objdir = '{}/Depth/'.format(self.basepath) trainlabels = '{}/{}.txt'.format(self.basepath, seqName) inputfile = open(trainlabels) txt = 'Loading {}'.format(seqName) pbar = pb.ProgressBar(maxval=len(inputfile.readlines()), widgets=[txt, pb.Percentage(), pb.Bar()]) pbar.start() inputfile.seek(0) data = [] i = 0 for line in inputfile: # early stop if len(data) >= Nmax: break part = line.split(' ') # check for subsequences and skip them if necessary subSeqName = '' if subSeq is not None: p = part[0].split('/') # handle original data (unrotated '0') separately if ('0' in subSeq) and len(p[0]) > 6: pass elif not('0' in subSeq) and len(p[0]) > 6: i += 1 continue elif (p[0] in subSeq) and len(p[0]) <= 6: pass elif not(p[0] in subSeq) and len(p[0]) <= 6: i += 1 continue if len(p[0]) <= 6: subSeqName = p[0] else: subSeqName = '0' dptFileName = '{}/{}'.format(objdir, part[0]) if not os.path.isfile(dptFileName): print("File {} does not exist!".format(dptFileName)) i += 1 continue dpt = self.loadDepthMap(dptFileName) if hand is not None: raise NotImplementedError() # joints in image coordinates gtorig = np.zeros((self.numJoints, 3), np.float32) for joint in range(self.numJoints): for xyz in range(0, 3): gtorig[joint, xyz] = part[joint*3+xyz+1] # normalized joints in 3D coordinates gt3Dorig = self.jointsImgTo3D(gtorig) # print gt3D # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,0,dptFileName,subSeqName,'')) # Detect hand hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self) if not hd.checkImage(1): print("Skipping image {}, no content".format(dptFileName)) i += 1 continue try: dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom) except UserWarning: print("Skipping image {}, no hand detected".format(dptFileName)) continue com3D = self.jointImgTo3D(com) gt3Dcrop = gt3Dorig - com3D # normalize to com gtcrop = transformPoints2D(gtorig, M) # print("{}".format(gt3Dorig)) # self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,subSeqName,'')) data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName, subSeqName, 'left', {})) pbar.update(i) i += 1 inputfile.close() pbar.finish() print("Loaded {} samples.".format(len(data))) if self.useCache: print("Save cache data to {}".format(pickleCache)) f = open(pickleCache, 'wb') cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL) f.close() # shuffle data if shuffle and rng is not None: print("Shuffling") rng.shuffle(data) return NamedImgSequence(seqName, data, config)
cPickle.dump(joints, open( "./eval/{}/result_{}_{}.pkl".format( eval_prefix, os.path.split(__file__)[1], eval_prefix), "wb"), protocol=cPickle.HIGHEST_PROTOCOL) print "Testing baseline" ################################# # BASELINE # Load the evaluation data_baseline = di.loadBaseline('../data/NYU/test/test_predictions.mat', numpy.asarray(gt3D)) hpe_base = NYUHandposeEvaluation(gt3D, data_baseline) hpe_base.subfolder += '/' + eval_prefix + '/' print("Mean error: {}mm".format(hpe_base.getMeanError())) hpe.plotEvaluation(eval_prefix, methodName='Our regr', baseline=[('Tompson et al.', hpe_base)]) ind = 0 for i in testSeqs[0].data: if ind % 20 != 0: ind += 1 continue jtI = transformPoints2D(di.joints3DToImg(joints[ind]), i.T) hpe.plotResult(i.dpt, i.gtcrop, jtI, "{}_{}".format(eval_prefix, ind)) ind += 1
subM = Ms_train[start:end].copy() subgt3Dcrop = gt3Dcrops_train[start:end].copy() resdata=np.ones_like(subdata) resgt3D=np.ones_like(subgt3Dcrop) hts=np.zeros(shape=(batch_size,24,24,14)) jtIs=np.zeros(shape=(batch_size,42)) for idx in range(batch_size): dm = norm_dm(subdata[idx], subcom[idx], subcube[idx]) s = augmentCrop(dm, subgt3Dcrop[idx], di_train.joint3DToImg(subcom[idx]), subcube[idx], subM[idx],['com', 'rot', 'none','sc'], hd, False, rng=rng) resdata[idx] = s[0] resgt3D[idx] = s[2] mode=s[7] gt3D_ = resgt3D[idx] * subcube[idx][0] / 2. + subcom[idx] jtI_ = transformPoints2D(di_train.joints3DToImg(gt3D_), subM[idx]) jtI_=np.reshape(jtI_,(1,42)) ht_=joints_heatmap_gen([1],jtI_,(24,24),points=14) hts[idx]=np.transpose(ht_,(0,2,3,1))/255. feed_dict = {inputs: resdata.reshape(-1, 96, 96, 1), label: resgt3D.reshape(-1, 42), gt_ht:hts, is_train:True, kp:kpv } _, losses,summs,steps = sess.run([optimizer, loss,summ,global_step], feed_dict=feed_dict) #writer.add_summary(summs,steps) print itrain,start/batch_size,losses,mode
f = open('res_msra_{}.txt'.format(MID), 'a+') for i in range(pred_hands.shape[0]): uvds=di.joints3DToImg(pred_hands[i]) uvds=np.reshape(uvds,(1,63)) for j in range(63): f.write(str(round(uvds[0,j],4))) f.write(' ') f.write('\n') f.close() if visual==True: import matplotlib.pyplot as plt for i in range(0,test_num,10): plt.imshow(np.squeeze(test_data[i]), cmap='gray') jtIp = transformPoints2D(di.joints3DToImg(pred_hands[i]), Ms_test[i]) plt.scatter(jtIp[:, 0], jtIp[:, 1], c='r') jtIt = transformPoints2D(di.joints3DToImg(gt_hands[i]), Ms_test[i]) plt.scatter(jtIt[:, 0], jtIt[:, 1], c='b') for edge in hd_edges: plt.plot(jtIp[:, 0][edge], jtIp[:, 1][edge], c='r',LineWidth=1.5) plt.plot(jtIt[:, 0][edge], jtIt[:, 1][edge], c='b',LineWidth=1.5) plt.pause(0.01) plt.axis('off') plt.savefig("../../image/MSRA/img_{}.png".format(i)) plt.cla()