def loadSequence(self, seqName, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if cube is None:
config = {'cube': self.default_cubes[seqName]}
else:
assert isinstance(cube, tuple)
assert len(cube) == 3
config = {'cube': cube}
pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, self.allJoints, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
self.loadRefineNetLazy(self.refineNet)
# Load the dataset
objdir = '{}/{}/'.format(self.basepath, seqName)
trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName)
mat = scipy.io.loadmat(trainlabels)
names = mat['joint_names'][0]
joints3D = mat['joint_xyz'][0]
joints2D = mat['joint_uvd'][0]
if self.allJoints:
eval_idxs = np.arange(36)
else:
eval_idxs = self.restrictedJointsEval
self.numJoints = len(eval_idxs)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=joints3D.shape[0], widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
data = []
i=0
for line in range(joints3D.shape[0]):
dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line+1)
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
if hand is not None:
raise NotImplementedError()
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for ii in range(joints2D.shape[1]):
if ii not in eval_idxs:
continue
gtorig[jt, 0] = joints2D[line, ii, 0]
gtorig[jt, 1] = joints2D[line, ii, 1]
gtorig[jt, 2] = joints2D[line, ii, 2]
jt += 1
# normalized joints in 3D coordinates
gt3Dorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for jj in range(joints3D.shape[1]):
if jj not in eval_idxs:
continue
gt3Dorig[jt, 0] = joints3D[line, jj, 0]
gt3Dorig[jt, 1] = joints3D[line, jj, 1]
gt3Dorig[jt, 2] = joints3D[line, jj, 2]
jt += 1
# print gt3D
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,'',''))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom)
except UserWarning:
print("Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = transformPoints2D(gtorig, M)
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'',''))
data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName,
'', self.sides[seqName], {}))
pbar.update(i)
i += 1
# early stop
if len(data) >= Nmax:
break
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
test_label[it] = gt3Dcrops_test[it] / (cubes_test[it][0] / 2.)
test_data = np.expand_dims(test_data, 3)
test_label = np.reshape(test_label, (-1, 21 * 3))
hd_edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8],
[0, 9], [9, 10], [10, 11], [11, 12], [0, 13], [13, 14], [14, 15],
[15, 16], [0, 17], [17, 18], [18, 19], [19, 20]]
visual = False
visual_aug = False
if visual == True:
import matplotlib.pyplot as plt
for i in range(0, test_num, 10):
plt.imshow(imgs_test[i], cmap='gray')
gt3D = gt3Dcrops_test[i] + coms_test[i]
jtI = transformPoints2D(di.joints3DToImg(gt3D), Ms_test[i])
plt.scatter(jtI[:, 0], jtI[:, 1])
for edge in hd_edges:
plt.plot(jtI[:, 0][edge], jtI[:, 1][edge], c='r')
plt.pause(0.001)
plt.cla()
print cubes_test[i]
hd = HandDetector(imgs_train[0].copy(),
abs(di.fx),
abs(di.fy),
importer=di,
refineNet=None)
inputs = tf.placeholder(dtype=tf.float32, shape=(None, 96, 96, 1))
label = tf.placeholder(dtype=tf.float32, shape=(None, 21 * 3))
gt_ht = tf.placeholder(dtype=tf.float32, shape=(None, 24, 24, 21))
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. subject1
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if (subSeq is not None) and (not isinstance(subSeq, list)):
raise TypeError("subSeq must be None or list")
if cube is None:
config = {'cube': self.default_cubes[seqName]}
else:
assert isinstance(cube, tuple)
assert len(cube) == 3
config = {'cube': cube}
if subSeq is None:
pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
else:
pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
if self.useCache & os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
self.loadRefineNetLazy(self.refineNet)
# Load the dataset
objdir = '{}/{}/'.format(self.basepath, seqName)
subdirs = sorted([name for name in os.listdir(objdir) if os.path.isdir(os.path.join(objdir, name))])
txt = 'Loading {}'.format(seqName)
nImgs = sum([len(files) for r, d, files in os.walk(objdir)]) // 2
pbar = pb.ProgressBar(maxval=nImgs, widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
data = []
pi = 0
for subdir in subdirs:
# check for subsequences and skip them if necessary
subSeqName = ''
if subSeq is not None:
if subdir not in subSeq:
continue
subSeqName = subdir
# iterate all subdirectories
trainlabels = '{}/{}/joint.txt'.format(objdir, subdir)
inputfile = open(trainlabels)
# read number of samples
nImgs = int(inputfile.readline())
for i in range(nImgs):
# early stop
if len(data) >= Nmax:
break
line = inputfile.readline()
part = line.split(' ')
dptFileName = '{}/{}/{}_depth.bin'.format(objdir, subdir, str(i).zfill(6))
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
continue
dpt = self.loadDepthMap(dptFileName)
if hand is not None:
raise NotImplementedError()
# joints in image coordinates
gt3Dorig = np.zeros((self.numJoints, 3), np.float32)
for joint in range(gt3Dorig.shape[0]):
for xyz in range(0, 3):
gt3Dorig[joint, xyz] = part[joint*3+xyz]
# invert axis
# gt3Dorig[:, 0] *= (-1.)
# gt3Dorig[:, 1] *= (-1.)
gt3Dorig[:, 2] *= (-1.)
# normalized joints in 3D coordinates
gtorig = self.joints3DToImg(gt3Dorig)
# print gt3D
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,com3D,dptFileName,'',''))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self)
if not hd.checkImage(1.):
print("Skipping image {}, no content".format(dptFileName))
continue
try:
dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom)
except UserWarning:
print("Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = transformPoints2D(gtorig, M)
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','',{}))
data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D,
dptFileName, subSeqName, self.sides[seqName], {}))
pbar.update(pi)
pi += 1
inputfile.close()
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None,
hand=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param subSeq: list of subsequence names, e.g. 0, 45, 122-5
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if (subSeq is not None) and (not isinstance(subSeq, list)):
raise TypeError("subSeq must be None or list")
if cube is None:
config = {'cube': self.default_cubes[seqName]}
else:
assert isinstance(cube, tuple)
assert len(cube) == 3
config = {'cube': cube}
if subSeq is None:
pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
else:
pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# check for multiple subsequences
if subSeq is not None:
if len(subSeq) > 1:
missing = False
for i in range(len(subSeq)):
if not os.path.isfile('{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None))):
missing = True
print("missing: {}".format(subSeq[i]))
break
if not missing:
# load first data
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[0], HandDetector.detectionModeToString(docom, self.refineNet is not None))
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, fullData, config) = cPickle.load(f)
f.close()
# load rest of data
for i in range(1, len(subSeq)):
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None))
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
fullData.extend(data)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(fullData)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, fullData[0:Nmax], config)
else:
return NamedImgSequence(seqName, fullData, config)
self.loadRefineNetLazy(self.refineNet)
# Load the dataset
objdir = '{}/Depth/'.format(self.basepath)
trainlabels = '{}/{}.txt'.format(self.basepath, seqName)
inputfile = open(trainlabels)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=len(inputfile.readlines()), widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
inputfile.seek(0)
data = []
i = 0
for line in inputfile:
# early stop
if len(data) >= Nmax:
break
part = line.split(' ')
# check for subsequences and skip them if necessary
subSeqName = ''
if subSeq is not None:
p = part[0].split('/')
# handle original data (unrotated '0') separately
if ('0' in subSeq) and len(p[0]) > 6:
pass
elif not('0' in subSeq) and len(p[0]) > 6:
i += 1
continue
elif (p[0] in subSeq) and len(p[0]) <= 6:
pass
elif not(p[0] in subSeq) and len(p[0]) <= 6:
i += 1
continue
if len(p[0]) <= 6:
subSeqName = p[0]
else:
subSeqName = '0'
dptFileName = '{}/{}'.format(objdir, part[0])
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
if hand is not None:
raise NotImplementedError()
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
for joint in range(self.numJoints):
for xyz in range(0, 3):
gtorig[joint, xyz] = part[joint*3+xyz+1]
# normalized joints in 3D coordinates
gt3Dorig = self.jointsImgTo3D(gtorig)
# print gt3D
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,0,dptFileName,subSeqName,''))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom)
except UserWarning:
print("Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = transformPoints2D(gtorig, M)
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,subSeqName,''))
data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName,
subSeqName, 'left', {}))
pbar.update(i)
i += 1
inputfile.close()
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
cPickle.dump(joints,
open(
"./eval/{}/result_{}_{}.pkl".format(
eval_prefix,
os.path.split(__file__)[1], eval_prefix), "wb"),
protocol=cPickle.HIGHEST_PROTOCOL)
print "Testing baseline"
#################################
# BASELINE
# Load the evaluation
data_baseline = di.loadBaseline('../data/NYU/test/test_predictions.mat',
numpy.asarray(gt3D))
hpe_base = NYUHandposeEvaluation(gt3D, data_baseline)
hpe_base.subfolder += '/' + eval_prefix + '/'
print("Mean error: {}mm".format(hpe_base.getMeanError()))
hpe.plotEvaluation(eval_prefix,
methodName='Our regr',
baseline=[('Tompson et al.', hpe_base)])
ind = 0
for i in testSeqs[0].data:
if ind % 20 != 0:
ind += 1
continue
jtI = transformPoints2D(di.joints3DToImg(joints[ind]), i.T)
hpe.plotResult(i.dpt, i.gtcrop, jtI, "{}_{}".format(eval_prefix, ind))
ind += 1
subM = Ms_train[start:end].copy()
subgt3Dcrop = gt3Dcrops_train[start:end].copy()
resdata=np.ones_like(subdata)
resgt3D=np.ones_like(subgt3Dcrop)
hts=np.zeros(shape=(batch_size,24,24,14))
jtIs=np.zeros(shape=(batch_size,42))
for idx in range(batch_size):
dm = norm_dm(subdata[idx], subcom[idx], subcube[idx])
s = augmentCrop(dm, subgt3Dcrop[idx], di_train.joint3DToImg(subcom[idx]),
subcube[idx], subM[idx],['com', 'rot', 'none','sc'], hd, False, rng=rng)
resdata[idx] = s[0]
resgt3D[idx] = s[2]
mode=s[7]
gt3D_ = resgt3D[idx] * subcube[idx][0] / 2. + subcom[idx]
jtI_ = transformPoints2D(di_train.joints3DToImg(gt3D_), subM[idx])
jtI_=np.reshape(jtI_,(1,42))
ht_=joints_heatmap_gen([1],jtI_,(24,24),points=14)
hts[idx]=np.transpose(ht_,(0,2,3,1))/255.
feed_dict = {inputs: resdata.reshape(-1, 96, 96, 1),
label: resgt3D.reshape(-1, 42),
gt_ht:hts,
is_train:True,
kp:kpv
}
_, losses,summs,steps = sess.run([optimizer, loss,summ,global_step], feed_dict=feed_dict)
#writer.add_summary(summs,steps)
print itrain,start/batch_size,losses,mode
f = open('res_msra_{}.txt'.format(MID), 'a+')
for i in range(pred_hands.shape[0]):
uvds=di.joints3DToImg(pred_hands[i])
uvds=np.reshape(uvds,(1,63))
for j in range(63):
f.write(str(round(uvds[0,j],4)))
f.write(' ')
f.write('\n')
f.close()
if visual==True:
import matplotlib.pyplot as plt
for i in range(0,test_num,10):
plt.imshow(np.squeeze(test_data[i]), cmap='gray')
jtIp = transformPoints2D(di.joints3DToImg(pred_hands[i]), Ms_test[i])
plt.scatter(jtIp[:, 0], jtIp[:, 1], c='r')
jtIt = transformPoints2D(di.joints3DToImg(gt_hands[i]), Ms_test[i])
plt.scatter(jtIt[:, 0], jtIt[:, 1], c='b')
for edge in hd_edges:
plt.plot(jtIp[:, 0][edge], jtIp[:, 1][edge], c='r',LineWidth=1.5)
plt.plot(jtIt[:, 0][edge], jtIt[:, 1][edge], c='b',LineWidth=1.5)
plt.pause(0.01)
plt.axis('off')
plt.savefig("../../image/MSRA/img_{}.png".format(i))
plt.cla()
