def frameToPCL(frame):
pointcloud = np.zeros((frame.dpt.shape[0] * frame.dpt.shape[1], 3),
dtype=float)
centerX = int(frame.dpt.shape[1] / 2.)
centerY = int(frame.dpt.shape[0] / 2.)
depth_idx = 0
for v in range(-centerY, centerY):
for u in range(-centerX, centerX):
# skip invalid points
if (frame.dpt[v + centerY, u + centerX] == 0.):
continue
t = transformPoint2D([u + centerX, v + centerY],
np.linalg.inv(frame.T))
pointcloud[depth_idx,
0] = float(t[0][0] -
160.) * frame.dpt[v + centerY,
u + centerX] / 241.42
pointcloud[depth_idx,
1] = float(t[1][0] -
120.) * frame.dpt[v + centerY,
u + centerX] / 241.42
pointcloud[depth_idx, 2] = frame.dpt[v + centerY, u + centerX]
depth_idx += 1
return pointcloud[0:depth_idx, :]
def replaceAnnotations(self, seq, new_gtorig2D):
"""
Replace annotations of sequence with new data from file
:param seq: sequence
:param new_gtorig2D: array with new 2D annotations
:return: new sequence
"""
if len(seq.data) != new_gtorig2D.shape[0]:
raise ValueError(
"Shape misfit: sequence has {} entries, but new annotation only {}"
.format(len(seq.data), new_gtorig2D.shape[0]))
if seq.data[0].gtorig.size != new_gtorig2D[0].size:
print "WARNING: Size misfit: seq has {}, but new annotations have {}".format(
seq.data[0].gtorig.size, new_gtorig2D[0].size)
new_gtorig2D = new_gtorig2D.reshape(
(len(seq.data), seq.data[0].gtorig.shape[0],
seq.data[0].gtorig.shape[1]))
for idx, el in enumerate(seq.data):
new_gtcrop = np.zeros((el.gtorig.shape[0], 3), np.float32)
for joint in xrange(el.gtorig.shape[0]):
t = transformPoint2D(new_gtorig2D[idx][joint], el.T)
new_gtcrop[joint, 0] = t[0]
new_gtcrop[joint, 1] = t[1]
new_gtcrop[joint, 2] = new_gtorig2D[idx][joint, 2]
new_gt3Dorig = self.jointsImgTo3D(new_gtorig2D[idx])
new_gt3Dcrop = new_gt3Dorig - el.com
seq.data[idx] = el._replace(gtorig=new_gtorig2D[idx],
gtcrop=new_gtcrop,
gt3Dorig=new_gt3Dorig,
gt3Dcrop=new_gt3Dcrop)
return seq
def loadSequence(self,
seqName,
camera=None,
Nmax=float('inf'),
shuffle=False,
rng=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. subject1
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
config = {'cube': (220, 220, 220)}
pickleCache = '{}/{}_{}_{}_cache.pkl'.format(self.cacheDir,
self.__class__.__name__,
seqName, camera)
if self.useCache & os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not (np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# Load the dataset
objdir = os.path.join(self.basepath, seqName)
if camera is not None:
dflt = "c{0:02d}_*.npy".format(camera)
else:
dflt = '*.npy'
dptFiles = sorted([
os.path.join(dirpath, f)
for dirpath, dirnames, files in os.walk(objdir)
for f in fnmatch.filter(files, dflt)
])
# dptFiles = sorted(glob.glob(os.path.join(objdir, '*exr')))
if camera is not None:
lflt = "c{0:02d}_*anno_blender.txt".format(camera)
else:
lflt = '*anno_blender.txt'
labelFiles = sorted([
os.path.join(dirpath, f)
for dirpath, dirnames, files in os.walk(objdir)
for f in fnmatch.filter(files, lflt)
])
#labelFiles = sorted(glob.glob(os.path.join(objdir, '*txt')))
# sync lists
newdpt = []
newlbl = []
if camera is not None:
number_idx = 1
else:
number_idx = 0
labelIdx = [
''.join(os.path.basename(lbl).split('_')[number_idx])
for lbl in labelFiles
]
for dpt in dptFiles:
if ''.join(
os.path.basename(dpt).split('_')[number_idx]) in labelIdx:
newdpt.append(dpt)
newlbl.append(labelFiles[labelIdx.index(''.join(
os.path.basename(dpt).split('_')[number_idx]))])
dptFiles = newdpt
labelFiles = newlbl
assert len(dptFiles) == len(labelFiles)
txt = 'Loading {}'.format(seqName)
nImgs = len(dptFiles)
pbar = pb.ProgressBar(maxval=nImgs,
widgets=[txt, pb.Percentage(),
pb.Bar()])
pbar.start()
data = []
for i in xrange(nImgs):
labelFileName = labelFiles[i]
dptFileName = dptFiles[i]
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
continue
dpt = self.loadDepthMap(dptFileName)
if not os.path.isfile(labelFileName):
print("File {} does not exist!".format(labelFileName))
continue
# joints in image coordinates
gtorig = np.genfromtxt(labelFileName)
gtorig = gtorig.reshape((gtorig.shape[0] / 3, 3))
gtorig[:, 2] *= 1000.
# normalized joints in 3D coordinates
gt3Dorig = self.jointsImgTo3D(gtorig)
### Shorting the tip bone, BUGFIX
tips = [3, 8, 13, 18, 23]
shrink_factor = 0.85
for joint_idx in tips:
t = gt3Dorig[joint_idx, :] - gt3Dorig[joint_idx - 1, :]
gt3Dorig[joint_idx, :] = gt3Dorig[joint_idx -
1, :] + shrink_factor * t
gtorig = self.joints3DToImg(gt3Dorig)
###
# print gtorig, gt3Dorig
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,0,0,dptFileName,'','',{}))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, importer=self)
if not hd.checkImage(1.):
print("Skipping image {}, no content".format(dptFileName))
continue
try:
dpt, M, com = hd.cropArea3D(gtorig[10], size=config['cube'])
except UserWarning:
print(
"Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = np.zeros((gt3Dorig.shape[0], 3), np.float32)
for joint in range(gtcrop.shape[0]):
t = transformPoint2D(gtorig[joint], M)
gtcrop[joint, 0] = t[0]
gtcrop[joint, 1] = t[1]
gtcrop[joint, 2] = gtorig[joint, 2]
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'',''))
data.append(
DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig,
gt3Dcrop, com3D, dptFileName, '',
self.sides[seqName], {}))
pbar.update(i)
# early stop
if len(data) >= Nmax:
break
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config),
f,
protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
protocol=cPickle.HIGHEST_PROTOCOL)
print "Testing baseline"
#################################
# BASELINE
# Load the evaluation
data_baseline = di.loadBaseline('../data/ICVL/LRF_Results_seq_1.txt')
hpe_base = ICVLHandposeEvaluation(gt3D, data_baseline)
hpe_base.subfolder += '/' + eval_prefix + '/'
print("Mean error: {}mm".format(hpe_base.getMeanError()))
hpe.plotEvaluation(eval_prefix,
methodName='Our regr',
baseline=[('Tang et al.', hpe_base)])
ind = 0
for i in testSeqs[0].data:
if ind % 20 != 0:
ind += 1
continue
jt = joints[ind]
jtI = di.joints3DToImg(jt)
for joint in range(jt.shape[0]):
t = transformPoint2D(jtI[joint], i.T)
jtI[joint, 0] = t[0]
jtI[joint, 1] = t[1]
hpe.plotResult(i.dpt, i.gtcrop, jtI, "{}_{}".format(eval_prefix, ind))
ind += 1
def saveVideo(self, filename, sequence, imp, showJoints=True, showDepth=True, fullFrame=True, plotFrameNumbers=False,
visibility=None, highlight=None, height=320, width=240, joints2D_override=None, showGT=False,
annoscale=1):
"""
Create a video with 2D annotations
:param filename: name of file to save
:param sequence: sequence data
:param imp: importer of data
:param showJoints: show joints
:param showDepth: show depth map as background
:param fullFrame: display full frame or only cropped frame
:param plotFrameNumbers: plot frame numbers and indicate reference frames
:param visibility: plot different markers for visible/non-visible joints, visible are marked as 1, non-visible 0
:param highlight: highlight different joints, set as 1 to highlight, otherwise 0
:return: None
"""
if visibility is not None:
assert visibility.shape == self.joints.shape, "{} == {}".format(visibility.shape, self.joints.shape)
if highlight is not None:
assert highlight.shape == self.joints.shape, "{} == {}".format(highlight.shape, self.joints.shape)
if joints2D_override is not None:
assert joints2D_override.shape == self.joints.shape, "{} == {}".format(joints2D_override.shape, self.joints.shape)
# check aspect ratio
if fullFrame:
dpt = imp.loadDepthMap(sequence.data[0].fileName)
else:
dpt = sequence.data[0].dpt
if dpt.shape[0] == dpt.shape[1]:
width = height
txt = 'Saving {}'.format(filename)
pbar = pb.ProgressBar(maxval=self.joints.shape[0], widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
# Define the codec and create VideoWriter object
fourcc = cv2.cv.CV_FOURCC(*'DIVX')
video = cv2.VideoWriter('{}/video_{}.avi'.format(self.subfolder, filename), fourcc, self.fps, (height, width))
if not video:
raise EnvironmentError("Error in creating video writer")
for i in xrange(self.joints.shape[0]):
if joints2D_override is not None:
jtI = joints2D_override[i]
else:
jt = self.joints[i]
jtI = imp.joints3DToImg(jt)
if fullFrame:
if not os.path.isfile(sequence.data[i].fileName):
raise EnvironmentError("file missing")
dm = imp.loadDepthMap(sequence.data[i].fileName)
dm[dm == imp.getDepthMapNV()] = 0
dpt = dm
gtcrop = imp.joints3DToImg(self.gt[i])
else:
for joint in xrange(jtI.shape[0]):
jtI[joint, 0:2] = transformPoint2D(jtI[joint], sequence.data[i].T).squeeze()[0:2]
dpt = sequence.data[i].dpt
gtcrop = imp.joints3DToImg(self.gt[i])
for joint in xrange(gtcrop.shape[0]):
gtcrop[joint, 0:2] = transformPoint2D(gtcrop[joint], sequence.data[i].T).squeeze()[0:2]
img = self.plotResult(dpt, gtcrop, jtI, showGT=showGT, niceColors=True, showJoints=showJoints,
showDepth=showDepth, visibility=None if visibility is None else visibility[i],
highlight=None if highlight is None else highlight[i], annoscale=annoscale)
img = img[:, :, [2, 1, 0]] # change color channels
img = cv2.resize(img, (height, width))
if plotFrameNumbers:
if sequence.data[i].subSeqName == 'ref':
cv2.putText(img, "Reference Frame {}".format(i), (20, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255))
# plot frame number
cv2.putText(img, "{}".format(i), (height-50, width-10), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 255))
# write frame
video.write(img)
pbar.update(i)
video.release()
del video
cv2.destroyAllWindows()
pbar.finish()
def loadSequence(self,
seqName,
allJoints=False,
Nmax=float('inf'),
shuffle=False,
rng=None,
docom=False):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param subSeq: list of subsequence names, e.g. 0, 45, 122-5
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
config = {'cube': (300, 300, 300)}
config['cube'] = [s * self.scales[seqName] for s in config['cube']]
refineNet = None
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__, seqName, allJoints, docom)
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not (np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# Load the dataset
objdir = '{}/{}/'.format(self.basepath, seqName)
trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName)
mat = scipy.io.loadmat(trainlabels)
names = mat['joint_names'][0]
joints3D = mat['joint_xyz'][0]
joints2D = mat['joint_uvd'][0]
if allJoints:
eval_idxs = np.arange(36)
else:
eval_idxs = self.restrictedJointsEval
self.numJoints = len(eval_idxs)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=joints3D.shape[0],
widgets=[txt, pb.Percentage(),
pb.Bar()])
pbar.start()
data = []
i = 0
for line in range(joints3D.shape[0]):
dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line + 1)
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for ii in range(joints2D.shape[1]):
if ii not in eval_idxs:
continue
gtorig[jt, 0] = joints2D[line, ii, 0]
gtorig[jt, 1] = joints2D[line, ii, 1]
gtorig[jt, 2] = joints2D[line, ii, 2]
jt += 1
# normalized joints in 3D coordinates
gt3Dorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for jj in range(joints3D.shape[1]):
if jj not in eval_idxs:
continue
gt3Dorig[jt, 0] = joints3D[line, jj, 0]
gt3Dorig[jt, 1] = joints3D[line, jj, 1]
gt3Dorig[jt, 2] = joints3D[line, jj, 2]
jt += 1
#print gt3D
#self.showAnnotatedDepth(ICVLFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,''))
# Detect hand
hd = HandDetector(dpt,
self.fx,
self.fy,
refineNet=refineNet,
importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
if allJoints:
dpt, M, com = hd.cropArea3D(gtorig[34],
size=config['cube'],
docom=True)
else:
dpt, M, com = hd.cropArea3D(gtorig[13],
size=config['cube'],
docom=True)
except UserWarning:
print(
"Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = np.zeros((gtorig.shape[0], 3), np.float32)
for joint in range(gtorig.shape[0]):
t = transformPoint2D(gtorig[joint], M)
gtcrop[joint, 0] = t[0]
gtcrop[joint, 1] = t[1]
gtcrop[joint, 2] = gtorig[joint, 2]
#print("{}".format(gt3Dorig))
#self.showAnnotatedDepth(ICVLFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,''))
data.append(
ICVLFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig,
gt3Dcrop, com3D, dptFileName, ''))
pbar.update(i)
i += 1
# early stop
if len(data) >= Nmax:
break
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config),
f,
protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
def loadSequence(self,
seqName,
subSeq=None,
Nmax=float('inf'),
shuffle=False,
rng=None,
docom=False):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param subSeq: list of subsequence names, e.g. 0, 45, 122-5
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if (subSeq is not None) and (not isinstance(subSeq, list)):
raise TypeError("subSeq must be None or list")
config = {'cube': (250, 250, 250)}
refineNet = None
if subSeq is None:
pickleCache = '{}/{}_{}_None_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__, seqName, docom)
else:
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__, seqName,
''.join(subSeq), docom)
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not (np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# check for multiple subsequences
if subSeq is not None:
if len(subSeq) > 1:
missing = False
for i in range(len(subSeq)):
if not os.path.isfile(
'{}/{}_{}_{}_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__,
seqName, subSeq[i], docom)):
missing = True
print("missing: {}".format(subSeq[i]))
break
if not missing:
# load first data
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__, seqName,
subSeq[0], docom)
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, fullData, config) = cPickle.load(f)
f.close()
# load rest of data
for i in range(1, len(subSeq)):
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__,
seqName, subSeq[i], docom)
print("Loading cache data from {}".format(
pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
fullData.extend(data)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(fullData)
if not (np.isinf(Nmax)):
return NamedImgSequence(seqName, fullData[0:Nmax],
config)
else:
return NamedImgSequence(seqName, fullData, config)
# Load the dataset
objdir = '{}/Depth/'.format(self.basepath)
trainlabels = '{}/{}.txt'.format(self.basepath, seqName)
inputfile = open(trainlabels)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=len(inputfile.readlines()),
widgets=[txt, pb.Percentage(),
pb.Bar()])
pbar.start()
inputfile.seek(0)
data = []
i = 0
for line in inputfile:
part = line.split(' ')
#print part[0]
# check for subsequences and skip them if necessary
subSeqName = ''
if subSeq is not None:
p = part[0].split('/')
# handle original data (unrotated '0') separately
if ('0' in subSeq) and len(p[0]) > 6:
pass
elif not ('0' in subSeq) and len(p[0]) > 6:
i += 1
continue
elif (p[0] in subSeq) and len(p[0]) <= 6:
pass
elif not (p[0] in subSeq) and len(p[0]) <= 6:
i += 1
continue
if len(p[0]) <= 6:
subSeqName = p[0]
else:
subSeqName = '0'
dptFileName = '{}/{}'.format(objdir, part[0])
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
for joint in range(self.numJoints):
for xyz in range(0, 3):
gtorig[joint, xyz] = part[joint * 3 + xyz + 1]
# normalized joints in 3D coordinates
gt3Dorig = self.jointsImgTo3D(gtorig)
#print gt3D
#self.showAnnotatedDepth(ICVLFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,0,dptFileName,subSeqName))
# Detect hand
hd = HandDetector(dpt,
self.fx,
self.fy,
refineNet=None,
importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
dpt, M, com = hd.cropArea3D(gtorig[0],
size=config['cube'],
docom=True)
except UserWarning:
print(
"Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D #normalize to com
gtcrop = np.zeros((gtorig.shape[0], 3), np.float32)
for joint in range(gtorig.shape[0]):
t = transformPoint2D(gtorig[joint], M)
gtcrop[joint, 0] = t[0]
gtcrop[joint, 1] = t[1]
gtcrop[joint, 2] = gtorig[joint, 2]
#print("{}".format(gt3Dorig))
#self.showAnnotatedDepth(ICVLFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,subSeqName))
data.append(
ICVLFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig,
gt3Dcrop, com3D, dptFileName, subSeqName))
pbar.update(i)
i += 1
# early stop
if len(data) >= Nmax:
break
inputfile.close()
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config),
f,
protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
def loadSequence(self,
seqName,
allJoints=False,
Nmax=float('inf'),
shuffle=False,
rng=None,
docom=False,
rotation=False,
dsize=(128, 128)):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param subSeq: list of subsequence names, e.g. 0, 45, 122-5
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
config = {'cube': (300, 300, 300)}
config['cube'] = [s * self.scales[seqName] for s in config['cube']]
refineNet = None
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(
self.cacheDir, self.__class__.__name__, seqName, allJoints, docom)
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not (np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# Load the dataset
objdir = '{}/{}/'.format(self.basepath, seqName)
trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName)
mat = scipy.io.loadmat(trainlabels)
names = mat['joint_names'][0]
joints3D = mat['joint_xyz'][0]
joints2D = mat['joint_uvd'][0]
if allJoints:
eval_idxs = np.arange(36)
else:
eval_idxs = self.restrictedJointsEval
self.numJoints = len(eval_idxs)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=joints3D.shape[0],
widgets=[txt, pb.Percentage(),
pb.Bar()])
pbar.start()
data = []
if rotation:
data_rotation = []
i = 0
for line in range(joints3D.shape[0]):
dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line + 1)
print dptFileName
# augment the training dataset
if rotation:
assert seqName == 'train', 'we only rotate the training data'
# 90', 180', or 270' rotation degree
rotation_degree = np.random.randint(1, 4) * 90
if not os.path.isfile(dptFileName):
if seqName == 'test_1' and line <= 2439 and line >= 0:
print("File {} does not exist!".format(dptFileName))
i += 1
continue
elif seqName == 'test_2' and line >= 2440 and line <= 8251:
print("File {} does not exist!".format(dptFileName))
i += 1
continue
else:
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
h, w = dpt.shape
if rotation:
import math
def rotate_about_center(src, angle, scale=1.):
w = src.shape[1]
h = src.shape[0]
rangle = np.deg2rad(angle) # angle in radians
# now calculate new image width and height
nw = (abs(np.sin(rangle) * h) +
abs(np.cos(rangle) * w)) * scale
nh = (abs(np.cos(rangle) * h) +
abs(np.sin(rangle) * w)) * scale
# ask OpenCV for the rotation matrix
rot_mat = cv2.getRotationMatrix2D((nw * 0.5, nh * 0.5),
angle, scale)
# calculate the move from the old center to the new center combined
# with the rotation
rot_move = np.dot(
rot_mat, np.array([(nw - w) * 0.5, (nh - h) * 0.5, 0]))
# the move only affects the translation, so update the translation
# part of the transform
rot_mat[0, 2] += rot_move[0]
rot_mat[1, 2] += rot_move[1]
return cv2.warpAffine(
src,
rot_mat, (int(math.ceil(nw)), int(math.ceil(nh))),
flags=cv2.INTER_LANCZOS4), rot_mat
# get the rotation matrix for warpAffine
dpt_rotation, rotMat = rotate_about_center(
dpt, rotation_degree)
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for ii in range(joints2D.shape[1]):
if ii not in eval_idxs:
continue
gtorig[jt, 0] = joints2D[line, ii, 0]
gtorig[jt, 1] = joints2D[line, ii, 1]
gtorig[jt, 2] = joints2D[line, ii, 2]
jt += 1
if rotation:
gtorig_rotation = np.zeros((self.numJoints, 3), np.float32)
for i, joint in enumerate(gtorig):
m11 = rotMat[0, 0]
m12 = rotMat[0, 1]
m13 = rotMat[0, 2]
m21 = rotMat[1, 0]
m22 = rotMat[1, 1]
m23 = rotMat[1, 2]
gtorig_rotation[
i, 0] = gtorig[i, 0] * m11 + gtorig[i, 1] * m12 + m13
gtorig_rotation[
i, 1] = gtorig[i, 0] * m21 + gtorig[i, 1] * m22 + m23
gtorig_rotation[i, 2] = gtorig[i, 2]
# normalized joints in 3D coordinates
gt3Dorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for jj in range(joints3D.shape[1]):
if jj not in eval_idxs:
continue
gt3Dorig[jt, 0] = joints3D[line, jj, 0]
gt3Dorig[jt, 1] = joints3D[line, jj, 1]
gt3Dorig[jt, 2] = joints3D[line, jj, 2]
jt += 1
# transform from gtorign
gt3Dorig_rotation = np.zeros((self.numJoints, 3), np.float32)
if rotation:
gt3Dorig_rotation = self.jointsImgTo3D(gtorig_rotation)
#print gt3D
#print("{}".format(gtorig))
#self.showAnnotatedDepth(ICVLFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,''))
#print("{}".format(gtorig_rotation))
#self.showAnnotatedDepth(ICVLFrame(dpt_rotation,gtorig_rotation,gtorig_rotation,0,gt3Dorig_rotation,gt3Dorig_rotation,0,dptFileName,''))
# Detect hand
hd = HandDetector(dpt,
self.fx,
self.fy,
refineNet=refineNet,
importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
if allJoints:
dpt, M, com = hd.cropArea3D(gtorig[34],
size=config['cube'],
docom=docom,
dsize=dsize)
else:
dpt, M, com = hd.cropArea3D(gtorig[13],
size=config['cube'],
docom=docom,
dsize=dsize)
except UserWarning:
print(
"Skipping image {}, no hand detected".format(dptFileName))
continue
if rotation:
# Detect hand
hd_rotation = HandDetector(dpt_rotation,
self.fx,
self.fy,
refineNet=refineNet,
importer=self)
try:
if allJoints:
dpt_rotation, M_rotation, com_rotation = hd_rotation.cropArea3D(
gtorig_rotation[34],
size=config['cube'],
docom=docom,
dsize=dsize)
else:
dpt_rotation, M_rotation, com_rotation = hd_rotation.cropArea3D(
gtorig_rotation[13],
size=config['cube'],
docom=docom,
dsize=dsize)
except UserWarning:
print("Skipping image {}, no hand detected".format(
dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = np.zeros((gtorig.shape[0], 3), np.float32)
for joint in range(gtorig.shape[0]):
t = transformPoint2D(gtorig[joint], M)
gtcrop[joint, 0] = t[0]
gtcrop[joint, 1] = t[1]
gtcrop[joint, 2] = gtorig[joint, 2]
# create 3D voxel
dpt3D = np.zeros((8, dpt.shape[0], dpt.shape[1]), dtype=np.uint8)
sorted_dpt = np.sort(dpt, axis=None)
iii = np.where(sorted_dpt != 0)[0][0]
min_d = sorted_dpt[iii]
max_d = np.max(dpt)
slice_range = []
slice_step = (max_d - min_d) / 8
for i in range(9):
slice_range.append(min_d + slice_step * i)
slice_range = np.array(slice_range)
lh, lw = np.where(dpt != 0)
for ii in xrange(lh.shape[0]):
ih, iw = lh[ii], lw[ii]
dptValue = dpt[ih, iw]
slice_layer = np.where(dptValue >= slice_range)[0][-1]
if slice_layer == 8:
slice_layer = 7
dpt3D[slice_layer, ih, iw] = 1
if rotation:
com3D_rotation = self.jointImgTo3D(com_rotation)
gt3Dcrop_rotation = gt3Dorig_rotation - com3D_rotation # normalize to com
gtcrop_rotation = np.zeros((gtorig_rotation.shape[0], 3),
np.float32)
for joint in range(gtorig_rotation.shape[0]):
t = transformPoint2D(gtorig_rotation[joint], M_rotation)
gtcrop_rotation[joint, 0] = t[0]
gtcrop_rotation[joint, 1] = t[1]
gtcrop_rotation[joint, 2] = gtorig_rotation[joint, 2]
dpt3D_rotation = np.zeros((8, dpt.shape[0], dpt.shape[1]),
dtype=np.uint8)
sorted_dpt_rotation = np.sort(dpt_rotation, axis=None)
iii = np.where(sorted_dpt_rotation != 0)[0][0]
min_d = sorted_dpt_rotation[iii]
max_d = np.max(dpt_rotation)
slice_range = []
slice_step = (max_d - min_d) / 8
for i in range(9):
slice_range.append(min_d + slice_step * i)
slice_range = np.array(slice_range)
lh, lw = np.where(dpt_rotation != 0)
for ii in xrange(lh.shape[0]):
ih, iw = lh[ii], lw[ii]
dptValue = dpt_rotation[ih, iw]
slice_layer = np.where(dptValue >= slice_range)[0][-1]
if slice_layer == 8:
slice_layer = 7
dpt3D_rotation[slice_layer, ih, iw] = 1
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(ICVLFrame(dpt,dpt3D,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,''))
# if rotation:
# print("{}".format(gt3Dorig_rotation))
# self.showAnnotatedDepth(ICVLFrame(dpt_rotation,dpt3D_rotation,gtorig_rotation,gtcrop_rotation,M_rotation,gt3Dorig_rotation,gt3Dcrop_rotation,com3D_rotation,dptFileName,''))
# visualize 3D
# from mpl_toolkits.mplot3d import Axes3D
# fig = plt.figure()
# ax = fig.add_subplot(111,projection='3d')
# d,x,y = np.where(dpt3D == 1)
# ax.scatter(x,y,8 - d)
# # ax.view_init(0,0)
# ax.set_xlabel('x')
# ax.set_ylabel('y')
# ax.set_zlabel('d')
# plt.show()
data.append(
ICVLFrame(dpt.astype(np.float32), dpt3D, gtorig, gtcrop, M,
gt3Dorig, gt3Dcrop, com3D, dptFileName, ''))
if rotation:
data_rotation.append(
ICVLFrame(dpt_rotation.astype(np.float32), dpt3D_rotation,
gtorig_rotation, gtcrop_rotation, M_rotation,
gt3Dorig_rotation, gt3Dcrop_rotation,
com3D_rotation, dptFileName, ''))
pbar.update(i)
i += 1
# early stop
if len(data) >= Nmax:
break
pbar.finish()
# augmentation
if rotation:
data.extend(data_rotation)
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config),
f,
protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)