def loadSequence(self, seqName, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if cube is None:
config = {'cube': self.default_cubes[seqName]}
else:
assert isinstance(cube, tuple)
assert len(cube) == 3
config = {'cube': cube}
pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, self.allJoints, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
self.loadRefineNetLazy(self.refineNet)
# Load the dataset
objdir = '{}/{}/'.format(self.basepath, seqName)
trainlabels = '{}/{}/joint_data.mat'.format(self.basepath, seqName)
mat = scipy.io.loadmat(trainlabels)
names = mat['joint_names'][0]
joints3D = mat['joint_xyz'][0]
joints2D = mat['joint_uvd'][0]
if self.allJoints:
eval_idxs = np.arange(36)
else:
eval_idxs = self.restrictedJointsEval
self.numJoints = len(eval_idxs)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=joints3D.shape[0], widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
data = []
i=0
for line in range(joints3D.shape[0]):
dptFileName = '{0:s}/depth_1_{1:07d}.png'.format(objdir, line+1)
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
if hand is not None:
raise NotImplementedError()
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for ii in range(joints2D.shape[1]):
if ii not in eval_idxs:
continue
gtorig[jt, 0] = joints2D[line, ii, 0]
gtorig[jt, 1] = joints2D[line, ii, 1]
gtorig[jt, 2] = joints2D[line, ii, 2]
jt += 1
# normalized joints in 3D coordinates
gt3Dorig = np.zeros((self.numJoints, 3), np.float32)
jt = 0
for jj in range(joints3D.shape[1]):
if jj not in eval_idxs:
continue
gt3Dorig[jt, 0] = joints3D[line, jj, 0]
gt3Dorig[jt, 1] = joints3D[line, jj, 1]
gt3Dorig[jt, 2] = joints3D[line, jj, 2]
jt += 1
# print gt3D
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dorig,0,dptFileName,'',''))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom)
except UserWarning:
print("Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = transformPoints2D(gtorig, M)
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'',''))
data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName,
'', self.sides[seqName], {}))
pbar.update(i)
i += 1
# early stop
if len(data) >= Nmax:
break
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None,
hand=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. train
:param subSeq: list of subsequence names, e.g. 0, 45, 122-5
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if (subSeq is not None) and (not isinstance(subSeq, list)):
raise TypeError("subSeq must be None or list")
if cube is None:
config = {'cube': self.default_cubes[seqName]}
else:
assert isinstance(cube, tuple)
assert len(cube) == 3
config = {'cube': cube}
if subSeq is None:
pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
else:
pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
if self.useCache:
if os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# check for multiple subsequences
if subSeq is not None:
if len(subSeq) > 1:
missing = False
for i in range(len(subSeq)):
if not os.path.isfile('{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None))):
missing = True
print("missing: {}".format(subSeq[i]))
break
if not missing:
# load first data
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[0], HandDetector.detectionModeToString(docom, self.refineNet is not None))
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, fullData, config) = cPickle.load(f)
f.close()
# load rest of data
for i in range(1, len(subSeq)):
pickleCache = '{}/{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, subSeq[i], HandDetector.detectionModeToString(docom, self.refineNet is not None))
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
fullData.extend(data)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(fullData)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, fullData[0:Nmax], config)
else:
return NamedImgSequence(seqName, fullData, config)
self.loadRefineNetLazy(self.refineNet)
# Load the dataset
objdir = '{}/Depth/'.format(self.basepath)
trainlabels = '{}/{}.txt'.format(self.basepath, seqName)
inputfile = open(trainlabels)
txt = 'Loading {}'.format(seqName)
pbar = pb.ProgressBar(maxval=len(inputfile.readlines()), widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
inputfile.seek(0)
data = []
i = 0
for line in inputfile:
# early stop
if len(data) >= Nmax:
break
part = line.split(' ')
# check for subsequences and skip them if necessary
subSeqName = ''
if subSeq is not None:
p = part[0].split('/')
# handle original data (unrotated '0') separately
if ('0' in subSeq) and len(p[0]) > 6:
pass
elif not('0' in subSeq) and len(p[0]) > 6:
i += 1
continue
elif (p[0] in subSeq) and len(p[0]) <= 6:
pass
elif not(p[0] in subSeq) and len(p[0]) <= 6:
i += 1
continue
if len(p[0]) <= 6:
subSeqName = p[0]
else:
subSeqName = '0'
dptFileName = '{}/{}'.format(objdir, part[0])
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
i += 1
continue
dpt = self.loadDepthMap(dptFileName)
if hand is not None:
raise NotImplementedError()
# joints in image coordinates
gtorig = np.zeros((self.numJoints, 3), np.float32)
for joint in range(self.numJoints):
for xyz in range(0, 3):
gtorig[joint, xyz] = part[joint*3+xyz+1]
# normalized joints in 3D coordinates
gt3Dorig = self.jointsImgTo3D(gtorig)
# print gt3D
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,0,dptFileName,subSeqName,''))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self)
if not hd.checkImage(1):
print("Skipping image {}, no content".format(dptFileName))
i += 1
continue
try:
dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom)
except UserWarning:
print("Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = transformPoints2D(gtorig, M)
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,subSeqName,''))
data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D, dptFileName,
subSeqName, 'left', {}))
pbar.update(i)
i += 1
inputfile.close()
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
def loadSequence(self, seqName, subSeq=None, Nmax=float('inf'), shuffle=False, rng=None, docom=False, cube=None, hand=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. subject1
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
if (subSeq is not None) and (not isinstance(subSeq, list)):
raise TypeError("subSeq must be None or list")
if cube is None:
config = {'cube': self.default_cubes[seqName]}
else:
assert isinstance(cube, tuple)
assert len(cube) == 3
config = {'cube': cube}
if subSeq is None:
pickleCache = '{}/{}_{}_None_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
else:
pickleCache = '{}/{}_{}_{}_{}_{}_cache.pkl'.format(self.cacheDir, self.__class__.__name__, seqName, ''.join(subSeq), HandDetector.detectionModeToString(docom, self.refineNet is not None), config['cube'][0])
if self.useCache & os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not(np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
self.loadRefineNetLazy(self.refineNet)
# Load the dataset
objdir = '{}/{}/'.format(self.basepath, seqName)
subdirs = sorted([name for name in os.listdir(objdir) if os.path.isdir(os.path.join(objdir, name))])
txt = 'Loading {}'.format(seqName)
nImgs = sum([len(files) for r, d, files in os.walk(objdir)]) // 2
pbar = pb.ProgressBar(maxval=nImgs, widgets=[txt, pb.Percentage(), pb.Bar()])
pbar.start()
data = []
pi = 0
for subdir in subdirs:
# check for subsequences and skip them if necessary
subSeqName = ''
if subSeq is not None:
if subdir not in subSeq:
continue
subSeqName = subdir
# iterate all subdirectories
trainlabels = '{}/{}/joint.txt'.format(objdir, subdir)
inputfile = open(trainlabels)
# read number of samples
nImgs = int(inputfile.readline())
for i in range(nImgs):
# early stop
if len(data) >= Nmax:
break
line = inputfile.readline()
part = line.split(' ')
dptFileName = '{}/{}/{}_depth.bin'.format(objdir, subdir, str(i).zfill(6))
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
continue
dpt = self.loadDepthMap(dptFileName)
if hand is not None:
raise NotImplementedError()
# joints in image coordinates
gt3Dorig = np.zeros((self.numJoints, 3), np.float32)
for joint in range(gt3Dorig.shape[0]):
for xyz in range(0, 3):
gt3Dorig[joint, xyz] = part[joint*3+xyz]
# invert axis
# gt3Dorig[:, 0] *= (-1.)
# gt3Dorig[:, 1] *= (-1.)
gt3Dorig[:, 2] *= (-1.)
# normalized joints in 3D coordinates
gtorig = self.joints3DToImg(gt3Dorig)
# print gt3D
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,gt3Dcrop,com3D,dptFileName,'',''))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, refineNet=self.refineNet, importer=self)
if not hd.checkImage(1.):
print("Skipping image {}, no content".format(dptFileName))
continue
try:
dpt, M, com = hd.cropArea3D(com=gtorig[self.crop_joint_idx], size=config['cube'], docom=docom)
except UserWarning:
print("Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = transformPoints2D(gtorig, M)
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'','',{}))
data.append(DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig, gt3Dcrop, com3D,
dptFileName, subSeqName, self.sides[seqName], {}))
pbar.update(pi)
pi += 1
inputfile.close()
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config), f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)
def loadSequence(self,
seqName,
camera=None,
Nmax=float('inf'),
shuffle=False,
rng=None):
"""
Load an image sequence from the dataset
:param seqName: sequence name, e.g. subject1
:param Nmax: maximum number of samples to load
:return: returns named image sequence
"""
config = {'cube': (220, 220, 220)}
pickleCache = '{}/{}_{}_{}_cache.pkl'.format(self.cacheDir,
self.__class__.__name__,
seqName, camera)
if self.useCache & os.path.isfile(pickleCache):
print("Loading cache data from {}".format(pickleCache))
f = open(pickleCache, 'rb')
(seqName, data, config) = cPickle.load(f)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
if not (np.isinf(Nmax)):
return NamedImgSequence(seqName, data[0:Nmax], config)
else:
return NamedImgSequence(seqName, data, config)
# Load the dataset
objdir = os.path.join(self.basepath, seqName)
if camera is not None:
dflt = "c{0:02d}_*.npy".format(camera)
else:
dflt = '*.npy'
dptFiles = sorted([
os.path.join(dirpath, f)
for dirpath, dirnames, files in os.walk(objdir)
for f in fnmatch.filter(files, dflt)
])
# dptFiles = sorted(glob.glob(os.path.join(objdir, '*exr')))
if camera is not None:
lflt = "c{0:02d}_*anno_blender.txt".format(camera)
else:
lflt = '*anno_blender.txt'
labelFiles = sorted([
os.path.join(dirpath, f)
for dirpath, dirnames, files in os.walk(objdir)
for f in fnmatch.filter(files, lflt)
])
#labelFiles = sorted(glob.glob(os.path.join(objdir, '*txt')))
# sync lists
newdpt = []
newlbl = []
if camera is not None:
number_idx = 1
else:
number_idx = 0
labelIdx = [
''.join(os.path.basename(lbl).split('_')[number_idx])
for lbl in labelFiles
]
for dpt in dptFiles:
if ''.join(
os.path.basename(dpt).split('_')[number_idx]) in labelIdx:
newdpt.append(dpt)
newlbl.append(labelFiles[labelIdx.index(''.join(
os.path.basename(dpt).split('_')[number_idx]))])
dptFiles = newdpt
labelFiles = newlbl
assert len(dptFiles) == len(labelFiles)
txt = 'Loading {}'.format(seqName)
nImgs = len(dptFiles)
pbar = pb.ProgressBar(maxval=nImgs,
widgets=[txt, pb.Percentage(),
pb.Bar()])
pbar.start()
data = []
for i in xrange(nImgs):
labelFileName = labelFiles[i]
dptFileName = dptFiles[i]
if not os.path.isfile(dptFileName):
print("File {} does not exist!".format(dptFileName))
continue
dpt = self.loadDepthMap(dptFileName)
if not os.path.isfile(labelFileName):
print("File {} does not exist!".format(labelFileName))
continue
# joints in image coordinates
gtorig = np.genfromtxt(labelFileName)
gtorig = gtorig.reshape((gtorig.shape[0] / 3, 3))
gtorig[:, 2] *= 1000.
# normalized joints in 3D coordinates
gt3Dorig = self.jointsImgTo3D(gtorig)
### Shorting the tip bone, BUGFIX
tips = [3, 8, 13, 18, 23]
shrink_factor = 0.85
for joint_idx in tips:
t = gt3Dorig[joint_idx, :] - gt3Dorig[joint_idx - 1, :]
gt3Dorig[joint_idx, :] = gt3Dorig[joint_idx -
1, :] + shrink_factor * t
gtorig = self.joints3DToImg(gt3Dorig)
###
# print gtorig, gt3Dorig
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtorig,0,gt3Dorig,0,0,dptFileName,'','',{}))
# Detect hand
hd = HandDetector(dpt, self.fx, self.fy, importer=self)
if not hd.checkImage(1.):
print("Skipping image {}, no content".format(dptFileName))
continue
try:
dpt, M, com = hd.cropArea3D(gtorig[10], size=config['cube'])
except UserWarning:
print(
"Skipping image {}, no hand detected".format(dptFileName))
continue
com3D = self.jointImgTo3D(com)
gt3Dcrop = gt3Dorig - com3D # normalize to com
gtcrop = np.zeros((gt3Dorig.shape[0], 3), np.float32)
for joint in range(gtcrop.shape[0]):
t = transformPoint2D(gtorig[joint], M)
gtcrop[joint, 0] = t[0]
gtcrop[joint, 1] = t[1]
gtcrop[joint, 2] = gtorig[joint, 2]
# print("{}".format(gt3Dorig))
# self.showAnnotatedDepth(DepthFrame(dpt,gtorig,gtcrop,M,gt3Dorig,gt3Dcrop,com3D,dptFileName,'',''))
data.append(
DepthFrame(dpt.astype(np.float32), gtorig, gtcrop, M, gt3Dorig,
gt3Dcrop, com3D, dptFileName, '',
self.sides[seqName], {}))
pbar.update(i)
# early stop
if len(data) >= Nmax:
break
pbar.finish()
print("Loaded {} samples.".format(len(data)))
if self.useCache:
print("Save cache data to {}".format(pickleCache))
f = open(pickleCache, 'wb')
cPickle.dump((seqName, data, config),
f,
protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
# shuffle data
if shuffle and rng is not None:
print("Shuffling")
rng.shuffle(data)
return NamedImgSequence(seqName, data, config)