def shardProc(dummy,
shard_id,
dataset_split,
dataset_mix,
numShards,
dsQueue,
itemType,
isRemoveBG=False,
fileListIn=None):
if dataset_mix == datasetMix.HO3D_MULTICAMERA:
dsCurr = datasetHo3dMultiCamera('',
0,
isRemoveBG=isRemoveBG,
fileListIn=fileListIn)
else:
raise NotImplementedError
num_images = dsCurr.getNumFiles()
num_per_shard = int(np.ceil(float(num_images) / float(numShards)))
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
dsCurr.setStartEndFiles(start_idx, end_idx)
print('Launching thread %d' % (shard_id))
for i in range(end_idx - start_idx):
_, ds = dsCurr.createTFExample(itemType=itemType)
dsQueue.put(ds)
def dataGen(w, h):
configFile = join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq, 'configs/configHandPose.json')
with open(configFile) as config_file:
data = yaml.safe_load(config_file)
base_dir = os.path.join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq)
objAliasLabelList = []
objLabel = data['objLabel']
if 'camID' in data:
camID = data['camID']
else:
camID = None
camera_dir = os.path.join(base_dir, 'camera')
# raw_seg_folder = os.path.join(base_dir, 'hand_only', 'mask')
raw_seg_folder = os.path.join(base_dir, 'segmentation', 'raw_seg_results')
handSegDir = os.path.join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq, 'segmentation', FLAGS.camID, 'raw_seg_results')
if useAutoInit:
if datasetName == datasetType.HO3D_MULTICAMERA:
autoInitDir = join(base_dir, 'handInit', camID, 'globalFit')
elif datasetName == datasetType.HO3D:
autoInitDir = join(base_dir, 'handInit', 'globalFit')
else:
raise NotImplementedError
files = os.listdir(autoInitDir)
files = [f[:-7] for f in files if 'pickle' in f]
fileTuple = {f: np.sum(cv2.imread(join(base_dir, 'segmentation', camID, 'raw_seg_results', f + '.png'))==objLabel) for f in files}
sorted_files = sorted(fileTuple.items(), key=lambda kv: kv[1], reverse=True)
files = [f[0] for f in sorted_files]
# files = sorted(files)
files = files[:numAutoInitFiles]
files = files
if datasetName == datasetType.HO3D_MULTICAMERA:
dataset = datasetHo3dMultiCamera(FLAGS.seq, FLAGS.camID)
# w = 640 // dscale
# h = 480 // dscale
frameIDList = []
handObjMaskList = []
handObjDepthList = []
handObjImgList = []
maskList = []
for file in files:
if datasetName == datasetType.HO3D:
imgName = os.path.join(camera_dir, 'color_' + file)
dptName = os.path.join(camera_dir, 'depth_' + file)
segName = os.path.join(raw_seg_folder, 'color_' + file)
segHandName = os.path.join(raw_seg_folder, 'color_' + file)
img = cv2.imread(imgName)[:, :, [2,1,0]]
dpt = cv2.imread(dptName)[:, :, [2,1,0]]
seg = cv2.imread(segName)[:, :, 0]
segH = cv2.imread(segHandName)[:, :, 0]
frameIDList.append(file[:-4])
elif datasetName == datasetType.HO3D_MULTICAMERA:
seq = base_dir.split('/')[-1]
_, ds = dataset.createTFExample(itemType='hand', fileIn=join(seq, camID, file))
img = ds.imgRaw[:, :, [2, 1, 0]]
dpt = ds.depth
# reading the seg from file because the seg in ds has no objAliasLabel info
seg = cv2.imread(join(base_dir, 'segmentation', camID, 'raw_seg_results', file + '.png'))[:, :, 0]
segH = cv2.imread(join(handSegDir, file + '.png'))[:,:,0]
assert seg.shape == segH.shape
segH[segH == 255] = handLabel
frameIDList.append(file)
else:
raise NotImplementedError
dpt = dpt[:, :, 0] + dpt[:, :, 1] * 256
dpt = dpt * depthScale
dptMask = np.logical_or(dpt > 0.75, dpt == 0.0)
dpt[dptMask] = bgDepth
seg[dptMask] = 0
segH[dptMask] = 0
seg[segH == handLabel] = handLabel # because object segmetation is not accurate always, might say hand is part of object
for alias in objAliasLabelList:
seg[seg == alias] = objLabel
objMask = (seg == objLabel)
handMask = (segH == handLabel)
objImg = img * np.expand_dims(objMask, 2)
handImg = img * np.expand_dims(handMask, 2)
objDepth = dpt * objMask
objDepth[np.logical_not(objMask)] = bgDepth
handDepth = dpt * handMask
handDepth[np.logical_not(handMask)] = bgDepth
# handDepth = handDepth*np.logical_or(handMask,objMask)
# handDepth[np.logical_not(np.logical_or(handMask,objMask))] = bgDepth
handMask = np.stack([handMask, handMask, handMask], axis=2).astype(np.float32) * handSegColor
objMask = np.stack([objMask, objMask, objMask], axis=2).astype(np.float32) * objSegColor
handObjMask = handMask + objMask
handObjDepth = dpt * (np.sum(handObjMask, 2)>0).astype(np.float32)
handObjDepth[np.sum(handObjMask, 2)==0] = bgDepth
handObjImg = img #* np.expand_dims((np.sum(handObjMask, 2)>0).astype(np.float32), 2)
mask = np.logical_not(np.zeros_like(objMask, dtype=np.bool))
handObjDepth = cv2.resize(handObjDepth, (w, h), interpolation=cv2.INTER_NEAREST)
handObjMask = cv2.resize(handObjMask, (w, h), interpolation=cv2.INTER_NEAREST)
mask = cv2.resize(mask.astype(np.float32), (w, h), interpolation=cv2.INTER_NEAREST)
handObjImg = cv2.resize(handObjImg, (w, h), interpolation=cv2.INTER_CUBIC)
handObjMaskList.append(handObjMask)
handObjDepth = np.stack([handObjDepth, handObjDepth, handObjDepth], axis=2)
handObjDepthList.append(handObjDepth)
maskList.append(mask)
handObjImg = handObjImg.astype(np.float32)/255.#np.zeros_like(mask, dtype=np.float32)
handObjImgList.append(handObjImg)
# realObservs = observables(frameID=file[:-4], seg=handMask, depth=handDepth, col=None, mask=mask, isReal=True)
yield (np.stack(frameIDList,0), np.stack(handObjMaskList,0), np.stack(handObjDepthList,0), np.stack(handObjImgList,0), np.stack(maskList,0))
def getJointsVisAndRend(fileList,
camMat,
hoPoseDir,
objModelPath,
camTransMat=np.eye(4, dtype=np.float32),
w=640,
h=480,
addBG=True,
addObject=True):
# camInd = cam2Ind
# camTransMat = np.linalg.inv(camPose12)
# python renderer for rendering object texture
pyRend = renderScene(h, w)
if addObject:
pyRend.addObjectFromMeshFile(objModelPath, 'obj')
pyRend.addCamera()
pyRend.creatcamProjMat([camMat[0, 0], camMat[1, 1]],
[camMat[0, 2], camMat[1, 2]], 0.001, 2.0)
bg = cv2.imread('/home/shreyas/Desktop/checkCrop.jpg')
bg = cv2.resize(bg, (w, h))
bg = np.zeros_like(bg) + 255
numImgs = len(fileList)
rendImgs = np.zeros((numImgs, h, w, 3), dtype=np.uint8)
jointVisImgs = np.zeros((numImgs, h, w, 3), dtype=np.uint8)
dataset = datasetHo3dMultiCamera(splitType.TEST, fileListIn=fileList)
for cntr, f in enumerate(fileList):
fileID = f.split('/')[-1]
_, ds = dataset.createTFExample()
img1 = ds.imgRaw
poseData = gutils.loadPickleData(join(hoPoseDir, fileID + '.pkl'))
# superimpose joints on image
coordChangMat = np.array([[1., 0., 0.], [0., -1., 0.], [0., 0., -1.]])
handJoints3DHomo = np.concatenate([
np.squeeze(poseData['JTransformed']),
np.ones((21, 1), dtype=np.float32)
],
axis=1).T
handJointProj = \
cv2.projectPoints(coordChangMat.dot(camTransMat.dot(handJoints3DHomo)[:3,:]).T,
np.zeros((3,)), np.zeros((3,)), camMat, np.zeros((4,)))[0][:, 0, :]
img1Joints = gutils.showHandJoints(
img1.copy(),
np.round(handJointProj[gutils.jointsMapManoToObman]).astype(
np.int32),
estIn=None,
filename=None,
upscale=1,
lineThickness=2)
objCorners3DHomo = np.concatenate([
np.squeeze(poseData['objCornersTransormed']),
np.ones((8, 1), dtype=np.float32)
],
axis=1).T
objCornersProj = \
cv2.projectPoints(coordChangMat.dot(camTransMat.dot(objCorners3DHomo)[:3,:]).T, np.zeros((3,)), np.zeros((3,)), camMat,
np.zeros((4,)))[0][:, 0, :]
img1Joints = gutils.showObjJoints(img1Joints,
objCornersProj,
lineThickness=2)
# render the synthetic image
if addObject:
poseMat = np.concatenate([
cv2.Rodrigues(poseData['rotObj'])[0],
np.reshape(poseData['transObj'], [3, 1])
],
axis=1)
poseMat = np.concatenate(
[poseMat, np.array([[0., 0., 0., 1.]])], axis=0)
pyRend.setObjectPose('obj', camTransMat.dot(poseMat))
if poseData['fullpose'].shape == (16, 3, 3):
fullpose = gutils.convertFullposeMatToVec(poseData['fullpose'])
else:
fullpose = poseData['fullpose']
# fullpose[5] = 0.8
# fullpose[12] = -1.6
# fullpose[14] = 0.4
# fullpose[23] = 0.8
# poseData['trans'][2] = poseData['trans'][2] + 0.05
# fullpose[32] = 0.8
_, handMesh = gutils.forwardKinematics(fullpose, poseData['trans'],
poseData['beta'])
handVertHomo = np.concatenate([
handMesh.r.copy(),
np.ones((handMesh.r.shape[0], 1), dtype=np.float32)
],
axis=1).T
pyRend.addObjectFromDict(
{
'vertices': camTransMat.dot(handVertHomo)[:3, :].T,
'faces': handMesh.f.copy(),
'vertex_colors': getHandVertexCols()[:, [2, 1, 0]]
}, 'hand')
# add background
cRend1, dRend1 = pyRend.render()
pyRend.scene.remove_node(pyRend.nodesDict['hand'])
if addBG:
mask = (dRend1 == 0)
mask = np.stack([mask, mask, mask], axis=2)
cRend1 = bg * mask + cRend1 * (1 - mask)
rendImgs[cntr] = cRend1.copy()
jointVisImgs[cntr] = img1Joints
# plt.imshow(img1Joints[:,:,[2,1,0]])
# plt.show()
print(f)
return rendImgs, jointVisImgs
def dataGen(w, h, batchSize):
with open(configFile) as config_file:
data = yaml.safe_load(config_file)
obj = data['obj']
objAliasLabelList = []
objLabel = data['objLabel']
startAt = data['startAt']
endAt = data['endAt']
skip = data['skip']
if datasetName == datasetType.HO3D_MULTICAMERA:
files = os.listdir(join(base_dir, 'rgb', camID))
files = [FLAGS.seq + '/' + camID + '/' + f1[:-4] for f1 in files if 'png' in f1]
files = sorted(files)
fileListIn = sorted(files)
dataset = datasetHo3dMultiCamera(FLAGS.seq, 0, fileListIn=fileListIn)
if isinstance(startAt, str):
for i, f in enumerate(files):
if f.split('/')[-1] == startAt:
break
files = files[i::skip]
# print(files)
else:
files = files[startAt:endAt:skip]
else:
raise NotImplementedError
numBatches = len(files)//batchSize
# w = 640 // dscale
# h = 480 // dscale
for i in range(numBatches):
frameIDList = []
handObjMaskList = []
handObjDepthList = []
handObjImgList = []
maskList = []
currBatchFiles = files[i*batchSize:(i+1)*batchSize]
for file in currBatchFiles:
if datasetName == datasetType.HO3D_MULTICAMERA:
seq = file.split('/')[0]
camInd = file.split('/')[1]
id = file.split('/')[2]
_, ds = dataset.createTFExample(itemType='hand', fileIn=file)
img = ds.imgRaw[:, :, [2, 1, 0]]
dpt = ds.depth
# reading the seg from file because the seg in ds has no objAliasLabel info
seg = cv2.imread(join(HO3D_MULTI_CAMERA_DIR, seq, 'segmentation', camInd, 'raw_seg_results', id + '.png'))[
:, :, 0]
frameID = np.array([id])
else:
raise NotImplementedError
dpt = dpt[:, :, 0] + dpt[:, :, 1] * 256
dpt = dpt * depthScale
dptMask = np.logical_or(dpt > DEPTH_THRESH, dpt == 0.0)
dpt[dptMask] = bgDepth
seg[dpt > DEPTH_THRESH] = 0
for alias in objAliasLabelList:
seg[seg == alias] = objLabel
objMask = (seg == objLabel)
handMask = (seg == handLabel)
objImg = img * np.expand_dims(objMask, 2)
handImg = img * np.expand_dims(handMask, 2)
objDepth = dpt * objMask
objDepth[np.logical_not(objMask)] = bgDepth
handDepth = dpt * handMask
handDepth[np.logical_not(handMask)] = bgDepth
# handDepth = handDepth*np.logical_or(handMask,objMask)
# handDepth[np.logical_not(np.logical_or(handMask,objMask))] = bgDepth
handMask = np.stack([handMask, handMask, handMask], axis=2).astype(np.float32) * handSegColor
objMask = np.stack([objMask, objMask, objMask], axis=2).astype(np.float32) * objSegColor
handObjMask = handMask + objMask
handObjDepth = dpt * (np.sum(handObjMask, 2)>0).astype(np.float32)
handObjDepth[np.sum(handObjMask, 2)==0] = bgDepth
handObjImg = img #* np.expand_dims((np.sum(handObjMask, 2)>0).astype(np.float32), 2)
mask = np.logical_not(np.zeros_like(objMask, dtype=np.bool))
handObjDepth = cv2.resize(handObjDepth, (w, h), interpolation=cv2.INTER_NEAREST)
handObjMask = cv2.resize(handObjMask, (w, h), interpolation=cv2.INTER_NEAREST)
mask = cv2.resize(mask.astype(np.float32), (w, h), interpolation=cv2.INTER_NEAREST)
handObjImg = cv2.resize(handObjImg, (w, h), interpolation=cv2.INTER_CUBIC)
handObjMaskList.append(handObjMask)
handObjDepth = np.stack([handObjDepth, handObjDepth, handObjDepth], axis=2)
handObjDepthList.append(handObjDepth)
maskList.append(mask)
frameIDList.append(frameID[0])
handObjImg = handObjImg.astype(np.float32)/255.#np.zeros_like(mask, dtype=np.float32)
handObjImgList.append(handObjImg)
# realObservs = observables(frameID=file[:-4], seg=handMask, depth=handDepth, col=None, mask=mask, isReal=True)
yield (np.stack(frameIDList,0), np.stack(handObjMaskList,0), np.stack(handObjDepthList,0), np.stack(handObjImgList,0), np.stack(maskList,0))
def dataGen(w, h):
with open(configFile) as config_file:
data = yaml.safe_load(config_file)
objAliasLabelList = []
objLabel = data['objLabel']
startAt = data['startAt']
endAt = data['endAt']
skip = data['skip']
if 'camID' in data:
camID = data['camID']
else:
camID = '0'
handSegDir = os.path.join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq, 'segmentation',
FLAGS.camID, 'raw_seg_results')
if datasetName == datasetType.HO3D_MULTICAMERA:
files = os.listdir(join(base_dir, 'rgb', camID))
files = [
FLAGS.seq + '/' + camID + '/' + f1[:-4] for f1 in files
if 'png' in f1
]
files = sorted(files)
fileListIn = sorted(files)
dataset = datasetHo3dMultiCamera(FLAGS.seq, 0, fileListIn=fileListIn)
if isinstance(startAt, str):
for i, f in enumerate(files):
if f.split('/')[-1] == startAt:
break
files = files[i::skip]
# print(files)
else:
files = files[startAt:endAt + 1:skip]
else:
raise NotImplementedError
for file in files:
if datasetName == datasetType.HO3D_MULTICAMERA:
seq = file.split('/')[0]
camInd = file.split('/')[1]
id = file.split('/')[2]
_, ds = dataset.createTFExample(itemType='hand', fileIn=file)
img = ds.imgRaw[:, :, [2, 1, 0]]
dpt = ds.depth
seg = cv2.imread(
join(HO3D_MULTI_CAMERA_DIR, seq, 'segmentation', camInd,
'raw_seg_results', id + '.png'))[:, :, 0]
frameID = np.array([join(id)])
else:
raise NotImplementedError
dpt = dpt[:, :, 0] + dpt[:, :, 1] * 256
dpt = dpt * depthScale
dptMask = np.logical_or(dpt > DEPTH_THRESH, dpt == 0.0)
dpt[dptMask] = bgDepth
seg[dpt > DEPTH_THRESH] = 0
objMask = (seg == objLabel)
handMask = (seg == handLabel)
objImg = img * np.expand_dims(objMask, 2)
handImg = img * np.expand_dims(handMask, 2)
objDepth = dpt * objMask
objDepth[np.logical_not(objMask)] = bgDepth
handDepth = dpt * handMask
handDepth[np.logical_not(handMask)] = bgDepth
handMask = np.stack([handMask, handMask, handMask], axis=2).astype(
np.float32) * handSegColor
objMask = np.stack([objMask, objMask, objMask], axis=2).astype(
np.float32) * objSegColor
handObjMask = handMask + objMask
handObjDepth = dpt * (np.sum(handObjMask, 2) > 0).astype(np.float32)
handObjDepth[np.sum(handObjMask, 2) == 0] = bgDepth
handObjImg = img #* np.expand_dims((np.sum(handObjMask, 2)>0).astype(np.float32), 2)
mask = np.logical_not(np.zeros_like(objMask, dtype=np.bool))
handObjDepth = cv2.resize(handObjDepth, (w, h),
interpolation=cv2.INTER_NEAREST)
handObjDepth = np.stack([handObjDepth, handObjDepth, handObjDepth],
axis=2)
handObjDepth = np.expand_dims(handObjDepth, 0).astype(np.float32)
handObjMask = cv2.resize(handObjMask, (w, h),
interpolation=cv2.INTER_NEAREST)
handObjMask = np.expand_dims(handObjMask, 0).astype(np.float32)
mask = cv2.resize(mask.astype(np.float32), (w, h),
interpolation=cv2.INTER_NEAREST)
mask = np.expand_dims(mask, 0).astype(np.float32)
handObjImg = cv2.resize(handObjImg, (w, h),
interpolation=cv2.INTER_CUBIC)
handObjImg = np.expand_dims(handObjImg, 0).astype(np.float32)
handObjImg = handObjImg.astype(
np.float32) / 255. #np.zeros_like(mask, dtype=np.float32)
yield (frameID, handObjMask, handObjDepth, handObjImg, mask)
def dataGen(w, h, datasetName):
'''
Generator which provides rgb, depth and segmentation data for each frame
'''
configFile = join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq,
'configs/configObjPose.json')
# read the config file
with open(configFile) as config_file:
data = yaml.safe_load(config_file)
base_dir = os.path.join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq)
obj = data['obj']
objAliasLabelList = []
objLabel = data['objLabel']
startAt = data['startAt']
endAt = data['endAt']
skip = data['skip']
# set some paths
modelPath = os.path.join(YCB_MODELS_DIR, obj)
plt.ion()
# get list of filenames
if datasetName == datasetType.HO3D_MULTICAMERA:
fileListIn = os.listdir(
join(HO3D_MULTI_CAMERA_DIR, FLAGS.seq, 'rgb', '0'))
fileListIn = [
join(FLAGS.seq, '0', f[:-4]) for f in fileListIn if 'png' in f
]
fileListIn = sorted(fileListIn)
dataset = datasetHo3dMultiCamera(FLAGS.seq, 0, fileListIn=fileListIn)
files = dataset.fileList
if isinstance(startAt, str):
for i, f in enumerate(files):
if f.split('/')[-1] == startAt:
break
files = files[i::skip]
# print(files)
else:
files = files[startAt:endAt + 1:skip]
else:
raise Exception('Unsupported datasetName')
for i, file in enumerate(files):
# read RGB. depth and segmentations for current fileID
seq = file.split('/')[0]
camInd = file.split('/')[1]
id = file.split('/')[2]
_, ds = dataset.createTFExample(itemType='hand', fileIn=file)
img = ds.imgRaw[:, :, [2, 1, 0]]
dpt = ds.depth
# reading the seg from file because the seg in ds has no objAliasLabel info
seg = cv2.imread(
join(HO3D_MULTI_CAMERA_DIR, seq, 'segmentation', camInd,
'raw_seg_results', id + '.png'))[:, :, 0]
frameID = np.array([join(camInd, id)])
# decode depth map
dpt = dpt[:, :, 0] + dpt[:, :, 1] * 256
dpt = dpt * depthScale
# clean up depth map
dptMask = np.logical_or(dpt > DEPTH_THRESH, dpt == 0.0)
dpt[dptMask] = bgDepth
# clean up seg map
seg[dptMask] = 0
for alias in objAliasLabelList:
seg[seg == alias] = objLabel
# Extract the object image in the image using the mask
objMask = (seg == objLabel)
handMask = (seg == handLabel)
objImg = img * np.expand_dims(objMask, 2)
handImg = img * np.expand_dims(handMask, 2)
# plt.imshow(seg)
# plt.show()
# Extract the object depth in the depth map using the mask
objDepth = dpt * objMask
objDepth[np.logical_not(objMask)] = bgDepth
handDepth = dpt * handMask
handDepth[np.logical_not(handMask)] = bgDepth
# skip the file if object is too small (occluded heavily)
maskPc = float(np.sum(objMask)) / float(
(objMask.shape[0] * objMask.shape[0]))
print('maskPC for Image %s is %f' % (file, maskPc))
if maskPc < .005:
continue
# resizing
objDepth = cv2.resize(objDepth, (w, h),
interpolation=cv2.INTER_NEAREST)
objMask = cv2.resize(objMask.astype(np.uint8), (w, h),
interpolation=cv2.INTER_NEAREST)
handMask = cv2.resize(handMask.astype(np.uint8), (w, h),
interpolation=cv2.INTER_NEAREST)
objImg = cv2.resize(objImg.astype(np.uint8), (w, h),
interpolation=cv2.INTER_CUBIC)
# minor changes to the dimensions as required by TF
objMask = np.stack([objMask, objMask, objMask], axis=2)
objMask = np.expand_dims(objMask, 0).astype(np.float32)
objDepth = np.stack([objDepth, objDepth, objDepth], axis=2)
objDepth = np.expand_dims(objDepth, 0)
handMask = np.stack([handMask, handMask, handMask], axis=2)
handMask = np.expand_dims(handMask, 0).astype(np.float32)
objImg = np.expand_dims(objImg, 0).astype(
np.float32) / 255. # np.zeros_like(mask, dtype=np.float32)
yield (frameID, objMask, objDepth, objImg, handMask)