def __getitem__(self, index):
img = self.LoadImage(index)
pts, c, s = self.GetPartInfo(index)
r = 0
if self.split == 'train':
s = s * (2 ** Rnd(ref.scale))
r = 0 if np.random.random() < 0.6 else Rnd(ref.rotate)
inp = Crop(img, c, s, r, ref.inputRes) / 256.
out = np.zeros((ref.nJoints, ref.outputRes, ref.outputRes))
out32 = np.zeros((ref.nJoints,32,32))
for i in range(ref.nJoints):
if pts[i][0] > 1:
pt = Transform(pts[i], c, s, r, ref.outputRes)
out[i] = DrawGaussian(out[i], pt, ref.hmGauss)
for i in range(ref.nJoints):
if pts[i][0] > 1:
pt = Transform(pts[i], c, s, r, 32)
out32[i] = DrawGaussian(out32[i], pt, ref.hmGauss)
if self.split == 'train':
if np.random.random() < 0.5:
inp = Flip(inp)
out = ShuffleLR(Flip(out))
out32 = ShuffleLR(Flip(out32))
inp[0] = np.clip(inp[0] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[1] = np.clip(inp[1] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[2] = np.clip(inp[2] * (np.random.random() * (0.4) + 0.6), 0, 1)
meta = np.zeros(1)
else:
meta = {'index' : index, 'center' : c, 'scale' : s, 'rotate': r}
return inp, out, out32, meta
def __getitem__(self, index):
img = self.LoadImage(index)
pts, c, s = self.GetPartInfo(index)
r = 0
if self.split == 'train':
s = s * (2 ** Rnd(ref.scale))
r = 0 if np.random.random() < 0.6 else Rnd(ref.rotate)
inp = Crop(img, c, s, r, ref.inputRes) / 256.
out = np.zeros((ref.nJoints, ref.outputRes, ref.outputRes))
Reg = np.zeros((ref.nJoints, 3))
for i in range(ref.nJoints):
if pts[i][0] > 1:
pt = Transform(pts[i], c, s, r, ref.outputRes)
out[i] = DrawGaussian(out[i], pt, ref.hmGauss)
Reg[i, :2] = pt
Reg[i, 2] = 1
if self.split == 'train':
if np.random.random() < 0.5:
inp = Flip(inp)
out = ShuffleLR(Flip(out))
Reg[:, 1] = Reg[:, 1] * -1
Reg = ShuffleLR(Reg)
#print 'before', inp[0].max(), inp[0].mean()
inp[0] = np.clip(inp[0] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[1] = np.clip(inp[1] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[2] = np.clip(inp[2] * (np.random.random() * (0.4) + 0.6), 0, 1)
#print 'after', inp[0].max(), inp[0].mean()
inp = torch.from_numpy(inp)
if self.returnMeta:
return inp, out, Reg, np.zeros((ref.nJoints, 3))
else:
return inp, out
def __getitem__(self, index):
if self.split == 'train':
index = np.random.randint(self.nSamples)
img = self.LoadImage(index)
pts, c, s, pts_3d, pts_3d_mono = self.GetPartInfo(index)
pts_3d[7] = (pts_3d[12] +
pts_3d[13]) / 2 # neck = average of shoulders
inp = Crop(img, c, s, 0,
ref.inputRes) / 256. # crop image to input resolution
outMap = np.zeros((ref.nJoints, ref.outputRes,
ref.outputRes)) # nJoints x 64 x 64 heatmap for 2D
outReg = np.zeros((ref.nJoints, 3)) # Regression target for 3D
for i in range(ref.nJoints):
pt = Transform3D(pts_3d[i], c, s, 0, ref.outputRes)
if pts[i][0] > 1:
outMap[i] = DrawGaussian(
outMap[i], pt[:2],
ref.hmGauss) # Draw 2D heat map for detection
outReg[i, 2] = pt[2] / ref.outputRes * 2 - 1
inp = torch.from_numpy(inp)
return inp, outMap, outReg, pts_3d_mono
def getitem(self, index, meta, whichFrames, frameWiseData):
img = cv2.imread(ref.posetrackDataDir + "/" + whichFrames[index])
#print(ref.posetrackDataDir + whichFrames[index])
frameData = frameWiseData[whichFrames[index]]
bboxmean, bboxdelta, joints = frameData[meta['personIndex']]
pts = joints
c = np.asarray(bboxmean)
s = bboxdelta * meta['s'] / 100.0
r = meta['r']
inp = Crop(img, c, s, r, ref.inputRes) / 256.
out = np.zeros((ref.nJoints, ref.outputRes, ref.outputRes))
Reg = np.zeros((ref.nJoints, 3))
for i in range(ref.nJoints):
if pts[i][0] > 1:
pt = Transform(pts[i], c, s, r, ref.outputRes)
out[i] = DrawGaussian(out[i], pt, ref.hmGauss)
Reg[i, :2] = pt
Reg[i, 2] = 1
if self.split == 'train':
if meta['flip']:
inp = Flip(inp)
out = ShuffleLR(Flip(out))
Reg[:, 1] = Reg[:, 1] * -1
Reg = ShuffleLR(Reg)
inp[0] = np.clip(inp[0] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[1] = np.clip(inp[1] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[2] = np.clip(inp[2] * (np.random.random() * (0.4) + 0.6), 0, 1)
whatever = (np.zeros((ref.nJoints, 3)))
return inp, out, Reg, whatever
def __getitem__(self, index):
img = self.LoadImage(index)
pts, action, c, s = self.GetPartInfo(index)
nb_pts = len(pts)
r = 0
flip = False
if self.split == 'train':
s = s * (2**Rnd(ref.scale))
r = 0 if np.random.random() < 0.6 else Rnd(ref.rotate)
inp = old_div(Crop(img, c, s, r, ref.inputRes), 256.)
if self.split == 'train':
if np.random.random() < 0.5:
inp = Flip(inp)
flip = True
inp[0] = np.clip(inp[0] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[1] = np.clip(inp[1] * (np.random.random() * (0.4) + 0.6), 0, 1)
inp[2] = np.clip(inp[2] * (np.random.random() * (0.4) + 0.6), 0, 1)
meta = np.zeros(1)
else:
meta = {'index': index, 'center': c, 'scale': s, 'rotate': r}
output = []
for k in range(nb_pts):
out = np.zeros((ref.nJoints, ref.outputRes, ref.outputRes))
for i in range(ref.nJoints):
if pts[k][i][0] > 1:
pt = Transform(pts[k][i], c, s, r, ref.outputRes)
out[i] = DrawGaussian(out[i], pt, ref.hmGauss)
if self.split == 'train':
out = ShuffleLR(Flip(out))
output.append(out)
return inp, output, action, meta
def __getitem__(self, index):
img = self.LoadImage(index)
pts2d, pts3d, emb, c, s = self.GetPartInfo(index)
s = min(s, max(img.shape[0], img.shape[1])) * 1.0
pts3d[:, 2] += s / 2
r = 0
if self.split == 'train':
s = s * (2**Rnd(ref.scale))
c[1] = c[1] + Rnd(ref.shiftY)
r = 0 if np.random.random() < 0.6 else Rnd(ref.rotate)
inp = Crop(img, c, s, r, ref.inputRes)
inp = inp.transpose(2, 0, 1).astype(np.float32) / 256.
starMap = np.zeros((1, ref.outputRes, ref.outputRes))
embMap = np.zeros((3, ref.outputRes, ref.outputRes))
depMap = np.zeros((1, ref.outputRes, ref.outputRes))
mask = np.concatenate([
np.ones((1, ref.outputRes, ref.outputRes)),
np.zeros((4, ref.outputRes, ref.outputRes))
])
for i in range(pts3d.shape[0]):
if self.annot['valid'][index][i] > ref.eps:
if (self.annot['vis'][index][i] > ref.eps):
pt3d = Transform3D(pts3d[i], c, s, r,
ref.outputRes).astype(np.int32)
pt2d = Transform(pts2d[i], c, s, r,
ref.outputRes).astype(np.int32)
if pt2d[0] >= 0 and pt2d[0] < ref.outputRes and pt2d[
1] >= 0 and pt2d[1] < ref.outputRes:
embMap[:, pt2d[1], pt2d[0]] = emb[i]
depMap[0, pt2d[1],
pt2d[0]] = 1.0 * pt3d[2] / ref.outputRes - 0.5
mask[1:, pt2d[1], pt2d[0]] = 1
starMap[0] = np.maximum(
starMap[0],
DrawGaussian(np.zeros((ref.outputRes, ref.outputRes)),
pt2d, ref.hmGauss).copy())
out = starMap
if 'emb' in self.opt.task:
out = np.concatenate([out, embMap])
if 'dep' in self.opt.task:
out = np.concatenate([out, depMap])
mask = mask[:out.shape[0]].copy()
if self.split == 'train':
if np.random.random() < 0.5:
inp = Flip(inp)
out = Flip(out)
mask = Flip(mask)
if 'emb' in self.opt.task:
out[1] = -out[1]
return inp, out, mask
def __getitem__(self, index):
if self.split == 'train':
index = np.random.randint(self.nSamples)
img = self.LoadImage(index)
pts, c, s, pts_3d, pts_3d_mono = self.GetPartInfo(index)
pts_3d[7] = (pts_3d[12] + pts_3d[13]) / 2
inp = Crop(img, c, s, 0, ref.inputRes) / 256.
outMap = np.zeros((ref.nJoints, ref.outputRes, ref.outputRes))
outReg = np.zeros((ref.nJoints, 3))
for i in range(ref.nJoints):
pt = Transform3D(pts_3d[i], c, s, 0, ref.outputRes)
if pts[i][0] > 1:
outMap[i] = DrawGaussian(outMap[i], pt[:2], ref.hmGauss)
outReg[i, 2] = pt[2] / ref.outputRes * 2 - 1
inp = torch.from_numpy(inp)
return inp, outMap, outReg, pts_3d_mono
def LoadFrameAndData(self, path, frameName):
frame = cv2.imread(path + frameName)
pts_2d, pts_3d, pts_3d_mono = pickle.load(open(
path + "data.pkl", 'rb'))[int(frameName[-10:-4])]
pts_2d = pts_2d
pts_3d = pts_3d
pts_3d_mono = pts_3d_mono
c = np.ones(2) * ref.h36mImgSize / 2
s = ref.h36mImgSize * 1.0
pts_3d = pts_3d - pts_3d[self.root]
s2d, s3d = 0, 0
for e in ref.edges:
s2d += ((pts_2d[e[0]] - pts_2d[e[1]])**2).sum()**0.5
s3d += ((pts_3d[e[0], :2] - pts_3d[e[1], :2])**2).sum()**0.5
scale = s2d / s3d
for j in range(ref.nJoints):
pts_3d[j, 0] = pts_3d[j, 0] * scale + pts_2d[self.root, 0]
pts_3d[j, 1] = pts_3d[j, 1] * scale + pts_2d[self.root, 1]
pts_3d[j, 2] = pts_3d[j, 2] * scale + ref.h36mImgSize / 2
pts_3d[7, :] = (pts_3d[12, :] + pts_3d[13, :]) / 2
frame = Crop(frame, c, s, 0, ref.inputRes) / 256.
outMap = np.zeros((ref.nJoints, ref.outputRes, ref.outputRes))
outReg = np.zeros((ref.nJoints, 3))
for i in range(ref.nJoints):
pt = Transform3D(pts_3d[i], c, s, 0, ref.outputRes)
if pts_2d[i][0] > 1:
outMap[i] = DrawGaussian(outMap[i], pt[:2], ref.hmGauss)
outReg[i, 2] = pt[2] / ref.outputRes * 2 - 1
return frame, outMap, pts_2d, outReg, pts_3d_mono
def __getitem__(self, index):
if self.split == 'train':
index = int(torch.randint(self.nVideos, ()))
whichFrames, frameWiseData = self.annotations[index]
numFrames = len(whichFrames)
if self.loadConsecutive:
startpt = random.randint(1, numFrames - (self.nFramesLoad))
inpFrames = np.zeros((3, self.nFramesLoad, 256, 256))
outPts_2ds = np.zeros((ref.nJoints, self.nFramesLoad, 2))
outOutRegs = np.ones((ref.nJoints, self.nFramesLoad, 1))
outPts_3d_monos = np.zeros((ref.nJoints, self.nFramesLoad, 3))
outOutMaps = np.zeros(
(ref.nJoints, self.nFramesLoad, ref.outputRes, ref.outputRes))
if self.split == 'val':
startPt = 0
oldnFramesLoad = self.nFramesLoad
self.nFramesLoad = min(150, self.nFramesLoad)
result = set(frameWiseData[whichFrames[startpt]].keys())
for i in range(self.nFramesLoad):
try:
result.intersection_update(
frameWiseData[whichFrames[startpt + i]].keys())
except:
print(startpt + i)
assert 1 != 1
if (len(result) == 0):
return self.__getitem__(index + 1)
personIndex = random.choice(list(result))
if self.augmentation & (self.split == 'train'):
angle = random.randint(0, 10)
scale = 1 + (random.random() - 0.5) * 0.4
flipOrNot = random.random() > 0
else:
angle = 0
scale = 1
flipOrNot = 0
if self.split == 'val':
startPt = 0
oldnFramesLoad = self.nFramesLoad
self.nFramesLoad = min(150, numFrames)
for i in range(self.nFramesLoad):
tempFrame = cv2.imread(ref.posetrackDataDir + '/' +
whichFrames[startpt + i])
#print(ref.posetrackDataDir + '/' + whichFrames[startpt + 0])
frameData = frameWiseData[whichFrames[startpt + i]]
bboxmean, bboxdelta, joints = frameData[personIndex]
tempFrame = F.to_pil_image(tempFrame)
croppedImage = F.crop(tempFrame, bboxmean[1] - bboxdelta[1],
bboxmean[0] - bboxdelta[0],
2 * bboxdelta[1], 2 * bboxdelta[0])
paddedCroppedImage = F.pad(
croppedImage, (int(max(0, bboxdelta[1] - bboxdelta[0])),
int(max(0, bboxdelta[0] - bboxdelta[1]))))
resizedSquarePaddedCropped = F.resize(paddedCroppedImage,
(256, 256))
joints = joints.copy()
for k in range(joints.shape[0]):
if (joints[k][0] < 0):
pass
else:
joints[k][0] -= bboxmean[0] - bboxdelta[0] - (max(
0, bboxdelta[1] - bboxdelta[0]))
joints[k][1] -= bboxmean[1] - bboxdelta[1] - (max(
0, bboxdelta[0] - bboxdelta[1]))
scaleFac = 256. / (2 * max(bboxdelta[1], bboxdelta[0]))
joints[k][0] *= scaleFac
joints[k][1] *= scaleFac
rotated = F.affine(resizedSquarePaddedCropped, angle, (0, 0),
scale, 0)
if flipOrNot:
flipped = hflip(rotated)
thisFrame = flipped
else:
thisFrame = rotated
inpFrames[:, i, :, :] = F.to_tensor(thisFrame)
outPts_2ds[:, i, :] = joints
for j in range(ref.nJoints):
if (joints[j, :] == -1).all():
continue
else:
outOutMaps[j, i, :, :] = DrawGaussian(
outOutMaps[j, i, :, :], joints[j, :] / 4,
ref.hmGauss)
if self.split == 'val':
self.nFramesLoad = oldnFramesLoad
return (inpFrames, outOutMaps, outPts_2ds, outOutRegs, outPts_3d_monos)
def __getitem__(self, index):
seqIdx = self.getSeq(index)
"""
input: predSeqLen x 3 x inputRes x inputRes Input image After Crop and transform
hmap: predSeqLen x numJoints x outputRes x outputRes
gtpts: predSeqLen x numJoints x 2 Joints Positions BEFORE crop and transform
proj: predSeqLen x numJoints x 2 Joints Positions AFTER crop and transform
"""
input = np.zeros((self.nPhase, 3, self.inputRes, self.inputRes))
hmap = np.zeros(
(self.nPhase, self.nJoints, self.outputRes, self.outputRes))
gtpts = np.zeros((self.nPhase, self.nJoints, 2))
repos, trans, focal, proj = {}, {}, {}, {}
for i in range(len(seqIdx)):
sid = seqIdx[i]
im = self.LoadImage(int(sid))
if i == 0:
center, scale = self.getCenterScale(im)
inp = Crop(im, center, scale, 0, self.inputRes)
pts = self.part[int(sid)]
pj = np.zeros(np.shape(pts))
for j in range(len(pts)):
if pts[j][0] != 0 and pts[j][1] != 0:
pj[j] = Transform(pts[j], center, scale, 0, self.outputRes,
False)
hm = np.zeros((np.shape(pts)[0], self.outputRes, self.outputRes))
for j in range(len(pts)):
if pts[j][0] != 0 and pts[j][1] != 0:
DrawGaussian(hm[j], np.round(pj[j]), 2)
inp = inp.transpose(2, 1, 0)
input[i] = inp
repos[i] = np.zeros((np.size(1), 3))
trans[i] = np.zeros(3)
focal[i] = np.zeros(1)
hmap[i] = hm
proj[i] = pj
gtpts[i] = pts
if self.split == 'train':
m1 = np.random.uniform(0.8, 1.2)
m2 = np.random.uniform(0.8, 1.2)
m3 = np.random.uniform(0.8, 1.2)
for i in range(len(input)):
input[i][:, :, 0] = input[i][:, :, 0] * m1
np.clip(input[i][:, :, 0], 0, 1, out=input[i][:, :, 0])
input[i][:, :, 1] = input[i][:, :, 1] * m2
np.clip(input[i][:, :, 1], 0, 1, out=input[i][:, :, 1])
input[i][:, :, 2] = input[i][:, :, 2] * m3
np.clip(input[i][:, :, 2], 0, 1, out=input[i][:, :, 2])
if np.random.uniform() <= 0.5:
for i in range(len(input)):
input[i] = cv2.flip(input[i], 1)
hmap[i] = Flip(ShuffleLR(hmap[i]))
proj[i] = ShuffleLR(proj[i])
ind = np.where(proj[i] == 0)
proj[i][:, 0] = self.outputRes - proj[i][:, 0] + 1
if len(ind[0]) != 0:
proj[i][ind[0][0]] = 0
return {
'input': input,
'label': hmap,
'gtpts': gtpts,
'center': center,
'scale': scale,
'proj': proj
}