def generateSamples(self, frame, position, samples):
# Generating the transformataions
frameDims = frame.shape
frame = frame.transpose(2, 0, 1)
position = Preprocess.scalePosition(position, frameDims[:2])
theta, thetaInv = self.generateTheta(position, samples,
self.transRange)
# Generating the sampled frames
frames = TH.shared(frame)
frames = TH.tensor.tile(frames, (samples, 1, 1, 1))
output = self.transformer.get_output_for((frames, theta[:, :2, :]))
frames = output.eval()
frames = frames.transpose(0, 2, 3, 1)
# Generating the sampled positions
positions = NP.vstack(
[NP.ravel(position).reshape((2, -1), order="F"),
NP.ones((1, 2))])
positions = NP.dot(thetaInv, positions)[:, :2, :].T
positions = NP.ravel(positions).reshape((samples, -1), order="F")
positions = Preprocess.rescalePosition(positions, frameDims[:2])
return frames, positions
def track(tracker, processor, frame, position, size):
x, y, w, h = position
x1, y1 = x + w, y + h
position = NP.array([x, y, x1, y1])
position = NP.expand_dims(position, axis=0)
position = NP.expand_dims(position, axis=1)
originalSize = frame.shape[:2][::-1] # imageSize must be (width, height)
frame = SCPM.imresize(frame, size)
frame = NP.expand_dims(frame, axis=0)
frame = NP.expand_dims(frame, axis=1)
position = Preprocess.scalePosition(position, originalSize)
position = Preprocess.rescalePosition(position, size)
frame, position = processor.preprocess(frame, position)
position = tracker.forward([frame], position[:, 0, :])
x, y, x1, y1 = position[0, 0, :]
#logging.info("Tracker prediction: [%s, %s, %s, %s]", x, y, x1, y1)
return vot.Rectangle(x, y, x1 - x, y1 - y)
def loadSequence(path, extension, boxesFile, size):
# Load frames
framesPath = [
os.path.join(path, fn) for fn in os.listdir(path)
if fn.endswith(extension)
]
framesPath = sorted(framesPath)
frame = []
for i, framePath in enumerate(framesPath):
tmpFrame = loadFrame(framePath)
originalSize = tmpFrame.shape[:
2][::
-1] # imageSize must be (width, height)
frame.append(SCPM.imresize(tmpFrame, size))
frame = NP.array(frame, dtype=NP.float)
# Load bounding boxes information
boxesPath = os.path.join(path, boxesFile)
position = loadPosition(boxesPath)
position = Preprocess.scalePosition(position, originalSize)
position = Preprocess.rescalePosition(position, size)
return frame, position
def postprocess(self, frame, position):
frame = Preprocess.rescaleFrame(frame)
#frame = frame[:,:,::-1,:,:] # Make RGB
frameDims = frame.shape[-2:]
frame = frame.transpose(0, 1, 3, 4, 2)
position = self.positionModel.toTwoCorners(position)
position = Preprocess.rescalePosition(position, frameDims)
return frame, position
def preprocess(self, frame, position):
frame = frame.transpose(0, 1, 4, 2, 3)
#frame = frame[:,:,::-1,:,:] # Make BGR
frame = Preprocess.scaleFrame(frame)
frameDims = frame.shape[-2:]
position = Preprocess.scalePosition(position, frameDims)
position = self.positionModel.fromTwoCorners(position)
return frame, position
def generateSamples(self, frame, position, samples):
# Generating the sampled positions
frameDims = frame.shape
position = Preprocess.scalePosition(position, frameDims[:2])
targetDim = position.shape[0]
trans = NP.random.uniform(-self.transRange, self.transRange, size=(samples, targetDim))
sampledPosition = position + trans
sampledPosition = Preprocess.rescalePosition(sampledPosition, frameDims[:2])
return NP.tile(frame, (samples, 1, 1, 1)), sampledPosition
def generateSamples(self, frame, position, samples):
# Generating the sampled positions
frameDims = frame.shape
position = Preprocess.scalePosition(position, frameDims[:2])
targetDim = position.shape[0]
trans = NP.random.uniform(-self.transRange,
self.transRange,
size=(samples, targetDim))
sampledPosition = position + trans
sampledPosition = Preprocess.rescalePosition(sampledPosition,
frameDims[:2])
return NP.tile(frame, (samples, 1, 1, 1)), sampledPosition
def getSample(self):
objData = self.randGen.choice(self.summary[self.summaryKey])
objPath = OS.path.join(self.framesPath, objData['file_name'].strip())
frame = Image.open(objPath)
if frame.mode != "RGB":
frame = frame.convert("RGB")
frame = NP.array(frame)
originalSize = frame.shape[:2][::-1] # imageSize must be (width, height)
position = Preprocess.scalePosition(NP.array(objData["bbox"]), originalSize)
position = Preprocess.rescalePosition(position, self.frameShape[1:])
frame = SCPM.imresize(frame, self.frameShape[1:3])
return frame, position
def getSample(self):
objData = self.randGen.choice(self.summary[self.summaryKey])
objPath = OS.path.join(self.framesPath, objData['file_name'].strip())
frame = Image.open(objPath)
if frame.mode != "RGB":
frame = frame.convert("RGB")
frame = NP.array(frame)
frame.shape
originalSize = frame.shape[:2][::
-1] # imageSize must be (width, height)
position = Preprocess.scalePosition(NP.array(objData["bbox"]),
originalSize)
position = Preprocess.rescalePosition(position, self.frameShape[1:])
frame = SCPM.imresize(frame, self.frameShape[1:3])
return frame, position
def postprocess(self, frame, position):
frame += self.mean
frame = frame[:,:,::-1,:,:] # Make RGB
frame = frame.transpose(0, 1, 3, 4, 2)
position = self.positionModel.toTwoCorners(position)
position = Preprocess.rescalePosition(position, self.frameDims)
return frame, position
def preprocess(self, frame, position):
frame = frame.astype(float).transpose(0, 1, 4, 2, 3)
frame = frame[:,:,::-1,:,:] # Make BGR
frame -= self.mean
position = Preprocess.scalePosition(position, self.frameDims)
position = self.positionModel.fromTwoCorners(position)
return frame, position
def preprocess(self, frame, position):
frame = NP.copy(frame).transpose(0, 1, 4, 2, 3)
#frame = frame[:,:,::-1,:,:] # Make BGR
frame = Preprocess.scaleFrame(frame)
frameDims = frame.shape[-2:][::-1]
oRange = NP.array([[0.0, 0.0], frameDims]).T
position = self.positionModel.scale(position, oRange, self.tRange)
position = self.positionModel.fromTwoCorners(position)
return frame, position
def postprocess(self, frame, position):
frame = Preprocess.rescaleFrame(NP.copy(frame))
#frame = frame[:,:,::-1,:,:] # Make RGB
frameDims = frame.shape[-2:][::-1]
frame = frame.transpose(0, 1, 3, 4, 2)
oRange = NP.array([[0.0, 0.0], frameDims]).T
position = self.positionModel.toTwoCorners(position)
position = self.positionModel.scale(position, self.tRange, oRange)
return frame, position
def preprocess(self, frame, position):
attPosition = NP.roll(position, 1, axis=1) # Shift the time
frame, position = self.processor.preprocess(frame, position)
frameDims = frame.shape[-2:]
if position.shape[1] > 1:
attPosition[:, 0, :] = attPosition[:, 1, :] # First frame is ground truth
attPosition = Preprocess.scalePosition(attPosition, frameDims)
return [frame, attPosition], position
def track(tracker, frame, position, size):
x, y, w, h = position
x1, y1 = x + w, y + h
position = NP.array([x, y, x1, y1])
position = NP.expand_dims(position, axis=0)
position = NP.expand_dims(position, axis=1)
originalSize = frame.shape[:2][::-1] # imageSize must be (width, height)
frame = SCPM.imresize(frame, size)
frame = NP.expand_dims(frame, axis=0)
frame = NP.expand_dims(frame, axis=1)
position = Preprocess.scalePosition(position, originalSize)
position = Preprocess.rescalePosition(position, size)
position = tracker.forward(frame, position[:, :1, :])
x, y, x1, y1 = position[0, 0, :]
return vot.Rectangle(x, y, x1 - x, y1 - y)
def generateSamples(self, frame, position, samples):
# Generating the transformataions
frameDims = frame.shape
frame = frame.transpose(2, 0, 1)
position = Preprocess.scalePosition(position, frameDims[:2])
theta, thetaInv = self.generateTheta(position, samples, self.transRange)
# Generating the sampled frames
frames = TH.shared(frame)
frames = TH.tensor.tile(frames, (samples, 1, 1, 1))
output = self.transformer.get_output_for((frames, theta[:, :2, :]))
frames = output.eval()
frames = frames.transpose(0, 2, 3, 1)
# Generating the sampled positions
positions = NP.vstack([NP.ravel(position).reshape((2, -1), order="F"), NP.ones((1, 2))])
positions = NP.dot(thetaInv, positions)[:, :2, :].T
positions = NP.ravel(positions).reshape((samples, -1), order="F")
positions = Preprocess.rescalePosition(positions, frameDims[:2])
return frames, positions
def getBatch(self, batchSize):
batchF = NP.empty((batchSize, self.seqLength) + self.frameShape)
batchP = NP.empty((batchSize, self.seqLength, 4))
for i in range(batchSize):
frame, position = self.getSample()
# Preprocessing the data
frame = frame.transpose(2, 0, 1)
position = Preprocess.scalePosition(position, self.frameShape[1:])
# Executing the sequence generation
theta = self.trajectoryModel.generateTrajectory(self.seqLength)
frame = NP.tile(frame, (self.seqLength, 1, 1, 1))
frame = self.transformer.predict_on_batch([frame, theta])
batchF[i, ...] = frame
batchP[i, ...] = self.pm.transform(NLA.inv(theta), NP.tile(position, (self.seqLength, 1)))
batchF = batchF.transpose(0, 1, 3, 4, 2)
batchP = Preprocess.rescalePosition(batchP, self.frameShape[1:])
return batchF, batchP
def getBatch(self, batchSize):
batchF = NP.empty((batchSize, self.seqLength) + self.frameShape)
batchP = NP.empty((batchSize, self.seqLength, 4))
for i in range(batchSize):
frame, position = self.getSample()
# Preprocessing the data
frame = frame.transpose(2, 0, 1)
position = Preprocess.scalePosition(position, self.frameShape[1:])
# Executing the sequence generation
theta, position = self.trajectoryModel.generateTrajectory(
position, self.seqLength)
frame = NP.tile(frame, (self.seqLength, 1, 1, 1))
frame = self.transformer.predict_on_batch([frame, theta])
batchF[i, ...] = frame
batchP[i, ...] = position
batchF = batchF.transpose(0, 1, 3, 4, 2)
batchP = Preprocess.rescalePosition(batchP, self.frameShape[1:])
return batchF, batchP
print("iouFirstFrames = " + str(iou[0, :5]))
PLT.plot(iou[0, ...])
PLT.suptitle("{}, {} {} = {}".format(testVotSeqName, measure.name, "Mean",
iouMean))
PLT.xlabel("frame")
PLT.ylabel(measure.name)
PLT.show()
# ## GENERATING VOT VIDEO
# In[ ]:
gtPos = positionRepresentation.fromTwoCorners(testVotP)[0, ...].tolist()
predPos = positionRepresentation.fromTwoCorners(postPredTestVotP)[
0, ...].tolist()
images = Preprocess.getFrames(testVotF[0, ...])
sequence = Sequence(images, positionRepresentation)
sequence.addBoxes(gtPos, "Red")
sequence.addBoxes(predPos, "Blue")
output = os.path.join(outDir, trackerName + testVotSeqName + ".mp4")
sequence.exportToVideo(fps, output)
# ## GENERATING ONLINE SAMPLES
# In[2]:
def generateSamples():
sampledF, sampledP = sampler.generateSamples(testVotF[0, 100, ...],
testVotP[:,
100, :], onBatchSize)