def A(self, x):
# First pass
scope = 'first/'
feed_dict = self.problem.get_feed_dict(scope)
# add variable feed
for idx, v in enumerate(self.x):
feed_dict['{}x_{}'.format(scope, idx)] = v
# add p feed
for idx, v in enumerate(x):
feed_dict['{}p_{}'.format(scope, idx)] = v
dfdx_x = TensorList(
self.exe.run(self.compiled_prog,
feed=feed_dict,
fetch_list=[v.name for v in self.dfdx_x]))
# Second pass
scope = 'second/'
feed_dict = self.problem.get_feed_dict(scope)
# add variable feed
for idx, v in enumerate(self.x):
feed_dict['{}x_{}'.format(scope, idx)] = v
# add p feed
for idx, v in enumerate(dfdx_x):
feed_dict['{}dfdx_x_{}'.format(scope, idx)] = v
res = TensorList(
self.exe2.run(self.compiled_prog2,
feed=feed_dict,
fetch_list=[v.name for v in self.dfdx_dfdx]))
return res
def init_label_function(self, train_x):
# Allocate label function
self.y = TensorList([
np.zeros(
[self.params.sample_memory_size, 1, x.shape[2], x.shape[3]],
'float32') for x in train_x
])
# Output sigma factor
output_sigma_factor = self.fparams.attribute('output_sigma_factor')
self.sigma = output_sigma_factor * np.ones(
(2, ), 'float32') * (self.feature_sz / self.img_support_sz *
self.base_target_sz).apply(np.prod).apply(
np.sqrt)
# Center pos in normalized coords
target_center_norm = (self.pos - np.round(self.pos)) / (
self.target_scale * self.img_support_sz)
# Generate label functions
for y, sig, sz, ksz, x in zip(self.y, self.sigma, self.feature_sz,
self.kernel_size, train_x):
center_pos = sz * target_center_norm + 0.5 * np.array(
[(ksz[0] + 1) % 2, (ksz[1] + 1) % 2], 'float32')
for i, T in enumerate(self.transforms[:x.shape[0]]):
sample_center = center_pos + np.array(
T.shift, 'float32') / self.img_support_sz * sz
y[i] = dcf.label_function_spatial(sz, sig, sample_center)
# Return only the ones to use for initial training
return TensorList([y[:x.shape[0]] for y, x in zip(self.y, train_x)])
def local_update(self,
sample_pos,
translation_vec,
scale_ind,
sample_scales,
s,
test_x,
update_flag=None):
# Check flags and set learning rate if hard negative
if update_flag is None:
update_flag = self.flag not in ['not_found', 'uncertain']
hard_negative = (self.flag == 'hard_negative')
learning_rate = self.local_Tracker.params.hard_negative_learning_rate if hard_negative else None
if update_flag:
# Get train sample
train_x = TensorList(
[x[scale_ind:scale_ind + 1, ...] for x in test_x])
# Create label for sample
train_y = self.local_Tracker.get_label_function(
sample_pos, sample_scales[scale_ind])
# Update memory
self.local_Tracker.update_memory(train_x, train_y, learning_rate)
# Train filter
if hard_negative:
self.local_Tracker.filter_optimizer.run(
self.local_Tracker.params.hard_negative_CG_iter)
elif (self.local_Tracker.frame_num -
1) % self.local_Tracker.params.train_skipping == 0:
self.local_Tracker.filter_optimizer.run(
self.local_Tracker.params.CG_iter)
def init_iou_net(self):
# Setup IoU net
self.iou_predictor = self.params.features.get_unique_attribute(
'iou_predictor')
# Get target boxes for the different augmentations
self.iou_target_box = self.get_iounet_box(self.pos, self.target_sz,
self.pos.round(),
self.target_scale)
target_boxes = TensorList()
if self.params.iounet_augmentation:
for T in self.transforms:
if not isinstance(
T, (augmentation.Identity, augmentation.Translation,
augmentation.FlipHorizontal,
augmentation.FlipVertical, augmentation.Blur)):
break
target_boxes.append(self.iou_target_box +
np.array([T.shift[1], T.shift[0], 0, 0]))
else:
target_boxes.append(self.iou_target_box.copy())
target_boxes = np.concatenate(target_boxes.view(1, 4), 0)
# Get iou features
iou_backbone_features = self.get_iou_backbone_features()
# Remove other augmentations such as rotation
iou_backbone_features = TensorList(
[x[:target_boxes.shape[0], ...] for x in iou_backbone_features])
# Extract target feat
with fluid.dygraph.guard():
iou_backbone_features = iou_backbone_features.apply(n2p)
target_boxes = n2p(target_boxes)
target_feat = self.iou_predictor.get_filter(
iou_backbone_features, target_boxes)
self.target_feat = TensorList(
[layers.reduce_mean(x, 0).numpy() for x in target_feat])
if getattr(self.params, 'iounet_not_use_reference', False):
self.target_feat = TensorList([
np.full_like(tf,
tf.norm() / tf.numel()) for tf in self.target_feat
])
def get_label_function(self, sample_pos, sample_scale):
# Generate label function
train_y = TensorList()
target_center_norm = (self.pos - sample_pos) / (self.img_support_sz *
sample_scale)
for sig, sz, ksz in zip(self.sigma, self.feature_sz, self.kernel_size):
center = sz * target_center_norm + 0.5 * np.array(
[(ksz[0] + 1) % 2, (ksz[1] + 1) % 2], 'float32')
train_y.append(dcf.label_function_spatial(sz, sig, center))
return train_y
def get_dfdxt_g(self):
scope = 'first/'
feed_dict = self.problem.get_feed_dict(scope)
# add variable feed
for idx, v in enumerate(self.x):
feed_dict['{}x_{}'.format(scope, idx)] = v
for idx, v in enumerate(self.x):
feed_dict['{}p_{}'.format(scope, idx)] = v
res = self.exe.run(self.compiled_prog,
feed=feed_dict,
fetch_list=[v.name for v in self.dfdxt_g])
return TensorList(res)
def init_projection_matrix(self, x):
# Set if using projection matrix
self.params.use_projection_matrix = getattr(self.params,
'use_projection_matrix',
True)
if self.params.use_projection_matrix:
self.compressed_dim = self.fparams.attribute(
'compressed_dim', None)
proj_init_method = getattr(self.params, 'proj_init_method', 'pca')
if proj_init_method == 'pca':
raise NotImplementedError
elif proj_init_method == 'randn':
with fluid.dygraph.guard():
self.projection_matrix = TensorList([
None if cdim is None else layers.gaussian_random(
(cdim, ex.shape[1], 1, 1), 0.0, 1 /
math.sqrt(ex.shape[1])).numpy()
for ex, cdim in zip(x, self.compressed_dim)
])
elif proj_init_method == 'np_randn':
rng = np.random.RandomState(0)
self.projection_matrix = TensorList([
None if cdim is None else rng.normal(
size=(cdim, ex.shape[1], 1, 1),
loc=0.0,
scale=1 / math.sqrt(ex.shape[1])).astype('float32')
for ex, cdim in zip(x, self.compressed_dim)
])
elif proj_init_method == 'ones':
self.projection_matrix = TensorList([
None if cdim is None else np.ones(
(cdim, ex.shape[1], 1, 1), 'float32') /
math.sqrt(ex.shape[1])
for ex, cdim in zip(x, self.compressed_dim)
])
else:
self.compressed_dim = x.size(1)
self.projection_matrix = TensorList([None] * len(x))
def _construct_graph(self):
train_program = fluid.Program()
start_program = fluid.Program()
with fluid.program_guard(train_program, start_program):
self.x_ph = TensorList([
fluid.layers.data('x_{}'.format(idx),
v.shape,
append_batch_size=False,
stop_gradient=False)
for idx, v in enumerate(self.x)
])
# problem forward
self.f0 = self.problem(self.x_ph)
self.loss = self.problem.ip_output(self.f0, self.f0)
# problem backward
self.grad = TensorList(fluid.gradients(self.loss, self.x_ph))
place = fluid.CUDAPlace(0)
self.exe = fluid.Executor(place)
self.exe.run(program=fluid.default_startup_program())
self.compiled_prog = fluid.compiler.CompiledProgram(train_program)
def evaluate_CG_iteration(self, delta_x):
if self.analyze_convergence:
scope = 'first/'
x = self.x + delta_x
feed_dict = self.problem.get_feed_dict(scope)
for idx, v in enumerate(x):
feed_dict['{}x_{}'.format(scope, idx)] = v
for idx, v in enumerate(x):
feed_dict['{}p_{}'.format(scope, idx)] = v
res = self.exe.run(self.compiled_prog,
feed=feed_dict,
fetch_list=[v.name for v in self.f0])
res = TensorList(res)
loss = self.problem.ip_output(res, res)
def init_memory(self, train_x):
# Initialize first-frame training samples
self.num_init_samples = train_x.size(0)
self.init_sample_weights = TensorList(
[np.ones(x.shape[0], 'float32') / x.shape[0] for x in train_x])
self.init_training_samples = train_x
# Sample counters and weights
self.num_stored_samples = self.num_init_samples.copy()
self.previous_replace_ind = [None] * len(self.num_stored_samples)
self.sample_weights = TensorList([
np.zeros(self.params.sample_memory_size, 'float32')
for x in train_x
])
for sw, init_sw, num in zip(self.sample_weights,
self.init_sample_weights,
self.num_init_samples):
sw[:num] = init_sw
# Initialize memory
self.training_samples = TensorList(
[[np.zeros([cdim, x.shape[2], x.shape[3]], 'float32')] *
self.params.sample_memory_size
for x, cdim in zip(train_x, self.compressed_dim)])
def init_learning(self):
# Get window function
self.feature_window = TensorList(
[dcf.hann2d(sz) for sz in self.feature_sz])
# Filter regularization
self.filter_reg = self.fparams.attribute('filter_reg')
# Activation function after the projection matrix (phi_1 in the paper)
projection_activation = getattr(self.params, 'projection_activation',
'none')
if isinstance(projection_activation, tuple):
projection_activation, act_param = projection_activation
if projection_activation == 'none':
self.projection_activation = lambda x: x
elif projection_activation == 'relu':
self.projection_activation = layers.relu
elif projection_activation == 'elu':
self.projection_activation = layers.elu
elif projection_activation == 'mlu':
self.projection_activation = lambda x: layers.elu(
leaky_relu(x, 1 / act_param), act_param)
else:
raise ValueError('Unknown activation')
# Activation function after the output scores (phi_2 in the paper)
response_activation = getattr(self.params, 'response_activation',
'none')
if isinstance(response_activation, tuple):
response_activation, act_param = response_activation
if response_activation == 'none':
self.response_activation = lambda x: x
elif response_activation == 'relu':
self.response_activation = layers.relu
elif response_activation == 'elu':
self.response_activation = layers.elu
elif response_activation == 'mlu':
self.response_activation = lambda x: layers.elu(
leaky_relu(x, 1 / act_param), act_param)
else:
raise ValueError('Unknown activation')
def run(self, num_iter, dummy=None):
if num_iter == 0:
return
lossvec = None
if self.debug:
lossvec = np.zeros((num_iter + 1, ))
grad_mags = np.zeros((num_iter + 1, ))
for i in range(num_iter):
self.x.stop_gradient = False
# Evaluate function at current estimate
loss = self.problem(self.x)
# Compute grad
loss.backward()
grad = TensorList(self.x.gradient()).apply(n2p)
self.x.clear_gradient()
# Update direction
if self.dir is None:
self.dir = grad
else:
self.dir = grad + self.momentum * self.dir
self.x = self.x.detach()
self.x -= self.step_legnth * self.dir
if self.debug:
lossvec[i] = loss.numpy()
grad_mags[i] = self.problem.ip_input(grad, grad).apply(
layers.sqrt).numpy()
self.problem.training_samples_stack = None
self.x = self.x.detach()
self.x.stop_gradient = True
self.clear_temp()
def run(self, num_iter, dummy=None):
if num_iter == 0:
return
grad_names = [v.name for v in self.grad]
for i in range(num_iter):
res = self.exe.run(self.compiled_prog,
feed=self.get_feed_dict(self.x),
fetch_list=[self.loss.name] + grad_names)
if self.debug:
loss = res[0]
#print('Paddle Loss: {}'.format(loss))
grad = TensorList(res[1:])
# update parameters
if self.dir is None:
self.dir = grad
else:
self.dir = grad + self.momentum * self.dir
self.x = self.x - self.step_legnth * self.dir
# reset problem training samples
self.problem.training_samples_stack = None
def refine_target_box(self,
sample_pos,
sample_scale,
scale_ind,
update_scale=True):
# Initial box for refinement
init_box = self.get_iounet_box(self.pos, self.target_sz, sample_pos,
sample_scale)
# Extract features from the relevant scale
iou_features = self.get_iou_features()
iou_features = TensorList(
[x[scale_ind:scale_ind + 1, ...] for x in iou_features])
init_boxes = np.reshape(init_box, (1, 4)).copy()
rand_fn = lambda a, b: np.random.rand(a, b).astype('float32')
if self.params.num_init_random_boxes > 0:
# Get random initial boxes
square_box_sz = np.sqrt(init_box[2:].prod())
rand_factor = square_box_sz * np.concatenate([
self.params.box_jitter_pos * np.ones(2),
self.params.box_jitter_sz * np.ones(2)
])
minimal_edge_size = init_box[2:].min() / 3
rand_bb = (rand_fn(self.params.num_init_random_boxes, 4) -
0.5) * rand_factor
new_sz = np.clip(init_box[2:] + rand_bb[:, 2:], minimal_edge_size,
1e10)
new_center = (init_box[:2] + init_box[2:] / 2) + rand_bb[:, :2]
init_boxes = np.concatenate([new_center - new_sz / 2, new_sz], 1)
init_boxes = np.concatenate(
[np.reshape(init_box, (1, 4)), init_boxes])
# Refine boxes by maximizing iou
output_boxes, output_iou = self.optimize_boxes(iou_features,
init_boxes)
# Remove weird boxes with extreme aspect ratios
output_boxes[:, 2:] = np.clip(output_boxes[:, 2:], 1, 1e10)
aspect_ratio = output_boxes[:, 2] / output_boxes[:, 3]
keep_ind = (aspect_ratio < self.params.maximal_aspect_ratio) * \
(aspect_ratio > 1 / self.params.maximal_aspect_ratio)
output_boxes = output_boxes[keep_ind, :]
output_iou = output_iou[keep_ind]
# If no box found
if output_boxes.shape[0] == 0:
return
# Take average of top k boxes
k = getattr(self.params, 'iounet_k', 5)
topk = min(k, output_boxes.shape[0])
inds = np.argsort(-output_iou)[:topk]
predicted_box = np.mean(output_boxes[inds, :], axis=0)
predicted_iou = np.mean(np.reshape(output_iou, (-1, 1))[inds, :],
axis=0)
# Update position
new_pos = predicted_box[:2] + predicted_box[2:] / 2 - (
self.iou_img_sample_sz - 1) / 2
new_pos = np.flip(new_pos, 0) * sample_scale + sample_pos
new_target_sz = np.flip(predicted_box[2:], 0) * sample_scale
new_scale = np.sqrt(
np.prod(new_target_sz) / np.prod(self.base_target_sz))
self.pos_iounet = new_pos.copy()
if getattr(self.params, 'use_iounet_pos_for_learning', True):
self.pos = new_pos.copy()
self.target_sz = new_target_sz
if update_scale:
self.target_scale = new_scale
def init_optimization(self, train_x, init_y):
# Initialize filter
filter_init_method = getattr(self.params, 'filter_init_method',
'zeros')
self.filter = TensorList([
np.zeros([1, cdim, sz[0], sz[1]], 'float32') for x, cdim, sz in
zip(train_x, self.compressed_dim, self.kernel_size)
])
if filter_init_method == 'zeros':
pass
elif filter_init_method == 'ones':
for idx, f in enumerate(self.filter):
self.filter[idx] = np.ones(f.shape, 'float32') / np.prod(
f.shape)
elif filter_init_method == 'np_randn':
rng = np.random.RandomState(0)
for idx, f in enumerate(self.filter):
self.filter[idx] = rng.normal(
size=f.shape, loc=0,
scale=1 / np.prod(f.shape)).astype('float32')
elif filter_init_method == 'randn':
for idx, f in enumerate(self.filter):
with fluid.dygraph.guard():
self.filter[idx] = layers.gaussian_random(
f.shape, std=1 / np.prod(f.shape)).numpy()
else:
raise ValueError('Unknown "filter_init_method"')
# Get parameters
self.params.update_projection_matrix = getattr(
self.params, 'update_projection_matrix',
True) and self.params.use_projection_matrix
optimizer = getattr(self.params, 'optimizer', 'GaussNewtonCG')
# Setup factorized joint optimization
if self.params.update_projection_matrix:
self.joint_problem = FactorizedConvProblem(
self.init_training_samples, init_y, self.filter_reg,
self.fparams.attribute('projection_reg'), self.params,
self.init_sample_weights, self.projection_activation,
self.response_activation)
# Variable containing both filter and projection matrix
joint_var = self.filter.concat(self.projection_matrix)
# Initialize optimizer
analyze_convergence = getattr(self.params, 'analyze_convergence',
False)
if optimizer == 'GaussNewtonCG':
self.joint_optimizer = GaussNewtonCG(
self.joint_problem,
joint_var,
plotting=(self.params.debug >= 3),
analyze=True,
fig_num=(12, 13, 14))
elif optimizer == 'GradientDescentL2':
self.joint_optimizer = GradientDescentL2(
self.joint_problem,
joint_var,
self.params.optimizer_step_length,
self.params.optimizer_momentum,
plotting=(self.params.debug >= 3),
debug=analyze_convergence,
fig_num=(12, 13))
# Do joint optimization
if isinstance(self.params.init_CG_iter, (list, tuple)):
self.joint_optimizer.run(self.params.init_CG_iter)
else:
self.joint_optimizer.run(
self.params.init_CG_iter // self.params.init_GN_iter,
self.params.init_GN_iter)
# Get back filter and optimizer
len_x = len(self.joint_optimizer.x)
self.filter = self.joint_optimizer.x[:len_x // 2] # w2 in paper
self.projection_matrix = self.joint_optimizer.x[len_x //
2:] # w1 in paper
if analyze_convergence:
opt_name = 'CG' if getattr(self.params, 'CG_optimizer',
True) else 'GD'
for val_name, values in zip(['loss', 'gradient'], [
self.joint_optimizer.losses,
self.joint_optimizer.gradient_mags
]):
val_str = ' '.join(
['{:.8e}'.format(v.item()) for v in values])
file_name = '{}_{}.txt'.format(opt_name, val_name)
with open(file_name, 'a') as f:
f.write(val_str + '\n')
raise RuntimeError('Exiting')
# Re-project samples with the new projection matrix
compressed_samples = self.project_sample(self.init_training_samples,
self.projection_matrix)
for train_samp, init_samp in zip(self.training_samples,
compressed_samples):
for idx in range(init_samp.shape[0]):
train_samp[idx] = init_samp[idx]
self.hinge_mask = None
# Initialize optimizer
self.conv_problem = ConvProblem(self.training_samples, self.y,
self.filter_reg, self.sample_weights,
self.response_activation)
if optimizer == 'GaussNewtonCG':
self.filter_optimizer = ConjugateGradient(
self.conv_problem,
self.filter,
fletcher_reeves=self.params.fletcher_reeves,
direction_forget_factor=self.params.direction_forget_factor,
debug=(self.params.debug >= 3),
fig_num=(12, 13))
elif optimizer == 'GradientDescentL2':
self.filter_optimizer = GradientDescentL2(
self.conv_problem,
self.filter,
self.params.optimizer_step_length,
self.params.optimizer_momentum,
debug=(self.params.debug >= 3),
fig_num=12)
# Transfer losses from previous optimization
if self.params.update_projection_matrix:
self.filter_optimizer.residuals = self.joint_optimizer.residuals
self.filter_optimizer.losses = self.joint_optimizer.losses
if not self.params.update_projection_matrix:
self.filter_optimizer.run(self.params.init_CG_iter)
# Post optimization
self.filter_optimizer.run(self.params.post_init_CG_iter)
self.filter = self.filter_optimizer.x
# Free memory
del self.init_training_samples
if self.params.use_projection_matrix:
del self.joint_problem, self.joint_optimizer
def local_track(self, image):
state, score_map, test_x, scale_ind, sample_pos, sample_scales, flag, s = self.local_Tracker.track_updater(
image)
# Check flags and set learning rate if hard negative
score_map = cv2.resize(np.squeeze(score_map), (19, 19))
update_flag = flag not in ['not_found', 'uncertain']
update = update_flag
max_score = max(score_map.flatten())
self.all_map.append(score_map)
local_state = np.array(state).reshape((1, 4))
ap_dis = self.metric_eval(image, local_state, self.anchor_feature)
self.dis_record.append(ap_dis.data.cpu().numpy()[0])
h = image.shape[0]
w = image.shape[1]
self.state_record.append([
local_state[0][0] / w, local_state[0][1] / h,
(local_state[0][0] + local_state[0][2]) / w,
(local_state[0][1] + local_state[0][3]) / h
])
self.rv_record.append(max_score)
if len(self.state_record) >= tcopts['time_steps']:
dis = np.array(self.dis_record[-tcopts["time_steps"]:]).reshape(
(tcopts["time_steps"], 1))
rv = np.array(self.rv_record[-tcopts["time_steps"]:]).reshape(
(tcopts["time_steps"], 1))
state_tc = np.array(self.state_record[-tcopts["time_steps"]:])
map_input = np.array(self.all_map[-tcopts["time_steps"]:])
map_input = np.reshape(map_input,
[tcopts['time_steps'], 1, 19, 19])
map_input = map_input.transpose((0, 2, 3, 1))
X_input = np.concatenate((state_tc, rv, dis), axis=1)
logits = self.sess.run(self.logits,
feed_dict={
self.X_input:
np.expand_dims(X_input, axis=0),
self.maps:
map_input
})
update = logits[0][0] < logits[0][1]
print(logits[0])
hard_negative = (flag == 'hard_negative')
learning_rate = self.local_Tracker.params.hard_negative_learning_rate if hard_negative else None
if update:
# [AL] Modification
# if update_flag:
# Get train sample
train_x = TensorList(
[x[scale_ind:scale_ind + 1, ...] for x in test_x])
# Create label for sample
train_y = self.local_Tracker.get_label_function(
sample_pos, sample_scales[scale_ind])
# Update memory
self.local_Tracker.update_memory(train_x, train_y, learning_rate)
# Train filter
if hard_negative:
self.local_Tracker.filter_optimizer.run(
self.local_Tracker.params.hard_negative_CG_iter)
elif (self.local_Tracker.frame_num -
1) % self.local_Tracker.params.train_skipping == 0:
self.local_Tracker.filter_optimizer.run(
self.local_Tracker.params.CG_iter)
self.last_gt = [
state[1], state[0], state[1] + state[3], state[0] + state[2]
]
return state, score_map, update, max_score, ap_dis.data.cpu().numpy(
)[0]
def _construct_graph(self):
train_program = fluid.Program()
start_program = fluid.Program()
with fluid.program_guard(train_program, start_program):
scope = 'first/'
self.x_ph = TensorList([
fluid.layers.data('{}x_{}'.format(scope, idx),
v.shape,
append_batch_size=False,
stop_gradient=False)
for idx, v in enumerate(self.x)
])
self.p_ph = TensorList([
fluid.layers.data('{}p_{}'.format(scope, idx),
v.shape,
append_batch_size=False,
stop_gradient=False)
for idx, v in enumerate(self.x)
])
# problem forward
self.f0 = self.problem(self.x_ph, scope)
self.g = self.f0.apply(static_clone)
# Get df/dx^t @ f0
self.dfdxt_g = TensorList(
fluid.gradients(self.f0, self.x_ph, self.g))
# For computing A
tmp = [a * b for a, b in zip(self.dfdxt_g, self.p_ph)]
self.dfdx_x = TensorList(fluid.gradients(tmp, self.g))
# self.dfdx_x = TensorList(fluid.gradients(self.dfdxt_g, self.g, self.p_ph))
train_program2 = fluid.Program()
start_program2 = fluid.Program()
with fluid.program_guard(train_program2, start_program2):
scope = 'second/'
self.x_ph_2 = TensorList([
fluid.layers.data('{}x_{}'.format(scope, idx),
v.shape,
append_batch_size=False,
stop_gradient=False)
for idx, v in enumerate(self.x)
])
self.dfdx_x_ph = TensorList([
fluid.layers.data('{}dfdx_x_{}'.format(scope, idx),
v.shape,
append_batch_size=False,
stop_gradient=False)
for idx, v in enumerate(self.g)
])
self.f0_2 = self.problem(self.x_ph_2, scope)
self.dfdx_dfdx = TensorList(
fluid.gradients(self.f0_2 * self.dfdx_x_ph, self.x_ph_2))
place = fluid.CUDAPlace(0)
self.exe = fluid.Executor(place)
self.exe.run(program=fluid.default_startup_program())
self.compiled_prog = fluid.compiler.CompiledProgram(train_program)
place2 = fluid.CUDAPlace(0)
self.exe2 = fluid.Executor(place2)
self.exe2.run(program=fluid.default_startup_program())
self.compiled_prog2 = fluid.compiler.CompiledProgram(train_program2)
def track(self, image):
self.frame_num += 1
# Convert image
# im = numpy_to_paddle(image)
im = image.astype('float32')
self.im = im # For debugging only
# ------- LOCALIZATION ------- #
# Get sample
sample_pos = self.pos.round()
sample_scales = self.target_scale * self.params.scale_factors
test_x = self.extract_processed_sample(im, self.pos, sample_scales,
self.img_sample_sz)
# Compute scores
scores_raw = self.apply_filter(test_x)
translation_vec, scale_ind, s, flag = self.localize_target(scores_raw)
# Update position and scale
if flag != 'not_found':
if self.use_iou_net:
update_scale_flag = getattr(self.params,
'update_scale_when_uncertain',
True) or flag != 'uncertain'
if getattr(self.params, 'use_classifier', True):
self.update_state(sample_pos + translation_vec)
self.refine_target_box(sample_pos, sample_scales[scale_ind],
scale_ind, update_scale_flag)
elif getattr(self.params, 'use_classifier', True):
self.update_state(sample_pos + translation_vec,
sample_scales[scale_ind])
# ------- UPDATE ------- #
# Check flags and set learning rate if hard negative
update_flag = flag not in ['not_found', 'uncertain']
hard_negative = (flag == 'hard_negative')
learning_rate = self.params.hard_negative_learning_rate if hard_negative else None
if update_flag:
# Get train sample
train_x = TensorList([x[scale_ind:scale_ind + 1] for x in test_x])
# Create label for sample
train_y = self.get_label_function(sample_pos,
sample_scales[scale_ind])
# Update memory
self.update_memory(train_x, train_y, learning_rate)
# Train filter
if hard_negative:
self.filter_optimizer.run(self.params.hard_negative_CG_iter)
elif (self.frame_num - 1) % self.params.train_skipping == 0:
self.filter_optimizer.run(self.params.CG_iter)
self.filter = self.filter_optimizer.x
# Set the pos of the tracker to iounet pos
if self.use_iou_net and flag != 'not_found':
self.pos = self.pos_iounet.copy()
# Return new state
yx = self.pos - (self.target_sz - 1) / 2
new_state = np.array(
[yx[1], yx[0], self.target_sz[1], self.target_sz[0]], 'float32')
return new_state.tolist()