def build_examplar_ini(self):
model_config = self.model_config
track_config = self.track_config
# Exemplar image lies at the center of the search image in the first frame
exemplar_images = get_exemplar_images(
self.search_images,
[model_config['z_image_size'], model_config['z_image_size']])
[templates_ini, templates] = self.get_image_embedding(exemplar_images)
center_scale = int(get_center(track_config['num_scales']))
center_template = tf.identity(templates[center_scale])
templates = tf.stack(
[center_template for _ in range(track_config['num_scales'])])
center_template_ini = tf.identity(templates_ini[center_scale])
templates_ini = tf.stack(
[center_template_ini for _ in range(track_config['num_scales'])])
with tf.variable_scope('target_examplar'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state = tf.get_variable(
'exemplar',
initializer=tf.zeros(templates_ini.get_shape().as_list(),
dtype=templates_ini.dtype),
trainable=False)
with tf.control_dependencies([templates_ini]):
self.init_examplar = tf.assign(state,
templates_ini,
validate_shape=True)
self.examplar_ini = state
def build_examplar(self):
model_config = self.model_config
track_config = self.track_config
# Exemplar image lies at the center of the search image in the first frame
exemplar_images = get_exemplar_images(
self.search_images,
[model_config['z_image_size'], model_config['z_image_size']])
[_, templates] = self.get_image_embedding(exemplar_images,
reuse=tf.AUTO_REUSE)
center_scale = int(get_center(track_config['num_scales']))
self.examplar = tf.identity(templates[center_scale])
self.examplar = tf.nn.relu(self.examplar)
def build_template(self):
model_config = self.model_config
track_config = self.track_config
# Exemplar image lies at the center of the search image in the first frame
exemplar_images = get_exemplar_images(self.search_images, [model_config['z_image_size'],
model_config['z_image_size']])
templates = self.get_image_embedding(exemplar_images)
center_scale = int(get_center(track_config['num_scales']))
center_template = tf.identity(templates[center_scale])
templates = tf.stack([center_template for _ in range(track_config['num_scales'])])
with tf.variable_scope('target_template'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state = tf.get_variable('exemplar',
initializer=tf.zeros(templates.get_shape().as_list(), dtype=templates.dtype),
trainable=False)
with tf.control_dependencies([templates]):
self.init = tf.assign(state, templates, validate_shape=True)
self.templates = state
def build_template(self):
# Exemplar image lies at the center of the search image in the first frame
exemplar_images = get_exemplar_images(self.search_images, [127, 127])
templates = self.get_image_embedding(exemplar_images, self.classid)
center_scale = int(get_center(3))
center_template = tf.identity(templates[center_scale])
templates = tf.stack([center_template for _ in range(3)])
with tf.variable_scope('target_template'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state = tf.get_variable('exemplar',
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False)
with tf.control_dependencies([templates]):
self.init = tf.assign(state,
templates,
validate_shape=True)
self.templates = state
def build_template(self):
model_config = self.model_config
track_config = self.track_config
# Exemplar image lies at the center of the search image in the first frame
exemplar_images = get_exemplar_images(
self.search_images,
[model_config['z_image_size'], model_config['z_image_size']])
templates_s_c5, templates_s_c4, templates_s_c3, templates_a_c5, templates_a_c4, templates_a_c3 = self.get_image_embedding(
exemplar_images)
# =============================================================================
# templates_s_c5, templates_s_c4, templates_s_c3 = self.get_image_embedding(exemplar_images)
# =============================================================================
# =============================================================================
# templates_a_c5, templates_a_c4, templates_a_c3 = self.get_image_embedding(exemplar_images)
# =============================================================================
center_scale = int(get_center(track_config['num_scales']))
center_template_s_c5 = tf.identity(templates_s_c5[center_scale])
center_template_s_c4 = tf.identity(templates_s_c4[center_scale])
center_template_s_c3 = tf.identity(templates_s_c3[center_scale])
templates_s_c5 = tf.stack(
[center_template_s_c5 for _ in range(track_config['num_scales'])])
templates_s_c4 = tf.stack(
[center_template_s_c4 for _ in range(track_config['num_scales'])])
templates_s_c3 = tf.stack(
[center_template_s_c3 for _ in range(track_config['num_scales'])])
center_template_a_c5 = tf.identity(templates_a_c5[center_scale])
center_template_a_c4 = tf.identity(templates_a_c4[center_scale])
center_template_a_c3 = tf.identity(templates_a_c3[center_scale])
templates_a_c5 = tf.stack(
[center_template_a_c5 for _ in range(track_config['num_scales'])])
templates_a_c4 = tf.stack(
[center_template_a_c4 for _ in range(track_config['num_scales'])])
templates_a_c3 = tf.stack(
[center_template_a_c3 for _ in range(track_config['num_scales'])])
with tf.variable_scope('target_template'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state_s_c5 = tf.get_variable(
'exemplar_s_c5',
initializer=tf.zeros(templates_s_c5.get_shape().as_list(),
dtype=templates_s_c5.dtype),
trainable=False)
state_s_c4 = tf.get_variable(
'exemplar_s_c4',
initializer=tf.zeros(templates_s_c4.get_shape().as_list(),
dtype=templates_s_c4.dtype),
trainable=False)
state_s_c3 = tf.get_variable(
'exemplar_s_c3',
initializer=tf.zeros(templates_s_c3.get_shape().as_list(),
dtype=templates_s_c3.dtype),
trainable=False)
state_a_c5 = tf.get_variable(
'exemplar_a_c5',
initializer=tf.zeros(templates_a_c5.get_shape().as_list(),
dtype=templates_a_c5.dtype),
trainable=False)
state_a_c4 = tf.get_variable(
'exemplar_a_c4',
initializer=tf.zeros(templates_a_c4.get_shape().as_list(),
dtype=templates_a_c4.dtype),
trainable=False)
state_a_c3 = tf.get_variable(
'exemplar_a_c3',
initializer=tf.zeros(templates_a_c3.get_shape().as_list(),
dtype=templates_a_c3.dtype),
trainable=False)
with tf.control_dependencies([templates_s_c5]):
self.init_s_c5 = tf.assign(state_s_c5,
templates_s_c5,
validate_shape=True)
with tf.control_dependencies([templates_s_c4]):
self.init_s_c4 = tf.assign(state_s_c4,
templates_s_c4,
validate_shape=True)
with tf.control_dependencies([templates_s_c3]):
self.init_s_c3 = tf.assign(state_s_c3,
templates_s_c3,
validate_shape=True)
with tf.control_dependencies([templates_a_c5]):
self.init_a_c5 = tf.assign(state_a_c5,
templates_a_c5,
validate_shape=True)
with tf.control_dependencies([templates_a_c4]):
self.init_a_c4 = tf.assign(state_a_c4,
templates_a_c4,
validate_shape=True)
with tf.control_dependencies([templates_a_c3]):
self.init_a_c3 = tf.assign(state_a_c3,
templates_a_c3,
validate_shape=True)
self.templates_s_c5 = state_s_c5
self.templates_s_c4 = state_s_c4
self.templates_s_c3 = state_s_c3
self.templates_a_c5 = state_a_c5
self.templates_a_c4 = state_a_c4
self.templates_a_c3 = state_a_c3
def build_template(self):
model_config = self.model_config
track_config = self.track_config
size_z = model_config['z_image_size']
ratio = self.target_bbox_feed[2] / self.target_bbox_feed[3]
# Exemplar image lies at the center of the search image in the first frame
search_images = self.search_images
shape = search_images.get_shape().as_list()
exemplar_images = get_exemplar_images(
search_images, [size_z, size_z],
tf.tile([[
get_center(track_config['x_image_size']),
get_center(track_config['x_image_size'])
]], [shape[0], 1]))
center_scale = int(get_center(track_config['num_scales']))
exemplar_images = tf.expand_dims(exemplar_images[center_scale], 0)
def background_suppression(embeds, embeds2, ratio):
offsets = tf.cond(
tf.greater(ratio, 1.5), lambda: [0, 2, 0],
lambda: tf.cond(tf.less(ratio, 0.66), lambda: [2, 0, 0],
lambda: [1, 1, 0]))
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, t_shape[1] - offsets[0], t_shape[2] - offsets[1])
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, t_shape[1], t_shape[2])
embeds2 = tf.image.resize_image_with_crop_or_pad(
embeds2, t_shape2[1] - offsets[0] * 2,
t_shape2[2] - offsets[1] * 2)
embeds2 = tf.image.resize_image_with_crop_or_pad(
embeds2, t_shape2[1], t_shape2[2])
return embeds, embeds2
self.exemplar_images = exemplar_images
templates, templates2 = self.get_image_embedding(exemplar_images)
t_shape = templates.get_shape().as_list()
t_shape2 = templates2.get_shape().as_list()
templates, templates2 = tf.map_fn(
lambda x: background_suppression(x[0], x[1], x[2]),
(templates, templates2, tf.expand_dims(ratio, 0)),
dtype=(templates.dtype, templates2.dtype))
with tf.variable_scope('target_template'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state = tf.get_variable('exemplar',
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False)
state2 = tf.get_variable('exemplar2',
initializer=tf.zeros(
templates2.get_shape().as_list(),
dtype=templates2.dtype),
trainable=False)
with tf.control_dependencies([templates, templates2]):
self.init = tf.assign(state,
templates,
validate_shape=True)
self.init2 = tf.assign(state2,
templates2,
validate_shape=True)
self.templates = state
self.templates2 = state2
# Store Pseudo Templates
def _euc_distance(x, z):
z = tf.expand_dims(z, 0)
return tf.reduce_sum(tf.abs(x - z), -1)
n_mem = 5
temp1 = tf.concat([templates for _ in range(n_mem)], 0)
temp2 = tf.concat([templates2 for _ in range(n_mem)], 0)
temp3 = tf.concat([exemplar_images for _ in range(n_mem)], 0)
state_mem1 = tf.get_variable('exemplar_mem',
initializer=tf.zeros(
temp1.get_shape().as_list(),
dtype=temp1.dtype),
trainable=False)
state_mem2 = tf.get_variable('exemplar2_mem',
initializer=tf.zeros(
temp2.get_shape().as_list(),
dtype=temp2.dtype),
trainable=False)
image_mem = tf.get_variable('exemplar_image_mem',
initializer=tf.zeros(
temp3.get_shape().as_list(),
dtype=temp3.dtype),
trainable=False)
with tf.control_dependencies([temp1, temp2, temp3]):
self.init_mem = tf.assign(state_mem1,
temp1,
validate_shape=True)
self.init_mem2 = tf.assign(state_mem2,
temp2,
validate_shape=True)
self.init_img_mem = tf.assign(image_mem,
temp3,
validate_shape=True)
up_mem = tf.scatter_update(state_mem1, self.mem_id_feed,
templates[0])
up_mem2 = tf.scatter_update(state_mem2, self.mem_id_feed,
templates2[0])
up_img_mem = tf.scatter_update(image_mem, self.mem_id_feed,
exemplar_images[0])
with tf.control_dependencies([up_mem, up_mem2, up_img_mem]):
self.up_mem = up_mem
self.up_mem2 = up_mem2
self.up_img_mem = up_img_mem
state_pseu = []
state_pseu2 = []
image_pseu = []
self.init_pseu = []
self.init2_pseu = []
self.init_pseu_img = []
for i in range(3):
state_pseu.append(
tf.get_variable('exemplar_pseu' + str(i),
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False))
state_pseu2.append(
tf.get_variable('exemplar2_pseu' + str(i),
initializer=tf.zeros(
templates2.get_shape().as_list(),
dtype=templates2.dtype),
trainable=False))
image_pseu.append(
tf.get_variable(
'exemplar_pseu_image' + str(i),
initializer=tf.zeros(
exemplar_images.get_shape().as_list(),
dtype=exemplar_images.dtype),
trainable=False))
with tf.control_dependencies(
[templates, templates2, exemplar_images]):
self.init_pseu.append(
tf.assign(state_pseu[i],
templates,
validate_shape=True))
self.init2_pseu.append(
tf.assign(state_pseu2[i],
templates2,
validate_shape=True))
self.init_pseu_img.append(
tf.assign(image_pseu[i],
exemplar_images,
validate_shape=True))
self.image_pseu = image_pseu
self.pseu_temp = state_pseu
self.pseu_temp2 = state_pseu2
state_pseus = tf.concat([self.templates] + state_pseu +
[state_mem1], 0)
sp_shape = state_pseus.get_shape().as_list()[0]
state_pseus_c = tf.reshape(state_pseus, [sp_shape, -1])
state_pseus_dis = tf.map_fn(
lambda x: _euc_distance(state_pseus_c, x),
state_pseus_c,
dtype=state_pseus_c.dtype)
state_pseus_dis = tf.reshape(state_pseus_dis,
[sp_shape, sp_shape])[1:, :]
state_pseus_dis = tf.reduce_sum(state_pseus_dis, -1)
self.state_pseus_dis = state_pseus_dis
_, state_pseus_idx = tf.nn.top_k(state_pseus_dis,
k=len(state_pseu))
image_pseu_extra = tf.concat(image_pseu + [image_mem], 0)
state_pseus2 = tf.concat(state_pseu2 + [state_mem2], 0)
self.up_img = []
self.up_pseu = []
self.up2_pseu = []
for i in range(len(state_pseu)):
with tf.control_dependencies([
state_pseus_idx, image_pseu_extra, state_pseus,
state_pseus2
]):
self.up_pseu.append(
tf.assign(state_pseu[i],
tf.expand_dims(
state_pseus[state_pseus_idx[i] + 1],
0),
validate_shape=True))
self.up2_pseu.append(
tf.assign(state_pseu2[i],
tf.expand_dims(
state_pseus2[state_pseus_idx[i]], 0),
validate_shape=True))
self.up_img.append(
tf.assign(image_pseu[i],
tf.expand_dims(
image_pseu_extra[state_pseus_idx[i]],
0),
validate_shape=True))
def build_template(self):
model_config = self.model_config
track_config = self.track_config
size_z = model_config['z_image_size']
ratio = self.target_size[0] / self.target_size[1]
# Exemplar image lies at the center of the search image in the first frame
center_scale = int(get_center(track_config['num_scales']))
search_images = tf.expand_dims(self.search_images[center_scale], 0)
exemplar_images = get_exemplar_images(
search_images, [size_z, size_z],
np.array([[
get_center(track_config['x_image_size']),
get_center(track_config['x_image_size'])
]]))
def boundary_suppression(embeds, embeds2, ratio):
offsets = tf.cond(
tf.greater(ratio, 1.5), lambda: [0, 4, 0, 4],
lambda: tf.cond(tf.less(ratio, 0.67), lambda: [4, 0, 4, 0],
lambda: [2, 2, 2, 2]))
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, t_shape[1] - offsets[0], t_shape[2] - offsets[1])
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, t_shape[1], t_shape[2])
embeds2 = tf.image.resize_image_with_crop_or_pad(
embeds2, t_shape2[1] - offsets[2], t_shape2[2] - offsets[3])
embeds2 = tf.image.resize_image_with_crop_or_pad(
embeds2, t_shape2[1], t_shape2[2])
return embeds, embeds2
def background_suppression(embeds, ratio):
offsets = tf.cond(
tf.greater(ratio, 1.5), # 1.2 / 0.83; 1.5 / 0.67
lambda: [1., 1.2 / ratio],
lambda: tf.cond(
tf.less(ratio, 0.67), lambda: [1.2 * ratio, 1.], lambda: tf
.cond(
tf.greater(ratio, 1.2), lambda: [1., 1.1 / ratio],
lambda: tf.cond(tf.less(ratio, 0.83), lambda: [
1.1 * ratio, 1.
], lambda: [0.7, 0.7]))))
h = tf.cast(size_z * offsets[0], tf.int32)
w = tf.cast(size_z * offsets[1], tf.int32)
embeds_mean = tf.reduce_mean(embeds, axis=(0, 1), keepdims=True)
embeds = embeds - embeds_mean
embeds = tf.image.resize_image_with_crop_or_pad(embeds, h, w)
embeds = tf.image.resize_image_with_crop_or_pad(
embeds, size_z, size_z)
return embeds + embeds_mean
exemplar_images = tf.map_fn(
lambda x: background_suppression(x[0], x[1]),
(exemplar_images, tf.expand_dims(ratio, 0)),
dtype=exemplar_images.dtype)
self.exemplar_images = exemplar_images
templates, templates2 = self.get_image_embedding(exemplar_images)
t_shape = templates.get_shape().as_list()
t_shape2 = templates2.get_shape().as_list()
templates, templates2 = tf.map_fn(
lambda x: boundary_suppression(x[0], x[1], x[2]),
(templates, templates2, tf.expand_dims(ratio, 0)),
dtype=(templates.dtype, templates2.dtype))
templates = templates
templates2 = templates2
with tf.variable_scope('target_template'):
# Store template in Variable such that we don't have to feed this template every time.
with tf.variable_scope('State'):
state = tf.get_variable('exemplar',
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False)
state2 = tf.get_variable('exemplar2',
initializer=tf.zeros(
templates2.get_shape().as_list(),
dtype=templates2.dtype),
trainable=False)
with tf.control_dependencies([templates, templates2]):
self.init = tf.assign(state,
templates,
validate_shape=True)
self.init2 = tf.assign(state2,
templates2,
validate_shape=True)
self.templates = state
self.templates2 = state2
# Store Pseudo Templates
def _euc_distance(x, z):
z = tf.expand_dims(z, 0)
return tf.reduce_sum(tf.abs(x - z), -1)
num_k = 3 # 3
state_pseu = []
state_pseu2 = []
image_pseu = []
self.init_pseu = []
self.init2_pseu = []
self.init_pseu_img = []
for i in range(num_k):
state_pseu.append(
tf.get_variable('exemplar_pseu' + str(i),
initializer=tf.zeros(
templates.get_shape().as_list(),
dtype=templates.dtype),
trainable=False))
state_pseu2.append(
tf.get_variable('exemplar2_pseu' + str(i),
initializer=tf.zeros(
templates2.get_shape().as_list(),
dtype=templates2.dtype),
trainable=False))
image_pseu.append(
tf.get_variable(
'exemplar_pseu_image' + str(i),
initializer=tf.zeros(
exemplar_images.get_shape().as_list(),
dtype=exemplar_images.dtype),
trainable=False))
with tf.control_dependencies(
[templates, templates2, exemplar_images]):
self.init_pseu.append(
tf.assign(state_pseu[i],
templates,
validate_shape=True))
self.init2_pseu.append(
tf.assign(state_pseu2[i],
templates2,
validate_shape=True))
self.init_pseu_img.append(
tf.assign(image_pseu[i],
exemplar_images,
validate_shape=True))
self.image_pseu = image_pseu
self.pseu_temp = state_pseu
self.pseu_temp2 = state_pseu2
state_pseus = tf.concat([self.templates] + state_pseu +
[templates], 0)
sp_shape = state_pseus.get_shape().as_list()[0]
state_pseus_c = tf.reshape(state_pseus, [sp_shape, -1])
state_pseus_dis = tf.map_fn(
lambda x: _euc_distance(state_pseus_c, x),
state_pseus_c,
dtype=state_pseus_c.dtype)
state_pseus_dis = tf.reshape(state_pseus_dis,
[sp_shape, sp_shape])[1:, :]
state_pseus_dis = tf.reduce_sum(state_pseus_dis, -1)
self.state_pseus_dis = state_pseus_dis
_, state_pseus_idx = tf.nn.top_k(state_pseus_dis,
k=len(state_pseu))
image_pseu_extra = tf.concat(image_pseu + [exemplar_images], 0)
state_pseus2 = tf.concat(state_pseu2 + [templates2], 0)
self.up_img = []
self.up_pseu = []
self.up2_pseu = []
for i in range(len(state_pseu)):
with tf.control_dependencies([
state_pseus_idx, image_pseu_extra, state_pseus,
state_pseus2
]):
self.up_pseu.append(
tf.assign(state_pseu[i],
tf.expand_dims(
state_pseus[state_pseus_idx[i] + 1],
0),
validate_shape=True))
self.up2_pseu.append(
tf.assign(state_pseu2[i],
tf.expand_dims(
state_pseus2[state_pseus_idx[i]], 0),
validate_shape=True))
self.up_img.append(
tf.assign(image_pseu[i],
tf.expand_dims(
image_pseu_extra[state_pseus_idx[i]],
0),
validate_shape=True))