def single_investigation(lang, distrib, h_layers, h_units, window, sample_size,
n_epochs, exp_num):
acc_per_d = []
loss_per_d = []
generator = SampleGenerator(lang)
## If you would like to fix your training set during the entire course of investigation,
## you should uncomment the following line (and comment the same line in the subsequent "for" loop);
## otherwise, each training set will come from the same distribution and same window but be different.
inputs, outputs, s_dst = generator.generate_sample(sample_size, window[0],
window[1], distrib,
False)
for _ in range(exp_num):
# inputs, outputs, s_dst = generator.generate_sample (sample_size, window[0], window[1], distrib, False)
e_vals, loss_vals = train(generator, distrib, h_layers, h_units,
inputs, outputs, n_epochs,
1) # each experiment is unique
acc_per_d.append(e_vals)
loss_per_d.append(loss_vals)
filename = '{}_{}_{}_{}_{}_{}_{}'.format(lang, 'single', distrib, h_layers,
h_units, window[0], window[1])
## Uncomment the following line if you would like to save the e_i and loss values.
# np.savez('./results/result_{}.npz'.format(filename), errors = np.array(e_vals), losses = np.array (loss_vals))
trials_label = [
'Experiment {}'.format(elt) for elt in range(1, exp_num + 1)
]
plot_graphs(lang, 'trials', trials_label, acc_per_d, loss_per_d, window,
filename)
return acc_per_d, loss_vals
def approx_rbfn_iterative(self):
residuals = 0
t = 0
for i in range(100):
max_iter = 500
model = RBFN(nb_features=15)
start = time.process_time()
# Generate a batch of data and store it
self.reset_batch()
g = SampleGenerator()
for i in range(max_iter):
# Draw a random sample on the interval [0,1]
x = np.random.random()
y = g.generate_non_linear_samples(x)
self.x_data.append(x)
self.y_data.append(y)
# Comment the ones you don't want to use
# model.train_gd(x, y, alpha=0.5)
# model.train_rls(x, y)
model.train_rls_sherman_morrison(x, y)
# if(i == 500):
# model.plot(self.x_data, self.y_data)
print("RBFN Incr time:", time.process_time() - start)
t += time.process_time() - start
residuals += model.residuals(self.x_data, self.y_data)
print(residuals / 100, t / 100)
model.plot(self.x_data, self.y_data)
def hidden_units_investigation(lang, distrib, h_layers, h_units, window,
sample_size, n_epochs, exp_num):
acc_per_d = []
loss_per_d = []
generator = SampleGenerator(lang)
## If you would like to fix your training set during the entire course of investigation,
## you should uncomment the following line (and comment the same line in the subsequent "for" loop);
## otherwise, each training set will come from the same distribution and same window but be different.
inputs, outputs, s_dst = generator.generate_sample(sample_size, window[0],
window[1], distrib,
False)
for hidden_dim in h_units:
# inputs, outputs, s_dst = generator.generate_sample (sample_size, window[0], window[1], distrib, False)
e_vals, loss_vals = train(generator, distrib, h_layers, hidden_dim,
inputs, outputs, n_epochs, exp_num)
acc_per_d.append(e_vals)
loss_per_d.append(loss_vals)
filename = '{}_{}_{}_{}_{}_{}'.format(lang, 'hidden', distrib, h_layers,
window[0], window[1])
hunits_label = ['{} Hidden Units'.format(val) for val in h_units]
plot_graphs(lang, 'hiddenunits', hunits_label, acc_per_d, loss_per_d,
window, filename)
return acc_per_d, loss_vals
def approx_rbfn_iterative_TestNB(self):
idx = []
res = np.zeros((10))
for x in range(200):
for i in range(10):
max_iter = 100
model = RBFN(15)
start = time.process_time()
# Generate a batch of data and store it
self.reset_batch()
g = SampleGenerator()
for j in range(max_iter):
# Draw a random sample on the interval [0,1]
x = np.random.random()
y = g.generate_non_linear_samples(x)
self.x_data.append(x)
self.y_data.append(y)
# Comment the ones you don't want to use
model.train_gd(x, y, alpha=i * 0.02 + 1)
#model.train_rls(x, y)
# model.train_rls_sherman_morrison(x, y)
res[i] += model.residuals(self.x_data, self.y_data)
for i in range(10):
idx.append(i * 0.02 + 1)
res[i] /= 200
plt.plot(idx, res, color="red")
plt.show()
print("RBFN Incr time:", time.process_time() - start)
def approx_rbfn_iterative_Test_Iter(self):
model = RBFN(50)
start = time.process_time()
# Generate a batch of data and store it
self.reset_batch()
g = SampleGenerator()
idx = []
res = []
for i in range(20, 1000):
# Draw a random sample on the interval [0,1]
x = np.random.random()
y = g.generate_non_linear_samples(x)
self.x_data.append(x)
self.y_data.append(y)
# Comment the ones you don't want to use
# model.train_gd(x, y, alpha=0.5)
model.train_rls(x, y)
# model.train_rls_sherman_morrison(x, y)
residuals = model.residuals(self.x_data, self.y_data)
idx.append(i)
print(residuals)
res.append(residuals / i)
plt.plot(idx, res)
plt.show()
def window_investigation(lang, distrib, h_layers, h_units, windows,
sample_size, n_epochs, exp_num, device):
acc_per_d = []
loss_per_d = []
loss_vals = None
generator = SampleGenerator(lang)
for window in windows:
print(window, flush=True)
filename = 'results/{}_{}_{}_{}_{}.npz'.format(lang, window, distrib,
h_layers, h_units)
if os.path.exists(filename):
d = np.load(filename)
e_vals = d["errors"]
loss_vals = d["losses"]
else:
inputs, outputs, s_dst = generator.generate_sample(
sample_size, window[0], window[1], distrib, False)
e_vals, loss_vals = train(generator, distrib, h_layers, h_units,
inputs, outputs, n_epochs, exp_num,
device)
np.savez(filename,
errors=np.array(e_vals),
losses=np.array(loss_vals))
acc_per_d.append(e_vals)
loss_per_d.append(loss_vals)
filename = '{}_{}_{}_{}_{}'.format(lang, 'window', distrib, h_layers,
h_units)
window_label = [
'Window [{}, {}]'.format(elt[0], elt[1]) for elt in windows
]
plot_graphs(lang, window_label, acc_per_d, None, filename)
return acc_per_d, loss_vals
def make_linear_batch_data(self):
"""
Generate a batch of linear data and store it into numpy structures
"""
self.reset_batch()
g = SampleGenerator()
for i in range(self.batch_size):
# Draw a random sample on the interval [0,1]
x = np.random.random()
y = g.generate_linear_samples(x)
self.x_data.append(x)
self.y_data.append(y)
def distribution_investigation(lang, distribution, h_layers, h_units, window,
sample_size, n_epochs, exp_num):
acc_per_d = []
loss_per_d = []
generator = SampleGenerator(lang)
for distrib in distribution:
inputs, outputs, s_dst = generator.generate_sample(
sample_size, window[0], window[1], distrib, False)
e_vals, loss_vals = train(generator, distrib, h_layers, h_units,
inputs, outputs, n_epochs, exp_num)
acc_per_d.append(e_vals)
loss_per_d.append(loss_vals)
filename = '{}_{}_{}_{}_{}_{}'.format(lang, 'distrib', h_layers, h_units,
window[0], window[1])
distrib_label = [elt.capitalize() for elt in distribution]
plot_graphs(lang, 'distrib', distrib_label, acc_per_d, loss_per_d, window,
filename)
return acc_per_d, loss_vals
def window_investigation(lang, distrib, h_layers, h_units, windows,
sample_size, n_epochs, exp_num):
acc_per_d = []
loss_per_d = []
generator = SampleGenerator(lang)
for window in windows:
inputs, outputs, s_dst = generator.generate_sample(
sample_size, window[0], window[1], distrib, False)
e_vals, loss_vals = train(generator, distrib, h_layers, h_units,
inputs, outputs, n_epochs, exp_num)
acc_per_d.append(e_vals)
loss_per_d.append(loss_vals)
filename = '{}_{}_{}_{}_{}'.format(lang, 'window', distrib, h_layers,
h_units)
window_label = [
'Window [{}, {}]'.format(elt[0], elt[1]) for elt in windows
]
plot_graphs(lang, 'window', window_label, acc_per_d, loss_per_d, [1, 30],
filename) # [1, 30] is a random window. We'll ignore it later.
return acc_per_d, loss_vals
def main():
# Arguments to main
parser = argparse.ArgumentParser()
parser.add_argument("-t", "--train_type", type=str, required=True, \
help="Training data set type, options = \
single batch_ind, short num_data, full, load_from_ckp ckp_path" )
parser.add_argument("-e", "--epochs", type=int, default=20, \
help="-e or --epochs number of epochs (default 20)")
parser.add_argument("-bs", "--batch_size", type=int, default=10, \
help="-bs or --batch_size training batch size (default 10)")
parser.add_argument("-s", "--save_model", type=str, \
help="-s save_path or --save_model save_path (save_path is relative from current dir)")
parser.add_argument("-cp", "--checkpoint", type=str, \
help="-cp or --checkpoint check_point_path to save checkpoints -- relative path")
parser.add_argument("-pl", "--print_loss", type=str, \
help="-pl or --print_loss plot_save_path (str) to save loss vs epochs")
parser.add_argument("-load", type=str, \
help="-load ckp_path (str) to load model from checkpoint before training")
args = parser.parse_args()
train_type, train_opts = " ", " "
if (len(args.train_type.split()) == 2):
train_type, train_opts = args.train_type.split()
else:
train_type = args.train_type.split()[0]
logger = Logger('log')
# Reading in data
logger.log(
'Reading in word: to word embedding -- mapping words to vectors...')
data_folder = "data_folder/created_data"
f = open(os.path.join(data_folder, "word_embed.pkl"), "rb")
word_embed = pickle.load(f)
f.close()
logger.log(
'Reading in raw text (tokenized) -- question ID maps to question (title + body)...'
)
f = open(os.path.join(data_folder, "question_id.pkl"), "rb")
question_id = pickle.load(f)
f.close()
logger.log(
'Reading in training data -- query question ID, similar questions ID (pos), random questions ID (neg)...'
)
train_q, train_pos, train_neg = read_question_data(
'data_folder/data/train_random.txt')
dev_q, dev_pos, dev_neg = read_question_data('data_folder/data/dev.txt')
# Creating model
logger.log('Creating Model ...')
data_obj = DataStore(question_id, word_embed)
n = 120 # number of sample questions per query question
N = 100 # number of words per question
opveclen = 100
wlen = len(word_embed['the'])
dims = n, N, wlen, opveclen
model = create_model(dims, data_obj.embedding_matrix)
print(model.summary())
train_sample_generator = SampleGenerator(train_q, train_pos, train_neg,
dims, data_obj)
dev_sample_generator = SampleGenerator(dev_q, dev_pos, dev_neg, dims,
data_obj)
logger.log('Model inputs and outputs')
loss_fn = loss_fn_wrap2(dims)
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(optimizer=opt, loss=loss_fn)
if (args.load):
model.load_weights(args.load)
# Training
cp_path = 0
if (args.checkpoint):
cp_path = args.checkpoint
if (train_type == "load_from_ckp"):
model.load_weights(train_opts)
logger.log("Model loaded")
elif train_type == "single":
batch_ind = int(train_opts)
model, loss_over_epochs = fit_model_single_data_point(
model,
train_sample_generator,
epochs=args.epochs,
batch_ind=batch_ind,
dims=dims,
logger=logger)
elif train_type == "short":
num_data = int(train_opts)
model, loss_over_epochs = fit_model(model, train_sample_generator, dev_sample_generator,\
batch_size=args.batch_size, epochs=args.epochs, dims=dims, \
checkpoint_path=cp_path, num_data = num_data)
elif train_type == "full":
model, loss_over_epochs = fit_model(model, train_sample_generator, dev_sample_generator,\
batch_size=args.batch_size, epochs=args.epochs, dims=dims, \
checkpoint_path=cp_path, num_data=[])
else:
logger.error("Invalid train type entered.")
# !mkdir -p saved_model
if (args.save_model):
model.save(args.save_model)
if (args.print_loss):
plt.plot(loss_over_epochs)
plt.show()
plt.savefig(args.print_loss, format='png')
def run_(model,
criterion,
target,
init_optimizer,
img_files,
init_rect,
video,
gt=None):
# Init bbox
box_ = np.array(init_rect)
result = np.zeros((len(img_files), 4))
result_bb = np.zeros((len(img_files), 4))
result[0] = box_
bbreg_bbox = box_
result_bb[0] = bbreg_bbox
tic = time.time()
# Load first image
image = Image.open(img_files[0]).convert('RGB') #[W,H] RGB
image_np = np.asarray(image) #[H, W, 3]
# Init bbox regressor
# Give the cropped region
# bbreg_examples_roi is [x_min,y_min,x_max,y_max]
region, crop_region_sz, bbreg_examples_roi, bbreg_im_index, box__roi, box__crop, coe = acquire_rois_bb(
SampleGenerator('uniform', 0.3, 1.5, 1.1, True), image_np, opts, box_,
opts['n_bbreg'], opts['overlap_bbreg'], opts['scale_bbreg'])
# bbreg_examples_reg is [x_min,y_min,w,h]
bbreg_examples_reg = np.hstack(
((bbreg_examples_roi[:, 0]).reshape(-1, 1),
(bbreg_examples_roi[:, 1]).reshape(-1, 1),
(bbreg_examples_roi[:, 2:] - bbreg_examples_roi[:, :2])))
bbreg_feats = extract_feat(model,
region,
bbreg_examples_roi,
bbreg_im_index,
fea_view=True)
bbreg = BBRegressor((np.array(region.shape[2:])).reshape(-1, 2),
overlap=opts['overlap_bbreg'],
scale=opts['scale_bbreg'])
bbreg.train(bbreg_feats, bbreg_examples_reg, box__roi)
# Draw pos/neg samples
pos_examples, pos_examples_roi, pos_im_index = acquire_roi_samples(
SampleGenerator('gaussian', 0.1, 1.2, valid=True), coe, box__crop,
crop_region_sz, opts['n_pos_init'], opts['overlap_pos_init'])
neg_examples, neg_examples_roi, neg_im_index = acquire_roi_samples(
SampleGenerator('uniform', 1, 2, 1.1, valid=True), coe, box__crop,
crop_region_sz, opts['n_neg_init'] // 2, opts['overlap_neg_init'])
neg_examples_whole, neg_examples_roi_whole, neg_im_index_whole = acquire_roi_samples(
SampleGenerator('whole', 0, 1.2, 1.1, valid=True), coe, box__crop,
crop_region_sz, opts['n_neg_init'] // 2, opts['overlap_neg_init'])
neg_examples_roi = np.concatenate(
(neg_examples_roi, neg_examples_roi_whole), axis=0)
neg_examples_roi = np.random.permutation(neg_examples_roi)
neg_im_index = np.concatenate((neg_im_index, neg_im_index_whole), axis=0)
# Extract pos/neg features
pos_feats = extract_feat(model, region, pos_examples_roi, pos_im_index)
neg_feats = extract_feat(model, region, neg_examples_roi, neg_im_index)
feat_dim = pos_feats.size(-1)
channel_dim = pos_feats.size(-3)
# Initial training
train(model, criterion, target, init_optimizer, pos_feats, neg_feats,
opts['maxiter_init'])
model.stop_learnable_params(opts['stop_layers'])
update_optimizer = set_optimizer(model, opts['lr_update'])
# Init sample generators
sample_generator = SampleGenerator('gaussian',
opts['trans_f'],
opts['scale_f'],
valid=True)
pos_generator = SampleGenerator('gaussian', 0.1, 1.2, valid=True)
neg_generator = SampleGenerator('uniform', 1.5, 1.2, valid=True)
# Init pos/neg features for update
pos_feats_all = [pos_feats[:opts['n_pos_update']]]
neg_feats_all = [neg_feats[:opts['n_neg_update']]]
spf_total = time.time() - tic
# Start tracking
unsuccess_num = 0
for i in range(1, len(img_files)):
tic = time.time()
# Load image
image = Image.open(img_files[i]).convert('RGB')
image_np = np.asarray(image) #[H, W, 3]
# Cropping
region, crop_region_sz, coe, box__crop = acquire_region(
image_np, box_, opts)
samples, samples_roi, samples_im_index = acquire_roi_samples(
sample_generator, coe, box__crop, crop_region_sz,
opts['n_samples'])
sample_scores, sta_g_weight, sta_penalty, atten_map, conv3_fea = extract_feat(
model, region, samples_roi, samples_im_index, out_layer='capsule')
top_scores, top_idx = sample_scores[:, 1].topk(5)
top_idx = top_idx.cpu().numpy()
target_score = top_scores.mean()
samples_topk = samples[top_idx]
samples_topk[:, :2] = samples_topk[:, :2] - box__crop[:2].reshape(
-1, 2) + box_[:2].reshape(-1, 2)
box__copy = box_.copy()
box_ = samples_topk.mean(
axis=0) #Take the mean value of top 5 as the tracking result
success = target_score > opts['success_thr']
# Expand search area when failure occurs
if success:
unsuccess_num = 0
sample_generator.set_trans_f(opts['trans_f'])
else:
unsuccess_num += 1
sample_generator.set_trans_f(opts['trans_f_expand'])
# Bbox regression
if success:
bbreg_samples_roi = samples_roi[top_idx]
bbreg_samples_reg = np.hstack(
((bbreg_samples_roi[:, 0]).reshape(-1, 1),
(bbreg_samples_roi[:, 1]).reshape(-1, 1),
(bbreg_samples_roi[:, 2:] - bbreg_samples_roi[:, :2])))
bbreg_feats = extract_feat(model,
region,
bbreg_samples_roi,
samples_im_index[top_idx],
fea_view=True)
bbreg_samples = bbreg.predict(bbreg_feats, bbreg_samples_reg)
bbreg_bbox = bbreg_samples.mean(axis=0)
bbreg_bbox = np.array([
bbreg_bbox[0] * coe[0], bbreg_bbox[1] * coe[1],
bbreg_bbox[2] * coe[0], bbreg_bbox[3] * coe[1]
])
bbreg_bbox[:2] = np.array(bbreg_bbox[:2] - box__crop[:2] +
box__copy[:2])
else:
bbreg_bbox = box_
# Copy previous result at failure
if not success:
box_ = result[i - 1]
bbreg_bbox = result_bb[i - 1]
# Save result
result[i] = box_
result_bb[i] = bbreg_bbox
# Data collect
if success:
# Draw pos/neg samples
region, crop_region_sz, coe, box__crop = acquire_region(
image_np, box_, opts)
pos_examples, pos_examples_roi, pos_im_index = acquire_roi_samples(
pos_generator, coe, box__crop, crop_region_sz,
opts['n_pos_update'], opts['overlap_pos_update'])
neg_examples, neg_examples_roi, neg_im_index = acquire_roi_samples(
neg_generator, coe, box__crop, crop_region_sz,
opts['n_neg_update'], opts['overlap_neg_update'])
# Extract pos/neg features
pos_feats = extract_feat(model, region, pos_examples_roi,
pos_im_index)
neg_feats = extract_feat(model, region, neg_examples_roi,
neg_im_index)
pos_feats_all.append(pos_feats)
neg_feats_all.append(neg_feats)
if len(pos_feats_all
) > opts['n_frames_long']: # Accumulate updating features
del pos_feats_all[
1] # Keep the information of the first frame 1 or 0
if len(neg_feats_all) > opts['n_frames_short']:
del neg_feats_all[
0] # Keep the information of the first frame, but it will hurt ironman
# Short term update
if (not success) & (unsuccess_num < 15):
nframes = min(opts['n_frames_short'], len(pos_feats_all))
pos_data = torch.stack(pos_feats_all[-nframes:],
0).view(-1, channel_dim, feat_dim,
feat_dim) # [20*50, 512,7,7]
neg_data = torch.stack(neg_feats_all, 0).view(
-1, channel_dim, feat_dim,
feat_dim) # [20 or less *200, 512,7,7]
train(model, criterion, target, update_optimizer, pos_data,
neg_data, opts['maxiter_update'])
# Long term update
elif i % opts['long_interval'] == 0:
pos_data = torch.stack(pos_feats_all,
0).view(-1, channel_dim, feat_dim, feat_dim)
neg_data = torch.stack(neg_feats_all,
0).view(-1, channel_dim, feat_dim, feat_dim)
train(model, criterion, target, update_optimizer, pos_data,
neg_data, opts['maxiter_update'])
spf = time.time() - tic
spf_total += spf
fps = len(img_files) / spf_total
print("Speed: %.3f" % (fps))
return result, result_bb, fps, spf_total
def run_mdnet(imgs,
init_bbox,
gt=None,
seq='',
label_id=0,
savefig_dir='',
display=False):
# Init bbox
target_bbox = np.array(init_bbox[0], dtype='float')
result = np.zeros((len(imgs), 4))
result[0] = np.copy(target_bbox)
iou_result = np.zeros((len(imgs), 1))
savefig_dir = os.path.join("dump/{}/{}".format(seq, label_id))
os.makedirs(savefig_dir, exist_ok=True)
# Init model and optimizer
mdnet = MDNet()
mdnet.load_fclayer_weights(opts['model_path'])
mdnet.set_learnable_params(opts['ft_layers'])
mdnet.OPN.eval()
mdnet.cuda()
gaussian_sampler = SampleGenerator("gaussian", opts["img_size"], 0.1, 1.2)
backend_sampler = SampleGenerator("gaussian", opts["img_size"], 0.2, 1.2)
uniform_sampler = SampleGenerator("uniform", opts["img_size"], 1, 2, 1.1)
FPN_RoIAlign = Pooler(output_size=(7, 7),
scales=(0.25, 0.125, 0.0625, 0.03125),
sampling_ratio=2)
criterion = BinaryLoss()
roi_scene_size = opts["img_size"] # (w, h)
init_optimizer = set_optimizer(mdnet, opts['lr_init'])
update_optimizer = set_optimizer(mdnet, opts['lr_update'])
pos_feats_all, neg_feats_all = [], []
tic = time.time()
# Load first image and finetune
init_image = imgs[0:1, :]
init_opn_feat, init_opn_rois = mdnet.forward_OPN(init_image)
init_roi_feats = FPN_RoIAlign(init_opn_feat, init_opn_rois)
init_pos_idx, init_neg_idx = sample_pos_neg_idxs(
init_bbox,
init_opn_rois,
fg_thres=opts['overlap_pos_init'][0],
bg_thres=opts['overlap_neg_init'][1],
fg_num=opts['n_pos_init'],
bg_num=opts['n_neg_init'])
if init_pos_idx is not None and init_neg_idx is not None:
init_pos_feats = init_roi_feats[init_pos_idx]
init_neg_feats = init_roi_feats[init_neg_idx]
init_pos_rois_visual = torch.cat([x.bbox for x in init_opn_rois
])[init_pos_idx]
else:
if boxarea(init_bbox[0]) < 50:
norm_init_bbox = scalebox(init_bbox[0], 5)
else:
norm_init_bbox = init_bbox[0]
init_pos_rois = gen_samples(gaussian_sampler,
norm_init_bbox,
opts['n_pos_init'],
overlap_range=opts['overlap_pos_init'])
init_neg_rois = gen_samples(uniform_sampler,
norm_init_bbox,
opts['n_neg_init'],
overlap_range=opts['overlap_neg_init'])
init_pos_rois = [
BoxList(torch.from_numpy(init_pos_rois).cuda(),
roi_scene_size,
mode="xyxy")
]
init_neg_rois = [
BoxList(torch.from_numpy(init_neg_rois).cuda(),
roi_scene_size,
mode="xyxy")
]
init_pos_feats = FPN_RoIAlign(init_opn_feat, init_pos_rois)
init_neg_feats = FPN_RoIAlign(init_opn_feat, init_neg_rois)
init_pos_rois_visual = init_pos_rois[0].bbox
init_pos_feats = init_pos_feats.view(opts['n_pos_init'], -1)
init_neg_feats = init_neg_feats.view(opts['n_neg_init'], -1)
feat_dim = init_pos_feats.size(-1)
print(feat_dim)
torch.cuda.empty_cache()
init_optimizer.zero_grad()
train(mdnet, criterion, init_optimizer, init_pos_feats, init_neg_feats,
opts['maxiter_init'])
# Memory
pos_idx = np.asarray(range(init_pos_feats.size(0)))
np.random.shuffle(pos_idx)
pos_feats_all = [
init_pos_feats.index_select(
0,
torch.from_numpy(pos_idx[0:opts['n_pos_update']]).cuda())
]
neg_idx = np.asarray(range(init_neg_feats.size(0)))
np.random.shuffle(neg_idx)
neg_feats_all = [
init_neg_feats.index_select(
0,
torch.from_numpy(neg_idx[0:opts['n_neg_update']]).cuda())
]
spf_total = time.time() - tic
# Visual
savefig_path = "{}/00000.jpg".format(savefig_dir)
print("Dump {}...".format(savefig_path))
#init_rois = torch.cat([x.bbox for x in init_pos_rois])
overlay_box(depreprocess(imgs[0]),
init_pos_rois_visual.cpu().numpy(),
gt[0:1],
savefig_path=savefig_path)
for i in range(1, len(imgs)):
tic = time.time()
cur_img = imgs[i:i + 1, :]
cur_opn_feat, cur_opn_rois = mdnet.forward_OPN(cur_img)
cur_roi_feats = FPN_RoIAlign(cur_opn_feat, cur_opn_rois)
cur_cand_idx = sample_cand_idxs(target_bbox[None, :],
torch.cat(
[x.bbox for x in cur_opn_rois]),
size=roi_scene_size,
thres=0.2)
if cur_cand_idx is not None:
cur_cand_feats = cur_roi_feats[cur_cand_idx].view(
cur_cand_idx.size(0), -1)
cur_cand_rois = torch.cat([x.bbox
for x in cur_opn_rois])[cur_cand_idx]
else:
backend_rois = gen_samples(backend_sampler,
target_bbox,
200,
overlap_range=(0, 0.3))
backend_rois = [
BoxList(torch.from_numpy(backend_rois).cuda(),
roi_scene_size,
mode="xyxy")
]
cur_cand_rois = torch.cat([x.bbox for x in backend_rois])
cur_cand_feats = FPN_RoIAlign(cur_opn_feat, backend_rois)
cur_cand_feats = cur_cand_feats.view(cur_cand_rois.size(0), -1)
cur_cand_scores = mdnet.forward(cur_cand_feats, in_layer='fc4')
top_scores, top_idx = cur_cand_scores[:, 1].topk(5)
top_idx = top_idx.cpu().numpy()
target_score = top_scores.data.mean().item()
success = target_score > 0
# Save result
if success:
target_bbox = cur_cand_rois[top_idx].data.cpu().numpy().mean(
axis=0)
print("success")
else:
target_bbox = result[i - 1]
print("failed")
result[i] = target_bbox
# Data collect
if success:
cur_pos_idx, cur_neg_idx = sample_pos_neg_idxs_4tensor(
target_bbox[None, :],
cur_cand_rois,
size=roi_scene_size,
fg_thres=opts['overlap_pos_update'][0],
bg_thres=opts['overlap_neg_update'][1],
fg_num=opts['n_pos_update'],
bg_num=opts['n_neg_update'])
if cur_pos_idx is not None and cur_neg_idx is not None:
cur_pos_feats = cur_cand_feats[cur_pos_idx].view(
opts['n_pos_update'], -1)
cur_neg_feats = cur_cand_feats[cur_neg_idx].view(
opts['n_neg_update'], -1)
pos_feats_all.append(cur_pos_feats)
neg_feats_all.append(cur_neg_feats)
if len(pos_feats_all) > opts['n_frames_long']:
del pos_feats_all[0]
if len(neg_feats_all) > opts['n_frames_short']:
del neg_feats_all[0]
# Short term update
if not success:
nframes = min(opts['n_frames_short'], len(pos_feats_all))
pos_data = torch.stack(pos_feats_all[-nframes:],
0).view(-1, feat_dim)
neg_data = torch.stack(neg_feats_all, 0).view(-1, feat_dim)
train(mdnet, criterion, update_optimizer, pos_data, neg_data,
opts['maxiter_update'])
# Long term update
elif i % opts['long_interval'] == 0:
pos_data = torch.stack(pos_feats_all, 0).view(-1, feat_dim)
neg_data = torch.stack(neg_feats_all, 0).view(-1, feat_dim)
train(mdnet, criterion, update_optimizer, pos_data, neg_data,
opts['maxiter_update'])
spf = time.time() - tic
spf_total += spf
# Visual
savefig_path = "{}/{:05d}.jpg".format(savefig_dir, i)
print("Dump {}...".format(savefig_path))
#debug_scores, debug_idx = cur_cand_scores[:, 1].topk(10)
overlay_box(depreprocess(imgs[i]),
target_bbox,
gt[i:i + 1],
savefig_path=savefig_path)
fps = len(imgs) / spf_total
return result, fps
def create_data_objs():
android_data_folder = 'Android-master'
f = open(os.path.join(android_data_folder, "corpus.pkl"), "rb")
android_question_id = pickle.load(f)
f.close()
data_folder = "data_folder/created_data"
f = open(os.path.join(data_folder, "word_embed.pkl"), "rb")
word_embed = pickle.load(f)
f.close()
f = open(os.path.join(data_folder, "question_id.pkl"), "rb")
ubuntu_question_id = pickle.load(f)
f.close()
n = 120
N = 100
opveclen = 100
wlen = len(word_embed['the'])
dims = n, N, wlen, opveclen
f = open(os.path.join(android_data_folder, "dev_pos.pkl"), "rb")
android_dev_pos = pickle.load(f)
f.close()
f = open(os.path.join(android_data_folder, "dev_neg.pkl"), "rb")
android_dev_neg = pickle.load(f)
f.close()
f = open(os.path.join(android_data_folder, "test_pos.pkl"), "rb")
android_test_pos = pickle.load(f)
f.close()
f = open(os.path.join(android_data_folder, "test_neg.pkl"), "rb")
android_test_neg = pickle.load(f)
f.close()
train_q, train_pos, train_neg = read_question_data(
'data_folder/data/train_random.txt')
dev_q, dev_pos, dev_neg = read_question_data('data_folder/data/dev.txt')
test_q, test_pos, test_neg = read_question_data(
'data_folder/data/test.txt')
android_data_obj = DataStore(android_question_id, word_embed)
ubuntu_data_obj = DataStore(ubuntu_question_id, word_embed)
android_dev_generator = AndroidSampleGenerator(android_dev_pos,\
android_dev_neg, dims, android_data_obj)
android_test_generator = AndroidSampleGenerator(android_test_pos,\
android_test_neg, dims, android_data_obj)
ubuntu_train_generator = SampleGenerator(train_q, train_pos, train_neg,
dims, ubuntu_data_obj)
ubuntu_dev_generator = SampleGenerator(dev_q, dev_pos, dev_neg, dims,
ubuntu_data_obj)
ubuntu_test_generator = SampleGenerator(test_q, test_pos, test_neg, dims,
ubuntu_data_obj)
return (dims, android_data_obj, android_dev_generator,
android_test_generator, ubuntu_data_obj, ubuntu_train_generator,
ubuntu_dev_generator, ubuntu_test_generator)
def RTMDNet_track(state, img_file):
cur_image = Image.open(img_file).convert('RGB')
cur_image = np.asarray(cur_image)
trans_f = state['trans_f']
model = state['model']
target_bbox = state['target_bbox']
img_crop_model = state['img_crop_model']
bbreg = state['bbreg']
criterion = state['criterion']
update_optimizer = state['update_optimizer']
pos_feats_all = state['pos_feats_all']
neg_feats_all = state['neg_feats_all']
feat_dim = state['feat_dim']
count = state['count']
ishape = cur_image.shape
samples = gen_samples(
SampleGenerator('gaussian', (ishape[1], ishape[0]),
trans_f,
opts['scale_f'],
valid=True), target_bbox, opts['n_samples'])
padded_x1 = (samples[:, 0] - samples[:, 2] *
(opts['padding'] - 1.) / 2.).min()
padded_y1 = (samples[:, 1] - samples[:, 3] *
(opts['padding'] - 1.) / 2.).min()
padded_x2 = (samples[:, 0] + samples[:, 2] *
(opts['padding'] + 1.) / 2.).max()
padded_y2 = (samples[:, 1] + samples[:, 3] *
(opts['padding'] + 1.) / 2.).max()
padded_scene_box = np.asarray(
(padded_x1, padded_y1, padded_x2 - padded_x1, padded_y2 - padded_y1))
if padded_scene_box[0] > cur_image.shape[1]:
padded_scene_box[0] = cur_image.shape[1] - 1
if padded_scene_box[1] > cur_image.shape[0]:
padded_scene_box[1] = cur_image.shape[0] - 1
if padded_scene_box[0] + padded_scene_box[2] < 0:
padded_scene_box[2] = -padded_scene_box[0] + 1
if padded_scene_box[1] + padded_scene_box[3] < 0:
padded_scene_box[3] = -padded_scene_box[1] + 1
crop_img_size = (padded_scene_box[2:4] *
((opts['img_size'], opts['img_size']) /
target_bbox[2:4])).astype('int64')
cropped_image, cur_image_var = img_crop_model.crop_image(
cur_image, np.reshape(padded_scene_box, (1, 4)), crop_img_size)
cropped_image = cropped_image - 128.
model.eval()
feat_map = model(cropped_image, out_layer='conv3')
# relative target bbox with padded_scene_box
rel_target_bbox = np.copy(target_bbox)
rel_target_bbox[0:2] -= padded_scene_box[0:2]
# Extract sample features and get target location
batch_num = np.zeros((samples.shape[0], 1))
sample_rois = np.copy(samples)
sample_rois[:, 0:2] -= np.repeat(np.reshape(padded_scene_box[0:2], (1, 2)),
sample_rois.shape[0],
axis=0)
sample_rois = samples2maskroi(sample_rois, model.receptive_field,
(opts['img_size'], opts['img_size']),
target_bbox[2:4], opts['padding'])
sample_rois = np.concatenate((batch_num, sample_rois), axis=1)
sample_rois = Variable(torch.from_numpy(
sample_rois.astype('float32'))).cuda()
sample_feats = model.roi_align_model(feat_map, sample_rois)
sample_feats = sample_feats.view(sample_feats.size(0), -1).clone()
sample_scores = model(sample_feats, in_layer='fc4')
top_scores, top_idx = sample_scores[:, 1].topk(5)
top_idx = top_idx.data.cpu().numpy()
target_score = top_scores.data.mean()
target_bbox = samples[top_idx].mean(axis=0)
success = target_score > opts['success_thr']
logging.info('target_score: %s, success: %s', target_score, success)
# Expand search area at failure
if success:
trans_f = opts['trans_f']
else:
trans_f = opts['trans_f_expand']
## Bbox regression
if success:
bbreg_feats = sample_feats[top_idx, :]
bbreg_samples = samples[top_idx]
bbreg_samples = bbreg.predict(bbreg_feats.data, bbreg_samples)
bbreg_bbox = bbreg_samples.mean(axis=0)
else:
bbreg_bbox = target_bbox
# Data collect
if success:
# Draw pos/neg samples
pos_examples = gen_samples(
SampleGenerator('gaussian', (ishape[1], ishape[0]), 0.1, 1.2),
target_bbox, opts['n_pos_update'], opts['overlap_pos_update'])
neg_examples = gen_samples(
SampleGenerator('uniform', (ishape[1], ishape[0]), 1.5, 1.2),
target_bbox, opts['n_neg_update'], opts['overlap_neg_update'])
padded_x1 = (neg_examples[:, 0] - neg_examples[:, 2] *
(opts['padding'] - 1.) / 2.).min()
padded_y1 = (neg_examples[:, 1] - neg_examples[:, 3] *
(opts['padding'] - 1.) / 2.).min()
padded_x2 = (neg_examples[:, 0] + neg_examples[:, 2] *
(opts['padding'] + 1.) / 2.).max()
padded_y2 = (neg_examples[:, 1] + neg_examples[:, 3] *
(opts['padding'] + 1.) / 2.).max()
padded_scene_box = np.reshape(
np.asarray((padded_x1, padded_y1, padded_x2 - padded_x1,
padded_y2 - padded_y1)), (1, 4))
scene_boxes = np.reshape(np.copy(padded_scene_box), (1, 4))
jitter_scale = [1.]
for bidx in range(0, scene_boxes.shape[0]):
crop_img_size = (scene_boxes[bidx, 2:4] * (
(opts['img_size'], opts['img_size']) / target_bbox[2:4])
).astype('int64') * jitter_scale[bidx]
cropped_image, cur_image_var = img_crop_model.crop_image(
cur_image, np.reshape(scene_boxes[bidx], (1, 4)),
crop_img_size)
cropped_image = cropped_image - 128.
feat_map = model(cropped_image, out_layer='conv3')
rel_target_bbox = np.copy(target_bbox)
rel_target_bbox[0:2] -= scene_boxes[bidx, 0:2]
batch_num = np.zeros((pos_examples.shape[0], 1))
cur_pos_rois = np.copy(pos_examples)
cur_pos_rois[:,
0:2] -= np.repeat(np.reshape(scene_boxes[bidx, 0:2],
(1, 2)),
cur_pos_rois.shape[0],
axis=0)
scaled_obj_size = float(opts['img_size']) * jitter_scale[bidx]
cur_pos_rois = samples2maskroi(cur_pos_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size),
target_bbox[2:4], opts['padding'])
cur_pos_rois = np.concatenate((batch_num, cur_pos_rois), axis=1)
cur_pos_rois = Variable(
torch.from_numpy(cur_pos_rois.astype('float32'))).cuda()
cur_pos_feats = model.roi_align_model(feat_map, cur_pos_rois)
cur_pos_feats = cur_pos_feats.view(cur_pos_feats.size(0),
-1).data.clone()
batch_num = np.zeros((neg_examples.shape[0], 1))
cur_neg_rois = np.copy(neg_examples)
cur_neg_rois[:,
0:2] -= np.repeat(np.reshape(scene_boxes[bidx, 0:2],
(1, 2)),
cur_neg_rois.shape[0],
axis=0)
cur_neg_rois = samples2maskroi(cur_neg_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size),
target_bbox[2:4], opts['padding'])
cur_neg_rois = np.concatenate((batch_num, cur_neg_rois), axis=1)
cur_neg_rois = Variable(
torch.from_numpy(cur_neg_rois.astype('float32'))).cuda()
cur_neg_feats = model.roi_align_model(feat_map, cur_neg_rois)
cur_neg_feats = cur_neg_feats.view(cur_neg_feats.size(0),
-1).data.clone()
feat_dim = cur_pos_feats.size(-1)
if bidx == 0:
pos_feats = cur_pos_feats ##index select
neg_feats = cur_neg_feats
else:
pos_feats = torch.cat((pos_feats, cur_pos_feats), dim=0)
neg_feats = torch.cat((neg_feats, cur_neg_feats), dim=0)
if pos_feats.size(0) > opts['n_pos_update']:
pos_idx = np.asarray(range(pos_feats.size(0)))
np.random.shuffle(pos_idx)
pos_feats = pos_feats.index_select(
0,
torch.from_numpy(pos_idx[0:opts['n_pos_update']]).cuda())
if neg_feats.size(0) > opts['n_neg_update']:
neg_idx = np.asarray(range(neg_feats.size(0)))
np.random.shuffle(neg_idx)
neg_feats = neg_feats.index_select(
0,
torch.from_numpy(neg_idx[0:opts['n_neg_update']]).cuda())
pos_feats_all.append(pos_feats)
neg_feats_all.append(neg_feats)
if len(pos_feats_all) > opts['n_frames_long']:
del pos_feats_all[0]
if len(neg_feats_all) > opts['n_frames_short']:
del neg_feats_all[0]
# Short term update
if not success:
nframes = min(opts['n_frames_short'], len(pos_feats_all))
pos_data = torch.stack(pos_feats_all[-nframes:], 0).view(-1, feat_dim)
neg_data = torch.stack(neg_feats_all, 0).view(-1, feat_dim)
train(model, criterion, update_optimizer, pos_data, neg_data,
opts['maxiter_update'])
# Long term update
elif count % opts['long_interval'] == 0:
pos_data = torch.stack(pos_feats_all, 0).view(-1, feat_dim)
neg_data = torch.stack(neg_feats_all, 0).view(-1, feat_dim)
train(model, criterion, update_optimizer, pos_data, neg_data,
opts['maxiter_update'])
state['trans_f'] = trans_f
state['model'] = model
state['target_bbox'] = target_bbox
state['img_crop_model'] = img_crop_model
state['bbreg'] = bbreg
state['criterion'] = criterion
state['update_optimizer'] = update_optimizer
state['pos_feats_all'] = pos_feats_all
state['neg_feats_all'] = neg_feats_all
state['count'] = count + 1
return state
def RTMDNet_init(model_path, image_file, init_bbox):
state = dict()
target_bbox = np.array(init_bbox)
model = MDNet(model_path)
if opts['adaptive_align']:
align_h = model.roi_align_model.aligned_height
align_w = model.roi_align_model.aligned_width
spatial_s = model.roi_align_model.spatial_scale
model.roi_align_model = RoIAlignAdaMax(align_h, align_w, spatial_s)
if opts['use_gpu']:
model = model.cuda()
model.set_learnable_params(opts['ft_layers'])
# Init image crop model
img_crop_model = imgCropper(1.)
if opts['use_gpu']:
img_crop_model.gpuEnable()
# Init criterion and optimizer
criterion = BinaryLoss()
init_optimizer = set_optimizer(model, opts['lr_init'])
update_optimizer = set_optimizer(model, opts['lr_update'])
cur_image = Image.open(image_file).convert('RGB')
cur_image = np.asarray(cur_image)
# Draw pos/neg samples
ishape = cur_image.shape
# logging.info('ishape: %s, n_pos_init: %s, overlap_pos_init: %s, target_bbox: %s', ishape, opts['n_pos_init'], opts['overlap_pos_init'], target_bbox)
pos_examples = gen_samples(
SampleGenerator('gaussian', (ishape[1], ishape[0]), 0.1, 1.2),
target_bbox, opts['n_pos_init'], opts['overlap_pos_init'])
neg_examples = gen_samples(
SampleGenerator('uniform', (ishape[1], ishape[0]), 1, 2, 1.1),
target_bbox, opts['n_neg_init'], opts['overlap_neg_init'])
neg_examples = np.random.permutation(neg_examples)
cur_bbreg_examples = gen_samples(
SampleGenerator('uniform', (ishape[1], ishape[0]), 0.3, 1.5,
1.1), target_bbox, opts['n_bbreg'],
opts['overlap_bbreg'], opts['scale_bbreg'])
# compute padded sample
padded_x1 = (neg_examples[:, 0] - neg_examples[:, 2] *
(opts['padding'] - 1.) / 2.).min()
padded_y1 = (neg_examples[:, 1] - neg_examples[:, 3] *
(opts['padding'] - 1.) / 2.).min()
padded_x2 = (neg_examples[:, 0] + neg_examples[:, 2] *
(opts['padding'] + 1.) / 2.).max()
padded_y2 = (neg_examples[:, 1] + neg_examples[:, 3] *
(opts['padding'] + 1.) / 2.).max()
padded_scene_box = np.reshape(
np.asarray((padded_x1, padded_y1, padded_x2 - padded_x1,
padded_y2 - padded_y1)), (1, 4))
scene_boxes = np.reshape(np.copy(padded_scene_box), (1, 4))
if opts['jitter']:
## horizontal shift
jittered_scene_box_horizon = np.copy(padded_scene_box)
jittered_scene_box_horizon[0, 0] -= 4.
jitter_scale_horizon = 1.
## vertical shift
jittered_scene_box_vertical = np.copy(padded_scene_box)
jittered_scene_box_vertical[0, 1] -= 4.
jitter_scale_vertical = 1.
jittered_scene_box_reduce1 = np.copy(padded_scene_box)
jitter_scale_reduce1 = 1.1**(-1)
## vertical shift
jittered_scene_box_enlarge1 = np.copy(padded_scene_box)
jitter_scale_enlarge1 = 1.1**(1)
## scale reduction
jittered_scene_box_reduce2 = np.copy(padded_scene_box)
jitter_scale_reduce2 = 1.1**(-2)
## scale enlarge
jittered_scene_box_enlarge2 = np.copy(padded_scene_box)
jitter_scale_enlarge2 = 1.1**(2)
scene_boxes = np.concatenate([
scene_boxes, jittered_scene_box_horizon,
jittered_scene_box_vertical, jittered_scene_box_reduce1,
jittered_scene_box_enlarge1, jittered_scene_box_reduce2,
jittered_scene_box_enlarge2
],
axis=0)
jitter_scale = [
1., jitter_scale_horizon, jitter_scale_vertical,
jitter_scale_reduce1, jitter_scale_enlarge1, jitter_scale_reduce2,
jitter_scale_enlarge2
]
else:
jitter_scale = [1.]
model.eval()
for bidx in range(0, scene_boxes.shape[0]):
crop_img_size = (scene_boxes[bidx, 2:4] * (
(opts['img_size'], opts['img_size']) / target_bbox[2:4])
).astype('int64') * jitter_scale[bidx]
cropped_image, cur_image_var = img_crop_model.crop_image(
cur_image, np.reshape(scene_boxes[bidx], (1, 4)), crop_img_size)
cropped_image = cropped_image - 128.
feat_map = model(cropped_image, out_layer='conv3')
rel_target_bbox = np.copy(target_bbox)
rel_target_bbox[0:2] -= scene_boxes[bidx, 0:2]
batch_num = np.zeros((pos_examples.shape[0], 1))
cur_pos_rois = np.copy(pos_examples)
# logging.info('cur_pos_rois from copy: %s', cur_pos_rois.shape)
cur_pos_rois[:, 0:2] -= np.repeat(np.reshape(scene_boxes[bidx, 0:2],
(1, 2)),
cur_pos_rois.shape[0],
axis=0)
# logging.info('cur_pos_rois after reshape: %s', cur_pos_rois.shape)
scaled_obj_size = float(opts['img_size']) * jitter_scale[bidx]
# logging.info('scaled_obj_size: %s ', scaled_obj_size)
cur_pos_rois = samples2maskroi(cur_pos_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size),
target_bbox[2:4], opts['padding'])
# logging.info('cur_pos_rois after after samples2maskroi: %s', cur_pos_rois.shape)
cur_pos_rois = np.concatenate((batch_num, cur_pos_rois), axis=1)
# logging.info('cur_pos_rois after after concatenate: %s', cur_pos_rois.shape)
cur_pos_rois = Variable(
torch.from_numpy(cur_pos_rois.astype('float32'))).cuda()
cur_pos_feats = model.roi_align_model(feat_map, cur_pos_rois)
cur_pos_feats = cur_pos_feats.view(cur_pos_feats.size(0),
-1).data.clone()
batch_num = np.zeros((neg_examples.shape[0], 1))
cur_neg_rois = np.copy(neg_examples)
cur_neg_rois[:, 0:2] -= np.repeat(np.reshape(scene_boxes[bidx, 0:2],
(1, 2)),
cur_neg_rois.shape[0],
axis=0)
cur_neg_rois = samples2maskroi(cur_neg_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size),
target_bbox[2:4], opts['padding'])
cur_neg_rois = np.concatenate((batch_num, cur_neg_rois), axis=1)
cur_neg_rois = Variable(
torch.from_numpy(cur_neg_rois.astype('float32'))).cuda()
cur_neg_feats = model.roi_align_model(feat_map, cur_neg_rois)
cur_neg_feats = cur_neg_feats.view(cur_neg_feats.size(0),
-1).data.clone()
## bbreg rois
batch_num = np.zeros((cur_bbreg_examples.shape[0], 1))
cur_bbreg_rois = np.copy(cur_bbreg_examples)
cur_bbreg_rois[:, 0:2] -= np.repeat(np.reshape(scene_boxes[bidx, 0:2],
(1, 2)),
cur_bbreg_rois.shape[0],
axis=0)
scaled_obj_size = float(opts['img_size']) * jitter_scale[bidx]
cur_bbreg_rois = samples2maskroi(cur_bbreg_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size),
target_bbox[2:4], opts['padding'])
cur_bbreg_rois = np.concatenate((batch_num, cur_bbreg_rois), axis=1)
cur_bbreg_rois = Variable(
torch.from_numpy(cur_bbreg_rois.astype('float32'))).cuda()
cur_bbreg_feats = model.roi_align_model(feat_map, cur_bbreg_rois)
cur_bbreg_feats = cur_bbreg_feats.view(cur_bbreg_feats.size(0),
-1).data.clone()
feat_dim = cur_pos_feats.size(-1)
if bidx == 0:
pos_feats = cur_pos_feats
neg_feats = cur_neg_feats
##bbreg feature
bbreg_feats = cur_bbreg_feats
bbreg_examples = cur_bbreg_examples
else:
pos_feats = torch.cat((pos_feats, cur_pos_feats), dim=0)
neg_feats = torch.cat((neg_feats, cur_neg_feats), dim=0)
##bbreg feature
bbreg_feats = torch.cat((bbreg_feats, cur_bbreg_feats), dim=0)
bbreg_examples = np.concatenate(
(bbreg_examples, cur_bbreg_examples), axis=0)
if pos_feats.size(0) > opts['n_pos_init']:
pos_idx = np.asarray(range(pos_feats.size(0)))
np.random.shuffle(pos_idx)
pos_feats = pos_feats[pos_idx[0:opts['n_pos_init']], :]
if neg_feats.size(0) > opts['n_neg_init']:
neg_idx = np.asarray(range(neg_feats.size(0)))
np.random.shuffle(neg_idx)
neg_feats = neg_feats[neg_idx[0:opts['n_neg_init']], :]
##bbreg
if bbreg_feats.size(0) > opts['n_bbreg']:
bbreg_idx = np.asarray(range(bbreg_feats.size(0)))
np.random.shuffle(bbreg_idx)
bbreg_feats = bbreg_feats[bbreg_idx[0:opts['n_bbreg']], :]
bbreg_examples = bbreg_examples[bbreg_idx[0:opts['n_bbreg']], :]
#print bbreg_examples.shape
## open images and crop patch from obj
extra_obj_size = np.array((opts['img_size'], opts['img_size']))
extra_crop_img_size = extra_obj_size * (opts['padding'] + 0.6)
replicateNum = 100
for iidx in range(replicateNum):
extra_target_bbox = np.copy(target_bbox)
extra_scene_box = np.copy(extra_target_bbox)
extra_scene_box_center = extra_scene_box[
0:2] + extra_scene_box[2:4] / 2.
extra_scene_box_size = extra_scene_box[2:4] * (opts['padding'] + 0.6)
extra_scene_box[
0:2] = extra_scene_box_center - extra_scene_box_size / 2.
extra_scene_box[2:4] = extra_scene_box_size
extra_shift_offset = np.clip(2. * np.random.randn(2), -4, 4)
cur_extra_scale = 1.1**np.clip(np.random.randn(1), -2, 2)
extra_scene_box[0] += extra_shift_offset[0]
extra_scene_box[1] += extra_shift_offset[1]
extra_scene_box[2:4] *= cur_extra_scale[0]
scaled_obj_size = float(opts['img_size']) / cur_extra_scale[0]
cur_extra_cropped_image, _ = img_crop_model.crop_image(
cur_image, np.reshape(extra_scene_box, (1, 4)),
extra_crop_img_size)
cur_extra_cropped_image = cur_extra_cropped_image.detach()
cur_extra_pos_examples = gen_samples(
SampleGenerator('gaussian', (ishape[1], ishape[0]), 0.1,
1.2), extra_target_bbox,
opts['n_pos_init'] / replicateNum, opts['overlap_pos_init'])
cur_extra_neg_examples = gen_samples(
SampleGenerator('uniform', (ishape[1], ishape[0]), 0.3, 2,
1.1), extra_target_bbox,
opts['n_neg_init'] / replicateNum / 4, opts['overlap_neg_init'])
##bbreg sample
cur_extra_bbreg_examples = gen_samples(
SampleGenerator('uniform', (ishape[1], ishape[0]), 0.3, 1.5, 1.1),
extra_target_bbox, opts['n_bbreg'] / replicateNum / 4,
opts['overlap_bbreg'], opts['scale_bbreg'])
batch_num = iidx * np.ones((cur_extra_pos_examples.shape[0], 1))
cur_extra_pos_rois = np.copy(cur_extra_pos_examples)
cur_extra_pos_rois[:, 0:2] -= np.repeat(np.reshape(
extra_scene_box[0:2], (1, 2)),
cur_extra_pos_rois.shape[0],
axis=0)
cur_extra_pos_rois = samples2maskroi(
cur_extra_pos_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size), extra_target_bbox[2:4],
opts['padding'])
cur_extra_pos_rois = np.concatenate((batch_num, cur_extra_pos_rois),
axis=1)
batch_num = iidx * np.ones((cur_extra_neg_examples.shape[0], 1))
cur_extra_neg_rois = np.copy(cur_extra_neg_examples)
cur_extra_neg_rois[:, 0:2] -= np.repeat(np.reshape(
extra_scene_box[0:2], (1, 2)),
cur_extra_neg_rois.shape[0],
axis=0)
cur_extra_neg_rois = samples2maskroi(
cur_extra_neg_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size), extra_target_bbox[2:4],
opts['padding'])
cur_extra_neg_rois = np.concatenate((batch_num, cur_extra_neg_rois),
axis=1)
## bbreg rois
batch_num = iidx * np.ones((cur_extra_bbreg_examples.shape[0], 1))
cur_extra_bbreg_rois = np.copy(cur_extra_bbreg_examples)
cur_extra_bbreg_rois[:,
0:2] -= np.repeat(np.reshape(
extra_scene_box[0:2], (1, 2)),
cur_extra_bbreg_rois.shape[0],
axis=0)
cur_extra_bbreg_rois = samples2maskroi(
cur_extra_bbreg_rois, model.receptive_field,
(scaled_obj_size, scaled_obj_size), extra_target_bbox[2:4],
opts['padding'])
cur_extra_bbreg_rois = np.concatenate(
(batch_num, cur_extra_bbreg_rois), axis=1)
if iidx == 0:
extra_cropped_image = cur_extra_cropped_image
extra_pos_rois = np.copy(cur_extra_pos_rois)
extra_neg_rois = np.copy(cur_extra_neg_rois)
##bbreg rois
extra_bbreg_rois = np.copy(cur_extra_bbreg_rois)
extra_bbreg_examples = np.copy(cur_extra_bbreg_examples)
else:
extra_cropped_image = torch.cat(
(extra_cropped_image, cur_extra_cropped_image), dim=0)
extra_pos_rois = np.concatenate(
(extra_pos_rois, np.copy(cur_extra_pos_rois)), axis=0)
extra_neg_rois = np.concatenate(
(extra_neg_rois, np.copy(cur_extra_neg_rois)), axis=0)
##bbreg rois
extra_bbreg_rois = np.concatenate(
(extra_bbreg_rois, np.copy(cur_extra_bbreg_rois)), axis=0)
extra_bbreg_examples = np.concatenate(
(extra_bbreg_examples, np.copy(cur_extra_bbreg_examples)),
axis=0)
extra_pos_rois = Variable(
torch.from_numpy(extra_pos_rois.astype('float32'))).cuda()
extra_neg_rois = Variable(
torch.from_numpy(extra_neg_rois.astype('float32'))).cuda()
##bbreg rois
extra_bbreg_rois = Variable(
torch.from_numpy(extra_bbreg_rois.astype('float32'))).cuda()
extra_cropped_image -= 128.
extra_feat_maps = model(extra_cropped_image, out_layer='conv3')
# Draw pos/neg samples
ishape = cur_image.shape
extra_pos_feats = model.roi_align_model(extra_feat_maps, extra_pos_rois)
extra_pos_feats = extra_pos_feats.view(extra_pos_feats.size(0),
-1).data.clone()
extra_neg_feats = model.roi_align_model(extra_feat_maps, extra_neg_rois)
extra_neg_feats = extra_neg_feats.view(extra_neg_feats.size(0),
-1).data.clone()
##bbreg feat
extra_bbreg_feats = model.roi_align_model(extra_feat_maps,
extra_bbreg_rois)
extra_bbreg_feats = extra_bbreg_feats.view(extra_bbreg_feats.size(0),
-1).data.clone()
## concatenate extra features to original_features
pos_feats = torch.cat((pos_feats, extra_pos_feats), dim=0)
neg_feats = torch.cat((neg_feats, extra_neg_feats), dim=0)
## concatenate extra bbreg feats to original_bbreg_feats
bbreg_feats = torch.cat((bbreg_feats, extra_bbreg_feats), dim=0)
bbreg_examples = np.concatenate((bbreg_examples, extra_bbreg_examples),
axis=0)
torch.cuda.empty_cache()
model.zero_grad()
# Initial training
train(model, criterion, init_optimizer, pos_feats, neg_feats,
opts['maxiter_init'])
##bbreg train
if bbreg_feats.size(0) > opts['n_bbreg']:
bbreg_idx = np.asarray(range(bbreg_feats.size(0)))
np.random.shuffle(bbreg_idx)
bbreg_feats = bbreg_feats[bbreg_idx[0:opts['n_bbreg']], :]
bbreg_examples = bbreg_examples[bbreg_idx[0:opts['n_bbreg']], :]
bbreg = BBRegressor((ishape[1], ishape[0]))
bbreg.train(bbreg_feats, bbreg_examples, target_bbox)
if pos_feats.size(0) > opts['n_pos_update']:
pos_idx = np.asarray(range(pos_feats.size(0)))
np.random.shuffle(pos_idx)
pos_feats_all = [
pos_feats.index_select(
0,
torch.from_numpy(pos_idx[0:opts['n_pos_update']]).cuda())
]
if neg_feats.size(0) > opts['n_neg_update']:
neg_idx = np.asarray(range(neg_feats.size(0)))
np.random.shuffle(neg_idx)
neg_feats_all = [
neg_feats.index_select(
0,
torch.from_numpy(neg_idx[0:opts['n_neg_update']]).cuda())
]
state['trans_f'] = opts['trans_f']
state['count'] = 1
state['model'] = model
state['target_bbox'] = target_bbox
state['img_crop_model'] = img_crop_model
state['bbreg'] = bbreg
state['criterion'] = criterion
state['update_optimizer'] = update_optimizer
state['pos_feats_all'] = pos_feats_all
state['neg_feats_all'] = neg_feats_all
state['feat_dim'] = feat_dim
return state
def __init__(self, init_bbox, first_frame):
self.frame_idx = 0
self.target_bbox = np.array(init_bbox)
self.bbreg_bbox = self.target_bbox
# Init model
self.model = MDNet(opts['model_path'])
if opts['use_gpu']:
self.model = self.model.cuda()
self.model.set_learnable_params(opts['ft_layers'])
# Init criterion and optimizer
self.criterion = BinaryLoss()
self.init_optimizer = set_optimizer(self.model, opts['lr_init'])
self.update_optimizer = set_optimizer(self.model, opts['lr_update'])
# Train bbox regressor
bbreg_examples = gen_samples(
SampleGenerator('uniform', first_frame.size, 0.3, 1.5,
1.1), self.target_bbox, opts['n_bbreg'],
opts['overlap_bbreg'], opts['scale_bbreg'])
assert len(bbreg_examples) > 0
bbreg_feats = forward_samples(self.model, first_frame, bbreg_examples)
assert len(bbreg_feats) > 0
self.bbreg = BBRegressor(first_frame.size)
self.bbreg.train(bbreg_feats, bbreg_examples, self.target_bbox)
# Draw pos/neg samples
pos_examples = gen_samples(
SampleGenerator('gaussian', first_frame.size, 0.1, 1.2),
self.target_bbox, opts['n_pos_init'], opts['overlap_pos_init'])
neg_examples = np.concatenate([
gen_samples(
SampleGenerator('uniform', first_frame.size, 1, 2,
1.1), self.target_bbox,
opts['n_neg_init'] // 2, opts['overlap_neg_init']),
gen_samples(
SampleGenerator('whole', first_frame.size, 0, 1.2,
1.1), self.target_bbox,
opts['n_neg_init'] // 2, opts['overlap_neg_init'])
])
neg_examples = np.random.permutation(neg_examples)
# Extract pos/neg features
pos_feats = forward_samples(self.model, first_frame, pos_examples)
neg_feats = forward_samples(self.model, first_frame, neg_examples)
self.feat_dim = pos_feats.size(-1)
# Initial training
train(self.model, self.criterion, self.init_optimizer, pos_feats,
neg_feats, opts['maxiter_init'])
# Init sample generators
self.sample_generator = SampleGenerator('gaussian',
first_frame.size,
opts['trans_f'],
opts['scale_f'],
valid=True)
self.pos_generator = SampleGenerator('gaussian', first_frame.size, 0.1,
1.2)
self.neg_generator = SampleGenerator('uniform', first_frame.size, 1.5,
1.2)
# Init pos/neg features for update
self.pos_feats_all = [pos_feats[:opts['n_pos_update']]]
self.neg_feats_all = [neg_feats[:opts['n_neg_update']]]
def __init__(self, pos_data, neg_data, dims, data_obj):
qset, pos_set, neg_set = self.format_data(pos_data, neg_data)
SampleGenerator.__init__(self, qset, pos_set, neg_set, dims, data_obj)
def generate_sample(view):
sample = SampleGenerator(view)
sample.run()
print('foo')
def main():
# Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--data_set", type=str, required=True, \
choices = ["train", "dev", "test"], \
help="Specify if evaluation to be done on train/dev/test")
parser.add_argument("-m", "--model", type=str, required=True, \
help="Model path")
parser.add_argument("-n", "--num_data", type=int, \
help="Number of data points to evaluate")
parser.add_argument("-po", "--print_outputs", type=str, default="1 1", \
help="-po batch_size num_pos outputs")
parser.add_argument("-met", "--metrics", action='store_false', \
help="-met (True/False) to print metrics")
parser.add_argument("-ver_s", "--verify_samples", action='store_false', \
help="-met (True/False) to print metrics")
parser.add_argument("-ver_i", "--verify_input", action='store_false', \
help="-met (True/False) to print metrics")
args = parser.parse_args()
model_path = args.model
num_data = []
if (args.num_data):
num_data = args.num_data
data_set = args.data_set
batch_size, num_pos = args.print_outputs.split()
batch_size = int(batch_size)
num_pos = int(num_pos)
f = open("data_folder/created_data/word_embed.pkl", "rb")
word_embed = pickle.load(f)
f.close()
f = open("data_folder/created_data/question_id.pkl", "rb")
question_id = pickle.load(f)
f.close()
data_obj = DataStore(question_id, word_embed)
n = 120
N = 100
opveclen = 100
wlen = len(word_embed['the'])
dims = n, N, wlen, opveclen
loss_fn = loss_fn_wrap(dims)
model = tf.keras.models.load_model(model_path, \
custom_objects={'loss': loss_fn_wrap(dims)}, compile=False)
model.compile(optimizer='adam', loss=loss_fn)
loaded_data = (word_embed, question_id, model)
data_folder = 'data_folder/data'
train_file = 'train_random.txt'
dev_file = 'dev.txt'
test_file = 'heldout.txt'
dev_path = os.path.join(data_folder, dev_file)
test_path = os.path.join(data_folder, test_file)
train_path = os.path.join(data_folder, train_file)
sample_gen = dict()
train_q, train_pos, train_neg = read_question_data(train_path)
sample_gen["train"] = SampleGenerator(train_q, train_pos, \
train_neg, dims, data_obj)
dev_q, dev_pos, dev_neg = read_question_data(dev_path)
sample_gen["dev"] = SampleGenerator(dev_q, dev_pos, \
dev_neg, dims, data_obj)
test_q, test_pos, test_neg = read_question_data(test_path)
sample_gen["test"] = SampleGenerator(test_q, test_pos, \
test_neg, dims, data_obj)
if (args.metrics):
print(get_metrics(model, sample_gen[data_set], dims, num_data=num_data))
if (args.verify_samples):
verify_samples(sample_gen[data_set], model, dims, batch_size=batch_size, num_pos=num_pos, randomly=False)
if (args.verify_input):
verify_inputs(sample_gen[data_set], dims, batch_ind=1)
if __name__ == '__main__':
if len(sys.argv) != 2:
print('word2vec.py train & sim')
sys.exit(0)
vocab = Vocabulary('../data/vocab.txt')
vocab.load()
word2vec = Word2Vec()
what = sys.argv[1]
if what == 'train':
generator = SampleGenerator('../data/3b_token.txt.final', vocab)
dataset = tf.data.Dataset.from_generator(
generator,
output_types=((tf.int32, tf.int32), tf.float32),
output_shapes=(((), ()), ())
#((tf.int32, tf.int32), tf.int32))
)
batch_size = 1024
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(buffer_size=5 * batch_size)
word2vec.build_model(vocab.get_size(), 200)
word2vec.train(dataset, epoch=1)
word2vec.save('../data/word2vec.h5')
class Tracker:
def __init__(self, init_bbox, first_frame):
self.frame_idx = 0
self.target_bbox = np.array(init_bbox)
self.bbreg_bbox = self.target_bbox
# Init model
self.model = MDNet(opts['model_path'])
if opts['use_gpu']:
self.model = self.model.cuda()
self.model.set_learnable_params(opts['ft_layers'])
# Init criterion and optimizer
self.criterion = BinaryLoss()
self.init_optimizer = set_optimizer(self.model, opts['lr_init'])
self.update_optimizer = set_optimizer(self.model, opts['lr_update'])
# Train bbox regressor
bbreg_examples = gen_samples(
SampleGenerator('uniform', first_frame.size, 0.3, 1.5,
1.1), self.target_bbox, opts['n_bbreg'],
opts['overlap_bbreg'], opts['scale_bbreg'])
assert len(bbreg_examples) > 0
bbreg_feats = forward_samples(self.model, first_frame, bbreg_examples)
assert len(bbreg_feats) > 0
self.bbreg = BBRegressor(first_frame.size)
self.bbreg.train(bbreg_feats, bbreg_examples, self.target_bbox)
# Draw pos/neg samples
pos_examples = gen_samples(
SampleGenerator('gaussian', first_frame.size, 0.1, 1.2),
self.target_bbox, opts['n_pos_init'], opts['overlap_pos_init'])
neg_examples = np.concatenate([
gen_samples(
SampleGenerator('uniform', first_frame.size, 1, 2,
1.1), self.target_bbox,
opts['n_neg_init'] // 2, opts['overlap_neg_init']),
gen_samples(
SampleGenerator('whole', first_frame.size, 0, 1.2,
1.1), self.target_bbox,
opts['n_neg_init'] // 2, opts['overlap_neg_init'])
])
neg_examples = np.random.permutation(neg_examples)
# Extract pos/neg features
pos_feats = forward_samples(self.model, first_frame, pos_examples)
neg_feats = forward_samples(self.model, first_frame, neg_examples)
self.feat_dim = pos_feats.size(-1)
# Initial training
train(self.model, self.criterion, self.init_optimizer, pos_feats,
neg_feats, opts['maxiter_init'])
# Init sample generators
self.sample_generator = SampleGenerator('gaussian',
first_frame.size,
opts['trans_f'],
opts['scale_f'],
valid=True)
self.pos_generator = SampleGenerator('gaussian', first_frame.size, 0.1,
1.2)
self.neg_generator = SampleGenerator('uniform', first_frame.size, 1.5,
1.2)
# Init pos/neg features for update
self.pos_feats_all = [pos_feats[:opts['n_pos_update']]]
self.neg_feats_all = [neg_feats[:opts['n_neg_update']]]
def track(self, image):
self.frame_idx += 1
# Estimate target bbox
samples = gen_samples(self.sample_generator, self.target_bbox,
opts['n_samples'])
sample_scores = forward_samples(self.model,
image,
samples,
out_layer='fc6')
top_scores, top_idx = sample_scores[:, 1].topk(5)
top_idx = top_idx.cpu().numpy()
target_score = top_scores.mean()
success = target_score > opts['success_thr']
# Expand search area at failure
if success:
self.sample_generator.set_trans_f(opts['trans_f'])
else:
self.sample_generator.set_trans_f(opts['trans_f_expand'])
# Save result at success.
if success:
self.target_bbox = samples[top_idx].mean(axis=0)
# Bbox regression
bbreg_samples = samples[top_idx]
bbreg_feats = forward_samples(self.model, image, bbreg_samples)
bbreg_samples = self.bbreg.predict(bbreg_feats, bbreg_samples)
self.bbreg_bbox = bbreg_samples.mean(axis=0)
# Data collect
if success:
# Draw pos/neg samples
pos_examples = gen_samples(self.pos_generator, self.target_bbox,
opts['n_pos_update'],
opts['overlap_pos_update'])
neg_examples = gen_samples(self.neg_generator, self.target_bbox,
opts['n_neg_update'],
opts['overlap_neg_update'])
# Extract pos/neg features
pos_feats = forward_samples(self.model, image, pos_examples)
neg_feats = forward_samples(self.model, image, neg_examples)
self.pos_feats_all.append(pos_feats)
self.neg_feats_all.append(neg_feats)
if len(self.pos_feats_all) > opts['n_frames_long']:
del self.pos_feats_all[0]
if len(self.neg_feats_all) > opts['n_frames_short']:
del self.neg_feats_all[0]
# Short term update
if not success:
nframes = min(opts['n_frames_short'], len(self.pos_feats_all))
pos_data = torch.stack(self.pos_feats_all[-nframes:],
0).view(-1, self.feat_dim)
neg_data = torch.stack(self.neg_feats_all,
0).view(-1, self.feat_dim)
train(self.model, self.criterion, self.update_optimizer, pos_data,
neg_data, opts['maxiter_update'])
# Long term update
elif self.frame_idx % opts['long_interval'] == 0:
pos_data = torch.stack(self.pos_feats_all,
0).view(-1, self.feat_dim)
neg_data = torch.stack(self.neg_feats_all,
0).view(-1, self.feat_dim)
train(self.model, self.criterion, self.update_optimizer, pos_data,
neg_data, opts['maxiter_update'])
return self.bbreg_bbox, target_score