def grad_check_tf_wb(num_samples, h_sizes):
tf.enable_eager_execution()
(x_set, y_set), _ = gen_samples(num_samples)
layer_sizes = gen_layer_sizes(len(h_sizes)+1, 1, 1, h_sizes)
mus_W, rhos_W, mus_b, rhos_b = init_distributions(layer_sizes)
eps_W, eps_b = get_eps(layer_sizes)
W, b, sigmas_W, sigmas_b = get_params(mus_W, rhos_W, mus_b, rhos_b, eps_W, eps_b)
w0 = tf.get_variable('w0', layer_sizes[0][0], initializer=tf.constant_initializer(W[0]), dtype=tf.float64)
b0 = tf.get_variable('b0', layer_sizes[0][1], initializer=tf.constant_initializer(b[0]), dtype=tf.float64)
w1 = tf.get_variable('w1', layer_sizes[1][0], initializer=tf.constant_initializer(W[1]), dtype=tf.float64)
b1 = tf.get_variable('b1', layer_sizes[1][1], initializer=tf.constant_initializer(b[1]), dtype=tf.float64)
x = tf.constant(x_set[0].reshape(-1,1), dtype=tf.float64)
y = tf.constant(y_set[0].reshape(-1,1), dtype=tf.float64)
with tf.GradientTape() as tape:
z0 = tf.matmul(x,w0)+b0
a0 = tf.nn.relu(z0)
y_hat = tf.matmul(a0,w1)+b1
loss = -tf_calc_log_likelihood(y, y_hat)
gw0, gb0, gw1, gb1 = tape.gradient(loss,[w0,b0,w1,b1])
x = x.numpy()
y = y.numpy()
z0 = [z0.numpy()]
a0 = [a0.numpy()]
y_hat = y_hat.numpy()
gW, gb = W_grad_log_likelihood(W, b, z0, a0, y, y_hat, x, num_samples)
print "grad_check w_0:", np.linalg.norm(gW[0]-gw0)
print "grad_check w_1:", np.linalg.norm(gW[1]-gw1)
print "grad_check b_0:", np.linalg.norm(gb[0]-gb0)
print "grad_check b_1:", np.linalg.norm(gb[1]-gb1)
train_cm = np.zeros((2, 2))
test_cm = np.zeros((2, 2))
for fold, data, path_info in cross_validation(vars(FLAGS)):
model = load_model('Models/%d/%05d.h5' %
(FLAGS.sherpa_trial, fold + 1),
custom_objects={'auc': auc})
x_train, x_test, y_train, y_test = data
train, test = path_info
train['split'] = 'train'
test['split'] = 'test'
x_train_samples, x_test_samples, = gen_samples(args, x_train, x_test,
y_train, y_test)
test_probabilities = model.predict(x_test_samples)
train_probabilities = model.predict(x_train_samples)
predictions = np.argmax(train_probabilities, axis=1)
targets = np.argmax(y_train, axis=1)
train_cm += confusion_matrix(targets, predictions)
train['prediction'] = predictions
train['target'] = targets
train['prob'] = train_probabilities[:, 1]
predictions = np.argmax(test_probabilities, axis=1)
targets = np.argmax(y_test, axis=1)
test_cm += confusion_matrix(targets, predictions)
test['prediction'] = predictions
test['target'] = targets
def mdnet_run(sess, model, region, images, config, display):
targetLoc = region
n_frames = len(images)
img = proc.load_image(images[0])
tres = open('tres.txt', 'w')
tres.close()
plt.ion()
######################### bbox regressor #########################
if config.bbreg:
pos_examples = utils.gen_samples('uniform_aspect', targetLoc, config.bbreg_n_samples*10, 0.3, 10, config)
r = overlap_ratio(pos_examples, targetLoc)
pos_examples = pos_examples[r>0.6]
pos_examples = pos_examples[np.random.choice(pos_examples.shape[0], min(pos_examples.shape[0], config.bbreg_n_samples))]
if pos_examples.shape[0] < config.bbreg_n_samples:
pos_examples = pos_examples[:pos_examples[0] // config.batch_size * config.batch_size]
# evaluate candidates
feats = np.array([])
for i in range(pos_examples.shape[0] // config.batch_size):
sample = pos_examples[config.batch_size * i: config.batch_size * (i+1)]
sample_im = proc.load_box(img, sample, img_input=True)
feat = sess.run(model.layers['conv3'], feed_dict={model.layers['input']:sample_im})
feat = feat.reshape(config.batch_size, -1)
if feats.size == 0:
feats = feat
else:
feats = np.r_[feats, feat]
print('bbox regression features extracted')
bbox_reg = train_bbox_regressor(feats, pos_examples, targetLoc)
################## finetune on the first frame ###################
print('Finetune on the first frame...')
# generate positive examples
pos_examples = utils.gen_samples('gaussian', targetLoc, config.n_pos_init*2, 0.1, 0.5, config)
r = overlap_ratio(pos_examples, targetLoc)
pos_examples = pos_examples[r>config.pos_thr_init]
pos_examples = pos_examples[np.random.choice(pos_examples.shape[0], min(pos_examples.shape[0], config.n_pos_init))]
neg_examples = np.r_[utils.gen_samples('uniform', targetLoc, config.n_neg_init, 1, 10, config), \
utils.gen_samples('whole', targetLoc, config.n_neg_init, 0, 0, config)]
r = overlap_ratio(neg_examples, targetLoc)
neg_examples = neg_examples[r<config.neg_thr_init]
neg_examples = neg_examples[np.random.choice(neg_examples.shape[0], min(neg_examples.shape[0], config.n_neg_init))]
# prepare patches
pos_data = proc.load_box(img, pos_examples, img_input=True)
neg_data = proc.load_box(img, neg_examples, img_input=True)
config.maxiter = config.maxiter_init
config.lr_rate = config.lr_rate_init
finetune.finetune(sess, model, pos_data, neg_data, config)
############# Prepare training data for online update ##############
print('Preparing online updating data...')
neg_examples = utils.gen_samples('uniform', targetLoc, config.n_neg_update*2, 2, 5, config)
r = overlap_ratio(neg_examples, targetLoc)
neg_examples = neg_examples[r<config.neg_thr_init]
neg_examples = neg_examples[np.random.choice(neg_examples.shape[0], min(neg_examples.shape[0], config.n_neg_update))]
total_pos_data = []
total_neg_data = []
total_pos_data.append(proc.load_box(img, pos_examples, img_input=True))
total_neg_data.append(proc.load_box(img, neg_examples, img_input=True))
############################ tracking ##############################
success_frames = np.array([0]).astype(np.int)
result = np.array([targetLoc]).reshape(-1, 4)
trans_f = config.trans_f
scale_f = config.scale_f
for To in range(1,len(images)):
print(targetLoc)
t = time.time()
print('Processing frame %d/%d...'%(To+1,n_frames))
img = proc.load_image(images[To])
## estimation
# draw target candidates
samples = utils.gen_samples('gaussian', targetLoc, config.n_samples, trans_f, scale_f, config)
# evaluate candidates
remain = config.n_samples
scores = np.array([])
feats = np.array([])
while(remain>0):
sample = samples[scores.shape[0]:scores.shape[0]+config.batch_size]
sample_im = proc.load_box(img, sample, img_input=True)
score, feat = sess.run([model.layers['fc6'], model.layers['conv3']],
feed_dict={model.layers['input']:sample_im})
score = score[:, 0, 0, 0]
scores = np.r_[scores, score]
feat = feat.reshape(config.batch_size, -1)
if feats.size == 0:
feats = feat
else:
feats = np.r_[feats, feat]
remain = config.n_samples - scores.shape[0]
# sort the bboxes
inds = np.argsort(scores)[::-1]
# generate prediction
target_score = np.mean(scores[inds[:5]])
targetLoc = np.round(np.mean(samples[inds[:5]], axis=0))
result = np.r_[result, targetLoc.reshape(1, -1)]
# extend search space in case of failure
if target_score < 0:
trans_f = min(1.5, 1.1*trans_f)
else:
trans_f = config.trans_f
# bbox regression
if config.bbreg and target_score > 0:
X_ = feats[inds[:5]]
bbox_ = samples[inds[:5]]
pred_bboxes = predict_bbox_regressor(bbox_reg.model, X_, bbox_)
targetLoc = np.round(np.mean(pred_bboxes, axis=0))
result[-1] = targetLoc.reshape(1, -1)
print(targetLoc)
if display:
im = Image.open(images[To])
print(images[To])
fig, ax = plt.subplots(1)
ax.imshow(im)
for i in range(10):
rect = patches.Rectangle(samples[i, :2], samples[i, 2], samples[i, 3],linewidth=0.5,edgecolor='b',facecolor='none')
ax.add_patch(rect)
rect = patches.Rectangle(targetLoc[:2], targetLoc[2], targetLoc[3],linewidth=1,edgecolor='r',facecolor='none')
ax.add_patch(rect)
plt.imshow(im)
fig.savefig(os.path.join('res', os.path.basename(images[To])))
plt.close(fig)
# prepare training data
print(target_score)
if target_score > config.update_thr:
pos_examples = utils.gen_samples('gaussian', targetLoc, config.n_pos_init*2, 0.1, 0.5, config)
r = overlap_ratio(pos_examples, targetLoc)
pos_examples = pos_examples[r>config.pos_thr_update]
pos_examples = pos_examples[np.random.choice(pos_examples.shape[0], min(pos_examples.shape[0], config.n_pos_update))]
neg_examples = utils.gen_samples('uniform', targetLoc, config.n_neg_update*2, 2, 5, config)
r = overlap_ratio(neg_examples, targetLoc)
neg_examples = neg_examples[r<config.neg_thr_update]
neg_examples = neg_examples[np.random.choice(neg_examples.shape[0], min(neg_examples.shape[0], config.n_neg_update))]
total_pos_data.append(proc.load_box(img, pos_examples, img_input=True))
total_neg_data.append(proc.load_box(img, neg_examples, img_input=True))
success_frames = np.r_[success_frames, To]
if success_frames.shape[0] > config.n_frames_long:
tmp = total_pos_data[success_frames[-config.n_frames_long-1]]
total_pos_data[success_frames[-config.n_frames_long-1]] = np.array([])
del tmp
#if success_frames.shape[0] > config.n_frames_short:
# tmp = total_neg_data[success_frames[-config.n_frames_short-1]]
# total_neg_data[success_frames[-config.n_frames_short-1]] = np.array([])
# del tmp
else:
total_pos_data.append(np.array([]).reshape(-1,4))
total_neg_data.append(np.array([]).reshape(-1,4))
# network update
if ((To+1) % config.update_interval == 0 or target_score <= config.update_thr) and To != n_frames-1:
print('##################### finetuning #######################')
if target_score < config.update_thr: # short-term update
pos_inds = success_frames[max(0,success_frames.shape[0]-config.n_frames_short):]
else: # long-term update
pos_inds = success_frames[max(0,success_frames.shape[0]-config.n_frames_long):]
neg_inds = success_frames[max(0,success_frames.shape[0]-config.n_frames_short):]
pos_data = np.concatenate([total_pos_data[pos_ind] for pos_ind in pos_inds])
neg_data = np.concatenate([total_neg_data[neg_ind] for neg_ind in neg_inds])
config.maxiter = config.maxiter_update
config.lr_rate = config.lr_rate_update
finetune.finetune(sess, model, pos_data, neg_data, config)
elapsed = time.time() - t
print("Elapsed Time: ", elapsed)
with open('tres.txt', 'a') as tres:
tres.write("Elapsed Time: "+str(elapsed))
tres.write('\n')
with open('res.txt', 'w') as f:
for i in range(result.shape[0]):
f.write(','.join([str(num) for num in result[i]]))
f.write('\n')
def grad_check_tf(num_samples, h_sizes):
tf.enable_eager_execution()
(x_set, y_set), prior_std = gen_samples(num_samples)
layer_sizes = gen_layer_sizes(len(h_sizes)+1, 1, 1, h_sizes)
mus_W, rhos_W, mus_b, rhos_b = init_distributions(layer_sizes)
eps_W, eps_b = get_eps(layer_sizes)
W, b, sigmas_W, sigmas_b = get_params(mus_W, rhos_W, mus_b, rhos_b, eps_W, eps_b)
mu_W_0 = tf.get_variable('mu_W_0', layer_sizes[0][0], initializer=tf.constant_initializer(mus_W[0]) , dtype=tf.float64)
rho_W_0 = tf.get_variable('rho_W_0', layer_sizes[0][0], initializer=tf.constant_initializer(rhos_W[0]), dtype=tf.float64)
mu_b_0 = tf.get_variable('mu_b_0', layer_sizes[0][1], initializer=tf.constant_initializer(mus_b[0]) , dtype=tf.float64)
rho_b_0 = tf.get_variable('rho_b_0', layer_sizes[0][1], initializer=tf.constant_initializer(rhos_b[0]), dtype=tf.float64)
mu_W_1 = tf.get_variable('mu_W_1', layer_sizes[1][0], initializer=tf.constant_initializer(mus_W[1]) , dtype=tf.float64)
rho_W_1 = tf.get_variable('rho_W_1', layer_sizes[1][0], initializer=tf.constant_initializer(rhos_W[1]), dtype=tf.float64)
mu_b_1 = tf.get_variable('mu_b_1', layer_sizes[1][1], initializer=tf.constant_initializer(mus_b[1]) , dtype=tf.float64)
rho_b_1 = tf.get_variable('rho_b_1', layer_sizes[1][1], initializer=tf.constant_initializer(rhos_b[1]), dtype=tf.float64)
eps_w0 = tf.constant(eps_W[0], dtype=tf.float64)
eps_b0 = tf.constant(eps_b[0], dtype=tf.float64)
eps_w1 = tf.constant(eps_W[1], dtype=tf.float64)
eps_b1 = tf.constant(eps_b[1], dtype=tf.float64)
x = tf.constant(x_set[0].reshape(-1,1), dtype=tf.float64)
y = tf.constant(y_set[0].reshape(-1,1), dtype=tf.float64)
with tf.GradientTape() as tape:
sigma_w0 = tf.nn.softplus(rho_W_0)
sigma_b0 = tf.nn.softplus(rho_b_0)
sigma_w1 = tf.nn.softplus(rho_W_1)
sigma_b1 = tf.nn.softplus(rho_b_1)
w0 = mu_W_0 + sigma_w0*eps_w0
b0 = mu_b_0 + sigma_b0*eps_b0
w1 = mu_W_1 + sigma_w1*eps_w1
b1 = mu_b_1 + sigma_b1*eps_b1
z0 = tf.matmul(x,w0)+b0
a0 = tf.nn.relu(z0)
y_hat = tf.matmul(a0,w1)+b1
tf_loss = tf_variational_free_energy(w0,b0,w1,b1,sigma_w0,sigma_b0,sigma_w1,sigma_b1,eps_w0,eps_b0,eps_w1,eps_b1, y, y_hat, num_samples,prior_std)
g_mu_W_0, g_rho_W_0, g_mu_b_0, g_rho_b_0, g_mu_W_1, g_rho_W_1, g_mu_b_1, g_rho_b_1 = tape.gradient(tf_loss,[mu_W_0,rho_W_0,mu_b_0,rho_b_0,mu_W_1,rho_W_1,mu_b_1,rho_b_1])
x = x.numpy()
y = y.numpy()
z0 = [z0.numpy()]
a0 = [a0.numpy()]
y_hat = y_hat.numpy()
loss = variational_free_energy(W, b, sigmas_W, sigmas_b, eps_W, eps_b, y, y_hat, num_samples,prior_std)
print "Loss check:", np.linalg.norm(loss-tf_loss)
gW, gb = W_grad_log_likelihood(W, b, z0, a0, y, y_hat, x, num_samples)
g_mus_W = mu_grad(gW, W, num_samples, prior_std)
g_rhos_W = rho_grad(gW, W, rhos_W, sigmas_W, eps_W, num_samples, prior_std)
g_mus_b = mu_grad(gb, b, num_samples, prior_std)
g_rhos_b = rho_grad(gb, b, rhos_b, sigmas_b, eps_b, num_samples, prior_std)
print "grad_check mu_w_0", np.linalg.norm(g_mus_W[0]-g_mu_W_0)
print "grad_check rho_w_0", np.linalg.norm(g_rhos_W[0]-g_rho_W_0)
print "grad_check mu_b_0", np.linalg.norm(g_mus_b[0]-g_mu_b_0)
print "grad_check rho_b_0", np.linalg.norm(g_rhos_b[0]-g_rho_b_0)
print "grad_check mu_w_1", np.linalg.norm(g_mus_W[1]-g_mu_W_1)
print "grad_check rho_w_1", np.linalg.norm(g_rhos_W[1]-g_rho_W_1)
print "grad_check mu_b_1", np.linalg.norm(g_mus_b[1]-g_mu_b_1)
print "grad_check rho_b_1", np.linalg.norm(g_rhos_b[1]-g_rho_b_1)
val_fid = pfw.fid(fake_images=fake, real_images=real_cpu)
j += 1
print(
"[%d/%d][%d/%d]\tLoss_D: %.4f\tLoss_G: %.4f\tD(x): %.4f\tD(G(z)): %.4f / %.4f FID: %.4f"
% (epoch, num_epochs, i, len(dataloader), errD.item(), errG.item(),
D_x, D_G_z1, D_G_z2, val_fid))
# Save Losses for plotting later
epoch_G_loss.append(errG.item())
epoch_D_loss.append(errD.item())
epoch_FID_log.append(val_fid)
if (iters % 20 == 0) or ((epoch == num_epochs - 1) and
(i == len(dataloader) - 1)):
gen_samples(netG,
latent_dim=nz,
run_name=RUN_NAME,
batch_count=iters)
iters += 1
G_losses.append(sum(epoch_G_loss) / len(epoch_G_loss))
D_losses.append(sum(epoch_D_loss) / len(epoch_D_loss))
FID_log.append(sum(epoch_FID_log) / len(epoch_FID_log))
plt.plot(np.linspace(1, num_epochs, num_epochs).astype(int), G_losses)
plt.savefig("./plots/nathan_G")
plt.plot(np.linspace(1, num_epochs, num_epochs).astype(int), D_losses)
plt.savefig("./plots/nathan_D")
plt.plot(np.linspace(1, num_epochs, num_epochs).astype(int), FID_log)
plt.savefig("./plots/nathan_FID")
def main():
N = 100
n = 100
mu = 0.0
sigma_sq = 0.1
S = utils.gen_samples(N, n, mu, sigma_sq)
ls = [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0]
predictions = {x: [] for x in ls}
for i in range(S.shape[0]):
x = S[i][:, 0]
y = S[i][:, 1]
ex = utils.expand_features(x, 3)
ex = np.append(np.ones(shape=(ex.shape[0], 1)), ex, axis=1)
for l in ls:
rr = RR.RidgeRegression()
rr.fit(ex, y, l)
y_hat = rr.predict(ex)
predictions[l].append(y_hat)
#return predictions
Y = S[:, :, 1].ravel()
E_Y = Y.mean()
bias = get_bias(predictions, Y, n, N)
vars = get_variance(predictions, S, n, N)
"""
bias = {l:0.0 for l in ls}
vars = {l:0.0 for l in ls}
for l in predictions.keys():
preds = predictions[l]
for i, sample_prediction in enumerate(preds):
ED_HD = sum(sample_prediction)/n
#b = (ED_HD - E_Y)**2
v = 0.0
b = 0.0
for j, hx in enumerate(sample_prediction):
v += pow(ED_HD - hx, 2)
b += pow(ED_HD - 2*S[i, j, 0], 2)
vars[l] += v/n
bias[l] += b/n
vars = {k:v/N for k, v in vars.items()}
bias = {k:b/N for k, b in bias.items()}
"""
print "\nRidge Regression"
print "{:^20}|{:^20}|{:^20}".format("lambda", "Var[y]", "Bias2")
print "-" * 60
for l in ls:
print "{:^20}|{:^20}|{:^20}".format(l, vars[l], bias[l])
#import matplotlib.pyplot as plt
#plt.plot(ls, [vars[l] for l in ls], "--")
#plt.plot(ls, [bias[l] for l in ls])
#plt.show()
#plt.plot(ls, [bias[l] for l in ls])
#plt.show()
"""
# result will not be plot if the regressor is active
# if result is needed along with regressor results:
# it should be plot explicitly
conv_net, fc_net, net_body= initialize_model()
#####################################
# Train Bounding Box Regressor
#####################################
if regress:
# create BBoxRegressor object
bbox_reg = BBoxRegressor()
# extract rectangles for training
pos_examples = gen_samples('uniform', targetLoc, opts.bbreg_nSamples*10, imgSize, 0.3, 1.5, 1.1)
scores = iou_score(pos_examples, targetLoc)
a = targetLoc[2]*targetLoc[3] # ground truth area
ar= np.prod(pos_examples[:, 2:], axis = 1)/a # areas ratio to gt
pos_examples = pos_examples[(scores>0.6) & (ar>0.6) & (ar<1)]
s = min(opts.bbreg_nSamples, len(pos_examples))
idx = np.random.choice(np.arange(len(pos_examples)), size = s, replace = False)
pos_examples = pos_examples[idx]
# extract conv3 features
n = len(pos_examples)
regions= extract_regions(img, pos_examples, opts.input_size) #shape (n, 107, 107, 3)
regions = regions - 128.
nBatches = np.ceil(n/opts.batchSize_test)
feat = np.zeros((n, 4608))
for i in range(int(nBatches)):