def main():
# ---------- Upload video frames: -----------------------------------------
video_name = "movies/test_video.mp4"
data = DataProvider(video_name)
# ---------- Create set of edges E: ---------------------------------------
print("Create set of edges E ...")
E = create_E(data)
print("Finished.")
# ---------- Compute relative transformation for each edge in E: ----------
print("Compute relative transformations for each edge in E ...")
if os.path.exists('output_file'):
os.remove('output_file')
f = open('output_file', 'w+')
for e in E:
# Compute features correspondence by running SIFT:
p, q, w = get_FC_by_SIFT(data.images[e.src], data.images[e.dst])
# center the points around zero:
img_sz = data.img_sz
p[:, 0] = p[:, 0] - img_sz[1] / 2
q[:, 0] = q[:, 0] - img_sz[1] / 2
p[:, 1] = p[:, 1] - img_sz[0] / 2
q[:, 1] = q[:, 1] - img_sz[0] / 2
# Compute relative transformation (theta):
E[e] = compute_LS_rigid_motion(p, q, w)
# Add this measurement to the output file:
pose = ' '.join([str(p) for p in np.reshape(E[e], (6, ))])
f.write('EDGE_SE2 ' + str(e.src) + ' ' + str(e.dst) + ' ' + pose +
'\n')
print("Finished.")
# ---------- view the relative transformation of a few edges:
imgs = []
titles = []
for i, e in enumerate(E):
if i in (200, 201, 203):
imgs.append(data.images[e.src])
imgs.append(data.images[e.dst])
transformed_I = warp_image(data.images[e.dst], E[e],
cv2.INTER_CUBIC)
imgs.append(transformed_I)
titles.append('I1')
titles.append('I2')
titles.append('theta(I2)')
fig1 = open_figure(1, '', (5, 3))
PlotImages(1, 3, 3, 1, imgs, titles, 'gray', axis=False, colorbar=False)
plt.show()
fig1.savefig('relative_trans_results.png', dpi=1000)
def train():
# Load data (frames) from video - generate embedded frames of size: (img_sz x img_sz x 4)
video_name = "movies/BG.mp4"
data = DataProvider(video_name, img_sz)
# Learn the subspace using Autoencoder
device = '/cpu:0' # '/gpu:0' OR '/cpu:0'
with tf.device(device):
# build the network
vae = VAE(latent_dim, batch_size, img_sz, d_sz)
for ep in range(epoch_num): # epochs loop
for i in range(iterations): # batches loop
# read a batch -> batch_img
batch_img = data.next_batch(batch_size, 'train')
loss, kl, recon_loss = vae.update(batch_img)
if not i % 10:
print('Epoch {0}: Iteration: {1} Loss: {2:.5f} kl: {3:.5f} recon_loss: {4:.5f}'.format((ep + 1), i, loss, kl, recon_loss))
# # test the trained network
batch_img = data.next_batch(batch_size, 'test')
batch_img[4, ...] = generate_outliers(batch_img[4,...],50,80)
batch_img[7, ...] = generate_outliers(batch_img[7,...],40,110)
test_results = vae.test(batch_img)[0]
# # plot the reconstructed images and their ground truths (inputs)
imgs = []
imgs_test = []
titles = []
for i in range(10):
imgs.append(batch_img[i, ...])
imgs_test.append(np.abs(test_results[i, ...]))
titles.append('')
fig1 = open_figure(1, 'Original Images', (7, 3))
PlotImages(1, 2, 5, 1, imgs, titles, 'gray', axis=True, colorbar=False)
fig2 = open_figure(2, 'Test Results', (7, 3))
PlotImages(2, 2, 5, 1, imgs_test, titles, 'gray', axis=True, colorbar=False)
plt.show()
fig1.savefig('f1.png')
fig2.savefig('f2.png')
def main():
# ---------- Upload video frames: -----------------------------------------
crop_size_x = 600
crop_size_y = 400
data = DataProvider(
crop_size_x,
crop_size_y) # take just one photo and create synthetic movement
# ---------- Create set of edges E: ---------------------------------------
print("Create set of edges E ...")
E = create_E(data)
print("Finished.")
# ---------- Compute relative transformation for each edge in E: ----------
print("Compute relative transformations for each edge in E ...")
if os.path.exists('output_file'):
os.remove('output_file')
f = open('output_file', 'w+')
for e in E:
# Add this measurement to the output file:
pose = ' '.join([str(p) for p in np.reshape(E[e], (6, ))])
f.write('EDGE_SE2 ' + str(e.src) + ' ' + str(e.dst) + ' ' + pose +
'\n')
print("Finished.")
# ---------- view the relative transformation of a few edges:
imgs = []
titles = []
for i, e in enumerate(E):
if i in (120, 121, 122):
imgs.append(data.imgs[e.src])
imgs.append(data.imgs[e.dst])
transformed_I = warp_image(data.imgs[e.dst], E[e], cv2.INTER_CUBIC)
#transformed_I = cv2.warpAffine(data.imgs[e.src], E[e], (crop_size_x, crop_size_y), cv2.INTER_LANCZOS4)
imgs.append(transformed_I)
titles.append('I1')
titles.append('I2')
titles.append('theta(I2)')
fig1 = open_figure(1, '', (5, 3))
PlotImages(1, 3, 3, 1, imgs, titles, 'gray', axis=False, colorbar=False)
plt.show()
fig1.savefig('relative_trans_results.png', dpi=1000)
def main():
k, N, data_path = update_params()
""" Build X in shape dxN: Read N frames from video """
X, d, m, n = build_data_matrix(data_path, k, N)
"""Compute PCA"""
X_zeromean, X_mean, V = learn_subspace_by_pca(X, k)
""" Generate outliers (moving object) for one frame and keep the ground truth W """
X_o, W = generate_outliers(X_zeromean, d, m, n)
""" Solve WLS (w=1 for background pixels, w=0 of object pixels) - this is the ground truth alpha"""
alpha_o = solve_wls(W, V, X_o)
# print(alpha_o)
# alpha_o += (100000)*np.random.standard_normal(alpha_o.size).reshape(k,1)
# print(alpha_o)
print("X_norm shape:", X_zeromean.shape, " V shape:", V.shape,
" alpha_o shape:", alpha_o.shape, " W shape:", W.shape)
""" Projection on the subspace """
proj, error = project_on_subspace(X_zeromean[:, :1], V,
V.T @ X_zeromean[:, :1])
proj_o, error_o = project_on_subspace(X_o, V, V.T @ X_o)
proj_robust, error_robust = project_on_subspace(X_o, V, alpha_o)
""" Compute Graph Cut to get W """
b = compute_graphcut(X_o, proj_robust, m, n)
proj += X_mean
proj_o += X_mean
proj_robust += X_mean
X_o += X_mean
X_ = X_zeromean[:, :1] + X_mean
""" Plot projections and errors """
plt_image, plt_proj, plt_error = prepare_projection_to_display(
X_, proj, error, m, n)
plt_image_o, plt_proj_o, plt_error_o = prepare_projection_to_display(
X_o, proj_o, error_o, m, n)
plt_image_robust, plt_proj_robust, plt_error_robust = prepare_projection_to_display(
X_o, proj_robust, error_robust, m, n)
""" Display results """
open_figure(1, 'Projections', (7, 7))
PlotImages(
1,
3,
3,
1, [
plt_image, plt_image_o, plt_image_robust, plt_proj, plt_proj_o,
plt_proj_robust, plt_error, plt_error_o, plt_error_robust
], [
'Direct Proj., X', 'Direct Proj., X_o', 'Robust Proj., X_o',
'Proj.', 'Proj.', 'Proj.', 'Error', 'Error', 'Error'
],
'gray',
axis=True,
colorbar=True,
m=200)
b.source_weight.shape = m, n
b.sink_weight.shape = m, n
open_figure(2, 'Graph Cut', (5, 5))
PlotImages(2,
4,
1,
1, [
b.out, b.source_weight, b.sink_weight,
b.source_weight < b.sink_weight
], ['Graph Cut', '', '', ''],
'gray',
axis=True,
colorbar=True)
plt.show()
def run_session(data,iter_per_epoch,X_train,X_test,n_epochs,batch_size,loss,logits,transformations,b_s,x,y,keep_prob,
optimizer,start_time,digit_to_align,img_sz,num_channels, alignment_loss, trans_regularizer, a, b):
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
#find the indexes needed for splitting the train and test sets into batchs with the desired batch size
#iter_per_epoch,indices,test_iter_per_epoch,test_indices = prepare_splitting_data(X_train,X_test,batch_size)
for epoch_i in range(n_epochs):
for iter_i in range(iter_per_epoch):
batch_xs = data.next_batch(batch_size,'train') #X_train[indices[iter_i]:indices[iter_i + 1]]
# batch_ys = y_train[indices[iter_i]:indices[iter_i + 1]]
# batch_size = batch_ys.size
loss_val, theta_val, alignment_loss_val, trans_regularizer_val, a_val, b_val = sess.run([loss,transformations, alignment_loss, trans_regularizer, a,b],
feed_dict={
b_s:[batch_size],
x:batch_xs,
keep_prob:1.0
})
if iter_i % 20 == 0:
print('Iteration: ' + str(iter_i) + ' Loss: ' + str(loss_val))
print("theta row 1 is: " + str(theta_val[0,:]))
sess.run(optimizer,feed_dict={
b_s:[batch_size],x:batch_xs,keep_prob:1.0})
#Find align_loss on test data
print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nrunning test data...")
align_loss = 0.
for iter_i in range(batch_size):
batch_xs = data.next_batch(batch_size,
'train') #X_test[test_indices[iter_i]:test_indices[iter_i+1]]
loss_val,theta_val,test_imgs = sess.run([loss,transformations,logits],
feed_dict={
b_s:[batch_size],
x:batch_xs,
keep_prob:1.0
})
align_loss += loss_val
align_loss /= batch_size
# plot the reconstructed images and their ground truths (inputs)
imgs = []
imgs_test = []
Ws = []
Ws_test = []
titles = []
for i in range(10):
I = batch_xs[i,...]
I = np.reshape(I,(img_sz,img_sz,num_channels))
imgs.append(I[:,:,0:3])
#tmp = np.reshape(I[:,:,3:],(img_sz,img_sz))
#Ws.append(tmp)
I = test_imgs[i,...]
I = np.reshape(I,(img_sz,img_sz,num_channels))
imgs_test.append(np.abs(I[:,:,0:3]))
#tmp = np.reshape(I[:,:,3:],(img_sz,img_sz))
#Ws_test.append(tmp)
titles.append('')
fig1 = open_figure(1,'Original Images',(7,3))
PlotImages(1,2,5,1,imgs,titles,'gray',axis=True,colorbar=False)
fig2 = open_figure(2,'Test Results',(7,3))
PlotImages(2,2,5,1,imgs_test,titles,'gray',axis=True,colorbar=False)
#fig3 = open_figure(3,'Original W',(7,3))
#PlotImages(3,2,5,1,Ws,titles,'gray',axis=True,colorbar=False)
#fig4 = open_figure(4,'Test W',(7,3))
#PlotImages(4,2,5,1,Ws_test,titles,'gray',axis=True,colorbar=False)
plt.show()
fig1.savefig('f1.png')
fig2.savefig('f2.png')
#fig3.savefig('f3.png')
#fig4.savefig('f4.png')
#print ("theta row 1 is: "+str(theta_val[0,:]))
#print ("theta row 10 is: "+str(theta_val[9,:]))
print('Alignment loss (%d): ' % (epoch_i + 1) + str(align_loss) + "\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
if np.isnan(align_loss):
duration = time.time() - start_time
print("Total runtime is " + "%02d" % (duration) + " seconds.")
raise SystemExit
# if update_transformed_mnist:
# #Run one forward pass again on the training data, in ordr to create a transformed mnist data
# all_training_imgs = prepare_new_mnist(sess,X_train,y_train,iter_per_epoch,indices,batch_ys,batch_xs,loss,logits,
# transformations,b_s,x,y,keep_prob)
#
# #show some of the test data before and after running the model which was learned
# all_test_imgs = None #Find align_loss on test data
# print("\n\nPreparing test images...")
# for iter_i in range(test_iter_per_epoch):
# batch_xs = X_test[test_indices[iter_i]:test_indices[iter_i + 1]]
# batch_ys = y_test[test_indices[iter_i]:test_indices[iter_i + 1]]
# batch_size = batch_ys.size
#
# loss_val,test_imgs = sess.run([loss,logits],
# feed_dict={
# b_s:[batch_size],
# x:batch_xs,
# y:batch_ys,
# keep_prob:1.0
# })
# if all_test_imgs is not None:
# all_test_imgs = np.concatenate((all_test_imgs,test_imgs))
# else:
# all_test_imgs = test_imgs
# show_test_imgs(all_test_imgs,X_test,height,width)
#
# if update_transformed_mnist: #create a new mnist data set with the newly aligned data
# create_mnist_dataset(y_train,y_test,path_to_new_mnist,digit_to_align,all_training_imgs,all_test_imgs)
sess.close()
def main():
# ---------- Upload video frames: -----------------------------------------
video_name = "movies/test_video.mp4"
data = DataProvider(video_name)
# ---------- Read the SE-Sync output: ---------------------------------------
V = create_V(data)
f = open("SE_Sync_output.txt", "r")
for line in f:
tokens = line[:-1].split(' ')
tokens = [float(i) for i in tokens]
v = int(tokens[0])
V[v] = extract_optimal_pose(tokens[1:])
# ---------- Transform the images and create a panorama: --------------------
big_I_sz = ((1000, 2000))
imgs_orig = []
imgs_trans = []
imgs_trans_all = []
titles = []
panoramas = []
T1_inv = invert_T(V[0])
for v in V:
if v in range(0, 9):
img = data.images[v]
# print(V[v])
T = revert_T(V[v], T1_inv)
img_sz = img.shape
# embed the image:
I = np.zeros((big_I_sz[0], big_I_sz[1], 3))
I[big_I_sz[0] // 4:big_I_sz[0] // 4 + img_sz[0],
big_I_sz[1] // 4:big_I_sz[1] // 4 + img_sz[1], :] = img
I = I / 255.
if np.isfinite(np.linalg.cond(T)):
I_Rt = warp_image(I, T, cv2.INTER_CUBIC)
I_Rt = np.abs(I_Rt)
imgs_trans_all.append(I_Rt)
# view the global transformation of a few vertices:
if v in range(0, 9):
print(T)
imgs_orig.append(data.images[v])
imgs_trans.append(I_Rt)
titles.append('')
# build panoramic image:
y = nan_if(imgs_trans_all)
panoramic_img = np.nanmean(y, axis=0)
panoramas.append(panoramic_img)
fig1 = open_figure(1, '', (5, 3))
PlotImages(1,
3,
3,
1,
imgs_orig,
titles,
'gray',
axis=False,
colorbar=False)
fig2 = open_figure(2, '', (5, 3))
PlotImages(2,
3,
3,
1,
imgs_trans,
titles,
'gray',
axis=False,
colorbar=False)
fig3 = open_figure(3, 'Panoramic Image', (5, 5))
PlotImages(3,
1,
1,
1,
panoramas,
titles,
'gray',
axis=True,
colorbar=False)
fig1.savefig('optimal_trans_results_orig.png', dpi=1000)
fig2.savefig('optimal_trans_results.png', dpi=1000)
fig3.savefig('Panorama.png', dpi=1000)
plt.show()
def main():
d, k, N = update_params()
"""Read one frame"""
X, height, width = prepare_image('background.jpeg')
d = height * width
"""Generate outliers (moving object)"""
X_ol = X.copy()
start_idx = np.array([20, 20])
end_idx = np.array([90, 90])
W = np.eye(d)
for i in range(start_idx[0], end_idx[0]):
for j in range(start_idx[1], end_idx[1]):
X_ol[i][j] = -1
idx = np.ravel_multi_index((i, j), X_ol.shape)
W[idx, idx] = 0
X = np.reshape(X, (d, 1))
X_ol = np.reshape(X_ol, (d, 1))
"""Compute PCA (SVD on X)"""
V, S, U = np.linalg.svd(X)
V = V[:, :k]
"""Solve WLS (w=1 for background pixels, w=0 of object pixels)"""
#scipy.linalg.lstsq OR scipy.sparse.linalg.lsqr
WV = W @ V
WX = W @ X_ol
alpha = np.linalg.lstsq(WV, WX, rcond=None)[0]
#print('alpha: ', alpha)
"""Projection on the subspace"""
proj = V @ V.T @ X
error = X - proj
proj_ol = V @ V.T @ X_ol
error_ol = X_ol - proj_ol
proj_wls = V @ alpha
error_wls = X_ol - proj_wls
"""Plot one frame"""
plt_image = np.reshape(X, (height, width))
plt_proj = np.reshape(proj, (height, width))
plt_error = np.reshape(error, (height, width))
plt_image_ol = np.reshape(X_ol, (height, width))
plt_proj_ol = np.reshape(proj_ol, (height, width))
plt_error_ol = np.reshape(error_ol, (height, width))
plt_image_ = np.reshape(X_ol, (height, width))
plt_proj_wls = np.reshape(proj_wls, (height, width))
plt_error_wls = np.reshape(error_wls, (height, width))
open_figure(1, 'Original Image', (7, 7))
PlotImages(
1,
3,
3,
1, [
plt_image, plt_image_ol, plt_image_ol, plt_proj, plt_proj_ol,
plt_proj_wls, plt_error, plt_error_ol, plt_error_wls
], [
'Original Image', 'Image with outliers', 'Image with outliers',
'Projection', 'Projection', 'Projection', 'Error', 'Error', 'Error'
],
'gray',
axis=True,
colorbar=True,
m=300)
def train():
all_images = load_data()
device = '/cpu:0' # '/gpu:0' OR '/cpu:0'
with tf.device(device):
# build the network
net = Network()
input_layer = tf.placeholder('float', [None, 784])
output_layer = net.autoencoder2(input_layer)
# output_true shall have the original image for error calculations
# define our cost function
meansq = tf.reduce_mean(tf.square(output_layer - input_layer))
# define our optimizer
optimizer = tf.train.AdagradOptimizer(learn_rate).minimize(meansq)
# initialising stuff and starting the session
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
batch_size = 100
epoch_num = 10
tot_images = 60000 # total number of images
for ep in range(epoch_num): # epochs loop
for i in range(int(tot_images / batch_size)): # batches loop
# read a batch -> batch_img
batch_img = all_images[i * batch_size:(i + 1) * batch_size]
# print("batch size: ", batch_img.shape)
_, c = sess.run([optimizer, meansq],
feed_dict={input_layer: batch_img})
if not i % 10:
print('Epoch {0}: Iteration: {1} Loss: {2:.5f}'.format(
(ep + 1), i, c))
# test the trained network, with outliers
batch_img = all_images[0:batch_size]
print(batch_img.shape)
# batch_img[0,...] = generate_outliers(batch_img[0,...],4,10)
# batch_img[4,...] = generate_outliers(batch_img[4,...],10,15)
# batch_img[7,...] = generate_outliers(batch_img[7,...],10,25)
test_results = sess.run([output_layer],
feed_dict={input_layer: batch_img})[0]
# test the trained network
imgs = []
imgs_test = []
titles = []
for i in range(10):
I = np.reshape(batch_img[i, ...], (28, 28))
imgs.append(I)
I = np.reshape(test_results[i, ...], (28, 28))
imgs_test.append(I)
titles.append('')
fig1 = open_figure(1, 'Original Images', (7, 3))
PlotImages(1, 2, 5, 1, imgs, titles, 'gray', axis=True, colorbar=False)
fig2 = open_figure(2, 'Test Results', (7, 3))
PlotImages(2,
2,
5,
1,
imgs_test,
titles,
'gray',
axis=True,
colorbar=False)
fig1.savefig('f1.png')
fig2.savefig('f2.png')
def train():
# Load data (frames) from video - generate embedded frames of size: (img_sz x img_sz x 4)
video_name = "movies/BG.mp4"
data = DataProvider(video_name, img_sz)
# STN - align the frames to the right position in the panorama
# align_frames(data)
# Learn the subspace using Autoencoder
device = '/cpu:0' # '/gpu:0' OR '/cpu:0'
with tf.device(device):
# build the network
net = Network(img_sz, d_sz)
# calculate the number of batches per epoch
# batch_per_ep = data.train_size // batch_size
# print('Data size: ', data.train_size, ' Num of epochs: ', epoch_num, ' Batches per epoch: ', batch_per_ep)
ae_inputs = tf.placeholder(
tf.float32,
(batch_size, img_sz, img_sz, 3)) # input to the network
ae_outputs = net.simple1(
ae_inputs) # fusion , simple1 , simple2, vae, fully_connected
# calculate the loss and optimize the network
loss = tf.reduce_mean(
tf.square(ae_outputs -
ae_inputs)) # claculate the mean square error loss
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)
# initialize the network
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for ep in range(epoch_num): # epochs loop
for i in range(iterations): # batches loop
# read a batch -> batch_img
batch_img = data.next_batch(batch_size, 'train')
# print("batch size: ", batch_img.shape)
_, c = sess.run([train_op, loss],
feed_dict={ae_inputs: batch_img})
if not i % 10:
print('Epoch {0}: Iteration: {1} Loss: {2:.5f}'.format(
(ep + 1), i, c))
# test the trained network
batch_img = data.next_batch(batch_size, 'train')
#batch_img[4, ...] = generate_outliers(batch_img[4,...],50,80)
# batch_img[7, ...] = generate_outliers(batch_img[7,...],40,110)
test_results = sess.run([ae_outputs], feed_dict={ae_inputs:
batch_img})[0]
# plot the reconstructed images and their ground truths (inputs)
imgs = []
imgs_test = []
titles = []
for i in range(10):
imgs.append(batch_img[i, ...])
imgs_test.append(np.abs(test_results[i, ...]))
titles.append('')
fig1 = open_figure(1, 'Original Images', (7, 3))
PlotImages(1, 2, 5, 1, imgs, titles, 'gray', axis=True, colorbar=False)
fig2 = open_figure(2, 'Test Results', (7, 3))
PlotImages(2,
2,
5,
1,
imgs_test,
titles,
'gray',
axis=True,
colorbar=False)
plt.show()
fig1.savefig('f1.png')
fig2.savefig('f2.png')
def train():
# read MNIST dataset
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
# calculate the number of batches per epoch
batch_per_ep = mnist.train.num_examples // batch_size
print(batch_per_ep, mnist.train.num_examples)
ae_inputs = tf.placeholder(
tf.float32, (None, 32, 32, 1)) # input to the network (MNIST images)
ae_outputs = autoencoder(ae_inputs) # create the Autoencoder network
# calculate the loss and optimize the network
loss = tf.reduce_mean(
tf.square(ae_outputs -
ae_inputs)) # claculate the mean square error loss
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)
# initialize the network
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for ep in range(epoch_num): # epochs loop
for batch_n in range(1000): # range(batch_per_ep): # batches loop
batch_img, batch_label = mnist.train.next_batch(
batch_size) # read a batch
batch_img = batch_img.reshape(
(-1, 28, 28,
1)) # reshape each sample to an (28, 28) image
batch_img = resize_batch(
batch_img) # reshape the images to (32, 32)
_, c = sess.run([train_op, loss],
feed_dict={ae_inputs: batch_img})
print('Epoch: {} - cost= {:.5f}'.format((ep + 1), c))
# test the trained network, with outliers
batch_img, batch_label = mnist.test.next_batch(50)
batch_img = resize_batch(batch_img)
batch_img[0, ...] = generate_outliers(batch_img[0, ...], 4, 10)
batch_img[4, ...] = generate_outliers(batch_img[4, ...], 10, 15)
batch_img[7, ...] = generate_outliers(batch_img[7, ...], 10, 25)
test_results = sess.run([ae_outputs], feed_dict={ae_inputs:
batch_img})[0]
# test the trained network
imgs = []
imgs_test = []
titles = []
for i in range(30):
imgs.append(batch_img[i, ..., 0])
imgs_test.append(test_results[i, ..., 0])
titles.append('')
fig1 = open_figure(1, 'Original Images', (7, 5))
PlotImages(1,
3,
10,
1,
imgs,
titles,
'gray',
axis=True,
colorbar=False)
fig2 = open_figure(2, 'Test Results', (7, 5))
PlotImages(2,
3,
10,
1,
imgs_test,
titles,
'gray',
axis=True,
colorbar=False)
fig1.savefig('f1.png')
fig2.savefig('f2.png')
def main():
# ---------- Upload video frames: -----------------------------------------
crop_size_x = 600
crop_size_y = 400
big_I_sz = ((1000, 2000))
data = DataProvider(
crop_size_x,
crop_size_y) # take just one photo and create synthetic movement
# ---------- Read the SE-Sync output: ---------------------------------------
V = create_V(data)
f = open("SE_Sync_output.txt", "r")
for line in f:
tokens = line[:-1].split(' ')
tokens = [float(i) for i in tokens]
v = int(tokens[0])
V[v] = extract_optimal_pose(tokens[1:], big_I_sz)
# ---------- Transform the images and create a panorama: --------------------
imgs_orig = []
imgs_trans = []
imgs_trans_all = []
titles = []
panoramas = []
T1_inv = invert_T(V[0])
for v in V:
img = data.imgs[v]
T = revert_T(V[v], T1_inv)
img_sz = img.shape
# embed the image:
I = np.zeros((big_I_sz[0], big_I_sz[1], 3))
I[big_I_sz[0] // 4:big_I_sz[0] // 4 + img_sz[0],
big_I_sz[1] // 4:big_I_sz[1] // 4 + img_sz[1], :] = img
I = I / 255.
I_Rt = warp_image(I, T, cv2.INTER_CUBIC)
I_Rt = np.abs(I_Rt)
imgs_trans_all.append(I_Rt)
# view the global transformation of a few vertices:
if v in (0, 5, 10, 15, 20, 25, 120, 123):
print(T)
imgs_orig.append(data.imgs[v])
imgs_trans.append(I_Rt)
titles.append('')
# build panoramic image:
panoramic_img = np.nanmean(nan_if(imgs_trans), axis=0)
panoramas.append(panoramic_img)
fig1 = open_figure(1, '', (5, 3))
PlotImages(1,
3,
3,
1,
imgs_orig,
titles,
'gray',
axis=False,
colorbar=False)
fig2 = open_figure(2, '', (5, 3))
PlotImages(2,
3,
3,
1,
imgs_trans,
titles,
'gray',
axis=False,
colorbar=False)
fig3 = open_figure(3, 'Panoramic Image', (5, 5))
PlotImages(3,
1,
1,
1,
panoramas,
titles,
'gray',
axis=True,
colorbar=False)
fig1.savefig('optimal_trans_results_orig.png', dpi=1000)
fig2.savefig('optimal_trans_results.png', dpi=1000)
fig3.savefig('Panorama.png', dpi=1000)
plt.show()
def main():
d, k, N = update_params()
"""Read one frame"""
X, m, n = prepare_image('background.jpeg')
d = m * n
"""Generate outliers (moving object)"""
X_ol = X.copy()
start_idx = np.array([20, 20])
end_idx = np.array([90, 90])
W = np.eye(d)
for i in range(start_idx[0], end_idx[0]):
for j in range(start_idx[1], end_idx[1]):
X_ol[i][j] = -1
idx = np.ravel_multi_index((i, j), X_ol.shape)
W[idx, idx] = 0
X = np.reshape(X, (d, 1))
X_ol = np.reshape(X_ol, (d, 1))
"""Compute PCA (SVD on X)"""
V, S, U = np.linalg.svd(X)
V = V[:, :k]
print(V.size)
"""Solve WLS (w=1 for background pixels, w=0 of object pixels)"""
WV = W @ V
WX = W @ X_ol
alpha = np.linalg.lstsq(WV, WX, rcond=None)[0]
print(alpha)
alpha += (4164) * np.random.standard_normal(alpha.size)
print(alpha)
"""Projection on the subspace"""
proj = V @ V.T @ X
error = X - proj
proj_ol = V @ V.T @ X_ol
error_ol = X_ol - proj_ol
proj_wls = V @ alpha
error_wls = X_ol - proj_wls
"""Plot one frame"""
plt_image = np.reshape(X, (m, n))
plt_proj = np.reshape(proj, (m, n))
plt_error = np.reshape(error, (m, n))
plt_image_ol = np.reshape(X_ol, (m, n))
plt_proj_ol = np.reshape(proj_ol, (m, n))
plt_error_ol = np.reshape(error_ol, (m, n))
plt_image_ = np.reshape(X_ol, (m, n))
plt_proj_wls = np.reshape(proj_wls, (m, n))
plt_error_wls = np.reshape(error_wls, (m, n))
""" Compute weights by Graph-Cut """
# Create the graph
g = maxflow.Graph[int](m, n)
nodes = g.add_nodes(m * n)
sigma1 = 0.4
sigma2 = 6
# go through all nodes and add edges
for i in range(m * n):
# add edge source->pixels and pixels->sink:
source_weight = ((1. / (sigma1)**2) * (X_ol[i] - proj_wls[i])**2
) # X_ol[i] # BG probability
sink_weight = (X_ol[i]**2) / (sigma2**2
) # 255 - X_ol[i] # FG probability
g.add_tedge(i, source_weight, sink_weight)
if (i == 0 or i == 6900):
print("sink/src links: ", i, source_weight, sink_weight,
X_ol[i] - proj_wls[i])
# add edges between pixels neighbors
if i % n != 0: # left exists
edge_wt = 2
g.add_edge(i, i - 1, edge_wt, edge_wt)
if (i + 1) % n != 0: # right exists
edge_wt = 2
g.add_edge(i, i + 1, edge_wt, edge_wt)
if i // n != 0: # up exists
edge_wt = 2
g.add_edge(i, i - n, edge_wt, edge_wt)
if i // n != m - 1: # down exists
edge_wt = 2
g.add_edge(i, i + n, edge_wt, edge_wt)
# compute min-cut / max-a-posterior
flow = g.maxflow()
sgm = g.get_grid_segments(nodes) # Get the segments.
sgm_ = np.reshape(sgm, (m, n))
print(sgm_)
print("flow: ", flow)
print('\n0', sgm[0])
print('6900', sgm[6900])
out = np.ones((m, n))
for i in range(m):
for j in range(n): # converting the True/False to Pixel intensity
if sgm_[i, j]:
out[i, j] = -1 # background
else:
out[i, j] = 1 # foreground
open_figure(1, 'Original Image', (7, 7))
PlotImages(
1,
4,
3,
1, [
plt_image, plt_image_ol, plt_image_ol, plt_proj, plt_proj_ol,
plt_proj_wls, plt_error, plt_error_ol, plt_error_wls
], [
'Original Image', 'Image with outliers', 'Image with outliers',
'Projection', 'Projection', 'Projection', 'Error', 'Error', 'Error'
],
'gray',
axis=True,
colorbar=True,
m=300)
open_figure(2, 'Graph Cut', (5, 5))
PlotImages(2,
1,
1,
1, [out], ['Graph Cut'],
'gray',
axis=True,
colorbar=True)
plt.show()