def decompose(It, Vt_O, Vt_B, I_O_init, I_B_init, A_init):
tf.reset_default_graph()
It = tf.constant(It, tf.float32)
Vt_O = tf.constant(Vt_O, tf.float32)
Vt_B = tf.constant(Vt_B, tf.float32)
I_O = tf.Variable(I_O_init, name='I_O', dtype=tf.float32)
I_B = tf.Variable(I_B_init, name='I_B', dtype=tf.float32)
warp_I_O = tf_warp(tf.tile(tf.expand_dims(I_O, 0), [5, 1, 1, 1]), Vt_O)
warp_I_B = tf_warp(tf.tile(tf.expand_dims(I_B, 0), [5, 1, 1, 1]), Vt_B)
g_O = spatial_gradient(tf.expand_dims(I_O, 0))
g_B = spatial_gradient(tf.expand_dims(I_B, 0))
A = None
if A_init is not None:
A = tf.Variable(A_init, name='A', dtype=tf.float32)
warp_A = tf_warp(tf.tile(tf.expand_dims(A, 0), [5, 1, 1, 1]), Vt_O)
residual = l1_norm(It - warp_I_O - tf.multiply(warp_A, warp_I_B))
else:
residual = l1_norm(It - warp_I_O - warp_I_B)
loss = l1_norm(residual)
if A is not None:
loss += LAMBDA_1 * \
tf.norm(spatial_gradient(tf.expand_dims(A, 0)), ord=2)**2
loss += LAMBDA_2 * (l1_norm(g_O) +
l1_norm(g_B))
loss += LAMBDA_3 * tf.norm(g_O*g_O*g_B*g_B, ord=2)**2
loss += constraint_penalty(I_O) + \
constraint_penalty(I_B)
if A is not None:
loss += constraint_penalty(A)
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
train = optimizer.minimize(loss)
with tf.Session() as session:
session.run(tf.initialize_all_variables())
for step in range(1000):
_, loss_val = session.run([train, loss])
print("step {}:loss = {}".format(step, loss_val))
if A is not None:
I_O, I_B, A = session.run([I_O, I_B, A])
else:
I_O, I_B = session.run([I_O, I_B])
visualize_image(I_O, 'obstruction')
visualize_image(I_B, 'background')
if A is not None:
visualize_image(A, 'alpha')
return I_O, I_B, A
def optimize_motion_based_decomposition(It, I_O_init, I_B_init, A_init, Vt_O_init, Vt_B_init, params):
"""
Optimize motion based decomposition problem.
Args:
It[list(Image)]: input image
I_O_init[Image]: init of image of obstruction
I_B_init[Image]: init of image of background
A_init[Image]: init of image of occlusion mask, None means it is for reflection
Vt_O_init[Image]: init of dense motion field of obstruction
Vt_B_init[Image]: init of dense motion field of background
params[OptimizationParams]: params for the optimization
"""
original_shape = It.shape[1:3]
previous_shape = original_shape
# initialize all values
I_O = I_O_init
I_B = I_B_init
A = A_init
Vt_O = Vt_O_init
Vt_B = Vt_B_init
for current_scale, num_iterations in zip(params.scales, params.num_iterations_by_scale):
# Scale values to proper scale.
current_shape = (int(current_scale * original_shape[0]), int(current_scale * original_shape[1]))
It_scaled = scale_images(
It, from_shape=original_shape, to_shape=current_shape)
I_O = scale_image(I_O, from_shape=previous_shape,
to_shape=current_shape)
I_B = scale_image(I_B, from_shape=previous_shape,
to_shape=current_shape)
if A is not None:
A = scale_image(A, from_shape=previous_shape, to_shape=current_shape)
Vt_O = scale_images(Vt_O, from_shape=previous_shape,
to_shape=current_shape)
Vt_B = scale_images(Vt_B, from_shape=previous_shape,
to_shape=current_shape)
visualize_image(I_O, 'obstruction_init')
visualize_image(I_B, 'background_init')
if A is not None:
visualize_image(A, 'alpha_init')
for _ in range(num_iterations):
Vt_O, Vt_B = estimate_motion(It_scaled, I_O, I_B, A, Vt_O, Vt_B)
I_O, I_B, A = decompose(
It_scaled, Vt_O, Vt_B, I_O, I_B, A)
previous_shape = current_shape
# TODO: check return value
return
def run_fusionseg():
image_list = fnmatch.filter(os.listdir(image_dir),'*.'+ext)
image_list.sort()
for img_name in image_list:
im_prefix = img_name.split('.')[0]
im_path = os.path.join(image_dir,img_name)
labels,fg_prob,raw_probs,img_h,img_w = segment_image(im_path)
labels_file = os.path.join(results_dir,im_prefix+'_mask.npy')
probs_file = os.path.join(results_dir,im_prefix+'_probs.npy')
raw_probs_file = os.path.join(results_dir,im_prefix+'_probs_raw.npy')
np.save(labels_file,labels)
np.save(probs_file,fg_prob)
np.save(raw_probs_file,raw_probs)
if visualize==True:
viz_img = visualize_image(im_path,labels,fg_prob,cmap_pascal,None)
viz_file = os.path.join(viz_dir,im_prefix+'_viz.png')
imsave(viz_file,viz_img);
def run_fusionseg():
for extension in ['jpg', 'jpeg', 'bmp', 'png']:
image_list = fnmatch.filter(os.listdir(image_dir), '*.' + extension)
if len(image_list) > 0:
break
image_list.sort()
appearance_dir = os.path.join(base_dir, 'results', 'appearance')
motion_dir = os.path.join(base_dir, 'results', 'motion')
for image_name in image_list:
im_path = os.path.join(image_dir, image_name)
im_prefix = image_name.split('.')[0]
print(im_path)
app_probs_file = os.path.join(appearance_dir,
im_prefix + '_probs_raw.npy')
mot_probs_file = os.path.join(motion_dir, im_prefix + '_probs_raw.npy')
app_prob = np.load(app_probs_file)
mot_prob = np.load(mot_probs_file)
img_h = app_prob.shape[1]
img_w = app_prob.shape[2]
seg_data = np.zeros((1, 4, img_h, img_w))
seg_data[0, 0:2, :, :] = app_prob
seg_data[0, 2:4, :, :] = mot_prob
labels, fg_prob = segment_image(seg_data)
labels_file = os.path.join(results_dir, im_prefix + '_mask.npy')
probs_file = os.path.join(results_dir, im_prefix + '_probs.npy')
np.save(labels_file, labels)
np.save(probs_file, fg_prob)
if visualize == True:
viz_img = visualize_image(im_path, labels, fg_prob, cmap_pascal,
None)
viz_file = os.path.join(viz_dir, im_prefix + '_viz.png')
imsave(viz_file, viz_img)
def training(self, epoch):
train_loss = 0.0
self.model.train()
#tbar = tqdm(self.train_loader) # Instantly make your loops show a smart progress meter
for i, sample in enumerate(self.train_loader):
if i >= 10000:
break
else:
image, target = sample['image'], sample['label']
if self.args.cuda:
image, target = image.cuda(), target.cuda()
self.scheduler(self.optimizer, i, epoch, self.best_pred)
self.optimizer.zero_grad()
output = self.model(image)
loss = self.criterion(output, target)
loss.backward()
self.optimizer.step()
self.trainloss_history.append(loss.data.cpu().numpy())
if epoch == 3 and i == 1:
print("after 3 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 9 and i == 1:
print("after 10 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 29 and i == 1:
print("after 20 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 29 and i == 3:
visualize_image("coco", image, target, output)
if epoch == 29 and i == 5:
visualize_image("coco", image, target, output)
if epoch == 29 and i == 7:
visualize_image("coco", image, target, output)
if epoch == 29 and i == 9:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 1:
print("after 30 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 49 and i == 3:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 5:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 7:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 9:
visualize_image("coco", image, target, output)
pred = output.data.cpu().numpy()
pred = np.argmax(pred, axis=1)
self.evaluator.add_batch(target.cpu().numpy(), pred)
acc = self.evaluator.Pixel_Accuracy()
#acc_class = self.evaluator.Pixel_Accuracy_Class()
mIoU = self.evaluator.Mean_Intersection_over_Union()
last_loss = self.trainloss_history[-i:]
train_loss = np.mean(last_loss)
self.train_plot.append(train_loss)
print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size ))
print('Loss: %.3f' % train_loss)
print("Train_Acc:{}".format(acc), "Train_mIoU:{}".format(mIoU))
if epoch == 20:
torch.save(self.model.state_dict(), '/home/wan/Segmentation/model_20')
if epoch == 30:
torch.save(self.model.state_dict(), '/home/wan/Segmentation/model_30')
if epoch == 40:
torch.save(self.model.state_dict(), '/home/wan/Segmentation/model_40')
if epoch == 50:
torch.save(self.model.state_dict(), '/home/wan/Segmentation/model_50')
def validation(self, epoch):
self.model.eval()
self.evaluator.reset()
#tbar = tqdm(self.val_loader, desc='\r')
val_loss = 0.0
for i, sample in enumerate(self.val_loader):
if i >= 750:
break
else:
image, target = sample['image'], sample['label']
if self.args.cuda:
image, target = image.cuda(), target.cuda()
with torch.no_grad():
output = self.model(image)
loss = self.criterion(output, target)
self.valloss_history.append(loss.data.cpu().numpy())
pred = output.data.cpu().numpy()
pred = np.argmax(pred, axis=1)
# visualize the prediction and target
if epoch == 3 and i == 0:
print("after 3 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch ==9 and i == 0:
print("after 10 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 29 and i == 0:
print("after 20 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 29 and i == 1:
visualize_image("coco", image, target, output)
if epoch == 29 and i == 2:
visualize_image("coco", image, target, output)
if epoch == 29 and i == 3:
visualize_image("coco", image, target, output)
if epoch == 29 and i == 4:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 0:
print("after 30 epochs, the result of training data:")
visualize_image("coco", image, target, output)
if epoch == 49 and i == 1:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 2:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 3:
visualize_image("coco", image, target, output)
if epoch == 49 and i == 4:
visualize_image("coco", image, target, output)
self.evaluator.add_batch(target.cpu().numpy(), pred)
acc = self.evaluator.Pixel_Accuracy()
#acc_class = self.evaluator.Pixel_Accuracy_Class()
mIoU = self.evaluator.Mean_Intersection_over_Union()
last_loss = self.valloss_history[-i:]
val_loss = np.mean(last_loss)
self.val_plot.append(val_loss)
print('Validation: ')
print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size ))
print("Acc:{}".format(acc), "mIoU:{}".format(mIoU))
print('Loss: %.3f' % val_loss)
#x, y = data_sample
#if x < 3 or x > 7 or y<3:
# break
#for i in range(3):
# data_sample[0] *= x / matrix_max_xy
# data_sample[1] *= y / matrix_max_xy
#break
data_sample = dataset.next_batch(1)[0]
#if len(data) < data_count:
# data.append(data_sample)
kohonen.alg_step(matrix, data_sample, step)
if step % visualize_after == 0:
text = "Step={}\nalpha={:.2f}\nsigma={:.2f}".format(
step, kohonen.alpha(step), kohonen.sigma(step))
print(text.replace("\n", ", "))
visualize.visualize_grid(matrix, visualized_data, "out_grid.png", text)
visualize.visualize_palette(matrix, "out_palette.png")
if step % visualize_image_after == 0:
print("Image visualization")
visualize.visualize_image(data, dataset.image_size, "out_image.png",
matrix)
# TODO: image compressor, better visualization of 3D, then maybe visualise the map also with colors