def backward_impl(self, inputs, outputs, prop_down, accum):
# inputs: [inputs_fwd_graph] + [inputs_bwd_graph] or
# [inputs_fwd_graph] + [outputs_fwd_graph] + [inputs_bwd_graph]
# Args
alpha = self.forward_func.info.args["alpha"]
# Inputs
x0 = inputs[0].data
dy = inputs[1].data
# Outputs
dx0 = outputs[0].data
# Grads of inputs
g_x0 = inputs[0].grad
g_dy = inputs[1].grad
# Grads of outputs
g_dx0 = outputs[0].grad
# Computation
if prop_down[0]:
if accum[0]:
g_x0 += g_dx0 * dx0 * F.less_equal_scalar(x0, 0.0)
else:
g_x0.copy_from(g_dx0 * dx0 * F.less_equal_scalar(x0, 0.0))
if prop_down[1]:
inp = nn.Variable(x0.shape).apply(data=x0,
grad=g_dy,
need_grad=True)
out = nn.Variable(dy.shape).apply(grad=g_dx0)
self.forward_func.backward([inp], [out], accum=[accum[1]])
def ray_march(self, camloc, raydir, t0, t1, N, n_chunks, t_argmin=False):
# Points computation
BR, _ = t0.shape
t0 = F.reshape(t0, (BR, 1, 1))
t1 = F.reshape(t1, (BR, 1, 1))
camloc = F.reshape(camloc, (BR, 1, 3))
raydir = F.reshape(raydir, (BR, 1, 3))
step = (t1 - t0) / (N - 1)
intervals = F.reshape(F.arange(0, N), (1, N, 1))
ts = t0 + step * intervals
points = camloc + ts * raydir
points = F.reshape(points, (BR * N, 3))
# SDF computation
sdf_points = []
batch = (BR * N) // n_chunks
for r in range(0, BR * N, batch):
sdf_points.append(self.sdf(points[r:r + batch, :]))
sdf_points = F.reshape(F.concatenate(*sdf_points, axis=0), (BR, N, 1)) if n_chunks != 1 else \
F.reshape(sdf_points[0], (BR, N, 1))
# t_argmin computation
if t_argmin:
idx_min = F.min(sdf_points, axis=1, keepdims=True, only_index=True)
t_argmin = F.reshape(F.gather(ts, idx_min, axis=1, batch_dims=1),
(BR, 1))
return t_argmin
# Intersection check
points = F.reshape(points, (BR, N, 3))
sdf_pos = F.greater_equal_scalar(sdf_points[:, :-1, :], 0)
sdf_neg = F.less_equal_scalar(sdf_points[:, 1:, :], 0)
mask_hit = sdf_pos * sdf_neg
decreasing_consts = F.reshape(F.arange(N, 1, -1), (1, N - 1, 1))
vals = mask_hit * decreasing_consts
idx_max = F.max(vals, axis=1, only_index=True)
points = points[:, :-1, :]
x_hit = F.gather(points, idx_max, axis=1, batch_dims=1)
x_hit = F.reshape(x_hit, (BR, 3))
mask_hit = F.greater_scalar(F.sum(mask_hit, axis=1), 0)
mask_hit = F.reshape(mask_hit, (BR, 1))
x_hit_rm0 = x_hit
step = F.reshape(step, (BR, 1))
raydir = F.reshape(raydir, (BR, 3))
x_hit_rm1 = x_hit_rm0 + step * raydir
return x_hit_rm0, x_hit_rm1, mask_hit
def hard_tanh_backward(inputs):
"""
Args:
inputs (list of nn.Variable): Incomming grads/inputs to/of the forward function.
kwargs (dict of arguments): Dictionary of the corresponding function arguments.
Return:
list of Variable: Return the gradients wrt inputs of the corresponding function.
"""
dy = inputs[0]
x0 = inputs[1]
m0 = F.greater_equal_scalar(x0, -1)
m1 = F.less_equal_scalar(x0, 1)
m01 = m0 * m1
m01 = no_grad(m01)
dx0 = dy * m01
return dx0
def main():
random.seed(args.seed)
np.random.seed(args.seed)
# Prepare for CUDA.
ctx = get_extension_context('cudnn', device_id=args.gpus)
nn.set_default_context(ctx)
start_full_time = time.time()
from iterator import data_iterator
# Data list for sceneflow data set
train_list = "./dataset/kitti_train.csv"
test_list = "./dataset/kitti_test.csv"
train = True
validation = False
# Set monitor path.
monitor_path = './nnmonitor' + str(datetime.now().strftime("%Y%m%d%H%M%S"))
img_left, img_right, disp_img = read_csv(train_list)
img_left_test, img_right_test, disp_img_test = read_csv(test_list)
train_samples = len(img_left)
test_samples = len(img_left_test)
train_size = int(len(img_left) / args.batchsize_train)
test_size = int(len(img_left_test) / args.batchsize_test)
# Create data iterator.
data_iterator_train = data_iterator(
train_samples, args.batchsize_train, img_left, img_right, disp_img, train, shuffle=True, dataset=args.dataset)
data_iterator_test = data_iterator(
test_samples, args.batchsize_test, img_left_test, img_right_test, disp_img_test, validation, shuffle=False, dataset=args.dataset)
# Set data size
print(train_size, test_size)
# Clrear patameters
nn.clear_parameters()
# Define data shape for training.
var_left = nn.Variable(
(args.batchsize_train, 3, args.crop_height, args.crop_width))
var_right = nn.Variable(
(args.batchsize_train, 3, args.crop_height, args.crop_width))
var_disp = nn.Variable(
(args.batchsize_train, 1, args.crop_height, args.crop_width))
# Define data shape for testing.
var_left_test = nn.Variable(
(args.batchsize_test, 3, args.im_height, args.im_width))
var_right_test = nn.Variable(
(args.batchsize_test, 3, args.im_height, args.im_width))
var_disp_test = nn.Variable(
(args.batchsize_test, 1, args.im_height, args.im_width))
if args.loadmodel is not None:
# Loading CNN pretrained parameters.
nn.load_parameters(args.loadmodel)
# === for Training ===
# Definition of pred
pred1, pred2, pred3 = psm_net(var_left, var_right, args.maxdisp, True)
mask_train = F.greater_scalar(var_disp, 0)
sum_mask = F.maximum_scalar(F.sum(mask_train), 1)
print(sum_mask.d, "sum_mask_first")
# Definition of loss
loss = 0.5 * (0.5 * F.sum(F.huber_loss(pred1, var_disp)*mask_train)/(sum_mask) + 0.7 * F.sum(F.huber_loss(
pred2, var_disp)*mask_train)/(sum_mask) + F.sum(F.huber_loss(pred3, var_disp)*mask_train)/(sum_mask))
# === for Testing ===
# Definition of pred
pred_test = psm_net(var_left_test, var_right_test, args.maxdisp, False)
var_gt = var_disp_test + F.less_equal_scalar(var_disp_test, 0) * -1
var_pred = pred_test + F.less_equal_scalar(pred_test, 0) * -1
E = F.abs(var_pred - var_gt)
n_err = F.sum(F.logical_and(F.logical_and(F.greater_scalar(var_gt, 0.0), F.greater_scalar(E, 3.0)),
F.greater_scalar(F.div2(E, F.abs(var_gt)), 0.05)))
n_total = F.sum(F.greater_scalar(var_gt, 0))
test_loss = F.div2(n_err, n_total)
# Prepare monitors.
monitor = Monitor(monitor_path)
monitor_train = MonitorSeries('Training loss', monitor, interval=1)
monitor_test = MonitorSeries('Validation loss', monitor, interval=1)
monitor_time_train = MonitorTimeElapsed(
"Training time/epoch", monitor, interval=1)
# Create a solver (parameter updater)
solver = S.Adam(alpha=0.001, beta1=0.9, beta2=0.999)
# Set Parameters
params = nn.get_parameters()
solver.set_parameters(params)
params2 = nn.get_parameters(grad_only=False)
solver.set_parameters(params2)
for epoch in range(1, args.epochs+1):
print('This is %d-th epoch' % (epoch))
total_train_loss = 0
index = 0
lr = adjust_learning_rate(epoch)
###Training###
while index < train_size:
# Get mini batch
# Preprocess
var_left.d, var_right.d, var_disp.d = data_iterator_train.next()
loss.forward(clear_no_need_grad=True)
# Initialize gradients
solver.zero_grad()
# Backward execution
loss.backward(clear_buffer=True)
# Update parameters by computed gradients
solver.set_learning_rate(lr)
solver.update()
print('Iter %d training loss = %.3f' % (index, loss.d))
total_train_loss += loss.d
index += 1
train_error = total_train_loss/train_size
print('epoch %d total training loss = %.3f' % (epoch, train_error))
monitor_time_train.add(epoch)
# ## teting ##
total_test_loss = 0
max_acc = 0
index_test = 0
while index_test < test_size:
var_left_test.d, var_right_test.d, var_disp_test.d = data_iterator_test.next()
test_loss.forward(clear_buffer=True)
total_test_loss += test_loss.d
print('Iter %d test loss = %.3f' % (index_test, test_loss.d*100))
index_test += 1
test_error = total_test_loss/test_size
print('epoch %d total 3-px error in val = %.3f' %
(epoch, test_error*100))
if test_error > max_acc:
max_acc = test_error*100
print('MAX epoch %d total test error = %.3f' % (epoch, max_acc))
# Pass validation loss to a monitor.
monitor_test.add(epoch, test_error*100)
# Pass training loss to a monitor.
monitor_train.add(epoch, train_error)
print('full training time = %.2f HR' %
((time.time() - start_full_time)/3600))
# Save Parameter
out_param_file = os.path.join(
args.savemodel, 'psmnet_trained_param_' + str(epoch) + '.h5')
nn.save_parameters(out_param_file)
def bidirectional_sphere_trace(self, camloc, raydir, t_start, t_finish):
t_f = F.identity(t_start)
x_f = camloc + t_f * raydir
s_f = self.sdf(x_f)
mask_hit_eps_f = 0 * F.identity(t_f)
t_b = F.identity(t_finish)
x_b = camloc + t_b * raydir
s_b = self.sdf(x_b)
mask_hit_eps_b = 0 * F.identity(t_b)
for i in range(self.sphere_trace_itr - 1):
# Forward direction
mask_hit_eps_f_i = F.less_equal_scalar(F.abs(s_f), self.eps)
mask_hit_eps_f += (1 - mask_hit_eps_f) * mask_hit_eps_f_i
t_f += (1 - mask_hit_eps_f) * s_f
x_f = camloc + t_f * raydir
s_f_prev = F.identity(s_f)
s_f = self.sdf(x_f)
mask_pos_f_prev = (1 - mask_hit_eps_f) * \
F.greater_scalar(s_f_prev, 0)
mask_neg_f = (1 - mask_hit_eps_f) * F.less_scalar(s_f, 0)
mask_revert_f = mask_pos_f_prev * mask_neg_f
t_f -= mask_revert_f * s_f_prev
s_f = mask_revert_f * s_f_prev + (1 - mask_revert_f) * s_f
# Backward direction
mask_hit_eps_b_i = F.less_equal_scalar(F.abs(s_b), self.eps)
mask_hit_eps_b += (1 - mask_hit_eps_b) * mask_hit_eps_b_i
t_b -= (1 - mask_hit_eps_b) * s_b
x_b = camloc + t_b * raydir
s_b_prev = F.identity(s_b)
s_b = self.sdf(x_b)
mask_pos_b_prev = (1 - mask_hit_eps_b) * \
F.greater_scalar(s_b_prev, 0)
mask_neg_b = (1 - mask_hit_eps_b) * F.less_scalar(s_b, 0)
mask_revert_b = mask_pos_b_prev * mask_neg_b
t_b += mask_revert_b * s_b_prev
s_b = mask_revert_b * s_b_prev + (1 - mask_revert_b) * s_b
## print("s_f neg", np.sum(s_f.data < 0))
## print("s_b neg", np.sum(s_b.data < 0))
# Fine grained start/finish points
t_f0 = t_f
t_f1 = t_f + mask_revert_f * s_f_prev
x_hit_st0 = camloc + t_f0 * raydir
## x0, x1 = self.post_method(x_hit_st0, camloc + t_f1 * raydir)
## t_f0 = F.norm((x0 - camloc), axis=(x0.ndim - 1), keepdims=True)
## t_f1 = F.norm((x1 - camloc), axis=(x1.ndim - 1), keepdims=True)
mask_hit_f1b = mask_revert_f * F.less(t_f1, t_b)
t_b = t_f1 * mask_hit_f1b + t_b * (1 - mask_hit_f1b)
# Reverse the opposite case
mask_fb = F.less(t_f, t_b)
t_f = t_f * mask_fb + t_start * (1 - mask_fb)
t_b = t_b * mask_fb + t_finish * (1 - mask_fb)
return x_hit_st0, t_f, t_b, mask_hit_eps_f
