def prepare(self, input):
'''Prepares calculating. This function must be run before
any others.'''
self.p = len(input.obs)
self.cams = tuple(to_tf_tensor(cam) for cam in input.cams)
self.x = tuple(to_tf_tensor(x) for x in input.x)
self.w = tuple(to_tf_tensor(w) for w in input.w)
self.obs = to_tf_tensor(input.obs, dtype = tf.int64)
self.feats = to_tf_tensor(input.feats)
self.reproj_error = tf.zeros(2 * self.p, dtype = tf.float64)
self.w_err = tf.zeros(len(input.w))
self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def prepare(self, input):
'''Prepares calculating. This function must be run before any others.'''
self.p = len(input.obs)
self.cams = input.cams
self.x = input.x
self.w = input.w
self.obs = input.obs
self.feats = input.feats
graph = tf.compat.v1.Graph()
with graph.as_default():
self.prepare_operations()
self.session = tf.compat.v1.Session(graph=graph)
self.first_running()
self.r_err = np.zeros(2 * self.p, dtype=np.float64)
self.w_err = np.zeros(len(input.w))
self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def prepare(self, input):
'''Prepares calculating. This function must be run before
any others.'''
self.p = len(input.obs)
# we use tuple of tensors instead of multidimensional tensor
# because torch doesn't differentiate by non-leaf tensors
self.cams = tuple(
to_torch_tensor(cam, grad_req=True) for cam in input.cams)
self.x = tuple(to_torch_tensor(x, grad_req=True) for x in input.x)
self.w = tuple(to_torch_tensor(w, grad_req=True) for w in input.w)
self.obs = to_torch_tensor(input.obs, dtype=torch.int64)
self.feats = to_torch_tensor(input.feats)
self.reproj_error = torch.zeros(2 * self.p, dtype=torch.float64)
self.w_err = torch.zeros(len(input.w))
self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
class BAOutput:
reproj_err: np.ndarray = field(default=np.empty(0, dtype=np.float64))
w_err: np.ndarray = field(default=np.empty(0, dtype=np.float64))
J: BASparseMat = field(default=BASparseMat())
def save_output_to_file(self, output_prefix, input_basename,
module_basename):
save_errors_to_file(
objective_file_name(output_prefix, input_basename,
module_basename), self.reproj_err, self.w_err)
save_sparse_j_to_file(
jacobian_file_name(output_prefix, input_basename, module_basename),
self.J)
class TensorflowGraphBA(ITest):
'''Test class for BA diferentiation by Tensorflow using computational
graphs.'''
def prepare(self, input):
'''Prepares calculating. This function must be run before any others.'''
self.p = len(input.obs)
self.cams = input.cams
self.x = input.x
self.w = input.w
self.obs = input.obs
self.feats = input.feats
graph = tf.compat.v1.Graph()
with graph.as_default():
self.prepare_operations()
self.session = tf.compat.v1.Session(graph=graph)
self.first_running()
self.r_err = np.zeros(2 * self.p, dtype=np.float64)
self.w_err = np.zeros(len(input.w))
self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def prepare_operations(self):
'''Prepares computational graph for needed operations.'''
# creating holders for storing needed input for calculating a part of
# the BA objective and its derivative (the current camera, point,
# weight, and feat)
self.cam_holder = tf.compat.v1.placeholder(dtype=tf.float64,
shape=self.cams[0].shape)
self.x_holder = tf.compat.v1.placeholder(dtype=tf.float64,
shape=self.x[0].shape)
self.w_holder = tf.compat.v1.placeholder(dtype=tf.float64,
shape=self.w[0].shape)
self.feat_holder = tf.compat.v1.placeholder(dtype=tf.float64,
shape=self.feats[0].shape)
self.create_operations()
def create_operations(self):
'''Creates operations for calculating the part of the objective and its
derivative.'''
with tf.GradientTape(persistent=True) as grad_tape:
grad_tape.watch(self.cam_holder)
grad_tape.watch(self.x_holder)
grad_tape.watch(self.w_holder)
self.w_err_operation = compute_w_err(self.w_holder)
self.r_err_operation = compute_reproj_err(self.cam_holder,
self.x_holder,
self.w_holder,
self.feat_holder)
dc, dx, dw = grad_tape.jacobian(
self.r_err_operation,
(self.cam_holder, self.x_holder, self.w_holder),
experimental_use_pfor=False)
self.r_err_grad_operation = flatten(tf.concat(
(dc, dx, tf.reshape(dw, [2, 1])), axis=1),
column_major=True)
self.w_err_grad_operation = grad_tape.gradient(self.w_err_operation,
self.w_holder)
def first_running(self):
'''Performs the first session running.'''
self.session.run((self.w_err_operation, self.r_err_operation),
feed_dict=self.get_feed_dict(0))
self.session.run(
(self.w_err_grad_operation, self.r_err_grad_operation),
feed_dict=self.get_feed_dict(0))
def output(self):
'''Returns calculation result.'''
return BAOutput(self.r_err, self.w_err, self.jacobian)
def get_feed_dict(self, i):
'''Returns feed dictionary for the needed jacobian part calculation.'''
return {
self.cam_holder: self.cams[self.obs[i, 0]],
self.x_holder: self.x[self.obs[i, 1]],
self.w_holder: self.w[i],
self.feat_holder: self.feats[i]
}
def calculate_objective(self, times):
'''Calculates objective function many times.'''
for _ in range(times):
# calculate reprojection and weight error part by part and
# then combine the parts together
result = tuple(
self.session.run((self.r_err_operation, self.w_err_operation),
feed_dict=self.get_feed_dict(i))
for i in range(self.p))
result = zip(*result)
self.r_err = np.concatenate(result.__next__(), 0)
self.w_err = np.stack(result.__next__(), 0)
def calculate_jacobian(self, times):
''' Calculates objective function jacobian many times.'''
for _ in range(times):
# calculate reprojection and weight error derivatives part by
# part. Note, that weight error should be added to the sparse
# matrix only after the last reprojection error jacobian part
# adding, otherwise the result will be wrong
dws = []
for i in range(self.p):
dr, dw = self.session.run(
(self.r_err_grad_operation, self.w_err_grad_operation),
feed_dict=self.get_feed_dict(i))
self.jacobian.insert_reproj_err_block(i, self.obs[i, 0],
self.obs[i, 1], dr)
dws.append(dw)
for i in range(self.p):
self.jacobian.insert_w_err_block(i, dws[i])
class TensorflowBA(ITest):
'''Test class for BA diferentiation by Tensorflow using eager execution.'''
def prepare(self, input):
'''Prepares calculating. This function must be run before
any others.'''
self.p = len(input.obs)
self.cams = tuple(to_tf_tensor(cam) for cam in input.cams)
self.x = tuple(to_tf_tensor(x) for x in input.x)
self.w = tuple(to_tf_tensor(w) for w in input.w)
self.obs = to_tf_tensor(input.obs, dtype = tf.int64)
self.feats = to_tf_tensor(input.feats)
self.reproj_error = tf.zeros(2 * self.p, dtype = tf.float64)
self.w_err = tf.zeros(len(input.w))
self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def output(self):
'''Returns calculation result.'''
return BAOutput(
self.reproj_error.numpy(),
self.w_err.numpy(),
self.jacobian
)
def calculate_objective(self, times):
'''Calculates objective function many times.'''
for _ in range(times):
reproj = []
w_err = []
for j in range(self.p):
reproj_err = compute_reproj_err(
self.cams[self.obs[j, 0]],
self.x[self.obs[j, 1]],
self.w[j],
self.feats[j]
)
reproj.append(reproj_err)
w_err.append(compute_w_err(self.w[j]))
self.reproj_error = tf.concat(reproj, 0)
self.w_err = tf.stack(w_err, 0)
def calculate_jacobian(self, times):
''' Calculates objective function jacobian many times.'''
for _ in range(times):
# reprojection error processing
reproj_err = []
for j in range(self.p):
camIdx = self.obs[j, 0]
ptIdx = self.obs[j, 1]
with tf.GradientTape(persistent = True) as t:
t.watch(self.cams[camIdx])
t.watch(self.x[ptIdx])
t.watch(self.w[j])
rej = compute_reproj_err(
self.cams[camIdx],
self.x[ptIdx],
self.w[j],
self.feats[j]
)
reproj_err.append(rej)
dc, dx, dw = t.jacobian(
rej,
(self.cams[camIdx], self.x[ptIdx], self.w[j]),
experimental_use_pfor = False
)
J = tf.concat(
( dc, dx, tf.reshape(dw, [2, 1]) ),
axis = 1
)
J = flatten(J, column_major = True).numpy()
self.jacobian.insert_reproj_err_block(j, camIdx, ptIdx, J)
self.reproj_error = tf.concat(reproj_err, 0)
# weight error processing
w_err = []
for j in range(self.p):
with tf.GradientTape(persistent = True) as t:
t.watch(self.w[j])
wj = compute_w_err(self.w[j])
w_err.append(wj)
dwj = t.gradient(wj, self.w[j])
self.jacobian.insert_w_err_block(j, dwj.numpy())
self.w_err = tf.stack(w_err, 0)
class PyTorchBA(ITest):
'''Test class for BA diferentiation by PyTorch.'''
def prepare(self, input):
'''Prepares calculating. This function must be run before
any others.'''
self.p = len(input.obs)
# we use tuple of tensors instead of multidimensional tensor
# because torch doesn't differentiate by non-leaf tensors
self.cams = tuple(
to_torch_tensor(cam, grad_req=True) for cam in input.cams)
self.x = tuple(to_torch_tensor(x, grad_req=True) for x in input.x)
self.w = tuple(to_torch_tensor(w, grad_req=True) for w in input.w)
self.obs = to_torch_tensor(input.obs, dtype=torch.int64)
self.feats = to_torch_tensor(input.feats)
self.reproj_error = torch.zeros(2 * self.p, dtype=torch.float64)
self.w_err = torch.zeros(len(input.w))
self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def output(self):
'''Returns calculation result.'''
return BAOutput(self.reproj_error.detach().numpy(),
self.w_err.detach().numpy(), self.jacobian)
def calculate_objective(self, times):
'''Calculates objective function many times.'''
reproj_error = torch.empty((self.p, 2), dtype=torch.float64)
for i in range(times):
for j in range(self.p):
reproj_error[j] = compute_reproj_err(self.cams[self.obs[j, 0]],
self.x[self.obs[j, 1]],
self.w[j], self.feats[j])
self.w_err[j] = compute_w_err(self.w[j])
self.reproj_error = reproj_error.flatten()
def calculate_jacobian(self, times):
''' Calculates objective function jacobian many times.'''
reproj_error = torch.empty((self.p, 2), dtype=torch.float64)
for i in range(times):
# reprojection error jacobian calculation
for j in range(self.p):
camIdx = self.obs[j, 0]
ptIdx = self.obs[j, 1]
cam = self.cams[camIdx]
x = self.x[ptIdx]
w = self.w[j]
reproj_error[j], J = torch_jacobian(compute_reproj_err,
(cam, x, w),
(self.feats[j], ))
self.jacobian.insert_reproj_err_block(j, camIdx, ptIdx, J)
# weight error jacobian calculation
for j in range(self.p):
self.w_err[j], J = torch_jacobian(compute_w_err, (self.w[j], ))
self.jacobian.insert_w_err_block(j, J)
self.reproj_error = reproj_error.flatten()