def prepare(self, input): '''Prepares calculating. This function must be run before any others.''' self.p = len(input.obs) self.cams = tuple(to_tf_tensor(cam) for cam in input.cams) self.x = tuple(to_tf_tensor(x) for x in input.x) self.w = tuple(to_tf_tensor(w) for w in input.w) self.obs = to_tf_tensor(input.obs, dtype = tf.int64) self.feats = to_tf_tensor(input.feats) self.reproj_error = tf.zeros(2 * self.p, dtype = tf.float64) self.w_err = tf.zeros(len(input.w)) self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def prepare(self, input): '''Prepares calculating. This function must be run before any others.''' self.p = len(input.obs) self.cams = input.cams self.x = input.x self.w = input.w self.obs = input.obs self.feats = input.feats graph = tf.compat.v1.Graph() with graph.as_default(): self.prepare_operations() self.session = tf.compat.v1.Session(graph=graph) self.first_running() self.r_err = np.zeros(2 * self.p, dtype=np.float64) self.w_err = np.zeros(len(input.w)) self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
def prepare(self, input): '''Prepares calculating. This function must be run before any others.''' self.p = len(input.obs) # we use tuple of tensors instead of multidimensional tensor # because torch doesn't differentiate by non-leaf tensors self.cams = tuple( to_torch_tensor(cam, grad_req=True) for cam in input.cams) self.x = tuple(to_torch_tensor(x, grad_req=True) for x in input.x) self.w = tuple(to_torch_tensor(w, grad_req=True) for w in input.w) self.obs = to_torch_tensor(input.obs, dtype=torch.int64) self.feats = to_torch_tensor(input.feats) self.reproj_error = torch.zeros(2 * self.p, dtype=torch.float64) self.w_err = torch.zeros(len(input.w)) self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p)
class BAOutput: reproj_err: np.ndarray = field(default=np.empty(0, dtype=np.float64)) w_err: np.ndarray = field(default=np.empty(0, dtype=np.float64)) J: BASparseMat = field(default=BASparseMat()) def save_output_to_file(self, output_prefix, input_basename, module_basename): save_errors_to_file( objective_file_name(output_prefix, input_basename, module_basename), self.reproj_err, self.w_err) save_sparse_j_to_file( jacobian_file_name(output_prefix, input_basename, module_basename), self.J)
class TensorflowGraphBA(ITest): '''Test class for BA diferentiation by Tensorflow using computational graphs.''' def prepare(self, input): '''Prepares calculating. This function must be run before any others.''' self.p = len(input.obs) self.cams = input.cams self.x = input.x self.w = input.w self.obs = input.obs self.feats = input.feats graph = tf.compat.v1.Graph() with graph.as_default(): self.prepare_operations() self.session = tf.compat.v1.Session(graph=graph) self.first_running() self.r_err = np.zeros(2 * self.p, dtype=np.float64) self.w_err = np.zeros(len(input.w)) self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p) def prepare_operations(self): '''Prepares computational graph for needed operations.''' # creating holders for storing needed input for calculating a part of # the BA objective and its derivative (the current camera, point, # weight, and feat) self.cam_holder = tf.compat.v1.placeholder(dtype=tf.float64, shape=self.cams[0].shape) self.x_holder = tf.compat.v1.placeholder(dtype=tf.float64, shape=self.x[0].shape) self.w_holder = tf.compat.v1.placeholder(dtype=tf.float64, shape=self.w[0].shape) self.feat_holder = tf.compat.v1.placeholder(dtype=tf.float64, shape=self.feats[0].shape) self.create_operations() def create_operations(self): '''Creates operations for calculating the part of the objective and its derivative.''' with tf.GradientTape(persistent=True) as grad_tape: grad_tape.watch(self.cam_holder) grad_tape.watch(self.x_holder) grad_tape.watch(self.w_holder) self.w_err_operation = compute_w_err(self.w_holder) self.r_err_operation = compute_reproj_err(self.cam_holder, self.x_holder, self.w_holder, self.feat_holder) dc, dx, dw = grad_tape.jacobian( self.r_err_operation, (self.cam_holder, self.x_holder, self.w_holder), experimental_use_pfor=False) self.r_err_grad_operation = flatten(tf.concat( (dc, dx, tf.reshape(dw, [2, 1])), axis=1), column_major=True) self.w_err_grad_operation = grad_tape.gradient(self.w_err_operation, self.w_holder) def first_running(self): '''Performs the first session running.''' self.session.run((self.w_err_operation, self.r_err_operation), feed_dict=self.get_feed_dict(0)) self.session.run( (self.w_err_grad_operation, self.r_err_grad_operation), feed_dict=self.get_feed_dict(0)) def output(self): '''Returns calculation result.''' return BAOutput(self.r_err, self.w_err, self.jacobian) def get_feed_dict(self, i): '''Returns feed dictionary for the needed jacobian part calculation.''' return { self.cam_holder: self.cams[self.obs[i, 0]], self.x_holder: self.x[self.obs[i, 1]], self.w_holder: self.w[i], self.feat_holder: self.feats[i] } def calculate_objective(self, times): '''Calculates objective function many times.''' for _ in range(times): # calculate reprojection and weight error part by part and # then combine the parts together result = tuple( self.session.run((self.r_err_operation, self.w_err_operation), feed_dict=self.get_feed_dict(i)) for i in range(self.p)) result = zip(*result) self.r_err = np.concatenate(result.__next__(), 0) self.w_err = np.stack(result.__next__(), 0) def calculate_jacobian(self, times): ''' Calculates objective function jacobian many times.''' for _ in range(times): # calculate reprojection and weight error derivatives part by # part. Note, that weight error should be added to the sparse # matrix only after the last reprojection error jacobian part # adding, otherwise the result will be wrong dws = [] for i in range(self.p): dr, dw = self.session.run( (self.r_err_grad_operation, self.w_err_grad_operation), feed_dict=self.get_feed_dict(i)) self.jacobian.insert_reproj_err_block(i, self.obs[i, 0], self.obs[i, 1], dr) dws.append(dw) for i in range(self.p): self.jacobian.insert_w_err_block(i, dws[i])
class TensorflowBA(ITest): '''Test class for BA diferentiation by Tensorflow using eager execution.''' def prepare(self, input): '''Prepares calculating. This function must be run before any others.''' self.p = len(input.obs) self.cams = tuple(to_tf_tensor(cam) for cam in input.cams) self.x = tuple(to_tf_tensor(x) for x in input.x) self.w = tuple(to_tf_tensor(w) for w in input.w) self.obs = to_tf_tensor(input.obs, dtype = tf.int64) self.feats = to_tf_tensor(input.feats) self.reproj_error = tf.zeros(2 * self.p, dtype = tf.float64) self.w_err = tf.zeros(len(input.w)) self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p) def output(self): '''Returns calculation result.''' return BAOutput( self.reproj_error.numpy(), self.w_err.numpy(), self.jacobian ) def calculate_objective(self, times): '''Calculates objective function many times.''' for _ in range(times): reproj = [] w_err = [] for j in range(self.p): reproj_err = compute_reproj_err( self.cams[self.obs[j, 0]], self.x[self.obs[j, 1]], self.w[j], self.feats[j] ) reproj.append(reproj_err) w_err.append(compute_w_err(self.w[j])) self.reproj_error = tf.concat(reproj, 0) self.w_err = tf.stack(w_err, 0) def calculate_jacobian(self, times): ''' Calculates objective function jacobian many times.''' for _ in range(times): # reprojection error processing reproj_err = [] for j in range(self.p): camIdx = self.obs[j, 0] ptIdx = self.obs[j, 1] with tf.GradientTape(persistent = True) as t: t.watch(self.cams[camIdx]) t.watch(self.x[ptIdx]) t.watch(self.w[j]) rej = compute_reproj_err( self.cams[camIdx], self.x[ptIdx], self.w[j], self.feats[j] ) reproj_err.append(rej) dc, dx, dw = t.jacobian( rej, (self.cams[camIdx], self.x[ptIdx], self.w[j]), experimental_use_pfor = False ) J = tf.concat( ( dc, dx, tf.reshape(dw, [2, 1]) ), axis = 1 ) J = flatten(J, column_major = True).numpy() self.jacobian.insert_reproj_err_block(j, camIdx, ptIdx, J) self.reproj_error = tf.concat(reproj_err, 0) # weight error processing w_err = [] for j in range(self.p): with tf.GradientTape(persistent = True) as t: t.watch(self.w[j]) wj = compute_w_err(self.w[j]) w_err.append(wj) dwj = t.gradient(wj, self.w[j]) self.jacobian.insert_w_err_block(j, dwj.numpy()) self.w_err = tf.stack(w_err, 0)
class PyTorchBA(ITest): '''Test class for BA diferentiation by PyTorch.''' def prepare(self, input): '''Prepares calculating. This function must be run before any others.''' self.p = len(input.obs) # we use tuple of tensors instead of multidimensional tensor # because torch doesn't differentiate by non-leaf tensors self.cams = tuple( to_torch_tensor(cam, grad_req=True) for cam in input.cams) self.x = tuple(to_torch_tensor(x, grad_req=True) for x in input.x) self.w = tuple(to_torch_tensor(w, grad_req=True) for w in input.w) self.obs = to_torch_tensor(input.obs, dtype=torch.int64) self.feats = to_torch_tensor(input.feats) self.reproj_error = torch.zeros(2 * self.p, dtype=torch.float64) self.w_err = torch.zeros(len(input.w)) self.jacobian = BASparseMat(len(input.cams), len(input.x), self.p) def output(self): '''Returns calculation result.''' return BAOutput(self.reproj_error.detach().numpy(), self.w_err.detach().numpy(), self.jacobian) def calculate_objective(self, times): '''Calculates objective function many times.''' reproj_error = torch.empty((self.p, 2), dtype=torch.float64) for i in range(times): for j in range(self.p): reproj_error[j] = compute_reproj_err(self.cams[self.obs[j, 0]], self.x[self.obs[j, 1]], self.w[j], self.feats[j]) self.w_err[j] = compute_w_err(self.w[j]) self.reproj_error = reproj_error.flatten() def calculate_jacobian(self, times): ''' Calculates objective function jacobian many times.''' reproj_error = torch.empty((self.p, 2), dtype=torch.float64) for i in range(times): # reprojection error jacobian calculation for j in range(self.p): camIdx = self.obs[j, 0] ptIdx = self.obs[j, 1] cam = self.cams[camIdx] x = self.x[ptIdx] w = self.w[j] reproj_error[j], J = torch_jacobian(compute_reproj_err, (cam, x, w), (self.feats[j], )) self.jacobian.insert_reproj_err_block(j, camIdx, ptIdx, J) # weight error jacobian calculation for j in range(self.p): self.w_err[j], J = torch_jacobian(compute_w_err, (self.w[j], )) self.jacobian.insert_w_err_block(j, J) self.reproj_error = reproj_error.flatten()