def cycle_back_classification_fast(batch1, batch2): u, u_indices = batch1 v, v_indices = batch2 len_u = len(u) len_v = len(v) t = cupy.arange(len(u)) u_repeat = F.repeat(F.expand_dims(u, 0), len_v, 0).transpose(1, 0, 2) length1 = -F.sum((u_repeat - v)**2, axis=2) sim_12 = length1 / u.shape[1] sim_12 /= CONFIG.softmax_tmp alpha = F.softmax(sim_12) v_tilde = F.matmul(alpha, v) v_tilde_repeat = F.repeat(F.expand_dims(v_tilde, 0), len_u, 0).transpose(1, 0, 2) length2 = -F.sum((v_tilde_repeat - u)**2, axis=2) sim_21 = length2 / u.shape[1] sim_21 /= CONFIG.softmax_tmp loss = (F.softmax_cross_entropy(sim_21, t)) return loss * len_u
def __call__(self, x, t1, t2): h1 = F.relu(self.l1(x)) h2 = F.relu(self.l2(x)) b1 = self.b1(self.one1) b2 = self.b2(self.one2) b1 = F.expand_dims(F.repeat(b1, h1.shape[0]), axis=1) b2 = F.expand_dims(F.repeat(b2, h2.shape[0]), axis=1) loss = F.sum(F.exp(-b1) * (h1 - t1)**2. + b1) + \ F.sum(F.exp(-b2) * (h2 - t2)**2. + b2) loss = F.mean(loss) if self.iteration % 100 == 0: print('loss:', loss.data, ', sigma1:', F.exp(b1[0]).data[0]**0.5, ', sigma2:', F.exp(b2[0]).data[0]**0.5) chainer.reporter.report( { 'loss': loss, 'b1': float(F.exp(b1[0]).data**0.5), 'b2': float(F.exp(b2[0]).data**0.5), 'Lb1': float(self.l1.b.data), 'Lb2': float(self.l2.b.data), }, self) self.iteration += 1 return loss
def outer(a, b): assert a.shape[0] == b.shape[0] assert a.ndim == 2 assert b.ndim == 2 M = a.shape[1] N = b.shape[1] a = F.repeat(a[:, :, None], N, axis=2) b = F.repeat(b[:, None, :], M, axis=1) return a * b
def blend_image(*image_pairs): result_image = [] for fore, back in zip(image_pairs[::2], image_pairs[1::2]): fore = utils.unwrapped(fore) back = utils.unwrapped(back) if back.shape[1] > fore.shape[1]: fore = F.repeat(fore, back.shape[1] // fore.shape[1], axis=1) elif back.shape[1] < fore.shape[1]: back = F.repeat(back, fore.shape[1] // back.shape[1], axis=1) result_image.append(normalize(fore) * 0.5 + normalize(back) * 0.5) return result_image
def forward(self, stories): """Compute the forward inference pass for given stories.""" self.log = dict() # --------------------------- vctx, vq, va, supps = stories # (B, R, P, C), (B, Q), (B,), (B, I) # Embed stories # ectx = F.embed_id(vctx, wordeye, ignore_label=0) # (B, R, P, C, V) # eq = F.embed_id(vq, wordeye, ignore_label=0) # (B, Q, V) ectx = self.embed(vctx) # (B, R, P, C, V) eq = self.embed(vq) # (B, Q, V) # --------------------------- # Embed predicates embedded_preds = seq_rnn_embed(vctx, ectx, self.pred_rnn, reverse=True) # (B, R, P, E) vector_preds = vctx[ ..., 0] # (B, R, P) first character to check if pred is empty embedded_query = seq_rnn_embed(vq, eq, self.pred_rnn, reverse=True) # (B, E) embedded_rules = embedded_preds[:, :, 0] # (B, R, E) head of rule # --------------------------- # Perform iterative updates state = embedded_query # (B, E) repeated_query = F.repeat(embedded_query[:, None], vctx.shape[1], 1) # (B, R, E) rule_mask = np.all(vctx == 0, (2, 3)) # (B, R) for _ in range(supps.shape[-1]): # Compute attention over memory repeated_state = F.repeat(state[:, None], vctx.shape[1], 1) # (B, R, E) combined = F.concat([ repeated_state, embedded_rules, repeated_query, F.squared_difference(repeated_state, embedded_rules), embedded_rules * repeated_state ], -1) # (B, R, 5*E) att = F.tanh(self.att_dense1(combined, n_batch_axes=2)) # (B, R, E//2) att = self.att_dense2(att, n_batch_axes=2) # (B, R, 1) att = F.squeeze(att, -1) # (B, R) att += rule_mask * MINUS_INF # (B, R) self.tolog('raw_att', att) att = F.softmax(att) # (B, R) self.tolog('att', att) # Iterate state new_states = seq_rnn_embed( vector_preds, embedded_preds, self.unifier, initial_state=repeated_state) # (B, R, E) # Update state # (B, R) x (B, R, E) -> (B, E) state = F.einsum('br,bre->be', att, new_states) # (B, E) return self.out_linear(state)[:, 0] # (B,)
def get_all_prob_or_log_prob(self, is_log=False): segments = self.parametric_segments num_of_action_types = len(segments) action_types_logits = self.beta * self.logits[:, :num_of_action_types] if is_log: action_types_probs = F.log_softmax(action_types_logits) else: action_types_probs = F.softmax(action_types_logits) # action_types_probs = F.softmax(action_types_logits) # if is_log: # print("LOG") # print(action_types_probs) # action_types_probs = action_types_probs.data * np.power(self.segments_sizes, 1/4) # action_types_probs = action_types_probs / np.expand_dims(np.sum(action_types_probs, axis=1), axis=1) # action_types_probs = chainer.Variable(action_types_probs.astype(np.float32)) # if is_log: # action_types_probs = F.log(action_types_probs) # print(action_types_probs) result = [] logits_offset = num_of_action_types for i in range(num_of_action_types): action_type_prob = action_types_probs[:, i:i + 1] if not segments[i]: # if no parameters for this action type result.append(action_type_prob) else: segments_factor = 1 for sub_seg_size in segments[i]: segments_factor *= sub_seg_size if is_log: sub_seg_probs = F.log_softmax( self.beta * self.logits[:, logits_offset:logits_offset + sub_seg_size]) else: sub_seg_probs = F.softmax( self.beta * self.logits[:, logits_offset:logits_offset + sub_seg_size]) if is_log: action_type_prob = F.repeat( action_type_prob, sub_seg_size, axis=1) + F.tile( sub_seg_probs, segments_factor // sub_seg_size) else: action_type_prob = F.repeat( action_type_prob, sub_seg_size, axis=1) * F.tile( sub_seg_probs, segments_factor // sub_seg_size) logits_offset += sub_seg_size result.append(action_type_prob) res = F.concat(tuple(result)) return res
def __call__(self, imgs, questions): feat = self.feat_extractor(imgs) # Append relative coordinates to each location in the feature maps. n, c, h, w = feat.shape spatial_area = h * w xp = self.xp coords_h = xp.linspace(-1, 1, h, dtype=feat.dtype) coords_w = xp.linspace(-1, 1, w, dtype=feat.dtype) coords_hh, coords_ww = xp.meshgrid(coords_h, coords_w) coords_hh = coords_hh[None] coords_ww = coords_ww[None] coords = xp.concatenate((coords_hh, coords_ww), axis=0) coords = coords.reshape(2, -1) coords = coords[None] # (1, 2, spatial_area * spatial_area) coords = xp.repeat(coords, n, axis=0) # Coordinates may be cached here but the performance gain is not # significant so it is skipped in favor of readability. feat = feat.reshape(n, c, spatial_area) h = F.concat((feat, coords), axis=1) # (n, c + 2, spatial_area) # Create coordinate pairs (differentiable meshgrid). h_hh = F.expand_dims(h, 2) h_ww = F.expand_dims(h, 3) h_hh = F.repeat(h_hh, spatial_area, axis=2) h_ww = F.repeat(h_ww, spatial_area, axis=3) h = F.concat((h_hh, h_ww), axis=1) # Append questions to each coordinate pair. questions = questions.astype(imgs.dtype) questions = questions[:, :, None, None] questions = F.tile(questions, (1, 1, spatial_area, spatial_area)) h = F.concat((h, questions), axis=1) # (n, (c + 2) * 2 + questions_length, spatial_area, spatial_area) # g. h = F.transpose(h, (0, 2, 3, 1)) h = F.reshape(h, (n * spatial_area * spatial_area, -1)) h = self.g(h) h = F.reshape(h, (n, spatial_area * spatial_area, -1)) h = F.sum(h, axis=1) h = self.f(h) # Logits. h = self.fc(h) return h
def __call__(self, x, x_=None, **kwargs): if x_ is None: x_ = x if self.k > 1: x_ = F.repeat(x_, self.k, axis=0) q_z = self.encode(x, **kwargs) z = self.sample(q_z) p_x = self.decode(z, **kwargs) p_z = self.prior() # 追加誤差関数 # mse_vel = F.mean_squared_error(x_, p_x.mean) # mse_vor = F.mean_squared_error(*map(vorticity, (x_, p_x.mean))) # y = F.sigmoid(p_x.mean) # mse_vel = self.batch_mean(F.squared_error(*map(logit, (x_, y)))) # mse_vor = self.batch_mean(F.squared_error(*map(vorticity_logit15, # (x_, y)))) # reporter.report({'mse_vel': mse_vel}, self) # reporter.report({'mse_vor': mse_vor}, self) # 誤差関数 reconstr = self.batch_mean(p_x.log_prob(x_)) # reconstr = -self.batch_mean((p_x.mean - x_) ** 2) kl_penalty = self.batch_mean(chainer.kl_divergence(q_z, p_z)) loss = self.beta * kl_penalty - reconstr reporter.report({'loss': loss}, self) reporter.report({'reconstr': reconstr}, self) reporter.report({'kl_penalty': kl_penalty}, self) return loss
def return_injected(self, h, z, n_layer, mult_until_exec=None): """ Performs the Hadamard products with z. """ # # check whether to skip the hadamard. skip_injection = False if self.thresh_skip is not None and self.thresh_skip[n_layer - 1] > 0: # # skip the hadamard, iff the random number is smaller than the threshold. skip_injection = np.random.uniform() < self.thresh_skip[n_layer - 1] if not skip_injection and mult_until_exec is not None: skip_injection = mult_until_exec <= n_layer if self.mult_lat and not skip_injection: if self.use_localz: # # apply local transformation. z1 = getattr(self, 'locz{}'.format(n_layer))(z) else: z1 = z # # appropriately reshape z for the elementwise multiplication. sh = h.shape z1 = F.reshape(z1, (sh[0], sh[1], 1)) if self.normalize_preinject: z1 /= F.sqrt(F.mean(z1 * z1, axis=1, keepdims=True) + 1e-8) z2 = F.repeat(z1, sh[3] * sh[2], axis=2) z2 = F.reshape(z2, sh) ret = h * z2 + h if self.add_h_injection else h * z2 return ret return h
def forward(self, equery, vmemory, ememory, mask, iteration=0): """Compute an attention over memory given the query.""" # equery.shape == (..., E) # vmemory.shape == (..., Ms, M) # ememory.shape == (..., Ms, E) # mask.shape == (..., Ms) # Setup memory embedding eq = F.repeat(equery[..., None, :], vmemory.shape[-2], -2) # (..., Ms, E) # Compute content based attention merged = F.concat( [eq, ememory, eq * ememory, F.squared_difference(eq, ememory)], -1) # (..., Ms, 4*E) inter = self.att_linear(merged, n_batch_axes=len(vmemory.shape) - 1) # (..., Ms, E) inter = F.tanh(inter) # (..., Ms, E) inter = F.dropout(inter, DROPOUT) # (..., Ms, E) # Split into sentences lengths = np.sum(np.any((vmemory != 0), -1), -1) # (...,) mems = [s[..., :l, :] for s, l in zip(F.separate(inter, 0), lengths) ] # B x [(M1, E), (M2, E), ...] _, bimems = self.att_birnn(None, mems) # B x [(M1, 2*E), (M2, 2*E), ...] bimems = F.pad_sequence(bimems) # (..., Ms, 2*E) att = self.att_score(bimems, n_batch_axes=len(vmemory.shape) - 1) # (..., Ms, 1) att = F.squeeze(att, -1) # (..., Ms) if mask is not None: att += mask * MINUS_INF # (..., Ms) return att
def _decode_multiple(self, s, z=None, decode_num=10): if z is None: xp = chainer.backend.get_array_module(s) z = chainer.Variable( xp.random.normal(0, 1, size=(s.shape[0], decode_num, self._latent_dim))) z = F.cast(z, typ=xp.float32) z = F.clip(z, -0.5, 0.5) s = F.expand_dims(s, axis=0) s = F.repeat(s, repeats=decode_num, axis=0) s = F.transpose(s, axes=(1, 0, 2)) x = F.concat((s, z), axis=2) x = F.reshape(x, shape=(-1, x.shape[-1])) h = self._linear3(x) h = F.relu(h) h = self._linear4(h) h = F.relu(h) h = self._linear5(h) h = F.reshape(h, shape=(-1, decode_num, h.shape[-1])) return F.tanh(h), h
def _compute_target_q_value(self, batch): with chainer.using_config('train', False), \ chainer.using_config('enable_backprop', False): (_, _, r, s_next, non_terminal) = batch r = F.reshape(r, shape=(*r.shape, 1)) non_terminal = F.reshape(non_terminal, shape=(*non_terminal.shape, 1)) s_next_rep = F.repeat(x=s_next, repeats=self._num_action_samples, axis=0) a_next_rep = self._vae._decode(s_next_rep) perturbed_action = self._target_perturbator(s_next_rep, a_next_rep) q_values = F.stack([ q_target(s_next_rep, perturbed_action) for q_target in self._target_q_ensembles ]) assert q_values.shape == (self._num_q_ensembles, self._batch_size * self._num_action_samples, 1) weighted_q_minmax = self._lambda * F.min(q_values, axis=0) \ + (1 - self._lambda) * F.max(q_values, axis=0) assert weighted_q_minmax.shape == (self._batch_size * self._num_action_samples, 1) next_q_value = F.max(F.reshape(weighted_q_minmax, shape=(self._batch_size, -1)), axis=1, keepdims=True) assert next_q_value.shape == (self._batch_size, 1) target_q_value = r + self._gamma * next_q_value * non_terminal target_q_value.unchain() assert target_q_value.shape == (self._batch_size, 1) return target_q_value
def expand_time_batch(m, time, n_batch): """ add [time, n_batch] dimension :param m: input chainer variable :param time: :param n_batch: :return: time times batch times m """ # expand two dimensions m = F.expand_dims(F.expand_dims(m, 0), 0) # repeat along batch dimension m = F.repeat(m, n_batch, axis=1) # repeat along time dimension m = F.repeat(m, time, axis=0) assert list(m.shape)[0] == time assert list(m.shape)[1] == n_batch return m
def create_2d_window(window_size, channel, xp): _1D_window = F.reshape(gaussian(window_size, 1.5, xp), (-1, 1)) _2D_window = F.reshape( F.tensordot(_1D_window, _1D_window.transpose(), axes=1), (1, 1, -1, window_size)) window = F.repeat(_2D_window, channel, axis=0) return window
def attention_layer(masked_weights): h1 = attention_layer1(masked_weights) attention_layer_output = attention_layer2(h1) attention_layer_output = F.softmax(attention_layer_output) print(attention_layer_output) attention_layer_output = F.repeat(attention_layer_output, (1, 1, 1, 1, 1, 1), axis=1) return masked_weights * attention_layer_output
def attention_layer(masked_weights): h1 = attention_layer1(masked_weights) h2 = attention_layer2(h1) attention_layer_output = attention_layer3(h2) attention_layer_output = F.softmax(attention_layer_output) rep_attention_layer_output = F.repeat(attention_layer_output, (1, 1, 1, 1, 1, 1), axis=1) return masked_weights * rep_attention_layer_output, attention_layer_output
def cycle_back_regression_fast(batch1, batch2): var_lambda = CONFIG.variance_lambda u, u_indices = batch1 v, v_indices = batch2 len_u = len(u) len_v = len(v) t = u_indices import ipdb ipdb.set_trace() if CONFIG.regression.normalize_indices: t = t / len(u) u_repeat = F.repeat(F.expand_dims(u, 0), len_v, 0).transpose(1, 0, 2) length1 = -F.sum((u_repeat - v)**2, axis=2) sim_12 = length1 / u.shape[1] sim_12 /= CONFIG.softmax_tmp alpha = F.softmax(sim_12) v_tilde = F.matmul(alpha, v) v_tilde_repeat = F.repeat(F.expand_dims(v_tilde, 0), len_u, 0).transpose(1, 0, 2) length2 = -F.sum((v_tilde_repeat - u)**2, axis=2) sim_21 = length2 / u.shape[1] sim_21 /= CONFIG.softmax_tmp beta = F.softmax(sim_21) myu = F.sum(beta * t, axis=1) var = F.sum(((beta - t)**2 * beta), axis=1) log_var = F.log(var) loss = (myu - t)**2 * F.exp(-log_var) + var_lambda * log_var return F.sum(loss)
def loss( self, class_id, quaternion_true, translation_true, quaternion_pred, translation_pred, confidence_pred, ): xp = self.xp B = class_id.shape[0] # prepare quaternion_true = quaternion_true.astype(np.float32) translation_true = translation_true.astype(np.float32) loss = 0 for i in range(B): n_point = quaternion_pred[i].shape[0] T_cad2cam_pred = morefusion.functions.transformation_matrix( quaternion_pred[i], translation_pred[i] ) # (M, 4, 4) T_cad2cam_true = morefusion.functions.transformation_matrix( quaternion_true[i], translation_true[i] ) # (4, 4) T_cad2cam_true = F.repeat(T_cad2cam_true[None], n_point, axis=0) class_id_i = int(class_id[i]) is_symmetric = ( class_id_i in morefusion.datasets.ycb_video.class_ids_symmetric ) cad_pcd = self._models.get_pcd(class_id=class_id_i) cad_pcd = xp.asarray(cad_pcd, dtype=np.float32) add = morefusion.functions.average_distance_l1( cad_pcd, T_cad2cam_true, T_cad2cam_pred, symmetric=is_symmetric, ) loss_i = F.mean( add * confidence_pred[i] - self._lambda_confidence * F.log(confidence_pred[i]) ) loss += loss_i loss /= B values = {"loss": loss} chainer.report(values, observer=self) return loss
def expand_batch(m, n_batch): """ add [n_batch] dimension :param m: input chainer variable :param n_batch: :return: batch times m """ # expand two dimensions m = F.expand_dims(m, 0) # repeat along batch dimension m = F.repeat(m, n_batch, axis=0) assert list(m.shape)[0] == n_batch return m
def batch_skew(vec, batch_size=None): """ vec is N x 3, batch_size is int returns N x 3 x 3. Skew_sym version of each matrix. """ xp = vec.xp if batch_size is None: batch_size = vec.shape[0] col_inds = xp.array([1, 2, 3, 5, 6, 7]) indices = F.reshape( F.repeat(col_inds.reshape(1, -1), batch_size, axis=0) + F.repeat(F.reshape(xp.arange(0, batch_size) * 9, [-1, 1]), 6, axis=1), [-1, 1]) updates = F.reshape( F.stack( [-vec[:, 2], vec[:, 1], vec[:, 2], -vec[:, 0], -vec[:, 1], vec[:, 0]], axis=1), [-1]) res = Variable(xp.zeros((batch_size * 3 * 3), 'f')) res.data[indices.reshape(-1).data] = updates.data res = F.reshape(res, [batch_size, 3, 3]) return res
def __call__(self, src_seq, dst_seq): """ [[9, 20, 18, ..., ], [0, 2, 101, ..., ], ... ] などIDのリストがbatch分くると想定 ただしnp.arrayで、np.int32. :param src_seq: :param dst_seq: 先頭に必ずBOSを付与 :return: 出力は(ch, height, width) """ batch_input_image = None for src, dst in zip(src_seq, dst_seq): # train用 embedded_src_tokens = self.emb_src(src) embedded_dst_tokens = self.emb_dst(dst) input_src_array = F.reshape(embedded_src_tokens, (-1, len(src), 1)) input_dst_array = F.reshape(embedded_dst_tokens, (-1, 1, len(dst))) input_src_array = F.repeat(input_src_array, len(dst), axis=2) input_dst_array = F.repeat(input_dst_array, len(src), axis=1) concat = F.reshape( F.concat([input_src_array, input_dst_array], axis=0), (1, -1, len(src), len(dst))) if not batch_input_image: batch_input_image = concat else: batch_input_image = F.concat([batch_input_image, concat], axis=0) return batch_input_image
def __call__(self, x, mask): xp = cuda.get_array_module(x) # point-wise feauture pwf = self.fcn(x) # locally aggregated feature laf = F.repeat(F.expand_dims(F.max(pwf, axis=2), 2), self.T, axis=2) # point-wise concat feature pwcf = F.concat((pwf, laf), axis=-1) # apply mask mask = xp.expand_dims(mask, -1).repeat(self.units * 2, axis=-1) pwcf *= mask return pwcf
def forward(self, x): b_ = cf.tanh(self.b) # Not sure if implementation is wrong m_ = cf.softplus(self.m) # m = cf.repeat(m, 8, axis=2) # m = cf.repeat(m, 8, axis=1) m_ = cf.repeat(m_, 16, axis=2) m_ = cf.repeat(m_, 16, axis=1) b_ = b_ * m_ x_ = cf.add(x, b_) x_ = cf.clip(x_, -0.5, 0.5) z = [] zs, logdet = self.encoder.forward_step(x_) for (zi, mean, ln_var) in zs: z.append(zi) z = merge_factorized_z(z) return z, zs, logdet, xp.sum(xp.abs(b_.data)), xp.tanh( self.b.data * 1), m_, x_
def gradient_correlation(x, t, normalize=True, absolute=False): assert x.ndim == t.ndim xp = chainer.cuda.get_array_module(x) n_grad_dim = x.ndim - 2 gc = [] for i in range(n_grad_dim): kernel_shape = tuple(np.roll((3, ) + (1, ) * (n_grad_dim - 1), shift=i)) w = xp.array([-1.0, 0.0, 1.0]).reshape(( 1, 1, ) + kernel_shape) x_grad = F.convolution_nd(x, w) t_grad = F.convolution_nd(t, w) x_grad_mean = F.mean(x_grad, axis=tuple(range(1, x.ndim))) t_grad_mean = F.mean(t_grad, axis=tuple(range(1, x.ndim))) repeat_shape = (1, ) + x_grad.shape[1:] x_grad_mean = F.reshape( F.repeat(x_grad_mean, functools.reduce(lambda x, y: x * y, repeat_shape)), x_grad.shape) t_grad_mean = F.reshape( F.repeat(t_grad_mean, functools.reduce(lambda x, y: x * y, repeat_shape)), x_grad.shape) x_norm_grad = x_grad - x_grad_mean t_norm_grad = t_grad - t_grad_mean gc.append(1.0 - F.sum(x_norm_grad * t_norm_grad) / (F.sqrt(F.sum(x_norm_grad**2)) * F.sqrt(F.sum(t_norm_grad**2)) + 1e-8)) return F.absolute(sum(gc) / len(gc))
def __call__(self, task_encoding=None, image_encoding=None, data_out=None, seed=None, return_sample=False, train=True): with chainer.using_config('train', train), chainer.using_config( 'enable_backprop', train): xp = cuda.cupy sequence_len = len(image_encoding) x = image_encoding y = data_out if self.AUTO_REGRESSIVE: y = F.swapaxes(y, 0, 1) y = F.reshape(y, (y.shape[0], y.shape[1] * y.shape[2], 1)) y = F.swapaxes(y, 0, 1) x = F.swapaxes(x, 0, 1) x = F.repeat(x, (1, self.OUT_DIM, 1)) x = F.swapaxes(x, 0, 1) sequence_len = x.shape[0] cost = 0 # if type(seed) == Variable: # last_joint = seed # else: last_joint = Variable(xp.zeros(y.shape[1:], dtype=np.float32)) sample_toRet = np.zeros(self.OUT_DIM, dtype=np.float32) for j in range(sequence_len): batch_cost, sample = self.process_batch( x[j], y[j], task_encoding, last_joint, return_sample=return_sample) last_joint = y[j] if self.AUTO_REGRESSIVE: sample_toRet[j % self.OUT_DIM] = cuda.to_cpu( sample.data)[0, 0] cost += batch_cost cost /= x.shape[0] if self.AUTO_REGRESSIVE: sample = sample_toRet return cost, sample
def read(self, k, b): # k: (1, M_DIM*kr), b: (1, kr) kr = b.shape[1] K = k.reshape(kr, M_DIM) C = F.matmul(F.normalize(K + EPS), F.transpose(F.normalize(self.M + EPS))) B = F.repeat(b, N_mem).reshape(kr, N_mem) # beta if kr == Kr: self.W_predictor = F.softmax(B * C) # B*C: elementwise multiplication M = F.matmul(self.W_predictor, self.M) elif kr == Krp: self.W_policy = F.softmax(B * C) M = F.matmul(self.W_policy, self.M) else: raise (ValueError) return M.reshape((1, -1))
def average_distance(points, transform_true, transforms_pred, symmetric=False): """Translation introduced pose_loss proposed in DenseFusion paper. Original pose_loss looks like below: .. math:: m = |M| PLOSS(\\tilde{q}, q) = 1 / m \\sum_{x \\in M} | R(\\tilde{q})x - R(q)x | where M is set of point_xyz, q~ and q are quaternion, R(q~) and R(q) are rotation matrix of ground truth and predicted, and m is size of the set M. If we introduce translation here, it will be: .. math:: PLOSS2(\\tilde{T}, T) = 1 / m \\sum_{x \\in M} | \\tilde{T}x - Tx | """ n_points = points.shape[0] n_pred = transforms_pred.shape[0] assert points.shape == (n_points, 3) assert transform_true.shape == (4, 4) assert transforms_pred.shape == (n_pred, 4, 4) points_true = transform_points(points, transform_true) points_pred = transform_points(points, transforms_pred) assert points_true.shape == (n_points, 3) assert points_pred.shape == (n_pred, n_points, 3) if symmetric: ref = points_true.array query = points_pred.array.reshape(n_pred * n_points, 3) indices = geometry_module.nn(ref, query) points_true = points_true[indices] points_true = points_true.reshape(n_pred, n_points, 3) else: points_true = F.repeat(points_true[None], n_pred, axis=0) return F.mean( F.sqrt(F.sum((points_true - points_pred) ** 2, axis=2)), axis=1 )
def prepare(self, images, downscale=1): if self.use_hist_eq: # Histogram equalization images = images.astype(int, copy=False) values, counts = self.xp.unique(images, return_counts=True) p = self.xp.zeros(256, dtype=int) p[values] = counts d1 = 255 / images.size * p.cumsum().astype(np.float32, copy=False) images = d1[images] images = F.repeat(images, 3, axis=1) if downscale > 1: output_shape = [x // downscale for x in images.shape[2:]] images = F.resize_images(images, tuple(output_shape)) images -= self.xp.array([103.063, 115.903, 123.152], dtype=np.float32).reshape(1, 3, 1, 1) return images
def __call__(self, h_rgb, pcd): B, _, n_point = h_rgb.shape # conv1 h_rgb = F.relu(self.conv1_rgb(h_rgb)) h_pcd = F.relu(self.conv1_pcd(pcd)) feat1 = F.concat((h_rgb, h_pcd), axis=1) # conv2 h_rgb = F.relu(self.conv2_rgb(h_rgb)) h_pcd = F.relu(self.conv2_pcd(h_pcd)) feat2 = F.concat((h_rgb, h_pcd), axis=1) # conv3, conv4 h = F.relu(self.conv3(feat2)) h = F.relu(self.conv4(h)) h = F.average_pooling_1d(h, n_point) h = h.reshape((B, 1024, 1)) feat3 = F.repeat(h, n_point, axis=2) feat = F.concat((feat1, feat2, feat3), axis=1) return feat
def __call__(self, h_img, pcd): B = h_img.shape[0] # conv1 h_img = F.relu(self.conv1_img(h_img)) h_pcd = F.relu(self.conv1_pcd(pcd)) feat1 = F.concat((h_pcd, h_img), axis=1) # conv2 h_img = F.relu(self.conv2_img(h_img)) h_pcd = F.relu(self.conv2_pcd(h_pcd)) feat2 = F.concat((h_pcd, h_img), axis=1) # conv3, conv4 h = F.relu(self.conv3(feat2)) h = F.relu(self.conv4(h)) h = F.average_pooling_1d(h, self.n_point) h = h.reshape((B, 1024, 1)) feat3 = F.repeat(h, self.n_point, axis=2) feat = F.concat((feat1, feat2, feat3), axis=1) return feat
def __call__(self, beta, theta, get_skin=False, with_a=False): batch_size = beta.shape[0] # 1. Add shape blend shapes # (N x 10) x (10 x 6890*3) = N x 6890 x 3 self.beta_shapedirs = F.matmul(beta, self.shapedirs) v_shaped = F.reshape( F.matmul(beta, self.shapedirs), [-1, self.size[0], self.size[1]]) + \ F.repeat(self.v_template[None, ], batch_size, axis=0) self.v_shaped = v_shaped # 2. Infer shape-dependent joint locations. Jx = F.matmul(v_shaped[:, :, 0], self.J_regressor) Jy = F.matmul(v_shaped[:, :, 1], self.J_regressor) Jz = F.matmul(v_shaped[:, :, 2], self.J_regressor) J = F.stack([Jx, Jy, Jz], axis=2) self.J = J # 3. Add pose blend shapes # N x 24 x 3 x 3 Rs = F.reshape( batch_rodrigues(F.reshape(theta, [-1, 3])), [-1, 24, 3, 3]) self.Rs = Rs # Ignore global rotation. pose_feature = F.reshape(Rs[:, 1:, :, :] - F.repeat(F.repeat(Variable(self.xp.array(self.xp.eye(3), 'f'))[ None, ], 23, axis=0)[None, ], batch_size, axis=0), [-1, 207]) self.pose_feature = pose_feature # (N x 207) x (207, 20670) -> N x 6890 x 3 v_posed = F.reshape( F.matmul(pose_feature, self.posedirs), [-1, self.size[0], self.size[1]]) + v_shaped # 4. Get the global joint location self.J_transformed, A = batch_global_rigid_transformation( Rs, J, self.parents) # 5. Do skinning: # W is N x 6890 x 24 W = F.reshape( F.tile(self.weights, (batch_size, 1)), [batch_size, -1, 24]) # (N x 6890 x 24) x (N x 24 x 16) T = F.reshape( F.matmul(W, F.reshape(A, [batch_size, 24, 16])), [batch_size, -1, 4, 4]) v_posed_homo = F.concat( [v_posed, self.xp.ones([batch_size, v_posed.shape[1], 1], 'f')], 2) v_homo = F.matmul(T, F.expand_dims(v_posed_homo, -1)) verts = v_homo[:, :, :3, 0] # Get cocoplus or lsp joints: joint_x = F.matmul(verts[:, :, 0], self.joint_regressor) joint_y = F.matmul(verts[:, :, 1], self.joint_regressor) joint_z = F.matmul(verts[:, :, 2], self.joint_regressor) joints = F.stack([joint_x, joint_y, joint_z], axis=2) return verts, joints, Rs, A
def test_value_error(self): x = numpy.random.uniform(-1, 1, (2,)).astype(numpy.float32) with self.assertRaises(ValueError): functions.repeat(x, self.repeats, self.axis)
def test_type_error_axis_bool(self): x = numpy.random.uniform(-1, 1, (2,)).astype(numpy.float32) with self.assertRaises(TypeError): functions.repeat(x, 1, True)
def f(x): return functions.repeat(x, self.repeats, self.axis)
def test_type_error_repeats_str(self): x = numpy.random.uniform(-1, 1, (2,)).astype(numpy.float32) with self.assertRaises(TypeError): functions.repeat(x, 'a')
def f(x): y = functions.repeat(x, self.repeats, self.axis) return y * y
def forward(self, inputs, devices): x, = inputs y = functions.repeat(x, self.repeats, self.axis) return y,
def check_forward(self, x_data): y = functions.repeat(x_data, self.repeats, self.axis) y_expected = self._repeat(self.x, self.repeats, self.axis) self.assertEqual(y.dtype, y_expected.dtype) testing.assert_allclose( y.data, y_expected, **self.check_forward_options)