def bbox_iou(box1, box2, x1y1x2y2=True): """ Returns the IoU of two bounding boxes """ if not x1y1x2y2: # Transform from center and width to exact coordinates b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2 b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2 b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2 b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2 else: # Get the coordinates of bounding boxes b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3] b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3] # get the corrdinates of the intersection rectangle inter_rect_x1 = torch.max(b1_x1, b2_x1) inter_rect_y1 = torch.max(b1_y1, b2_y1) inter_rect_x2 = torch.min(b1_x2, b2_x2) inter_rect_y2 = torch.min(b1_y2, b2_y2) # Intersection area inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp( inter_rect_y2 - inter_rect_y1 + 1, min=0 ) # Union Area b1_area = (b1_x2 - b1_x1 + 1) * (b1_y2 - b1_y1 + 1) b2_area = (b2_x2 - b2_x1 + 1) * (b2_y2 - b2_y1 + 1) iou = inter_area / (b1_area + b2_area - inter_area + 1e-16) return iou
def F_bilinear_interp2d(input, coords): """ bilinear interpolation of 2d torch.autograd.Variable """ x = torch.clamp(coords[:,:,0], 0, input.size(1)-2) x0 = x.floor() x1 = x0 + 1 y = torch.clamp(coords[:,:,1], 0, input.size(2)-2) y0 = y.floor() y1 = y0 + 1 stride = torch.LongTensor(input.stride()) x0_ix = x0.mul(stride[1]).long() x1_ix = x1.mul(stride[1]).long() y0_ix = y0.mul(stride[2]).long() y1_ix = y1.mul(stride[2]).long() input_flat = input.view(input.size(0),-1).contiguous() vals_00 = input_flat.gather(1, x0_ix.add(y0_ix).detach()) vals_10 = input_flat.gather(1, x1_ix.add(y0_ix).detach()) vals_01 = input_flat.gather(1, x0_ix.add(y1_ix).detach()) vals_11 = input_flat.gather(1, x1_ix.add(y1_ix).detach()) xd = x - x0 yd = y - y0 xm = 1 - xd ym = 1 - yd x_mapped = (vals_00.mul(xm).mul(ym) + vals_10.mul(xd).mul(ym) + vals_01.mul(xm).mul(yd) + vals_11.mul(xd).mul(yd)) return x_mapped.view_as(input)
def bbox_iou(box1, box2): """ Returns the IoU of two bounding boxes """ #Get the coordinates of bounding boxes b1_x1, b1_y1, b1_x2, b1_y2 = box1[:,0], box1[:,1], box1[:,2], box1[:,3] b2_x1, b2_y1, b2_x2, b2_y2 = box2[:,0], box2[:,1], box2[:,2], box2[:,3] #get the corrdinates of the intersection rectangle inter_rect_x1 = torch.max(b1_x1, b2_x1) inter_rect_y1 = torch.max(b1_y1, b2_y1) inter_rect_x2 = torch.min(b1_x2, b2_x2) inter_rect_y2 = torch.min(b1_y2, b2_y2) #Intersection area inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(inter_rect_y2 - inter_rect_y1 + 1, min=0) #Union Area b1_area = (b1_x2 - b1_x1 + 1)*(b1_y2 - b1_y1 + 1) b2_area = (b2_x2 - b2_x1 + 1)*(b2_y2 - b2_y1 + 1) iou = inter_area / (b1_area + b2_area - inter_area) return iou
def sample_from_discretized_mix_logistic_1d(l, nr_mix): # Pytorch ordering l = l.permute(0, 2, 3, 1) ls = [int(y) for y in l.size()] xs = ls[:-1] + [1] #[3] # unpack parameters logit_probs = l[:, :, :, :nr_mix] l = l[:, :, :, nr_mix:].contiguous().view(xs + [nr_mix * 2]) # for mean, scale # sample mixture indicator from softmax temp = torch.FloatTensor(logit_probs.size()) if l.is_cuda : temp = temp.cuda() temp.uniform_(1e-5, 1. - 1e-5) temp = logit_probs.data - torch.log(- torch.log(temp)) _, argmax = temp.max(dim=3) one_hot = to_one_hot(argmax, nr_mix) sel = one_hot.view(xs[:-1] + [1, nr_mix]) # select logistic parameters means = torch.sum(l[:, :, :, :, :nr_mix] * sel, dim=4) log_scales = torch.clamp(torch.sum( l[:, :, :, :, nr_mix:2 * nr_mix] * sel, dim=4), min=-7.) u = torch.FloatTensor(means.size()) if l.is_cuda : u = u.cuda() u.uniform_(1e-5, 1. - 1e-5) u = Variable(u) x = means + torch.exp(log_scales) * (torch.log(u) - torch.log(1. - u)) x0 = torch.clamp(torch.clamp(x[:, :, :, 0], min=-1.), max=1.) out = x0.unsqueeze(1) return out
def forward(self, input) -> torch.FloatTensor: """ Preprocess the input matrix :param input tensor """ if isinstance(input, np.ndarray): input = torch.from_numpy(input).type(self.dtype) if isinstance(input, rlt.FeatureVector): input = input.float_features.type(self.dtype) # ONNX doesn't support != yet not_missing_input = ( self.one_tensor.float() - (input == self.missing_tensor).float() ) feature_starts = self._get_type_boundaries() outputs = [] for i, feature_type in enumerate(FEATURE_TYPES): begin_index = feature_starts[i] if (i + 1) == len(FEATURE_TYPES): end_index = len(self.normalization_parameters) else: end_index = feature_starts[i + 1] if begin_index == end_index: continue # No features of this type if feature_type == ENUM: # Process one-at-a-time for j in range(begin_index, end_index): norm_params = self.normalization_parameters[self.sorted_features[j]] new_output = self._preprocess_feature_single_column( j, input[:, j : j + 1], norm_params ) new_output *= not_missing_input[:, j : j + 1] self._check_preprocessing_output(new_output, [norm_params]) outputs.append(new_output) else: norm_params = [] for f in self.sorted_features[begin_index:end_index]: norm_params.append(self.normalization_parameters[f]) new_output = self._preprocess_feature_multi_column( begin_index, input[:, begin_index:end_index], norm_params ) new_output *= not_missing_input[:, begin_index:end_index] self._check_preprocessing_output(new_output, norm_params) outputs.append(new_output) def wrap(output): if self.typed_output: return rlt.FeatureVector(float_features=output) else: return output if len(outputs) == 1: return wrap(torch.clamp(outputs[0], MIN_FEATURE_VALUE, MAX_FEATURE_VALUE)) return wrap( torch.clamp(torch.cat(outputs, dim=1), MIN_FEATURE_VALUE, MAX_FEATURE_VALUE) )
def train_actor_critic(actor, critic, memory, actor_optim, critic_optim, args): memory = np.array(memory) states = np.vstack(memory[:, 0]) actions = list(memory[:, 1]) rewards = list(memory[:, 2]) masks = list(memory[:, 3]) old_values = critic(torch.Tensor(states)) returns, advants = get_gae(rewards, masks, old_values, args) mu, std = actor(torch.Tensor(states)) old_policy = log_prob_density(torch.Tensor(actions), mu, std) criterion = torch.nn.MSELoss() n = len(states) arr = np.arange(n) for _ in range(args.ppo_update_num): np.random.shuffle(arr) for i in range(n // args.batch_size): batch_index = arr[args.batch_size * i : args.batch_size * (i + 1)] batch_index = torch.LongTensor(batch_index) inputs = torch.Tensor(states)[batch_index] actions_samples = torch.Tensor(actions)[batch_index] returns_samples = returns.unsqueeze(1)[batch_index] advants_samples = advants.unsqueeze(1)[batch_index] oldvalue_samples = old_values[batch_index].detach() values = critic(inputs) clipped_values = oldvalue_samples + \ torch.clamp(values - oldvalue_samples, -args.clip_param, args.clip_param) critic_loss1 = criterion(clipped_values, returns_samples) critic_loss2 = criterion(values, returns_samples) critic_loss = torch.max(critic_loss1, critic_loss2).mean() loss, ratio, entropy = surrogate_loss(actor, advants_samples, inputs, old_policy.detach(), actions_samples, batch_index) clipped_ratio = torch.clamp(ratio, 1.0 - args.clip_param, 1.0 + args.clip_param) clipped_loss = clipped_ratio * advants_samples actor_loss = -torch.min(loss, clipped_loss).mean() loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy critic_optim.zero_grad() loss.backward(retain_graph=True) critic_optim.step() actor_optim.zero_grad() loss.backward() actor_optim.step()
def forward(self, x=None, warmup=1., inf_net=None): #, k=1): #, marginf_type=0): outputs = {} B = x.shape[0] if inf_net is None: # mu, logvar = self.inference_net(x) z, logits = self.q.sample(x) else: # mu, logvar = inf_net.inference_net(x) z, logqz = inf_net.sample(x) # print (z[0]) # b = harden(z) # print (b[0]) # logpz = torch.sum( self.prior.log_prob(b), dim=1) # print (logpz[0]) # print (logpz.shape) # fdasf probs_q = torch.sigmoid(logits) probs_q = torch.clamp(probs_q, min=.00000001, max=.9999999) probs_p = torch.ones(B, self.z_size).cuda() *.5 KL = probs_q*torch.log(probs_q/probs_p) + (1-probs_q)*torch.log((1-probs_q)/(1-probs_p)) KL = torch.sum(KL, dim=1) # print (z.shape) # Decode Image x_hat = self.generator.forward(z) alpha = torch.sigmoid(x_hat) beta = Beta(alpha*self.beta_scale, (1.-alpha)*self.beta_scale) x_noise = torch.clamp(x + torch.FloatTensor(x.shape).uniform_(0., 1./256.).cuda(), min=1e-5, max=1-1e-5) logpx = beta.log_prob(x_noise) #[120,3,112,112] # add uniform noise here logpx = torch.sum(logpx.view(B, -1),1) # [PB] * self.w_logpx # print (logpx.shape,logpz.shape,logqz.shape) # fsdfda log_ws = logpx - KL #+ logpz - logqz outputs['logpx'] = torch.mean(logpx) outputs['x_recon'] = alpha # outputs['welbo'] = torch.mean(logpx + warmup*( logpz - logqz)) outputs['welbo'] = torch.mean(logpx + warmup*(KL)) outputs['elbo'] = torch.mean(log_ws) outputs['logws'] = log_ws outputs['z'] = z outputs['logpz'] = torch.zeros(1) #torch.mean(logpz) outputs['logqz'] = torch.mean(KL) # outputs['logvar'] = logvar return outputs
def F_trilinear_interp3d(input, coords): """ trilinear interpolation of 3D image """ # take clamp then floor/ceil of x coords x = torch.clamp(coords[:,0], 0, input.size(1)-2) x0 = x.floor() x1 = x0 + 1 # take clamp then floor/ceil of y coords y = torch.clamp(coords[:,1], 0, input.size(2)-2) y0 = y.floor() y1 = y0 + 1 # take clamp then floor/ceil of z coords z = torch.clamp(coords[:,2], 0, input.size(3)-2) z0 = z.floor() z1 = z0 + 1 stride = torch.LongTensor(input.stride())[1:] x0_ix = x0.mul(stride[0]).long() x1_ix = x1.mul(stride[0]).long() y0_ix = y0.mul(stride[1]).long() y1_ix = y1.mul(stride[1]).long() z0_ix = z0.mul(stride[2]).long() z1_ix = z1.mul(stride[2]).long() input_flat = th_flatten(input) vals_000 = input_flat[x0_ix.add(y0_ix).add(z0_ix).detach()] vals_100 = input_flat[x1_ix.add(y0_ix).add(z0_ix).detach()] vals_010 = input_flat[x0_ix.add(y1_ix).add(z0_ix).detach()] vals_001 = input_flat[x0_ix.add(y0_ix).add(z1_ix).detach()] vals_101 = input_flat[x1_ix.add(y0_ix).add(z1_ix).detach()] vals_011 = input_flat[x0_ix.add(y1_ix).add(z1_ix).detach()] vals_110 = input_flat[x1_ix.add(y1_ix).add(z0_ix).detach()] vals_111 = input_flat[x1_ix.add(y1_ix).add(z1_ix).detach()] xd = x - x0 yd = y - y0 zd = z - z0 xm = 1 - xd ym = 1 - yd zm = 1 - zd x_mapped = (vals_000.mul(xm).mul(ym).mul(zm) + vals_100.mul(xd).mul(ym).mul(zm) + vals_010.mul(xm).mul(yd).mul(zm) + vals_001.mul(xm).mul(ym).mul(zd) + vals_101.mul(xd).mul(ym).mul(zd) + vals_011.mul(xm).mul(yd).mul(zd) + vals_110.mul(xd).mul(yd).mul(zm) + vals_111.mul(xd).mul(yd).mul(zd)) return x_mapped.view_as(input)
def F_batch_trilinear_interp3d(input, coords): """ input : torch.Tensor size = (N,H,W,C) coords : torch.Tensor size = (N,H*W*C,2) """ x = torch.clamp(coords[:,:,0], 0, input.size(2)-2) x0 = x.floor() x1 = x0 + 1 y = torch.clamp(coords[:,:,1], 0, input.size(3)-2) y0 = y.floor() y1 = y0 + 1 z = torch.clamp(coords[:,:,2], 0, input.size(4)-2) z0 = z.floor() z1 = z0 + 1 stride = torch.LongTensor(input.stride()) x0_ix = x0.mul(stride[2]).long() x1_ix = x1.mul(stride[2]).long() y0_ix = y0.mul(stride[3]).long() y1_ix = y1.mul(stride[3]).long() z0_ix = z0.mul(stride[4]).long() z1_ix = z1.mul(stride[4]).long() input_flat = input.contiguous().view(input.size(0),-1) vals_000 = input_flat.gather(1,x0_ix.add(y0_ix).add(z0_ix).detach()) vals_100 = input_flat.gather(1,x1_ix.add(y0_ix).add(z0_ix).detach()) vals_010 = input_flat.gather(1,x0_ix.add(y1_ix).add(z0_ix).detach()) vals_001 = input_flat.gather(1,x0_ix.add(y0_ix).add(z1_ix).detach()) vals_101 = input_flat.gather(1,x1_ix.add(y0_ix).add(z1_ix).detach()) vals_011 = input_flat.gather(1,x0_ix.add(y1_ix).add(z1_ix).detach()) vals_110 = input_flat.gather(1,x1_ix.add(y1_ix).add(z0_ix).detach()) vals_111 = input_flat.gather(1,x1_ix.add(y1_ix).add(z1_ix).detach()) xd = x - x0 yd = y - y0 zd = z - z0 xm = 1 - xd ym = 1 - yd zm = 1 - zd x_mapped = (vals_000.mul(xm).mul(ym).mul(zm) + vals_100.mul(xd).mul(ym).mul(zm) + vals_010.mul(xm).mul(yd).mul(zm) + vals_001.mul(xm).mul(ym).mul(zd) + vals_101.mul(xd).mul(ym).mul(zd) + vals_011.mul(xm).mul(yd).mul(zd) + vals_110.mul(xd).mul(yd).mul(zm) + vals_111.mul(xd).mul(yd).mul(zd)) return x_mapped.view_as(input)
def calculate_variance_term(pred, gt, means, n_objects, delta_v, norm=2): """pred: bs, height * width, n_filters gt: bs, height * width, n_instances means: bs, n_instances, n_filters""" bs, n_loc, n_filters = pred.size() n_instances = gt.size(2) # bs, n_loc, n_instances, n_filters means = means.unsqueeze(1).expand(bs, n_loc, n_instances, n_filters) # bs, n_loc, n_instances, n_filters pred = pred.unsqueeze(2).expand(bs, n_loc, n_instances, n_filters) # bs, n_loc, n_instances, n_filters gt = gt.unsqueeze(3).expand(bs, n_loc, n_instances, n_filters) _var = (torch.clamp(torch.norm((pred - means), norm, 3) - delta_v, min=0.0) ** 2) * gt[:, :, :, 0] var_term = 0.0 for i in range(bs): _var_sample = _var[i, :, :n_objects[i]] # n_loc, n_objects _gt_sample = gt[i, :, :n_objects[i], 0] # n_loc, n_objects var_term += torch.sum(_var_sample) / torch.sum(_gt_sample) var_term = var_term / bs return var_term
def calculate_distance_term(means, n_objects, delta_d, norm=2, usegpu=True): """means: bs, n_instances, n_filters""" bs, n_instances, n_filters = means.size() dist_term = 0.0 for i in range(bs): _n_objects_sample = n_objects[i] if _n_objects_sample <= 1: continue _mean_sample = means[i, : _n_objects_sample, :] # n_objects, n_filters means_1 = _mean_sample.unsqueeze(1).expand( _n_objects_sample, _n_objects_sample, n_filters) means_2 = means_1.permute(1, 0, 2) diff = means_1 - means_2 # n_objects, n_objects, n_filters _norm = torch.norm(diff, norm, 2) margin = 2 * delta_d * (1.0 - torch.eye(_n_objects_sample)) if usegpu: margin = margin.cuda() margin = Variable(margin) _dist_term_sample = torch.sum( torch.clamp(margin - _norm, min=0.0) ** 2) _dist_term_sample = _dist_term_sample / \ (_n_objects_sample * (_n_objects_sample - 1)) dist_term += _dist_term_sample dist_term = dist_term / bs return dist_term
def encode(self, x): # x = x.view(-1, 1, self.x_size, self.x_size) # print (x.shape) x = self.act_func(self.conv1(x)) # print (x.shape) x = self.act_func(self.conv2(x)) x = self.act_func(self.conv3(x)) # print (x.size()) x = x.view(-1, self.intermediate_size) h1 = self.act_func(self.fc1(x)) h2 = self.fc2(h1) mean = h2[:,:self.z_size] logvar = h2[:,self.z_size:] #this solves the nan grad problem. logvar = torch.clamp(logvar, min=-20.) self.mean = mean self.logvar = logvar return mean, logvar
def inference_net(self, x): mean_logvar = self.image_encoder2(x) mean = mean_logvar[:,:6] logvar = mean_logvar[:,6:6*2] xenc = mean_logvar[:,6*2:] logvar = torch.clamp(logvar, min=-15., max=10.) return mean, logvar, xenc
def _get_state_cost(self, worlds: List[WikiTablesWorld], state: CoverageState) -> torch.Tensor: if not state.is_finished(): raise RuntimeError("_get_state_cost() is not defined for unfinished states!") world = worlds[state.batch_indices[0]] # Our checklist cost is a sum of squared error from where we want to be, making sure we # take into account the mask. We clamp the lower limit of the balance at 0 to avoid # penalizing agenda actions produced multiple times. checklist_balance = torch.clamp(state.checklist_state[0].get_balance(), min=0.0) checklist_cost = torch.sum((checklist_balance) ** 2) # This is the number of items on the agenda that we want to see in the decoded sequence. # We use this as the denotation cost if the path is incorrect. denotation_cost = torch.sum(state.checklist_state[0].checklist_target.float()) checklist_cost = self._checklist_cost_weight * checklist_cost action_history = state.action_history[0] batch_index = state.batch_indices[0] action_strings = [state.possible_actions[batch_index][i][0] for i in action_history] logical_form = world.get_logical_form(action_strings) lisp_string = state.extras[batch_index] if self._executor.evaluate_logical_form(logical_form, lisp_string): cost = checklist_cost else: cost = checklist_cost + (1 - self._checklist_cost_weight) * denotation_cost return cost
def log_Bernoulli(x, mean, average=False, dim=None): probs = torch.clamp( mean, min=min_epsilon, max=max_epsilon ) log_bernoulli = x * torch.log( probs ) + (1. - x ) * torch.log( 1. - probs ) if average: return torch.mean( log_bernoulli, dim ) else: return torch.sum( log_bernoulli, dim )
def pdist(self, fX): """Compute pdist à-la scipy.spatial.distance.pdist Parameters ---------- fX : (n, d) torch.Tensor Embeddings. Returns ------- distances : (n * (n-1) / 2,) torch.Tensor Condensed pairwise distance matrix """ n_sequences, _ = fX.size() distances = [] for i in range(n_sequences - 1): if self.metric in ('cosine', 'angular'): d = 1. - F.cosine_similarity( fX[i, :].expand(n_sequences - 1 - i, -1), fX[i+1:, :], dim=1, eps=1e-8) if self.metric == 'angular': d = torch.acos(torch.clamp(1. - d, -1 + 1e-6, 1 - 1e-6)) elif self.metric == 'euclidean': d = F.pairwise_distance( fX[i, :].expand(n_sequences - 1 - i, -1), fX[i+1:, :], p=2, eps=1e-06).view(-1) distances.append(d) return torch.cat(distances)
def get_triplet_loss(image_a_pred, image_b_pred, matches_a, matches_b, non_matches_a, non_matches_b, alpha): """ Computes the loss function \sum_{triplets} ||D(I_a, u_a, I_b, u_{b,match})||_2^2 - ||D(I_a, u_a, I_b, u_{b,non-match)||_2^2 + alpha """ num_matches = matches_a.size()[0] num_non_matches = non_matches_a.size()[0] multiplier = num_non_matches / num_matches ## non_matches_a is already replicated up to be the right size ## non_matches_b is also that side ## matches_a is just a smaller version of non_matches_a ## matches_b is the only thing that needs to be replicated up in size matches_b_long = torch.t(matches_b.repeat(multiplier, 1)).contiguous().view(-1) matches_a_descriptors = torch.index_select(image_a_pred, 1, non_matches_a) matches_b_descriptors = torch.index_select(image_b_pred, 1, matches_b_long) non_matches_b_descriptors = torch.index_select(image_b_pred, 1, non_matches_b) triplet_losses = (matches_a_descriptors - matches_b_descriptors).pow(2) - (matches_a_descriptors - non_matches_b_descriptors).pow(2) + alpha triplet_loss = 1.0 / num_non_matches * torch.clamp(triplet_losses, min=0).sum() return triplet_loss
def l2_pixel_loss(self, matches_b, non_matches_b, M_pixel=None): """ Apply l2 loss in pixel space. This weights non-matches more if they are "far away" in pixel space. :param matches_b: A torch.LongTensor with shape torch.Shape([num_matches]) :param non_matches_b: A torch.LongTensor with shape torch.Shape([num_non_matches]) :return l2 loss per sample: A torch.FloatTensorof with shape torch.Shape([num_matches]) """ if M_pixel is None: M_pixel = self._config['M_pixel'] num_non_matches_per_match = len(non_matches_b)/len(matches_b) ground_truth_pixels_for_non_matches_b = torch.t(matches_b.repeat(num_non_matches_per_match,1)).contiguous().view(-1,1) ground_truth_u_v_b = self.flattened_pixel_locations_to_u_v(ground_truth_pixels_for_non_matches_b) sampled_u_v_b = self.flattened_pixel_locations_to_u_v(non_matches_b.unsqueeze(1)) # each element is always within [0,1], you have 1 if you are at least M_pixel away in # L2 norm in pixel space norm_degree = 2 squared_l2_pixel_loss = 1.0/M_pixel * torch.clamp((ground_truth_u_v_b - sampled_u_v_b).float().norm(norm_degree,1), max=M_pixel) return squared_l2_pixel_loss, ground_truth_u_v_b, sampled_u_v_b
def hinge_loss(positive_predictions, negative_predictions, mask=None): """ Hinge pairwise loss function. Parameters ---------- positive_predictions: tensor Tensor containing predictions for known positive items. negative_predictions: tensor Tensor containing predictions for sampled negative items. mask: tensor, optional A binary tensor used to zero the loss from some entries of the loss tensor. Returns ------- loss, float The mean value of the loss function. """ loss = torch.clamp(negative_predictions - positive_predictions + 1.0, 0.0) if mask is not None: mask = mask.float() loss = loss * mask return loss.sum() / mask.sum() return loss.mean()
def project_to_2d(X, camera_params): """ Project 3D points to 2D using the Human3.6M camera projection function. This is a differentiable and batched reimplementation of the original MATLAB script. Arguments: X -- 3D points in *camera space* to transform (N, *, 3) camera_params -- intrinsic parameteres (N, 2+2+3+2=9) """ assert X.shape[-1] == 3 assert len(camera_params.shape) == 2 assert camera_params.shape[-1] == 9 assert X.shape[0] == camera_params.shape[0] while len(camera_params.shape) < len(X.shape): camera_params = camera_params.unsqueeze(1) f = camera_params[..., :2] c = camera_params[..., 2:4] k = camera_params[..., 4:7] p = camera_params[..., 7:] XX = torch.clamp(X[..., :2] / X[..., 2:], min=-1, max=1) r2 = torch.sum(XX[..., :2]**2, dim=len(XX.shape)-1, keepdim=True) radial = 1 + torch.sum(k * torch.cat((r2, r2**2, r2**3), dim=len(r2.shape)-1), dim=len(r2.shape)-1, keepdim=True) tan = torch.sum(p*XX, dim=len(XX.shape)-1, keepdim=True) XXX = XX*(radial + tan) + p*r2 return f*XXX + c
def discretized_mix_logistic_loss_1d(x, l): """ log-likelihood for mixture of discretized logistics, assumes the data has been rescaled to [-1,1] interval """ # Pytorch ordering x = x.permute(0, 2, 3, 1) l = l.permute(0, 2, 3, 1) xs = [int(y) for y in x.size()] ls = [int(y) for y in l.size()] # here and below: unpacking the params of the mixture of logistics nr_mix = int(ls[-1] / 3) logit_probs = l[:, :, :, :nr_mix] l = l[:, :, :, nr_mix:].contiguous().view(xs + [nr_mix * 2]) # 2 for mean, scale means = l[:, :, :, :, :nr_mix] log_scales = torch.clamp(l[:, :, :, :, nr_mix:2 * nr_mix], min=-7.) # here and below: getting the means and adjusting them based on preceding # sub-pixels x = x.contiguous() x = x.unsqueeze(-1) + Variable(torch.zeros(xs + [nr_mix]).cuda(), requires_grad=False) # means = torch.cat((means[:, :, :, 0, :].unsqueeze(3), m2, m3), dim=3) centered_x = x - means inv_stdv = torch.exp(-log_scales) plus_in = inv_stdv * (centered_x + 1. / 255.) cdf_plus = F.sigmoid(plus_in) min_in = inv_stdv * (centered_x - 1. / 255.) cdf_min = F.sigmoid(min_in) # log probability for edge case of 0 (before scaling) log_cdf_plus = plus_in - F.softplus(plus_in) # log probability for edge case of 255 (before scaling) log_one_minus_cdf_min = -F.softplus(min_in) cdf_delta = cdf_plus - cdf_min # probability for all other cases mid_in = inv_stdv * centered_x # log probability in the center of the bin, to be used in extreme cases # (not actually used in our code) log_pdf_mid = mid_in - log_scales - 2. * F.softplus(mid_in) inner_inner_cond = (cdf_delta > 1e-5).float() inner_inner_out = inner_inner_cond * torch.log(torch.clamp(cdf_delta, min=1e-12)) + (1. - inner_inner_cond) * (log_pdf_mid - np.log(127.5)) inner_cond = (x > 0.999).float() inner_out = inner_cond * log_one_minus_cdf_min + (1. - inner_cond) * inner_inner_out cond = (x < -0.999).float() log_probs = cond * log_cdf_plus + (1. - cond) * inner_out log_probs = torch.sum(log_probs, dim=3) + log_prob_from_logits(logit_probs) #Don't sum over batch dimension lse = log_sum_exp(log_probs) return -torch.sum(lse.view(lse.size(0), -1), dim=1)
def feed(self, Q, pi_s, actions, stats, old_pi_s=dict()): """One iteration of policy gradient. rho nabla_w log p_w(a|s) Q + entropy_ratio * nabla H(pi(.|s)) Args: Q(tensor): estimated return actions(tensor): action pi_s(variable): policy old_pi_s(tensor, optional): old policy, in order to get importance factor. If you specify multiple policies, then all the log prob of these policies are added, and their importance factors are multiplied. Feed to stats: policy error and nll error """ # We need to set it beforehand. # Note that the samples we collect might be off-policy, so we need # to do importance sampling. pg_weights = Q.clone() policy_err = None entropy_err = None log_pi_s = [] for pi_node, a_node in self.policy_action_nodes: pi = pi_s[pi_node] a = actions[a_node].squeeze() if pi_node in old_pi_s: old_pi = old_pi_s[pi_node].squeeze() # Cap it. clamped_ratios = torch.clamp( pi.data.div(old_pi), max=self.options.ratio_clamp) coeff = clamped_ratios.gather(1, a.view(-1, 1)).squeeze() pg_weights.mul_(coeff) # There is another term (to compensate clamping), but we omit # it for now. # Compute policy gradient error: errs = self._compute_policy_entropy_err(pi, Variable(a)) policy_err = add_err(policy_err, errs["policy_err"]) entropy_err = add_err(entropy_err, errs["entropy_err"]) log_pi_s.append(errs["logpi"]) stats["nll_" + pi_node].feed(errs["policy_err"].data[0]) stats["entropy_" + pi_node].feed(errs["entropy_err"].data[0]) for log_pi in log_pi_s: self._reg_backward(log_pi, Variable(pg_weights)) if len(self.policy_action_nodes) > 1: stats["total_nll"].feed(policy_err.data[0]) stats["total_entropy"].feed(entropy_err.data[0]) return policy_err + entropy_err * self.options.entropy_ratio
def __call__(self, *inputs): outputs = [] for idx, _input in enumerate(inputs): channel_means = _input.mean(1).mean(2) channel_means = channel_means.expand_as(_input) _input = th.clamp((_input - channel_means) * self.value + channel_means,0,1) outputs.append(_input) return outputs if idx > 1 else outputs[0]
def th_nearest_interp3d(input, coords): """ 2d nearest neighbor interpolation th.Tensor """ # take clamp of coords so they're in the image bounds coords[:,0] = th.clamp(coords[:,0], 0, input.size(1)-1).round() coords[:,1] = th.clamp(coords[:,1], 0, input.size(2)-1).round() coords[:,2] = th.clamp(coords[:,2], 0, input.size(3)-1).round() stride = th.LongTensor(input.stride())[1:].float() idx = coords.mv(stride).long() input_flat = th_flatten(input) mapped_vals = input_flat[idx] return mapped_vals.view_as(input)
def forward(self, x, y, xidx=None, yidx=None): K = torch.sqrt(l2_distance(x, y)) u, v = self._get_uv(x, y, xidx, yidx) if self.regularization == 'entropy': return torch.exp((u[:, None] + v[None, :] - K) / self.alpha) else: return torch.clamp((u[:, None] + v[None, :] - K), min=0) / (2 * self.alpha)
def __call__(self, *inputs): outputs = [] for idx, _input in enumerate(inputs): _in_gs = Grayscale(keep_channels=True)(_input) alpha = 1.0 + self.value _in = th.clamp(_blend(_input, _in_gs, alpha), 0, 1) outputs.append(_in) return outputs if idx > 1 else outputs[0]
def th_nearest_interp2d(input, coords): """ 2d nearest neighbor interpolation th.Tensor """ # take clamp of coords so they're in the image bounds x = th.clamp(coords[:,:,0], 0, input.size(1)-1).round() y = th.clamp(coords[:,:,1], 0, input.size(2)-1).round() stride = th.LongTensor(input.stride()) x_ix = x.mul(stride[1]).long() y_ix = y.mul(stride[2]).long() input_flat = input.view(input.size(0),-1) mapped_vals = input_flat.gather(1, x_ix.add(y_ix)) return mapped_vals.view_as(input)
def batch_iou_pair(yx_min1, yx_max1, yx_min2, yx_max2, min=float(np.finfo(np.float32).eps)): """ Pairwisely calculates the IoU of two lists (at the same size M) of bounding boxes for N independent batches. :author 申瑞珉 (Ruimin Shen) :param yx_min1: The top left coordinates (y, x) of the first lists (size [N, M, 2]) of bounding boxes. :param yx_max1: The bottom right coordinates (y, x) of the first lists (size [N, M, 2]) of bounding boxes. :param yx_min2: The top left coordinates (y, x) of the second lists (size [N, M, 2]) of bounding boxes. :param yx_max2: The bottom right coordinates (y, x) of the second lists (size [N, M, 2]) of bounding boxes. :return: The lists (size [N, M]) of the IoU. """ yx_min = torch.max(yx_min1, yx_min2) yx_max = torch.min(yx_max1, yx_max2) size = torch.clamp(yx_max - yx_min, min=0) intersect_area = torch.prod(size, -1) area1 = torch.prod(yx_max1 - yx_min1, -1) area2 = torch.prod(yx_max2 - yx_min2, -1) union_area = torch.clamp(area1 + area2 - intersect_area, min=min) return intersect_area / union_area
def part_loss(pred_part, gt_seg_part, gt_seg_object, object_label, valid): mask_object = (gt_seg_object == object_label) loss = F.nll_loss(pred_part, gt_seg_part * mask_object.long(), reduction='none') loss = loss * mask_object.float() loss = torch.sum(loss.view(loss.size(0), -1), dim=1) nr_pixel = torch.sum(mask_object.view(mask_object.shape[0], -1), dim=1) sum_pixel = (nr_pixel * valid).sum() loss = (loss * valid.float()).sum() / torch.clamp(sum_pixel, 1).float() return loss
def forward(self, anchor, positive, negative): sim1 = torch.sum(anchor*positive,1) sim2 = torch.sum(anchor*negative,1) #print sim1 #print sim2 dist = sim2-sim1+self.margin dist_hinge = torch.clamp(dist, min=0.0) loss = torch.mean(dist_hinge) return loss
def convert_to_multimnist(d, t, t_len=14): d = d.permute(1, 0) d = d.reshape((64, 1, t_len, t_len)) perm = torch.randperm(d.shape[0]) drand = d[perm] #save_image(d.data.cpu(), 'multimnist_0.png') #save_image(drand.data.cpu(), 'multimnist_1_preshift.png') for i in range(0, 64): dr = drand[i] rshiftx1 = random.randint(0, 3) rshiftx2 = random.randint(0, 3) rshifty1 = random.randint(0, 3) rshifty2 = random.randint(0, 3) dr = dr[:, rshiftx1:t_len - rshiftx2, rshifty1:t_len - rshifty2] if random.uniform(0, 1) < 0.5: padl = rshiftx1 + rshiftx2 padr = 0 else: padl = 0 padr = rshiftx1 + rshiftx2 if random.uniform(0, 1) < 0.5: padt = rshifty1 + rshifty2 padb = 0 else: padt = 0 padb = rshifty1 + rshifty2 dr = torch.cat([ torch.zeros(1, padl, dr.shape[2]).long(), dr, torch.zeros(1, padr, dr.shape[2]).long() ], 1) dr = torch.cat([ torch.zeros(1, dr.shape[1], padt).long(), dr, torch.zeros(1, dr.shape[1], padb).long() ], 2) drand[i] = dr * 1.0 #save_image(drand.data.cpu(), 'multimnist_1.png') d = torch.clamp((d + drand), 0, 1) tr = t[perm] #print(t[0:5], tr[0:5]) new_target = t * 0.0 for i in range(t.shape[0]): if t[i] >= tr[i]: nt = int(str(t[i].item()) + str(tr[i].item())) else: nt = int(str(tr[i].item()) + str(t[i].item())) new_target[i] += nt #print('new target i', new_target[0:5]) #save_image(d.data.cpu(), 'multimnist.png') #raise Exception('done') d = d.reshape((64, t_len * t_len)) d = d.permute(1, 0) return d, new_target
def _do_paste_mask(masks, boxes, img_h, img_w, skip_empty=True): """Paste instance masks acoording to boxes. This implementation is modified from https://github.com/facebookresearch/detectron2/ Args: masks (Tensor): N, 1, H, W boxes (Tensor): N, 4 img_h (int): Height of the image to be pasted. img_w (int): Width of the image to be pasted. skip_empty (bool): Only paste masks within the region that tightly bound all boxes, and returns the results this region only. An important optimization for CPU. Returns: tuple: (Tensor, tuple). The first item is mask tensor, the second one is the slice object. If skip_empty == False, the whole image will be pasted. It will return a mask of shape (N, img_h, img_w) and an empty tuple. If skip_empty == True, only area around the mask will be pasted. A mask of shape (N, h', w') and its start and end coordinates in the original image will be returned. """ # On GPU, paste all masks together (up to chunk size) # by using the entire image to sample the masks # Compared to pasting them one by one, # this has more operations but is faster on COCO-scale dataset. device = masks.device if skip_empty: x0_int, y0_int = torch.clamp( boxes.min(dim=0).values.floor()[:2] - 1, min=0).to(dtype=torch.int32) x1_int = torch.clamp( boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32) y1_int = torch.clamp( boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32) else: x0_int, y0_int = 0, 0 x1_int, y1_int = img_w, img_h x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1 N = masks.shape[0] img_y = torch.arange( y0_int, y1_int, device=device, dtype=torch.float32) + 0.5 img_x = torch.arange( x0_int, x1_int, device=device, dtype=torch.float32) + 0.5 img_y = (img_y - y0) / (y1 - y0) * 2 - 1 img_x = (img_x - x0) / (x1 - x0) * 2 - 1 # img_x, img_y have shapes (N, w), (N, h) if torch.isinf(img_x).any(): inds = torch.where(torch.isinf(img_x)) img_x[inds] = 0 if torch.isinf(img_y).any(): inds = torch.where(torch.isinf(img_y)) img_y[inds] = 0 gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1)) gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1)) grid = torch.stack([gx, gy], dim=3) img_masks = F.grid_sample( masks.to(dtype=torch.float32), grid, align_corners=False) if skip_empty: return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) else: return img_masks[:, 0], ()
def find_dynamic_lmk_idx_and_bcoords(vertices, pose, dynamic_lmk_faces_idx, dynamic_lmk_b_coords, neck_kin_chain, dtype=torch.float32): ''' Compute the faces, barycentric coordinates for the dynamic landmarks To do so, we first compute the rotation of the neck around the y-axis and then use a pre-computed look-up table to find the faces and the barycentric coordinates that will be used. Special thanks to Soubhik Sanyal ([email protected]) for providing the original TensorFlow implementation and for the LUT. Parameters ---------- vertices: torch.tensor BxVx3, dtype = torch.float32 The tensor of input vertices pose: torch.tensor Bx(Jx3), dtype = torch.float32 The current pose of the body model dynamic_lmk_faces_idx: torch.tensor L, dtype = torch.long The look-up table from neck rotation to faces dynamic_lmk_b_coords: torch.tensor Lx3, dtype = torch.float32 The look-up table from neck rotation to barycentric coordinates neck_kin_chain: list A python list that contains the indices of the joints that form the kinematic chain of the neck. dtype: torch.dtype, optional Returns ------- dyn_lmk_faces_idx: torch.tensor, dtype = torch.long A tensor of size BxL that contains the indices of the faces that will be used to compute the current dynamic landmarks. dyn_lmk_b_coords: torch.tensor, dtype = torch.float32 A tensor of size BxL that contains the indices of the faces that will be used to compute the current dynamic landmarks. ''' batch_size = vertices.shape[0] aa_pose = torch.index_select(pose.view(batch_size, -1, 3), 1, neck_kin_chain) rot_mats = batch_rodrigues( aa_pose.view(-1, 3), dtype=dtype).view(batch_size, -1, 3, 3) rel_rot_mat = torch.eye(3, device=vertices.device, dtype=dtype).unsqueeze_(dim=0) for idx in range(len(neck_kin_chain)): rel_rot_mat = torch.bmm(rot_mats[:, idx], rel_rot_mat) y_rot_angle = torch.round( torch.clamp(-rot_mat_to_euler(rel_rot_mat) * 180.0 / np.pi, max=39)).to(dtype=torch.long) neg_mask = y_rot_angle.lt(0).to(dtype=torch.long) mask = y_rot_angle.lt(-39).to(dtype=torch.long) neg_vals = mask * 78 + (1 - mask) * (39 - y_rot_angle) y_rot_angle = (neg_mask * neg_vals + (1 - neg_mask) * y_rot_angle) dyn_lmk_faces_idx = torch.index_select(dynamic_lmk_faces_idx, 0, y_rot_angle) dyn_lmk_b_coords = torch.index_select(dynamic_lmk_b_coords, 0, y_rot_angle) return dyn_lmk_faces_idx, dyn_lmk_b_coords
def amp_to_db(x): o = 20 * torch.log10(torch.clamp(x, min=1e-5)) return o
def funcInf(a, b): return torch.clamp(a + b, min=0, max=float('inf'))
def build_targets(pred_boxes, pred_conf, pred_cls, target, anchor_wh, nA, nC, nG, batch_report): """ returns nT, nCorrect, tx, ty, tw, th, tconf, tcls """ nG_w = nG[0] nG_h = nG[1] nB = len(target) # number of images in batch nT = [len(x) for x in target] # torch.argmin(target[:, :, 4], 1) # targets per image tx = torch.zeros(nB, nA, nG_h, nG_w) # batch size (4), number of anchors (3), number of grid points (13) ty = torch.zeros(nB, nA, nG_h, nG_w) tw = torch.zeros(nB, nA, nG_h, nG_w) th = torch.zeros(nB, nA, nG_h, nG_w) tconf = torch.ByteTensor(nB, nA, nG_h, nG_w).fill_(0) tcls = torch.ByteTensor(nB, nA, nG_h, nG_w, nC).fill_(0) # nC = number of classes TP = torch.ByteTensor(nB, max(nT)).fill_(0) FP = torch.ByteTensor(nB, max(nT)).fill_(0) FN = torch.ByteTensor(nB, max(nT)).fill_(0) TC = torch.ShortTensor(nB, max(nT)).fill_(-1) # target category for b in range(nB): nTb = nT[b] # number of targets if nTb == 0: continue t = target[b] # Convert to position relative to box "Mijenjano" TC[b, :nTb], gx, gy, gw, gh = t[:, 0].long(), t[:, 1] * nG_w, t[:, 2] * nG_h, t[:, 3] * nG_w, t[:, 4] * nG_h # Get grid box indices and prevent overflows (i.e. 13.01 on 13 anchors) gi = torch.clamp(gx.long(), min=0, max=nG_w - 1) gj = torch.clamp(gy.long(), min=0, max=nG_h - 1) # iou of targets-anchors (using wh only) "Mijenjano" box1_1 = (t[:, 3] * nG_w).view(-1,1) box1_2 = (t[:, 4] * nG_h).view(-1,1) box1 = torch.cat((box1_1, box1_2), 1) box2 = anchor_wh.unsqueeze(1) inter_area = torch.min(box1, box2).prod(2) iou = inter_area / (gw * gh + box2.prod(2) - inter_area + 1e-16) # Select best iou_pred and anchor iou_best, a = iou.max(0) # best anchor [0-2] for each target # Select best unique target-anchor combinations if nTb > 1: iou_order = torch.argsort(-iou_best) # best to worst # Unique anchor selection u = torch.cat((gi, gj, a), 0).view(3, -1) _, first_unique = np.unique(u[:, iou_order], axis=1, return_index=True) # first unique indices # _, first_unique = torch.unique(u[:, iou_order], dim=1, return_inverse=True) # different than numpy? i = iou_order[first_unique] # best anchor must share significant commonality (iou) with target i = i[iou_best[i] > 0.10] if len(i) == 0: continue a, gj, gi, t = a[i], gj[i], gi[i], t[i] if len(t.shape) == 1: t = t.view(1, 5) else: if iou_best < 0.10: continue i = 0 tc, gx, gy, gw, gh = t[:, 0].long(), t[:, 1] * nG_w, t[:, 2] * nG_h, t[:, 3] * nG_w, t[:, 4] * nG_h # Coordinates tx[b, a, gj, gi] = gx - gi.float() ty[b, a, gj, gi] = gy - gj.float() # Width and height (yolo method) tw[b, a, gj, gi] = torch.log(gw / anchor_wh[a, 0]) th[b, a, gj, gi] = torch.log(gh / anchor_wh[a, 1]) # Width and height (power method) # tw[b, a, gj, gi] = torch.sqrt(gw / anchor_wh[a, 0]) / 2 # th[b, a, gj, gi] = torch.sqrt(gh / anchor_wh[a, 1]) / 2 # One-hot encoding of label tcls[b, a, gj, gi, tc] = 1 tconf[b, a, gj, gi] = 1 if batch_report: # predicted classes and confidence tb = torch.cat((gx - gw / 2, gy - gh / 2, gx + gw / 2, gy + gh / 2)).view(4, -1).t() # target boxes pcls = torch.argmax(pred_cls[b, a, gj, gi], 1).cpu() pconf = torch.sigmoid(pred_conf[b, a, gj, gi]).cpu() iou_pred = bbox_iou(tb, pred_boxes[b, a, gj, gi].cpu()) TP[b, i] = (pconf > 0.5) & (iou_pred > 0.5) & (pcls == tc) FP[b, i] = (pconf > 0.5) & (TP[b, i] == 0) # coordinates or class are wrong FN[b, i] = pconf <= 0.5 # confidence score is too low (set to zero) return tx, ty, tw, th, tconf, tcls, TP, FP, FN, TC
if args.norm_adv: mb_advantages = (mb_advantages - mb_advantages.mean()) / ( mb_advantages.std() + 1e-8) _, newlogproba, entropy = agent.get_action( b_obs[minibatch_ind], b_actions.long()[minibatch_ind].T, b_invalid_action_masks[minibatch_ind]) ratio = (newlogproba - b_logprobs[minibatch_ind]).exp() # Stats approx_kl = (b_logprobs[minibatch_ind] - newlogproba).mean() # Policy loss pg_loss1 = -mb_advantages * ratio pg_loss2 = -mb_advantages * torch.clamp(ratio, 1 - args.clip_coef, 1 + args.clip_coef) pg_loss = torch.max(pg_loss1, pg_loss2).mean() entropy_loss = entropy.mean() # Value loss new_values = agent.get_value(b_obs[minibatch_ind]).view(-1) if args.clip_vloss: v_loss_unclipped = ((new_values - b_returns[minibatch_ind])**2) v_clipped = b_values[minibatch_ind] + torch.clamp( new_values - b_values[minibatch_ind], -args.clip_coef, args.clip_coef) v_loss_clipped = (v_clipped - b_returns[minibatch_ind])**2 v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) v_loss = 0.5 * v_loss_max.mean() else: v_loss = 0.5 * ((new_values - b_returns[minibatch_ind])**2)
def sampleIsotropic(self, w): theta = 2 * self.PI * self.random() phi = torch.acos(torch.clamp(1 - 2 * self.random(), -1, 1)) return self.vec( torch.sin(phi) * torch.cos(theta), torch.sin(phi) * torch.sin(theta), torch.cos(phi))
def disagg_fold(dataset, fold_num, lr, p): train, test = get_train_test(dataset, num_folds=num_folds, fold_num=fold_num) valid = train[int(0.8 * len(train)):].copy() train = train[:int(0.8 * len(train))].copy() train_aggregate = train[:, 0, :, :].reshape(train.shape[0], 1, -1, 24) valid_aggregate = valid[:, 0, :, :].reshape(valid.shape[0], 1, -1, 24) test_aggregate = test[:, 0, :, :].reshape(test.shape[0], 1, -1, 24) out_train, out_valid, out_test = preprocess(train, valid, test) loss_func = nn.L1Loss() model = AppliancesCNN(len(ORDER)) optimizer = torch.optim.Adam(model.parameters(), lr=lr) if cuda_av: model = model.cuda() loss_func = loss_func.cuda() inp = Variable(torch.Tensor(train_aggregate), requires_grad=False) valid_inp = Variable(torch.Tensor(valid_aggregate), requires_grad=False) test_inp = Variable(torch.Tensor(test_aggregate), requires_grad=False) if cuda_av: inp = inp.cuda() valid_inp = valid_inp.cuda() test_inp = test_inp.cuda() valid_out = torch.cat([out_valid[appliance_num] for appliance_num, appliance in enumerate(ORDER)]) test_out = torch.cat([out_test[appliance_num] for appliance_num, appliance in enumerate(ORDER)]) train_out = torch.cat([out_train[appliance_num] for appliance_num, appliance in enumerate(ORDER)]) valid_pred = {} train_pred = {} test_pred = {} train_losses = {} test_losses = {} valid_losses = {} params = [inp, p] for a_num, appliance in enumerate(ORDER): params.append(out_train[a_num]) if cuda_av: train_out = train_out.cuda() for t in range(1, num_iterations + 1): pred = model(*params) optimizer.zero_grad() loss = loss_func(pred, train_out) if t % 500 == 0: if cuda_av: valid_inp = valid_inp.cuda() valid_params = [valid_inp, -2] for i in range(len(ORDER)): valid_params.append(None) valid_pr = model(*valid_params) valid_loss = loss_func(valid_pr, valid_out) if cuda_av: test_inp = test_inp.cuda() test_params = [test_inp, -2] for i in range(len(ORDER)): test_params.append(None) test_pr = model(*test_params) test_loss = loss_func(test_pr, test_out) test_losses[t] = test_loss.data[0] valid_losses[t] = valid_loss.data[0] train_losses[t] = loss.data[0] # np.save("./baseline/p_50_loss") if t % 1000 == 0: valid_pr = torch.clamp(valid_pr, min=0.) valid_pred[t] = valid_pr test_pr = torch.clamp(test_pr, min=0.) test_pred[t] = test_pr train_pr = pred train_pr = torch.clamp(train_pr, min=0.) train_pred[t] = train_pr print("Round:", t, "Training Error:", loss.data[0], "Validation Error:", valid_loss.data[0], "Test Error:", test_loss.data[0]) loss.backward() optimizer.step() train_fold = [None for x in range(len(ORDER))] train_fold = {} for t in range(1000, num_iterations + 1, 1000): train_pred[t] = torch.split(train_pred[t], train_aggregate.shape[0]) train_fold[t] = [None for x in range(len(ORDER))] if cuda_av: for appliance_num, appliance in enumerate(ORDER): train_fold[t][appliance_num] = train_pred[t][appliance_num].cpu().data.numpy().reshape(-1, 24) else: for appliance_num, appliance in enumerate(ORDER): train_fold[t][appliance_num] = train_pred[t][appliance_num].data.numpy().reshape(-1, 24) valid_fold = {} for t in range(1000, num_iterations + 1, 1000): valid_pred[t] = torch.split(valid_pred[t], valid_aggregate.shape[0]) valid_fold[t] = [None for x in range(len(ORDER))] if cuda_av: for appliance_num, appliance in enumerate(ORDER): valid_fold[t][appliance_num] = valid_pred[t][appliance_num].cpu().data.numpy().reshape(-1, 24) else: for appliance_num, appliance in enumerate(ORDER): valid_fold[t][appliance_num] = valid_pred[t][appliance_num].data.numpy().reshape(-1, 24) test_fold = {} for t in range(1000, num_iterations + 1, 1000): test_pred[t] = torch.split(test_pred[t], test_aggregate.shape[0]) test_fold[t] = [None for x in range(len(ORDER))] if cuda_av: for appliance_num, appliance in enumerate(ORDER): test_fold[t][appliance_num] = test_pred[t][appliance_num].cpu().data.numpy().reshape(-1, 24) else: for appliance_num, appliance in enumerate(ORDER): test_fold[t][appliance_num] = test_pred[t][appliance_num].data.numpy().reshape(-1, 24) # store ground truth of validation set train_gt_fold = [None for x in range(len(ORDER))] for appliance_num, appliance in enumerate(ORDER): train_gt_fold[appliance_num] = train[:, APPLIANCE_ORDER.index(appliance), :, :].reshape( train_aggregate.shape[0], -1, 1).reshape(-1, 24) valid_gt_fold = [None for x in range(len(ORDER))] for appliance_num, appliance in enumerate(ORDER): valid_gt_fold[appliance_num] = valid[:, APPLIANCE_ORDER.index(appliance), :, :].reshape( valid_aggregate.shape[0], -1, 1).reshape(-1, 24) test_gt_fold = [None for x in range(len(ORDER))] for appliance_num, appliance in enumerate(ORDER): test_gt_fold[appliance_num] = test[:, APPLIANCE_ORDER.index(appliance), :, :].reshape( test_aggregate.shape[0], -1, 1).reshape(-1, 24) # calcualte the error of validation set train_error = {} for t in range(1000, num_iterations + 1, 1000): train_error[t] = {} for appliance_num, appliance in enumerate(ORDER): train_error[t][appliance] = mean_absolute_error(train_fold[t][appliance_num], train_gt_fold[appliance_num]) valid_error = {} for t in range(1000, num_iterations + 1, 1000): valid_error[t] = {} for appliance_num, appliance in enumerate(ORDER): valid_error[t][appliance] = mean_absolute_error(valid_fold[t][appliance_num], valid_gt_fold[appliance_num]) test_error = {} for t in range(1000, num_iterations + 1, 1000): test_error[t] = {} for appliance_num, appliance in enumerate(ORDER): test_error[t][appliance] = mean_absolute_error(test_fold[t][appliance_num], test_gt_fold[appliance_num]) return train_fold, valid_fold, test_fold, train_error, valid_error, test_error, train_losses, valid_losses, test_losses
def _pgd_whitebox(model, X, y, args): device = X.device out = model(X) acc = (out.data.max(1)[1] == y.data).float().mean() X_pgd = Variable(X.data, requires_grad=True) batch_size = X.shape[0] if args.pgd_loss == "baseline": criterion_ce = nn.CrossEntropyLoss().to(device) else: criterion_ce = nn.CrossEntropyLoss(reduction='none').to(device) max_loss_pgd = torch.zeros(y.shape[0]).cuda() max_delta = torch.zeros_like(X).cuda() for zz in range(10): if args.random_init: random_noise = torch.FloatTensor(*X_pgd.shape).uniform_(-args.epsilon_eval, args.epsilon_eval).to(device) X_pgd = Variable(X_pgd.data + random_noise, requires_grad=True) # #################################################################### # ########################### PGD LOSS ############################### # #################################################################### for _ in range(args.num_steps_eval): opt = optim.SGD([X_pgd], lr=1e-3) opt.zero_grad() perturbation = X_pgd - X with torch.enable_grad(): outputs_adv = model(X_pgd) outputs = model(X) loss_pgd, loss_individual_pgd = pgd_loss_ce(args, outputs_adv, outputs, y, perturbation, criterion_ce) loss_pgd.backward() eta = args.step_size_eval * X_pgd.grad.data.sign() X_pgd = Variable(X_pgd.data + eta, requires_grad=True) eta = torch.clamp(X_pgd.data - X.data, -args.epsilon_eval, args.epsilon_eval) X_pgd = Variable(X.data + eta, requires_grad=True) X_pgd = Variable(torch.clamp(X_pgd, 0, 1.0), requires_grad=True) """ margin loss, if enabled """ # outputs_adv = model(X_pgd) # label_mask = nn.functional.one_hot(y, 10).to(torch.bool) # label_logit = outputs_adv[label_mask] # others = outputs_adv[~label_mask].reshape(-1, 9) # top_other_logit, _ = torch.max(others, dim=1) # all_loss = (top_other_logit - label_logit).detach() """ cross entropy loss, if enabled """ all_loss = F.cross_entropy(model(X_pgd), y, reduction='none').detach() delta = X_pgd - X max_delta[all_loss >= max_loss_pgd] = delta.detach()[all_loss >= max_loss_pgd] max_loss_pgd = torch.max(max_loss_pgd, all_loss) outputs_adv = model(X + max_delta) one_hot_adv = (outputs_adv.data.max(1)[1] == y.data).float().detach().cpu().numpy() """ margin loss, if enabled """ # label_mask = nn.functional.one_hot(y, 10).to(torch.bool) # label_logit = outputs_adv[label_mask] # others = outputs_adv[~label_mask].reshape(-1, 9) # top_other_logit, _ = torch.max(others, dim=1) # loss_individual_pgd = (top_other_logit - label_logit).detach().cpu().numpy() """ cross entropy loss, if enabled """ loss_individual_pgd = F.cross_entropy(outputs_adv, y, reduction='none').detach().cpu().numpy() outputs = model(X) one_hot = (outputs.data.max(1)[1] == y.data).float().detach().cpu().numpy() """ margin loss, if enabled """ # label_mask = nn.functional.one_hot(y, 10).to(torch.bool) # label_logit = outputs[label_mask] # others = outputs[~label_mask].reshape(-1, 9) # top_other_logit, _ = torch.max(others, dim=1) # loss_individual = (top_other_logit - label_logit).detach().cpu().numpy() """ cross entropy loss, if enabled """ loss_individual = F.cross_entropy(outputs, y, reduction='none').detach().cpu().numpy() perturbation = max_delta margin, weights_margin = compute_weights(args, outputs_adv, perturbation, y) acc = np.mean(one_hot) return margin, weights_margin, one_hot, one_hot_adv, acc, loss_individual, loss_individual_pgd
def test_model_fit(model_fit: bool): """ Test that controls that scVI inferred distributions make sense on a non-trivial synthetic dataset. We define technical zeros of the synthetic dataset as the zeros that result from highly expressed genes (relatively to the considered cell) and the biological zeros as the rest of the zeros :return: None """ print("model_fit set to : ", model_fit) folder = "/tmp/scVI_zeros_test" print("Saving graphs in : {}".format(folder)) if not os.path.exists(folder): os.makedirs(folder) n_epochs = 150 if model_fit else 1 n_mc_sim_total = 100 if model_fit else 1 n_cells_cluster = 1000 if model_fit else 100 torch.manual_seed(seed=42) synth_data = ZISyntheticDatasetCorr( n_clusters=8, n_genes_high=15, n_overlap=8, lam_0=320, n_cells_cluster=n_cells_cluster, weight_high=1.714286, weight_low=1, dropout_coef_low=0.08, dropout_coef_high=0.05, ) is_high = synth_data.is_highly_exp.squeeze() poisson_params_gt = synth_data.exprs_param.squeeze() # Step 2: Training scVI model mdl = VAE( n_input=synth_data.nb_genes, n_batch=synth_data.n_batches, reconstruction_loss="zinb", n_latent=5, ) trainer = UnsupervisedTrainer( model=mdl, gene_dataset=synth_data, use_cuda=True, train_size=1.0 ) trainer.train(n_epochs=n_epochs, lr=1e-3) full = trainer.create_posterior( trainer.model, synth_data, indices=np.arange(len(synth_data)) ) # Step 3: Inference poisson_params = [] p_dropout_infered = [] latent_reps = [] bio_zero_p = [] tech_zero_p = [] with torch.no_grad(): for tensors in full.sequential(): # TODO: Properly sample posterior sample_batch, _, _, batch_index, labels = tensors px_scale, px_dispersion, px_rate, px_dropout, qz_m, qz_v, z, ql_m, ql_v, library = mdl.inference( sample_batch, batch_index ) p_zero = 1.0 / (1.0 + torch.exp(-px_dropout)) p_dropout_infered.append(p_zero.cpu().numpy()) l_train_batch = torch.zeros( (sample_batch.size(0), sample_batch.size(1), n_mc_sim_total), device=sample_batch.device, ) for n_mc_sim in range(n_mc_sim_total): p = px_rate / (px_rate + px_dispersion) r = px_dispersion l_train = torch.distributions.Gamma( concentration=r, rate=(1 - p) / p ).sample() l_train = torch.clamp(l_train, max=1e18) X = torch.distributions.Poisson(l_train).sample() l_train_batch[:, :, n_mc_sim] = l_train p_zero = 1.0 / (1.0 + torch.exp(-px_dropout)) random_prob = torch.rand_like(p_zero) X[random_prob <= p_zero] = 0 l_train_batch = torch.mean(l_train_batch, dim=(-1)) bio_zero_prob_batch = torch.exp(-l_train_batch) tech_zero_prob_batch = p_zero bio_zero_p.append(bio_zero_prob_batch.cpu().numpy()) tech_zero_p.append(tech_zero_prob_batch.cpu().numpy()) latent_reps.append(z.cpu().numpy()) poisson_params.append(l_train_batch.cpu().numpy()) latent_reps = np.concatenate(latent_reps) bio_zero_p = np.concatenate(bio_zero_p) tech_zero_p = np.concatenate(tech_zero_p) bio_zero_tech_no = bio_zero_p * (1.0 - tech_zero_p) tech_zero_bio_no = (1.0 - bio_zero_p) * tech_zero_p # Final Step: Checking predictions # Dropout checks p_dropout_infered_all = np.concatenate(p_dropout_infered) p_dropout_gt = synth_data.p_dropout.squeeze() vmin = 0.0 vmax = 2.0 * p_dropout_gt.max() fig, axes = plt.subplots(ncols=2, nrows=2, figsize=(10, 10)) sns.heatmap(p_dropout_infered_all, vmin=vmin, vmax=vmax, ax=axes[0, 1]) axes[0, 1].set_title("Dropout Rate Predicted") sns.heatmap(p_dropout_gt, vmin=vmin, vmax=vmax, ax=axes[0, 0]) axes[0, 0].set_title("Dropout Rate GT") # Poisson Params checks poisson_params = np.concatenate(poisson_params) vmin = min(poisson_params_gt.min(), poisson_params.min()) vmax = max(poisson_params_gt.max(), poisson_params.max()) sns.heatmap(poisson_params, vmin=vmin, vmax=vmax, ax=axes[1, 1]) axes[1, 1].set_title("Poisson Distribution Parameter Predicted") sns.heatmap(poisson_params_gt, vmin=vmin, vmax=vmax, ax=axes[1, 0]) axes[1, 0].set_title("Poisson Distribution Parameter GT") plt.savefig(os.path.join(folder, "params_comparison.png")) plt.close() # TODO: Decrease test tolerances l1_poisson = np.abs(poisson_params - poisson_params_gt).mean() if model_fit: print("Average Poisson L1 error: ", l1_poisson) assert l1_poisson <= 0.75, "High Error on Poisson parameter inference" l1_dropout = np.abs(p_dropout_infered_all - synth_data.p_dropout).mean() print("Average Dropout L1 error: ", l1_dropout) assert l1_dropout <= 5e-2, "High Error on Dropout parameter inference" # tSNE plot print("Computing tSNE rep ...") x_rep = TSNE(n_components=2).fit_transform(latent_reps) print("Done!") pos = np.random.permutation(len(x_rep))[:1000] labels = ["c_{}".format(idx) for idx in synth_data.labels[pos].squeeze()] sns.scatterplot(x=x_rep[pos, 0], y=x_rep[pos, 1], hue=labels, palette="Set2") plt.title("Synthetic Dataset latent space") plt.savefig(os.path.join(folder, "t_sne.png")) plt.close() # Tech/Bio Classif checks # --For high expressed genes # ---Poisson nul and ZI non null print(bio_zero_tech_no[is_high].mean(), synth_data.probas_zero_bio_tech_high[1, 0]) # ---Poisson non nul and . print(tech_zero_bio_no[is_high].mean(), synth_data.probas_zero_bio_tech_high[0, 1]) # --Low expressed expressend # ---Poisson nul and ZI non null print(bio_zero_tech_no[~is_high].mean(), synth_data.probas_zero_bio_tech_low[1, 0]) # ---Poisson non nul and . print(tech_zero_bio_no[~is_high].mean(), synth_data.probas_zero_bio_tech_low[0, 1]) diff1 = np.abs( bio_zero_tech_no[is_high].mean() - synth_data.probas_zero_bio_tech_high[1, 0] ) diff2 = np.abs( tech_zero_bio_no[is_high].mean() - synth_data.probas_zero_bio_tech_high[0, 1] ) diff3 = np.abs( bio_zero_tech_no[~is_high].mean() - synth_data.probas_zero_bio_tech_low[1, 0] ) diff4 = np.abs( tech_zero_bio_no[~is_high].mean() - synth_data.probas_zero_bio_tech_low[0, 1] ) if model_fit: assert diff1 <= 2e-2 assert diff2 <= 2e-2 assert diff3 <= 2e-2 assert diff4 <= 2e-2
def nms(boxes, scores, overlap=0.5, top_k=200): """Apply non-maximum suppression at test time to avoid detecting too many overlapping bounding boxes for a given object. Args: boxes: (tensor) The location preds for the img, Shape: [num_priors,4]. scores: (tensor) The class predscores for the img, Shape:[num_priors]. overlap: (float) The overlap thresh for suppressing unnecessary boxes. top_k: (int) The Maximum number of box preds to consider. Return: The indices of the kept boxes with respect to num_priors. """ keep = scores.new(scores.size(0)).zero_().long() if boxes.numel() == 0: return keep x1 = boxes[:, 0] y1 = boxes[:, 1] x2 = boxes[:, 2] y2 = boxes[:, 3] area = torch.mul(x2 - x1, y2 - y1) v, idx = scores.sort(0) # sort in ascending order # I = I[v >= 0.01] idx = idx[-top_k:] # indices of the top-k largest vals xx1 = boxes.new() yy1 = boxes.new() xx2 = boxes.new() yy2 = boxes.new() w = boxes.new() h = boxes.new() # keep = torch.Tensor() count = 0 while idx.numel() > 0: i = idx[-1] # index of current largest val # keep.append(i) keep[count] = i count += 1 if idx.size(0) == 1: break idx = idx[:-1] # remove kept element from view # load bboxes of next highest vals torch.index_select(x1, 0, idx, out=xx1) torch.index_select(y1, 0, idx, out=yy1) torch.index_select(x2, 0, idx, out=xx2) torch.index_select(y2, 0, idx, out=yy2) # store element-wise max with next highest score xx1 = torch.clamp(xx1, min=x1[i]) yy1 = torch.clamp(yy1, min=y1[i]) xx2 = torch.clamp(xx2, max=x2[i]) yy2 = torch.clamp(yy2, max=y2[i]) w.resize_as_(xx2) h.resize_as_(yy2) w = xx2 - xx1 h = yy2 - yy1 # check sizes of xx1 and xx2.. after each iteration w = torch.clamp(w, min=0.0) h = torch.clamp(h, min=0.0) inter = w * h # IoU = i / (area(a) + area(b) - i) rem_areas = torch.index_select(area, 0, idx) # load remaining areas) union = (rem_areas - inter) + area[i] IoU = inter / union # store result in iou # keep only elements with an IoU <= overlap idx = idx[IoU.le(overlap)] return keep, count
optimizer.zero_grad() #Prepare sample and target image = torch.autograd.Variable(sample_batched['image'].cpu()) depth = torch.autograd.Variable(sample_batched['depth'].cpu()) # print("input", image.shape) # Normalize depth depth_n = DepthNorm( depth ) # Predict output = model(image) # print("prediction output", output.shape) # print("GT", depth_n.shape) # Compute the loss l_depth = l1_criterion(output, depth_n) l_ssim = torch.clamp((1 - ssim(output, depth_n, val_range = 1000.0 / 10.0)) * 0.5, 0, 1) loss = (1.0 * l_ssim.mean().item()) + (0.1 * l_depth) # Update step losses.update(loss.data.item(), image.size(0)) loss.backward() optimizer.step() # Measure elapsed time batch_time.update(time.time() - end) end = time.time() eta = str(datetime.timedelta(seconds=int(batch_time.val*(N - i)))) # Log progress
def funcOptMax(a, b): return torch.clamp(a + b, min=0)
def detect(self, sample): input_pair = [] detection_pair = [] camera = sample[30][0].cuda() for indexOffset in [0, ]: images, image_metas, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, gt_parameters, gt_depth, extrinsics, planes, gt_segmentation = sample[indexOffset + 0].cuda(), sample[indexOffset + 1].numpy(), sample[indexOffset + 2].cuda(), sample[indexOffset + 3].cuda(), sample[indexOffset + 4].cuda(), sample[indexOffset + 5].cuda(), sample[indexOffset + 6].cuda(), sample[indexOffset + 7].cuda(), sample[indexOffset + 8].cuda(), sample[indexOffset + 9].cuda(), sample[indexOffset + 10].cuda(), sample[indexOffset + 11].cuda() rpn_class_logits, rpn_pred_bbox, target_class_ids, mrcnn_class_logits, target_deltas, mrcnn_bbox, target_mask, mrcnn_mask, target_parameters, mrcnn_parameters, detections, detection_masks, detection_gt_parameters, detection_gt_masks, rpn_rois, roi_features, roi_indices, depth_np_pred = self.model.predict([images, image_metas, gt_class_ids, gt_boxes, gt_masks, gt_parameters, camera], mode='inference_detection', use_nms=2, use_refinement=True) if len(detections) > 0: detections, detection_masks = unmoldDetections(self.config, camera, detections, detection_masks, depth_np_pred, debug=False) pass XYZ_pred, detection_mask, plane_XYZ = calcXYZModule(self.config, camera, detections, detection_masks, depth_np_pred, return_individual=True) detection_mask = detection_mask.unsqueeze(0) input_pair.append({'image': images, 'depth': gt_depth, 'mask': gt_masks, 'bbox': gt_boxes, 'extrinsics': extrinsics, 'segmentation': gt_segmentation, 'camera': camera}) if 'nyu_dorn_only' in self.options.dataset: XYZ_pred[1:2] = sample[27].cuda() pass detection_pair.append({'XYZ': XYZ_pred, 'depth': XYZ_pred[1:2], 'mask': detection_mask, 'detection': detections, 'masks': detection_masks, 'depth_np': depth_np_pred, 'plane_XYZ': plane_XYZ}) continue if ('refine' in self.modelType or 'refine' in self.options.suffix): pose = sample[26][0].cuda() pose = torch.cat([pose[0:3], pose[3:6] * pose[6]], dim=0) pose_gt = torch.cat([pose[0:1], -pose[2:3], pose[1:2], pose[3:4], -pose[5:6], pose[4:5]], dim=0).unsqueeze(0) camera = camera.unsqueeze(0) for c in range(1): detection_dict, input_dict = detection_pair[c], input_pair[c] new_input_dict = {k: v for k, v in input_dict.items()} new_input_dict['image'] = (input_dict['image'] + self.config.MEAN_PIXEL_TENSOR.view((-1, 1, 1))) / 255.0 - 0.5 new_input_dict['image_2'] = (sample[13].cuda() + self.config.MEAN_PIXEL_TENSOR.view((-1, 1, 1))) / 255.0 - 0.5 detections = detection_dict['detection'] detection_masks = detection_dict['masks'] depth_np = detection_dict['depth_np'] image = new_input_dict['image'] image_2 = new_input_dict['image_2'] depth_gt = new_input_dict['depth'].unsqueeze(1) masks_inp = torch.cat([detection_masks.unsqueeze(1), detection_dict['plane_XYZ']], dim=1) segmentation = new_input_dict['segmentation'] detection_masks = torch.nn.functional.interpolate(detection_masks[:, 80:560].unsqueeze(1), size=(192, 256), mode='nearest').squeeze(1) image = torch.nn.functional.interpolate(image[:, :, 80:560], size=(192, 256), mode='bilinear') image_2 = torch.nn.functional.interpolate(image_2[:, :, 80:560], size=(192, 256), mode='bilinear') masks_inp = torch.nn.functional.interpolate(masks_inp[:, :, 80:560], size=(192, 256), mode='bilinear') depth_np = torch.nn.functional.interpolate(depth_np[:, 80:560].unsqueeze(1), size=(192, 256), mode='bilinear').squeeze(1) plane_depth = torch.nn.functional.interpolate(detection_dict['depth'][:, 80:560].unsqueeze(1), size=(192, 256), mode='bilinear').squeeze(1) segmentation = torch.nn.functional.interpolate(segmentation[:, 80:560].unsqueeze(1).float(), size=(192, 256), mode='nearest').squeeze().long() new_input_dict['image'] = image new_input_dict['image_2'] = image_2 results = self.refine_model(image, image_2, camera, masks_inp, detection_dict['detection'][:, 6:9], plane_depth, depth_np) masks = results[-1]['mask'].squeeze(1) all_masks = torch.softmax(masks, dim=0) masks_small = all_masks[1:] all_masks = torch.nn.functional.interpolate(all_masks.unsqueeze(1), size=(480, 640), mode='bilinear').squeeze(1) all_masks = (all_masks.max(0, keepdim=True)[1] == torch.arange(len(all_masks)).cuda().long().view((-1, 1, 1))).float() masks = all_masks[1:] detection_masks = torch.zeros(detection_dict['masks'].shape).cuda() detection_masks[:, 80:560] = masks detection_dict['masks'] = detection_masks detection_dict['depth_ori'] = detection_dict['depth'].clone() detection_dict['mask'][:, 80:560] = (masks.max(0, keepdim=True)[0] > (1 - masks.sum(0, keepdim=True))).float() if self.options.modelType == 'fitting': masks_cropped = masks_small ranges = self.config.getRanges(camera).transpose(1, 2).transpose(0, 1) XYZ = torch.nn.functional.interpolate(ranges.unsqueeze(1), size=(192, 256), mode='bilinear').squeeze(1) * results[-1]['depth'].squeeze(1) detection_areas = masks_cropped.sum(-1).sum(-1) A = masks_cropped.unsqueeze(1) * XYZ b = masks_cropped Ab = (A * b.unsqueeze(1)).sum(-1).sum(-1) AA = (A.unsqueeze(2) * A.unsqueeze(1)).sum(-1).sum(-1) plane_parameters = torch.stack([torch.matmul(torch.inverse(AA[planeIndex]), Ab[planeIndex]) if detection_areas[planeIndex] else detection_dict['detection'][planeIndex, 6:9] for planeIndex in range(len(AA))], dim=0) plane_offsets = torch.norm(plane_parameters, dim=-1, keepdim=True) plane_parameters = plane_parameters / torch.clamp(torch.pow(plane_offsets, 2), 1e-4) detection_dict['detection'][:, 6:9] = plane_parameters XYZ_pred, detection_mask, plane_XYZ = calcXYZModule(self.config, camera, detection_dict['detection'], detection_masks, detection_dict['depth'], return_individual=True) detection_dict['depth'] = XYZ_pred[1:2] pass continue pass return detection_pair
def postprocess(im, invert=False): im = th.clamp((im + 1.0) / 2.0, 0, 1) if invert: im = (1.0 - im) im = ttools.tensor2image(im) return im
def forward(self, x, target=None): # backbone c2, c3, c4, c5 = self.backbone(x) B = c5.size(0) # bottom-up p5 = self.spp(c5) p4 = self.deconv5(p5) p3 = self.deconv4(p4) p2 = self.deconv3(p3) # head cls_pred = self.cls_pred(p2) txty_pred = self.txty_pred(p2) twth_pred = self.twth_pred(p2) # train if self.trainable: # [B, H*W, num_classes] cls_pred = cls_pred.permute(0, 2, 3, 1).contiguous().view(B, -1, self.num_classes) # [B, H*W, 2] txty_pred = txty_pred.permute(0, 2, 3, 1).contiguous().view(B, -1, 2) # [B, H*W, 2] twth_pred = twth_pred.permute(0, 2, 3, 1).contiguous().view(B, -1, 2) # compute loss cls_loss, txty_loss, twth_loss, total_loss = tools.loss(pred_cls=cls_pred, pred_txty=txty_pred, pred_twth=twth_pred, label=target, num_classes=self.num_classes ) return cls_loss, txty_loss, twth_loss, total_loss # test else: with torch.no_grad(): # batch_size = 1 cls_pred = torch.sigmoid(cls_pred) # simple nms hmax = F.max_pool2d(cls_pred, kernel_size=5, padding=2, stride=1) keep = (hmax == cls_pred).float() cls_pred *= keep # decode box txtytwth_pred = torch.cat([txty_pred, twth_pred], dim=1).permute(0, 2, 3, 1).contiguous().view(B, -1, 4) # [B, H*W, 4] -> [H*W, 4] bbox_pred = torch.clamp((self.decode_boxes(txtytwth_pred) / self.scale_torch)[0], 0., 1.) # topk topk_scores, topk_inds, topk_clses = self._topk(cls_pred) topk_scores = topk_scores[0].cpu().numpy() topk_ind = topk_clses[0].cpu().numpy() topk_bbox_pred = bbox_pred[topk_inds[0]].cpu().numpy() if self.use_nms: # nms keep = np.zeros(len(topk_bbox_pred), dtype=np.int) for i in range(self.num_classes): inds = np.where(topk_ind == i)[0] if len(inds) == 0: continue c_bboxes = topk_bbox_pred[inds] c_scores = topk_scores[inds] c_keep = self.nms(c_bboxes, c_scores) keep[inds[c_keep]] = 1 keep = np.where(keep > 0) topk_bbox_pred = topk_bbox_pred[keep] topk_scores = topk_scores[keep] topk_ind = topk_ind[keep] return topk_bbox_pred, topk_scores, topk_ind
def earlystop(model, data, target, step_size, epsilon, perturb_steps, tau, randominit_type, loss_fn, rand_init=True, omega=0): ''' The implematation of early-stopped PGD Following the Alg.1 in our FAT paper <https://arxiv.org/abs/2002.11242> :param step_size: the PGD step size :param epsilon: the perturbation bound :param perturb_steps: the maximum PGD step :param tau: the step controlling how early we should stop interations when wrong adv data is found :param randominit_type: To decide the type of random inirialization (random start for searching adv data) :param rand_init: To decide whether to initialize adversarial sample with random noise (random start for searching adv data) :param omega: random sample parameter for adv data generation (this is for escaping the local minimum.) :return: output_adv (friendly adversarial data) output_target (targets), output_natural (the corresponding natrual data), count (average backword propagations count) ''' model.eval() K = perturb_steps count = 0 output_target = [] output_adv = [] output_natural = [] control = (torch.ones(len(target)) * tau).cuda() # Initialize the adversarial data with random noise if rand_init: if randominit_type == "normal_distribution_randominit": iter_adv = data.detach() + 0.001 * torch.randn( data.shape).cuda().detach() iter_adv = torch.clamp(iter_adv, 0.0, 1.0) if randominit_type == "uniform_randominit": iter_adv = data.detach() + torch.from_numpy( np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() iter_adv = torch.clamp(iter_adv, 0.0, 1.0) else: iter_adv = data.cuda().detach() iter_clean_data = data.cuda().detach() iter_target = target.cuda().detach() output_iter_clean_data = model(data) while K > 0: iter_adv.requires_grad_() output = model(iter_adv) pred = output.max(1, keepdim=True)[1] output_index = [] iter_index = [] # Calculate the indexes of adversarial data those still needs to be iterated for idx in range(len(pred)): if pred[idx] != target[idx]: if control[idx] == 0: output_index.append(idx) else: control[idx] -= 1 iter_index.append(idx) else: iter_index.append(idx) # Add adversarial data those do not need any more iteration into set output_adv if len(output_index) != 0: if len(output_target) == 0: # incorrect adv data should not keep iterated output_adv = iter_adv[output_index].reshape(-1, 3, 32, 32).cuda() output_natural = iter_clean_data[output_index].reshape( -1, 3, 32, 32).cuda() output_target = iter_target[output_index].reshape(-1).cuda() else: # incorrect adv data should not keep iterated output_adv = torch.cat( (output_adv, iter_adv[output_index].reshape(-1, 3, 32, 32).cuda()), dim=0) output_natural = torch.cat( (output_natural, iter_clean_data[output_index].reshape( -1, 3, 32, 32).cuda()), dim=0) output_target = torch.cat( (output_target, iter_target[output_index].reshape(-1).cuda()), dim=0) # calculate gradient model.zero_grad() with torch.enable_grad(): if loss_fn == "cent": loss_adv = nn.CrossEntropyLoss(reduction='mean')(output, iter_target) if loss_fn == "kl": criterion_kl = nn.KLDivLoss(size_average=False).cuda() loss_adv = criterion_kl( F.log_softmax(output, dim=1), F.softmax(output_iter_clean_data, dim=1)) loss_adv.backward(retain_graph=True) grad = iter_adv.grad # update iter adv if len(iter_index) != 0: control = control[iter_index] iter_adv = iter_adv[iter_index] iter_clean_data = iter_clean_data[iter_index] iter_target = iter_target[iter_index] output_iter_clean_data = output_iter_clean_data[iter_index] grad = grad[iter_index] eta = step_size * grad.sign() iter_adv = iter_adv.detach() + eta + omega * torch.randn( iter_adv.shape).detach().cuda() iter_adv = torch.min( torch.max(iter_adv, iter_clean_data - epsilon), iter_clean_data + epsilon) iter_adv = torch.clamp(iter_adv, 0, 1) count += len(iter_target) else: return output_adv, output_target, output_natural, count K = K - 1 if len(output_target) == 0: output_target = iter_target.reshape(-1).squeeze().cuda() output_adv = iter_adv.reshape(-1, 3, 32, 32).cuda() output_natural = iter_clean_data.reshape(-1, 3, 32, 32).cuda() else: output_adv = torch.cat((output_adv, iter_adv.reshape(-1, 3, 32, 32)), dim=0).cuda() output_target = torch.cat((output_target, iter_target.reshape(-1)), dim=0).squeeze().cuda() output_natural = torch.cat( (output_natural, iter_clean_data.reshape(-1, 3, 32, 32).cuda()), dim=0).cuda() return output_adv, output_target, output_natural, count
def update_policy(args, get_loss, get_kl, policy_net, policy_optimizer=None, value_net=None, likelihood=None): """Updates parameters of policy network according to optimization scheme.""" # This method is used for updating the policy parameters based on the selected policy gradient setting. grads = torch.autograd.grad(get_loss(), policy_net.parameters()) loss_grad = torch.cat([grad.view(-1) for grad in grads]).data step_size = None def Fvp(v): # Computes Fisher vector product as a Hessian vector product of KL divergence (same as the trick used in TRPO) kl = get_kl() kl = kl.mean() grads = torch.autograd.grad(kl, policy_net.parameters(), create_graph=True) flat_grad_kl = torch.cat([grad.view(-1) for grad in grads]) kl_v = (flat_grad_kl * Variable(v)).sum() grads = torch.autograd.grad(kl_v, policy_net.parameters()) flat_grad_grad_kl = torch.cat( [grad.contiguous().view(-1) for grad in grads]).data return flat_grad_grad_kl + v * args.damping # damping ensures numerical stability and faster convergence if args.pg_algorithm == "vanilla": # Computes conventional policy gradient if args.pg_estimator == 'BQ' and args.UAPG_flag: # Computes the low-rank SVD of the covariance matrix for vanilla PG, without constructing it in the first place. # Specifically, we utilize fast covariance vector products for fast low-rank SVD computation. u_cov, s_cov, v_cov = fast_svd.pca_Cov(args, conjugate_gradients, Fvp, policy_net, value_net, likelihood) # Lowering the step-size of the gradient components with high estimation uncertainty. new_s_cov = 1 - torch.sqrt(s_cov.min()) / torch.sqrt(s_cov) # Final UAPG update for vanilla policy gradient loss_grad = loss_grad - torch.matmul( u_cov, torch.matmul(torch.diag(new_s_cov), torch.matmul(v_cov, loss_grad))) policy_optimizer.zero_grad() set_flat_grad_to(policy_net, loss_grad) policy_optimizer.step() else: # Computes natural policy gradient, for NPG and TRPO neg_stepdir = conjugate_gradients(Fvp, loss_grad, 50, device=args.device) if args.pg_estimator == 'BQ' and args.UAPG_flag: # Computes the low-rank SVD of the inverse Covariance matrix for the natural gradient. u_cov, s_cov, v_cov = fast_svd.pca_NPG_InvCov( args, conjugate_gradients, Fvp, policy_net, value_net, likelihood) # Increasing the step-size of the gradient components with low estimation uncertainty (most confident directions). new_s_cov = torch.clamp(torch.sqrt(s_cov / s_cov.min()), 1, args.UAPG_epsilon) - 1 # Final UAPG update for natural policy gradient neg_stepdir = neg_stepdir + torch.matmul( u_cov, torch.matmul(torch.diag(new_s_cov), torch.matmul(v_cov, neg_stepdir))) if args.pg_algorithm == "NPG": # NPG update old_params = get_flat_params_from(policy_net) policy_optimizer.zero_grad() set_flat_grad_to(policy_net, neg_stepdir) policy_optimizer.step() new_params = get_flat_params_from(policy_net) else: # TRPO update stepdir = -neg_stepdir # Search direction after solving the constrained optimization problem, same as natural gradient shs = 0.5 * (stepdir * Fvp(stepdir)).sum(0, keepdim=True) ## Important, here is the TRPO magic lm = torch.sqrt(shs / args.max_kl) # One over largest step size/ trust region size beta = torch.sqrt(args.max_kl/shs) step_size = 1 / lm fullstep = stepdir / lm[ 0] # Naive trust region based update corresponding to the largest step size # print(fullstep) neggdotstepdir = (stepdir * Fvp(stepdir)).sum(0, keepdim=True) # print(neggdotstepdir) prev_params = get_flat_params_from(policy_net) # Line search avoids large policy steps that result in catastrophic performance degradation. success, new_params = linesearch(policy_net, get_loss, prev_params, fullstep, neggdotstepdir / lm[0]) set_flat_params_to(policy_net, new_params) return step_size
def run(self, frame, frames): if self.skip_flag == 0: inp_dim = int(self.model.net_info["height"]) assert inp_dim % 32 == 0 assert inp_dim > 32 self.model.eval() img, orig_im, dim = prep_image(frame, inp_dim) im_dim = torch.FloatTensor(dim).repeat(1, 2) if self.CUDA: im_dim = im_dim.cuda() img = img.cuda() with torch.no_grad(): output = self.model(Variable(img), self.CUDA) output = write_results(output, self.confidence, self.num_classes, nms=True, nms_conf=self.nms_thesh) if type(output) == int: return frame im_dim = im_dim.repeat(output.size(0), 1) scaling_factor = torch.min(inp_dim / im_dim, 1)[0].view(-1, 1) output[:, [1, 3]] -= (inp_dim - scaling_factor * im_dim[:, 0].view(-1, 1)) / 2 output[:, [2, 4]] -= (inp_dim - scaling_factor * im_dim[:, 1].view(-1, 1)) / 2 output[:, 1:5] /= scaling_factor for i in range(output.shape[0]): output[i, [1, 3]] = torch.clamp(output[i, [1, 3]], 0.0, im_dim[i, 0]) output[i, [2, 4]] = torch.clamp(output[i, [2, 4]], 0.0, im_dim[i, 1]) classes = load_classes('data/coco.names') colors = pkl.load(open("pallete", "rb")) #print('output: {}'.format(output.shape[0])) for i in range(output.shape[0]): data_list = np.array([[ frames, int(output[i, 1]), int(output[i, 3]), int(output[i, 2]), int(output[i, 4]), classes[int(output[i, 7])] ]]) self.data = np.vstack([self.data, data_list]) #print(self.data) #print(self.data) list( map(lambda x: write( x, orig_im, classes, colors, frames, ), output)) return orig_im
def forward(self, article_sentences, article_sentences_lengths, num_codes, codes=None, code_description=None, code_description_length=None, linearized_codes=None, linearized_codes_lengths=None, linearized_descriptions=None, linearized_descriptions_lengths=None): nq = num_codes.max() nq_temp = self.codes_per_checkpoint scores, word_level_attentions, traceback_word_level_attentions, sentence_level_scores = [], [], [], [] for offset in range(0, nq, nq_temp): if codes is not None: codes_temp = codes[:, offset:offset + nq_temp] else: codes_temp = torch.zeros(0) if code_description is not None: code_description_temp = code_description[:, offset:offset + nq_temp] code_description_length_temp = code_description_length[:, offset: offset + nq_temp] else: code_description_temp = torch.zeros(0) code_description_length_temp = torch.zeros(0) if linearized_codes is not None: linearized_codes_temp = linearized_codes[:, offset:offset + nq_temp] linearized_codes_lengths_temp = linearized_codes_lengths[:, offset: offset + nq_temp] else: linearized_codes_temp = torch.zeros(0) linearized_codes_lengths_temp = torch.zeros(0) num_codes_temp = torch.clamp(num_codes - offset, 0, nq_temp) scores_temp, word_level_attentions_temp, traceback_word_level_attentions_temp, sentence_level_scores_temp = checkpoint( self.inner_forward, article_sentences, article_sentences_lengths, num_codes_temp, codes_temp, code_description_temp, code_description_length_temp, linearized_codes_temp, linearized_codes_lengths_temp, *self.parameters()) scores.append(scores_temp) word_level_attentions.append(word_level_attentions_temp) traceback_word_level_attentions.append( traceback_word_level_attentions_temp) sentence_level_scores.append(sentence_level_scores_temp) scores = torch.cat(scores, 1) word_level_attentions = torch.cat(word_level_attentions, 1) traceback_word_level_attentions = torch.cat( traceback_word_level_attentions, 1) sentence_level_scores = torch.cat(sentence_level_scores, 1) return_dict = dict( scores=scores, num_codes=num_codes, word_level_attentions=word_level_attentions, traceback_word_level_attentions=traceback_word_level_attentions, sentence_level_scores=sentence_level_scores, article_sentences_lengths=article_sentences_lengths) if codes is not None: return_dict['codes'] = codes return return_dict
def clip(self, x, clipping_value): return torch.clamp(x, min=-clipping_value, max=clipping_value)
def funcOptMin(a, b): return torch.clamp(a + b, max=2)
def f_p(self, p): # Calculate activation for inferred grounded location, using a leaky relu for sparsity. Either apply to full multi-frequency grounded location or single frequency module activation = [utils.leaky_relu(torch.clamp(p_f, min=-1, max=1)) for p_f in p] if type(p) is list else utils.leaky_relu(torch.clamp(p, min=-1, max=1)) return activation
def func2(a, b): return torch.clamp(a + b, min=0, max=2)
def f_g_clamp(self, g): # Calculate activation for abstract location, thresholding between -1 and 1 activation = [torch.clamp(g_f, min=-1, max=1) for g_f in g] return activation
def acquire_scores(base_cfg, samples_to_score, all_samples, model_file, depth_ifp_w=0, verbose=False): calc_depth_distances = depth_ifp_w > 0 depth_lambda = base_cfg["label_selection"]["depth_lambda"] entropy_lambda = base_cfg["label_selection"]["entropy_lambda"] dist_bias_weight = base_cfg["label_selection"]["bias_weight"] ifp_args = base_cfg["label_selection"]["ifp_args"] if not verbose: if isinstance(depth_lambda, list): for dl, el in zip(depth_lambda, entropy_lambda): assert dl + el > 0 else: assert depth_lambda + entropy_lambda > 0 or calc_depth_distances if calc_depth_distances and ifp_args["m"] in ["aspp", "u4", "u3", "bn"]: depth_teacher = build_depth_trainer_model(base_cfg) cfg = deepcopy(base_cfg) cfg['data']['augmentations'] = {} cfg['monodepth_options'].pop('crop_h') cfg['monodepth_options'].pop('crop_w') cfg['training']['batch_size'] = 1 cfg['data']['shuffle_trainset'] = False restrict_subset = all_samples if calc_depth_distances else samples_to_score cfg['data']['restrict_to_subset'] = { "mode": "fixed", "n_subset": len(restrict_subset), "subset": restrict_subset, } cfg["training"]["resume"] = model_file trainer = build_trainer(cfg, "label_selection_scoring") if cfg["training"]["resume"] is not None: trainer.load_resume(strict=True, load_model_only=True) else: print("LABEL_SELECTION: Warning - Evaluated model is None. This might happen when using ifp.") scores = [] all_depth_features = [] dist_i_to_img_idx = {} img_idx_to_dist_i = {} dist_bias = [] trainer.model.eval() with torch.no_grad(): depth_loss_mask = None for inputs in tqdm(trainer.train_data_loader): for k, v in inputs.items(): cuda_tensor_names = [("color_aug", 0, 0), "pseudo_depth"] if verbose: cuda_tensor_names.extend(["lbl", ("color", 0, 0)]) if torch.is_tensor(v) and k in cuda_tensor_names: inputs[k] = v.to(trainer.device, non_blocking=True) if calc_depth_distances: if ifp_args["pool"] == "avg": pool_fn = torch.nn.functional.adaptive_avg_pool2d elif ifp_args["pool"] == "max": pool_fn = torch.nn.functional.adaptive_max_pool2d else: raise NotImplementedError(ifp_args["pool"]) if ifp_args["m"] in ["aspp", "u3", "u4", "bn"]: teacher_outputs = depth_teacher(inputs) if ifp_args["m"] == "u3": depth_features = teacher_outputs[("upconv", 3)] elif ifp_args["m"] == "u4": depth_features = teacher_outputs[("upconv", 4)] elif ifp_args["m"] == "bn": depth_features = teacher_outputs["bottleneck"] else: raise NotImplementedError(ifp_args["m"]) depth_features = pool_fn(depth_features, (ifp_args["h"], 2 * ifp_args["h"])) elif ifp_args["m"] == "logdepth": depth_features = inputs["pseudo_depth"][0] depth_features = torch.log(torch.clamp(1 / depth_features, 0.1, 80)) depth_features = pool_fn(depth_features, (ifp_args["h"], 2 * ifp_args["h"])) depth_features.unsqueeze_(0) elif ifp_args["m"] == "depth": depth_features = inputs["pseudo_depth"][0] depth_features = torch.clamp(1 / depth_features, 0.1, 80) depth_features = pool_fn(depth_features, (ifp_args["h"], 2 * ifp_args["h"])) depth_features.unsqueeze_(0) else: raise NotImplementedError(ifp_args["m"]) assert depth_features.shape[0] == 1 dist_i_to_img_idx[len(all_depth_features)] = inputs["idx"].item() img_idx_to_dist_i[inputs["idx"].item()] = len(all_depth_features) all_depth_features.append(depth_features.detach()) if not verbose and dist_bias_weight == 0: scores.append({ "idx": inputs["idx"], "label_criterion": [0], "depth_error": [0], "entropy_mean": 0, }) continue with autocast(enabled=trainer.cfg["training"]["amp"]): outputs = trainer.model(inputs) if inputs["idx"] not in samples_to_score: dist_bias.append(0) continue entropy_imgs = pixel_wise_entropy(outputs["semantics"]) disp_pred = outputs["disp", 0][0][0] disp_pseudo = inputs["pseudo_depth"][0][0] depth_error_maps = [] depth_errors = [] depth_error_types = cfg["label_selection"].get("depth_error_types", "abs") if not isinstance(depth_error_types, list): depth_error_types = [depth_error_types] for depth_error_type in depth_error_types: if depth_error_type == "abs": depth_error_map = torch.abs(disp_pred - disp_pseudo) elif depth_error_type == "abs_inv_log": depth_pred = torch.log(torch.clamp(1 / disp_pred, 0.1, 80)) depth_pseudo = torch.log(torch.clamp(1 / disp_pseudo, 0.1, 80)) depth_error_map = torch.abs(depth_pseudo - depth_pred) elif depth_error_type == "abs_inv": depth_pred = torch.clamp(1 / disp_pred, 0.1, 80) depth_pseudo = torch.clamp(1 / disp_pseudo, 0.1, 80) depth_error_map = torch.abs(depth_pseudo - depth_pred) elif depth_error_type == "sq": depth_error_map = (disp_pred - disp_pseudo) ** 2 elif depth_error_type == "abs_rel": depth_error_map = torch.abs(disp_pred - disp_pseudo) / (disp_pseudo + 1e-1) elif depth_error_type == "sq_rel": depth_error_map = ((disp_pred - disp_pseudo) ** 2) / (disp_pseudo + 1e-1) elif depth_error_type == "abs_log": depth_error_map = torch.abs(torch.log(1 + disp_pred) - torch.log(1 + disp_pseudo)) else: raise NotImplementedError(depth_error_type) # Mask out cars moving in front with very small disparity mask = dilate((disp_pseudo < 0.07).float(), 7, 3) depth_error_map *= (1 - mask) # Mask out own car depth_error_map[int(0.87 * depth_error_map.shape[0]):, :] = 0 depth_error = torch.mean(depth_error_map) depth_error_maps.append(depth_error_map.detach()) depth_errors.append(depth_error.detach()) entropy_mean = torch.mean(entropy_imgs[0]) assert not (isinstance(depth_lambda, list) and len(depth_error_types) > 1) if isinstance(depth_lambda, list): label_criterion = [] for dl, el in zip(depth_lambda, entropy_lambda): label_criterion.append((dl * depth_error + el * entropy_mean).detach()) depth_error_maps.append(depth_error_map) depth_errors.append(depth_error) elif isinstance(depth_error_types, list): label_criterion = [] for depth_error in depth_errors: label_criterion.append((depth_lambda * depth_error + entropy_lambda * entropy_mean).detach()) else: label_criterion = (depth_lambda * depth_error + entropy_lambda * entropy_mean).detach() if dist_bias_weight > 0: assert len(label_criterion) == 1 dist_bias.append(dist_bias_weight * label_criterion[0]) scores.append({ "idx": inputs["idx"], "label_criterion": label_criterion, "depth_error": depth_errors, "entropy_mean": entropy_mean.detach(), }) if verbose: segmentation_loss = trainer.loss_fn( input=outputs["semantics"], target=inputs["lbl"], pixel_weights=None ) preds = outputs["semantics"].data.max(1)[1].cpu().numpy() gts = inputs["lbl"].data.cpu().numpy() for k, v in outputs.items(): if "depth" in k or "cam_T_cam" in k: outputs[k] = v.to(torch.float32) # trainer.monodepth_loss_calculator_train.generate_images_pred(all_inputs, outputs) # mono_losses = trainer.monodepth_loss_calculator_train.compute_losses(all_inputs, outputs) # mono_loss = mono_losses["loss"] mono_loss = torch.tensor([0]) mono_outputs = trainer.model.predict_test_disp(inputs) trainer.monodepth_loss_calculator_val.generate_depth_test_pred(mono_outputs) # Crop away bottom of image with own car if depth_loss_mask is None: depth_loss_mask = torch.ones(outputs["disp", 0].shape, device=trainer.device) depth_loss_mask[:, :, int(outputs["disp", 0].shape[2] * 0.9):, :] = 0 pseudo_depth_loss = berhu(outputs["disp", 0], inputs["pseudo_depth"], depth_loss_mask) running_metrics_val = runningScore(trainer.n_classes) running_metrics_val.update(gts, preds) score, class_iou = running_metrics_val.get_scores() scores[-1].update({ "image": inputs["color_aug", 0, 0][0].detach().cpu(), "segmentation_entropy": entropy_imgs[0].detach().cpu(), "disparity": torch.log(torch.clamp(1 / outputs["disp", 0][0], 0.1, 80)).detach().cpu(), "teacher_depth": torch.log(torch.clamp(1 / inputs["pseudo_depth"][0], 0.1, 80)).detach().cpu(), "depth_error_map": depth_error_maps, "mIoU": score["Mean IoU : \t"], "fwAcc": score["FreqW Acc : \t"], "mAcc": score["Mean Acc : \t"], "tAcc": score["Overall Acc: \t"], "cIoU": class_iou, "segmentation_loss": segmentation_loss.item(), "mono_loss": mono_loss.item(), "pseudo_depth_loss": pseudo_depth_loss.item(), "segmentation_pred": preds[0], "segmentation_gt": gts[0], # "reprojection_error_map": outputs["to_optimise/0"][0].detach().cpu(), }) depth_feature_distances = 0 if calc_depth_distances: depth_feature_distances = _calc_feature_distance(all_depth_features, dist_bias, dist_bias_weight, p=ifp_args["p"], normalize_features=ifp_args.get("norm", False), patch_wise=ifp_args.get("pw", False)) feature_distances = depth_ifp_w * depth_feature_distances return scores, {'distances': feature_distances, 'dist_i_to_img_idx': dist_i_to_img_idx, 'img_idx_to_dist_i': img_idx_to_dist_i}
def backward(self, sample_): sample_["logp"] = self.pd.log_prob(self.action) sample_["value"] = self.Q self.replay_buffer.push(sample_) self.running_step += 1 """""" """""" "" "training part" "in each step, we train for batch batch_training_times" """""" """""" "" if self.step > self.learning_starts: if self.running_step % self.run_step == 0 and self.training_step == 0: " sample advantage generate " with torch.no_grad(): sample = self.replay_buffer.recent_step_sample( self.running_step) last_value = self.value_model.forward(sample["s_"][-1]) self.record_sample = gae(sample, last_value, self.gamma, self.lam) self.running_step = 0 if self.training_step < self.sample_training_step and self.record_sample is not None: pg_loss_re = 0 entropy_re = 0 vf_loss_re = 0 loss_re = 0 for _ in range(self.batch_training_round): index = self.train_ticks[self.training_step] S = self.record_sample["s"][index].detach() A = self.record_sample["a"][index].detach() old_log = self.record_sample["logp"][index].detach() advs = self.record_sample["advs"][index].detach() value = self.record_sample["value"][index].detach() returns = self.record_sample["return"][index].detach() # generate Policy gradient loss outcome = self.run_policy.forward(S) new_policy = self.dist(outcome) new_lop = new_policy.log_prob(A) ratio = torch.exp(new_lop - old_log) pg_loss1 = advs * ratio pg_loss2 = advs * torch.clamp(ratio, 1.0 - self.cliprange, 1.0 + self.cliprange) pg_loss = -.5 * torch.min(pg_loss1, pg_loss2).mean() # value loss value_now = self.run_value.forward(S) value_clip = value + torch.clamp( value_now - value, min=-self.cliprange, max=self.cliprange) # Clipped value vf_loss1 = self.loss_cal(value_now, returns) # Unclipped loss vf_loss2 = self.loss_cal(value_clip, returns) # clipped loss vf_loss = .5 * torch.max(vf_loss1, vf_loss2) # vf_loss = 0.5 * vf_loss1 # entropy entropy = new_policy.entropy().mean() loss = pg_loss - entropy * self.ent_coef + vf_loss * self.vf_coef # approxkl = self.loss_cal(neg_log_pac, self.record_sample["neglogp"]) # self.cliprange = torch.gt(torch.abs(ratio - 1.0).mean(), self.cliprange) self.value_model_optim.zero_grad() loss.backward(retain_graph=True) self.value_model_optim.step() self.policy_model_optim.zero_grad() loss.backward() self.policy_model_optim.step() self.training_step += 1 pg_loss_re += pg_loss.data.numpy() entropy_re += entropy.data.numpy() vf_loss_re += vf_loss.data.numpy() loss_re += loss.data.numpy() if self.training_step == self.sample_training_step: print("the" + str(self.episode) + " round have training finished") self.run_policy.load_state_dict( self.policy_model.state_dict()) self.run_value.load_state_dict( self.value_model.state_dict()) self.training_step = 0 self.record_sample = None return loss_re, { "pg_loss": pg_loss_re, "entropy": entropy_re, "vf_loss": vf_loss_re } return 0, {"pg_loss": 0, "entropy": 0, "vf_loss": 0}
init_states[:, n_link + which] = mesh[1].reshape(-1) n_steps = 2000 ROApoints = [] grads = [] for n in range(n_samples): print("Sample # {}".format(n)) x = init_states[n] Trj = [x.copy()] running_V = 100. inROA = True for t in range(n_steps): x_torch = numpy2torch(x.copy()) with torch.no_grad(): _, u, V = net(x_torch.unsqueeze(0)) u = torch.clamp(u, -10., 10.) # Limit the amount of control input u = torch2numpy(u[0]) times = np.arange(2) * dt x_orig = x.copy() x_orig[:n_link] = wrapToPi(x_orig[:n_link] + target) y = odeint(system.gradient, x_orig, times, args=(u, )) y = y[1] y[:n_link] = wrapToPi(y[:n_link] - target) if t == 0: grads.append(system.gradient(x_orig, times, u)) within = (y.copy() <= domain).all() within = within & (y.copy() >= -domain).all()
def safe_log(x, eps=1e-6): """ Prevents :math:`log(0)` by using :math:`log(max(x, eps))`.""" return th.log(th.clamp(x, min=eps))