def test_train(self): self._metric.train() calls = [[torch.FloatTensor([0.0]), torch.LongTensor([0])], [torch.FloatTensor([0.0, 0.1, 0.2, 0.3]), torch.LongTensor([0, 1, 2, 3])]] for i in range(len(self._states)): self._metric.process(self._states[i]) self.assertEqual(2, len(self._metric_function.call_args_list)) for i in range(len(self._metric_function.call_args_list)): self.assertTrue(torch.eq(self._metric_function.call_args_list[i][0][0], calls[i][0]).all) self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[i][0][1], -calls[i][1])), 1e-12).all) self._metric_function.reset_mock() self._metric.process_final({}) self._metric_function.assert_called_once() self.assertTrue(torch.eq(self._metric_function.call_args_list[0][0][1], torch.LongTensor([0, 1, 2, 3, 4])).all) self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[0][0][0], -torch.FloatTensor([0.0, 0.1, 0.2, 0.3, 0.4]))), 1e-12).all)
def test_validate(self): self._metric.eval() for i in range(len(self._states)): self._metric.process(self._states[i]) self._metric_function.assert_not_called() self._metric.process_final_validate({}) self._metric_function.assert_called_once() self.assertTrue(torch.eq(self._metric_function.call_args_list[0][0][1], torch.LongTensor([0, 1, 2, 3, 4])).all) self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[0][0][0], -torch.FloatTensor([0.0, 0.1, 0.2, 0.3, 0.4]))), 1e-12).all)
def __init__(self, low, high, validate_args=None): self.low, self.high = broadcast_all(low, high) if isinstance(low, Number) and isinstance(high, Number): batch_shape = torch.Size() else: batch_shape = self.low.size() super(Uniform, self).__init__(batch_shape, validate_args=validate_args) if self._validate_args and not torch.lt(self.low, self.high).all(): raise ValueError("Uniform is not defined when low>= high")
def sample_with_temperature(logits, sampling_temp, keep_topk): """Select next tokens randomly from the top k possible next tokens. Samples from a categorical distribution over the ``keep_topk`` words using the category probabilities ``logits / sampling_temp``. Args: logits (FloatTensor): Shaped ``(batch_size, vocab_size)``. These can be logits (``(-inf, inf)``) or log-probs (``(-inf, 0]``). (The distribution actually uses the log-probabilities ``logits - logits.logsumexp(-1)``, which equals the logits if they are log-probabilities summing to 1.) sampling_temp (float): Used to scale down logits. The higher the value, the more likely it is that a non-max word will be sampled. keep_topk (int): This many words could potentially be chosen. The other logits are set to have probability 0. Returns: (LongTensor, FloatTensor): * topk_ids: Shaped ``(batch_size, 1)``. These are the sampled word indices in the output vocab. * topk_scores: Shaped ``(batch_size, 1)``. These are essentially ``(logits / sampling_temp)[topk_ids]``. """ if sampling_temp == 0.0 or keep_topk == 1: # For temp=0.0, take the argmax to avoid divide-by-zero errors. # keep_topk=1 is also equivalent to argmax. topk_scores, topk_ids = logits.topk(1, dim=-1) if sampling_temp > 0: topk_scores /= sampling_temp else: logits = torch.div(logits, sampling_temp) if keep_topk > 0: top_values, top_indices = torch.topk(logits, keep_topk, dim=1) kth_best = top_values[:, -1].view([-1, 1]) kth_best = kth_best.repeat([1, logits.shape[1]]).float() # Set all logits that are not in the top-k to -10000. # This puts the probabilities close to 0. ignore = torch.lt(logits, kth_best) logits = logits.masked_fill(ignore, -10000) dist = torch.distributions.Multinomial( logits=logits, total_count=1) topk_ids = torch.argmax(dist.sample(), dim=1, keepdim=True) topk_scores = logits.gather(dim=1, index=topk_ids) return topk_ids, topk_scores
def get_unsupervised_loss(self, examples): samples, sample_scores, enc_states = self.infer(examples) reconstruction_scores = self.decoder.score(samples) # compute prior probability prior_scores = self.prior([e.tgt_code for e in samples]) if isinstance(self.prior, UniformPrior): prior_scores = Variable(sample_scores.data.new(prior_scores)) kl_term = self.args.alpha * (sample_scores - prior_scores) raw_learning_signal = reconstruction_scores - kl_term baseline = self.baseline(samples, enc_states) learning_signal = raw_learning_signal.detach() - baseline # clip learning signal if self.args.clip_learning_signal is not None: mask = torch.lt(learning_signal, self.args.clip_learning_signal).float() clipped_learning_signal = learning_signal * (1. - mask) + mask * self.args.clip_learning_signal else: clipped_learning_signal = learning_signal encoder_loss = -clipped_learning_signal.detach() * sample_scores decoder_loss = -reconstruction_scores # compute baseline loss baseline_loss = learning_signal ** 2 meta_data = {'samples': samples, 'reconstruction_scores': reconstruction_scores, 'encoding_scores': sample_scores, 'raw_learning_signal': raw_learning_signal, 'learning_signal': learning_signal, 'baseline': baseline, 'kl_term': kl_term, 'prior': prior_scores} return encoder_loss, decoder_loss, baseline_loss, meta_data
def Focal_Loss(classifications, regressions, anchors, annotations): alpha = 0.25 gamma = 2.0 batch_size = classifications.shape[0] classification_losses = [] regression_losses = [] anchor = anchors[0, :, :] anchor_width = anchor[:, 2] - anchor[:, 0] anchor_height = anchor[:, 3] - anchor[:, 1] anchor_cx = anchor[:, 0] + 0.5 * anchor_width anchor_cy = anchor[:, 1] + 0.5 * anchor_height for j in range(batch_size): classification = classifications[j, :, :] regression = regressions[j, :, :] bbox_annotation = annotations[j, :, :] bbox_annotation = bbox_annotation[bbox_annotation[:, 4] != -1] if bbox_annotation.shape[0] == 0: regression_losses.append(torch.tensor(0).float().cuda()) classification_losses.append(torch.tensor(0).float().cuda()) continue classification = torch.clamp(classification, 1e-4, 1.0 - 1e-4) Iou = calc_iou(anchors[0, :, :], bbox_annotation[:, :4]) Iou_max, Iou_argmax = torch.max(Iou, dim=1) # 类别损失 targets = torch.ones(classification.shape) * -1 targets = targets.cuda() targets[torch.lt(Iou_max, 0.4), :] = 0 positive_indices = torch.ge(Iou_max, 0.5) num_positive_anchors = positive_indices.sum() assigned_annotations = bbox_annotation[Iou_argmax, :] targets[positive_indices, :] = 0 targets[positive_indices, assigned_annotations[positive_indices, 4].long()] = 1 alpha_factor = torch.ones(targets.shape).cuda() * alpha alpha_factor = torch.where(torch.eq(targets, 1.), alpha_factor, 1. - alpha_factor) focal_weight = torch.where(torch.eq(targets, 1.), 1. - classification, classification) focal_weight = alpha_factor * torch.pow(focal_weight, gamma) bce = -(targets * torch.log(classification) + (1.0 - targets) * torch.log(1.0 - classification)) cls_loss = focal_weight * bce cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss, torch.zeros(cls_loss.shape).cuda()) classification_losses.append( cls_loss.sum() / torch.clamp(num_positive_anchors.float(), min=1.0)) # 回归检测框损失 if positive_indices.sum() > 0: assigned_annotations = assigned_annotations[positive_indices, :] anchor_width_pi = anchor_width[positive_indices] anchor_height_pi = anchor_height[positive_indices] anchor_cx_pi = anchor_cx[positive_indices] anchor_cy_pi = anchor_cy[positive_indices] gt_width = assigned_annotations[:, 2] - assigned_annotations[:, 0] gt_height = assigned_annotations[:, 3] - assigned_annotations[:, 1] gt_cx = assigned_annotations[:, 0] + 0.5 * gt_width gt_cy = assigned_annotations[:, 1] + 0.5 * gt_height gt_width = torch.clamp(gt_width, min=1) gt_height = torch.clamp(gt_height, min=1) targets_dx = (gt_cx - anchor_cx_pi) / anchor_width_pi targets_dy = (gt_cy - anchor_cy_pi) / anchor_height_pi targets_dw = torch.log(gt_width / anchor_width_pi) targets_dh = torch.log(gt_height / anchor_height_pi) targets = torch.stack( (targets_dx, targets_dy, targets_dw, targets_dh)) targets = targets.t() targets = targets / torch.Tensor([[0.1, 0.1, 0.2, 0.2]]).cuda() negative_indices = 1 - positive_indices regression_diff = torch.abs(targets - regression[positive_indices, :]) regression_loss = torch.where( torch.le(regression_diff, 1.0 / 9.0), 0.5 * 9.0 * torch.pow(regression_diff, 2), regression_diff - 0.5 / 9.0) regression_losses.append(regression_loss.mean()) else: regression_losses.append(torch.tensor(0).float().cuda()) c_losses = torch.stack(classification_losses).mean(dim=0, keepdim=True) r_losses = torch.stack(regression_losses).mean(dim=0, keepdim=True) return c_losses, r_losses
def torch_all_close(a, b, tolerance=1e-12): """ Check if all elements in two tensors are equal within a tolerance. """ return torch.all(torch.lt(torch.abs(a - b), tolerance))
def easy(x, y): c = torch.lt(x, y) return c
def forward(self, operation, x=None, w=None, ws=None, wP=None, wc=None, wC=None, msk_s=None, msk_c=None, mode=None): # x: T(bat, max_word_num) # w: T(bat) # x_embedding: T(bat, max_word_num, embed_dim) x_embedding = self.embedding(x) x_embedding = self.embedding_dropout(x_embedding) # mask: T(bat, max_word_num) mask = torch.gt(x, 0).to(torch.int64) # x_len: T(bat) x_len = torch.sum(mask, dim=1) # h: T(bat, max_word_num, hid*2) # ht: T(num_layers*2, bat, hid) float32 h, (ht, _) = self.encoder(x_embedding, x_len) # ht: T(bat, hid*2) ht = torch.transpose(ht[ht.shape[0] - 2:, :, :], 0, 1).contiguous().view(x_len.shape[0], self.hidden_dim * 2) # alpha: T(bat, max_word_num, 1) alpha = (h.bmm(ht.unsqueeze(2))) # mask_3: T(bat, max_word_num, 1) mask_3 = mask.to(torch.float32).unsqueeze(2) ## word prediction # vd: T(bat, embed_dim) #h_ = self.fc(h) #vd = torch.sum(h_, 1) #vd = self.fc(torch.max(h, dim=1)[0]) h_1 = torch.sum(h * alpha, 1) vd = self.fc( h_1 ) #+ torch.sum(self.embedding(x), 1)#+ torch.sum(x_embedding, 1) #ok #vd = self.fc(torch.cat([torch.sum(h*alpha, 1), torch.sum(self.embedding(x), 1)], 1)) #ok #vd = self.fc(torch.sum(torch.cat([h, self.embedding(x)], 2)*alpha, 1)) #best #vd = self.fc(torch.max(torch.cat([h, self.embedding(x)], 2), 1)[0]) #bad #vd = torch.sum(self.embedding(x), 1) #vd = torch.max(self.embedding.weight(x), 1)[0] #vd = self.fc(ht) # score: T(bat, calss_num) score0 = vd.mm(self.embedding.weight[[range(self.class_num)]].t()) score = score0 if 'C' in mode: # scC[i]: T(bat, Ci_size) # 词林的层次分类训练的慢,其实这样不公平,不平衡,因为词预测先收敛了,而cilin的分类还没效果,其他信息的利用也有同样的问题,不一定同时收敛!!! scC = [ self.fc_C1(h_1), self.fc_C2(h_1), self.fc_C3(h_1), self.fc_C4(h_1) ] score2 = torch.zeros((score0.shape[0], score0.shape[1]), dtype=torch.float32, device=device) rank = 0.6 for i in range(4): # wC[i]: T(class_num, Ci_size) # C_sc: T(bat, class_num) score2 += self.relu(scC[i].mm(wC[i].t()) * (rank**i)) #----------add mean cilin-class score to those who have no cilin-class mean_cilin_sc = torch.mean(score2, 1) score2 = score2 * (1 - msk_c) + mean_cilin_sc.unsqueeze(1).mm( msk_c.unsqueeze(0)) #---------- score = score + score2 / 3 if 'P' in mode: ## POS prediction # score_POS: T(bat, 13) pos_num=12+1 score_POS = self.fc2(torch.sum(h * alpha, 1)) # s: (class_num, 13) multi-hot # weight_sc: T(bat, class_num) = [bat, 13] .mm [class_num, 13].t() weight_sc = self.relu(score_POS.mm(wP.t())) #print(torch.max(weight_sc), torch.min(weight_sc)) score = score + weight_sc if 's' in mode: ## sememe prediction # pos_score: T(bat, max_word_num, sememe_num) pos_score = self.fc1(h) pos_score = pos_score * mask_3 + (-1e7) * (1 - mask_3) # sem_score: T(bat, sememe_num) sem_score, _ = torch.max(pos_score, dim=1) #sem_score = torch.sum(pos_score * alpha, 1) # score: T(bat, class_num) = [bat, sememe_num] .mm [class_num, sememe_num].t() score1 = self.relu(sem_score.mm(ws.t())) #----------add mean sememe score to those who have no sememes # mean_sem_sc: T(bat) mean_sem_sc = torch.mean(score1, 1) # msk: T(class_num) score1 = score1 + mean_sem_sc.unsqueeze(1).mm(msk_s.unsqueeze(0)) #---------- score = score + score1 if 'c' in mode: ## character prediction # pos_score: T(bat, max_word_num, sememe_num) pos_score = self.fc3(h) pos_score = pos_score * mask_3 + (-1e7) * (1 - mask_3) # chara_score: T(bat, chara_num) chara_score, _ = torch.max(pos_score, dim=1) #chara_score = torch.sum(pos_score * alpha, 1) # score: T(bat, class_num) = [bat, sememe_num] .mm [class_num, sememe_num].t() score3 = self.relu(chara_score.mm(wc.t())) score = score + score3 # fine-tune depended on the target word shouldn't exist in the definition. #score_res = score.clone().detach() mask1 = torch.lt(x, self.class_num).to(torch.int64) mask2 = torch.ones((score.shape[0], score.shape[1]), dtype=torch.float32, device=device) for i in range(x.shape[0]): mask2[i][x[i] * mask1[i]] = 0. score = score * mask2 + (-1e6) * (1 - mask2) _, indices = torch.sort(score, descending=True) if operation == 'train': loss = self.loss(score, w) return loss, score, indices elif operation == 'test': return indices
def act_step(self, exper, meta_optimizer): """ Subtleties of the batch processing. TODO: NEED TO EXPLAIN THIS!!! :param exper: :param meta_optimizer: :return: IMPORTANT RETURNS LOSS STEP THAT IS NOT MULTIPLIED BY QT-VALUE! NUMPY FLOAT32 """ self.init_trunc_bptt(exper, meta_optimizer) loss = get_func_loss(exper, self.functions, average=False) # make the forward step, which depends heavily on the experiment we're performing (MLP or Regression(T)) delta_param, rho_probs, eval_par_new = self.forward( loss, exper, meta_optimizer) # then, apply the previous batch mask, although we did the forward pass with all functions, we filter here delta_param = torch.mul(delta_param, self.float_mask) # moved to method >>> compute_probs <<< # rho_new = torch.mul(rho_probs, self.float_mask) # compute new probability values, based on the cumulative probs construct new batch mask # note that we also save the new rho_t values in the array self.rho_t (in the method compute_probs) if self.learner == "meta_act": # in Graves ACT model, the qt values are probabilities already : sigmoid(W^T h_t + bias) values # so we don't use any stick-breaking here (meaning the rho_t values that we transfer in the act_sb new_probs = torch.mul(rho_probs.double(), self.float_mask.double()) else: # stick-breaking approach: transform RNN output rho_t values to probabilities # method compute_probs multiplies with the float_mask object! new_probs = self.compute_probs(rho_probs) # we need to determine the indices of the functions that "stop" in this time step (new_funcs_mask) funcs_that_stop = torch.le(self.compare_probs + new_probs, self.one_minus_eps) less_or_equal = LessOrEqual() # NOTE: ALSO for the Graves ACT model we increase all functions that participated in THIS ITERATION and # therefore we compare with self.compare_probs BEFORE we increase them with the new probs iterations = less_or_equal(self.compare_probs, self.one_minus_eps) if self.learner == "meta_act": self.halting_steps += iterations else: # and we increase the number of time steps TAKEN with the previous float_mask object to keep track of the steps self.halting_steps += self.float_mask self.iterations += iterations new_batch_mask = tensor_and(funcs_that_stop, self.bool_mask) new_float_mask = new_batch_mask.float() # IMPORTANT: although above we constructed the new mask (for the next time step) we use the previous mask for the # increase of the cumulative probs which we use in the WHILE loop to determine when to stop the complete batch self.compare_probs += torch.mul(new_probs, self.float_mask.double()) # IMPORTANT UPDATE OF OPTIMIZEE PARAMETERS if self.func_is_nn_module: # awkward but true: delta_param has [batch_dim, num_params] due to multiplication with float masks, but # must have transpose shape for this update par_new = self.functions.get_flat_params() - delta_param.permute( 1, 0) else: par_new = self.functions.params - delta_param # increase the number of steps taken for the functions that are still in the race, we need these to compare # against the maximum allowed number of steps self.counter_compare += new_float_mask # generate object that holds time_step_condition (which function has taken more than MAX time steps allowed) # Note that self.max_T = HORIZON time_step_condition = torch.lt(self.counter_compare, self.max_T) # we then generate the mask that determines which functions will finally be part of the next round next_iteration_condition = tensor_and(new_batch_mask, time_step_condition) final_func_mask = (self.bool_mask - next_iteration_condition) == 1 # set the cumulative_probs for all functions that participate in the next time step # Subtlety here: so for an optimizee stopping in this step, we don't want the cum-prob to be increased # because we'll use the previous (step-1) cum-probs to determine the q_T value...the final # q_t value for the halting step which gets assigned the REST prob mass # we use this object for determining the rest-probability for the functions that stop after this time step # the mask "final_func_mask" holds the indices of the functions that stop after this time step # the mask "next_iteration_condition" holds the indices of the functions that continue in the next time step self.cumulative_probs += torch.mul(new_probs, next_iteration_condition.double()) self.set_qt_values(new_probs, next_iteration_condition, final_func_mask) # set the masks for the next time step self.float_mask = new_float_mask self.bool_mask = new_float_mask.type(torch.cuda.ByteTensor) if exper.args.cuda else \ new_float_mask.type(torch.ByteTensor) # compute the new step loss if self.is_train: loss_step = self.step_loss(par_new, exper, average_batch=True) else: loss_step = self.step_loss(eval_par_new, exper, average_batch=True) batch_loss = self.batch_step_losses[-1] # we don't "have" any variance when batch size is 1 if self.batch_size > 1: exper.add_step_loss_variance(batch_loss, self.step + 1) elif exper.args.problem == "mlp": # batch_size is always one, but we want to compute stddev later, so just save all losses for the function # we calculate the stddev in Experiment.eval() method self.np_step_losses[ self.step + 1] = batch_loss.data.cpu().squeeze().numpy()[0] # update batch functions parameter for next step if self.is_train: self.functions.set_parameters(par_new) else: self.functions.set_parameters(eval_par_new) # MLP which is a torch.nn.module or regression function (which is just a normal Class object) if self.func_is_nn_module: self.functions.zero_grad() else: self.functions.params.grad.data.zero_() return loss_step.data.cpu().squeeze().numpy()[0]
def sample_from_mean_field_model(Q, batch_size): random_sample = torch.rand((batch_size, Q.dim_latent)) return torch.lt(random_sample, Q.parameters.repeat(batch_size, 1)).float()
def forward(self, classifications, regressions, anchors, annotations, **kwargs): alpha = 0.25 gamma = 2.0 batch_size = classifications.shape[0] classification_losses = [] regression_losses = [] anchor = anchors[ 0, :, :] # assuming all image sizes are the same, which it is dtype = anchors.dtype anchor_widths = anchor[:, 3] - anchor[:, 1] anchor_heights = anchor[:, 2] - anchor[:, 0] anchor_ctr_x = anchor[:, 1] + 0.5 * anchor_widths anchor_ctr_y = anchor[:, 0] + 0.5 * anchor_heights for j in range(batch_size): classification = classifications[j, :, :] regression = regressions[j, :, :] bbox_annotation = annotations[j] bbox_annotation = bbox_annotation[bbox_annotation[:, 4] != -1] classification = torch.clamp(classification, 1e-4, 1.0 - 1e-4) if bbox_annotation.shape[0] == 0: if torch.cuda.is_available(): alpha_factor = torch.ones_like(classification) * alpha alpha_factor = alpha_factor.cuda() alpha_factor = 1. - alpha_factor focal_weight = classification focal_weight = alpha_factor * torch.pow( focal_weight, gamma) bce = -(torch.log(1.0 - classification)) cls_loss = focal_weight * bce regression_losses.append(torch.tensor(0).to(dtype).cuda()) classification_losses.append(cls_loss.sum()) else: alpha_factor = torch.ones_like(classification) * alpha alpha_factor = 1. - alpha_factor focal_weight = classification focal_weight = alpha_factor * torch.pow( focal_weight, gamma) bce = -(torch.log(1.0 - classification)) cls_loss = focal_weight * bce regression_losses.append(torch.tensor(0).to(dtype)) classification_losses.append(cls_loss.sum()) continue IoU = calc_iou(anchor[:, :], bbox_annotation[:, :4]) IoU_max, IoU_argmax = torch.max(IoU, dim=1) # compute the loss for classification targets = torch.ones_like(classification) * -1 if torch.cuda.is_available(): targets = targets.cuda() targets[torch.lt(IoU_max, 0.4), :] = 0 positive_indices = torch.ge(IoU_max, 0.5) num_positive_anchors = positive_indices.sum() assigned_annotations = bbox_annotation[IoU_argmax, :] targets[positive_indices, :] = 0 targets[positive_indices, assigned_annotations[positive_indices, 4].long()] = 1 alpha_factor = torch.ones_like(targets) * alpha if torch.cuda.is_available(): alpha_factor = alpha_factor.cuda() alpha_factor = torch.where(torch.eq(targets, 1.), alpha_factor, 1. - alpha_factor) focal_weight = torch.where(torch.eq(targets, 1.), 1. - classification, classification) focal_weight = alpha_factor * torch.pow(focal_weight, gamma) bce = -(targets * torch.log(classification) + (1.0 - targets) * torch.log(1.0 - classification)) cls_loss = focal_weight * bce zeros = torch.zeros_like(cls_loss) if torch.cuda.is_available(): zeros = zeros.cuda() cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss, zeros) classification_losses.append( cls_loss.sum() / torch.clamp(num_positive_anchors.to(dtype), min=1.0)) if positive_indices.sum() > 0: assigned_annotations = assigned_annotations[ positive_indices, :] anchor_widths_pi = anchor_widths[positive_indices] anchor_heights_pi = anchor_heights[positive_indices] anchor_ctr_x_pi = anchor_ctr_x[positive_indices] anchor_ctr_y_pi = anchor_ctr_y[positive_indices] gt_widths = assigned_annotations[:, 2] - assigned_annotations[:, 0] gt_heights = assigned_annotations[:, 3] - assigned_annotations[:, 1] gt_ctr_x = assigned_annotations[:, 0] + 0.5 * gt_widths gt_ctr_y = assigned_annotations[:, 1] + 0.5 * gt_heights # efficientdet style gt_widths = torch.clamp(gt_widths, min=1) gt_heights = torch.clamp(gt_heights, min=1) targets_dx = (gt_ctr_x - anchor_ctr_x_pi) / anchor_widths_pi targets_dy = (gt_ctr_y - anchor_ctr_y_pi) / anchor_heights_pi targets_dw = torch.log(gt_widths / anchor_widths_pi) targets_dh = torch.log(gt_heights / anchor_heights_pi) targets = torch.stack( (targets_dy, targets_dx, targets_dh, targets_dw)) targets = targets.t() regression_diff = torch.abs(targets - regression[positive_indices, :]) regression_loss = torch.where( torch.le(regression_diff, 1.0 / 9.0), 0.5 * 9.0 * torch.pow(regression_diff, 2), regression_diff - 0.5 / 9.0) regression_losses.append(regression_loss.mean()) else: if torch.cuda.is_available(): regression_losses.append(torch.tensor(0).to(dtype).cuda()) else: regression_losses.append(torch.tensor(0).to(dtype)) # debug imgs = kwargs.get('imgs', None) if imgs is not None: regressBoxes = BBoxTransform() clipBoxes = ClipBoxes() obj_list = kwargs.get('obj_list', None) out = postprocess( imgs.detach(), torch.stack([anchors[0]] * imgs.shape[0], 0).detach(), regressions.detach(), classifications.detach(), regressBoxes, clipBoxes, 0.5, 0.3) imgs = imgs.permute(0, 2, 3, 1).cpu().numpy() imgs = ((imgs * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]) * 255).astype(np.uint8) imgs = [cv2.cvtColor(img, cv2.COLOR_RGB2BGR) for img in imgs] display(out, imgs, obj_list, imshow=False, imwrite=True) return torch.stack(classification_losses).mean(dim=0, keepdim=True), \ torch.stack(regression_losses).mean(dim=0, keepdim=True)
def forward(self, marker_data, time_data, mask_data, embedding): size = time_data.size() self.index = 0 # Initialize Data Inputs and Outputs marker_res = torch.zeros(size, dtype=torch.long).cuda(self.cuda_id) time_res = time_data[:, 0:1] mask_res = torch.zeros(size, dtype=torch.float).cuda(self.cuda_id) marker_res[:, 0:1] = marker_data[:, 0:1] mask_res[:, 0:1] = mask_data[:, 0:1] data_input = torch.zeros(size[0], 0, self.d_model).cuda(self.cuda_id) intense_res = torch.zeros(size[0], 0, self.d_model).cuda(self.cuda_id) candidate_list = marker_data[:, 0:1] prob_list = torch.ones(size[0], 1).cuda(self.cuda_id) chosen_index = torch.zeros(size[0], 1, dtype=torch.long).cuda(self.cuda_id) # Hidden States of RNN Encoder hidden_state = torch.zeros(1, size[0], self.d_model).cuda(self.cuda_id) # Output Probabilities for Expectation Maximization neighbor_prob_record = torch.ones(size[0], 1).cuda(self.cuda_id) total_neighbor_prob = torch.ones(size[0], 1).cuda(self.cuda_id) total_sample_prob = torch.ones(size[0], 1).cuda(self.cuda_id) # Sequence Generation while self.index < size[1] - 1: last_marker = marker_res[:, self.index:self.index + 1] last_time = time_res[:, self.index:self.index + 1] new_vector = self.get_embedding(last_marker, last_time, embedding) model_dim_vector = self.embed_ac(self.embed_linear(new_vector)) data_input = torch.cat((data_input, model_dim_vector), 1) rnn_output = self.encoder( data_input[:, self.index:self.index + 1, :], hidden_state) intensity = rnn_output[0] hidden_state = rnn_output[1] # Time Decoding new_time = last_time + F.softplus( self.time_linear(intensity.squeeze(1))) time_res = torch.cat((time_res, new_time), 1) new_mask = torch.lt(new_time, self.max_time).long() intense_res = torch.cat((intense_res, intensity), 1) neighbor = self.neighbor_list[last_marker].squeeze(1) prob_neighbor = self.neighbor_prob[last_marker].squeeze(1) # Intensity Function Stacks neighbor_prob_record = torch.cat( (neighbor_prob_record, prob_neighbor), 1) neighbor_inf = embedding[neighbor] intensity_inf = torch.stack([(intensity.squeeze(1)) for _ in range(self.sample_size)], 1) inf_matrix = torch.cat((neighbor_inf, intensity_inf), 2) # Marker Decoding marker_weight = self.marker_linear(inf_matrix).squeeze(2) marker_prob = torch.softmax(marker_weight, 1) # Marker Sampling candidate_list = torch.cat((candidate_list, neighbor[:, 1:]), 1) chosen_prob = torch.gather(prob_list, 1, chosen_index) attached_prob = chosen_prob * marker_prob for i in range(size[0]): prob_list[i][chosen_index[i]] = attached_prob[i][0] prob_list = torch.cat((prob_list, attached_prob[:, 1:]), 1) chosen_index = torch.multinomial(prob_list, 1) new_markers = torch.gather(candidate_list, 1, chosen_index) # Record the marker probabilities for optimization selected_neighbor_prob = torch.gather(neighbor_prob_record, 1, chosen_index) total_neighbor_prob = torch.cat( (total_neighbor_prob, selected_neighbor_prob), 1) selected_sample_prob = torch.gather(prob_list, 1, chosen_index) total_sample_prob = torch.cat( (total_sample_prob, selected_sample_prob), 1) self.index += 1 marker_res = torch.cat((marker_res, new_markers), 1) mask_res[:, self.index:self.index + 1] = new_mask return marker_res, time_res, mask_res, total_neighbor_prob, total_sample_prob
def forward(self, marker_data, time_data, mask_data, embedding): # Forward Propagation d_size = time_data.size() # Initialize data self.index = 0 marker_res = marker_data[:, 0:1].clone() time_res = time_data[:, 0:1].clone() mask_res = mask_data[:, 0:1].clone() data_input = torch.zeros(d_size[0], 0, self.d_model).cuda(self.cuda_id) candidate_list = marker_data[:, 0:1] prob_list = torch.ones(d_size[0], 1).cuda(self.cuda_id) chosen_index = torch.zeros(d_size[0], 1, dtype=torch.long).cuda(self.cuda_id) neighbor_prob_record = torch.ones(d_size[0], 1).cuda(self.cuda_id) total_neighbor_prob = torch.ones(d_size[0], 1).cuda(self.cuda_id) total_sample_prob = torch.ones(d_size[0], 1).cuda(self.cuda_id) # Generating Cascades while self.index < d_size[1] - 1: last_marker = marker_res[:, self.index:self.index + 1] last_time = time_res[:, self.index:self.index + 1] new_vector = self.get_embedding(last_marker, last_time, embedding) model_dim_vector = self.embed_ac(self.embed_linear(new_vector)) data_input = torch.cat((data_input, model_dim_vector), 1) intensity = self.encoder.forward(data_input, self.index) # Time Decoding new_time = last_time + F.softplus( self.time_linear(intensity.squeeze(1))) # Causal Descendants time_res = torch.cat((time_res, new_time), 1) new_mask = torch.lt(new_time, self.max_time).float() neighbor = self.neighbor_list[last_marker].squeeze(1) prob_neighbor = self.neighbor_prob[last_marker].squeeze(1) neighbor_prob_record = torch.cat( (neighbor_prob_record, prob_neighbor), 1) # Intensity Function neighbor_inf = embedding[neighbor] intensity_inf = torch.stack([(intensity.squeeze(1)) for _ in range(self.sample_size)], 1) inf_matrix = torch.cat((neighbor_inf, intensity_inf), 2) # Marker Decoding marker_weight = self.marker_linear(inf_matrix).squeeze(2) marker_prob = torch.softmax(marker_weight, 1) candidate_list = torch.cat((candidate_list, neighbor[:, 1:]), 1) chosen_prob = torch.gather(prob_list, 1, chosen_index) attached_prob = chosen_prob * marker_prob for i in range(d_size[0]): prob_list[i][chosen_index[i]] = attached_prob[i][0] prob_list = torch.cat((prob_list, attached_prob[:, 1:]), 1) chosen_index = torch.multinomial(prob_list, 1) new_markers = torch.gather(candidate_list, 1, chosen_index) # Record Probabilities for BP selected_neighbor_prob = torch.gather(neighbor_prob_record, 1, chosen_index) total_neighbor_prob = torch.cat( (total_neighbor_prob, selected_neighbor_prob), 1) selected_sample_prob = torch.gather(prob_list, 1, chosen_index) total_sample_prob = torch.cat( (total_sample_prob, selected_sample_prob), 1) self.index += 1 # Mark down the Results marker_res = torch.cat((marker_res, new_markers), 1) mask_res = torch.cat((mask_res, new_mask), 1) return marker_res, time_res, mask_res, total_neighbor_prob, total_sample_prob
def test_comparison_ops_with_type_promotion(self, device): value_for_type = { torch.uint8: (1 << 5), torch.int8: (1 << 5), torch.int16: (1 << 10), torch.int32: (1 << 20), torch.int64: (1 << 35), torch.float16: (1 << 10), torch.float32: (1 << 20), torch.float64: (1 << 35), torch.complex64: (1 << 20), torch.complex128: (1 << 35) } comparison_ops = [ dict( name="lt", out_op=lambda x, y, d: torch.lt( x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.lt(x, y), compare_op=lambda x, y: x < y, ), dict( name="le", out_op=lambda x, y, d: torch.le( x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.le(x, y), compare_op=lambda x, y: x <= y, ), dict( name="gt", out_op=lambda x, y, d: torch.gt( x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.gt(x, y), compare_op=lambda x, y: x > y, ), dict( name="ge", out_op=lambda x, y, d: torch.ge( x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.ge(x, y), compare_op=lambda x, y: x >= y, ), dict( name="eq", out_op=lambda x, y, d: torch.eq( x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.eq(x, y), compare_op=lambda x, y: x == y, ), dict( name="ne", out_op=lambda x, y, d: torch.ne( x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.ne(x, y), compare_op=lambda x, y: x != y, ), ] for op in comparison_ops: for dt1 in torch.testing.get_all_math_dtypes(device): for dt2 in torch.testing.get_all_math_dtypes(device): if (dt1.is_complex or dt2.is_complex ) and not (op["name"] == "eq" or op["name"] == "ne"): continue val1 = value_for_type[dt1] val2 = value_for_type[dt2] t1 = torch.tensor([val1], dtype=dt1, device=device) t2 = torch.tensor([val2], dtype=dt2, device=device) expected = torch.tensor([op["compare_op"](val1, val2)], dtype=torch.bool) out_res = op["out_op"](t1, t2, device) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2) out_res = op["ret_op"](t1, t2) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2) # test that comparing a zero dim tensor with another zero dim tensor has type promotion behavior t1 = torch.tensor(val1, dtype=dt1, device=device) t2 = torch.tensor(val2, dtype=dt2, device=device) expected = torch.tensor(op["compare_op"](val1, val2), dtype=torch.bool) out_res = op["out_op"](t1, t2, device) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2) out_res = op["ret_op"](t1, t2) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2)
def initialize(self, opt): BaseModel.initialize(self, opt) self.opt = opt self.isTrain = opt.isTrain # specify the training losses you want to print out. The program will call base_model.get_current_losses self.loss_names = ['G_GAN', 'G_L1', 'D_real', 'D_fake'] # specify the images you want to save/display. The program will call base_model.get_current_visuals self.visual_names = ['real_A', 'fake_B', 'real_B'] # specify the models you want to save to the disk. The program will call base_model.save_networks and base_model.load_networks if self.isTrain: self.model_names = ['G', 'D'] else: # during test time, only load Gs self.model_names = ['G'] # batchsize should be 1 for mask_global self.mask_global = torch.ByteTensor(1, 1, \ opt.fineSize, opt.fineSize) # Here we need to set an artificial mask_global(not to make it broken, so center hole is ok.) self.mask_global.zero_() self.mask_global[:, :, int(self.opt.fineSize/4) + self.opt.overlap : int(self.opt.fineSize/2) + int(self.opt.fineSize/4) - self.opt.overlap,\ int(self.opt.fineSize/4) + self.opt.overlap: int(self.opt.fineSize/2) + int(self.opt.fineSize/4) - self.opt.overlap] = 1 self.mask_type = opt.mask_type self.gMask_opts = {} self.fixed_mask = opt.fixed_mask if opt.mask_type == 'center' else 0 if opt.mask_type == 'center': assert opt.fixed_mask == 1, "Center mask must be fixed mask!" if self.mask_type == 'random': res = 0.06 # the lower it is, the more continuous the output will be. 0.01 is too small and 0.1 is too large density = 0.25 MAX_SIZE = 10000 maxPartition = 30 low_pattern = torch.rand(1, 1, int(res*MAX_SIZE), int(res*MAX_SIZE)).mul(255) pattern = F.functional.interpolate(low_pattern, (MAX_SIZE, MAX_SIZE), mode='bilinear').detach() low_pattern = None pattern.div_(255) pattern = torch.lt(pattern,density).byte() # 25% 1s and 75% 0s pattern = torch.squeeze(pattern).byte() print('...Random pattern generated') self.gMask_opts['pattern'] = pattern self.gMask_opts['MAX_SIZE'] = MAX_SIZE self.gMask_opts['fineSize'] = opt.fineSize self.gMask_opts['maxPartition'] = maxPartition self.gMask_opts['mask_global'] = self.mask_global self.mask_global = util.create_gMask(self.gMask_opts) # create an initial random mask. self.wgan_gp = False # added for wgan-gp if opt.gan_type == 'wgan_gp': self.gp_lambda = opt.gp_lambda self.ncritic = opt.ncritic self.wgan_gp = True if len(opt.gpu_ids) > 0: self.use_gpu = True self.mask_global = self.mask_global.to(self.device) # load/define networks # self.ng_innerCos_list is the constraint list in netG inner layers. # self.ng_mask_list is the mask list constructing shift operation. self.netG, self.ng_innerCos_list, self.ng_shift_list = networks.define_G(opt.input_nc, opt.output_nc, opt.ngf, opt.which_model_netG, opt, self.mask_global, opt.norm, opt.use_dropout, opt.init_type, self.gpu_ids, opt.init_gain) # add opt, we need opt.shift_sz and other stuffs if self.isTrain: use_sigmoid = False if opt.gan_type == 'vanilla': use_sigmoid = True # only vanilla GAN using BCECriterion # don't use cGAN self.netD = networks.define_D(opt.input_nc, opt.ndf, opt.which_model_netD, opt.n_layers_D, opt.norm, use_sigmoid, opt.init_type, self.gpu_ids, opt.init_gain) if self.isTrain: self.old_lr = opt.lr # define loss functions self.criterionGAN = networks.GANLoss(gan_type=opt.gan_type).to(self.device) self.criterionL1 = torch.nn.L1Loss() # initialize optimizers self.schedulers = [] self.optimizers = [] if self.wgan_gp: opt.beta1 = 0 self.optimizer_G = torch.optim.Adam(self.netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) self.optimizer_D = torch.optim.Adam(self.netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) else: self.optimizer_G = torch.optim.Adam(self.netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) self.optimizer_D = torch.optim.Adam(self.netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) self.optimizers.append(self.optimizer_G) self.optimizers.append(self.optimizer_D) for optimizer in self.optimizers: self.schedulers.append(networks.get_scheduler(optimizer, opt)) if not self.isTrain or opt.continue_train: self.load_networks(opt.which_epoch) self.print_networks(opt.verbose)
def lower_bound_softplus_inv(x, low=0., high=0.): return torch.log(torch.exp(x - low) - 1) def upper_bound_softplus(x, low=0., high=0.): return -torch.log(1 + torch.exp(-x + high)) + high def upper_bound_softplus_inv(x, low=0., high=0.): return -torch.log(torch.exp(-x + high) - 1) + high if __name__ == "__main__": # Sanity Check: lists = torch.arange(-5, 5).float() if torch.all(torch.lt(torch.abs(torch.add(softplus_inv(softplus(lists)), -lists)), 1e-3)): print("success") else: print("fail") if torch.all(torch.lt(torch.abs(torch.add(softplus_inv(softplus(lists, low=3, high=5), low=3, high=5), -lists)), 1e-3)): print("success") else: print("fail") if torch.all(torch.lt(torch.abs(torch.add(exp_inv(exp(lists)), -lists)), 1e-3)): print("success") else: print("fail") if torch.all(torch.lt(torch.abs(torch.add(exp_inv(exp(lists, low=3, high=5), low=3, high=5), -lists)), 1e-3)): print("success")
def train(self): dataset = samp.FaceDataset(self.rootpath, islandmark=True) self.dataloader = data.DataLoader(dataset, batch_size=512, shuffle=True, num_workers=4) self.optimer = torch.optim.Adam( self.net.parameters(), lr=1e-4, weight_decay=0.00005) #weight_decay 正则化因子 if os.path.exists(self.parampath): # 恢复参数继续训练 self.net.load_state_dict(torch.load(self.parampath)) if not os.path.exists(self.parampath): #创建参数文件 dirs, file = os.path.split(self.parampath) if not os.path.exists(dirs): os.makedirs(dirs) #训练 while True: for epoch, (imgdata, con, offset, landmarks) in enumerate(self.dataloader): # batchsize if self.isCuda == True: #GPU 运算 imgdata, con, offset, landmarks = imgdata.cuda(), con.cuda( ), offset.cuda(), landmarks.cuda() con_out, offset_out, landmarks_out = self.net( imgdata) # imgdata为网络输入, 得到网络输出 #如何保证con_out > 0??? # Assertion `x >= 0. && x <= 1.' failed. input value should be between 0~1, but got -0.157099 at # 加signmod 函数 转化成概率 # 计算置信度损失 # print(con) # print("==================") # print(con_out) con_mask = torch.lt(con, 2) #部分样本不参与分类损失 2定义为部分样本 掩码操作 con_ = con[con_mask[:, 0]] # print("con_",con_.shape) con_out_ = con_out[con_mask[:, 0]] con_loss = self.con_lossfun(con_out_, con_) #只有正负样本参与置信度损失计算 #计算坐标偏移损失 offset_mask = torch.gt(con, 0) #选出非负样本 offset_ = offset[offset_mask[:, 0]] offset_out_ = offset_out[offset_mask[:, 0]] offset_loss = self.offset_lossfun(offset_out_, offset_) # 只有非负样本参与坐标偏移损失计算 #计算关键点损失 landmarks_mask = torch.gt(con, 0) #正样本和部分样本训练关键点 landmarks_ = landmarks[landmarks_mask[:, 0]] landmarks_out_ = landmarks_out[landmarks_mask[:, 0]] # print(landmarks_mask) landmarks_loss = self.landmarks_lossfun( landmarks_out_, landmarks_) #总loss total_loss = con_loss + 0.5 * offset_loss + landmarks_loss self.optimer.zero_grad() total_loss.backward() self.optimer.step() # break if epoch % 100 == 0 and epoch > 0: print("epoch==>", epoch, "total_loss==>", total_loss) if epoch % 500 == 0 and epoch > 0: torch.save(self.net.state_dict(), self.parampath) #保存模型参数 print("===========save module===============") print("epoch==>", epoch, "total_loss==>", total_loss)
def create_mask(self, lengths, max_length): r = torch.unsqueeze(torch.arange(0, max_length), 0).long().to(DEVICE) # (1, 82) l = torch.unsqueeze(lengths, 1).expand(lengths.size(0), max_length) # (20, 82) mask = torch.lt(r.expand_as(l), l) return mask
def step_rescale(self, values): y = cudaify(torch.ones(values.size()).double() / (10 * self.n)) y[torch.lt(values.data, 5)] = 1.0 return Variable(y, requires_grad=False)
def forward(self, classifications, bbox_regressions, ldm_regressions, anchors, annotations): batch_size = classifications.shape[0] classification_losses = [] bbox_regression_losses = [] ldm_regression_losses = [] anchor = anchors[0, :, :] anchor_widths = anchor[:, 2] - anchor[:, 0] anchor_heights = anchor[:, 3] - anchor[:, 1] anchor_ctr_x = anchor[:, 0] + 0.5 * anchor_widths anchor_ctr_y = anchor[:, 1] + 0.5 * anchor_heights #temp positive_indices_list = [] for j in range(batch_size): classification = classifications[j, :, :] bbox_regression = bbox_regressions[j, :, :] ldm_regression = ldm_regressions[j, :, :] annotation = annotations[j, :, :] # annotation = annotation[annotation[:,0] != -1] annotation = annotation[annotation[:, 0] > 0] bbox_annotation = annotation[:, :4] ldm_annotation = annotation[:, 4:] if bbox_annotation.shape[0] == 0: bbox_regression_losses.append( torch.tensor(0., requires_grad=True).cuda()) classification_losses.append( torch.tensor(0., requires_grad=True).cuda()) ldm_regression_losses.append( torch.tensor(0., requires_grad=True).cuda()) # temp positive_indices_list.append([]) continue IoU = calc_iou(anchors[0, :, :], bbox_annotation[:, :4]) #IoU, filt_iou = filt_IoU(anchors[0, :, :], bbox_annotation, ldm_annotation) IoU_max, IoU_argmax = torch.max(IoU, dim=1) #取每个类型anchor匹配的最大值 targets = torch.ones(classification.shape) * -1 targets = targets.cuda() # those whose iou<0.3 have no object negative_indices = torch.lt(IoU_max, 0.3) #iou小于0.3的为负anchor targets[negative_indices, :] = 0 targets[negative_indices, 1] = 1 #负anchor对应targets位置存0, 1 # those whose iou>0.5 have object positive_indices = torch.ge(IoU_max, 0.5) #取iou大于0.5的为正样本位置 #temp positive_indices_list.append(positive_indices) num_positive_anchors = positive_indices.sum() #keep positive and negative ratios with 1:3 keep_negative_anchors = num_positive_anchors * 3 bbox_assigned_annotations = bbox_annotation[IoU_argmax, :] ldm_assigned_annotations = ldm_annotation[IoU_argmax, :] targets[positive_indices, :] = 0 targets[positive_indices, 0] = 1 #正anchor对应targets位置存1,0 # ignore targets with no landmarks # f_IoU_max ,f_IoU_argmax = torch.max(filt_iou, dim=1) # ldm_positive_indices = torch.ge(f_IoU_max, 0.5) ldm_sum = ldm_assigned_annotations.sum(dim=1) ge0_mask = ldm_sum > 0 ldm_positive_indices = ge0_mask & positive_indices # OHEM negative_losses = classification[ negative_indices, 1] * -1 #classification为LogSoftmax输出,需要x轴翻转,所以乘以-1 sorted_losses, _ = torch.sort(negative_losses, descending=True) if sorted_losses.numel() > keep_negative_anchors: sorted_losses = sorted_losses[:keep_negative_anchors] positive_losses = classification[ positive_indices, 0] * -1 #classification为LogSoftmax输出,需要x轴翻转,所以乘以-1 focal_loss = True #yzk use focal loss # focal loss if focal_loss: alpha = 0.25 gamma = 2.0 alpha_factor = torch.ones(targets.shape).cuda() * alpha focal_weight = targets * torch.where( torch.eq(targets, 1), 1. - torch.exp(classification), torch.exp(classification)) focal_weight = alpha_factor * torch.pow(focal_weight, gamma) bce = -( classification ) #+ (1.0 - targets) * torch.log(1.0 - classification)) #add by yzk cls_loss = focal_weight * bce cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss, torch.zeros(cls_loss.shape).cuda()) classification_losses.append( cls_loss.sum() / torch.clamp(num_positive_anchors.float(), min=1.0)) else: if positive_indices.sum() > 0: classification_losses.append(positive_losses.mean() + sorted_losses.mean()) else: classification_losses.append( torch.tensor(0., requires_grad=True).cuda()) # compute bboxes loss if positive_indices.sum() > 0: # bbox bbox_assigned_annotations = bbox_assigned_annotations[ positive_indices, :] #取正样本对应的anchor box anchor_widths_pi = anchor_widths[positive_indices] anchor_heights_pi = anchor_heights[positive_indices] anchor_ctr_x_pi = anchor_ctr_x[positive_indices] anchor_ctr_y_pi = anchor_ctr_y[positive_indices] gt_widths = bbox_assigned_annotations[:, 2] - bbox_assigned_annotations[:, 0] gt_heights = bbox_assigned_annotations[:, 3] - bbox_assigned_annotations[:, 1] gt_ctr_x = bbox_assigned_annotations[:, 0] + 0.5 * gt_widths gt_ctr_y = bbox_assigned_annotations[:, 1] + 0.5 * gt_heights targets_dx = (gt_ctr_x - anchor_ctr_x_pi) / (anchor_widths_pi + 1e-14) targets_dy = (gt_ctr_y - anchor_ctr_y_pi) / (anchor_heights_pi + 1e-14) targets_dw = torch.log(gt_widths / anchor_widths_pi) targets_dh = torch.log(gt_heights / anchor_heights_pi) bbox_targets = torch.stack( (targets_dx, targets_dy, targets_dw, targets_dh)) #根据annotation数据构造预测数据格式的数据 bbox_targets = bbox_targets.t() # Rescale bbox_targets = bbox_targets / torch.Tensor( [[0.1, 0.1, 0.2, 0.2]]).cuda() # smooth L1 # box losses bbox_regression_loss = self.smoothl1( bbox_targets, bbox_regression[positive_indices, :]) bbox_regression_losses.append(bbox_regression_loss) else: bbox_regression_losses.append( torch.tensor(0., requires_grad=True).cuda()) # compute landmarks loss #changed by yzktmp test # if ldm_positive_indices.sum() > 0 : # ldm_assigned_annotations = ldm_assigned_annotations[ldm_positive_indices, :] # # anchor_widths_l = anchor_widths[ldm_positive_indices] # anchor_heights_l = anchor_heights[ldm_positive_indices] # anchor_ctr_x_l = anchor_ctr_x[ldm_positive_indices] # anchor_ctr_y_l = anchor_ctr_y[ldm_positive_indices] # # l0_x = (ldm_assigned_annotations[:,0] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14) # l0_y = (ldm_assigned_annotations[:,1] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14) # l1_x = (ldm_assigned_annotations[:,2] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14) # l1_y = (ldm_assigned_annotations[:,3] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14) # l2_x = (ldm_assigned_annotations[:,4] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14) # l2_y = (ldm_assigned_annotations[:,5] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14) # l3_x = (ldm_assigned_annotations[:,6] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14) # l3_y = (ldm_assigned_annotations[:,7] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14) # l4_x = (ldm_assigned_annotations[:,8] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14) # l4_y = (ldm_assigned_annotations[:,9] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14) # # ldm_targets = torch.stack((l0_x,l0_y,l1_x,l1_y,l2_x,l2_y,l3_x,l3_y,l4_x,l4_y)) # ldm_targets = ldm_targets.t() # # # Rescale # scale = torch.ones(1,10)*0.1 # ldm_targets = ldm_targets/scale.cuda() # # ldm_regression_loss = self.smoothl1(ldm_targets, ldm_regression[ldm_positive_indices, :]) # ldm_regression_losses.append(ldm_regression_loss) # else: ldm_regression_losses.append( torch.tensor(0., requires_grad=True).cuda()) return torch.stack(classification_losses), torch.stack( bbox_regression_losses), torch.stack(ldm_regression_losses)
def __getitem__(self, idx): if self.seg_level: env_id = self.seg_list[idx][0] set_idx = self.seg_list[idx][1] seg_idx = self.seg_list[idx][2] else: env_id = self.env_list[idx] env_conf_json = load_env_config(env_id) landmark_names, landmark_indices, landmark_positions = get_landmark_locations_airsim( env_conf_json) top_down_image = load_env_img(env_id) path = load_path(env_id) img_x = top_down_image.shape[0] img_y = top_down_image.shape[1] path_in_img_coords = self.cf_to_img(img_x, path) landmark_pos_in_img = self.as_to_img( img_x, np.asarray(landmark_positions)[:, 0:2]) self.pos_rand_image = self.pos_rand_range * img_x #self.plot_path_on_img(top_down_image, path_in_img_coords) #self.plot_path_on_img(top_down_image, landmark_pos_in_img) #cv2.imshow("top_down", top_down_image) #cv2.waitKey() input_images = [] input_instructions = [] label_images = [] aux_labels = [] # Somehow load the instruction with the start and end indices for each of the N segments if self.seg_level: instruction_segments = [ self.all_instr[env_id][set_idx]["instructions"][seg_idx] ] else: instruction_segments = self.all_instr[env_id][0]["instructions"] for seg_idx, seg in enumerate(instruction_segments): start_idx = seg["start_idx"] end_idx = seg["end_idx"] instruction = seg["instruction"] # TODO: Check for overflowz seg_path = path_in_img_coords[start_idx:end_idx] seg_img = top_down_image.copy() #test_plot = self.plot_path_on_img(seg_img, seg_path) # TODO: Validate the 0.5 choice, should it be 2? affine, cropsize = self.get_affine_matrix( seg_path, 0, [int(img_x / 2), int(img_y / 2)], 0.5) if affine is None: continue seg_img_rot = self.apply_affine(seg_img, affine, cropsize) seg_labels = np.zeros_like(seg_img[:, :, 0:1]).astype(float) seg_labels = self.plot_path_on_img(seg_labels, seg_path) seg_labels = gaussian_filter(seg_labels, 4) seg_labels_rot = self.apply_affine(seg_labels, affine, cropsize) #seg_labels_rot = gaussian_filter(seg_labels_rot, 4) seg_labels_rot = self.normalize_0_1(seg_labels_rot) # Change to true to visualize the paths / labels if False: cv2.imshow("rot_img", seg_img_rot) cv2.imshow("seg_labels", seg_labels_rot) rot_viz = seg_img_rot.astype(np.float64) / 512 rot_viz[:, :, 0] += seg_labels_rot.squeeze() cv2.imshow("rot_viz", rot_viz) cv2.waitKey(0) tok_instruction = tokenize_instruction(instruction, self.word2token) instruction_t = torch.LongTensor(tok_instruction).unsqueeze(0) # Get landmark classification labels landmark_pos_in_seg_img = self.apply_affine_on_pts( landmark_pos_in_img, affine) # Down-size images and labels if requested by the model if self.img_scale != 1.0: seg_img_rot = transform.resize(seg_img_rot, [ seg_img_rot.shape[0] * self.img_scale, seg_img_rot.shape[1] * self.img_scale ], mode="constant") seg_labels_rot = transform.resize(seg_labels_rot, [ seg_labels_rot.shape[0] * self.img_scale, seg_labels_rot.shape[1] * self.img_scale ], mode="constant") landmark_pos_in_seg_img = landmark_pos_in_seg_img * self.img_scale seg_img_rot = standardize_image(seg_img_rot) seg_labels_rot = standardize_image(seg_labels_rot) seg_img_t = torch.from_numpy(seg_img_rot).unsqueeze(0).float() seg_labels_t = torch.from_numpy(seg_labels_rot).unsqueeze( 0).float() landmark_pos_t = torch.from_numpy( landmark_pos_in_seg_img).unsqueeze(0) landmark_indices_t = torch.LongTensor(landmark_indices).unsqueeze( 0) mask1 = torch.gt(landmark_pos_t, 0) mask2 = torch.lt(landmark_pos_t, seg_img_t.size(2)) mask = mask1 * mask2 mask = mask[:, :, 0] * mask[:, :, 1] mask = mask landmark_pos_t = torch.masked_select( landmark_pos_t, mask.unsqueeze(2).expand_as(landmark_pos_t)).view([-1, 2]) landmark_indices_t = torch.masked_select(landmark_indices_t, mask).view([-1]) mentioned_names, mentioned_indices = get_mentioned_landmarks( self.thesaurus, instruction) mentioned_labels_t = empty_float_tensor( list(landmark_indices_t.size())).long() for i, landmark_idx_present in enumerate(landmark_indices_t): if landmark_idx_present in mentioned_indices: mentioned_labels_t[i] = 1 aux_label = { "landmark_pos": landmark_pos_t, "landmark_indices": landmark_indices_t, "landmark_mentioned": mentioned_labels_t, "visible_mask": mask, } if self.include_instr_negatives: # If we are to be using similar instructions according to the json file, then # initialize choices with similar instructions. Otherwise let choices be empty, and they will # be filled in the following lines. if self.instr_negatives_similar_only: choices = self.similar_instruction_map[str(env_id)][str( seg_idx)] else: choices = [] # If there are no similar instructions to this instruction, pick a completely random instruction if len(choices) == 0: while len(choices) == 0: env_options = list(self.similar_instruction_map.keys()) random_env = random.choice(env_options) seg_options = list( self.similar_instruction_map[random_env].keys()) if len(seg_options) == 0: continue random_seg = random.choice(seg_options) choices = self.similar_instruction_map[random_env][ random_seg] pick = random.choice(choices) picked_env = pick["env_id"] picked_seg = pick["seg_idx"] picked_set = pick["set_idx"] picked_instruction = self.all_instr[picked_env][picked_set][ "instructions"][picked_seg]["instruction"] tok_fake_instruction = tokenize_instruction( picked_instruction, self.word2token) aux_label["negative_instruction"] = torch.LongTensor( tok_fake_instruction).unsqueeze(0) input_images.append(seg_img_t) input_instructions.append(instruction_t) label_images.append(seg_labels_t) aux_labels.append(aux_label) return [input_images, input_instructions, label_images, aux_labels]
def vtln_warp_freq(vtln_low_cutoff: float, vtln_high_cutoff: float, low_freq: float, high_freq: float, vtln_warp_factor: float, freq: Tensor) -> Tensor: r"""This computes a VTLN warping function that is not the same as HTK's one, but has similar inputs (this function has the advantage of never producing empty bins). This function computes a warp function F(freq), defined between low_freq and high_freq inclusive, with the following properties: F(low_freq) == low_freq F(high_freq) == high_freq The function is continuous and piecewise linear with two inflection points. The lower inflection point (measured in terms of the unwarped frequency) is at frequency l, determined as described below. The higher inflection point is at a frequency h, determined as described below. If l <= f <= h, then F(f) = f/vtln_warp_factor. If the higher inflection point (measured in terms of the unwarped frequency) is at h, then max(h, F(h)) == vtln_high_cutoff. Since (by the last point) F(h) == h/vtln_warp_factor, then max(h, h/vtln_warp_factor) == vtln_high_cutoff, so h = vtln_high_cutoff / max(1, 1/vtln_warp_factor). = vtln_high_cutoff * min(1, vtln_warp_factor). If the lower inflection point (measured in terms of the unwarped frequency) is at l, then min(l, F(l)) == vtln_low_cutoff This implies that l = vtln_low_cutoff / min(1, 1/vtln_warp_factor) = vtln_low_cutoff * max(1, vtln_warp_factor) Args: vtln_low_cutoff (float): Lower frequency cutoffs for VTLN vtln_high_cutoff (float): Upper frequency cutoffs for VTLN low_freq (float): Lower frequency cutoffs in mel computation high_freq (float): Upper frequency cutoffs in mel computation vtln_warp_factor (float): Vtln warp factor freq (Tensor): given frequency in Hz Returns: Tensor: Freq after vtln warp """ assert vtln_low_cutoff > low_freq, 'be sure to set the vtln_low option higher than low_freq' assert vtln_high_cutoff < high_freq, 'be sure to set the vtln_high option lower than high_freq [or negative]' l = vtln_low_cutoff * max(1.0, vtln_warp_factor) h = vtln_high_cutoff * min(1.0, vtln_warp_factor) scale = 1.0 / vtln_warp_factor Fl = scale * l # F(l) Fh = scale * h # F(h) assert l > low_freq and h < high_freq # slope of left part of the 3-piece linear function scale_left = (Fl - low_freq) / (l - low_freq) # [slope of center part is just "scale"] # slope of right part of the 3-piece linear function scale_right = (high_freq - Fh) / (high_freq - h) res = torch.empty_like(freq) outside_low_high_freq = torch.lt(freq, low_freq) | torch.gt( freq, high_freq) # freq < low_freq || freq > high_freq before_l = torch.lt(freq, l) # freq < l before_h = torch.lt(freq, h) # freq < h after_h = torch.ge(freq, h) # freq >= h # order of operations matter here (since there is overlapping frequency regions) res[after_h] = high_freq + scale_right * (freq[after_h] - high_freq) res[before_h] = scale * freq[before_h] res[before_l] = low_freq + scale_left * (freq[before_l] - low_freq) res[outside_low_high_freq] = freq[outside_low_high_freq] return res
def forward(self, classifications, regressions, anchors, annotations): alpha = 0.25 gamma = 2.0 batch_size = classifications.shape[0] classification_losses = [] regression_losses = [] anchor = anchors[0, :, :] anchor_widths = anchor[:, 2] - anchor[:, 0] anchor_heights = anchor[:, 3] - anchor[:, 1] anchor_ctr_x = anchor[:, 0] + 0.5 * anchor_widths anchor_ctr_y = anchor[:, 1] + 0.5 * anchor_heights for j in range(batch_size): classification = classifications[j, :, :] regression = regressions[j, :, :] bbox_annotation = annotations[j, :, :] bbox_annotation = bbox_annotation[bbox_annotation[:, 4] != -1] classification = torch.clamp(classification, 1e-4, 1.0 - 1e-4) if bbox_annotation.shape[0] == 0: if torch.cuda.is_available(): alpha_factor = torch.ones( classification.shape).cuda() * alpha alpha_factor = 1. - alpha_factor focal_weight = classification focal_weight = alpha_factor * torch.pow( focal_weight, gamma) bce = -(torch.log(1.0 - classification)) # cls_loss = focal_weight * torch.pow(bce, gamma) cls_loss = focal_weight * bce classification_losses.append(cls_loss.sum()) regression_losses.append(torch.tensor(0).float()) else: alpha_factor = torch.ones(classification.shape) * alpha alpha_factor = 1. - alpha_factor focal_weight = classification focal_weight = alpha_factor * torch.pow( focal_weight, gamma) bce = -(torch.log(1.0 - classification)) # cls_loss = focal_weight * torch.pow(bce, gamma) cls_loss = focal_weight * bce classification_losses.append(cls_loss.sum()) regression_losses.append(torch.tensor(0).float()) continue # Num_anchors x num_annotations IoU = calc_iou(anchors[0, :, :], bbox_annotation[:, :4]) # Num_anchors x 1 IoU_max, IoU_argmax = torch.max(IoU, dim=1) # compute the loss for classification targets = torch.ones(classification.shape) * -1 if torch.cuda.is_available(): targets = targets.cuda() targets[torch.lt(IoU_max, 0.4), :] = 0 positive_indices = torch.ge(IoU_max, 0.5) num_positive_anchors = positive_indices.sum() assigned_annotations = bbox_annotation[IoU_argmax, :] targets[positive_indices, :] = 0 targets[positive_indices, assigned_annotations[positive_indices, 4].long()] = 1 if torch.cuda.is_available(): alpha_factor = torch.ones(targets.shape).cuda() * alpha else: alpha_factor = torch.ones(targets.shape) * alpha alpha_factor = torch.where(torch.eq(targets, 1.), alpha_factor, 1. - alpha_factor) focal_weight = torch.where(torch.eq(targets, 1.), 1. - classification, classification) focal_weight = alpha_factor * torch.pow(focal_weight, gamma) bce = -(targets * torch.log(classification) + (1.0 - targets) * torch.log(1.0 - classification)) # cls_loss = focal_weight * torch.pow(bce, gamma) cls_loss = focal_weight * bce if torch.cuda.is_available(): cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss, torch.zeros(cls_loss.shape).cuda()) else: cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss, torch.zeros(cls_loss.shape)) classification_losses.append( cls_loss.sum() / torch.clamp(num_positive_anchors.float(), min=1.0)) # compute the loss for regression if positive_indices.sum() > 0: assigned_annotations = assigned_annotations[ positive_indices, :] anchor_widths_pi = anchor_widths[positive_indices] anchor_heights_pi = anchor_heights[positive_indices] anchor_ctr_x_pi = anchor_ctr_x[positive_indices] anchor_ctr_y_pi = anchor_ctr_y[positive_indices] gt_widths = (assigned_annotations[:, 2] - assigned_annotations[:, 0]) gt_heights = (assigned_annotations[:, 3] - assigned_annotations[:, 1]) gt_ctr_x = assigned_annotations[:, 0] + 0.5 * gt_widths gt_ctr_y = assigned_annotations[:, 1] + 0.5 * gt_heights # clip widths to 1 gt_widths = torch.clamp(gt_widths, min=1) gt_heights = torch.clamp(gt_heights, min=1) targets_dx = (gt_ctr_x - anchor_ctr_x_pi) / anchor_widths_pi targets_dy = (gt_ctr_y - anchor_ctr_y_pi) / anchor_heights_pi targets_dw = torch.log(gt_widths / anchor_widths_pi) targets_dh = torch.log(gt_heights / anchor_heights_pi) targets = torch.stack( (targets_dx, targets_dy, targets_dw, targets_dh)) targets = targets.t() if torch.cuda.is_available(): targets = targets / torch.Tensor([[0.1, 0.1, 0.2, 0.2] ]).cuda() else: targets = targets / torch.Tensor([[0.1, 0.1, 0.2, 0.2]]) negative_indices = 1 + (~positive_indices) regression_diff = torch.abs(targets - regression[positive_indices, :]) regression_loss = torch.where( torch.le(regression_diff, 1.0 / 9.0), 0.5 * 9.0 * torch.pow(regression_diff, 2), regression_diff - 0.5 / 9.0) regression_losses.append(regression_loss.mean()) else: if torch.cuda.is_available(): regression_losses.append(torch.tensor(0).float().cuda()) else: regression_losses.append(torch.tensor(0).float()) return (torch.stack(classification_losses).mean(dim=0, keepdim=True), torch.stack(regression_losses).mean(dim=0, keepdim=True))
def get_mel_banks(num_bins: int, window_length_padded: int, sample_freq: float, low_freq: float, high_freq: float, vtln_low: float, vtln_high: float, vtln_warp_factor: float) -> Tuple[Tensor, Tensor]: """ Returns: (Tensor, Tensor): The tuple consists of ``bins`` (which is melbank of size (``num_bins``, ``num_fft_bins``)) and ``center_freqs`` (which is center frequencies of bins of size (``num_bins``)). """ assert num_bins > 3, 'Must have at least 3 mel bins' assert window_length_padded % 2 == 0 num_fft_bins = window_length_padded / 2 nyquist = 0.5 * sample_freq if high_freq <= 0.0: high_freq += nyquist assert (0.0 <= low_freq < nyquist) and (0.0 < high_freq <= nyquist) and (low_freq < high_freq), \ ('Bad values in options: low-freq {} and high-freq {} vs. nyquist {}'.format(low_freq, high_freq, nyquist)) # fft-bin width [think of it as Nyquist-freq / half-window-length] fft_bin_width = sample_freq / window_length_padded mel_low_freq = mel_scale_scalar(low_freq) mel_high_freq = mel_scale_scalar(high_freq) # divide by num_bins+1 in next line because of end-effects where the bins # spread out to the sides. mel_freq_delta = (mel_high_freq - mel_low_freq) / (num_bins + 1) if vtln_high < 0.0: vtln_high += nyquist assert vtln_warp_factor == 1.0 or ((low_freq < vtln_low < high_freq) and (0.0 < vtln_high < high_freq) and (vtln_low < vtln_high)), \ ('Bad values in options: vtln-low {} and vtln-high {}, versus ' 'low-freq {} and high-freq {}'.format(vtln_low, vtln_high, low_freq, high_freq)) bin = torch.arange(num_bins).unsqueeze(1) left_mel = mel_low_freq + bin * mel_freq_delta # size(num_bins, 1) center_mel = mel_low_freq + (bin + 1.0) * mel_freq_delta # size(num_bins, 1) right_mel = mel_low_freq + (bin + 2.0) * mel_freq_delta # size(num_bins, 1) if vtln_warp_factor != 1.0: left_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, left_mel) center_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, center_mel) right_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, right_mel) center_freqs = inverse_mel_scale(center_mel) # size (num_bins) # size(1, num_fft_bins) mel = mel_scale(fft_bin_width * torch.arange(num_fft_bins)).unsqueeze(0) # size (num_bins, num_fft_bins) up_slope = (mel - left_mel) / (center_mel - left_mel) down_slope = (right_mel - mel) / (right_mel - center_mel) if vtln_warp_factor == 1.0: # left_mel < center_mel < right_mel so we can min the two slopes and clamp negative values bins = torch.max(torch.zeros(1), torch.min(up_slope, down_slope)) else: # warping can move the order of left_mel, center_mel, right_mel anywhere bins = torch.zeros_like(up_slope) up_idx = torch.gt(mel, left_mel) & torch.le( mel, center_mel) # left_mel < mel <= center_mel down_idx = torch.gt(mel, center_mel) & torch.lt( mel, right_mel) # center_mel < mel < right_mel bins[up_idx] = up_slope[up_idx] bins[down_idx] = down_slope[down_idx] return bins, center_freqs
def __call__(self, exper, epoch_obj, meta_optimizer, final_batch=False): self.step = 0 if self.is_train: do_continue = tensor_any( tensor_and(torch.le(self.compare_probs, self.one_minus_eps), torch.lt(self.counter_compare, self.max_T))).data.cpu().numpy()[0] else: do_continue = True if self.step < self.horizon - 1 else False while do_continue: # IMPORTANT! avg_loss_step is NOT MULTIPLIED BY THE qt-value!!!! avg_loss_step = self.act_step(exper, meta_optimizer) if self.is_train: do_continue = tensor_any( tensor_and( torch.le(self.compare_probs, self.one_minus_eps), torch.lt(self.counter_compare, self.max_T))).data.cpu().numpy()[0] else: do_continue = True if self.step < self.horizon - 1 else False if self.eval_last_step_taken == 0: num_next_iter = torch.sum( self.float_mask).data.cpu().squeeze().numpy()[0] if num_next_iter == 0. and self.eval_last_step_taken == 0.: self.eval_last_step_taken = self.step + 1 if self.is_train and exper.args.trunc_bptt and \ (self.forward_steps == exper.args.truncated_bptt_step or not do_continue): loss_sum, sum_grads = self.backward(epoch_obj, meta_optimizer, exper.optimizer, retain_graph=False, trunc_bptt=do_continue) self.num_of_backwards += 1 self.total_opt_loss += loss_sum self.total_kl_term += self.kl_term self.total_sum_grads += sum_grads self.last_backward_step = self.step + 1 # important increase step after "do_continue" stuff but before adding step losses self.step += 1 exper.add_step_loss(avg_loss_step, self.step, is_train=self.is_train) exper.add_opt_steps(self.step, is_train=self.is_train) epoch_obj.add_step_loss(avg_loss_step, last_time_step=not do_continue) # print("final qt-probs") # print(self.q_t[0:2, 0:self.step].data.cpu().numpy()) exper.add_step_qts(self.q_t[:, 0:self.step].data.cpu().numpy(), is_train=self.is_train) exper.add_halting_steps(self.halting_steps, is_train=self.is_train) # set the class variable if we reached a new maximum time steps if epoch_obj.get_max_time_steps_taken(self.is_train) < self.step: epoch_obj.set_max_time_steps_taken(self.step, self.is_train) if not self.is_train: if self.eval_last_step_taken == 0: self.eval_last_step_taken = self.step epoch_obj.set_max_time_steps_taken(self.eval_last_step_taken, self.is_train) if exper.args.problem == "mlp" and final_batch: # evaluate the last MLP that we optimized accuracy = self.functions.test_model(exper.dta_set, exper.args.cuda, quick_test=True) # exper.meta_logger.info("Epoch {}: last batch - accuracy of last MLP {:.4f}".format(exper.epoch, accuracy)) self.test_result_scores.append(accuracy)
dataframe subset = dataframe[[ 'Overall', 'Age', 'International Reputation', 'Weak Foot', 'Skill Moves' ]].dropna( axis=0, how='any' ) #esto significa que eliminará las filas osea los jugadores que tengan algún valor en NAN o no tengan un numero para este caso... #cogemos desde el 1 porque no queremos el overall columns = subset.columns[1:] players = torch.tensor(subset.values).float() playersOverall = players[:, 0] meanPlayersOverall = torch.mean(playersOverall) meanPlayersData = torch.mean(players, dim=0) stdPlayersData = torch.std(players, dim=0) norm = (players - meanPlayersData) / torch.sqrt(stdPlayersData) good = players[torch.ge(playersOverall, 85)] average = players[torch.gt(playersOverall, 70) & torch.lt(playersOverall, 85)] notSoGood = players[torch.le(playersOverall, 70)] goodMean = torch.mean(good, dim=0) averageMean = torch.mean(average, dim=0) notSoGoodMean = torch.mean(notSoGood, dim=0) print("media de cada aspecto de todos los jugadores:") print(meanPlayersData) print("---------------------------------------------------") print("promedio de cada jugador:") print(playersOverall) print("---------------------------------------------------") print("media del puntaje general del jugador:") print(meanPlayersOverall) print("---------------------------------------------------") print("Estructura de los datos de los jugadores:") print(players.shape)
def mask_from_lens(lens, max_len: Optional[int] = None): if max_len is None: max_len = int(lens.max().item()) ids = torch.arange(0, max_len, device=lens.device, dtype=lens.dtype) mask = torch.lt(ids, lens.unsqueeze(1)) return mask
def forward(self, ctgs, regs, ancs, anns): batchsz = ctgs.shape[0] ctglosses, reglosses = [], [] anc = ancs[0, :, :] ancw = anc[:, 2] - anc[:, 0] anch = anc[:, 3] - anc[:, 1] ancx = anc[:, 0] + 0.5 * ancw ancy = anc[:, 1] + 0.5 * anch for j in range(batchsz): ctg = ctgs[j, :, :] reg = regs[j, :, :] ann = anns[j, :, :] ann = ann[ann[:, 4] != -1] if ann.shape[0] == 0: reglosses.append(torch.tensor(0).float().cuda()) ctglosses.append(torch.tensor(0).float().cuda()) continue ctg = torch.clamp(ctg, 1e-4, 1.0 - 1e-4) ious = iou(ancs[0, :, :], ann[:, :4]) # ancnum x annnum maxiou, argmaxiou = torch.max(ious, dim=1) # ancnum x 1 # compute the loss for classification tgts = torch.ones(ctg.shape) * -1 tgts = tgts.cuda() tgts[torch.lt(maxiou, 0.4), :] = 0 posinds = torch.ge(maxiou, 0.5) posancnum = posinds.sum() asianns = ann[argmaxiou, :] tgts[posinds, :] = 0 tgts[posinds, asianns[posinds, 4].long()] = 1 alpha = torch.ones(tgts.shape).cuda() * self.alpha alpha = torch.where(torch.eq(tgts, 1.), alpha, 1. - alpha) focal = torch.where(torch.eq(tgts, 1.), 1. - ctg, ctg) focal = alpha * torch.pow(focal, self.gamma) bce = -(tgts * torch.log(ctg) + (1.0 - tgts) * torch.log(1.0 - ctg)) # ctgloss = focal_weight * torch.pow(bce, gamma) ctgloss = focal * bce ctgloss = torch.where(torch.ne(tgts, -1.0), ctgloss, torch.zeros(ctgloss.shape).cuda()) ctglosses.append(ctgloss.sum() / torch.clamp(posancnum.float(), min=1.0)) # compute the loss for regression if posinds.sum() > 0: asianns = asianns[posinds, :] posancw = ancw[posinds] posanch = anch[posinds] posancx = ancx[posinds] posancy = ancy[posinds] gtw = asianns[:, 2] - asianns[:, 0] gth = asianns[:, 3] - asianns[:, 1] gtx = asianns[:, 0] + 0.5 * gtw gty = asianns[:, 1] + 0.5 * gth # clip widths to 1 gtw = torch.clamp(gtw, min=1) gth = torch.clamp(gth, min=1) tgtdx = (gtx - posancx) / posancw tgtdy = (gty - posancy) / posanch tgtdw = torch.log(gtw / posancw) tgtdh = torch.log(gth / posanch) tgts = torch.stack((tgtdx, tgtdy, tgtdw, tgtdh)) tgts = tgts.t() tgts = tgts / torch.Tensor([[0.1, 0.1, 0.2, 0.2]]).cuda() neginds = 1 - posinds regdiff = torch.abs(tgts - reg[posinds, :]) regloss = torch.where(torch.le(regdiff, 1.0 / 9.0), 0.5 * 9.0 * torch.pow(regdiff, 2), regdiff - 0.5 / 9.0) reglosses.append(regloss.mean()) else: reglosses.append(torch.tensor(0).float().cuda()) return torch.stack(ctglosses).mean( dim=0, keepdim=True), torch.stack(reglosses).mean(dim=0, keepdim=True)
def test_lt(x, y): c = torch.lt(torch.add(x, y), y) return c
def apply_token_dropout(sentence_tensors, p): dropout = torch.rand_like(sentence_tensors, dtype=torch.float).fill_(p) rand = torch.rand_like(sentence_tensors, dtype=torch.float) mask = torch.lt(dropout, rand).type(torch.long) return torch.mul(sentence_tensors, mask)
def compute_projection(self, depth, camera_to_world, world_to_grid): # compute projection by voxels -> image #print 'camera_to_world', camera_to_world #print 'intrinsic', self.intrinsic #print(world_to_grid) world_to_camera = torch.inverse(camera_to_world) grid_to_world = torch.inverse(world_to_grid) voxel_bounds_min, voxel_bounds_max = self.compute_frustum_bounds(world_to_grid, camera_to_world) voxel_bounds_min = np.maximum(voxel_bounds_min, 0).cuda().float() if depth.is_cuda else np.maximum(voxel_bounds_min, 0).cpu().float() voxel_bounds_max = np.minimum(voxel_bounds_max, self.volume_dims).cuda().float() if depth.is_cuda else np.minimum(voxel_bounds_max, self.volume_dims).cpu().float() # coordinates within frustum bounds # TODO python opt for this part instead of lua/torch opt? lin_ind_volume = torch.arange(0, self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2], out=torch.LongTensor()) lin_ind_volume = lin_ind_volume.cuda() if depth.is_cuda else lin_ind_volume.cpu() coords = camera_to_world.new(4, lin_ind_volume.size(0)) coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1]) tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long() coords[1] = tmp / self.volume_dims[0] coords[0] = torch.remainder(tmp, self.volume_dims[0]) coords[3].fill_(1) mask_frustum_bounds = torch.ge(coords[0], voxel_bounds_min[0]) * torch.ge(coords[1], voxel_bounds_min[1]) * torch.ge(coords[2], voxel_bounds_min[2]) mask_frustum_bounds = mask_frustum_bounds * torch.lt(coords[0], voxel_bounds_max[0]) * torch.lt(coords[1], voxel_bounds_max[1]) * torch.lt(coords[2], voxel_bounds_max[2]) if not mask_frustum_bounds.any(): print('error: nothing in frustum bounds') return None lin_ind_volume = lin_ind_volume[mask_frustum_bounds] coords = coords.resize_(4, lin_ind_volume.size(0)) coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1]) tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long() coords[1] = tmp / self.volume_dims[0] coords[0] = torch.remainder(tmp, self.volume_dims[0]) coords[3].fill_(1) # transform to current frame p = torch.mm(world_to_camera, torch.mm(grid_to_world, coords)) # project into image p[0] = (p[0] * self.intrinsic[0][0]) / p[2] + self.intrinsic[0][2] p[1] = (p[1] * self.intrinsic[1][1]) / p[2] + self.intrinsic[1][2] pi = torch.round(p).long() valid_ind_mask = torch.ge(pi[0], 0) * torch.ge(pi[1], 0) * torch.lt(pi[0], self.image_dims[0]) * torch.lt(pi[1], self.image_dims[1]) if not valid_ind_mask.any(): print('error: no valid image indices') return None valid_image_ind_x = pi[0][valid_ind_mask] valid_image_ind_y = pi[1][valid_ind_mask] valid_image_ind_lin = valid_image_ind_y * self.image_dims[0] + valid_image_ind_x depth_vals = torch.index_select(depth.view(-1), 0, valid_image_ind_lin) depth_mask = depth_vals.ge(self.depth_min) * depth_vals.le(self.depth_max) * torch.abs(depth_vals - p[2][valid_ind_mask]).le(self.voxel_size) if not depth_mask.any(): print('error: no valid depths') return None lin_ind_update = lin_ind_volume[valid_ind_mask] lin_ind_update = lin_ind_update[depth_mask] lin_indices_3d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size) lin_indices_2d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size) lin_indices_3d[0] = lin_ind_update.shape[0] lin_indices_2d[0] = lin_ind_update.shape[0] lin_indices_3d[1:1+lin_indices_3d[0]] = lin_ind_update lin_indices_2d[1:1+lin_indices_2d[0]] = torch.index_select(valid_image_ind_lin, 0, torch.nonzero(depth_mask)[:,0]) num_ind = lin_indices_3d[0] #print '[proj] #ind = ', lin_indices_3d[0] #print '2d', torch.min(lin_indices_2d[1:1+num_ind]), torch.max(lin_indices_2d[1:1+num_ind]) #print '3d', torch.min(lin_indices_3d[1:1+num_ind]), torch.max(lin_indices_3d[1:1+num_ind]) return lin_indices_3d, lin_indices_2d
def equal(x, y, prec=1e-4): return torch.all(torch.lt(torch.abs(torch.add(x, -y)), prec))
def get_adj(batch): #Not very nice but we do not have access to value comming from opt.gpuid command line parameter here. use_cuda = batch.src[0].is_cuda node1_index = batch.node1.data.transpose( 0, 1) # [ seqLen x batch_size ] ==> [ batch_size x seqLen ] node2_index = batch.node2.data.transpose(0, 1) label_index = batch.label.data.transpose(0, 1) node1_voc = batch.dataset.fields['node1'].vocab.itos node2_voc = batch.dataset.fields['node2'].vocab.itos label_voc = batch.dataset.fields['label'].vocab.itos batch_size = batch.batch_size _MAX_BATCH_LEN = batch.src[0].data.size()[ 0] # data is [ seqLen x batch_size ] _MAX_DEGREE = 10 # If the average degree is much higher than this, it must be changed. sent_mask = torch.lt(torch.eq(batch.src[0].data, 1), 1) # if there is a sent, then mask=1 # each batch has _MAX_BATCH_LEN nodes # each node has _MAX_DEGREE lines as the adj list # the first row is the batch id, the second row is the neighbor id (or the label id) # mask indicates whether or not one slot in the adj_list is used, because each node has 10 slots adj_arc_in = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 2), dtype='int32') adj_lab_in = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 1), dtype='int32') adj_arc_out = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 2), dtype='int32') adj_lab_out = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 1), dtype='int32') mask_in = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE), dtype='float32') mask_out = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE), dtype='float32') mask_loop = np.ones((batch_size * _MAX_BATCH_LEN, 1), dtype='float32') tmp_in = {} tmp_out = {} for d, de in enumerate(node1_index): # iterates over the batch for a, arc in enumerate(de): # d: batch index, a: triple index triple: node1, node2, label arc_0 = label_voc[label_index[d, a]] if arc_0 == '<unk>' or arc_0 == '<pad>': pass else: # arc_1, arc2, arc_0: node1, node2, label arc_1 = int(node1_voc[arc]) # arc = node1_index[d, a] arc_2 = int(node2_voc[node2_index[d, a]]) if arc_1 in tmp_in: tmp_in[arc_1] += 1 else: tmp_in[arc_1] = 0 if arc_2 in tmp_out: tmp_out[arc_2] += 1 else: tmp_out[arc_2] = 0 # (d * _MAX_BATCH_LEN * _MAX_DEGREE): offset of current batch # arc_1 * _MAX_DEGREE: offset of current node # tmp_in[arc_1]: idx_in = (d * _MAX_BATCH_LEN * _MAX_DEGREE) + arc_1 * _MAX_DEGREE + tmp_in[arc_1] idx_out = (d * _MAX_BATCH_LEN * _MAX_DEGREE) + arc_2 * _MAX_DEGREE + tmp_out[arc_2] if tmp_in[arc_1] < _MAX_DEGREE: # store the batch_id, dest node, label adj_arc_in[idx_in] = np.array([d, arc_2]) # incoming arcs adj_lab_in[idx_in] = np.array([label_index[d, a] ]) # incoming arcs mask_in[idx_in] = 1. if tmp_out[arc_2] < _MAX_DEGREE: adj_arc_out[idx_out] = np.array([d, arc_1]) # outgoing arcs adj_lab_out[idx_out] = np.array([label_index[d, a] ]) # outgoing arcs mask_out[idx_out] = 1. tmp_in = {} tmp_out = {} adj_arc_in = autograd.Variable( torch.LongTensor(np.transpose(adj_arc_in).tolist())) adj_arc_out = autograd.Variable( torch.LongTensor(np.transpose(adj_arc_out).tolist())) adj_lab_in = autograd.Variable( torch.LongTensor(np.transpose(adj_lab_in).tolist())) adj_lab_out = autograd.Variable( torch.LongTensor(np.transpose(adj_lab_out).tolist())) mask_in = autograd.Variable(torch.FloatTensor( mask_in.reshape( (_MAX_BATCH_LEN * node1_index.size()[0], _MAX_DEGREE)).tolist()), requires_grad=False) mask_out = autograd.Variable(torch.FloatTensor( mask_out.reshape( (_MAX_BATCH_LEN * node1_index.size()[0], _MAX_DEGREE)).tolist()), requires_grad=False) mask_loop = autograd.Variable(torch.FloatTensor(mask_loop.tolist()), requires_grad=False) sent_mask = autograd.Variable(torch.FloatTensor(sent_mask.tolist()), requires_grad=False) if use_cuda: adj_arc_in = adj_arc_in.cuda() adj_arc_out = adj_arc_out.cuda() adj_lab_in = adj_lab_in.cuda() adj_lab_out = adj_lab_out.cuda() mask_in = mask_in.cuda() mask_out = mask_out.cuda() mask_loop = mask_loop.cuda() sent_mask = sent_mask.cuda() return adj_arc_in, adj_arc_out, adj_lab_in, adj_lab_out, mask_in, mask_out, mask_loop, sent_mask
self.stride, padding=self.padding) self.alpha_frequencies = np.zeros(2) self.alpha = random.uniform(0, 1) def forward(self, x): #######训练阶段,混合加权池化 if self.training: self.alpha_frequencies[0] += self.alpha self.alpha_frequencies[1] += 1 - self.alpha out = self.alpha * self.maxpool(x) + (1 - self.alpha) * self.avgpool(x) else: #######测试阶段,根据频次,选择池化方法 if (self.alpha_frequencies[0] < self.alpha_frequencies[1]): out = self.avgpool(x) else: out = self.maxpool(x) return out if __name__ == "__main__": print("=" * 10 + "MixPool2D" + "=" * 10) avgpool = "avgpool" maxpool = "maxpool" a = torch.Tensor([[1], [2]], [[3], [4]]) b = torch.Tensor([[5], [6]], [[1], [3]]) mixpool = MixPooling2D() print(mixpool.training) print(torch.lt(a, b))
def rprop(opfunc, x, config, state=None): """ A plain implementation of RPROP ARGS: - `opfunc` : a function that takes a single input (X), the point of evaluation, and returns f(X) and df/dX - `x` : the initial point - `state` : a table describing the state of the optimizer; after each call the state is modified - `state['stepsize']` : initial step size, common to all components - `state['etaplus']` : multiplicative increase factor, > 1 (default 1.2) - `state['etaminus']` : multiplicative decrease factor, < 1 (default 0.5) - `state['stepsizemax']` : maximum stepsize allowed (default 50) - `state['stepsizemin']` : minimum stepsize allowed (default 1e-6) - `state['niter']` : number of iterations (default 1) RETURN: - `x` : the new x vector - `f(x)` : the function, evaluated before the update (Martin Riedmiller, Koray Kavukcuoglu 2013) """ if config is None and state is None: raise ValueError("rprop requires a dictionary to retain state between iterations") # (0) get/update state state = state if state is not None else config stepsize = config.get('stepsize', 0.1) etaplus = config.get('etaplus', 1.2) etaminus = config.get('etaminus', 0.5) stepsizemax = config.get('stepsizemax', 50.0) stepsizemin = config.get('stepsizemin', 1e-06) niter = config.get('niter', 1) hfx = [] for i in range(niter): # (1) evaluate f(x) and df/dx fx, dfdx = opfunc(x) # init temp storage if 'delta' not in state: state['delta'] = dfdx.new(dfdx.size()).zero_() state['stepsize'] = dfdx.new(dfdx.size()).fill_(stepsize) state['sign'] = dfdx.new(dfdx.size()) state['bytesign'] = torch.ByteTensor(dfdx.size()) state['psign'] = torch.ByteTensor(dfdx.size()) state['nsign'] = torch.ByteTensor(dfdx.size()) state['zsign'] = torch.ByteTensor(dfdx.size()) state['dminmax'] = torch.ByteTensor(dfdx.size()) if str(type(x)).find('Cuda') > -1: # Push to GPU state['psign'] = state['psign'].cuda() state['nsign'] = state['nsign'].cuda() state['zsign'] = state['zsign'].cuda() state['dminmax'] = state['dminmax'].cuda() # sign of derivative from last step to this one torch.mul(dfdx, state['delta'], out=state['sign']).sign_() # get indices of >0, <0 and ==0 entries torch.gt(state['sign'], 0, out=state['psign']) torch.lt(state['sign'], 0, out=state['nsign']) torch.eq(state['sign'], 0, out=state['zsign']) # get step size updates state['sign'][state['psign']] = etaplus state['sign'][state['nsign']] = etaminus state['sign'][state['zsign']] = 1 # update stepsizes with step size updates state['stepsize'].mul_(state['sign']) # threshold step sizes # >50 => 50 torch.gt(state['stepsize'], stepsizemax, out=state['dminmax']) state['stepsize'][state['dminmax']] = stepsizemax # <1e-6 ==> 1e-6 torch.lt(state['stepsize'], stepsizemin, out=state['dminmax']) state['stepsize'][state['dminmax']] = stepsizemin # for dir<0, dfdx=0 # for dir>=0 dfdx=dfdx dfdx[state['nsign']] = 0 torch.sign(dfdx, out=state['sign']) # update weights x.addcmul_(-1, state['sign'], state['stepsize']) # update state['dfdx'] with current dfdx state['delta'].copy_(dfdx) hfx.append(fx) # return x*, table of f(x) values from each step return x, hfx
def get_dgl_batched_graph(batch): edge_to_id = { e: i for i, e in enumerate(['sp', 'po', 'op', 'ps', 'll', 'ne']) } #Not very nice but we do not have access to value comming from opt.gpuid command line parameter here. use_cuda = batch.src[0].is_cuda nodes_id = batch.src[0].transpose(0, 1) nodes_size = batch.src[1].cpu().numpy() node1_index = batch.node1.data.transpose( 0, 1) # [ seqLen x batch_size ] ==> [ batch_size x seqLen ] node2_index = batch.node2.data.transpose(0, 1) label_index = batch.label.data.transpose(0, 1) node1_voc = batch.dataset.fields['node1'].vocab.itos node2_voc = batch.dataset.fields['node2'].vocab.itos label_voc = batch.dataset.fields['label'].vocab.itos batch_size = batch.batch_size _MAX_BATCH_LEN = batch.src[0].data.size()[ 0] # data is [ seqLen x batch_size ] _MAX_DEGREE = 10 # If the average degree is much higher than this, it must be changed. sent_mask = torch.lt(torch.eq(batch.src[0].data, 1), 1) # if there is a sent, then mask=1 g_list = [] for d, de in enumerate(node1_index): # iterates over the batch g = dgl.DGLGraph() node_size = nodes_size[d] g.add_nodes(int(node_size)) g.ndata['global_id'] = nodes_id[d][:node_size] g.ndata['spo'] = torch.zeros(node_size, 3).float() edge_list = [] edge_type_list = [] # we guess the max_node_size is no longer than 20 ... node_spo_feature = torch.zeros(node_size, 3).float() node_set = set() for a, arc in enumerate(de): # d: batch index, a: triple index triple: node1, node2, label arc_0 = label_voc[label_index[d, a]] if arc_0 == '<unk>' or arc_0 == '<pad>': pass else: # arc_1, arc2, arc_0: node1, node2, label # by indexing we get the local id of each node/edge arc_1 = int(node1_voc[arc]) # arc = node1_index[d, a] arc_2 = int(node2_voc[node2_index[d, a]]) if arc_0 == "A0": g.nodes[arc_1].data['spo'] += torch.tensor([1.0, 0, 0]) g.nodes[arc_2].data['spo'] += torch.tensor([0, 0.5, 0]) edge_list.append((arc_1, arc_2)) edge_list.append((arc_2, arc_1)) edge_list.append((arc_1, arc_1)) edge_list.append((arc_2, arc_2)) edge_type_list += [ edge_to_id[e] for e in ['sp', 'ps', 'll', 'll'] ] elif arc_0 == "A1": g.nodes[arc_1].data['spo'] += torch.tensor([0, 0, 1.0]) g.nodes[arc_2].data['spo'] += torch.tensor([0, 0.5, 0]) edge_list.append((arc_1, arc_2)) edge_list.append((arc_2, arc_1)) edge_list.append((arc_1, arc_1)) edge_list.append((arc_2, arc_2)) edge_type_list += [ edge_to_id[e] for e in ['op', 'po', 'll', 'll'] ] elif arc_0 == 'NE': edge_list.append((arc_1, arc_2)) edge_type_list += [edge_to_id["ne"]] else: raise ValueError( "Do not support the edge type {}".format(arc_0)) new_edge_list, new_edge_type_list = [], [] added_edge = set() for edge, edge_type in zip(edge_list, edge_type_list): # if edge_type == edge_to_id["ll"]: # continue edge_id = "{}-{}-{}".format(edge[0], edge[1], edge_type) if edge_id not in added_edge: added_edge.add(edge_id) new_edge_list.append(edge) new_edge_type_list.append(edge_type) src, dst = tuple(zip(*new_edge_list)) g.add_edges(src, dst) g.ndata['spo'] = torch.gt(g.ndata['spo'], 0) g.edata['type'] = torch.tensor(new_edge_type_list).reshape(-1, 1) dst_in_deg = {} for dst_node in set(dst): if dst_node not in dst_in_deg: dst_in_deg[dst_node] = {} for edge_type in range(len(edge_to_id)): if edge_type not in dst_in_deg[dst_node]: dst_in_deg[dst_node][edge_type] = 0 for dst_node, edge_type in zip(dst, new_edge_type_list): dst_in_deg[dst_node][edge_type] += 1 e_norm = [ 1.0 / dst_in_deg[dst_node][e_type] for dst_node, e_type in zip(dst, new_edge_type_list) ] g.edata['norm'] = torch.tensor(e_norm).reshape(-1, 1) g_list.append(g) return dgl.batch(g_list)