def mrcnn_bbox_loss(target_bbox, target_class_ids, pred_bbox): """Loss for Mask R-CNN bounding box refinement. target_bbox: [batch, num_rois, (dy, dx, log(dh), log(dw))] target_class_ids: [batch, num_rois]. Integer class IDs. pred_bbox: [batch, num_rois, num_classes, (dy, dx, log(dh), log(dw))] """ # Reshape to merge batch and roi dimensions for simplicity. target_class_ids = target_class_ids.contiguous().view(-1) target_bbox = target_bbox.contiguous().view(-1, 4) pred_bbox = pred_bbox.contiguous().view(-1, pred_bbox.size()[2], 4) # print(target_class_ids) # Only positive ROIs contribute to the loss. And only # the right class_id of each ROI. Get their indicies. positive_roi_ix = torch.gt(target_class_ids , 0) # print(positive_roi_ix) positive_roi_class_ids = torch.masked_select(target_class_ids, positive_roi_ix) indices = target_class_ids # indices = torch.stack([positive_roi_ix, positive_roi_class_ids], dim=1) # print(indices) # Gather the deltas (predicted and true) that contribute to loss # target_bbox = torch.gather(target_bbox, positive_roi_ix) # pred_bbox = torch.gather(pred_bbox, indices) loss = F.smooth_l1_loss(pred_bbox, target_bbox, size_average=True) return loss
def forward(self, tgt_seq, tgt_pos, src_seq, enc_output, return_attns=False): # Word embedding look up dec_input = self.tgt_word_emb(tgt_seq) # Position Encoding addition dec_input += self.position_enc(tgt_pos) # Decode dec_slf_attn_pad_mask = get_attn_padding_mask(tgt_seq, tgt_seq) dec_slf_attn_sub_mask = get_attn_subsequent_mask(tgt_seq) dec_slf_attn_mask = torch.gt(dec_slf_attn_pad_mask + dec_slf_attn_sub_mask, 0) dec_enc_attn_pad_mask = get_attn_padding_mask(tgt_seq, src_seq) if return_attns: dec_slf_attns, dec_enc_attns = [], [] dec_output = dec_input for dec_layer in self.layer_stack: dec_output, dec_slf_attn, dec_enc_attn = dec_layer( dec_output, enc_output, slf_attn_mask=dec_slf_attn_mask, dec_enc_attn_mask=dec_enc_attn_pad_mask) if return_attns: dec_slf_attns += [dec_slf_attn] dec_enc_attns += [dec_enc_attn] if return_attns: return dec_output, dec_slf_attns, dec_enc_attns else: return dec_output,
def sparsify(model, sparsity_level=50.): for name, param in model.named_parameters(): if 'weight' in name: threshold = calculate_threshold(param.data, sparsity_level) mask = torch.gt(torch.abs(param.data), threshold).float() param.data = param.data * mask return model
def updateGradInput(self, input, y): self.gradInput.resize_as_(input).copy_(y) self.gradInput[torch.mul(torch.eq(y, -1), torch.gt(input, self.margin))] = 0 if self.sizeAverage: self.gradInput.mul_(1. / input.nelement()) return self.gradInput
def backward(self, grad_output): input, target = self.saved_tensors grad_input = input.new().resize_as_(input).copy_(target) grad_input[torch.mul(torch.eq(target, -1), torch.gt(input, self.margin))] = 0 if self.size_average: grad_input.mul_(1. / input.nelement()) if grad_output[0] != 1: grad_input.mul_(grad_output[0]) return grad_input, None
def forward(ctx, input, target, grad_output, margin, size_average): ctx.margin = margin ctx.size_average = size_average ctx.save_for_backward(input, target, grad_output) grad_input = input.new().resize_as_(input).copy_(target) grad_input[torch.mul(torch.eq(target, -1), torch.gt(input, ctx.margin))] = 0 if ctx.size_average: grad_input.mul_(1. / input.nelement()) if grad_output[0] != 1: grad_input.mul_(grad_output[0]) return grad_input
def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, layer_cache=None, step=None): """ Args: inputs (FloatTensor): ``(batch_size, 1, model_dim)`` memory_bank (FloatTensor): ``(batch_size, src_len, model_dim)`` src_pad_mask (LongTensor): ``(batch_size, 1, src_len)`` tgt_pad_mask (LongTensor): ``(batch_size, 1, 1)`` Returns: (FloatTensor, FloatTensor): * output ``(batch_size, 1, model_dim)`` * attn ``(batch_size, 1, src_len)`` """ dec_mask = None if step is None: tgt_len = tgt_pad_mask.size(-1) future_mask = torch.ones( [tgt_len, tgt_len], device=tgt_pad_mask.device, dtype=torch.uint8) future_mask = future_mask.triu_(1).view(1, tgt_len, tgt_len) dec_mask = torch.gt(tgt_pad_mask + future_mask, 0) input_norm = self.layer_norm_1(inputs) if isinstance(self.self_attn, MultiHeadedAttention): query, attn = self.self_attn(input_norm, input_norm, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self") elif isinstance(self.self_attn, AverageAttention): query, attn = self.self_attn(input_norm, mask=dec_mask, layer_cache=layer_cache, step=step) query = self.drop(query) + inputs query_norm = self.layer_norm_2(query) mid, attn = self.context_attn(memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context") output = self.feed_forward(self.drop(mid) + query) return output, attn
def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None): # Args Checks input_batch, input_len, _ = inputs.size() if previous_input is not None: pi_batch, _, _ = previous_input.size() aeq(pi_batch, input_batch) contxt_batch, contxt_len, _ = memory_bank.size() aeq(input_batch, contxt_batch) src_batch, t_len, s_len = src_pad_mask.size() tgt_batch, t_len_, t_len__ = tgt_pad_mask.size() aeq(input_batch, contxt_batch, src_batch, tgt_batch) # aeq(t_len, t_len_, t_len__, input_len) aeq(s_len, contxt_len) # END Args Checks dec_mask = torch.gt(tgt_pad_mask + self.mask[:, :tgt_pad_mask.size(1), :tgt_pad_mask.size(1)], 0) input_norm = self.layer_norm_1(inputs) all_input = input_norm if previous_input is not None: all_input = torch.cat((previous_input, input_norm), dim=1) dec_mask = None query, attn = self.self_attn(all_input, all_input, input_norm, mask=dec_mask) query = self.drop(query) + inputs query_norm = self.layer_norm_2(query) mid, attn = self.context_attn(memory_bank, memory_bank, query_norm, mask=src_pad_mask) output = self.feed_forward(self.drop(mid) + query) # CHECKS output_batch, output_len, _ = output.size() aeq(input_len, output_len) aeq(contxt_batch, output_batch) n_batch_, t_len_, s_len_ = attn.size() aeq(input_batch, n_batch_) aeq(contxt_len, s_len_) aeq(input_len, t_len_) # END CHECKS return output, attn, all_input
def forward(ctx, gboxes, qboxes, aligned=False): assert gboxes.shape[0] == qboxes.shape[0] indicator = torch.gt(gboxes[:, 3], 0) & torch.gt(gboxes[:, 4], 0) & torch.gt(gboxes[:, 5], 0) \ & torch.gt(qboxes[:, 3], 0) & torch.gt(qboxes[:, 4], 0) & torch.gt(qboxes[:, 5], 0) index_loc = torch.nonzero(indicator) ## if we want to compute the gious of two aligned rectangles gious = torch.zeros([gboxes.shape[0], ], device=gboxes.device, dtype=torch.float32) if (aligned): align_inter_aligned_object = align_inter_aligned() inter_area_xoz, mbr_area_xoz, inter_area_xoy, mbr_area_xoy, inter_area_yoz, mbr_area_yoz = align_inter_aligned_object( gboxes, qboxes) volume_gboxes = gboxes[:, 3].mul(gboxes[:, 4]).mul(gboxes[:, 5]) volume_qboxes = qboxes[:, 3].mul(qboxes[:, 4]).mul(qboxes[:, 5]) ## for three different views xoz plane # inter_h = (torch.min(gboxes[:, 1], qboxes[:, 1]) - torch.max(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4])) # oniou_h = (torch.max(gboxes[:, 1], qboxes[:, 1]) - torch.min(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4])) # inter_h[inter_h < 0] = 0 # oniou_h[oniou_h < 0] = 0 # inter_area_xoz_cuda = inter_area_xoz.to(torch.device(gboxes.device)) # mbr_area_xoz_cuda = mbr_area_xoz.to(torch.device(gboxes.device)) # volume_inc = inter_h.mul(inter_area_xoz_cuda) # volume_con = oniou_h.mul(mbr_area_xoz_cuda) # volume_union = (volume_gboxes + volume_qboxes - volume_inc) # volume_ca = volume_con - volume_union # ious = torch.div(volume_inc, volume_union) union_xoz = gboxes[:, 3].mul(gboxes[:, 5]) + qboxes[:, 3].mul(qboxes[:, 5]) - inter_area_xoz iou_xoz = torch.div(inter_area_xoz, union_xoz) iou_bis_xoz = torch.div(mbr_area_xoz - union_xoz, mbr_area_xoz) gious_xoz = iou_xoz - iou_bis_xoz ## for xoy plane union_xoy = gboxes[:, 3].mul(gboxes[:, 4]) + qboxes[:, 3].mul(qboxes[:, 4]) - inter_area_xoy iou_xoy = torch.div(inter_area_xoy, union_xoy) iou_bis_xoy = torch.div(mbr_area_xoy - union_xoy, mbr_area_xoy) gious_xoy = iou_xoy - iou_bis_xoy ## for yoz plane union_yoz = gboxes[:, 4].mul(gboxes[:, 5]) + qboxes[:, 4].mul(qboxes[:, 5]) - inter_area_xoy iou_yoz = torch.div(inter_area_yoz, union_yoz) iou_bis_yoz = torch.div(mbr_area_yoz - union_yoz, mbr_area_yoz) gious_xoy = iou_yoz - iou_bis_yoz gious[index_loc[:, 0]] = (gious_xoz[index_loc[:, 0]] + gious_xoy[index_loc[:, 0]] + gious_xoy[ index_loc[:, 0]]) / 3.0 # for i in range(inter_area_xoz.shape[0]): # if (gious[i] > 1): # print("infor: (%.4f %.4f %.4f %.4f %.4f %.4f %.4f,%.4f %.4f %.4f %.4f)" # % (i, inter_h[i], oniou_h[i], inter_area_xoz[i], mbr_area_xoz[i], ious[i], gious[i], volume_inc[i], # volume_con[i], volume_union[i], volume_ca[i])) # elif (gious[i] < -1): # print("infor: (%.4f %.4f %.4f %.4f %.4f %.4f %.4f,%.4f %.4f %.4f %.4f)" # % (i, inter_h[i], oniou_h[i], inter_area_xoz[i], mbr_area_xoz[i], ious[i], gious[i], volume_inc[i], # volume_con[i], volume_union[i], volume_ca[i])) else: rbbox_to_corners_object = rbbox_to_corners() corners_gboxes = rbbox_to_corners_object(gboxes[:, [0, 2, 3, 5, 6]]) corners_qboxes = rbbox_to_corners_object(qboxes[:, [0, 2, 3, 5, 6]]) # compute the inter area rinter_area_compute_object = rinter_area_compute() inter_area = rinter_area_compute_object(corners_gboxes, corners_qboxes) corners_gboxes_1 = torch.stack((corners_gboxes[:, [0, 2, 4, 6]], corners_gboxes[:, [1, 3, 5, 7]]), 2) corners_qboxes_1 = torch.stack((corners_qboxes[:, [0, 2, 4, 6]], corners_qboxes[:, [1, 3, 5, 7]]), 2) corners_pts = torch.cat((corners_gboxes_1, corners_qboxes_1), 1) # compute the mbr area mbr_area_compute_object = mbr_area_compute() mbr_area = mbr_area_compute_object(corners_pts) ## Compute the gious for 3D inter_h = (torch.min(gboxes[:, 1], qboxes[:, 1]) - torch.max(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4])) oniou_h = (torch.max(gboxes[:, 1], qboxes[:, 1]) - torch.min(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4])) inter_h[inter_h < 0] = 0 volume_gboxes = gboxes[:, 3].mul(gboxes[:, 4]).mul(gboxes[:, 5]) volume_qboxes = qboxes[:, 3].mul(qboxes[:, 4]).mul(qboxes[:, 5]) inter_area_cuda = inter_area.to(torch.device(gboxes.device)) mbr_area_cuda = mbr_area.to(torch.device(gboxes.device)) volume_inc = inter_h.mul(inter_area_cuda) volume_con = oniou_h.mul(mbr_area_cuda) volume_union = (volume_gboxes + volume_qboxes - volume_inc) volume_ca = volume_con - volume_union ious = torch.div(volume_inc, volume_union) gious = torch.zeros([gboxes.shape[0],], device=gboxes.device, dtype=torch.float32) gious[index_loc[:, 0]] = ious[index_loc[:, 0]] - torch.div(volume_ca[index_loc[:, 0]], volume_con[index_loc[:, 0]]) # for i in range(inter_area.shape[0]): # if(gious[i] < -1): # print("infor: (%.4f %.4f %.4f %.4f %.4f %.4f %.4f,%.4f %.4f %.4f %.4f)" # %(i,inter_h[i], oniou_h[i], inter_area[i], mbr_area[i], ious[i], gious[i],volume_inc[i],volume_con[i],volume_union[i],volume_ca[i])) return torch.unsqueeze(gious, 1)
def _forward( self, inputs, tgt_pad_mask, layer_cache=None, step=None, future=False ): """A naive forward pass for transformer decoder. # T: could be 1 in the case of stepwise decoding or tgt_len Args: inputs (FloatTensor): ``(batch_size, T, model_dim)`` tgt_pad_mask (bool): ``(batch_size, 1, T)`` layer_cache (dict or None): cached layer info when stepwise decode step (int or None): stepwise decoding counter future (bool): If set True, do not apply future_mask. Returns: (FloatTensor, FloatTensor): * output ``(batch_size, T, model_dim)`` * attns ``(batch_size, head, T, T)`` """ dec_mask = None if step is None: tgt_len = tgt_pad_mask.size(-1) if not future: # apply future_mask, result mask in (B, T, T) future_mask = torch.ones( [tgt_len, tgt_len], device=tgt_pad_mask.device, dtype=torch.uint8, ) future_mask = future_mask.triu_(1).view(1, tgt_len, tgt_len) # BoolTensor was introduced in pytorch 1.2 try: future_mask = future_mask.bool() except AttributeError: pass dec_mask = torch.gt(tgt_pad_mask + future_mask, 0) else: # only mask padding, result mask in (B, 1, T) dec_mask = tgt_pad_mask inputs_norm = self.layer_norm_1(inputs) if isinstance(self.self_attn, MultiHeadedAttention): query, attns = self.self_attn( inputs_norm, inputs_norm, inputs_norm, mask=dec_mask, layer_cache=layer_cache, attn_type="self", ) elif isinstance(self.self_attn, AverageAttention): query, attns = self.self_attn( inputs_norm, mask=dec_mask, layer_cache=layer_cache, step=step ) output = self.drop(query) + inputs output_feedforward = self.feed_forward(self.layer_norm_2(output)) output_norm = self.drop(output_feedforward) + output return output_norm, attns
def gt(*args, **kwargs): return torch.gt(*args, **kwargs)
def loss(self, H1, H2): """ It is the loss function of CCA as introduced in the original paper. There can be other formulations. """ r1 = 1e-3 r2 = 1e-3 eps = 1e-9 H1, H2 = H1.t(), H2.t() # assert torch.isnan(H1).sum().item() == 0 # assert torch.isnan(H2).sum().item() == 0 o1 = o2 = H1.size(0) m = H1.size(1) # print(H1.size()) H1bar = H1 - H1.mean(dim=1).unsqueeze(dim=1) H2bar = H2 - H2.mean(dim=1).unsqueeze(dim=1) # assert torch.isnan(H1bar).sum().item() == 0 # assert torch.isnan(H2bar).sum().item() == 0 SigmaHat12 = (1.0 / (m - 1)) * torch.matmul(H1bar, H2bar.t()) SigmaHat11 = (1.0 / (m - 1)) * torch.matmul( H1bar, H1bar.t()) + r1 * torch.eye(o1, device=self.device) SigmaHat22 = (1.0 / (m - 1)) * torch.matmul( H2bar, H2bar.t()) + r2 * torch.eye(o2, device=self.device) # assert torch.isnan(SigmaHat11).sum().item() == 0 # assert torch.isnan(SigmaHat12).sum().item() == 0 # assert torch.isnan(SigmaHat22).sum().item() == 0 # Calculating the root inverse of covariance matrices by using eigen decomposition [D1, V1] = torch.symeig(SigmaHat11, eigenvectors=True) [D2, V2] = torch.symeig(SigmaHat22, eigenvectors=True) # assert torch.isnan(D1).sum().item() == 0 # assert torch.isnan(D2).sum().item() == 0 # assert torch.isnan(V1).sum().item() == 0 # assert torch.isnan(V2).sum().item() == 0 # Added to increase stability posInd1 = torch.gt(D1, eps).nonzero()[:, 0] D1 = D1[posInd1] V1 = V1[:, posInd1] posInd2 = torch.gt(D2, eps).nonzero()[:, 0] D2 = D2[posInd2] V2 = V2[:, posInd2] # print(posInd1.size()) # print(posInd2.size()) SigmaHat11RootInv = torch.matmul( torch.matmul(V1, torch.diag(D1**-0.5)), V1.t()) SigmaHat22RootInv = torch.matmul( torch.matmul(V2, torch.diag(D2**-0.5)), V2.t()) Tval = torch.matmul(torch.matmul(SigmaHat11RootInv, SigmaHat12), SigmaHat22RootInv) # print(Tval.size()) if self.use_all_singular_values: # all singular values are used to calculate the correlation tmp = torch.trace(torch.matmul(Tval.t(), Tval)) # print(tmp) corr = torch.sqrt(tmp) # assert torch.isnan(corr).item() == 0 else: # just the top self.outdim_size singular values are used U, V = torch.symeig(torch.matmul(Tval.t(), Tval), eigenvectors=True) # U = U[torch.gt(U, eps).nonzero()[:, 0]] U = U.topk(self.outdim_size)[0] corr = torch.sum(torch.sqrt(U)) return -corr
def cosine_similarity_selector_IQP_Exact(x1, solver, nb_selected, eps=1e-3, slack=0.01): """ Integer programming """ # x1=gradient memories x2 = None w1 = x1.norm(p=2, dim=1, keepdim=True) inds = torch.nonzero(torch.gt(w1, slack))[:, 0] print("removed due to gradients", w1.size(0) - inds.size(0)) if inds.size(0) < nb_selected: print("WARNING GRADIENTS ARE TOO SMALL!!!!!!!!") inds = torch.arange(0, x1.size(0)) w1 = w1[inds] x1 = x1[inds] x2 = x1 if x2 is None else x2 w2 = w1 if x2 is x1 else x2.norm(p=2, dim=1, keepdim=True) G = torch.mm(x1, x2.t()) / (w1 * w2.t()) # .clamp(min=eps) t = G.size(0) G = G.double().numpy() a = np.zeros(t) # a=np.ones(t)*-1 # a=((w1-torch.min(w1))/(torch.max(w1)-torch.min(w1))).squeeze().double().numpy()*-0.01 C = np.ones((t, 1)) h = np.zeros(1) + nb_selected C2 = np.eye(t) hlower = np.zeros(t) hupper = np.ones(t) idx = np.arange(t) ################# C = np.concatenate((C2, C), axis=1) C = np.transpose(C) h_final_lower = np.concatenate((hlower, h), axis=0) h_final_upper = np.concatenate((hupper, h), axis=0) ################# G = spa.csc_matrix(G) C = spa.csc_matrix(C) solver.setup(G, a, C, h_final_lower, h_final_upper, idx, hlower, hupper, miosqp_settings, osqp_settings) results = solver.solve() print("STATUS", results.status) coeffiecents_np = results.x coeffiecents = torch.nonzero(torch.Tensor(coeffiecents_np)) print("number of selected items is", sum(coeffiecents_np)) if "Infeasible" in results.status: return inds return inds[coeffiecents.squeeze()]
def forward(self, tgt_seq, enc_output, enc_mask, enc_attn_caches=None, self_attn_caches=None, sample_K=0, seed=0): batch_size, tgt_len = tgt_seq.size() query_len = tgt_len key_len = tgt_len src_len = enc_output.size(1) # Run the forward pass of the TransformerDecoder. emb = self.embeddings(tgt_seq) if not self.layer_norm_first: emb = self.layer_norm(emb) if self_attn_caches is not None: emb = emb[:, -1:].contiguous() query_len = 1 # Decode mask dec_slf_attn_pad_mask = tgt_seq.detach().eq(PAD).unsqueeze(1).expand( batch_size, query_len, key_len) dec_slf_attn_sub_mask = get_attn_causal_mask(emb) dec_slf_attn_mask = torch.gt( dec_slf_attn_pad_mask + dec_slf_attn_sub_mask, 0) dec_enc_attn_mask = enc_mask.unsqueeze(1).expand( batch_size, query_len, src_len) output = emb new_self_attn_caches = [] new_enc_attn_caches = [] for i in range(self.num_layers): # copy head occur when last layer if i == self.num_layers - 1: layer_sample_K = sample_K else: layer_sample_K = 0 output, attn, self_attn_cache, enc_attn_cache \ = self.layer_stack[i](output, enc_output, dec_slf_attn_mask, dec_enc_attn_mask, enc_attn_cache=enc_attn_caches[i] if enc_attn_caches is not None else None, self_attn_cache=self_attn_caches[i] if self_attn_caches is not None else None, sample_K=layer_sample_K, seed=seed) new_self_attn_caches += [self_attn_cache] new_enc_attn_caches += [enc_attn_cache] if self.layer_norm_first: output = self.layer_norm(output) return output, new_self_attn_caches, new_enc_attn_caches
def common_target_masks_generation( self, metrics: Dict[str, Tensor]) -> Dict[str, Dict[str, Tensor]]: masks = {} if not metrics: return masks # TODO: support more target type in wrapper & config list refactor target_name = 'weight' # validate all wrapper setting the same sparsity # TODO: move validation logic to pruner global_sparsity_rate = self.pruner.get_modules_wrapper()[list( metrics.keys())[0]].config['total_sparsity'] for module_name, target_metric in metrics.items(): wrapper = self.pruner.get_modules_wrapper()[module_name] assert global_sparsity_rate == wrapper.config['total_sparsity'] # find the largest metric value among all metrics max_metric_value = list(metrics.values())[0].max() for module_name, target_metric in metrics.items(): max_metric_value = max_metric_value if max_metric_value >= target_metric.max( ) else target_metric.max() # prevent each module from being over-pruned, prevent ratio is 'max_sparsity_per_layer' for module_name, target_metric in metrics.items(): wrapper = self.pruner.get_modules_wrapper()[module_name] max_sparsity = wrapper.config.get('max_sparsity_per_layer', {}).get(module_name, 0.99) assert 0 <= max_sparsity <= 1 old_target_mask: Tensor = getattr(wrapper, f'{target_name}_mask') expand_times = old_target_mask.numel() // target_metric.numel() max_pruning_numel = int( max_sparsity * target_metric.numel()) * expand_times threshold = torch.topk(target_metric.reshape(-1), max_pruning_numel, largest=False)[0].max() metrics[module_name] = torch.where(target_metric <= threshold, target_metric, max_metric_value) # build the global_matric & calculate global threshold metric_list = [] for module_name, target_metric in metrics.items(): wrapper = self.pruner.get_modules_wrapper()[module_name] old_target_mask: Tensor = getattr(wrapper, f'{target_name}_mask') expand_times = old_target_mask.numel() // target_metric.numel() metric_list.append( target_metric.reshape(-1).unsqueeze(0).expand( expand_times, -1).reshape(-1)) global_metric = torch.cat(metric_list) max_pruning_num = int((global_metric != max_metric_value).sum().item()) total_pruning_num = min( int(global_sparsity_rate * global_metric.numel()), max_pruning_num) global_threshold = torch.topk(global_metric.reshape(-1), total_pruning_num, largest=False)[0].max() # generate masks for each target for module_name, target_metric in metrics.items(): masks[module_name] = {} wrapper = self.pruner.get_modules_wrapper()[module_name] shrinked_mask = torch.gt(target_metric, global_threshold).type_as(target_metric) masks[module_name][target_name] = self._expand_mask( module_name, target_name, shrinked_mask) return masks
def decode(self, pred, source): #pred = pred.squeeze(dim=0) for idx in range(self.num_label): name, chooce = self.label_list[idx], self.attribute_dict[ self.label_list[idx]][pred[idx]] #if chooce print('{}: {} source: {}'.format(name, chooce, source[idx])) #return name, chooce if __name__ == '__main__': checkpoint = torch.load("") model = resnet50(class_num=11) model.load_state_dict(checkpoint['model_state_dict']) model.cuda() img = Image.open("") img = img.convert("RGB") img = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), ])(img) img = img.unsqueeze(dim=0) out = model(img.cuda()) pred = torch.gt(out, torch.ones_like(out) * 0.8) dec = predict_decoder(dataset="CelebA", list=[6, 7, 8, 9, 11, 15, 20, 31, 32, 25, 33]) dec.decode(pred=pred, source=out.data.cpu().numpy())
def test_comparison_ops_with_type_promotion(self, device): value_for_type = { torch.uint8: (1 << 5), torch.int8: (1 << 5), torch.int16: (1 << 10), torch.int32: (1 << 20), torch.int64: (1 << 35), torch.float16: (1 << 10), torch.float32: (1 << 20), torch.float64: (1 << 35) } comparison_ops = [ dict( name="lt", out_op=lambda x, y, d: torch.lt(x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.lt(x, y), compare_op=lambda x, y: x < y, ), dict( name="le", out_op=lambda x, y, d: torch.le(x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.le(x, y), compare_op=lambda x, y: x <= y, ), dict( name="gt", out_op=lambda x, y, d: torch.gt(x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.gt(x, y), compare_op=lambda x, y: x > y, ), dict( name="ge", out_op=lambda x, y, d: torch.ge(x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.ge(x, y), compare_op=lambda x, y: x >= y, ), dict( name="eq", out_op=lambda x, y, d: torch.eq(x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.eq(x, y), compare_op=lambda x, y: x == y, ), dict( name="ne", out_op=lambda x, y, d: torch.ne(x, y, out=torch.empty(1, dtype=torch.bool, device=d)), ret_op=lambda x, y: torch.ne(x, y), compare_op=lambda x, y: x != y, ), ] for op in comparison_ops: for dt1 in torch.testing.get_all_math_dtypes(device): for dt2 in torch.testing.get_all_math_dtypes(device): val1 = value_for_type[dt1] val2 = value_for_type[dt2] t1 = torch.tensor([val1], dtype=dt1, device=device) t2 = torch.tensor([val2], dtype=dt2, device=device) expected = torch.tensor([op["compare_op"](val1, val2)], dtype=torch.bool) out_res = op["out_op"](t1, t2, device) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2) out_res = op["ret_op"](t1, t2) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2) # test that comparing a zero dim tensor with another zero dim tensor has type promotion behavior t1 = torch.tensor(val1, dtype=dt1, device=device) t2 = torch.tensor(val2, dtype=dt2, device=device) expected = torch.tensor(op["compare_op"](val1, val2), dtype=torch.bool) out_res = op["out_op"](t1, t2, device) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2) out_res = op["ret_op"](t1, t2) self.assertEqual(out_res, expected) self.assertTrue(out_res.dtype == torch.bool) self.assertTrue(t1.dtype == dt1) self.assertTrue(t2.dtype == dt2)
def binary_class_score(pred, target, thresh=0.5): predLabel = torch.gt(pred, thresh) print[predLabel, target] classScoreTest = torch.eq(predLabel, target.type_as(predLabel)) return classScoreTest.float().sum() / target.size(0)
def topkextra(x, y, model, loss_fct, k=25, epsilon=0.02, alpha=0.5, is_report_loss_diff=False, use_sample=False): x_next = topk(x, y, model, loss_fct, k, epsilon, alpha, False, use_sample) # some book-keeping if next(model.parameters()).is_cuda: x = x.cuda() y = y.cuda() x_next = x_next.cuda() y = Variable(y) # compute natural loss y_model = model(Variable(x)) loss_natural = loss_fct(y_model, y).data for i in range(1): x_best = x_next.clone() y_model = model(Variable(x_next)) best_loss = loss_fct(y_model, y) factor = 2 * len(x) no_improve = 0 for n in range(k): # forward pass # x_old = x_next.clone() xn_var = Variable(x_next, requires_grad=True) y_model = model(xn_var) loss = loss_fct(y_model, y) # compute gradient grads = torch.autograd.grad(loss.mean(), xn_var)[0].data # topk signs = torch.gt(grads, 0).float() grads = (signs - x_next) * grads grads -= x * grads rand_vars = torch.rand(len(grads), len(grads[0])) if next(model.parameters()).is_cuda: rand_vars = rand_vars.cuda() grads = rand_vars * grads kvals, kidx = grads.topk(k=min(10000, max(1, int(factor))), dim=1) x_next.scatter_(dim=1, index=kidx, src=signs.gather(dim=1, index=kidx)) # projection x_next = clip_tensor(x_next) x_next = or_float_tensors(x_next, x) # compute adversarial loss loss_adv = loss_fct(model(Variable(x_next)), y) factor = random.random() * 2 * len(x) + 1.0 #if loss.data.mean() > loss_adv.data.mean(): # x_next = x_old # factor = max(1,factor * 0.5) #else: # factor = factor * 2.0 #print(loss_adv.data.mean()) found = False for r in range(len(x)): if loss_adv.data[r] > best_loss.data[r]: x_best[r] = x_next[r].clone() best_loss.data[r] = loss_adv.data[r] found = True if found is True: # if loss_adv.data.mean() > best_loss.data.mean(): #x_best = x_next.clone() #best_loss = loss_adv #x_next = x_best.clone() no_improve = 0 else: no_improve += 1 if no_improve > len(x): break x_next = topk(x, y.data, model, loss_fct, k, epsilon, alpha, False, use_sample, x_best).cuda() # projection #x_next = clip_tensor(x_next) #x_next = or_float_tensors(x_next, x) #x_next = x_best # compute adversarial loss y_model = model(Variable(x_next)) loss_adv = loss_fct(y_model, y).data if is_report_loss_diff: print( "Natural loss (%.4f) vs Adversarial loss (%.4f), Difference: (%.4f)" % (loss_natural.mean(), loss_adv.mean(), loss_adv.mean() - loss_natural.mean())) replace_flag = (loss_adv < loss_natural).unsqueeze(1).expand_as(x_next) x_next[replace_flag] = x[replace_flag] if x_next.is_cuda: x_next = x_next.cpu() return x_next
def topk(x, y, model, loss_fct, k=25, epsilon=0.02, alpha=0.5, is_report_loss_diff=False, use_sample=False, start_x=None): # some book-keeping if next(model.parameters()).is_cuda: x = x.cuda() y = y.cuda() y = Variable(y) # compute natural loss y_model = model(Variable(x)) loss_natural = loss_fct(y_model, y).data # initialize starting point x_next = get_x0(x, use_sample).clone() if start_x is not None: x_next = start_x.clone() x_best = x_next.clone() best_loss = loss_fct(y_model, y) factor = 1.0 for n in range(k): # forward pass x_old = x_next.clone() xn_var = Variable(x_next, requires_grad=True) y_model = model(xn_var) loss = loss_fct(y_model, y) # compute gradient grads = torch.autograd.grad(loss.mean(), xn_var)[0].data # topk signs = torch.gt(grads, 0).float() grads = (signs - x_next) * grads grads -= x * grads rand_vars = torch.rand(len(grads), len(grads[0])) kvals, kidx = grads.topk(k=min(10000, max(1, int(factor))), dim=1) x_next.scatter_(dim=1, index=kidx, src=signs.gather(dim=1, index=kidx)) # projection x_next = clip_tensor(x_next) x_next = or_float_tensors(x_next, x) # compute adversarial loss loss_adv = loss_fct(model(Variable(x_next)), y) found = 0 for r in range(len(x)): if loss_adv.data[r] > best_loss.data[r]: x_best[r] = x_next[r].clone() best_loss.data[r] = loss_adv.data[r] found += 1 #x_next = x_best.clone() if found < len(x) * 0.5 and loss.data.mean() > loss_adv.data.mean(): #if loss.data.mean() > loss_adv.data.mean(): factor = max(factor * 0.5, 0.25) x_next = x_old #if found is False: if factor < 0.5: break else: factor = factor * 2.0 #x_next = x_best.clone() x_next = x_best # compute adversarial loss y_model = model(Variable(x_next)) loss_adv = loss_fct(y_model, y).data if is_report_loss_diff: print( "Natural loss (%.4f) vs Adversarial loss (%.4f), Difference: (%.4f)" % (loss_natural.mean(), loss_adv.mean(), loss_adv.mean() - loss_natural.mean())) replace_flag = (loss_adv < loss_natural).unsqueeze(1).expand_as(x_next) x_next[replace_flag] = x[replace_flag] if x_next.is_cuda: x_next = x_next.cpu() return x_next
def accuracy(score_pos, score_neg): """Computes the % of correctly ordered pairs""" pred1 = score_pos pred2 = score_neg correct = torch.gt(pred1, pred2) return float(correct.sum()) / correct.size(0), int(correct.sum())
def step(self, closure=None): """Performs a single optimization step. Arguments: closure (callable, optional): A closure that reevaluates the model and returns the loss. """ loss = None if closure is not None: loss = closure() for group in self.param_groups: for p in group['params']: if p.grad is None: continue grad = p.grad.data if grad.is_sparse: raise RuntimeError( 'cosangulargrad does not support sparse gradients, please consider SparseAdam instead' ) state = self.state[p] # State initialization if len(state) == 0: state['step'] = 0 # Exponential moving average of gradient values state['exp_avg'] = torch.zeros_like(p.data) # Exponential moving average of squared gradient values state['exp_avg_sq'] = torch.zeros_like(p.data) # Previous gradient state['previous_grad'] = torch.zeros_like(p.data) # temporary minimum value for comparison state['min'] = torch.zeros_like(p.data) # temporary difference between gradients for comparison state['diff'] = torch.zeros_like(p.data) # final cos value to be used state['final_cos_theta'] = torch.zeros_like(p.data) exp_avg, exp_avg_sq, previous_grad, min, diff, final_cos_theta = state['exp_avg'], state['exp_avg_sq'], \ state['previous_grad'], state['min'], \ state['diff'], state['final_cos_theta'] beta1, beta2 = group['betas'] state['step'] += 1 if group['weight_decay'] != 0: grad.add_(group['weight_decay'], p.data) # Decay the first and second moment running average coefficient exp_avg.mul_(beta1).add_(1 - beta1, grad) exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) denom = exp_avg_sq.sqrt().add_(group['eps']) bias_correction1 = 1 - beta1**state['step'] bias_correction2 = 1 - beta2**state['step'] tan_theta = abs( (previous_grad - grad) / (1 + previous_grad * grad)) cos_theta = 1 / torch.sqrt(1 + torch.square(tan_theta)) angle = torch.atan(tan_theta) * (180 / 3.141592653589793238) ans = torch.gt(angle, min) ans1, count = torch.unique(ans, return_counts=True) try: if (count[1] < count[0]): min = angle diff = abs(previous_grad - grad) final_cos_theta = cos_theta.clone() except: if (ans1[0].item() == False): min = angle diff = abs(previous_grad - grad) final_cos_theta = cos_theta.clone() angular_coeff = torch.tanh( abs(final_cos_theta )) * 0.5 + 0.5 # Calculating Angular coefficient state['previous_grad'] = grad.clone() state['min'] = min.clone() state['diff'] = diff.clone() state['final_cos_theta'] = final_cos_theta.clone() # update momentum with angular_coeff exp_avg1 = exp_avg * angular_coeff step_size = group['lr'] * math.sqrt( bias_correction2) / bias_correction1 p.data.addcdiv_(-step_size, exp_avg1, denom) return loss
def forward(self, output, target, loss_history, epoch, batch_idx): # output: B x As * (6 + 1 + num_classes) * H * W t0 = time.time() nB = output.data.size(0) nA = self.num_anchors # num_anchors = 5 nC = self.num_classes # num_classes = 8 nH = output.data.size(2) # nH 16 nW = output.data.size(3) # nW 32 output = output.view(nB, nA, (7 + nC), nH, nW) x = torch.sigmoid( output.index_select(2, Variable(torch.cuda.LongTensor([0]))).view( nB, nA, nH, nW)) y = torch.sigmoid( output.index_select(2, Variable(torch.cuda.LongTensor([1]))).view( nB, nA, nH, nW)) w = output.index_select(2, Variable(torch.cuda.LongTensor([2]))).view( nB, nA, nH, nW) l = output.index_select(2, Variable(torch.cuda.LongTensor([3]))).view( nB, nA, nH, nW) im = output.index_select(2, Variable(torch.cuda.LongTensor([4]))).view( nB, nA, nH, nW) re = output.index_select(2, Variable(torch.cuda.LongTensor([5]))).view( nB, nA, nH, nW) conf = torch.sigmoid( output.index_select(2, Variable(torch.cuda.LongTensor([6]))).view( nB, nA, nH, nW)) cls = output.index_select( 2, Variable(torch.linspace(7, 7 + nC - 1, nC).long().cuda())) cls = cls.view(nB * nA, nC, nH * nW).transpose(1, 2).contiguous().view( nB * nA * nH * nW, nC) t1 = time.time() pred_boxes = torch.cuda.FloatTensor(6, nB * nA * nH * nW) grid_x = torch.linspace(0, nW - 1, nW).repeat(nH, 1).repeat( nB * nA, 1, 1).view(nB * nA * nH * nW).cuda() grid_y = torch.linspace(0, nH - 1, nH).repeat(nW, 1).t().repeat( nB * nA, 1, 1).view(nB * nA * nH * nW).cuda() anchor_w = torch.Tensor(anchors).view( nA, self.anchor_step).index_select(1, torch.LongTensor([0])).cuda() anchor_l = torch.Tensor(anchors).view( nA, self.anchor_step).index_select(1, torch.LongTensor([1])).cuda() anchor_w = anchor_w.repeat(nB, 1).repeat(1, 1, nH * nW).view(nB * nA * nH * nW) anchor_l = anchor_l.repeat(nB, 1).repeat(1, 1, nH * nW).view(nB * nA * nH * nW) pred_boxes[0] = x.data.view(nB * nA * nH * nW).cuda() + grid_x pred_boxes[1] = y.data.view(nB * nA * nH * nW).cuda() + grid_y pred_boxes[2] = torch.exp(w.data).view( nB * nA * nH * nW).cuda() * anchor_w pred_boxes[3] = torch.exp(l.data).view( nB * nA * nH * nW).cuda() * anchor_l pred_boxes[4] = im.data.view(nB * nA * nH * nW).cuda() pred_boxes[5] = re.data.view(nB * nA * nH * nW).cuda() pred_boxes = convert2cpu( pred_boxes.transpose(0, 1).contiguous().view(-1, 6)) t2 = time.time() nGT, nCorrect, coord_mask, conf_mask, cls_mask, tx, ty, tw, tl, tim, tre, tconf,tcls, loss_iou = build_targets(pred_boxes, target.data, self.anchors, nA, nC, \ nH, nW, self.noobject_scale, self.object_scale, self.thresh) cls_mask = (cls_mask == 1) nProposals = int(torch.sum(torch.gt(conf, 0.25))) tx = Variable(tx.cuda()) ty = Variable(ty.cuda()) tw = Variable(tw.cuda()) tl = Variable(tl.cuda()) tim = Variable(tim.cuda()) tre = Variable(tre.cuda()) tconf = Variable(tconf.cuda()) cls_mask = cls_mask.view(nB * nA * nH * nW) tcls = Variable(tcls.view(-1)[cls_mask].long().cuda()) coord_mask = Variable(coord_mask.cuda()) conf_mask = Variable(conf_mask.cuda()) cls_mask = Variable(cls_mask.view(-1, 1).repeat(1, nC).cuda()) cls = cls[cls_mask].view(-1, nC) t3 = time.time() loss_x = self.coord_scale * nn.MSELoss(reduction='sum')( x * coord_mask, tx * coord_mask) loss_y = self.coord_scale * nn.MSELoss(reduction='sum')( y * coord_mask, ty * coord_mask) loss_w = self.coord_scale * nn.MSELoss(reduction='sum')( w * coord_mask, tw * coord_mask) loss_l = self.coord_scale * nn.MSELoss(reduction='sum')( l * coord_mask, tl * coord_mask) loss_im = self.coord_scale * nn.MSELoss(reduction='sum')( im * coord_mask, tim * coord_mask) loss_re = self.coord_scale * nn.MSELoss(reduction='sum')( re * coord_mask, tre * coord_mask) loss_Euler = loss_im + loss_re loss_conf = nn.MSELoss(reduction='sum')(conf * conf_mask, tconf * conf_mask) loss_cls = self.class_scale * nn.CrossEntropyLoss(reduction='sum')( cls, tcls) loss = loss_x + loss_y + loss_w + loss_l + loss_conf + loss_cls + loss_Euler + torch.tensor( loss_iou).cuda() loss_history[epoch, batch_idx, :] = [ loss_x, loss_y, loss_w, loss_l, loss_conf, loss_cls, loss_Euler, loss_iou ] t4 = time.time() if False: print('-----------------------------------') print(' activation : %f' % (t1 - t0)) print(' create pred_boxes : %f' % (t2 - t1)) print(' build targets : %f' % (t3 - t2)) print(' create loss : %f' % (t4 - t3)) print(' total : %f' % (t4 - t0)) logging.info( 'nGT %d, recall %d, proposals %d, loss: x %f, y %f, w %f, h %f, conf %f, cls %f, Euler %f, total %f' % (nGT, nCorrect, nProposals, loss_x.data, loss_y.data, loss_w.data, loss_l.data, loss_conf.data, loss_cls.data, loss_Euler.data, loss.data)) return loss
def beam_search(models, features, params): if not isinstance(models, (list, tuple)): raise ValueError("'models' must be a list or tuple") beam_size = params.beam_size top_beams = params.top_beams alpha = params.decode_alpha decode_ratio = params.decode_ratio decode_length = params.decode_length pad_id = params.lookup["target"][params.pad.encode("utf-8")] bos_id = params.lookup["target"][params.bos.encode("utf-8")] eos_id = params.lookup["target"][params.eos.encode("utf-8")] min_val = -1e9 shape = features["source"].shape device = features["source"].device batch_size = shape[0] seq_length = shape[1] # Compute initial state if necessary states = [] funcs = [] for model in models: state = model.empty_state(batch_size, device) states.append(model.encode(features, state)) funcs.append(model.decode) # For source sequence length max_length = features["source_mask"].sum(1) * decode_ratio max_length = max_length.long() + decode_length max_step = max_length.max() # [batch, beam_size] max_length = torch.unsqueeze(max_length, 1).repeat([1, beam_size]) # Expand the inputs # [batch, length] => [batch * beam_size, length] features["source"] = torch.unsqueeze(features["source"], 1) features["source"] = features["source"].repeat([1, beam_size, 1]) features["source"] = torch.reshape(features["source"], [batch_size * beam_size, seq_length]) features["source_mask"] = torch.unsqueeze(features["source_mask"], 1) features["source_mask"] = features["source_mask"].repeat([1, beam_size, 1]) features["source_mask"] = torch.reshape( features["source_mask"], [batch_size * beam_size, seq_length]) decoding_fn = _get_inference_fn(funcs, features) states = map_structure(lambda x: _tile_to_beam_size(x, beam_size), states) # Initial beam search state init_seqs = torch.full([batch_size, beam_size, 1], bos_id, device=device, dtype=torch.long) #init_seqs = init_seqs.long() init_log_probs = init_seqs.new_tensor([[0.] + [min_val] * (beam_size - 1)], dtype=torch.float32) init_log_probs = init_log_probs.repeat([batch_size, 1]) init_scores = torch.zeros_like(init_log_probs) fin_seqs = torch.zeros([batch_size, beam_size, 1], dtype=torch.int64, device=device) fin_scores = torch.full([batch_size, beam_size], min_val, dtype=torch.float32, device=device) fin_flags = torch.zeros([batch_size, beam_size], dtype=torch.bool, device=device) state = BeamSearchState( inputs=(init_seqs, init_log_probs, init_scores), state=states, finish=(fin_flags, fin_seqs, fin_scores), ) for time in range(max_step): state = _beam_search_step(time, decoding_fn, state, batch_size, beam_size, alpha, pad_id, eos_id, max_length) max_penalty = ((5.0 + max_step) / 6.0)**alpha best_alive_score = torch.max(state.inputs[1][:, 0] / max_penalty) worst_finished_score = torch.min(state.finish[2]) cond = torch.gt(worst_finished_score, best_alive_score) is_finished = bool(cond) if is_finished: break final_state = state alive_seqs = final_state.inputs[0] alive_scores = final_state.inputs[2] final_flags = final_state.finish[0].byte() final_seqs = final_state.finish[1] final_scores = final_state.finish[2] final_seqs = torch.where(final_flags[:, :, None], final_seqs, alive_seqs) final_scores = torch.where(final_flags, final_scores, alive_scores) final_mask = final_seqs[:, 0, 1:].eq(pad_id).eq(0) # Append extra <eos> final_seqs = torch.nn.functional.pad(final_seqs, (0, 1, 0, 0, 0, 0), value=eos_id) first_beam_states = map_structure(lambda x: x[:, 0], final_state.state) # update target starts according to actuall length for i, fb_state in enumerate(first_beam_states): first_beam_states[i]["target_mask"] = final_mask return final_seqs[:, :top_beams, 1:], final_scores[:, :top_beams], first_beam_states
[shape_features, skel_xyz, skel_r], 2).detach() A_init = A_init.detach() # train GAE A_pred = model_gae(skel_node_features, A_init) loss_MBCE = model_gae.compute_loss(A_pred, A_init, known_mask.detach()) optimizer_gae.zero_grad() loss_MBCE.backward() optimizer_gae.step() A_final = model_gae.recover_A(A_pred, valid_mask) if iter % save_result_iter == 0: output_results(save_result_path, batch_id, epoch, batch_pc, skel_xyz, skel_r, A_init, torch.gt(A_pred, 0.5), A_final) if iter % save_net_iter == 0: torch.save(model_gae.state_dict(), save_net_path + 'para-gae.pth') print('A init:', torch.sum(A_init).item() / conf.BATCH_SIZE / 2.0) print('A known:', torch.sum(known_mask).item() / conf.BATCH_SIZE / 2.0) print( 'A pred:', torch.sum(torch.gt(A_pred, 0.5)).item() / conf.BATCH_SIZE / 2.0) ###################################### # Train two networks jointly
def forward(self, q_pts, s_pts, neighb_inds, x): ################### # Offset generation ################### if self.deformable: # Get offsets with a KPConv that only takes part of the features self.offset_features = self.offset_conv(q_pts, s_pts, neighb_inds, x) + self.offset_bias if self.modulated: # Get offset (in normalized scale) from features unscaled_offsets = self.offset_features[:, :self.p_dim * self.K] unscaled_offsets = unscaled_offsets.view(-1, self.K, self.p_dim) # Get modulations modulations = 2 * torch.sigmoid(self.offset_features[:, self.p_dim * self.K:]) else: # Get offset (in normalized scale) from features unscaled_offsets = self.offset_features.view(-1, self.K, self.p_dim) # No modulations modulations = None # Rescale offset for this layer offsets = unscaled_offsets * self.KP_extent else: offsets = None modulations = None ###################### # Deformed convolution ###################### # Add a fake point in the last row for shadow neighbors s_pts = torch.cat((s_pts, torch.zeros_like(s_pts[:1, :]) + 1e6), 0) # Get neighbor points [n_points, n_neighbors, dim] neighbors = s_pts[neighb_inds, :] # Center every neighborhood neighbors = neighbors - q_pts.unsqueeze(1) # Apply offsets to kernel points [n_points, n_kpoints, dim] if self.deformable: self.deformed_KP = offsets + self.kernel_points deformed_K_points = self.deformed_KP.unsqueeze(1) else: deformed_K_points = self.kernel_points # Get all difference matrices [n_points, n_neighbors, n_kpoints, dim] neighbors.unsqueeze_(2) differences = neighbors - deformed_K_points # Get the square distances [n_points, n_neighbors, n_kpoints] sq_distances = torch.sum(differences ** 2, dim=3) # Optimization by ignoring points outside a deformed KP range if self.deformable: # Save distances for loss self.min_d2, _ = torch.min(sq_distances, dim=1) # Boolean of the neighbors in range of a kernel point [n_points, n_neighbors] in_range = torch.any(sq_distances < self.KP_extent ** 2, dim=2).type(torch.int32) # New value of max neighbors new_max_neighb = torch.max(torch.sum(in_range, dim=1)) # For each row of neighbors, indices of the ones that are in range [n_points, new_max_neighb] neighb_row_bool, neighb_row_inds = torch.topk(in_range, new_max_neighb.item(), dim=1) # Gather new neighbor indices [n_points, new_max_neighb] new_neighb_inds = neighb_inds.gather(1, neighb_row_inds, sparse_grad=False) # Gather new distances to KP [n_points, new_max_neighb, n_kpoints] neighb_row_inds.unsqueeze_(2) neighb_row_inds = neighb_row_inds.expand(-1, -1, self.K) sq_distances = sq_distances.gather(1, neighb_row_inds, sparse_grad=False) # New shadow neighbors have to point to the last shadow point new_neighb_inds *= neighb_row_bool new_neighb_inds -= (neighb_row_bool.type(torch.int64) - 1) * int(s_pts.shape[0] - 1) else: new_neighb_inds = neighb_inds # Get Kernel point influences [n_points, n_kpoints, n_neighbors] if self.KP_influence == 'constant': # Every point get an influence of 1. all_weights = torch.ones_like(sq_distances) all_weights = torch.transpose(all_weights, 1, 2) elif self.KP_influence == 'linear': # Influence decrease linearly with the distance, and get to zero when d = KP_extent. all_weights = torch.clamp(1 - torch.sqrt(sq_distances) / self.KP_extent, min=0.0) all_weights = torch.transpose(all_weights, 1, 2) elif self.KP_influence == 'gaussian': # Influence in gaussian of the distance. sigma = self.KP_extent * 0.3 all_weights = radius_gaussian(sq_distances, sigma) all_weights = torch.transpose(all_weights, 1, 2) else: raise ValueError('Unknown influence function type (config.KP_influence)') # In case of closest mode, only the closest KP can influence each point if self.aggregation_mode == 'closest': neighbors_1nn = torch.argmin(sq_distances, dim=2) all_weights *= torch.transpose(nn.functional.one_hot(neighbors_1nn, self.K), 1, 2) elif self.aggregation_mode != 'sum': raise ValueError("Unknown convolution mode. Should be 'closest' or 'sum'") # Add a zero feature for shadow neighbors x = torch.cat((x, torch.zeros_like(x[:1, :])), 0) # Get the features of each neighborhood [n_points, n_neighbors, in_fdim] neighb_x = gather(x, new_neighb_inds) # Apply distance weights [n_points, n_kpoints, in_fdim] weighted_features = torch.matmul(all_weights, neighb_x) # Apply modulations if self.deformable and self.modulated: weighted_features *= modulations.unsqueeze(2) # Apply network weights [n_kpoints, n_points, out_fdim] weighted_features = weighted_features.permute((1, 0, 2)) kernel_outputs = torch.matmul(weighted_features, self.weights) # Convolution sum [n_points, out_fdim] # return torch.sum(kernel_outputs, dim=0) output_features = torch.sum(kernel_outputs, dim=0, keepdim=False) # normalization term. neighbor_features_sum = torch.sum(neighb_x, dim=-1) neighbor_num = torch.sum(torch.gt(neighbor_features_sum, 0.0), dim=-1) neighbor_num = torch.max(neighbor_num, torch.ones_like(neighbor_num)) output_features = output_features / neighbor_num.unsqueeze(1) return output_features
def mask(var): return torch.gt(var, 0).float()
def _common_channel_to_prune(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): """ Calculate the common channels should be pruned by all the layers in this group. This function is for filter pruning of Conv layers. if want to support the dependency-aware mode for others ops, you need to inherit this class and overwrite `_common_channel_to_prune`. Parameters ---------- sparsities : list List of float that specify the sparsity for each conv layer. wrappers : list List of wrappers groups : list The number of the filter groups of each layer. wrappers_idx : list The indexes of the wrappers """ # sparsity configs for each wrapper # sparsities = [_w.config['sparsity'] for _w in wrappers] # check the type of the input wrappers for _w in wrappers: msg = 'module type {} is not supported!'.format(_w.type) assert _w.type == 'Conv2d', msg # Among the dependent layers, the layer with smallest # sparsity determines the final benefit of the speedup # module. To better harvest the speed benefit, we need # to ensure that these dependent layers have at least # `min_sparsity` pruned channel are the same. if len(channel_dsets) == len(wrappers): # all the layers in the dependency sets are pruned min_sparsity = min(sparsities) else: # not all the layers in the dependency set # are pruned min_sparsity = 0 # donnot prune the channels that we cannot harvest the speed from sparsities = [min_sparsity] * len(sparsities) # find the max number of the filter groups of the dependent # layers. The group constraint of this dependency set is decided # by the layer with the max groups. # should use the least common multiple for all the groups # the max_group is lower than the channel_count, because # the number of the filter is always divisible by the number of the group max_group = np.lcm.reduce(groups) channel_count = wrappers[0].module.weight.data.size(0) device = wrappers[0].module.weight.device channel_sum = torch.zeros(channel_count).to(device) for _w, _w_idx in zip(wrappers, wrappers_idx): # calculate the L1/L2 sum for all channels c_sum = self.get_channel_sum(_w, _w_idx) if c_sum is None: # if the channel sum cannot be calculated # now, return None return None channel_sum += c_sum # prune the same `min_sparsity` channels based on channel_sum # for all the layers in the channel sparsity target_pruned = int(channel_count * min_sparsity) # pruned_per_group may be zero, for example dw conv pruned_per_group = int(target_pruned / max_group) group_step = int(channel_count / max_group) channel_masks = [] for gid in range(max_group): _start = gid * group_step _end = (gid + 1) * group_step if pruned_per_group > 0: threshold = torch.topk( channel_sum[_start: _end], pruned_per_group, largest=False)[0].max() group_mask = torch.gt(channel_sum[_start:_end], threshold) else: group_mask = torch.ones(group_step).to(device) channel_masks.append(group_mask) channel_masks = torch.cat(channel_masks, dim=0) pruned_channel_index = ( channel_masks == False).nonzero().squeeze(1).tolist() logger.info('Prune the %s channels for all dependent', ','.join([str(x) for x in pruned_channel_index])) return channel_masks
def mode(self): return torch.gt(self.probs, 0.5).float()
def cosine_similarity_selector_IQP(x1, solver, nb_selected, eps=1e-3, slack=0.01): """ Integer programming """ # x1=gradient memories x2 = None w1 = x1.norm(p=2, dim=1, keepdim=True) inds = torch.nonzero(torch.gt(w1, slack))[:, 0] w1 = w1[inds] x1 = x1[inds] x2 = x1 if x2 is None else x2 w2 = w1 if x2 is x1 else x2.norm(p=2, dim=1, keepdim=True) G = torch.mm(x1, x2.t()) / (w1 * w2.t()) # .clamp(min=eps) t = G.size(0) # a=torch.sum(G-torch.eye(t),1)/(t-1)*-0.1 # a=torch.max(G-torch.eye(t),1)[0]*-0.1 # a=a.double().numpy() G = G.double().numpy() # G=G+ np.eye(t) * eps # G = 0.5 * (G + G.transpose()) +np.eye(t) * eps # a=(w1*-1).view(t).numpy() # a=np.zeros(t) a = np.ones(t) * -1 # a=((w1-torch.min(w1))/(torch.max(w1)-torch.min(w1))).squeeze().double().numpy()*-0.01 # h = np.zeros(1) + nb_selected C2 = np.eye(t) hlower = np.zeros(t) hupper = np.ones(t) idx = np.arange(t) # np.concatenate((a, b), axis=0) ################# # C=np.concatenate((C2, C), axis=1) # C=np.transpose(C) # h_final_lower=np.concatenate((hlower,h), axis=0) # h_final_upper=np.concatenate((hupper,h), axis=0) ################# G = spa.csc_matrix(G) C2 = spa.csc_matrix(C2) solver.setup(G, a, C2, hlower, hupper, idx, hlower, hupper, miosqp_settings, osqp_settings) results = solver.solve() print("STATUS", results.status) coeffiecents_np = results.x coeffiecents = torch.nonzero(torch.Tensor(coeffiecents_np)) print("number of selected items is", sum(coeffiecents_np)) # _,inds=torch.sort(coeffiecents,descending=True) return inds[coeffiecents.squeeze()]
def rprop(opfunc, x, config, state=None): """ A plain implementation of RPROP ARGS: - `opfunc` : a function that takes a single input (X), the point of evaluation, and returns f(X) and df/dX - `x` : the initial point - `state` : a table describing the state of the optimizer; after each call the state is modified - `state['stepsize']` : initial step size, common to all components - `state['etaplus']` : multiplicative increase factor, > 1 (default 1.2) - `state['etaminus']` : multiplicative decrease factor, < 1 (default 0.5) - `state['stepsizemax']` : maximum stepsize allowed (default 50) - `state['stepsizemin']` : minimum stepsize allowed (default 1e-6) - `state['niter']` : number of iterations (default 1) RETURN: - `x` : the new x vector - `f(x)` : the function, evaluated before the update (Martin Riedmiller, Koray Kavukcuoglu 2013) """ if config is None and state is None: raise ValueError("rprop requires a dictionary to retain state between iterations") # (0) get/update state state = state if state is not None else config stepsize = config.get('stepsize', 0.1) etaplus = config.get('etaplus', 1.2) etaminus = config.get('etaminus', 0.5) stepsizemax = config.get('stepsizemax', 50.0) stepsizemin = config.get('stepsizemin', 1e-06) niter = config.get('niter', 1) hfx = [] for i in range(niter): # (1) evaluate f(x) and df/dx fx, dfdx = opfunc(x) # init temp storage if 'delta' not in state: state['delta'] = dfdx.new(dfdx.size()).zero_() state['stepsize'] = dfdx.new(dfdx.size()).fill_(stepsize) state['sign'] = dfdx.new(dfdx.size()) state['bytesign'] = torch.ByteTensor(dfdx.size()) state['psign'] = torch.ByteTensor(dfdx.size()) state['nsign'] = torch.ByteTensor(dfdx.size()) state['zsign'] = torch.ByteTensor(dfdx.size()) state['dminmax'] = torch.ByteTensor(dfdx.size()) if str(type(x)).find('Cuda') > -1: # Push to GPU state['psign'] = state['psign'].cuda() state['nsign'] = state['nsign'].cuda() state['zsign'] = state['zsign'].cuda() state['dminmax'] = state['dminmax'].cuda() # sign of derivative from last step to this one torch.mul(dfdx, state['delta'], out=state['sign']).sign_() # get indices of >0, <0 and ==0 entries torch.gt(state['sign'], 0, out=state['psign']) torch.lt(state['sign'], 0, out=state['nsign']) torch.eq(state['sign'], 0, out=state['zsign']) # get step size updates state['sign'][state['psign']] = etaplus state['sign'][state['nsign']] = etaminus state['sign'][state['zsign']] = 1 # update stepsizes with step size updates state['stepsize'].mul_(state['sign']) # threshold step sizes # >50 => 50 torch.gt(state['stepsize'], stepsizemax, out=state['dminmax']) state['stepsize'][state['dminmax']] = stepsizemax # <1e-6 ==> 1e-6 torch.lt(state['stepsize'], stepsizemin, out=state['dminmax']) state['stepsize'][state['dminmax']] = stepsizemin # for dir<0, dfdx=0 # for dir>=0 dfdx=dfdx dfdx[state['nsign']] = 0 torch.sign(dfdx, out=state['sign']) # update weights x.addcmul_(-1, state['sign'], state['stepsize']) # update state['dfdx'] with current dfdx state['delta'].copy_(dfdx) hfx.append(fx) # return x*, table of f(x) values from each step return x, hfx
def greater(a, b): return torch.gt(a, b)
def gt_select(self, input: str, other) -> 'TensorGroup': """ NOTE: tensor.dim() must all be 1 """ mask = torch.gt(self.tensors[input], other) return self.masked_select(mask)
def forward(self, x1, x1_f, x1_pos, x1_ner, x1_mask, x2, x2_mask): """Inputs: x1 = document word indices [batch * len_d] x1_f = document word features indices [batch * len_d * nfeat] x1_pos = document POS tags [batch * len_d] x1_ner = document entity tags [batch * len_d] x1_mask = document padding mask [batch * len_d] x2 = question word indices [batch * len_q] x2_mask = question padding mask [batch * len_q] """ # Embed both document and question x1_emb = self.embedding(x1) x2_emb = self.embedding(x2) if self.opt['dropout_emb'] > 0: x1_emb = nn.functional.dropout(x1_emb, p=self.opt['dropout_emb'], training=self.training) x2_emb = nn.functional.dropout(x2_emb, p=self.opt['dropout_emb'], training=self.training) drnn_input_list = [x1_emb, x1_f] # Add attention-weighted question representation if self.opt['use_qemb']: x2_weighted_emb = self.qemb_match(x1_emb, x2_emb, x2_mask) drnn_input_list.append(x2_weighted_emb) if self.opt['pos']: x1_pos_emb = self.pos_embedding(x1_pos) if self.opt['dropout_emb'] > 0: x1_pos_emb = nn.functional.dropout(x1_pos_emb, p=self.opt['dropout_emb'], training=self.training) drnn_input_list.append(x1_pos_emb) if self.opt['ner']: x1_ner_emb = self.ner_embedding(x1_ner) if self.opt['dropout_emb'] > 0: x1_ner_emb = nn.functional.dropout(x1_ner_emb, p=self.opt['dropout_emb'], training=self.training) drnn_input_list.append(x1_ner_emb) drnn_input = torch.cat(drnn_input_list, 2) # Encode document with RNN doc_hiddens = self.doc_rnn(drnn_input, x1_mask) # Encode question with RNN + merge hiddens question_hiddens = self.question_rnn(x2_emb, x2_mask) #if self.opt['question_merge'] == 'avg': # q_merge_weights = layers.uniform_weights(question_hiddens, x2_mask) #elif self.opt['question_merge'] == 'self_attn': # q_merge_weights = self.self_attn(question_hiddens, x2_mask) #question_hidden = layers.weighted_avg(question_hiddens, q_merge_weights) enc_slf_attn_pad_mask = get_attn_padding_mask(x1, x1) enc_slf_attn_sub_mask = get_attn_subsequent_mask(x1) enc_slf_attn_mask = torch.gt(enc_slf_attn_pad_mask + enc_slf_attn_sub_mask, 0) enc_dec_attn_pad_mask = get_attn_padding_mask(x1, x2) start_scores, *_ = self.decoder_start.forward(dec_input=doc_hiddens, enc_output=question_hiddens, slf_attn_mask=enc_slf_attn_mask, dec_enc_attn_mask=enc_dec_attn_pad_mask ) end_scores, *_ = self.decoder_end.forward(dec_input=doc_hiddens, enc_output=question_hiddens, slf_attn_mask=enc_slf_attn_mask, dec_enc_attn_mask=enc_dec_attn_pad_mask ) if self.training: start_scores = nn.functional.log_softmax(start_scores) end_scores = nn.functional.log_softmax(end_scores) else: start_scores = nn.functional.softmax(start_scores) end_scores = nn.functional.softmax(end_scores) return start_scores, end_scores
def MaskHelper(seg, color): # green mask = torch.Tensor() if (color == 'green'): mask = torch.lt(seg[0], 0.1) mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1)) mask = torch.mul(mask, torch.lt(seg[2], 0.1)) elif (color == 'black'): mask = torch.lt(seg[0], 0.1) mask = torch.mul(mask, torch.lt(seg[1], 0.1)) mask = torch.mul(mask, torch.lt(seg[2], 0.1)) elif (color == 'white'): mask = torch.gt(seg[0], 1 - 0.1) mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1)) mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1)) elif (color == 'red'): mask = torch.gt(seg[0], 1 - 0.1) mask = torch.mul(mask, torch.lt(seg[1], 0.1)) mask = torch.mul(mask, torch.lt(seg[2], 0.1)) elif (color == 'blue'): mask = torch.lt(seg[0], 0.1) mask = torch.mul(mask, torch.lt(seg[1], 0.1)) mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1)) elif (color == 'yellow'): mask = torch.gt(seg[0], 1 - 0.1) mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1)) mask = torch.mul(mask, torch.lt(seg[2], 0.1)) elif (color == 'grey'): mask = torch.lt(seg[0], 0.1) mask = torch.mul(mask, torch.lt(seg[1], 0.1)) mask = torch.mul(mask, torch.lt(seg[2], 0.1)) elif (color == 'lightblue'): mask = torch.lt(seg[0], 0.1) mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1)) mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1)) elif (color == 'purple'): mask = torch.gt(seg[0], 1 - 0.1) mask = torch.mul(mask, torch.lt(seg[1], 0.1)) mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1)) else: print('MaskHelper(): color not recognized, color = ' + color) return mask.float()
def vtln_warp_freq(vtln_low_cutoff, vtln_high_cutoff, low_freq, high_freq, vtln_warp_factor, freq): r"""This computes a VTLN warping function that is not the same as HTK's one, but has similar inputs (this function has the advantage of never producing empty bins). This function computes a warp function F(freq), defined between low_freq and high_freq inclusive, with the following properties: F(low_freq) == low_freq F(high_freq) == high_freq The function is continuous and piecewise linear with two inflection points. The lower inflection point (measured in terms of the unwarped frequency) is at frequency l, determined as described below. The higher inflection point is at a frequency h, determined as described below. If l <= f <= h, then F(f) = f/vtln_warp_factor. If the higher inflection point (measured in terms of the unwarped frequency) is at h, then max(h, F(h)) == vtln_high_cutoff. Since (by the last point) F(h) == h/vtln_warp_factor, then max(h, h/vtln_warp_factor) == vtln_high_cutoff, so h = vtln_high_cutoff / max(1, 1/vtln_warp_factor). = vtln_high_cutoff * min(1, vtln_warp_factor). If the lower inflection point (measured in terms of the unwarped frequency) is at l, then min(l, F(l)) == vtln_low_cutoff This implies that l = vtln_low_cutoff / min(1, 1/vtln_warp_factor) = vtln_low_cutoff * max(1, vtln_warp_factor) Args: vtln_low_cutoff (float): Lower frequency cutoffs for VTLN vtln_high_cutoff (float): Upper frequency cutoffs for VTLN low_freq (float): Lower frequency cutoffs in mel computation high_freq (float): Upper frequency cutoffs in mel computation vtln_warp_factor (float): Vtln warp factor freq (torch.Tensor): given frequency in Hz Returns: torch.Tensor: Freq after vtln warp """ assert vtln_low_cutoff > low_freq, 'be sure to set the vtln_low option higher than low_freq' assert vtln_high_cutoff < high_freq, 'be sure to set the vtln_high option lower than high_freq [or negative]' l = vtln_low_cutoff * max(1.0, vtln_warp_factor) h = vtln_high_cutoff * min(1.0, vtln_warp_factor) scale = 1.0 / vtln_warp_factor Fl = scale * l # F(l) Fh = scale * h # F(h) assert l > low_freq and h < high_freq # slope of left part of the 3-piece linear function scale_left = (Fl - low_freq) / (l - low_freq) # [slope of center part is just "scale"] # slope of right part of the 3-piece linear function scale_right = (high_freq - Fh) / (high_freq - h) res = torch.empty_like(freq) outside_low_high_freq = torch.lt(freq, low_freq) | torch.gt( freq, high_freq) # freq < low_freq || freq > high_freq before_l = torch.lt(freq, l) # freq < l before_h = torch.lt(freq, h) # freq < h after_h = torch.ge(freq, h) # freq >= h # order of operations matter here (since there is overlapping frequency regions) res[after_h] = high_freq + scale_right * (freq[after_h] - high_freq) res[before_h] = scale * freq[before_h] res[before_l] = low_freq + scale_left * (freq[before_l] - low_freq) res[outside_low_high_freq] = freq[outside_low_high_freq] return res
def get_mel_banks(num_bins, window_length_padded, sample_freq, low_freq, high_freq, vtln_low, vtln_high, vtln_warp_factor): # type: (int, int, float, float, float, float, float) """ Returns: Tuple[torch.Tensor, torch.Tensor]: The tuple consists of ``bins`` (which is melbank of size (``num_bins``, ``num_fft_bins``)) and ``center_freqs`` (which is center frequencies of bins of size (``num_bins``)). """ assert num_bins > 3, 'Must have at least 3 mel bins' assert window_length_padded % 2 == 0 num_fft_bins = window_length_padded / 2 nyquist = 0.5 * sample_freq if high_freq <= 0.0: high_freq += nyquist assert (0.0 <= low_freq < nyquist) and (0.0 < high_freq <= nyquist) and (low_freq < high_freq), \ ('Bad values in options: low-freq %f and high-freq %f vs. nyquist %f' % (low_freq, high_freq, nyquist)) # fft-bin width [think of it as Nyquist-freq / half-window-length] fft_bin_width = sample_freq / window_length_padded mel_low_freq = mel_scale_scalar(low_freq) mel_high_freq = mel_scale_scalar(high_freq) # divide by num_bins+1 in next line because of end-effects where the bins # spread out to the sides. mel_freq_delta = (mel_high_freq - mel_low_freq) / (num_bins + 1) if vtln_high < 0.0: vtln_high += nyquist assert vtln_warp_factor == 1.0 or ((low_freq < vtln_low < high_freq) and (0.0 < vtln_high < high_freq) and (vtln_low < vtln_high)), \ ('Bad values in options: vtln-low %f and vtln-high %f, versus low-freq %f and high-freq %f' % (vtln_low, vtln_high, low_freq, high_freq)) bin = torch.arange(num_bins).unsqueeze(1) left_mel = mel_low_freq + bin * mel_freq_delta # size(num_bins, 1) center_mel = mel_low_freq + (bin + 1.0) * mel_freq_delta # size(num_bins, 1) right_mel = mel_low_freq + (bin + 2.0) * mel_freq_delta # size(num_bins, 1) if vtln_warp_factor != 1.0: left_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, left_mel) center_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, center_mel) right_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, right_mel) center_freqs = inverse_mel_scale(center_mel) # size (num_bins) # size(1, num_fft_bins) mel = mel_scale(fft_bin_width * torch.arange(num_fft_bins)).unsqueeze(0) # size (num_bins, num_fft_bins) up_slope = (mel - left_mel) / (center_mel - left_mel) down_slope = (right_mel - mel) / (right_mel - center_mel) if vtln_warp_factor == 1.0: # left_mel < center_mel < right_mel so we can min the two slopes and clamp negative values bins = torch.max(torch.zeros(1), torch.min(up_slope, down_slope)) else: # warping can move the order of left_mel, center_mel, right_mel anywhere bins = torch.zeros_like(up_slope) up_idx = torch.gt(mel, left_mel) & torch.le( mel, center_mel) # left_mel < mel <= center_mel down_idx = torch.gt(mel, center_mel) & torch.lt( mel, right_mel) # center_mel < mel < right_mel bins[up_idx] = up_slope[up_idx] bins[down_idx] = down_slope[down_idx] return bins, center_freqs
def step(self, blob): ''' This function should be called at every timestep to perform tracking with a blob containing the image information. ''' for t in self.tracks: # add current position to last_pos list t.last_pos.append(t.pos.clone()) # 1. Look for new detections if self.public_detections: boxes, scores = blob['boxes'], blob['scores'] if boxes.nelement() == 0: boxes = scores = torch.zeros(0).cuda() else: boxes = boxes.cuda() scores = scores.cuda() else: boxes, scores = self.obj_detect.detect(blob['img']) if boxes.nelement() > 0: boxes = clip_boxes_to_image(boxes, blob['img'].shape[-2:]) inds = torch.gt(scores, self.detection_person_thresh).nonzero().view(-1) else: inds = torch.zeros(0).cuda() if inds.nelement() > 0: det_pos = boxes[inds] det_scores = scores[inds] else: det_pos = torch.zeros(0).cuda() det_scores = torch.zeros(0).cuda() # 2. Predict tracks num_tracks = 0 nms_inp_reg = torch.zeros(0).cuda() if len(self.tracks): # 2.1 Align if self.do_align: self.align(blob) # 2.2 Apply motion model if self.motion_model_cfg['enabled']: self.motion() self.tracks = [t for t in self.tracks if t.has_positive_area()] # 2.3 Regress person_scores = self.regress_tracks(blob) if len(self.tracks): # nms here if tracks overlap keep = nms(self.get_pos(), person_scores, self.regression_nms_thresh) self.tracks_to_inactive([ self.tracks[i] for i in list(range(len(self.tracks))) if i not in keep ]) if keep.nelement() > 0: if self.do_reid: new_features = self.get_appearances(blob) self.add_features(new_features) # 3. Create new tracks # !!! Here NMS is used to filter out detections that are already covered by tracks. This is # !!! done by iterating through the active tracks one by one, assigning them a bigger score # !!! than 1 (maximum score for detections) and then filtering the detections with NMS. # !!! In the paper this is done by calculating the overlap with existing tracks, but the # !!! result stays the same. if det_pos.nelement() > 0: keep = nms(det_pos, det_scores, self.detection_nms_thresh) det_pos = det_pos[keep] det_scores = det_scores[keep] # Check with every track in a single run (problem if tracks delete each other) for t in self.tracks: nms_track_pos = torch.cat([t.pos, det_pos]) nms_track_scores = torch.cat( [torch.tensor([2.0]).to(det_scores.device), det_scores]) keep = nms(nms_track_pos, nms_track_scores, self.detection_nms_thresh) keep = keep[torch.ge(keep, 1)] - 1 det_pos = det_pos[keep] det_scores = det_scores[keep] if keep.nelement() == 0: break if det_pos.nelement() > 0: new_det_pos = det_pos new_det_scores = det_scores # Try to reidentify tracks new_det_pos, new_det_scores, new_det_features = self.reid( blob, new_det_pos, new_det_scores) if new_det_pos.nelement() > 0: self.add(new_det_pos, new_det_scores, new_det_features) # 4. Generate results for t in self.tracks: if t.id not in self.results.keys(): self.results[t.id] = {} self.results[t.id][self.im_index] = np.concatenate([ t.pos[0].clone().cpu().numpy(), np.array([t.score.clone().cpu().numpy().item()]) ]) for t in self.inactive_tracks: t.count_inactive += 1 self.inactive_tracks = [ t for t in self.inactive_tracks if t.has_positive_area() and t.count_inactive <= self.inactive_patience ] self.im_index += 1 self.last_image = blob['img'][0]
def Generate_SQL(cond_op_score, cond_col_score, cond_num_score, cond_str_out, predicted_gate, truth_seq, SQL_YN, fp, args=None): t_fp, p_fp, sql_fp = fp batch_size = predicted_gate.size(0) sigm = nn.Sigmoid() cond_col_prob = sigm(cond_col_score) sigmoid_matrix = torch.ones_like(cond_col_prob) * 0.5 predicted_cond_col = torch.gt(cond_col_prob, sigmoid_matrix).type(torch.cuda.FloatTensor) for i in range(batch_size): last_index = predicted_gate[i].size(0) - 1 output_query = 'None' pred = 'None' # # Write ground truth answer # truth_seq_i = unicodedata.normalize('NFKD', truth_seq[i]).encode('ascii','ignore') # simple_truth_sql = translate_query_to_simple(truth_seq_i) # t_fp.write(str(SQL_YN[i].data.item()) + ' | ' + truth_seq_i + ' | ' + str(simple_truth_sql) + '\n') if predicted_gate[i, last_index] == 1: # <t2> : gate open col_list = predicted_cond_col[i, last_index] output_query = 'SELECT * FROM Airdialogue_Table WHERE ' pred = '' for c in range(11): if col_list[c] == 1: _, prediction = torch.max( cond_str_out[c][i][last_index].unsqueeze(0).data, 1) pred += str(prediction.item()) + ' ' else: pred += str(-1) + ' ' # None if col_list[11] == 1: pred += str(0) + ' ' else: pred += str(-1) + ' ' if col_list[0] == 1: _, departure_airport = torch.max( cond_str_out[0][i][last_index].unsqueeze(0).data, 1) output_query = output_query + "departure_airport" _, op = torch.max( cond_op_score[i][last_index][0].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(departure_airport.item()) if col_list[1] == 1: _, return_airport = torch.max( cond_str_out[1][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND return_airport" _, op = torch.max( cond_op_score[i][last_index][1].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(return_airport.item()) if col_list[2] == 1: _, departure_month = torch.max( cond_str_out[2][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND departure_month" _, op = torch.max( cond_op_score[i][last_index][2].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(departure_month.item()) if col_list[3] == 1: _, return_month = torch.max( cond_str_out[3][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND return_month" _, op = torch.max( cond_op_score[i][last_index][3].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(return_month.item()) if col_list[4] == 1: _, departure_day = torch.max( cond_str_out[4][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND departure_day" _, op = torch.max( cond_op_score[i][last_index][4].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(departure_day.item()) if col_list[5] == 1: _, return_day = torch.max( cond_str_out[5][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND return_day" _, op = torch.max( cond_op_score[i][last_index][5].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(return_day.item()) if col_list[6] == 1: _, departure_time_num = torch.max( cond_str_out[6][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND departure_time_num" _, op = torch.max( cond_op_score[i][last_index][6].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(departure_time_num.item()) if col_list[7] == 1: _, return_time_num = torch.max( cond_str_out[7][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND return_time_num" _, op = torch.max( cond_op_score[i][last_index][7].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(return_time_num.item()) if col_list[8] == 1: _, class_ = torch.max( cond_str_out[8][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND class" _, op = torch.max( cond_op_score[i][last_index][8].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(class_.item()) if col_list[9] == 1: _, price = torch.max( cond_str_out[9][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND price" _, op = torch.max( cond_op_score[i][last_index][9].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(price.item()) if col_list[10] == 1: _, num_connections = torch.max( cond_str_out[10][i][last_index].unsqueeze(0).data, 1) output_query = output_query + " AND num_connections" _, op = torch.max( cond_op_score[i][last_index][10].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(num_connections.item()) if col_list[11] == 1: output_query = output_query + " AND airline_preference" _, op = torch.max( cond_op_score[i][last_index][11].unsqueeze(0).data, 1) if op.item() == 0: output_query = output_query + " = " if op.item() == 1: output_query = output_query + " <= " output_query = output_query + str(0) # p_fp.write(str(int(predicted_gate[i, last_index].data.item())) + ' | ' + output_query + ' | ' + str(pred) + '\n') # sql_fp.write(output_query + '\n') return output_query, str(pred), str( int(predicted_gate[i, last_index].data.item()))
normal_entropy = FixedNormal.entropy FixedNormal.entropy = lambda self: normal_entropy(self).sum(-1) FixedNormal.mode = lambda self: self.mean # Bernoulli FixedBernoulli = torch.distributions.Bernoulli log_prob_bernoulli = FixedBernoulli.log_prob FixedBernoulli.log_probs = lambda self, actions: log_prob_bernoulli( self, actions).view(actions.size(0), -1).sum(-1).unsqueeze(-1) bernoulli_entropy = FixedBernoulli.entropy FixedBernoulli.entropy = lambda self: bernoulli_entropy(self).sum(-1) FixedBernoulli.mode = lambda self: torch.gt(self.probs, 0.5).float() class Categorical(nn.Module): def __init__(self, num_inputs, num_outputs, device): super(Categorical, self).__init__() self.device = device init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init.constant_(x, 0), gain=0.01) self.linear = init_(nn.Linear(num_inputs, num_outputs)) def forward(self, x):
print(torch.sum(a, 0)) # tensor([5, 7, 9]) print(torch.sum(a, 1)) # tensor([ 6, 15]) print(a.sum(1)) # sum 中 dim 是几, 就是将对应的维度消掉 ## max compare argsort print(b.argsort(1)) # tensor([[0, 2, 1], [0, 2, 1]]) print(b.argsort(1)[:, -2:]) # tensor([[2, 1], [2, 1]]) print(b.max(1)) # (tensor([3, 6]), tensor([1, 1])); max only for the maximum. ## gt(input, other, out=None) # input (Tensor): the tensor to compare # other (Tensor or float): the tensor or value to compare # out (Tensor, optional): the output tensor that must be a `ByteTensor` a = torch.tensor([[1, 2], [3, 4]]) b = torch.tensor([[1, 1], [4, 4]]) print(torch.gt(a, b)) # tensor([[0, 1], [0, 0]], dtype=torch.uint8) print(a.gt(1.5)) # tensor([[0, 1], [1, 1]], dtype=torch.uint8) ## eq, ne, masked_select gold = torch.Tensor([[1, 2], [3, 0]]) pred = torch.Tensor([[1, 3], [3, 4]]) print(gold.ne(0)) # tensor([[1, 1], [1, 0]], dtype=torch.uint8) print(gold.eq(0)) # tensor([[0, 0], [0, 1]], dtype=torch.uint8) non_pad_mask = gold.ne(0); print(not_pad_mask) # tensor([[1, 1], [1, 0]], dtype=torch.uint8) n_correct = pred.eq(gold); print(n_correct) # tensor([[1, 0], [1, 0]], dtype=torch.uint8) n_correct = n_correct.masked_select(non_pad_mask); print(n_correct) # tensor([1, 0, 1], dtype=torch.uint8) print(n_correct.sum()) # tensor(2) ## prod print(torch.prod(torch.tensor([1, 2, 3]))) # tensor(6)