def process_scores_( self, hrt_batch: MappedTriples, target: Target, scores: torch.FloatTensor, true_scores: Optional[torch.FloatTensor] = None, dense_positive_mask: Optional[torch.FloatTensor] = None, ) -> None: # noqa: D102 if dense_positive_mask is None: raise KeyError("Sklearn evaluators need the positive mask!") # Transfer to cpu and convert to numpy scores = scores.detach().cpu().numpy() dense_positive_mask = dense_positive_mask.detach().cpu().numpy() remaining = [i for i in range(hrt_batch.shape[1]) if i != TARGET_TO_INDEX[target]] keys = hrt_batch[:, remaining].detach().cpu().numpy() # Ensure that each key gets counted only once for i in range(keys.shape[0]): # include head_side flag into key to differentiate between (h, r) and (r, t) key_suffix = tuple(map(int, keys[i])) assert len(key_suffix) == 2 key_suffix = cast(Tuple[int, int], key_suffix) key = (target,) + key_suffix self.all_scores[key] = scores[i] self.all_positives[key] = dense_positive_mask[i]
def process_slcwa_scores( self, positive_scores: torch.FloatTensor, negative_scores: torch.FloatTensor, label_smoothing: Optional[float] = None, batch_filter: Optional[torch.BoolTensor] = None, num_entities: Optional[int] = None, ) -> torch.FloatTensor: # noqa: D102 # Sanity check if label_smoothing: raise UnsupportedLabelSmoothingError(self) if batch_filter is not None: # negative_scores have already been filtered in the sampler! # (dense) softmax requires unfiltered scores / masking negative_scores_ = torch.zeros_like(batch_filter, dtype=positive_scores.dtype) negative_scores_[batch_filter] = negative_scores # we need to fill the scores with -inf for all filtered negative examples # EXCEPT if all negative samples are filtered (since softmax over only -inf yields nan) fill_mask = ~batch_filter fill_mask = fill_mask & ~(fill_mask.all(dim=1, keepdim=True)) negative_scores_[fill_mask] = float("-inf") # use filled negatives scores negative_scores = negative_scores_ # compute weights (without gradient tracking) weights = negative_scores.detach().mul( self.inverse_softmax_temperature).softmax(dim=-1) return self( pos_scores=positive_scores, neg_scores=negative_scores, neg_weights=weights, )
def process_lcwa_scores( self, predictions: torch.FloatTensor, labels: torch.FloatTensor, label_smoothing: Optional[float] = None, num_entities: Optional[int] = None, ) -> torch.FloatTensor: # noqa: D102 # Sanity check if label_smoothing: raise UnsupportedLabelSmoothingError(self) pos_mask = labels == 1 # compute negative weights (without gradient tracking) # clone is necessary since we modify in-place weights = predictions.detach().clone() weights[pos_mask] = float("-inf") weights = weights.mul(self.inverse_softmax_temperature).softmax(dim=1) # Split positive and negative scores positive_scores = predictions[pos_mask] negative_scores = predictions[~pos_mask] return self( pos_scores=positive_scores, neg_scores=negative_scores, neg_weights=weights[~pos_mask], )
def _to_labels_n_scores( predictions:torch.FloatTensor, label:int )->List[Tuple[float, int]]: return [ (label, score) for score in predictions.detach().cpu().numpy() ]
def xtrans(self, Xc: FloatTensor, Xe: LongTensor) -> np.ndarray: if self.num_enum == 0: return Xc.detach().numpy() else: Xe_one_hot = self.one_hot(Xe) if Xc is None: Xc = torch.zeros(Xe.shape[0], 0) return torch.cat([Xc, Xe_one_hot], dim=1).numpy()
def _process_scores( self, keys: torch.LongTensor, scores: torch.FloatTensor, positive_mask: torch.FloatTensor, head_side: bool, ) -> None: # Transfer to cpu and convert to numpy scores = scores.detach().cpu().numpy() positive_mask = positive_mask.detach().cpu().numpy() keys = keys.detach().cpu().numpy() # Ensure that each key gets counted only once for i in range(keys.shape[0]): # include head_side flag into key to differentiate between (h, r) and (r, t) key = (head_side, ) + tuple(map(int, keys[i])) self.all_scores[key] = scores[i] self.all_positives[key] = positive_mask[i]
def fit(self, x: torch.FloatTensor): assert (x.dim() == 2) with torch.no_grad(): scaler = MinMaxScaler((self.range_lb, self.range_ub)) scaler.fit(x.detach().numpy()) self.scale_ = torch.FloatTensor(scaler.scale_) self.min_ = torch.FloatTensor(scaler.min_) self.fitted = True return self
def fit(self, x: torch.FloatTensor): assert (x.dim() == 2) with torch.no_grad(): scaler = StandardScaler() scaler.fit(x.detach().numpy()) self.mean = torch.FloatTensor(scaler.mean_.copy()).view(-1) self.std = torch.FloatTensor(scaler.scale_.copy()).view(-1) invalid = ~(torch.isfinite(self.mean) & torch.isfinite(self.std)) self.mean[ invalid] = 0. # somethime we face data with some all-NaN columns self.std[invalid] = 1. return self
def get_saliency_map(self, _input: torch.FloatTensor, _class: list[int]) -> torch.Tensor: if isinstance(_class, int): _class = [_class] * len(_input) _class: torch.Tensor = torch.tensor(_class).to(_input.device) x: torch.FloatTensor = _input.detach() x.requires_grad_() _output: torch.FloatTensor = self(x) _output: torch.FloatTensor = _output.gather( dim=1, index=_class.unsqueeze(1)).sum() grad: torch.FloatTensor = torch.autograd.grad(_output, x)[0] # (N,C,H,W) x.requires_grad_(False) heatmap = grad.clamp(min=0).max(dim=1)[0] # (N,H,W) heatmap.sub_( heatmap.min(dim=-2, keepdim=True)[0].min(dim=-1, keepdim=True)[0]) heatmap.div_( heatmap.max(dim=-2, keepdim=True)[0].max(dim=-1, keepdim=True)[0]) return heatmap
def torch_to_tf(torch_tensor: torch.FloatTensor) -> tf.Tensor: torch_tensor = torch_tensor.permute([0, 2, 3, 4, 1]) # channels last np_tensor = torch_tensor.detach().cpu().numpy() tf_tensor = tf.convert_to_tensor(np_tensor) return tf_tensor
class SGNS(nn.Module): def __init__(self, embedding, vocab_size=20000, n_negs=20, weights=None, tie_weights=False, fake_indices=None): super(SGNS, self).__init__() self.embedding = embedding self.vocab_size = vocab_size self.n_negs = n_negs self.weights = None if weights is not None: wf = np.power(weights, 0.75) wf = wf / wf.sum() self.weights = FT(wf) self.tie_weights = tie_weights if weights is not None and fake_indices is not None: is_fake = t.zeros(4000).type(t.bool) is_fake[t.LongTensor(list(fake_indices))] = True # adjust weights here and zero them out self.weights_real = self.weights.detach().clone() self.weights_real[is_fake] = 0.0 self.weights_fake = self.weights.detach().clone() self.weights_fake[~is_fake] = 0.0 self.fake_indices = t.LongTensor(list(fake_indices)) def forward(self, iword, owords): batch_size = iword.size()[0] context_size = owords.size()[1] if self.fake_indices is None: if self.weights is not None: nwords = t.multinomial(self.weights, batch_size * context_size * self.n_negs, replacement=True).view(batch_size, -1) else: nwords = FT(batch_size, context_size * self.n_negs).uniform_( 0, self.vocab_size - 1).long() else: if self.weights is not None: # do broadcasting to check the values is_fake = iword.view(-1, 1).eq(self.fake_indices).sum(1).type( t.bool) n_fake = is_fake.sum() n_real = batch_size - n_fake # two times sampling nwords_fake = t.multinomial(self.weights_fake, n_fake * context_size * self.n_negs, replacement=True).view(n_fake, -1) nwords_real = t.multinomial(self.weights_real, n_real * context_size * self.n_negs, replacement=True).view(n_real, -1) # create empty tensor and use is_fake to assign the sampled words to it nwords = t.zeros(batch_size, context_size * self.n_negs).type(t.long) nwords[is_fake] = nwords_fake nwords[~is_fake] = nwords_real else: raise NotImplementedError() ivectors = self.embedding.forward_i(iword).unsqueeze(2) if self.tie_weights: ovectors = self.embedding.forward_i(owords) nvectors = self.embedding.forward_i(nwords).neg() else: ovectors = self.embedding.forward_o(owords) nvectors = self.embedding.forward_o(nwords).neg() oloss = t.bmm(ovectors, ivectors).squeeze().sigmoid().log().mean(1) nloss = t.bmm(nvectors, ivectors).squeeze().sigmoid().log().view( -1, context_size, self.n_negs).sum(2).mean(1) return -(oloss + nloss).mean()
def to_y(h: FloatTensor, x_rec: FloatTensor): return decide(h.detach())