def evaluate(self):
acc = torch.diag(self.hist).sum() / (self.hist.sum() + self.eps)
acc_cls = torch.diag(self.hist) / (self.hist.sum(axis=1) + +self.eps)
acc_cls = torch.nansum(acc_cls) / len(acc_cls)
iu = torch.diag(self.hist) / (self.hist.sum(axis=1) + self.hist.sum(axis=0) - torch.diag(self.hist) + self.eps)
mean_iu = torch.nansum(iu) / len(iu)
freq = self.hist.sum(axis=1) / (self.hist.sum() + self.eps)
fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
return acc, acc_cls, iu, mean_iu, fwavacc
def constrain_weight(self, weight_arr):
square_weight_arr = weight_arr * weight_arr
while torch.nansum(square_weight_arr) > self.weight_limit:
weight_arr = weight_arr * 0.9
square_weight_arr = weight_arr * weight_arr
return weight_arr
def elbo(self, mean, var, y):
y_masked = y.masked_fill(y.isnan(), 0.)
v = self.log_var.exp()
elbo = (var + mean**2 - 2. * y_masked * mean + y_masked**2) / v
elbo += self.log_var + np.log(2. * np.pi)
elbo.masked_fill(y.isnan(), 0.)
return -0.5 * torch.nansum(elbo)
def reduction_ops(self):
a = torch.randn(4)
b = torch.randn(4)
return (
torch.argmax(a),
torch.argmin(a),
torch.amax(a),
torch.amin(a),
torch.aminmax(a),
torch.all(a),
torch.any(a),
torch.max(a),
torch.min(a),
torch.dist(a, b),
torch.logsumexp(a, 0),
torch.mean(a),
torch.nanmean(a),
torch.median(a),
torch.nanmedian(a),
torch.mode(a),
torch.norm(a),
torch.nansum(a),
torch.prod(a),
torch.quantile(a, torch.tensor([0.25, 0.5, 0.75])),
torch.nanquantile(a, torch.tensor([0.25, 0.5, 0.75])),
torch.std(a),
torch.std_mean(a),
torch.sum(a),
torch.unique(a),
torch.unique_consecutive(a),
torch.var(a),
torch.var_mean(a),
torch.count_nonzero(a),
)
def compute_cams(self,
class_idx: int,
scores: Optional[Tensor] = None,
normalized: bool = True) -> Tensor:
"""Compute the CAM for a specific output class
Args:
class_idx (int): output class index of the target class whose CAM will be computed
scores (torch.Tensor[1, K], optional): forward output scores of the hooked model
normalized (bool, optional): whether the CAM should be normalized
Returns:
torch.Tensor[M, N]: class activation map of hooked conv layer
"""
# Get map weight & unsqueeze it
weights = self._get_weights(class_idx, scores)
weights = weights[
(..., ) + (None, ) *
(self.hook_a.ndim - 2)] # type: ignore[operator, union-attr]
# Perform the weighted combination to get the CAM
batch_cams = torch.nansum(weights * self.hook_a.squeeze(0),
dim=0) # type: ignore[union-attr]
if self._relu:
batch_cams = F.relu(batch_cams, inplace=True)
# Normalize the CAM
if normalized:
batch_cams = self._normalize(batch_cams)
return batch_cams
def log_likelihood(self, proba=None, A=None):
nll = torch.tensor(0.).cuda()
if (A is not None) and (proba is not None):
A = A.unsqueeze(1)
llk = A * torch.log(proba) + (1. - A) * torch.log(1. - proba)
nll += -torch.nansum(llk) / llk.size(1)
return -nll
def kl_divergence(log_prediction: Tensor, log_target: Tensor) -> Tensor:
sum_dim = [i for i in range(1, log_prediction.ndim)]
return torch.nansum(F.kl_div(log_prediction,
log_target,
reduction='none',
log_target=True),
dim=sum_dim)
def _cdist_nb_full(self, x, cutoff=9.0, mask=False):
dmat = torch.cdist(x.permute(0, 2, 1), x.permute(0, 2, 1))
dmat6 = (self._warp_domain(dmat, 1.9)**6)
LJpB = self.vdw_B / dmat6
LJpA = self.vdw_A / (dmat6**2)
Cp = (self.q1q2 / self._warp_domain(dmat, 0.4))
return torch.nansum(LJpA - LJpB + Cp)
def forward(self, g):
# input embedding
h = self.embedding_h(g.ndata['feat'])
e = self.embedding_e_real(g.edata['feat'])
PosEnc = g.ndata['pos_enc'] # (Num nodes) x (Num Eigenvectors) x 2
empty_mask = torch.isnan(
PosEnc) # (Num nodes) x (Num Eigenvectors) x 2
PosEnc[empty_mask] = 0 # (Num nodes) x (Num Eigenvectors) x 2
PosEnc = torch.transpose(
PosEnc, 0, 1).float() # (Num Eigenvectors) x (Num nodes) x 2
PosEnc = self.linear_A(
PosEnc) # (Num Eigenvectors) x (Num nodes) x PE_dim
# 1st Transformer: Learned PE
PosEnc = self.PE_Transformer(src=PosEnc,
src_key_padding_mask=empty_mask[:, :, 0])
# remove masked sequences
PosEnc[torch.transpose(empty_mask, 0, 1)[:, :, 0]] = float('nan')
# Sum pooling
PosEnc = torch.nansum(PosEnc, 0, keepdim=False)
# Concatenate learned PE to input embedding
h = torch.cat((h, PosEnc), 1)
h = self.in_feat_dropout(h)
# Second Transformer
for conv in self.layers:
h, e = conv(g, h, e)
g.ndata['feat'] = h
def forward(self, x):
self._check_init()
inputs = self.drop_(-(x.unsqueeze(dim=2) - self.centers) ** 2 * (self.h / self.sigmas ** 2 + self.eps))
inputs = torch.nansum(
inputs, dim=1
)
frs = F.softmax(inputs, dim=1)
return frs
def jensen_shannon_divergence(p, log_p: Tensor, q, log_q: Tensor) -> Tensor:
sum_dim = [i for i in range(1, log_p.ndim)]
with torch.no_grad():
log_m = torch.log((p + q) / 2)
kl_p = F.kl_div(log_m, log_p, reduction='none', log_target=True)
kl_q = F.kl_div(log_m, log_q, reduction='none', log_target=True)
return torch.nansum(kl_p + kl_q, dim=sum_dim) / 2
def log_likelihood(self,
mean_cts=None,
X_cts=None,
proba_bin=None,
X_bin=None):
nll = torch.tensor(0.).cuda()
if (X_cts is not None) and (mean_cts is not None):
X_cts = X_cts.unsqueeze(1)
var = torch.exp(self.cts_logvar)
X_cts_masked = X_cts.masked_fill(X_cts.isnan(), 0.)
llk = (mean_cts - X_cts_masked)**2 / (2. * var) + 0.5 * torch.log(
2. * math.pi * var)
llk = llk.masked_fill(X_cts.isnan(), 0.)
nll += torch.nansum(llk) / llk.size(-1)
if (X_bin is not None) and (proba_bin is not None):
X_bin = X_bin.unsqueeze(1)
X_bin_masked = X_bin.masked_fill(X_bin.isnan(), 0.)
llk = X_bin_masked * torch.log(proba_bin) + (
1. - X_bin_masked) * torch.log(1. - proba_bin)
llk = llk.masked_fill(X_bin.isnan(), 0.)
nll += -torch.nansum(llk) / llk.size(-1)
return -nll
def mean_average_precision(self, y_que, X_que, y_pool, X_pool):
if self.rank == None:
raise ("rank function is not provided.")
n_que = len(y_que)
n_pool = len(y_pool)
ap = th.zeros(n_que, device=self.device)
for i in range(n_que):
y = y_que[i]
ranks = self.rank(X_que[i], X_pool)
rel = y_pool[ranks] == y
pre_k = th.cumsum(rel, dim=0) / th.arange(
1, n_pool + 1, device=self.device)
ap[i] = th.divide(th.sum(pre_k * rel), th.sum(rel))
return th.nansum(ap) / (n_que - th.sum(th.isnan(ap)))
def area_under_the_curve(self, y_que, X_que, y_pool, X_pool):
if self.rank == None:
raise ("rank function is not provided.")
n_que = len(y_que)
auc = th.zeros(n_que, device=self.device)
for i in range(n_que):
y = y_que[i]
ranks = self.rank(X_que[i], X_pool)
y_count = th.sum(y_pool == y)
swapped_pairs = th.sum(
(y_pool[ranks] != y) *
(y_count - th.cumsum(y_pool[ranks] == y, dim=0)))
auc[i] = 1 - swapped_pairs / (y_count * (len(y_pool) - y_count))
return th.nansum(auc) / (n_que - th.sum(th.isnan(auc)))
def _cdist_nb(self, x, cutoff=9.0, mask=False):
dmat = torch.cdist(x.permute(0, 2, 1), x.permute(0, 2, 1))
LJp = self.vdw_A / (self._warp_domain(dmat, 1.9)**12)
Cp = (self.q1q2 / self._warp_domain(dmat, 0.4))
return torch.nansum(LJp + Cp)
def product(self, dim, keepdim=False):
return Gaussian(
torch.nansum(self._precision, dim, keepdim=keepdim),
torch.nansum(self._mean_times_precision, dim, keepdim=keepdim)
)
def compute(self) -> torch.Tensor:
avg_precision = torch.Tensor(super().compute())
total = torch.nansum(avg_precision)
count = (~torch.isnan(avg_precision)).sum()
return total / count if count != 0 else torch.Tensor(
[0.], device=avg_precision.device)
print(
torch.bernoulli(a)
) # draw binary random numbers (0 or 1) from a Bernoulli distribution
weights = torch.tensor([0, 10, 3, 0],
dtype=torch.float) # create a tensor of weights
print(torch.multinomial(weights, 2))
x = torch.randn(3)
print(x)
print(torch.mean(x))
print(torch.sum(x))
print(torch.median(x))
# print(torch.nanmedian(x)) # ignoring NaN values
print(torch.min(x))
print(torch.max(x))
print(torch.mode(x))
print(torch.std(x))
print(torch.var(x))
print(torch.quantile(x, 0.1))
print(x.nanquantile(0.1))
print(torch.nansum(x)) # treating NaN as zero
print(torch.unique(torch.tensor([1, 3, 2, 3], dtype=torch.long)))
# count the frequency of each value in an array of non-negative int
print(torch.bincount(torch.randint(0, 8, (5, ), dtype=torch.int64)))
print(torch.histc(torch.tensor([1., 2, 1]), bins=4, min=0, max=3))
x = torch.zeros(3, 3)
x[torch.randn(3, 3) > 0.5] = 1
print(torch.count_nonzero(x))
def elbo(self, mean, var, y):
y_masked = y.masked_fill(y.isnan(), 0.)
t = torch.sqrt(mean**2 + var)
elbo = nn.LogSigmoid()(t) + (y_masked - 0.5) * mean - 0.5 * t
elbo.masked_fill(y.isnan(), 0.)
return torch.nansum(elbo)
def forward(
self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
relevance_score=None,
labels=None,
mlm_mask=None,
output_attentions=None,
output_hidden_states=None,
return_dict=None,
):
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
(
flattened_input_ids,
flattened_attention_mask,
flattened_token_type_ids,
) = self._flatten_inputs(input_ids, attention_mask, token_type_ids)
joint_outputs = self.realm(
flattened_input_ids,
attention_mask=flattened_attention_mask,
token_type_ids=flattened_token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
)
# [batch_size * num_candidates, joint_seq_len, d_model]
joint_output = joint_outputs[0]
# [batch_size * num_candidates, joint_seq_len, s_vocab]
prediction_scores = self.cls(joint_output)
# [batch_size, num_candidates]
candidate_score = relevance_score
masked_lm_loss = None
if labels is not None:
if candidate_score is None:
raise ValueError(
"You have to specify `relevance_score` when `labels` is specified in order to compute loss."
)
batch_size, seq_length = labels.size()
if mlm_mask is None:
mlm_mask = torch.ones_like(labels, dtype=torch.float32)
else:
mlm_mask = mlm_mask.type(torch.float32)
# Compute marginal log-likelihood
loss_fct = CrossEntropyLoss(
reduction="none") # -100 index = padding token
# [batch_size * num_candidates * joint_seq_len, s_vocab]
mlm_logits = prediction_scores.view(-1, self.config.s_vocab)
# [batch_size * num_candidates * joint_seq_len]
mlm_targets = labels.tile(1, self.config.num_candidates).view(-1)
# [batch_size, num_candidates, joint_seq_len]
masked_lm_log_prob = -loss_fct(mlm_logits, mlm_targets).view(
batch_size, self.config.num_candidates, seq_length)
# [batch_size, num_candidates, 1]
candidate_log_prob = candidate_score.log_softmax(-1).unsqueeze(-1)
# [batch_size, num_candidates, joint_seq_len]
joint_gold_log_prob = candidate_log_prob + masked_lm_log_prob
# [batch_size, joint_seq_len]
marginal_gold_log_probs = joint_gold_log_prob.logsumexp(1)
# []
masked_lm_loss = -torch.nansum(
torch.sum(marginal_gold_log_probs * mlm_mask) /
torch.sum(mlm_mask))
if not return_dict:
output = (prediction_scores, ) + joint_outputs[2:4]
return ((masked_lm_loss, ) +
output) if masked_lm_loss is not None else output
return MaskedLMOutput(
loss=masked_lm_loss,
logits=prediction_scores,
hiddens=joint_outputs.hiddens,
attns=joint_outputs.attns,
)
