def test_log(test_case):
input = flow.Tensor(np.random.randn(2, 3, 4, 5), dtype=flow.float32)
of_out = flow.log(input)
np_out = np.log(input.numpy())
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
test_case.assertTrue(
np.allclose(input.log().numpy(), np_out, equal_nan=True))
arr = np.array([-0.7168, -0.5471, -0.8933, -1.4428, -0.1190])
input2 = flow.Tensor(arr, dtype=flow.float32)
np_out = np.full((5, ), np.nan)
of_out2 = flow.log(input2)
test_case.assertTrue(
np.allclose(of_out2.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def test_log_nan_value(test_case):
arr = np.array([-0.7168, -0.5471, -0.8933, -1.4428, -0.1190])
input = flow.Tensor(arr, dtype=flow.float32)
np_out = np.full((5, ), np.nan)
of_out = flow.log(input)
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def cal_loss(pred, gold, smoothing=0.0):
"""Calculate cross entropy loss, apply label smoothing if needed.
"""
if smoothing > 0.0:
eps = smoothing
n_class = pred.size(1)
# Generate one-hot matrix: N x C.
# Only label position is 1 and all other positions are 0
# gold include -1 value (IGNORE_ID) and this will lead to assert error
gold_for_scatter = gold.ne(IGNORE_ID).long() * gold
one_hot = F.one_hot(gold_for_scatter, n_class).to(dtype=flow.float32)
one_hot = one_hot * (1 - eps) + (1 - one_hot) * eps / n_class
sof_prb = F.softmax(pred)
log_prb = flow.log(sof_prb)
non_pad_mask = gold.ne(IGNORE_ID)
n_word = float(non_pad_mask.sum().numpy())
loss = -(one_hot * log_prb).sum(dim=1)
loss = loss.masked_select(non_pad_mask).sum() / n_word
else:
loss_fn = nn.CrossEntropyLoss(ignore_index=IGNORE_ID).to(pred.device)
loss = loss_fn(pred, gold)
return loss
def test_log(test_case):
input = flow.Tensor(np.random.randn(2, 3, 4, 5), dtype=flow.float32)
of_out = flow.log(input)
np_out = np.log(input.numpy())
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
test_case.assertTrue(
np.allclose(input.log().numpy(), np_out, equal_nan=True))
def forward(self, input, target, weight=None):
assert (input.shape == target.shape
), "The Input shape must be the same as Target shape"
_cross_entropy_loss = flow.negative(target * flow.log(input) +
(1 - target) * flow.log(1 - input))
if weight is not None:
assert (weight.shape == input.shape
), "The weight shape must be the same as Input shape"
_weighted_loss = weight * _cross_entropy_loss
else:
_weighted_loss = _cross_entropy_loss
if self.reduction == "mean":
return flow.mean(_weighted_loss)
elif self.reduction == "sum":
return flow.sum(_weighted_loss)
else:
return _weighted_loss
def __call__(self, outputs, targets):
loss = (1 - self.jaccard_weight) * self.nll_loss(outputs, targets)
if self.jaccard_weight:
eps = 1e-15
jaccard_target = flow.Tensor(
(targets.numpy() == 1)).to(flow.device("cuda"))
jaccard_output = flow.sigmoid(outputs)
intersection = (jaccard_output * jaccard_target).sum()
union = jaccard_output.sum() + jaccard_target.sum()
loss -= self.jaccard_weight * flow.log(
(intersection + eps) / (union - intersection + eps))
return loss
def forward(self, x):
need_transpose, permute = _softmax_need_transpose(x, self.dim)
if need_transpose:
x = x.transpose(perm=permute)
x = flow.softmax(x)
res = flow.log(x)
if need_transpose:
res = res.transpose(perm=permute)
return res
def forward(self, predicted, target):
# ------------ AM Softmax ------------ #
predicted = predicted / (predicted.norm(dim=0) + self.epsilon)
indexes = flow.Tensor(range(predicted.size(0))).long().to(
predicted.device)
cos_theta_y = predicted[indexes, target]
cos_theta_y_m = cos_theta_y - self.m
exp_s = (flow.ones_like(cos_theta_y_m) * np.e)**(self.s *
cos_theta_y_m)
sum_cos_theta_j = ((flow.ones_like(predicted) * np.e)
**(predicted * self.s)).sum(dim=1) - (
(flow.ones_like(predicted[indexes, target]) *
np.e)**(predicted[indexes, target] * self.s))
log = -flow.log(exp_s /
(exp_s + sum_cos_theta_j + self.epsilon)).mean()
return log
def sisnr(self, x, s, eps=1e-8):
"""
Arguments:
x: separated signal, N x S tensor
s: reference signal, N x S tensor
Return:
sisnr: N tensor
"""
def l2norm(mat, keepdim=False):
return flow.linalg.norm(mat, dim=-1, keepdim=keepdim)
if x.shape != s.shape:
raise RuntimeError(
"Dimention mismatch when calculate si-snr, {} vs {}".format(
x.shape, s.shape))
x_zm = x - flow.mean(x, dim=-1, keepdim=True)
s_zm = s - flow.mean(s, dim=-1, keepdim=True)
t = (flow.sum(x_zm * s_zm, dim=-1, keepdim=True) * s_zm /
(l2norm(s_zm, keepdim=True)**2 + eps))
res = 20 * flow.log(eps + l2norm(t) /
(l2norm(x_zm - t) + eps)) / 2.3025851
return res
def _log(self):
return flow.log(self)
def forward(self, x):
return flow.log(x)
def recognize_beam(self, encoder_outputs, char_list, args):
"""
Beam search, decode one utterence now.
Args:
encoder_outputs: T x H #418 x 512
char_list: list of character #4233
args: args.beam #5
Returns:
nbest_hyps:
"""
# search params
beam = args.beam_size
nbest = args.nbest
if args.decode_max_len == 0:
maxlen = encoder_outputs.size(0)
else:
maxlen = args.decode_max_len
encoder_outputs = encoder_outputs.unsqueeze(0)
# prepare sos
ys = flow.ones(1, 1).fill_(self.sos_id).type_as(encoder_outputs).long()
hyp = {"score": 0.0, "yseq": ys}
hyps = [hyp]
ended_hyps = []
for i in range(maxlen):
hyps_best_kept = []
for hyp in hyps:
ys = hyp["yseq"]
ys = ys.to(device=encoder_outputs.device)
# -- Prepare masks
non_pad_mask = flow.ones_like(ys).to(
dtype=flow.float32).unsqueeze(-1)
slf_attn_mask = get_subsequent_mask(ys)
# -- Forward
dec_output = self.dropout(
self.tgt_word_emb(ys) * self.x_logit_scale +
self.positional_encoding(ys))
for dec_layer in self.layer_stack:
dec_output, _, _ = dec_layer(
dec_output,
encoder_outputs,
non_pad_mask=non_pad_mask,
slf_attn_mask=slf_attn_mask,
dec_enc_attn_mask=None,
)
seq_logit = self.tgt_word_prj(dec_output[:, -1])
local_logit = F.softmax(seq_logit)
local_scores = flow.log(local_logit)
# topk scores
local_best_scores, local_best_ids = flow.topk(local_scores,
beam,
dim=1)
for j in range(beam):
new_hyp = {}
new_hyp["score"] = hyp["score"] + local_best_scores[0, j]
new_hyp["yseq"] = (flow.ones(
1, (1 + ys.size(1))).type_as(encoder_outputs).long())
new_hyp["yseq"][:, :ys.size(1)] = hyp["yseq"]
new_hyp["yseq"][:, ys.size(1)] = int(
float(local_best_ids[0, j].numpy()))
hyps_best_kept.append(new_hyp)
hyps_best_kept = sorted(hyps_best_kept,
key=lambda x: x["score"],
reverse=True)[:beam]
# end for hyp in hyps
hyps = hyps_best_kept
# add eos in the final loop to avoid that there are no ended hyps
if i == maxlen - 1:
for hyp in hyps:
hyp["yseq"] = flow.cat(
[
hyp["yseq"],
flow.ones(1, 1).fill_(
self.eos_id).type_as(encoder_outputs).long(),
],
dim=1,
)
# add ended hypothes to a final list, and removed them from current hypothes
# (this will be a probmlem, number of hyps < beam)
remained_hyps = []
for hyp in hyps:
if hyp["yseq"][0, -1] == self.eos_id:
ended_hyps.append(hyp)
else:
remained_hyps.append(hyp)
hyps = remained_hyps
if len(hyps) > 0:
print("remeined hypothes: " + str(len(hyps)))
else:
print("no hypothesis. Finish decoding.")
break
for hyp in hyps:
print("hypo: " + "".join(
[char_list[int(x.numpy())] for x in hyp["yseq"][0, 1:]]))
nbest_hyps = sorted(ended_hyps, key=lambda x: x["score"],
reverse=True)[:min(len(ended_hyps), nbest)]
for hyp in nbest_hyps:
hyp["yseq"] = hyp["yseq"][0].cpu().numpy().tolist()
return nbest_hyps
def forward(self, x):
return flow.where(
x * self.beta > self.threshold,
x,
1 / self.beta * flow.log(1.0 + flow.exp(self.beta * x)),
)