def test_train(self):
self._metric.train()
calls = [[torch.FloatTensor([0.0]), torch.LongTensor([0])],
[torch.FloatTensor([0.0, 0.1, 0.2, 0.3]), torch.LongTensor([0, 1, 2, 3])]]
for i in range(len(self._states)):
self._metric.process(self._states[i])
self.assertEqual(2, len(self._metric_function.call_args_list))
for i in range(len(self._metric_function.call_args_list)):
self.assertTrue(torch.eq(self._metric_function.call_args_list[i][0][0], calls[i][0]).all)
self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[i][0][1], -calls[i][1])), 1e-12).all)
self._metric_function.reset_mock()
self._metric.process_final({})
self._metric_function.assert_called_once()
self.assertTrue(torch.eq(self._metric_function.call_args_list[0][0][1], torch.LongTensor([0, 1, 2, 3, 4])).all)
self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[0][0][0], -torch.FloatTensor([0.0, 0.1, 0.2, 0.3, 0.4]))), 1e-12).all)
def test_validate(self):
self._metric.eval()
for i in range(len(self._states)):
self._metric.process(self._states[i])
self._metric_function.assert_not_called()
self._metric.process_final_validate({})
self._metric_function.assert_called_once()
self.assertTrue(torch.eq(self._metric_function.call_args_list[0][0][1], torch.LongTensor([0, 1, 2, 3, 4])).all)
self.assertTrue(torch.lt(torch.abs(torch.add(self._metric_function.call_args_list[0][0][0], -torch.FloatTensor([0.0, 0.1, 0.2, 0.3, 0.4]))), 1e-12).all)
def __init__(self, low, high, validate_args=None):
self.low, self.high = broadcast_all(low, high)
if isinstance(low, Number) and isinstance(high, Number):
batch_shape = torch.Size()
else:
batch_shape = self.low.size()
super(Uniform, self).__init__(batch_shape, validate_args=validate_args)
if self._validate_args and not torch.lt(self.low, self.high).all():
raise ValueError("Uniform is not defined when low>= high")
def sample_with_temperature(logits, sampling_temp, keep_topk):
"""Select next tokens randomly from the top k possible next tokens.
Samples from a categorical distribution over the ``keep_topk`` words using
the category probabilities ``logits / sampling_temp``.
Args:
logits (FloatTensor): Shaped ``(batch_size, vocab_size)``.
These can be logits (``(-inf, inf)``) or log-probs (``(-inf, 0]``).
(The distribution actually uses the log-probabilities
``logits - logits.logsumexp(-1)``, which equals the logits if
they are log-probabilities summing to 1.)
sampling_temp (float): Used to scale down logits. The higher the
value, the more likely it is that a non-max word will be
sampled.
keep_topk (int): This many words could potentially be chosen. The
other logits are set to have probability 0.
Returns:
(LongTensor, FloatTensor):
* topk_ids: Shaped ``(batch_size, 1)``. These are
the sampled word indices in the output vocab.
* topk_scores: Shaped ``(batch_size, 1)``. These
are essentially ``(logits / sampling_temp)[topk_ids]``.
"""
if sampling_temp == 0.0 or keep_topk == 1:
# For temp=0.0, take the argmax to avoid divide-by-zero errors.
# keep_topk=1 is also equivalent to argmax.
topk_scores, topk_ids = logits.topk(1, dim=-1)
if sampling_temp > 0:
topk_scores /= sampling_temp
else:
logits = torch.div(logits, sampling_temp)
if keep_topk > 0:
top_values, top_indices = torch.topk(logits, keep_topk, dim=1)
kth_best = top_values[:, -1].view([-1, 1])
kth_best = kth_best.repeat([1, logits.shape[1]]).float()
# Set all logits that are not in the top-k to -10000.
# This puts the probabilities close to 0.
ignore = torch.lt(logits, kth_best)
logits = logits.masked_fill(ignore, -10000)
dist = torch.distributions.Multinomial(
logits=logits, total_count=1)
topk_ids = torch.argmax(dist.sample(), dim=1, keepdim=True)
topk_scores = logits.gather(dim=1, index=topk_ids)
return topk_ids, topk_scores
def get_unsupervised_loss(self, examples):
samples, sample_scores, enc_states = self.infer(examples)
reconstruction_scores = self.decoder.score(samples)
# compute prior probability
prior_scores = self.prior([e.tgt_code for e in samples])
if isinstance(self.prior, UniformPrior):
prior_scores = Variable(sample_scores.data.new(prior_scores))
kl_term = self.args.alpha * (sample_scores - prior_scores)
raw_learning_signal = reconstruction_scores - kl_term
baseline = self.baseline(samples, enc_states)
learning_signal = raw_learning_signal.detach() - baseline
# clip learning signal
if self.args.clip_learning_signal is not None:
mask = torch.lt(learning_signal, self.args.clip_learning_signal).float()
clipped_learning_signal = learning_signal * (1. - mask) + mask * self.args.clip_learning_signal
else:
clipped_learning_signal = learning_signal
encoder_loss = -clipped_learning_signal.detach() * sample_scores
decoder_loss = -reconstruction_scores
# compute baseline loss
baseline_loss = learning_signal ** 2
meta_data = {'samples': samples,
'reconstruction_scores': reconstruction_scores,
'encoding_scores': sample_scores,
'raw_learning_signal': raw_learning_signal,
'learning_signal': learning_signal,
'baseline': baseline,
'kl_term': kl_term,
'prior': prior_scores}
return encoder_loss, decoder_loss, baseline_loss, meta_data
def Focal_Loss(classifications, regressions, anchors, annotations):
alpha = 0.25
gamma = 2.0
batch_size = classifications.shape[0]
classification_losses = []
regression_losses = []
anchor = anchors[0, :, :]
anchor_width = anchor[:, 2] - anchor[:, 0]
anchor_height = anchor[:, 3] - anchor[:, 1]
anchor_cx = anchor[:, 0] + 0.5 * anchor_width
anchor_cy = anchor[:, 1] + 0.5 * anchor_height
for j in range(batch_size):
classification = classifications[j, :, :]
regression = regressions[j, :, :]
bbox_annotation = annotations[j, :, :]
bbox_annotation = bbox_annotation[bbox_annotation[:, 4] != -1]
if bbox_annotation.shape[0] == 0:
regression_losses.append(torch.tensor(0).float().cuda())
classification_losses.append(torch.tensor(0).float().cuda())
continue
classification = torch.clamp(classification, 1e-4, 1.0 - 1e-4)
Iou = calc_iou(anchors[0, :, :], bbox_annotation[:, :4])
Iou_max, Iou_argmax = torch.max(Iou, dim=1)
# 类别损失
targets = torch.ones(classification.shape) * -1
targets = targets.cuda()
targets[torch.lt(Iou_max, 0.4), :] = 0
positive_indices = torch.ge(Iou_max, 0.5)
num_positive_anchors = positive_indices.sum()
assigned_annotations = bbox_annotation[Iou_argmax, :]
targets[positive_indices, :] = 0
targets[positive_indices, assigned_annotations[positive_indices,
4].long()] = 1
alpha_factor = torch.ones(targets.shape).cuda() * alpha
alpha_factor = torch.where(torch.eq(targets, 1.), alpha_factor,
1. - alpha_factor)
focal_weight = torch.where(torch.eq(targets, 1.), 1. - classification,
classification)
focal_weight = alpha_factor * torch.pow(focal_weight, gamma)
bce = -(targets * torch.log(classification) +
(1.0 - targets) * torch.log(1.0 - classification))
cls_loss = focal_weight * bce
cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss,
torch.zeros(cls_loss.shape).cuda())
classification_losses.append(
cls_loss.sum() /
torch.clamp(num_positive_anchors.float(), min=1.0))
# 回归检测框损失
if positive_indices.sum() > 0:
assigned_annotations = assigned_annotations[positive_indices, :]
anchor_width_pi = anchor_width[positive_indices]
anchor_height_pi = anchor_height[positive_indices]
anchor_cx_pi = anchor_cx[positive_indices]
anchor_cy_pi = anchor_cy[positive_indices]
gt_width = assigned_annotations[:, 2] - assigned_annotations[:, 0]
gt_height = assigned_annotations[:, 3] - assigned_annotations[:, 1]
gt_cx = assigned_annotations[:, 0] + 0.5 * gt_width
gt_cy = assigned_annotations[:, 1] + 0.5 * gt_height
gt_width = torch.clamp(gt_width, min=1)
gt_height = torch.clamp(gt_height, min=1)
targets_dx = (gt_cx - anchor_cx_pi) / anchor_width_pi
targets_dy = (gt_cy - anchor_cy_pi) / anchor_height_pi
targets_dw = torch.log(gt_width / anchor_width_pi)
targets_dh = torch.log(gt_height / anchor_height_pi)
targets = torch.stack(
(targets_dx, targets_dy, targets_dw, targets_dh))
targets = targets.t()
targets = targets / torch.Tensor([[0.1, 0.1, 0.2, 0.2]]).cuda()
negative_indices = 1 - positive_indices
regression_diff = torch.abs(targets -
regression[positive_indices, :])
regression_loss = torch.where(
torch.le(regression_diff,
1.0 / 9.0), 0.5 * 9.0 * torch.pow(regression_diff, 2),
regression_diff - 0.5 / 9.0)
regression_losses.append(regression_loss.mean())
else:
regression_losses.append(torch.tensor(0).float().cuda())
c_losses = torch.stack(classification_losses).mean(dim=0, keepdim=True)
r_losses = torch.stack(regression_losses).mean(dim=0, keepdim=True)
return c_losses, r_losses
def torch_all_close(a, b, tolerance=1e-12):
"""
Check if all elements in two tensors are equal within a tolerance.
"""
return torch.all(torch.lt(torch.abs(a - b), tolerance))
def easy(x, y):
c = torch.lt(x, y)
return c
def forward(self,
operation,
x=None,
w=None,
ws=None,
wP=None,
wc=None,
wC=None,
msk_s=None,
msk_c=None,
mode=None):
# x: T(bat, max_word_num)
# w: T(bat)
# x_embedding: T(bat, max_word_num, embed_dim)
x_embedding = self.embedding(x)
x_embedding = self.embedding_dropout(x_embedding)
# mask: T(bat, max_word_num)
mask = torch.gt(x, 0).to(torch.int64)
# x_len: T(bat)
x_len = torch.sum(mask, dim=1)
# h: T(bat, max_word_num, hid*2)
# ht: T(num_layers*2, bat, hid) float32
h, (ht, _) = self.encoder(x_embedding, x_len)
# ht: T(bat, hid*2)
ht = torch.transpose(ht[ht.shape[0] - 2:, :, :], 0,
1).contiguous().view(x_len.shape[0],
self.hidden_dim * 2)
# alpha: T(bat, max_word_num, 1)
alpha = (h.bmm(ht.unsqueeze(2)))
# mask_3: T(bat, max_word_num, 1)
mask_3 = mask.to(torch.float32).unsqueeze(2)
## word prediction
# vd: T(bat, embed_dim)
#h_ = self.fc(h)
#vd = torch.sum(h_, 1)
#vd = self.fc(torch.max(h, dim=1)[0])
h_1 = torch.sum(h * alpha, 1)
vd = self.fc(
h_1
) #+ torch.sum(self.embedding(x), 1)#+ torch.sum(x_embedding, 1) #ok
#vd = self.fc(torch.cat([torch.sum(h*alpha, 1), torch.sum(self.embedding(x), 1)], 1)) #ok
#vd = self.fc(torch.sum(torch.cat([h, self.embedding(x)], 2)*alpha, 1)) #best
#vd = self.fc(torch.max(torch.cat([h, self.embedding(x)], 2), 1)[0]) #bad
#vd = torch.sum(self.embedding(x), 1)
#vd = torch.max(self.embedding.weight(x), 1)[0]
#vd = self.fc(ht)
# score: T(bat, calss_num)
score0 = vd.mm(self.embedding.weight[[range(self.class_num)]].t())
score = score0
if 'C' in mode:
# scC[i]: T(bat, Ci_size)
# 词林的层次分类训练的慢，其实这样不公平，不平衡，因为词预测先收敛了，而cilin的分类还没效果，其他信息的利用也有同样的问题，不一定同时收敛！！！
scC = [
self.fc_C1(h_1),
self.fc_C2(h_1),
self.fc_C3(h_1),
self.fc_C4(h_1)
]
score2 = torch.zeros((score0.shape[0], score0.shape[1]),
dtype=torch.float32,
device=device)
rank = 0.6
for i in range(4):
# wC[i]: T(class_num, Ci_size)
# C_sc: T(bat, class_num)
score2 += self.relu(scC[i].mm(wC[i].t()) * (rank**i))
#----------add mean cilin-class score to those who have no cilin-class
mean_cilin_sc = torch.mean(score2, 1)
score2 = score2 * (1 - msk_c) + mean_cilin_sc.unsqueeze(1).mm(
msk_c.unsqueeze(0))
#----------
score = score + score2 / 3
if 'P' in mode:
## POS prediction
# score_POS: T(bat, 13) pos_num=12+1
score_POS = self.fc2(torch.sum(h * alpha, 1))
# s: (class_num, 13) multi-hot
# weight_sc: T(bat, class_num) = [bat, 13] .mm [class_num, 13].t()
weight_sc = self.relu(score_POS.mm(wP.t()))
#print(torch.max(weight_sc), torch.min(weight_sc))
score = score + weight_sc
if 's' in mode:
## sememe prediction
# pos_score: T(bat, max_word_num, sememe_num)
pos_score = self.fc1(h)
pos_score = pos_score * mask_3 + (-1e7) * (1 - mask_3)
# sem_score: T(bat, sememe_num)
sem_score, _ = torch.max(pos_score, dim=1)
#sem_score = torch.sum(pos_score * alpha, 1)
# score: T(bat, class_num) = [bat, sememe_num] .mm [class_num, sememe_num].t()
score1 = self.relu(sem_score.mm(ws.t()))
#----------add mean sememe score to those who have no sememes
# mean_sem_sc: T(bat)
mean_sem_sc = torch.mean(score1, 1)
# msk: T(class_num)
score1 = score1 + mean_sem_sc.unsqueeze(1).mm(msk_s.unsqueeze(0))
#----------
score = score + score1
if 'c' in mode:
## character prediction
# pos_score: T(bat, max_word_num, sememe_num)
pos_score = self.fc3(h)
pos_score = pos_score * mask_3 + (-1e7) * (1 - mask_3)
# chara_score: T(bat, chara_num)
chara_score, _ = torch.max(pos_score, dim=1)
#chara_score = torch.sum(pos_score * alpha, 1)
# score: T(bat, class_num) = [bat, sememe_num] .mm [class_num, sememe_num].t()
score3 = self.relu(chara_score.mm(wc.t()))
score = score + score3
# fine-tune depended on the target word shouldn't exist in the definition.
#score_res = score.clone().detach()
mask1 = torch.lt(x, self.class_num).to(torch.int64)
mask2 = torch.ones((score.shape[0], score.shape[1]),
dtype=torch.float32,
device=device)
for i in range(x.shape[0]):
mask2[i][x[i] * mask1[i]] = 0.
score = score * mask2 + (-1e6) * (1 - mask2)
_, indices = torch.sort(score, descending=True)
if operation == 'train':
loss = self.loss(score, w)
return loss, score, indices
elif operation == 'test':
return indices
def act_step(self, exper, meta_optimizer):
"""
Subtleties of the batch processing. TODO: NEED TO EXPLAIN THIS!!!
:param exper:
:param meta_optimizer:
:return: IMPORTANT RETURNS LOSS STEP THAT IS NOT MULTIPLIED BY QT-VALUE! NUMPY FLOAT32
"""
self.init_trunc_bptt(exper, meta_optimizer)
loss = get_func_loss(exper, self.functions, average=False)
# make the forward step, which depends heavily on the experiment we're performing (MLP or Regression(T))
delta_param, rho_probs, eval_par_new = self.forward(
loss, exper, meta_optimizer)
# then, apply the previous batch mask, although we did the forward pass with all functions, we filter here
delta_param = torch.mul(delta_param, self.float_mask)
# moved to method >>> compute_probs <<<
# rho_new = torch.mul(rho_probs, self.float_mask)
# compute new probability values, based on the cumulative probs construct new batch mask
# note that we also save the new rho_t values in the array self.rho_t (in the method compute_probs)
if self.learner == "meta_act":
# in Graves ACT model, the qt values are probabilities already : sigmoid(W^T h_t + bias) values
# so we don't use any stick-breaking here (meaning the rho_t values that we transfer in the act_sb
new_probs = torch.mul(rho_probs.double(), self.float_mask.double())
else:
# stick-breaking approach: transform RNN output rho_t values to probabilities
# method compute_probs multiplies with the float_mask object!
new_probs = self.compute_probs(rho_probs)
# we need to determine the indices of the functions that "stop" in this time step (new_funcs_mask)
funcs_that_stop = torch.le(self.compare_probs + new_probs,
self.one_minus_eps)
less_or_equal = LessOrEqual()
# NOTE: ALSO for the Graves ACT model we increase all functions that participated in THIS ITERATION and
# therefore we compare with self.compare_probs BEFORE we increase them with the new probs
iterations = less_or_equal(self.compare_probs, self.one_minus_eps)
if self.learner == "meta_act":
self.halting_steps += iterations
else:
# and we increase the number of time steps TAKEN with the previous float_mask object to keep track of the steps
self.halting_steps += self.float_mask
self.iterations += iterations
new_batch_mask = tensor_and(funcs_that_stop, self.bool_mask)
new_float_mask = new_batch_mask.float()
# IMPORTANT: although above we constructed the new mask (for the next time step) we use the previous mask for the
# increase of the cumulative probs which we use in the WHILE loop to determine when to stop the complete batch
self.compare_probs += torch.mul(new_probs, self.float_mask.double())
# IMPORTANT UPDATE OF OPTIMIZEE PARAMETERS
if self.func_is_nn_module:
# awkward but true: delta_param has [batch_dim, num_params] due to multiplication with float masks, but
# must have transpose shape for this update
par_new = self.functions.get_flat_params() - delta_param.permute(
1, 0)
else:
par_new = self.functions.params - delta_param
# increase the number of steps taken for the functions that are still in the race, we need these to compare
# against the maximum allowed number of steps
self.counter_compare += new_float_mask
# generate object that holds time_step_condition (which function has taken more than MAX time steps allowed)
# Note that self.max_T = HORIZON
time_step_condition = torch.lt(self.counter_compare, self.max_T)
# we then generate the mask that determines which functions will finally be part of the next round
next_iteration_condition = tensor_and(new_batch_mask,
time_step_condition)
final_func_mask = (self.bool_mask - next_iteration_condition) == 1
# set the cumulative_probs for all functions that participate in the next time step
# Subtlety here: so for an optimizee stopping in this step, we don't want the cum-prob to be increased
# because we'll use the previous (step-1) cum-probs to determine the q_T value...the final
# q_t value for the halting step which gets assigned the REST prob mass
# we use this object for determining the rest-probability for the functions that stop after this time step
# the mask "final_func_mask" holds the indices of the functions that stop after this time step
# the mask "next_iteration_condition" holds the indices of the functions that continue in the next time step
self.cumulative_probs += torch.mul(new_probs,
next_iteration_condition.double())
self.set_qt_values(new_probs, next_iteration_condition,
final_func_mask)
# set the masks for the next time step
self.float_mask = new_float_mask
self.bool_mask = new_float_mask.type(torch.cuda.ByteTensor) if exper.args.cuda else \
new_float_mask.type(torch.ByteTensor)
# compute the new step loss
if self.is_train:
loss_step = self.step_loss(par_new, exper, average_batch=True)
else:
loss_step = self.step_loss(eval_par_new, exper, average_batch=True)
batch_loss = self.batch_step_losses[-1]
# we don't "have" any variance when batch size is 1
if self.batch_size > 1:
exper.add_step_loss_variance(batch_loss, self.step + 1)
elif exper.args.problem == "mlp":
# batch_size is always one, but we want to compute stddev later, so just save all losses for the function
# we calculate the stddev in Experiment.eval() method
self.np_step_losses[
self.step + 1] = batch_loss.data.cpu().squeeze().numpy()[0]
# update batch functions parameter for next step
if self.is_train:
self.functions.set_parameters(par_new)
else:
self.functions.set_parameters(eval_par_new)
# MLP which is a torch.nn.module or regression function (which is just a normal Class object)
if self.func_is_nn_module:
self.functions.zero_grad()
else:
self.functions.params.grad.data.zero_()
return loss_step.data.cpu().squeeze().numpy()[0]
def sample_from_mean_field_model(Q, batch_size):
random_sample = torch.rand((batch_size, Q.dim_latent))
return torch.lt(random_sample, Q.parameters.repeat(batch_size, 1)).float()
def forward(self, classifications, regressions, anchors, annotations,
**kwargs):
alpha = 0.25
gamma = 2.0
batch_size = classifications.shape[0]
classification_losses = []
regression_losses = []
anchor = anchors[
0, :, :] # assuming all image sizes are the same, which it is
dtype = anchors.dtype
anchor_widths = anchor[:, 3] - anchor[:, 1]
anchor_heights = anchor[:, 2] - anchor[:, 0]
anchor_ctr_x = anchor[:, 1] + 0.5 * anchor_widths
anchor_ctr_y = anchor[:, 0] + 0.5 * anchor_heights
for j in range(batch_size):
classification = classifications[j, :, :]
regression = regressions[j, :, :]
bbox_annotation = annotations[j]
bbox_annotation = bbox_annotation[bbox_annotation[:, 4] != -1]
classification = torch.clamp(classification, 1e-4, 1.0 - 1e-4)
if bbox_annotation.shape[0] == 0:
if torch.cuda.is_available():
alpha_factor = torch.ones_like(classification) * alpha
alpha_factor = alpha_factor.cuda()
alpha_factor = 1. - alpha_factor
focal_weight = classification
focal_weight = alpha_factor * torch.pow(
focal_weight, gamma)
bce = -(torch.log(1.0 - classification))
cls_loss = focal_weight * bce
regression_losses.append(torch.tensor(0).to(dtype).cuda())
classification_losses.append(cls_loss.sum())
else:
alpha_factor = torch.ones_like(classification) * alpha
alpha_factor = 1. - alpha_factor
focal_weight = classification
focal_weight = alpha_factor * torch.pow(
focal_weight, gamma)
bce = -(torch.log(1.0 - classification))
cls_loss = focal_weight * bce
regression_losses.append(torch.tensor(0).to(dtype))
classification_losses.append(cls_loss.sum())
continue
IoU = calc_iou(anchor[:, :], bbox_annotation[:, :4])
IoU_max, IoU_argmax = torch.max(IoU, dim=1)
# compute the loss for classification
targets = torch.ones_like(classification) * -1
if torch.cuda.is_available():
targets = targets.cuda()
targets[torch.lt(IoU_max, 0.4), :] = 0
positive_indices = torch.ge(IoU_max, 0.5)
num_positive_anchors = positive_indices.sum()
assigned_annotations = bbox_annotation[IoU_argmax, :]
targets[positive_indices, :] = 0
targets[positive_indices, assigned_annotations[positive_indices,
4].long()] = 1
alpha_factor = torch.ones_like(targets) * alpha
if torch.cuda.is_available():
alpha_factor = alpha_factor.cuda()
alpha_factor = torch.where(torch.eq(targets, 1.), alpha_factor,
1. - alpha_factor)
focal_weight = torch.where(torch.eq(targets, 1.),
1. - classification, classification)
focal_weight = alpha_factor * torch.pow(focal_weight, gamma)
bce = -(targets * torch.log(classification) +
(1.0 - targets) * torch.log(1.0 - classification))
cls_loss = focal_weight * bce
zeros = torch.zeros_like(cls_loss)
if torch.cuda.is_available():
zeros = zeros.cuda()
cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss, zeros)
classification_losses.append(
cls_loss.sum() /
torch.clamp(num_positive_anchors.to(dtype), min=1.0))
if positive_indices.sum() > 0:
assigned_annotations = assigned_annotations[
positive_indices, :]
anchor_widths_pi = anchor_widths[positive_indices]
anchor_heights_pi = anchor_heights[positive_indices]
anchor_ctr_x_pi = anchor_ctr_x[positive_indices]
anchor_ctr_y_pi = anchor_ctr_y[positive_indices]
gt_widths = assigned_annotations[:,
2] - assigned_annotations[:,
0]
gt_heights = assigned_annotations[:,
3] - assigned_annotations[:,
1]
gt_ctr_x = assigned_annotations[:, 0] + 0.5 * gt_widths
gt_ctr_y = assigned_annotations[:, 1] + 0.5 * gt_heights
# efficientdet style
gt_widths = torch.clamp(gt_widths, min=1)
gt_heights = torch.clamp(gt_heights, min=1)
targets_dx = (gt_ctr_x - anchor_ctr_x_pi) / anchor_widths_pi
targets_dy = (gt_ctr_y - anchor_ctr_y_pi) / anchor_heights_pi
targets_dw = torch.log(gt_widths / anchor_widths_pi)
targets_dh = torch.log(gt_heights / anchor_heights_pi)
targets = torch.stack(
(targets_dy, targets_dx, targets_dh, targets_dw))
targets = targets.t()
regression_diff = torch.abs(targets -
regression[positive_indices, :])
regression_loss = torch.where(
torch.le(regression_diff, 1.0 / 9.0),
0.5 * 9.0 * torch.pow(regression_diff, 2),
regression_diff - 0.5 / 9.0)
regression_losses.append(regression_loss.mean())
else:
if torch.cuda.is_available():
regression_losses.append(torch.tensor(0).to(dtype).cuda())
else:
regression_losses.append(torch.tensor(0).to(dtype))
# debug
imgs = kwargs.get('imgs', None)
if imgs is not None:
regressBoxes = BBoxTransform()
clipBoxes = ClipBoxes()
obj_list = kwargs.get('obj_list', None)
out = postprocess(
imgs.detach(),
torch.stack([anchors[0]] * imgs.shape[0], 0).detach(),
regressions.detach(), classifications.detach(), regressBoxes,
clipBoxes, 0.5, 0.3)
imgs = imgs.permute(0, 2, 3, 1).cpu().numpy()
imgs = ((imgs * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]) *
255).astype(np.uint8)
imgs = [cv2.cvtColor(img, cv2.COLOR_RGB2BGR) for img in imgs]
display(out, imgs, obj_list, imshow=False, imwrite=True)
return torch.stack(classification_losses).mean(dim=0, keepdim=True), \
torch.stack(regression_losses).mean(dim=0, keepdim=True)
def forward(self, marker_data, time_data, mask_data, embedding):
size = time_data.size()
self.index = 0
# Initialize Data Inputs and Outputs
marker_res = torch.zeros(size, dtype=torch.long).cuda(self.cuda_id)
time_res = time_data[:, 0:1]
mask_res = torch.zeros(size, dtype=torch.float).cuda(self.cuda_id)
marker_res[:, 0:1] = marker_data[:, 0:1]
mask_res[:, 0:1] = mask_data[:, 0:1]
data_input = torch.zeros(size[0], 0, self.d_model).cuda(self.cuda_id)
intense_res = torch.zeros(size[0], 0, self.d_model).cuda(self.cuda_id)
candidate_list = marker_data[:, 0:1]
prob_list = torch.ones(size[0], 1).cuda(self.cuda_id)
chosen_index = torch.zeros(size[0], 1,
dtype=torch.long).cuda(self.cuda_id)
# Hidden States of RNN Encoder
hidden_state = torch.zeros(1, size[0], self.d_model).cuda(self.cuda_id)
# Output Probabilities for Expectation Maximization
neighbor_prob_record = torch.ones(size[0], 1).cuda(self.cuda_id)
total_neighbor_prob = torch.ones(size[0], 1).cuda(self.cuda_id)
total_sample_prob = torch.ones(size[0], 1).cuda(self.cuda_id)
# Sequence Generation
while self.index < size[1] - 1:
last_marker = marker_res[:, self.index:self.index + 1]
last_time = time_res[:, self.index:self.index + 1]
new_vector = self.get_embedding(last_marker, last_time, embedding)
model_dim_vector = self.embed_ac(self.embed_linear(new_vector))
data_input = torch.cat((data_input, model_dim_vector), 1)
rnn_output = self.encoder(
data_input[:, self.index:self.index + 1, :], hidden_state)
intensity = rnn_output[0]
hidden_state = rnn_output[1]
# Time Decoding
new_time = last_time + F.softplus(
self.time_linear(intensity.squeeze(1)))
time_res = torch.cat((time_res, new_time), 1)
new_mask = torch.lt(new_time, self.max_time).long()
intense_res = torch.cat((intense_res, intensity), 1)
neighbor = self.neighbor_list[last_marker].squeeze(1)
prob_neighbor = self.neighbor_prob[last_marker].squeeze(1)
# Intensity Function Stacks
neighbor_prob_record = torch.cat(
(neighbor_prob_record, prob_neighbor), 1)
neighbor_inf = embedding[neighbor]
intensity_inf = torch.stack([(intensity.squeeze(1))
for _ in range(self.sample_size)], 1)
inf_matrix = torch.cat((neighbor_inf, intensity_inf), 2)
# Marker Decoding
marker_weight = self.marker_linear(inf_matrix).squeeze(2)
marker_prob = torch.softmax(marker_weight, 1)
# Marker Sampling
candidate_list = torch.cat((candidate_list, neighbor[:, 1:]), 1)
chosen_prob = torch.gather(prob_list, 1, chosen_index)
attached_prob = chosen_prob * marker_prob
for i in range(size[0]):
prob_list[i][chosen_index[i]] = attached_prob[i][0]
prob_list = torch.cat((prob_list, attached_prob[:, 1:]), 1)
chosen_index = torch.multinomial(prob_list, 1)
new_markers = torch.gather(candidate_list, 1, chosen_index)
# Record the marker probabilities for optimization
selected_neighbor_prob = torch.gather(neighbor_prob_record, 1,
chosen_index)
total_neighbor_prob = torch.cat(
(total_neighbor_prob, selected_neighbor_prob), 1)
selected_sample_prob = torch.gather(prob_list, 1, chosen_index)
total_sample_prob = torch.cat(
(total_sample_prob, selected_sample_prob), 1)
self.index += 1
marker_res = torch.cat((marker_res, new_markers), 1)
mask_res[:, self.index:self.index + 1] = new_mask
return marker_res, time_res, mask_res, total_neighbor_prob, total_sample_prob
def forward(self, marker_data, time_data, mask_data, embedding):
# Forward Propagation
d_size = time_data.size()
# Initialize data
self.index = 0
marker_res = marker_data[:, 0:1].clone()
time_res = time_data[:, 0:1].clone()
mask_res = mask_data[:, 0:1].clone()
data_input = torch.zeros(d_size[0], 0, self.d_model).cuda(self.cuda_id)
candidate_list = marker_data[:, 0:1]
prob_list = torch.ones(d_size[0], 1).cuda(self.cuda_id)
chosen_index = torch.zeros(d_size[0], 1,
dtype=torch.long).cuda(self.cuda_id)
neighbor_prob_record = torch.ones(d_size[0], 1).cuda(self.cuda_id)
total_neighbor_prob = torch.ones(d_size[0], 1).cuda(self.cuda_id)
total_sample_prob = torch.ones(d_size[0], 1).cuda(self.cuda_id)
# Generating Cascades
while self.index < d_size[1] - 1:
last_marker = marker_res[:, self.index:self.index + 1]
last_time = time_res[:, self.index:self.index + 1]
new_vector = self.get_embedding(last_marker, last_time, embedding)
model_dim_vector = self.embed_ac(self.embed_linear(new_vector))
data_input = torch.cat((data_input, model_dim_vector), 1)
intensity = self.encoder.forward(data_input, self.index)
# Time Decoding
new_time = last_time + F.softplus(
self.time_linear(intensity.squeeze(1)))
# Causal Descendants
time_res = torch.cat((time_res, new_time), 1)
new_mask = torch.lt(new_time, self.max_time).float()
neighbor = self.neighbor_list[last_marker].squeeze(1)
prob_neighbor = self.neighbor_prob[last_marker].squeeze(1)
neighbor_prob_record = torch.cat(
(neighbor_prob_record, prob_neighbor), 1)
# Intensity Function
neighbor_inf = embedding[neighbor]
intensity_inf = torch.stack([(intensity.squeeze(1))
for _ in range(self.sample_size)], 1)
inf_matrix = torch.cat((neighbor_inf, intensity_inf), 2)
# Marker Decoding
marker_weight = self.marker_linear(inf_matrix).squeeze(2)
marker_prob = torch.softmax(marker_weight, 1)
candidate_list = torch.cat((candidate_list, neighbor[:, 1:]), 1)
chosen_prob = torch.gather(prob_list, 1, chosen_index)
attached_prob = chosen_prob * marker_prob
for i in range(d_size[0]):
prob_list[i][chosen_index[i]] = attached_prob[i][0]
prob_list = torch.cat((prob_list, attached_prob[:, 1:]), 1)
chosen_index = torch.multinomial(prob_list, 1)
new_markers = torch.gather(candidate_list, 1, chosen_index)
# Record Probabilities for BP
selected_neighbor_prob = torch.gather(neighbor_prob_record, 1,
chosen_index)
total_neighbor_prob = torch.cat(
(total_neighbor_prob, selected_neighbor_prob), 1)
selected_sample_prob = torch.gather(prob_list, 1, chosen_index)
total_sample_prob = torch.cat(
(total_sample_prob, selected_sample_prob), 1)
self.index += 1
# Mark down the Results
marker_res = torch.cat((marker_res, new_markers), 1)
mask_res = torch.cat((mask_res, new_mask), 1)
return marker_res, time_res, mask_res, total_neighbor_prob, total_sample_prob
def test_comparison_ops_with_type_promotion(self, device):
value_for_type = {
torch.uint8: (1 << 5),
torch.int8: (1 << 5),
torch.int16: (1 << 10),
torch.int32: (1 << 20),
torch.int64: (1 << 35),
torch.float16: (1 << 10),
torch.float32: (1 << 20),
torch.float64: (1 << 35),
torch.complex64: (1 << 20),
torch.complex128: (1 << 35)
}
comparison_ops = [
dict(
name="lt",
out_op=lambda x, y, d: torch.lt(
x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.lt(x, y),
compare_op=lambda x, y: x < y,
),
dict(
name="le",
out_op=lambda x, y, d: torch.le(
x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.le(x, y),
compare_op=lambda x, y: x <= y,
),
dict(
name="gt",
out_op=lambda x, y, d: torch.gt(
x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.gt(x, y),
compare_op=lambda x, y: x > y,
),
dict(
name="ge",
out_op=lambda x, y, d: torch.ge(
x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.ge(x, y),
compare_op=lambda x, y: x >= y,
),
dict(
name="eq",
out_op=lambda x, y, d: torch.eq(
x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.eq(x, y),
compare_op=lambda x, y: x == y,
),
dict(
name="ne",
out_op=lambda x, y, d: torch.ne(
x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.ne(x, y),
compare_op=lambda x, y: x != y,
),
]
for op in comparison_ops:
for dt1 in torch.testing.get_all_math_dtypes(device):
for dt2 in torch.testing.get_all_math_dtypes(device):
if (dt1.is_complex or dt2.is_complex
) and not (op["name"] == "eq" or op["name"] == "ne"):
continue
val1 = value_for_type[dt1]
val2 = value_for_type[dt2]
t1 = torch.tensor([val1], dtype=dt1, device=device)
t2 = torch.tensor([val2], dtype=dt2, device=device)
expected = torch.tensor([op["compare_op"](val1, val2)],
dtype=torch.bool)
out_res = op["out_op"](t1, t2, device)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
out_res = op["ret_op"](t1, t2)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
# test that comparing a zero dim tensor with another zero dim tensor has type promotion behavior
t1 = torch.tensor(val1, dtype=dt1, device=device)
t2 = torch.tensor(val2, dtype=dt2, device=device)
expected = torch.tensor(op["compare_op"](val1, val2),
dtype=torch.bool)
out_res = op["out_op"](t1, t2, device)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
out_res = op["ret_op"](t1, t2)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
def initialize(self, opt):
BaseModel.initialize(self, opt)
self.opt = opt
self.isTrain = opt.isTrain
# specify the training losses you want to print out. The program will call base_model.get_current_losses
self.loss_names = ['G_GAN', 'G_L1', 'D_real', 'D_fake']
# specify the images you want to save/display. The program will call base_model.get_current_visuals
self.visual_names = ['real_A', 'fake_B', 'real_B']
# specify the models you want to save to the disk. The program will call base_model.save_networks and base_model.load_networks
if self.isTrain:
self.model_names = ['G', 'D']
else: # during test time, only load Gs
self.model_names = ['G']
# batchsize should be 1 for mask_global
self.mask_global = torch.ByteTensor(1, 1, \
opt.fineSize, opt.fineSize)
# Here we need to set an artificial mask_global(not to make it broken, so center hole is ok.)
self.mask_global.zero_()
self.mask_global[:, :, int(self.opt.fineSize/4) + self.opt.overlap : int(self.opt.fineSize/2) + int(self.opt.fineSize/4) - self.opt.overlap,\
int(self.opt.fineSize/4) + self.opt.overlap: int(self.opt.fineSize/2) + int(self.opt.fineSize/4) - self.opt.overlap] = 1
self.mask_type = opt.mask_type
self.gMask_opts = {}
self.fixed_mask = opt.fixed_mask if opt.mask_type == 'center' else 0
if opt.mask_type == 'center':
assert opt.fixed_mask == 1, "Center mask must be fixed mask!"
if self.mask_type == 'random':
res = 0.06 # the lower it is, the more continuous the output will be. 0.01 is too small and 0.1 is too large
density = 0.25
MAX_SIZE = 10000
maxPartition = 30
low_pattern = torch.rand(1, 1, int(res*MAX_SIZE), int(res*MAX_SIZE)).mul(255)
pattern = F.functional.interpolate(low_pattern, (MAX_SIZE, MAX_SIZE), mode='bilinear').detach()
low_pattern = None
pattern.div_(255)
pattern = torch.lt(pattern,density).byte() # 25% 1s and 75% 0s
pattern = torch.squeeze(pattern).byte()
print('...Random pattern generated')
self.gMask_opts['pattern'] = pattern
self.gMask_opts['MAX_SIZE'] = MAX_SIZE
self.gMask_opts['fineSize'] = opt.fineSize
self.gMask_opts['maxPartition'] = maxPartition
self.gMask_opts['mask_global'] = self.mask_global
self.mask_global = util.create_gMask(self.gMask_opts) # create an initial random mask.
self.wgan_gp = False
# added for wgan-gp
if opt.gan_type == 'wgan_gp':
self.gp_lambda = opt.gp_lambda
self.ncritic = opt.ncritic
self.wgan_gp = True
if len(opt.gpu_ids) > 0:
self.use_gpu = True
self.mask_global = self.mask_global.to(self.device)
# load/define networks
# self.ng_innerCos_list is the constraint list in netG inner layers.
# self.ng_mask_list is the mask list constructing shift operation.
self.netG, self.ng_innerCos_list, self.ng_shift_list = networks.define_G(opt.input_nc, opt.output_nc, opt.ngf,
opt.which_model_netG, opt, self.mask_global, opt.norm, opt.use_dropout, opt.init_type, self.gpu_ids, opt.init_gain) # add opt, we need opt.shift_sz and other stuffs
if self.isTrain:
use_sigmoid = False
if opt.gan_type == 'vanilla':
use_sigmoid = True # only vanilla GAN using BCECriterion
# don't use cGAN
self.netD = networks.define_D(opt.input_nc, opt.ndf,
opt.which_model_netD,
opt.n_layers_D, opt.norm, use_sigmoid, opt.init_type, self.gpu_ids, opt.init_gain)
if self.isTrain:
self.old_lr = opt.lr
# define loss functions
self.criterionGAN = networks.GANLoss(gan_type=opt.gan_type).to(self.device)
self.criterionL1 = torch.nn.L1Loss()
# initialize optimizers
self.schedulers = []
self.optimizers = []
if self.wgan_gp:
opt.beta1 = 0
self.optimizer_G = torch.optim.Adam(self.netG.parameters(),
lr=opt.lr, betas=(opt.beta1, 0.999))
self.optimizer_D = torch.optim.Adam(self.netD.parameters(),
lr=opt.lr, betas=(opt.beta1, 0.999))
else:
self.optimizer_G = torch.optim.Adam(self.netG.parameters(),
lr=opt.lr, betas=(opt.beta1, 0.999))
self.optimizer_D = torch.optim.Adam(self.netD.parameters(),
lr=opt.lr, betas=(opt.beta1, 0.999))
self.optimizers.append(self.optimizer_G)
self.optimizers.append(self.optimizer_D)
for optimizer in self.optimizers:
self.schedulers.append(networks.get_scheduler(optimizer, opt))
if not self.isTrain or opt.continue_train:
self.load_networks(opt.which_epoch)
self.print_networks(opt.verbose)
def lower_bound_softplus_inv(x, low=0., high=0.):
return torch.log(torch.exp(x - low) - 1)
def upper_bound_softplus(x, low=0., high=0.):
return -torch.log(1 + torch.exp(-x + high)) + high
def upper_bound_softplus_inv(x, low=0., high=0.):
return -torch.log(torch.exp(-x + high) - 1) + high
if __name__ == "__main__":
# Sanity Check:
lists = torch.arange(-5, 5).float()
if torch.all(torch.lt(torch.abs(torch.add(softplus_inv(softplus(lists)), -lists)), 1e-3)):
print("success")
else:
print("fail")
if torch.all(torch.lt(torch.abs(torch.add(softplus_inv(softplus(lists, low=3, high=5),
low=3, high=5), -lists)), 1e-3)):
print("success")
else:
print("fail")
if torch.all(torch.lt(torch.abs(torch.add(exp_inv(exp(lists)), -lists)), 1e-3)):
print("success")
else:
print("fail")
if torch.all(torch.lt(torch.abs(torch.add(exp_inv(exp(lists, low=3, high=5),
low=3, high=5), -lists)), 1e-3)):
print("success")
def train(self):
dataset = samp.FaceDataset(self.rootpath, islandmark=True)
self.dataloader = data.DataLoader(dataset,
batch_size=512,
shuffle=True,
num_workers=4)
self.optimer = torch.optim.Adam(
self.net.parameters(), lr=1e-4,
weight_decay=0.00005) #weight_decay 正则化因子
if os.path.exists(self.parampath): # 恢复参数继续训练
self.net.load_state_dict(torch.load(self.parampath))
if not os.path.exists(self.parampath): #创建参数文件
dirs, file = os.path.split(self.parampath)
if not os.path.exists(dirs):
os.makedirs(dirs)
#训练
while True:
for epoch, (imgdata, con, offset,
landmarks) in enumerate(self.dataloader): # batchsize
if self.isCuda == True: #GPU 运算
imgdata, con, offset, landmarks = imgdata.cuda(), con.cuda(
), offset.cuda(), landmarks.cuda()
con_out, offset_out, landmarks_out = self.net(
imgdata) # imgdata为网络输入, 得到网络输出
#如何保证con_out > 0??? # Assertion `x >= 0. && x <= 1.' failed. input value should be between 0~1, but got -0.157099 at
# 加signmod 函数 转化成概率
# 计算置信度损失
# print(con)
# print("==================")
# print(con_out)
con_mask = torch.lt(con, 2) #部分样本不参与分类损失 2定义为部分样本 掩码操作
con_ = con[con_mask[:, 0]]
# print("con_",con_.shape)
con_out_ = con_out[con_mask[:, 0]]
con_loss = self.con_lossfun(con_out_, con_) #只有正负样本参与置信度损失计算
#计算坐标偏移损失
offset_mask = torch.gt(con, 0) #选出非负样本
offset_ = offset[offset_mask[:, 0]]
offset_out_ = offset_out[offset_mask[:, 0]]
offset_loss = self.offset_lossfun(offset_out_,
offset_) # 只有非负样本参与坐标偏移损失计算
#计算关键点损失
landmarks_mask = torch.gt(con, 0) #正样本和部分样本训练关键点
landmarks_ = landmarks[landmarks_mask[:, 0]]
landmarks_out_ = landmarks_out[landmarks_mask[:, 0]]
# print(landmarks_mask)
landmarks_loss = self.landmarks_lossfun(
landmarks_out_, landmarks_)
#总loss
total_loss = con_loss + 0.5 * offset_loss + landmarks_loss
self.optimer.zero_grad()
total_loss.backward()
self.optimer.step()
# break
if epoch % 100 == 0 and epoch > 0:
print("epoch==>", epoch, "total_loss==>", total_loss)
if epoch % 500 == 0 and epoch > 0:
torch.save(self.net.state_dict(), self.parampath) #保存模型参数
print("===========save module===============")
print("epoch==>", epoch, "total_loss==>", total_loss)
def create_mask(self, lengths, max_length):
r = torch.unsqueeze(torch.arange(0, max_length), 0).long().to(DEVICE) # (1, 82)
l = torch.unsqueeze(lengths, 1).expand(lengths.size(0), max_length) # (20, 82)
mask = torch.lt(r.expand_as(l), l)
return mask
def step_rescale(self, values):
y = cudaify(torch.ones(values.size()).double() / (10 * self.n))
y[torch.lt(values.data, 5)] = 1.0
return Variable(y, requires_grad=False)
def forward(self, classifications, bbox_regressions, ldm_regressions,
anchors, annotations):
batch_size = classifications.shape[0]
classification_losses = []
bbox_regression_losses = []
ldm_regression_losses = []
anchor = anchors[0, :, :]
anchor_widths = anchor[:, 2] - anchor[:, 0]
anchor_heights = anchor[:, 3] - anchor[:, 1]
anchor_ctr_x = anchor[:, 0] + 0.5 * anchor_widths
anchor_ctr_y = anchor[:, 1] + 0.5 * anchor_heights
#temp
positive_indices_list = []
for j in range(batch_size):
classification = classifications[j, :, :]
bbox_regression = bbox_regressions[j, :, :]
ldm_regression = ldm_regressions[j, :, :]
annotation = annotations[j, :, :]
# annotation = annotation[annotation[:,0] != -1]
annotation = annotation[annotation[:, 0] > 0]
bbox_annotation = annotation[:, :4]
ldm_annotation = annotation[:, 4:]
if bbox_annotation.shape[0] == 0:
bbox_regression_losses.append(
torch.tensor(0., requires_grad=True).cuda())
classification_losses.append(
torch.tensor(0., requires_grad=True).cuda())
ldm_regression_losses.append(
torch.tensor(0., requires_grad=True).cuda())
# temp
positive_indices_list.append([])
continue
IoU = calc_iou(anchors[0, :, :], bbox_annotation[:, :4])
#IoU, filt_iou = filt_IoU(anchors[0, :, :], bbox_annotation, ldm_annotation)
IoU_max, IoU_argmax = torch.max(IoU, dim=1) #取每个类型anchor匹配的最大值
targets = torch.ones(classification.shape) * -1
targets = targets.cuda()
# those whose iou<0.3 have no object
negative_indices = torch.lt(IoU_max, 0.3) #iou小于0.3的为负anchor
targets[negative_indices, :] = 0
targets[negative_indices, 1] = 1 #负anchor对应targets位置存0， 1
# those whose iou>0.5 have object
positive_indices = torch.ge(IoU_max, 0.5) #取iou大于0.5的为正样本位置
#temp
positive_indices_list.append(positive_indices)
num_positive_anchors = positive_indices.sum()
#keep positive and negative ratios with 1:3
keep_negative_anchors = num_positive_anchors * 3
bbox_assigned_annotations = bbox_annotation[IoU_argmax, :]
ldm_assigned_annotations = ldm_annotation[IoU_argmax, :]
targets[positive_indices, :] = 0
targets[positive_indices, 0] = 1 #正anchor对应targets位置存1，0
# ignore targets with no landmarks
# f_IoU_max ,f_IoU_argmax = torch.max(filt_iou, dim=1)
# ldm_positive_indices = torch.ge(f_IoU_max, 0.5)
ldm_sum = ldm_assigned_annotations.sum(dim=1)
ge0_mask = ldm_sum > 0
ldm_positive_indices = ge0_mask & positive_indices
# OHEM
negative_losses = classification[
negative_indices,
1] * -1 #classification为LogSoftmax输出，需要x轴翻转，所以乘以-1
sorted_losses, _ = torch.sort(negative_losses, descending=True)
if sorted_losses.numel() > keep_negative_anchors:
sorted_losses = sorted_losses[:keep_negative_anchors]
positive_losses = classification[
positive_indices,
0] * -1 #classification为LogSoftmax输出，需要x轴翻转，所以乘以-1
focal_loss = True #yzk use focal loss
# focal loss
if focal_loss:
alpha = 0.25
gamma = 2.0
alpha_factor = torch.ones(targets.shape).cuda() * alpha
focal_weight = targets * torch.where(
torch.eq(targets, 1), 1. - torch.exp(classification),
torch.exp(classification))
focal_weight = alpha_factor * torch.pow(focal_weight, gamma)
bce = -(
classification
) #+ (1.0 - targets) * torch.log(1.0 - classification))
#add by yzk
cls_loss = focal_weight * bce
cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss,
torch.zeros(cls_loss.shape).cuda())
classification_losses.append(
cls_loss.sum() /
torch.clamp(num_positive_anchors.float(), min=1.0))
else:
if positive_indices.sum() > 0:
classification_losses.append(positive_losses.mean() +
sorted_losses.mean())
else:
classification_losses.append(
torch.tensor(0., requires_grad=True).cuda())
# compute bboxes loss
if positive_indices.sum() > 0:
# bbox
bbox_assigned_annotations = bbox_assigned_annotations[
positive_indices, :] #取正样本对应的anchor box
anchor_widths_pi = anchor_widths[positive_indices]
anchor_heights_pi = anchor_heights[positive_indices]
anchor_ctr_x_pi = anchor_ctr_x[positive_indices]
anchor_ctr_y_pi = anchor_ctr_y[positive_indices]
gt_widths = bbox_assigned_annotations[:,
2] - bbox_assigned_annotations[:,
0]
gt_heights = bbox_assigned_annotations[:,
3] - bbox_assigned_annotations[:,
1]
gt_ctr_x = bbox_assigned_annotations[:, 0] + 0.5 * gt_widths
gt_ctr_y = bbox_assigned_annotations[:, 1] + 0.5 * gt_heights
targets_dx = (gt_ctr_x - anchor_ctr_x_pi) / (anchor_widths_pi +
1e-14)
targets_dy = (gt_ctr_y -
anchor_ctr_y_pi) / (anchor_heights_pi + 1e-14)
targets_dw = torch.log(gt_widths / anchor_widths_pi)
targets_dh = torch.log(gt_heights / anchor_heights_pi)
bbox_targets = torch.stack(
(targets_dx, targets_dy, targets_dw,
targets_dh)) #根据annotation数据构造预测数据格式的数据
bbox_targets = bbox_targets.t()
# Rescale
bbox_targets = bbox_targets / torch.Tensor(
[[0.1, 0.1, 0.2, 0.2]]).cuda()
# smooth L1
# box losses
bbox_regression_loss = self.smoothl1(
bbox_targets, bbox_regression[positive_indices, :])
bbox_regression_losses.append(bbox_regression_loss)
else:
bbox_regression_losses.append(
torch.tensor(0., requires_grad=True).cuda())
# compute landmarks loss #changed by yzktmp test
# if ldm_positive_indices.sum() > 0 :
# ldm_assigned_annotations = ldm_assigned_annotations[ldm_positive_indices, :]
#
# anchor_widths_l = anchor_widths[ldm_positive_indices]
# anchor_heights_l = anchor_heights[ldm_positive_indices]
# anchor_ctr_x_l = anchor_ctr_x[ldm_positive_indices]
# anchor_ctr_y_l = anchor_ctr_y[ldm_positive_indices]
#
# l0_x = (ldm_assigned_annotations[:,0] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14)
# l0_y = (ldm_assigned_annotations[:,1] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14)
# l1_x = (ldm_assigned_annotations[:,2] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14)
# l1_y = (ldm_assigned_annotations[:,3] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14)
# l2_x = (ldm_assigned_annotations[:,4] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14)
# l2_y = (ldm_assigned_annotations[:,5] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14)
# l3_x = (ldm_assigned_annotations[:,6] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14)
# l3_y = (ldm_assigned_annotations[:,7] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14)
# l4_x = (ldm_assigned_annotations[:,8] - anchor_ctr_x_l) / (anchor_widths_l + 1e-14)
# l4_y = (ldm_assigned_annotations[:,9] - anchor_ctr_y_l) / (anchor_heights_l + 1e-14)
#
# ldm_targets = torch.stack((l0_x,l0_y,l1_x,l1_y,l2_x,l2_y,l3_x,l3_y,l4_x,l4_y))
# ldm_targets = ldm_targets.t()
#
# # Rescale
# scale = torch.ones(1,10)*0.1
# ldm_targets = ldm_targets/scale.cuda()
#
# ldm_regression_loss = self.smoothl1(ldm_targets, ldm_regression[ldm_positive_indices, :])
# ldm_regression_losses.append(ldm_regression_loss)
# else:
ldm_regression_losses.append(
torch.tensor(0., requires_grad=True).cuda())
return torch.stack(classification_losses), torch.stack(
bbox_regression_losses), torch.stack(ldm_regression_losses)
def __getitem__(self, idx):
if self.seg_level:
env_id = self.seg_list[idx][0]
set_idx = self.seg_list[idx][1]
seg_idx = self.seg_list[idx][2]
else:
env_id = self.env_list[idx]
env_conf_json = load_env_config(env_id)
landmark_names, landmark_indices, landmark_positions = get_landmark_locations_airsim(
env_conf_json)
top_down_image = load_env_img(env_id)
path = load_path(env_id)
img_x = top_down_image.shape[0]
img_y = top_down_image.shape[1]
path_in_img_coords = self.cf_to_img(img_x, path)
landmark_pos_in_img = self.as_to_img(
img_x,
np.asarray(landmark_positions)[:, 0:2])
self.pos_rand_image = self.pos_rand_range * img_x
#self.plot_path_on_img(top_down_image, path_in_img_coords)
#self.plot_path_on_img(top_down_image, landmark_pos_in_img)
#cv2.imshow("top_down", top_down_image)
#cv2.waitKey()
input_images = []
input_instructions = []
label_images = []
aux_labels = []
# Somehow load the instruction with the start and end indices for each of the N segments
if self.seg_level:
instruction_segments = [
self.all_instr[env_id][set_idx]["instructions"][seg_idx]
]
else:
instruction_segments = self.all_instr[env_id][0]["instructions"]
for seg_idx, seg in enumerate(instruction_segments):
start_idx = seg["start_idx"]
end_idx = seg["end_idx"]
instruction = seg["instruction"]
# TODO: Check for overflowz
seg_path = path_in_img_coords[start_idx:end_idx]
seg_img = top_down_image.copy()
#test_plot = self.plot_path_on_img(seg_img, seg_path)
# TODO: Validate the 0.5 choice, should it be 2?
affine, cropsize = self.get_affine_matrix(
seg_path, 0, [int(img_x / 2), int(img_y / 2)], 0.5)
if affine is None:
continue
seg_img_rot = self.apply_affine(seg_img, affine, cropsize)
seg_labels = np.zeros_like(seg_img[:, :, 0:1]).astype(float)
seg_labels = self.plot_path_on_img(seg_labels, seg_path)
seg_labels = gaussian_filter(seg_labels, 4)
seg_labels_rot = self.apply_affine(seg_labels, affine, cropsize)
#seg_labels_rot = gaussian_filter(seg_labels_rot, 4)
seg_labels_rot = self.normalize_0_1(seg_labels_rot)
# Change to true to visualize the paths / labels
if False:
cv2.imshow("rot_img", seg_img_rot)
cv2.imshow("seg_labels", seg_labels_rot)
rot_viz = seg_img_rot.astype(np.float64) / 512
rot_viz[:, :, 0] += seg_labels_rot.squeeze()
cv2.imshow("rot_viz", rot_viz)
cv2.waitKey(0)
tok_instruction = tokenize_instruction(instruction,
self.word2token)
instruction_t = torch.LongTensor(tok_instruction).unsqueeze(0)
# Get landmark classification labels
landmark_pos_in_seg_img = self.apply_affine_on_pts(
landmark_pos_in_img, affine)
# Down-size images and labels if requested by the model
if self.img_scale != 1.0:
seg_img_rot = transform.resize(seg_img_rot, [
seg_img_rot.shape[0] * self.img_scale,
seg_img_rot.shape[1] * self.img_scale
],
mode="constant")
seg_labels_rot = transform.resize(seg_labels_rot, [
seg_labels_rot.shape[0] * self.img_scale,
seg_labels_rot.shape[1] * self.img_scale
],
mode="constant")
landmark_pos_in_seg_img = landmark_pos_in_seg_img * self.img_scale
seg_img_rot = standardize_image(seg_img_rot)
seg_labels_rot = standardize_image(seg_labels_rot)
seg_img_t = torch.from_numpy(seg_img_rot).unsqueeze(0).float()
seg_labels_t = torch.from_numpy(seg_labels_rot).unsqueeze(
0).float()
landmark_pos_t = torch.from_numpy(
landmark_pos_in_seg_img).unsqueeze(0)
landmark_indices_t = torch.LongTensor(landmark_indices).unsqueeze(
0)
mask1 = torch.gt(landmark_pos_t, 0)
mask2 = torch.lt(landmark_pos_t, seg_img_t.size(2))
mask = mask1 * mask2
mask = mask[:, :, 0] * mask[:, :, 1]
mask = mask
landmark_pos_t = torch.masked_select(
landmark_pos_t,
mask.unsqueeze(2).expand_as(landmark_pos_t)).view([-1, 2])
landmark_indices_t = torch.masked_select(landmark_indices_t,
mask).view([-1])
mentioned_names, mentioned_indices = get_mentioned_landmarks(
self.thesaurus, instruction)
mentioned_labels_t = empty_float_tensor(
list(landmark_indices_t.size())).long()
for i, landmark_idx_present in enumerate(landmark_indices_t):
if landmark_idx_present in mentioned_indices:
mentioned_labels_t[i] = 1
aux_label = {
"landmark_pos": landmark_pos_t,
"landmark_indices": landmark_indices_t,
"landmark_mentioned": mentioned_labels_t,
"visible_mask": mask,
}
if self.include_instr_negatives:
# If we are to be using similar instructions according to the json file, then
# initialize choices with similar instructions. Otherwise let choices be empty, and they will
# be filled in the following lines.
if self.instr_negatives_similar_only:
choices = self.similar_instruction_map[str(env_id)][str(
seg_idx)]
else:
choices = []
# If there are no similar instructions to this instruction, pick a completely random instruction
if len(choices) == 0:
while len(choices) == 0:
env_options = list(self.similar_instruction_map.keys())
random_env = random.choice(env_options)
seg_options = list(
self.similar_instruction_map[random_env].keys())
if len(seg_options) == 0:
continue
random_seg = random.choice(seg_options)
choices = self.similar_instruction_map[random_env][
random_seg]
pick = random.choice(choices)
picked_env = pick["env_id"]
picked_seg = pick["seg_idx"]
picked_set = pick["set_idx"]
picked_instruction = self.all_instr[picked_env][picked_set][
"instructions"][picked_seg]["instruction"]
tok_fake_instruction = tokenize_instruction(
picked_instruction, self.word2token)
aux_label["negative_instruction"] = torch.LongTensor(
tok_fake_instruction).unsqueeze(0)
input_images.append(seg_img_t)
input_instructions.append(instruction_t)
label_images.append(seg_labels_t)
aux_labels.append(aux_label)
return [input_images, input_instructions, label_images, aux_labels]
def vtln_warp_freq(vtln_low_cutoff: float, vtln_high_cutoff: float,
low_freq: float, high_freq: float, vtln_warp_factor: float,
freq: Tensor) -> Tensor:
r"""This computes a VTLN warping function that is not the same as HTK's one,
but has similar inputs (this function has the advantage of never producing
empty bins).
This function computes a warp function F(freq), defined between low_freq
and high_freq inclusive, with the following properties:
F(low_freq) == low_freq
F(high_freq) == high_freq
The function is continuous and piecewise linear with two inflection
points.
The lower inflection point (measured in terms of the unwarped
frequency) is at frequency l, determined as described below.
The higher inflection point is at a frequency h, determined as
described below.
If l <= f <= h, then F(f) = f/vtln_warp_factor.
If the higher inflection point (measured in terms of the unwarped
frequency) is at h, then max(h, F(h)) == vtln_high_cutoff.
Since (by the last point) F(h) == h/vtln_warp_factor, then
max(h, h/vtln_warp_factor) == vtln_high_cutoff, so
h = vtln_high_cutoff / max(1, 1/vtln_warp_factor).
= vtln_high_cutoff * min(1, vtln_warp_factor).
If the lower inflection point (measured in terms of the unwarped
frequency) is at l, then min(l, F(l)) == vtln_low_cutoff
This implies that l = vtln_low_cutoff / min(1, 1/vtln_warp_factor)
= vtln_low_cutoff * max(1, vtln_warp_factor)
Args:
vtln_low_cutoff (float): Lower frequency cutoffs for VTLN
vtln_high_cutoff (float): Upper frequency cutoffs for VTLN
low_freq (float): Lower frequency cutoffs in mel computation
high_freq (float): Upper frequency cutoffs in mel computation
vtln_warp_factor (float): Vtln warp factor
freq (Tensor): given frequency in Hz
Returns:
Tensor: Freq after vtln warp
"""
assert vtln_low_cutoff > low_freq, 'be sure to set the vtln_low option higher than low_freq'
assert vtln_high_cutoff < high_freq, 'be sure to set the vtln_high option lower than high_freq [or negative]'
l = vtln_low_cutoff * max(1.0, vtln_warp_factor)
h = vtln_high_cutoff * min(1.0, vtln_warp_factor)
scale = 1.0 / vtln_warp_factor
Fl = scale * l # F(l)
Fh = scale * h # F(h)
assert l > low_freq and h < high_freq
# slope of left part of the 3-piece linear function
scale_left = (Fl - low_freq) / (l - low_freq)
# [slope of center part is just "scale"]
# slope of right part of the 3-piece linear function
scale_right = (high_freq - Fh) / (high_freq - h)
res = torch.empty_like(freq)
outside_low_high_freq = torch.lt(freq, low_freq) | torch.gt(
freq, high_freq) # freq < low_freq || freq > high_freq
before_l = torch.lt(freq, l) # freq < l
before_h = torch.lt(freq, h) # freq < h
after_h = torch.ge(freq, h) # freq >= h
# order of operations matter here (since there is overlapping frequency regions)
res[after_h] = high_freq + scale_right * (freq[after_h] - high_freq)
res[before_h] = scale * freq[before_h]
res[before_l] = low_freq + scale_left * (freq[before_l] - low_freq)
res[outside_low_high_freq] = freq[outside_low_high_freq]
return res
def forward(self, classifications, regressions, anchors, annotations):
alpha = 0.25
gamma = 2.0
batch_size = classifications.shape[0]
classification_losses = []
regression_losses = []
anchor = anchors[0, :, :]
anchor_widths = anchor[:, 2] - anchor[:, 0]
anchor_heights = anchor[:, 3] - anchor[:, 1]
anchor_ctr_x = anchor[:, 0] + 0.5 * anchor_widths
anchor_ctr_y = anchor[:, 1] + 0.5 * anchor_heights
for j in range(batch_size):
classification = classifications[j, :, :]
regression = regressions[j, :, :]
bbox_annotation = annotations[j, :, :]
bbox_annotation = bbox_annotation[bbox_annotation[:, 4] != -1]
classification = torch.clamp(classification, 1e-4, 1.0 - 1e-4)
if bbox_annotation.shape[0] == 0:
if torch.cuda.is_available():
alpha_factor = torch.ones(
classification.shape).cuda() * alpha
alpha_factor = 1. - alpha_factor
focal_weight = classification
focal_weight = alpha_factor * torch.pow(
focal_weight, gamma)
bce = -(torch.log(1.0 - classification))
# cls_loss = focal_weight * torch.pow(bce, gamma)
cls_loss = focal_weight * bce
classification_losses.append(cls_loss.sum())
regression_losses.append(torch.tensor(0).float())
else:
alpha_factor = torch.ones(classification.shape) * alpha
alpha_factor = 1. - alpha_factor
focal_weight = classification
focal_weight = alpha_factor * torch.pow(
focal_weight, gamma)
bce = -(torch.log(1.0 - classification))
# cls_loss = focal_weight * torch.pow(bce, gamma)
cls_loss = focal_weight * bce
classification_losses.append(cls_loss.sum())
regression_losses.append(torch.tensor(0).float())
continue
# Num_anchors x num_annotations
IoU = calc_iou(anchors[0, :, :], bbox_annotation[:, :4])
# Num_anchors x 1
IoU_max, IoU_argmax = torch.max(IoU, dim=1)
# compute the loss for classification
targets = torch.ones(classification.shape) * -1
if torch.cuda.is_available():
targets = targets.cuda()
targets[torch.lt(IoU_max, 0.4), :] = 0
positive_indices = torch.ge(IoU_max, 0.5)
num_positive_anchors = positive_indices.sum()
assigned_annotations = bbox_annotation[IoU_argmax, :]
targets[positive_indices, :] = 0
targets[positive_indices, assigned_annotations[positive_indices,
4].long()] = 1
if torch.cuda.is_available():
alpha_factor = torch.ones(targets.shape).cuda() * alpha
else:
alpha_factor = torch.ones(targets.shape) * alpha
alpha_factor = torch.where(torch.eq(targets, 1.), alpha_factor,
1. - alpha_factor)
focal_weight = torch.where(torch.eq(targets, 1.),
1. - classification, classification)
focal_weight = alpha_factor * torch.pow(focal_weight, gamma)
bce = -(targets * torch.log(classification) +
(1.0 - targets) * torch.log(1.0 - classification))
# cls_loss = focal_weight * torch.pow(bce, gamma)
cls_loss = focal_weight * bce
if torch.cuda.is_available():
cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss,
torch.zeros(cls_loss.shape).cuda())
else:
cls_loss = torch.where(torch.ne(targets, -1.0), cls_loss,
torch.zeros(cls_loss.shape))
classification_losses.append(
cls_loss.sum() /
torch.clamp(num_positive_anchors.float(), min=1.0))
# compute the loss for regression
if positive_indices.sum() > 0:
assigned_annotations = assigned_annotations[
positive_indices, :]
anchor_widths_pi = anchor_widths[positive_indices]
anchor_heights_pi = anchor_heights[positive_indices]
anchor_ctr_x_pi = anchor_ctr_x[positive_indices]
anchor_ctr_y_pi = anchor_ctr_y[positive_indices]
gt_widths = (assigned_annotations[:, 2] -
assigned_annotations[:, 0])
gt_heights = (assigned_annotations[:, 3] -
assigned_annotations[:, 1])
gt_ctr_x = assigned_annotations[:, 0] + 0.5 * gt_widths
gt_ctr_y = assigned_annotations[:, 1] + 0.5 * gt_heights
# clip widths to 1
gt_widths = torch.clamp(gt_widths, min=1)
gt_heights = torch.clamp(gt_heights, min=1)
targets_dx = (gt_ctr_x - anchor_ctr_x_pi) / anchor_widths_pi
targets_dy = (gt_ctr_y - anchor_ctr_y_pi) / anchor_heights_pi
targets_dw = torch.log(gt_widths / anchor_widths_pi)
targets_dh = torch.log(gt_heights / anchor_heights_pi)
targets = torch.stack(
(targets_dx, targets_dy, targets_dw, targets_dh))
targets = targets.t()
if torch.cuda.is_available():
targets = targets / torch.Tensor([[0.1, 0.1, 0.2, 0.2]
]).cuda()
else:
targets = targets / torch.Tensor([[0.1, 0.1, 0.2, 0.2]])
negative_indices = 1 + (~positive_indices)
regression_diff = torch.abs(targets -
regression[positive_indices, :])
regression_loss = torch.where(
torch.le(regression_diff, 1.0 / 9.0),
0.5 * 9.0 * torch.pow(regression_diff, 2),
regression_diff - 0.5 / 9.0)
regression_losses.append(regression_loss.mean())
else:
if torch.cuda.is_available():
regression_losses.append(torch.tensor(0).float().cuda())
else:
regression_losses.append(torch.tensor(0).float())
return (torch.stack(classification_losses).mean(dim=0, keepdim=True),
torch.stack(regression_losses).mean(dim=0, keepdim=True))
def get_mel_banks(num_bins: int, window_length_padded: int, sample_freq: float,
low_freq: float, high_freq: float, vtln_low: float,
vtln_high: float,
vtln_warp_factor: float) -> Tuple[Tensor, Tensor]:
"""
Returns:
(Tensor, Tensor): The tuple consists of ``bins`` (which is
melbank of size (``num_bins``, ``num_fft_bins``)) and ``center_freqs`` (which is
center frequencies of bins of size (``num_bins``)).
"""
assert num_bins > 3, 'Must have at least 3 mel bins'
assert window_length_padded % 2 == 0
num_fft_bins = window_length_padded / 2
nyquist = 0.5 * sample_freq
if high_freq <= 0.0:
high_freq += nyquist
assert (0.0 <= low_freq < nyquist) and (0.0 < high_freq <= nyquist) and (low_freq < high_freq), \
('Bad values in options: low-freq {} and high-freq {} vs. nyquist {}'.format(low_freq, high_freq, nyquist))
# fft-bin width [think of it as Nyquist-freq / half-window-length]
fft_bin_width = sample_freq / window_length_padded
mel_low_freq = mel_scale_scalar(low_freq)
mel_high_freq = mel_scale_scalar(high_freq)
# divide by num_bins+1 in next line because of end-effects where the bins
# spread out to the sides.
mel_freq_delta = (mel_high_freq - mel_low_freq) / (num_bins + 1)
if vtln_high < 0.0:
vtln_high += nyquist
assert vtln_warp_factor == 1.0 or ((low_freq < vtln_low < high_freq) and
(0.0 < vtln_high < high_freq) and (vtln_low < vtln_high)), \
('Bad values in options: vtln-low {} and vtln-high {}, versus '
'low-freq {} and high-freq {}'.format(vtln_low, vtln_high, low_freq, high_freq))
bin = torch.arange(num_bins).unsqueeze(1)
left_mel = mel_low_freq + bin * mel_freq_delta # size(num_bins, 1)
center_mel = mel_low_freq + (bin +
1.0) * mel_freq_delta # size(num_bins, 1)
right_mel = mel_low_freq + (bin +
2.0) * mel_freq_delta # size(num_bins, 1)
if vtln_warp_factor != 1.0:
left_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq,
vtln_warp_factor, left_mel)
center_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq,
high_freq, vtln_warp_factor,
center_mel)
right_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq,
high_freq, vtln_warp_factor, right_mel)
center_freqs = inverse_mel_scale(center_mel) # size (num_bins)
# size(1, num_fft_bins)
mel = mel_scale(fft_bin_width * torch.arange(num_fft_bins)).unsqueeze(0)
# size (num_bins, num_fft_bins)
up_slope = (mel - left_mel) / (center_mel - left_mel)
down_slope = (right_mel - mel) / (right_mel - center_mel)
if vtln_warp_factor == 1.0:
# left_mel < center_mel < right_mel so we can min the two slopes and clamp negative values
bins = torch.max(torch.zeros(1), torch.min(up_slope, down_slope))
else:
# warping can move the order of left_mel, center_mel, right_mel anywhere
bins = torch.zeros_like(up_slope)
up_idx = torch.gt(mel, left_mel) & torch.le(
mel, center_mel) # left_mel < mel <= center_mel
down_idx = torch.gt(mel, center_mel) & torch.lt(
mel, right_mel) # center_mel < mel < right_mel
bins[up_idx] = up_slope[up_idx]
bins[down_idx] = down_slope[down_idx]
return bins, center_freqs
def __call__(self, exper, epoch_obj, meta_optimizer, final_batch=False):
self.step = 0
if self.is_train:
do_continue = tensor_any(
tensor_and(torch.le(self.compare_probs, self.one_minus_eps),
torch.lt(self.counter_compare,
self.max_T))).data.cpu().numpy()[0]
else:
do_continue = True if self.step < self.horizon - 1 else False
while do_continue:
# IMPORTANT! avg_loss_step is NOT MULTIPLIED BY THE qt-value!!!!
avg_loss_step = self.act_step(exper, meta_optimizer)
if self.is_train:
do_continue = tensor_any(
tensor_and(
torch.le(self.compare_probs, self.one_minus_eps),
torch.lt(self.counter_compare,
self.max_T))).data.cpu().numpy()[0]
else:
do_continue = True if self.step < self.horizon - 1 else False
if self.eval_last_step_taken == 0:
num_next_iter = torch.sum(
self.float_mask).data.cpu().squeeze().numpy()[0]
if num_next_iter == 0. and self.eval_last_step_taken == 0.:
self.eval_last_step_taken = self.step + 1
if self.is_train and exper.args.trunc_bptt and \
(self.forward_steps == exper.args.truncated_bptt_step or not do_continue):
loss_sum, sum_grads = self.backward(epoch_obj,
meta_optimizer,
exper.optimizer,
retain_graph=False,
trunc_bptt=do_continue)
self.num_of_backwards += 1
self.total_opt_loss += loss_sum
self.total_kl_term += self.kl_term
self.total_sum_grads += sum_grads
self.last_backward_step = self.step + 1
# important increase step after "do_continue" stuff but before adding step losses
self.step += 1
exper.add_step_loss(avg_loss_step,
self.step,
is_train=self.is_train)
exper.add_opt_steps(self.step, is_train=self.is_train)
epoch_obj.add_step_loss(avg_loss_step,
last_time_step=not do_continue)
# print("final qt-probs")
# print(self.q_t[0:2, 0:self.step].data.cpu().numpy())
exper.add_step_qts(self.q_t[:, 0:self.step].data.cpu().numpy(),
is_train=self.is_train)
exper.add_halting_steps(self.halting_steps, is_train=self.is_train)
# set the class variable if we reached a new maximum time steps
if epoch_obj.get_max_time_steps_taken(self.is_train) < self.step:
epoch_obj.set_max_time_steps_taken(self.step, self.is_train)
if not self.is_train:
if self.eval_last_step_taken == 0:
self.eval_last_step_taken = self.step
epoch_obj.set_max_time_steps_taken(self.eval_last_step_taken,
self.is_train)
if exper.args.problem == "mlp" and final_batch:
# evaluate the last MLP that we optimized
accuracy = self.functions.test_model(exper.dta_set,
exper.args.cuda,
quick_test=True)
# exper.meta_logger.info("Epoch {}: last batch - accuracy of last MLP {:.4f}".format(exper.epoch, accuracy))
self.test_result_scores.append(accuracy)
dataframe
subset = dataframe[[
'Overall', 'Age', 'International Reputation', 'Weak Foot', 'Skill Moves'
]].dropna(
axis=0, how='any'
) #esto significa que eliminará las filas osea los jugadores que tengan algún valor en NAN o no tengan un numero para este caso...
#cogemos desde el 1 porque no queremos el overall
columns = subset.columns[1:]
players = torch.tensor(subset.values).float()
playersOverall = players[:, 0]
meanPlayersOverall = torch.mean(playersOverall)
meanPlayersData = torch.mean(players, dim=0)
stdPlayersData = torch.std(players, dim=0)
norm = (players - meanPlayersData) / torch.sqrt(stdPlayersData)
good = players[torch.ge(playersOverall, 85)]
average = players[torch.gt(playersOverall, 70) & torch.lt(playersOverall, 85)]
notSoGood = players[torch.le(playersOverall, 70)]
goodMean = torch.mean(good, dim=0)
averageMean = torch.mean(average, dim=0)
notSoGoodMean = torch.mean(notSoGood, dim=0)
print("media de cada aspecto de todos los jugadores:")
print(meanPlayersData)
print("---------------------------------------------------")
print("promedio de cada jugador:")
print(playersOverall)
print("---------------------------------------------------")
print("media del puntaje general del jugador:")
print(meanPlayersOverall)
print("---------------------------------------------------")
print("Estructura de los datos de los jugadores:")
print(players.shape)
def mask_from_lens(lens, max_len: Optional[int] = None):
if max_len is None:
max_len = int(lens.max().item())
ids = torch.arange(0, max_len, device=lens.device, dtype=lens.dtype)
mask = torch.lt(ids, lens.unsqueeze(1))
return mask
def forward(self, ctgs, regs, ancs, anns):
batchsz = ctgs.shape[0]
ctglosses, reglosses = [], []
anc = ancs[0, :, :]
ancw = anc[:, 2] - anc[:, 0]
anch = anc[:, 3] - anc[:, 1]
ancx = anc[:, 0] + 0.5 * ancw
ancy = anc[:, 1] + 0.5 * anch
for j in range(batchsz):
ctg = ctgs[j, :, :]
reg = regs[j, :, :]
ann = anns[j, :, :]
ann = ann[ann[:, 4] != -1]
if ann.shape[0] == 0:
reglosses.append(torch.tensor(0).float().cuda())
ctglosses.append(torch.tensor(0).float().cuda())
continue
ctg = torch.clamp(ctg, 1e-4, 1.0 - 1e-4)
ious = iou(ancs[0, :, :], ann[:, :4]) # ancnum x annnum
maxiou, argmaxiou = torch.max(ious, dim=1) # ancnum x 1
# compute the loss for classification
tgts = torch.ones(ctg.shape) * -1
tgts = tgts.cuda()
tgts[torch.lt(maxiou, 0.4), :] = 0
posinds = torch.ge(maxiou, 0.5)
posancnum = posinds.sum()
asianns = ann[argmaxiou, :]
tgts[posinds, :] = 0
tgts[posinds, asianns[posinds, 4].long()] = 1
alpha = torch.ones(tgts.shape).cuda() * self.alpha
alpha = torch.where(torch.eq(tgts, 1.), alpha, 1. - alpha)
focal = torch.where(torch.eq(tgts, 1.), 1. - ctg, ctg)
focal = alpha * torch.pow(focal, self.gamma)
bce = -(tgts * torch.log(ctg) +
(1.0 - tgts) * torch.log(1.0 - ctg))
# ctgloss = focal_weight * torch.pow(bce, gamma)
ctgloss = focal * bce
ctgloss = torch.where(torch.ne(tgts, -1.0), ctgloss,
torch.zeros(ctgloss.shape).cuda())
ctglosses.append(ctgloss.sum() /
torch.clamp(posancnum.float(), min=1.0))
# compute the loss for regression
if posinds.sum() > 0:
asianns = asianns[posinds, :]
posancw = ancw[posinds]
posanch = anch[posinds]
posancx = ancx[posinds]
posancy = ancy[posinds]
gtw = asianns[:, 2] - asianns[:, 0]
gth = asianns[:, 3] - asianns[:, 1]
gtx = asianns[:, 0] + 0.5 * gtw
gty = asianns[:, 1] + 0.5 * gth
# clip widths to 1
gtw = torch.clamp(gtw, min=1)
gth = torch.clamp(gth, min=1)
tgtdx = (gtx - posancx) / posancw
tgtdy = (gty - posancy) / posanch
tgtdw = torch.log(gtw / posancw)
tgtdh = torch.log(gth / posanch)
tgts = torch.stack((tgtdx, tgtdy, tgtdw, tgtdh))
tgts = tgts.t()
tgts = tgts / torch.Tensor([[0.1, 0.1, 0.2, 0.2]]).cuda()
neginds = 1 - posinds
regdiff = torch.abs(tgts - reg[posinds, :])
regloss = torch.where(torch.le(regdiff, 1.0 / 9.0),
0.5 * 9.0 * torch.pow(regdiff, 2),
regdiff - 0.5 / 9.0)
reglosses.append(regloss.mean())
else:
reglosses.append(torch.tensor(0).float().cuda())
return torch.stack(ctglosses).mean(
dim=0, keepdim=True), torch.stack(reglosses).mean(dim=0,
keepdim=True)
def test_lt(x, y):
c = torch.lt(torch.add(x, y), y)
return c
def apply_token_dropout(sentence_tensors, p):
dropout = torch.rand_like(sentence_tensors, dtype=torch.float).fill_(p)
rand = torch.rand_like(sentence_tensors, dtype=torch.float)
mask = torch.lt(dropout, rand).type(torch.long)
return torch.mul(sentence_tensors, mask)
def compute_projection(self, depth, camera_to_world, world_to_grid):
# compute projection by voxels -> image
#print 'camera_to_world', camera_to_world
#print 'intrinsic', self.intrinsic
#print(world_to_grid)
world_to_camera = torch.inverse(camera_to_world)
grid_to_world = torch.inverse(world_to_grid)
voxel_bounds_min, voxel_bounds_max = self.compute_frustum_bounds(world_to_grid, camera_to_world)
voxel_bounds_min = np.maximum(voxel_bounds_min, 0).cuda().float() if depth.is_cuda else np.maximum(voxel_bounds_min, 0).cpu().float()
voxel_bounds_max = np.minimum(voxel_bounds_max, self.volume_dims).cuda().float() if depth.is_cuda else np.minimum(voxel_bounds_max, self.volume_dims).cpu().float()
# coordinates within frustum bounds
# TODO python opt for this part instead of lua/torch opt?
lin_ind_volume = torch.arange(0, self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2], out=torch.LongTensor())
lin_ind_volume = lin_ind_volume.cuda() if depth.is_cuda else lin_ind_volume.cpu()
coords = camera_to_world.new(4, lin_ind_volume.size(0))
coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1])
tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long()
coords[1] = tmp / self.volume_dims[0]
coords[0] = torch.remainder(tmp, self.volume_dims[0])
coords[3].fill_(1)
mask_frustum_bounds = torch.ge(coords[0], voxel_bounds_min[0]) * torch.ge(coords[1], voxel_bounds_min[1]) * torch.ge(coords[2], voxel_bounds_min[2])
mask_frustum_bounds = mask_frustum_bounds * torch.lt(coords[0], voxel_bounds_max[0]) * torch.lt(coords[1], voxel_bounds_max[1]) * torch.lt(coords[2], voxel_bounds_max[2])
if not mask_frustum_bounds.any():
print('error: nothing in frustum bounds')
return None
lin_ind_volume = lin_ind_volume[mask_frustum_bounds]
coords = coords.resize_(4, lin_ind_volume.size(0))
coords[2] = lin_ind_volume / (self.volume_dims[0]*self.volume_dims[1])
tmp = lin_ind_volume - (coords[2]*self.volume_dims[0]*self.volume_dims[1]).long()
coords[1] = tmp / self.volume_dims[0]
coords[0] = torch.remainder(tmp, self.volume_dims[0])
coords[3].fill_(1)
# transform to current frame
p = torch.mm(world_to_camera, torch.mm(grid_to_world, coords))
# project into image
p[0] = (p[0] * self.intrinsic[0][0]) / p[2] + self.intrinsic[0][2]
p[1] = (p[1] * self.intrinsic[1][1]) / p[2] + self.intrinsic[1][2]
pi = torch.round(p).long()
valid_ind_mask = torch.ge(pi[0], 0) * torch.ge(pi[1], 0) * torch.lt(pi[0], self.image_dims[0]) * torch.lt(pi[1], self.image_dims[1])
if not valid_ind_mask.any():
print('error: no valid image indices')
return None
valid_image_ind_x = pi[0][valid_ind_mask]
valid_image_ind_y = pi[1][valid_ind_mask]
valid_image_ind_lin = valid_image_ind_y * self.image_dims[0] + valid_image_ind_x
depth_vals = torch.index_select(depth.view(-1), 0, valid_image_ind_lin)
depth_mask = depth_vals.ge(self.depth_min) * depth_vals.le(self.depth_max) * torch.abs(depth_vals - p[2][valid_ind_mask]).le(self.voxel_size)
if not depth_mask.any():
print('error: no valid depths')
return None
lin_ind_update = lin_ind_volume[valid_ind_mask]
lin_ind_update = lin_ind_update[depth_mask]
lin_indices_3d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size)
lin_indices_2d = lin_ind_update.new(self.volume_dims[0]*self.volume_dims[1]*self.volume_dims[2] + 1) #needs to be same size for all in batch... (first element has size)
lin_indices_3d[0] = lin_ind_update.shape[0]
lin_indices_2d[0] = lin_ind_update.shape[0]
lin_indices_3d[1:1+lin_indices_3d[0]] = lin_ind_update
lin_indices_2d[1:1+lin_indices_2d[0]] = torch.index_select(valid_image_ind_lin, 0, torch.nonzero(depth_mask)[:,0])
num_ind = lin_indices_3d[0]
#print '[proj] #ind = ', lin_indices_3d[0]
#print '2d', torch.min(lin_indices_2d[1:1+num_ind]), torch.max(lin_indices_2d[1:1+num_ind])
#print '3d', torch.min(lin_indices_3d[1:1+num_ind]), torch.max(lin_indices_3d[1:1+num_ind])
return lin_indices_3d, lin_indices_2d
def equal(x, y, prec=1e-4):
return torch.all(torch.lt(torch.abs(torch.add(x, -y)), prec))
def get_adj(batch):
#Not very nice but we do not have access to value comming from opt.gpuid command line parameter here.
use_cuda = batch.src[0].is_cuda
node1_index = batch.node1.data.transpose(
0, 1) # [ seqLen x batch_size ] ==> [ batch_size x seqLen ]
node2_index = batch.node2.data.transpose(0, 1)
label_index = batch.label.data.transpose(0, 1)
node1_voc = batch.dataset.fields['node1'].vocab.itos
node2_voc = batch.dataset.fields['node2'].vocab.itos
label_voc = batch.dataset.fields['label'].vocab.itos
batch_size = batch.batch_size
_MAX_BATCH_LEN = batch.src[0].data.size()[
0] # data is [ seqLen x batch_size ]
_MAX_DEGREE = 10 # If the average degree is much higher than this, it must be changed.
sent_mask = torch.lt(torch.eq(batch.src[0].data, 1),
1) # if there is a sent, then mask=1
# each batch has _MAX_BATCH_LEN nodes
# each node has _MAX_DEGREE lines as the adj list
# the first row is the batch id, the second row is the neighbor id (or the label id)
# mask indicates whether or not one slot in the adj_list is used, because each node has 10 slots
adj_arc_in = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 2),
dtype='int32')
adj_lab_in = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 1),
dtype='int32')
adj_arc_out = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 2),
dtype='int32')
adj_lab_out = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE, 1),
dtype='int32')
mask_in = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE),
dtype='float32')
mask_out = np.zeros((batch_size * _MAX_BATCH_LEN * _MAX_DEGREE),
dtype='float32')
mask_loop = np.ones((batch_size * _MAX_BATCH_LEN, 1), dtype='float32')
tmp_in = {}
tmp_out = {}
for d, de in enumerate(node1_index): # iterates over the batch
for a, arc in enumerate(de):
# d: batch index, a: triple index triple: node1, node2, label
arc_0 = label_voc[label_index[d, a]]
if arc_0 == '<unk>' or arc_0 == '<pad>':
pass
else:
# arc_1, arc2, arc_0: node1, node2, label
arc_1 = int(node1_voc[arc]) # arc = node1_index[d, a]
arc_2 = int(node2_voc[node2_index[d, a]])
if arc_1 in tmp_in:
tmp_in[arc_1] += 1
else:
tmp_in[arc_1] = 0
if arc_2 in tmp_out:
tmp_out[arc_2] += 1
else:
tmp_out[arc_2] = 0
# (d * _MAX_BATCH_LEN * _MAX_DEGREE): offset of current batch
# arc_1 * _MAX_DEGREE: offset of current node
# tmp_in[arc_1]:
idx_in = (d * _MAX_BATCH_LEN *
_MAX_DEGREE) + arc_1 * _MAX_DEGREE + tmp_in[arc_1]
idx_out = (d * _MAX_BATCH_LEN *
_MAX_DEGREE) + arc_2 * _MAX_DEGREE + tmp_out[arc_2]
if tmp_in[arc_1] < _MAX_DEGREE:
# store the batch_id, dest node, label
adj_arc_in[idx_in] = np.array([d, arc_2]) # incoming arcs
adj_lab_in[idx_in] = np.array([label_index[d, a]
]) # incoming arcs
mask_in[idx_in] = 1.
if tmp_out[arc_2] < _MAX_DEGREE:
adj_arc_out[idx_out] = np.array([d,
arc_1]) # outgoing arcs
adj_lab_out[idx_out] = np.array([label_index[d, a]
]) # outgoing arcs
mask_out[idx_out] = 1.
tmp_in = {}
tmp_out = {}
adj_arc_in = autograd.Variable(
torch.LongTensor(np.transpose(adj_arc_in).tolist()))
adj_arc_out = autograd.Variable(
torch.LongTensor(np.transpose(adj_arc_out).tolist()))
adj_lab_in = autograd.Variable(
torch.LongTensor(np.transpose(adj_lab_in).tolist()))
adj_lab_out = autograd.Variable(
torch.LongTensor(np.transpose(adj_lab_out).tolist()))
mask_in = autograd.Variable(torch.FloatTensor(
mask_in.reshape(
(_MAX_BATCH_LEN * node1_index.size()[0], _MAX_DEGREE)).tolist()),
requires_grad=False)
mask_out = autograd.Variable(torch.FloatTensor(
mask_out.reshape(
(_MAX_BATCH_LEN * node1_index.size()[0], _MAX_DEGREE)).tolist()),
requires_grad=False)
mask_loop = autograd.Variable(torch.FloatTensor(mask_loop.tolist()),
requires_grad=False)
sent_mask = autograd.Variable(torch.FloatTensor(sent_mask.tolist()),
requires_grad=False)
if use_cuda:
adj_arc_in = adj_arc_in.cuda()
adj_arc_out = adj_arc_out.cuda()
adj_lab_in = adj_lab_in.cuda()
adj_lab_out = adj_lab_out.cuda()
mask_in = mask_in.cuda()
mask_out = mask_out.cuda()
mask_loop = mask_loop.cuda()
sent_mask = sent_mask.cuda()
return adj_arc_in, adj_arc_out, adj_lab_in, adj_lab_out, mask_in, mask_out, mask_loop, sent_mask
self.stride,
padding=self.padding)
self.alpha_frequencies = np.zeros(2)
self.alpha = random.uniform(0, 1)
def forward(self, x):
#######训练阶段，混合加权池化
if self.training:
self.alpha_frequencies[0] += self.alpha
self.alpha_frequencies[1] += 1 - self.alpha
out = self.alpha * self.maxpool(x) + (1 -
self.alpha) * self.avgpool(x)
else:
#######测试阶段，根据频次，选择池化方法
if (self.alpha_frequencies[0] < self.alpha_frequencies[1]):
out = self.avgpool(x)
else:
out = self.maxpool(x)
return out
if __name__ == "__main__":
print("=" * 10 + "MixPool2D" + "=" * 10)
avgpool = "avgpool"
maxpool = "maxpool"
a = torch.Tensor([[1], [2]], [[3], [4]])
b = torch.Tensor([[5], [6]], [[1], [3]])
mixpool = MixPooling2D()
print(mixpool.training)
print(torch.lt(a, b))
def rprop(opfunc, x, config, state=None):
""" A plain implementation of RPROP
ARGS:
- `opfunc` : a function that takes a single input (X), the point of
evaluation, and returns f(X) and df/dX
- `x` : the initial point
- `state` : a table describing the state of the optimizer; after each
call the state is modified
- `state['stepsize']` : initial step size, common to all components
- `state['etaplus']` : multiplicative increase factor, > 1 (default 1.2)
- `state['etaminus']` : multiplicative decrease factor, < 1 (default 0.5)
- `state['stepsizemax']` : maximum stepsize allowed (default 50)
- `state['stepsizemin']` : minimum stepsize allowed (default 1e-6)
- `state['niter']` : number of iterations (default 1)
RETURN:
- `x` : the new x vector
- `f(x)` : the function, evaluated before the update
(Martin Riedmiller, Koray Kavukcuoglu 2013)
"""
if config is None and state is None:
raise ValueError("rprop requires a dictionary to retain state between iterations")
# (0) get/update state
state = state if state is not None else config
stepsize = config.get('stepsize', 0.1)
etaplus = config.get('etaplus', 1.2)
etaminus = config.get('etaminus', 0.5)
stepsizemax = config.get('stepsizemax', 50.0)
stepsizemin = config.get('stepsizemin', 1e-06)
niter = config.get('niter', 1)
hfx = []
for i in range(niter):
# (1) evaluate f(x) and df/dx
fx, dfdx = opfunc(x)
# init temp storage
if 'delta' not in state:
state['delta'] = dfdx.new(dfdx.size()).zero_()
state['stepsize'] = dfdx.new(dfdx.size()).fill_(stepsize)
state['sign'] = dfdx.new(dfdx.size())
state['bytesign'] = torch.ByteTensor(dfdx.size())
state['psign'] = torch.ByteTensor(dfdx.size())
state['nsign'] = torch.ByteTensor(dfdx.size())
state['zsign'] = torch.ByteTensor(dfdx.size())
state['dminmax'] = torch.ByteTensor(dfdx.size())
if str(type(x)).find('Cuda') > -1:
# Push to GPU
state['psign'] = state['psign'].cuda()
state['nsign'] = state['nsign'].cuda()
state['zsign'] = state['zsign'].cuda()
state['dminmax'] = state['dminmax'].cuda()
# sign of derivative from last step to this one
torch.mul(dfdx, state['delta'], out=state['sign']).sign_()
# get indices of >0, <0 and ==0 entries
torch.gt(state['sign'], 0, out=state['psign'])
torch.lt(state['sign'], 0, out=state['nsign'])
torch.eq(state['sign'], 0, out=state['zsign'])
# get step size updates
state['sign'][state['psign']] = etaplus
state['sign'][state['nsign']] = etaminus
state['sign'][state['zsign']] = 1
# update stepsizes with step size updates
state['stepsize'].mul_(state['sign'])
# threshold step sizes
# >50 => 50
torch.gt(state['stepsize'], stepsizemax, out=state['dminmax'])
state['stepsize'][state['dminmax']] = stepsizemax
# <1e-6 ==> 1e-6
torch.lt(state['stepsize'], stepsizemin, out=state['dminmax'])
state['stepsize'][state['dminmax']] = stepsizemin
# for dir<0, dfdx=0
# for dir>=0 dfdx=dfdx
dfdx[state['nsign']] = 0
torch.sign(dfdx, out=state['sign'])
# update weights
x.addcmul_(-1, state['sign'], state['stepsize'])
# update state['dfdx'] with current dfdx
state['delta'].copy_(dfdx)
hfx.append(fx)
# return x*, table of f(x) values from each step
return x, hfx
def get_dgl_batched_graph(batch):
edge_to_id = {
e: i
for i, e in enumerate(['sp', 'po', 'op', 'ps', 'll', 'ne'])
}
#Not very nice but we do not have access to value comming from opt.gpuid command line parameter here.
use_cuda = batch.src[0].is_cuda
nodes_id = batch.src[0].transpose(0, 1)
nodes_size = batch.src[1].cpu().numpy()
node1_index = batch.node1.data.transpose(
0, 1) # [ seqLen x batch_size ] ==> [ batch_size x seqLen ]
node2_index = batch.node2.data.transpose(0, 1)
label_index = batch.label.data.transpose(0, 1)
node1_voc = batch.dataset.fields['node1'].vocab.itos
node2_voc = batch.dataset.fields['node2'].vocab.itos
label_voc = batch.dataset.fields['label'].vocab.itos
batch_size = batch.batch_size
_MAX_BATCH_LEN = batch.src[0].data.size()[
0] # data is [ seqLen x batch_size ]
_MAX_DEGREE = 10 # If the average degree is much higher than this, it must be changed.
sent_mask = torch.lt(torch.eq(batch.src[0].data, 1),
1) # if there is a sent, then mask=1
g_list = []
for d, de in enumerate(node1_index): # iterates over the batch
g = dgl.DGLGraph()
node_size = nodes_size[d]
g.add_nodes(int(node_size))
g.ndata['global_id'] = nodes_id[d][:node_size]
g.ndata['spo'] = torch.zeros(node_size, 3).float()
edge_list = []
edge_type_list = []
# we guess the max_node_size is no longer than 20 ...
node_spo_feature = torch.zeros(node_size, 3).float()
node_set = set()
for a, arc in enumerate(de):
# d: batch index, a: triple index triple: node1, node2, label
arc_0 = label_voc[label_index[d, a]]
if arc_0 == '<unk>' or arc_0 == '<pad>':
pass
else:
# arc_1, arc2, arc_0: node1, node2, label
# by indexing we get the local id of each node/edge
arc_1 = int(node1_voc[arc]) # arc = node1_index[d, a]
arc_2 = int(node2_voc[node2_index[d, a]])
if arc_0 == "A0":
g.nodes[arc_1].data['spo'] += torch.tensor([1.0, 0, 0])
g.nodes[arc_2].data['spo'] += torch.tensor([0, 0.5, 0])
edge_list.append((arc_1, arc_2))
edge_list.append((arc_2, arc_1))
edge_list.append((arc_1, arc_1))
edge_list.append((arc_2, arc_2))
edge_type_list += [
edge_to_id[e] for e in ['sp', 'ps', 'll', 'll']
]
elif arc_0 == "A1":
g.nodes[arc_1].data['spo'] += torch.tensor([0, 0, 1.0])
g.nodes[arc_2].data['spo'] += torch.tensor([0, 0.5, 0])
edge_list.append((arc_1, arc_2))
edge_list.append((arc_2, arc_1))
edge_list.append((arc_1, arc_1))
edge_list.append((arc_2, arc_2))
edge_type_list += [
edge_to_id[e] for e in ['op', 'po', 'll', 'll']
]
elif arc_0 == 'NE':
edge_list.append((arc_1, arc_2))
edge_type_list += [edge_to_id["ne"]]
else:
raise ValueError(
"Do not support the edge type {}".format(arc_0))
new_edge_list, new_edge_type_list = [], []
added_edge = set()
for edge, edge_type in zip(edge_list, edge_type_list):
# if edge_type == edge_to_id["ll"]:
# continue
edge_id = "{}-{}-{}".format(edge[0], edge[1], edge_type)
if edge_id not in added_edge:
added_edge.add(edge_id)
new_edge_list.append(edge)
new_edge_type_list.append(edge_type)
src, dst = tuple(zip(*new_edge_list))
g.add_edges(src, dst)
g.ndata['spo'] = torch.gt(g.ndata['spo'], 0)
g.edata['type'] = torch.tensor(new_edge_type_list).reshape(-1, 1)
dst_in_deg = {}
for dst_node in set(dst):
if dst_node not in dst_in_deg:
dst_in_deg[dst_node] = {}
for edge_type in range(len(edge_to_id)):
if edge_type not in dst_in_deg[dst_node]:
dst_in_deg[dst_node][edge_type] = 0
for dst_node, edge_type in zip(dst, new_edge_type_list):
dst_in_deg[dst_node][edge_type] += 1
e_norm = [
1.0 / dst_in_deg[dst_node][e_type]
for dst_node, e_type in zip(dst, new_edge_type_list)
]
g.edata['norm'] = torch.tensor(e_norm).reshape(-1, 1)
g_list.append(g)
return dgl.batch(g_list)
