def mrcnn_bbox_loss(target_bbox, target_class_ids, pred_bbox):
"""Loss for Mask R-CNN bounding box refinement.
target_bbox: [batch, num_rois, (dy, dx, log(dh), log(dw))]
target_class_ids: [batch, num_rois]. Integer class IDs.
pred_bbox: [batch, num_rois, num_classes, (dy, dx, log(dh), log(dw))]
"""
# Reshape to merge batch and roi dimensions for simplicity.
target_class_ids = target_class_ids.contiguous().view(-1)
target_bbox = target_bbox.contiguous().view(-1, 4)
pred_bbox = pred_bbox.contiguous().view(-1, pred_bbox.size()[2], 4)
# print(target_class_ids)
# Only positive ROIs contribute to the loss. And only
# the right class_id of each ROI. Get their indicies.
positive_roi_ix = torch.gt(target_class_ids , 0)
# print(positive_roi_ix)
positive_roi_class_ids = torch.masked_select(target_class_ids, positive_roi_ix)
indices = target_class_ids
# indices = torch.stack([positive_roi_ix, positive_roi_class_ids], dim=1)
# print(indices)
# Gather the deltas (predicted and true) that contribute to loss
# target_bbox = torch.gather(target_bbox, positive_roi_ix)
# pred_bbox = torch.gather(pred_bbox, indices)
loss = F.smooth_l1_loss(pred_bbox, target_bbox, size_average=True)
return loss
def forward(self, tgt_seq, tgt_pos, src_seq, enc_output, return_attns=False):
# Word embedding look up
dec_input = self.tgt_word_emb(tgt_seq)
# Position Encoding addition
dec_input += self.position_enc(tgt_pos)
# Decode
dec_slf_attn_pad_mask = get_attn_padding_mask(tgt_seq, tgt_seq)
dec_slf_attn_sub_mask = get_attn_subsequent_mask(tgt_seq)
dec_slf_attn_mask = torch.gt(dec_slf_attn_pad_mask + dec_slf_attn_sub_mask, 0)
dec_enc_attn_pad_mask = get_attn_padding_mask(tgt_seq, src_seq)
if return_attns:
dec_slf_attns, dec_enc_attns = [], []
dec_output = dec_input
for dec_layer in self.layer_stack:
dec_output, dec_slf_attn, dec_enc_attn = dec_layer(
dec_output, enc_output,
slf_attn_mask=dec_slf_attn_mask,
dec_enc_attn_mask=dec_enc_attn_pad_mask)
if return_attns:
dec_slf_attns += [dec_slf_attn]
dec_enc_attns += [dec_enc_attn]
if return_attns:
return dec_output, dec_slf_attns, dec_enc_attns
else:
return dec_output,
def sparsify(model, sparsity_level=50.):
for name, param in model.named_parameters():
if 'weight' in name:
threshold = calculate_threshold(param.data, sparsity_level)
mask = torch.gt(torch.abs(param.data), threshold).float()
param.data = param.data * mask
return model
def updateGradInput(self, input, y):
self.gradInput.resize_as_(input).copy_(y)
self.gradInput[torch.mul(torch.eq(y, -1), torch.gt(input, self.margin))] = 0
if self.sizeAverage:
self.gradInput.mul_(1. / input.nelement())
return self.gradInput
def backward(self, grad_output):
input, target = self.saved_tensors
grad_input = input.new().resize_as_(input).copy_(target)
grad_input[torch.mul(torch.eq(target, -1), torch.gt(input, self.margin))] = 0
if self.size_average:
grad_input.mul_(1. / input.nelement())
if grad_output[0] != 1:
grad_input.mul_(grad_output[0])
return grad_input, None
def forward(ctx, input, target, grad_output, margin, size_average):
ctx.margin = margin
ctx.size_average = size_average
ctx.save_for_backward(input, target, grad_output)
grad_input = input.new().resize_as_(input).copy_(target)
grad_input[torch.mul(torch.eq(target, -1), torch.gt(input, ctx.margin))] = 0
if ctx.size_average:
grad_input.mul_(1. / input.nelement())
if grad_output[0] != 1:
grad_input.mul_(grad_output[0])
return grad_input
def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask,
layer_cache=None, step=None):
"""
Args:
inputs (FloatTensor): ``(batch_size, 1, model_dim)``
memory_bank (FloatTensor): ``(batch_size, src_len, model_dim)``
src_pad_mask (LongTensor): ``(batch_size, 1, src_len)``
tgt_pad_mask (LongTensor): ``(batch_size, 1, 1)``
Returns:
(FloatTensor, FloatTensor):
* output ``(batch_size, 1, model_dim)``
* attn ``(batch_size, 1, src_len)``
"""
dec_mask = None
if step is None:
tgt_len = tgt_pad_mask.size(-1)
future_mask = torch.ones(
[tgt_len, tgt_len],
device=tgt_pad_mask.device,
dtype=torch.uint8)
future_mask = future_mask.triu_(1).view(1, tgt_len, tgt_len)
dec_mask = torch.gt(tgt_pad_mask + future_mask, 0)
input_norm = self.layer_norm_1(inputs)
if isinstance(self.self_attn, MultiHeadedAttention):
query, attn = self.self_attn(input_norm, input_norm, input_norm,
mask=dec_mask,
layer_cache=layer_cache,
type="self")
elif isinstance(self.self_attn, AverageAttention):
query, attn = self.self_attn(input_norm, mask=dec_mask,
layer_cache=layer_cache, step=step)
query = self.drop(query) + inputs
query_norm = self.layer_norm_2(query)
mid, attn = self.context_attn(memory_bank, memory_bank, query_norm,
mask=src_pad_mask,
layer_cache=layer_cache,
type="context")
output = self.feed_forward(self.drop(mid) + query)
return output, attn
def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask,
previous_input=None):
# Args Checks
input_batch, input_len, _ = inputs.size()
if previous_input is not None:
pi_batch, _, _ = previous_input.size()
aeq(pi_batch, input_batch)
contxt_batch, contxt_len, _ = memory_bank.size()
aeq(input_batch, contxt_batch)
src_batch, t_len, s_len = src_pad_mask.size()
tgt_batch, t_len_, t_len__ = tgt_pad_mask.size()
aeq(input_batch, contxt_batch, src_batch, tgt_batch)
# aeq(t_len, t_len_, t_len__, input_len)
aeq(s_len, contxt_len)
# END Args Checks
dec_mask = torch.gt(tgt_pad_mask +
self.mask[:, :tgt_pad_mask.size(1),
:tgt_pad_mask.size(1)], 0)
input_norm = self.layer_norm_1(inputs)
all_input = input_norm
if previous_input is not None:
all_input = torch.cat((previous_input, input_norm), dim=1)
dec_mask = None
query, attn = self.self_attn(all_input, all_input, input_norm,
mask=dec_mask)
query = self.drop(query) + inputs
query_norm = self.layer_norm_2(query)
mid, attn = self.context_attn(memory_bank, memory_bank, query_norm,
mask=src_pad_mask)
output = self.feed_forward(self.drop(mid) + query)
# CHECKS
output_batch, output_len, _ = output.size()
aeq(input_len, output_len)
aeq(contxt_batch, output_batch)
n_batch_, t_len_, s_len_ = attn.size()
aeq(input_batch, n_batch_)
aeq(contxt_len, s_len_)
aeq(input_len, t_len_)
# END CHECKS
return output, attn, all_input
def forward(ctx, gboxes, qboxes, aligned=False):
assert gboxes.shape[0] == qboxes.shape[0]
indicator = torch.gt(gboxes[:, 3], 0) & torch.gt(gboxes[:, 4], 0) & torch.gt(gboxes[:, 5], 0) \
& torch.gt(qboxes[:, 3], 0) & torch.gt(qboxes[:, 4], 0) & torch.gt(qboxes[:, 5], 0)
index_loc = torch.nonzero(indicator)
## if we want to compute the gious of two aligned rectangles
gious = torch.zeros([gboxes.shape[0], ], device=gboxes.device, dtype=torch.float32)
if (aligned):
align_inter_aligned_object = align_inter_aligned()
inter_area_xoz, mbr_area_xoz, inter_area_xoy, mbr_area_xoy, inter_area_yoz, mbr_area_yoz = align_inter_aligned_object(
gboxes, qboxes)
volume_gboxes = gboxes[:, 3].mul(gboxes[:, 4]).mul(gboxes[:, 5])
volume_qboxes = qboxes[:, 3].mul(qboxes[:, 4]).mul(qboxes[:, 5])
## for three different views xoz plane
# inter_h = (torch.min(gboxes[:, 1], qboxes[:, 1]) - torch.max(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4]))
# oniou_h = (torch.max(gboxes[:, 1], qboxes[:, 1]) - torch.min(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4]))
# inter_h[inter_h < 0] = 0
# oniou_h[oniou_h < 0] = 0
# inter_area_xoz_cuda = inter_area_xoz.to(torch.device(gboxes.device))
# mbr_area_xoz_cuda = mbr_area_xoz.to(torch.device(gboxes.device))
# volume_inc = inter_h.mul(inter_area_xoz_cuda)
# volume_con = oniou_h.mul(mbr_area_xoz_cuda)
# volume_union = (volume_gboxes + volume_qboxes - volume_inc)
# volume_ca = volume_con - volume_union
# ious = torch.div(volume_inc, volume_union)
union_xoz = gboxes[:, 3].mul(gboxes[:, 5]) + qboxes[:, 3].mul(qboxes[:, 5]) - inter_area_xoz
iou_xoz = torch.div(inter_area_xoz, union_xoz)
iou_bis_xoz = torch.div(mbr_area_xoz - union_xoz, mbr_area_xoz)
gious_xoz = iou_xoz - iou_bis_xoz
## for xoy plane
union_xoy = gboxes[:, 3].mul(gboxes[:, 4]) + qboxes[:, 3].mul(qboxes[:, 4]) - inter_area_xoy
iou_xoy = torch.div(inter_area_xoy, union_xoy)
iou_bis_xoy = torch.div(mbr_area_xoy - union_xoy, mbr_area_xoy)
gious_xoy = iou_xoy - iou_bis_xoy
## for yoz plane
union_yoz = gboxes[:, 4].mul(gboxes[:, 5]) + qboxes[:, 4].mul(qboxes[:, 5]) - inter_area_xoy
iou_yoz = torch.div(inter_area_yoz, union_yoz)
iou_bis_yoz = torch.div(mbr_area_yoz - union_yoz, mbr_area_yoz)
gious_xoy = iou_yoz - iou_bis_yoz
gious[index_loc[:, 0]] = (gious_xoz[index_loc[:, 0]] + gious_xoy[index_loc[:, 0]] + gious_xoy[
index_loc[:, 0]]) / 3.0
# for i in range(inter_area_xoz.shape[0]):
# if (gious[i] > 1):
# print("infor: (%.4f %.4f %.4f %.4f %.4f %.4f %.4f,%.4f %.4f %.4f %.4f)"
# % (i, inter_h[i], oniou_h[i], inter_area_xoz[i], mbr_area_xoz[i], ious[i], gious[i], volume_inc[i],
# volume_con[i], volume_union[i], volume_ca[i]))
# elif (gious[i] < -1):
# print("infor: (%.4f %.4f %.4f %.4f %.4f %.4f %.4f,%.4f %.4f %.4f %.4f)"
# % (i, inter_h[i], oniou_h[i], inter_area_xoz[i], mbr_area_xoz[i], ious[i], gious[i], volume_inc[i],
# volume_con[i], volume_union[i], volume_ca[i]))
else:
rbbox_to_corners_object = rbbox_to_corners()
corners_gboxes = rbbox_to_corners_object(gboxes[:, [0, 2, 3, 5, 6]])
corners_qboxes = rbbox_to_corners_object(qboxes[:, [0, 2, 3, 5, 6]])
# compute the inter area
rinter_area_compute_object = rinter_area_compute()
inter_area = rinter_area_compute_object(corners_gboxes, corners_qboxes)
corners_gboxes_1 = torch.stack((corners_gboxes[:, [0, 2, 4, 6]], corners_gboxes[:, [1, 3, 5, 7]]), 2)
corners_qboxes_1 = torch.stack((corners_qboxes[:, [0, 2, 4, 6]], corners_qboxes[:, [1, 3, 5, 7]]), 2)
corners_pts = torch.cat((corners_gboxes_1, corners_qboxes_1), 1)
# compute the mbr area
mbr_area_compute_object = mbr_area_compute()
mbr_area = mbr_area_compute_object(corners_pts)
## Compute the gious for 3D
inter_h = (torch.min(gboxes[:, 1], qboxes[:, 1]) - torch.max(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4]))
oniou_h = (torch.max(gboxes[:, 1], qboxes[:, 1]) - torch.min(gboxes[:, 1] - gboxes[:, 4], qboxes[:, 1] - qboxes[:, 4]))
inter_h[inter_h < 0] = 0
volume_gboxes = gboxes[:, 3].mul(gboxes[:, 4]).mul(gboxes[:, 5])
volume_qboxes = qboxes[:, 3].mul(qboxes[:, 4]).mul(qboxes[:, 5])
inter_area_cuda = inter_area.to(torch.device(gboxes.device))
mbr_area_cuda = mbr_area.to(torch.device(gboxes.device))
volume_inc = inter_h.mul(inter_area_cuda)
volume_con = oniou_h.mul(mbr_area_cuda)
volume_union = (volume_gboxes + volume_qboxes - volume_inc)
volume_ca = volume_con - volume_union
ious = torch.div(volume_inc, volume_union)
gious = torch.zeros([gboxes.shape[0],], device=gboxes.device, dtype=torch.float32)
gious[index_loc[:, 0]] = ious[index_loc[:, 0]] - torch.div(volume_ca[index_loc[:, 0]], volume_con[index_loc[:, 0]])
# for i in range(inter_area.shape[0]):
# if(gious[i] < -1):
# print("infor: (%.4f %.4f %.4f %.4f %.4f %.4f %.4f,%.4f %.4f %.4f %.4f)"
# %(i,inter_h[i], oniou_h[i], inter_area[i], mbr_area[i], ious[i], gious[i],volume_inc[i],volume_con[i],volume_union[i],volume_ca[i]))
return torch.unsqueeze(gious, 1)
def _forward(
self, inputs, tgt_pad_mask, layer_cache=None, step=None, future=False
):
"""A naive forward pass for transformer decoder.
# T: could be 1 in the case of stepwise decoding or tgt_len
Args:
inputs (FloatTensor): ``(batch_size, T, model_dim)``
tgt_pad_mask (bool): ``(batch_size, 1, T)``
layer_cache (dict or None): cached layer info when stepwise decode
step (int or None): stepwise decoding counter
future (bool): If set True, do not apply future_mask.
Returns:
(FloatTensor, FloatTensor):
* output ``(batch_size, T, model_dim)``
* attns ``(batch_size, head, T, T)``
"""
dec_mask = None
if step is None:
tgt_len = tgt_pad_mask.size(-1)
if not future: # apply future_mask, result mask in (B, T, T)
future_mask = torch.ones(
[tgt_len, tgt_len],
device=tgt_pad_mask.device,
dtype=torch.uint8,
)
future_mask = future_mask.triu_(1).view(1, tgt_len, tgt_len)
# BoolTensor was introduced in pytorch 1.2
try:
future_mask = future_mask.bool()
except AttributeError:
pass
dec_mask = torch.gt(tgt_pad_mask + future_mask, 0)
else: # only mask padding, result mask in (B, 1, T)
dec_mask = tgt_pad_mask
inputs_norm = self.layer_norm_1(inputs)
if isinstance(self.self_attn, MultiHeadedAttention):
query, attns = self.self_attn(
inputs_norm,
inputs_norm,
inputs_norm,
mask=dec_mask,
layer_cache=layer_cache,
attn_type="self",
)
elif isinstance(self.self_attn, AverageAttention):
query, attns = self.self_attn(
inputs_norm, mask=dec_mask, layer_cache=layer_cache, step=step
)
output = self.drop(query) + inputs
output_feedforward = self.feed_forward(self.layer_norm_2(output))
output_norm = self.drop(output_feedforward) + output
return output_norm, attns
def gt(*args, **kwargs):
return torch.gt(*args, **kwargs)
def loss(self, H1, H2):
"""
It is the loss function of CCA as introduced in the original paper. There can be other formulations.
"""
r1 = 1e-3
r2 = 1e-3
eps = 1e-9
H1, H2 = H1.t(), H2.t()
# assert torch.isnan(H1).sum().item() == 0
# assert torch.isnan(H2).sum().item() == 0
o1 = o2 = H1.size(0)
m = H1.size(1)
# print(H1.size())
H1bar = H1 - H1.mean(dim=1).unsqueeze(dim=1)
H2bar = H2 - H2.mean(dim=1).unsqueeze(dim=1)
# assert torch.isnan(H1bar).sum().item() == 0
# assert torch.isnan(H2bar).sum().item() == 0
SigmaHat12 = (1.0 / (m - 1)) * torch.matmul(H1bar, H2bar.t())
SigmaHat11 = (1.0 / (m - 1)) * torch.matmul(
H1bar, H1bar.t()) + r1 * torch.eye(o1, device=self.device)
SigmaHat22 = (1.0 / (m - 1)) * torch.matmul(
H2bar, H2bar.t()) + r2 * torch.eye(o2, device=self.device)
# assert torch.isnan(SigmaHat11).sum().item() == 0
# assert torch.isnan(SigmaHat12).sum().item() == 0
# assert torch.isnan(SigmaHat22).sum().item() == 0
# Calculating the root inverse of covariance matrices by using eigen decomposition
[D1, V1] = torch.symeig(SigmaHat11, eigenvectors=True)
[D2, V2] = torch.symeig(SigmaHat22, eigenvectors=True)
# assert torch.isnan(D1).sum().item() == 0
# assert torch.isnan(D2).sum().item() == 0
# assert torch.isnan(V1).sum().item() == 0
# assert torch.isnan(V2).sum().item() == 0
# Added to increase stability
posInd1 = torch.gt(D1, eps).nonzero()[:, 0]
D1 = D1[posInd1]
V1 = V1[:, posInd1]
posInd2 = torch.gt(D2, eps).nonzero()[:, 0]
D2 = D2[posInd2]
V2 = V2[:, posInd2]
# print(posInd1.size())
# print(posInd2.size())
SigmaHat11RootInv = torch.matmul(
torch.matmul(V1, torch.diag(D1**-0.5)), V1.t())
SigmaHat22RootInv = torch.matmul(
torch.matmul(V2, torch.diag(D2**-0.5)), V2.t())
Tval = torch.matmul(torch.matmul(SigmaHat11RootInv, SigmaHat12),
SigmaHat22RootInv)
# print(Tval.size())
if self.use_all_singular_values:
# all singular values are used to calculate the correlation
tmp = torch.trace(torch.matmul(Tval.t(), Tval))
# print(tmp)
corr = torch.sqrt(tmp)
# assert torch.isnan(corr).item() == 0
else:
# just the top self.outdim_size singular values are used
U, V = torch.symeig(torch.matmul(Tval.t(), Tval),
eigenvectors=True)
# U = U[torch.gt(U, eps).nonzero()[:, 0]]
U = U.topk(self.outdim_size)[0]
corr = torch.sum(torch.sqrt(U))
return -corr
def cosine_similarity_selector_IQP_Exact(x1,
solver,
nb_selected,
eps=1e-3,
slack=0.01):
"""
Integer programming
"""
# x1=gradient memories
x2 = None
w1 = x1.norm(p=2, dim=1, keepdim=True)
inds = torch.nonzero(torch.gt(w1, slack))[:, 0]
print("removed due to gradients", w1.size(0) - inds.size(0))
if inds.size(0) < nb_selected:
print("WARNING GRADIENTS ARE TOO SMALL!!!!!!!!")
inds = torch.arange(0, x1.size(0))
w1 = w1[inds]
x1 = x1[inds]
x2 = x1 if x2 is None else x2
w2 = w1 if x2 is x1 else x2.norm(p=2, dim=1, keepdim=True)
G = torch.mm(x1, x2.t()) / (w1 * w2.t()) # .clamp(min=eps)
t = G.size(0)
G = G.double().numpy()
a = np.zeros(t)
# a=np.ones(t)*-1
# a=((w1-torch.min(w1))/(torch.max(w1)-torch.min(w1))).squeeze().double().numpy()*-0.01
C = np.ones((t, 1))
h = np.zeros(1) + nb_selected
C2 = np.eye(t)
hlower = np.zeros(t)
hupper = np.ones(t)
idx = np.arange(t)
#################
C = np.concatenate((C2, C), axis=1)
C = np.transpose(C)
h_final_lower = np.concatenate((hlower, h), axis=0)
h_final_upper = np.concatenate((hupper, h), axis=0)
#################
G = spa.csc_matrix(G)
C = spa.csc_matrix(C)
solver.setup(G, a, C, h_final_lower, h_final_upper, idx, hlower, hupper,
miosqp_settings, osqp_settings)
results = solver.solve()
print("STATUS", results.status)
coeffiecents_np = results.x
coeffiecents = torch.nonzero(torch.Tensor(coeffiecents_np))
print("number of selected items is", sum(coeffiecents_np))
if "Infeasible" in results.status:
return inds
return inds[coeffiecents.squeeze()]
def forward(self,
tgt_seq,
enc_output,
enc_mask,
enc_attn_caches=None,
self_attn_caches=None,
sample_K=0,
seed=0):
batch_size, tgt_len = tgt_seq.size()
query_len = tgt_len
key_len = tgt_len
src_len = enc_output.size(1)
# Run the forward pass of the TransformerDecoder.
emb = self.embeddings(tgt_seq)
if not self.layer_norm_first:
emb = self.layer_norm(emb)
if self_attn_caches is not None:
emb = emb[:, -1:].contiguous()
query_len = 1
# Decode mask
dec_slf_attn_pad_mask = tgt_seq.detach().eq(PAD).unsqueeze(1).expand(
batch_size, query_len, key_len)
dec_slf_attn_sub_mask = get_attn_causal_mask(emb)
dec_slf_attn_mask = torch.gt(
dec_slf_attn_pad_mask + dec_slf_attn_sub_mask, 0)
dec_enc_attn_mask = enc_mask.unsqueeze(1).expand(
batch_size, query_len, src_len)
output = emb
new_self_attn_caches = []
new_enc_attn_caches = []
for i in range(self.num_layers):
# copy head occur when last layer
if i == self.num_layers - 1:
layer_sample_K = sample_K
else:
layer_sample_K = 0
output, attn, self_attn_cache, enc_attn_cache \
= self.layer_stack[i](output,
enc_output,
dec_slf_attn_mask,
dec_enc_attn_mask,
enc_attn_cache=enc_attn_caches[i] if enc_attn_caches is not None else None,
self_attn_cache=self_attn_caches[i] if self_attn_caches is not None else None,
sample_K=layer_sample_K, seed=seed)
new_self_attn_caches += [self_attn_cache]
new_enc_attn_caches += [enc_attn_cache]
if self.layer_norm_first:
output = self.layer_norm(output)
return output, new_self_attn_caches, new_enc_attn_caches
def common_target_masks_generation(
self, metrics: Dict[str, Tensor]) -> Dict[str, Dict[str, Tensor]]:
masks = {}
if not metrics:
return masks
# TODO: support more target type in wrapper & config list refactor
target_name = 'weight'
# validate all wrapper setting the same sparsity
# TODO: move validation logic to pruner
global_sparsity_rate = self.pruner.get_modules_wrapper()[list(
metrics.keys())[0]].config['total_sparsity']
for module_name, target_metric in metrics.items():
wrapper = self.pruner.get_modules_wrapper()[module_name]
assert global_sparsity_rate == wrapper.config['total_sparsity']
# find the largest metric value among all metrics
max_metric_value = list(metrics.values())[0].max()
for module_name, target_metric in metrics.items():
max_metric_value = max_metric_value if max_metric_value >= target_metric.max(
) else target_metric.max()
# prevent each module from being over-pruned, prevent ratio is 'max_sparsity_per_layer'
for module_name, target_metric in metrics.items():
wrapper = self.pruner.get_modules_wrapper()[module_name]
max_sparsity = wrapper.config.get('max_sparsity_per_layer',
{}).get(module_name, 0.99)
assert 0 <= max_sparsity <= 1
old_target_mask: Tensor = getattr(wrapper, f'{target_name}_mask')
expand_times = old_target_mask.numel() // target_metric.numel()
max_pruning_numel = int(
max_sparsity * target_metric.numel()) * expand_times
threshold = torch.topk(target_metric.reshape(-1),
max_pruning_numel,
largest=False)[0].max()
metrics[module_name] = torch.where(target_metric <= threshold,
target_metric, max_metric_value)
# build the global_matric & calculate global threshold
metric_list = []
for module_name, target_metric in metrics.items():
wrapper = self.pruner.get_modules_wrapper()[module_name]
old_target_mask: Tensor = getattr(wrapper, f'{target_name}_mask')
expand_times = old_target_mask.numel() // target_metric.numel()
metric_list.append(
target_metric.reshape(-1).unsqueeze(0).expand(
expand_times, -1).reshape(-1))
global_metric = torch.cat(metric_list)
max_pruning_num = int((global_metric != max_metric_value).sum().item())
total_pruning_num = min(
int(global_sparsity_rate * global_metric.numel()), max_pruning_num)
global_threshold = torch.topk(global_metric.reshape(-1),
total_pruning_num,
largest=False)[0].max()
# generate masks for each target
for module_name, target_metric in metrics.items():
masks[module_name] = {}
wrapper = self.pruner.get_modules_wrapper()[module_name]
shrinked_mask = torch.gt(target_metric,
global_threshold).type_as(target_metric)
masks[module_name][target_name] = self._expand_mask(
module_name, target_name, shrinked_mask)
return masks
def decode(self, pred, source):
#pred = pred.squeeze(dim=0)
for idx in range(self.num_label):
name, chooce = self.label_list[idx], self.attribute_dict[
self.label_list[idx]][pred[idx]]
#if chooce
print('{}: {} source: {}'.format(name, chooce, source[idx]))
#return name, chooce
if __name__ == '__main__':
checkpoint = torch.load("")
model = resnet50(class_num=11)
model.load_state_dict(checkpoint['model_state_dict'])
model.cuda()
img = Image.open("")
img = img.convert("RGB")
img = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
])(img)
img = img.unsqueeze(dim=0)
out = model(img.cuda())
pred = torch.gt(out, torch.ones_like(out) * 0.8)
dec = predict_decoder(dataset="CelebA",
list=[6, 7, 8, 9, 11, 15, 20, 31, 32, 25, 33])
dec.decode(pred=pred, source=out.data.cpu().numpy())
def test_comparison_ops_with_type_promotion(self, device):
value_for_type = {
torch.uint8: (1 << 5),
torch.int8: (1 << 5),
torch.int16: (1 << 10),
torch.int32: (1 << 20),
torch.int64: (1 << 35),
torch.float16: (1 << 10),
torch.float32: (1 << 20),
torch.float64: (1 << 35)
}
comparison_ops = [
dict(
name="lt",
out_op=lambda x, y, d: torch.lt(x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.lt(x, y),
compare_op=lambda x, y: x < y,
),
dict(
name="le",
out_op=lambda x, y, d: torch.le(x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.le(x, y),
compare_op=lambda x, y: x <= y,
),
dict(
name="gt",
out_op=lambda x, y, d: torch.gt(x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.gt(x, y),
compare_op=lambda x, y: x > y,
),
dict(
name="ge",
out_op=lambda x, y, d: torch.ge(x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.ge(x, y),
compare_op=lambda x, y: x >= y,
),
dict(
name="eq",
out_op=lambda x, y, d: torch.eq(x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.eq(x, y),
compare_op=lambda x, y: x == y,
),
dict(
name="ne",
out_op=lambda x, y, d: torch.ne(x, y, out=torch.empty(1, dtype=torch.bool, device=d)),
ret_op=lambda x, y: torch.ne(x, y),
compare_op=lambda x, y: x != y,
),
]
for op in comparison_ops:
for dt1 in torch.testing.get_all_math_dtypes(device):
for dt2 in torch.testing.get_all_math_dtypes(device):
val1 = value_for_type[dt1]
val2 = value_for_type[dt2]
t1 = torch.tensor([val1], dtype=dt1, device=device)
t2 = torch.tensor([val2], dtype=dt2, device=device)
expected = torch.tensor([op["compare_op"](val1, val2)], dtype=torch.bool)
out_res = op["out_op"](t1, t2, device)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
out_res = op["ret_op"](t1, t2)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
# test that comparing a zero dim tensor with another zero dim tensor has type promotion behavior
t1 = torch.tensor(val1, dtype=dt1, device=device)
t2 = torch.tensor(val2, dtype=dt2, device=device)
expected = torch.tensor(op["compare_op"](val1, val2), dtype=torch.bool)
out_res = op["out_op"](t1, t2, device)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
out_res = op["ret_op"](t1, t2)
self.assertEqual(out_res, expected)
self.assertTrue(out_res.dtype == torch.bool)
self.assertTrue(t1.dtype == dt1)
self.assertTrue(t2.dtype == dt2)
def binary_class_score(pred, target, thresh=0.5):
predLabel = torch.gt(pred, thresh)
print[predLabel, target]
classScoreTest = torch.eq(predLabel, target.type_as(predLabel))
return classScoreTest.float().sum() / target.size(0)
def topkextra(x,
y,
model,
loss_fct,
k=25,
epsilon=0.02,
alpha=0.5,
is_report_loss_diff=False,
use_sample=False):
x_next = topk(x, y, model, loss_fct, k, epsilon, alpha, False, use_sample)
# some book-keeping
if next(model.parameters()).is_cuda:
x = x.cuda()
y = y.cuda()
x_next = x_next.cuda()
y = Variable(y)
# compute natural loss
y_model = model(Variable(x))
loss_natural = loss_fct(y_model, y).data
for i in range(1):
x_best = x_next.clone()
y_model = model(Variable(x_next))
best_loss = loss_fct(y_model, y)
factor = 2 * len(x)
no_improve = 0
for n in range(k):
# forward pass
# x_old = x_next.clone()
xn_var = Variable(x_next, requires_grad=True)
y_model = model(xn_var)
loss = loss_fct(y_model, y)
# compute gradient
grads = torch.autograd.grad(loss.mean(), xn_var)[0].data
# topk
signs = torch.gt(grads, 0).float()
grads = (signs - x_next) * grads
grads -= x * grads
rand_vars = torch.rand(len(grads), len(grads[0]))
if next(model.parameters()).is_cuda:
rand_vars = rand_vars.cuda()
grads = rand_vars * grads
kvals, kidx = grads.topk(k=min(10000, max(1, int(factor))), dim=1)
x_next.scatter_(dim=1,
index=kidx,
src=signs.gather(dim=1, index=kidx))
# projection
x_next = clip_tensor(x_next)
x_next = or_float_tensors(x_next, x)
# compute adversarial loss
loss_adv = loss_fct(model(Variable(x_next)), y)
factor = random.random() * 2 * len(x) + 1.0
#if loss.data.mean() > loss_adv.data.mean():
# x_next = x_old
# factor = max(1,factor * 0.5)
#else:
# factor = factor * 2.0
#print(loss_adv.data.mean())
found = False
for r in range(len(x)):
if loss_adv.data[r] > best_loss.data[r]:
x_best[r] = x_next[r].clone()
best_loss.data[r] = loss_adv.data[r]
found = True
if found is True:
# if loss_adv.data.mean() > best_loss.data.mean():
#x_best = x_next.clone()
#best_loss = loss_adv
#x_next = x_best.clone()
no_improve = 0
else:
no_improve += 1
if no_improve > len(x):
break
x_next = topk(x, y.data, model, loss_fct, k, epsilon, alpha, False,
use_sample, x_best).cuda()
# projection
#x_next = clip_tensor(x_next)
#x_next = or_float_tensors(x_next, x)
#x_next = x_best
# compute adversarial loss
y_model = model(Variable(x_next))
loss_adv = loss_fct(y_model, y).data
if is_report_loss_diff:
print(
"Natural loss (%.4f) vs Adversarial loss (%.4f), Difference: (%.4f)"
% (loss_natural.mean(), loss_adv.mean(),
loss_adv.mean() - loss_natural.mean()))
replace_flag = (loss_adv < loss_natural).unsqueeze(1).expand_as(x_next)
x_next[replace_flag] = x[replace_flag]
if x_next.is_cuda:
x_next = x_next.cpu()
return x_next
def topk(x,
y,
model,
loss_fct,
k=25,
epsilon=0.02,
alpha=0.5,
is_report_loss_diff=False,
use_sample=False,
start_x=None):
# some book-keeping
if next(model.parameters()).is_cuda:
x = x.cuda()
y = y.cuda()
y = Variable(y)
# compute natural loss
y_model = model(Variable(x))
loss_natural = loss_fct(y_model, y).data
# initialize starting point
x_next = get_x0(x, use_sample).clone()
if start_x is not None:
x_next = start_x.clone()
x_best = x_next.clone()
best_loss = loss_fct(y_model, y)
factor = 1.0
for n in range(k):
# forward pass
x_old = x_next.clone()
xn_var = Variable(x_next, requires_grad=True)
y_model = model(xn_var)
loss = loss_fct(y_model, y)
# compute gradient
grads = torch.autograd.grad(loss.mean(), xn_var)[0].data
# topk
signs = torch.gt(grads, 0).float()
grads = (signs - x_next) * grads
grads -= x * grads
rand_vars = torch.rand(len(grads), len(grads[0]))
kvals, kidx = grads.topk(k=min(10000, max(1, int(factor))), dim=1)
x_next.scatter_(dim=1, index=kidx, src=signs.gather(dim=1, index=kidx))
# projection
x_next = clip_tensor(x_next)
x_next = or_float_tensors(x_next, x)
# compute adversarial loss
loss_adv = loss_fct(model(Variable(x_next)), y)
found = 0
for r in range(len(x)):
if loss_adv.data[r] > best_loss.data[r]:
x_best[r] = x_next[r].clone()
best_loss.data[r] = loss_adv.data[r]
found += 1
#x_next = x_best.clone()
if found < len(x) * 0.5 and loss.data.mean() > loss_adv.data.mean():
#if loss.data.mean() > loss_adv.data.mean():
factor = max(factor * 0.5, 0.25)
x_next = x_old
#if found is False:
if factor < 0.5:
break
else:
factor = factor * 2.0
#x_next = x_best.clone()
x_next = x_best
# compute adversarial loss
y_model = model(Variable(x_next))
loss_adv = loss_fct(y_model, y).data
if is_report_loss_diff:
print(
"Natural loss (%.4f) vs Adversarial loss (%.4f), Difference: (%.4f)"
% (loss_natural.mean(), loss_adv.mean(),
loss_adv.mean() - loss_natural.mean()))
replace_flag = (loss_adv < loss_natural).unsqueeze(1).expand_as(x_next)
x_next[replace_flag] = x[replace_flag]
if x_next.is_cuda:
x_next = x_next.cpu()
return x_next
def accuracy(score_pos, score_neg):
"""Computes the % of correctly ordered pairs"""
pred1 = score_pos
pred2 = score_neg
correct = torch.gt(pred1, pred2)
return float(correct.sum()) / correct.size(0), int(correct.sum())
def step(self, closure=None):
"""Performs a single optimization step.
Arguments:
closure (callable, optional): A closure that reevaluates the model
and returns the loss.
"""
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError(
'cosangulargrad does not support sparse gradients, please consider SparseAdam instead'
)
state = self.state[p]
# State initialization
if len(state) == 0:
state['step'] = 0
# Exponential moving average of gradient values
state['exp_avg'] = torch.zeros_like(p.data)
# Exponential moving average of squared gradient values
state['exp_avg_sq'] = torch.zeros_like(p.data)
# Previous gradient
state['previous_grad'] = torch.zeros_like(p.data)
# temporary minimum value for comparison
state['min'] = torch.zeros_like(p.data)
# temporary difference between gradients for comparison
state['diff'] = torch.zeros_like(p.data)
# final cos value to be used
state['final_cos_theta'] = torch.zeros_like(p.data)
exp_avg, exp_avg_sq, previous_grad, min, diff, final_cos_theta = state['exp_avg'], state['exp_avg_sq'], \
state['previous_grad'], state['min'], \
state['diff'], state['final_cos_theta']
beta1, beta2 = group['betas']
state['step'] += 1
if group['weight_decay'] != 0:
grad.add_(group['weight_decay'], p.data)
# Decay the first and second moment running average coefficient
exp_avg.mul_(beta1).add_(1 - beta1, grad)
exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
denom = exp_avg_sq.sqrt().add_(group['eps'])
bias_correction1 = 1 - beta1**state['step']
bias_correction2 = 1 - beta2**state['step']
tan_theta = abs(
(previous_grad - grad) / (1 + previous_grad * grad))
cos_theta = 1 / torch.sqrt(1 + torch.square(tan_theta))
angle = torch.atan(tan_theta) * (180 / 3.141592653589793238)
ans = torch.gt(angle, min)
ans1, count = torch.unique(ans, return_counts=True)
try:
if (count[1] < count[0]):
min = angle
diff = abs(previous_grad - grad)
final_cos_theta = cos_theta.clone()
except:
if (ans1[0].item() == False):
min = angle
diff = abs(previous_grad - grad)
final_cos_theta = cos_theta.clone()
angular_coeff = torch.tanh(
abs(final_cos_theta
)) * 0.5 + 0.5 # Calculating Angular coefficient
state['previous_grad'] = grad.clone()
state['min'] = min.clone()
state['diff'] = diff.clone()
state['final_cos_theta'] = final_cos_theta.clone()
# update momentum with angular_coeff
exp_avg1 = exp_avg * angular_coeff
step_size = group['lr'] * math.sqrt(
bias_correction2) / bias_correction1
p.data.addcdiv_(-step_size, exp_avg1, denom)
return loss
def forward(self, output, target, loss_history, epoch, batch_idx):
# output: B x As * (6 + 1 + num_classes) * H * W
t0 = time.time()
nB = output.data.size(0)
nA = self.num_anchors # num_anchors = 5
nC = self.num_classes # num_classes = 8
nH = output.data.size(2) # nH 16
nW = output.data.size(3) # nW 32
output = output.view(nB, nA, (7 + nC), nH, nW)
x = torch.sigmoid(
output.index_select(2, Variable(torch.cuda.LongTensor([0]))).view(
nB, nA, nH, nW))
y = torch.sigmoid(
output.index_select(2, Variable(torch.cuda.LongTensor([1]))).view(
nB, nA, nH, nW))
w = output.index_select(2, Variable(torch.cuda.LongTensor([2]))).view(
nB, nA, nH, nW)
l = output.index_select(2, Variable(torch.cuda.LongTensor([3]))).view(
nB, nA, nH, nW)
im = output.index_select(2, Variable(torch.cuda.LongTensor([4]))).view(
nB, nA, nH, nW)
re = output.index_select(2, Variable(torch.cuda.LongTensor([5]))).view(
nB, nA, nH, nW)
conf = torch.sigmoid(
output.index_select(2, Variable(torch.cuda.LongTensor([6]))).view(
nB, nA, nH, nW))
cls = output.index_select(
2, Variable(torch.linspace(7, 7 + nC - 1, nC).long().cuda()))
cls = cls.view(nB * nA, nC, nH * nW).transpose(1, 2).contiguous().view(
nB * nA * nH * nW, nC)
t1 = time.time()
pred_boxes = torch.cuda.FloatTensor(6, nB * nA * nH * nW)
grid_x = torch.linspace(0, nW - 1, nW).repeat(nH, 1).repeat(
nB * nA, 1, 1).view(nB * nA * nH * nW).cuda()
grid_y = torch.linspace(0, nH - 1, nH).repeat(nW, 1).t().repeat(
nB * nA, 1, 1).view(nB * nA * nH * nW).cuda()
anchor_w = torch.Tensor(anchors).view(
nA, self.anchor_step).index_select(1,
torch.LongTensor([0])).cuda()
anchor_l = torch.Tensor(anchors).view(
nA, self.anchor_step).index_select(1,
torch.LongTensor([1])).cuda()
anchor_w = anchor_w.repeat(nB,
1).repeat(1, 1,
nH * nW).view(nB * nA * nH * nW)
anchor_l = anchor_l.repeat(nB,
1).repeat(1, 1,
nH * nW).view(nB * nA * nH * nW)
pred_boxes[0] = x.data.view(nB * nA * nH * nW).cuda() + grid_x
pred_boxes[1] = y.data.view(nB * nA * nH * nW).cuda() + grid_y
pred_boxes[2] = torch.exp(w.data).view(
nB * nA * nH * nW).cuda() * anchor_w
pred_boxes[3] = torch.exp(l.data).view(
nB * nA * nH * nW).cuda() * anchor_l
pred_boxes[4] = im.data.view(nB * nA * nH * nW).cuda()
pred_boxes[5] = re.data.view(nB * nA * nH * nW).cuda()
pred_boxes = convert2cpu(
pred_boxes.transpose(0, 1).contiguous().view(-1, 6))
t2 = time.time()
nGT, nCorrect, coord_mask, conf_mask, cls_mask, tx, ty, tw, tl, tim, tre, tconf,tcls, loss_iou = build_targets(pred_boxes, target.data, self.anchors, nA, nC, \
nH, nW, self.noobject_scale, self.object_scale, self.thresh)
cls_mask = (cls_mask == 1)
nProposals = int(torch.sum(torch.gt(conf, 0.25)))
tx = Variable(tx.cuda())
ty = Variable(ty.cuda())
tw = Variable(tw.cuda())
tl = Variable(tl.cuda())
tim = Variable(tim.cuda())
tre = Variable(tre.cuda())
tconf = Variable(tconf.cuda())
cls_mask = cls_mask.view(nB * nA * nH * nW)
tcls = Variable(tcls.view(-1)[cls_mask].long().cuda())
coord_mask = Variable(coord_mask.cuda())
conf_mask = Variable(conf_mask.cuda())
cls_mask = Variable(cls_mask.view(-1, 1).repeat(1, nC).cuda())
cls = cls[cls_mask].view(-1, nC)
t3 = time.time()
loss_x = self.coord_scale * nn.MSELoss(reduction='sum')(
x * coord_mask, tx * coord_mask)
loss_y = self.coord_scale * nn.MSELoss(reduction='sum')(
y * coord_mask, ty * coord_mask)
loss_w = self.coord_scale * nn.MSELoss(reduction='sum')(
w * coord_mask, tw * coord_mask)
loss_l = self.coord_scale * nn.MSELoss(reduction='sum')(
l * coord_mask, tl * coord_mask)
loss_im = self.coord_scale * nn.MSELoss(reduction='sum')(
im * coord_mask, tim * coord_mask)
loss_re = self.coord_scale * nn.MSELoss(reduction='sum')(
re * coord_mask, tre * coord_mask)
loss_Euler = loss_im + loss_re
loss_conf = nn.MSELoss(reduction='sum')(conf * conf_mask,
tconf * conf_mask)
loss_cls = self.class_scale * nn.CrossEntropyLoss(reduction='sum')(
cls, tcls)
loss = loss_x + loss_y + loss_w + loss_l + loss_conf + loss_cls + loss_Euler + torch.tensor(
loss_iou).cuda()
loss_history[epoch, batch_idx, :] = [
loss_x, loss_y, loss_w, loss_l, loss_conf, loss_cls, loss_Euler,
loss_iou
]
t4 = time.time()
if False:
print('-----------------------------------')
print(' activation : %f' % (t1 - t0))
print(' create pred_boxes : %f' % (t2 - t1))
print(' build targets : %f' % (t3 - t2))
print(' create loss : %f' % (t4 - t3))
print(' total : %f' % (t4 - t0))
logging.info(
'nGT %d, recall %d, proposals %d, loss: x %f, y %f, w %f, h %f, conf %f, cls %f, Euler %f, total %f'
% (nGT, nCorrect, nProposals, loss_x.data, loss_y.data,
loss_w.data, loss_l.data, loss_conf.data, loss_cls.data,
loss_Euler.data, loss.data))
return loss
def beam_search(models, features, params):
if not isinstance(models, (list, tuple)):
raise ValueError("'models' must be a list or tuple")
beam_size = params.beam_size
top_beams = params.top_beams
alpha = params.decode_alpha
decode_ratio = params.decode_ratio
decode_length = params.decode_length
pad_id = params.lookup["target"][params.pad.encode("utf-8")]
bos_id = params.lookup["target"][params.bos.encode("utf-8")]
eos_id = params.lookup["target"][params.eos.encode("utf-8")]
min_val = -1e9
shape = features["source"].shape
device = features["source"].device
batch_size = shape[0]
seq_length = shape[1]
# Compute initial state if necessary
states = []
funcs = []
for model in models:
state = model.empty_state(batch_size, device)
states.append(model.encode(features, state))
funcs.append(model.decode)
# For source sequence length
max_length = features["source_mask"].sum(1) * decode_ratio
max_length = max_length.long() + decode_length
max_step = max_length.max()
# [batch, beam_size]
max_length = torch.unsqueeze(max_length, 1).repeat([1, beam_size])
# Expand the inputs
# [batch, length] => [batch * beam_size, length]
features["source"] = torch.unsqueeze(features["source"], 1)
features["source"] = features["source"].repeat([1, beam_size, 1])
features["source"] = torch.reshape(features["source"],
[batch_size * beam_size, seq_length])
features["source_mask"] = torch.unsqueeze(features["source_mask"], 1)
features["source_mask"] = features["source_mask"].repeat([1, beam_size, 1])
features["source_mask"] = torch.reshape(
features["source_mask"], [batch_size * beam_size, seq_length])
decoding_fn = _get_inference_fn(funcs, features)
states = map_structure(lambda x: _tile_to_beam_size(x, beam_size), states)
# Initial beam search state
init_seqs = torch.full([batch_size, beam_size, 1],
bos_id,
device=device,
dtype=torch.long)
#init_seqs = init_seqs.long()
init_log_probs = init_seqs.new_tensor([[0.] + [min_val] * (beam_size - 1)],
dtype=torch.float32)
init_log_probs = init_log_probs.repeat([batch_size, 1])
init_scores = torch.zeros_like(init_log_probs)
fin_seqs = torch.zeros([batch_size, beam_size, 1],
dtype=torch.int64,
device=device)
fin_scores = torch.full([batch_size, beam_size],
min_val,
dtype=torch.float32,
device=device)
fin_flags = torch.zeros([batch_size, beam_size],
dtype=torch.bool,
device=device)
state = BeamSearchState(
inputs=(init_seqs, init_log_probs, init_scores),
state=states,
finish=(fin_flags, fin_seqs, fin_scores),
)
for time in range(max_step):
state = _beam_search_step(time, decoding_fn, state, batch_size,
beam_size, alpha, pad_id, eos_id, max_length)
max_penalty = ((5.0 + max_step) / 6.0)**alpha
best_alive_score = torch.max(state.inputs[1][:, 0] / max_penalty)
worst_finished_score = torch.min(state.finish[2])
cond = torch.gt(worst_finished_score, best_alive_score)
is_finished = bool(cond)
if is_finished:
break
final_state = state
alive_seqs = final_state.inputs[0]
alive_scores = final_state.inputs[2]
final_flags = final_state.finish[0].byte()
final_seqs = final_state.finish[1]
final_scores = final_state.finish[2]
final_seqs = torch.where(final_flags[:, :, None], final_seqs, alive_seqs)
final_scores = torch.where(final_flags, final_scores, alive_scores)
final_mask = final_seqs[:, 0, 1:].eq(pad_id).eq(0)
# Append extra <eos>
final_seqs = torch.nn.functional.pad(final_seqs, (0, 1, 0, 0, 0, 0),
value=eos_id)
first_beam_states = map_structure(lambda x: x[:, 0], final_state.state)
# update target starts according to actuall length
for i, fb_state in enumerate(first_beam_states):
first_beam_states[i]["target_mask"] = final_mask
return final_seqs[:, :top_beams,
1:], final_scores[:, :top_beams], first_beam_states
[shape_features, skel_xyz, skel_r], 2).detach()
A_init = A_init.detach()
# train GAE
A_pred = model_gae(skel_node_features, A_init)
loss_MBCE = model_gae.compute_loss(A_pred, A_init,
known_mask.detach())
optimizer_gae.zero_grad()
loss_MBCE.backward()
optimizer_gae.step()
A_final = model_gae.recover_A(A_pred, valid_mask)
if iter % save_result_iter == 0:
output_results(save_result_path, batch_id, epoch, batch_pc,
skel_xyz, skel_r, A_init,
torch.gt(A_pred, 0.5), A_final)
if iter % save_net_iter == 0:
torch.save(model_gae.state_dict(),
save_net_path + 'para-gae.pth')
print('A init:',
torch.sum(A_init).item() / conf.BATCH_SIZE / 2.0)
print('A known:',
torch.sum(known_mask).item() / conf.BATCH_SIZE / 2.0)
print(
'A pred:',
torch.sum(torch.gt(A_pred, 0.5)).item() / conf.BATCH_SIZE /
2.0)
######################################
# Train two networks jointly
def forward(self, q_pts, s_pts, neighb_inds, x):
###################
# Offset generation
###################
if self.deformable:
# Get offsets with a KPConv that only takes part of the features
self.offset_features = self.offset_conv(q_pts, s_pts, neighb_inds, x) + self.offset_bias
if self.modulated:
# Get offset (in normalized scale) from features
unscaled_offsets = self.offset_features[:, :self.p_dim * self.K]
unscaled_offsets = unscaled_offsets.view(-1, self.K, self.p_dim)
# Get modulations
modulations = 2 * torch.sigmoid(self.offset_features[:, self.p_dim * self.K:])
else:
# Get offset (in normalized scale) from features
unscaled_offsets = self.offset_features.view(-1, self.K, self.p_dim)
# No modulations
modulations = None
# Rescale offset for this layer
offsets = unscaled_offsets * self.KP_extent
else:
offsets = None
modulations = None
######################
# Deformed convolution
######################
# Add a fake point in the last row for shadow neighbors
s_pts = torch.cat((s_pts, torch.zeros_like(s_pts[:1, :]) + 1e6), 0)
# Get neighbor points [n_points, n_neighbors, dim]
neighbors = s_pts[neighb_inds, :]
# Center every neighborhood
neighbors = neighbors - q_pts.unsqueeze(1)
# Apply offsets to kernel points [n_points, n_kpoints, dim]
if self.deformable:
self.deformed_KP = offsets + self.kernel_points
deformed_K_points = self.deformed_KP.unsqueeze(1)
else:
deformed_K_points = self.kernel_points
# Get all difference matrices [n_points, n_neighbors, n_kpoints, dim]
neighbors.unsqueeze_(2)
differences = neighbors - deformed_K_points
# Get the square distances [n_points, n_neighbors, n_kpoints]
sq_distances = torch.sum(differences ** 2, dim=3)
# Optimization by ignoring points outside a deformed KP range
if self.deformable:
# Save distances for loss
self.min_d2, _ = torch.min(sq_distances, dim=1)
# Boolean of the neighbors in range of a kernel point [n_points, n_neighbors]
in_range = torch.any(sq_distances < self.KP_extent ** 2, dim=2).type(torch.int32)
# New value of max neighbors
new_max_neighb = torch.max(torch.sum(in_range, dim=1))
# For each row of neighbors, indices of the ones that are in range [n_points, new_max_neighb]
neighb_row_bool, neighb_row_inds = torch.topk(in_range, new_max_neighb.item(), dim=1)
# Gather new neighbor indices [n_points, new_max_neighb]
new_neighb_inds = neighb_inds.gather(1, neighb_row_inds, sparse_grad=False)
# Gather new distances to KP [n_points, new_max_neighb, n_kpoints]
neighb_row_inds.unsqueeze_(2)
neighb_row_inds = neighb_row_inds.expand(-1, -1, self.K)
sq_distances = sq_distances.gather(1, neighb_row_inds, sparse_grad=False)
# New shadow neighbors have to point to the last shadow point
new_neighb_inds *= neighb_row_bool
new_neighb_inds -= (neighb_row_bool.type(torch.int64) - 1) * int(s_pts.shape[0] - 1)
else:
new_neighb_inds = neighb_inds
# Get Kernel point influences [n_points, n_kpoints, n_neighbors]
if self.KP_influence == 'constant':
# Every point get an influence of 1.
all_weights = torch.ones_like(sq_distances)
all_weights = torch.transpose(all_weights, 1, 2)
elif self.KP_influence == 'linear':
# Influence decrease linearly with the distance, and get to zero when d = KP_extent.
all_weights = torch.clamp(1 - torch.sqrt(sq_distances) / self.KP_extent, min=0.0)
all_weights = torch.transpose(all_weights, 1, 2)
elif self.KP_influence == 'gaussian':
# Influence in gaussian of the distance.
sigma = self.KP_extent * 0.3
all_weights = radius_gaussian(sq_distances, sigma)
all_weights = torch.transpose(all_weights, 1, 2)
else:
raise ValueError('Unknown influence function type (config.KP_influence)')
# In case of closest mode, only the closest KP can influence each point
if self.aggregation_mode == 'closest':
neighbors_1nn = torch.argmin(sq_distances, dim=2)
all_weights *= torch.transpose(nn.functional.one_hot(neighbors_1nn, self.K), 1, 2)
elif self.aggregation_mode != 'sum':
raise ValueError("Unknown convolution mode. Should be 'closest' or 'sum'")
# Add a zero feature for shadow neighbors
x = torch.cat((x, torch.zeros_like(x[:1, :])), 0)
# Get the features of each neighborhood [n_points, n_neighbors, in_fdim]
neighb_x = gather(x, new_neighb_inds)
# Apply distance weights [n_points, n_kpoints, in_fdim]
weighted_features = torch.matmul(all_weights, neighb_x)
# Apply modulations
if self.deformable and self.modulated:
weighted_features *= modulations.unsqueeze(2)
# Apply network weights [n_kpoints, n_points, out_fdim]
weighted_features = weighted_features.permute((1, 0, 2))
kernel_outputs = torch.matmul(weighted_features, self.weights)
# Convolution sum [n_points, out_fdim]
# return torch.sum(kernel_outputs, dim=0)
output_features = torch.sum(kernel_outputs, dim=0, keepdim=False)
# normalization term.
neighbor_features_sum = torch.sum(neighb_x, dim=-1)
neighbor_num = torch.sum(torch.gt(neighbor_features_sum, 0.0), dim=-1)
neighbor_num = torch.max(neighbor_num, torch.ones_like(neighbor_num))
output_features = output_features / neighbor_num.unsqueeze(1)
return output_features
def mask(var):
return torch.gt(var, 0).float()
def _common_channel_to_prune(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups):
"""
Calculate the common channels should be pruned by all the layers in this group.
This function is for filter pruning of Conv layers. if want to support the dependency-aware
mode for others ops, you need to inherit this class and overwrite `_common_channel_to_prune`.
Parameters
----------
sparsities : list
List of float that specify the sparsity for each conv layer.
wrappers : list
List of wrappers
groups : list
The number of the filter groups of each layer.
wrappers_idx : list
The indexes of the wrappers
"""
# sparsity configs for each wrapper
# sparsities = [_w.config['sparsity'] for _w in wrappers]
# check the type of the input wrappers
for _w in wrappers:
msg = 'module type {} is not supported!'.format(_w.type)
assert _w.type == 'Conv2d', msg
# Among the dependent layers, the layer with smallest
# sparsity determines the final benefit of the speedup
# module. To better harvest the speed benefit, we need
# to ensure that these dependent layers have at least
# `min_sparsity` pruned channel are the same.
if len(channel_dsets) == len(wrappers):
# all the layers in the dependency sets are pruned
min_sparsity = min(sparsities)
else:
# not all the layers in the dependency set
# are pruned
min_sparsity = 0
# donnot prune the channels that we cannot harvest the speed from
sparsities = [min_sparsity] * len(sparsities)
# find the max number of the filter groups of the dependent
# layers. The group constraint of this dependency set is decided
# by the layer with the max groups.
# should use the least common multiple for all the groups
# the max_group is lower than the channel_count, because
# the number of the filter is always divisible by the number of the group
max_group = np.lcm.reduce(groups)
channel_count = wrappers[0].module.weight.data.size(0)
device = wrappers[0].module.weight.device
channel_sum = torch.zeros(channel_count).to(device)
for _w, _w_idx in zip(wrappers, wrappers_idx):
# calculate the L1/L2 sum for all channels
c_sum = self.get_channel_sum(_w, _w_idx)
if c_sum is None:
# if the channel sum cannot be calculated
# now, return None
return None
channel_sum += c_sum
# prune the same `min_sparsity` channels based on channel_sum
# for all the layers in the channel sparsity
target_pruned = int(channel_count * min_sparsity)
# pruned_per_group may be zero, for example dw conv
pruned_per_group = int(target_pruned / max_group)
group_step = int(channel_count / max_group)
channel_masks = []
for gid in range(max_group):
_start = gid * group_step
_end = (gid + 1) * group_step
if pruned_per_group > 0:
threshold = torch.topk(
channel_sum[_start: _end], pruned_per_group, largest=False)[0].max()
group_mask = torch.gt(channel_sum[_start:_end], threshold)
else:
group_mask = torch.ones(group_step).to(device)
channel_masks.append(group_mask)
channel_masks = torch.cat(channel_masks, dim=0)
pruned_channel_index = (
channel_masks == False).nonzero().squeeze(1).tolist()
logger.info('Prune the %s channels for all dependent',
','.join([str(x) for x in pruned_channel_index]))
return channel_masks
def mode(self):
return torch.gt(self.probs, 0.5).float()
def cosine_similarity_selector_IQP(x1,
solver,
nb_selected,
eps=1e-3,
slack=0.01):
"""
Integer programming
"""
# x1=gradient memories
x2 = None
w1 = x1.norm(p=2, dim=1, keepdim=True)
inds = torch.nonzero(torch.gt(w1, slack))[:, 0]
w1 = w1[inds]
x1 = x1[inds]
x2 = x1 if x2 is None else x2
w2 = w1 if x2 is x1 else x2.norm(p=2, dim=1, keepdim=True)
G = torch.mm(x1, x2.t()) / (w1 * w2.t()) # .clamp(min=eps)
t = G.size(0)
# a=torch.sum(G-torch.eye(t),1)/(t-1)*-0.1
# a=torch.max(G-torch.eye(t),1)[0]*-0.1
# a=a.double().numpy()
G = G.double().numpy()
# G=G+ np.eye(t) * eps
# G = 0.5 * (G + G.transpose()) +np.eye(t) * eps
# a=(w1*-1).view(t).numpy()
# a=np.zeros(t)
a = np.ones(t) * -1
# a=((w1-torch.min(w1))/(torch.max(w1)-torch.min(w1))).squeeze().double().numpy()*-0.01
# h = np.zeros(1) + nb_selected
C2 = np.eye(t)
hlower = np.zeros(t)
hupper = np.ones(t)
idx = np.arange(t)
# np.concatenate((a, b), axis=0)
#################
# C=np.concatenate((C2, C), axis=1)
# C=np.transpose(C)
# h_final_lower=np.concatenate((hlower,h), axis=0)
# h_final_upper=np.concatenate((hupper,h), axis=0)
#################
G = spa.csc_matrix(G)
C2 = spa.csc_matrix(C2)
solver.setup(G, a, C2, hlower, hupper, idx, hlower, hupper,
miosqp_settings, osqp_settings)
results = solver.solve()
print("STATUS", results.status)
coeffiecents_np = results.x
coeffiecents = torch.nonzero(torch.Tensor(coeffiecents_np))
print("number of selected items is", sum(coeffiecents_np))
# _,inds=torch.sort(coeffiecents,descending=True)
return inds[coeffiecents.squeeze()]
def rprop(opfunc, x, config, state=None):
""" A plain implementation of RPROP
ARGS:
- `opfunc` : a function that takes a single input (X), the point of
evaluation, and returns f(X) and df/dX
- `x` : the initial point
- `state` : a table describing the state of the optimizer; after each
call the state is modified
- `state['stepsize']` : initial step size, common to all components
- `state['etaplus']` : multiplicative increase factor, > 1 (default 1.2)
- `state['etaminus']` : multiplicative decrease factor, < 1 (default 0.5)
- `state['stepsizemax']` : maximum stepsize allowed (default 50)
- `state['stepsizemin']` : minimum stepsize allowed (default 1e-6)
- `state['niter']` : number of iterations (default 1)
RETURN:
- `x` : the new x vector
- `f(x)` : the function, evaluated before the update
(Martin Riedmiller, Koray Kavukcuoglu 2013)
"""
if config is None and state is None:
raise ValueError("rprop requires a dictionary to retain state between iterations")
# (0) get/update state
state = state if state is not None else config
stepsize = config.get('stepsize', 0.1)
etaplus = config.get('etaplus', 1.2)
etaminus = config.get('etaminus', 0.5)
stepsizemax = config.get('stepsizemax', 50.0)
stepsizemin = config.get('stepsizemin', 1e-06)
niter = config.get('niter', 1)
hfx = []
for i in range(niter):
# (1) evaluate f(x) and df/dx
fx, dfdx = opfunc(x)
# init temp storage
if 'delta' not in state:
state['delta'] = dfdx.new(dfdx.size()).zero_()
state['stepsize'] = dfdx.new(dfdx.size()).fill_(stepsize)
state['sign'] = dfdx.new(dfdx.size())
state['bytesign'] = torch.ByteTensor(dfdx.size())
state['psign'] = torch.ByteTensor(dfdx.size())
state['nsign'] = torch.ByteTensor(dfdx.size())
state['zsign'] = torch.ByteTensor(dfdx.size())
state['dminmax'] = torch.ByteTensor(dfdx.size())
if str(type(x)).find('Cuda') > -1:
# Push to GPU
state['psign'] = state['psign'].cuda()
state['nsign'] = state['nsign'].cuda()
state['zsign'] = state['zsign'].cuda()
state['dminmax'] = state['dminmax'].cuda()
# sign of derivative from last step to this one
torch.mul(dfdx, state['delta'], out=state['sign']).sign_()
# get indices of >0, <0 and ==0 entries
torch.gt(state['sign'], 0, out=state['psign'])
torch.lt(state['sign'], 0, out=state['nsign'])
torch.eq(state['sign'], 0, out=state['zsign'])
# get step size updates
state['sign'][state['psign']] = etaplus
state['sign'][state['nsign']] = etaminus
state['sign'][state['zsign']] = 1
# update stepsizes with step size updates
state['stepsize'].mul_(state['sign'])
# threshold step sizes
# >50 => 50
torch.gt(state['stepsize'], stepsizemax, out=state['dminmax'])
state['stepsize'][state['dminmax']] = stepsizemax
# <1e-6 ==> 1e-6
torch.lt(state['stepsize'], stepsizemin, out=state['dminmax'])
state['stepsize'][state['dminmax']] = stepsizemin
# for dir<0, dfdx=0
# for dir>=0 dfdx=dfdx
dfdx[state['nsign']] = 0
torch.sign(dfdx, out=state['sign'])
# update weights
x.addcmul_(-1, state['sign'], state['stepsize'])
# update state['dfdx'] with current dfdx
state['delta'].copy_(dfdx)
hfx.append(fx)
# return x*, table of f(x) values from each step
return x, hfx
def greater(a, b):
return torch.gt(a, b)
def gt_select(self, input: str, other) -> 'TensorGroup':
"""
NOTE: tensor.dim() must all be 1
"""
mask = torch.gt(self.tensors[input], other)
return self.masked_select(mask)
def forward(self, x1, x1_f, x1_pos, x1_ner, x1_mask, x2, x2_mask):
"""Inputs:
x1 = document word indices [batch * len_d]
x1_f = document word features indices [batch * len_d * nfeat]
x1_pos = document POS tags [batch * len_d]
x1_ner = document entity tags [batch * len_d]
x1_mask = document padding mask [batch * len_d]
x2 = question word indices [batch * len_q]
x2_mask = question padding mask [batch * len_q]
"""
# Embed both document and question
x1_emb = self.embedding(x1)
x2_emb = self.embedding(x2)
if self.opt['dropout_emb'] > 0:
x1_emb = nn.functional.dropout(x1_emb, p=self.opt['dropout_emb'],
training=self.training)
x2_emb = nn.functional.dropout(x2_emb, p=self.opt['dropout_emb'],
training=self.training)
drnn_input_list = [x1_emb, x1_f]
# Add attention-weighted question representation
if self.opt['use_qemb']:
x2_weighted_emb = self.qemb_match(x1_emb, x2_emb, x2_mask)
drnn_input_list.append(x2_weighted_emb)
if self.opt['pos']:
x1_pos_emb = self.pos_embedding(x1_pos)
if self.opt['dropout_emb'] > 0:
x1_pos_emb = nn.functional.dropout(x1_pos_emb, p=self.opt['dropout_emb'],
training=self.training)
drnn_input_list.append(x1_pos_emb)
if self.opt['ner']:
x1_ner_emb = self.ner_embedding(x1_ner)
if self.opt['dropout_emb'] > 0:
x1_ner_emb = nn.functional.dropout(x1_ner_emb, p=self.opt['dropout_emb'],
training=self.training)
drnn_input_list.append(x1_ner_emb)
drnn_input = torch.cat(drnn_input_list, 2)
# Encode document with RNN
doc_hiddens = self.doc_rnn(drnn_input, x1_mask)
# Encode question with RNN + merge hiddens
question_hiddens = self.question_rnn(x2_emb, x2_mask)
#if self.opt['question_merge'] == 'avg':
# q_merge_weights = layers.uniform_weights(question_hiddens, x2_mask)
#elif self.opt['question_merge'] == 'self_attn':
# q_merge_weights = self.self_attn(question_hiddens, x2_mask)
#question_hidden = layers.weighted_avg(question_hiddens, q_merge_weights)
enc_slf_attn_pad_mask = get_attn_padding_mask(x1, x1)
enc_slf_attn_sub_mask = get_attn_subsequent_mask(x1)
enc_slf_attn_mask = torch.gt(enc_slf_attn_pad_mask + enc_slf_attn_sub_mask, 0)
enc_dec_attn_pad_mask = get_attn_padding_mask(x1, x2)
start_scores, *_ = self.decoder_start.forward(dec_input=doc_hiddens,
enc_output=question_hiddens,
slf_attn_mask=enc_slf_attn_mask,
dec_enc_attn_mask=enc_dec_attn_pad_mask
)
end_scores, *_ = self.decoder_end.forward(dec_input=doc_hiddens,
enc_output=question_hiddens,
slf_attn_mask=enc_slf_attn_mask,
dec_enc_attn_mask=enc_dec_attn_pad_mask
)
if self.training:
start_scores = nn.functional.log_softmax(start_scores)
end_scores = nn.functional.log_softmax(end_scores)
else:
start_scores = nn.functional.softmax(start_scores)
end_scores = nn.functional.softmax(end_scores)
return start_scores, end_scores
def MaskHelper(seg, color):
# green
mask = torch.Tensor()
if (color == 'green'):
mask = torch.lt(seg[0], 0.1)
mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1))
mask = torch.mul(mask, torch.lt(seg[2], 0.1))
elif (color == 'black'):
mask = torch.lt(seg[0], 0.1)
mask = torch.mul(mask, torch.lt(seg[1], 0.1))
mask = torch.mul(mask, torch.lt(seg[2], 0.1))
elif (color == 'white'):
mask = torch.gt(seg[0], 1 - 0.1)
mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1))
mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1))
elif (color == 'red'):
mask = torch.gt(seg[0], 1 - 0.1)
mask = torch.mul(mask, torch.lt(seg[1], 0.1))
mask = torch.mul(mask, torch.lt(seg[2], 0.1))
elif (color == 'blue'):
mask = torch.lt(seg[0], 0.1)
mask = torch.mul(mask, torch.lt(seg[1], 0.1))
mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1))
elif (color == 'yellow'):
mask = torch.gt(seg[0], 1 - 0.1)
mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1))
mask = torch.mul(mask, torch.lt(seg[2], 0.1))
elif (color == 'grey'):
mask = torch.lt(seg[0], 0.1)
mask = torch.mul(mask, torch.lt(seg[1], 0.1))
mask = torch.mul(mask, torch.lt(seg[2], 0.1))
elif (color == 'lightblue'):
mask = torch.lt(seg[0], 0.1)
mask = torch.mul(mask, torch.gt(seg[1], 1 - 0.1))
mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1))
elif (color == 'purple'):
mask = torch.gt(seg[0], 1 - 0.1)
mask = torch.mul(mask, torch.lt(seg[1], 0.1))
mask = torch.mul(mask, torch.gt(seg[2], 1 - 0.1))
else:
print('MaskHelper(): color not recognized, color = ' + color)
return mask.float()
def vtln_warp_freq(vtln_low_cutoff, vtln_high_cutoff, low_freq, high_freq,
vtln_warp_factor, freq):
r"""This computes a VTLN warping function that is not the same as HTK's one,
but has similar inputs (this function has the advantage of never producing
empty bins).
This function computes a warp function F(freq), defined between low_freq
and high_freq inclusive, with the following properties:
F(low_freq) == low_freq
F(high_freq) == high_freq
The function is continuous and piecewise linear with two inflection
points.
The lower inflection point (measured in terms of the unwarped
frequency) is at frequency l, determined as described below.
The higher inflection point is at a frequency h, determined as
described below.
If l <= f <= h, then F(f) = f/vtln_warp_factor.
If the higher inflection point (measured in terms of the unwarped
frequency) is at h, then max(h, F(h)) == vtln_high_cutoff.
Since (by the last point) F(h) == h/vtln_warp_factor, then
max(h, h/vtln_warp_factor) == vtln_high_cutoff, so
h = vtln_high_cutoff / max(1, 1/vtln_warp_factor).
= vtln_high_cutoff * min(1, vtln_warp_factor).
If the lower inflection point (measured in terms of the unwarped
frequency) is at l, then min(l, F(l)) == vtln_low_cutoff
This implies that l = vtln_low_cutoff / min(1, 1/vtln_warp_factor)
= vtln_low_cutoff * max(1, vtln_warp_factor)
Args:
vtln_low_cutoff (float): Lower frequency cutoffs for VTLN
vtln_high_cutoff (float): Upper frequency cutoffs for VTLN
low_freq (float): Lower frequency cutoffs in mel computation
high_freq (float): Upper frequency cutoffs in mel computation
vtln_warp_factor (float): Vtln warp factor
freq (torch.Tensor): given frequency in Hz
Returns:
torch.Tensor: Freq after vtln warp
"""
assert vtln_low_cutoff > low_freq, 'be sure to set the vtln_low option higher than low_freq'
assert vtln_high_cutoff < high_freq, 'be sure to set the vtln_high option lower than high_freq [or negative]'
l = vtln_low_cutoff * max(1.0, vtln_warp_factor)
h = vtln_high_cutoff * min(1.0, vtln_warp_factor)
scale = 1.0 / vtln_warp_factor
Fl = scale * l # F(l)
Fh = scale * h # F(h)
assert l > low_freq and h < high_freq
# slope of left part of the 3-piece linear function
scale_left = (Fl - low_freq) / (l - low_freq)
# [slope of center part is just "scale"]
# slope of right part of the 3-piece linear function
scale_right = (high_freq - Fh) / (high_freq - h)
res = torch.empty_like(freq)
outside_low_high_freq = torch.lt(freq, low_freq) | torch.gt(
freq, high_freq) # freq < low_freq || freq > high_freq
before_l = torch.lt(freq, l) # freq < l
before_h = torch.lt(freq, h) # freq < h
after_h = torch.ge(freq, h) # freq >= h
# order of operations matter here (since there is overlapping frequency regions)
res[after_h] = high_freq + scale_right * (freq[after_h] - high_freq)
res[before_h] = scale * freq[before_h]
res[before_l] = low_freq + scale_left * (freq[before_l] - low_freq)
res[outside_low_high_freq] = freq[outside_low_high_freq]
return res
def get_mel_banks(num_bins, window_length_padded, sample_freq, low_freq,
high_freq, vtln_low, vtln_high, vtln_warp_factor):
# type: (int, int, float, float, float, float, float)
"""
Returns:
Tuple[torch.Tensor, torch.Tensor]: The tuple consists of ``bins`` (which is
melbank of size (``num_bins``, ``num_fft_bins``)) and ``center_freqs`` (which is
center frequencies of bins of size (``num_bins``)).
"""
assert num_bins > 3, 'Must have at least 3 mel bins'
assert window_length_padded % 2 == 0
num_fft_bins = window_length_padded / 2
nyquist = 0.5 * sample_freq
if high_freq <= 0.0:
high_freq += nyquist
assert (0.0 <= low_freq < nyquist) and (0.0 < high_freq <= nyquist) and (low_freq < high_freq), \
('Bad values in options: low-freq %f and high-freq %f vs. nyquist %f' % (low_freq, high_freq, nyquist))
# fft-bin width [think of it as Nyquist-freq / half-window-length]
fft_bin_width = sample_freq / window_length_padded
mel_low_freq = mel_scale_scalar(low_freq)
mel_high_freq = mel_scale_scalar(high_freq)
# divide by num_bins+1 in next line because of end-effects where the bins
# spread out to the sides.
mel_freq_delta = (mel_high_freq - mel_low_freq) / (num_bins + 1)
if vtln_high < 0.0:
vtln_high += nyquist
assert vtln_warp_factor == 1.0 or ((low_freq < vtln_low < high_freq) and
(0.0 < vtln_high < high_freq) and (vtln_low < vtln_high)), \
('Bad values in options: vtln-low %f and vtln-high %f, versus low-freq %f and high-freq %f' %
(vtln_low, vtln_high, low_freq, high_freq))
bin = torch.arange(num_bins).unsqueeze(1)
left_mel = mel_low_freq + bin * mel_freq_delta # size(num_bins, 1)
center_mel = mel_low_freq + (bin +
1.0) * mel_freq_delta # size(num_bins, 1)
right_mel = mel_low_freq + (bin +
2.0) * mel_freq_delta # size(num_bins, 1)
if vtln_warp_factor != 1.0:
left_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq, high_freq,
vtln_warp_factor, left_mel)
center_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq,
high_freq, vtln_warp_factor,
center_mel)
right_mel = vtln_warp_mel_freq(vtln_low, vtln_high, low_freq,
high_freq, vtln_warp_factor, right_mel)
center_freqs = inverse_mel_scale(center_mel) # size (num_bins)
# size(1, num_fft_bins)
mel = mel_scale(fft_bin_width * torch.arange(num_fft_bins)).unsqueeze(0)
# size (num_bins, num_fft_bins)
up_slope = (mel - left_mel) / (center_mel - left_mel)
down_slope = (right_mel - mel) / (right_mel - center_mel)
if vtln_warp_factor == 1.0:
# left_mel < center_mel < right_mel so we can min the two slopes and clamp negative values
bins = torch.max(torch.zeros(1), torch.min(up_slope, down_slope))
else:
# warping can move the order of left_mel, center_mel, right_mel anywhere
bins = torch.zeros_like(up_slope)
up_idx = torch.gt(mel, left_mel) & torch.le(
mel, center_mel) # left_mel < mel <= center_mel
down_idx = torch.gt(mel, center_mel) & torch.lt(
mel, right_mel) # center_mel < mel < right_mel
bins[up_idx] = up_slope[up_idx]
bins[down_idx] = down_slope[down_idx]
return bins, center_freqs
def step(self, blob):
'''
This function should be called at every timestep to perform tracking
with a blob containing the image information.
'''
for t in self.tracks:
# add current position to last_pos list
t.last_pos.append(t.pos.clone())
# 1. Look for new detections
if self.public_detections:
boxes, scores = blob['boxes'], blob['scores']
if boxes.nelement() == 0:
boxes = scores = torch.zeros(0).cuda()
else:
boxes = boxes.cuda()
scores = scores.cuda()
else:
boxes, scores = self.obj_detect.detect(blob['img'])
if boxes.nelement() > 0:
boxes = clip_boxes_to_image(boxes, blob['img'].shape[-2:])
inds = torch.gt(scores,
self.detection_person_thresh).nonzero().view(-1)
else:
inds = torch.zeros(0).cuda()
if inds.nelement() > 0:
det_pos = boxes[inds]
det_scores = scores[inds]
else:
det_pos = torch.zeros(0).cuda()
det_scores = torch.zeros(0).cuda()
# 2. Predict tracks
num_tracks = 0
nms_inp_reg = torch.zeros(0).cuda()
if len(self.tracks):
# 2.1 Align
if self.do_align:
self.align(blob)
# 2.2 Apply motion model
if self.motion_model_cfg['enabled']:
self.motion()
self.tracks = [t for t in self.tracks if t.has_positive_area()]
# 2.3 Regress
person_scores = self.regress_tracks(blob)
if len(self.tracks):
# nms here if tracks overlap
keep = nms(self.get_pos(), person_scores,
self.regression_nms_thresh)
self.tracks_to_inactive([
self.tracks[i] for i in list(range(len(self.tracks)))
if i not in keep
])
if keep.nelement() > 0:
if self.do_reid:
new_features = self.get_appearances(blob)
self.add_features(new_features)
# 3. Create new tracks
# !!! Here NMS is used to filter out detections that are already covered by tracks. This is
# !!! done by iterating through the active tracks one by one, assigning them a bigger score
# !!! than 1 (maximum score for detections) and then filtering the detections with NMS.
# !!! In the paper this is done by calculating the overlap with existing tracks, but the
# !!! result stays the same.
if det_pos.nelement() > 0:
keep = nms(det_pos, det_scores, self.detection_nms_thresh)
det_pos = det_pos[keep]
det_scores = det_scores[keep]
# Check with every track in a single run (problem if tracks delete each other)
for t in self.tracks:
nms_track_pos = torch.cat([t.pos, det_pos])
nms_track_scores = torch.cat(
[torch.tensor([2.0]).to(det_scores.device), det_scores])
keep = nms(nms_track_pos, nms_track_scores,
self.detection_nms_thresh)
keep = keep[torch.ge(keep, 1)] - 1
det_pos = det_pos[keep]
det_scores = det_scores[keep]
if keep.nelement() == 0:
break
if det_pos.nelement() > 0:
new_det_pos = det_pos
new_det_scores = det_scores
# Try to reidentify tracks
new_det_pos, new_det_scores, new_det_features = self.reid(
blob, new_det_pos, new_det_scores)
if new_det_pos.nelement() > 0:
self.add(new_det_pos, new_det_scores, new_det_features)
# 4. Generate results
for t in self.tracks:
if t.id not in self.results.keys():
self.results[t.id] = {}
self.results[t.id][self.im_index] = np.concatenate([
t.pos[0].clone().cpu().numpy(),
np.array([t.score.clone().cpu().numpy().item()])
])
for t in self.inactive_tracks:
t.count_inactive += 1
self.inactive_tracks = [
t for t in self.inactive_tracks if t.has_positive_area()
and t.count_inactive <= self.inactive_patience
]
self.im_index += 1
self.last_image = blob['img'][0]
def Generate_SQL(cond_op_score,
cond_col_score,
cond_num_score,
cond_str_out,
predicted_gate,
truth_seq,
SQL_YN,
fp,
args=None):
t_fp, p_fp, sql_fp = fp
batch_size = predicted_gate.size(0)
sigm = nn.Sigmoid()
cond_col_prob = sigm(cond_col_score)
sigmoid_matrix = torch.ones_like(cond_col_prob) * 0.5
predicted_cond_col = torch.gt(cond_col_prob,
sigmoid_matrix).type(torch.cuda.FloatTensor)
for i in range(batch_size):
last_index = predicted_gate[i].size(0) - 1
output_query = 'None'
pred = 'None'
# # Write ground truth answer
# truth_seq_i = unicodedata.normalize('NFKD', truth_seq[i]).encode('ascii','ignore')
# simple_truth_sql = translate_query_to_simple(truth_seq_i)
# t_fp.write(str(SQL_YN[i].data.item()) + ' | ' + truth_seq_i + ' | ' + str(simple_truth_sql) + '\n')
if predicted_gate[i, last_index] == 1: # <t2> : gate open
col_list = predicted_cond_col[i, last_index]
output_query = 'SELECT * FROM Airdialogue_Table WHERE '
pred = ''
for c in range(11):
if col_list[c] == 1:
_, prediction = torch.max(
cond_str_out[c][i][last_index].unsqueeze(0).data, 1)
pred += str(prediction.item()) + ' '
else:
pred += str(-1) + ' ' # None
if col_list[11] == 1:
pred += str(0) + ' '
else:
pred += str(-1) + ' '
if col_list[0] == 1:
_, departure_airport = torch.max(
cond_str_out[0][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + "departure_airport"
_, op = torch.max(
cond_op_score[i][last_index][0].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(departure_airport.item())
if col_list[1] == 1:
_, return_airport = torch.max(
cond_str_out[1][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND return_airport"
_, op = torch.max(
cond_op_score[i][last_index][1].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(return_airport.item())
if col_list[2] == 1:
_, departure_month = torch.max(
cond_str_out[2][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND departure_month"
_, op = torch.max(
cond_op_score[i][last_index][2].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(departure_month.item())
if col_list[3] == 1:
_, return_month = torch.max(
cond_str_out[3][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND return_month"
_, op = torch.max(
cond_op_score[i][last_index][3].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(return_month.item())
if col_list[4] == 1:
_, departure_day = torch.max(
cond_str_out[4][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND departure_day"
_, op = torch.max(
cond_op_score[i][last_index][4].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(departure_day.item())
if col_list[5] == 1:
_, return_day = torch.max(
cond_str_out[5][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND return_day"
_, op = torch.max(
cond_op_score[i][last_index][5].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(return_day.item())
if col_list[6] == 1:
_, departure_time_num = torch.max(
cond_str_out[6][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND departure_time_num"
_, op = torch.max(
cond_op_score[i][last_index][6].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(departure_time_num.item())
if col_list[7] == 1:
_, return_time_num = torch.max(
cond_str_out[7][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND return_time_num"
_, op = torch.max(
cond_op_score[i][last_index][7].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(return_time_num.item())
if col_list[8] == 1:
_, class_ = torch.max(
cond_str_out[8][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND class"
_, op = torch.max(
cond_op_score[i][last_index][8].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(class_.item())
if col_list[9] == 1:
_, price = torch.max(
cond_str_out[9][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND price"
_, op = torch.max(
cond_op_score[i][last_index][9].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(price.item())
if col_list[10] == 1:
_, num_connections = torch.max(
cond_str_out[10][i][last_index].unsqueeze(0).data, 1)
output_query = output_query + " AND num_connections"
_, op = torch.max(
cond_op_score[i][last_index][10].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(num_connections.item())
if col_list[11] == 1:
output_query = output_query + " AND airline_preference"
_, op = torch.max(
cond_op_score[i][last_index][11].unsqueeze(0).data, 1)
if op.item() == 0:
output_query = output_query + " = "
if op.item() == 1:
output_query = output_query + " <= "
output_query = output_query + str(0)
# p_fp.write(str(int(predicted_gate[i, last_index].data.item())) + ' | ' + output_query + ' | ' + str(pred) + '\n')
# sql_fp.write(output_query + '\n')
return output_query, str(pred), str(
int(predicted_gate[i, last_index].data.item()))
normal_entropy = FixedNormal.entropy
FixedNormal.entropy = lambda self: normal_entropy(self).sum(-1)
FixedNormal.mode = lambda self: self.mean
# Bernoulli
FixedBernoulli = torch.distributions.Bernoulli
log_prob_bernoulli = FixedBernoulli.log_prob
FixedBernoulli.log_probs = lambda self, actions: log_prob_bernoulli(
self, actions).view(actions.size(0), -1).sum(-1).unsqueeze(-1)
bernoulli_entropy = FixedBernoulli.entropy
FixedBernoulli.entropy = lambda self: bernoulli_entropy(self).sum(-1)
FixedBernoulli.mode = lambda self: torch.gt(self.probs, 0.5).float()
class Categorical(nn.Module):
def __init__(self, num_inputs, num_outputs, device):
super(Categorical, self).__init__()
self.device = device
init_ = lambda m: init(m,
nn.init.orthogonal_,
lambda x: nn.init.constant_(x, 0),
gain=0.01)
self.linear = init_(nn.Linear(num_inputs, num_outputs))
def forward(self, x):
print(torch.sum(a, 0)) # tensor([5, 7, 9]) print(torch.sum(a, 1)) # tensor([ 6, 15]) print(a.sum(1)) # sum 中 dim 是几, 就是将对应的维度消掉 ## max compare argsort print(b.argsort(1)) # tensor([[0, 2, 1], [0, 2, 1]]) print(b.argsort(1)[:, -2:]) # tensor([[2, 1], [2, 1]]) print(b.max(1)) # (tensor([3, 6]), tensor([1, 1])); max only for the maximum. ## gt(input, other, out=None) # input (Tensor): the tensor to compare # other (Tensor or float): the tensor or value to compare # out (Tensor, optional): the output tensor that must be a `ByteTensor` a = torch.tensor([[1, 2], [3, 4]]) b = torch.tensor([[1, 1], [4, 4]]) print(torch.gt(a, b)) # tensor([[0, 1], [0, 0]], dtype=torch.uint8) print(a.gt(1.5)) # tensor([[0, 1], [1, 1]], dtype=torch.uint8) ## eq, ne, masked_select gold = torch.Tensor([[1, 2], [3, 0]]) pred = torch.Tensor([[1, 3], [3, 4]]) print(gold.ne(0)) # tensor([[1, 1], [1, 0]], dtype=torch.uint8) print(gold.eq(0)) # tensor([[0, 0], [0, 1]], dtype=torch.uint8) non_pad_mask = gold.ne(0); print(not_pad_mask) # tensor([[1, 1], [1, 0]], dtype=torch.uint8) n_correct = pred.eq(gold); print(n_correct) # tensor([[1, 0], [1, 0]], dtype=torch.uint8) n_correct = n_correct.masked_select(non_pad_mask); print(n_correct) # tensor([1, 0, 1], dtype=torch.uint8) print(n_correct.sum()) # tensor(2) ## prod print(torch.prod(torch.tensor([1, 2, 3]))) # tensor(6)
