def forward(self, x):
'''input: (batch_size, mic_channels, time_steps, mel_bins)'''
for conv_cnt in range(len(self.conv_block_list)):
x = self.conv_block_list[conv_cnt](x)
'''(batch_size, feature_maps, time_steps, mel_bins)'''
x = x.transpose(1, 2).contiguous()
x = x.view(x.shape[0], x.shape[1], -1).contiguous()
''' (batch_size, time_steps, feature_maps):'''
(x, _) = self.gru(x)
x = torch.tanh(x)
x = x[:, :, x.shape[-1]//2:] * x[:, :, :x.shape[-1]//2]
'''(batch_size, time_steps, feature_maps)'''
if self.attn is not None:
x = self.attn.forward(x, x, x)
# out - batch x hidden x seq
x = torch.tanh(x)
x_rnn = x
for fnn_cnt in range(len(self.fnn_list)-1):
x = torch.relu_(self.fnn_list[fnn_cnt](x))
doa = torch.tanh(self.fnn_list[-1](x))
'''(batch_size, time_steps, label_dim)'''
if self.use_hnet and self.use_activity_out:
for fnn_cnt in range(len(self.fnn_act_list)-1):
x_rnn = torch.relu_(self.fnn_act_list[fnn_cnt](x_rnn))
activity = torch.tanh(self.fnn_act_list[-1](x_rnn))
return doa, activity
else:
return doa
def test_batch_mm(n: int, use_t1: bool):
T1 = torch.zeros((n, n))
T2 = torch.zeros((n, n))
T3 = torch.zeros((n, n))
T4 = torch.zeros((n, n))
T5 = torch.zeros((n, n))
T6 = torch.zeros((n, n))
T7 = torch.zeros((n, n))
T8 = torch.zeros((n, n))
T9 = torch.zeros((n, n))
T10 = torch.zeros((n, n))
if use_t1:
torch.relu_(T1)
return (
torch.mm(T1, T2)
+ torch.mm(T3, T4)
+ torch.mm(T5, T6)
+ torch.mm(T7, T8)
+ torch.mm(T9, T10)
)
else:
return (
torch.mm(T2, T3)
+ torch.mm(T4, T5)
+ torch.mm(T6, T7)
+ torch.mm(T8, T9)
)
def test_batch_mm(n: int):
A = torch.zeros((n, n))
T1 = torch.zeros((n, n))
T2 = torch.zeros((n, n))
T3 = torch.zeros((n, n))
T4 = torch.zeros((n, n))
T5 = torch.zeros((n, n))
T6 = torch.zeros((n, n))
T7 = torch.zeros((n, n))
T8 = torch.zeros((n, n))
T9 = torch.zeros((n, n))
T10 = torch.zeros((n, n))
torch.relu_(A)
result = {}
result["T1"] = torch.mm(A, T1)
result["T2"] = torch.mm(A, T2)
result["T3"] = torch.mm(A, T3)
result["T4"] = torch.mm(A, T4)
result["T5"] = torch.mm(A, T5)
result["T6"] = torch.mm(A, T6)
result["T7"] = torch.mm(A, T7)
result["T8"] = torch.mm(A, T8)
result["T9"] = torch.mm(A, T9)
result["T10"] = torch.mm(A, T10)
return result
def test_batch_mm(n: int):
T1 = torch.zeros((n, n))
T2 = torch.zeros((n, n))
T3 = torch.zeros((n, n))
T4 = torch.zeros((n, n))
T5 = torch.zeros((n, n))
T6 = torch.zeros((n, n))
T7 = torch.zeros((n, n))
T8 = torch.zeros((n, n))
T9 = torch.zeros((n, n))
T10 = torch.zeros((n, n))
torch.relu_(T1)
result = {}
result["no_mutated_parameters"] = (
torch.mm(T2, T3)
+ torch.mm(T4, T5)
+ torch.mm(T6, T7)
+ torch.mm(T8, T9)
)
result["all_parameters"] = (
torch.mm(T1, T2)
+ torch.mm(T3, T4)
+ torch.mm(T5, T6)
+ torch.mm(T7, T8)
+ torch.mm(T9, T10)
)
return result
def jojo_0(self, input_vars_12, var_220, input_vars_14, input_vars_11,
input_vars_13):
var_221 = torch.relu_(var_220)
var_239 = torch._convolution(var_221, input_vars_11, input_vars_12, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_240 = torch.relu_(var_239)
var_258 = torch._convolution(var_240, input_vars_13, input_vars_14, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
return var_258
def forward(self, query_users, target_items, auxiliary_items=None):
only_target = False
if auxiliary_items is None:
only_target = True
auxiliary_items = [
random.choice(self.source_ratings[user_id.item()]) if len(
self.source_ratings[user_id.item()]) > 0 else self.item_padding_idx for user_id in query_users[0]]
auxiliary_items = (torch.tensor(auxiliary_items, dtype=torch.long, device=query_users[0].device),)
query_users = list(map(lambda x: x.expand(target_items[0].size(0)), query_users))
auxiliary_items = list(map(lambda x: x.expand(target_items[0].size(0)), auxiliary_items))
query_users = self.useritem_embeds(*query_users, is_user=True)
target_items, auxiliary_items = self.useritem_embeds(*target_items, is_user=False), self.useritem_embeds(
*auxiliary_items, is_user=False)
target_x = torch.cat((*query_users, *target_items), dim=1)
auxiliary_x = torch.cat((*query_users, *auxiliary_items), dim=1)
for target_layer, auxiliary_layer, transfer_layer in zip(self.target_layers, self.auxiliary_layers,
self.transfer_layers):
new_target_x = target_layer(target_x) + transfer_layer(auxiliary_x)
new_auxiliary_x = auxiliary_layer(auxiliary_x) + transfer_layer(target_x)
target_x, auxiliary_x = new_target_x, new_auxiliary_x
target_x, auxiliary_x = torch.relu_(target_x), torch.relu_(auxiliary_x)
if only_target:
return self.target_output(target_x).squeeze(-1)
else:
return self.target_output(target_x).squeeze(-1), self.auxiliary_output(auxiliary_x).squeeze(-1)
def forward(self, x):
# Linear layers to regress parameters
# Note: cnf is the predicted variance and must always be positive
x = self.conv_ops(x)
x = x.flatten(start_dim=-2).mean(dim=-1)
x = self.l1(x)
torch.selu_(x)
if self.l2_conf:
cnf = self.l2_conf(x)
else:
cnf = []
out = self.l2_out(x)
if self.ellipse_pred:
# Ellipse centers should be between -1 and 1
torch.nn.functional.hardtanh_(out[:, [0, 1, 5, 6]])
# Ellipse axes should be between 0 and 1
torch.relu_(out[:, [2, 3, 7, 8]])
# Ellipse orientation should be between 0 and pi
out[:, [4, 9]] = np.pi * torch.clamp(out[:, [4, 9]], 0., 1.)
else:
pass
return out, cnf
def forward(self, inputs):
x, u = inputs # state, action
x = self.bn0(x)
x = torch.relu_(self.linear1(x))
x = torch.relu_(self.linear2(x))
V = self.V(x) # applying linear1, linear2 on x and finally V.
mu = torch.tanh(self.mu(
x)) # applying linear1, linear2 on x and finally F.than(mu(x)).
Q = None
# calculating the advantage function
if u is not None:
num_outputs = mu.size(1)
L = self.L(x).view(-1, num_outputs, num_outputs)
L = L * self.tril_mask.expand_as(L) + torch.exp(
L) * self.diag_mask.expand_as(L)
P = torch.bmm(L, L.transpose(2, 1))
u_mu = (u - mu).unsqueeze(2)
A = -0.5 * torch.bmm(torch.bmm(u_mu.transpose(2, 1), P),
u_mu)[:, :, 0]
Q = A + V
return mu, Q, V
def fixed_circle_loss_clean(output, target, mask, num_classes,
softplus: torch.nn.Softplus):
"""
output: (B, N)
target: (B, X)
mask: (B, N)
num_classes = N
loss = log(1 + \sum_i exp(s^{neg}_i) \sum_j exp(s^{pos}_j))
\gamma = 1, m = 0
"""
output = output.float()
# seq_len = output.size(1)
target_mask = (target == -1)
target[target_mask] = num_classes
one_hot_target = F.one_hot(target, num_classes + 1)
one_hot_target = one_hot_target.sum(dim=1)
one_hot_target = one_hot_target[:, :-1].to(dtype=output.dtype)
mask = mask.to(dtype=output.dtype)
all_mask = (one_hot_target.sum(dim=-1) == 0)
mask_for_pos = torch.clamp_max(1 - one_hot_target + mask, max=1.0)
mask_for_neg = torch.clamp_max(one_hot_target + mask, max=1.0)
_flag = -1e12
ap = torch.clamp_min(1 - output.detach(), min=0.)
an = torch.clamp_min(output.detach(), min=0.)
delta_p = 1
delta_n = 0
output = (1 - 2 * one_hot_target) * output # Positive: -1 // Negative: +1
logit_pos = ap * (delta_p + output) + mask_for_pos * _flag
x = logit_pos.max(dim=-1, keepdim=True)[0].detach()
x = torch.relu_(x)
logit_pos = logit_pos - x
# assert output_pos.size() == output.size(), (output_pos.size(), output.size())
logit_neg = an * (output - delta_n) + mask_for_neg * _flag
y = logit_neg.max(dim=-1, keepdim=True)[0].detach()
y = torch.relu_(y)
logit_neg = logit_neg - y
loss = softplus(x + y + torch.logsumexp(logit_pos, dim=-1) +
torch.logsumexp(logit_neg, dim=-1))
masked_loss = loss.masked_fill(all_mask, 0.)
return masked_loss
def forward(self, x):
x = self.fc1(x)
x = torch.relu_(x)
x = self.fc2(x)
x = torch.relu_(x)
x = self.fc3(x)
x = torch.relu_(x)
return x
def jojo_20(self, input_vars_45, input_vars_44, var_848, var_799, input_vars_47, input_vars_46, input_vars_49, input_vars_43):
var_850 = torch.add(var_848, var_799, alpha=1)
var_851 = torch.relu_(var_850)
var_870 = torch._convolution(var_851, input_vars_43, None, [2, 2, ], [1, 1, ], [1, 1, ], False, [0, 0, ], 1, False, False, True)
var_875 = torch.batch_norm(var_870, input_vars_44, input_vars_45, input_vars_46, input_vars_47, False, 0.1, 1e-05, True)
var_876 = torch.relu_(var_875)
var_895 = torch._convolution(var_876, input_vars_49, None, [1, 1, ], [1, 1, ], [1, 1, ], False, [0, 0, ], 1, False, False, True)
return var_851, var_895
def jojo_6(self, input_vars_44, input_vars_34, input_vars_37, var_459,
input_vars_45, var_508, input_vars_31, input_vars_38,
input_vars_41, input_vars_47, input_vars_40, input_vars_43,
input_vars_33, input_vars_35, input_vars_46, input_vars_32,
input_vars_39):
var_510 = torch.add(var_508, var_459, alpha=1)
var_511 = torch.relu_(var_510)
var_530 = torch._convolution(var_511, input_vars_31, None, [
2,
2,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_535 = torch.batch_norm(var_530, input_vars_32, input_vars_33,
input_vars_34, input_vars_35, False, 0.1,
1e-05, True)
var_536 = torch.relu_(var_535)
var_555 = torch._convolution(var_536, input_vars_37, None, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_560 = torch.batch_norm(var_555, input_vars_38, input_vars_39,
input_vars_40, input_vars_41, False, 0.1,
1e-05, True)
var_579 = torch._convolution(var_511, input_vars_43, None, [
2,
2,
], [
0,
0,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_584 = torch.batch_norm(var_579, input_vars_44, input_vars_45,
input_vars_46, input_vars_47, False, 0.1,
1e-05, True)
return var_560, var_584
def jojo_5(self, input_vars_50, var_586, input_vars_51, input_vars_49,
input_vars_57, input_vars_64, input_vars_52, input_vars_55,
input_vars_61, input_vars_63, input_vars_58, input_vars_56,
input_vars_53, input_vars_65, input_vars_62, input_vars_59):
var_587 = torch.relu_(var_586)
var_606 = torch._convolution(var_587, input_vars_49, None, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_611 = torch.batch_norm(var_606, input_vars_50, input_vars_51,
input_vars_52, input_vars_53, False, 0.1,
1e-05, True)
var_612 = torch.relu_(var_611)
var_631 = torch._convolution(var_612, input_vars_55, None, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_636 = torch.batch_norm(var_631, input_vars_56, input_vars_57,
input_vars_58, input_vars_59, False, 0.1,
1e-05, True)
var_638 = torch.add(var_636, var_587, alpha=1)
var_639 = torch.relu_(var_638)
var_658 = torch._convolution(var_639, input_vars_61, None, [
2,
2,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_663 = torch.batch_norm(var_658, input_vars_62, input_vars_63,
input_vars_64, input_vars_65, False, 0.1,
1e-05, True)
return var_663, var_639
def forward(self, x):
x = x
x = self.bn_0(torch.relu_(self.fc_0(x)))
# x = F.dropout(x)
x = self.bn_1(torch.relu_(self.fc_1(x)))
# x = F.dropout(x)
x = self.bn_2(torch.relu_(self.fc_2(x)))
# x = F.dropout(x)
return self.output(x).squeeze()
def feed_forward_fn(qp_vec):
qp_dict = quant_params_vec2dict(keys, qp_vec, search_clipping)
quantized_layer.update_linear_quant_params(qp_dict)
# Using cloned input, required if the layer is inplace
y = layer(input.clone().detach())
if getattr(quantized_layer, 'clip_half_range', False):
torch.relu_(y)
q_y = quantized_layer(input.clone().detach())
loss = loss_fn(y, q_y)
return loss
def _compute_scales(A):
"""Compute optimal parameters for scaling-and-squaring algorithm.
The constants used in this function are determined by the MATLAB
function found in
https://github.com/cetmann/pytorch_expm/blob/master/determine_frechet_scaling_constant.m
"""
norm = matrix_1_norm(A)
max_norm = torch.max(norm)
s = torch.zeros_like(norm)
if A.dtype == torch.float64:
if A.requires_grad:
ell = { 3: 0.010813385777848,
5: 0.199806320697895,
7: 0.783460847296204,
9: 1.782448623969279,
13: 4.740307543765127}
else:
ell = { 3: 0.014955852179582,
5: 0.253939833006323,
7: 0.950417899616293,
9: 2.097847961257068,
13: 5.371920351148152}
if max_norm >= ell[9]:
m = 13
magic_number = ell[m]
s = torch.relu_(torch.ceil(torch.log2_(norm / magic_number)))
else:
for m in [3,5,7,9]:
if max_norm < ell[m]:
magic_number = ell[m]
# results in s = torch.tensor([0,...,0])
break
elif A.dtype == torch.float32:
if A.requires_grad:
ell = {3: 0.308033041845330,
5: 1.482532614793145,
7: 3.248671755200478}
else:
ell = {3: 0.425873001692283,
5: 1.880152677804762,
7: 3.925724783138660}
if max_norm >= ell[5]:
m = 7
magic_number = ell[m]
s = torch.relu_(torch.ceil(torch.log2_(norm / magic_number)))
else:
for m in [3,5]:
if max_norm < ell[m]:
# results in s = torch.tensor([0,...,0])
magic_number = ell[m]
break
return s, m
def forward(cxt, input, bias, dilations, blocksize, *weights):
with torch.no_grad():
cxt.stride = 1
cxt.paddings = dilations
cxt.groups = 1
cxt.dilations = dilations
cxt.n_conv = len(dilations)
cxt.blocksize = blocksize
cxt.ndim = input.dim() - 2
if cxt.ndim == 1:
conv = torch.nn.functional.conv1d
elif cxt.ndim == 2:
conv = torch.nn.functional.conv2d
elif cxt.ndim == 3:
conv = torch.nn.functional.conv3d
else:
raise ValueError('Only supports 1 2 or 3 dimensions')
result = torch.empty(input.shape[0],
input.shape[1] + blocksize * cxt.n_conv,
*input.shape[2:],
device=input.device)
result[:, :input.shape[1]] = input
result_start = input.shape[1]
bias_start = 0
for i in range(cxt.n_conv):
# Extract variables
sub_weight = weights[i]
sub_bias = bias[i * blocksize:(i + 1) *
blocksize] if bias is not None else None
padding = cxt.paddings[i]
dilation = cxt.dilations[i]
# Compute convolution
sub_result = conv(result[:, :result_start], sub_weight,
sub_bias, cxt.stride, padding, dilation,
cxt.groups)
# Apply ReLU
torch.relu_(sub_result)
# Update result array
result[:, result_start:result_start + blocksize] = sub_result
# Update steps etc
result_start += blocksize
bias_start += blocksize
cxt.save_for_backward(bias, result, *weights)
return result
def jojo_7(self, input_vars_25, var_431, input_vars_16, input_vars_13,
input_vars_15, input_vars_23, input_vars_19, input_vars_21,
input_vars_22, var_407, input_vars_17, input_vars_14,
input_vars_20):
var_432 = torch.relu_(var_431)
var_451 = torch._convolution(var_432, input_vars_13, None, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_456 = torch.batch_norm(var_451, input_vars_14, input_vars_15,
input_vars_16, input_vars_17, False, 0.1,
1e-05, True)
var_458 = torch.add(var_456, var_407, alpha=1)
var_459 = torch.relu_(var_458)
var_478 = torch._convolution(var_459, input_vars_19, None, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_483 = torch.batch_norm(var_478, input_vars_20, input_vars_21,
input_vars_22, input_vars_23, False, 0.1,
1e-05, True)
var_484 = torch.relu_(var_483)
var_503 = torch._convolution(var_484, input_vars_25, None, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
return var_459, var_503
def test_relu_(self):
x = torch.randn((4, 5), dtype=torch.float32) * 10
x1 = x.clone().requires_grad_()
x2 = x.clone().to_mkldnn().requires_grad_()
y1 = torch.relu_(x1.clone())
y2 = torch.relu_(x2.clone()).to_dense()
loss1 = y1.sum()
loss2 = y2.sum()
loss1.backward()
loss2.backward()
self.assertEqual(y1, y2)
self.assertEqual(x1.grad, x2.grad.to_dense())
def test_batch_mm(n: int):
T1 = torch.zeros((n, n))
T2 = torch.zeros((n, n))
T3 = torch.zeros((n, n))
T4 = torch.zeros((n, n))
T5 = torch.zeros((n, n))
T6 = torch.zeros((n, n))
T7 = torch.zeros((n, n))
T8 = torch.zeros((n, n))
torch.relu_(T1)
result = (torch.mm(T1, T2) + torch.mm(T3, T4) + torch.mm(T5, T6) +
torch.mm(T7, T8))
return result
def forward(self, x: torch.Tensor) -> Dict[str, torch.Tensor]:
value = self.backbone(x)
if self.backbone_key is not None:
value = value[self.backbone_key]
features = torch.relu_(self.down_sampler(torch.relu_(value)))
value = {
"hm": self.head_heatmap(features),
"wh": self.head_width_height(features),
"reg": self.head_offset_regularizer(features),
}
return value
def jojo_19(self, var_851, input_vars_71, input_vars_70, input_vars_67, input_vars_55, input_vars_57, input_vars_56, var_900, input_vars_62, input_vars_65, input_vars_73, input_vars_61, input_vars_68, input_vars_63, input_vars_69, input_vars_59, input_vars_64, input_vars_58):
var_919 = torch._convolution(var_851, input_vars_55, None, [2, 2, ], [0, 0, ], [1, 1, ], False, [0, 0, ], 1, False, False, True)
var_924 = torch.batch_norm(var_919, input_vars_56, input_vars_57, input_vars_58, input_vars_59, False, 0.1, 1e-05, True)
var_926 = torch.add(var_900, var_924, alpha=1)
var_927 = torch.relu_(var_926)
var_946 = torch._convolution(var_927, input_vars_61, None, [1, 1, ], [1, 1, ], [1, 1, ], False, [0, 0, ], 1, False, False, True)
var_951 = torch.batch_norm(var_946, input_vars_62, input_vars_63, input_vars_64, input_vars_65, False, 0.1, 1e-05, True)
var_952 = torch.relu_(var_951)
var_971 = torch._convolution(var_952, input_vars_67, None, [1, 1, ], [1, 1, ], [1, 1, ], False, [0, 0, ], 1, False, False, True)
var_976 = torch.batch_norm(var_971, input_vars_68, input_vars_69, input_vars_70, input_vars_71, False, 0.1, 1e-05, True)
var_978 = torch.add(var_976, var_927, alpha=1)
var_979 = torch.relu_(var_978)
var_998 = torch._convolution(var_979, input_vars_73, None, [1, 1, ], [1, 1, ], [1, 1, ], False, [0, 0, ], 1, False, False, True)
return var_979, var_998
def forward(self, x):
# layer 1
h = self.conv_1(x)
h = self.bn_1(h)
torch.relu_(h)
h = self.se_1(h)
h = self.mp_1(h)
# layer 2
h = self.conv_2(h)
h = self.bn_2(h)
torch.relu_(h)
h = self.se_2(h)
h = self.mp_2(h)
# layer 3
h = self.conv_3(h)
h = self.bn_3(h)
torch.relu_(h)
h = self.se_3(h)
h = self.mp_3(h)
# flatten
h = h.view(x.shape[0], -1)
# hidden
h = self.dense_1(h)
h = self.bn_4(h)
torch.relu_(h)
logits = self.dense_2(h)
return logits
def sample_dose_distributions(data):
original = data["A"]
original_variance = data["A_var"]
target = data["B"]
target_variance = data["B_var"]
original += torch.sqrt(original_variance) * torch.randn_like(original_variance)
target += torch.sqrt(target_variance) * torch.randn_like(target_variance)
torch.relu_(original)
torch.relu_(target)
return data
def _compute_scales(A):
norm = matrix_1_norm(A)
max_norm = torch.max(norm)
s = torch.zeros_like(norm)
if A.dtype == torch.float64:
if A.requires_grad:
ell = { 3: 0.010813385777848,
5: 0.199806320697895,
7: 0.783460847296204,
9: 1.782448623969279,
13: 4.740307543765127}
else:
ell = { 3: 0.014955852179582,
5: 0.253939833006323,
7: 0.950417899616293,
9: 2.097847961257068,
13: 5.371920351148152}
if max_norm >= ell[9]:
m = 13
magic_number = ell[m]
s = torch.relu_(torch.ceil(torch.log2_(norm / magic_number)))
else:
for m in [3,5,7,9]:
if max_norm < ell[m]:
magic_number = ell[m]
# results in s = torch.tensor([0,...,0])
break
elif A.dtype == torch.float32:
if A.requires_grad:
ell = {3: 0.308033041845330,
5: 1.482532614793145,
7: 3.248671755200478}
else:
ell = {3: 0.425873001692283,
5: 1.880152677804762,
7: 3.925724783138660}
if max_norm >= ell[5]:
m = 7
magic_number = ell[m]
s = torch.relu_(torch.ceil(torch.log2_(norm / magic_number)))
else:
for m in [3,5]:
if max_norm < ell[m]:
# results in s = torch.tensor([0,...,0])
magic_number = ell[m]
break
return s, m
def forward(self, q, k, v, mask=None):
d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
sz_b, len_q, _ = q.size()
sz_b, len_k, _ = k.size()
sz_b, len_v, _ = v.size()
residual = q
q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)
q = q.permute(2, 0, 1, 3).contiguous().view(-1, len_q, d_k) # (n*b) x lq x dk
k = k.permute(2, 0, 1, 3).contiguous().view(-1, len_k, d_k) # (n*b) x lk x dk
v = v.permute(2, 0, 1, 3).contiguous().view(-1, len_v, d_v) # (n*b) x lv x dv
if mask is not None:
mask = mask.repeat(n_head, 1, 1) # (n*b) x .. x ..
output, attn = self.attention(q, k, v, mask=mask)
output = output.view(n_head, sz_b, len_q, d_v)
output = output.permute(1, 2, 0, 3).contiguous().view(sz_b, len_q, -1) # b x lq x (n*dv)
if self.use_star:
output = self.dropout(torch.relu_(self.fc(output)))
output = self.layer_norm(output)
else:
output = self.dropout(self.fc(output))
output = self.layer_norm(output + residual)
return output, attn
def forward(self, x):
x = self.linear1(x)
x = torch.relu_(x)
# x = torch.sigmoid(x)
x = self.linear2(x)
x = torch.sigmoid(x)
return x
def forward(self, input):
x = torch.relu_(self.conv1(input))
x = self.conv2(x)
x = self.conv3(x)
if self.scaling is not None:
x = self.scaling * x
return x + input
def function_hook(input, *args, **kwargs):
class ReLU(nn.Module):
def __init__(self):
super().__init__()
output = torch.relu_(input.tensor(), *args, **kwargs)
return forward_hook(ReLU(*args, **kwargs), (input, ), output)
def jojo_1(self, var_169, input_vars_8, input_vars_7):
var_187 = torch._convolution(var_169, input_vars_7, input_vars_8, [
1,
1,
], [
1,
1,
], [
1,
1,
], False, [
0,
0,
], 1, False, False, True)
var_188 = torch.relu_(var_187)
var_202 = torch.max_pool2d(var_188, [
2,
2,
], [
2,
2,
], [
0,
0,
], [
1,
1,
], False)
return var_202
