def forward(self, x):
x = F.celu(self.input(x))
x = F.celu(self.hidden_1(x))
x = F.celu(self.hidden_2(x))
x = F.celu(self.hidden_3(x))
x = self.output(x)
return x
def forward(self, x):
x = self.pool(F.celu(self.conv1(x)))
x = self.pool(F.celu(self.conv2(x)))
x = x.view(-1, 32 * 38 * 23)
x = F.relu(self.fc1(x))
x = self.fc3(x)
return x
def forward(self, x):
x_short = x
x = F.celu(self.bn1(self.conv1(x)), alpha=0.075)
x = F.celu(self.bn2(self.conv2(x)), alpha=0.075)
x = self.bn3(self.conv3(x))
x += self.shortcut(x_short)
x = F.celu(x, alpha=0.075)
return x
def forward(self, x):
"""
Applies the forward pass.
"""
x = self.pool(F.celu(self.conv1(x)))
x = self.pool(F.celu(self.conv2(x)))
x = x.view(-1, 16 * 5 * 5)
x = F.celu(self.fc1(x))
x = F.celu(self.fc2(x))
x = F.celu(self.fc3(x))
return x
def forward(self, x):
"""
Applies the forward pass.
"""
x = torch.unsqueeze(x[:, 0, :, :], 1) # unsqueeze for convl
x = self.pool1(F.celu(self.conv1(x)))
x = self.pool2(F.celu(self.conv2(x)))
x = x.view(-1, 16 * 5 * 5)
x = F.celu(self.fc1(x))
x = F.celu(self.fc2(x))
x = F.celu(self.fc3(x))
return x
def decode(self, z):
x = F.celu(self.recons(z)) # [batch, 256]
x = x.view(-1, 256, 1, 1)
x = F.celu(self.t_conv1(x))
x = F.celu(self.t_conv2(x))
x = F.celu(self.t_conv3(x))
x = F.celu(self.t_conv4(x))
x = torch.clamp(torch.sigmoid(self.t_conv5(x)), 0, 1)
return x
def forward(self, x):
# shape(x) = N, L, C
# conv1d needs N,C,L
x = x.transpose(1, 2)
x = self.conv_blocks(x)
x = x.flatten(start_dim=1)
h_phase = F.celu(self.fc_hp(x))
h_dm = F.celu(self.fc_hdm(x))
## add activation layers if desired:
# F.relu(self.out_p(h)) or F.sigmoid(self.out_p(h))
phase = self.out_p(h_phase)
dm = self.out_dm(h_dm)
return phase, dm
def encode(self, x):
assert len(x.shape) == 4
x = F.celu(self.conv1(x))
x = F.celu(self.conv2(x))
x = F.celu(self.conv3(x))
x = F.celu(self.conv4(x))
x = F.celu(self.conv5(x))
x = x.view(-1, 256 * 4)
mu = self.mu(x)
sigma = torch.exp(torch.clamp(self.log_std(x), -3, 3))
return mu, sigma
def forward(self, x):
#x = self.batch_norm1(x)
x = self.dropout1(x)
x = F.celu(self.dense1(x), alpha=0.06)
x = x.reshape(x.shape[0], self.cha_1, self.cha_1_reshape)
#x = self.batch_norm_c1(x)
x = self.dropout_c1(x)
x = F.relu(self.conv1(x))
x = self.ave_po_c1(x)
x = self.batch_norm_c2(x)
x = self.dropout_c2(x)
x = F.relu(self.conv2(x))
x_s = x
x = self.batch_norm_c2_1(x)
x = self.dropout_c2_1(x)
x = F.relu(self.conv2_1(x))
x = self.batch_norm_c2_2(x)
x = self.dropout_c2_2(x)
x = F.relu(self.conv2_2(x))
x = x * x_s
x = self.max_po_c2(x)
x = self.flt(x)
x = self.batch_norm3(x)
x = self.dropout3(x)
x = self.dense3(x)
return x
def forward(self, x, edge_index, edge_attr):
h = x.unsqueeze(0)
for i in range(self.time_step):
m = F.celu(self.conv(x, edge_index, edge_attr))
x, h = self.gru(m.unsqueeze(0), h)
x = self.ln(x.squeeze(0))
return x
def forward(self, data):
x = F.celu(self.lin0(data.x))
for conv in self.convs:
x = x + conv(x, data.edge_index, data.edge_attr)
x = self.set2set(x, data.batch)
x = self.lin1(x)
return x.view(-1)
def celu(input, *args, **kwargs):
output = F.celu(input.F, *args, **kwargs)
return SparseTensor(
output,
coordinate_map_key=input.coordinate_map_key,
coordinate_manager=input.coordinate_manager,
)
def forward(self, x):
x_celu = F.celu(x)
x_gap = F.relu(x) - x_celu
x_out = x_celu + x_gap.data
return x_out
def forward(self, data):
x = F.celu(self.lin0(data.x))
for conv in self.convs:
x = x + F.dropout(conv(x, data.edge_index, data.edge_attr),
p=self.dropout,
training=self.training)
x = self.set2set(x, data.batch)
x = self.out(F.dropout(x, p=self.dropout, training=self.training))
return x
def forward_batch(self, data, edge_index, batch, alpha=None):
x1 = F.celu(self.conv1(data, edge_index))
x1 = self.bn1(x1)
x2 = F.celu(self.conv2(x1, edge_index))
x2 = self.bn2(x2)
x3 = F.celu(self.conv3(x2, edge_index))
x3 = self.bn3(x3)
x_embedding = gmp(x3, batch)
x_embedding_mean = F.celu(self.linear_before(x_embedding))
x_embedding_drop = F.dropout(x_embedding_mean,
p=0.1,
training=self.training)
mean = self.linear_mean(x_embedding_drop)
mean = self.out_layer(mean)
return mean
def forward(self, inputs, T):
# inputs: B x T x nf_in
inputs = inputs.reshape(-1, T, self.nf_in)
inputs = inputs.transpose(1, 2)
# inputs: B x nf_in x T
x = F.celu(self.conv1(inputs))
x = self.bn1(x)
x = F.dropout(x, self.dropout_prob, training=self.training)
x = self.pool(x)
# x = F.celu(self.conv2(x))
# x = self.bn2(x)
# x = F.dropout(x, self.dropout_prob, training=self.training)
# x = self.pool(x)
x = F.celu(self.conv3(x))
x = self.bn3(x)
pred = self.conv_predict(x)
# ret: B x nf_out
ret = pred.max(dim=2)[0]
return ret
def forward(self, x):
assert len(x.shape) == 2
x1 = F.celu(self.q1_fc1(x))
x1_V = F.celu(self.q1_fc2_V(x1))
x1_A = F.celu(self.q1_fc2_A(x1))
x1_V = self.q1_V(x1_V)
x1_A = self.q1_A(x1_A)
x1_A_mean = torch.mean(x1_A, dim=1, keepdim=True)
x1_A_mean = torch.cat([x1_A_mean] * self.action_dim, dim=1)
x1_A = x1_A - x1_A_mean
x1 = x1_V + x1_A
x2 = F.celu(self.q2_fc1(x))
x2_V = F.celu(self.q2_fc2_V(x2))
x2_A = F.celu(self.q2_fc2_A(x2))
x2_V = self.q2_V(x2_V)
x2_A = self.q2_A(x2_A)
x2_A_mean = torch.mean(x2_A, dim=1, keepdim=True)
x2_A_mean = torch.cat([x2_A_mean] * self.action_dim, dim=1)
x2_A = x2_A - x2_A_mean
x2 = x2_V + x2_A
return x1, x2
def forward(self, x):
x = x.view(-1, self.input_size)
for lay_id, lay in enumerate(self.fc):
x = lay(x)
if self.nonlin == 'relu':
x = F.relu(x)
if self.nonlin == 'leakyrelu':
x = F.leaky_relu(x, negative_slope=0.03)
if self.nonlin == 'tanh':
x = F.tanh(x)
if self.nonlin == 'celu':
x = F.celu(x)
return x
def forward(self,
image_tensor: torch.Tensor,
agent_state_vector: torch.Tensor,
noise:torch.Tensor=None) -> torch.Tensor:
"""
Forward pass of the model.
:param image_tensor: Tensor of images shape [batch_size, n_channels, length, width].
:param agent_state_vector: Tensor of floats representing the agent state.
[batch_size, 3].
:returns: Tensor of dimension [batch_size, number_of_modes * number_of_predictions_per_mode + number_of_modes]
storing the predicted trajectory and mode probabilities. Mode probabilities are normalized to sum
to 1 during inference.
"""
backbone_features = self.backbone(input_tensor=image_tensor)
features = torch.cat([backbone_features, agent_state_vector], dim=1)
self.env_feature = features
if noise is not None:
features = torch.cat([features, noise], dim=1)
if self.output_activation == 'sigmoid':
predictions = f.sigmoid(self.fc2(f.celu(self.fc1(features))))
elif self.output_activation == 'linear':
predictions = self.fc2(f.celu(self.fc1(features)))
elif self.output_activation == 'tanh':
predictions = torch.tanh(self.fc2(f.celu(self.fc1(features))))
else:
raise ValueError(f"activation {self.output_activation} is not supported")
# Normalize the probabilities to sum to 1 for inference.
mode_probabilities = predictions[:, -self.num_modes:].clone()
if not self.training:
mode_probabilities = f.softmax(mode_probabilities, dim=-1)
predictions = predictions[:, :-self.num_modes]
return torch.cat((predictions, mode_probabilities), 1)
def forward(self, x, dropout_rate=0.5, dropout_mask=None):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.avgpool(x)
x = torch.flatten(x, 1)
x = F.celu(self.fc(x))
if dropout_mask is not None:
x = x * dropout_mask(x, dropout_rate, 0)
else:
x = F.dropout(x, self.dropout_rate)
x = F.celu(self.fc2(x))
x = F.dropout(x, self.dropout_rate)
x = self.fc3(x)
return x
def forward(self, x):
"""
Applies the forward pass.
"""
x0 = torch.unsqueeze(x[:, 0, :, :], 1) # unsqueeze for convl
x0 = self.pool1(F.celu(self.conv1(x0)))
x0 = self.pool2(F.celu(self.conv2(x0)))
x0 = x0.view(-1, 16 * 5 * 5)
x0 = F.celu(self.fc1(x0))
x0 = F.celu(self.fc2(x0))
x0 = F.celu(self.fc3(x0))
x1 = torch.unsqueeze(x[:, 1, :, :], 1)
x1 = self.pool1(F.celu(self.conv1(x1)))
x1 = self.pool2(F.celu(self.conv2(x1)))
x1 = x1.view(-1, 16 * 5 * 5)
x1 = F.celu(self.fc1(x1))
x1 = F.celu(self.fc2(x1))
x1 = F.celu(self.fc3(x1))
x = torch.cat((x0, x1), 1)
x = self.fc4(x)
return x
def forward(self, species_aev: Tuple[Tensor, Tensor]) -> SpeciesEnergies:
species, aev = species_aev
# Reshape: [num_mols, num_atoms, num_features] --> [num_mols, num_atoms, 1, num_features, 1]
vectors = aev.unsqueeze(-2).unsqueeze(-1)
vectors = batchedLinear(vectors, self.layer0_weights,
self.layer0_biases) # Linear 0
vectors = F.celu(vectors, alpha=0.1) # CELU 1
vectors = batchedLinear(vectors, self.layer2_weights,
self.layer2_biases) # Linear 2
vectors = F.celu(vectors, alpha=0.1) # CELU 3
vectors = batchedLinear(vectors, self.layer4_weights,
self.layer4_biases) # Linear 4
vectors = F.celu(vectors, alpha=0.1) # CELU 5
vectors = batchedLinear(vectors, self.layer6_weights,
self.layer6_biases) # Linear 6
# Sum: [num_mols, num_atoms, num_models, 1, 1] --> [num_mols, num_models]
# Mean: [num_mols, num_models] --> [num_mols]
energies = torch.mean(torch.sum(vectors, (1, 3, 4)), 1)
return SpeciesEnergies(species, energies)
def forward(self, inputs, T):
# inputs: B x T x nf_in
# inputs = inputs.reshape(-1, T, self.nf_in)
# inputs = inputs.transpose(1, 2)
x = self.cnn_enc(inputs)
x # [BS, T//4, 32, 32]
x = x.max(dim=1)[0] # Max pooling [BS, 1, 32, 32]
x = self.cnn_dec(x)
# inputs: B x nf_in x T
x = F.celu(self.conv1(inputs))
x = self.bn1(x)
x = F.dropout(x, self.dropout_prob, training=self.training)
x = self.pool(x)
# x = F.celu(self.conv2(x))
# x = self.bn2(x)
# x = F.dropout(x, self.dropout_prob, training=self.training)
# x = self.pool(x)
x = F.celu(self.conv3(x))
x = self.bn3(x)
pred = self.conv_predict(x)
# ret: B x nf_out
ret = pred.max(dim=2)[0]
return ret
def forward(self, x): # (B, n_in, N)
if self.conv is not None:
x = self.conv(x) # (B, C, N)
x = torch.transpose(x, 1, 2) # (B, N, C)
if self.lstm is not None:
x_a, _ = self.lstm(x)
x_a = self.lstm_ln(x_a)
x_a = self.dropout(F.celu(x_a)) # (B, N, H*2)
x = x + x_a if self.resnet and x.shape[2] == x_a.shape[2] else x_a
if self.att is not None:
x = torch.transpose(x, 0, 1)
x_a, _ = self.att(x, x, x)
x = x + x_a
x = torch.transpose(x, 0, 1)
return x
def forward(self, z_what):
"""
Args:
z_what: (B, D)
Returns:
glimpse: (B, 3, H, W)
"""
B, D = z_what.size()
x = F.celu(self.fc(z_what))
# (B, 128, E, E)
x = x.view(B, 128, self.embed_size, self.embed_size)
x = self.net(x)
x = torch.sigmoid(x)
# (B, 3, H, W), (B, 1, H, W)
o_att, alpha_att = x.split([3, 1], dim=1)
return o_att, alpha_att
def forward_batch(self, data, edge_index, batch, alpha=None):
x1 = F.celu(self.conv1(data, edge_index))
x1 = self.bn1(x1)
x2 = F.celu(self.conv2(x1, edge_index))
x2 = self.bn2(x2)
x3 = F.celu(self.conv3(x2, edge_index))
x3 = self.bn3(x3)
x_embedding = gmp(x3, batch)
x_embedding_mean = F.celu(self.linear_before(x_embedding))
x_embedding_drop = F.dropout(x_embedding_mean, p=0.1, training=self.training)
mean = self.linear_mean(x_embedding_drop)
x_embedding_std = F.celu(self.linear_before_std(x_embedding))
std = F.celu(self.linear_std(x_embedding_std))
std = torch.exp(std / 2)
eps = torch.randn_like(std)
x_sample = gaussian_layer(mean, std, eps)
return x_sample, mean, std
def test_celu(self):
inp = torch.randn(1, 3, 32, 32, device='cuda', dtype=self.dtype)
output = F.celu(inp, alpha=1.0, inplace=False)
def forward(self, x):
assert len(x.shape) == 4
x1 = F.celu(self.q1_conv1(x))
x1 = F.celu(self.q1_conv2(x1))
x1 = F.celu(self.q1_conv3(x1))
#x1 = F.celu(self.q1_conv4(x1))
x1 = x1.view(-1, self.fc_input_size)
x1 = F.celu(self.q1_fc1(x1))
x1_V = F.celu(self.q1_fc2_V(x1))
x1_A = F.celu(self.q1_fc2_A(x1))
x1_V = self.q1_V(x1_V)
x1_A = self.q1_A(x1_A)
x1_A_mean = torch.mean(x1_A, dim=1, keepdim=True)
x1_A_mean = torch.cat([x1_A_mean] * self.action_dim, dim=1)
x1_A = x1_A - x1_A_mean
x1 = x1_V + x1_A
x2 = F.celu(self.q2_conv1(x))
x2 = F.celu(self.q2_conv2(x2))
x2 = F.celu(self.q2_conv3(x2))
#x2 = F.celu(self.q2_conv4(x2))
x2 = x2.view(-1, self.fc_input_size)
x2 = F.celu(self.q2_fc1(x2))
x2_V = F.celu(self.q2_fc2_V(x2))
x2_A = F.celu(self.q2_fc2_A(x2))
x2_V = self.q2_V(x2_V)
x2_A = self.q2_A(x2_A)
x2_A_mean = torch.mean(x2_A, dim=1, keepdim=True)
x2_A_mean = torch.cat([x2_A_mean] * self.action_dim, dim=1)
x2_A = x2_A - x2_A_mean
x2 = x2_V + x2_A
return x1, x2
import torch.nn.functional as F
activations = {
"id": (lambda x: x),
"relu": F.relu_,
"hardtanh": F.hardtanh_,
"relu6": (lambda x: F.relu6_(x, inplace=True)),
"elu": F.elu_,
"selu": (lambda x: F.selu(x, inplace=True)),
"celu": (lambda x: F.celu(x, inplace=True)),
"leaky_relu": F.leaky_relu_,
"rrelu": F.rrelu_,
"gelu": F.gelu,
"logsigmoid": F.logsigmoid,
"hardshrink": F.hardshrink,
"tanhshrink": F.tanhshrink,
"softsign": F.softsign,
"softplus": F.softplus,
"softmin": (lambda x: F.softmin(x, 1)),
"softmax": (lambda x: F.softmax(x, 1)),
"softshrink": F.softshrink,
"gumbel_softmax": F.gumbel_softmax,
"log_softmax": (lambda x: F.log_softmax(x, 1)),
"tanh": F.tanh,
"sigmoid": F.sigmoid,
}
def forward(self, x):
for layer, layer_norm in zip(self.feature_layers, self.layer_norms):
x = layer_norm(F.celu(layer(x)))
return x
