def set_last_mlp(self, last_mlp_opt):
if len(last_mlp_opt.nn) > 2:
self.FC_layer = MLP(last_mlp_opt.nn[:len(last_mlp_opt.nn) - 1])
self.FC_layer.add_module(
"last", Lin(last_mlp_opt.nn[-2], last_mlp_opt.nn[-1]))
elif len(last_mlp_opt.nn) == 2:
self.FC_layer = Seq(Lin(last_mlp_opt.nn[-2], last_mlp_opt.nn[-1]))
else:
self.FC_layer = torch.nn.Identity()
def __init__(self, input_size, n_classes, k_graph=False):
super(GAT, self).__init__()
self.conv1 = GATConv(input_size, 64, heads=4, dropout=0.)
self.conv2 = GATConv(256, 256, heads=3, dropout=0.)
#self.conv3 = GATConv(1024, 256, heads=4, dropout=0., concat=False)
self.pool = global_max_pool
self.mlp = Seq(MLP([1024, 512]), Dropout(0.2), MLP([512, 256]),
Dropout(0.2), Lin(256, n_classes))
#self.mlp = Lin(256,n_classes)
self.k_graph = k_graph
def __init__(self, out_channels, k=5, aggr='max'):
super().__init__()
self.conv1 = DynamicEdgeConv(MLP([2 * 3, 64, 64, 64]), k, aggr)
self.conv2 = DynamicEdgeConv(MLP([2 * 64, 128]), k, aggr)
self.lin1 = MLP([128 + 64, 1024])
self.mlp = Seq(
MLP([1024, 512]), Dropout(0.5), MLP([512, 256]), Dropout(0.5),
Lin(256, out_channels))
def __init__(self, input_size, embedding_size, n_classes, dropout=True, k=5, aggr='max',pool_op='max'):
super(DECSeq, self).__init__()
# self.bn0 = BN(input_size)
# self.bn1 = BN(64)
# self.bn2 = BN(128)
self.conv1 = EdgeConv(MLP([2 * 3, 64, 64, 64], batch_norm=True), aggr)
self.conv2 = DynamicEdgeConv(MLP([2 * 64, 128], batch_norm=True), k, aggr)
self.lin1 = MLP([128 + 64, 1024])
if pool_op == 'max':
self.pool = global_max_pool
if dropout:
self.mlp = Seq(
MLP([1024, 512],batch_norm=True), Dropout(0.5), MLP([512, 256],batch_norm=True), Dropout(0.5),
Lin(256, n_classes))
else:
self.mlp = Seq(
MLP([1024, 512]), MLP([512, 256]),
Lin(256, n_classes))
def __init__(self, option, model_type, dataset, modules):
# Extract parameters from the dataset
# Assemble encoder / decoder
UnwrappedUnetBasedModel.__init__(self, option, model_type, dataset,
modules)
# Build final MLP
last_mlp_opt = option.mlp_cls
self.out_channels = option.out_channels
in_feat = last_mlp_opt.nn[0]
self.FC_layer = Seq()
for i in range(1, len(last_mlp_opt.nn)):
self.FC_layer.add_module(
str(i),
Seq(*[
Lin(in_feat, last_mlp_opt.nn[i], bias=False),
FastBatchNorm1d(last_mlp_opt.nn[i],
momentum=last_mlp_opt.bn_momentum),
LeakyReLU(0.2),
]),
)
in_feat = last_mlp_opt.nn[i]
if last_mlp_opt.dropout:
self.FC_layer.add_module("Dropout",
Dropout(p=last_mlp_opt.dropout))
self.FC_layer.add_module("Last",
Lin(in_feat, self.out_channels, bias=False))
self.mode = option.loss_mode
self.normalize_feature = option.normalize_feature
self.loss_names = ["loss_reg", "loss"]
self.lambda_reg = self.get_from_opt(option,
["loss_weights", "lambda_reg"])
if self.lambda_reg:
self.loss_names += ["loss_regul"]
self.lambda_internal_losses = self.get_from_opt(
option, ["loss_weights", "lambda_internal_losses"])
self.visual_names = ["data_visual"]
def __init__(self, opt):
super(DenseDeepGCN, self).__init__()
channels = opt.n_filters
k = opt.kernel_size
act = opt.act
norm = opt.norm
bias = opt.bias
epsilon = opt.epsilon
stochastic = opt.stochastic
conv = opt.conv
c_growth = channels
self.n_blocks = opt.n_blocks
self.knn = DenseDilatedKnnGraph(k, 1, stochastic, epsilon)
self.head = GraphConv2d(opt.in_channels, channels, conv, act, norm,
bias)
if opt.block.lower() == 'res':
self.backbone = Seq(*[
ResDynBlock2d(channels, k, 1 +
i, conv, act, norm, bias, stochastic, epsilon)
for i in range(self.n_blocks - 1)
])
elif opt.block.lower() == 'dense':
self.backbone = Seq(*[
DenseDynBlock2d(channels + c_growth * i, c_growth, k, 1 +
i, conv, act, norm, bias, stochastic, epsilon)
for i in range(self.n_blocks - 1)
])
else:
raise NotImplementedError(
'{} is not implemented. Please check.\n'.format(opt.block))
self.fusion_block = BasicConv(
[channels + c_growth * (self.n_blocks - 1), 1024], act, norm, bias)
self.prediction = Seq(*[
BasicConv([channels + c_growth *
(self.n_blocks - 1) + 1024, 512], act, norm, bias),
BasicConv([512, 256], act, norm, bias),
torch.nn.Dropout(p=opt.dropout),
BasicConv([256, opt.n_classes], None, None, bias)
])
self.model_init()
def __init__(self, input_size, n_classes=2, embedding_size=128, hidden_size=256):
super(BiLSTM, self).__init__()
self.emb_size = embedding_size
self.h_size = hidden_size
self.mlp = MLP([input_size, embedding_size])
self.lstm = nn.LSTM(embedding_size, hidden_size,
bidirectional=True, batch_first=True)
self.lin = Seq(
MLP([hidden_size, 256]), Dropout(0.5),
MLP([256, 128]), Dropout(0.5),
nn.Linear(128, n_classes))
def __init__(self, in_features, out_features, k):
super(DirectionalEdgeConv, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.k = k
self.mlp = Seq(Linear(3+2*in_features, out_features*2),
ReLU(),
Linear(out_features*2, out_features))
self.econv = EdgeConv(self.mlp, aggr='max')
def __init__(self, n_features, n_outputs, dim=95):
super(GINAttNet, self).__init__()
# Preparation of the Graph Isomorphism Convolutional Layer
nn1 = Seq(Linear(n_features, dim), ReLU(), Linear(dim, dim))
self.conv1 = GINConv(nn1)
self.bn1 = torch.nn.BatchNorm1d(dim)
# Preparation of the Attention Layer
self.conv2 = GATConv(dim, dim, heads=1, dropout=0.3)
# Preparation of the Fully Connected Layer
self.fc1 = Linear(dim, 2 * dim)
self.fc2 = Linear(2 * dim, n_outputs)
def __init__(self, config):
NodeModelBase.__init__(self, config)
self.dim_lh = config['node_model_latent_mlp_hidden_size']
self.dim_l = config['l_outc']
self.node_mlp_2 = Seq(Linear(self.mlp2_inc, self.mlp2_hs1),
LayerNorm(self.mlp2_hs1), ReLU(),
Linear(self.mlp2_hs1, self.dim_lh),
LayerNorm(self.dim_lh), ReLU())
self.mlp_m = Linear(self.dim_lh, self.dim_l)
self.mlp_v = Linear(self.dim_lh, self.dim_l)
self.mlp_x = Linear(self.dim_l, self.dim_out)
def __init__(self, dof=6, act="Sigmoid", dropout=0.5, graph_input=False, transform=None):
super(Net, self).__init__()
self.graph_input = graph_input
self.transform = transform
self.pointnet = PointNetXX(act=act, dropout=dropout)
self.fcs = Seq(
eval(act)(), Lin(64 * 2, 64), Dropout(dropout),
eval(act)(), Lin(64, dof)
)
self.sx = torch.nn.Parameter(torch.tensor(-2.5), requires_grad=True)
self.sq = torch.nn.Parameter(torch.tensor(-2.5), requires_grad=True)
def MLP(channels, enable_group_norm=True):
if enable_group_norm:
num_groups = [0]
for i in range(1, len(channels)):
if channels[i] >= 32:
num_groups.append(channels[i] // 32)
else:
num_groups.append(1)
return Seq(*[
Seq(torch.nn.utils.weight_norm(Lin(channels[i - 1], channels[i])),
LeakyReLU(
negative_slope=0.2), GroupNorm(num_groups[i], channels[i]))
for i in range(1, len(channels))
])
else:
return Seq(*[
Seq(torch.nn.utils.weight_norm(Lin(channels[i - 1]), channels[i]),
LeakyReLU(negative_slope=0.2))
for i in range(1, len(channels))
])
def __init__(self, input_size, embedding_size, n_classes, aggr='max', k=5, pool_op='max', same_size=False):
super(ECnet, self).__init__()
self.conv1 = EdgeConv(MLP([2 * 3, 64, 64, 64]), aggr)
self.conv2 = EdgeConv(MLP([2 * 64, 128]), aggr)
self.lin1 = MLP([128 + 64, 1024])
if pool_op == 'max':
self.pool = global_max_pool
self.mlp = Seq(
MLP([1024, 512]), Dropout(0.5), MLP([512, 256]), Dropout(0.5),
Lin(256, n_classes))
def __init__(self):
super(EdgeBlock, self).__init__()
#A sequential container. Modules will be added to it in the order they are passed in the constructor.
#Alternatively, an ordered dict of modules can also be passed in.
#Applies a linear transformation to the incoming data: y = xA^T + by=xA^T+b
self.edge_mlp = Seq(
Lin(48 * 2, 128), # changed 2 to 6
BatchNorm1d(128),
ReLU(),
Lin(128, 128))
def test_point_conv():
in_channels, out_channels = (16, 32)
edge_index = torch.tensor([[0, 0, 0, 1, 2, 3], [1, 2, 3, 0, 0, 0]])
num_nodes = edge_index.max().item() + 1
x = torch.randn((num_nodes, in_channels))
pos = torch.rand((num_nodes, 3))
norm = torch.nn.functional.normalize(torch.rand((num_nodes, 3)), dim=1)
local_nn = Seq(Lin(in_channels + 4, 32), ReLU(), Lin(32, out_channels))
global_nn = Seq(Lin(out_channels, out_channels))
conv = PPFConv(local_nn, global_nn)
assert conv.__repr__() == (
'PPFConv(local_nn=Sequential(\n'
' (0): Linear(in_features=20, out_features=32, bias=True)\n'
' (1): ReLU()\n'
' (2): Linear(in_features=32, out_features=32, bias=True)\n'
'), global_nn=Sequential(\n'
' (0): Linear(in_features=32, out_features=32, bias=True)\n'
'))')
assert conv(x, pos, norm, edge_index).size() == (num_nodes, out_channels)
def __init__(self):
super(GlobalModel_ONE, self).__init__()
hidden = HIDDEN_GRAPH_ONE
in_channels = ENCODING_NODE_1 + ENCODING_EDGE_1
self.global_mlp = Seq(
Lin(in_channels, hidden),
LeakyReLU(),
LayerNorm(hidden),
#Lin(hidden, hidden), LeakyReLU(),
Lin(hidden, NO_GRAPH_FEATURES_ONE)).apply(init_weights)
def test_reset():
nn = Lin(16, 16)
w = nn.weight.clone()
reset(nn)
assert not nn.weight.tolist() == w.tolist()
nn = Seq(Lin(16, 16), ReLU(), Lin(16, 16))
w_1, w_2 = nn[0].weight.clone(), nn[2].weight.clone()
reset(nn)
assert not nn[0].weight.tolist() == w_1.tolist()
assert not nn[2].weight.tolist() == w_2.tolist()
def __init__(self,
in_channels,
n_growth=32,
kernel_size=3,
bias=True,
act='relu',
norm=True,
res_scale=1.,
n_layers=3):
super(DenseBlock, self).__init__()
self.res_scale = res_scale
convs = Seq(*[
DenseConv(in_channels +
i * n_growth, n_growth, kernel_size, bias, act, norm)
for i in range(n_layers - 1)
])
tail = Conv(in_channels + (n_layers - 1) * n_growth, in_channels,
kernel_size, bias, act, norm)
self.body = Seq(*[convs, tail])
self.n_layers = n_layers
def __init__(self, node_channels, edge_channels, out_channels, aggr='add'):
super(NNConvLayer, self).__init__()
self.node_channels = node_channels
self.edge_channels = edge_channels
self.out_channels = out_channels
self.aggr = aggr
self.mlp_edge = Seq(nn.Linear(self.edge_channels, out_channels), nn.BatchNorm1d(out_channels), nn.ReLU(),
nn.Linear(out_channels, node_channels * out_channels))
self.mlp_output = nn.ReLU()
self.nn_conv = NNConv(self.node_channels, self.out_channels, self.mlp_edge, aggr=self.aggr)
def test_nn_conv():
in_channels, out_channels = (16, 32)
edge_index = torch.tensor([[0, 0, 0, 1, 2, 3], [1, 2, 3, 0, 0, 0]])
num_nodes = edge_index.max().item() + 1
x = torch.randn((num_nodes, in_channels))
pseudo = torch.rand((edge_index.size(1), 3))
nn = Seq(Lin(3, 32), ReLU(), Lin(32, in_channels * out_channels))
conv = NNConv(in_channels, out_channels, nn)
assert conv.__repr__() == 'NNConv(16, 32)'
assert conv(x, edge_index, pseudo).size() == (num_nodes, out_channels)
def test_gine_conv():
x1 = torch.randn(4, 16)
x2 = torch.randn(2, 16)
edge_index = torch.tensor([[0, 1, 2, 3], [0, 0, 1, 1]])
row, col = edge_index
value = torch.randn(row.size(0), 16)
adj = SparseTensor(row=row, col=col, value=value, sparse_sizes=(4, 4))
nn = Seq(Lin(16, 32), ReLU(), Lin(32, 32))
conv = GINEConv(nn, train_eps=True)
assert conv.__repr__() == (
'GINEConv(nn=Sequential(\n'
' (0): Linear(in_features=16, out_features=32, bias=True)\n'
' (1): ReLU()\n'
' (2): Linear(in_features=32, out_features=32, bias=True)\n'
'))')
out = conv(x1, edge_index, value)
assert out.size() == (4, 32)
assert conv(x1, edge_index, value, size=(4, 4)).tolist() == out.tolist()
assert conv(x1, adj.t()).tolist() == out.tolist()
if is_full_test():
t = '(Tensor, Tensor, OptTensor, Size) -> Tensor'
jit = torch.jit.script(conv.jittable(t))
assert jit(x1, edge_index, value).tolist() == out.tolist()
assert jit(x1, edge_index, value, size=(4, 4)).tolist() == out.tolist()
t = '(Tensor, SparseTensor, OptTensor, Size) -> Tensor'
jit = torch.jit.script(conv.jittable(t))
assert jit(x1, adj.t()).tolist() == out.tolist()
adj = adj.sparse_resize((4, 2))
out1 = conv((x1, x2), edge_index, value)
out2 = conv((x1, None), edge_index, value, (4, 2))
assert out1.size() == (2, 32)
assert out2.size() == (2, 32)
assert conv((x1, x2), edge_index, value, (4, 2)).tolist() == out1.tolist()
assert conv((x1, x2), adj.t()).tolist() == out1.tolist()
assert conv((x1, None), adj.t()).tolist() == out2.tolist()
if is_full_test():
t = '(OptPairTensor, Tensor, OptTensor, Size) -> Tensor'
jit = torch.jit.script(conv.jittable(t))
assert jit((x1, x2), edge_index, value).tolist() == out1.tolist()
assert jit((x1, x2), edge_index, value,
size=(4, 2)).tolist() == out1.tolist()
assert jit((x1, None), edge_index, value,
size=(4, 2)).tolist() == out2.tolist()
t = '(OptPairTensor, SparseTensor, OptTensor, Size) -> Tensor'
jit = torch.jit.script(conv.jittable(t))
assert jit((x1, x2), adj.t()).tolist() == out1.tolist()
assert jit((x1, None), adj.t()).tolist() == out2.tolist()
def __init__(self, config):
EdgeModelBase.__init__(self, config)
self.dim_x1 = self.dim_in
self.dim_lh = config['edge_model_latent_mlp_hidden_size']
self.dim_l = config['l_outc']
self.mlp_h = Seq(Linear(self.dim_x1, self.dim_h1),
LayerNorm(self.dim_h1), ReLU(),
Linear(self.dim_h1, self.dim_lh),
LayerNorm(self.dim_lh), ReLU())
self.mlp_m = Linear(self.dim_lh, self.dim_l)
self.mlp_v = Linear(self.dim_lh, self.dim_l)
self.mlp_w = Linear(self.dim_l, self.dim_out)
def MLP(channels, batch_norm=True):
"""
return a MLP of shape 'channels'
"""
mlps = []
for i in range(1, len(channels)):
mlp = nn.Conv2d(in_channels=channels[i - 1],
out_channels=channels[i],
kernel_size=(1, 1),
stride=(1, 1),
bias=True)
nn.init.kaiming_normal_(mlp.weight)
mlps.append(mlp)
if batch_norm:
mlp = Seq(*[
Seq(mlps[i - 1], BN(channels[i]), ReLU())
for i in range(1, len(channels))
])
else:
mlp = Seq(*[Seq(mlps[i - 1], ReLU()) for i in range(1, len(channels))])
return mlp
def __init__(self, config):
super(EdgeModel, self).__init__()
in_channels = config['e_inc'] + 2*config['n_inc'] + config['u_inc']
hs1 = config['edge_model_mlp1_hidden_sizes'][0]
hs2 = config['edge_model_mlp1_hidden_sizes'][1]
self.edge_mlp = Seq(Linear(in_channels, hs1),
ReLU(),
Linear(hs1, hs2),
ReLU(),
Linear(hs2, config['e_outc']))
def make_mlp(in_channels, mlp_channels, batch_norm=True):
assert len(mlp_channels) >= 1
layers = []
for c in mlp_channels:
layers += [Lin(in_channels, c)]
if batch_norm: layers += [BatchNorm1d(c)]
layers += [ReLU()]
in_channels = c
return Seq(*layers)
def __init__(self, in_channels, out_channels, dim_model, k=16):
super().__init__()
self.k = k
# dummy feature is created if there is none given
in_channels = max(in_channels, 1)
# first block
self.mlp_input = MLP([in_channels, dim_model[0]])
self.transformer_input = TransformerBlock(
in_channels=dim_model[0],
out_channels=dim_model[0],
)
# backbone layers
self.transformers_up = torch.nn.ModuleList()
self.transformers_down = torch.nn.ModuleList()
self.transition_up = torch.nn.ModuleList()
self.transition_down = torch.nn.ModuleList()
for i in range(0, len(dim_model) - 1):
# Add Transition Down block followed by a Point Transformer block
self.transition_down.append(
TransitionDown(in_channels=dim_model[i],
out_channels=dim_model[i + 1],
k=self.k))
self.transformers_down.append(
TransformerBlock(in_channels=dim_model[i + 1],
out_channels=dim_model[i + 1]))
# Add Transition Up block followed by Point Transformer block
self.transition_up.append(
TransitionUp(in_channels=dim_model[i + 1],
out_channels=dim_model[i]))
self.transformers_up.append(
TransformerBlock(in_channels=dim_model[i],
out_channels=dim_model[i]))
# summit layers
self.mlp_summit = MLP([dim_model[-1], dim_model[-1]], batch_norm=False)
self.transformer_summit = TransformerBlock(
in_channels=dim_model[-1],
out_channels=dim_model[-1],
)
# class score computation
self.mlp_output = Seq(Lin(dim_model[0], 64), ReLU(), Lin(64, 64),
ReLU(), Lin(64, out_channels))
def test_dynamic_edge_conv_conv():
x = torch.randn((4, 16))
nn = Seq(Lin(32, 16), ReLU(), Lin(16, 32))
conv = DynamicEdgeConv(nn, k=6)
assert conv.__repr__() == (
'DynamicEdgeConv(nn=Sequential(\n'
' (0): Linear(in_features=32, out_features=16, bias=True)\n'
' (1): ReLU()\n'
' (2): Linear(in_features=16, out_features=32, bias=True)\n'
'), k=6)')
assert conv(x).size() == (4, 32)
def __init__(self):
super(Net, self).__init__()
self.edge_mlp = Seq(Linear(128, 256), ReLU(), Linear(256, 128))
self.node_mlp = Seq(Linear(128, 256), ReLU(), Linear(256, 128))
self.global_mlp = Seq(Linear(128, 256).ReLU(), Linear(256, 128))
def edge_model(source, target, edge_attr, u):
out = torch.cat([source, target, edge_attr], dim=1)
return self.edge_mlp(out)
def node_model(x, edge_index, edge_attr, u):
row, col = edge_index
out = torch.cat([x[col], edge_attr], dim=1)
out = self.node_mlp(out)
return scatter_mean(out, row, dim=0, dim_size=x.size(0))
def global_model(x, edge_index, edge_attr, u, batch):
out = torch.cat([u, scatter_mean(x, batch, dim=0)], dim=1)
return self.global_mlp(out)
self.op = MetaLayer(edge_model, node_model, global_model)
def __init__(self, k=30, emb_dims=1024):
super(GlobalFeat, self).__init__()
self.k = k
self.conv1 = Seq(nn.Conv2d(6, 64, kernel_size=1, bias=False),
nn.BatchNorm2d(64),
nn.LeakyReLU(negative_slope=0.2))
self.conv2 = Seq(nn.Conv2d(128, 64, kernel_size=1, bias=False),
nn.BatchNorm2d(64),
nn.LeakyReLU(negative_slope=0.2))
self.conv3 = Seq(nn.Conv2d(128, 128, kernel_size=1, bias=False),
nn.BatchNorm2d(128),
nn.LeakyReLU(negative_slope=0.2))
self.conv4 = Seq(nn.Conv2d(256, 256, kernel_size=1, bias=False),
nn.BatchNorm2d(256),
nn.LeakyReLU(negative_slope=0.2))
self.conv5 = Seq(nn.Conv1d(512, emb_dims, kernel_size=1, bias=False),
nn.BatchNorm1d(emb_dims),
nn.LeakyReLU(negative_slope=0.2))
def __init__(self,
n_node_features,
n_edge_features,
n_global_features,
n_hiddens,
n_targets,
use_batch_norm=False):
super().__init__()
if use_batch_norm:
self.edge_mlp = Seq(
Lin(2 * n_node_features + n_edge_features + n_global_features,
n_hiddens), LeakyReLU(), Lin(n_hiddens, n_hiddens),
LeakyReLU(), Lin(n_hiddens, n_targets), BatchNorm1d(n_targets))
else:
self.edge_mlp = Seq(
Lin(2 * n_node_features + n_edge_features + n_global_features,
n_hiddens),
LeakyReLU(),
Lin(n_hiddens, n_hiddens),
LeakyReLU(),
Lin(n_hiddens, n_targets),
)
