def __init__(self, num_layers, num_mlp_layers, feature_dim, hidden_dim, num_classes,
final_dropout, graph_pooling_type, device, embedded_dim):
super(gcnn_new, self).__init__()
self.final_dropout = final_dropout
self.device = device
self.num_layers = num_layers
self.hidden_dim = hidden_dim
self.graph_pooling_type = graph_pooling_type
self.feature_dim = feature_dim
self.embedded_dim = embedded_dim
self.mlps = torch.nn.ModuleList()
###List of batchnorms applied to the output of MLP (input of the final prediction linear layer)
self.batch_norms = torch.nn.ModuleList()
for layer in range(self.num_layers - 1):
if layer == 0:
self.mlps.append(MLP(num_mlp_layers, feature_dim, hidden_dim, hidden_dim))
else:
self.mlps.append(MLP(num_mlp_layers, hidden_dim, hidden_dim, hidden_dim))
self.linears_prediction = torch.nn.ModuleList()
for layer in range(num_layers):
if layer == 0:
self.linears_prediction.append(nn.Linear(feature_dim, num_classes))
else:
self.linears_prediction.append(nn.Linear(hidden_dim, num_classes))
self.conv_kernel = torch.nn.ModuleList()
for layer in range(self.num_layers - 1):
if layer == 0:
self.conv_kernel.append(MLP(num_mlp_layers, embedded_dim, hidden_dim, feature_dim))
self.batch_norms.append(nn.BatchNorm1d(feature_dim))
else:
self.conv_kernel.append(MLP(num_mlp_layers, embedded_dim, hidden_dim, hidden_dim))
self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
def __init__(self, layers, in_features, hidden_features, out_features, prop_depth, dropout=0.0, model_name='DE-GNN'):
super(GNNModel, self).__init__()
self.layers, self.in_features, self.hidden_features, self.out_features, self.model_name = layers, in_features, hidden_features, out_features, model_name
Layer = self.get_layer_class()
self.act = nn.ReLU()
self.dropout = nn.Dropout(p=dropout)
self.layers = nn.ModuleList()
if self.model_name == 'DE-GNN':
self.layers.append(Layer(in_channels=in_features, out_channels=hidden_features, K=prop_depth))
elif self.model_name == 'GIN':
self.layers.append(
Layer(MLP(num_layers=2, input_dim=in_features, hidden_dim=hidden_features, output_dim=hidden_features)))
else:
self.layers.append(Layer(in_channels=in_features, out_channels=hidden_features))
if layers > 1:
for i in range(layers - 1):
if self.model_name == 'DE-GNN':
self.layers.append(Layer(in_channels=hidden_features, out_channels=hidden_features, K=prop_depth))
elif self.model_name == 'GIN':
self.layers.append(Layer(MLP(num_layers=2, input_dim=hidden_features, hidden_dim=hidden_features,
output_dim=hidden_features)))
elif self.model_name == 'GAT':
self.layers.append(Layer(in_channels=hidden_features, out_channels=hidden_features, heads=8))
else:
# for GCN and GraphSAGE
self.layers.append(Layer(in_channels=hidden_features, out_channels=hidden_features))
self.layer_norms = nn.ModuleList([nn.LayerNorm(hidden_features) for i in range(layers)])
self.merger = nn.Linear(3 * hidden_features, hidden_features)
self.feed_forward = FeedForwardNetwork(hidden_features, out_features)
def _create_model(model_name):
if model_name == "vgg":
model = cifar10vgg(n_reps=3,
train=False,
weight_file="data/cifar10vgg.h5")
if model_name == "vgg19_89acc":
model = general_vgg(
n_reps=4,
train=False,
weight_file="cifar10_vgg19weights_init130-50-40_5e-3lr_30.h5")
if model_name == "vgg19_highestacc":
model = general_vgg(
n_reps=4,
train=False,
weight_file=
"cifar10_vgg19weights/cifar10_vgg19weights_init130-50-40-5e-3lr30-30_250.h5"
)
elif model_name == "mlp5":
model = MLP(train=False,
num_layers=5,
hidden_dim=1000,
weight_file="models/weights/mlp_l5_h1000.h5")
elif model_name == "mlp8":
model = MLP(train=False,
num_layers=8,
hidden_dim=1000,
weight_file="models/weights/mlp_l8_h1000.h5")
elif model_name == "mlp12":
model = MLP(train=False,
num_layers=12,
hidden_dim=1000,
weight_file="data/mlp.h5")
return model
def __init__(self, state_dim: int, action_space, hidden_dims: List[int],
state_normalizer: Optional[nn.Module], use_limited_entropy=False, use_tanh_squash=False,
use_state_dependent_std=False, **kwargs):
super(Actor, self).__init__()
self.state_dim = state_dim
self.action_space = action_space
self.hidden_dims = hidden_dims
self.use_limited_entropy = use_limited_entropy
self.use_tanh_squash = use_tanh_squash
if isinstance(action_space, Box) or isinstance(action_space, MultiBinary):
self.action_dim = action_space.shape[0]
else:
assert isinstance(action_space, Discrete)
self.action_dim = action_space.n
mlp_kwargs = kwargs.copy()
mlp_kwargs['activation'] = kwargs.get('activation', 'relu')
mlp_kwargs['last_activation'] = kwargs.get('activation', 'relu')
self.actor_feature = MLP(state_dim, hidden_dims[-1], hidden_dims[:-1], **mlp_kwargs)
self.state_normalizer = state_normalizer or nn.Identity()
self.actor_layer = TanhGaussainActorLayer(hidden_dims[-1], self.action_dim,
use_state_dependent_std)
def init_(m): init(m, fanin_init, lambda x: nn.init.constant_(x, 0))
self.actor_feature.init(init_, init_)
def __init__(self, num_layers, in_features, hidden_features, out_features, prop_depth, dropout=0.0, model_name='DE-GNN',type_norm='group'):
super(GNNModel, self).__init__()
self.num_layers, self.in_features, self.hidden_features, self.out_features, self.model_name = num_layers, in_features, hidden_features, out_features, model_name
Layer = self.get_layer_class()
self.act = nn.ReLU()
self.dropout = nn.Dropout(p=dropout)
self.layers = nn.ModuleList()
self.type_norm = type_norm
if self.model_name == 'DE-GNN':
self.layers.append(Layer(in_channels=in_features, out_channels=hidden_features, K=prop_depth))
elif self.model_name == 'GIN':
self.layers.append(
Layer(MLP(num_layers=2, input_dim=in_features, hidden_dim=hidden_features, output_dim=hidden_features)))
else:
self.layers.append(Layer(in_channels=in_features, out_channels=hidden_features))
if self.num_layers > 1:
for i in range(self.num_layers - 1):
if self.model_name == 'DE-GNN':
self.layers.append(Layer(in_channels=hidden_features, out_channels=hidden_features, K=prop_depth))
elif self.model_name == 'GIN':
self.layers.append(Layer(MLP(num_layers=2, input_dim=hidden_features, hidden_dim=hidden_features,
output_dim=hidden_features)))
elif self.model_name == 'GAT':
self.layers.append(Layer(in_channels=hidden_features, out_channels=hidden_features, heads=8))
else:
# for GCN and GraphSAGE
self.layers.append(Layer(in_channels=hidden_features, out_channels=hidden_features))
# we're building up the normalization layers here.
if self.type_norm == 'group':
self.layer_norms = nn.ModuleList([batch_norm(hidden_features,'group',skip_connect = True) for i in range(self.num_layers)])
else:
self.layer_norms = nn.ModuleList([nn.LayerNorm(hidden_features) for i in range(self.num_layers)])
self.merger = nn.Linear(3 * hidden_features, hidden_features)
self.feed_forward = FeedForwardNetwork(hidden_features, out_features)
def __init__(self,
edge_in_dim=None,
node_in_dim=None,
edge_out_dim=None,
node_out_dim=None,
node_fc_dims=None,
edge_fc_dims=None,
dropout_p=None,
use_batchnorm=None):
super(MLPGraphIndependent, self).__init__()
if node_in_dim is not None:
self.node_mlp = MLP(input_dim=node_in_dim,
fc_dims=list(node_fc_dims) + [node_out_dim],
dropout_p=dropout_p,
use_batchnorm=use_batchnorm)
else:
self.node_mlp = None
if edge_in_dim is not None:
self.edge_mlp = MLP(input_dim=edge_in_dim,
fc_dims=list(edge_fc_dims) + [edge_out_dim],
dropout_p=dropout_p,
use_batchnorm=use_batchnorm)
else:
self.edge_mlp = None
def _build_model(self):
self.gconv = nn.ModuleList()
self.classifier = MLP(self.num_mlp_layers, self.hidden_dim, int(self.hidden_dim / 2), self.output_dim)
self.node_mlp = MLP(2, self.hidden_dim, self.hidden_dim, self.hidden_dim)
self.gconv.append(AGraphATLayer(self.input_dim, self.hidden_dim, self.embeddim))
for layer in range(self.num_layers - 1):
self.gconv.append(AGraphATLayer(self.hidden_dim, self.hidden_dim, self.embeddim,self.shortcut))
return
def __init__(self,
out_dim,
v_hdim,
cnn_fdim,
no_cnn=False,
frame_shape=(3, 64, 64),
mlp_dim=(300, 200),
cnn_type='resnet',
v_net_type='lstm',
v_net_param=None,
cnn_rs=True,
causal=False,
device=None):
super().__init__()
self.out_dim = out_dim
self.cnn_fdim = cnn_fdim
self.v_hdim = v_hdim
self.no_cnn = no_cnn
self.cnn_type = cnn_type
self.frame_shape = frame_shape
self.device = device
if no_cnn:
self.cnn = None
else:
self.frame_shape = (1, 32, 32, 64)
""" only for ResNet based models """
if v_net_param is None:
v_net_param = {}
spec = v_net_param.get('spec', 'resnet18')
self.cnn = P2PsfNet(cnn_fdim,
device=self.device,
running_stats=cnn_rs,
spec=spec)
self.v_net_type = v_net_type
if v_net_type == 'lstm':
self.v_net = RNN(cnn_fdim, v_hdim, v_net_type, bi_dir=not causal)
elif v_net_type == 'tcn':
if v_net_param is None:
v_net_param = {}
tcn_size = v_net_param.get('size', [64, 128])
dropout = v_net_param.get('dropout', 0.2)
kernel_size = v_net_param.get('kernel_size', 3)
assert tcn_size[-1] == v_hdim
self.v_net = TemporalConvNet(cnn_fdim,
tcn_size,
kernel_size=kernel_size,
dropout=dropout,
causal=causal)
if self.v_net_type is 'no_lstm':
self.mlp = MLP(self.cnn_fdim, mlp_dim, 'relu')
else:
self.mlp = MLP(v_hdim, mlp_dim, 'relu')
self.linear = nn.Linear(self.mlp.out_dim, out_dim)
def __init__(self, num_layers, num_mlp_layers, input_dim, hidden_dim,
output_dim, final_dropout, learn_eps, graph_pooling_type,
neighbor_pooling_type, device):
'''
num_layers: number of layers in the neural networks (INCLUDING the input layer)
num_mlp_layers: number of layers in mlps (EXCLUDING the input layer)
input_dim: dimensionality of input features
hidden_dim: dimensionality of hidden units at ALL layers
output_dim: number of classes for prediction
final_dropout: dropout ratio on the final linear layer
learn_eps: If True, learn epsilon to distinguish center nodes from neighboring nodes. If False, aggregate neighbors and center nodes altogether.
neighbor_pooling_type: how to aggregate neighbors (mean, average, or max)
graph_pooling_type: how to aggregate entire nodes in a graph (mean, average)
device: which device to use
'''
super(GraphCNN, self).__init__()
self.final_dropout = final_dropout
self.device = device
self.num_layers = num_layers
self.graph_pooling_type = graph_pooling_type
self.neighbor_pooling_type = neighbor_pooling_type
self.learn_eps = learn_eps
self.eps = nn.Parameter(torch.zeros(self.num_layers - 1))
###List of MLPs
self.mlps = torch.nn.ModuleList()
###List of batchnorms applied to the output of MLP (input of the final prediction linear layer)
self.batch_norms = torch.nn.ModuleList()
for layer in range(self.num_layers - 1):
if layer == 0:
self.mlps.append(
MLP(num_mlp_layers, input_dim, hidden_dim, hidden_dim))
else:
self.mlps.append(
MLP(num_mlp_layers, hidden_dim, hidden_dim, hidden_dim))
self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
#Linear function that maps the hidden representation at dofferemt layers into a prediction score
self.linears_prediction = torch.nn.ModuleList()
for layer in range(num_layers):
if layer == 0:
self.linears_prediction.append(nn.Linear(
input_dim, output_dim))
else:
self.linears_prediction.append(
nn.Linear(hidden_dim, output_dim))
def __init__(self, device, type_vocab, type_id_dict, word_vec_dim,
lstm_dim, mlp_hidden_dim, type_embed_dim):
super(SRLFET, self).__init__()
# TODO to other class
self.type_vocab, self.type_id_dict = type_vocab, type_id_dict
self.l1_type_indices, self.l1_type_vec, self.child_type_vecs = fetutils.build_hierarchy_vecs(
self.type_vocab, self.type_id_dict)
self.n_types = len(self.type_vocab)
self.device = device
self.word_vec_dim = word_vec_dim
self.lstm_dim = lstm_dim
self.lstm1 = nn.LSTM(input_size=self.word_vec_dim,
hidden_size=self.lstm_dim,
bidirectional=False)
self.lstm_hidden1 = None
self.lstm2 = nn.LSTM(input_size=self.word_vec_dim,
hidden_size=self.lstm_dim,
bidirectional=False)
self.lstm_hidden2 = None
self.type_embed_dim = type_embed_dim
self.type_embeddings = torch.tensor(np.random.normal(
scale=0.01,
size=(type_embed_dim, self.n_types)).astype(np.float32),
device=self.device,
requires_grad=True)
self.type_embeddings = nn.Parameter(self.type_embeddings)
self.mlp = MLP(2, 2 * self.word_vec_dim + 2 * self.lstm_dim,
self.type_embed_dim, mlp_hidden_dim)
def __init__(self, model, output_size, classif, end_classif=True):
super().__init__()
self.net = model
self.c_1 = MLP(**classif)
self.end_classif = end_classif
if self.end_classif:
self.c_2 = nn.Linear(output_size, output_size)
def DefaultCifar10CNN(device):
cnn = CNN(in_features=(32, 32, 3),
out_features=10,
conv_filters=[32, 16],
conv_kernel_size=[5, 5],
conv_strides=[1, 1],
conv_pad=[0, 0],
max_pool_kernels=[(2, 2), (2, 2)],
max_pool_strides=[2, 2],
use_dropout=False,
use_batch_norm=False,
actv_func=["leakyrelu", "leakyrelu"],
device=device)
# Create MLP
# Calculate the input shape
s = cnn.GetCurShape()
in_features = s[0] * s[1] * s[2]
mlp = MLP(in_features,
10, [120, 84], ["leakyrelu", "leakyrelu"],
use_batch_norm=False,
use_dropout=False,
use_softmax=False,
device=device)
# mlp = DefaultCifar10MLP(device=device, in_features=in_features)
cnn.AddMLP(mlp)
return cnn
def __init__(self, in_features, mlp_features):
"""
SetPartitionTri model.
"""
super().__init__()
cfg = dict(mlp_with_relu=False)
self.mlp = MLP(in_features=in_features, feats=mlp_features, cfg=cfg)
self.tensor_1 = torch.tensor(1., device='cuda')
def __init__(self, base_vqa):
super().__init__()
self.model = base_vqa
dimensions = [2048, 2048, 3000]
self.mlp = MLP(4800, dimensions)
self.slp = nn.Linear(3000, 3000)
def main(cfg):
device = set_env(cfg)
logging.info('Loading the dataset.')
train_criterion, val_criterion = get_criterion(cfg.optimization.criterion)
train_dataloader, val_dataloader = get_dataloader(cfg)
model = MLP(**cfg.network).to(device)
logging.info(f'Constructing model on the {device}:{cfg.CUDA_DEVICE}.')
logging.info(model)
# Set total steps for onecycleLR and cosineLR
cfg.optimization.total_steps = len(train_dataloader) * cfg.optimization.epoch
cfg.optimization.onecycle_scheduler.total_steps = \
cfg.optimization.cosine_scheduler.T_max = cfg.optimization.total_steps
optimizer, scheduler = get_optimization(cfg, model)
best_loss = float("inf")
for epoch in range(cfg.optimization.epoch):
train_epoch(model,
train_dataloader,
train_criterion,
optimizer,
scheduler,
device,
epoch,
cfg)
if cfg.optimization.scheduler in ['exp', 'step']:
scheduler.step()
val_loss = valid_epoch(model,
val_dataloader,
val_criterion,
device,
epoch,
cfg)
if val_loss < best_loss:
best_loss = val_loss
torch.save(model.state_dict(), '{}/best_model.pth'.format(cfg.log_dir))
logging.info(f'New Main Loss {best_loss}')
torch.save(model.state_dict(), '{}/last_model.pth'.format(cfg.log_dir))
logging.info('Best Main Loss {}'.format(best_loss))
logging.info(cfg)
pickle.dump(cfg, open('{}/config.pkl'.format(cfg.log_dir), 'wb'))
def _build_model(self):
self.gconv = nn.ModuleList()
self.classifier = MLP(self.num_mlp_layers, self.hidden_dim, int(self.hidden_dim/2), self.output_dim)
self.embedding_layer = nn.Linear(self.input_dim, self.hidden_dim, bias=True) #transform input 0 to feature
for layer in range(self.num_layers):
self.gconv.append(ASPGATLayer(self.hidden_dim, self.hidden_dim, self.embeddim, self.k_order))
return