def __init__(self):
super(Net, self).__init__()
self.block = nn.ModuleList([])
self.backward_interfaces = nn.ModuleList([])
self.block.append(
nn.Sequential(nn.Conv2d(3, 128, kernel_size=5, padding=2),
nn.BatchNorm2d(128), nn.ReLU(), nn.MaxPool2d(
(3, 3))))
self.block.append(
nn.Sequential(nn.Conv2d(128, 128, kernel_size=5, padding=2),
nn.BatchNorm2d(128), nn.ReLU(), nn.AvgPool2d(
(3, 3))))
self.block.append(
nn.Sequential(nn.Conv2d(128, 128, kernel_size=5, padding=2),
nn.BatchNorm2d(128), nn.ReLU(), nn.AvgPool2d(
(3, 3))))
if args.dni:
self.backward_interfaces[0] = dni.BackwardInterface(
ConvSynthesizer())
self.backward_interfaces[1] = dni.BackwardInterface(
ConvSynthesizer())
self.backward_interfaces[2] = dni.BackwardInterface(
ConvSynthesizer())
self.fc1 = nn.Linear(128, 10)
def __init__(self):
super(VGG16_SG2, self).__init__()
# modify the original VGG model to have a 10 unit output linear layer
self.model = models.vgg16(True)
self.model.classifier[0].in_features = 512
self.conv1 = nn.Conv2d(1, 3, 1)
self.block1 = self.model.features[:5]
self.block2 = self.model.features[5:10]
self.block3 = self.model.features[10:17]
self.block4 = self.model.features[17:24]
self.block5 = self.model.features[24:]
self.classifier = nn.Sequential(
nn.Linear(512, 128),
nn.ReLU(True),
nn.Linear(128, 64),
nn.ReLU(True),
nn.Linear(64, 10),
)
self.backward_interface_1 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=24, n_hidden=20))
self.backward_interface_2 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=12, n_hidden=20))
self.backward_interface_3 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=6, n_hidden=20))
self.backward_interface_4 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=3, n_hidden=20))
self.backward_interface_5 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=1, n_hidden=20))
def __init__(self, *args, **kwargs):
super(less, self).__init__(Bottleneck, [3, 4, 6, 3],
num_classes=num_classes,
*args,
**kwargs)
self.seq1 = nn.Sequential(self.conv1, self.bn1, self.relu,
self.maxpool, self.layer1, self.layer2).to(
'cuda:0') # changed from cpu to gpu
# self.input_context = nn.Linear(10, 16)
# self.input_trigger = nn.Conv2d(512, 512, kernel_size=5, padding=2)
# self.hidden = nn.Conv2d(512, 512, kernel_size=5, padding=2)
# self.output = nn.Conv2d(512, 512, kernel_size=5, padding=2)
context_dim = None
self.backward_interface = dni.BackwardInterface(Conv_less())
#self.backward_interface = dni.BackwardInterface( dni.BasicSynthesizer(mcf()))
self.seq2 = nn.Sequential(
self.layer3,
self.layer4,
self.avgpool,
).to('cuda:0')
self.fc.to('cuda:0')
def __init__(self):
super(Eight_Layer_SG, self).__init__()
self.block_1 = nn.Sequential(
nn.Conv2d(
in_channels=1,
out_channels=32,
kernel_size=(12, 12),
stride=(4, 4),
padding=(4, 4),
),
nn.BatchNorm2d(32),
nn.ReLU(),
)
self.block_2 = nn.Sequential(
nn.Conv2d(
in_channels=32,
out_channels=64,
kernel_size=(12, 12),
stride=(4, 4),
padding=(4, 4),
),
nn.BatchNorm2d(64),
nn.ReLU(),
)
self.classifier = nn.Sequential(
nn.Linear(576, 128),
nn.ReLU(True),
nn.Dropout(p=0.5),
nn.Linear(128, 64),
nn.ReLU(True),
nn.Dropout(p=0.5),
nn.Linear(64, 10),
)
self.backward_interface_1 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=12, n_hidden=256))
self.backward_interface_2 = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=3, n_hidden=128))
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv1_bn = nn.BatchNorm2d(10)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_bn = nn.BatchNorm2d(20)
self.conv2_drop = nn.Dropout2d()
if args.dni:
self.backward_interface = dni.BackwardInterface(ConvSynthesizer())
self.fc1 = nn.Linear(320, 50)
self.fc1_bn = nn.BatchNorm1d(50)
self.fc2 = nn.Linear(50, 10)
self.fc2_bn = nn.BatchNorm1d(10)
def _init_dni(self, synthesizer_type, freeze_synthesizer, non_zero_init,
scale_synth_grad, trained_net_file,
trained_net_initial_file):
if self.context:
context_dim = 10
else:
context_dim = None
if synthesizer_type == 'mlp':
synthesizer = dni.BasicSynthesizer(
output_dim=self.num_neurons,
context_dim=context_dim,
n_hidden=1,
non_zero_init=non_zero_init,
normalize_output=scale_synth_grad)
if synthesizer_type == 'local_mlp':
synthesizer = LocalSynthesizer(output_dim=self.num_neurons,
context_dim=context_dim,
non_zero_init=non_zero_init)
if synthesizer_type == 'conv':
pass #TODO
self.backward_interface = dni.BackwardInterface(synthesizer)
# Loading the specified initialization
if trained_net_initial_file is not None:
trained_net = torch.load(trained_net_initial_file)
dict_trained_params = dict(trained_net.named_parameters())
for name, param in self.named_parameters():
if name in dict_trained_params:
param.requires_grad = False
print('Copying pretrained ' + name)
param.copy_(dict_trained_params[name].data)
param.requires_grad = True
# Loading the trained synthesizer
if trained_net_file is not None:
trained_net = torch.load(trained_net_file)
dict_trained_params = dict(
trained_net.backward_interface.named_parameters())
for name, param in self.backward_interface.named_parameters():
if name in dict_trained_params:
param.requires_grad = False
print('Copying pretrained ' + name)
param.copy_(dict_trained_params[name].data)
param.requires_grad = True
if freeze_synthesizer:
for name, param in self.backward_interface.named_parameters():
param.requires_grad = False
def __init__(self):
super(Net, self).__init__()
self.hidden1 = nn.Linear(784, 256, bias=False)
self.hidden1_bn = nn.BatchNorm1d(256)
self.hidden2 = nn.Linear(256, 256, bias=False)
self.hidden2_bn = nn.BatchNorm1d(256)
if args.dni:
if args.context:
context_dim = 10
else:
context_dim = None
self.backward_interface = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim=256,
n_hidden=1,
context_dim=context_dim))
self.output = nn.Linear(256, 10, bias=False)
self.output_bn = nn.BatchNorm1d(10)
def __init__(self, *nkwargs, **kwargs):
super(ModelParallelResNet50, self).__init__(
Bottleneck, [3, 4, 6, 3], num_classes=10, *nkwargs, **kwargs)
self.seq1 = nn.Sequential(
self.conv1,
self.bn1,
self.relu,
self.maxpool,
self.layer1
) #.to('cpu')
if args.dni:
self.backward_interface = dni.BackwardInterface(ResNetConvSynthesizer())
self.seq2 = nn.Sequential(
self.layer2,
self.layer3,
self.layer4,
self.avgpool,
) # .to('cuda:0')
def __init__(self):
super(NetDNIPart, self).__init__()
if args.dni:
self.backward_interface = dni.BackwardInterface(ConvSynthesizer())
def __init__(self,
rnn_type,
ntoken,
ninp,
nhid,
nlayers,
dropout=0.5,
tie_weights=False,
use_dni=False):
super(RNNModel, self).__init__()
self.drop = nn.Dropout(dropout)
self.encoder = nn.Embedding(ntoken, ninp)
if rnn_type in ['LSTM', 'GRU']:
self.rnn = getattr(nn, rnn_type)(ninp,
nhid,
nlayers,
dropout=dropout)
else:
try:
nonlinearity = {
'RNN_TANH': 'tanh',
'RNN_RELU': 'relu'
}[rnn_type]
except KeyError:
raise ValueError(
"""An invalid option for `--model` was supplied,
options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']"""
)
self.rnn = nn.RNN(ninp,
nhid,
nlayers,
nonlinearity=nonlinearity,
dropout=dropout)
self.decoder = nn.Linear(nhid, ntoken)
# Optionally tie weights as in:
# "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016)
# https://arxiv.org/abs/1608.05859
# and
# "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
# https://arxiv.org/abs/1611.01462
if tie_weights:
if nhid != ninp:
raise ValueError(
'When using the tied flag, nhid must be equal to emsize')
self.decoder.weight = self.encoder.weight
self.init_weights()
self.rnn_type = rnn_type
self.nhid = nhid
self.nlayers = nlayers
if use_dni:
if rnn_type == 'LSTM':
output_dim = 2 * nhid
else:
output_dim = nhid
self.backward_interface = dni.BackwardInterface(
dni.BasicSynthesizer(output_dim, n_hidden=2))
else:
self.backward_interface = None
