def __init__(self): super(Net, self).__init__() self.block = nn.ModuleList([]) self.backward_interfaces = nn.ModuleList([]) self.block.append( nn.Sequential(nn.Conv2d(3, 128, kernel_size=5, padding=2), nn.BatchNorm2d(128), nn.ReLU(), nn.MaxPool2d( (3, 3)))) self.block.append( nn.Sequential(nn.Conv2d(128, 128, kernel_size=5, padding=2), nn.BatchNorm2d(128), nn.ReLU(), nn.AvgPool2d( (3, 3)))) self.block.append( nn.Sequential(nn.Conv2d(128, 128, kernel_size=5, padding=2), nn.BatchNorm2d(128), nn.ReLU(), nn.AvgPool2d( (3, 3)))) if args.dni: self.backward_interfaces[0] = dni.BackwardInterface( ConvSynthesizer()) self.backward_interfaces[1] = dni.BackwardInterface( ConvSynthesizer()) self.backward_interfaces[2] = dni.BackwardInterface( ConvSynthesizer()) self.fc1 = nn.Linear(128, 10)
def __init__(self): super(VGG16_SG2, self).__init__() # modify the original VGG model to have a 10 unit output linear layer self.model = models.vgg16(True) self.model.classifier[0].in_features = 512 self.conv1 = nn.Conv2d(1, 3, 1) self.block1 = self.model.features[:5] self.block2 = self.model.features[5:10] self.block3 = self.model.features[10:17] self.block4 = self.model.features[17:24] self.block5 = self.model.features[24:] self.classifier = nn.Sequential( nn.Linear(512, 128), nn.ReLU(True), nn.Linear(128, 64), nn.ReLU(True), nn.Linear(64, 10), ) self.backward_interface_1 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=24, n_hidden=20)) self.backward_interface_2 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=12, n_hidden=20)) self.backward_interface_3 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=6, n_hidden=20)) self.backward_interface_4 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=3, n_hidden=20)) self.backward_interface_5 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=1, n_hidden=20))
def __init__(self, *args, **kwargs): super(less, self).__init__(Bottleneck, [3, 4, 6, 3], num_classes=num_classes, *args, **kwargs) self.seq1 = nn.Sequential(self.conv1, self.bn1, self.relu, self.maxpool, self.layer1, self.layer2).to( 'cuda:0') # changed from cpu to gpu # self.input_context = nn.Linear(10, 16) # self.input_trigger = nn.Conv2d(512, 512, kernel_size=5, padding=2) # self.hidden = nn.Conv2d(512, 512, kernel_size=5, padding=2) # self.output = nn.Conv2d(512, 512, kernel_size=5, padding=2) context_dim = None self.backward_interface = dni.BackwardInterface(Conv_less()) #self.backward_interface = dni.BackwardInterface( dni.BasicSynthesizer(mcf())) self.seq2 = nn.Sequential( self.layer3, self.layer4, self.avgpool, ).to('cuda:0') self.fc.to('cuda:0')
def __init__(self): super(Eight_Layer_SG, self).__init__() self.block_1 = nn.Sequential( nn.Conv2d( in_channels=1, out_channels=32, kernel_size=(12, 12), stride=(4, 4), padding=(4, 4), ), nn.BatchNorm2d(32), nn.ReLU(), ) self.block_2 = nn.Sequential( nn.Conv2d( in_channels=32, out_channels=64, kernel_size=(12, 12), stride=(4, 4), padding=(4, 4), ), nn.BatchNorm2d(64), nn.ReLU(), ) self.classifier = nn.Sequential( nn.Linear(576, 128), nn.ReLU(True), nn.Dropout(p=0.5), nn.Linear(128, 64), nn.ReLU(True), nn.Dropout(p=0.5), nn.Linear(64, 10), ) self.backward_interface_1 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=12, n_hidden=256)) self.backward_interface_2 = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=3, n_hidden=128))
def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(1, 10, kernel_size=5) self.conv1_bn = nn.BatchNorm2d(10) self.conv2 = nn.Conv2d(10, 20, kernel_size=5) self.conv2_bn = nn.BatchNorm2d(20) self.conv2_drop = nn.Dropout2d() if args.dni: self.backward_interface = dni.BackwardInterface(ConvSynthesizer()) self.fc1 = nn.Linear(320, 50) self.fc1_bn = nn.BatchNorm1d(50) self.fc2 = nn.Linear(50, 10) self.fc2_bn = nn.BatchNorm1d(10)
def _init_dni(self, synthesizer_type, freeze_synthesizer, non_zero_init, scale_synth_grad, trained_net_file, trained_net_initial_file): if self.context: context_dim = 10 else: context_dim = None if synthesizer_type == 'mlp': synthesizer = dni.BasicSynthesizer( output_dim=self.num_neurons, context_dim=context_dim, n_hidden=1, non_zero_init=non_zero_init, normalize_output=scale_synth_grad) if synthesizer_type == 'local_mlp': synthesizer = LocalSynthesizer(output_dim=self.num_neurons, context_dim=context_dim, non_zero_init=non_zero_init) if synthesizer_type == 'conv': pass #TODO self.backward_interface = dni.BackwardInterface(synthesizer) # Loading the specified initialization if trained_net_initial_file is not None: trained_net = torch.load(trained_net_initial_file) dict_trained_params = dict(trained_net.named_parameters()) for name, param in self.named_parameters(): if name in dict_trained_params: param.requires_grad = False print('Copying pretrained ' + name) param.copy_(dict_trained_params[name].data) param.requires_grad = True # Loading the trained synthesizer if trained_net_file is not None: trained_net = torch.load(trained_net_file) dict_trained_params = dict( trained_net.backward_interface.named_parameters()) for name, param in self.backward_interface.named_parameters(): if name in dict_trained_params: param.requires_grad = False print('Copying pretrained ' + name) param.copy_(dict_trained_params[name].data) param.requires_grad = True if freeze_synthesizer: for name, param in self.backward_interface.named_parameters(): param.requires_grad = False
def __init__(self): super(Net, self).__init__() self.hidden1 = nn.Linear(784, 256, bias=False) self.hidden1_bn = nn.BatchNorm1d(256) self.hidden2 = nn.Linear(256, 256, bias=False) self.hidden2_bn = nn.BatchNorm1d(256) if args.dni: if args.context: context_dim = 10 else: context_dim = None self.backward_interface = dni.BackwardInterface( dni.BasicSynthesizer(output_dim=256, n_hidden=1, context_dim=context_dim)) self.output = nn.Linear(256, 10, bias=False) self.output_bn = nn.BatchNorm1d(10)
def __init__(self, *nkwargs, **kwargs): super(ModelParallelResNet50, self).__init__( Bottleneck, [3, 4, 6, 3], num_classes=10, *nkwargs, **kwargs) self.seq1 = nn.Sequential( self.conv1, self.bn1, self.relu, self.maxpool, self.layer1 ) #.to('cpu') if args.dni: self.backward_interface = dni.BackwardInterface(ResNetConvSynthesizer()) self.seq2 = nn.Sequential( self.layer2, self.layer3, self.layer4, self.avgpool, ) # .to('cuda:0')
def __init__(self): super(NetDNIPart, self).__init__() if args.dni: self.backward_interface = dni.BackwardInterface(ConvSynthesizer())
def __init__(self, rnn_type, ntoken, ninp, nhid, nlayers, dropout=0.5, tie_weights=False, use_dni=False): super(RNNModel, self).__init__() self.drop = nn.Dropout(dropout) self.encoder = nn.Embedding(ntoken, ninp) if rnn_type in ['LSTM', 'GRU']: self.rnn = getattr(nn, rnn_type)(ninp, nhid, nlayers, dropout=dropout) else: try: nonlinearity = { 'RNN_TANH': 'tanh', 'RNN_RELU': 'relu' }[rnn_type] except KeyError: raise ValueError( """An invalid option for `--model` was supplied, options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""" ) self.rnn = nn.RNN(ninp, nhid, nlayers, nonlinearity=nonlinearity, dropout=dropout) self.decoder = nn.Linear(nhid, ntoken) # Optionally tie weights as in: # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016) # https://arxiv.org/abs/1608.05859 # and # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016) # https://arxiv.org/abs/1611.01462 if tie_weights: if nhid != ninp: raise ValueError( 'When using the tied flag, nhid must be equal to emsize') self.decoder.weight = self.encoder.weight self.init_weights() self.rnn_type = rnn_type self.nhid = nhid self.nlayers = nlayers if use_dni: if rnn_type == 'LSTM': output_dim = 2 * nhid else: output_dim = nhid self.backward_interface = dni.BackwardInterface( dni.BasicSynthesizer(output_dim, n_hidden=2)) else: self.backward_interface = None