def __init__(self, field_dims, embed_dim, num_layers, mlp_dims, dropout, training_method='dfa'): super().__init__() self.embedding = FeaturesEmbedding(field_dims, embed_dim) self.embed_output_dim = len(field_dims) * embed_dim self.cn = DFACrossNetwork(self.embed_output_dim, num_layers, dfa_output=False) self.mlp = DFAMultiLayerPerceptron(self.embed_output_dim, mlp_dims, dropout, output_layer=False, dfa_output=False) self.linear = torch.nn.Linear(mlp_dims[-1] + self.embed_output_dim, 1) self.dfa_embeddings = DFALayer() self.dfa_stack = DFALayer() self.dfa = DFA(dfa_layers=[ *self.cn.dfa_layers, *self.mlp.dfa_layers, self.dfa_stack, self.dfa_embeddings ], feedback_points_handling=FeedbackPointsHandling.LAST, no_training=training_method != 'dfa')
def __init__(self, in_channels, hidden_channels, out_channels, num_layers, heads=1, groups=1, training_method='dfa'): super(DFADNANet, self).__init__() self.hidden_channels = hidden_channels self.lin1 = torch.nn.Linear(in_channels, hidden_channels) self.dfa1 = DFALayer() self.convs = torch.nn.ModuleList() self.dfa_convs = [] for i in range(num_layers): self.convs.append(DNAConv(hidden_channels, heads, groups, dropout=0., cached=True)) self.dfa_convs.append(DFALayer()) self.lin2 = torch.nn.Linear(hidden_channels, out_channels) self.dfa = DFA([self.dfa1, *self.dfa_convs], no_training=training_method != 'dfa')
def __init__(self, hidden_size, training_method='DFA'): super(MNISTFullyConnected, self).__init__() self.fc1 = nn.Linear(784, hidden_size) self.fc2 = nn.Linear(hidden_size, hidden_size) self.fc3 = nn.Linear(hidden_size, 10) self.training_method = training_method if self.training_method in ['DFA', 'SHALLOW']: self.dfa1, self.dfa2 = DFALayer(), DFALayer() self.dfa = DFA( [self.dfa1, self.dfa2], feedback_points_handling=FeedbackPointsHandling.LAST, no_training=(self.training_method == 'SHALLOW'))
def __init__(self, d_model, n_heads, dim_feedforward=2048, dropout=0.1, activation="relu", layernorm=True, attn='standard', dfa='none'): super().__init__() if attn == 'standard': self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) elif attn == 'fixed': self.self_attn = attention.FixedAttention(d_model, language_model=True) elif attn in ['dense', 'random']: self.self_attn = attention.SynthesizerAttention(d_model, n_heads, synth=attn, dropout=dropout) else: raise ValueError( "attn must be in ['standard', 'fixed', 'dense', 'random']") # Implementation of Feedforward model self.linear1 = nn.Linear(d_model, dim_feedforward) self.dropout = nn.Dropout(dropout) self.linear2 = nn.Linear(dim_feedforward, d_model) if layernorm: self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) else: self.norm1 = nn.Identity() self.norm2 = nn.Identity() print("WARNING: layer normalization is deactivated") self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.activation = _get_activation_fn(activation) self.dfas = [] if dfa == 'simple': self.dfas = [DFALayer()] elif dfa == 'full': self.dfas = [DFALayer(), DFALayer(), DFALayer()]
def __init__(self, num_features, num_classes, training_method='dfa'): super(DFASplineNet, self).__init__() self.conv1 = SplineConv(num_features, 16, dim=1, kernel_size=2) self.dfa_1 = DFALayer() self.conv2 = SplineConv(16, num_classes, dim=1, kernel_size=2) self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
def __init__(self, input_dim, embed_dims, dropout, output_layer=True, dfa_output=True, training_method='dfa'): super().__init__() self.dfa_output = dfa_output layers = list() self.dfa_layers = [] for embed_dim in embed_dims: layers.append(torch.nn.Linear(input_dim, embed_dim)) layers.append(torch.nn.BatchNorm1d(embed_dim)) layers.append(torch.nn.ReLU()) layers.append(torch.nn.Dropout(p=dropout)) dfa_layer = DFALayer() layers.append(dfa_layer) self.dfa_layers.append(dfa_layer) input_dim = embed_dim if output_layer: layers.append(torch.nn.Linear(input_dim, 1)) if dfa_output: layers.append( DFA(dfa_layers=self.dfa_layers, feedback_points_handling=FeedbackPointsHandling.LAST, no_training=training_method != 'dfa')) self.mlp = torch.nn.Sequential(*layers)
def __init__(self, num_features, num_classes, second_layer_heads=8, training_method='dfa'): super(DFAGATNet, self).__init__() self.conv1 = GATConv(num_features, 8, heads=8, dropout=0.6) self.dfa_1 = DFALayer() # On the Pubmed dataset, use heads=8 in conv2. self.conv2 = GATConv(8 * 8, num_classes, heads=second_layer_heads, concat=True, dropout=0.) self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
def __init__(self, field_dims, embed_dim, atten_embed_dim, num_heads, num_layers, mlp_dims, dropouts, has_residual=True, training_method='dfa'): super().__init__() self.num_fields = len(field_dims) self.linear = FeaturesLinear(field_dims) self.embedding = FeaturesEmbedding(field_dims, embed_dim) self.dfa_embedding = DFALayer() self.atten_embedding = torch.nn.Linear(embed_dim, atten_embed_dim) self.dfa_atten_embedding = DFALayer() self.embed_output_dim = len(field_dims) * embed_dim self.atten_output_dim = len(field_dims) * atten_embed_dim self.has_residual = has_residual self.mlp = DFAMultiLayerPerceptron(self.embed_output_dim, mlp_dims, dropouts[1], dfa_output=False) self.self_attns = torch.nn.ModuleList([ torch.nn.MultiheadAttention(atten_embed_dim, num_heads, dropout=dropouts[0]) for _ in range(num_layers) ]) self.dfa_self_attns = [DFALayer() for _ in range(num_layers - 1)] self.attn_fc = torch.nn.Linear(self.atten_output_dim, 1) if self.has_residual: self.V_res_embedding = torch.nn.Linear(embed_dim, atten_embed_dim) self.dfa_cross = DFALayer() self.dfa = DFA(dfa_layers=[ self.dfa_atten_embedding, *self.dfa_self_attns, self.dfa_cross, *self.mlp.dfa_layers, self.dfa_embedding ], feedback_points_handling=FeedbackPointsHandling.LAST, no_training=training_method != 'dfa')
def __init__(self, num_features, num_classes, training_method='dfa'): super(DFAChebNet, self).__init__() self.conv1 = ChebConv(num_features, 16, K=2) self.dfa_1 = DFALayer() self.conv2 = ChebConv(16, num_classes, K=2) self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa') self.reg_params = self.conv1.parameters() self.non_reg_params = self.conv2.parameters()
def __init__(self, num_features, num_classes, training_method='dfa'): super(DFAGraphNet, self).__init__() self.conv1 = GCNConv(num_features, 16, cached=True) self.dfa_1 = DFALayer() self.conv2 = GCNConv(16, num_classes, cached=True) self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa') self.reg_params = self.conv1.parameters() self.non_reg_params = self.conv2.parameters()
def __init__(self, in_channels, n_classes, *args, **kwargs): super().__init__() self.training_method = None if 'training_method' in kwargs: self.training_method = kwargs['training_method'] self.use_dfa = self.training_method in ['DFA', 'SHALLOW', 'BLOCK'] if self.use_dfa: all_dfas = None if self.training_method == 'DFA': # Groups of Layers DFA all_dfas = dfas = [ DFALayer() for _ in range(len(kwargs['depths'])) ] # should I grab these from the EncoderLayers instead? layer_dfas = None elif self.training_method == 'BLOCK': dfas = None layer_dfas = [[DFALayer() for _ in range(number_of_blocks)] for number_of_blocks in kwargs['depths']] assert len(layer_dfas) == len(kwargs['depths']) # flat version of layer_dfas total_number_of_blocks = sum(kwargs['depths']) print(f'total_number_of_blocks={total_number_of_blocks}') all_dfas = [item for sublist in layer_dfas for item in sublist] assert len(all_dfas) == total_number_of_blocks elif self.training_method == 'SHALLOW': raise NotImplementedError('SHALLOW DFA not implemented') self.dfa = DFA( all_dfas, feedback_points_handling=FeedbackPointsHandling.LAST, no_training=(self.training_method == 'SHALLOW')) else: dfas = None layer_dfas = None self.encoder = ResNetEncoder(in_channels, dfas=dfas, layer_dfas=layer_dfas, *args, **kwargs) self.decoder = ResnetDecoder( self.encoder.blocks[-1].blocks[-1].expanded_channels, n_classes)
def __init__(self, field_dims, embed_dim, LNN_dim, mlp_dims, dropouts, training_method='dfa'): super().__init__() self.num_fields = len(field_dims) self.linear = FeaturesLinear(field_dims) # Linear self.embedding = FeaturesEmbedding(field_dims, embed_dim) # Embedding self.dfa_embed = DFALayer() self.LNN_dim = LNN_dim self.LNN_output_dim = self.LNN_dim * embed_dim self.LNN = LNN(self.num_fields, embed_dim, LNN_dim) self.dfa_lnn = DFALayer() self.mlp = DFAMultiLayerPerceptron(self.LNN_output_dim, mlp_dims, dropouts[0], dfa_output=False) self.dfa = DFA(dfa_layers=[self.dfa_lnn, *self.mlp.dfa_layers], feedback_points_handling=FeedbackPointsHandling.LAST, no_training=training_method != 'dfa')
def __init__(self, vocab_size, d_model, n_heads, dim_feedforward, n_layers, dropout=0.1, tie_embeddings=False, dfa='none', no_training=False, dfa_after_vocab=False, dfa_embed=False, **kwargs): super().__init__() self.embedding = nn.Embedding(vocab_size, d_model) self.pos_encoder = PositionalEncoding(d_model, dropout) layer = CustomTransformerEncoderLayer(d_model, n_heads, dim_feedforward, dropout, dfa=dfa, **kwargs) self.transformer_encoder = nn.TransformerEncoder(layer, n_layers) self.dfa = None self.dfaemb = None if dfa in ['simple', 'full']: self.dfaemb = [DFALayer()] if dfa_embed else [] dfas = self.dfaemb + [ dfalayer for layer in self.transformer_encoder.layers for dfalayer in layer.dfas ] self.dfa = DFA(dfas, no_training=no_training, batch_dims=(0, 1)) self.dfa_after_vocab = dfa_after_vocab self.decoder = None if not tie_embeddings: self.decoder = nn.Linear(d_model, vocab_size) self.tie_embeddings = tie_embeddings self.d_model = d_model self.src_mask = None self.init_weights()
def __init__(self, opu, binary_layer, opu_features, features, opu_input, dataset, sign_back): super(AblationCifarVGG16, self).__init__() self.features = features print( f'OPU: {opu}, binary_layer: {binary_layer}, oo: {opu_features}, oi: {opu_input}, sign back: {sign_back}' ) if dataset == 'cifar10': self.number_classes = 10 if dataset == 'cifar100': self.number_classes = 100 if sign_back: self.dfa1 = Identity() self.dfa = Identity() else: self.training_method = 'DFA' self.dfa1 = DFALayer() self.dfa = DFA([self.dfa1], no_training=(self.training_method == 'SHALLOW')) if opu: second_classifier_layer = SimulatedOpu(in_features=opu_input, out_features=opu_features) else: second_classifier_layer = nn.Linear(opu_input, opu_features) self.classifier = nn.Sequential( nn.Dropout(), nn.Linear(512, opu_input), nn.ReLU(True), self.dfa1, Sign(binary_layer, sign_back=sign_back), second_classifier_layer, nn.ReLU(True), nn.Linear(opu_features, self.number_classes), self.dfa) # Initialize weights for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) m.bias.data.zero_()
def __init__(self, input_dim, num_layers, dfa_output=True, training_method='dfa'): super().__init__() self.dfa_output = dfa_output self.num_layers = num_layers self.w = torch.nn.ModuleList([ torch.nn.Linear(input_dim, 1, bias=False) for _ in range(num_layers) ]) self.b = torch.nn.ParameterList([ torch.nn.Parameter(torch.zeros((input_dim, ))) for _ in range(num_layers) ]) self.dfa_layers = [DFALayer() for _ in range(num_layers - 1)] if self.dfa_output: self.dfa = DFA( dfa_layers=self.dfa_layers, feedback_points_handling=FeedbackPointsHandling.Last, no_training=training_method != 'dfa')
def __init__(self, in_channels, out_channels, training_method): super(DFAGraphEncoder, self).__init__() self.conv1 = GCNConv(in_channels, 2 * out_channels, cached=True) self.dfa_1 = DFALayer() self.conv2 = GCNConv(2 * out_channels, out_channels, cached=True) self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
def __init__(self, opu, binary_layer, opu_features, features, opu_input, sign_back, real_opu, device, dataset): super(CifarVGG16, self).__init__() self.features = features if dataset == 'cifar10': self.number_classes = 10 if dataset == 'cifar100': self.number_classes = 100 if binary_layer: if sign_back: print( f'- Binary layer = {binary_layer} but sign_back = {sign_back}--> attack method = FA!' ) self.dfa1 = Identity() self.dfa = Identity() else: self.training_method = 'DFA' print( f'- Binary layer = {binary_layer} --> training method = DFA!' ) self.dfa1 = DFALayer() self.dfa = DFA([self.dfa1], no_training=(self.training_method == 'SHALLOW')) else: if opu: print( f'- Binary layer = {binary_layer}, opu = {opu} --> training method = FA' ) else: print( f'- Binary layer = {binary_layer}, opu = {opu} --> training method = BP' ) self.dfa1 = nn.Dropout() self.dfa = Identity() if opu: print(f'- opu_input = {opu_input}, opu_output = {opu_features}') if real_opu: print('- VGG with real OPU') print(opu_input, opu_features) second_classifier_layer = RealOpu(in_features=opu_input, out_features=opu_features, device=device) else: print('- VGG with simulated OPU') second_classifier_layer = SimulatedOpu( in_features=opu_input, out_features=opu_features) else: print('- VGG without OPU') second_classifier_layer = nn.Linear(opu_input, opu_features) self.classifier = nn.Sequential( nn.Dropout(), nn.Linear(512, opu_input), nn.ReLU(True), self.dfa1, Sign(binary_layer, sign_back=sign_back), second_classifier_layer, nn.ReLU(True), nn.Linear(opu_features, self.number_classes), self.dfa) # Initialize weights for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) m.bias.data.zero_()