def __init__(self,
field_dims,
embed_dim,
num_layers,
mlp_dims,
dropout,
training_method='dfa'):
super().__init__()
self.embedding = FeaturesEmbedding(field_dims, embed_dim)
self.embed_output_dim = len(field_dims) * embed_dim
self.cn = DFACrossNetwork(self.embed_output_dim,
num_layers,
dfa_output=False)
self.mlp = DFAMultiLayerPerceptron(self.embed_output_dim,
mlp_dims,
dropout,
output_layer=False,
dfa_output=False)
self.linear = torch.nn.Linear(mlp_dims[-1] + self.embed_output_dim, 1)
self.dfa_embeddings = DFALayer()
self.dfa_stack = DFALayer()
self.dfa = DFA(dfa_layers=[
*self.cn.dfa_layers, *self.mlp.dfa_layers, self.dfa_stack,
self.dfa_embeddings
],
feedback_points_handling=FeedbackPointsHandling.LAST,
no_training=training_method != 'dfa')
def __init__(self, in_channels, hidden_channels, out_channels, num_layers, heads=1, groups=1,
training_method='dfa'):
super(DFADNANet, self).__init__()
self.hidden_channels = hidden_channels
self.lin1 = torch.nn.Linear(in_channels, hidden_channels)
self.dfa1 = DFALayer()
self.convs = torch.nn.ModuleList()
self.dfa_convs = []
for i in range(num_layers):
self.convs.append(DNAConv(hidden_channels, heads, groups, dropout=0., cached=True))
self.dfa_convs.append(DFALayer())
self.lin2 = torch.nn.Linear(hidden_channels, out_channels)
self.dfa = DFA([self.dfa1, *self.dfa_convs], no_training=training_method != 'dfa')
def __init__(self, hidden_size, training_method='DFA'):
super(MNISTFullyConnected, self).__init__()
self.fc1 = nn.Linear(784, hidden_size)
self.fc2 = nn.Linear(hidden_size, hidden_size)
self.fc3 = nn.Linear(hidden_size, 10)
self.training_method = training_method
if self.training_method in ['DFA', 'SHALLOW']:
self.dfa1, self.dfa2 = DFALayer(), DFALayer()
self.dfa = DFA(
[self.dfa1, self.dfa2],
feedback_points_handling=FeedbackPointsHandling.LAST,
no_training=(self.training_method == 'SHALLOW'))
def __init__(self,
d_model,
n_heads,
dim_feedforward=2048,
dropout=0.1,
activation="relu",
layernorm=True,
attn='standard',
dfa='none'):
super().__init__()
if attn == 'standard':
self.self_attn = nn.MultiheadAttention(d_model,
n_heads,
dropout=dropout)
elif attn == 'fixed':
self.self_attn = attention.FixedAttention(d_model,
language_model=True)
elif attn in ['dense', 'random']:
self.self_attn = attention.SynthesizerAttention(d_model,
n_heads,
synth=attn,
dropout=dropout)
else:
raise ValueError(
"attn must be in ['standard', 'fixed', 'dense', 'random']")
# Implementation of Feedforward model
self.linear1 = nn.Linear(d_model, dim_feedforward)
self.dropout = nn.Dropout(dropout)
self.linear2 = nn.Linear(dim_feedforward, d_model)
if layernorm:
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
else:
self.norm1 = nn.Identity()
self.norm2 = nn.Identity()
print("WARNING: layer normalization is deactivated")
self.dropout1 = nn.Dropout(dropout)
self.dropout2 = nn.Dropout(dropout)
self.activation = _get_activation_fn(activation)
self.dfas = []
if dfa == 'simple':
self.dfas = [DFALayer()]
elif dfa == 'full':
self.dfas = [DFALayer(), DFALayer(), DFALayer()]
def __init__(self, num_features, num_classes, training_method='dfa'):
super(DFASplineNet, self).__init__()
self.conv1 = SplineConv(num_features, 16, dim=1, kernel_size=2)
self.dfa_1 = DFALayer()
self.conv2 = SplineConv(16, num_classes, dim=1, kernel_size=2)
self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
def __init__(self,
input_dim,
embed_dims,
dropout,
output_layer=True,
dfa_output=True,
training_method='dfa'):
super().__init__()
self.dfa_output = dfa_output
layers = list()
self.dfa_layers = []
for embed_dim in embed_dims:
layers.append(torch.nn.Linear(input_dim, embed_dim))
layers.append(torch.nn.BatchNorm1d(embed_dim))
layers.append(torch.nn.ReLU())
layers.append(torch.nn.Dropout(p=dropout))
dfa_layer = DFALayer()
layers.append(dfa_layer)
self.dfa_layers.append(dfa_layer)
input_dim = embed_dim
if output_layer:
layers.append(torch.nn.Linear(input_dim, 1))
if dfa_output:
layers.append(
DFA(dfa_layers=self.dfa_layers,
feedback_points_handling=FeedbackPointsHandling.LAST,
no_training=training_method != 'dfa'))
self.mlp = torch.nn.Sequential(*layers)
def __init__(self, num_features, num_classes, second_layer_heads=8, training_method='dfa'):
super(DFAGATNet, self).__init__()
self.conv1 = GATConv(num_features, 8, heads=8, dropout=0.6)
self.dfa_1 = DFALayer()
# On the Pubmed dataset, use heads=8 in conv2.
self.conv2 = GATConv(8 * 8, num_classes, heads=second_layer_heads, concat=True, dropout=0.)
self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
def __init__(self,
field_dims,
embed_dim,
atten_embed_dim,
num_heads,
num_layers,
mlp_dims,
dropouts,
has_residual=True,
training_method='dfa'):
super().__init__()
self.num_fields = len(field_dims)
self.linear = FeaturesLinear(field_dims)
self.embedding = FeaturesEmbedding(field_dims, embed_dim)
self.dfa_embedding = DFALayer()
self.atten_embedding = torch.nn.Linear(embed_dim, atten_embed_dim)
self.dfa_atten_embedding = DFALayer()
self.embed_output_dim = len(field_dims) * embed_dim
self.atten_output_dim = len(field_dims) * atten_embed_dim
self.has_residual = has_residual
self.mlp = DFAMultiLayerPerceptron(self.embed_output_dim,
mlp_dims,
dropouts[1],
dfa_output=False)
self.self_attns = torch.nn.ModuleList([
torch.nn.MultiheadAttention(atten_embed_dim,
num_heads,
dropout=dropouts[0])
for _ in range(num_layers)
])
self.dfa_self_attns = [DFALayer() for _ in range(num_layers - 1)]
self.attn_fc = torch.nn.Linear(self.atten_output_dim, 1)
if self.has_residual:
self.V_res_embedding = torch.nn.Linear(embed_dim, atten_embed_dim)
self.dfa_cross = DFALayer()
self.dfa = DFA(dfa_layers=[
self.dfa_atten_embedding, *self.dfa_self_attns, self.dfa_cross,
*self.mlp.dfa_layers, self.dfa_embedding
],
feedback_points_handling=FeedbackPointsHandling.LAST,
no_training=training_method != 'dfa')
def __init__(self, num_features, num_classes, training_method='dfa'):
super(DFAChebNet, self).__init__()
self.conv1 = ChebConv(num_features, 16, K=2)
self.dfa_1 = DFALayer()
self.conv2 = ChebConv(16, num_classes, K=2)
self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
self.reg_params = self.conv1.parameters()
self.non_reg_params = self.conv2.parameters()
def __init__(self, num_features, num_classes, training_method='dfa'):
super(DFAGraphNet, self).__init__()
self.conv1 = GCNConv(num_features, 16, cached=True)
self.dfa_1 = DFALayer()
self.conv2 = GCNConv(16, num_classes, cached=True)
self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
self.reg_params = self.conv1.parameters()
self.non_reg_params = self.conv2.parameters()
def __init__(self, in_channels, n_classes, *args, **kwargs):
super().__init__()
self.training_method = None
if 'training_method' in kwargs:
self.training_method = kwargs['training_method']
self.use_dfa = self.training_method in ['DFA', 'SHALLOW', 'BLOCK']
if self.use_dfa:
all_dfas = None
if self.training_method == 'DFA': # Groups of Layers DFA
all_dfas = dfas = [
DFALayer() for _ in range(len(kwargs['depths']))
] # should I grab these from the EncoderLayers instead?
layer_dfas = None
elif self.training_method == 'BLOCK':
dfas = None
layer_dfas = [[DFALayer() for _ in range(number_of_blocks)]
for number_of_blocks in kwargs['depths']]
assert len(layer_dfas) == len(kwargs['depths'])
# flat version of layer_dfas
total_number_of_blocks = sum(kwargs['depths'])
print(f'total_number_of_blocks={total_number_of_blocks}')
all_dfas = [item for sublist in layer_dfas for item in sublist]
assert len(all_dfas) == total_number_of_blocks
elif self.training_method == 'SHALLOW':
raise NotImplementedError('SHALLOW DFA not implemented')
self.dfa = DFA(
all_dfas,
feedback_points_handling=FeedbackPointsHandling.LAST,
no_training=(self.training_method == 'SHALLOW'))
else:
dfas = None
layer_dfas = None
self.encoder = ResNetEncoder(in_channels,
dfas=dfas,
layer_dfas=layer_dfas,
*args,
**kwargs)
self.decoder = ResnetDecoder(
self.encoder.blocks[-1].blocks[-1].expanded_channels, n_classes)
def __init__(self,
field_dims,
embed_dim,
LNN_dim,
mlp_dims,
dropouts,
training_method='dfa'):
super().__init__()
self.num_fields = len(field_dims)
self.linear = FeaturesLinear(field_dims) # Linear
self.embedding = FeaturesEmbedding(field_dims, embed_dim) # Embedding
self.dfa_embed = DFALayer()
self.LNN_dim = LNN_dim
self.LNN_output_dim = self.LNN_dim * embed_dim
self.LNN = LNN(self.num_fields, embed_dim, LNN_dim)
self.dfa_lnn = DFALayer()
self.mlp = DFAMultiLayerPerceptron(self.LNN_output_dim,
mlp_dims,
dropouts[0],
dfa_output=False)
self.dfa = DFA(dfa_layers=[self.dfa_lnn, *self.mlp.dfa_layers],
feedback_points_handling=FeedbackPointsHandling.LAST,
no_training=training_method != 'dfa')
def __init__(self,
vocab_size,
d_model,
n_heads,
dim_feedforward,
n_layers,
dropout=0.1,
tie_embeddings=False,
dfa='none',
no_training=False,
dfa_after_vocab=False,
dfa_embed=False,
**kwargs):
super().__init__()
self.embedding = nn.Embedding(vocab_size, d_model)
self.pos_encoder = PositionalEncoding(d_model, dropout)
layer = CustomTransformerEncoderLayer(d_model,
n_heads,
dim_feedforward,
dropout,
dfa=dfa,
**kwargs)
self.transformer_encoder = nn.TransformerEncoder(layer, n_layers)
self.dfa = None
self.dfaemb = None
if dfa in ['simple', 'full']:
self.dfaemb = [DFALayer()] if dfa_embed else []
dfas = self.dfaemb + [
dfalayer for layer in self.transformer_encoder.layers
for dfalayer in layer.dfas
]
self.dfa = DFA(dfas, no_training=no_training, batch_dims=(0, 1))
self.dfa_after_vocab = dfa_after_vocab
self.decoder = None
if not tie_embeddings:
self.decoder = nn.Linear(d_model, vocab_size)
self.tie_embeddings = tie_embeddings
self.d_model = d_model
self.src_mask = None
self.init_weights()
def __init__(self, opu, binary_layer, opu_features, features, opu_input,
dataset, sign_back):
super(AblationCifarVGG16, self).__init__()
self.features = features
print(
f'OPU: {opu}, binary_layer: {binary_layer}, oo: {opu_features}, oi: {opu_input}, sign back: {sign_back}'
)
if dataset == 'cifar10':
self.number_classes = 10
if dataset == 'cifar100':
self.number_classes = 100
if sign_back:
self.dfa1 = Identity()
self.dfa = Identity()
else:
self.training_method = 'DFA'
self.dfa1 = DFALayer()
self.dfa = DFA([self.dfa1],
no_training=(self.training_method == 'SHALLOW'))
if opu:
second_classifier_layer = SimulatedOpu(in_features=opu_input,
out_features=opu_features)
else:
second_classifier_layer = nn.Linear(opu_input, opu_features)
self.classifier = nn.Sequential(
nn.Dropout(), nn.Linear(512, opu_input), nn.ReLU(True), self.dfa1,
Sign(binary_layer, sign_back=sign_back), second_classifier_layer,
nn.ReLU(True), nn.Linear(opu_features, self.number_classes),
self.dfa)
# Initialize weights
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
m.bias.data.zero_()
def __init__(self,
input_dim,
num_layers,
dfa_output=True,
training_method='dfa'):
super().__init__()
self.dfa_output = dfa_output
self.num_layers = num_layers
self.w = torch.nn.ModuleList([
torch.nn.Linear(input_dim, 1, bias=False)
for _ in range(num_layers)
])
self.b = torch.nn.ParameterList([
torch.nn.Parameter(torch.zeros((input_dim, )))
for _ in range(num_layers)
])
self.dfa_layers = [DFALayer() for _ in range(num_layers - 1)]
if self.dfa_output:
self.dfa = DFA(
dfa_layers=self.dfa_layers,
feedback_points_handling=FeedbackPointsHandling.Last,
no_training=training_method != 'dfa')
def __init__(self, in_channels, out_channels, training_method):
super(DFAGraphEncoder, self).__init__()
self.conv1 = GCNConv(in_channels, 2 * out_channels, cached=True)
self.dfa_1 = DFALayer()
self.conv2 = GCNConv(2 * out_channels, out_channels, cached=True)
self.dfa = DFA(dfa_layers=[self.dfa_1], no_training=training_method != 'dfa')
def __init__(self, opu, binary_layer, opu_features, features, opu_input,
sign_back, real_opu, device, dataset):
super(CifarVGG16, self).__init__()
self.features = features
if dataset == 'cifar10':
self.number_classes = 10
if dataset == 'cifar100':
self.number_classes = 100
if binary_layer:
if sign_back:
print(
f'- Binary layer = {binary_layer} but sign_back = {sign_back}--> attack method = FA!'
)
self.dfa1 = Identity()
self.dfa = Identity()
else:
self.training_method = 'DFA'
print(
f'- Binary layer = {binary_layer} --> training method = DFA!'
)
self.dfa1 = DFALayer()
self.dfa = DFA([self.dfa1],
no_training=(self.training_method == 'SHALLOW'))
else:
if opu:
print(
f'- Binary layer = {binary_layer}, opu = {opu} --> training method = FA'
)
else:
print(
f'- Binary layer = {binary_layer}, opu = {opu} --> training method = BP'
)
self.dfa1 = nn.Dropout()
self.dfa = Identity()
if opu:
print(f'- opu_input = {opu_input}, opu_output = {opu_features}')
if real_opu:
print('- VGG with real OPU')
print(opu_input, opu_features)
second_classifier_layer = RealOpu(in_features=opu_input,
out_features=opu_features,
device=device)
else:
print('- VGG with simulated OPU')
second_classifier_layer = SimulatedOpu(
in_features=opu_input, out_features=opu_features)
else:
print('- VGG without OPU')
second_classifier_layer = nn.Linear(opu_input, opu_features)
self.classifier = nn.Sequential(
nn.Dropout(), nn.Linear(512, opu_input), nn.ReLU(True), self.dfa1,
Sign(binary_layer, sign_back=sign_back), second_classifier_layer,
nn.ReLU(True), nn.Linear(opu_features, self.number_classes),
self.dfa)
# Initialize weights
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
m.bias.data.zero_()
