def residual_attention(x1):
x2 = HierarchicalAttention(self.attention_heads)(x1)
x2 = tf.keras.layers.Dropout(self.dropout)(x2)
x2 = tf.keras.layers.LayerNormalization()(x2)
if self.attention_heads == 'same':
return tf.keras.layers.add([x1, x2])
else:
return x2
def residual_attention(x1):
attention_heads = self.attention_heads
if self.feature_approach == "multilabel-attention":
attention_heads = self.nb_outputs
print(f"Attention heads {attention_heads}")
x2 = HierarchicalAttention(attention_heads)(x1)
x2 = tf.keras.layers.Dropout(self.dropout)(x2)
x2 = tf.keras.layers.LayerNormalization()(x2)
if attention_heads == 'same':
return tf.keras.layers.add([x1, x2])
else:
return x2