def __init__(self,
units,
num_head,
activation=None,
use_bias=False,
attention_dropout=0.0,
kernel_initializer='glorot_normal',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(RelativePartialMultiHeadSelfAttention, self).__init__(**kwargs)
self.supports_masking = True
self.units = units
self.num_head = num_head
self.units_head = units // num_head
self.activation = activation
self.activation = activations.get(activation)
self.use_bias = use_bias
self.attention_dropout = attention_dropout
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.kernel, self.bias = None, None
self.att_drop_layer = None
def __init__(self,
units,
bias_initializer='zeros',
bias_regularizer=None,
bias_constraint=None,
**kwargs):
super(RelativeBias, self).__init__(**kwargs)
self.supports_masking = True
self.units = units
self.bias_initializer = initializers.get(bias_initializer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.bias_constraint = constraints.get(bias_constraint)
self.bias_context, self.bias_relative = None, None
def __init__(self,
units,
initializer='uniform',
regularizer=None,
constraint=None,
**kwargs):
super(MaskEmbedding, self).__init__(**kwargs)
self.supports_masking = True
self.units = units
self.initializer = initializers.get(initializer)
self.regularizer = regularizers.get(regularizer)
self.constraint = constraints.get(constraint)
self.embeddings = None