def get_output_dim(self) -> int:
unidirectional_dim = int(self._input_dim / 2)
forward_combined_dim = util.get_combined_dim(self._forward_combination,
[unidirectional_dim, unidirectional_dim])
backward_combined_dim = util.get_combined_dim(self._backward_combination,
[unidirectional_dim, unidirectional_dim])
if self._span_width_embedding is not None:
return forward_combined_dim + backward_combined_dim + \
self._span_width_embedding.get_output_dim()
return forward_combined_dim + backward_combined_dim
def get_output_dim(self) -> int:
unidirectional_dim = int(self._input_dim / 2)
forward_combined_dim = util.get_combined_dim(
self._forward_combination, [unidirectional_dim, unidirectional_dim]
)
backward_combined_dim = util.get_combined_dim(
self._backward_combination, [unidirectional_dim, unidirectional_dim]
)
if self._span_width_embedding is not None:
return (
forward_combined_dim
+ backward_combined_dim
+ self._span_width_embedding.get_output_dim()
)
return forward_combined_dim + backward_combined_dim
def __init__(self,
input_dim: int,
projection_dim: int = None,
similarity_function: SimilarityFunction = DotProductSimilarity(),
num_attention_heads: int = 1,
combination: str = '1,2',
output_dim: int = None) -> None:
super(IntraSentenceAttentionEncoder, self).__init__()
self._input_dim = input_dim
if projection_dim:
self._projection = torch.nn.Linear(input_dim, projection_dim)
else:
self._projection = lambda x: x
projection_dim = input_dim
self._matrix_attention = LegacyMatrixAttention(similarity_function)
self._num_attention_heads = num_attention_heads
if isinstance(similarity_function, MultiHeadedSimilarity):
if num_attention_heads == 1:
raise ConfigurationError("Similarity function has multiple heads but encoder doesn't")
if num_attention_heads != similarity_function.num_heads:
raise ConfigurationError("Number of heads don't match between similarity function "
"and encoder: %d, %d" % (num_attention_heads,
similarity_function.num_heads))
elif num_attention_heads > 1:
raise ConfigurationError("Encoder has multiple heads but similarity function doesn't")
self._combination = combination
combined_dim = util.get_combined_dim(combination, [input_dim, projection_dim])
if output_dim:
self._output_projection = Linear(combined_dim, output_dim)
self._output_dim = output_dim
else:
self._output_projection = lambda x: x
self._output_dim = combined_dim
def __init__(self,
tensor_1_dim: int,
tensor_2_dim: int,
combination: str = 'x,y',
activation: Activation = Activation.by_name('linear')()) -> None:
super(LinearSimilarity, self).__init__()
self._combination = combination
combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim])
self._weight_vector = Parameter(torch.Tensor(combined_dim))
self._bias = Parameter(torch.Tensor(1))
self._activation = activation
self.reset_parameters()
def __init__(self,
tensor_1_dim: int,
tensor_2_dim: int,
combination: str = 'x,y',
activation: Activation = Activation.by_name('linear')()) -> None:
super(MyLinearSimilarity, self).__init__()
self._combination = combination
self._part = len(self._combination.split(','))
combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim])
self._weight_vector = Parameter(torch.Tensor(combined_dim))
self._bias = Parameter(torch.Tensor(1))
self._activation = activation
self.reset_parameters()
def __init__(self,
tensor_1_dim: int,
tensor_2_dim: int,
combination: str = 'x,y',
activation: Activation = None,
normalize: bool = True) -> None:
super().__init__(normalize)
self._combination = combination
combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim])
self._weight_vector = Parameter(torch.Tensor(combined_dim))
self._bias = Parameter(torch.Tensor(1))
self._activation = activation or Activation.by_name('linear')()
self.reset_parameters()
def __init__(self,
tensor_1_dim,
tensor_2_dim,
combination=u'x,y',
activation=None):
super(LinearMatrixAttention, self).__init__()
self._combination = combination
combined_dim = util.get_combined_dim(combination,
[tensor_1_dim, tensor_2_dim])
self._weight_vector = Parameter(torch.Tensor(combined_dim))
self._bias = Parameter(torch.Tensor(1))
self._activation = activation or Activation.by_name(u'linear')()
self.reset_parameters()
def __init__(
self,
tensor_1_dim: int,
tensor_2_dim: int,
combination: str = "x,y",
activation: Activation = None,
normalize: bool = True,
) -> None:
super().__init__(normalize)
self._combination = combination
combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim])
self._weight_vector = Parameter(torch.Tensor(combined_dim))
self._bias = Parameter(torch.Tensor(1))
self._activation = activation or Activation.by_name("linear")()
self.reset_parameters()
def test_combine_tensors_and_multiply_with_batch_size_one(self):
seq_len_1 = 10
seq_len_2 = 5
embedding_dim = 8
combination = "x,y,x*y"
t1 = torch.randn(1, seq_len_1, embedding_dim)
t2 = torch.randn(1, seq_len_2, embedding_dim)
combined_dim = util.get_combined_dim(combination,
[embedding_dim, embedding_dim])
weight = torch.Tensor(combined_dim)
result = util.combine_tensors_and_multiply(
combination, [t1.unsqueeze(2), t2.unsqueeze(1)], weight)
assert_almost_equal(result.size(), [1, seq_len_1, seq_len_2])
def __init__(
self,
tensor_1_dim: int,
tensor_2_dim: int,
attend_dim: int,
combination: str = 'x,y',
activation: Activation = Activation.by_name('linear')()
) -> None:
super(LinearVSimilarity, self).__init__()
self._combination = combination
combined_dim = util.get_combined_dim(combination,
[tensor_1_dim, tensor_2_dim])
self._weight_vector = Parameter(torch.Tensor(combined_dim, attend_dim))
self._bias = Parameter(torch.Tensor(attend_dim))
self._activation = activation
self._V = Parameter(torch.rand(attend_dim))
self.reset_parameters()
def test_combine_tensors_and_multiply_with_batch_size_one_and_seq_len_one(self):
seq_len_1 = 10
seq_len_2 = 1
embedding_dim = 8
combination = "x,y,x*y"
t1 = torch.randn(1, seq_len_1, embedding_dim)
t2 = torch.randn(1, seq_len_2, embedding_dim)
combined_dim = util.get_combined_dim(combination, [embedding_dim, embedding_dim])
weight = torch.Tensor(combined_dim)
result = util.combine_tensors_and_multiply(combination, [t1.unsqueeze(2), t2.unsqueeze(1)], weight)
assert_almost_equal(
result.size(),
[1, seq_len_1, seq_len_2]
)
def __init__(self,
tensor_1_dim: int,
tensor_2_dim: int,
combination: str = 'x,y',
activation: Activation = None,
prior = None) -> None:
super(LinearSimilarityVB, self).__init__()
self._combination = combination
combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim])
self.posterior_mean = False # Flag to know if we sample from the posterior mean or we actually sample
## If no prior is specified we just create it ourselves
if (type(prior) == type (None)):
prior = Vil.Prior(0.5, np.log(0.1),np.log(0.5))
size_combination = int(torch.Tensor(combined_dim).size()[0])
# print ("Combination size: ", size_combination)
prior = prior.get_standarized_Prior(size_combination)
self.prior = prior
"""
Mean and rhos of the parameters
"""
self.mu_weight = Parameter(torch.Tensor(combined_dim))# , requires_grad=True
self.rho_weight = Parameter(torch.Tensor(combined_dim))
self.rho_bias = Parameter(torch.Tensor(1))
self.mu_bias = Parameter(torch.Tensor(1))
"""
The sampled weights
"""
self.weight = torch.Tensor(combined_dim)
self.bias = torch.Tensor(1)
self._activation = activation or Activation.by_name('linear')()
## Initialize the Variational variables
self.reset_parameters()
def get_output_dim(self) -> int:
combined_dim = util.get_combined_dim(self._combination, [self._input_dim, self._input_dim])
if self._span_width_embedding is not None:
return combined_dim + self._span_width_embedding.get_output_dim()
return combined_dim
