def get_output_dim(self) -> int: unidirectional_dim = int(self._input_dim / 2) forward_combined_dim = util.get_combined_dim(self._forward_combination, [unidirectional_dim, unidirectional_dim]) backward_combined_dim = util.get_combined_dim(self._backward_combination, [unidirectional_dim, unidirectional_dim]) if self._span_width_embedding is not None: return forward_combined_dim + backward_combined_dim + \ self._span_width_embedding.get_output_dim() return forward_combined_dim + backward_combined_dim
def get_output_dim(self) -> int: unidirectional_dim = int(self._input_dim / 2) forward_combined_dim = util.get_combined_dim( self._forward_combination, [unidirectional_dim, unidirectional_dim] ) backward_combined_dim = util.get_combined_dim( self._backward_combination, [unidirectional_dim, unidirectional_dim] ) if self._span_width_embedding is not None: return ( forward_combined_dim + backward_combined_dim + self._span_width_embedding.get_output_dim() ) return forward_combined_dim + backward_combined_dim
def __init__(self, input_dim: int, projection_dim: int = None, similarity_function: SimilarityFunction = DotProductSimilarity(), num_attention_heads: int = 1, combination: str = '1,2', output_dim: int = None) -> None: super(IntraSentenceAttentionEncoder, self).__init__() self._input_dim = input_dim if projection_dim: self._projection = torch.nn.Linear(input_dim, projection_dim) else: self._projection = lambda x: x projection_dim = input_dim self._matrix_attention = LegacyMatrixAttention(similarity_function) self._num_attention_heads = num_attention_heads if isinstance(similarity_function, MultiHeadedSimilarity): if num_attention_heads == 1: raise ConfigurationError("Similarity function has multiple heads but encoder doesn't") if num_attention_heads != similarity_function.num_heads: raise ConfigurationError("Number of heads don't match between similarity function " "and encoder: %d, %d" % (num_attention_heads, similarity_function.num_heads)) elif num_attention_heads > 1: raise ConfigurationError("Encoder has multiple heads but similarity function doesn't") self._combination = combination combined_dim = util.get_combined_dim(combination, [input_dim, projection_dim]) if output_dim: self._output_projection = Linear(combined_dim, output_dim) self._output_dim = output_dim else: self._output_projection = lambda x: x self._output_dim = combined_dim
def __init__(self, tensor_1_dim: int, tensor_2_dim: int, combination: str = 'x,y', activation: Activation = Activation.by_name('linear')()) -> None: super(LinearSimilarity, self).__init__() self._combination = combination combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self._weight_vector = Parameter(torch.Tensor(combined_dim)) self._bias = Parameter(torch.Tensor(1)) self._activation = activation self.reset_parameters()
def __init__(self, tensor_1_dim: int, tensor_2_dim: int, combination: str = 'x,y', activation: Activation = Activation.by_name('linear')()) -> None: super(MyLinearSimilarity, self).__init__() self._combination = combination self._part = len(self._combination.split(',')) combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self._weight_vector = Parameter(torch.Tensor(combined_dim)) self._bias = Parameter(torch.Tensor(1)) self._activation = activation self.reset_parameters()
def __init__(self, tensor_1_dim: int, tensor_2_dim: int, combination: str = 'x,y', activation: Activation = None, normalize: bool = True) -> None: super().__init__(normalize) self._combination = combination combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self._weight_vector = Parameter(torch.Tensor(combined_dim)) self._bias = Parameter(torch.Tensor(1)) self._activation = activation or Activation.by_name('linear')() self.reset_parameters()
def __init__(self, tensor_1_dim, tensor_2_dim, combination=u'x,y', activation=None): super(LinearMatrixAttention, self).__init__() self._combination = combination combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self._weight_vector = Parameter(torch.Tensor(combined_dim)) self._bias = Parameter(torch.Tensor(1)) self._activation = activation or Activation.by_name(u'linear')() self.reset_parameters()
def __init__( self, tensor_1_dim: int, tensor_2_dim: int, combination: str = "x,y", activation: Activation = None, normalize: bool = True, ) -> None: super().__init__(normalize) self._combination = combination combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self._weight_vector = Parameter(torch.Tensor(combined_dim)) self._bias = Parameter(torch.Tensor(1)) self._activation = activation or Activation.by_name("linear")() self.reset_parameters()
def test_combine_tensors_and_multiply_with_batch_size_one(self): seq_len_1 = 10 seq_len_2 = 5 embedding_dim = 8 combination = "x,y,x*y" t1 = torch.randn(1, seq_len_1, embedding_dim) t2 = torch.randn(1, seq_len_2, embedding_dim) combined_dim = util.get_combined_dim(combination, [embedding_dim, embedding_dim]) weight = torch.Tensor(combined_dim) result = util.combine_tensors_and_multiply( combination, [t1.unsqueeze(2), t2.unsqueeze(1)], weight) assert_almost_equal(result.size(), [1, seq_len_1, seq_len_2])
def __init__( self, tensor_1_dim: int, tensor_2_dim: int, attend_dim: int, combination: str = 'x,y', activation: Activation = Activation.by_name('linear')() ) -> None: super(LinearVSimilarity, self).__init__() self._combination = combination combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self._weight_vector = Parameter(torch.Tensor(combined_dim, attend_dim)) self._bias = Parameter(torch.Tensor(attend_dim)) self._activation = activation self._V = Parameter(torch.rand(attend_dim)) self.reset_parameters()
def test_combine_tensors_and_multiply_with_batch_size_one_and_seq_len_one(self): seq_len_1 = 10 seq_len_2 = 1 embedding_dim = 8 combination = "x,y,x*y" t1 = torch.randn(1, seq_len_1, embedding_dim) t2 = torch.randn(1, seq_len_2, embedding_dim) combined_dim = util.get_combined_dim(combination, [embedding_dim, embedding_dim]) weight = torch.Tensor(combined_dim) result = util.combine_tensors_and_multiply(combination, [t1.unsqueeze(2), t2.unsqueeze(1)], weight) assert_almost_equal( result.size(), [1, seq_len_1, seq_len_2] )
def __init__(self, tensor_1_dim: int, tensor_2_dim: int, combination: str = 'x,y', activation: Activation = None, prior = None) -> None: super(LinearSimilarityVB, self).__init__() self._combination = combination combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim]) self.posterior_mean = False # Flag to know if we sample from the posterior mean or we actually sample ## If no prior is specified we just create it ourselves if (type(prior) == type (None)): prior = Vil.Prior(0.5, np.log(0.1),np.log(0.5)) size_combination = int(torch.Tensor(combined_dim).size()[0]) # print ("Combination size: ", size_combination) prior = prior.get_standarized_Prior(size_combination) self.prior = prior """ Mean and rhos of the parameters """ self.mu_weight = Parameter(torch.Tensor(combined_dim))# , requires_grad=True self.rho_weight = Parameter(torch.Tensor(combined_dim)) self.rho_bias = Parameter(torch.Tensor(1)) self.mu_bias = Parameter(torch.Tensor(1)) """ The sampled weights """ self.weight = torch.Tensor(combined_dim) self.bias = torch.Tensor(1) self._activation = activation or Activation.by_name('linear')() ## Initialize the Variational variables self.reset_parameters()
def get_output_dim(self) -> int: combined_dim = util.get_combined_dim(self._combination, [self._input_dim, self._input_dim]) if self._span_width_embedding is not None: return combined_dim + self._span_width_embedding.get_output_dim() return combined_dim