def embedding_matrix( vocab, embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=False, embedding_initializer=None, ): vocab_size = len(vocab) if representation == 'dense': if pretrained_embeddings is not None and pretrained_embeddings is not False: embeddings_matrix = load_pretrained_embeddings( pretrained_embeddings, vocab) if embeddings_matrix.shape[-1] != embedding_size: raise ValueError( 'The size of the pretrained embeddings is {}, ' 'but the specified embedding_size is {}. ' 'Please change the embedding_size accordingly.'.format( embeddings_matrix.shape[-1], embedding_size)) embedding_initializer_obj = tf.constant(embeddings_matrix, dtype=tf.float32) else: if vocab_size < embedding_size and not force_embedding_size: logger.info( ' embedding_size ({}) is greater than vocab_size ({}). ' 'Setting embedding size to be equal to vocab_size.'.format( embedding_size, vocab_size)) embedding_size = vocab_size if embedding_initializer is not None: embedding_initializer_obj_ref = get_initializer( embedding_initializer) else: embedding_initializer_obj_ref = get_initializer({ TYPE: 'uniform', 'minval': -1.0, 'maxval': 1.0 }) embedding_initializer_obj = embedding_initializer_obj_ref( [vocab_size, embedding_size]) embeddings = tf.Variable(embedding_initializer_obj, trainable=embeddings_trainable, name='embeddings') elif representation == 'sparse': embedding_size = vocab_size embeddings = tf.Variable( get_initializer('identity')([vocab_size, embedding_size]), trainable=False, name='embeddings') else: raise Exception('Embedding representation {} not supported.'.format( representation)) return embeddings, embedding_size
def embedding_matrix( vocab: List[str], embedding_size: int, representation: str = "dense", embeddings_trainable: bool = True, pretrained_embeddings: Optional[str] = None, force_embedding_size: bool = False, embedding_initializer: Optional[Union[str, Dict]] = None, ) -> Tuple[nn.Module, int]: """Returns initialized torch.nn.Embedding module and embedding size.""" vocab_size = len(vocab) if representation == "dense": if pretrained_embeddings: embeddings_matrix = load_pretrained_embeddings(pretrained_embeddings, vocab) if embeddings_matrix.shape[-1] != embedding_size: if not force_embedding_size: embedding_size = embeddings_matrix.shape[-1] logger.info(f"Setting embedding size to be equal to {embeddings_matrix.shape[-1]}.") else: raise ValueError( f"The size of the pretrained embeddings is " f"{embeddings_matrix.shape[-1]}, but the specified " f"embedding_size is {embedding_size}. Please change " f"the embedding_size accordingly." ) embedding_initializer_obj = torch.tensor(embeddings_matrix, dtype=torch.float32) else: if vocab_size < embedding_size and not force_embedding_size: logger.info( f" embedding_size ({embedding_size}) is greater than " f"vocab_size ({vocab_size}). Setting embedding size to be " f"equal to vocab_size." ) embedding_size = vocab_size if embedding_initializer is not None: embedding_initializer_obj_ref = get_initializer(embedding_initializer) else: embedding_initializer_obj_ref = get_initializer({TYPE: "uniform", "a": -1.0, "b": 1.0}) embedding_initializer_obj = embedding_initializer_obj_ref([vocab_size, embedding_size]) embeddings = embedding_initializer_obj elif representation == "sparse": embedding_size = vocab_size embeddings = get_initializer("identity")([vocab_size, embedding_size]) embeddings.requires_grad = False else: raise Exception(f"Embedding representation {representation} not supported.") embeddings = nn.Embedding.from_pretrained(embeddings, freeze=not embeddings_trainable) return embeddings, embedding_size
def test_get_initializer(): """Currently only checks for when the parameters are None.""" tensor_size = (2, 3) # Test for when the parameters are None torch.random.manual_seed(0) initialized_tensor = get_initializer(None)(*tensor_size, device=DEVICE) # Check that the tensor using the expected initialization and the same seed is identical default_initializer = nn.init.xavier_uniform_ torch.random.manual_seed(0) default_tensor = default_initializer( torch.empty(*tensor_size, device=DEVICE)) assert torch.equal(initialized_tensor, default_tensor)
def __init__( self, embedding_size=10, embeddings_on_cpu=False, should_softmax=False, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, **kwargs): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocaularyb_size, embedding_size)` for `dense` representations and exacly `vocaularyb_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embedings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memroy and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: Boolean :param initializer: the initializer to use. If `None`, the default initialized of each variable is used (`glorot_uniform` in most cases). Options are: `constant`, `identity`, `zeros`, `ones`, `orthogonal`, `normal`, `uniform`, `truncated_normal`, `variance_scaling`, `glorot_normal`, `glorot_uniform`, `xavier_normal`, `xavier_uniform`, `he_normal`, `he_uniform`, `lecun_normal`, `lecun_uniform`. Alternatively it is possible to specify a dictionary with a key `type` that identifies the type of initialzier and other keys for its parameters, e.g. `{type: normal, mean: 0, stddev: 0}`. To know the parameters of each initializer, please refer to TensorFlow's documentation. :type initializer: str :param regularize: if `True` the embedding wieghts are added to the set of weights that get reularized by a regularization loss (if the `regularization_lambda` in `training` is greater than 0). :type regularize: Boolean """ super(H3WeightedSum, self).__init__() logger.debug(' {}'.format(self.name)) self.should_softmax = should_softmax self.reduce_sequence = SequenceReducer(reduce_mode='sum') self.h3_embed = H3Embed( embedding_size, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, weights_initializer=weights_initializer, bias_initializer=bias_initializer, weights_regularizer=weights_regularizer, bias_regularizer=bias_regularizer, activity_regularizer=activity_regularizer, # weights_constraint=weights_constraint, # bias_constraint=bias_constraint, reduce_output=None) self.aggregation_weights = tf.Variable( get_initializer(weights_initializer)([19, 1])) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, embedding_size: int = 10, embeddings_on_cpu: bool = False, should_softmax: bool = False, fc_layers: Optional[List] = None, num_fc_layers: int = 0, output_size: int = 10, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: Optional[str] = None, norm_params: Dict = None, activation: str = "relu", dropout: float = 0, **kwargs, ): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocabulary_size, embedding_size)` for `dense` representations and exactly `vocabulary_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: Boolean """ super().__init__() logger.debug(f" {self.name}") self.should_softmax = should_softmax self.sum_sequence_reducer = SequenceReducer(reduce_mode="sum") self.h3_embed = H3Embed( embedding_size, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, weights_initializer=weights_initializer, bias_initializer=bias_initializer, reduce_output="None", ) self.register_buffer( "aggregation_weights", torch.Tensor( get_initializer(weights_initializer)([H3_INPUT_SIZE, 1]))) logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=self.h3_embed.output_shape[0], layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )