def __init__(self, config, **kwargs): super().__init__(**kwargs) self.dense = tf.keras.layers.Dense( config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense") self.dense_prediction = tf.keras.layers.Dense( 1, kernel_initializer=get_initializer(config.initializer_range), name="dense_prediction") self.config = config
def __init__(self, config, *inputs, **kwargs): super().__init__(*inputs, **kwargs) self.dense_0 = tf.keras.layers.Dense( config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="pooler_answer_class_dense_0" ) self.activation = tf.keras.layers.Activation('tanh') self.dense_1 = tf.keras.layers.Dense( 1, use_bias=False, kernel_initializer=get_initializer(config.initializer_range), name="pooler_answer_class_dense_1" )
def __init__(self, config, *inputs, **kwargs): super().__init__(*inputs, **kwargs) self.dense_0 = tf.keras.layers.Dense( config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="end_logit_pooler_dense_0" ) self.activation = tf.keras.layers.Activation('tanh') # nn.Tanh() self.LayerNorm = tf.keras.layers.LayerNormalization(axis=-1, epsilon=config.layer_norm_eps, name="end_logit_pooler_LayerNorm") self.dense_1 = tf.keras.layers.Dense( 1, kernel_initializer=get_initializer(config.initializer_range), name="end_logit_pooler_dense_1" )
def __init__(self, config, **kwargs): super().__init__(config, **kwargs) self.electra = TFElectraMainLayer(config, name="electra") self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense( config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="classifier")
def build(self, input_shape): """Build shared word embedding layer """ with tf.name_scope("word_embeddings"): # Create and initialize weights. The random normal initializer was chosen # arbitrarily, and works well. self.word_embeddings = self.add_weight( "weight", shape=[self.vocab_size, self.embedding_size], initializer=get_initializer(self.initializer_range), ) super().build(input_shape)
def __init__(self, config, **kwargs): super().__init__(**kwargs) self.vocab_size = config.vocab_size self.embedding_size = config.embedding_size self.initializer_range = config.initializer_range self.position_embeddings = tf.keras.layers.Embedding( config.max_position_embeddings, config.embedding_size, embeddings_initializer=get_initializer(self.initializer_range), name="position_embeddings", ) self.token_type_embeddings = tf.keras.layers.Embedding( config.type_vocab_size, config.embedding_size, embeddings_initializer=get_initializer(self.initializer_range), name="token_type_embeddings", ) # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load # any TensorFlow checkpoint file self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm") self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
def __init__(self, config, shared_embeddings=False, input_embeddings=None, **kwargs): super().__init__(config, **kwargs) if shared_embeddings and input_embeddings is not None: self.embeddings = input_embeddings else: self.embeddings = TFElectraEmbeddings(config, name="embeddings") if config.embedding_size != config.hidden_size: self.embeddings_project = tf.keras.layers.Dense( config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="embeddings_project") self.encoder = TFBertEncoder(config, name="encoder") self.config = config
def __init__(self, config, args): super().__init__(config, args) self.start_n_top = args.beam_size # config.start_n_top self.end_n_top = args.beam_size # config.end_n_top self.joint_head = args.joint_head self.v2 = args.version_2_with_negative self.electra = TFElectraMainLayer(config, name="electra") self.num_hidden_layers = config.num_hidden_layers self.amp = config.amp ##old head if not self.joint_head: self.qa_outputs = tf.keras.layers.Dense( 2, kernel_initializer=get_initializer(config.initializer_range), name="qa_outputs") else: self.start_logits = TFPoolerStartLogits(config, name='start_logits') self.end_logits = TFPoolerEndLogits(config, name='end_logits') if self.v2: self.answer_class = TFPoolerAnswerClass(config, name='answer_class')
def __init__(self, config, *inputs, **kwargs): super().__init__(*inputs, **kwargs) self.dense = tf.keras.layers.Dense( 1, kernel_initializer=get_initializer(config.initializer_range), name="start_logit_pooler_dense" )
def __init__(self, config, **kwargs): super().__init__(**kwargs) self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm") self.dense = tf.keras.layers.Dense( config.embedding_size, kernel_initializer=get_initializer(config.initializer_range), name="dense")