def create_labse_model(bert_tfhub_module: Text, bert_config: configs.BertConfig, normalize: bool) -> tf.keras.Model: """Creates a LaBSE keras core model from BERT configuration. Args: bert_tfhub_module: The bert tfhub module path. The LaBSE will be built upon the tfhub module if it is not empty. bert_config: A `BertConfig` to create the core model. Used if bert_tfhub_module is empty. normalize: Parameter of DualEncoder model, normalize the embedding ( pooled_output) if set to True. Returns: A keras model. """ if bert_tfhub_module: encoder_network = utils.get_encoder_from_hub(bert_tfhub_module) else: encoder_network = bert_models.get_transformer_encoder( bert_config, sequence_length=None) labse_model = models.DualEncoder(network=encoder_network, max_seq_length=None, normalize=normalize, output="predictions") return labse_model, encoder_network # pytype: disable=bad-return-type # typed-keras
def build_model(self): if self._hub_module: encoder_from_hub = utils.get_encoder_from_hub(self._hub_module) return bert.instantiate_bertpretrainer_from_cfg( self.task_config.model, encoder_network=encoder_from_hub) else: return bert.instantiate_bertpretrainer_from_cfg(self.task_config.model)
def build_model(self): if self._hub_module: encoder_network = utils.get_encoder_from_hub(self._hub_module) else: encoder_network = encoders.instantiate_encoder_from_cfg( self.task_config.network) return models.BertSpanLabeler( network=encoder_network, initializer=tf.keras.initializers.TruncatedNormal( stddev=self.task_config.network.initializer_range))
def build_model(self): if self._hub_module: encoder_network = utils.get_encoder_from_hub(self._hub_module) else: encoder_network = encoders.instantiate_encoder_from_cfg( self.task_config.model.encoder) # Currently, we only supports bert-style question answering finetuning. return models.BertSpanLabeler( network=encoder_network, initializer=tf.keras.initializers.TruncatedNormal( stddev=self.task_config.model.encoder.initializer_range))
def build_model(self): if self._hub_module: encoder_network = utils.get_encoder_from_hub(self._hub_module) else: encoder_network = encoders.build_encoder(self.task_config.model.encoder) encoder_cfg = self.task_config.model.encoder.get() # Currently, we only support bert-style sentence prediction finetuning. return models.BertClassifier( network=encoder_network, num_classes=self.task_config.model.num_classes, initializer=tf.keras.initializers.TruncatedNormal( stddev=encoder_cfg.initializer_range), use_encoder_pooler=self.task_config.model.use_encoder_pooler)
def build_model(self): if self._hub_module: encoder_network = utils.get_encoder_from_hub(self._hub_module) else: encoder_network = encoders.build_encoder(self.task_config.model.encoder) return models.BertTokenClassifier( network=encoder_network, num_classes=len(self.task_config.class_names), initializer=tf.keras.initializers.TruncatedNormal( stddev=self.task_config.model.head_initializer_range), dropout_rate=self.task_config.model.head_dropout, output='logits')
def build_model(self): if self._hub_module: encoder_network = utils.get_encoder_from_hub(self._hub_module) else: encoder_network = encoders.instantiate_encoder_from_cfg( self.task_config.model) return models.BertTokenClassifier( network=encoder_network, num_classes=self.task_config.num_classes, initializer=tf.keras.initializers.TruncatedNormal( stddev=self.task_config.model.initializer_range), dropout_rate=self.task_config.model.dropout_rate, output='logits')
def build_model(self): if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError('At most one of `hub_module_url` and ' '`init_checkpoint` can be specified.') if self.task_config.hub_module_url: encoder_network = utils.get_encoder_from_hub( self.task_config.hub_module_url) else: encoder_network = encoders.build_encoder(self.task_config.model.encoder) encoder_cfg = self.task_config.model.encoder.get() return models.BertSpanLabeler( network=encoder_network, initializer=tf.keras.initializers.TruncatedNormal( stddev=encoder_cfg.initializer_range))
def build_model(self): """Interface to build model. Refer to base_task.Task.build_model.""" if self._hub_module: encoder_network = utils.get_encoder_from_hub(self._hub_module) else: encoder_network = encoders.build_encoder(self.task_config.model.encoder) # Currently, we only supports bert-style dual encoder. return models.DualEncoder( network=encoder_network, max_seq_length=self.task_config.model.max_sequence_length, normalize=self.task_config.model.normalize, logit_scale=self.task_config.model.logit_scale, logit_margin=self.task_config.model.logit_margin, output='logits')
def build_model(self) -> tf.keras.Model: if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError('At most one of `hub_module_url` and ' '`init_checkpoint` can be specified.') if self.task_config.hub_module_url: encoder_network = utils.get_encoder_from_hub( self.task_config.hub_module_url) else: encoder_network = encoders.build_encoder( self.task_config.model.encoder) return models.BertClassifier( network=encoder_network, num_classes=len(self.task_config.class_names), initializer=tf.keras.initializers.TruncatedNormal( stddev=self.task_config.model.head_initializer_range), dropout_rate=self.task_config.model.head_dropout)
def build_model(self): if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError('At most one of `hub_module_url` and ' '`init_checkpoint` can be specified.') if self.task_config.hub_module_url: encoder_network = utils.get_encoder_from_hub( self.task_config.hub_module_url) else: encoder_network = encoders.build_encoder( self.task_config.model.encoder) encoder_cfg = self.task_config.model.encoder.get() # Currently, we only support bert-style sentence prediction finetuning. return models.BertClassifier( network=encoder_network, num_classes=self.task_config.model.num_classes, initializer=tf.keras.initializers.TruncatedNormal( stddev=encoder_cfg.initializer_range), use_encoder_pooler=self.task_config.model.use_encoder_pooler)
def build_model(self): if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError('At most one of `hub_module_url` and ' '`init_checkpoint` can be specified.') if self.task_config.hub_module_url: hub_module = hub.load(self.task_config.hub_module_url) else: hub_module = None if hub_module: encoder_network = utils.get_encoder_from_hub(hub_module) else: encoder_network = encoders.build_encoder( self.task_config.model.encoder) encoder_cfg = self.task_config.model.encoder.get() # Currently, we only supports bert-style question answering finetuning. return models.BertSpanLabeler( network=encoder_network, initializer=tf.keras.initializers.TruncatedNormal( stddev=encoder_cfg.initializer_range))
def build_model(self): """Interface to build model. Refer to base_task.Task.build_model.""" if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError('At most one of `hub_module_url` and ' '`init_checkpoint` can be specified.') if self.task_config.hub_module_url: encoder_network = utils.get_encoder_from_hub( self.task_config.hub_module_url) else: encoder_network = encoders.build_encoder( self.task_config.model.encoder) # Currently, we only supports bert-style dual encoder. return models.DualEncoder( network=encoder_network, max_seq_length=self.task_config.model.max_sequence_length, normalize=self.task_config.model.normalize, logit_scale=self.task_config.model.logit_scale, logit_margin=self.task_config.model.logit_margin, output='logits')
def build_model(self): if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError('At most one of `hub_module_url` and ' '`init_checkpoint` can be specified.') if self.task_config.hub_module_url: encoder_network = utils.get_encoder_from_hub( self.task_config.hub_module_url) else: encoder_network = encoders.build_encoder(self.task_config.model.encoder) encoder_cfg = self.task_config.model.encoder.get() if self.task_config.model.encoder.type == 'xlnet': return models.XLNetClassifier( network=encoder_network, num_classes=self.task_config.model.num_classes, initializer=tf.keras.initializers.RandomNormal( stddev=encoder_cfg.initializer_range)) else: return models.BertClassifier( network=encoder_network, num_classes=self.task_config.model.num_classes, initializer=tf.keras.initializers.TruncatedNormal( stddev=encoder_cfg.initializer_range), use_encoder_pooler=self.task_config.model.use_encoder_pooler)
def build_model(self, train_last_layer_only=False): """Modified version of official.nlp.tasks.tagging.build_model Allows to freeze the underlying bert encoder, such that only the dense layer is trained. """ if self.task_config.hub_module_url and self.task_config.init_checkpoint: raise ValueError("At most one of `hub_module_url` and " "`init_checkpoint` can be specified.") if self.task_config.hub_module_url: encoder_network = utils.get_encoder_from_hub( self.task_config.hub_module_url) else: encoder_network = encoders.build_encoder( self.task_config.model.encoder) encoder_network.trainable = not train_last_layer_only return models.BertTokenClassifier( network=encoder_network, num_classes=len(self.task_config.class_names), initializer=tf.keras.initializers.TruncatedNormal( stddev=self.task_config.model.head_initializer_range), dropout_rate=self.task_config.model.head_dropout, output="logits")