def test_network_invocation(self): config = bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig(vocab_size=10, num_layers=1)) _ = bert.instantiate_bertpretrainer_from_cfg(config) # Invokes with classification heads. config = bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig(vocab_size=10, num_layers=1), cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=2, name="next_sentence") ]) _ = bert.instantiate_bertpretrainer_from_cfg(config) with self.assertRaises(ValueError): config = bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig(vocab_size=10, num_layers=1), cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=2, name="next_sentence"), bert.ClsHeadConfig(inner_dim=10, num_classes=2, name="next_sentence") ]) _ = bert.instantiate_bertpretrainer_from_cfg(config)
class ELECTRAPretrainerConfig(base_config.Config): """ELECTRA pretrainer configuration.""" num_masked_tokens: int = 76 sequence_length: int = 512 num_classes: int = 2 discriminator_loss_weight: float = 50.0 generator_encoder: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) discriminator_encoder: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) cls_heads: List[bert.ClsHeadConfig] = dataclasses.field(default_factory=list)
def setUp(self): super(QuestionAnsweringTaskTest, self).setUp() self._encoder_config = encoders.TransformerEncoderConfig( vocab_size=30522, num_layers=1) self._train_data_config = bert.QADataConfig(input_path="dummy", seq_length=128, global_batch_size=1)
def test_task(self): config = masked_lm.MaskedLMConfig( init_checkpoint=self.get_temp_dir(), model=bert.BertPretrainerConfig( encoders.TransformerEncoderConfig(vocab_size=30522, num_layers=1), num_masked_tokens=20, cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=2, name="next_sentence") ]), train_data=pretrain_dataloader.BertPretrainDataConfig( input_path="dummy", max_predictions_per_seq=20, seq_length=128, global_batch_size=1)) task = masked_lm.MaskedLMTask(config) model = task.build_model() metrics = task.build_metrics() dataset = task.build_inputs(config.train_data) iterator = iter(dataset) optimizer = tf.keras.optimizers.SGD(lr=0.1) task.train_step(next(iterator), model, optimizer, metrics=metrics) task.validation_step(next(iterator), model, metrics=metrics) # Saves a checkpoint. ckpt = tf.train.Checkpoint(model=model, **model.checkpoint_items) ckpt.save(config.init_checkpoint) task.initialize(model)
def test_task(self): config = sentence_prediction.SentencePredictionConfig( init_checkpoint=self.get_temp_dir(), model=self.get_model_config(2), train_data=self._train_data_config) task = sentence_prediction.SentencePredictionTask(config) model = task.build_model() metrics = task.build_metrics() dataset = task.build_inputs(config.train_data) iterator = iter(dataset) optimizer = tf.keras.optimizers.SGD(lr=0.1) task.train_step(next(iterator), model, optimizer, metrics=metrics) task.validation_step(next(iterator), model, metrics=metrics) # Saves a checkpoint. pretrain_cfg = bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig( vocab_size=30522, num_layers=1), cls_heads=[ bert.ClsHeadConfig( inner_dim=10, num_classes=3, name="next_sentence") ]) pretrain_model = bert.instantiate_pretrainer_from_cfg(pretrain_cfg) ckpt = tf.train.Checkpoint( model=pretrain_model, **pretrain_model.checkpoint_items) ckpt.save(config.init_checkpoint) task.initialize(model)
class QuestionAnsweringConfig(cfg.TaskConfig): """The model config.""" # At most one of `init_checkpoint` and `hub_module_url` can be specified. init_checkpoint: str = '' hub_module_url: str = '' network: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) train_data: cfg.DataConfig = cfg.DataConfig() validation_data: cfg.DataConfig = cfg.DataConfig()
def get_model_config(self, num_classes): return bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig(vocab_size=30522, num_layers=1), num_masked_tokens=0, cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=num_classes, name="sentence_prediction") ])
def test_network_invocation(self): config = electra.ELECTRAPretrainerConfig( generator_encoder=encoders.TransformerEncoderConfig( vocab_size=10, num_layers=1), discriminator_encoder=encoders.TransformerEncoderConfig( vocab_size=10, num_layers=2), ) _ = electra.instantiate_pretrainer_from_cfg(config) # Invokes with classification heads. config = electra.ELECTRAPretrainerConfig( generator_encoder=encoders.TransformerEncoderConfig( vocab_size=10, num_layers=1), discriminator_encoder=encoders.TransformerEncoderConfig( vocab_size=10, num_layers=2), cls_heads=[ bert.ClsHeadConfig( inner_dim=10, num_classes=2, name="next_sentence") ]) _ = electra.instantiate_pretrainer_from_cfg(config)
class QuestionAnsweringConfig(cfg.TaskConfig): """The model config.""" # At most one of `init_checkpoint` and `hub_module_url` can be specified. init_checkpoint: str = '' hub_module_url: str = '' n_best_size: int = 20 max_answer_length: int = 30 null_score_diff_threshold: float = 0.0 model: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) train_data: cfg.DataConfig = cfg.DataConfig() validation_data: cfg.DataConfig = cfg.DataConfig()
def test_checkpoint_items(self): config = bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig(vocab_size=10, num_layers=1), cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=2, name="next_sentence") ]) encoder = bert.instantiate_bertpretrainer_from_cfg(config) self.assertSameElements(encoder.checkpoint_items.keys(), ["encoder", "next_sentence.pooler_dense"])
def setUp(self): super(SentencePredictionTaskTest, self).setUp() self._network_config = bert.BertPretrainerConfig( encoder=encoders.TransformerEncoderConfig(vocab_size=30522, num_layers=1), num_masked_tokens=0, cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=3, name="sentence_prediction") ]) self._train_data_config = bert.SentencePredictionDataConfig( input_path="dummy", seq_length=128, global_batch_size=1)
class TaggingConfig(cfg.TaskConfig): """The model config.""" # At most one of `init_checkpoint` and `hub_module_url` can be specified. init_checkpoint: str = '' hub_module_url: str = '' model: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) num_classes: int = 0 # The ignored label id will not contribute to loss. # A word may be tokenized into multiple word_pieces tokens, and we usually # assign the real label id for the first token of the word, and # `ignore_label_id` for the remaining tokens. ignore_label_id: int = 0 train_data: cfg.DataConfig = cfg.DataConfig() validation_data: cfg.DataConfig = cfg.DataConfig()
class TaggingConfig(cfg.TaskConfig): """The model config.""" # At most one of `init_checkpoint` and `hub_module_url` can be specified. init_checkpoint: str = '' hub_module_url: str = '' model: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) # The number of real labels. Note that a word may be tokenized into # multiple word_pieces tokens, and we asssume the real label id (non-negative) # is assigned to the first token of the word, and a negative label id is # assigned to the remaining tokens. The negative label id will not contribute # to loss and metrics. num_classes: int = 0 train_data: cfg.DataConfig = cfg.DataConfig() validation_data: cfg.DataConfig = cfg.DataConfig()
class TaggingConfig(cfg.TaskConfig): """The model config.""" # At most one of `init_checkpoint` and `hub_module_url` can be specified. init_checkpoint: str = '' hub_module_url: str = '' model: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) # The real class names, the order of which should match real label id. # Note that a word may be tokenized into multiple word_pieces tokens, and # we asssume the real label id (non-negative) is assigned to the first token # of the word, and a negative label id is assigned to the remaining tokens. # The negative label id will not contribute to loss and metrics. class_names: Optional[List[str]] = None train_data: cfg.DataConfig = cfg.DataConfig() validation_data: cfg.DataConfig = cfg.DataConfig()
def test_task_with_hub(self): hub_module_url = self._export_bert_tfhub() config = sentence_prediction.SentencePredictionConfig( hub_module_url=hub_module_url, network=bert.BertPretrainerConfig( encoders.TransformerEncoderConfig(vocab_size=30522, num_layers=1), num_masked_tokens=0, cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=3, name="sentence_prediction") ]), train_data=bert.BertSentencePredictionDataConfig( input_path="dummy", seq_length=128, global_batch_size=10)) self._run_task(config)
def setUp(self): super(QuestionAnsweringTaskTest, self).setUp() self._encoder_config = encoders.TransformerEncoderConfig( vocab_size=30522, num_layers=1) self._train_data_config = bert.QADataConfig(input_path="dummy", seq_length=128, global_batch_size=1) val_data = { "version": "1.1", "data": [{ "paragraphs": [{ "context": "Sky is blue.", "qas": [{ "question": "What is blue?", "id": "1234", "answers": [{ "text": "Sky", "answer_start": 0 }, { "text": "Sky", "answer_start": 0 }, { "text": "Sky", "answer_start": 0 }] }] }] }] } self._val_input_path = os.path.join(self.get_temp_dir(), "val_data.json") with tf.io.gfile.GFile(self._val_input_path, "w") as writer: writer.write(json.dumps(val_data, indent=4) + "\n") self._test_vocab = os.path.join(self.get_temp_dir(), "vocab.txt") with tf.io.gfile.GFile(self._test_vocab, "w") as writer: writer.write("[PAD]\n[UNK]\n[CLS]\n[SEP]\n[MASK]\nsky\nis\nblue\n")
def test_task(self): config = sentence_prediction.SentencePredictionConfig( network=bert.BertPretrainerConfig( encoders.TransformerEncoderConfig(vocab_size=30522, num_layers=1), num_masked_tokens=0, cls_heads=[ bert.ClsHeadConfig(inner_dim=10, num_classes=3, name="sentence_prediction") ]), train_data=bert.BertSentencePredictionDataConfig( input_path="dummy", seq_length=128, global_batch_size=1)) task = sentence_prediction.SentencePredictionTask(config) model = task.build_model() metrics = task.build_metrics() dataset = task.build_inputs(config.train_data) iterator = iter(dataset) optimizer = tf.keras.optimizers.SGD(lr=0.1) task.train_step(next(iterator), model, optimizer, metrics=metrics) task.validation_step(next(iterator), model, metrics=metrics)
class ModelConfig(base_config.Config): """A base span labeler configuration.""" encoder: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig())
class ModelConfig(base_config.Config): """A base span labeler configuration.""" encoder: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) head_dropout: float = 0.1 head_initializer_range: float = 0.02
class ModelConfig(base_config.Config): """A classifier/regressor configuration.""" num_classes: int = 0 use_encoder_pooler: bool = False encoder: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig())
class BertPretrainerConfig(base_config.Config): """BERT encoder configuration.""" num_masked_tokens: int = 76 encoder: encoders.TransformerEncoderConfig = ( encoders.TransformerEncoderConfig()) cls_heads: List[ClsHeadConfig] = dataclasses.field(default_factory=list)
def setUp(self): super(TaggingTest, self).setUp() self._encoder_config = encoders.TransformerEncoderConfig( vocab_size=30522, num_layers=1) self._train_data_config = tagging_data_loader.TaggingDataConfig( input_path="dummy", seq_length=128, global_batch_size=1)
def get_model_config(self, num_classes): return sentence_prediction.ModelConfig( encoder=encoders.TransformerEncoderConfig( vocab_size=30522, num_layers=1), num_classes=num_classes)