def test_network_invocation(self):
config = bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(vocab_size=10,
num_layers=1))
_ = bert.instantiate_bertpretrainer_from_cfg(config)
# Invokes with classification heads.
config = bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(vocab_size=10,
num_layers=1),
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=2,
name="next_sentence")
])
_ = bert.instantiate_bertpretrainer_from_cfg(config)
with self.assertRaises(ValueError):
config = bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(vocab_size=10,
num_layers=1),
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=2,
name="next_sentence"),
bert.ClsHeadConfig(inner_dim=10,
num_classes=2,
name="next_sentence")
])
_ = bert.instantiate_bertpretrainer_from_cfg(config)
def test_task(self):
# Saves a checkpoint.
pretrain_cfg = bert.BertPretrainerConfig(encoder=self._encoder_config,
num_masked_tokens=20,
cls_heads=[
bert.ClsHeadConfig(
inner_dim=10,
num_classes=3,
name="next_sentence")
])
pretrain_model = bert.instantiate_bertpretrainer_from_cfg(pretrain_cfg)
ckpt = tf.train.Checkpoint(model=pretrain_model,
**pretrain_model.checkpoint_items)
saved_path = ckpt.save(self.get_temp_dir())
config = question_answering.QuestionAnsweringConfig(
init_checkpoint=saved_path,
network=self._encoder_config,
train_data=self._train_data_config)
task = question_answering.QuestionAnsweringTask(config)
model = task.build_model()
metrics = task.build_metrics()
dataset = task.build_inputs(config.train_data)
iterator = iter(dataset)
optimizer = tf.keras.optimizers.SGD(lr=0.1)
task.train_step(next(iterator), model, optimizer, metrics=metrics)
task.validation_step(next(iterator), model, metrics=metrics)
task.initialize(model)
def test_task(self):
config = sentence_prediction.SentencePredictionConfig(
init_checkpoint=self.get_temp_dir(),
model=self.get_model_config(2),
train_data=self._train_data_config)
task = sentence_prediction.SentencePredictionTask(config)
model = task.build_model()
metrics = task.build_metrics()
dataset = task.build_inputs(config.train_data)
iterator = iter(dataset)
optimizer = tf.keras.optimizers.SGD(lr=0.1)
task.train_step(next(iterator), model, optimizer, metrics=metrics)
task.validation_step(next(iterator), model, metrics=metrics)
# Saves a checkpoint.
pretrain_cfg = bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(
vocab_size=30522, num_layers=1),
cls_heads=[
bert.ClsHeadConfig(
inner_dim=10, num_classes=3, name="next_sentence")
])
pretrain_model = bert.instantiate_pretrainer_from_cfg(pretrain_cfg)
ckpt = tf.train.Checkpoint(
model=pretrain_model, **pretrain_model.checkpoint_items)
ckpt.save(config.init_checkpoint)
task.initialize(model)
def test_task(self):
config = masked_lm.MaskedLMConfig(
init_checkpoint=self.get_temp_dir(),
model=bert.BertPretrainerConfig(
encoders.TransformerEncoderConfig(vocab_size=30522,
num_layers=1),
num_masked_tokens=20,
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=2,
name="next_sentence")
]),
train_data=pretrain_dataloader.BertPretrainDataConfig(
input_path="dummy",
max_predictions_per_seq=20,
seq_length=128,
global_batch_size=1))
task = masked_lm.MaskedLMTask(config)
model = task.build_model()
metrics = task.build_metrics()
dataset = task.build_inputs(config.train_data)
iterator = iter(dataset)
optimizer = tf.keras.optimizers.SGD(lr=0.1)
task.train_step(next(iterator), model, optimizer, metrics=metrics)
task.validation_step(next(iterator), model, metrics=metrics)
# Saves a checkpoint.
ckpt = tf.train.Checkpoint(model=model, **model.checkpoint_items)
ckpt.save(config.init_checkpoint)
task.initialize(model)
class MaskedLMConfig(cfg.TaskConfig):
"""The model config."""
network: bert.BertPretrainerConfig = bert.BertPretrainerConfig(cls_heads=[
bert.ClsHeadConfig(
inner_dim=768, num_classes=2, dropout_rate=0.1, name='next_sentence')
])
train_data: cfg.DataConfig = cfg.DataConfig()
validation_data: cfg.DataConfig = cfg.DataConfig()
def get_model_config(self, num_classes):
return bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(vocab_size=30522,
num_layers=1),
num_masked_tokens=0,
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=num_classes,
name="sentence_prediction")
])
def test_checkpoint_items(self):
config = bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(vocab_size=10,
num_layers=1),
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=2,
name="next_sentence")
])
encoder = bert.instantiate_bertpretrainer_from_cfg(config)
self.assertSameElements(encoder.checkpoint_items.keys(),
["encoder", "next_sentence.pooler_dense"])
def setUp(self):
super(SentencePredictionTaskTest, self).setUp()
self._network_config = bert.BertPretrainerConfig(
encoder=encoders.TransformerEncoderConfig(vocab_size=30522,
num_layers=1),
num_masked_tokens=0,
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=3,
name="sentence_prediction")
])
self._train_data_config = bert.SentencePredictionDataConfig(
input_path="dummy", seq_length=128, global_batch_size=1)
class SentencePredictionConfig(cfg.TaskConfig):
"""The model config."""
# At most one of `pretrain_checkpoint_dir` and `hub_module_url` can
# be specified.
pretrain_checkpoint_dir: str = ''
hub_module_url: str = ''
network: bert.BertPretrainerConfig = bert.BertPretrainerConfig(
num_masked_tokens=0,
cls_heads=[
bert.ClsHeadConfig(inner_dim=768,
num_classes=3,
dropout_rate=0.1,
name='sentence_prediction')
])
train_data: cfg.DataConfig = cfg.DataConfig()
validation_data: cfg.DataConfig = cfg.DataConfig()
def test_task_with_hub(self):
hub_module_url = self._export_bert_tfhub()
config = sentence_prediction.SentencePredictionConfig(
hub_module_url=hub_module_url,
network=bert.BertPretrainerConfig(
encoders.TransformerEncoderConfig(vocab_size=30522,
num_layers=1),
num_masked_tokens=0,
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=3,
name="sentence_prediction")
]),
train_data=bert.BertSentencePredictionDataConfig(
input_path="dummy", seq_length=128, global_batch_size=10))
self._run_task(config)
class SentencePredictionConfig(cfg.TaskConfig):
"""The model config."""
# At most one of `init_checkpoint` and `hub_module_url` can
# be specified.
init_checkpoint: str = ''
hub_module_url: str = ''
metric_type: str = 'accuracy'
network: bert.BertPretrainerConfig = bert.BertPretrainerConfig(
num_masked_tokens=0, # No masked language modeling head.
cls_heads=[
bert.ClsHeadConfig(inner_dim=768,
num_classes=3,
dropout_rate=0.1,
name='sentence_prediction')
])
train_data: cfg.DataConfig = cfg.DataConfig()
validation_data: cfg.DataConfig = cfg.DataConfig()
def test_task(self, version_2_with_negative, tokenization):
# Saves a checkpoint.
pretrain_cfg = bert.BertPretrainerConfig(
encoder=self._encoder_config,
cls_heads=[
bert.ClsHeadConfig(
inner_dim=10, num_classes=3, name="next_sentence")
])
pretrain_model = bert.instantiate_pretrainer_from_cfg(pretrain_cfg)
ckpt = tf.train.Checkpoint(
model=pretrain_model, **pretrain_model.checkpoint_items)
saved_path = ckpt.save(self.get_temp_dir())
config = question_answering.QuestionAnsweringConfig(
init_checkpoint=saved_path,
model=question_answering.ModelConfig(encoder=self._encoder_config),
train_data=self._train_data_config,
validation_data=self._get_validation_data_config(
version_2_with_negative))
self._run_task(config)
def test_task(self):
config = sentence_prediction.SentencePredictionConfig(
network=bert.BertPretrainerConfig(
encoders.TransformerEncoderConfig(vocab_size=30522,
num_layers=1),
num_masked_tokens=0,
cls_heads=[
bert.ClsHeadConfig(inner_dim=10,
num_classes=3,
name="sentence_prediction")
]),
train_data=bert.BertSentencePredictionDataConfig(
input_path="dummy", seq_length=128, global_batch_size=1))
task = sentence_prediction.SentencePredictionTask(config)
model = task.build_model()
metrics = task.build_metrics()
dataset = task.build_inputs(config.train_data)
iterator = iter(dataset)
optimizer = tf.keras.optimizers.SGD(lr=0.1)
task.train_step(next(iterator), model, optimizer, metrics=metrics)
task.validation_step(next(iterator), model, metrics=metrics)
from official.core import task_factory
>>>>>>> a811a3b7e640722318ad868c99feddf3f3063e36
from official.modeling.hyperparams import config_definitions as cfg
from official.nlp.configs import bert
from official.nlp.data import data_loader_factory
@dataclasses.dataclass
class MaskedLMConfig(cfg.TaskConfig):
"""The model config."""
<<<<<<< HEAD
init_checkpoint: str = ''
=======
>>>>>>> a811a3b7e640722318ad868c99feddf3f3063e36
model: bert.BertPretrainerConfig = bert.BertPretrainerConfig(cls_heads=[
bert.ClsHeadConfig(
inner_dim=768, num_classes=2, dropout_rate=0.1, name='next_sentence')
])
train_data: cfg.DataConfig = cfg.DataConfig()
validation_data: cfg.DataConfig = cfg.DataConfig()
<<<<<<< HEAD
@base_task.register_task_cls(MaskedLMConfig)
=======
@task_factory.register_task_cls(MaskedLMConfig)
>>>>>>> a811a3b7e640722318ad868c99feddf3f3063e36
class MaskedLMTask(base_task.Task):
"""Mock task object for testing."""
def build_model(self, params=None):
params = params or self.task_config.model
