def annotate(self, model : Model, instances: List[Instance],
batch_size: Optional[int] = None,
annotation_function: Optional[Callable[[Model, Dict[str, torch.Tensor]], torch.Tensor]] = None) -> List[Dict[str, Any]]:
dataset = AllennlpDataset(instances, model.vocab)
with torch.no_grad():
cuda_device = model._get_prediction_device()
data_loader = self.data_loader.construct(dataset=dataset)
if batch_size is not None:
if data_loader.batch_sampler is not None:
data_loader.batch_sampler.batch_size = batch_size #overwrite batch size
data_loader = DataLoader(dataset, batch_sampler=data_loader.batch_sampler)
else:
data_loader = DataLoader(dataset, batch_size=batch_size)
preds = []
for batch in data_loader:
model_input = util.move_to_device(batch, cuda_device)
if annotation_function is not None:
outputs = annotation_function(model, model_input)
else:
outputs = model.make_output_human_readable(model(**model_input))
output_dict = split_up(outputs, model_input["order_metadata"])
preds.extend(output_dict)
if self.decoder is not None:
preds = self.decoder.decode_batch(model.vocab, preds)
return OrderedDatasetReader.restore_order(preds)
def build_data_loaders(
train_data: torch.utils.data.Dataset,
dev_data: torch.utils.data.Dataset,
) -> Tuple[allennlp.data.DataLoader, allennlp.data.DataLoader]:
train_loader = DataLoader(train_data, batch_size=32, shuffle=True)
dev_loader = DataLoader(dev_data, batch_size=32, shuffle=False)
return train_loader, dev_loader
def run_config(config):
params = Params(json.loads(config))
params_copy = params.duplicate()
if 'dataset_reader' in params:
reader = DatasetReader.from_params(params.pop('dataset_reader'))
else:
raise RuntimeError('`dataset_reader` section is required')
all_instances = []
if 'train_data_path' in params:
print('Reading the training data...')
train_data = reader.read(params.pop('train_data_path'))
all_instances.extend(train_data)
else:
raise RuntimeError('`train_data_path` section is required')
validation_data = None
if 'validation_data_path' in params:
print('Reading the validation data...')
validation_data = reader.read(params.pop('validation_data_path'))
all_instances.extend(validation_data)
print('Building the vocabulary...')
vocab = Vocabulary.from_instances(all_instances)
model = None
iterator = None
if 'model' not in params:
# 'dataset' mode — just preview the (first 10) instances
print('Showing the first 10 instances:')
for inst in all_instances[:10]:
print(inst)
else:
model = Model.from_params(vocab=vocab, params=params.pop('model'))
loader_params = deepcopy(params.pop("data_loader"))
train_data_loader = DataLoader.from_params(dataset=train_data,
params=loader_params)
dev_data_loader = DataLoader.from_params(dataset=validation_data,
params=loader_params)
train_data.index_with(vocab)
# set up a temporary, empty directory for serialization
with tempfile.TemporaryDirectory() as serialization_dir:
trainer = Trainer.from_params(
model=model,
serialization_dir=serialization_dir,
data_loader=train_data_loader,
validation_data_loader=dev_data_loader,
params=params.pop('trainer'))
trainer.train()
return {
'params': params_copy,
'dataset_reader': reader,
'vocab': vocab,
'iterator': iterator,
'model': model
}
def build_data_loaders(
dataset_reader: DatasetReader, train_data: torch.utils.data.Dataset,
dev_data: torch.utils.data.Dataset,
args) -> Tuple[allennlp.data.DataLoader, allennlp.data.DataLoader]:
# Note that DataLoader is imported from allennlp above, *not* torch.
# We need to get the allennlp-specific collate function, which is
# what actually does indexing and batching.
# Sampling is handled exclusively from the dataset reader itself now. The actual datasets returned will be
# sampled appropriately. Hence, we don't need this anymore
train_sampler = None
val_sampler = None
if args.use_subsampling:
train_sampler = dataset_reader.train_sampler #note that dev_data should not be limited...
val_sampler = dataset_reader.test_sampler
# because now our sampling is done inside the reader, it is obsolete to use a constructed sampler
# the sampler should still be fine, since it is indeed just a list of indices, but for some reason, this will not work
train_loader = DataLoader(train_data,
batch_size=args.batch_size,
sampler=None)
dev_loader = DataLoader(
dev_data,
batch_size=args.batch_size) # the validation should not use a sampler
# expect: sampler to now balance things out. and also, we don't get too many examples
return train_loader, dev_loader
def create_dataloader(
dataset: InstancesDataset,
batch_size: int,
data_bucketing: bool = False,
batches_per_epoch: Optional[int] = None,
) -> DataLoader:
"""Returns a pytorch DataLoader for AllenNLP
Parameters
----------
dataset
The data set for the DataLoader
batch_size
Size of the batch.
data_bucketing
If enabled, try to apply data bucketing over training batches.
batches_per_epoch
Determines the number of batches after which an epoch ends.
If the number is smaller than the total amount of batches in your data,
the second "epoch" will take off where the first "epoch" ended.
If this is `None`, then an epoch is set to be one full pass through your data.
Returns
-------
data_loader
"""
return (DataLoader(
dataset,
batch_sampler=BucketBatchSampler(data_source=dataset,
batch_size=batch_size),
batches_per_epoch=batches_per_epoch,
) if data_bucketing and not isinstance(dataset, IterableDataset) else
DataLoader(dataset,
batch_size=batch_size,
batches_per_epoch=batches_per_epoch))
def build_data_loaders_from_reader(dataset_reader, vocab, batch_size=64):
train_data, dev_data = read_data(dataset_reader)
train_data.index_with(vocab)
dev_data.index_with(vocab)
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=False)
dev_loader = DataLoader(dev_data, batch_size=batch_size, shuffle=False)
return train_loader, dev_loader
def build_data_loaders(train_data: torch.utils.data.Dataset,
dev_data: torch.utils.data.Dataset,
batch_size: int = 16) -> Tuple[DataLoader, DataLoader]:
# Note that DataLoader is imported from allennlp above, *not* torch.
# We need to get the allennlp-specific collate function, which is
# what actually does indexing and batching.
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
dev_loader = DataLoader(dev_data, batch_size=batch_size, shuffle=False)
return train_loader, dev_loader
def test_from_params_in_trainer(self):
# This is more of an integration test, making sure that a bunch of pieces fit together
# correctly, but it matters most for this learning rate scheduler, so we're testing it here.
params = Params(
{
"num_epochs": 5,
"learning_rate_scheduler": {
"type": "slanted_triangular",
"gradual_unfreezing": True,
"discriminative_fine_tuning": True,
"decay_factor": 0.5,
},
}
)
# The method called in the logic below only checks the length of this list, not its
# contents, so this should be safe.
instances = AllennlpDataset([1] * 40)
optim = self._get_optimizer()
trainer = Trainer.from_params(
model=self.model,
optimizer=Lazy(lambda **kwargs: optim),
serialization_dir=self.TEST_DIR,
params=params,
data_loader=DataLoader(instances, batch_size=10),
)
assert isinstance(trainer._learning_rate_scheduler, SlantedTriangular)
# This is what we wrote this test for: to be sure that num_epochs is passed correctly, and
# that num_steps_per_epoch is computed and passed correctly. This logic happens inside of
# `Trainer.from_partial_objects`.
assert trainer._learning_rate_scheduler.num_epochs == 5
assert trainer._learning_rate_scheduler.num_steps_per_epoch == 4
# And we'll do one more to make sure that we can override num_epochs in the scheduler if we
# really want to. Not sure why you would ever want to in this case; this is just testing
# the functionality.
params = Params(
{
"num_epochs": 5,
"learning_rate_scheduler": {
"type": "slanted_triangular",
"num_epochs": 3,
"gradual_unfreezing": True,
"discriminative_fine_tuning": True,
"decay_factor": 0.5,
},
}
)
trainer = Trainer.from_params(
model=self.model,
optimizer=Lazy(lambda **kwargs: optim),
serialization_dir=self.TEST_DIR,
params=params,
data_loader=DataLoader(instances, batch_size=10),
)
assert trainer._learning_rate_scheduler.num_epochs == 3
def run_config(config):
params = Params(json.loads(config))
params_copy = params.duplicate()
if "dataset_reader" in params:
reader = DatasetReader.from_params(params.pop("dataset_reader"))
else:
raise RuntimeError("`dataset_reader` section is required")
loader_params = params.pop("data_loader")
train_data_loader = DataLoader.from_params(
reader=reader,
data_path=params.pop("train_data_path"),
params=loader_params.duplicate(),
)
dev_data_loader = DataLoader.from_params(
reader=reader,
data_path=params.pop("validation_data_path"),
params=loader_params,
)
print("Building the vocabulary...")
vocab = Vocabulary.from_instances(train_data_loader.iter_instances())
if "model" not in params:
# 'dataset' mode — just preview the (first 10) instances
print("Showing the first 10 instances:")
for inst in train_data_loader.iter_instances():
print(inst)
return None
model = Model.from_params(vocab=vocab, params=params.pop("model"))
train_data_loader.index_with(vocab)
dev_data_loader.index_with(vocab)
# set up a temporary, empty directory for serialization
with tempfile.TemporaryDirectory() as serialization_dir:
trainer = Trainer.from_params(
model=model,
serialization_dir=serialization_dir,
data_loader=train_data_loader,
validation_data_loader=dev_data_loader,
params=params.pop("trainer"),
)
trainer.train()
return {
"params": params_copy,
"dataset_reader": reader,
"vocab": vocab,
"model": model,
}
def build_data_loaders(
train_data: torch.utils.data.Dataset,
dev_data: torch.utils.data.Dataset = None,
batch_size: int = 8):
# Note that DataLoader is imported from allennlp above, *not* torch.
# We need to get the allennlp-specific collate function, which is
# what actually does indexing and batching.
batch_sampler = BucketBatchSampler(train_data, batch_size=batch_size, sorting_keys=["text"], padding_noise=0)
train_loader = DataLoader(train_data, batch_sampler=batch_sampler)
if dev_data:
dev_batch_sampler = BucketBatchSampler(dev_data, batch_size=batch_size, sorting_keys=["text"], padding_noise=0)
dev_loader = DataLoader(dev_data, batch_sampler=dev_batch_sampler)
return train_loader, dev_loader
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger("allennlp.common.params").disabled = True
logging.getLogger("allennlp.nn.initializers").disabled = True
logging.getLogger("allennlp.modules.token_embedders.embedding").setLevel(
logging.INFO)
# Load from archive
archive = load_archive(
args.archive_file,
weights_file=args.weights_file,
cuda_device=args.cuda_device,
overrides=args.overrides,
)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop("validation_dataset_reader",
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop("dataset_reader"))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
embedding_sources = (json.loads(args.embedding_sources_mapping)
if args.embedding_sources_mapping else {})
if args.extend_vocab:
logger.info("Vocabulary is being extended with test instances.")
model.vocab.extend_from_instances(instances=instances)
model.extend_embedder_vocab(embedding_sources)
instances.index_with(model.vocab)
data_loader_params = config.pop("validation_data_loader", None)
if data_loader_params is None:
data_loader_params = config.pop("data_loader")
if args.batch_size:
data_loader_params["batch_size"] = args.batch_size
data_loader = DataLoader.from_params(dataset=instances,
params=data_loader_params)
metrics = evaluate(model, data_loader, args.cuda_device,
args.batch_weight_key)
logger.info("Finished evaluating.")
dump_metrics(args.output_file, metrics, log=True)
return metrics
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
common_logging.FILE_FRIENDLY_LOGGING = args.file_friendly_logging
# Disable some of the more verbose logging statements
logging.getLogger("allennlp.common.params").disabled = True
logging.getLogger("allennlp.nn.initializers").disabled = True
logging.getLogger("allennlp.modules.token_embedders.embedding").setLevel(
logging.INFO)
# Load from archive
archive = load_archive(
args.archive_file,
weights_file=args.weights_file,
cuda_device=args.cuda_device,
overrides=args.overrides,
)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
dataset_reader = archive.validation_dataset_reader
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
data_loader_params = config.pop("validation_data_loader", None)
if data_loader_params is None:
data_loader_params = config.pop("data_loader")
if args.batch_size:
data_loader_params["batch_size"] = args.batch_size
data_loader = DataLoader.from_params(params=data_loader_params,
reader=dataset_reader,
data_path=evaluation_data_path)
embedding_sources = (json.loads(args.embedding_sources_mapping)
if args.embedding_sources_mapping else {})
if args.extend_vocab:
logger.info("Vocabulary is being extended with test instances.")
model.vocab.extend_from_instances(
instances=data_loader.iter_instances())
model.extend_embedder_vocab(embedding_sources)
data_loader.index_with(model.vocab)
metrics = evaluate(
model,
data_loader,
args.cuda_device,
args.batch_weight_key,
output_file=args.output_file,
predictions_output_file=args.predictions_output_file,
)
logger.info("Finished evaluating.")
return metrics
def val_dataloader(self):
batch_sampler = BucketBatchSampler(self.val_dataset,
batch_size=self.hparams['batch_size'],
sorting_keys=['source_tokens', 'target_tokens'],
drop_last=True)
val_data_loader = DataLoader(
self.val_dataset, batch_sampler=batch_sampler, num_workers=self.hparams['num_workers'])
return val_data_loader
def test_can_optimise_model_with_dense_and_sparse_params(self):
optimizer_params = Params({"type": "dense_sparse_adam"})
parameters = [[n, p] for n, p in self.model.named_parameters()
if p.requires_grad]
optimizer = Optimizer.from_params(model_parameters=parameters,
params=optimizer_params)
self.instances.index_with(self.vocab)
Trainer(self.model, optimizer, DataLoader(self.instances, 2)).train()
def main():
reader = SkipGramReader()
text8 = reader.read('data/text8/text8')
vocab = Vocabulary.from_instances(text8,
min_count={
'token_in': 5,
'token_out': 5
})
reader = SkipGramReader(vocab=vocab)
text8 = reader.read('data/text8/text8')
text8.index_with(vocab)
embedding_in = Embedding(num_embeddings=vocab.get_vocab_size('token_in'),
embedding_dim=EMBEDDING_DIM)
embedding_out = Embedding(num_embeddings=vocab.get_vocab_size('token_out'),
embedding_dim=EMBEDDING_DIM)
if CUDA_DEVICE > -1:
embedding_in = embedding_in.to(CUDA_DEVICE)
embedding_out = embedding_out.to(CUDA_DEVICE)
data_loader = DataLoader(text8, batch_size=BATCH_SIZE)
# model = SkipGramNegativeSamplingModel(
# vocab=vocab,
# embedding_in=embedding_in,
# embedding_out=embedding_out,
# neg_samples=10,
# cuda_device=CUDA_DEVICE)
model = SkipGramModel(vocab=vocab,
embedding_in=embedding_in,
cuda_device=CUDA_DEVICE)
optimizer = optim.Adam(model.parameters())
trainer = GradientDescentTrainer(model=model,
optimizer=optimizer,
data_loader=data_loader,
num_epochs=5,
cuda_device=CUDA_DEVICE)
trainer.train()
# write_embeddings(embedding_in, 'data/text8/embeddings.txt', vocab)
print(get_synonyms('one', embedding_in, vocab))
print(get_synonyms('december', embedding_in, vocab))
print(get_synonyms('flower', embedding_in, vocab))
print(get_synonyms('design', embedding_in, vocab))
print(get_synonyms('snow', embedding_in, vocab))
rho = evaluate_embeddings(embedding_in, vocab)
print('simlex999 speareman correlation: {}'.format(rho))
def get_bucket_iterator(dataset,
max_tokens_in_batch,
sort_key=("tokens", ),
sort=False,
sort_within_batch=False,
repeat=False,
is_trainingset=True,
device: torch.device = torch.device("cuda", 0),
**kwargs):
batch_sampler = MaxTokensBatchSampler(dataset, max_tokens_in_batch,
sort_key)
return DataLoader(dataset, batch_sampler=batch_sampler)
def setUp(self):
super().setUp()
param_file = self.FIXTURES_ROOT / "simple_tagger" / "experiment_with_regularization.json"
self.set_up_model(param_file,
self.FIXTURES_ROOT / "data" / "sequence_tagging.tsv")
params = Params.from_file(param_file)
self.reader = DatasetReader.from_params(params["dataset_reader"])
self.data_loader = DataLoader.from_params(dataset=self.instances,
params=params["data_loader"])
self.trainer = Trainer.from_params(
model=self.model,
data_loader=self.data_loader,
serialization_dir=self.TEST_DIR,
params=params.get("trainer"),
)
def test_multigpu_qanet(self):
params = Params.from_file(self.param_file)
vocab = Vocabulary.from_instances(self.instances)
model = Model.from_params(vocab=vocab, params=params["model"]).cuda()
optimizer = torch.optim.SGD(self.model.parameters(),
0.01,
momentum=0.9)
self.instances.index_with(model.vocab)
loader = DataLoader(self.instances, batch_size=4)
trainer = Trainer(model,
optimizer,
loader,
num_epochs=2,
cuda_device=[0, 1])
trainer.train()
def test_regularization(self):
penalty = self.model.get_regularization_penalty()
assert penalty == 0
data_loader = DataLoader(self.instances, batch_size=32)
trainer = GradientDescentTrainer(self.model, None, data_loader) # optimizer,
# You get a RuntimeError if you call `model.forward` twice on the same inputs.
# The data and config are such that the whole dataset is one batch.
training_batch = next(iter(data_loader))
validation_batch = next(iter(data_loader))
training_loss = trainer.batch_outputs(training_batch, for_training=True)["loss"].item()
validation_loss = trainer.batch_outputs(validation_batch, for_training=False)["loss"].item()
# Training loss should have the regularization penalty, but validation loss should not.
numpy.testing.assert_almost_equal(training_loss, validation_loss)
def setUp(self):
super().setUp()
params = Params(
{
"model": {
"type": "simple_tagger",
"text_field_embedder": {
"token_embedders": {"tokens": {"type": "embedding", "embedding_dim": 5}}
},
"encoder": {"type": "lstm", "input_size": 5, "hidden_size": 7, "num_layers": 2},
},
"dataset_reader": {"type": "sequence_tagging"},
"train_data_path": str(self.FIXTURES_ROOT / "data" / "sequence_tagging.tsv"),
"validation_data_path": str(self.FIXTURES_ROOT / "data" / "sequence_tagging.tsv"),
"data_loader": {"batch_size": 2},
"trainer": {"cuda_device": -1, "num_epochs": 2, "optimizer": "adam"},
}
)
all_datasets = datasets_from_params(params)
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
instances=(instance for dataset in all_datasets.values() for instance in dataset),
)
model = Model.from_params(vocab=vocab, params=params.pop("model"))
train_data = all_datasets["train"]
train_data.index_with(vocab)
data_loader = DataLoader.from_params(dataset=train_data, params=params.pop("data_loader"))
trainer_params = params.pop("trainer")
serialization_dir = os.path.join(self.TEST_DIR, "test_search_learning_rate")
self.trainer = TrainerBase.from_params(
model=model,
serialization_dir=serialization_dir,
data_loader=data_loader,
train_data=train_data,
params=trainer_params,
validation_data=None,
validation_iterator=None,
)
def test_evaluation(self) -> Dict[str, Any]:
"""
Evaluates the model against the test dataset (if defined)
Returns
-------
Test metrics information
"""
test_data = self._test
if not test_data:
return {}
self.__LOGGER.info(
"The model will be evaluated using the best epoch weights.")
return evaluate(
self._pipeline._model,
data_loader=DataLoader(test_data,
batch_size=self._trainer_config.batch_size),
cuda_device=self._trainer.cuda_device,
batch_weight_key=self._batch_weight_key,
)
def benchmark_xlmr_mdl():
from allennlp.data import DataLoader
from allennlp.training.util import evaluate
xlmr = load_xlmr_coref_model()
data_loader_params = xlmr.config.pop("data_loader")
instances = xlmr.dataset_reader.load_dataset(testset)
instances.index_with(xlmr.model.vocab)
data_loader = DataLoader.from_params(dataset=instances,
params=data_loader_params)
start = time.time()
metrics = evaluate(xlmr.model, data_loader)
print('**XLM-R model**')
print_speed_performance(start, num_sentences, num_tokens)
print('Precision : ', metrics['coref_precision'])
print('Recall : ', metrics['coref_recall'])
print('F1 : ', metrics['coref_f1'])
print('Mention Recall : ', metrics['mention_recall'])
def ensure_model_can_train_save_and_load(
self,
param_file: Union[PathLike, str],
tolerance: float = 1e-4,
cuda_device: int = -1,
gradients_to_ignore: Set[str] = None,
overrides: str = "",
metric_to_check: str = None,
metric_terminal_value: float = None,
metric_tolerance: float = 1e-4,
disable_dropout: bool = True,
):
save_dir = self.TEST_DIR / "save_and_load_test"
archive_file = save_dir / "model.tar.gz"
model = train_model_from_file(param_file, save_dir, overrides=overrides)
metrics_file = save_dir / "metrics.json"
if metric_to_check is not None:
metric_value = metrics.get(f"best_validation_{metric_to_check}") or metrics.get(
f"training_{metric_to_check}"
)
assert metric_value is not None, f"Cannot find {metric_to_check} in metrics.json file"
assert metric_terminal_value is not None, "Please specify metric terminal value"
assert abs(metric_value - metric_terminal_value) < metric_tolerance
loaded_model = load_archive(archive_file, cuda_device=cuda_device).model
assert state_keys == loaded_state_keys
for key in state_keys:
assert_allclose(
model.state_dict()[key].cpu().numpy(),
loaded_model.state_dict()[key].cpu().numpy(),
err_msg=key,
)
params = Params.from_file(param_file, params_overrides=overrides)
reader = DatasetReader.from_params(params["dataset_reader"])
print("Reading with original model")
model_dataset = reader.read(params["validation_data_path"])
print("Reading with loaded model")
loaded_dataset = reader.read(params["validation_data_path"])
data_loader_params = params["data_loader"]
data_loader_params["shuffle"] = False
data_loader_params2 = Params(copy.deepcopy(data_loader_params.as_dict()))
data_loader2 = DataLoader.from_params(dataset=loaded_dataset, params=data_loader_params2)
model_batch = next(iter(data_loader))
loaded_batch = next(iter(data_loader2))
self.check_model_computes_gradients_correctly(
model, model_batch, gradients_to_ignore, disable_dropout
)
for key in model_batch.keys():
self.assert_fields_equal(model_batch[key], loaded_batch[key], key, 1e-6)
for model_ in [model, loaded_model]:
for module in model_.modules():
if hasattr(module, "stateful") and module.stateful:
module.reset_states()
print("Predicting with original model")
model_predictions = model(**model_batch)
print("Predicting with loaded model")
loaded_model_predictions = loaded_model(**loaded_batch)
self.assert_fields_equal(
model_predictions[key], loaded_model_predictions[key], name=key, tolerance=tolerance
)
# 构建词表
print("加载词表.........")
vocab = Vocabulary(padding_token="[PAD]", oov_token="[UNK]")
vocab.set_from_file(bert_path + "vocab.txt",
is_padded=False, oov_token="[UNK]", namespace="bert_tags")
# 构架reader和模型
print("定义模型........")
reader = BertSpanResolutionReader(model_name=bert_path,
max_turn_len=max_turn_len,
max_length=max_length)
model = BertSpanPointerResolution(vocab=vocab,
model_name=bert_path,
max_turn_len=max_turn_len,
task_pretrained_file=Path(pretrained_file) / "best.th")
model = model.eval()
# 读取测试集数据
instances = reader.read(validation_data_path)
instances.vocab = vocab
datasampler = BucketBatchSampler(instances, batch_size=16)
dataloader = DataLoader(dataset=instances, batch_sampler=datasampler)
print("预测.........")
# 读取数据并前向传播
with torch.no_grad():
for i, batch in tqdm(enumerate(dataloader)):
output_dict = model(**batch)
print("所有指标：", model.get_metrics())
def ensure_model_can_train_save_and_load(
self,
param_file: str,
tolerance: float = 1e-4,
cuda_device: int = -1,
gradients_to_ignore: Set[str] = None,
overrides: str = "",
metric_to_check: str = None,
metric_terminal_value: float = None,
metric_tolerance: float = 1e-4,
disable_dropout: bool = True,
):
"""
# Parameters
param_file : `str`
Path to a training configuration file that we will use to train the model for this
test.
tolerance : `float`, optional (default=1e-4)
When comparing model predictions between the originally-trained model and the model
after saving and loading, we will use this tolerance value (passed as `rtol` to
`numpy.testing.assert_allclose`).
cuda_device : `int`, optional (default=-1)
The device to run the test on.
gradients_to_ignore : `Set[str]`, optional (default=None)
This test runs a gradient check to make sure that we're actually computing gradients
for all of the parameters in the model. If you really want to ignore certain
parameters when doing that check, you can pass their names here. This is not
recommended unless you're `really` sure you don't need to have non-zero gradients for
those parameters (e.g., some of the beam search / state machine models have
infrequently-used parameters that are hard to force the model to use in a small test).
overrides : `str`, optional (default = "")
A JSON string that we will use to override values in the input parameter file.
metric_to_check: `str`, optional (default = None)
We may want to automatically perform a check that model reaches given metric when
training (on validation set, if it is specified). It may be useful in CI, for example.
You can pass any metric that is in your model returned metrics.
metric_terminal_value: `str`, optional (default = None)
When you set `metric_to_check`, you need to set the value this metric must converge to
metric_tolerance: `float`, optional (default=1e-4)
Tolerance to check you model metric against metric terminal value. One can expect some
variance in model metrics when the training process is highly stochastic.
disable_dropout : `bool`, optional (default = True)
If True we will set all dropout to 0 before checking gradients. (Otherwise, with small
datasets, you may get zero gradients because of unlucky dropout.)
"""
save_dir = self.TEST_DIR / "save_and_load_test"
archive_file = save_dir / "model.tar.gz"
model = train_model_from_file(param_file,
save_dir,
overrides=overrides)
metrics_file = save_dir / "metrics.json"
if metric_to_check is not None:
metrics = json.loads(metrics_file.read_text())
metric_value = metrics.get(
f"best_validation_{metric_to_check}") or metrics.get(
f"training_{metric_to_check}")
assert metric_value is not None, f"Cannot find {metric_to_check} in metrics.json file"
assert metric_terminal_value is not None, "Please specify metric terminal value"
assert abs(metric_value - metric_terminal_value) < metric_tolerance
loaded_model = load_archive(archive_file,
cuda_device=cuda_device).model
state_keys = model.state_dict().keys()
loaded_state_keys = loaded_model.state_dict().keys()
assert state_keys == loaded_state_keys
# First we make sure that the state dict (the parameters) are the same for both models.
for key in state_keys:
assert_allclose(
model.state_dict()[key].cpu().numpy(),
loaded_model.state_dict()[key].cpu().numpy(),
err_msg=key,
)
params = Params.from_file(param_file, params_overrides=overrides)
reader = DatasetReader.from_params(params["dataset_reader"])
print("Reading with original model")
model_dataset = reader.read(params["validation_data_path"])
model_dataset.index_with(model.vocab)
print("Reading with loaded model")
loaded_dataset = reader.read(params["validation_data_path"])
loaded_dataset.index_with(loaded_model.vocab)
# Need to duplicate params because DataLoader.from_params will consume.
data_loader_params = params["data_loader"]
data_loader_params["shuffle"] = False
data_loader_params2 = Params(
copy.deepcopy(data_loader_params.as_dict()))
data_loader = DataLoader.from_params(dataset=model_dataset,
params=data_loader_params)
data_loader2 = DataLoader.from_params(dataset=loaded_dataset,
params=data_loader_params2)
# We'll check that even if we index the dataset with each model separately, we still get
# the same result out.
model_batch = next(iter(data_loader))
loaded_batch = next(iter(data_loader2))
# Check gradients are None for non-trainable parameters and check that
# trainable parameters receive some gradient if they are trainable.
self.check_model_computes_gradients_correctly(model, model_batch,
gradients_to_ignore,
disable_dropout)
# The datasets themselves should be identical.
assert model_batch.keys() == loaded_batch.keys()
for key in model_batch.keys():
self.assert_fields_equal(model_batch[key], loaded_batch[key], key,
1e-6)
# Set eval mode, to turn off things like dropout, then get predictions.
model.eval()
loaded_model.eval()
# Models with stateful RNNs need their states reset to have consistent
# behavior after loading.
for model_ in [model, loaded_model]:
for module in model_.modules():
if hasattr(module, "stateful") and module.stateful:
module.reset_states()
print("Predicting with original model")
model_predictions = model(**model_batch)
print("Predicting with loaded model")
loaded_model_predictions = loaded_model(**loaded_batch)
# Check loaded model's loss exists and we can compute gradients, for continuing training.
loaded_model_loss = loaded_model_predictions["loss"]
assert loaded_model_loss is not None
loaded_model_loss.backward()
# Both outputs should have the same keys and the values for these keys should be close.
for key in model_predictions.keys():
self.assert_fields_equal(model_predictions[key],
loaded_model_predictions[key],
name=key,
tolerance=tolerance)
return model, loaded_model
def find_learning_rate_model(
params: Params,
serialization_dir: str,
start_lr: float = 1e-5,
end_lr: float = 10,
num_batches: int = 100,
linear_steps: bool = False,
stopping_factor: float = None,
force: bool = False,
) -> None:
"""
Runs learning rate search for given `num_batches` and saves the results in ``serialization_dir``
# Parameters
params : `Params`
A parameter object specifying an AllenNLP Experiment.
serialization_dir : `str`
The directory in which to save results.
start_lr : `float`
Learning rate to start the search.
end_lr : `float`
Learning rate upto which search is done.
num_batches : `int`
Number of mini-batches to run Learning rate finder.
linear_steps : `bool`
Increase learning rate linearly if False exponentially.
stopping_factor : `float`
Stop the search when the current loss exceeds the best loss recorded by
multiple of stopping factor. If `None` search proceeds till the `end_lr`
force : `bool`
If True and the serialization directory already exists, everything in it will
be removed prior to finding the learning rate.
"""
create_serialization_dir(params,
serialization_dir,
recover=False,
force=force)
prepare_environment(params)
cuda_device = params.params.get("trainer").get("cuda_device", -1)
check_for_gpu(cuda_device)
distributed_params = params.params.get("distributed")
# See https://github.com/allenai/allennlp/issues/3658
assert not distributed_params, "find-lr is not compatible with DistributedDataParallel."
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation),
)
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
instances=(instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation),
)
train_data = all_datasets["train"]
train_data.index_with(vocab)
model = Model.from_params(vocab=vocab, params=params.pop("model"))
data_loader = DataLoader.from_params(dataset=train_data,
params=params.pop("data_loader"))
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
trainer_choice = trainer_params.pop("type", "gradient_descent")
if trainer_choice != "gradient_descent":
raise ConfigurationError(
"currently find-learning-rate only works with the GradientDescentTrainer"
)
trainer: GradientDescentTrainer = Trainer.from_params( # type: ignore
model=model,
serialization_dir=serialization_dir,
data_loader=data_loader,
params=trainer_params,
)
logger.info(
f"Starting learning rate search from {start_lr} to {end_lr} in {num_batches} iterations."
)
learning_rates, losses = search_learning_rate(
trainer,
start_lr=start_lr,
end_lr=end_lr,
num_batches=num_batches,
linear_steps=linear_steps,
stopping_factor=stopping_factor,
)
logger.info("Finished learning rate search.")
losses = _smooth(losses, 0.98)
_save_plot(learning_rates, losses,
os.path.join(serialization_dir, "lr-losses.png"))
def data_loaders_from_params(
params: Params,
train: bool = True,
validation: bool = True,
test: bool = True,
serialization_dir: Optional[Union[str, PathLike]] = None,
) -> Dict[str, DataLoader]:
"""
Instantiate data loaders specified by the config.
"""
data_loaders: Dict[str, DataLoader] = {}
train = train and ("train_data_path" in params)
validation = validation and ("validation_data_path" in params)
test = test and ("test_data_path" in params)
if not any((train, validation, test)):
# Return early so don't unnecessarily initialize the train data reader.
return data_loaders
dataset_reader_params = params.pop("dataset_reader")
dataset_reader = DatasetReader.from_params(
dataset_reader_params, serialization_dir=serialization_dir)
data_loader_params = params.pop("data_loader")
if train:
train_data_path = params.pop("train_data_path")
logger.info("Reading training data from %s", train_data_path)
data_loaders["train"] = DataLoader.from_params(
data_loader_params.duplicate(),
reader=dataset_reader,
data_path=train_data_path)
if not validation and not test:
# Return early so we don't unnecessarily initialize the validation/test data
# reader.
return data_loaders
validation_and_test_dataset_reader: DatasetReader = dataset_reader
validation_dataset_reader_params = params.pop("validation_dataset_reader",
None)
if validation_dataset_reader_params is not None:
logger.info(
"Using a separate dataset reader to load validation and test data."
)
validation_and_test_dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params,
serialization_dir=serialization_dir)
validation_data_loader_params = params.pop("validation_data_loader",
data_loader_params)
if validation:
validation_data_path = params.pop("validation_data_path")
logger.info("Reading validation data from %s", validation_data_path)
data_loaders["validation"] = DataLoader.from_params(
validation_data_loader_params.duplicate(),
reader=validation_and_test_dataset_reader,
data_path=validation_data_path,
)
if test:
test_data_path = params.pop("test_data_path")
logger.info("Reading test data from %s", test_data_path)
data_loaders["test"] = DataLoader.from_params(
validation_data_loader_params,
reader=validation_and_test_dataset_reader,
data_path=test_data_path,
)
return data_loaders
def get_domains(path):
all_intents = [
x[0].replace(path, "") for x in os.walk(path)
if len(x[0].replace(path, "")) > 0 and x[0].replace(path, "")[0] != "."
]
return all_intents
if __name__ == '__main__':
args = get_parameter()
input_path = args.input_path
out_path = args.out_path
dataset = args.dataset
domains = get_domains(input_path + dataset + "/")
for domain in domains:
print("starting", domain)
model, dataset_reader, test_data = run_training_loop(
domain, domains, input_path + dataset + "/", out_path)
test_data_loader = DataLoader(test_data, batch_size=32)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print("cpu/gpu? ", device)
results = evaluate(model, test_data_loader, cuda_device=device)
print("results for", domain)
print(results)
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger("allennlp.common.params").disabled = True
logging.getLogger("allennlp.nn.initializers").disabled = True
logging.getLogger("transformers.modeling_utils").disabled = True
logging.getLogger("transformers.tokenization_utils").disabled = True
logging.getLogger("transformers.configuration_utils").disabled = True
logging.basicConfig(level=logging.INFO)
# Load from archive
archive = load_archive(
args.archive_file,
weights_file=args.weights_file,
cuda_device=args.cuda_device,
overrides=args.overrides,
)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop("validation_dataset_reader", None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(config.pop("dataset_reader"))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
embedding_sources = (
json.loads(args.embedding_sources_mapping) if args.embedding_sources_mapping else {}
)
if args.extend_vocab:
logger.info("Vocabulary is being extended with test instances.")
model.vocab.extend_from_instances(instances=instances)
model.extend_embedder_vocab(embedding_sources)
instances.index_with(model.vocab)
data_loader_params = config.pop("validation_data_loader", None)
if data_loader_params is None:
data_loader_params = config.pop("data_loader")
if args.batch_size:
data_loader_params["batch_size"] = args.batch_size
data_loader = DataLoader.from_params(dataset=instances, params=data_loader_params)
if "iter_norm" in dir(model.text_field_embedder._token_embedders['tokens']):
iter_num = model.text_field_embedder._token_embedders['tokens'].iter_norm
else:
iter_num = None
if iter_num:
# Obtrain evaluation info for iterative normalization:
iter_mean_eval = []
for iter_norm_i in range(iter_num):
logging.info("This is the {} time during iterative normalization for evaluation".format(iter_norm_i))
mean, embeddings = get_iter_norm_mean_eval(model, data_loader, iter_mean_eval, args.cuda_device)
logger.info("The degree of isotropy of vectors is {} ".format(degree_anisotropy(embeddings.t(), args.cuda_device)))
iter_mean_eval.append(mean)
model.text_field_embedder._token_embedders['tokens'].iter_norm = None
model.text_field_embedder._token_embedders['tokens']._matched_embedder.mean_emb_eval = iter_mean_eval
model.text_field_embedder._token_embedders['tokens']._matched_embedder.is_train = False
metrics = evaluate(model, data_loader, args.cuda_device, args.batch_weight_key)
logger.info("Finished evaluating.")
dump_metrics(args.output_file, metrics, log=True)
return metrics
def evaluate(
model: Model,
data_loader: DataLoader,
cuda_device: int = -1,
batch_weight_key: str = None,
output_file: str = None,
predictions_output_file: str = None,
) -> Dict[str, Any]:
"""
# Parameters
model : `Model`
The model to evaluate
data_loader : `DataLoader`
The `DataLoader` that will iterate over the evaluation data (data loaders already contain
their data).
cuda_device : `int`, optional (default=`-1`)
The cuda device to use for this evaluation. The model is assumed to already be using this
device; this parameter is only used for moving the input data to the correct device.
batch_weight_key : `str`, optional (default=`None`)
If given, this is a key in the output dictionary for each batch that specifies how to weight
the loss for that batch. If this is not given, we use a weight of 1 for every batch.
metrics_output_file : `str`, optional (default=`None`)
Optional path to write the final metrics to.
predictions_output_file : `str`, optional (default=`None`)
Optional path to write the predictions to.
# Returns
`Dict[str, Any]`
The final metrics.
"""
check_for_gpu(cuda_device)
data_loader.set_target_device(int_to_device(cuda_device))
predictions_file = (None if predictions_output_file is None else open(
predictions_output_file, "w"))
with torch.no_grad():
model.eval()
iterator = iter(data_loader)
logger.info("Iterating over dataset")
generator_tqdm = Tqdm.tqdm(iterator)
# Number of batches in instances.
batch_count = 0
# Number of batches where the model produces a loss.
loss_count = 0
# Cumulative weighted loss
total_loss = 0.0
# Cumulative weight across all batches.
total_weight = 0.0
for batch in generator_tqdm:
batch_count += 1
batch = nn_util.move_to_device(batch, cuda_device)
output_dict = model(**batch)
loss = output_dict.get("loss")
metrics = model.get_metrics()
if loss is not None:
loss_count += 1
if batch_weight_key:
weight = output_dict[batch_weight_key].item()
else:
weight = 1.0
total_weight += weight
total_loss += loss.item() * weight
# Report the average loss so far.
metrics["loss"] = total_loss / total_weight
if not HasBeenWarned.tqdm_ignores_underscores and any(
metric_name.startswith("_") for metric_name in metrics):
logger.warning('Metrics with names beginning with "_" will '
"not be logged to the tqdm progress bar.")
HasBeenWarned.tqdm_ignores_underscores = True
description = (", ".join([
"%s: %.2f" % (name, value)
for name, value in metrics.items() if not name.startswith("_")
]) + " ||")
generator_tqdm.set_description(description, refresh=False)
if predictions_file is not None:
predictions = json.dumps(
sanitize(model.make_output_human_readable(output_dict)))
predictions_file.write(predictions + "\n")
if predictions_file is not None:
predictions_file.close()
final_metrics = model.get_metrics(reset=True)
if loss_count > 0:
# Sanity check
if loss_count != batch_count:
raise RuntimeError(
"The model you are trying to evaluate only sometimes produced a loss!"
)
final_metrics["loss"] = total_loss / total_weight
if output_file is not None:
dump_metrics(output_file, final_metrics, log=True)
return final_metrics
