def predict_json(self, _: JsonDict, cuda_device: int = -1) -> JsonDict:
parameter_filename = 'allennlp/seq2seq.json'
serialization_dir = 'retrained'
subprocess.check_call(['mkdir', '-p', serialization_dir])
params = Params.from_file(parameter_filename)
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(self._model.vocab)
parameters = [[n, p] for n, p in self._model.named_parameters()
if p.requires_grad]
trainer_params = params.pop('trainer')
optimizer = Optimizer.from_params(parameters,
trainer_params.pop("optimizer"))
all_datasets = datasets_from_params(params)
train_data = all_datasets['train']
trainer = SimpleTrainer(self._model, optimizer, train_data, iterator)
interpreter = Interpreter(self._model, self._dataset_reader, trainer)
while True:
try:
interpreter.cmdloop()
except Exception as e:
print(e)
traceback.print_exc()
print('Restarting interpreter cmdloop.')
def make_vocab_from_params(params: Params, serialization_dir: str):
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
os.makedirs(serialization_dir, exist_ok=True)
vocab_dir = os.path.join(serialization_dir, "vocabulary")
if os.path.isdir(vocab_dir) and os.listdir(vocab_dir) is not None:
raise ConfigurationError("The 'vocabulary' directory in the provided "
"serialization directory is non-empty")
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
instances = [
instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation
]
vocab = Vocabulary.from_params(vocab_params, instances)
logger.info(f"writing the vocabulary to {vocab_dir}.")
vocab.save_to_files(vocab_dir)
logger.info("done creating vocab")
def make_vocab_from_params(params: Params):
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
vocab_dir = vocab_params.get('directory_path')
if vocab_dir is None:
raise ConfigurationError(
"To use `make-vocab` your configuration must contain a value "
"at vocabulary.directory_path")
os.makedirs(vocab_dir, exist_ok=True)
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("Creating a vocabulary using %s data.",
", ".join(datasets_for_vocab_creation))
vocab = Vocabulary.from_params(Params(
{}), (instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation))
vocab.save_to_files(vocab_dir)
logger.info("done creating vocab")
def make_vocab_from_params(params: Params, serialization_dir: str):
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
os.makedirs(serialization_dir, exist_ok=True)
vocab_dir = os.path.join(serialization_dir, "vocabulary")
if os.path.isdir(vocab_dir) and os.listdir(vocab_dir) is not None:
raise ConfigurationError("The 'vocabulary' directory in the provided "
"serialization directory is non-empty")
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
instances = [instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation]
vocab = Vocabulary.from_params(vocab_params, instances)
logger.info(f"writing the vocabulary to {vocab_dir}.")
vocab.save_to_files(vocab_dir)
logger.info("done creating vocab")
def make_vocab_from_params(params: Params):
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
vocab_dir = vocab_params.get('directory_path')
if vocab_dir is None:
raise ConfigurationError("To use `make-vocab` your configuration must contain a value "
"at vocabulary.directory_path")
os.makedirs(vocab_dir, exist_ok=True)
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("Creating a vocabulary using %s data.", ", ".join(datasets_for_vocab_creation))
vocab = Vocabulary.from_params(Params({}),
(instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation))
vocab.save_to_files(vocab_dir)
logger.info("done creating vocab")
def setUp(self):
super().setUp()
params = Params({
"model": {
"type": "simple_tagger",
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 5
}
}
},
"encoder": {
"type": "lstm",
"input_size": 5,
"hidden_size": 7,
"num_layers": 2
}
},
"dataset_reader": {
"type": "sequence_tagging"
},
"train_data_path":
str(self.FIXTURES_ROOT / 'data' / 'sequence_tagging.tsv'),
"validation_data_path":
str(self.FIXTURES_ROOT / 'data' / 'sequence_tagging.tsv'),
"iterator": {
"type": "basic",
"batch_size": 2
},
"trainer": {
"cuda_device": -1,
"num_epochs": 2,
"optimizer": "adam"
}
})
all_datasets = datasets_from_params(params)
vocab = Vocabulary.from_params(params.pop("vocabulary", {}),
(instance
for dataset in all_datasets.values()
for instance in dataset))
model = Model.from_params(vocab=vocab, params=params.pop('model'))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
train_data = all_datasets['train']
trainer_params = params.pop("trainer")
serialization_dir = os.path.join(self.TEST_DIR,
'test_search_learning_rate')
self.trainer = Trainer.from_params(model,
serialization_dir,
iterator,
train_data,
params=trainer_params,
validation_data=None,
validation_iterator=None)
def main(params: Params, outdir: str):
os.makedirs(outdir, exist_ok=True)
params['dataset_reader']['include_table_metadata'] = True
if 'validation_dataset_reader' in params:
params['validation_dataset_reader']['include_table_metadata'] = True
all_datasets = datasets_from_params(params)
for name, dataset in all_datasets.items():
with open(outdir + name + '.jsonl', 'w') as outfile:
for instance in iter(dataset):
outfile.write(to_json_line(instance) + '\n')
def main(params: Params, outdir: str):
os.makedirs(outdir, exist_ok=True)
params["dataset_reader"]["include_table_metadata"] = True
if "validation_dataset_reader" in params:
params["validation_dataset_reader"]["include_table_metadata"] = True
all_datasets = datasets_from_params(params)
for name, dataset in all_datasets.items():
with open(outdir + name + ".jsonl", "w") as outfile:
for instance in iter(dataset):
outfile.write(to_json_line(instance) + "\n")
def main(params: Params, outdir: str):
os.makedirs(outdir, exist_ok=True)
params['dataset_reader']['include_table_metadata'] = True
if 'validation_dataset_reader' in params:
params['validation_dataset_reader']['include_table_metadata'] = True
all_datasets = datasets_from_params(params)
for name, dataset in all_datasets.items():
with open(outdir + name + '.jsonl', 'w') as outfile:
for instance in iter(dataset):
outfile.write(to_json_line(instance) + '\n')
def dry_run_from_params(params: Params, serialization_dir: str) -> None:
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
os.makedirs(serialization_dir, exist_ok=True)
vocab_dir = os.path.join(serialization_dir, "vocabulary")
if os.path.isdir(vocab_dir) and os.listdir(vocab_dir) is not None:
raise ConfigurationError("The 'vocabulary' directory in the provided "
"serialization directory is non-empty")
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
instances = [
instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation
]
vocab = Vocabulary.from_params(vocab_params, instances)
dataset = Batch(instances)
dataset.index_instances(vocab)
dataset.print_statistics()
vocab.print_statistics()
logger.info(f"writing the vocabulary to {vocab_dir}.")
vocab.save_to_files(vocab_dir)
model = Model.from_params(vocab=vocab, params=params.pop('model'))
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
def setUp(self):
super().setUp()
params = Params({
"model": {
"type": "simple_tagger",
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 5
}
}
},
"encoder": {
"type": "lstm",
"input_size": 5,
"hidden_size": 7,
"num_layers": 2
}
},
"dataset_reader": {"type": "sequence_tagging"},
"train_data_path": str(self.FIXTURES_ROOT / 'data' / 'sequence_tagging.tsv'),
"validation_data_path": str(self.FIXTURES_ROOT / 'data' / 'sequence_tagging.tsv'),
"iterator": {"type": "basic", "batch_size": 2},
"trainer": {
"cuda_device": -1,
"num_epochs": 2,
"optimizer": "adam"
}
})
all_datasets = datasets_from_params(params)
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
(instance for dataset in all_datasets.values()
for instance in dataset)
)
model = Model.from_params(vocab=vocab, params=params.pop('model'))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
train_data = all_datasets['train']
trainer_params = params.pop("trainer")
serialization_dir = os.path.join(self.TEST_DIR, 'test_search_learning_rate')
self.trainer = Trainer.from_params(model,
serialization_dir,
iterator,
train_data,
params=trainer_params,
validation_data=None,
validation_iterator=None)
def dry_run_from_params(params: Params, serialization_dir: str) -> None:
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
os.makedirs(serialization_dir, exist_ok=True)
vocab_dir = os.path.join(serialization_dir, "vocabulary")
if os.path.isdir(vocab_dir) and os.listdir(vocab_dir) is not None:
raise ConfigurationError("The 'vocabulary' directory in the provided "
"serialization directory is non-empty")
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
instances = [instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation]
vocab = Vocabulary.from_params(vocab_params, instances)
dataset = Batch(instances)
dataset.index_instances(vocab)
dataset.print_statistics()
vocab.print_statistics()
logger.info(f"writing the vocabulary to {vocab_dir}.")
vocab.save_to_files(vocab_dir)
model = Model.from_params(vocab=vocab, params=params.pop('model'))
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
def dry_run_from_params(params: Params, serialization_dir: str) -> None:
prepare_environment(params)
vocab_params = params.pop("vocabulary", {})
vocab_dir = vocab_params.pop('directory_path', None)
if vocab_dir is not None:
logger.info(
"Found a vocabulary.directory_path parameter in your config. "
"Also saving the vocab we create to that location.")
os.makedirs(vocab_dir, exist_ok=True)
os.makedirs(serialization_dir, exist_ok=True)
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("Creating a vocabulary using %s data.",
", ".join(datasets_for_vocab_creation))
instances = [
instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation
]
vocabulary = verbosely_create_vocabulary(vocab_params, instances)
logger.info(f"writing the vocabulary to {serialization_dir}.")
vocabulary.save_to_files(os.path.join(serialization_dir, "vocabulary"))
if vocab_dir is not None and os.path.exists(vocab_dir) and os.listdir(
vocab_dir) is not None:
logger.info(
f"You passed a vocabulary.directory_path in your config which already exists "
f"and is non-empty. Refusing to overwrite - we saved it to {serialization_dir} instead."
)
elif vocab_dir is not None:
logger.info(
f"You passed a vocabulary.directory_path in your config which was empty. Also "
f"writing the vocabulary to {vocab_dir}.")
vocabulary.save_to_files(vocab_dir)
def load_data_from_params(self, params: Params):
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
instances_for_vocab_creation = (
instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation)
self._instances_for_vocab_creation = instances_for_vocab_creation
self._datasets_for_vocab_creation = datasets_for_vocab_creation
if "train" in all_datasets.keys():
self._train_data = all_datasets["train"]
self._tr_instances = sum(
1 for e in self._train_data
) # This is horrible if lazy iterator (Iterable)
if "validation" in all_datasets.keys():
self._validation_data = all_datasets["validation"]
self._val_instances = sum(
1 for e in self._validation_data
) # This is horrible if lazy iterator (Iterable)
if "test" in all_datasets.keys():
self._test_data = all_datasets["test"]
self._test_instances = sum(
1 for e in self._test_data
) # This is horrible if lazy iterator (Iterable)
# If trying to evaluate on test set, make sure the dataset is loaded
if self._evaluate_on_test:
assert self._test_data is not None
# return instances_for_vocab_creation, datasets_for_vocab_creation, all_datasets
return instances_for_vocab_creation, datasets_for_vocab_creation
def train_model(params: Params,
serialization_dir: str,
file_friendly_logging: bool = False,
recover: bool = False,
force: bool = False) -> Model:
prepare_environment(params)
create_serialization_dir(params, serialization_dir, recover, force)
prepare_global_logging(serialization_dir, file_friendly_logging)
cuda_device = params.params.get('trainer').get('cuda_device', -1)
if isinstance(cuda_device, list):
for device in cuda_device:
check_for_gpu(device)
else:
check_for_gpu(cuda_device)
params.to_file(os.path.join(serialization_dir, CONFIG_NAME))
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
if recover and os.path.exists(os.path.join(serialization_dir,
"vocabulary")):
vocab = Vocabulary.from_files(
os.path.join(serialization_dir, "vocabulary"))
else:
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
(instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation))
model = Model.from_params(vocab=vocab, params=params.pop('model'))
# Initializing the model can have side effect of expanding the vocabulary
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
validation_iterator_params = params.pop("validation_iterator", None)
if validation_iterator_params:
validation_iterator = DataIterator.from_params(
validation_iterator_params)
validation_iterator.index_with(vocab)
else:
validation_iterator = None
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
trainer_choice = trainer_params.pop_choice("type",
Trainer.list_available(),
default_to_first_choice=True)
trainer = Trainer.by_name(trainer_choice).from_params(
model=model,
serialization_dir=serialization_dir,
iterator=iterator,
train_data=train_data,
validation_data=validation_data,
params=trainer_params,
validation_iterator=validation_iterator)
evaluate_on_test = params.pop_bool("evaluate_on_test", False)
params.assert_empty('base train command')
try:
metrics = trainer.train()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if os.path.exists(os.path.join(serialization_dir, DEFAULT_WEIGHTS)):
logging.info(
"Training interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(serialization_dir,
files_to_archive=params.files_to_archive)
raise
# Now tar up results
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
logger.info("Loading the best epoch weights.")
best_model_state_path = os.path.join(serialization_dir, 'best.th')
best_model_state = torch.load(best_model_state_path)
best_model = model
best_model.load_state_dict(best_model_state)
if test_data and evaluate_on_test:
logger.info(
"The model will be evaluated using the best epoch weights.")
test_metrics = evaluate(
best_model,
test_data,
validation_iterator or iterator,
cuda_device=trainer._cuda_devices[0] # pylint: disable=protected-access
)
for key, value in test_metrics.items():
metrics["test_" + key] = value
elif test_data:
logger.info(
"To evaluate on the test set after training, pass the "
"'evaluate_on_test' flag, or use the 'allennlp evaluate' command.")
dump_metrics(os.path.join(serialization_dir, "metrics.json"),
metrics,
log=True)
return best_model
def fine_tune_model(model: Model,
params: Params,
serialization_dir: str,
file_friendly_logging: bool = False) -> Model:
"""
Fine tunes the given model, using a set of parameters that is largely identical to those used
for :func:`~allennlp.commands.train.train_model`, except that the ``model`` section is ignored,
if it is present (as we are already given a ``Model`` here).
The main difference between the logic done here and the logic done in ``train_model`` is that
here we do not worry about vocabulary construction or creating the model object. Everything
else is the same.
Parameters
----------
archive : ``Archive``
A saved model archive that is the result of running the ``train`` command.
train_data_path : ``str``
Path to the training data to use for fine-tuning.
serialization_dir : ``str``
The directory in which to save results and logs.
validation_data_path : ``str``, optional
Path to the validation data to use while fine-tuning.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
"""
prepare_environment(params)
os.makedirs(serialization_dir)
# TODO(mattg): pull this block out into a separate function (maybe just add this to
# `prepare_environment`?)
Tqdm.set_slower_interval(file_friendly_logging)
sys.stdout = TeeLogger(
os.path.join(serialization_dir, "stdout.log"), # type: ignore
sys.stdout,
file_friendly_logging)
sys.stderr = TeeLogger(
os.path.join(serialization_dir, "stderr.log"), # type: ignore
sys.stderr,
file_friendly_logging)
handler = logging.FileHandler(
os.path.join(serialization_dir, "python_logging.log"))
handler.setLevel(logging.INFO)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(levelname)s - %(name)s - %(message)s'))
logging.getLogger().addHandler(handler)
serialization_params = deepcopy(params).as_dict(quiet=True)
with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file:
json.dump(serialization_params, param_file, indent=4)
if params.pop('model', None):
logger.warning(
"You passed parameters for the model in your configuration file, but we "
"are ignoring them, using instead the model parameters in the archive."
)
if params.pop('vocabulary', None):
logger.warning(
"You passed parameters for the vocabulary in your configuration file, but "
"we are ignoring them, using instead the vocabulary from the saved model."
)
vocab = model.vocab
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
all_datasets = datasets_from_params(params)
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
trainer = Trainer.from_params(model, serialization_dir, iterator,
train_data, validation_data, trainer_params)
evaluate_on_test = params.pop_bool("evaluate_on_test", False)
params.assert_empty('base train command')
metrics = trainer.train()
# Now tar up results
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
if test_data and evaluate_on_test:
test_metrics = evaluate(model,
test_data,
iterator,
cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access
for key, value in test_metrics.items():
metrics["test_" + key] = value
elif test_data:
logger.info(
"To evaluate on the test set after training, pass the "
"'evaluate_on_test' flag, or use the 'allennlp evaluate' command.")
metrics_json = json.dumps(metrics, indent=2)
with open(os.path.join(serialization_dir, "metrics.json"),
"w") as metrics_file:
metrics_file.write(metrics_json)
logger.info("Metrics: %s", metrics_json)
return model
def find_learning_rate_model(params: Params,
serialization_dir: str,
start_lr: float = 1e-5,
end_lr: float = 10,
num_batches: int = 100,
linear_steps: bool = False,
stopping_factor: float = None,
force: bool = False) -> None:
"""
Runs learning rate search for given `num_batches` and saves the results in ``serialization_dir``
Parameters
----------
trainer: :class:`~allennlp.common.registrable.Registrable`
params : ``Params``
A parameter object specifying an AllenNLP Experiment.
serialization_dir : ``str``
The directory in which to save results.
start_lr: ``float``
Learning rate to start the search.
end_lr: ``float``
Learning rate upto which search is done.
num_batches: ``int``
Number of mini-batches to run Learning rate finder.
linear_steps: ``bool``
Increase learning rate linearly if False exponentially.
stopping_factor: ``float``
Stop the search when the current loss exceeds the best loss recorded by
multiple of stopping factor. If ``None`` search proceeds till the ``end_lr``
force: ``bool``
If True and the serialization directory already exists, everything in it will
be removed prior to finding the learning rate.
"""
if os.path.exists(serialization_dir) and force:
shutil.rmtree(serialization_dir)
if os.path.exists(serialization_dir) and os.listdir(serialization_dir):
raise ConfigurationError(
f'Serialization directory {serialization_dir} already exists and is '
f'not empty.')
else:
os.makedirs(serialization_dir, exist_ok=True)
prepare_environment(params)
check_for_gpu(params.get('trainer').get('cuda_device', -1))
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
(instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation))
model = Model.from_params(vocab=vocab, params=params.pop('model'))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
train_data = all_datasets['train']
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
trainer = Trainer.from_params(model,
serialization_dir,
iterator,
train_data,
params=trainer_params,
validation_data=None,
validation_iterator=None)
logger.info(
f'Starting learning rate search from {start_lr} to {end_lr} in {num_batches} iterations.'
)
learning_rates, losses = search_learning_rate(
trainer,
start_lr=start_lr,
end_lr=end_lr,
num_batches=num_batches,
linear_steps=linear_steps,
stopping_factor=stopping_factor)
logger.info(f'Finished learning rate search.')
losses = _smooth(losses, 0.98)
_save_plot(learning_rates, losses,
os.path.join(serialization_dir, 'lr-losses.png'))
def fine_tune_model(model: Model,
params: Params,
serialization_dir: str,
file_friendly_logging: bool = False) -> Model:
"""
Fine tunes the given model, using a set of parameters that is largely identical to those used
for :func:`~allennlp.commands.train.train_model`, except that the ``model`` section is ignored,
if it is present (as we are already given a ``Model`` here).
The main difference between the logic done here and the logic done in ``train_model`` is that
here we do not worry about vocabulary construction or creating the model object. Everything
else is the same.
Parameters
----------
archive : ``Archive``
A saved model archive that is the result of running the ``train`` command.
train_data_path : ``str``
Path to the training data to use for fine-tuning.
serialization_dir : ``str``
The directory in which to save results and logs.
validation_data_path : ``str``, optional
Path to the validation data to use while fine-tuning.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
"""
prepare_environment(params)
os.makedirs(serialization_dir)
prepare_global_logging(serialization_dir, file_friendly_logging)
serialization_params = deepcopy(params).as_dict(quiet=True)
with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file:
json.dump(serialization_params, param_file, indent=4)
if params.pop('model', None):
logger.warning("You passed parameters for the model in your configuration file, but we "
"are ignoring them, using instead the model parameters in the archive.")
vocabulary_params = params.pop('vocabulary', {})
if vocabulary_params.get('directory_path', None):
logger.warning("You passed `directory_path` in parameters for the vocabulary in "
"your configuration file, but it will be ignored. "
"Vocabulary from the saved model will be extended with current data.")
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("Extending model vocabulary using %s data.", ", ".join(datasets_for_vocab_creation))
vocab = model.vocab
vocab.extend_from_instances(vocabulary_params,
(instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation))
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
trainer = Trainer.from_params(model,
serialization_dir,
iterator,
train_data,
validation_data,
trainer_params)
evaluate_on_test = params.pop_bool("evaluate_on_test", False)
params.assert_empty('base train command')
try:
metrics = trainer.train()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)):
logging.info("Fine-tuning interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
raise
# Now tar up results
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
if test_data and evaluate_on_test:
test_metrics = evaluate(model, test_data, iterator, cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access
for key, value in test_metrics.items():
metrics["test_" + key] = value
elif test_data:
logger.info("To evaluate on the test set after training, pass the "
"'evaluate_on_test' flag, or use the 'allennlp evaluate' command.")
metrics_json = json.dumps(metrics, indent=2)
with open(os.path.join(serialization_dir, "metrics.json"), "w") as metrics_file:
metrics_file.write(metrics_json)
logger.info("Metrics: %s", metrics_json)
return model
def find_learning_rate_model(params: Params, serialization_dir: str,
start_lr: float = 1e-5,
end_lr: float = 10,
num_batches: int = 100,
linear_steps: bool = False,
stopping_factor: float = None,
force: bool = False) -> None:
"""
Runs learning rate search for given `num_batches` and saves the results in ``serialization_dir``
Parameters
----------
trainer: :class:`~allennlp.common.registrable.Registrable`
params : ``Params``
A parameter object specifying an AllenNLP Experiment.
serialization_dir : ``str``
The directory in which to save results.
start_lr: ``float``
Learning rate to start the search.
end_lr: ``float``
Learning rate upto which search is done.
num_batches: ``int``
Number of mini-batches to run Learning rate finder.
linear_steps: ``bool``
Increase learning rate linearly if False exponentially.
stopping_factor: ``float``
Stop the search when the current loss exceeds the best loss recorded by
multiple of stopping factor. If ``None`` search proceeds till the ``end_lr``
force: ``bool``
If True and the serialization directory already exists, everything in it will
be removed prior to finding the learning rate.
"""
if os.path.exists(serialization_dir) and force:
shutil.rmtree(serialization_dir)
if os.path.exists(serialization_dir) and os.listdir(serialization_dir):
raise ConfigurationError(f'Serialization directory {serialization_dir} already exists and is '
f'not empty.')
else:
os.makedirs(serialization_dir, exist_ok=True)
prepare_environment(params)
cuda_device = params.params.get('trainer').get('cuda_device', -1)
if isinstance(cuda_device, list):
for device in cuda_device:
check_for_gpu(device)
else:
check_for_gpu(cuda_device)
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
(instance for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation)
)
model = Model.from_params(vocab=vocab, params=params.pop('model'))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
train_data = all_datasets['train']
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
trainer = Trainer.from_params(model,
serialization_dir,
iterator,
train_data,
params=trainer_params,
validation_data=None,
validation_iterator=None)
logger.info(f'Starting learning rate search from {start_lr} to {end_lr} in {num_batches} iterations.')
learning_rates, losses = search_learning_rate(trainer,
start_lr=start_lr,
end_lr=end_lr,
num_batches=num_batches,
linear_steps=linear_steps,
stopping_factor=stopping_factor)
logger.info(f'Finished learning rate search.')
losses = _smooth(losses, 0.98)
_save_plot(learning_rates, losses, os.path.join(serialization_dir, 'lr-losses.png'))
def fine_tune_model(model: Model,
params: Params,
serialization_dir: str,
file_friendly_logging: bool = False) -> Model:
"""
Fine tunes the given model, using a set of parameters that is largely identical to those used
for :func:`~allennlp.commands.train.train_model`, except that the ``model`` section is ignored,
if it is present (as we are already given a ``Model`` here).
The main difference between the logic done here and the logic done in ``train_model`` is that
here we do not worry about vocabulary construction or creating the model object. Everything
else is the same.
Parameters
----------
archive : ``Archive``
A saved model archive that is the result of running the ``train`` command.
train_data_path : ``str``
Path to the training data to use for fine-tuning.
serialization_dir : ``str``
The directory in which to save results and logs.
validation_data_path : ``str``, optional
Path to the validation data to use while fine-tuning.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
"""
prepare_environment(params)
os.makedirs(serialization_dir)
prepare_global_logging(serialization_dir, file_friendly_logging)
serialization_params = deepcopy(params).as_dict(quiet=True)
with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file:
json.dump(serialization_params, param_file, indent=4)
if params.pop('model', None):
logger.warning(
"You passed parameters for the model in your configuration file, but we "
"are ignoring them, using instead the model parameters in the archive."
)
if params.pop('vocabulary', None):
logger.warning(
"You passed parameters for the vocabulary in your configuration file, but "
"we are ignoring them, using instead the vocabulary from the saved model."
)
vocab = model.vocab
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
all_datasets = datasets_from_params(params)
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
trainer = Trainer.from_params(model, serialization_dir, iterator,
train_data, validation_data, trainer_params)
evaluate_on_test = params.pop_bool("evaluate_on_test", False)
params.assert_empty('base train command')
try:
metrics = trainer.train()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)):
logging.info(
"Fine-tuning interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(serialization_dir,
files_to_archive=params.files_to_archive)
raise
# Now tar up results
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
if test_data and evaluate_on_test:
test_metrics = evaluate(model,
test_data,
iterator,
cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access
for key, value in test_metrics.items():
metrics["test_" + key] = value
elif test_data:
logger.info(
"To evaluate on the test set after training, pass the "
"'evaluate_on_test' flag, or use the 'allennlp evaluate' command.")
metrics_json = json.dumps(metrics, indent=2)
with open(os.path.join(serialization_dir, "metrics.json"),
"w") as metrics_file:
metrics_file.write(metrics_json)
logger.info("Metrics: %s", metrics_json)
return model
def fine_tune_model(model: Model,
params: Params,
serialization_dir: str,
extend_vocab: bool = False,
file_friendly_logging: bool = False) -> Model:
"""
Fine tunes the given model, using a set of parameters that is largely identical to those used
for :func:`~allennlp.commands.train.train_model`, except that the ``model`` section is ignored,
if it is present (as we are already given a ``Model`` here).
The main difference between the logic done here and the logic done in ``train_model`` is that
here we do not worry about vocabulary construction or creating the model object. Everything
else is the same.
Parameters
----------
archive : ``Archive``
A saved model archive that is the result of running the ``train`` command.
train_data_path : ``str``
Path to the training data to use for fine-tuning.
serialization_dir : ``str``
The directory in which to save results and logs.
validation_data_path : ``str``, optional
Path to the validation data to use while fine-tuning.
extend_vocab: ``bool``, optional (default=False)
If ``True``, we use the new instances to extend your vocabulary.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
"""
prepare_environment(params)
if os.path.exists(serialization_dir) and os.listdir(serialization_dir):
raise ConfigurationError(
f"Serialization directory ({serialization_dir}) "
f"already exists and is not empty.")
os.makedirs(serialization_dir, exist_ok=True)
prepare_global_logging(serialization_dir, file_friendly_logging)
serialization_params = deepcopy(params).as_dict(quiet=True)
with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file:
json.dump(serialization_params, param_file, indent=4)
if params.pop('model', None):
logger.warning(
"You passed parameters for the model in your configuration file, but we "
"are ignoring them, using instead the model parameters in the archive."
)
vocabulary_params = params.pop('vocabulary', {})
if vocabulary_params.get('directory_path', None):
logger.warning(
"You passed `directory_path` in parameters for the vocabulary in "
"your configuration file, but it will be ignored. ")
all_datasets = datasets_from_params(params)
vocab = model.vocab
if extend_vocab:
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info("Extending model vocabulary using %s data.",
", ".join(datasets_for_vocab_creation))
vocab.extend_from_instances(
vocabulary_params,
(instance for key, dataset in all_datasets.items()
for instance in dataset if key in datasets_for_vocab_creation))
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(model.vocab)
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
trainer = Trainer.from_params(model, serialization_dir, iterator,
train_data, validation_data, trainer_params)
evaluate_on_test = params.pop_bool("evaluate_on_test", False)
params.assert_empty('base train command')
try:
metrics = trainer.train()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)):
logging.info(
"Fine-tuning interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(serialization_dir,
files_to_archive=params.files_to_archive)
raise
# Now tar up results
archive_model(serialization_dir, files_to_archive=params.files_to_archive)
if test_data and evaluate_on_test:
test_metrics = evaluate(model,
test_data,
iterator,
cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access
for key, value in test_metrics.items():
metrics["test_" + key] = value
elif test_data:
logger.info(
"To evaluate on the test set after training, pass the "
"'evaluate_on_test' flag, or use the 'allennlp evaluate' command.")
metrics_json = json.dumps(metrics, indent=2)
with open(os.path.join(serialization_dir, "metrics.json"),
"w") as metrics_file:
metrics_file.write(metrics_json)
logger.info("Metrics: %s", metrics_json)
return model
