def test_registry_has_builtin_dataset_readers(self):
assert DatasetReader.by_name(u'snli').__name__ == u'SnliReader'
assert DatasetReader.by_name(
u'sequence_tagging').__name__ == u'SequenceTaggingDatasetReader'
assert DatasetReader.by_name(
u'language_modeling').__name__ == u'LanguageModelingReader'
assert DatasetReader.by_name(u'squad').__name__ == u'SquadReader'
def datasets_from_params(params: Params) -> Dict[str, Iterable[Instance]]:
"""
Load all the datasets specified by the config.
"""
dataset_reader = DatasetReader.from_params(params.pop('dataset_reader'))
validation_dataset_reader_params = params.pop("validation_dataset_reader", None)
validation_and_test_dataset_reader: DatasetReader = dataset_reader
if validation_dataset_reader_params is not None:
logger.info("Using a separate dataset reader to load validation and test data.")
validation_and_test_dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
train_data_path = params.pop('train_data_path')
logger.info("Reading training data from %s", train_data_path)
train_data = dataset_reader.read(train_data_path)
datasets: Dict[str, Iterable[Instance]] = {"train": train_data}
validation_data_path = params.pop('validation_data_path', None)
if validation_data_path is not None:
logger.info("Reading validation data from %s", validation_data_path)
validation_data = validation_and_test_dataset_reader.read(validation_data_path)
datasets["validation"] = validation_data
test_data_path = params.pop("test_data_path", None)
if test_data_path is not None:
logger.info("Reading test data from %s", test_data_path)
test_data = validation_and_test_dataset_reader.read(test_data_path)
datasets["test"] = test_data
return datasets
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger("allennlp.common.params").disabled = True
logging.getLogger("allennlp.nn.initializers").disabled = True
logging.getLogger("allennlp.modules.token_embedders.embedding").setLevel(
logging.INFO)
# Load from archive
archive = load_archive(
args.archive_file,
weights_file=args.weights_file,
cuda_device=args.cuda_device,
overrides=args.overrides,
)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop("validation_dataset_reader",
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop("dataset_reader"))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
embedding_sources = (json.loads(args.embedding_sources_mapping)
if args.embedding_sources_mapping else {})
if args.extend_vocab:
logger.info("Vocabulary is being extended with test instances.")
model.vocab.extend_from_instances(instances=instances)
model.extend_embedder_vocab(embedding_sources)
instances.index_with(model.vocab)
data_loader_params = config.pop("validation_data_loader", None)
if data_loader_params is None:
data_loader_params = config.pop("data_loader")
if args.batch_size:
data_loader_params["batch_size"] = args.batch_size
data_loader = DataLoader.from_params(dataset=instances,
params=data_loader_params)
metrics = evaluate(model, data_loader, args.cuda_device,
args.batch_weight_key)
logger.info("Finished evaluating.")
dump_metrics(args.output_file, metrics, log=True)
return metrics
def datasets_from_params(params: Params) -> Dict[str, Iterable[Instance]]:
"""
Load all the datasets specified by the config.
"""
dataset_reader = DatasetReader.from_params(params.pop('dataset_reader'))
validation_dataset_reader_params = params.pop("validation_dataset_reader", None)
validation_and_test_dataset_reader: DatasetReader = dataset_reader
if validation_dataset_reader_params is not None:
logger.info("Using a separate dataset reader to load validation and test data.")
validation_and_test_dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
train_data_path = params.pop('train_data_path')
logger.info("Reading training data from %s", train_data_path)
train_data = dataset_reader.read(train_data_path)
datasets: Dict[str, Iterable[Instance]] = {"train": train_data}
validation_data_path = params.pop('validation_data_path', None)
if validation_data_path is not None:
logger.info("Reading validation data from %s", validation_data_path)
validation_data = validation_and_test_dataset_reader.read(validation_data_path)
datasets["validation"] = validation_data
test_data_path = params.pop("test_data_path", None)
if test_data_path is not None:
logger.info("Reading test data from %s", test_data_path)
test_data = validation_and_test_dataset_reader.read(test_data_path)
datasets["test"] = test_data
return datasets
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides, args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader', None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(config.pop('dataset_reader'))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
iterator_params = config.pop("validation_iterator", None)
if iterator_params is None:
iterator_params = config.pop("iterator")
iterator = DataIterator.from_params(iterator_params)
iterator.index_with(model.vocab)
def test_registry_has_builtin_dataset_readers(self):
assert DatasetReader.by_name("snli").__name__ == "SnliReader"
assert DatasetReader.by_name(
"sequence_tagging").__name__ == "SequenceTaggingDatasetReader"
assert DatasetReader.by_name(
"language_modeling").__name__ == "LanguageModelingReader"
assert DatasetReader.by_name("squad").__name__ == "SquadReader"
def datasets_from_params(params ) :
u"""
Load all the datasets specified by the config.
"""
dataset_reader = DatasetReader.from_params(params.pop(u'dataset_reader'))
validation_dataset_reader_params = params.pop(u"validation_dataset_reader", None)
validation_and_test_dataset_reader = dataset_reader
if validation_dataset_reader_params is not None:
logger.info(u"Using a separate dataset reader to load validation and test data.")
validation_and_test_dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
train_data_path = params.pop(u'train_data_path')
logger.info(u"Reading training data from %s", train_data_path)
train_data = dataset_reader.read(train_data_path)
datasets = {u"train": train_data}
validation_data_path = params.pop(u'validation_data_path', None)
if validation_data_path is not None:
logger.info(u"Reading validation data from %s", validation_data_path)
validation_data = validation_and_test_dataset_reader.read(validation_data_path)
datasets[u"validation"] = validation_data
test_data_path = params.pop(u"test_data_path", None)
if test_data_path is not None:
logger.info(u"Reading test data from %s", test_data_path)
test_data = validation_and_test_dataset_reader.read(test_data_path)
datasets[u"test"] = test_data
return datasets
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides,
args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader',
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop('dataset_reader'))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
embedding_sources: Dict[str,
str] = (json.loads(args.embedding_sources_mapping)
if args.embedding_sources_mapping else {})
if args.extend_vocab:
logger.info("Vocabulary is being extended with test instances.")
model.vocab.extend_from_instances(Params({}), instances=instances)
model.extend_embedder_vocab(embedding_sources)
iterator_params = config.pop("validation_iterator", None)
if iterator_params is None:
iterator_params = config.pop("iterator")
iterator = DataIterator.from_params(iterator_params)
iterator.index_with(model.vocab)
metrics = evaluate(model, instances, iterator, args.cuda_device,
args.batch_weight_key)
logger.info("Finished evaluating.")
logger.info("Metrics:")
for key, metric in metrics.items():
logger.info("%s: %s", key, metric)
output_file = args.output_file
if output_file:
with open(output_file, "w") as file:
json.dump(metrics, file, indent=4)
return metrics
def test_registry_has_builtin_dataset_readers(self):
assert DatasetReader.by_name('snli').__name__ == 'SnliReader'
assert DatasetReader.by_name(
'sequence_tagging').__name__ == 'SequenceTaggingDatasetReader'
assert DatasetReader.by_name(
'language_modeling').__name__ == 'LanguageModelingReader'
assert DatasetReader.by_name(
'squad_sentence_selection'
).__name__ == 'SquadSentenceSelectionReader'
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Load model from archive
model_archive = load_archive(args.model_archive_file, args.cuda_device,
args.overrides, args.weights_file)
config = model_archive.config
prepare_environment(config)
model = model_archive.model
model.eval()
# Load sampler
sampler_archive = load_archive(args.sampler_archive_file, args.cuda_device,
args.overrides, args.weights_file)
sampler = sampler_archive.model
sampler.eval()
# Load the evaluation data. NOTE: We are using the model's reader!
validation_dataset_reader_params = config.pop('validation_dataset_reader',
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop('dataset_reader'))
evaluation_data_path = args.input_file
logger.info('Reading evaluation data from: %s', evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
# To avoid hairy issues with splitting, we opt to use a basic iterator so that we can
# generate samples for entire sequences.
iterator_params = config.pop('iterator', 'None')
iterator = DataIterator.from_params(iterator_params)
iterator.index_with(model.vocab)
# iterator.eval()
metrics = evaluate_perplexity(model, sampler, args.num_samples, instances,
iterator, args.cuda_device)
logger.info('Finished evaluating.')
logger.info('Metrics:')
for key, metric in metrics.items():
logger.info('%s: %s', key, metric)
output_file = args.output_file
if output_file:
with open(output_file, 'w') as f:
json.dump(metrics, f, indent=4)
return metrics
def __init__(
self,
token_indexers: Dict[str, TokenIndexer] = None,
domain_identifier: str = None,
model_name: str = None,
**kwargs,
) -> None:
DatasetReader.__init__(self, **kwargs)
self._token_indexers = token_indexers or {
"tokens": PretrainedTransformerIndexer(model_name)
}
self._domain_identifier = domain_identifier
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.lowercase_input = "uncased" in model_name
def datasets_from_params(params: Params) -> Dict[str, Iterable[Instance]]:
"""
Load all the datasets specified by the config.
"""
fully_labelled_threshold = 3000 if 'fully_labelled_threshold' not in params[
'dataset_reader'] else params['dataset_reader'][
'fully_labelled_threshold']
dataset_reader = DatasetReader.from_params(
params.pop("dataset_reader", None))
validation_dataset_reader_params = params.pop("validation_dataset_reader",
None)
validation_and_test_dataset_reader: DatasetReader = dataset_reader
if validation_dataset_reader_params is not None:
logger.info(
"Using a separate dataset reader to load validation and test data."
)
validation_and_test_dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
# Split train data into held out/not held out, initializing to 10% non-held-out
# non-held-out training data will have 100% of labels (using dataset_reader)
# held-out training data will have only 50% of labels (using held_out_dataset_reader)
train_data_path = params.pop('train_data_path')
logger.info("Reading training data from %s", train_data_path)
train_data = dataset_reader.read(train_data_path)
held_out_train_data = train_data[
fully_labelled_threshold:] # after threshold
train_data = train_data[:fully_labelled_threshold] # before threshold
datasets: Dict[str, Iterable[Instance]] = {
"train": train_data,
"held_out_train": held_out_train_data
}
validation_data_path = params.pop('validation_data_path', None)
if validation_data_path is not None:
logger.info("Reading validation data from %s", validation_data_path)
validation_data = validation_and_test_dataset_reader.read(
validation_data_path)
datasets["validation"] = validation_data
test_data_path = params.pop("test_data_path", None)
if test_data_path is not None:
logger.info("Reading test data from %s", test_data_path)
test_data = validation_and_test_dataset_reader.read(test_data_path)
datasets["test"] = test_data
return datasets
def evaluate_from_args(args):
# Disable some of the more verbose logging statements
logging.getLogger(u'allennlp.common.params').disabled = True
logging.getLogger(u'allennlp.nn.initializers').disabled = True
logging.getLogger(u'allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides,
args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop(u'validation_dataset_reader',
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop(u'dataset_reader'))
evaluation_data_path = args.input_file
logger.info(u"Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
iterator_params = config.pop(u"validation_iterator", None)
if iterator_params is None:
iterator_params = config.pop(u"iterator")
iterator = DataIterator.from_params(iterator_params)
iterator.index_with(model.vocab)
metrics = evaluate(model, instances, iterator, args.cuda_device)
logger.info(u"Finished evaluating.")
logger.info(u"Metrics:")
for key, metric in list(metrics.items()):
logger.info(u"%s: %s", key, metric)
output_file = args.output_file
if output_file:
with open(output_file, u"w") as file:
json.dump(metrics, file, indent=4)
return metrics
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides,
args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Set the model to error analysis mode
model.error_analysis = True
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader',
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop('dataset_reader'))
evaluation_data_path = args.evaluation_data_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
iterator = DataIterator.from_params(config.pop("iterator"))
iterator.index_with(model.vocab)
metrics = evaluate(model, instances, iterator, args.cuda_device)
logger.info("Finished evaluating.")
print("All Metrics")
print("=" * 79)
for key, metric in metrics.items():
print("{}\t{}".format(key, metric))
# Turn off error analysis mode
model.error_analysis = False
return metrics
def _evaluate_nn(self, model_path: str, evaluation_data_file: str,
cuda_device: int):
"""
:param model_path:
:param evaluation_data_file:
:param cuda_device:
:return:
"""
# import allennlp ontoemma classes (to register -- necessary, do not remove)
from emma.allennlp_classes.ontoemma_dataset_reader import OntologyMatchingDatasetReader
from emma.allennlp_classes.ontoemma_model import OntoEmmaNN
# Load from archive
archive = load_archive(model_path, cuda_device)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
dataset_reader = DatasetReader.from_params(
config.pop('dataset_reader'))
evaluation_data_path = evaluation_data_file
dataset = dataset_reader.read(evaluation_data_path)
# compute metrics
dataset.index_instances(model.vocab)
iterator = DataIterator.from_params(config.pop("iterator"))
metrics = evaluate_allennlp(model, dataset, iterator, cuda_device)
return metrics
def __init__(self,
target_namespace: str,
span_predictor_model,
source_tokenizer: Tokenizer = None,
target_tokenizer: Tokenizer = None,
source_token_indexers: Dict[str, TokenIndexer] = None,
lazy: bool = False,
add_rule = True,
embed_span = True,
add_question = True,
add_followup_ques = True,
train_using_gold = True)-> None:
super().__init__(lazy)
self._target_namespace = target_namespace
self._source_tokenizer = source_tokenizer or WordTokenizer()
self._target_tokenizer = target_tokenizer or self._source_tokenizer
self._source_token_indexers = source_token_indexers or {"tokens": SingleIdTokenIndexer()}
self.add_rule = add_rule
self.embed_span = embed_span
self.add_question = add_question
self.add_followup_ques = add_followup_ques
self.train_using_gold = train_using_gold
if "tokens" not in self._source_token_indexers or \
not isinstance(self._source_token_indexers["tokens"], SingleIdTokenIndexer):
raise ConfigurationError("CopyNetDatasetReader expects 'source_token_indexers' to contain "
"a 'single_id' token indexer called 'tokens'.")
self._target_token_indexers: Dict[str, TokenIndexer] = {
"tokens": SingleIdTokenIndexer(namespace=self._target_namespace)
}
archive = load_archive(span_predictor_model)
self.dataset_reader = DatasetReader.from_params(archive.config.duplicate()["dataset_reader"])
self.span_predictor = Predictor.from_archive(archive, 'sharc_predictor')
def load_predictor(serialization_dir, device):
## Load the model
archive = load_archive(join(serialization_dir, 'model.tar.gz'))
model = archive.model.eval()
if device >= 0:
model.to(0)
## Load the dataset reader
dataset_reader_params = archive.config.pop('dataset_reader')
model_name = archive.config.pop('model')['type']
# Turn off truncation of the inputs
if model_name == 'gnli':
pass
# dataset_reader_params.params['max_premise_length'] = None
# dataset_reader_params.params['max_hypothesis_length'] = None
elif model_name == 'bertnli':
dataset_reader_params.params['max_seq_length'] = None
else:
raise ValueError()
reader = DatasetReader.by_name(dataset_reader_params.pop('type')).from_params(dataset_reader_params)
predictor = Predictor(model, reader)
return predictor
def run(model_path, test_path, config_path, output_path, batch_size):
params_path = config_path or os.path.join(model_path, "config.json")
params = Params.from_file(params_path)
is_subwords = "tokenizer" in params["reader"] and params["reader"][
"tokenizer"]["type"] == "subword"
reader = DatasetReader.from_params(params.pop("reader"))
device = 0 if torch.cuda.is_available() else -1
model = Model.load(params, model_path, cuda_device=device)
model.training = False
predictor = Seq2SeqPredictor(model, reader)
with open(output_path, "wt", encoding="utf-8") as w:
for batch_number, batch in enumerate(get_batches(
test_path, batch_size)):
outputs = predictor.predict_batch_json(batch)
assert len(outputs) == len(batch)
for output in outputs:
decoded_words = output["predicted_tokens"]
if not decoded_words:
decoded_words = ["заявил"]
if not is_subwords:
hyp = " ".join(decoded_words)
else:
hyp = "".join(decoded_words).replace("▁", " ").replace(
"\n", "").strip()
if len(hyp) <= 3:
hyp = "заявил"
w.write(hyp + "\n")
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(logging.INFO)
# Load parameter file
with open(args.config_file) as config_file:
config = Params(replace_none(json.loads(config_file.read())))
model = Model.load(config,
weights_file=args.weights_file,
cuda_device=args.cuda_device)
model.eval()
vocab = model._vocab # pylint: disable=protected-access
# Load the evaluation data
dataset_reader = DatasetReader.from_params(config.pop('dataset_reader'))
evaluation_data_path = args.evaluation_data_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
dataset = dataset_reader.read(evaluation_data_path)
dataset.index_instances(vocab)
iterator = DataIterator.from_params(config.pop("iterator"))
metrics = evaluate(model, dataset, iterator, args.cuda_device)
logger.info("Finished evaluating.")
logger.info("Metrics:")
for key, metric in metrics.items():
logger.info("%s: %s", key, metric)
return metrics
def train(model_path, train_path, val_path, seed, vocabulary_path=None, config_path=None):
assert os.path.isdir(model_path), "Model directory does not exist"
set_seed(seed)
config_path = config_path or os.path.join(model_path, "config.json")
assert os.path.isfile(config_path), "Config file does not exist"
params = Params.from_file(config_path)
vocabulary_path = vocabulary_path or os.path.join(model_path, "vocabulary")
assert os.path.exists(vocabulary_path), "Vocabulary is not ready, do not forget to run preprocess.py first"
vocabulary = Vocabulary.from_files(vocabulary_path)
reader_params = params.duplicate().pop("reader", default=Params({}))
reader = DatasetReader.from_params(reader_params)
train_dataset = reader.read(train_path)
val_dataset = reader.read(val_path) if val_path else None
model_params = params.pop("model")
model = Model.from_params(model_params, vocab=vocabulary)
print(model)
print("Trainable params count: ", sum(p.numel() for p in model.parameters() if p.requires_grad))
iterator = DataIterator.from_params(params.pop('iterator'))
iterator.index_with(vocabulary)
trainer = Trainer.from_params(model, model_path, iterator,
train_dataset, val_dataset, params.pop('trainer'))
trainer.train()
def datasets_from_params(params: Params) -> Dict[str, InstanceCollection]:
"""
Load all the datasets specified by the config.
"""
dataset_reader = DatasetReader.from_params(params.pop('dataset_reader'))
train_data_path = params.pop('train_data_path')
logger.info("Reading training data from %s", train_data_path)
train_data = dataset_reader.read(train_data_path)
datasets: Dict[str, InstanceCollection] = {"train": train_data}
validation_data_path = params.pop('validation_data_path', None)
if validation_data_path is not None:
logger.info("Reading validation data from %s", validation_data_path)
validation_data = dataset_reader.read(validation_data_path)
datasets["validation"] = validation_data
test_data_path = params.pop("test_data_path", None)
if test_data_path is not None:
logger.info("Reading test data from %s", test_data_path)
test_data = dataset_reader.read(test_data_path)
datasets["test"] = test_data
return datasets
def test_can_read_partial_instances(self, partial_data_path, params: Params) -> None:
reader = DatasetReader.from_params(params)
instances = reader.read_partial(partial_data_path)
vocab = Vocabulary.from_instances(instances)
assert "partial_labels" in vocab._token_to_index
assert set(vocab.get_token_to_index_vocabulary("partial_labels").keys()) == \
set(["I-<UNK>", "O", "I-type", "I-attr", "I-location"])
def _load_reader_and_predictor(basedir):
config_path = os.path.join(basedir, 'config.json')
config = Params.from_file(config_path)
model = Model.load(config=config, serialization_dir=basedir)
reader = DatasetReader.from_params(config.get('dataset_reader'))
predictor = Predictor(model=model, dataset_reader=reader)
return reader, predictor
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
dataset_reader = DatasetReader.from_params(config.pop('dataset_reader'))
evaluation_data_path = args.evaluation_data_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
dataset = dataset_reader.read(evaluation_data_path)
dataset.index_instances(model.vocab)
iterator = DataIterator.from_params(config.pop("iterator"))
metrics = evaluate(model, dataset, iterator, args.cuda_device)
logger.info("Finished evaluating.")
logger.info("Metrics:")
for key, metric in metrics.items():
logger.info("%s: %s", key, metric)
return metrics
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Import any additional modules needed (to register custom classes)
for package_name in args.include_package:
import_submodules(package_name)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides,
args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader',
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(
config.pop('dataset_reader'))
evaluation_data_path = args.evaluation_data_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
iterator = DataIterator.from_params(config.pop("iterator"))
iterator.index_with(model.vocab)
metrics = evaluate(model, instances, iterator, args.cuda_device)
logger.info("Finished evaluating.")
logger.info("Metrics:")
for key, metric in metrics.items():
logger.info("%s: %s", key, metric)
return metrics
def evaluate(test_path,
batch_size,
metric,
max_count,
report_every,
is_multiple_ref=False,
model_path=None,
model_config_path=None,
baseline=None,
reader_config_path=None,
detokenize_after=False):
reader_params = get_reader_params(reader_config_path, model_config_path,
model_path)
is_subwords = "tokenizer" in reader_params and reader_params["tokenizer"][
"type"] == "subword"
reader = DatasetReader.from_params(reader_params)
run_model = get_model_runner(model_path, reader) if not baseline else None
hyps = []
refs = []
for batch in get_batches(reader, test_path, batch_size):
batch_refs, batch_hyps = run_model(
batch) if not baseline else run_baseline(batch, baseline)
for ref, hyp in zip(batch_refs, batch_hyps):
hyp = hyp if not is_subwords else "".join(hyp.split(" ")).replace(
"▁", " ")
if is_multiple_ref:
reference_sents = ref.split(" s_s ")
decoded_sents = hyp.split("s_s")
hyp = [
w.replace("<", "<").replace(">", ">").strip()
for w in decoded_sents
]
ref = [
w.replace("<", "<").replace(">", ">").strip()
for w in reference_sents
]
hyp = " ".join(hyp)
ref = " ".join(ref)
ref = ref.strip()
hyp = hyp.strip()
if detokenize_after:
hyp = detokenize(hyp)
ref = detokenize(ref)
if isinstance(ref, str) and len(ref) <= 1:
ref = "some content"
print("Empty ref")
if isinstance(hyp, str) and len(hyp) <= 1:
hyp = "some content"
print("Empty hyp")
refs.append(ref)
hyps.append(hyp)
if len(hyps) % report_every == 0:
calc_metrics(refs, hyps, metric)
if max_count and len(hyps) >= max_count:
break
calc_metrics(refs, hyps, metric)
def datasets_from_params(params: Params) -> Dict[str, Iterable[Instance]]:
"""
Load all the datasets specified by the config.
"""
dataset_reader = DatasetReader.from_params(params.pop('dataset_reader'))
validation_dataset_reader_params = params.pop("validation_dataset_reader",
None)
validation_and_test_dataset_reader: DatasetReader = dataset_reader
if validation_dataset_reader_params is not None:
logger.info(
"Using a separate dataset reader to load validation and test data."
)
validation_and_test_dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
# train_data_path = params.pop('train_data_path')
# logger.info("Reading training data from %s", train_data_path)
# train_data = dataset_reader.read(train_data_path)
# datasets: Dict[str, Iterable[Instance]] = {"train": train_data}
datasets: Dict[str, Iterable[Instance]] = {}
validation_data_path = params.pop('validation_data_path', None)
if validation_data_path is not None:
logger.info("Reading validation data from %s", validation_data_path)
validation_data = validation_and_test_dataset_reader.read(
validation_data_path)
datasets["validation"] = validation_data
test_data_path = validation_data_path.replace(
'development', 'test') if validation_data_path else None
if test_data_path is not None:
logger.info("Reading test data from %s", test_data_path)
test_data = validation_and_test_dataset_reader.read(test_data_path)
datasets["test"] = test_data
#other_data_path = params.pop("other_data_path",None)
#if other_data_path is not None:
# logger.info("Reading other data from %s", other_data_path)
# other_data_path = validation_and_test_dataset_reader.read(other_data_path)
# datasets["other"] = other_data_path
return datasets
def test_sentence_markers(self, data_path: str, sentence_marker_params: Params) -> None:
reader = DatasetReader.from_params(sentence_marker_params)
instances = reader.read(data_path)
# vocab = Vocabulary.from_instances(instances)
for instance in instances:
tokens = instance["tokens"]
sentence_markers = instance["metadata"]["sentence_markers"]
sentences = get_sentences_from_markers(tokens, sentence_markers)
assert sum(len(x) for x in sentences) == len(tokens) == sentence_markers[-1]
def target_to_lines(archive_file, input_file, output_file, lowercase=True):
archive = load_archive(archive_file)
reader = DatasetReader.from_params(archive.config.pop("dataset_reader"))
with open(output_file, "w") as w:
for t in reader.parse_set(input_file):
target = t[1]
target = target.strip()
target = target.lower() if lowercase else target
w.write(target.replace("\n", " ") + "\n")
def evaluate_from_args(args: argparse.Namespace) -> Dict[str, Any]:
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides, args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader', None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(config.pop('dataset_reader'))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
iterator_params = config.pop("validation_iterator", None)
if iterator_params is None:
iterator_params = config.pop("iterator")
iterator = DataIterator.from_params(iterator_params)
iterator.index_with(model.vocab)
metrics = evaluate(model, instances, iterator, args.cuda_device, args.batch_weight_key)
logger.info("Finished evaluating.")
logger.info("Metrics:")
for key, metric in metrics.items():
logger.info("%s: %s", key, metric)
output_file = args.output_file
if output_file:
with open(output_file, "w") as file:
json.dump(metrics, file, indent=4)
return metrics
def main(config: str, model_th: str, dataset: str, hypo_file: str, ref_file: str,
batch_size: int, no_gpu: bool):
logger = logging.getLogger(__name__)
logger.info("Loading configuration parameters")
params = Params.from_file(config)
vocab_params = params.pop("vocabulary")
vocab = Vocabulary.from_params(vocab_params)
reader_params = params.pop("dataset_reader")
reader_name = reader_params.pop("type")
# reader_params["lazy"] = True # make sure we do not load the entire dataset
reader = DatasetReader.by_name(reader_name).from_params(reader_params)
logger.info("Reading data from {}".format(dataset))
data = reader.read(dataset)
iterator = BasicIterator(batch_size=batch_size)
iterator.index_with(vocab)
batches = iterator._create_batches(data, shuffle=False)
logger.info("Loading model")
model_params = params.pop("model")
model_name = model_params.pop("type")
model = Model.by_name(model_name).from_params(model_params, vocab=vocab)
if not no_gpu:
model.cuda(0)
with open(model_th, 'rb') as f:
if no_gpu:
state_dict = torch.load(f, map_location=torch.device('cpu'))
else:
state_dict = torch.load(f)
model.load_state_dict(state_dict)
predictor = Seq2SeqPredictor(model, reader)
model.eval()
with open(hypo_file, 'w') as hf, open(ref_file, 'w') as rf:
logger.info("Generating predictions")
for sample in tqdm(batches):
s = list(sample)
pred = predictor.predict_batch_instance(s)
for inst, p in zip(s, pred):
print(
" ".join(p["predicted_tokens"][0]),
file=hf
)
print(
" ".join(t.text for t in inst["target_tokens"][1:-1]),
file=rf
)
def preprocess(train_path, vocabulary_path, config_path):
params = Params.from_file(config_path)
reader_params = params.pop("reader", default=Params({}))
reader = DatasetReader.from_params(reader_params)
dataset = reader.read(train_path)
vocabulary_params = params.pop("vocabulary", default=Params({}))
vocabulary = Vocabulary.from_params(vocabulary_params, instances=dataset)
vocabulary.save_to_files(vocabulary_path)
def _worker(reader: DatasetReader,
input_queue: Queue,
output_queue: Queue,
index: int) -> None:
"""
A worker that pulls filenames off the input queue, uses the dataset reader
to read them, and places the generated instances on the output queue.
When there are no filenames left on the input queue, it puts its ``index``
on the output queue and doesn't do anything else.
"""
# Keep going until you get a file_path that's None.
while True:
file_path = input_queue.get()
if file_path is None:
# Put my index on the queue to signify that I'm finished
output_queue.put(index)
break
logger.info(f"reading instances from {file_path}")
for instance in reader.read(file_path):
output_queue.put(instance)
def test_registry_has_builtin_dataset_readers(self):
assert DatasetReader.by_name('snli').__name__ == 'SnliReader'
assert DatasetReader.by_name('sequence_tagging').__name__ == 'SequenceTaggingDatasetReader'
assert DatasetReader.by_name('language_modeling').__name__ == 'LanguageModelingReader'
assert DatasetReader.by_name('squad').__name__ == 'SquadReader'
