def from_archive(cls,
generative_archive: Archive,
discriminative_archive: Archive,
predictor_name: str = None) -> 'Predictor':
"""
Instantiate a :class:`CompleteTheSentencePredictor` from a :class:`~allennlp.models.archival.Archive`;
that is, from the result of training a model. Optionally specify which `Predictor`
subclass; otherwise, the default one for the model will be used.
"""
# We need to duplicate the configs so that they do not get consumed inside the archive
generative_config = generative_archive.config.duplicate()
discriminative_config = discriminative_archive.config.duplicate()
model = generative_archive.model
sampler = discriminative_archive.model
dataset_reader_params_model = generative_config['dataset_reader']
dataset_reader_params_sampler = discriminative_config['dataset_reader']
dataset_reader_model = DatasetReader.from_params(
dataset_reader_params_model)
dataset_reader_sampler = DatasetReader.from_params(
dataset_reader_params_sampler)
model.eval()
sampler.eval()
return Predictor.by_name(predictor_name)(model, sampler,
dataset_reader_model,
dataset_reader_sampler)
def read_data(
reader: DatasetReader
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read(TRAIN_PATH)
validation_data = reader.read(DEV_PATH)
return training_data, validation_data
def read_data(
reader: DatasetReader
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read("/path/to/your/training/data")
validation_data = reader.read("/path/to/your/validation/data")
return training_data, validation_data
def read_data(
reader: DatasetReader
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read("quick_start/data/movie_review/train.tsv")
validation_data = reader.read("quick_start/data/movie_review/dev.tsv")
return training_data, validation_data
def read_data(reader: DatasetReader, tgt_domain: str, input_path: str,
domains: List) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = None
for domain in domains:
if domain != tgt_domain:
if training_data == None:
training_data = reader.read(input_path + domain + '/' +
domain + '_neg.txt')
else:
training_data += reader.read(input_path + domain + '/' +
domain + '_neg.txt')
valid_test_data = reader.read(input_path + tgt_domain + '/' + tgt_domain +
'_neg.txt')
as_per_percent = int(len(valid_test_data) * 0.25)
valid_size = 2000 if as_per_percent >= 2000 else as_per_percent
validation_data = valid_test_data[:valid_size]
test_data = valid_test_data[valid_size:]
training_data = AllennlpDataset(training_data)
validation_data = AllennlpDataset(validation_data)
test_data = AllennlpDataset(test_data)
print("train:", len(training_data), "validation:", len(validation_data),
"test:", len(test_data))
return training_data, validation_data, test_data
def read_data(
reader: DatasetReader,
train_data_path:
str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv",
valid_data_path:
str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/test/listfile.csv"
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
logger.critical("Reading the data. Lazy variable set to {}".format(
reader.lazy))
start_time = time.time()
'''Expect: that this is the only time it is called'''
reader.mode = "train"
training_data = reader.read(train_data_path)
# instead, we will set the examples differently here
reader.mode = "valid"
validation_data = reader.read(
valid_data_path) #need to unlimit the examples here...
logger.critical(
"Finished the call to read the data. Time took {}".format(time.time() -
start_time))
return training_data, validation_data
def read_data(
reader: DatasetReader
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read("data/train.tsv")
validation_data = reader.read("data/dev.tsv")
return training_data, validation_data
def read_data(
reader: DatasetReader
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read("./data/sample/train_dataset.json")
validation_data = reader.read(
"./data/sample/train_dataset.json") # TODO: same data
return training_data, validation_data
def main(span_model_path: str, span_to_question_model_path: str,
cuda_device: int, input_file: str, output_file: str,
span_min_prob: float, question_min_prob: float,
question_beam_size: int) -> None:
check_for_gpu(cuda_device)
span_model_archive = load_archive_from_folder(
span_model_path,
cuda_device=cuda_device,
overrides=
'{ "model": { "span_selector": {"span_decoding_threshold": 0.00} } }',
weights_file=os.path.join(span_model_path, "best.th"))
# override span detection threshold to be low enough so we can reasonably approximate bad spans
# as having probability 0.
span_to_question_model_archive = load_archive_from_folder(
span_to_question_model_path,
cuda_device=cuda_device,
weights_file=os.path.join(span_to_question_model_path, "best.th"))
span_model_dataset_reader_params = span_model_archive.config[
"dataset_reader"].duplicate()
span_model_dataset_reader_params["qasrl_filter"]["allow_all"] = True
span_to_question_model_dataset_reader_params = span_to_question_model_archive.config[
"dataset_reader"].duplicate()
span_to_question_model_dataset_reader_params["qasrl_filter"][
"allow_all"] = True
pipeline = AFirstPipelineSequential(
span_model=span_model_archive.model,
span_model_dataset_reader=DatasetReader.from_params(
span_model_dataset_reader_params),
span_to_question_model=span_to_question_model_archive.model,
span_to_question_model_dataset_reader=DatasetReader.from_params(
span_to_question_model_dataset_reader_params),
span_minimum_threshold=span_min_prob,
question_minimum_threshold=question_min_prob,
question_beam_size=question_beam_size)
if output_file is None:
for line in tqdm(read_lines(cached_path(input_file))):
input_json = json.loads(line)
output_json = pipeline.predict(input_json)
print(json.dumps(output_json))
elif output_file.endswith('.gz'):
with gzip.open(output_file, 'wt') as f:
for line in tqdm(read_lines(cached_path(input_file))):
input_json = json.loads(line)
output_json = pipeline.predict(input_json)
f.write(json.dumps(output_json))
f.write('\n')
else:
with open(output_file, 'w', encoding='utf8') as out:
for line in tqdm(read_lines(cached_path(input_file))):
input_json = json.loads(line)
output_json = pipeline.predict(input_json)
print(json.dumps(output_json), file=out)
def read_data(
reader: DatasetReader,
train_data_path: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv",
valid_data_path: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/test/listfile.csv"
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read(train_data_path)
validation_data = reader.read(valid_data_path)
return training_data, validation_data
def read_data(reader: DatasetReader) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
training_data = reader.read('../data/snips/utterances_train_features.txt')
validation_data = reader.read('../data/snips/utterances_valid_features.txt')
training_data = AllennlpDataset(training_data)
validation_data = AllennlpDataset(validation_data)
print("train:",len(training_data), "validation:", len(validation_data))
return training_data, validation_data
def evaluate_from_args(args: argparse.Namespace):
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides, args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader', None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(validation_dataset_reader_params)
else:
dataset_reader = DatasetReader.from_params(config.pop('dataset_reader'))
evaluation_data_path = args.input_file
logger.info("Reading evaluation data from %s", evaluation_data_path)
instances = dataset_reader.read(evaluation_data_path)
embedding_sources: Dict[str, str] = (json.loads(args.embedding_sources_mapping)
if args.embedding_sources_mapping else {})
if args.extend_vocab:
logger.info("Vocabulary is being extended with test instances.")
model.vocab.extend_from_instances(Params({}), instances=instances)
model.extend_embedder_vocab(embedding_sources)
iterator_params = config.pop("validation_iterator", None)
if iterator_params is None:
iterator_params = config.pop("iterator")
iterator = DataIterator.from_params(iterator_params)
iterator.index_with(model.vocab)
csv_writer = csv.writer(args.output_file)
keys = None
for instance in instances:
metrics = evaluate(model, [instance], iterator, args.cuda_device, args.batch_weight_key)
if keys is None:
keys = sorted(metrics.keys())
csv_writer.writerow(['instance_id', *keys])
instance_id = instance.fields['metadata']['id']
values = [metrics[key] for key in keys]
csv_writer.writerow([instance_id, *values])
def test_from_params(self):
squad1_reader = DatasetReader.from_params(Params({"type": "squad1"}))
assert squad1_reader.no_answer_token is None
squad2_reader = DatasetReader.from_params(Params({"type": "squad2"}))
assert squad2_reader.no_answer_token is not None
with pytest.warns(DeprecationWarning):
squad_reader = DatasetReader.from_params(
Params({
"type": "squad1",
"skip_invalid_examples": True
}))
assert squad_reader.skip_impossible_questions is True
def __init__(self,
question_model_archive: Archive,
# question_model_dataset_reader: QasrlReader,
question_to_span_model_archive: QuestionToSpanModel,
# question_to_span_model_dataset_reader: QasrlReader,
tan_model_archive: Optional[Archive] = None,
span_to_tan_model_archive: Optional[Archive] = None,
animacy_model_archive: Optional[Archive] = None,
question_minimum_threshold: float = question_minimum_threshold_default,
span_minimum_threshold: float = span_minimum_threshold_default,
tan_minimum_threshold: float = tan_minimum_threshold_default,
question_beam_size: int = question_beam_size_default,
clause_mode: bool = False) -> None:
self._question_model = question_model_archive.model
self._question_model_dataset_reader = DatasetReader.from_params(question_model_archive.config["dataset_reader"].duplicate())
print("Question model loaded.", flush = True)
self._question_to_span_model = question_to_span_model_archive.model
self._question_to_span_model_dataset_reader = DatasetReader.from_params(question_to_span_model_archive.config["dataset_reader"].duplicate())
print("Question-to-span model loaded.", flush = True)
if tan_model_archive is not None:
self._tan_model = tan_model_archive.model
self._tan_model_dataset_reader = DatasetReader.from_params(tan_model_archive.config["dataset_reader"].duplicate())
print("TAN model loaded.", flush = True)
else:
self._tan_model = None
if span_to_tan_model_archive is not None:
self._span_to_tan_model = span_to_tan_model_archive.model
self._span_to_tan_model_dataset_reader = DatasetReader.from_params(span_to_tan_model_archive.config["dataset_reader"].duplicate())
print("Span-to-TAN model loaded.", flush = True)
else:
self._span_to_tan_model = None
if animacy_model_archive is not None:
self._animacy_model = animacy_model_archive.model
self._animacy_model_dataset_reader = DatasetReader.from_params(animacy_model_archive.config["dataset_reader"].duplicate())
print("Animacy model loaded.", flush = True)
else:
self._animacy_model = None
print("All models loaded.", flush = True)
self._span_minimum_threshold = span_minimum_threshold
self._question_minimum_threshold = question_minimum_threshold
self._tan_minimum_threshold = tan_minimum_threshold
self._question_beam_size = question_beam_size
self._clause_mode = clause_mode
qg_slots = set(self._question_model.get_slot_names())
qa_slots = set(self._question_to_span_model.get_slot_names())
if not qa_slots.issubset(qg_slots):
raise ConfigurationError(
"Question Answerer must read in a subset of question slots generated by the Question Generator. " + \
("QG slots: %s; QA slots: %s" % (qg_slots, qa_slots)))
def read_data(
train_path: str,
val_path: str,
train_reader: DatasetReader,
val_reader: DatasetReader = None
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
print("Reading data")
print(type(train_reader), train_path)
training_data = train_reader.read(train_path)
if val_reader is None:
validation_data = train_reader.read(val_path)
else:
validation_data = val_reader.read(val_path)
return training_data, validation_data
def setUp(self):
super(TestCopyNetReader, self).setUp()
params = Params.from_file(self.FIXTURES_ROOT / "encoder_decoder" / "copynet_seq2seq" / "experiment.json")
self.reader = DatasetReader.from_params(params["dataset_reader"])
instances = self.reader.read(self.FIXTURES_ROOT / "data" / "copynet" / "copyover.tsv")
self.instances = ensure_list(instances)
self.vocab = Vocabulary.from_params(params=params["vocabulary"], instances=instances)
def test_batch_predictions_are_consistent(self):
# The CNN encoder has problems with this kind of test - it's not properly masked yet, so
# changing the amount of padding in the batch will result in small differences in the
# output of the encoder. Because BiDAF is so deep, these differences get magnified through
# the network and make this test impossible. So, we'll remove the CNN encoder entirely
# from the model for this test. If/when we fix the CNN encoder to work correctly with
# masking, we can change this back to how the other models run this test, with just a
# single line.
# pylint: disable=protected-access,attribute-defined-outside-init
# Save some state.
saved_model = self.model
saved_instances = self.instances
# Modify the state, run the test with modified state.
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
reader._token_indexers = {'tokens': reader._token_indexers['tokens']}
self.instances = reader.read('tests/fixtures/data/squad.json')
vocab = Vocabulary.from_instances(self.instances)
for instance in self.instances:
instance.index_fields(vocab)
del params['model']['text_field_embedder']['token_characters']
params['model']['phrase_layer']['input_size'] = 2
self.model = Model.from_params(vocab, params['model'])
self.ensure_batch_predictions_are_consistent()
# Restore the state.
self.model = saved_model
self.instances = saved_instances
def test_batch_predictions_are_consistent(self):
# The CNN encoder has problems with this kind of test - it's not properly masked yet, so
# changing the amount of padding in the batch will result in small differences in the
# output of the encoder. Because BiDAF is so deep, these differences get magnified through
# the network and make this test impossible. So, we'll remove the CNN encoder entirely
# from the model for this test. If/when we fix the CNN encoder to work correctly with
# masking, we can change this back to how the other models run this test, with just a
# single line.
# pylint: disable=protected-access,attribute-defined-outside-init
# Save some state.
saved_model = self.model
saved_instances = self.instances
# Modify the state, run the test with modified state.
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
reader._token_indexers = {'tokens': reader._token_indexers['tokens']}
self.instances = reader.read(self.FIXTURES_ROOT / 'data' /
'squad.json')
vocab = Vocabulary.from_instances(self.instances)
for instance in self.instances:
instance.index_fields(vocab)
del params['model']['text_field_embedder']['token_embedders'][
'token_characters']
params['model']['phrase_layer']['input_size'] = 2
self.model = Model.from_params(vocab=vocab, params=params['model'])
self.ensure_batch_predictions_are_consistent()
# Restore the state.
self.model = saved_model
self.instances = saved_instances
def from_archive(cls,
archive: Archive,
predictor_name: str = None) -> 'Predictor':
"""
Instantiate a :class:`Predictor` from an :class:`~allennlp.models.archival.Archive`;
that is, from the result of training a model. Optionally specify which `Predictor`
subclass; otherwise, the default one for the model will be used.
"""
# Duplicate the config so that the config inside the archive doesn't get consumed
config = archive.config.duplicate()
if not predictor_name:
model_type = config.get("model").get("type")
if not model_type in DEFAULT_PREDICTORS:
raise ConfigurationError(f"No default predictor for model type {model_type}.\n"\
f"Please specify a predictor explicitly.")
predictor_name = DEFAULT_PREDICTORS[model_type]
dataset_reader_params = config["dataset_reader"]
dataset_reader = DatasetReader.from_params(dataset_reader_params)
model = archive.model
model.eval()
return Predictor.by_name(predictor_name)(model, dataset_reader)
def test_create_models_from_allennlp_configs(self, config_path):
params = Params.from_file(
str(config_path),
ext_vars={
"CLF_TRAIN_DATA_PATH": "",
"CLF_VALID_DATA_PATH": "",
"DISCRETIZER_PATH": str(DISCRETIZER_PATH),
"VOCAB_PATH": str(VOCAB_PATH),
},
)
reader = DatasetReader.from_params(params["dataset_reader"])
instances = reader.read(DATA_PATH)
vocab = Vocabulary.from_instances(instances)
num_labels = vocab.get_vocab_size(namespace="labels")
batch = Batch(instances)
batch.index_instances(vocab)
try:
model = Model.from_params(params=params["model"], vocab=vocab)
except Exception as e:
raise AssertionError(f"unable to load params from {config_path}") from e
output_dict = model(**batch.as_tensor_dict())
assert "probs" in output_dict
assert len(output_dict["probs"].shape) == 2
assert output_dict["probs"].shape[0] == len(instances)
assert output_dict["probs"].shape[1] == num_labels
def test_from_params(self, data_path: str, sentence_marker_params: Params, ccm_params: Params) -> None:
reader = DatasetReader.from_params(sentence_marker_params)
instances = reader.read(data_path)
vocab = Vocabulary.from_instances(instances)
ccm_module = ConstrainedConditionalModule.from_params(vocab=vocab, params=ccm_params)
index = vocab.get_token_index("I-type", "labels")
assert ccm_module._sentence_penalty_map == (index, 50.)
def test_batch_predictions_are_consistent(self):
# The same issue as the bidaf test case.
# The CNN encoder has problems with this kind of test - it's not properly masked yet, so
# changing the amount of padding in the batch will result in small differences in the
# output of the encoder. So, we'll remove the CNN encoder entirely from the model for this test.
# Save some state.
# pylint: disable=protected-access,attribute-defined-outside-init
saved_model = self.model
saved_instances = self.instances
# Modify the state, run the test with modified state.
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
reader._token_indexers = {'tokens': reader._token_indexers['tokens']}
self.instances = reader.read(self.FIXTURES_ROOT / 'data' /
'squad.json')
vocab = Vocabulary.from_instances(self.instances)
for instance in self.instances:
instance.index_fields(vocab)
del params['model']['text_field_embedder']['token_embedders'][
'token_characters']
params['model']['phrase_layer']['num_convs_per_block'] = 0
params['model']['modeling_layer']['num_convs_per_block'] = 0
self.model = Model.from_params(vocab=vocab, params=params['model'])
self.ensure_batch_predictions_are_consistent()
# Restore the state.
self.model = saved_model
self.instances = saved_instances
def setUp(self):
super().setUp()
params = Params.from_file(self.FIXTURES_ROOT / "encoder_decoder" / "copynet_seq2seq" / "experiment.json")
self.reader = DatasetReader.from_params(params["dataset_reader"])
instances = self.reader.read(self.FIXTURES_ROOT / "data" / "copynet" / "copyover.tsv")
self.instances = ensure_list(instances)
self.vocab = Vocabulary.from_params(params=params["vocabulary"], instances=instances)
def from_archive(
cls,
archive: Archive,
predictor_name: str = None,
dataset_reader_to_load: str = "validation",
frozen: bool = True,
language: str = "en_core_web_sm",
restrict_frames: bool = False,
restrict_roles: bool = False,
) -> "Predictor":
# Duplicate the config so that the config inside the archive doesn't get consumed
config = archive.config.duplicate()
if not predictor_name:
model_type = config.get("model").get("type")
model_class, _ = Model.resolve_class_name(model_type)
predictor_name = model_class.default_predictor
predictor_class: Type[Predictor] = (
Predictor.by_name(predictor_name) if predictor_name is not None else cls # type: ignore
)
if dataset_reader_to_load == "validation" and "validation_dataset_reader" in config:
dataset_reader_params = config["validation_dataset_reader"]
else:
dataset_reader_params = config["dataset_reader"]
dataset_reader = DatasetReader.from_params(dataset_reader_params)
model = archive.model
if frozen:
model.restrict_frames = restrict_frames
model.restrict_roles = restrict_roles
model.eval()
return predictor_class(model, dataset_reader, language)
def test_batch_predictions_are_consistent(self):
# The same issue as the bidaf test case.
# The CNN encoder has problems with this kind of test - it's not properly masked yet, so
# changing the amount of padding in the batch will result in small differences in the
# output of the encoder. So, we'll remove the CNN encoder entirely from the model for this test.
# Save some state.
saved_model = self.model
saved_instances = self.instances
# Modify the state, run the test with modified state.
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params["dataset_reader"])
reader._token_indexers = {"tokens": reader._token_indexers["tokens"]}
self.instances = reader.read(FIXTURES_ROOT / "data" / "squad.json")
vocab = Vocabulary.from_instances(self.instances)
for instance in self.instances:
instance.index_fields(vocab)
del params["model"]["text_field_embedder"]["token_embedders"][
"token_characters"]
params["model"]["phrase_layer"]["num_convs_per_block"] = 0
params["model"]["modeling_layer"]["num_convs_per_block"] = 0
self.model = Model.from_params(vocab=vocab, params=params["model"])
self.ensure_batch_predictions_are_consistent()
# Restore the state.
self.model = saved_model
self.instances = saved_instances
def write_for_official_eval(model_archive_file, test_file, output_file,
label_ids_to_label):
archive = load_archive(model_archive_file)
model = archive.model
reader = DatasetReader.from_params(archive.config['dataset_reader'])
iterator = DataIterator.from_params(Params({"type": "basic", "batch_size": 4}))
vocab = Vocabulary.from_params(archive.config['vocabulary'])
iterator.index_with(vocab)
model.cuda()
model.eval()
instances = reader.read(test_file)
predictions = []
for batch in iterator(instances, num_epochs=1, shuffle=False):
batch = move_to_device(batch, cuda_device=0)
output = model(**batch)
batch_labels = [
label_ids_to_label[i]
for i in output['predictions'].cpu().numpy().tolist()
]
predictions.extend(batch_labels)
with open(output_file, 'w') as fout:
for p in predictions:
fout.write("{}\n".format(p))
def from_archive(
cls,
archive: Archive,
predictor_name: str = None,
dataset_reader_to_load: str = "validation",
) -> "Predictor":
"""
Instantiate a :class:`Predictor` from an :class:`~allennlp.models.archival.Archive`;
that is, from the result of training a model. Optionally specify which `Predictor`
subclass; otherwise, we try to find a corresponding predictor in `DEFAULT_PREDICTORS`, or if
one is not found, the base class (i.e. :class:`Predictor`) will be used. Optionally specify
which :class:`DatasetReader` should be loaded; otherwise, the validation one will be used
if it exists followed by the training dataset reader.
"""
# Duplicate the config so that the config inside the archive doesn't get consumed
config = archive.config.duplicate()
if not predictor_name:
model_type = config.get("model").get("type")
if model_type in DEFAULT_PREDICTORS:
predictor_name = DEFAULT_PREDICTORS[model_type]
predictor_class: Type[Predictor] = Predictor.by_name( # type: ignore
predictor_name
) if predictor_name is not None else cls
if dataset_reader_to_load == "validation" and "validation_dataset_reader" in config:
dataset_reader_params = config["validation_dataset_reader"]
else:
dataset_reader_params = config["dataset_reader"]
dataset_reader = DatasetReader.from_params(dataset_reader_params)
model = archive.model
model.eval()
return predictor_class(model, dataset_reader)
def set_up_model(self, param_file, dataset_file):
# pylint: disable=attribute-defined-outside-init
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
# The dataset reader might be lazy, but a lazy list here breaks some of our tests.
instances = list(reader.read(str(dataset_file)))
# Use parameters for vocabulary if they are present in the config file, so that choices like
# "non_padded_namespaces", "min_count" etc. can be set if needed.
if 'vocabulary' in params:
vocab_params = params['vocabulary']
vocab = Vocabulary.from_params(params=vocab_params,
instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.model = Model.from_params(vocab=self.vocab,
params=params['model'])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(self.instances)
self.dataset.index_instances(self.vocab)
def __init__(self) -> None:
super().__init__(lazy=True)
self.reader = DatasetReader.from_params(
Params({
"type": "sequence_tagging",
"lazy": True
}))
def get_reader():
params = {
"type": "kg_probe",
"tokenizer_and_candidate_generator": {
"type": "bert_tokenizer_and_candidate_generator",
"entity_candidate_generators": {
"wordnet": {"type": "wordnet_mention_generator",
"entity_file": "tests/fixtures/wordnet/entities_fixture.jsonl"}
},
"entity_indexers": {
"wordnet": {
"type": "characters_tokenizer",
"tokenizer": {
"type": "word",
"word_splitter": {"type": "just_spaces"},
},
"namespace": "entity"
}
},
"bert_model_type": "tests/fixtures/bert/vocab.txt",
"do_lower_case": True,
},
}
return DatasetReader.from_params(Params(params))
def __init__(self) -> None:
super().__init__(lazy=True)
self.reader = DatasetReader.from_params(
Params({
'type': 'sequence_tagging',
'lazy': True
}))
def set_up_model(self, param_file, dataset_file):
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params["dataset_reader"])
# The dataset reader might be lazy, but a lazy list here breaks some of our tests.
instances = reader.read(str(dataset_file))
# Use parameters for vocabulary if they are present in the config file, so that choices like
# "non_padded_namespaces", "min_count" etc. can be set if needed.
if "vocabulary" in params:
vocab_params = params["vocabulary"]
vocab = Vocabulary.from_params(params=vocab_params,
instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.instances.index_with(vocab)
self.model = Model.from_params(vocab=self.vocab,
params=params["model"])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(list(self.instances))
self.dataset.index_instances(self.vocab)
def set_up_model(self, param_file, dataset_file):
# pylint: disable=attribute-defined-outside-init
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
instances = reader.read(dataset_file)
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.model = Model.from_params(self.vocab, params['model'])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(self.instances)
self.dataset.index_instances(self.vocab)
def main(serialization_directory, device):
"""
serialization_directory : str, required.
The directory containing the serialized weights.
device: int, default = -1
The device to run the evaluation on.
"""
config = Params.from_file(os.path.join(serialization_directory, "config.json"))
dataset_reader = DatasetReader.from_params(config['dataset_reader'])
evaluation_data_path = config['validation_data_path']
model = Model.load(config, serialization_dir=serialization_directory, cuda_device=device)
prediction_file_path = os.path.join(serialization_directory, "predictions.txt")
gold_file_path = os.path.join(serialization_directory, "gold.txt")
prediction_file = open(prediction_file_path, "w+")
gold_file = open(gold_file_path, "w+")
# Load the evaluation data and index it.
print("Reading evaluation data from {}".format(evaluation_data_path))
instances = dataset_reader.read(evaluation_data_path)
iterator = BasicIterator(batch_size=32)
iterator.index_with(model.vocab)
model_predictions = []
batches = iterator(instances, num_epochs=1, shuffle=False, cuda_device=device, for_training=False)
for batch in Tqdm.tqdm(batches):
result = model(**batch)
predictions = model.decode(result)
model_predictions.extend(predictions["tags"])
for instance, prediction in zip(instances, model_predictions):
fields = instance.fields
try:
# Most sentences have a verbal predicate, but not all.
verb_index = fields["verb_indicator"].labels.index(1)
except ValueError:
verb_index = None
gold_tags = fields["tags"].labels
sentence = fields["tokens"].tokens
write_to_conll_eval_file(prediction_file, gold_file,
verb_index, sentence, prediction, gold_tags)
prediction_file.close()
gold_file.close()
def set_up_model(self, param_file, dataset_file):
# pylint: disable=attribute-defined-outside-init
self.param_file = param_file
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
instances = reader.read(dataset_file)
# Use parameters for vocabulary if they are present in the config file, so that choices like
# "non_padded_namespaces", "min_count" etc. can be set if needed.
if 'vocabulary' in params:
vocab_params = params['vocabulary']
vocab = Vocabulary.from_params(params=vocab_params, instances=instances)
else:
vocab = Vocabulary.from_instances(instances)
self.vocab = vocab
self.instances = instances
self.model = Model.from_params(vocab=self.vocab, params=params['model'])
# TODO(joelgrus) get rid of these
# (a lot of the model tests use them, so they'll have to be changed)
self.dataset = Batch(self.instances)
self.dataset.index_instances(self.vocab)
def from_archive(cls, archive: Archive, predictor_name: str = None) -> 'Predictor':
"""
Instantiate a :class:`Predictor` from an :class:`~allennlp.models.archival.Archive`;
that is, from the result of training a model. Optionally specify which `Predictor`
subclass; otherwise, the default one for the model will be used.
"""
# Duplicate the config so that the config inside the archive doesn't get consumed
config = archive.config.duplicate()
if not predictor_name:
model_type = config.get("model").get("type")
if not model_type in DEFAULT_PREDICTORS:
raise ConfigurationError(f"No default predictor for model type {model_type}.\n"\
f"Please specify a predictor explicitly.")
predictor_name = DEFAULT_PREDICTORS[model_type]
dataset_reader_params = config["dataset_reader"]
dataset_reader = DatasetReader.from_params(dataset_reader_params)
model = archive.model
model.eval()
return Predictor.by_name(predictor_name)(model, dataset_reader)
def main(serialization_directory: int,
device: int,
data: str,
prefix: str,
domain: str = None):
"""
serialization_directory : str, required.
The directory containing the serialized weights.
device: int, default = -1
The device to run the evaluation on.
data: str, default = None
The data to evaluate on. By default, we use the validation data from
the original experiment.
prefix: str, default=""
The prefix to prepend to the generated gold and prediction files, to distinguish
different models/data.
domain: str, optional (default = None)
If passed, filters the ontonotes evaluation/test dataset to only contain the
specified domain. This overwrites the domain in the config file from the model,
to allow evaluation on domains other than the one the model was trained on.
"""
config = Params.from_file(os.path.join(serialization_directory, "config.json"))
if domain is not None:
# Hack to allow evaluation on different domains than the
# model was trained on.
config["dataset_reader"]["domain_identifier"] = domain
prefix = f"{domain}_{prefix}"
else:
config["dataset_reader"].pop("domain_identifier", None)
dataset_reader = DatasetReader.from_params(config['dataset_reader'])
evaluation_data_path = data if data else config['validation_data_path']
archive = load_archive(os.path.join(serialization_directory, "model.tar.gz"), cuda_device=device)
model = archive.model
model.eval()
prediction_file_path = os.path.join(serialization_directory, prefix + "_predictions.txt")
gold_file_path = os.path.join(serialization_directory, prefix + "_gold.txt")
prediction_file = open(prediction_file_path, "w+")
gold_file = open(gold_file_path, "w+")
# Load the evaluation data and index it.
print("reading evaluation data from {}".format(evaluation_data_path))
instances = dataset_reader.read(evaluation_data_path)
with torch.autograd.no_grad():
iterator = BasicIterator(batch_size=32)
iterator.index_with(model.vocab)
model_predictions = []
batches = iterator(instances, num_epochs=1, shuffle=False, cuda_device=device)
for batch in Tqdm.tqdm(batches):
result = model(**batch)
predictions = model.decode(result)
model_predictions.extend(predictions["tags"])
for instance, prediction in zip(instances, model_predictions):
fields = instance.fields
try:
# Most sentences have a verbal predicate, but not all.
verb_index = fields["verb_indicator"].labels.index(1)
except ValueError:
verb_index = None
gold_tags = fields["tags"].labels
sentence = [x.text for x in fields["tokens"].tokens]
write_to_conll_eval_file(prediction_file, gold_file,
verb_index, sentence, prediction, gold_tags)
prediction_file.close()
gold_file.close()
def __init__(self) -> None:
super().__init__(lazy=True)
self.reader = DatasetReader.from_params(Params({'type': 'sequence_tagging'}))
def ensure_model_can_train_save_and_load(self,
param_file: str,
tolerance: float = 1e-4,
cuda_device: int = -1,
gradients_to_ignore: Set[str] = None,
overrides: str = ""):
"""
Parameters
----------
param_file : ``str``
Path to a training configuration file that we will use to train the model for this
test.
tolerance : ``float``, optional (default=1e-4)
When comparing model predictions between the originally-trained model and the model
after saving and loading, we will use this tolerance value (passed as ``rtol`` to
``numpy.testing.assert_allclose``).
cuda_device : ``int``, optional (default=-1)
The device to run the test on.
gradients_to_ignore : ``Set[str]``, optional (default=None)
This test runs a gradient check to make sure that we're actually computing gradients
for all of the parameters in the model. If you really want to ignore certain
parameters when doing that check, you can pass their names here. This is not
recommended unless you're `really` sure you don't need to have non-zero gradients for
those parameters (e.g., some of the beam search / state machine models have
infrequently-used parameters that are hard to force the model to use in a small test).
overrides : ``str``, optional (default = "")
A JSON string that we will use to override values in the input parameter file.
"""
save_dir = self.TEST_DIR / "save_and_load_test"
archive_file = save_dir / "model.tar.gz"
model = train_model_from_file(param_file, save_dir, overrides=overrides)
loaded_model = load_archive(archive_file, cuda_device=cuda_device).model
state_keys = model.state_dict().keys()
loaded_state_keys = loaded_model.state_dict().keys()
assert state_keys == loaded_state_keys
# First we make sure that the state dict (the parameters) are the same for both models.
for key in state_keys:
assert_allclose(model.state_dict()[key].cpu().numpy(),
loaded_model.state_dict()[key].cpu().numpy(),
err_msg=key)
params = Params.from_file(param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
# Need to duplicate params because Iterator.from_params will consume.
iterator_params = params['iterator']
iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict()))
iterator = DataIterator.from_params(iterator_params)
iterator2 = DataIterator.from_params(iterator_params2)
# We'll check that even if we index the dataset with each model separately, we still get
# the same result out.
model_dataset = reader.read(params['validation_data_path'])
iterator.index_with(model.vocab)
model_batch = next(iterator(model_dataset, shuffle=False))
loaded_dataset = reader.read(params['validation_data_path'])
iterator2.index_with(loaded_model.vocab)
loaded_batch = next(iterator2(loaded_dataset, shuffle=False))
# Check gradients are None for non-trainable parameters and check that
# trainable parameters receive some gradient if they are trainable.
self.check_model_computes_gradients_correctly(model, model_batch, gradients_to_ignore)
# The datasets themselves should be identical.
assert model_batch.keys() == loaded_batch.keys()
for key in model_batch.keys():
self.assert_fields_equal(model_batch[key], loaded_batch[key], key, 1e-6)
# Set eval mode, to turn off things like dropout, then get predictions.
model.eval()
loaded_model.eval()
# Models with stateful RNNs need their states reset to have consistent
# behavior after loading.
for model_ in [model, loaded_model]:
for module in model_.modules():
if hasattr(module, 'stateful') and module.stateful:
module.reset_states()
model_predictions = model(**model_batch)
loaded_model_predictions = loaded_model(**loaded_batch)
# Check loaded model's loss exists and we can compute gradients, for continuing training.
loaded_model_loss = loaded_model_predictions["loss"]
assert loaded_model_loss is not None
loaded_model_loss.backward()
# Both outputs should have the same keys and the values for these keys should be close.
for key in model_predictions.keys():
self.assert_fields_equal(model_predictions[key],
loaded_model_predictions[key],
name=key,
tolerance=tolerance)
return model, loaded_model
def ensure_model_can_train_save_and_load(self,
param_file: str,
tolerance: float = 1e-4,
cuda_device: int = -1):
save_dir = os.path.join(self.TEST_DIR, "save_and_load_test")
archive_file = os.path.join(save_dir, "model.tar.gz")
model = train_model_from_file(param_file, save_dir)
loaded_model = load_archive(archive_file, cuda_device=cuda_device).model
state_keys = model.state_dict().keys()
loaded_state_keys = loaded_model.state_dict().keys()
assert state_keys == loaded_state_keys
# First we make sure that the state dict (the parameters) are the same for both models.
for key in state_keys:
assert_allclose(model.state_dict()[key].cpu().numpy(),
loaded_model.state_dict()[key].cpu().numpy(),
err_msg=key)
params = Params.from_file(self.param_file)
reader = DatasetReader.from_params(params['dataset_reader'])
# Need to duplicate params because Iterator.from_params will consume.
iterator_params = params['iterator']
iterator_params2 = Params(copy.deepcopy(iterator_params.as_dict()))
iterator = DataIterator.from_params(iterator_params)
iterator2 = DataIterator.from_params(iterator_params2)
# We'll check that even if we index the dataset with each model separately, we still get
# the same result out.
model_dataset = reader.read(params['validation_data_path'])
iterator.index_with(model.vocab)
model_batch = next(iterator(model_dataset, shuffle=False, cuda_device=cuda_device))
loaded_dataset = reader.read(params['validation_data_path'])
iterator2.index_with(loaded_model.vocab)
loaded_batch = next(iterator2(loaded_dataset, shuffle=False, cuda_device=cuda_device))
# Check gradients are None for non-trainable parameters and check that
# trainable parameters receive some gradient if they are trainable.
self.check_model_computes_gradients_correctly(model, model_batch)
# The datasets themselves should be identical.
assert model_batch.keys() == loaded_batch.keys()
for key in model_batch.keys():
self.assert_fields_equal(model_batch[key], loaded_batch[key], key, 1e-6)
# Set eval mode, to turn off things like dropout, then get predictions.
model.eval()
loaded_model.eval()
# Models with stateful RNNs need their states reset to have consistent
# behavior after loading.
for model_ in [model, loaded_model]:
for module in model_.modules():
if hasattr(module, 'stateful') and module.stateful:
module.reset_states()
model_predictions = model(**model_batch)
loaded_model_predictions = loaded_model(**loaded_batch)
# Check loaded model's loss exists and we can compute gradients, for continuing training.
loaded_model_loss = loaded_model_predictions["loss"]
assert loaded_model_loss is not None
loaded_model_loss.backward()
# Both outputs should have the same keys and the values for these keys should be close.
for key in model_predictions.keys():
self.assert_fields_equal(model_predictions[key],
loaded_model_predictions[key],
name=key,
tolerance=tolerance)
return model, loaded_model
if (load_pretrained_BiDAF):
archive = load_archive("https://s3-us-west-2.amazonaws.com/allennlp/models/bidaf-model-2017.09.15-charpad.tar.gz")
# Get the model and the config file
model = archive.model
config = archive.config.duplicate()
keys_config = list(config.keys())
print ("Key config list: ", keys_config)
for key in keys_config:
print ("Params of %s"%(key))
print (config[key])
### Get the elements
## Data Readers ##
dataset_reader_params = config["dataset_reader"]
dataset_reader = DatasetReader.from_params(dataset_reader_params)
## Vocabulary ##
vocab = model.vocab
"""
############ Propagate an instance text #############
"""
instance = dataset_reader.text_to_instance("What kind of test succeeded on its first attempt?",
"One time I was writing a unit test, and it succeeded on the first attempt.",
char_spans=[(6, 10)])
print ("Keys instance: ", instance.fields.keys())
# Batch intances and convert to index using the vocabulary.
instances = [instance]
dataset = Batch(instances)
