def test_samples_properly(self):
reader = SemiSupervisedTextClassificationJsonReader(
sample=1, max_sequence_length=5)
ag_path = self.FIXTURES_ROOT / "imdb" / "train.jsonl"
params = Params({"random_seed": 5, "numpy_seed": 5, "pytorch_seed": 5})
prepare_environment(params)
instances = reader.read(ag_path)
instances = ensure_list(instances)
instance = {
"tokens": ['The', 'fight', 'scenes', 'were', 'great'],
"label": "pos"
}
assert len(instances) == 1
fields = instances[0].fields
assert [t.text for t in fields["tokens"].tokens] == instance["tokens"]
assert fields["label"].label == instance["label"]
def test_from_params_adds_tokens_to_vocab(self):
vocab = Vocabulary.from_params(Params(
{"tokens_to_add": {
"tokens": ["q", "x", "z"]
}}),
instances=self.dataset)
assert vocab.get_index_to_token_vocabulary("tokens") == {
0: "@@PADDING@@",
1: "@@UNKNOWN@@",
2: "a",
3: "c",
4: "b",
5: "q",
6: "x",
7: "z",
}
def test_openai_can_run_with_no_offsets(self):
params = Params({
"transformer": {
"model_path":
"allennlp/tests/fixtures/openai_transformer/transformer_small.tar.gz",
"embedding_dim": 10,
"num_heads": 2,
"num_layers": 2,
"vocab_size": 50,
"n_ctx": 50
},
})
embedder = OpenaiTransformerEmbedder.from_params(params)
training_tensors = self._get_training_tensors()
output = embedder(training_tensors['tokens']['openai_transformer'])
assert list(output.shape) == [2, 2, 10]
def setup_method(self):
super().setup_method()
self.instances = SequenceTaggingDatasetReader().read(
self.FIXTURES_ROOT / "data" / "sequence_tagging.tsv"
)
vocab = Vocabulary.from_instances(self.instances)
self.model_params = Params(
{
"text_field_embedder": {
"token_embedders": {"tokens": {"type": "embedding", "embedding_dim": 5}}
},
"encoder": {"type": "lstm", "input_size": 5, "hidden_size": 7, "num_layers": 2},
}
)
self.model = SimpleTagger.from_params(vocab=vocab, params=self.model_params)
def test_max_vocab_size_partial_dict(self):
indexers = {"tokens": SingleIdTokenIndexer(),
"token_characters": TokenCharactersIndexer(min_padding_length=3)}
instance = Instance({
'text': TextField([Token(w) for w in 'Abc def ghi jkl mno pqr stu vwx yz'.split(' ')], indexers)
})
dataset = Batch([instance])
params = Params({
"max_vocab_size": {
"tokens": 1
}
})
vocab = Vocabulary.from_params(params=params, instances=dataset)
assert len(vocab.get_index_to_token_vocabulary("tokens").values()) == 3 # 1 + 2
assert len(vocab.get_index_to_token_vocabulary("token_characters").values()) == 28 # 26 + 2
def _load(
cls, config: Params, serialization_dir: str, weights_file: str = None, cuda_device: int = -1
) -> "Model":
"""
Instantiates an already-trained model, based on the experiment
configuration and some optional overrides.
"""
weights_file = weights_file or os.path.join(serialization_dir, _DEFAULT_WEIGHTS)
# Load vocabulary from file
vocab_dir = os.path.join(serialization_dir, "vocabulary")
# If the config specifies a vocabulary subclass, we need to use it.
vocab_params = config.get("vocabulary", Params({}))
vocab_choice = vocab_params.pop_choice("type", Vocabulary.list_available(), True)
vocab_class, _ = Vocabulary.resolve_class_name(vocab_choice)
vocab = vocab_class.from_files(
vocab_dir, vocab_params.get("padding_token", None), vocab_params.get("oov_token", None)
)
model_params = config.get("model")
# The experiment config tells us how to _train_ a model, including where to get pre-trained
# embeddings from. We're now _loading_ the model, so those embeddings will already be
# stored in our weights. We don't need any pretrained weight file anymore, and we don't
# want the code to look for it, so we remove it from the parameters here.
remove_pretrained_embedding_params(model_params)
model = Model.from_params(vocab=vocab, params=model_params)
# If vocab+embedding extension was done, the model initialized from from_params
# and one defined by state dict in weights_file might not have same embedding shapes.
# Eg. when model embedder module was transferred along with vocab extension, the
# initialized embedding weight shape would be smaller than one in the state_dict.
# So calling model embedding extension is required before load_state_dict.
# If vocab and model embeddings are in sync, following would be just a no-op.
model.extend_embedder_vocab()
model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device))
model.load_state_dict(model_state)
# Force model to cpu or gpu, as appropriate, to make sure that the embeddings are
# in sync with the weights
if cuda_device >= 0:
model.cuda(cuda_device)
else:
model.cpu()
return model
def test_dropout_version_is_different_to_no_dropout(self):
augmented_lstm = AugmentedLstm(10, 11)
dropped_augmented_lstm = AugmentedLstm(
10, 11, recurrent_dropout_probability=0.9)
# Initialize all weights to be == 1.
constant_init = Initializer.from_params(
Params({
"type": "constant",
"val": 0.5
}))
initializer = InitializerApplicator([(".*", constant_init)])
initializer(augmented_lstm)
initializer(dropped_augmented_lstm)
initial_state = torch.randn([1, 5, 11])
initial_memory = torch.randn([1, 5, 11])
# If we use too bigger number like in the PyTorch test the dropout has no affect
sorted_tensor, sorted_sequence, _, _ = sort_batch_by_length(
self.random_tensor, self.sequence_lengths)
lstm_input = pack_padded_sequence(sorted_tensor,
sorted_sequence.data.tolist(),
batch_first=True)
augmented_output, augmented_state = augmented_lstm(
lstm_input, (initial_state, initial_memory))
dropped_output, dropped_state = dropped_augmented_lstm(
lstm_input, (initial_state, initial_memory))
dropped_output_sequence, _ = pad_packed_sequence(dropped_output,
batch_first=True)
augmented_output_sequence, _ = pad_packed_sequence(augmented_output,
batch_first=True)
with pytest.raises(AssertionError):
numpy.testing.assert_array_almost_equal(
dropped_output_sequence.data.numpy(),
augmented_output_sequence.data.numpy(),
decimal=4)
with pytest.raises(AssertionError):
numpy.testing.assert_array_almost_equal(
dropped_state[0].data.numpy(),
augmented_state[0].data.numpy(),
decimal=4)
with pytest.raises(AssertionError):
numpy.testing.assert_array_almost_equal(
dropped_state[1].data.numpy(),
augmented_state[1].data.numpy(),
decimal=4)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
context_layer: Seq2SeqEncoder,
modules: Params,
loss_weights: Dict[str, int],
lexical_dropout: float = 0.2,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
display_metrics: List[str] = None,
) -> None:
super(ScirexModel, self).__init__(vocab, regularizer)
self._text_field_embedder = text_field_embedder
self._context_layer = context_layer
self._lexical_dropout = torch.nn.Dropout(p=lexical_dropout)
modules = Params(modules)
self._ner = NERTagger.from_params(vocab=vocab,
params=modules.pop("ner"))
self._saliency_classifier = SpanClassifier.from_params(
vocab=vocab, params=modules.pop("saliency_classifier"))
self._cluster_n_ary_relation = NAryRelationExtractor.from_params(
vocab=vocab, params=modules.pop("n_ary_relation"))
self._endpoint_span_extractor = EndpointSpanExtractor(
context_layer.get_output_dim(), combination="x,y")
self._attentive_span_extractor = SelfAttentiveSpanExtractor(
input_dim=context_layer.get_output_dim())
for k in loss_weights:
loss_weights[k] = float(loss_weights[k])
self._loss_weights = loss_weights
self._permanent_loss_weights = copy.deepcopy(self._loss_weights)
self._display_metrics = display_metrics
self._multi_task_loss_metrics = {
k: Average()
for k in ["ner", "saliency", "n_ary_relation"]
}
self.training_mode = True
self.prediction_mode = False
initializer(self)
def test_invalid_vocab_extension(self):
vocab_dir = self.TEST_DIR / 'vocab_save'
original_vocab = Vocabulary(non_padded_namespaces=["tokens1"])
original_vocab.add_token_to_namespace("a", namespace="tokens1")
original_vocab.add_token_to_namespace("b", namespace="tokens1")
original_vocab.add_token_to_namespace("p", namespace="tokens2")
original_vocab.save_to_files(vocab_dir)
text_field1 = TextField([Token(t) for t in ["a"
"c"]],
{"tokens1": SingleIdTokenIndexer("tokens1")})
text_field2 = TextField([Token(t) for t in ["p", "q", "r"]],
{"tokens2": SingleIdTokenIndexer("tokens2")})
instances = Batch(
[Instance({
"text1": text_field1,
"text2": text_field2
})])
# Following 2 should give error: token1 is non-padded in original_vocab but not in instances
params = Params({
"directory_path": vocab_dir,
"extend": True,
"non_padded_namespaces": []
})
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(params, instances)
with pytest.raises(ConfigurationError):
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": []})
extended_vocab.extend_from_instances(params, instances)
# Following 2 should not give error: overlapping namespaces have same padding setting
params = Params({
"directory_path": vocab_dir,
"extend": True,
"non_padded_namespaces": ["tokens1"]
})
Vocabulary.from_params(params, instances)
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": ["tokens1"]})
extended_vocab.extend_from_instances(params, instances)
# Following 2 should give error: token1 is padded in instances but not in original_vocab
params = Params({
"directory_path": vocab_dir,
"extend": True,
"non_padded_namespaces": ["tokens1", "tokens2"]
})
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(params, instances)
with pytest.raises(ConfigurationError):
extended_vocab = copy.copy(original_vocab)
params = Params({"non_padded_namespaces": ["tokens1", "tokens2"]})
extended_vocab.extend_from_instances(params, instances)
def test_from_params_tar_gz(self):
with tempfile.NamedTemporaryFile(suffix=".tar.gz") as f:
with tarfile.open(fileobj=f, mode="w:gz") as archive:
archive.add(self.temp_file,
arcname=os.path.basename(self.temp_file))
f.flush()
params = Params({
"type": "pretrained",
"weights_file_path": f.name
})
initializer = Initializer.from_params(params)
assert initializer.weights
assert initializer.parameter_name_overrides == {}
for name, parameter in self.net2.state_dict().items():
assert torch.equal(parameter, initializer.weights[name])
def test_from_params2():
params = Params({
"dataset_reader": {
"type": "openke-rank-validation-dataset",
"dataset_name": 'FB15K237',
"all_datadir":
'/Users/dhruv/UnsyncedDocuments/IESL/kb_completion/datasets/.data/test',
"file_reader": {
"type": "rank-val-id-reader"
}
},
"train_data_path": None
})
dataset = util.datasets_from_params(params.duplicate())
for i, instance in enumerate(dataset['train']):
print(i, instance)
def test_registrability(self):
class MyVocabulary(object):
@classmethod
def from_params(cls, params, instances=None):
# pylint: disable=unused-argument
return MyVocabulary()
MyVocabulary = Vocabulary.register(u'my-vocabulary')(
MyVocabulary)
params = Params({u'type': u'my-vocabulary'})
instance = Instance(fields={})
vocab = Vocabulary.from_params(params=params, instances=[instance])
assert isinstance(vocab, MyVocabulary)
def train_func(config, reporter):
logger.debug(f"CUDA_VISIBLE_DEVICES: {os.environ['CUDA_VISIBLE_DEVICES']}")
for package_name in getattr(run_args, "include_package", ()):
import_submodules(package_name)
run_parameters = {k: json.dumps(v) for k, v in config.items()}
search_config = SEARCH_ENVIRONMENTS[default_args.search_config]
search_space = HyperparameterSearch(**search_config)
sample = search_space.sample()
logger.info(f"Hyperparameter Configuration: {sample}")
for k, v in sample.items():
sample[k] = str(v)
os.environ[k] = str(v)
params_dict = json.loads(
_jsonnet.evaluate_snippet(
"config", parameter_file_snippet, tla_codes=run_parameters, ext_vars=sample
)
)
if default_args.num_gpus == 0:
logger.warning(f"No GPU specified, using CPU.")
params_dict["trainer"]["cuda_device"] = -1
# Make sure path is absolute (as Ray workers do not use the same working dir)
train_data_path = params_dict["train_data_path"]
validation_data_path = params_dict.get("validation_data_path")
# if not os.path.isabs(train_data_path) and not is_s3_url(train_data_path):
# params_dict["train_data_path"] = os.path.abspath(
# os.path.join(default_args.cwd, train_data_path)
# )
# if validation_data_path and not os.path.isabs(validation_data_path) and not is_s3_url(validation_data_path):
# params_dict["validation_data_path"] = os.path.abspath(
# os.path.join(default_args.cwd, validation_data_path)
# )
params = Params(params_dict)
logger.debug(f"AllenNLP Configuration: {params.as_dict()}")
train_model(params=params, serialization_dir="./trial/")
reporter(done=True)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
inp_dim,
hid_dim,
dropout: float = 0.4,
dropout_emb: float = 0.2,
pretrain_embedding_file=None,
gather='sum'):
super(EncDoc, self).__init__(vocab, regularizer)
token_embedding = Embedding(
num_embeddings=vocab.get_vocab_size('tokens'),
embedding_dim=inp_dim)
if dropout_emb > 0:
self._lexical_dropout = torch.nn.Dropout(p=dropout_emb)
else:
self._lexical_dropout = lambda x: x
self.hid_dim = hid_dim
self.sent_enc = EncWord2Sent(inp_dim=inp_dim,
hid_dim=hid_dim,
dropout=dropout,
gather=gather)
if pretrain_embedding_file is not None:
logger = logging.getLogger()
logger.info(
"Loading word embedding: {}".format(pretrain_embedding_file))
token_embedding.from_params(vocab=vocab,
params=Params({
"pretrained_file":
pretrain_embedding_file,
"embedding_dim": inp_dim
}))
print("token_embedding size: {}".format(
token_embedding.num_embeddings))
self._text_field_embedder = BasicTextFieldEmbedder(
{"tokens": token_embedding})
self.sent2doc = EncWord2Sent(inp_dim=self.sent_enc.get_output_dim(),
hid_dim=hid_dim,
nenc_lay=1,
dropout=dropout)
def test_trainer_can_run_with_lr_scheduler(self):
lr_params = Params({"type": "reduce_on_plateau"})
lr_scheduler = LearningRateScheduler.from_params(
self.optimizer, lr_params)
callbacks = self.default_callbacks() + [
UpdateLearningRate(lr_scheduler)
]
trainer = CallbackTrainer(
model=self.model,
training_data=self.instances,
iterator=self.iterator,
optimizer=self.optimizer,
callbacks=callbacks,
num_epochs=2,
)
trainer.train()
def test_registered_subclass(self):
"""
Tests that registering Checkpointer subclasses works correctly.
"""
@Checkpointer.register("checkpointer_subclass")
class CheckpointerSubclass(Checkpointer):
def __init__(self, x: int, y: int) -> None:
super().__init__()
self.x = x
self.y = y
sub_inst = Checkpointer.from_params(
Params({"type": "checkpointer_subclass", "x": 1, "y": 3})
)
assert sub_inst.__class__ == CheckpointerSubclass
assert sub_inst.x == 1 and sub_inst.y == 3
def test_invalid_datasets_for_vocab_creation(self):
params = Params({
"dataset_reader": {
"type": "train-util-test-reader"
},
"train_data_path":
"path-to-training-file",
"validation_data_path":
"path-to-validation-file",
"datasets_for_vocab_creation": ["train", "validation", "test"],
"data_loader": {
"batch_size": 2
},
})
with pytest.raises(ConfigurationError,
match="invalid 'datasets_for_vocab_creation' test"):
make_vocab_from_params(params, str(self.TEST_DIR))
def test_raise_error_if_directory_non_empty(self):
params = Params({
"dataset_reader": {
"type": "train-util-test-reader"
},
"train_data_path": "path-to-training-file",
"validation_data_path": "path-to-validation-file",
"data_loader": {
"batch_size": 2
},
})
os.makedirs(self.TEST_DIR / "vocabulary")
with open(self.TEST_DIR / "vocabulary" / "blah", "w") as random_file:
random_file.write("BLAH!")
with pytest.raises(ConfigurationError,
match="The 'vocabulary' directory in the provided"):
make_vocab_from_params(params, str(self.TEST_DIR))
def test_all_datasets_read_for_vocab(self, caplog):
params = Params({
"dataset_reader": {
"type": "train-util-test-reader"
},
"train_data_path": "path-to-training-file",
"validation_data_path": "path-to-validation-file",
"test_data_path": "path-to-test-file",
})
_ = make_vocab_from_params(params, str(self.TEST_DIR))
log_messages = "\n".join([rec.message for rec in caplog.records])
assert "...train-util-test-reader reading from path-to-training-file" in log_messages
assert "...train-util-test-reader reading from path-to-validation-file" in log_messages
assert "...train-util-test-reader reading from path-to-test-file" in log_messages
assert "Reading training data" in log_messages
assert "Reading validation data" in log_messages
assert "Reading test data" in log_messages
def test_using_seperate_validation_reader(self, caplog):
params = Params({
"dataset_reader": {
"type": "train-util-test-reader"
},
"validation_dataset_reader": {
"type": "train-util-test-reader"
},
"train_data_path": "path-to-training-file",
"validation_data_path": "path-to-validation-file",
"data_loader": {
"batch_size": 2
},
})
_ = make_vocab_from_params(params, str(self.TEST_DIR))
log_messages = "\n".join([rec.message for rec in caplog.records])
assert "Using a separate dataset reader to load validation and test data" in log_messages
def test_frozen_params(self):
model = torch.nn.Sequential(torch.nn.Linear(5, 10),
torch.nn.Linear(10, 5))
constant_init = Initializer.from_params(
Params({
"type": "constant",
"val": -1
}))
initializer = InitializerApplicator([(".*", constant_init)])
initializer(model)
# freeze the parameters of the first linear
for name, param in model.named_parameters():
if re.search(r"0.*$", name):
param.requires_grad = False
value = RegularizerApplicator([("", L1Regularizer(1.0))])(model)
# 55 because of bias (5*10 + 5)
assert value.data.numpy() == 55
def test_to_file(self):
# Test to_file works with or without preference orders
params_dict = {"keyA": "valA", "keyB": "valB"}
expected_ordered_params_dict = OrderedDict({
"keyB": "valB",
"keyA": "valA"
})
params = Params(params_dict)
file_path = self.TEST_DIR / 'config.jsonnet'
# check with preference orders
params.to_file(file_path, [["keyB", "keyA"]])
with open(file_path, "r") as handle:
ordered_params_dict = OrderedDict(json.load(handle))
assert json.dumps(expected_ordered_params_dict) == json.dumps(
ordered_params_dict)
# check without preference orders doesn't give error
params.to_file(file_path)
def train_subwords(train_path, model_path, model_type, vocab_size,
config_path):
temp = tempfile.NamedTemporaryFile(mode="w", delete=False)
params = Params.from_file(config_path)
reader_params = params.pop("reader", default=Params({}))
reader = DatasetReader.from_params(reader_params)
for text, summary in reader.parse_set(train_path):
temp.write(text + "\n")
temp.write(summary + "\n")
temp.close()
if not os.path.exists(model_path):
os.makedirs(model_path)
cmd = "--input={} --model_prefix={} --vocab_size={} --model_type={}".format(
temp.name, os.path.join(model_path, model_type), vocab_size,
model_type)
sp_trainer.Train(cmd)
os.unlink(temp.name)
def test_openai_can_run_with_top_layer(self):
params = Params({
"transformer": {
"model_path":
"allennlp/tests/fixtures/openai_transformer/transformer_small.tar.gz",
"embedding_dim": 10,
"num_heads": 2,
"num_layers": 2,
"vocab_size": 50,
"n_ctx": 50,
},
"top_layer_only": True,
})
embedder = OpenaiTransformerEmbedder.from_params(params)
training_tensors = self._get_training_tensors()
output = embedder(**training_tensors["tokens"]["openai_transformer"])
assert list(output.shape) == [2, 7, 10]
def test_sort_every_batch_actually_adds_noise_every_batch(self):
# We're just going to get two epoch's worth of batches, and make sure that they're
# different.
params = Params({
'padding_noise': 0.8,
'sort_every_epoch': True,
'dynamic_padding': True,
})
generator = DataGenerator(self.text_trainer, params)
batches = generator.create_generator(IndexedDataset(self.instances))
assert generator.last_num_batches == 4
first_epoch_arrays = [next(batches) for _ in range(4)]
second_epoch_arrays = [next(batches) for _ in range(4)]
first_epoch_arrays.sort(key=lambda x: x[0][0])
second_epoch_arrays.sort(key=lambda x: x[0][0])
first_epoch = [self.as_list(x[0]) for x in first_epoch_arrays]
second_epoch = [self.as_list(x[0]) for x in second_epoch_arrays]
assert first_epoch != second_epoch
def train(model_path,
train_path,
val_path=None,
vocabulary_path=None,
config_path=None):
vocabulary_path = vocabulary_path or os.path.join(model_path, "vocabulary")
assert os.path.isdir(
vocabulary_path), "Can't find vocab, run preprocess.py first"
vocabulary = Vocabulary.from_files(vocabulary_path)
config_path = config_path or os.path.join(model_path, "config.json")
params = Params.from_file(config_path)
train_params = params.pop("train", Params({}))
model = LanguageModel.from_params(params, vocab=vocabulary)
model.train(train_path,
train_params,
serialization_dir=model_path,
valid_file_name=val_path)
def load_config(serialization_dir: str,
config_path: str = None,
overrides_dict: Dict = None) -> Params:
if serialization_dir is None and (config_path is None):
raise ValueError("Both cannot be None")
if config_path is None:
config_path = Path(serialization_dir) / "config.json" # type: ignore
config = Params.from_file(config_path)
if overrides_dict is not None:
config_dict = (config.as_dict())
config = with_fallback(preferred=overrides_dict, fallback=config_dict)
return Params(config)
else:
return config
def test_augmented_lstm_computes_same_function_as_pytorch_lstm(self):
augmented_lstm = AugmentedLstm(10, 11)
pytorch_lstm = LSTM(10, 11, num_layers=1, batch_first=True)
# Initialize all weights to be == 1.
constant_init = Initializer.from_params(
Params({
"type": "constant",
"val": 1.
}))
initializer = InitializerApplicator([(".*", constant_init)])
initializer(augmented_lstm)
initializer(pytorch_lstm)
initial_state = torch.zeros([1, 5, 11])
initial_memory = torch.zeros([1, 5, 11])
# Use bigger numbers to avoid floating point instability.
sorted_tensor, sorted_sequence, _, _ = sort_batch_by_length(
self.random_tensor * 5., self.sequence_lengths)
lstm_input = pack_padded_sequence(sorted_tensor,
sorted_sequence.data.tolist(),
batch_first=True)
augmented_output, augmented_state = augmented_lstm(
lstm_input, (initial_state, initial_memory))
pytorch_output, pytorch_state = pytorch_lstm(
lstm_input, (initial_state, initial_memory))
pytorch_output_sequence, _ = pad_packed_sequence(pytorch_output,
batch_first=True)
augmented_output_sequence, _ = pad_packed_sequence(augmented_output,
batch_first=True)
numpy.testing.assert_array_almost_equal(
pytorch_output_sequence.data.numpy(),
augmented_output_sequence.data.numpy(),
decimal=4)
numpy.testing.assert_array_almost_equal(
pytorch_state[0].data.numpy(),
augmented_state[0].data.numpy(),
decimal=4)
numpy.testing.assert_array_almost_equal(
pytorch_state[1].data.numpy(),
augmented_state[1].data.numpy(),
decimal=4)
def tasks():
params = {
"id":
"ner",
"name":
"Negation",
"description":
"Negation is the tasks of detecting a negation keyword and its corresponding scope.",
"expected_inputs":
"The task expects an input sentence.",
"expected_outputs":
"The output is all the identified named entities (which can be one or more words) in the text.",
"scope_and_limitations":
None,
"examples": [{
"sentence":
"This shirt was bought at Grandpa Joe's in downtown Deep Learning."
}, {
"sentence":
"AllenNLP is a PyTorch-based natural language processing library developed at the Allen Institute for Artificial Intelligence in Seattle."
}, {
"sentence":
"Did Uriah honestly think he could beat The Legend of Zelda in under three hours?"
}, {
"sentence":
"Michael Jordan is a professor at Berkeley."
}, {
"sentence":
"My preferred candidate is Cary Moon, but she won't be the next mayor of Seattle."
}, {
"sentence":
"If you like Paul McCartney you should listen to the first Wings album."
}, {
"sentence":
"When I told John that I wanted to move to Alaska, he warned me that I'd have trouble finding a Starbucks there."
}, {
"sentence":
"This is a negation sentence for the demo, wait, no it isn't. Hi Michael Jordan."
}],
}
tasks = get_tasks()
nerparams = Params(params)
tasks['ner'] = TaskCard.from_params(nerparams)
return flask.jsonify(tasks)
def test_stacked_bidirectional_lstm_dropout_version_is_different(self, dropout_name: str):
stacked_lstm = StackedBidirectionalLstm(input_size=10, hidden_size=11,
num_layers=3)
if dropout_name == 'layer_dropout_probability':
dropped_stacked_lstm = StackedBidirectionalLstm(input_size=10, hidden_size=11,
num_layers=3,
layer_dropout_probability=0.9)
elif dropout_name == 'recurrent_dropout_probability':
dropped_stacked_lstm = StackedBidirectionalLstm(input_size=10, hidden_size=11,
num_layers=3,
recurrent_dropout_probability=0.9)
else:
raise ValueError('Do not recognise the following dropout name '
f'{dropout_name}')
# Initialize all weights to be == 1.
constant_init = Initializer.from_params(Params({"type": "constant", "val": 0.5}))
initializer = InitializerApplicator([(".*", constant_init)])
initializer(stacked_lstm)
initializer(dropped_stacked_lstm)
initial_state = torch.randn([3, 5, 11])
initial_memory = torch.randn([3, 5, 11])
tensor = torch.rand([5, 7, 10])
sequence_lengths = torch.LongTensor([7, 7, 7, 7, 7])
sorted_tensor, sorted_sequence, _, _ = sort_batch_by_length(tensor, sequence_lengths)
lstm_input = pack_padded_sequence(sorted_tensor, sorted_sequence.data.tolist(), batch_first=True)
stacked_output, stacked_state = stacked_lstm(lstm_input, (initial_state, initial_memory))
dropped_output, dropped_state = dropped_stacked_lstm(lstm_input, (initial_state, initial_memory))
dropped_output_sequence, _ = pad_packed_sequence(dropped_output, batch_first=True)
stacked_output_sequence, _ = pad_packed_sequence(stacked_output, batch_first=True)
if dropout_name == 'layer_dropout_probability':
with pytest.raises(AssertionError):
numpy.testing.assert_array_almost_equal(dropped_output_sequence.data.numpy(),
stacked_output_sequence.data.numpy(), decimal=4)
if dropout_name == 'recurrent_dropout_probability':
with pytest.raises(AssertionError):
numpy.testing.assert_array_almost_equal(dropped_state[0].data.numpy(),
stacked_state[0].data.numpy(), decimal=4)
with pytest.raises(AssertionError):
numpy.testing.assert_array_almost_equal(dropped_state[1].data.numpy(),
stacked_state[1].data.numpy(), decimal=4)