def test_read_creates_cache_file_when_not_present(self): snli_file = AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl" reader = SnliReader(cache_directory=self.cache_directory) cache_file = reader._get_cache_location_for_file_path(snli_file) assert not os.path.exists(cache_file) reader.read(snli_file) assert os.path.exists(cache_file)
def main(): args = parse_args() checkpoint_path = Path(args.checkpoint) checkpoint_dir = checkpoint_path.parent params_path = checkpoint_dir / 'params.json' vocab_dir = checkpoint_dir / 'vocab' params = Params.from_file(params_path) train_params, model_params = params.pop('train'), params.pop('model') tokenizer = WordTokenizer( start_tokens=['<s>'], end_tokens=['</s>'], ) token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) dataset_reader = SnliReader(tokenizer=tokenizer, token_indexers={'tokens': token_indexer}) valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path')) if not args.test_dataset: test_dataset_path = train_params.pop('test_dataset_path') else: test_dataset_path = args.test_dataset test_dataset = dataset_reader.read(test_dataset_path) if args.only_label: test_dataset = [ d for d in test_dataset if d.fields['label'].label == args.only_label ] vocab = Vocabulary.from_files(vocab_dir) random.shuffle(valid_dataset) model_params['token_embedder']['pretrained_file'] = None model = SNLIModel(params=model_params, vocab=vocab) model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'), strict=False) model.to(args.cuda_device) model.eval() torch.set_grad_enabled(False) iterator = BasicIterator(batch_size=32) iterator.index_with(vocab) for dataset in (valid_dataset, test_dataset): generator = iterator(dataset, shuffle=False, num_epochs=1) model.get_metrics(reset=True) for batch in tqdm(generator): batch = move_to_device(batch, cuda_device=args.cuda_device) model(premise=batch['premise'], hypothesis=batch['hypothesis'], label=batch['label']) metrics = model.get_metrics() pprint(metrics)
def test_read_uses_existing_cache_file_when_present(self): snli_file = AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl" snli_copy_file = str(snli_file) + ".copy" shutil.copyfile(snli_file, snli_copy_file) reader = SnliReader(cache_directory=self.cache_directory) # The first read will create the cache. instances = reader.read(snli_copy_file) # Now we _remove_ the data file, to be sure we're reading from the cache. os.remove(snli_copy_file) cached_instances = reader.read(snli_copy_file) # We should get the same instances both times. assert len(instances) == len(cached_instances) for instance, cached_instance in zip(instances, cached_instances): assert instance.fields == cached_instance.fields
def test_read_from_file(self): reader = SnliReader() dataset = reader.read('tests/fixtures/data/snli.jsonl') instance1 = {"premise": ["A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", "."], "hypothesis": ["A", "person", "is", "training", "his", "horse", "for", "a", "competition", "."], "label": "neutral"} instance2 = {"premise": ["A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", "."], "hypothesis": ["A", "person", "is", "at", "a", "diner", ",", "ordering", "an", "omelette", "."], "label": "contradiction"} instance3 = {"premise": ["A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", "."], "hypothesis": ["A", "person", "is", "outdoors", ",", "on", "a", "horse", "."], "label": "entailment"} assert len(dataset.instances) == 3 fields = dataset.instances[0].fields() assert fields["premise"].tokens() == instance1["premise"] assert fields["hypothesis"].tokens() == instance1["hypothesis"] assert fields["label"].label() == instance1["label"] fields = dataset.instances[1].fields() assert fields["premise"].tokens() == instance2["premise"] assert fields["hypothesis"].tokens() == instance2["hypothesis"] assert fields["label"].label() == instance2["label"] fields = dataset.instances[2].fields() assert fields["premise"].tokens() == instance3["premise"] assert fields["hypothesis"].tokens() == instance3["hypothesis"] assert fields["label"].label() == instance3["label"]
def test_cached_max_instances(self, lazy): snli_file = AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl" # The first read will create the cache if it's not there already. reader = SnliReader(cache_directory=self.cache_directory, lazy=lazy) instances = reader.read(snli_file) instance_count = sum(1 for _ in instances) assert instance_count > 2 # The second read should only return two instances, even though it's from the cache. reader = SnliReader(cache_directory=self.cache_directory, max_instances=2, lazy=lazy) instances = reader.read(snli_file) instance_count = sum(1 for _ in instances) assert instance_count == 2
def test_read_from_file(self, lazy): reader = SnliReader(lazy=lazy) instances = reader.read('tests/fixtures/multinli_1.0_train.jsonl') instances = ensure_list(instances) instance0 = {"premise": ["Conceptually", "cream", "skimming", "has", "two", "basic", "dimensions", "-", "product", "and", "geography", "."], "hypothesis": ["Product", "and", "geography", "are", "what", "make", "cream", "skimming", "work", "."], "label": "neutral"} instance1 = {"premise": ["you", "know", "during", "the", "season", "and", "i", "guess", "at", "at", "your", "level", "uh", "you", "lose", "them", "to", "the", "next", "level", "if", "if", "they", "decide", "to", "recall", "the", "the", "parent", "team", "the", "Braves", "decide", "to", "call", "to", "recall", "a", "guy", "from", "triple", "A", "then", "a", "double", "A", "guy", "goes", "up", "to", "replace", "him", "and", "a", "single", "A", "guy", "goes", "up", "to", "replace", "him"], "hypothesis": ["You", "lose", "the", "things", "to", "the", "following", "level", "if", "the", "people", "recall", "."], "label": "entailment"} instance2 = {"premise": ["One", "of", "our", "number", "will", "carry", "out", "your", "instructions", "minutely", "."], "hypothesis": ["A", "member", "of", "my", "team", "will", "execute", "your", "orders", "with", "immense", "precision", "."], "label": "entailment"} assert len(instances) == 3 def equals(fields, instance): assert [t.text for t in fields["premise"].tokens] == instance["premise"] assert [t.text for t in fields["hypothesis"].tokens] == instance["hypothesis"] assert fields["label"].label == instance["label"] equals(instances[0].fields, instance0) equals(instances[1].fields, instance1) equals(instances[2].fields, instance2)
def test_read_from_file(self, lazy): reader = SnliReader(lazy=lazy) instances = reader.read(AllenNlpTestCase.FIXTURES_ROOT / u'data' / u'snli.jsonl') instances = ensure_list(instances) instance1 = {u"premise": [u"A", u"person", u"on", u"a", u"horse", u"jumps", u"over", u"a", u"broken", u"down", u"airplane", u"."], u"hypothesis": [u"A", u"person", u"is", u"training", u"his", u"horse", u"for", u"a", u"competition", u"."], u"label": u"neutral"} instance2 = {u"premise": [u"A", u"person", u"on", u"a", u"horse", u"jumps", u"over", u"a", u"broken", u"down", u"airplane", u"."], u"hypothesis": [u"A", u"person", u"is", u"at", u"a", u"diner", u",", u"ordering", u"an", u"omelette", u"."], u"label": u"contradiction"} instance3 = {u"premise": [u"A", u"person", u"on", u"a", u"horse", u"jumps", u"over", u"a", u"broken", u"down", u"airplane", u"."], u"hypothesis": [u"A", u"person", u"is", u"outdoors", u",", u"on", u"a", u"horse", u"."], u"label": u"entailment"} assert len(instances) == 3 fields = instances[0].fields assert [t.text for t in fields[u"premise"].tokens] == instance1[u"premise"] assert [t.text for t in fields[u"hypothesis"].tokens] == instance1[u"hypothesis"] assert fields[u"label"].label == instance1[u"label"] fields = instances[1].fields assert [t.text for t in fields[u"premise"].tokens] == instance2[u"premise"] assert [t.text for t in fields[u"hypothesis"].tokens] == instance2[u"hypothesis"] assert fields[u"label"].label == instance2[u"label"] fields = instances[2].fields assert [t.text for t in fields[u"premise"].tokens] == instance3[u"premise"] assert [t.text for t in fields[u"hypothesis"].tokens] == instance3[u"hypothesis"] assert fields[u"label"].label == instance3[u"label"]
def test_read_from_file(self, lazy): reader = SnliReader(lazy=lazy) instances = reader.read('tests/fixtures/data/snli.jsonl') instances = ensure_list(instances) instance1 = {"premise": ["A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", "."], "hypothesis": ["A", "person", "is", "training", "his", "horse", "for", "a", "competition", "."], "label": "neutral"} instance2 = {"premise": ["A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", "."], "hypothesis": ["A", "person", "is", "at", "a", "diner", ",", "ordering", "an", "omelette", "."], "label": "contradiction"} instance3 = {"premise": ["A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", "."], "hypothesis": ["A", "person", "is", "outdoors", ",", "on", "a", "horse", "."], "label": "entailment"} assert len(instances) == 3 fields = instances[0].fields assert [t.text for t in fields["premise"].tokens] == instance1["premise"] assert [t.text for t in fields["hypothesis"].tokens] == instance1["hypothesis"] assert fields["label"].label == instance1["label"] fields = instances[1].fields assert [t.text for t in fields["premise"].tokens] == instance2["premise"] assert [t.text for t in fields["hypothesis"].tokens] == instance2["hypothesis"] assert fields["label"].label == instance2["label"] fields = instances[2].fields assert [t.text for t in fields["premise"].tokens] == instance3["premise"] assert [t.text for t in fields["hypothesis"].tokens] == instance3["hypothesis"] assert fields["label"].label == instance3["label"]
def test_caching_works_with_lazy_reading(self): snli_file = self.FIXTURES_ROOT / "data" / "snli.jsonl" snli_copy_file = str(snli_file) + ".copy" shutil.copyfile(snli_file, snli_copy_file) reader = SnliReader(lazy=True) reader.cache_data(self.cache_directory) cache_file = reader._get_cache_location_for_file_path(snli_copy_file) # The call to read() will give us an _iterator_. We'll iterate over it multiple times, # and the caching behavior should change as we go. instances = reader.read(snli_copy_file) assert isinstance(instances, _LazyInstances) # The first iteration will create the cache assert not os.path.exists(cache_file) first_pass_instances = [] for instance in instances: first_pass_instances.append(instance) assert os.path.exists(cache_file) # Now we _remove_ the data file, to be sure we're reading from the cache. os.remove(snli_copy_file) second_pass_instances = [] for instance in instances: second_pass_instances.append(instance) # We should get the same instances both times. assert len(first_pass_instances) == len(second_pass_instances) for instance, cached_instance in zip(first_pass_instances, second_pass_instances): assert instance.fields == cached_instance.fields # And just to be super paranoid, in case the second pass somehow bypassed the cache # because of a bug in `_CachedLazyInstance` that's hard to detect, we'll read the # instances from the cache with a non-lazy iterator and make sure they're the same. reader = SnliReader(lazy=False) reader.cache_data(self.cache_directory) cached_instances = reader.read(snli_copy_file) assert len(first_pass_instances) == len(cached_instances) for instance, cached_instance in zip(first_pass_instances, cached_instances): assert instance.fields == cached_instance.fields
def main(): args = parse_args() checkpoint_path = Path(args.checkpoint) checkpoint_dir = checkpoint_path.parent params_path = checkpoint_dir / 'params.json' vocab_dir = checkpoint_dir / 'vocab' params = Params.from_file(params_path) train_params, model_params = params.pop('train'), params.pop('model') tokenizer = WordTokenizer( start_tokens=['<s>'], end_tokens=['</s>'], ) token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) dataset_reader = SnliReader(tokenizer=tokenizer, token_indexers={'tokens': token_indexer}) valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path')) vocab = Vocabulary.from_files(vocab_dir) random.shuffle(valid_dataset) model_params['token_embedder']['pretrained_file'] = None model = SNLIModel(params=model_params, vocab=vocab) model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'), strict=False) model.eval() iterator = BasicIterator(batch_size=1) iterator.index_with(vocab) generator = iterator(valid_dataset) for i in range(10): batch = next(generator) label_token_to_index = vocab.get_token_to_index_vocabulary('labels') print('----') print(' '.join( model.convert_to_readable_text(batch['premise']['tokens'])[0])) for label, label_index in label_token_to_index.items(): label_tensor = torch.tensor([label_index]) enc_embs = model.embed(batch['premise']['tokens']) enc_mask = get_text_field_mask(batch['premise']) enc_hidden = model.encode(inputs=enc_embs, mask=enc_mask, drop_start_token=True) code, kld = model.sample_code_and_compute_kld(enc_hidden) generated = model.generate(code=code, label=label_tensor, max_length=enc_mask.sum(1) * 2, beam_size=10, lp_alpha=args.lp_alpha) text = model.convert_to_readable_text(generated[:, 0])[0] print(label) print(' '.join(text))
def main(): args = parse_args() checkpoint_path = Path(args.checkpoint) checkpoint_dir = checkpoint_path.parent params_path = checkpoint_dir / 'params.json' vocab_dir = checkpoint_dir / 'vocab' params = Params.from_file(params_path) train_params, model_params = params.pop('train'), params.pop('model') tokenizer = WordTokenizer(start_tokens=['<s>'], end_tokens=['</s>'],) token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) dataset_reader = SnliReader( tokenizer=tokenizer, token_indexers={'tokens': token_indexer}) valid_dataset = dataset_reader.read( train_params.pop('valid_dataset_path')) vocab = Vocabulary.from_files(vocab_dir) model_params['token_embedder']['pretrained_file'] = None model = SNLIModel(params=model_params, vocab=vocab) model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'), strict=False) model.to(args.device) model.eval() iterator = BasicIterator(batch_size=args.batch_size) iterator.index_with(vocab) generator = iterator(valid_dataset, num_epochs=1, shuffle=False) label_index_to_token = vocab.get_index_to_token_vocabulary('labels') out_file = open(args.out, 'w') for batch in tqdm(generator): premise_tokens = batch['premise']['tokens'] enc_embs = model.embed(premise_tokens.to(args.device)) enc_mask = get_text_field_mask(batch['premise']).to(args.device) enc_hidden = model.encode(inputs=enc_embs, mask=enc_mask, drop_start_token=True) code, kld = model.sample_code_and_compute_kld(enc_hidden) pre_text = model.convert_to_readable_text(premise_tokens[:, 1:]) label_tensor = batch['label'].to(args.device) generated = model.generate( code=code, label=label_tensor, max_length=25, beam_size=10, lp_alpha=args.lp_alpha) text = model.convert_to_readable_text(generated[:, 0]) for pre_text_b, text_b, label_index_b in zip(pre_text, text, label_tensor): obj = {'sentence1': ' '.join(pre_text_b), 'sentence2': ' '.join(text_b), 'gold_label': label_index_to_token[label_index_b.item()]} out_file.write(json.dumps(obj)) out_file.write('\n')
def test_combine_input_fields(self): reader = SnliReader( tokenizer=PretrainedTransformerTokenizer("bert-base-uncased"), combine_input_fields=True) instances = reader.read(AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl") instances = ensure_list(instances) instance1 = { "tokens": [ "[CLS]", "a", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", ".", "[SEP]", "a", "person", "is", "training", "his", "horse", "for", "a", "competition", ".", "[SEP]", ], "label": "neutral", } assert len(instances) == 3 fields = instances[0].fields assert [t.text for t in fields["tokens"].tokens] == instance1["tokens"] assert fields["label"].label == instance1["label"]
def test_read_only_creates_cache_file_once(self): snli_file = AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl" reader = SnliReader(cache_directory=self.cache_directory) cache_file = reader._get_cache_location_for_file_path(snli_file) # The first read will create the cache. reader.read(snli_file) assert os.path.exists(cache_file) with open(cache_file, "r") as in_file: cache_contents = in_file.read() # The second and all subsequent reads should _use_ the cache, not modify it. I looked # into checking file modification times, but this test will probably be faster than the # granularity of `os.path.getmtime()` (which only returns values in seconds). reader.read(snli_file) reader.read(snli_file) reader.read(snli_file) reader.read(snli_file) with open(cache_file, "r") as in_file: final_cache_contents = in_file.read() assert cache_contents == final_cache_contents
serialization_dir = 'checkpoints/' device = 5 torch.cuda.set_device(device) batch_size = 64 hid_dim = 100 embed_dim = 300 lr = 1e-3 grad_clipping = 5 dropout = 0.1 # lazy=False将数据一次性加入内存, lazy=True边训练边加载 print('data loading, please wait...') reader = SnliReader(lazy=True) train_dataset = reader.read("snli_1.0/snli_1.0_train.jsonl") dev_dataset = reader.read("snli_1.0/snli_1.0_dev.jsonl") test_dataset = reader.read("snli_1.0/snli_1.0_test.jsonl") if os.path.exists(vocab_dir): vocab = Vocabulary.from_files(vocab_dir) else: vocab = Vocabulary.from_instances(chain(train_dataset, dev_dataset), max_vocab_size=max_vocab_size) vocab.save_to_files(vocab_dir) print("vocab_size: {}".format(vocab.get_vocab_size())) train_iterator = BucketIterator(batch_size=batch_size, sorting_keys=[("premise", "num_tokens")]) dev_iterator = BucketIterator(batch_size=batch_size,
def main(): args = parse_args() params = Params.from_file(args.params) save_dir = Path(args.save) save_dir.mkdir(parents=True) params.to_file(save_dir / 'params.json') train_params, model_params = params.pop('train'), params.pop('model') random_seed = train_params.pop_int('random_seed', 2019) torch.manual_seed(random_seed) random.seed(random_seed) log_filename = save_dir / 'stdout.log' sys.stdout = TeeLogger(filename=log_filename, terminal=sys.stdout, file_friendly_terminal_output=False) sys.stderr = TeeLogger(filename=log_filename, terminal=sys.stderr, file_friendly_terminal_output=False) tokenizer = WordTokenizer(start_tokens=['<s>'], end_tokens=['</s>']) token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) dataset_reader = SnliReader(tokenizer=tokenizer, token_indexers={'tokens': token_indexer}) train_labeled_dataset_path = train_params.pop('train_labeled_dataset_path') train_unlabeled_dataset_path = train_params.pop( 'train_unlabeled_dataset_path', None) train_labeled_dataset = dataset_reader.read(train_labeled_dataset_path) train_labeled_dataset = filter_dataset_by_length( dataset=train_labeled_dataset, max_length=30) if train_unlabeled_dataset_path is not None: train_unlabeled_dataset = dataset_reader.read( train_unlabeled_dataset_path) train_unlabeled_dataset = filter_dataset_by_length( dataset=train_unlabeled_dataset, max_length=30) else: train_unlabeled_dataset = [] valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path')) vocab = Vocabulary.from_instances( instances=train_labeled_dataset + train_unlabeled_dataset, max_vocab_size=train_params.pop_int('max_vocab_size', None)) vocab.save_to_files(save_dir / 'vocab') labeled_batch_size = train_params.pop_int('labeled_batch_size') unlabeled_batch_size = train_params.pop_int('unlabeled_batch_size') labeled_iterator = BasicIterator(batch_size=labeled_batch_size) unlabeled_iterator = BasicIterator(batch_size=unlabeled_batch_size) labeled_iterator.index_with(vocab) unlabeled_iterator.index_with(vocab) if not train_unlabeled_dataset: unlabeled_iterator = None pretrained_checkpoint_path = train_params.pop('pretrained_checkpoint_path', None) model = SNLIModel(params=model_params, vocab=vocab) if pretrained_checkpoint_path: model.load_state_dict( torch.load(pretrained_checkpoint_path, map_location='cpu')) model.add_finetune_parameters( con_autoweight=train_params.pop_bool('con_autoweight', False), con_y_weight=train_params.pop_float('con_y_weight'), con_z_weight=train_params.pop_float('con_z_weight'), con_z2_weight=train_params.pop_float('con_z2_weight')) main_optimizer = optim.Adam(params=model.finetune_main_parameters( exclude_generator=train_params.pop_bool('exclude_generator')), lr=train_params.pop_float('lr', 1e-3)) aux_optimizer = optim.Adam(params=model.finetune_aux_parameters(), lr=train_params.pop_float('aux_lr', 1e-4)) summary_writer = SummaryWriter(log_dir=save_dir / 'log') kl_anneal_rate = train_params.pop_float('kl_anneal_rate', None) if kl_anneal_rate is None: kl_weight_scheduler = None else: kl_weight_scheduler = (lambda step: min(1.0, kl_anneal_rate * step)) model.kl_weight = 0.0 gumbel_anneal_rate = train_params.pop_float('gumbel_anneal_rate', None) if gumbel_anneal_rate is None: gumbel_temperature_scheduler = None else: gumbel_temperature_scheduler = ( lambda step: max(0.1, 1.0 - gumbel_anneal_rate * step)) model.gumbel_temperature = 1.0 iters_per_epoch = train_params.pop_int( 'iters_per_epoch', len(train_labeled_dataset) // labeled_batch_size) trainer = FineTuningTrainer( model=model, main_optimizer=main_optimizer, aux_optimizer=aux_optimizer, labeled_iterator=labeled_iterator, unlabeled_iterator=unlabeled_iterator, train_labeled_dataset=train_labeled_dataset, train_unlabeled_dataset=train_unlabeled_dataset, validation_dataset=valid_dataset, summary_writer=summary_writer, serialization_dir=save_dir, num_epochs=train_params.pop_int('num_epochs', 50), iters_per_epoch=iters_per_epoch, write_summary_every=100, validate_every=1000, patience=train_params.pop_int('patience', 5), clip_grad_max_norm=train_params.pop_float('grad_max_norm', 5.0), kl_weight_scheduler=kl_weight_scheduler, gumbel_temperature_scheduler=gumbel_temperature_scheduler, cuda_device=train_params.pop_int('cuda_device', 0), ) trainer.train()
from allennlp.data import Vocabulary from allennlp.data.dataset_readers import SnliReader from allennlp.data.iterators.basic_iterator import BasicIterator from allennlp.modules.token_embedders.embedding import Embedding from allennlp.training.util import evaluate from model import BowmanEtAlRNN, BowmanEtAlSumOfWords, NLIPredictor, RocktaschelEtAlConditionalEncoding, RocktaschelEtAlAttention, ChenEtAlESIM from utils import grad_zero, comb_to_str, re_read_embeddings_from_text_file t = SnliReader() ### You can choose train/val/test datasets here train_dataset = t.read('.data/snli_1.0/snli_1.0_train.jsonl') val_dataset = t.read('.data/snli_1.0/snli_1.0_dev.jsonl') test_dataset = t.read('.data/snli_1.0/snli_1.0_test.jsonl') vocab = Vocabulary.from_instances(train_dataset + val_dataset) vocab = Vocabulary.from_files('./.vocab/snli_vocab') glove = Embedding(vocab.get_vocab_size(), 300) ### Choose and load model here model = RocktaschelEtAlAttention(vocab, glove, word_by_word=False).to("cuda") with open( './.serialization_data/C.E. Attention_Adam_32_0.1_0.0003_5e-05_True/best.th', 'rb') as f: model.load_state_dict(torch.load(f))
def test_read_from_file(self, lazy): reader = SnliReader(lazy=lazy) instances = reader.read(AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl") instances = ensure_list(instances) instance1 = { "premise": [ "A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", ".", ], "hypothesis": [ "A", "person", "is", "training", "his", "horse", "for", "a", "competition", ".", ], "label": "neutral", } instance2 = { "premise": [ "A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", ".", ], "hypothesis": [ "A", "person", "is", "at", "a", "diner", ",", "ordering", "an", "omelette", ".", ], "label": "contradiction", } instance3 = { "premise": [ "A", "person", "on", "a", "horse", "jumps", "over", "a", "broken", "down", "airplane", ".", ], "hypothesis": ["A", "person", "is", "outdoors", ",", "on", "a", "horse", "."], "label": "entailment", } assert len(instances) == 3 fields = instances[0].fields assert [t.text for t in fields["premise"].tokens] == instance1["premise"] assert [t.text for t in fields["hypothesis"].tokens ] == instance1["hypothesis"] assert fields["label"].label == instance1["label"] fields = instances[1].fields assert [t.text for t in fields["premise"].tokens] == instance2["premise"] assert [t.text for t in fields["hypothesis"].tokens ] == instance2["hypothesis"] assert fields["label"].label == instance2["label"] fields = instances[2].fields assert [t.text for t in fields["premise"].tokens] == instance3["premise"] assert [t.text for t in fields["hypothesis"].tokens ] == instance3["hypothesis"] assert fields["label"].label == instance3["label"]
def test_max_instances(self, lazy): snli_file = AllenNlpTestCase.FIXTURES_ROOT / "data" / "snli.jsonl" reader = SnliReader(max_instances=2, lazy=lazy) instances = reader.read(snli_file) instance_count = sum(1 for _ in instances) assert instance_count == 2
def main(): args = parse_args() params = Params.from_file(args.params) save_dir = Path(args.save) save_dir.mkdir(parents=True) params.to_file(save_dir / 'params.json') train_params, model_params = params.pop('train'), params.pop('model') random_seed = train_params.pop_int('random_seed', 2019) torch.manual_seed(random_seed) random.seed(random_seed) log_filename = save_dir / 'stdout.log' sys.stdout = TeeLogger(filename=log_filename, terminal=sys.stdout, file_friendly_terminal_output=False) sys.stderr = TeeLogger(filename=log_filename, terminal=sys.stderr, file_friendly_terminal_output=False) tokenizer = WordTokenizer( start_tokens=['<s>'], end_tokens=['</s>'], ) token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) dataset_reader = SnliReader(tokenizer=tokenizer, token_indexers={'tokens': token_indexer}) train_labeled_dataset_path = train_params.pop('train_labeled_dataset_path') train_unlabeled_dataset_path = train_params.pop( 'train_unlabeled_dataset_path', None) train_labeled_dataset = dataset_reader.read(train_labeled_dataset_path) train_labeled_dataset = filter_dataset_by_length( dataset=train_labeled_dataset, max_length=30) if train_unlabeled_dataset_path is not None: train_unlabeled_dataset = dataset_reader.read( train_unlabeled_dataset_path) train_unlabeled_dataset = filter_dataset_by_length( dataset=train_unlabeled_dataset, max_length=30) else: train_unlabeled_dataset = [] valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path')) vocab = Vocabulary.from_instances( instances=train_labeled_dataset + train_unlabeled_dataset, max_vocab_size=train_params.pop_int('max_vocab_size', None)) vocab.save_to_files(save_dir / 'vocab') labeled_batch_size = train_params.pop_int('labeled_batch_size') unlabeled_batch_size = train_params.pop_int('unlabeled_batch_size') labeled_iterator = BasicIterator(batch_size=labeled_batch_size) unlabeled_iterator = BasicIterator(batch_size=unlabeled_batch_size) labeled_iterator.index_with(vocab) unlabeled_iterator.index_with(vocab) if not train_unlabeled_dataset: unlabeled_iterator = None model = SNLIModel(params=model_params, vocab=vocab) optimizer = optim.Adam(params=model.parameters(), lr=train_params.pop_float('lr', 1e-3)) summary_writer = SummaryWriter(log_dir=save_dir / 'log') kl_anneal_rate = train_params.pop_float('kl_anneal_rate', None) if kl_anneal_rate is None: kl_weight_scheduler = None else: kl_weight_scheduler = (lambda step: min(1.0, kl_anneal_rate * step)) model.kl_weight = 0.0 trainer = Trainer(model=model, optimizer=optimizer, labeled_iterator=labeled_iterator, unlabeled_iterator=unlabeled_iterator, train_labeled_dataset=train_labeled_dataset, train_unlabeled_dataset=train_unlabeled_dataset, validation_dataset=valid_dataset, summary_writer=summary_writer, serialization_dir=save_dir, num_epochs=train_params.pop('num_epochs', 50), iters_per_epoch=len(train_labeled_dataset) // labeled_batch_size, write_summary_every=100, validate_every=2000, patience=2, clip_grad_max_norm=5, kl_weight_scheduler=kl_weight_scheduler, cuda_device=train_params.pop_int('cuda_device', 0), early_stop=train_params.pop_bool('early_stop', True)) trainer.train()
def getMnliBow(dataset, vocab, bow_type='groundBow'): premises, hypothesis, labs = mnliToList(dataset, vocab) premises = getBow(premises, vocab, bow_type) hypothesis = getBow(hypothesis, vocab, bow_type) labels = np.zeros((len(dataset), vocab.get_vocab_size(namespace='labels'))) for i in range(len(dataset)): labels[i, labs[i]] = 1 return (premises, hypothesis, labels) reader = SnliReader() # train_dataset = reader.read(cached_path('datasets/multinli_1.0/multinli_1.0_train.jsonl')) train_dataset = reader.read('tests/fixtures/train1000.jsonl') # Fixture validation_dataset = reader.read('tests/fixtures/val1000.jsonl') # Fixture #validation_dataset = reader.read('datasets/multinli_1.0/multinli_1.0_dev_matched.jsonl') # print(train_dataset) vocab = Vocabulary.from_instances(train_dataset + validation_dataset) # vocab.print_statistics() t_premises, t_hypothesis, t_labels = getMnliBow(train_dataset, vocab, 'freqBow') v_premises, v_hypothesis, v_labels = getMnliBow(validation_dataset, vocab, 'freqBow') # for i in range(3): # print(i) # print(t_premises[i]) # print(t_hypothesis[i])
def main(): args = parse_args() params = Params.from_file(args.params) save_dir = Path(args.save) save_dir.mkdir(parents=True) params.to_file(save_dir / 'params.json') train_params, model_params = params.pop('train'), params.pop('model') random_seed = train_params.pop_int('random_seed', 2019) torch.manual_seed(random_seed) random.seed(random_seed) log_filename = save_dir / 'stdout.log' sys.stdout = TeeLogger( filename=log_filename, terminal=sys.stdout, file_friendly_terminal_output=False) sys.stderr = TeeLogger( filename=log_filename, terminal=sys.stderr, file_friendly_terminal_output=False) tokenizer = WordTokenizer() token_indexer = SingleIdTokenIndexer(lowercase_tokens=True) dataset_reader = SnliReader( tokenizer=tokenizer, token_indexers={'tokens': token_indexer}) train_labeled_dataset_path = train_params.pop( 'train_labeled_dataset_path') train_unlabeled_dataset_path = train_params.pop( 'train_unlabeled_dataset_path', None) train_labeled_dataset = dataset_reader.read(train_labeled_dataset_path) truncate_or_pad_dataset(dataset=train_labeled_dataset, length=29) if train_unlabeled_dataset_path is not None: train_unlabeled_dataset = dataset_reader.read( train_unlabeled_dataset_path) truncate_or_pad_dataset( dataset=train_unlabeled_dataset, length=29) else: train_unlabeled_dataset = [] valid_dataset = dataset_reader.read( train_params.pop('valid_dataset_path')) truncate_or_pad_dataset(valid_dataset, length=29) vocab = Vocabulary.from_instances( instances=train_labeled_dataset + train_unlabeled_dataset, max_vocab_size=train_params.pop_int('max_vocab_size', None)) vocab.save_to_files(save_dir / 'vocab') labeled_batch_size = train_params.pop_int('labeled_batch_size') unlabeled_batch_size = train_params.pop_int('unlabeled_batch_size') labeled_iterator = BasicIterator(batch_size=labeled_batch_size) unlabeled_iterator = BasicIterator(batch_size=unlabeled_batch_size) labeled_iterator.index_with(vocab) unlabeled_iterator.index_with(vocab) if not train_unlabeled_dataset: unlabeled_iterator = None model = DeconvSNLIModel(params=model_params, vocab=vocab) optimizer = optim.Adam(params=model.parameters()) summary_writer = SummaryWriter(log_dir=save_dir / 'log') trainer = SeparatedLVMTrainer( model=model, optimizer=optimizer, labeled_iterator=labeled_iterator, unlabeled_iterator=unlabeled_iterator, train_labeled_dataset=train_labeled_dataset, train_unlabeled_dataset=train_unlabeled_dataset, validation_dataset=valid_dataset, summary_writer=summary_writer, serialization_dir=save_dir, num_epochs=train_params.pop('num_epochs', 50), iters_per_epoch=len(train_labeled_dataset) // labeled_batch_size, write_summary_every=100, validate_every=2000, patience=2, clip_grad_max_norm=5, cuda_device=train_params.pop_int('cuda_device', 0) ) trainer.train()
import torch from allennlp.data.dataset_readers import SnliReader from allennlp.predictors.predictor import Predictor from allennlp.data.tokenizers import CharacterTokenizer from allennlp.data.token_indexers import ELMoTokenCharactersIndexer torch.manual_seed(1) DATA_PT = 'data/multinli_1.0/multinli_1.0_dev_matched.jsonl' reader = SnliReader(CharacterTokenizer(), {"elmo": ELMoTokenCharactersIndexer()}) data = reader.read(DATA_PT) MODEL_PT = "models/decomposable-attention-elmo-2018.02.19.tar.gz" predictor = Predictor.from_path(MODEL_PT) result = predictor.predict_instance(data[0]) print(result)