def __init__(self,
tokenizer: Callable[[str], List[str]] = lambda x: x.split(),
token_indexers: Dict[str, TokenIndexer] = None,
max_seq_length: Optional[int] = MAX_SEQ_LENGTH) -> None:
super().__init__(lazy=False)
self.tokenizer = tokenizer
self.token_indexers = token_indexers or {
"tokens": SingleIdTokenIndexer()
}
self.max_seq_length = max_seq_length
def main():
args = parse_args()
checkpoint_path = Path(args.checkpoint)
checkpoint_dir = checkpoint_path.parent
params_path = checkpoint_dir / 'params.json'
vocab_dir = checkpoint_dir / 'vocab'
params = Params.from_file(params_path)
train_params, model_params = params.pop('train'), params.pop('model')
tokenizer = WordTokenizer(
start_tokens=['<s>'],
end_tokens=['</s>'],
)
token_indexer = SingleIdTokenIndexer(lowercase_tokens=True)
dataset_reader = SnliReader(tokenizer=tokenizer,
token_indexers={'tokens': token_indexer})
valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path'))
if not args.test_dataset:
test_dataset_path = train_params.pop('test_dataset_path')
else:
test_dataset_path = args.test_dataset
test_dataset = dataset_reader.read(test_dataset_path)
if args.only_label:
test_dataset = [
d for d in test_dataset
if d.fields['label'].label == args.only_label
]
vocab = Vocabulary.from_files(vocab_dir)
random.shuffle(valid_dataset)
model_params['token_embedder']['pretrained_file'] = None
model = SNLIModel(params=model_params, vocab=vocab)
model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'),
strict=False)
model.to(args.cuda_device)
model.eval()
torch.set_grad_enabled(False)
iterator = BasicIterator(batch_size=32)
iterator.index_with(vocab)
for dataset in (valid_dataset, test_dataset):
generator = iterator(dataset, shuffle=False, num_epochs=1)
model.get_metrics(reset=True)
for batch in tqdm(generator):
batch = move_to_device(batch, cuda_device=args.cuda_device)
model(premise=batch['premise'],
hypothesis=batch['hypothesis'],
label=batch['label'])
metrics = model.get_metrics()
pprint(metrics)
def main():
args = parse_args()
checkpoint_path = Path(args.checkpoint)
checkpoint_dir = checkpoint_path.parent
params_path = checkpoint_dir / 'params.json'
vocab_dir = checkpoint_dir / 'vocab'
params = Params.from_file(params_path)
train_params, model_params = params.pop('train'), params.pop('model')
tokenizer = WordTokenizer(
start_tokens=['<s>'],
end_tokens=['</s>'],
)
token_indexer = SingleIdTokenIndexer(lowercase_tokens=True)
dataset_reader = SnliReader(tokenizer=tokenizer,
token_indexers={'tokens': token_indexer})
valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path'))
vocab = Vocabulary.from_files(vocab_dir)
random.shuffle(valid_dataset)
model_params['token_embedder']['pretrained_file'] = None
model = SNLIModel(params=model_params, vocab=vocab)
model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'),
strict=False)
model.eval()
iterator = BasicIterator(batch_size=1)
iterator.index_with(vocab)
generator = iterator(valid_dataset)
for i in range(10):
batch = next(generator)
label_token_to_index = vocab.get_token_to_index_vocabulary('labels')
print('----')
print(' '.join(
model.convert_to_readable_text(batch['premise']['tokens'])[0]))
for label, label_index in label_token_to_index.items():
label_tensor = torch.tensor([label_index])
enc_embs = model.embed(batch['premise']['tokens'])
enc_mask = get_text_field_mask(batch['premise'])
enc_hidden = model.encode(inputs=enc_embs,
mask=enc_mask,
drop_start_token=True)
code, kld = model.sample_code_and_compute_kld(enc_hidden)
generated = model.generate(code=code,
label=label_tensor,
max_length=enc_mask.sum(1) * 2,
beam_size=10,
lp_alpha=args.lp_alpha)
text = model.convert_to_readable_text(generated[:, 0])[0]
print(label)
print(' '.join(text))
def __init__(self, tokenizer,
token_indexers,
max_seq_len,
label_cols,
feature_type,
testing):
super().__init__(lazy=False)
self.tokenizer = tokenizer
self.token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
self.max_seq_len = max_seq_len
self.label_cols = label_cols
self.feature_type = feature_type
self.testing = testing
def main():
args = parse_args()
checkpoint_path = Path(args.checkpoint)
checkpoint_dir = checkpoint_path.parent
params_path = checkpoint_dir / 'params.json'
vocab_dir = checkpoint_dir / 'vocab'
params = Params.from_file(params_path)
train_params, model_params = params.pop('train'), params.pop('model')
tokenizer = WordTokenizer(start_tokens=['<s>'], end_tokens=['</s>'],)
token_indexer = SingleIdTokenIndexer(lowercase_tokens=True)
dataset_reader = SnliReader(
tokenizer=tokenizer, token_indexers={'tokens': token_indexer})
valid_dataset = dataset_reader.read(
train_params.pop('valid_dataset_path'))
vocab = Vocabulary.from_files(vocab_dir)
model_params['token_embedder']['pretrained_file'] = None
model = SNLIModel(params=model_params, vocab=vocab)
model.load_state_dict(torch.load(checkpoint_path, map_location='cpu'),
strict=False)
model.to(args.device)
model.eval()
iterator = BasicIterator(batch_size=args.batch_size)
iterator.index_with(vocab)
generator = iterator(valid_dataset, num_epochs=1, shuffle=False)
label_index_to_token = vocab.get_index_to_token_vocabulary('labels')
out_file = open(args.out, 'w')
for batch in tqdm(generator):
premise_tokens = batch['premise']['tokens']
enc_embs = model.embed(premise_tokens.to(args.device))
enc_mask = get_text_field_mask(batch['premise']).to(args.device)
enc_hidden = model.encode(inputs=enc_embs, mask=enc_mask,
drop_start_token=True)
code, kld = model.sample_code_and_compute_kld(enc_hidden)
pre_text = model.convert_to_readable_text(premise_tokens[:, 1:])
label_tensor = batch['label'].to(args.device)
generated = model.generate(
code=code, label=label_tensor, max_length=25,
beam_size=10, lp_alpha=args.lp_alpha)
text = model.convert_to_readable_text(generated[:, 0])
for pre_text_b, text_b, label_index_b in zip(pre_text, text, label_tensor):
obj = {'sentence1': ' '.join(pre_text_b), 'sentence2': ' '.join(text_b),
'gold_label': label_index_to_token[label_index_b.item()]}
out_file.write(json.dumps(obj))
out_file.write('\n')
def get_vocab_and_both_elmo_indexed_ids(batch: List[List[str]]):
instances = []
indexer = ELMoTokenCharactersIndexer()
indexer2 = SingleIdTokenIndexer()
for sentence in batch:
tokens = [Token(token) for token in sentence]
field = TextField(tokens, {"character_ids": indexer, "tokens": indexer2})
instance = Instance({"elmo": field})
instances.append(instance)
dataset = Batch(instances)
vocab = Vocabulary.from_instances(instances)
dataset.index_instances(vocab)
return vocab, dataset.as_tensor_dict()["elmo"]
def __init__(self,
max_seq_length: int,
lazy: bool = False,
coding_scheme: str = "IOB1",
label_namespace: str = "labels",
tag_namespace: str = "tags",
tokenizer: Tokenizer = None,
token_indexers: Dict[str, TokenIndexer] = None) -> None:
super().__init__(lazy)
self.max_seq_length = max_seq_length
self.lazy = lazy
self.coding_scheme = coding_scheme
self.label_namespace = label_namespace
self.tag_namespace = tag_namespace
self._original_coding_scheme = "IOB1"
self._tokenizer = tokenizer or WordTokenizer()
self._token_indexers = token_indexers or {
"tokens": SingleIdTokenIndexer()
}
self.rng = random.Random(time.time())
def __init__(self,
chunker_path: str,
segmental_vocabulary: Vocabulary,
preprocessed_chunk_file: str = None,
max_span_width: int = 89,
update_chunker_params: bool = False,
remove_dropout: bool = False,
bos_token: str = '<S>',
eos_token: str = '</S>',
namespace: str = 'chunky_elmo') -> None:
self._namespace = namespace
self._max_span_width = max_span_width
# First initialize the chunker.
if preprocessed_chunk_file is not None:
self.chunks_dict: Dict(str, List[str]) = {}
self.read_predicted_chunks(preprocessed_chunk_file)
else:
self.chunks_dict = None
logger.info("Reading Chunker from %s", chunker_path)
from allennlp.models.archival import load_archive
chunker_archive = load_archive(chunker_path)
self.chunker = chunker_archive.model
if not update_chunker_params:
for param in self.chunker.parameters():
param.requires_grad_(False)
if remove_dropout:
# Setting dropout to 0.0 for all parameters in chunker.
self.chunker.dropout.p = 0.0
self.chunker.encoder._module.dropout = 0.0
self.chunker.text_field_embedder.token_embedder_elmo._elmo._dropout.p = 0.0
self.elmo_indexer = ELMoTokenCharactersIndexer(
namespace='elmo_characters')
self.token_indexer = SingleIdTokenIndexer()
self.seglm_vocab = segmental_vocabulary #load_archive(segmental_path).model.vocab
self.bos_token = bos_token
self.eos_token = eos_token
def main():
args = parse_args()
params = Params.from_file(args.params)
save_dir = Path(args.save)
save_dir.mkdir(parents=True)
params.to_file(save_dir / 'params.json')
train_params, model_params = params.pop('train'), params.pop('model')
random_seed = train_params.pop_int('random_seed', 2019)
torch.manual_seed(random_seed)
random.seed(random_seed)
log_filename = save_dir / 'stdout.log'
sys.stdout = TeeLogger(filename=log_filename,
terminal=sys.stdout,
file_friendly_terminal_output=False)
sys.stderr = TeeLogger(filename=log_filename,
terminal=sys.stderr,
file_friendly_terminal_output=False)
tokenizer = WordTokenizer(word_splitter=JustSpacesWordSplitter(),
start_tokens=['<s>'],
end_tokens=['</s>'])
token_indexer = SingleIdTokenIndexer(lowercase_tokens=True)
dataset_reader = QuoraParaphraseDatasetReader(
tokenizer=tokenizer, token_indexers={'tokens': token_indexer})
train_labeled_dataset_path = train_params.pop('train_labeled_dataset_path')
train_unlabeled_dataset_path = train_params.pop(
'train_unlabeled_dataset_path', None)
train_labeled_dataset = dataset_reader.read(train_labeled_dataset_path)
train_labeled_dataset = filter_dataset_by_length(
dataset=train_labeled_dataset, max_length=35)
if train_unlabeled_dataset_path is not None:
train_unlabeled_dataset = dataset_reader.read(
train_unlabeled_dataset_path)
train_unlabeled_dataset = filter_dataset_by_length(
dataset=train_unlabeled_dataset, max_length=35)
else:
train_unlabeled_dataset = []
valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path'))
vocab = Vocabulary.from_instances(
instances=train_labeled_dataset + train_unlabeled_dataset,
max_vocab_size=train_params.pop_int('max_vocab_size', None))
vocab.save_to_files(save_dir / 'vocab')
batch_size = train_params.pop_int('batch_size')
iterator = BasicIterator(batch_size=batch_size)
iterator.index_with(vocab)
model = BaselineModel(params=model_params, vocab=vocab)
optimizer = optim.Adam(params=model.parameters())
summary_writer = SummaryWriter(log_dir=save_dir / 'log')
trainer = SupervisedTrainer(
model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=train_labeled_dataset,
validation_dataset=valid_dataset,
summary_writer=summary_writer,
serialization_dir=save_dir,
num_epochs=train_params.pop('num_epochs', 20),
iters_per_epoch=len(train_labeled_dataset) // batch_size,
write_summary_every=100,
validate_every=2000,
patience=2,
clip_grad_max_norm=5,
cuda_device=train_params.pop_int('cuda_device', 0))
trainer.train()
def main():
args = parse_args()
params = Params.from_file(args.params)
save_dir = Path(args.save)
save_dir.mkdir(parents=True)
params.to_file(save_dir / 'params.json')
train_params, model_params = params.pop('train'), params.pop('model')
random_seed = train_params.pop_int('random_seed', 2019)
torch.manual_seed(random_seed)
random.seed(random_seed)
log_filename = save_dir / 'stdout.log'
sys.stdout = TeeLogger(filename=log_filename,
terminal=sys.stdout,
file_friendly_terminal_output=False)
sys.stderr = TeeLogger(filename=log_filename,
terminal=sys.stderr,
file_friendly_terminal_output=False)
tokenizer = WordTokenizer(start_tokens=['<s>'], end_tokens=['</s>'])
token_indexer = SingleIdTokenIndexer(lowercase_tokens=True)
dataset_reader = SnliReader(tokenizer=tokenizer,
token_indexers={'tokens': token_indexer})
train_labeled_dataset_path = train_params.pop('train_labeled_dataset_path')
train_unlabeled_dataset_path = train_params.pop(
'train_unlabeled_dataset_path', None)
train_labeled_dataset = dataset_reader.read(train_labeled_dataset_path)
train_labeled_dataset = filter_dataset_by_length(
dataset=train_labeled_dataset, max_length=30)
if train_unlabeled_dataset_path is not None:
train_unlabeled_dataset = dataset_reader.read(
train_unlabeled_dataset_path)
train_unlabeled_dataset = filter_dataset_by_length(
dataset=train_unlabeled_dataset, max_length=30)
else:
train_unlabeled_dataset = []
valid_dataset = dataset_reader.read(train_params.pop('valid_dataset_path'))
vocab = Vocabulary.from_instances(
instances=train_labeled_dataset + train_unlabeled_dataset,
max_vocab_size=train_params.pop_int('max_vocab_size', None))
vocab.save_to_files(save_dir / 'vocab')
labeled_batch_size = train_params.pop_int('labeled_batch_size')
unlabeled_batch_size = train_params.pop_int('unlabeled_batch_size')
labeled_iterator = BasicIterator(batch_size=labeled_batch_size)
unlabeled_iterator = BasicIterator(batch_size=unlabeled_batch_size)
labeled_iterator.index_with(vocab)
unlabeled_iterator.index_with(vocab)
if not train_unlabeled_dataset:
unlabeled_iterator = None
pretrained_checkpoint_path = train_params.pop('pretrained_checkpoint_path',
None)
model = SNLIModel(params=model_params, vocab=vocab)
if pretrained_checkpoint_path:
model.load_state_dict(
torch.load(pretrained_checkpoint_path, map_location='cpu'))
model.add_finetune_parameters(
con_autoweight=train_params.pop_bool('con_autoweight', False),
con_y_weight=train_params.pop_float('con_y_weight'),
con_z_weight=train_params.pop_float('con_z_weight'),
con_z2_weight=train_params.pop_float('con_z2_weight'))
main_optimizer = optim.Adam(params=model.finetune_main_parameters(
exclude_generator=train_params.pop_bool('exclude_generator')),
lr=train_params.pop_float('lr', 1e-3))
aux_optimizer = optim.Adam(params=model.finetune_aux_parameters(),
lr=train_params.pop_float('aux_lr', 1e-4))
summary_writer = SummaryWriter(log_dir=save_dir / 'log')
kl_anneal_rate = train_params.pop_float('kl_anneal_rate', None)
if kl_anneal_rate is None:
kl_weight_scheduler = None
else:
kl_weight_scheduler = (lambda step: min(1.0, kl_anneal_rate * step))
model.kl_weight = 0.0
gumbel_anneal_rate = train_params.pop_float('gumbel_anneal_rate', None)
if gumbel_anneal_rate is None:
gumbel_temperature_scheduler = None
else:
gumbel_temperature_scheduler = (
lambda step: max(0.1, 1.0 - gumbel_anneal_rate * step))
model.gumbel_temperature = 1.0
iters_per_epoch = train_params.pop_int(
'iters_per_epoch',
len(train_labeled_dataset) // labeled_batch_size)
trainer = FineTuningTrainer(
model=model,
main_optimizer=main_optimizer,
aux_optimizer=aux_optimizer,
labeled_iterator=labeled_iterator,
unlabeled_iterator=unlabeled_iterator,
train_labeled_dataset=train_labeled_dataset,
train_unlabeled_dataset=train_unlabeled_dataset,
validation_dataset=valid_dataset,
summary_writer=summary_writer,
serialization_dir=save_dir,
num_epochs=train_params.pop_int('num_epochs', 50),
iters_per_epoch=iters_per_epoch,
write_summary_every=100,
validate_every=1000,
patience=train_params.pop_int('patience', 5),
clip_grad_max_norm=train_params.pop_float('grad_max_norm', 5.0),
kl_weight_scheduler=kl_weight_scheduler,
gumbel_temperature_scheduler=gumbel_temperature_scheduler,
cuda_device=train_params.pop_int('cuda_device', 0),
)
trainer.train()
