def load_data(filename, file_dir, vocab, depend_rels, n_classes):
if os.path.isfile(filename):
dataset = torch.load(filename)
else:
dataset = SICKDataset(file_dir, vocab, depend_rels, n_classes)
torch.save(dataset, filename)
return dataset
def get_data(self, flag):
data_dir = os.path.join(self.cfg.input_dir(), flag)
# load SICK dataset splits
data_file = os.path.join(data_dir, 'sick_' + flag + '.pth')
# dataset = torch.load(data_file)
#else:
dataset = SICKDataset(data_dir, self.vocab, self.cfg.num_classes())
torch.save(dataset, data_file)
return dataset
def main():
global args
args = parse_args()
# global logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"[%(asctime)s] %(levelname)s:%(name)s:%(message)s")
# file logger
fh = logging.FileHandler(os.path.join(args.save, args.expname) + '.log',
mode='w')
fh.setLevel(logging.INFO)
fh.setFormatter(formatter)
logger.addHandler(fh)
# console logger
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
# argument validation
args.cuda = args.cuda and torch.cuda.is_available()
if args.sparse and args.wd != 0:
logger.error('Sparsity and weight decay are incompatible, pick one!')
exit()
logger.debug(args)
torch.manual_seed(args.seed)
random.seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
if not os.path.exists(args.save):
os.makedirs(args.save)
train_dir = os.path.join(args.data, 'train/')
dev_dir = os.path.join(args.data, 'dev/')
test_dir = os.path.join(args.data, 'test/')
# write unique words from all token files
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
if not os.path.isfile(sick_vocab_file):
token_files_a = [
os.path.join(split, 'a.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files_b = [
os.path.join(split, 'b.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files = token_files_a + token_files_b
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
build_vocab(token_files, sick_vocab_file)
# get vocab object from vocab file previously written
vocab = Vocab(filename=sick_vocab_file,
data=[
Constants.PAD_WORD, Constants.UNK_WORD,
Constants.BOS_WORD, Constants.EOS_WORD
])
logger.debug('==> SICK vocabulary size : %d ' % vocab.size())
# load SICK dataset splits
train_file = os.path.join(args.data, 'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
torch.save(train_dataset, train_file)
logger.debug('==> Size of train data : %d ' % len(train_dataset))
dev_file = os.path.join(args.data, 'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
logger.debug('==> Size of dev data : %d ' % len(dev_dataset))
test_file = os.path.join(args.data, 'sick_test.pth')
if os.path.isfile(test_file):
test_dataset = torch.load(test_file)
else:
test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
torch.save(test_dataset, test_file)
logger.debug('==> Size of test data : %d ' % len(test_dataset))
# initialize model, criterion/loss_function, optimizer
model = SimilarityTreeLSTM(args.cuda, vocab.size(), args.input_dim,
args.mem_dim, args.hidden_dim, args.num_classes,
args.sparse)
criterion = nn.KLDivLoss()
if args.cuda:
model.cuda(), criterion.cuda()
if args.optim == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adagrad':
optimizer = optim.Adagrad(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
metrics = Metrics(args.num_classes)
# for words common to dataset vocab and GLOVE, use GLOVE vectors
# for other words in dataset vocab, use random normal vectors
emb_file = os.path.join(args.data, 'sick_embed.pth')
if os.path.isfile(emb_file):
emb = torch.load(emb_file)
else:
# load glove embeddings and vocab
glove_vocab, glove_emb = load_word_vectors(
os.path.join(args.glove, 'glove.840B.300d'))
logger.debug('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
emb = torch.Tensor(vocab.size(),
glove_emb.size(1)).normal_(-0.05, 0.05)
# zero out the embeddings for padding and other special words if they are absent in vocab
for idx, item in enumerate([
Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD,
Constants.EOS_WORD
]):
emb[idx].zero_()
for word in vocab.labelToIdx.keys():
if glove_vocab.getIndex(word):
emb[vocab.getIndex(word)] = glove_emb[glove_vocab.getIndex(
word)]
torch.save(emb, emb_file)
# plug these into embedding matrix inside model
if args.cuda:
emb = emb.cuda()
model.childsumtreelstm.emb.state_dict()['weight'].copy_(emb)
# create trainer object for training and testing
trainer = Trainer(args, model, criterion, optimizer)
best = -float('inf')
for epoch in range(args.epochs):
train_loss = trainer.train(train_dataset)
train_loss, train_pred = trainer.test(train_dataset)
dev_loss, dev_pred = trainer.test(dev_dataset)
test_loss, test_pred = trainer.test(test_dataset)
train_pearson = metrics.pearson(train_pred, train_dataset.labels)
train_mse = metrics.mse(train_pred, train_dataset.labels)
logger.info(
'==> Epoch {}, Train \tLoss: {}\tPearson: {}\tMSE: {}'.format(
epoch, train_loss, train_pearson, train_mse))
dev_pearson = metrics.pearson(dev_pred, dev_dataset.labels)
dev_mse = metrics.mse(dev_pred, dev_dataset.labels)
logger.info(
'==> Epoch {}, Dev \tLoss: {}\tPearson: {}\tMSE: {}'.format(
epoch, dev_loss, dev_pearson, dev_mse))
test_pearson = metrics.pearson(test_pred, test_dataset.labels)
test_mse = metrics.mse(test_pred, test_dataset.labels)
logger.info(
'==> Epoch {}, Test \tLoss: {}\tPearson: {}\tMSE: {}'.format(
epoch, test_loss, test_pearson, test_mse))
if best < test_pearson:
best = test_pearson
checkpoint = {
'model': trainer.model.state_dict(),
'optim': trainer.optimizer,
'pearson': test_pearson,
'mse': test_mse,
'args': args,
'epoch': epoch
}
logger.debug(
'==> New optimum found, checkpointing everything now...')
torch.save(checkpoint,
'%s.pt' % os.path.join(args.save, args.expname))
def main():
global args
args = parse_args()
mkl.set_num_threads(1)
args.cuda = args.cuda and torch.cuda.is_available()
if args.sparse and args.wd != 0:
print('Sparsity and weight decay are incompatible, pick one!')
exit()
print(args)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
train_dir = os.path.join(args.data, 'train/')
dev_dir = os.path.join(args.data, 'dev/')
test_dir = os.path.join(args.data, 'test/')
# write unique words from all token files
token_files_a = [
os.path.join(split, 'a.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files_b = [
os.path.join(split, 'b.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files = token_files_a + token_files_b
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
build_vocab(token_files, sick_vocab_file)
# get vocab object from vocab file previously written
vocab = Vocab(filename=sick_vocab_file,
data=[
Constants.PAD_WORD, Constants.UNK_WORD,
Constants.BOS_WORD, Constants.EOS_WORD
])
print('==> SICK vocabulary size : %d ' % vocab.size())
# load SICK dataset splits
train_file = os.path.join(args.data, 'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
torch.save(train_dataset, train_file)
print('==> Size of train data : %d ' % len(train_dataset))
dev_file = os.path.join(args.data, 'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
print('==> Size of dev data : %d ' % len(dev_dataset))
test_file = os.path.join(args.data, 'sick_test.pth')
if os.path.isfile(test_file):
test_dataset = torch.load(test_file)
else:
test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
torch.save(test_dataset, test_file)
print('==> Size of test data : %d ' % len(test_dataset))
# initialize model, criterion/loss_function, optimizer
model = SimilarityTreeLSTM(args.encoder_type, args.cuda, vocab.size(),
args.input_dim, args.mem_dim, args.hidden_dim,
args.num_classes, args.sparse, args)
criterion = nn.KLDivLoss()
if args.cuda:
model.cuda(), criterion.cuda()
trainable_parameters = [
param for param in model.parameters() if param.requires_grad
]
if args.optim == 'adam':
optimizer = optim.Adam(trainable_parameters,
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adagrad':
optimizer = optim.Adagrad(trainable_parameters,
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'sgd':
optimizer = optim.SGD(trainable_parameters,
lr=args.lr,
weight_decay=args.wd)
metrics = Metrics(args.num_classes)
# for words common to dataset vocab and GLOVE, use GLOVE vectors
# for other words in dataset vocab, use random normal vectors
emb_file = os.path.join(args.data, 'sick_embed.pth')
if os.path.isfile(emb_file):
emb = torch.load(emb_file)
else:
# load glove embeddings and vocab
glove_vocab, glove_emb = load_word_vectors(
os.path.join(args.glove, 'glove.840B.300d'))
print('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
emb = torch.Tensor(vocab.size(),
glove_emb.size(1)).normal_(-0.05, 0.05)
# zero out the embeddings for padding and other special words if they are absent in vocab
for idx, item in enumerate([
Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD,
Constants.EOS_WORD
]):
# TODO '<s>', '</s>' these tokens present in glove w2v but probably with different meaning.
# though they are not currently used
emb[idx].zero_()
for word in vocab.labelToIdx.keys():
if word in glove_vocab.labelToIdx.keys():
emb[vocab.getIndex(word)] = glove_emb[glove_vocab.getIndex(
word)]
torch.save(emb, emb_file)
# plug these into embedding matrix inside model
if args.cuda:
emb = emb.cuda()
model.encoder.emb.state_dict()['weight'].copy_(emb)
# create trainer object for training and testing
trainer = Trainer(args, model, criterion, optimizer)
metric_functions = [metrics.pearson, metrics.mse]
for epoch in range(args.epochs):
train_loss = trainer.train(train_dataset)
train_loss, train_pred = trainer.test(train_dataset)
dev_loss, dev_pred = trainer.test(dev_dataset)
test_loss, test_pred = trainer.test(test_dataset)
pearson_stats, mse_stats = get_median_and_confidence_interval(
train_pred, train_dataset.labels, metric_functions)
print_results("Train", train_loss, pearson_stats, mse_stats)
pearson_stats, mse_stats = get_median_and_confidence_interval(
dev_pred, dev_dataset.labels, metric_functions)
print_results("Dev", dev_loss, pearson_stats, mse_stats)
pearson_stats, mse_stats = get_median_and_confidence_interval(
test_pred, test_dataset.labels, metric_functions)
print_results("Test", test_loss, pearson_stats, mse_stats)
def main():
global args
args = parse_args()
args.input_dim, args.mem_dim = 300, 150
args.hidden_dim, args.num_classes = 20, 2
args.cuda = args.cuda and torch.cuda.is_available()
if args.sparse and args.wd != 0:
print('Sparsity and weight decay are incompatible, pick one!')
exit()
print(args)
torch.manual_seed(args.seed)
random.seed(args.seed)
numpy.random.seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
if not os.path.exists(args.save):
os.makedirs(args.save)
train_dir = os.path.join(args.data, 'train/')
dev_dir = os.path.join(args.data, 'dev/')
test_dir = os.path.join(args.data, 'test/')
# write unique words from all token files
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
if not os.path.isfile(sick_vocab_file):
token_files_a = [
os.path.join(split, 'toks.a')
for split in [train_dir, dev_dir, test_dir]
]
token_files_b = [
os.path.join(split, 'toks.b')
for split in [train_dir, dev_dir, test_dir]
]
token_files = token_files_a + token_files_b
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
build_vocab(token_files, sick_vocab_file)
# get vocab object from vocab file previously written
vocab = Vocab(filename=sick_vocab_file,
data=[
Constants.PAD_WORD, Constants.UNK_WORD,
Constants.BOS_WORD, Constants.EOS_WORD
])
print('==> SICK vocabulary size : %d ' % vocab.size())
# load SICK dataset splits
train_file = os.path.join(args.data, 'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
torch.save(train_dataset, train_file)
print('==> Size of train data : %d ' % len(train_dataset))
dev_file = os.path.join(args.data, 'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
print('==> Size of dev data : %d ' % len(dev_dataset))
test_file = os.path.join(args.data, 'sick_test.pth')
if os.path.isfile(test_file):
test_dataset = torch.load(test_file)
else:
test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
torch.save(test_dataset, test_file)
print('==> Size of test data : %d ' % len(test_dataset))
# initialize model, criterion/loss_function, optimizer
model = SimilarityTreeLSTM(args.cuda, vocab.size(), args.input_dim,
args.mem_dim, args.hidden_dim, args.num_classes,
args.sparse)
criterion = nn.KLDivLoss()
if args.cuda:
model.cuda(), criterion.cuda()
if args.optim == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adagrad':
optimizer = optim.Adagrad(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
metrics = Metrics(args.num_classes)
# for words common to dataset vocab and GLOVE, use GLOVE vectors
# for other words in dataset vocab, use random normal vectors
emb_file = os.path.join(args.data, 'sick_embed.pth')
if os.path.isfile(emb_file):
emb = torch.load(emb_file)
else:
# load glove embeddings and vocab
glove_vocab, glove_emb = load_word_vectors(
os.path.join(args.glove, 'glove.840B.300d'))
print('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
emb = torch.Tensor(vocab.size(),
glove_emb.size(1)).normal_(-0.05, 0.05)
# zero out the embeddings for padding and other special words if they are absent in vocab
for idx, item in enumerate([
Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD,
Constants.EOS_WORD
]):
emb[idx].zero_()
for word in vocab.labelToIdx.keys():
word_new = word.decode("utf8")
idx_set = [
glove_vocab.getIndex(token)
for token in word_tokenize(word_new)
]
idx_set = [id for id in idx_set if id is not None]
if len(idx_set) != 0:
idx_set = torch.LongTensor(idx_set)
sum_emb = F.torch.sum(glove_emb.index_select(0, idx_set), 0)
else:
sum_emb = glove_emb[1] * 0
# for token in word_tokenize(word_new):
# idx = glove_vocab.getIndex(token)
# if idx is not None:
# if sum_emb is None:
# sum_emb = glove_emb[idx]
# else:
# sum_emb += glove_emb[idx]
emb[vocab.getIndex(word)] = sum_emb
torch.save(emb, emb_file)
# plug these into embedding matrix inside model
if args.cuda:
emb = emb.cuda()
model.childsumtreelstm.emb.state_dict()['weight'].copy_(emb)
# create trainer object for training and testing
trainer = Trainer(args, model, criterion, optimizer)
best = -float('inf')
for epoch in range(args.epochs):
train_loss = trainer.train(train_dataset)
train_loss, train_pred = trainer.test(train_dataset)
print(train_pred)
dev_loss, dev_pred = trainer.test(dev_dataset)
print(dev_pred)
test_loss, test_pred = trainer.test(test_dataset)
train_pearson = metrics.pearson(train_pred, train_dataset.labels)
train_mse = metrics.accuracy(train_pred, train_dataset.labels)
print('==> Train Loss: {}\tPearson: {}\tL1: {}'.format(
train_loss, train_pearson, train_mse))
dev_pearson = metrics.pearson(dev_pred, dev_dataset.labels)
dev_mse = metrics.accuracy(dev_pred, dev_dataset.labels)
print('==> Dev Loss: {}\tPearson: {}\tL1: {}'.format(
dev_loss, dev_pearson, dev_mse))
test_pearson = metrics.pearson(test_pred, test_dataset.labels)
test_mse = metrics.accuracy(test_pred, test_dataset.labels)
print('==> Test Loss: {}\tPearson: {}\tL1: {}'.format(
test_loss, test_pearson, test_mse))
if best < test_pearson:
best = test_pearson
checkpoint = {
'model': trainer.model.state_dict(),
'optim': trainer.optimizer,
'pearson': test_pearson,
'mse': test_mse,
'args': args,
'epoch': epoch
}
print('==> New optimum found, checkpointing everything now...')
torch.save(
checkpoint,
'%s.pt' % os.path.join(args.save, args.expname + '.pth'))
def main():
global args
args = parse_args(type=10)
args.input_dim = 300
args.hidden_dim = 50
# args.input_dim, args.mem_dim = 300, 150
# args.hidden_dim, args.num_classes = 50, 5
if args.model_name == 'dependency':
args.mem_dim = 150
elif args.model_name == 'constituency':
args.mem_dim = 142
args.num_classes = 5
args.cuda = args.cuda and torch.cuda.is_available()
print(args)
# torch.manual_seed(args.seed)
# if args.cuda:
# torch.cuda.manual_seed(args.seed)
train_dir = os.path.join(args.data,'train/')
dev_dir = os.path.join(args.data,'dev/')
test_dir = os.path.join(args.data,'test/')
# write unique words from all token files
token_files_a = [os.path.join(split,'a.toks') for split in [train_dir,dev_dir,test_dir]]
token_files_b = [os.path.join(split,'b.toks') for split in [train_dir,dev_dir,test_dir]]
token_files = token_files_a+token_files_b
sick_vocab_file = os.path.join(args.data,'vocab-cased.txt')
# build_vocab(token_files, sick_vocab_file)
# get vocab object from vocab file previously written
vocab = Vocab(filename=sick_vocab_file, data=[Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD, Constants.EOS_WORD])
print('==> SICK vocabulary size : %d ' % vocab.size())
# load SICK dataset splits
train_file = os.path.join(args.data,'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
torch.save(train_dataset, train_file)
print('==> Size of train data : %d ' % len(train_dataset))
dev_file = os.path.join(args.data,'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
print('==> Size of dev data : %d ' % len(dev_dataset))
test_file = os.path.join(args.data,'sick_test.pth')
if os.path.isfile(test_file):
test_dataset = torch.load(test_file)
else:
test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
torch.save(test_dataset, test_file)
print('==> Size of test data : %d ' % len(test_dataset))
# initialize model, criterion/loss_function, optimizer
model = SimilarityTreeLSTM(
args.cuda, vocab.size(),
args.input_dim, args.mem_dim,
args.hidden_dim, args.num_classes
)
embedding_model = nn.Embedding(vocab.size(), args.input_dim)
if args.cuda:
embedding_model = embedding_model.cuda()
criterion = nn.KLDivLoss()
if args.cuda:
model.cuda(), criterion.cuda()
if args.optim=='adam':
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)
elif args.optim=='adagrad':
optimizer = optim.Adagrad(model.parameters(), lr=args.lr, weight_decay=args.wd)
metrics = Metrics(args.num_classes)
# for words common to dataset vocab and GLOVE, use GLOVE vectors
# for other words in dataset vocab, use random normal vectors
emb_file = os.path.join(args.data, 'sick_embed.pth')
if os.path.isfile(emb_file):
emb = torch.load(emb_file)
else:
# load glove embeddings and vocab
glove_vocab, glove_emb = load_word_vectors(os.path.join(args.glove,'glove.840B.300d'))
print('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
emb = torch.Tensor(vocab.size(),glove_emb.size(1)).normal_(-0.05,0.05)
# zero out the embeddings for padding and other special words if they are absent in vocab
for idx, item in enumerate([Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD, Constants.EOS_WORD]):
emb[idx].zero_()
for word in vocab.labelToIdx.keys():
if glove_vocab.getIndex(word):
emb[vocab.getIndex(word)] = glove_emb[glove_vocab.getIndex(word)]
torch.save(emb, emb_file)
# plug these into embedding matrix inside model
if args.cuda:
emb = emb.cuda()
embedding_model.state_dict()['weight'].copy_(emb)
# model.childsumtreelstm.emb.state_dict()['weight'].copy_(emb)
# create trainer object for training and testing
trainer = SimilarityTrainer(args, model, embedding_model, criterion, optimizer)
for epoch in range(args.epochs):
train_loss = trainer.train(train_dataset)
train_loss, train_pred = trainer.test(train_dataset)
dev_loss, dev_pred = trainer.test(dev_dataset)
test_loss, test_pred = trainer.test(test_dataset)
print('==> Train loss : %f \t' % train_loss, end="")
print('Train Pearson : %f \t' % metrics.pearson(train_pred,train_dataset.labels), end="")
print('Train MSE : %f \t' % metrics.mse(train_pred,train_dataset.labels), end="\n")
print('==> Dev loss : %f \t' % dev_loss, end="")
print('Dev Pearson : %f \t' % metrics.pearson(dev_pred,dev_dataset.labels), end="")
print('Dev MSE : %f \t' % metrics.mse(dev_pred,dev_dataset.labels), end="\n")
print('==> Test loss : %f \t' % test_loss, end="")
print('Test Pearson : %f \t' % metrics.pearson(test_pred,test_dataset.labels), end="")
print('Test MSE : %f \t' % metrics.mse(test_pred,test_dataset.labels), end="\n")
data=[Constants.PAD_WORD, Constants.UNK_WORD,
Constants.BOS_WORD, Constants.EOS_WORD])
logger.debug('==> SICK vocabulary size : %d ' % vocab.size())
#ADDED
#get relation vocab object from relation
rel_vocab = Vocab(filename=sick_rel_file)
logger.debug('==> RELATION vocabulary size : %d ' % rel_vocab.size())
# load SICK dataset splits
train_file = os.path.join(args.data, 'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, rel_vocab, args.num_classes)
torch.save(train_dataset, train_file)
logger.debug('==> Size of train data : %d ' % len(train_dataset))
dev_file = os.path.join(args.data, 'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, rel_vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
logger.debug('==> Size of dev data : %d ' % len(dev_dataset))
test_file = os.path.join(args.data, 'sick_test.pth')
def main():
args = parse_args()
print(args)
args.cuda = args.cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
random.seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
if not os.path.exists(args.save):
os.makedirs(args.save)
train_dir = os.path.join(args.data, 'train/')
dev_dir = os.path.join(args.data, 'dev/')
test_dir = os.path.join(args.data, 'test/')
# write unique words from all token files
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
if not os.path.isfile(sick_vocab_file):
token_files_a = [
os.path.join(split, 'a.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files_b = [
os.path.join(split, 'b.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files = token_files_a + token_files_b
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
build_vocab(token_files, sick_vocab_file)
# get vocab object from vocab file previously written
vocab = Vocab(filename=sick_vocab_file,
data=[
Constants.PAD_WORD, Constants.UNK_WORD,
Constants.BOS_WORD, Constants.EOS_WORD
])
# load SICK dataset splits
train_file = os.path.join(args.data, 'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
torch.save(train_dataset, train_file)
dev_file = os.path.join(args.data, 'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
test_file = os.path.join(args.data, 'sick_test.pth')
if os.path.isfile(test_file):
test_dataset = torch.load(test_file)
else:
test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
torch.save(test_dataset, test_file)
# initialize model, criterion/loss_function, optimizer
model = SimilarityTreeLSTM(args.cuda,
vocab.size(),
args.input_dim,
args.mem_dim,
args.hidden_dim1,
args.hidden_dim2,
args.hidden_dim3,
args.num_classes,
args.sparse,
args.att_hops,
args.att_units,
args.maxlen,
args.dropout1,
args.dropout2,
args.dropout3,
freeze_emb=True)
criterion = nn.KLDivLoss()
if args.cuda:
model.cuda(), criterion.cuda()
if args.optim == 'adam':
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adagrad':
optimizer = optim.Adagrad(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'sgd':
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adadelta':
optimizer = optim.Adadelta(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'asgd':
optimizer = optim.ASGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
weight_decay=args.wd)
metrics = Metrics(args.num_classes)
# for words common to dataset vocab and GLOVE, use GLOVE vectors
# for other words in dataset vocab, use random normal vectors
emb_file = os.path.join(args.data, 'sick_embed.pth')
if os.path.isfile(emb_file):
emb = torch.load(emb_file)
else:
# load glove embeddings and vocab
glove_vocab, glove_emb = load_word_vectors(
os.path.join(args.glove, 'glove.840B.300d'))
emb = torch.Tensor(vocab.size(),
glove_emb.size(1)).normal_(-0.05, 0.05)
# zero out the embeddings for padding and other special words if they are absent in vocab
for idx, item in enumerate([
Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD,
Constants.EOS_WORD
]):
emb[idx].zero_()
for word in vocab.labelToIdx.keys():
if glove_vocab.getIndex(word):
emb[vocab.getIndex(word)] = glove_emb[glove_vocab.getIndex(
word)]
torch.save(emb, emb_file)
# plug these into embedding matrix inside model
if args.cuda:
emb = emb.cuda()
model.emb.weight.data.copy_(emb)
# create trainer object for training and testing
trainer = Trainer(args, model, criterion, optimizer)
best = -float('inf')
def adjust_learning_rate(optimizer, epoch):
"""Sets the learning rate to the initial LR decayed by 5 every 3 epochs"""
lr = args.lr * (0.01**(epoch // 15))
for param_group in optimizer.param_groups:
param_group['lr'] = lr
for epoch in range(args.epochs):
adjust_learning_rate(optimizer, epoch)
train_loss = trainer.train(train_dataset)
dev_loss, dev_pred = trainer.test(dev_dataset, mode='test')
test_pearson = metrics.pearson(dev_pred, dev_dataset.labels)
test_mse = metrics.mse(dev_pred, dev_dataset.labels)
if best < test_pearson:
best = test_pearson
checkpoint = {
'model': trainer.model.state_dict(),
'optim': trainer.optimizer,
'pearson': test_pearson,
'mse': test_mse,
'args': args,
'epoch': epoch,
'vocab': vocab
}
torch.save(
checkpoint, '%s.pt' % os.path.join(
args.save, args.expname + '_' + str(test_pearson)))
# Evaluate
trainer.model.load_state_dict(checkpoint['model'])
# trainer.train(train_dataset)
test_loss, test_pred = trainer.test(test_dataset, mode='test')
test_pearson = metrics.pearson(test_pred, test_dataset.labels)
test_mse = metrics.mse(test_pred, test_dataset.labels)
# Final read out
checkpoint = {
'model': trainer.model.state_dict(),
'optim': trainer.optimizer,
'pearson': test_pearson,
'mse': test_mse,
'args': args,
'vocab': vocab
}
torch.save(
checkpoint, '%s.pt' %
os.path.join(args.save, 'end_model_test' + str(test_pearson) + '.pt'))
def prepare_to_train(data=None, glove=None):
args = parse_args()
if data is not None:
args.data = data
if glove is not None:
args.glove = glove
args.input_dim, args.mem_dim = 300, 150
args.hidden_dim, args.num_classes = 50, 5
args.cuda = args.cuda and torch.cuda.is_available()
if args.sparse and args.wd != 0:
print('Sparsity and weight decay are incompatible, pick one!')
exit()
print(args)
torch.manual_seed(args.seed)
random.seed(args.seed)
numpy.random.seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
if not os.path.exists(args.save):
os.makedirs(args.save)
train_dir = os.path.join(args.data, 'train/')
dev_dir = os.path.join(args.data, 'dev/')
test_dir = os.path.join(args.data, 'test/')
# write unique words from all token files
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
if not os.path.isfile(sick_vocab_file):
token_files_a = [
os.path.join(split, 'a.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files_b = [
os.path.join(split, 'b.toks')
for split in [train_dir, dev_dir, test_dir]
]
token_files = token_files_a + token_files_b
sick_vocab_file = os.path.join(args.data, 'sick.vocab')
build_vocab(token_files, sick_vocab_file)
# get vocab object from vocab file previously written
vocab = Vocab(filename=sick_vocab_file,
data=[
Constants.PAD_WORD, Constants.UNK_WORD,
Constants.BOS_WORD, Constants.EOS_WORD
])
print('==> SICK vocabulary size : %d ' % vocab.size())
# load SICK dataset splits
train_file = os.path.join(args.data, 'sick_train.pth')
if os.path.isfile(train_file):
train_dataset = torch.load(train_file)
else:
train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
torch.save(train_dataset, train_file)
print('==> Size of train data : %d ' % len(train_dataset))
dev_file = os.path.join(args.data, 'sick_dev.pth')
if os.path.isfile(dev_file):
dev_dataset = torch.load(dev_file)
else:
dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
torch.save(dev_dataset, dev_file)
print('==> Size of dev data : %d ' % len(dev_dataset))
test_file = os.path.join(args.data, 'sick_test.pth')
if os.path.isfile(test_file):
test_dataset = torch.load(test_file)
else:
test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
torch.save(test_dataset, test_file)
print('==> Size of test data : %d ' % len(test_dataset))
# initialize model, criterion/loss_function, optimizer
model = SimilarityTreeLSTM(args.cuda, vocab.size(), args.input_dim,
args.mem_dim, args.hidden_dim, args.num_classes,
args.sparse)
criterion = nn.KLDivLoss()
if args.cuda:
model.cuda(), criterion.cuda()
if args.optim == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adagrad':
optimizer = optim.Adagrad(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
weight_decay=args.wd)
metrics = Metrics(args.num_classes)
# for words common to dataset vocab and GLOVE, use GLOVE vectors
# for other words in dataset vocab, use random normal vectors
emb_file = os.path.join(args.data, 'sick_embed.pth')
if os.path.isfile(emb_file):
emb = torch.load(emb_file)
else:
# load glove embeddings and vocab
glove_vocab, glove_emb = load_word_vectors(
os.path.join(args.glove, 'glove.840B.300d'))
print('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
emb = torch.Tensor(vocab.size(),
glove_emb.size(1)).normal_(-0.05, 0.05)
# zero out the embeddings for padding and other special words if they are absent in vocab
for idx, item in enumerate([
Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD,
Constants.EOS_WORD
]):
emb[idx].zero_()
for word in vocab.labelToIdx.keys():
if glove_vocab.get_index(word):
emb[vocab.get_index(word)] = glove_emb[glove_vocab.get_index(
word)]
torch.save(emb, emb_file)
# plug these into embedding matrix inside model
if args.cuda:
emb = emb.cuda()
model.childsumtreelstm.emb.state_dict()['weight'].copy_(emb)
# create trainer object for training and testing
#trainer = Trainer(args, model, criterion, optimizer)
best = -float('inf')
return (args, best, train_dataset, dev_dataset, test_dataset, metrics,
optimizer, criterion, model)