def main(model_dir, epoch, input_csv, batch_size):
try:
cfg = Box.from_yaml(filename=model_dir / 'config.yaml')
except FileNotFoundError:
print('[!] Model directory({}) must contain config.yaml'.format(
model_dir))
exit(1)
csv_magic(input_csv)
device = torch.device('{}:{}'.format(cfg.device.type, cfg.device.ordinal))
random.seed(cfg.random_seed)
np.random.seed(cfg.random_seed)
torch.manual_seed(cfg.random_seed)
torch.cuda.manual_seed_all(cfg.random_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
prediction_dir = model_dir / 'predictions'
if not prediction_dir.exists():
prediction_dir.mkdir()
print('[-] Directory {} created'.format(prediction_dir))
dataset_dir = Path(cfg.dataset_dir)
test_dataset_path = dataset_dir / 'test.pkl'
ckpt_path = model_dir / 'ckpts' / 'epoch-{}.ckpt'.format(epoch)
print('[-] Test dataset: {}'.format(test_dataset_path))
print('[-] Model checkpoint: {}\n'.format(ckpt_path))
print('[*] Loading vocabs and test dataset from {}'.format(dataset_dir))
word_vocab = load_pkl(dataset_dir / 'word.pkl')
char_vocab = load_pkl(dataset_dir / 'char.pkl')
test_dataset = load_pkl(test_dataset_path)
print('[*] Creating test data loader\n')
if batch_size:
cfg.data_loader.batch_size = batch_size
data_loader = create_data_loader(test_dataset,
word_vocab,
char_vocab,
**cfg.data_loader,
shuffle=False)
if cfg.use_elmo:
print('[*] Creating ELMo embedder')
elmo_embedder = Embedder(**cfg.elmo_embedder)
else:
elmo_embedder = None
print('[*] Creating model\n')
cfg.net.n_ctx_embs = cfg.elmo_embedder.n_ctx_embs if cfg.use_elmo else 0
cfg.net.ctx_emb_dim = cfg.elmo_embedder.ctx_emb_dim if cfg.use_elmo else 0
model = Model(device, word_vocab, char_vocab, cfg.net, cfg.optim, False)
model.load_state(ckpt_path)
Ids, predictions = predict(device, data_loader,
cfg.data_loader.max_sent_len, elmo_embedder,
model)
save_predictions(Ids, predictions,
prediction_dir / 'epoch-{}.csv'.format(epoch))
def main(model_dir):
try:
cfg = Box.from_yaml(filename=model_dir / 'config.yaml')
except FileNotFoundError:
print('[!] Model directory({}) must contain config.yaml'.format(
model_dir))
exit(1)
print(
'[-] Model checkpoints and training log will be saved to {}\n'.format(
model_dir))
device = torch.device('{}:{}'.format(cfg.device.type, cfg.device.ordinal))
random.seed(cfg.random_seed)
np.random.seed(cfg.random_seed)
torch.manual_seed(cfg.random_seed)
torch.cuda.manual_seed_all(cfg.random_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
log_path = model_dir / 'log.csv'
ckpt_dir = model_dir / 'ckpts'
if any([p.exists() for p in [log_path, ckpt_dir]]):
print('[!] Directory already contains saved ckpts/log')
exit(1)
ckpt_dir.mkdir()
print('[*] Loading vocabs and datasets from {}'.format(cfg.dataset_dir))
dataset_dir = Path(cfg.dataset_dir)
word_vocab = load_pkl(dataset_dir / 'word.pkl')
char_vocab = load_pkl(dataset_dir / 'char.pkl')
train_dataset = load_pkl(dataset_dir / 'train.pkl')
dev_dataset = load_pkl(dataset_dir / 'dev.pkl')
print('[*] Creating train/dev data loaders\n')
if cfg.data_loader.batch_size % cfg.train.n_gradient_accumulation_steps != 0:
print(
'[!] n_gradient_accumulation_steps({}) is not a divider of '.
format(cfg.train.n_gradient_accumulation_steps),
'batch_size({})'.format(cfg.data_loader.batch_size))
exit(1)
cfg.data_loader.batch_size //= cfg.train.n_gradient_accumulation_steps
train_data_loader = create_data_loader(train_dataset, word_vocab,
char_vocab, **cfg.data_loader)
dev_data_loader = create_data_loader(dev_dataset, word_vocab, char_vocab,
**cfg.data_loader)
print('[*] Creating model\n')
# cfg.net.n_ctx_embs = cfg.elmo_embedder.n_ctx_embs if cfg.use_elmo else 0
# cfg.net.ctx_emb_dim = cfg.elmo_embedder.ctx_emb_dim if cfg.use_elmo else 0
model = Model(device, word_vocab, char_vocab, cfg.net, cfg.optim)
trainer = Trainer(
cfg.data_loader.max_sent_len, device, cfg.train, train_data_loader,
dev_data_loader, model, [
NLLloss(
device, 'logits', 'label', ignore_index=word_vocab.sp.pad.idx)
], [Accuracy(device, 'label')], log_path, ckpt_dir)
trainer.start()
def main(model_dir):
try:
cfg = Box.from_yaml(filename=model_dir / 'config.yaml')
except FileNotFoundError:
print('[!] Model directory({}) must contain config.yaml'.format(model_dir))
exit(1)
print(
'[-] Model checkpoints and training log will be saved to {}\n'
.format(model_dir))
#device = torch.device('{}:{}'.format(cfg.device.type, cfg.device.ordinal))
device = torch.device("cuda:0")
random.seed(cfg.random_seed)
np.random.seed(cfg.random_seed)
torch.manual_seed(cfg.random_seed)
torch.cuda.manual_seed_all(cfg.random_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
log_path = model_dir / 'log.csv'
ckpt_dir = model_dir / 'ckpts'
if any([p.exists() for p in [log_path, ckpt_dir]]):
print('[!] Directory already contains saved ckpts/log')
exit(1)
ckpt_dir.mkdir()
print('[*] Loading datasets from {}'.format(cfg.dataset_dir))
dataset_dir = Path(cfg.dataset_dir)
train_dataset = load_pkl(dataset_dir / 'train.pkl')
dev_dataset = load_pkl(dataset_dir / 'dev.pkl')
print('[*] Creating train/dev data loaders\n')
if cfg.data_loader.batch_size % cfg.train.n_gradient_accumulation_steps != 0:
print(
'[!] n_gradient_accumulation_steps({}) is not a divider of '
.format(cfg.train.n_gradient_accumulation_steps),
'batch_size({})'.format(cfg.data_loader.batch_size))
exit(1)
cfg.data_loader.batch_size //= cfg.train.n_gradient_accumulation_steps
train_data_loader = create_data_loader(train_dataset, **cfg.data_loader)
dev_data_loader = create_data_loader(dev_dataset, **cfg.data_loader)
num_train_optimization_steps = \
int(len(train_data_loader) / cfg.train.n_gradient_accumulation_steps) * cfg.train.n_epochs
print('[*] Creating model\n')
model1 = Model(device, cfg.random_seed, cfg.net, cfg.optim, num_train_optimization_steps)
torch.manual_seed(8981)
torch.cuda.manual_seed_all(8981)
model2 = Model(device, 8981, cfg.net, cfg.optim, num_train_optimization_steps)
trainer = Trainer(
cfg.data_loader.max_sent_len, device, cfg.train,
train_data_loader, dev_data_loader, [model1, model2],
[CrossEntropyLoss(device, 'logits', 'label')], [Accuracy(device, 'label')],
log_path, ckpt_dir)
trainer.start()
def main(model_dir, batch_size, pred_path):
try:
cfg = Box.from_yaml(filename=model_dir / 'config.yaml')
except FileNotFoundError:
print('[!] Model directory({}) must contain config.yaml'.format(
model_dir))
exit(1)
device = torch.device('{}:{}'.format(cfg.device.type, cfg.device.ordinal))
random.seed(cfg.random_seed)
np.random.seed(cfg.random_seed)
torch.manual_seed(cfg.random_seed)
torch.cuda.manual_seed_all(cfg.random_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
prediction_dir = model_dir / 'predictions'
if not prediction_dir.exists():
prediction_dir.mkdir()
print('[-] Directory {} created'.format(prediction_dir))
dataset_dir = Path(cfg.dataset_dir)
test_dataset_path = dataset_dir / 'test.pkl'
print('[-] Test dataset: {}'.format(test_dataset_path))
print('[*] Loading vocabs and test dataset from {}'.format(dataset_dir))
test_dataset = load_pkl(test_dataset_path)
print('[*] Creating test data loader\n')
if batch_size:
cfg.data_loader.batch_size = batch_size
test_data_loader = create_data_loader(test_dataset, **cfg.data_loader)
print('[*] Creating model\n')
model_list = []
ckpt = cfg.models
for model in ckpt:
path = ckpt[model]['path']
seed = ckpt[model]['seed']
_model = Model(device, seed, cfg.net, cfg.optim, None)
_model.load_state(path, True)
model_list.append(_model)
Ids, predictions = predict(device, test_data_loader,
cfg.data_loader.max_sent_len, model_list)
save_predictions(Ids, predictions, pred_path)
def main(model_dir):
try:
cfg = Box.from_yaml(filename=model_dir / 'config.yaml')
except FileNotFoundError:
print('[!] Model directory({}) must contain config.yaml'.format(model_dir))
exit(1)
print(
'[-] Model checkpoints and training log will be saved to {}\n'
.format(model_dir))
device = torch.device('{}:{}'.format(cfg.device.type, cfg.device.ordinal))
random.seed(cfg.random_seed)
np.random.seed(cfg.random_seed)
torch.manual_seed(cfg.random_seed)
torch.cuda.manual_seed_all(cfg.random_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
log_path = model_dir / 'log.csv'
ckpt_dir = model_dir / 'ckpts'
if any([p.exists() for p in [log_path, ckpt_dir]]):
print('[!] Directory already contains saved ckpts/log')
exit(1)
ckpt_dir.mkdir()
print('[*] Loading vocabs and datasets from {}'.format(cfg.dataset_dir))
dataset_dir = Path(cfg.dataset_dir)
word_vocab = load_pkl(dataset_dir / 'word.pkl')
char_vocab = load_pkl(dataset_dir / 'char.pkl')
train_dataset = load_pkl(dataset_dir / 'train.pkl')
dev_dataset = load_pkl(dataset_dir / 'dev.pkl')
print('[*] Creating train/dev data loaders\n')
if cfg.data_loader.batch_size % cfg.train.n_gradient_accumulation_steps != 0:
print(
'[!] n_gradient_accumulation_steps({}) is not a divider of '
.format(cfg.train.n_gradient_accumulation_steps),
'batch_size({})'.format(cfg.data_loader.batch_size))
exit(1)
cfg.data_loader.batch_size //= cfg.train.n_gradient_accumulation_steps
train_data_loader = create_data_loader(
train_dataset, word_vocab, char_vocab, **cfg.data_loader)
dev_data_loader = create_data_loader(
dev_dataset, word_vocab, char_vocab, **cfg.data_loader)
if cfg.use_elmo:
print('[*] Creating ELMo embedder')
elmo_embedder = Embedder(**cfg.elmo_embedder)
else:
elmo_embedder = None
print('[*] Creating model\n')
cfg.net.n_ctx_embs = cfg.elmo_embedder.n_ctx_embs if cfg.use_elmo else 0
cfg.net.ctx_emb_dim = cfg.elmo_embedder.ctx_emb_dim if cfg.use_elmo else 0
model = Model(device, word_vocab, char_vocab, cfg.net, cfg.optim)
trainer = Trainer(
cfg.data_loader.max_sent_len, elmo_embedder, device, cfg.train,
train_data_loader, dev_data_loader, model,
[CrossEntropyLoss(device, 'logits', 'label')], [Accuracy(device, 'label')],
log_path, ckpt_dir)
train_acc, eval_acc = trainer.start()
plt.plot(range(10), [1 - i for i in train_acc], range(10), [1 - i for i in eval_acc])
plt.legend(['Training perplexity', 'Validation perplexity'], loc='upper left')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.title('Perplexity from training and validation')
plt.show()