longest_sequence=85,
shortest_sequence=55)
print('Building dataloaders..')
eval_dataloader = data.DataLoader(dataset, batch_size=batch_size)
model = RNN(num_features, hidden_dimension, num_classes,
num_layers=2).to(device)
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint['model_state_dict'])
true_positives = []
false_positives = []
temp = np.ones((85, 128))
model.eval()
true_positives = []
false_positives = []
true_negatives = []
false_negatives = []
for batch_idx, val_data in enumerate(eval_dataloader):
gt_piano_roll = val_data['piano_roll'].permute(
0, 2, 1).squeeze(0).detach().numpy()
sequence = val_data['sequence'].permute(1, 0, 2)
seq_len = val_data['sequence_length']
# gt = gt_piano_roll.detach().numpy()
# -----------------------------------------------------------------------------------
h = torch.zeros([1, 128])
c = torch.zeros([1, 128])
# Create an instance of the model with given hyperparameters
rnn = RNN(dataset.vocab_size,
hidden_size=MODEL_HIDDEN_SIZE,
embedding_size=MODEL_EMBEDDING_SIZE,
embeddings_dropout=MODEL_EMBEDDINGS_DROPOUT,
lstm_dropout=MODEL_LSTM_DROPOUT,
num_decode_layers=MODEL_NUM_HIDDEN_LAYERS)
else:
rnn = MultiTaskRNN(dataset.vocab_size,
dataset.tag_vocab_size,
hidden_size=MODEL_HIDDEN_SIZE_MULTI,
embedding_size=MODEL_EMBEDDING_SIZE_MULTI,
embeddings_dropout=MODEL_EMBEDDINGS_DROPOUT_MULTI,
lstm_dropout=MODEL_LSTM_DROPOUT_MULTI,
num_decode_layers=MODEL_NUM_HIDDEN_LAYERS_MULTI)
rnn.eval()
# Load state dict
try:
if USE_MULTI_TASK:
state_dict_path = STATE_DICT_PATH_MULTI
else:
state_dict_path = STATE_DICT_PATH
state_dict, epoch, batch, best_loss = load_state_dict(
device, state_dict_path)
rnn.load_state_dict(state_dict)
print("Successfully loaded model state from {}.".format(state_dict_path))
print("Loaded model at epoch {}, batch {}.".format(epoch, batch))
print("Best recorded loss was {}.".format(best_loss))
def predict(cfg, model_path_c1, model_path_c2, model_path_c3, model_path_c4,
loader, device, save_path):
print(f'Saving predictions @ {save_path}')
model_c1 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c2 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c3 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c4 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c1 = model_c1.to(device)
model_c2 = model_c2.to(device)
model_c3 = model_c3.to(device)
model_c4 = model_c4.to(device)
# load the model
model_c1.load_state_dict(torch.load(model_path_c1))
model_c2.load_state_dict(torch.load(model_path_c2))
model_c3.load_state_dict(torch.load(model_path_c3))
model_c4.load_state_dict(torch.load(model_path_c4))
# just to be sure
model_c1.eval()
model_c2.eval()
model_c3.eval()
model_c4.eval()
predictions = {
'Object': [],
'Sequence': [],
}
for c in range(cfg.output_dim):
predictions[f'{cfg.task}_prob_{c}'] = []
for batch in loader:
inputs = batch['inputs'].to(device)
with torch.set_grad_enabled(False):
# (B, T, D)
outputs_c1, hiddens = model_c1(inputs[:, 0, :, :])
outputs_c2, hiddens = model_c2(inputs[:, 1, :, :])
outputs_c3, hiddens = model_c3(inputs[:, 2, :, :])
outputs_c4, hiddens = model_c4(inputs[:, 3, :, :])
outputs = outputs_c1 + outputs_c2 + outputs_c3 + outputs_c4
_, preds = torch.max(outputs, 1)
softmaxed = torch.nn.functional.softmax(outputs, dim=-1)
for i in range(len(batch['paths'])):
sequence = pathlib.Path(batch['paths'][i]).stem
predictions['Object'].append(batch['containers'][i])
predictions['Sequence'].append(sequence)
for c in range(cfg.output_dim):
predictions[f'{cfg.task}_prob_{c}'].append(softmaxed[i,
c].item())
predictions_dataset = pd.DataFrame.from_dict(predictions).sort_values(
['Object', 'Sequence'])
predictions_dataset.to_csv(save_path, index=False)
# returning the dataset because it will be useful for test-time prediction averaging
return predictions_dataset
print_every = 50
# training interface
step = 0
tracker = {'NLL': []}
start_time = time.time()
for ep in range(epoch):
# learning rate decay
if ep >= 10 and ep % 2 == 0:
learning_rate = learning_rate * 0.5
for param_group in optimizer.param_groups:
param_group['lr'] = learning_rate
for split in splits:
dataloader = dataloaders[split]
model.train() if split == 'train' else model.eval()
totals = {'NLL': 0., 'words': 0}
for itr, (_, dec_inputs, targets, lengths) in enumerate(dataloader):
bsize = dec_inputs.size(0)
dec_inputs = dec_inputs.to(device)
targets = targets.to(device)
lengths = lengths.to(device)
# forward
logp = model(dec_inputs, lengths)
# calculate loss
NLL_loss = NLL(logp, targets, lengths + 1)
loss = NLL_loss / bsize
def train(cfg, datasets, dataloaders, device, save_model_path_c1,
save_model_path_c2, save_model_path_c3, save_model_path_c4):
model_c1 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c2 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c3 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c4 = RNN(cfg.model_type, cfg.input_dim, cfg.hidden_dim, cfg.n_layers,
cfg.drop_p, cfg.output_dim, cfg.bi_dir)
model_c1 = model_c1.to(device)
model_c2 = model_c2.to(device)
model_c3 = model_c3.to(device)
model_c4 = model_c4.to(device)
params = list(model_c1.parameters()) + list(model_c2.parameters()) + list(
model_c3.parameters()) + list(model_c4.parameters())
optimizer = torch.optim.Adam(params, lr=1e-4)
criterion = torch.nn.CrossEntropyLoss()
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
best_metric = 0.0
best_epoch = 0
best_model_wts_c1 = copy.deepcopy(model_c1.state_dict())
best_model_wts_c2 = copy.deepcopy(model_c2.state_dict())
best_model_wts_c3 = copy.deepcopy(model_c3.state_dict())
best_model_wts_c4 = copy.deepcopy(model_c4.state_dict())
for epoch in range(cfg.num_epochs):
for phase in ['train', 'valid']:
if phase == 'train':
model_c1.train() # Set model to training mode
model_c2.train() # Set model to training mode
model_c3.train() # Set model to training mode
model_c4.train() # Set model to training mode
else:
model_c1.eval() # Set model to training mode
model_c2.eval() # Set model to training mode
model_c3.eval() # Set model to training mode
model_c4.eval() # Set model to training mode
running_loss = 0.0
# running_corrects = 0
y_pred = []
y_true = []
# Iterate over data.
for batch in dataloaders[phase]:
# (B, num_cam, T, D)
inputs = batch['inputs'].to(device)
targets = batch['targets'][cfg.task].to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# (B, T, D)
outputs_c1, hiddens = model_c1(inputs[:, 0, :, :])
outputs_c2, hiddens = model_c2(inputs[:, 1, :, :])
outputs_c3, hiddens = model_c3(inputs[:, 2, :, :])
outputs_c4, hiddens = model_c4(inputs[:, 3, :, :])
outputs = outputs_c1 + outputs_c2 + outputs_c3 + outputs_c4
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, targets)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
# running_corrects += torch.sum(preds == targets.data)
y_pred.extend(preds.tolist())
y_true.extend(targets.tolist())
# if phase == 'train':
# scheduler.step()
# epoch_acc = running_corrects.double() / len(datasets[phase])
epoch_loss = running_loss / len(datasets[phase])
f1_ep = f1_score(y_true, y_pred, average='weighted')
precision_ep = precision_score(y_true, y_pred, average='weighted')
recall_ep = recall_score(y_true, y_pred, average='weighted')
accuracy_ep = accuracy_score(y_true, y_pred)
# print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
print(
f'({phase} @ {epoch+1}): L: {epoch_loss:3f}; A: {accuracy_ep:3f}; R: {recall_ep:3f}; '
+ f'P: {precision_ep:3f}; F1: {f1_ep:3f}')
# deep copy the model
if phase == 'valid' and f1_ep > best_metric:
best_metric = f1_ep
best_epoch = epoch
best_model_wts_c1 = copy.deepcopy(model_c1.state_dict())
best_model_wts_c2 = copy.deepcopy(model_c2.state_dict())
best_model_wts_c3 = copy.deepcopy(model_c3.state_dict())
best_model_wts_c4 = copy.deepcopy(model_c4.state_dict())
print(f'Best val Metric {best_metric:3f} @ {best_epoch+1}\n')
# load best model weights
model_c1.load_state_dict(best_model_wts_c1)
model_c2.load_state_dict(best_model_wts_c2)
model_c3.load_state_dict(best_model_wts_c3)
model_c4.load_state_dict(best_model_wts_c4)
torch.save(model_c1.state_dict(), save_model_path_c1)
torch.save(model_c2.state_dict(), save_model_path_c2)
torch.save(model_c3.state_dict(), save_model_path_c3)
torch.save(model_c4.state_dict(), save_model_path_c4)
print(f'models are saved @ {save_model_path_c1}')
return best_metric
