def evaluate(outputs, labels):
def zscore(x):
return (x - x.mean()) / np.std(x)
preds = zscore(outputs) > 1.5
acc = metrics.accuracy_score(labels, preds)
print("Accuracy:", acc)
print("Percent positive:", preds.mean())
roc = metrics.roc_auc_score(labels, outputs)
print("ROC-AUC:", roc)
precision, recall, thresh = metrics.precision_recall_curve(labels, outputs)
prauc = metrics.auc(recall, precision)
print("PR-AUC:", prauc)
f1 = metrics.f1_score(labels, preds)
print("F1 score:", f1)
print("Smoothed preds results:")
smoothed_outputs = post_processing.smooth_outputs(outputs)
evaluate(smoothed_outputs, labels)
print("Naive preds results:")
evaluate(outputs, labels)
utils.notify("DONE")
def fit(self,
data_loader,
n_epochs,
batch_size,
checkpoint_freq=None,
checkpoint_dir=None):
criterion = self.get_criterion()
optimizer = self.get_optim()
batch_timer = 0
batch_samples = 50
loss_progress = []
st = time.time()
checkpoint_time = time.time()
for epoch in range(n_epochs):
print("Starting epoch", epoch + 1)
for i, batch_data in enumerate(data_loader):
# Prepare data and take step.
data, labels = self.prepare_data(batch_data)
optimizer.zero_grad()
with torch.autograd.detect_anomaly():
outputs = super().__call__(*data)
loss = self.compute_loss(criterion, outputs, labels)
loss.backward()
optimizer.step()
# Progress tracking.
if i % 25 == 0:
loss_progress.append(loss.item())
if i % 500 == 499:
print('[%d, %5d] loss: %.3f' %
(epoch + 1, i + 1, np.mean(loss_progress[-10:])))
if checkpoint_dir and time.time(
) - checkpoint_time > checkpoint_freq:
torch.save(model.state_dict(),
f'{checkpoint_dir}/StepPlacement.torch')
batch_timer += 1
if batch_timer == batch_samples:
batch_time = (time.time() - st) / batch_samples
batches_per_epoch = len(data_loader)
expected_time = \
batch_time * batches_per_epoch * n_epochs * 0.8
print("Run time per batch: %s" %
utils.format_time(batch_time))
print("Crude expected runtime: %s" %
utils.format_time(expected_time))
print("TEMPORARY BREAK")
break # Temporary
if epoch == 0:
epoch_time = time.time() - st
remaining_time = epoch_time * (n_epochs - 1)
epoch_time_str = utils.format_time(epoch_time)
remaining_time_str = utils.format_time(remaining_time)
print("Epoch time: %s" % epoch_time_str)
print("Expected remaining running time: %s" %
remaining_time_str)
utils.notify("First epoch done! \nEpoch time: %s" %
epoch_time_str +
"\nRemaining time: %s" % remaining_time_str)
# Done training.
final_loss = np.mean(loss_progress[-10:])
print("Training complete.")
print(f"Final loss: {final_loss:.5f}")
utils.notify(f"Training done! Final loss: {final_loss:.5f}")
# PR-AUC: Per class type.
for step_type in range(5):
step_scores = scores[:, :, step_type]
precision, recall, thresh = metrics.precision_recall_curve(
labels.reshape(-1) == step_type, step_scores.reshape(-1))
prauc = metrics.auc(recall, precision)
print(f"PR-AUC for step type {step_type}: {prauc:.3f}")
# F1 score.
for step_type in range(5):
step_scores = scores[:, :, step_type]
f1 = metrics.f1_score(
labels.reshape(-1) == step_type,
preds.reshape(-1) == step_type)
print(f"F1 score for step type {step_type}: {f1:.3f}")
# Perplexity: Per step.
entropy = np.sum(scores * np.log2(scores + 1e-8), axis=2)
perplexity = np.mean(2**-entropy)
print(f"Average perplexity: {perplexity:.3f}")
# Blank notes
blanks = (preds.sum(axis=1) == 0).mean()
print(f"Percent blank: {blanks:.3f}")
# Triple or more
triples = (preds.sum(axis=1) >= 3).mean()
print(f"Percent triple+: {blanks:.3f}")
utils.notify('DONE')
def train(args):
print("Begin training.")
train_dataset = SMDUtils.get_dataset_from_file(args.train)
test_dataset = SMDUtils.get_dataset_from_file(args.test)
train_loader = datautils.DataLoader(
train_dataset,
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True)
test_loader = datautils.DataLoader(
test_dataset,
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True)
print('N train:', len(train_loader))
print('N test:', len(test_loader))
train_ts = utils.timestamp()
model = StepPlacement.RecurrentStepPlacementModel()
checkpoint_path = f'{args.output_data_dir}/model.cpt'
if os.path.exists(checkpoint_path):
print("Loading weights from" checkpoint_path)
model.load_state_dict(torch.load(checkpoint_path))
model.cuda()
model.fit(train_loader,
args.epochs,
args.batch_size,
args.checkpoint_freq,
args.output_data_dir)
torch.save(model.state_dict(), f'{args.model_dir}/StepPlacement.torch')
outputs, labels = model.predict(test_loader, max_batches = 2000)
s3_bucket = 'sagemaker-us-west-1-284801879240'
sm_env = json.loads(os.environ['SM_TRAINING_ENV'])
s3_path = sm_env['job_name']
def sigmoid(x):
return 1 / (1 + np.exp(-x))
preds = sigmoid(outputs) > 0.1
accuracy = 1 - np.mean(np.abs(preds - labels))
print("Accuracy:", accuracy)
percent_pos = np.mean(preds)
print("Percent positive:", percent_pos)
fpr, tpr, _ = roc_curve(labels, outputs)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr)
roc_buf = io.BytesIO()
plt.savefig(roc_buf, format='png')
roc_buf.seek(0)
utils.upload_image_obj(roc_buf, s3_bucket, f'{s3_path}/roc_auc.png')
print("AUC ROC:", roc_auc)
precision, recall, _ = precision_recall_curve(labels, outputs)
prauc = auc(recall, precision)
print("AUC PR:", prauc)
plt.plot(recall, precision)
pr_buf = io.BytesIO()
plt.savefig(pr_buf, format='png')
pr_buf.seek(0)
utils.upload_image_obj(pr_buf, s3_bucket, f'{s3_path}/pr_auc.png')
f1 = f1_score(labels, preds)
print("F1 score:", f1)
output_metrics = [
'Training done.',
f'Accuracy: {accuracy}',
f'Percent pos: {percent_pos}',
f'ROC AUC: {roc_auc}',
f'PRAUC: {prauc}',
f'F1 score: {f1_score}'
]
output_str = output_metrics.join('\\n')
utils.notify(output_str)
outputs = np.concatenate(list(
map(lambda output: output.reshape((-1, 4, 5)).numpy(), outputs_list)),
axis=0)
print(labels_list[0].shape)
labels = np.concatenate(labels_list, axis=0).reshape((-1, 4))
preds = np.argmax(outputs, axis=2)
print(labels.shape)
print(preds.shape)
accuracy = {}
overall_acc = np.mean(labels == preds)
for i in range(5):
acc = np.mean(labels[labels == i] == preds[labels == i])
accuracy[i] = acc if not np.isnan(acc) else None
accuracy['overall'] = overall_acc
print("Accuracy:", accuracy)
utils.notify(accuracy)
model_save = f"{output_dir}/models/{model_name}_{train_ts}.sd"
torch.save(model.state_dict(), model_save)
total_time = time.time() - st_time
utils.notify(f"Done. Total time: {utils.format_time(total_time)}")