def evaluate(outputs, labels): def zscore(x): return (x - x.mean()) / np.std(x) preds = zscore(outputs) > 1.5 acc = metrics.accuracy_score(labels, preds) print("Accuracy:", acc) print("Percent positive:", preds.mean()) roc = metrics.roc_auc_score(labels, outputs) print("ROC-AUC:", roc) precision, recall, thresh = metrics.precision_recall_curve(labels, outputs) prauc = metrics.auc(recall, precision) print("PR-AUC:", prauc) f1 = metrics.f1_score(labels, preds) print("F1 score:", f1) print("Smoothed preds results:") smoothed_outputs = post_processing.smooth_outputs(outputs) evaluate(smoothed_outputs, labels) print("Naive preds results:") evaluate(outputs, labels) utils.notify("DONE")
def fit(self, data_loader, n_epochs, batch_size, checkpoint_freq=None, checkpoint_dir=None): criterion = self.get_criterion() optimizer = self.get_optim() batch_timer = 0 batch_samples = 50 loss_progress = [] st = time.time() checkpoint_time = time.time() for epoch in range(n_epochs): print("Starting epoch", epoch + 1) for i, batch_data in enumerate(data_loader): # Prepare data and take step. data, labels = self.prepare_data(batch_data) optimizer.zero_grad() with torch.autograd.detect_anomaly(): outputs = super().__call__(*data) loss = self.compute_loss(criterion, outputs, labels) loss.backward() optimizer.step() # Progress tracking. if i % 25 == 0: loss_progress.append(loss.item()) if i % 500 == 499: print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, np.mean(loss_progress[-10:]))) if checkpoint_dir and time.time( ) - checkpoint_time > checkpoint_freq: torch.save(model.state_dict(), f'{checkpoint_dir}/StepPlacement.torch') batch_timer += 1 if batch_timer == batch_samples: batch_time = (time.time() - st) / batch_samples batches_per_epoch = len(data_loader) expected_time = \ batch_time * batches_per_epoch * n_epochs * 0.8 print("Run time per batch: %s" % utils.format_time(batch_time)) print("Crude expected runtime: %s" % utils.format_time(expected_time)) print("TEMPORARY BREAK") break # Temporary if epoch == 0: epoch_time = time.time() - st remaining_time = epoch_time * (n_epochs - 1) epoch_time_str = utils.format_time(epoch_time) remaining_time_str = utils.format_time(remaining_time) print("Epoch time: %s" % epoch_time_str) print("Expected remaining running time: %s" % remaining_time_str) utils.notify("First epoch done! \nEpoch time: %s" % epoch_time_str + "\nRemaining time: %s" % remaining_time_str) # Done training. final_loss = np.mean(loss_progress[-10:]) print("Training complete.") print(f"Final loss: {final_loss:.5f}") utils.notify(f"Training done! Final loss: {final_loss:.5f}")
# PR-AUC: Per class type. for step_type in range(5): step_scores = scores[:, :, step_type] precision, recall, thresh = metrics.precision_recall_curve( labels.reshape(-1) == step_type, step_scores.reshape(-1)) prauc = metrics.auc(recall, precision) print(f"PR-AUC for step type {step_type}: {prauc:.3f}") # F1 score. for step_type in range(5): step_scores = scores[:, :, step_type] f1 = metrics.f1_score( labels.reshape(-1) == step_type, preds.reshape(-1) == step_type) print(f"F1 score for step type {step_type}: {f1:.3f}") # Perplexity: Per step. entropy = np.sum(scores * np.log2(scores + 1e-8), axis=2) perplexity = np.mean(2**-entropy) print(f"Average perplexity: {perplexity:.3f}") # Blank notes blanks = (preds.sum(axis=1) == 0).mean() print(f"Percent blank: {blanks:.3f}") # Triple or more triples = (preds.sum(axis=1) >= 3).mean() print(f"Percent triple+: {blanks:.3f}") utils.notify('DONE')
def train(args): print("Begin training.") train_dataset = SMDUtils.get_dataset_from_file(args.train) test_dataset = SMDUtils.get_dataset_from_file(args.test) train_loader = datautils.DataLoader( train_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=True, pin_memory=True) test_loader = datautils.DataLoader( test_dataset, num_workers=args.num_workers, batch_size=args.batch_size, shuffle=True, pin_memory=True) print('N train:', len(train_loader)) print('N test:', len(test_loader)) train_ts = utils.timestamp() model = StepPlacement.RecurrentStepPlacementModel() checkpoint_path = f'{args.output_data_dir}/model.cpt' if os.path.exists(checkpoint_path): print("Loading weights from" checkpoint_path) model.load_state_dict(torch.load(checkpoint_path)) model.cuda() model.fit(train_loader, args.epochs, args.batch_size, args.checkpoint_freq, args.output_data_dir) torch.save(model.state_dict(), f'{args.model_dir}/StepPlacement.torch') outputs, labels = model.predict(test_loader, max_batches = 2000) s3_bucket = 'sagemaker-us-west-1-284801879240' sm_env = json.loads(os.environ['SM_TRAINING_ENV']) s3_path = sm_env['job_name'] def sigmoid(x): return 1 / (1 + np.exp(-x)) preds = sigmoid(outputs) > 0.1 accuracy = 1 - np.mean(np.abs(preds - labels)) print("Accuracy:", accuracy) percent_pos = np.mean(preds) print("Percent positive:", percent_pos) fpr, tpr, _ = roc_curve(labels, outputs) roc_auc = auc(fpr, tpr) plt.plot(fpr, tpr) roc_buf = io.BytesIO() plt.savefig(roc_buf, format='png') roc_buf.seek(0) utils.upload_image_obj(roc_buf, s3_bucket, f'{s3_path}/roc_auc.png') print("AUC ROC:", roc_auc) precision, recall, _ = precision_recall_curve(labels, outputs) prauc = auc(recall, precision) print("AUC PR:", prauc) plt.plot(recall, precision) pr_buf = io.BytesIO() plt.savefig(pr_buf, format='png') pr_buf.seek(0) utils.upload_image_obj(pr_buf, s3_bucket, f'{s3_path}/pr_auc.png') f1 = f1_score(labels, preds) print("F1 score:", f1) output_metrics = [ 'Training done.', f'Accuracy: {accuracy}', f'Percent pos: {percent_pos}', f'ROC AUC: {roc_auc}', f'PRAUC: {prauc}', f'F1 score: {f1_score}' ] output_str = output_metrics.join('\\n') utils.notify(output_str)
outputs = np.concatenate(list( map(lambda output: output.reshape((-1, 4, 5)).numpy(), outputs_list)), axis=0) print(labels_list[0].shape) labels = np.concatenate(labels_list, axis=0).reshape((-1, 4)) preds = np.argmax(outputs, axis=2) print(labels.shape) print(preds.shape) accuracy = {} overall_acc = np.mean(labels == preds) for i in range(5): acc = np.mean(labels[labels == i] == preds[labels == i]) accuracy[i] = acc if not np.isnan(acc) else None accuracy['overall'] = overall_acc print("Accuracy:", accuracy) utils.notify(accuracy) model_save = f"{output_dir}/models/{model_name}_{train_ts}.sd" torch.save(model.state_dict(), model_save) total_time = time.time() - st_time utils.notify(f"Done. Total time: {utils.format_time(total_time)}")