def evaluate(split, model_path, diagnosis, use_gpu): train_loader, valid_loader, test_loader = load_data(diagnosis, use_gpu) model = MRNet() state_dict = torch.load(model_path, map_location=(None if use_gpu else 'cpu')) model.load_state_dict(state_dict) if use_gpu: model = model.cuda() if split == 'train': loader = train_loader elif split == 'valid': loader = valid_loader elif split == 'test': loader = test_loader else: raise ValueError("split must be 'train', 'valid', or 'test'") loss, auc, preds, labels = run_model(model, loader) print(f'{split} loss: {loss:0.4f}') print(f'{split} AUC: {auc:0.4f}') return preds, labels
def train(rundir, diagnosis, epochs, learning_rate, use_gpu): train_loader, valid_loader, test_loader = load_data(diagnosis, use_gpu) model = MRNet() if use_gpu: model = model.cuda() optimizer = torch.optim.Adam(model.parameters(), learning_rate, weight_decay=.01) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=.3, threshold=1e-4) best_val_loss = float('inf') start_time = datetime.now() for epoch in range(epochs): change = datetime.now() - start_time print('starting epoch {}. time passed: {}'.format(epoch+1, str(change))) train_loss, train_auc, _, _ = run_model(model, train_loader, train=True, optimizer=optimizer) print(f'train loss: {train_loss:0.4f}') print(f'train AUC: {train_auc:0.4f}') val_loss, val_auc, _, _ = run_model(model, valid_loader) print(f'valid loss: {val_loss:0.4f}') print(f'valid AUC: {val_auc:0.4f}') scheduler.step(val_loss) if val_loss < best_val_loss: best_val_loss = val_loss file_name = f'val{val_loss:0.4f}_train{train_loss:0.4f}_epoch{epoch+1}' save_path = Path(rundir) / file_name torch.save(model.state_dict(), save_path)
def get_model(): model = MRNet() state_dict = torch.load( "/mnt/g/Grad Projects/MRNet-interface/model weights/f1score_val0.7155_train0.8096_epoch14", map_location=(torch.device('cpu'))) model.load_state_dict(state_dict) return model
def main(data_dir, models_dir, choose_16): device = 'cuda' if torch.cuda.is_available() else 'cpu' planes = ['axial', 'coronal', 'sagittal'] conditions = ['abnormal', 'acl', 'meniscus'] models = [] print(f'Loading best CNN models from {models_dir}...') for condition in conditions: models_per_condition = [] for plane in planes: checkpoint_pattern = glob(f'{models_dir}/*{plane}*{condition}*.pt') checkpoint_path = sorted(checkpoint_pattern)[-1] checkpoint = torch.load(checkpoint_path, map_location=device) model = MRNet().to(device) model.load_state_dict(checkpoint['state_dict']) models_per_condition.append(model) models.append(models_per_condition) print(f'Creating data loaders...') axial_loader = make_data_loader(data_dir, 'train', 'axial', choose_16) coronal_loader = make_data_loader(data_dir, 'train', 'coronal', choose_16) sagittal_loader = make_data_loader(data_dir, 'train', 'sagittal', choose_16) print(f'Collecting predictions on train dataset from the models...') ys = [] Xs = [[], [], []] # Abnormal, ACL, Meniscus with tqdm(total=len(axial_loader)) as pbar: for (axial_inputs, labels), (coronal_inputs, _), (sagittal_inputs, _) in \ zip(axial_loader, coronal_loader, sagittal_loader): axial_inputs, coronal_inputs, sagittal_inputs = \ axial_inputs.to(device), coronal_inputs.to(device), sagittal_inputs.to(device) ys.append(labels[0].cpu().tolist()) for i, model in enumerate(models): axial_pred = model[0](axial_inputs).detach().cpu().item() coronal_pred = model[1](coronal_inputs).detach().cpu().item() sagittal_pred = model[2](sagittal_inputs).detach().cpu().item() X = [axial_pred, coronal_pred, sagittal_pred] Xs[i].append(X) pbar.update(1) ys = np.asarray(ys).transpose() Xs = np.asarray(Xs) print(f'Training logistic regression models for each condition...') clfs = [] for X, y in zip(Xs, ys): clf = LogisticRegressionCV(cv=5, random_state=0).fit(X, y) clfs.append(clf) for i, clf in enumerate(clfs): print( f'Cross validation score for {conditions[i]}: {clf.score(X, y):.3f}' ) clf_path = f'{models_dir}/lr_{conditions[i]}.pkl' joblib.dump(clf, clf_path) print(f'Logistic regression models saved to {models_dir}')
def main(data_dir, plane, epochs, lr, weight_decay, device=None): diagnoses = ['abnormal', 'acl', 'meniscus'] exp = f'{datetime.now():%Y-%m-%d_%H-%M}' out_dir, losses_path = create_output_dir(exp, plane) if device is None: device = 'cuda' if torch.cuda.is_available() else 'cpu' print('Creating data loaders...') train_loader = make_data_loader(data_dir, 'train', plane, device, shuffle=True) valid_loader = make_data_loader(data_dir, 'valid', plane, device) print(f'Creating models...') # Create a model for each diagnosis models = [MRNet().to(device), MRNet().to(device), MRNet().to(device)] # Calculate loss weights based on the prevalences in train set pos_weights = calculate_weights(data_dir, 'train', device) criterions = [nn.BCEWithLogitsLoss(pos_weight=weight) \ for weight in pos_weights] optimizers = [make_adam_optimizer(model, lr, weight_decay) \ for model in models] lr_schedulers = [make_lr_scheduler(optimizer) for optimizer in optimizers] min_valid_losses = [np.inf, np.inf, np.inf] print(f'Training a model using {plane} series...') print(f'Checkpoints and losses will be save to {out_dir}') for epoch, _ in enumerate(range(epochs), 1): print(f'=== Epoch {epoch}/{epochs} ===') batch_train_losses = np.array([0.0, 0.0, 0.0]) batch_valid_losses = np.array([0.0, 0.0, 0.0]) for inputs, labels in train_loader: inputs, labels = inputs.to(device), labels.to(device) batch_loss = batch_forward_backprop(models, inputs, labels, criterions, optimizers) batch_train_losses += batch_loss valid_preds = [] valid_labels = [] for inputs, labels in valid_loader: inputs, labels = inputs.to(device), labels.to(device) batch_preds, batch_loss = \ batch_forward(models, inputs, labels, criterions) batch_valid_losses += batch_loss valid_labels.append(labels.detach().cpu().numpy().squeeze()) valid_preds.append(batch_preds) batch_train_losses /= len(train_loader) batch_valid_losses /= len(valid_loader) print_stats(batch_train_losses, batch_valid_losses, valid_labels, valid_preds) save_losses(batch_train_losses, batch_valid_losses, losses_path) update_lr_schedulers(lr_schedulers, batch_valid_losses) for i, (batch_v_loss, min_v_loss) in \ enumerate(zip(batch_valid_losses, min_valid_losses)): if batch_v_loss < min_v_loss: save_checkpoint(epoch, plane, diagnoses[i], models[i], optimizers[i], out_dir) min_valid_losses[i] = batch_v_loss
def main(valid_paths_csv, output_dir): device = 'cuda' if torch.cuda.is_available() else 'cpu' input_files_df = pd.read_csv(valid_paths_csv, header=None) cnn_models_paths = 'src/cnn_models_paths.txt' lr_models_paths = 'src/lr_models_paths.txt' if not os.path.exists(output_dir): os.makedirs(output_dir) output_file = f'{output_dir}/predictions.csv' if os.path.exists(output_file): os.rename(output_file, f'{output_file}.back') print(f'!! {output_file} already exists, renamed to {output_file}.bak') # Load MRNet models print(f'Loading CNN models listed in {cnn_models_paths}...') cnn_models_paths = [ line.rstrip('\n') for line in open(cnn_models_paths, 'r') ] abnormal_mrnets = [] acl_mrnets = [] meniscus_mrnets = [] for i, mrnet_path in enumerate(cnn_models_paths): model = MRNet().to(device) checkpoint = torch.load(mrnet_path, map_location=device) model.load_state_dict(checkpoint['state_dict']) if i < 3: abnormal_mrnets.append(model) elif i >= 3 and i < 6: acl_mrnets.append(model) else: meniscus_mrnets.append(model) mrnets = [abnormal_mrnets, acl_mrnets, meniscus_mrnets] # Load logistic regression models print(f'Loading logistic regression models listed in {lr_models_paths}...') lr_models_paths = [ line.rstrip('\n') for line in open(lr_models_paths, 'r') ] lrs = [joblib.load(lr_path) for lr_path in lr_models_paths] # Parse input, 3 rows at a time (i.e. per case) npy_paths = [row.values[0] for _, row in input_files_df.iterrows()] transform = transforms.Compose( [transforms.ToPILImage(), transforms.ToTensor()]) print(f'Generating predictions per case...') print(f'Predictions will be saved as {output_file}') for i in tqdm(range(0, len(npy_paths), 3)): case_paths = [npy_paths[i], npy_paths[i + 1], npy_paths[i + 2]] data = [] for case_path in case_paths: series = preprocess_data(case_path, transform) data.append(series.unsqueeze(0).to(device)) # Make predictions per case case_preds = [] for i, mrnet in enumerate(mrnets): # For each condition (mrnet) # Based on each plane (data) sagittal_pred = mrnet[0](data[0]).detach().cpu().item() coronal_pred = mrnet[1](data[1]).detach().cpu().item() axial_pred = mrnet[2](data[2]).detach().cpu().item() # Combine predictions to make a final prediction X = [[axial_pred, coronal_pred, sagittal_pred]] case_preds.append(np.float64(lrs[i].predict_proba(X)[:, 1])) # Write to output csv - append if it exists already with open(output_file, 'a+') as csv_file: writer = csv.writer(csv_file) writer.writerow(case_preds)
def train(rundir, diagnosis, dataset, epochs, learning_rate, use_gpu, attention): models = [] if (dataset == 0): train_loader, valid_loader, test_loader = external_load_data( diagnosis, use_gpu) models.append((MRNet(useMultiHead=attention), train_loader, valid_loader, 'external_validation')) elif (dataset == 1): train_loaders, valid_loaders = mr_load_data(diagnosis, use_gpu) train_loader_sag, train_loader_ax, train_loader_cor = train_loaders valid_loader_sag, valid_loader_ax, valid_loader_cor = valid_loaders models = [(MRNet(max_layers=51), train_loader_sag, valid_loader_sag, 'sagittal'), (MRNet(max_layers=61), train_loader_ax, valid_loader_ax, 'axial'), (MRNet(max_layers=58), train_loader_cor, valid_loader_cor, 'coronal')] for model, train_loader, valid_loader, fname in models: if use_gpu: model = model.cuda() optimizer = torch.optim.Adam(model.parameters(), learning_rate, weight_decay=.01) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=.3, threshold=1e-4) best_val_loss = float('inf') start_time = datetime.now() for epoch in range(epochs): change = datetime.now() - start_time print('starting epoch {}. time passed: {}'.format( epoch + 1, str(change))) train_loss, train_auc, _, _ = run_model(model, train_loader, train=True, optimizer=optimizer) print(f'train loss: {train_loss:0.4f}') print(f'train AUC: {train_auc:0.4f}') val_loss, val_auc, _, _ = run_model(model, valid_loader) print(f'valid loss: {val_loss:0.4f}') print(f'valid AUC: {val_auc:0.4f}') scheduler.step(val_loss) if val_loss < best_val_loss: best_val_loss = val_loss file_name = f'val{val_loss:0.4f}_train{train_loss:0.4f}_epoch{epoch+1}' save_path = Path(rundir) / fname / file_name folder = rundir + '/' + fname try: for f in os.listdir(folder): os.remove(folder + '/' + f) torch.save(model, save_path) except: os.makedirs(rundir + '/' + fname) torch.save(model, save_path)
def compute_attributions(algo, inputs, **kwargs): ''' A common function for computing captum attributions ''' return algo.attribute(inputs, **kwargs) # load model """ gc_model = torchvision.models.squeezenet1_1(pretrained=True) for param in gc_model.parameters(): param.requires_grad = True """ gc_model = MRNet(useMultiHead=True) weights = torch.load("weights", map_location=torch.device('cpu')) gc_model.load_state_dict(weights) #print(gc_model) # Grad cam code conv_module = gc_model.model.features[10] #conv_module = gc_model.features[12] gradient_value = None # Stores gradient of the module you chose above during a backwards pass. activation_value = None # Stores the activation of the module you chose above during a forwards pass. def gradient_hook(a, b, gradient):
def evaluate(split, model_path, diagnosis, dataset, use_gpu, attention): preds = None labels = None if dataset == 0: train_loader, valid_loader, test_loader = external_load_data(diagnosis, use_gpu) #model = MRNet(useMultiHead = attention) #state_dict = torch.load(model_path, map_location=(None if use_gpu else 'cpu')) #model.load_state_dict(state_dict) model = torch.load(model_path) if use_gpu: model = model.cuda() if split == 'train': loader = train_loader elif split == 'valid': loader = valid_loader elif split == 'test': loader = test_loader else: raise ValueError("split must be 'train', 'valid', or 'test'") loss, auc, preds, labels = run_model(model, loader) print(f'{split} loss: {loss:0.4f}') print(f'{split} AUC: {auc:0.4f}') if dataset == 1: train_loaders, valid_loaders = mr_load_data(diagnosis, use_gpu, train_shuffle = True) path_s = os.listdir(model_path + '/sagittal') path_a = os.listdir(model_path + '/axial') path_c = os.listdir(model_path + '/coronal') ps = [int(x.split("epoch")[1]) for x in path_s] pa = [int(x.split("epoch")[1]) for x in path_a] pc = [int(x.split("epoch")[1]) for x in path_c] model_path_sag = path_s[ps.index(max(ps))] model_path_ax = path_a[pa.index(max(pa))] model_path_cor = path_c[pc.index(max(pc))] print("{} {} {}".format(model_path_sag, model_path_ax, model_path_cor)) state_dict_sag = torch.load(model_path + '/sagittal/' + model_path_sag, map_location=(None if use_gpu else 'cpu')) state_dict_ax = torch.load(model_path + '/axial/' + model_path_ax, map_location=(None if use_gpu else 'cpu')) state_dict_cor = torch.load(model_path + '/coronal/' + model_path_cor, map_location=(None if use_gpu else 'cpu')) model_sag = MRNet(useMultiHead=attention, max_layers=51) model_sag.load_state_dict(state_dict_sag) model_ax = MRNet(useMultiHead=attention, max_layers=61) model_ax.load_state_dict(state_dict_ax) model_cor = MRNet(useMultiHead=attention, max_layers=58) model_cor.load_state_dict(state_dict_cor) #model_sag = torch.load(model_path + '/sagittal/' + model_path_sag) #model_ax = torch.load(model_path + '/axial/' + model_path_ax) #model_cor = torch.load(model_path + '/coronal/' + model_path_cor) if use_gpu: model_sag = model_sag.cuda() model_ax = model_ax.cuda() model_cor = model_cor.cuda() loss_sag, auc_sag, t_preds_sag, labels_sag = run_model(model_sag, train_loaders[0]) _, _, preds_sag, _ = run_model(model_sag, valid_loaders[0]) print(f'sagittal {split} loss: {loss_sag:0.4f}') print(f'sagittal {split} AUC: {auc_sag:0.4f}') loss_ax, auc_ax, t_preds_ax, labels_ax = run_model(model_ax, train_loaders[1]) _, _, preds_ax, _ = run_model(model_ax, valid_loaders[1]) print(f'axial {split} loss: {loss_ax:0.4f}') print(f'axial {split} AUC: {auc_ax:0.4f}') loss_cor, auc_cor, t_preds_cor, labels_cor = run_model(model_cor, train_loaders[2]) _, _, preds_cor, valid_labels = run_model(model_cor, valid_loaders[2]) print(f'coronal {split} loss: {loss_cor:0.4f}') print(f'coronal {split} AUC: {auc_cor:0.4f}') X = np.zeros((len(t_preds_cor), 3)) X[:, 0] = t_preds_sag X[:, 1] = t_preds_ax X[:, 2] = t_preds_cor y = np.array(labels_cor) lgr = LogisticRegression(solver='lbfgs') lgr.fit(X,y) X_valid = np.zeros((len(preds_cor), 3)) X_valid[:, 0] = preds_sag X_valid[:, 1] = preds_ax X_valid[:, 2] = preds_cor y_preds = lgr.predict(X_valid) y_true = np.array(valid_labels) print(metrics.roc_auc_score(y_true, y_preds)) print(metrics.classification_report(y_true, y_preds, target_names=['class 0', 'class 1'])) return preds, labels