def train(args): if args.ckpt_path: model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 else: model_fn = models.__dict__[args.model] model = model_fn(**vars(args)) model = nn.DataParallel(model, args.gpu_ids) model = model.to(args.device) model.train() # Set up population-based training client pbt_client = PBTClient(args.pbt_server_url, args.pbt_server_port, args.pbt_server_key, args.pbt_config_path) # Get optimizer and scheduler parameters = model.module.parameters() optimizer = optim.get_optimizer(parameters, args, pbt_client) ModelSaver.load_optimizer(args.ckpt_path, args.gpu_ids, optimizer) # Get logger, evaluator, saver train_loader = DataLoader(args, 'train', is_training_set=True) eval_loaders = [DataLoader(args, 'valid', is_training_set=False)] evaluator = ModelEvaluator(eval_loaders, args.epochs_per_eval, args.max_eval, args.num_visuals, use_ten_crop=args.use_ten_crop) saver = ModelSaver(**vars(args)) for _ in range(args.num_epochs): optim.update_hyperparameters(model.module, optimizer, pbt_client.hyperparameters()) for inputs, targets in train_loader: with torch.set_grad_enabled(True): logits = model.forward(inputs.to(args.device)) loss = F.binary_cross_entropy_with_logits(logits, targets.to(args.device)) optimizer.zero_grad() loss.backward() optimizer.step() metrics = evaluator.evaluate(model, args.device) metric_val = metrics.get(args.metric_name, None) ckpt_path = saver.save(model, args.model, optimizer, args.device, metric_val) pbt_client.save(ckpt_path, metric_val) if pbt_client.should_exploit(): # Exploit pbt_client.exploit() # Load model and optimizer parameters from exploited network model, ckpt_info = ModelSaver.load_model(pbt_client.parameters_path(), args.gpu_ids) model = model.to(args.device) model.train() ModelSaver.load_optimizer(pbt_client.parameters_path(), args.gpu_ids, optimizer) # Explore pbt_client.explore()
def test(args): model_fn = model_dict[args_.model] model = model_fn(num_classes=1 if args_.model == 'fd' else 10) model = nn.DataParallel(model, args.gpu_ids) ckpt_info = ModelSaver.load_model(args.ckpt_path, model) args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) model.eval() test_set = FilterDataset('alexnet', './filters', is_training=False) test_loader = torch.utils.data.DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) logger = TestLogger(args) logger.start_epoch() for inputs, labels in test_loader: logger.start_iter() with torch.set_grad_enabled(True): # Forward logits = model.forward(inputs.to(args.device)) logger.end_iter(inputs, labels, logits) logger.end_epoch()
def train(args): # Get model model = models.__dict__[args.model](args) if args.ckpt_path: model = ModelSaver.load_model(model, args.ckpt_path, args.gpu_ids, is_training=True) model = model.to(args.device) model.train() # Get loader, logger, and saver train_loader, val_loader = get_data_loaders(args) logger = TrainLogger(args, model, dataset_len=len(train_loader.dataset)) saver = ModelSaver(args.save_dir, args.max_ckpts, metric_name=args.metric_name, maximize_metric=args.maximize_metric, keep_topk=True) # Train while not logger.is_finished_training(): logger.start_epoch() for batch in train_loader: logger.start_iter() # Train over one batch model.set_inputs(batch['src'], batch['tgt']) model.train_iter() logger.end_iter() # Evaluate if logger.global_step % args.iters_per_eval < args.batch_size: criteria = {'MSE_src2tgt': mse, 'MSE_tgt2src': mse} stats = evaluate(model, val_loader, criteria) logger.log_scalars({'val_' + k: v for k, v in stats.items()}) saver.save(logger.global_step, model, stats[args.metric_name], args.device) logger.end_epoch()
def main(args): # create npy from dicom print("Reading input dicom...") study = util.dicom_2_npy(args.input_study, args.series_description) # normalize and convert to tensor print("Formatting input for model...") study_windows = util.format_img(study) print("Loading saved model...") model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) print("Sending model to GPU device...") #start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) print("Evaluating study...") model.eval() predicted_probabilities = [] # window wise for a single study with torch.no_grad(): for window in study_windows: cls_logits = model.forward( window.to(args.device, dtype=torch.float)) cls_probs = torch.sigmoid(cls_logits).to('cpu').numpy() predicted_probabilities.append(cls_probs[0][0]) print( f"Probablity of having Pulmonary Embolism: {max(predicted_probabilities)}" )
def predict(args): model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) model.eval() # Predict outputs data_loader = WhiteboardLoader(args.data_dir, args.phase, args.batch_size, shuffle=False, do_augment=False, num_workers=args.num_workers) all_probs, all_paths = [], [] with tqdm(total=len(data_loader.dataset), unit=' ' + args.phase) as progress_bar: for inputs, targets, paths in data_loader: bs, n_crops, c, h, w = inputs.size() inputs = inputs.view(-1, c, h, w) # Fuse batch size and n_crops with torch.no_grad(): logits = model.forward(inputs.to(args.device)) logits = logits.view(bs, n_crops, -1).mean(1) # Average over n_crops probs = F.softmax(logits, -1) # Take probability of whiteboard all_probs += [p[1] for p in probs] all_paths += list(paths) progress_bar.update(targets.size(0)) # Write CSV record_ids = [os.path.basename(p)[:-4] for p in all_paths] # Convert to record_id df = pd.DataFrame([{'record_id': r, 'output': '{:.5f}'.format(prob.item()), 'has_whiteboard_@{:.2f}'.format(args.prob_threshold): int(prob > args.prob_threshold), 'url': get_url(r)} for r, prob in zip(record_ids, all_probs)]) df.to_csv(os.path.join(args.results_dir, 'outputs.csv'), index=False)
def extract_filters(ckpt_path, model_name): """Get numpy array, val_loss from a saved checkpoint.""" model_dict = { 'alexnet': models.alexnet, 'resnet50': models.resnet50, 'vgg19': models.vgg19_bn, } model_fn = model_dict[model_name] model = model_fn(num_classes=10) model = nn.DataParallel(model) ckpt_info = ModelSaver.load_model(ckpt_path, model) filter_dict = { 'alexnet': 'module.features.0', 'resnet50': 'module.conv1', 'vgg19': 'module.features.0' } target_layer = filter_dict[model_name] filters_np = None for name, module in model.named_modules(): if name == target_layer: filters_np = module.weight.data.cpu().numpy() break if filters_np is None: raise RuntimeError( 'Could not find filters for layer {}'.format(target_layer)) return filters_np, ckpt_info['epoch'], ckpt_info['val_loss']
def train(args): if args.ckpt_path: model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 else: model_fn = models.__dict__[args.model] model = model_fn(**vars(args)) model = nn.DataParallel(model, args.gpu_ids) model = model.to(args.device) model.train() # Get optimizer and scheduler optimizer = optim.get_optimizer( filter(lambda p: p.requires_grad, model.parameters()), args) lr_scheduler = optim.get_scheduler(optimizer, args) if args.ckpt_path: ModelSaver.load_optimizer(args.ckpt_path, optimizer, lr_scheduler) # Get logger, evaluator, saver loss_fn = nn.CrossEntropyLoss() train_loader = CIFARLoader('train', args.batch_size, args.num_workers) logger = TrainLogger(args, len(train_loader.dataset)) eval_loaders = [CIFARLoader('val', args.batch_size, args.num_workers)] evaluator = ModelEvaluator(eval_loaders, logger, args.max_eval, args.epochs_per_eval) saver = ModelSaver(**vars(args)) # Train model while not logger.is_finished_training(): logger.start_epoch() for inputs, targets in train_loader: logger.start_iter() with torch.set_grad_enabled(True): logits = model.forward(inputs.to(args.device)) loss = loss_fn(logits, targets.to(args.device)) logger.log_iter(loss) optimizer.zero_grad() loss.backward() optimizer.step() logger.end_iter() metrics = evaluator.evaluate(model, args.device, logger.epoch) saver.save(logger.epoch, model, optimizer, lr_scheduler, args.device, metric_val=metrics.get(args.metric_name, None)) logger.end_epoch(metrics) optim.step_scheduler(lr_scheduler, metrics, logger.epoch)
def train(args): train_loader = get_loader(args=args) if args.ckpt_path: model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 else: model_fn = models.__dict__[args.model] args.D_in = train_loader.D_in model = model_fn(**vars(args)) model = model.to(args.device) model.train() # Get optimizer and scheduler optimizer = optim.get_optimizer( filter(lambda p: p.requires_grad, model.parameters()), args) lr_scheduler = optim.get_scheduler(optimizer, args) if args.ckpt_path: ModelSaver.load_optimizer(args.ckpt_path, optimizer, lr_scheduler) # Get logger, evaluator, saver loss_fn = optim.get_loss_fn(args.loss_fn, args) logger = TrainLogger(args, len(train_loader.dataset)) eval_loaders = [ get_loader(args, phase='train', is_training=False), get_loader(args, phase='valid', is_training=False) ] evaluator = ModelEvaluator(args, eval_loaders, logger, args.max_eval, args.epochs_per_eval) saver = ModelSaver(**vars(args)) # Train model while not logger.is_finished_training(): logger.start_epoch() for src, tgt in train_loader: logger.start_iter() with torch.set_grad_enabled(True): pred_params = model.forward(src.to(args.device)) ages = src[:, 1] loss = loss_fn(pred_params, tgt.to(args.device), ages.to(args.device), args.use_intvl) #loss = loss_fn(pred_params, tgt.to(args.device), src.to(args.device), args.use_intvl) logger.log_iter(src, pred_params, tgt, loss) optimizer.zero_grad() loss.backward() optimizer.step() logger.end_iter() metrics = evaluator.evaluate(model, args.device, logger.epoch) # print(metrics) saver.save(logger.epoch, model, optimizer, lr_scheduler, args.device,\ metric_val=metrics.get(args.metric_name, None)) logger.end_epoch(metrics=metrics)
def load_multi_model(multi_args, model_args, data_args, gpu_ids): """Load multi lodel (a frontal model and a lateral model).""" model_ap, ckpt_info_ap = ModelSaver.load_model(multi_args.ap_ckpt_path, gpu_ids, model_args, data_args) model_pa, ckpt_info_pa = ModelSaver.load_model(multi_args.pa_ckpt_path, gpu_ids, model_args, data_args) model_lateral, lateral_ckpt_info = ModelSaver.load_model( multi_args.lateral_ckpt_path, args.gpu_ids, args) # Make sure all models used the same task sequence assert model_ap.task_sequence == model_pa.task_sequence assert model_pa.task_sequence == model_lateral.task_sequence models = {'ap': model_ap, 'pa': model_pa, 'lateral': model_lateral} model = MultiModelWrapper(models) model.task_sequence = model_ap.task_sequence return model
def loaded_model_iterator(self, task): from saver import ModelSaver model_dicts = self.task2model_dicts[task] for model_dict in model_dicts: ckpt_path = model_dict['ckpt_path'] self.model_args.model_uncertainty = model_dict['is_3class'] model, ckpt_info = ModelSaver.load_model(ckpt_path, self.gpu_ids, self.model_args, self.data_args) yield model
def test(args): model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) model.eval() # Run a single evaluation eval_loader = WhiteboardLoader(args.data_dir, args.phase, args.batch_size, shuffle=False, do_augment=False, num_workers=args.num_workers) logger = TestLogger(args, len(eval_loader.dataset)) logger.start_epoch() evaluator = ModelEvaluator([eval_loader], logger, num_visuals=args.num_visuals, prob_threshold=args.prob_threshold) metrics = evaluator.evaluate(model, args.device, logger.epoch) logger.end_epoch(metrics)
def __init__(self): root_path = os.path.dirname(os.path.dirname(__file__)) if torch.cuda.is_available(): self.device = torch.device("cuda") gpu_ids = list(range(torch.cuda.device_count())) else: self.device = torch.device("cpu") gpu_ids = [] self.model, _ = ModelSaver.load_model( os.path.join(root_path, "penet_best.pth.tar"), gpu_ids) self.model = self.model.to(self.device) self.grad_cam = CustomGradCAM(self.model, self.device, is_binary=True, is_3d=True)
def test(args): model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) model.eval() data_loader = get_loader(args, phase=args.phase, is_training=False) logger = TestLogger(args, len(data_loader.dataset)) # Get model outputs, log to TensorBoard, write masks to disk window-by-window util.print_err('Writing model outputs to {}...'.format(args.results_dir)) all_gender = [] all_age = [] all_tte = [] all_is_alive = [] all_mu = [] all_s2 = [] with tqdm(total=len(data_loader.dataset), unit=' windows') as progress_bar: for i, (src, tgt) in enumerate(data_loader): all_gender.extend([int(x) for x in src[:, 0]]) all_age.extend([float(x) for x in src[:, 1]]) all_tte.extend([float(x) for x in tgt[:, 0]]) all_is_alive.extend([int(x) for x in tgt[:, 1]]) with torch.no_grad(): pred_params = model.forward(src.to(args.device)) # import pdb # pdb.set_trace() outputs = pred_params.cpu().numpy() all_mu.extend([float(x) for x in outputs[:, 0]]) all_s2.extend([float(x) for x in outputs[:, 1]]) progress_bar.update(src.size(0)) # print pred_params (mu, s) to file fd = open(args.results_dir + '/test_stats.csv', 'w') fd.write('gender, age, tte, is_alive, mu, s2\n') for gender, age, tte, is_alive, mu, s2 \ in zip(all_gender, all_age, all_tte, all_is_alive, all_mu, all_s2): fd.write('%d, %f, %f, %d, %f, %f\n' % (gender, age, tte, is_alive, mu, s2)) fd.close()
def test(args): # Get dataset dataset = PairedDataset(args.data_dir, phase=args.phase, resize_shape=args.resize_shape, crop_shape=args.crop_shape, direction=args.direction) data_loader = DataLoader(dataset, args.batch_size, shuffle=False, num_workers=args.num_workers) # Get model model = models.__dict__[args.model](args) model = ModelSaver.load_model(model, args.ckpt_path, args.gpu_ids) model.train() # Set up image saving if args.save_images is None: save_hook = None else: saver = ImageSaver(args.save_images, args.results_dir, args.name, args.phase) save_hook = saver.save # Test model criteria = {'MSE_src2tgt': mse, 'MSE_tgt2src': mse} stats = evaluate(model, data_loader, criteria, batch_hook=save_hook) # Add model info to stats stats.update({'name': args.name, 'ckpt_path': args.ckpt_path}) # Write stats to disk stats_path = os.path.join(args.results_dir, 'stats.json') print('Saving stats to {}...'.format(stats_path)) with open(stats_path, 'w') as json_fh: json.dump(stats, json_fh, sort_keys=True, indent=4) # Copy training args for reference args_src = os.path.join(args.save_dir, 'args.json') args_dst = os.path.join(args.results_dir, 'args.json') print('Copying args to {}...'.format(args_dst)) shutil.copy(args_src, args_dst)
def test(args): model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) model.eval() data_loader = CIFARLoader('val', args.batch_size, args.num_workers) # Get model outputs, log to TensorBoard, write masks to disk window-by-window util.print_err('Writing model outputs to {}...'.format(args.results_dir)) with tqdm(total=len(data_loader.dataset), unit=' examples') as progress_bar: for i, (inputs, info_dict) in enumerate(data_loader): with torch.no_grad(): logits = model.forward(inputs.to(args.device)) probs = F.softmax(logits) # TODO: Test script is incomplete. Does nothing with the outputs. progress_bar.update(inputs.size(0))
def run_model(ckpt_path, ckpt_args, has_gpu, custom_tasks=None): """Run a model with the specified args and output predictions. Args: ckpt_path (Path): path specifying the checkpoint ckpt_args (dict): args associated with the corresponding run Returns: pred_df (pandas.DataFrame): model predictions gt_df (pandas.DataFrame): corresponding ground-truth labels """ ckpt_save_dir = ckpt_path.parent model_args = Namespace(**ckpt_args['model_args']) # JBY: The samed model will not be moco model_args.moco = False transform_args = Namespace(**ckpt_args['transform_args']) data_args = Namespace(**ckpt_args['data_args']) print("in select_ensemble.py: data_args: {}".format(data_args)) data_args.custom_tasks = custom_tasks if has_gpu: gpu_ids = util.args_to_list(ckpt_args['gpu_ids'], allow_empty=True, arg_type=int, allow_negative=False) else: # TODO: JBY: HACK! CHANGING GPU ID TO NONE gpu_ids = [] device = util.setup_gpus(gpu_ids) model, _ = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=gpu_ids, model_args=model_args, is_training=False) predictor = Predictor(model=model, device=device) loader = get_loader(phase='valid', data_args=data_args, transform_args=transform_args, is_training=False, return_info_dict=False, logger=None) pred_df, gt_df = predictor.predict(loader) return pred_df, gt_df
def test(args): model_fn = models.__dict__[args_.model] model = model_fn(args.num_classes) model = nn.DataParallel(model, args.gpu_ids) ckpt_info = ModelSaver.load_model(args.ckpt_path, model) args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) model.eval() _, test_loader, _ = get_cifar_loaders(args.batch_size, args.num_workers) logger = TestLogger(args) logger.start_epoch() for inputs, labels in test_loader: logger.start_iter() with torch.set_grad_enabled(True): # Forward logits = model.forward(inputs.to(args.device)) logger.end_iter(inputs, labels, logits) logger.end_epoch()
def get_cams(args): print('Loading model...') model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) model = model.to(args.device) args.start_epoch = ckpt_info['epoch'] + 1 print('Last layer in model.features is named "{}"...'.format([k for k in model.module.encoders._modules.keys()][-1])) print('Extracting feature maps from layer named "{}"...'.format(args.target_layer)) grad_cam = GradCAM(model, args.device, is_binary=True, is_3d=True) print(grad_cam) gbp = GuidedBackPropagation(model, args.device, is_binary=True, is_3d=True) print(gbp) num_generated = 0 data_loader = CTDataLoader(args, phase=args.phase, is_training=False) print(data_loader) study_idx_dict = {} study_count = 1 for inputs, target_dict in data_loader: #print(inputs, target_dict) #print('target_dict dir={}'.format(dir(target_dict))) #print('\ntarget_dict[study_num]={}'.format(target_dict['study_num'])) probs, idx = grad_cam.forward(inputs) grad_cam.backward(idx=idx[0]) # Just take top prediction cam = grad_cam.get_cam(args.target_layer) labels = target_dict['is_abnormal'] if labels.item() == 0: # Keep going until we get an aneurysm study print('Skipping a normal example...') continue print('Generating CAM...') study_num = 1 with torch.set_grad_enabled(True): probs, idx = grad_cam.forward(inputs) print(probs, idx) grad_cam.backward(idx=idx[0]) # Just take top prediction cam = grad_cam.get_cam(args.target_layer) guided_backprop = None if args.use_gbp: inputs2 = torch.autograd.Variable(inputs, requires_grad=True) probs2, idx2 = gbp.forward(inputs2) gbp.backward(idx=idx2[0]) guided_backprop = np.squeeze(gbp.generate()) print('Overlaying CAM...') print(cam.shape) new_cam = util.resize(cam, inputs[0]) print(new_cam.shape) input_np = util.un_normalize(inputs[0], args.img_format, data_loader.dataset.pixel_dict) input_np = np.transpose(input_np, (1, 2, 3, 0)) input_frames = list(input_np) input_normed = np.float32(input_np) / 255 cam_frames = list(util.add_heat_map(input_normed, new_cam)) gbp_frames = None if args.use_gbp: gbp_np = util.normalize_to_image(guided_backprop * new_cam) gbp_frames = [] for dim in range(gbp_np.shape[0]): slice_ = gbp_np[dim, :, :] gbp_frames.append(slice_[..., None]) # Write to a GIF file output_path_input = os.path.join(os.path.join(args.cam_dir, '{}_{}_input_fn.gif'.format(target_dict['study_num'], study_count))) output_path_cam = os.path.join(args.cam_dir, '{}_{}_cam_fn.gif'.format(target_dict['study_num'], study_count)) output_path_combined = os.path.join(args.cam_dir, '{}_{}_combined_fn.gif'.format(target_dict['study_num'], study_count)) print('Writing set {}/{} of CAMs to {}...'.format(num_generated + 1, args.num_cams, args.cam_dir)) input_clip = mpy.ImageSequenceClip(input_frames, fps=4) input_clip.write_gif(output_path_input, verbose=False) cam_clip = mpy.ImageSequenceClip(cam_frames, fps=4) cam_clip.write_gif(output_path_cam, verbose=False) combined_clip = mpy.clips_array([[input_clip, cam_clip]]) combined_clip.write_gif(output_path_combined, verbose=False) if args.use_gbp: output_path_gcam = os.path.join(args.cam_dir, 'gbp_{}.gif'.format(num_generated + 1)) gbp_clip = mpy.ImageSequenceClip(gbp_frames, fps=4) gbp_clip.write_gif(output_path_gcam, verbose=False) study_count += 1 num_generated += 1 if num_generated == args.num_cams: return
def test(args): """Run model testing.""" model_args = args.model_args data_args = args.data_args logger_args = args.logger_args # import pdb; pdb.set_trace() # Get logger. logger = Logger(logger_args.log_path, logger_args.save_dir, logger_args.results_dir) # Get image paths corresponding to predictions for logging paths = None if model_args.config_path is not None: # Instantiate the EnsemblePredictor class for obtaining # model predictions. predictor = EnsemblePredictor(config_path=model_args.config_path, model_args=model_args, data_args=data_args, gpu_ids=args.gpu_ids, device=args.device, logger=logger) # Obtain ensemble predictions. # Caches both individual and ensemble predictions. # We always turn off caching to ensure that we write the Path column. predictions, groundtruth, paths = predictor.predict(cache=False, return_paths=True, all_gt_tasks=True) else: # Load the model at ckpt_path. ckpt_path = model_args.ckpt_path ckpt_save_dir = Path(ckpt_path).parent model_uncertainty = model_args.model_uncertainty # Get model args from checkpoint and add them to # command-line specified model args. model_args, transform_args\ = ModelSaver.get_args(cl_model_args=model_args, dataset=data_args.dataset, ckpt_save_dir=ckpt_save_dir, model_uncertainty=model_uncertainty) # TODO JBY: in test moco should never be true. model_args.moco = args.model_args.moco model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=args.gpu_ids, model_args=model_args, is_training=False) # Instantiate the Predictor class for obtaining model predictions. predictor = Predictor(model=model, device=args.device) # Get phase loader object. return_info_dict = True loader = get_loader(phase=data_args.phase, data_args=data_args, transform_args=transform_args, is_training=False, return_info_dict=return_info_dict, logger=logger) # Obtain model predictions. if return_info_dict: predictions, groundtruth, paths = predictor.predict(loader) else: predictions, groundtruth = predictor.predict(loader) # print(predictions[CHEXPERT_COMPETITION_TASKS]) if model_args.calibrate: #open the json file which has the saved parameters import json with open(CALIBRATION_FILE) as f: data = json.load(f) i = 0 #print(predictions) import math def sigmoid(x): return 1 / (1 + math.exp(-x)) for column in predictions: predictions[column] = predictions[column].apply \ (lambda x: sigmoid(x * data[i][0][0][0] \ + data[i][1][0])) i += 1 # print(predictions[CHEXPERT_COMPETITION_TASKS]) #run forward on all the predictions in each row of predictions # Log predictions and groundtruth to file in CSV format. logger.log_predictions_groundtruth(predictions, groundtruth, paths) if not args.inference_only: # Instantiate the evaluator class for evaluating models. evaluator = Evaluator(logger, operating_points_path=CHEXPERT_RAD_PATH) # Get model metrics and curves on the phase dataset. metrics, curves = evaluator.evaluate_tasks(groundtruth, predictions) # Log metrics to stdout and file. logger.log_stdout(f"Writing metrics to {logger.metrics_path}.") logger.log_metrics(metrics, save_csv=True) # TODO: make this work with ensemble # TODO: investigate if the eval_loader can just be the normal loader here if logger_args.save_cams: cams_dir = logger_args.save_dir / 'cams' print(f'Save cams to {cams_dir}') save_grad_cams(args, loader, model, cams_dir, only_competition=logger_args.only_competition_cams, only_top_task=False) logger.log("=== Testing Complete ===")
def test(args): print("Stage 1") model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) print("Stage 2") args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) print("Stage 3") model.eval() print("Stage 4") data_loader = CTDataLoader(args, phase=args.phase, is_training=False) print(data_loader.dataset.ctpe_list) study2slices = defaultdict(list) study2probs = defaultdict(list) study2labels = {} logger = TestLogger(args, len(data_loader.dataset), data_loader.dataset.pixel_dict) minimum = [] means = [] maximum = [] data = [] # Get model outputs, log to TensorBoard, write masks to disk window-by-window util.print_err('Writing model outputs to {}...'.format(args.results_dir)) with tqdm(total=len(data_loader.dataset), unit=' windows') as progress_bar: for i, (inputs, targets_dict) in enumerate(data_loader): with torch.no_grad(): cls_logits = model.forward(inputs.to(args.device)) cls_probs = F.sigmoid(cls_logits) if args.visualize_all: logger.visualize(inputs, cls_logits, targets_dict=None, phase=args.phase, unique_id=i) max_probs = cls_probs.to('cpu').numpy() for study_num, slice_idx, prob in \ zip(targets_dict['study_num'], targets_dict['slice_idx'], list(max_probs)): # Convert to standard python data types study_num = int(study_num) slice_idx = int(slice_idx) # Save series num for aggregation study2slices[study_num].append(slice_idx) study2probs[study_num].append(prob.item()) series = data_loader.get_series(study_num) if study_num not in study2labels: study2labels[study_num] = int(series.is_positive) progress_bar.update(inputs.size(0)) # Combine masks util.print_err('Combining masks...') max_probs = [] labels = [] study_nums = [] predictions = {} print("Get max prob") for study_num in tqdm(study2slices): # Sort by slice index and get max probability slice_list, prob_list = (list(t) for t in zip( *sorted(zip(study2slices[study_num], study2probs[study_num]), key=lambda slice_and_prob: slice_and_prob[0]))) study2slices[study_num] = slice_list study2probs[study_num] = prob_list max_prob = max(prob_list) max_probs.append(max_prob) label = study2labels[study_num] labels.append(label) study_nums.append(study_num) predictions[study_num] = {'label': label, 'pred': max_prob} #Save predictions to file, indexed by study number print("Saving predictions to pickle files") with open('{}/preds.pickle'.format(args.results_dir), "wb") as fp: pickle.dump(predictions, fp) # Compute AUROC and AUPRC using max aggregation, write to files max_probs, labels = np.array(max_probs), np.array(labels) fpr, tpr, threshold = roc_curve(labels, max_probs) i = np.arange(len(tpr)) roc = pd.DataFrame({ 'tf': pd.Series(tpr - (1 - fpr), index=i), 'threshold': pd.Series(threshold, index=i) }) roc_t = roc.ix[(roc.tf - 0).abs().argsort()[:1]] threshold = 0.5 pred = [1 if p > threshold else 0 for p in max_probs] tn, fp, fn, tp = confusion_matrix(labels, pred).ravel() print("\nTrue Positive Examples\n" + "-" * 80) for i, study_num in enumerate(study_nums): if labels[i] == 1 and pred[i] == 1: print(study_num) print("\nTrue Negative Examples\n" + "-" * 80) for i, study_num in enumerate(study_nums): if labels[i] == 0 and pred[i] == 0: print(study_num) print("\nFalse Negative Examples\n" + "-" * 80) for i, study_num in enumerate(study_nums): if labels[i] == 1 and pred[i] == 0: print(study_num) print("\nFalse Positive Examples\n" + "-" * 80) for i, study_num in enumerate(study_nums): if labels[i] == 0 and pred[i] == 1: print(study_num) print("Total number of data :", len(labels)) print("Total number of positives :", len([l for l in labels if l == 1])) print("Total number of negatives :", len([l for l in labels if l == 0])) print("# True Negative : ", tn) print("# True Positive : ", tp) print("# False Negative : ", fn) print("# False Positive : ", fp) metrics = { args.phase + '_' + 'AUPRC': sk_metrics.average_precision_score(labels, max_probs), args.phase + '_' + 'AUROC': sk_metrics.roc_auc_score(labels, max_probs), } for k, v in metrics.items(): print('{}: {:.5f}\n'.format(k, v)) print("Saving metrics to file") with open(os.path.join(args.results_dir, 'metrics.txt'), 'w') as metrics_fh: for k, v in metrics.items(): metrics_fh.write('{}: {:.5f}\n'.format(k, v)) curves = { args.phase + '_' + 'PRC': sk_metrics.precision_recall_curve(labels, max_probs), args.phase + '_' + 'ROC': sk_metrics.roc_curve(labels, max_probs) } roc = sk_metrics.roc_curve(labels, max_probs) with open("intermountain_roc.pkl", 'wb') as f: pickle.dump(roc, f) for name, curve in curves.items(): curve_np = util.get_plot(name, curve) curve_img = Image.fromarray(curve_np) curve_img.save(os.path.join(args.results_dir, '{}.png'.format(name)))
def calibrate(args): """Run model testing.""" model_args = args.model_args data_args = args.data_args logger_args = args.logger_args # Get logger. logger = Logger(logger_args.log_path, logger_args.save_dir, logger_args.results_dir) # Get image paths corresponding to predictions for logging paths = None if model_args.config_path is not None: # Instantiate the EnsemblePredictor class for obtaining # model predictions. predictor = EnsemblePredictor(config_path=model_args.config_path, model_args=model_args, data_args=data_args, gpu_ids=args.gpu_ids, device=args.device, logger=logger) # Obtain ensemble predictions. # Caches both individual and ensemble predictions. # We always turn off caching to ensure that we write the Path column. predictions, groundtruth, paths = predictor.predict(cache=False, return_paths=True, all_gt_tasks=True) else: # Load the model at ckpt_path. ckpt_path = model_args.ckpt_path ckpt_save_dir = Path(ckpt_path).parent model_uncertainty = model_args.model_uncertainty # Get model args from checkpoint and add them to # command-line specified model args. model_args, transform_args\ = ModelSaver.get_args(cl_model_args=model_args, dataset=data_args.dataset, ckpt_save_dir=ckpt_save_dir, model_uncertainty=model_uncertainty) model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=args.gpu_ids, model_args=model_args, is_training=False) # Instantiate the Predictor class for obtaining model predictions. predictor = Predictor(model=model, device=args.device) # Get phase loader object. return_info_dict = True loader = get_loader(phase=data_args.phase, data_args=data_args, transform_args=transform_args, is_training=False, return_info_dict=return_info_dict, logger=logger) # Obtain model predictions if return_info_dict: predictions, groundtruth, paths = predictor.predict(loader) else: predictions, groundtruth = predictor.predict(loader) #print(groundtruth) # custom function from sklearn.linear_model import LogisticRegression as LR params = [] for column in predictions: #print(predictions[column].values) #print(groundtruth[column].values) #drop corresponding rows where gt is -1 and lr = LR(C=15) to_drop = groundtruth.index[groundtruth[column] == -1].tolist() lr.fit(predictions[column].drop(to_drop).values.reshape(-1, 1), groundtruth[column].drop( to_drop).values) # LR needs X to be 2-dimensional print("num_rows_used", predictions[column].drop(to_drop).values.size) #print(groundtruth[column].drop(to_drop).values.size) #print(predictions[column].values) print("coeffs", lr.coef_, lr.intercept_) p_calibrated = lr.predict_proba(predictions[column].values.reshape( -1, 1)) params.append((lr.coef_, lr.intercept_)) import json with open('calibration_params.json', 'w') as f: import pandas as pd pd.Series(params).to_json(f, orient='values')
def test(args): """Run model testing.""" test_args = args.test_args logger_args = args.logger_args # Load the model at ckpt_path. ckpt_path = test_args.ckpt_path ckpt_save_dir = Path(ckpt_path).parent # Get model args from checkpoint and add them to # command-line specified model args. model_args, data_args, optim_args, logger_args\ = ModelSaver.get_args(cl_logger_args=logger_args, ckpt_save_dir=ckpt_save_dir) model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=args.gpu_ids, model_args=model_args, is_training=False) # Get logger. logger = Logger(logger_args=logger_args, data_args=data_args, optim_args=optim_args, test_args=test_args) # Instantiate the Predictor class for obtaining model predictions. predictor = Predictor(model=model, device=args.device) phase = test_args.phase is_test = False if phase == 'test': is_test = True phase = 'valid' # Run valid first to get threshold print(f"======================{phase}=======================") # Get phase loader object. loader = get_loader(phase=phase, data_args=data_args, is_training=False, logger=logger) # Obtain model predictions. predictions, groundtruth = predictor.predict(loader) # Instantiate the evaluator class for evaluating models. evaluator = Evaluator(logger=logger, tune_threshold=True) # Get model metrics and curves on the phase dataset. metrics = evaluator.evaluate(groundtruth, predictions) # Log metrics to stdout and file. logger.log_stdout(f"Writing metrics to {logger.metrics_path}.") logger.log_metrics(metrics, phase=phase) # Evaluate dense to get back thresholds dense_loader = get_loader(phase=phase, data_args=data_args, is_training=False, logger=logger) dense_predictions, dense_groundtruth = predictor.predict(dense_loader) dense_metrics = evaluator.dense_evaluate(dense_groundtruth, dense_predictions) # Log metrics to stdout and file. logger.log_stdout(f"Writing metrics to {logger.metrics_path}.") logger.log_metrics(dense_metrics, phase=phase) if is_test: phase = 'test' threshold = metrics['threshold'] print(f"======================{phase}=======================") # Get phase loader object. loader = get_loader(phase=phase, data_args=data_args, is_training=False, test_args=test_args, logger=logger) # Obtain model predictions. predictions, groundtruth = predictor.predict(loader) # Instantiate the evaluator class for evaluating models. evaluator = Evaluator(logger=logger, threshold=threshold, tune_threshold=False) # Get model metrics and curves on the phase dataset. metrics = evaluator.evaluate(groundtruth, predictions) # Log metrics to stdout and file. logger.log_stdout(f"Writing metrics to {logger.metrics_path}.") logger.log_metrics(metrics, phase=phase) # Dense test phase = 'dense_test' dense_loader = get_loader(phase=phase, data_args=data_args, is_training=False, test_args=test_args, logger=logger) threshold_dense = dense_metrics["threshold_dense"] threshold_tunef1_dense = dense_metrics["threshold_tunef1_dense"] dense_predictions, dense_groundtruth = predictor.predict(dense_loader) dense_metrics = evaluator.dense_evaluate( dense_groundtruth, dense_predictions, threshold=threshold_dense, threshold_tunef1=threshold_tunef1_dense) logger.log_stdout(f"Writing metrics to {logger.metrics_path}.") logger.log_metrics(dense_metrics, phase=phase)
def test(args): print ("Stage 1") model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) print ("Stage 2") args.start_epoch = ckpt_info['epoch'] + 1 model = model.to(args.device) print ("Stage 3") model.eval() print ('This should be false: {}'.format(model.training)) print ("Stage 4") data_loader = CTDataLoader(args, phase=args.phase, is_training=False) #print('data_loader={}'.format(data_loader)) #print('data_loader.dataset={}'.format(data_loader.dataset)) study2slices = defaultdict(list) study2probs = defaultdict(list) study2labels = {} logger = TestLogger(args, len(data_loader.dataset), data_loader.dataset.pixel_dict) print("Stage 5") f = open('/projectnb/ece601/kaggle-pulmonary-embolism/meganmp/train/series_list.pkl','rb') data_labels = pickle.load(f) # Create list to manually process labels #with open('positive.txt') as f: #pos_labels = f.readlines() #pos_labels = [x.strip() for x in pos_labels] ispos = [x.is_positive for x in data_labels] isposidx = [x.study_num for x in data_labels] label_dict = {} for i in range(len(ispos)): label_dict[isposidx[i]] = ispos[i] for key in label_dict.keys(): print('label_dict={}\t{}'.format(key, label_dict[key])) # Get model outputs, log to TensorBoard, write masks to disk window-by-window util.print_err('Writing model outputs to {}...'.format(args.results_dir)) with tqdm(total=len(data_loader.dataset), unit=' windows') as progress_bar: for i, (inputs, targets_dict) in enumerate(data_loader): with torch.no_grad(): cls_logits = model.forward(inputs.to(args.device)) cls_probs = torch.sigmoid(cls_logits) if args.visualize_all: logger.visualize(inputs, cls_logits, targets_dict=None, phase=args.phase, unique_id=i) max_probs = cls_probs.to('cpu').numpy() for study_num, slice_idx, prob in \ zip(targets_dict['study_num'], targets_dict['slice_idx'], list(max_probs)): #print('targets_dict[studynum]={}'.format(targets_dict['study_num'])) #print('targets_dict[sliceidx]={}'.format(targets_dict['slice_idx'])) # Convert to standard python data types study_num = study_num #.item() #study_num = int(study_num) slice_idx = int(slice_idx) # Save series num for aggregation study2slices[study_num].append(slice_idx) study2probs[study_num].append(prob.item()) series = data_loader.get_series(study_num) if study_num not in study2labels: print('study_num={}'.format(study_num)) print('series.is_positive={}'.format(label_dict[study_num])) study2labels[study_num] = label_dict[study_num] #if study_num in pos_labels: #print('DEBUG -------=1?-------------------') #print('POS LABEL') #print('study_num={}'.format(study_num)) #study2labels[study_num] = 1 #else: #print('Not in study2labels. series = {}'.format(study_num)) #print('series.is_positive={}'.format(series.is_positive)) #study2labels[study_num] = int(series.is_positive) #print('study2labels: {}'.format(study2labels[study_num])) progress_bar.update(inputs.size(0)) print('study2labels={}'.format(study2labels)) # Combine masks util.print_err('Combining masks...') max_probs = [] labels = [] predictions = {} print("Get max prob") for study_num in tqdm(study2slices): # Sort by slice index and get max probability slice_list, prob_list = (list(t) for t in zip(*sorted(zip(study2slices[study_num], study2probs[study_num]), key=lambda slice_and_prob: slice_and_prob[0]))) study2slices[study_num] = slice_list study2probs[study_num] = prob_list max_prob = max(prob_list) print('study={}\tmax_prob={}'.format(study_num, max_prob)) max_probs.append(max_prob) label = study2labels[study_num] labels.append(label) predictions[study_num] = {'label':label, 'pred':max_prob} #Save predictions to file, indexed by study number print("Saving predictions to pickle files") with open('{}/preds.pickle'.format(args.results_dir),"wb") as fp: pickle.dump(predictions,fp) results_series = [k for k,_ in predictions.items()] results_pred = [v['pred'] for _,v in predictions.items()] results_label = [v['label'] for _,v in predictions.items()] print('roc_auc_score={}'.format(roc_auc_score(results_label, results_pred))) # Create dataframe summary TRAIN_CSV = '/projectnb/ece601/kaggle-pulmonary-embolism/rsna-str-pulmonary-embolism-detection/train.csv' train_df = pd.read_csv(TRAIN_CSV) train_df = train_df[['SeriesInstanceUID', 'negative_exam_for_pe']] train_df = train_df.groupby('SeriesInstanceUID').aggregate(list) train_df['pe_label'] = train_df['negative_exam_for_pe'].apply(lambda x: 0 if 1 in x else 1) results_dict = { 'series': results_series, 'pred': results_pred } results_df = pd.DataFrame.from_dict(results_dict) results_df = results_df.set_index('series') results_df = results_df.join(train_df, how='left').reset_index().rename({'index': 'series'}) print('roc_auc_score={}'.format(roc_auc_score(results_df['pe_label'], results_df['pred']))) # Calculate confusion matrix results_df['interpretation'] = results_df['pred'].apply(lambda x: 0 if x < 0.5 else 1) print(results_df.head(10)) tn, fp, fn, tp = confusion_matrix(results_df['pe_label'], results_df['interpretation']).ravel() print('confusion_matrix: [{} {} {} {}]'.format(tp, fp, fn, tn))
def test(args): """Run testing with the given args. The function consists of the following steps: 1. Get model for evaluation. 2. Get task sequence and class weights. 3. Get data eval loaders and evaluator. 4. Evaluate and save model performance (metrics and curves). """ model_args = args.model_args logger_args = args.logger_args data_args = args.data_args transform_args = args.transform_args # Get model if args.use_multi_model: model = load_multi_model(args.multi, model_args, data_args, args.gpu_ids) elif model_args.use_csv_probs: model = CSVReaderModel(model_args.ckpt_path, TASK_SEQUENCES[data_args.task_sequence]) ckpt_info = {'epoch': 0} elif args.config_path is not None: task2models, aggregation_fn = get_config(args.config_path) model = EnsembleModel(task2models, aggregation_fn, args.gpu_ids, model_args, data_args) ckpt_info = {'epoch': 0} else: model, ckpt_info = ModelSaver.load_model(model_args.ckpt_path, args.gpu_ids, model_args, data_args) model = model.to(args.device) model.eval() # Get the task sequence that the model outputs. # Newer models have an attribute called 'task_sequence'. # For older models we need to specify what # task sequence was used. if hasattr(model.module, 'task_sequence'): task_sequence = model.module.task_sequence else: task_sequence = TASK_SEQUENCES[data_args.task_sequence] print( f'WARNING: assuming that the models task sequence is \n {task_sequence}' ) # Get train loader in order to get the class weights train_csv_name = 'train' if data_args.uncertain_map_path is not None: train_csv_name = data_args.uncertain_map_path ''' loader = get_loader(data_args, args.transform_args, train_csv_name, task_sequence, su_frac = 1 if data_args.eval_su else 0, nih_frac = 1 if data_args.eval_nih else 0, pocus_frac = 1 if data_args.eval_pocus else 0, tcga_frac = 1 if data_args.eval_tcga else 0, batch_size=args.batch_size) ''' class_weights = [[0.5], [0.5]] #loader.dataset.class_weights ''' # Get eval loaders and radiologist performance eval_loader = get_eval_loaders(data_args, transform_args, task_sequence, args.batch_size, frontal_lateral=model_args.frontal_lateral, return_info_dict=model_args.use_csv_probs or logger_args.save_cams)[-1] # Evaluate only on valid ''' rad_perf = pd.read_csv( data_args.su_rad_perf_path ) if data_args.su_rad_perf_path is not None else None eval_loader = get_loader( data_args, transform_args, data_args.split, task_sequence, su_frac=1 if data_args.eval_su else 0, nih_frac=1 if data_args.eval_nih else 0, pocus_frac=1 if data_args.eval_pocus else 0, tcga_frac=1 if data_args.eval_tcga else 0, tcga_google_frac=1 if data_args.eval_tcga_google else 0, tcga_stanford_frac=1 if data_args.eval_tcga_stanford else 0, batch_size=args.batch_size, normalize=model_args.normalize, study_level=not model_args.frontal_lateral, frontal_lateral=model_args.frontal_lateral, return_info_dict=model_args.use_csv_probs or logger_args.save_cams) results_dir = os.path.join(logger_args.results_dir, data_args.split) visuals_dir = Path(results_dir) / 'visuals' if args.config_path is None: # Get evaluator eval_args = {} eval_args['num_visuals'] = None eval_args['iters_per_eval'] = None eval_args['has_missing_tasks'] = args.has_tasks_missing eval_args['model_uncertainty'] = model_args.model_uncertainty eval_args['class_weights'] = class_weights eval_args['max_eval'] = None eval_args['device'] = args.device evaluator = get_evaluator('classification', [eval_loader], None, eval_args) metrics, curves = evaluator.evaluate(model, args.device) print(metrics) # TODO: Generalize the plot function. Remove hard-coded values. # Plot Results # print(f"Plotting to {visuals_dir}") # TODO: uncomment once we have curves to plot # plot(curves, metrics, visuals_dir, rad_perf) eval_metrics = [ 'AUPRC', 'AUROC', 'log_loss', 'rads_below_ROC', 'rads_below_PR' ] if logger_args.write_results: results_path = os.path.join(results_dir, f'scores.csv') write_results(data_args.dataset_name, data_args.split, eval_metrics, metrics, results_path, logger_args.name, ckpt_info) # Save visuals if logger_args.save_cams: cams_dir = visuals_dir / 'cams' save_grad_cams(args, eval_loader, model, cams_dir, only_competition=logger_args.only_competition_cams, only_top_task=False)
def train(args): """Run model training.""" print("Start Training ...") # Get nested namespaces. model_args = args.model_args logger_args = args.logger_args optim_args = args.optim_args data_args = args.data_args transform_args = args.transform_args # Get logger. print('Getting logger... log to path: {}'.format(logger_args.log_path)) logger = Logger(logger_args.log_path, logger_args.save_dir) # For conaug, point to the MOCO pretrained weights. if model_args.ckpt_path and model_args.ckpt_path != 'None': print("pretrained checkpoint specified : {}".format( model_args.ckpt_path)) # CL-specified args are used to load the model, rather than the # ones saved to args.json. model_args.pretrained = False ckpt_path = model_args.ckpt_path model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=args.gpu_ids, model_args=model_args, is_training=True) if not model_args.moco: optim_args.start_epoch = ckpt_info['epoch'] + 1 else: optim_args.start_epoch = 1 else: print( 'Starting without pretrained training checkpoint, random initialization.' ) # If no ckpt_path is provided, instantiate a new randomly # initialized model. model_fn = models.__dict__[model_args.model] if data_args.custom_tasks is not None: tasks = NamedTasks[data_args.custom_tasks] else: tasks = model_args.__dict__[TASKS] # TASKS = "tasks" print("Tasks: {}".format(tasks)) model = model_fn(tasks, model_args) model = nn.DataParallel(model, args.gpu_ids) # Put model on gpu or cpu and put into training mode. model = model.to(args.device) model.train() print("========= MODEL ==========") print(model) # Get train and valid loader objects. train_loader = get_loader(phase="train", data_args=data_args, transform_args=transform_args, is_training=True, return_info_dict=False, logger=logger) valid_loader = get_loader(phase="valid", data_args=data_args, transform_args=transform_args, is_training=False, return_info_dict=False, logger=logger) # Instantiate the predictor class for obtaining model predictions. predictor = Predictor(model, args.device) # Instantiate the evaluator class for evaluating models. evaluator = Evaluator(logger) # Get the set of tasks which will be used for saving models # and annealing learning rate. eval_tasks = EVAL_METRIC2TASKS[optim_args.metric_name] # Instantiate the saver class for saving model checkpoints. saver = ModelSaver(save_dir=logger_args.save_dir, iters_per_save=logger_args.iters_per_save, max_ckpts=logger_args.max_ckpts, metric_name=optim_args.metric_name, maximize_metric=optim_args.maximize_metric, keep_topk=logger_args.keep_topk) # TODO: JBY: handle threshold for fine tuning if model_args.fine_tuning == 'full': # Fine tune all layers. pass else: # Freeze other layers. models.PretrainedModel.set_require_grad_for_fine_tuning( model, model_args.fine_tuning.split(',')) # Instantiate the optimizer class for guiding model training. optimizer = Optimizer(parameters=model.parameters(), optim_args=optim_args, batch_size=data_args.batch_size, iters_per_print=logger_args.iters_per_print, iters_per_visual=logger_args.iters_per_visual, iters_per_eval=logger_args.iters_per_eval, dataset_len=len(train_loader.dataset), logger=logger) if model_args.ckpt_path and not model_args.moco: # Load the same optimizer as used in the original training. optimizer.load_optimizer(ckpt_path=model_args.ckpt_path, gpu_ids=args.gpu_ids) model_uncertainty = model_args.model_uncertainty loss_fn = evaluator.get_loss_fn( loss_fn_name=optim_args.loss_fn, model_uncertainty=model_args.model_uncertainty, mask_uncertain=True, device=args.device) # Run training while not optimizer.is_finished_training(): optimizer.start_epoch() # TODO: JBY, HACK WARNING # What is the hack? metrics = None for inputs, targets in train_loader: optimizer.start_iter() if optimizer.global_step and optimizer.global_step % optimizer.iters_per_eval == 0 or len( train_loader.dataset ) - optimizer.iter < optimizer.batch_size: # Only evaluate every iters_per_eval examples. predictions, groundtruth = predictor.predict(valid_loader) # print("predictions: {}".format(predictions)) metrics, curves = evaluator.evaluate_tasks( groundtruth, predictions) # Log metrics to stdout. logger.log_metrics(metrics) # Add logger for all the metrics for valid_loader logger.log_scalars(metrics, optimizer.global_step) # Get the metric used to save model checkpoints. average_metric = evaluator.evaluate_average_metric( metrics, eval_tasks, optim_args.metric_name) if optimizer.global_step % logger_args.iters_per_save == 0: # Only save every iters_per_save examples directly # after evaluation. print("Save global step: {}".format(optimizer.global_step)) saver.save(iteration=optimizer.global_step, epoch=optimizer.epoch, model=model, optimizer=optimizer, device=args.device, metric_val=average_metric) # Step learning rate scheduler. optimizer.step_scheduler(average_metric) with torch.set_grad_enabled(True): logits, embedding = model(inputs.to(args.device)) loss = loss_fn(logits, targets.to(args.device)) optimizer.log_iter(inputs, logits, targets, loss) optimizer.zero_grad() loss.backward() optimizer.step() optimizer.end_iter() optimizer.end_epoch(metrics) logger.log('=== Training Complete ===')
def predict(self, cache=False): """Get model predictions on the evaluation set. Args: cache: Bool indicating whether to cache ensemble predictions. If true, first tries to load already cached files, then writes all predictions (and groundtruth) which have not been cached. Return: ensemble probabilities Pandas DataFrame, ensemble groundtruth Pandas DataFrame """ is_cached = False if cache and self.logger is not None: results_dir = self.logger.results_dir self.predictions_path = results_dir / "ensemble_predictions.csv" self.groundtruth_path = results_dir / "groundtruth.csv" if (self.predictions_path.exists() and self.groundtruth_path.exists()): self.logger.log(f"Predictions at {self.predictions_path} " + "already exist. Loading from this file.") ensemble_probs_df = pd.read_csv(self.predictions_path) ensemble_gt_df = pd.read_csv(self.groundtruth_path) is_cached = True elif cache: raise ValueError("Must instantiate Predictor with logger" + "if caching.") if not is_cached: model2probs = {} model2gt = {} task2ensemble_probs = {} task2gt = {} self.save_config() for task, model_dicts in self.task2models.items(): for model_dict in model_dicts: ckpt_path = Path(model_dict[CFG_CKPT_PATH]) is_3class = model_dict[CFG_IS_3CLASS] if (ckpt_path in model2probs): # We've already computed predictions for this model, # skip it! continue ckpt_save_dir = Path(ckpt_path).parent results_parent_dir = ckpt_save_dir / "results" results_dir = results_parent_dir / self.data_args.phase results_dir.mkdir(parents=True, exist_ok=True) ckpt_iter = ckpt_path.stem.split(".")[0] predictions_name = f"{ckpt_iter}-predictions.csv" groundtruth_name = f"{ckpt_iter}-groundtruth.csv" predictions_path = results_dir / predictions_name groundtruth_path = results_dir / groundtruth_name if cache and (predictions_path.exists() and groundtruth_path.exists()): self.logger.log(f"Predictions at {predictions_path}" + " already exist. Loading from this " + "file.") probs_df = pd.read_csv(predictions_path, dtype=np.float32) gt_df = pd.read_csv(groundtruth_path, dtype=np.float32) else: dataset = self.data_args.dataset # Get model args from checkpoint and add them to # command-line specified model args. model_args, transform_args =\ ModelSaver.get_args(cl_model_args=self.model_args, dataset=dataset, ckpt_save_dir=ckpt_save_dir, model_uncertainty=is_3class) model, ckpt_info =\ ModelSaver.load_model(ckpt_path=ckpt_path, gpu_ids=self.gpu_ids, model_args=model_args, is_training=False) predictor = Predictor(model=model, device=self.device) loader = get_loader(phase=self.data_args.phase, data_args=self.data_args, transform_args=transform_args, is_training=False, return_info_dict=False, logger=self.logger) probs_df, gt_df = predictor.predict(loader) if cache: self.logger.log("Writing predictions to " + f"{predictions_path}.") probs_df.astype(np.float64).to_csv( predictions_path, index=False) self.logger.log("Writing groundtruth to " + f"{groundtruth_path}.") gt_df.astype(np.float64).to_csv(groundtruth_path, index=False) model2probs[ckpt_path] = probs_df model2gt[ckpt_path] = gt_df task_ckpt_probs =\ [model2probs[Path(model_dict[CFG_CKPT_PATH])][task] for model_dict in model_dicts] task2ensemble_probs[task] =\ self.aggregation_fn(task_ckpt_probs, axis=0) first_gt = model2gt[Path(model_dicts[0][CFG_CKPT_PATH])][task] assert all([ model2gt[Path( model_dict[CFG_CKPT_PATH])][task].equals(first_gt) for model_dict in model_dicts ]) task2gt[task] =\ model2gt[Path(model_dicts[0][CFG_CKPT_PATH])][task] ensemble_probs_df = pd.DataFrame( {task: task2ensemble_probs[task] for task in self.task2models}) ensemble_gt_df = pd.DataFrame( {task: task2gt[task] for task in self.task2models}) if cache: self.logger.log("Writing predictions to " f"{self.predictions_path}.") ensemble_probs_df.astype(np.float64).to_csv( self.predictions_path, index=False) self.logger.log("Writing groundtruth to " f"{self.groundtruth_path}.") ensemble_gt_df.astype(np.float64).to_csv(self.groundtruth_path, index=False) return ensemble_probs_df, ensemble_gt_df
def train(args): if args.ckpt_path and not args.use_pretrained: model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids) args.start_epoch = ckpt_info['epoch'] + 1 else: model_fn = models.__dict__[args.model] model = model_fn(**vars(args)) if args.use_pretrained: model.load_pretrained(args.ckpt_path, args.gpu_ids) model = nn.DataParallel(model, args.gpu_ids) model = model.to(args.device) model.train() # Get optimizer and scheduler if args.use_pretrained or args.fine_tune: parameters = model.module.fine_tuning_parameters( args.fine_tuning_boundary, args.fine_tuning_lr) else: parameters = model.parameters() optimizer = util.get_optimizer(parameters, args) lr_scheduler = util.get_scheduler(optimizer, args) if args.ckpt_path and not args.use_pretrained and not args.fine_tune: ModelSaver.load_optimizer(args.ckpt_path, optimizer, lr_scheduler) # Get logger, evaluator, saver cls_loss_fn = util.get_loss_fn(is_classification=True, dataset=args.dataset, size_average=False) data_loader_fn = data_loader.__dict__[args.data_loader] train_loader = data_loader_fn(args, phase='train', is_training=True) logger = TrainLogger(args, len(train_loader.dataset), train_loader.dataset.pixel_dict) eval_loaders = [data_loader_fn(args, phase='val', is_training=False)] evaluator = ModelEvaluator(args.do_classify, args.dataset, eval_loaders, logger, args.agg_method, args.num_visuals, args.max_eval, args.epochs_per_eval) saver = ModelSaver(args.save_dir, args.epochs_per_save, args.max_ckpts, args.best_ckpt_metric, args.maximize_metric) # Train model while not logger.is_finished_training(): logger.start_epoch() for inputs, target_dict in train_loader: logger.start_iter() with torch.set_grad_enabled(True): inputs.to(args.device) cls_logits = model.forward(inputs) cls_targets = target_dict['is_abnormal'] cls_loss = cls_loss_fn(cls_logits, cls_targets.to(args.device)) loss = cls_loss.mean() logger.log_iter(inputs, cls_logits, target_dict, cls_loss.mean(), optimizer) optimizer.zero_grad() loss.backward() optimizer.step() logger.end_iter() util.step_scheduler(lr_scheduler, global_step=logger.global_step) metrics, curves = evaluator.evaluate(model, args.device, logger.epoch) saver.save(logger.epoch, model, optimizer, lr_scheduler, args.device, metric_val=metrics.get(args.best_ckpt_metric, None)) logger.end_epoch(metrics, curves) util.step_scheduler(lr_scheduler, metrics, epoch=logger.epoch, best_ckpt_metric=args.best_ckpt_metric)
def test(args): """Run testing with the given args. The function consists of the following steps: 1. Get model for evaluation. 2. Get task sequence and class weights. 3. Get data eval loaders and evaluator. 4. Evaluate and save model performance (metrics and curves). """ model_args = args.model_args logger_args = args.logger_args data_args = args.data_args transform_args = args.transform_args # Get model if args.use_multi_model: model = load_multi_model(args.multi, model_args, data_args, args.gpu_ids) elif model_args.use_csv_probs: model = CSVReaderModel(model_args.ckpt_path, TASK_SEQUENCES[data_args.task_sequence]) ckpt_info = {'epoch': 0} elif args.config_path is not None: task2models, aggregation_fn = get_config(args.config_path) model = EnsembleModel(task2models, aggregation_fn, args.gpu_ids, model_args, data_args) ckpt_info = {'epoch': 0} elif model_args.ckpt_paths: model, ckpt_info = ModelSaver.load_ensemble(model_args.ckpt_paths, args.gpu_ids, model_args, data_args) else: model_args.pretrained = False model, ckpt_info = ModelSaver.load_model(model_args.ckpt_path, args.gpu_ids, model_args, data_args) model = model.to(args.device) model.eval() # Get the task sequence that the model outputs. # Newer models have an attribute called 'task_sequence'. # For older models we need to specify what # task sequence was used. # if hasattr(model.module, 'task_sequence'): # task_sequence = model.module.task_sequence # else: # task_sequence = TASK_SEQUENCES[data_args.task_sequence] # print(f'WARNING: assuming that the models task sequence is \n {task_sequence}') task_sequence = TASK_SEQUENCES[data_args.task_sequence] cxr_frac = { 'pocus': data_args.eval_pocus, 'hocus': data_args.eval_hocus, 'pulm': data_args.eval_pulm } # Get train loader in order to get the class weights train_csv_name = 'train' if data_args.uncertain_map_path is not None: train_csv_name = data_args.uncertain_map_path loader = get_loader(data_args, args.transform_args, train_csv_name, task_sequence, su_frac=1 if data_args.eval_su else 0, nih_frac=1 if data_args.eval_nih else 0, cxr_frac=cxr_frac, tcga_frac=1 if data_args.eval_tcga else 0, batch_size=args.batch_size, covar_list=model_args.covar_list, fold_num=data_args.fold_num) class_weights = loader.dataset.class_weights # Get eval loaders and radiologist performance eval_loader = get_eval_loaders( data_args, transform_args, task_sequence, args.batch_size, frontal_lateral=model_args.frontal_lateral, return_info_dict=model_args.use_csv_probs or logger_args.save_cams, covar_list=model_args.covar_list, fold_num=data_args.fold_num)[-1] # Evaluate only on valid rad_perf = pd.read_csv( data_args.su_rad_perf_path ) if data_args.su_rad_perf_path is not None else None if data_args.split != 'valid': eval_loader = get_loader(data_args, args.transform_args, data_args.split, task_sequence, su_frac=1 if data_args.eval_su else 0, nih_frac=1 if data_args.eval_nih else 0, cxr_frac=cxr_frac, tcga_frac=1 if data_args.eval_tcga else 0, batch_size=args.batch_size, covar_list=model_args.covar_list, fold_num=data_args.fold_num) results_dir = os.path.join(logger_args.results_dir, data_args.split) os.makedirs(results_dir, exist_ok=True) write_model_paths(results_dir, model_args.ckpt_path, model_args.ckpt_paths) visuals_dir = Path(results_dir) / 'visuals' if args.config_path is None and not model_args.ckpt_paths: # Get evaluator eval_args = {} eval_args['num_visuals'] = None eval_args['iters_per_eval'] = None eval_args['has_missing_tasks'] = args.has_tasks_missing eval_args['model_uncertainty'] = model_args.model_uncertainty eval_args['class_weights'] = class_weights eval_args['max_eval'] = None eval_args['device'] = args.device eval_args['optimizer'] = None evaluator = get_evaluator('classification', [eval_loader], None, eval_args) metrics, curves = evaluator.evaluate(model, args.device, results_dir=results_dir, report_probabilities=True) # TODO: Generalize the plot function. Remove hard-coded values. # plot(curves, metrics, visuals_dir, rad_perf) eval_metrics = [ 'AUPRC', 'AUROC', 'log_loss', 'rads_below_ROC', 'rads_below_PR', 'accuracy' ] if logger_args.write_results: results_path = os.path.join(results_dir, f'scores.csv') evaluate_task_sequence = 'competition' if data_args.dataset_name == 'stanford' else data_args.task_sequence write_results(data_args.dataset_name, data_args.split, eval_metrics, metrics, results_path, logger_args.name, ckpt_info, evaluate_task_sequence) # Save visuals if logger_args.save_cams: cams_dir = visuals_dir / 'cams' save_grad_cams(args, eval_loader, model, cams_dir, only_competition=logger_args.only_competition_cams, only_top_task=False, probabilities_csv=logger_args.probabilities_csv)
ckpt_path = 'penet_best.pth.tar' device = 'cuda' gpu_ids = 1 #map_location = 'cpu' print("Reading input dicom...") study = util.dicom_2_npy(input_study) print('is study empty') print(study) # normalize and convert to tensor print("Formatting input for model...") study_windows = util.format_img(study) print("is study window empty") print(study_windows) print ("Loading saved model...") model, ckpt_info = ModelSaver.load_model(ckpt_path, [0]) print ("Sending model to GPU device...") #start_epoch = ckpt_info['epoch'] + 1 model = model.to(device) print ("Evaluating study...") model.eval() predicted_probabilities = [] # window wise for a single study with torch.no_grad(): for window in study_windows: cls_logits = model.forward(window.to(device, dtype=torch.float)) cls_probs = torch.sigmoid(cls_logits).to('cpu').numpy() predicted_probabilities.append(cls_probs[0][0])
def test(args): data_args = args.data_args model_args = args.model_args logger_args = args.logger_args transform_args = args.transform_args # Get model model, ckpt_info = ModelSaver.load_model(model_args.ckpt_path, args.gpu_ids, model_args, data_args) model = model.to(args.device) model.eval() # Get the task sequence that the model outputs. # Newer models have an attribute called 'task_sequence'. # For older models we need to specify what # task sequence was used. if hasattr(model.module, 'task_sequence'): task_sequence = model.module.task_sequence else: task_sequence = TASK_SEQUENCES[data_args.task_sequence] print( f'WARNING: assuming that the models task sequence is \n {task_sequence}' ) # Get eval loader loader = get_loader(data_args, transform_args, data_args.split, task_sequence, su_frac=1, nih_frac=0, batch_size=args.batch_size, is_training=False, shuffle=False, study_level=not model_args.frontal_lateral, frontal_lateral=model_args.frontal_lateral, return_info_dict=True) if model_args.frontal_lateral: task_sequence = ["Frontal/Lateral"] # Sample from the data loader and record model outputs csv_rows = [] num_evaluated = 0 num_examples = len(loader.dataset) with tqdm(total=num_examples, unit=' ' + data_args.split + ' ' + data_args.dataset_name) as progress_bar: for data in loader: if num_evaluated >= num_examples: break with torch.no_grad(): if loader.dataset.study_level: inputs, targets, mask, info_dict = data # Fuse batch size `b` and study length `s` b, s, c, h, w = inputs.size() inputs = inputs.view(-1, c, h, w) # Predict logits = model.forward(inputs.to(args.device)) logits = logits.view(b, s, -1) # Mask padding to negative infinity ignore_where = (mask == 0).unsqueeze(-1).repeat( 1, 1, logits.size(-1)).to(args.device) logits = torch.where(ignore_where, torch.full_like(logits, NEG_INF), logits) logits, _ = torch.max(logits, 1) else: inputs, targets, info_dict = data logits = model.forward(inputs.to(args.device)) # Save study path and probabilities for each example if model_args.model_uncertainty: batch_probs = util.uncertain_logits_to_probs(logits) else: batch_probs = torch.sigmoid(logits) study_paths = info_dict['paths'] for probs, path in zip(batch_probs, study_paths): csv_row = {COL_PATH: path, COL_SPLIT: data_args.split} csv_row.update({ task: prob.item() for task, prob in zip(task_sequence, probs) }) csv_rows.append(csv_row) progress_bar.update(targets.size(0)) num_evaluated += targets.size(0) # Write CSV file to disk df = pd.DataFrame(csv_rows) if model_args.frontal_lateral: int2str = {0: "Frontal", 1: "Lateral"} df["Frontal/Lateral"] = df["Frontal/Lateral"].apply( lambda x: int2str[int(x > 0.5)]) csv_cols = [COL_PATH] + [task for task in task_sequence] output_path = os.path.join(logger_args.results_dir, logger_args.output_csv_name) print('Saving outputs to: {}'.format(output_path)) df[csv_cols].to_csv(output_path, index=False)