def train(args):
# Get model
model = models.__dict__[args.model](args)
if args.ckpt_path:
model = ModelSaver.load_model(model, args.ckpt_path, args.gpu_ids, is_training=True)
model = model.to(args.device)
model.train()
# Get loader, logger, and saver
train_loader, val_loader = get_data_loaders(args)
logger = TrainLogger(args, model, dataset_len=len(train_loader.dataset))
saver = ModelSaver(args.save_dir, args.max_ckpts, metric_name=args.metric_name,
maximize_metric=args.maximize_metric, keep_topk=True)
# Train
while not logger.is_finished_training():
logger.start_epoch()
for batch in train_loader:
logger.start_iter()
# Train over one batch
model.set_inputs(batch['src'], batch['tgt'])
model.train_iter()
logger.end_iter()
# Evaluate
if logger.global_step % args.iters_per_eval < args.batch_size:
criteria = {'MSE_src2tgt': mse, 'MSE_tgt2src': mse}
stats = evaluate(model, val_loader, criteria)
logger.log_scalars({'val_' + k: v for k, v in stats.items()})
saver.save(logger.global_step, model,
stats[args.metric_name], args.device)
logger.end_epoch()
def test(args):
print(args.save_dir)
model_args, data_args = load_args(Path(args.save_dir))
assert not model_args.modelfree, "Code only evaluates on model based models"
saver = ModelSaver(Path(args.save_dir), None)
power_constraint = PowerConstraint()
possible_inputs = get_md_set(model_args.md_len)
# TODO: change to batch size and batch per epoch to 1000
data_args.batch_size = 5000
data_args.batches_per_epoch = 1000
dataset_size = data_args.batch_size * data_args.batches_per_epoch
loader = InputDataloader(data_args.batch_size, data_args.block_length,
dataset_size)
loader = loader.example_generator()
SNRs = [.5, 1, 2, 3, 4]
BER = []
loss = []
for SNR in SNRs:
print(f"Testing {SNR} SNR level")
data_args.SNR = SNR
accuracy = []
losses = []
print(data_args.channel)
print(model_args.modelfree)
channel = get_channel(data_args.channel, model_args.modelfree,
data_args)
model = AutoEncoder(model_args, data_args, power_constraint, channel,
possible_inputs)
saver.load(model)
for step in tqdm(range(data_args.batches_per_epoch)):
msg = next(loader)
metrics = model.trainable_encoder.test_on_batch(msg, msg)
losses.append(metrics[0])
accuracy.append(metrics[1])
mean_loss = sum(losses) / len(losses)
mean_BER = 1 - sum(accuracy) / len(accuracy)
loss.append(mean_loss)
BER.append(mean_BER)
print(f"mean BER: {mean_BER}")
print(f"mean loss: {mean_loss}")
# create plots for results
plt.plot(SNRs, BER)
plt.ylabel("BER")
plt.xlabel("SNR")
plt.yscale('log')
plt.savefig('figures/AWGN_modelaware.png')
plt.show()
def train(args):
if args.ckpt_path:
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
else:
model_fn = models.__dict__[args.model]
model = model_fn(**vars(args))
model = nn.DataParallel(model, args.gpu_ids)
model = model.to(args.device)
model.train()
# Set up population-based training client
pbt_client = PBTClient(args.pbt_server_url, args.pbt_server_port, args.pbt_server_key, args.pbt_config_path)
# Get optimizer and scheduler
parameters = model.module.parameters()
optimizer = optim.get_optimizer(parameters, args, pbt_client)
ModelSaver.load_optimizer(args.ckpt_path, args.gpu_ids, optimizer)
# Get logger, evaluator, saver
train_loader = DataLoader(args, 'train', is_training_set=True)
eval_loaders = [DataLoader(args, 'valid', is_training_set=False)]
evaluator = ModelEvaluator(eval_loaders, args.epochs_per_eval,
args.max_eval, args.num_visuals, use_ten_crop=args.use_ten_crop)
saver = ModelSaver(**vars(args))
for _ in range(args.num_epochs):
optim.update_hyperparameters(model.module, optimizer, pbt_client.hyperparameters())
for inputs, targets in train_loader:
with torch.set_grad_enabled(True):
logits = model.forward(inputs.to(args.device))
loss = F.binary_cross_entropy_with_logits(logits, targets.to(args.device))
optimizer.zero_grad()
loss.backward()
optimizer.step()
metrics = evaluator.evaluate(model, args.device)
metric_val = metrics.get(args.metric_name, None)
ckpt_path = saver.save(model, args.model, optimizer, args.device, metric_val)
pbt_client.save(ckpt_path, metric_val)
if pbt_client.should_exploit():
# Exploit
pbt_client.exploit()
# Load model and optimizer parameters from exploited network
model, ckpt_info = ModelSaver.load_model(pbt_client.parameters_path(), args.gpu_ids)
model = model.to(args.device)
model.train()
ModelSaver.load_optimizer(pbt_client.parameters_path(), args.gpu_ids, optimizer)
# Explore
pbt_client.explore()
def predict(args):
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
model.eval()
# Predict outputs
data_loader = WhiteboardLoader(args.data_dir, args.phase, args.batch_size,
shuffle=False, do_augment=False, num_workers=args.num_workers)
all_probs, all_paths = [], []
with tqdm(total=len(data_loader.dataset), unit=' ' + args.phase) as progress_bar:
for inputs, targets, paths in data_loader:
bs, n_crops, c, h, w = inputs.size()
inputs = inputs.view(-1, c, h, w) # Fuse batch size and n_crops
with torch.no_grad():
logits = model.forward(inputs.to(args.device))
logits = logits.view(bs, n_crops, -1).mean(1) # Average over n_crops
probs = F.softmax(logits, -1)
# Take probability of whiteboard
all_probs += [p[1] for p in probs]
all_paths += list(paths)
progress_bar.update(targets.size(0))
# Write CSV
record_ids = [os.path.basename(p)[:-4] for p in all_paths] # Convert to record_id
df = pd.DataFrame([{'record_id': r,
'output': '{:.5f}'.format(prob.item()),
'has_whiteboard_@{:.2f}'.format(args.prob_threshold): int(prob > args.prob_threshold),
'url': get_url(r)}
for r, prob in zip(record_ids, all_probs)])
df.to_csv(os.path.join(args.results_dir, 'outputs.csv'), index=False)
def test(args):
model_fn = model_dict[args_.model]
model = model_fn(num_classes=1 if args_.model == 'fd' else 10)
model = nn.DataParallel(model, args.gpu_ids)
ckpt_info = ModelSaver.load_model(args.ckpt_path, model)
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
model.eval()
test_set = FilterDataset('alexnet', './filters', is_training=False)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
logger = TestLogger(args)
logger.start_epoch()
for inputs, labels in test_loader:
logger.start_iter()
with torch.set_grad_enabled(True):
# Forward
logits = model.forward(inputs.to(args.device))
logger.end_iter(inputs, labels, logits)
logger.end_epoch()
def main(args):
# create npy from dicom
print("Reading input dicom...")
study = util.dicom_2_npy(args.input_study, args.series_description)
# normalize and convert to tensor
print("Formatting input for model...")
study_windows = util.format_img(study)
print("Loading saved model...")
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
print("Sending model to GPU device...")
#start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
print("Evaluating study...")
model.eval()
predicted_probabilities = [] # window wise for a single study
with torch.no_grad():
for window in study_windows:
cls_logits = model.forward(
window.to(args.device, dtype=torch.float))
cls_probs = torch.sigmoid(cls_logits).to('cpu').numpy()
predicted_probabilities.append(cls_probs[0][0])
print(
f"Probablity of having Pulmonary Embolism: {max(predicted_probabilities)}"
)
def extract_filters(ckpt_path, model_name):
"""Get numpy array, val_loss from a saved checkpoint."""
model_dict = {
'alexnet': models.alexnet,
'resnet50': models.resnet50,
'vgg19': models.vgg19_bn,
}
model_fn = model_dict[model_name]
model = model_fn(num_classes=10)
model = nn.DataParallel(model)
ckpt_info = ModelSaver.load_model(ckpt_path, model)
filter_dict = {
'alexnet': 'module.features.0',
'resnet50': 'module.conv1',
'vgg19': 'module.features.0'
}
target_layer = filter_dict[model_name]
filters_np = None
for name, module in model.named_modules():
if name == target_layer:
filters_np = module.weight.data.cpu().numpy()
break
if filters_np is None:
raise RuntimeError(
'Could not find filters for layer {}'.format(target_layer))
return filters_np, ckpt_info['epoch'], ckpt_info['val_loss']
def __init__(self,
data_processor,
num_labels,
bert_config_file,
max_seq_length,
vocab_file,
logdir,
init_checkpoint,
keep_checkpoint_max,
use_GPU=False,
label_smoothing=0.0,
cycle=1):
config = tf.ConfigProto(allow_soft_placement=True)
# config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
self.output_dropout_keep_prob = np.array([0.9])
self.hidden_dropout_prob = np.array([0.1])
self.attention_probs_dropout_prob = np.array([0.1])
self.init_checkpoint = init_checkpoint
bert_config = modeling.BertConfig.from_json_file(bert_config_file)
# self.train_op, self.loss, self.logits, self.probabilities, self.feed_dict, self.attention_probs = create_model(
self.train_op, self.loss, self.logits, self.probabilities, self.feed_dict = create_model(
bert_config,
num_labels,
max_seq_length,
self.sess,
init_checkpoint=self.init_checkpoint,
use_GPU=use_GPU,
label_smoothing=label_smoothing,
cycle=cycle)
self.max_seq_length = max_seq_length
self.tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
do_lower_case=True)
if not os.path.exists(logdir):
os.makedirs(logdir)
self.summary_writer = tf.summary.FileWriter(logdir, self.sess.graph)
self.prob_hist = None
self.logits_hist = None
self.eval_iterator = None
self.num_eval_steps = None
self.num_labels = num_labels
self.model_saver = ModelSaver(keep_checkpoint_max=keep_checkpoint_max)
self.data_processor = data_processor
def train(args):
"""Train the model on the dataset."""
Dataset = collections.namedtuple('Dataset', 'X y')
assert args.random or args.csv_path, "Please choose either random data or pass a data path"
if args.random:
train_features = sp.random(100, 100)
train_costs = np.random.random((100, ))
test_features = np.random.random((100, 100))
test_costs = np.random.random((100, ))
else:
train_path = Path(args.train_path)
train_df = pd.read_csv(train_path)
train_features, train_costs = preprocess(train_df, args.sdh)
test_path = Path(args.test_path)
test_df = pd.read_csv(test_path)
test_features, test_costs = preprocess(test_df, args.sdh)
train_dataset = Dataset(train_features, train_costs)
test_dataset = Dataset(test_features, test_costs)
# Load the model.
saver = ModelSaver(args.save_dir)
model = get_model(args)
# Instantiate the model saver.
# Train model on dataset, cross-validating on validation set.
model.fit(train_dataset)
saver.save(model)
preds, targets = predict(model, test_dataset)
metrics = {
"R2-score": r2_score(targets, preds),
"MAE": mean_absolute_error(targets, preds)
}
# Print metrics to stdout.
print(metrics)
def train(args):
if args.ckpt_path:
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
else:
model_fn = models.__dict__[args.model]
model = model_fn(**vars(args))
model = nn.DataParallel(model, args.gpu_ids)
model = model.to(args.device)
model.train()
# Get optimizer and scheduler
optimizer = optim.get_optimizer(
filter(lambda p: p.requires_grad, model.parameters()), args)
lr_scheduler = optim.get_scheduler(optimizer, args)
if args.ckpt_path:
ModelSaver.load_optimizer(args.ckpt_path, optimizer, lr_scheduler)
# Get logger, evaluator, saver
loss_fn = nn.CrossEntropyLoss()
train_loader = CIFARLoader('train', args.batch_size, args.num_workers)
logger = TrainLogger(args, len(train_loader.dataset))
eval_loaders = [CIFARLoader('val', args.batch_size, args.num_workers)]
evaluator = ModelEvaluator(eval_loaders, logger, args.max_eval,
args.epochs_per_eval)
saver = ModelSaver(**vars(args))
# Train model
while not logger.is_finished_training():
logger.start_epoch()
for inputs, targets in train_loader:
logger.start_iter()
with torch.set_grad_enabled(True):
logits = model.forward(inputs.to(args.device))
loss = loss_fn(logits, targets.to(args.device))
logger.log_iter(loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
logger.end_iter()
metrics = evaluator.evaluate(model, args.device, logger.epoch)
saver.save(logger.epoch,
model,
optimizer,
lr_scheduler,
args.device,
metric_val=metrics.get(args.metric_name, None))
logger.end_epoch(metrics)
optim.step_scheduler(lr_scheduler, metrics, logger.epoch)
def train(args):
train_loader = get_loader(args=args)
if args.ckpt_path:
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
else:
model_fn = models.__dict__[args.model]
args.D_in = train_loader.D_in
model = model_fn(**vars(args))
model = model.to(args.device)
model.train()
# Get optimizer and scheduler
optimizer = optim.get_optimizer(
filter(lambda p: p.requires_grad, model.parameters()), args)
lr_scheduler = optim.get_scheduler(optimizer, args)
if args.ckpt_path:
ModelSaver.load_optimizer(args.ckpt_path, optimizer, lr_scheduler)
# Get logger, evaluator, saver
loss_fn = optim.get_loss_fn(args.loss_fn, args)
logger = TrainLogger(args, len(train_loader.dataset))
eval_loaders = [
get_loader(args, phase='train', is_training=False),
get_loader(args, phase='valid', is_training=False)
]
evaluator = ModelEvaluator(args, eval_loaders, logger, args.max_eval,
args.epochs_per_eval)
saver = ModelSaver(**vars(args))
# Train model
while not logger.is_finished_training():
logger.start_epoch()
for src, tgt in train_loader:
logger.start_iter()
with torch.set_grad_enabled(True):
pred_params = model.forward(src.to(args.device))
ages = src[:, 1]
loss = loss_fn(pred_params, tgt.to(args.device),
ages.to(args.device), args.use_intvl)
#loss = loss_fn(pred_params, tgt.to(args.device), src.to(args.device), args.use_intvl)
logger.log_iter(src, pred_params, tgt, loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
logger.end_iter()
metrics = evaluator.evaluate(model, args.device, logger.epoch)
# print(metrics)
saver.save(logger.epoch, model, optimizer, lr_scheduler, args.device,\
metric_val=metrics.get(args.metric_name, None))
logger.end_epoch(metrics=metrics)
def load_multi_model(multi_args, model_args, data_args, gpu_ids):
"""Load multi lodel (a frontal model and a lateral model)."""
model_ap, ckpt_info_ap = ModelSaver.load_model(multi_args.ap_ckpt_path,
gpu_ids, model_args,
data_args)
model_pa, ckpt_info_pa = ModelSaver.load_model(multi_args.pa_ckpt_path,
gpu_ids, model_args,
data_args)
model_lateral, lateral_ckpt_info = ModelSaver.load_model(
multi_args.lateral_ckpt_path, args.gpu_ids, args)
# Make sure all models used the same task sequence
assert model_ap.task_sequence == model_pa.task_sequence
assert model_pa.task_sequence == model_lateral.task_sequence
models = {'ap': model_ap, 'pa': model_pa, 'lateral': model_lateral}
model = MultiModelWrapper(models)
model.task_sequence = model_ap.task_sequence
return model
def loaded_model_iterator(self, task):
from saver import ModelSaver
model_dicts = self.task2model_dicts[task]
for model_dict in model_dicts:
ckpt_path = model_dict['ckpt_path']
self.model_args.model_uncertainty = model_dict['is_3class']
model, ckpt_info = ModelSaver.load_model(ckpt_path, self.gpu_ids,
self.model_args,
self.data_args)
yield model
def test(args):
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
model.eval()
# Run a single evaluation
eval_loader = WhiteboardLoader(args.data_dir, args.phase, args.batch_size,
shuffle=False, do_augment=False, num_workers=args.num_workers)
logger = TestLogger(args, len(eval_loader.dataset))
logger.start_epoch()
evaluator = ModelEvaluator([eval_loader], logger, num_visuals=args.num_visuals, prob_threshold=args.prob_threshold)
metrics = evaluator.evaluate(model, args.device, logger.epoch)
logger.end_epoch(metrics)
def __init__(self):
root_path = os.path.dirname(os.path.dirname(__file__))
if torch.cuda.is_available():
self.device = torch.device("cuda")
gpu_ids = list(range(torch.cuda.device_count()))
else:
self.device = torch.device("cpu")
gpu_ids = []
self.model, _ = ModelSaver.load_model(
os.path.join(root_path, "penet_best.pth.tar"), gpu_ids)
self.model = self.model.to(self.device)
self.grad_cam = CustomGradCAM(self.model,
self.device,
is_binary=True,
is_3d=True)
def test(args):
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
model.eval()
data_loader = get_loader(args, phase=args.phase, is_training=False)
logger = TestLogger(args, len(data_loader.dataset))
# Get model outputs, log to TensorBoard, write masks to disk window-by-window
util.print_err('Writing model outputs to {}...'.format(args.results_dir))
all_gender = []
all_age = []
all_tte = []
all_is_alive = []
all_mu = []
all_s2 = []
with tqdm(total=len(data_loader.dataset), unit=' windows') as progress_bar:
for i, (src, tgt) in enumerate(data_loader):
all_gender.extend([int(x) for x in src[:, 0]])
all_age.extend([float(x) for x in src[:, 1]])
all_tte.extend([float(x) for x in tgt[:, 0]])
all_is_alive.extend([int(x) for x in tgt[:, 1]])
with torch.no_grad():
pred_params = model.forward(src.to(args.device))
# import pdb
# pdb.set_trace()
outputs = pred_params.cpu().numpy()
all_mu.extend([float(x) for x in outputs[:, 0]])
all_s2.extend([float(x) for x in outputs[:, 1]])
progress_bar.update(src.size(0))
# print pred_params (mu, s) to file
fd = open(args.results_dir + '/test_stats.csv', 'w')
fd.write('gender, age, tte, is_alive, mu, s2\n')
for gender, age, tte, is_alive, mu, s2 \
in zip(all_gender, all_age, all_tte, all_is_alive, all_mu, all_s2):
fd.write('%d, %f, %f, %d, %f, %f\n' %
(gender, age, tte, is_alive, mu, s2))
fd.close()
def test(args):
# Get dataset
dataset = PairedDataset(args.data_dir,
phase=args.phase,
resize_shape=args.resize_shape,
crop_shape=args.crop_shape,
direction=args.direction)
data_loader = DataLoader(dataset,
args.batch_size,
shuffle=False,
num_workers=args.num_workers)
# Get model
model = models.__dict__[args.model](args)
model = ModelSaver.load_model(model, args.ckpt_path, args.gpu_ids)
model.train()
# Set up image saving
if args.save_images is None:
save_hook = None
else:
saver = ImageSaver(args.save_images, args.results_dir, args.name,
args.phase)
save_hook = saver.save
# Test model
criteria = {'MSE_src2tgt': mse, 'MSE_tgt2src': mse}
stats = evaluate(model, data_loader, criteria, batch_hook=save_hook)
# Add model info to stats
stats.update({'name': args.name, 'ckpt_path': args.ckpt_path})
# Write stats to disk
stats_path = os.path.join(args.results_dir, 'stats.json')
print('Saving stats to {}...'.format(stats_path))
with open(stats_path, 'w') as json_fh:
json.dump(stats, json_fh, sort_keys=True, indent=4)
# Copy training args for reference
args_src = os.path.join(args.save_dir, 'args.json')
args_dst = os.path.join(args.results_dir, 'args.json')
print('Copying args to {}...'.format(args_dst))
shutil.copy(args_src, args_dst)
def test(args):
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
model.eval()
data_loader = CIFARLoader('val', args.batch_size, args.num_workers)
# Get model outputs, log to TensorBoard, write masks to disk window-by-window
util.print_err('Writing model outputs to {}...'.format(args.results_dir))
with tqdm(total=len(data_loader.dataset), unit=' examples') as progress_bar:
for i, (inputs, info_dict) in enumerate(data_loader):
with torch.no_grad():
logits = model.forward(inputs.to(args.device))
probs = F.softmax(logits)
# TODO: Test script is incomplete. Does nothing with the outputs.
progress_bar.update(inputs.size(0))
def run_model(ckpt_path, ckpt_args, has_gpu, custom_tasks=None):
"""Run a model with the specified args and output predictions.
Args:
ckpt_path (Path): path specifying the checkpoint
ckpt_args (dict): args associated with the corresponding run
Returns:
pred_df (pandas.DataFrame): model predictions
gt_df (pandas.DataFrame): corresponding ground-truth labels
"""
ckpt_save_dir = ckpt_path.parent
model_args = Namespace(**ckpt_args['model_args'])
# JBY: The samed model will not be moco
model_args.moco = False
transform_args = Namespace(**ckpt_args['transform_args'])
data_args = Namespace(**ckpt_args['data_args'])
print("in select_ensemble.py: data_args: {}".format(data_args))
data_args.custom_tasks = custom_tasks
if has_gpu:
gpu_ids = util.args_to_list(ckpt_args['gpu_ids'], allow_empty=True,
arg_type=int, allow_negative=False)
else:
# TODO: JBY: HACK! CHANGING GPU ID TO NONE
gpu_ids = []
device = util.setup_gpus(gpu_ids)
model, _ = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=gpu_ids,
model_args=model_args,
is_training=False)
predictor = Predictor(model=model, device=device)
loader = get_loader(phase='valid',
data_args=data_args,
transform_args=transform_args,
is_training=False,
return_info_dict=False,
logger=None)
pred_df, gt_df = predictor.predict(loader)
return pred_df, gt_df
def test(args):
model_fn = models.__dict__[args_.model]
model = model_fn(args.num_classes)
model = nn.DataParallel(model, args.gpu_ids)
ckpt_info = ModelSaver.load_model(args.ckpt_path, model)
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
model.eval()
_, test_loader, _ = get_cifar_loaders(args.batch_size, args.num_workers)
logger = TestLogger(args)
logger.start_epoch()
for inputs, labels in test_loader:
logger.start_iter()
with torch.set_grad_enabled(True):
# Forward
logits = model.forward(inputs.to(args.device))
logger.end_iter(inputs, labels, logits)
logger.end_epoch()
def train(args):
if args.ckpt_path and not args.use_pretrained:
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
args.start_epoch = ckpt_info['epoch'] + 1
else:
model_fn = models.__dict__[args.model]
model = model_fn(**vars(args))
if args.use_pretrained:
model.load_pretrained(args.ckpt_path, args.gpu_ids)
model = nn.DataParallel(model, args.gpu_ids)
model = model.to(args.device)
model.train()
# Get optimizer and scheduler
if args.use_pretrained or args.fine_tune:
parameters = model.module.fine_tuning_parameters(
args.fine_tuning_boundary, args.fine_tuning_lr)
else:
parameters = model.parameters()
optimizer = util.get_optimizer(parameters, args)
lr_scheduler = util.get_scheduler(optimizer, args)
if args.ckpt_path and not args.use_pretrained and not args.fine_tune:
ModelSaver.load_optimizer(args.ckpt_path, optimizer, lr_scheduler)
# Get logger, evaluator, saver
cls_loss_fn = util.get_loss_fn(is_classification=True,
dataset=args.dataset,
size_average=False)
data_loader_fn = data_loader.__dict__[args.data_loader]
train_loader = data_loader_fn(args, phase='train', is_training=True)
logger = TrainLogger(args, len(train_loader.dataset),
train_loader.dataset.pixel_dict)
eval_loaders = [data_loader_fn(args, phase='val', is_training=False)]
evaluator = ModelEvaluator(args.do_classify, args.dataset, eval_loaders,
logger, args.agg_method, args.num_visuals,
args.max_eval, args.epochs_per_eval)
saver = ModelSaver(args.save_dir, args.epochs_per_save, args.max_ckpts,
args.best_ckpt_metric, args.maximize_metric)
# Train model
while not logger.is_finished_training():
logger.start_epoch()
for inputs, target_dict in train_loader:
logger.start_iter()
with torch.set_grad_enabled(True):
inputs.to(args.device)
cls_logits = model.forward(inputs)
cls_targets = target_dict['is_abnormal']
cls_loss = cls_loss_fn(cls_logits, cls_targets.to(args.device))
loss = cls_loss.mean()
logger.log_iter(inputs, cls_logits, target_dict,
cls_loss.mean(), optimizer)
optimizer.zero_grad()
loss.backward()
optimizer.step()
logger.end_iter()
util.step_scheduler(lr_scheduler, global_step=logger.global_step)
metrics, curves = evaluator.evaluate(model, args.device, logger.epoch)
saver.save(logger.epoch,
model,
optimizer,
lr_scheduler,
args.device,
metric_val=metrics.get(args.best_ckpt_metric, None))
logger.end_epoch(metrics, curves)
util.step_scheduler(lr_scheduler,
metrics,
epoch=logger.epoch,
best_ckpt_metric=args.best_ckpt_metric)
def train(args):
# Get loader for outer loop training
loader = get_loader(args)
target_image_shape = loader.dataset.target_image_shape
setattr(args, 'target_image_shape', target_image_shape)
# Load model
model_fn = models.__dict__[args.model]
model = model_fn(**vars(args))
model = nn.DataParallel(model, args.gpu_ids)
model = model.to(args.device)
model.train()
# Print model parameters
print('Model parameters: name, size, mean, std')
for name, param in model.named_parameters():
print(name, param.size(), torch.mean(param), torch.std(param))
# Get optimizer and loss
parameters = model.parameters()
optimizer = util.get_optimizer(parameters, args)
loss_fn = util.get_loss_fn(args.loss_fn, args)
z_loss_fn = util.get_loss_fn(args.loss_fn, args)
# Get logger, saver
logger = TrainLogger(args)
saver = ModelSaver(args)
print(f'Logs: {logger.log_dir}')
print(f'Ckpts: {args.save_dir}')
# Train model
logger.log_hparams(args)
batch_size = args.batch_size
while not logger.is_finished_training():
logger.start_epoch()
for input_noise, target_image, mask, z_test_target, z_test in loader:
logger.start_iter()
if torch.cuda.is_available():
input_noise = input_noise.to(args.device) #.cuda()
target_image = target_image.cuda()
mask = mask.cuda()
z_test = z_test.cuda()
z_test_target = z_test_target.cuda()
masked_target_image = target_image * mask
obscured_target_image = target_image * (1.0 - mask)
# Input is noise tensor, target is image
model.train()
with torch.set_grad_enabled(True):
if args.use_intermediate_logits:
logits = model.forward(input_noise).float()
probs = F.sigmoid(logits)
# Debug logits and diffs
logger.debug_visualize(
[logits, logits * mask, logits * (1.0 - mask)],
unique_suffix='logits-train')
else:
probs = model.forward(input_noise).float()
# With backprop, calculate (1) masked loss, loss when mask is applied.
# Loss is done elementwise without reduction, so must take mean after.
# Easier for debugging.
masked_probs = probs * mask
masked_loss = torch.zeros(1,
requires_grad=True).to(args.device)
masked_loss = loss_fn(masked_probs, masked_target_image).mean()
masked_loss.backward()
optimizer.step()
optimizer.zero_grad()
# Without backprop, calculate (2) full loss on the entire image,
# And (3) the obscured loss, region obscured by mask.
model.eval()
with torch.no_grad():
if args.use_intermediate_logits:
logits_eval = model.forward(input_noise).float()
probs_eval = F.sigmoid(logits_eval)
# Debug logits and diffs
logger.debug_visualize([
logits_eval, logits_eval * mask, logits_eval *
(1.0 - mask)
],
unique_suffix='logits-eval')
else:
probs_eval = model.forward(input_noise).float()
masked_probs_eval = probs_eval * mask
masked_loss_eval = torch.zeros(1)
masked_loss_eval = loss_fn(masked_probs_eval,
masked_target_image).mean()
full_loss_eval = torch.zeros(1)
full_loss_eval = loss_fn(probs_eval, target_image).mean()
obscured_probs_eval = probs_eval * (1.0 - mask)
obscured_loss_eval = torch.zeros(1)
obscured_loss_eval = loss_fn(obscured_probs_eval,
obscured_target_image).mean()
# With backprop on only the input z, (4) run one step of z-test and get z-loss
z_optimizer = util.get_optimizer([z_test.requires_grad_()], args)
with torch.set_grad_enabled(True):
if args.use_intermediate_logits:
z_logits = model.forward(z_test).float()
z_probs = F.sigmoid(z_logits)
else:
z_probs = model.forward(z_test).float()
z_loss = torch.zeros(1, requires_grad=True).to(args.device)
z_loss = z_loss_fn(z_probs, z_test_target).mean()
z_loss.backward()
z_optimizer.step()
z_optimizer.zero_grad()
if z_loss < args.max_z_test_loss: # TODO: include this part into the metrics/saver stuff below
# Save MSE on obscured region
final_metrics = {'final/score': obscured_loss_eval.item()}
logger._log_scalars(final_metrics)
print('z loss', z_loss)
print('Final MSE value', obscured_loss_eval)
# TODO: Make a function for metrics - or at least make sure dict includes all possible best ckpt metrics
metrics = {'masked_loss': masked_loss.item()}
saver.save(logger.global_step,
model,
optimizer,
args.device,
metric_val=metrics.get(args.best_ckpt_metric, None))
# Log both train and eval model settings, and visualize their outputs
logger.log_status(
inputs=input_noise,
targets=target_image,
probs=probs,
masked_probs=masked_probs,
masked_loss=masked_loss,
probs_eval=probs_eval,
masked_probs_eval=masked_probs_eval,
obscured_probs_eval=obscured_probs_eval,
masked_loss_eval=masked_loss_eval,
obscured_loss_eval=obscured_loss_eval,
full_loss_eval=full_loss_eval,
z_target=z_test_target,
z_probs=z_probs,
z_loss=z_loss,
save_preds=args.save_preds,
)
logger.end_iter()
logger.end_epoch()
# Last log after everything completes
logger.log_status(
inputs=input_noise,
targets=target_image,
probs=probs,
masked_probs=masked_probs,
masked_loss=masked_loss,
probs_eval=probs_eval,
masked_probs_eval=masked_probs_eval,
obscured_probs_eval=obscured_probs_eval,
masked_loss_eval=masked_loss_eval,
obscured_loss_eval=obscured_loss_eval,
full_loss_eval=full_loss_eval,
z_target=z_test_target,
z_probs=z_probs,
z_loss=z_loss,
save_preds=args.save_preds,
force_visualize=True,
)
def get_cams(args):
print('Loading model...')
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
model = model.to(args.device)
args.start_epoch = ckpt_info['epoch'] + 1
print('Last layer in model.features is named "{}"...'.format([k for k in model.module.encoders._modules.keys()][-1]))
print('Extracting feature maps from layer named "{}"...'.format(args.target_layer))
grad_cam = GradCAM(model, args.device, is_binary=True, is_3d=True)
print(grad_cam)
gbp = GuidedBackPropagation(model, args.device, is_binary=True, is_3d=True)
print(gbp)
num_generated = 0
data_loader = CTDataLoader(args, phase=args.phase, is_training=False)
print(data_loader)
study_idx_dict = {}
study_count = 1
for inputs, target_dict in data_loader:
#print(inputs, target_dict)
#print('target_dict dir={}'.format(dir(target_dict)))
#print('\ntarget_dict[study_num]={}'.format(target_dict['study_num']))
probs, idx = grad_cam.forward(inputs)
grad_cam.backward(idx=idx[0]) # Just take top prediction
cam = grad_cam.get_cam(args.target_layer)
labels = target_dict['is_abnormal']
if labels.item() == 0:
# Keep going until we get an aneurysm study
print('Skipping a normal example...')
continue
print('Generating CAM...')
study_num = 1
with torch.set_grad_enabled(True):
probs, idx = grad_cam.forward(inputs)
print(probs, idx)
grad_cam.backward(idx=idx[0]) # Just take top prediction
cam = grad_cam.get_cam(args.target_layer)
guided_backprop = None
if args.use_gbp:
inputs2 = torch.autograd.Variable(inputs, requires_grad=True)
probs2, idx2 = gbp.forward(inputs2)
gbp.backward(idx=idx2[0])
guided_backprop = np.squeeze(gbp.generate())
print('Overlaying CAM...')
print(cam.shape)
new_cam = util.resize(cam, inputs[0])
print(new_cam.shape)
input_np = util.un_normalize(inputs[0], args.img_format, data_loader.dataset.pixel_dict)
input_np = np.transpose(input_np, (1, 2, 3, 0))
input_frames = list(input_np)
input_normed = np.float32(input_np) / 255
cam_frames = list(util.add_heat_map(input_normed, new_cam))
gbp_frames = None
if args.use_gbp:
gbp_np = util.normalize_to_image(guided_backprop * new_cam)
gbp_frames = []
for dim in range(gbp_np.shape[0]):
slice_ = gbp_np[dim, :, :]
gbp_frames.append(slice_[..., None])
# Write to a GIF file
output_path_input = os.path.join(os.path.join(args.cam_dir, '{}_{}_input_fn.gif'.format(target_dict['study_num'], study_count)))
output_path_cam = os.path.join(args.cam_dir, '{}_{}_cam_fn.gif'.format(target_dict['study_num'], study_count))
output_path_combined = os.path.join(args.cam_dir, '{}_{}_combined_fn.gif'.format(target_dict['study_num'], study_count))
print('Writing set {}/{} of CAMs to {}...'.format(num_generated + 1, args.num_cams, args.cam_dir))
input_clip = mpy.ImageSequenceClip(input_frames, fps=4)
input_clip.write_gif(output_path_input, verbose=False)
cam_clip = mpy.ImageSequenceClip(cam_frames, fps=4)
cam_clip.write_gif(output_path_cam, verbose=False)
combined_clip = mpy.clips_array([[input_clip, cam_clip]])
combined_clip.write_gif(output_path_combined, verbose=False)
if args.use_gbp:
output_path_gcam = os.path.join(args.cam_dir, 'gbp_{}.gif'.format(num_generated + 1))
gbp_clip = mpy.ImageSequenceClip(gbp_frames, fps=4)
gbp_clip.write_gif(output_path_gcam, verbose=False)
study_count += 1
num_generated += 1
if num_generated == args.num_cams:
return
def test(args):
"""Run model testing."""
model_args = args.model_args
data_args = args.data_args
logger_args = args.logger_args
# import pdb; pdb.set_trace()
# Get logger.
logger = Logger(logger_args.log_path, logger_args.save_dir,
logger_args.results_dir)
# Get image paths corresponding to predictions for logging
paths = None
if model_args.config_path is not None:
# Instantiate the EnsemblePredictor class for obtaining
# model predictions.
predictor = EnsemblePredictor(config_path=model_args.config_path,
model_args=model_args,
data_args=data_args,
gpu_ids=args.gpu_ids,
device=args.device,
logger=logger)
# Obtain ensemble predictions.
# Caches both individual and ensemble predictions.
# We always turn off caching to ensure that we write the Path column.
predictions, groundtruth, paths = predictor.predict(cache=False,
return_paths=True,
all_gt_tasks=True)
else:
# Load the model at ckpt_path.
ckpt_path = model_args.ckpt_path
ckpt_save_dir = Path(ckpt_path).parent
model_uncertainty = model_args.model_uncertainty
# Get model args from checkpoint and add them to
# command-line specified model args.
model_args, transform_args\
= ModelSaver.get_args(cl_model_args=model_args,
dataset=data_args.dataset,
ckpt_save_dir=ckpt_save_dir,
model_uncertainty=model_uncertainty)
# TODO JBY: in test moco should never be true.
model_args.moco = args.model_args.moco
model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=args.gpu_ids,
model_args=model_args,
is_training=False)
# Instantiate the Predictor class for obtaining model predictions.
predictor = Predictor(model=model, device=args.device)
# Get phase loader object.
return_info_dict = True
loader = get_loader(phase=data_args.phase,
data_args=data_args,
transform_args=transform_args,
is_training=False,
return_info_dict=return_info_dict,
logger=logger)
# Obtain model predictions.
if return_info_dict:
predictions, groundtruth, paths = predictor.predict(loader)
else:
predictions, groundtruth = predictor.predict(loader)
# print(predictions[CHEXPERT_COMPETITION_TASKS])
if model_args.calibrate:
#open the json file which has the saved parameters
import json
with open(CALIBRATION_FILE) as f:
data = json.load(f)
i = 0
#print(predictions)
import math
def sigmoid(x):
return 1 / (1 + math.exp(-x))
for column in predictions:
predictions[column] = predictions[column].apply \
(lambda x: sigmoid(x * data[i][0][0][0] \
+ data[i][1][0]))
i += 1
# print(predictions[CHEXPERT_COMPETITION_TASKS])
#run forward on all the predictions in each row of predictions
# Log predictions and groundtruth to file in CSV format.
logger.log_predictions_groundtruth(predictions, groundtruth, paths)
if not args.inference_only:
# Instantiate the evaluator class for evaluating models.
evaluator = Evaluator(logger, operating_points_path=CHEXPERT_RAD_PATH)
# Get model metrics and curves on the phase dataset.
metrics, curves = evaluator.evaluate_tasks(groundtruth, predictions)
# Log metrics to stdout and file.
logger.log_stdout(f"Writing metrics to {logger.metrics_path}.")
logger.log_metrics(metrics, save_csv=True)
# TODO: make this work with ensemble
# TODO: investigate if the eval_loader can just be the normal loader here
if logger_args.save_cams:
cams_dir = logger_args.save_dir / 'cams'
print(f'Save cams to {cams_dir}')
save_grad_cams(args,
loader,
model,
cams_dir,
only_competition=logger_args.only_competition_cams,
only_top_task=False)
logger.log("=== Testing Complete ===")
def test(args):
print("Stage 1")
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
print("Stage 2")
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
print("Stage 3")
model.eval()
print("Stage 4")
data_loader = CTDataLoader(args, phase=args.phase, is_training=False)
print(data_loader.dataset.ctpe_list)
study2slices = defaultdict(list)
study2probs = defaultdict(list)
study2labels = {}
logger = TestLogger(args, len(data_loader.dataset),
data_loader.dataset.pixel_dict)
minimum = []
means = []
maximum = []
data = []
# Get model outputs, log to TensorBoard, write masks to disk window-by-window
util.print_err('Writing model outputs to {}...'.format(args.results_dir))
with tqdm(total=len(data_loader.dataset), unit=' windows') as progress_bar:
for i, (inputs, targets_dict) in enumerate(data_loader):
with torch.no_grad():
cls_logits = model.forward(inputs.to(args.device))
cls_probs = F.sigmoid(cls_logits)
if args.visualize_all:
logger.visualize(inputs,
cls_logits,
targets_dict=None,
phase=args.phase,
unique_id=i)
max_probs = cls_probs.to('cpu').numpy()
for study_num, slice_idx, prob in \
zip(targets_dict['study_num'], targets_dict['slice_idx'], list(max_probs)):
# Convert to standard python data types
study_num = int(study_num)
slice_idx = int(slice_idx)
# Save series num for aggregation
study2slices[study_num].append(slice_idx)
study2probs[study_num].append(prob.item())
series = data_loader.get_series(study_num)
if study_num not in study2labels:
study2labels[study_num] = int(series.is_positive)
progress_bar.update(inputs.size(0))
# Combine masks
util.print_err('Combining masks...')
max_probs = []
labels = []
study_nums = []
predictions = {}
print("Get max prob")
for study_num in tqdm(study2slices):
# Sort by slice index and get max probability
slice_list, prob_list = (list(t) for t in zip(
*sorted(zip(study2slices[study_num], study2probs[study_num]),
key=lambda slice_and_prob: slice_and_prob[0])))
study2slices[study_num] = slice_list
study2probs[study_num] = prob_list
max_prob = max(prob_list)
max_probs.append(max_prob)
label = study2labels[study_num]
labels.append(label)
study_nums.append(study_num)
predictions[study_num] = {'label': label, 'pred': max_prob}
#Save predictions to file, indexed by study number
print("Saving predictions to pickle files")
with open('{}/preds.pickle'.format(args.results_dir), "wb") as fp:
pickle.dump(predictions, fp)
# Compute AUROC and AUPRC using max aggregation, write to files
max_probs, labels = np.array(max_probs), np.array(labels)
fpr, tpr, threshold = roc_curve(labels, max_probs)
i = np.arange(len(tpr))
roc = pd.DataFrame({
'tf': pd.Series(tpr - (1 - fpr), index=i),
'threshold': pd.Series(threshold, index=i)
})
roc_t = roc.ix[(roc.tf - 0).abs().argsort()[:1]]
threshold = 0.5
pred = [1 if p > threshold else 0 for p in max_probs]
tn, fp, fn, tp = confusion_matrix(labels, pred).ravel()
print("\nTrue Positive Examples\n" + "-" * 80)
for i, study_num in enumerate(study_nums):
if labels[i] == 1 and pred[i] == 1: print(study_num)
print("\nTrue Negative Examples\n" + "-" * 80)
for i, study_num in enumerate(study_nums):
if labels[i] == 0 and pred[i] == 0: print(study_num)
print("\nFalse Negative Examples\n" + "-" * 80)
for i, study_num in enumerate(study_nums):
if labels[i] == 1 and pred[i] == 0: print(study_num)
print("\nFalse Positive Examples\n" + "-" * 80)
for i, study_num in enumerate(study_nums):
if labels[i] == 0 and pred[i] == 1: print(study_num)
print("Total number of data :", len(labels))
print("Total number of positives :", len([l for l in labels if l == 1]))
print("Total number of negatives :", len([l for l in labels if l == 0]))
print("# True Negative : ", tn)
print("# True Positive : ", tp)
print("# False Negative : ", fn)
print("# False Positive : ", fp)
metrics = {
args.phase + '_' + 'AUPRC':
sk_metrics.average_precision_score(labels, max_probs),
args.phase + '_' + 'AUROC':
sk_metrics.roc_auc_score(labels, max_probs),
}
for k, v in metrics.items():
print('{}: {:.5f}\n'.format(k, v))
print("Saving metrics to file")
with open(os.path.join(args.results_dir, 'metrics.txt'),
'w') as metrics_fh:
for k, v in metrics.items():
metrics_fh.write('{}: {:.5f}\n'.format(k, v))
curves = {
args.phase + '_' + 'PRC':
sk_metrics.precision_recall_curve(labels, max_probs),
args.phase + '_' + 'ROC':
sk_metrics.roc_curve(labels, max_probs)
}
roc = sk_metrics.roc_curve(labels, max_probs)
with open("intermountain_roc.pkl", 'wb') as f:
pickle.dump(roc, f)
for name, curve in curves.items():
curve_np = util.get_plot(name, curve)
curve_img = Image.fromarray(curve_np)
curve_img.save(os.path.join(args.results_dir, '{}.png'.format(name)))
def calibrate(args):
"""Run model testing."""
model_args = args.model_args
data_args = args.data_args
logger_args = args.logger_args
# Get logger.
logger = Logger(logger_args.log_path, logger_args.save_dir,
logger_args.results_dir)
# Get image paths corresponding to predictions for logging
paths = None
if model_args.config_path is not None:
# Instantiate the EnsemblePredictor class for obtaining
# model predictions.
predictor = EnsemblePredictor(config_path=model_args.config_path,
model_args=model_args,
data_args=data_args,
gpu_ids=args.gpu_ids,
device=args.device,
logger=logger)
# Obtain ensemble predictions.
# Caches both individual and ensemble predictions.
# We always turn off caching to ensure that we write the Path column.
predictions, groundtruth, paths = predictor.predict(cache=False,
return_paths=True,
all_gt_tasks=True)
else:
# Load the model at ckpt_path.
ckpt_path = model_args.ckpt_path
ckpt_save_dir = Path(ckpt_path).parent
model_uncertainty = model_args.model_uncertainty
# Get model args from checkpoint and add them to
# command-line specified model args.
model_args, transform_args\
= ModelSaver.get_args(cl_model_args=model_args,
dataset=data_args.dataset,
ckpt_save_dir=ckpt_save_dir,
model_uncertainty=model_uncertainty)
model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=args.gpu_ids,
model_args=model_args,
is_training=False)
# Instantiate the Predictor class for obtaining model predictions.
predictor = Predictor(model=model, device=args.device)
# Get phase loader object.
return_info_dict = True
loader = get_loader(phase=data_args.phase,
data_args=data_args,
transform_args=transform_args,
is_training=False,
return_info_dict=return_info_dict,
logger=logger)
# Obtain model predictions
if return_info_dict:
predictions, groundtruth, paths = predictor.predict(loader)
else:
predictions, groundtruth = predictor.predict(loader)
#print(groundtruth)
# custom function
from sklearn.linear_model import LogisticRegression as LR
params = []
for column in predictions:
#print(predictions[column].values)
#print(groundtruth[column].values)
#drop corresponding rows where gt is -1 and
lr = LR(C=15)
to_drop = groundtruth.index[groundtruth[column] == -1].tolist()
lr.fit(predictions[column].drop(to_drop).values.reshape(-1, 1),
groundtruth[column].drop(
to_drop).values) # LR needs X to be 2-dimensional
print("num_rows_used", predictions[column].drop(to_drop).values.size)
#print(groundtruth[column].drop(to_drop).values.size)
#print(predictions[column].values)
print("coeffs", lr.coef_, lr.intercept_)
p_calibrated = lr.predict_proba(predictions[column].values.reshape(
-1, 1))
params.append((lr.coef_, lr.intercept_))
import json
with open('calibration_params.json', 'w') as f:
import pandas as pd
pd.Series(params).to_json(f, orient='values')
def train(args):
"""Run model training."""
print("Start Training ...")
# Get nested namespaces.
model_args = args.model_args
logger_args = args.logger_args
optim_args = args.optim_args
data_args = args.data_args
transform_args = args.transform_args
# Get logger.
print('Getting logger... log to path: {}'.format(logger_args.log_path))
logger = Logger(logger_args.log_path, logger_args.save_dir)
# For conaug, point to the MOCO pretrained weights.
if model_args.ckpt_path and model_args.ckpt_path != 'None':
print("pretrained checkpoint specified : {}".format(
model_args.ckpt_path))
# CL-specified args are used to load the model, rather than the
# ones saved to args.json.
model_args.pretrained = False
ckpt_path = model_args.ckpt_path
model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=args.gpu_ids,
model_args=model_args,
is_training=True)
if not model_args.moco:
optim_args.start_epoch = ckpt_info['epoch'] + 1
else:
optim_args.start_epoch = 1
else:
print(
'Starting without pretrained training checkpoint, random initialization.'
)
# If no ckpt_path is provided, instantiate a new randomly
# initialized model.
model_fn = models.__dict__[model_args.model]
if data_args.custom_tasks is not None:
tasks = NamedTasks[data_args.custom_tasks]
else:
tasks = model_args.__dict__[TASKS] # TASKS = "tasks"
print("Tasks: {}".format(tasks))
model = model_fn(tasks, model_args)
model = nn.DataParallel(model, args.gpu_ids)
# Put model on gpu or cpu and put into training mode.
model = model.to(args.device)
model.train()
print("========= MODEL ==========")
print(model)
# Get train and valid loader objects.
train_loader = get_loader(phase="train",
data_args=data_args,
transform_args=transform_args,
is_training=True,
return_info_dict=False,
logger=logger)
valid_loader = get_loader(phase="valid",
data_args=data_args,
transform_args=transform_args,
is_training=False,
return_info_dict=False,
logger=logger)
# Instantiate the predictor class for obtaining model predictions.
predictor = Predictor(model, args.device)
# Instantiate the evaluator class for evaluating models.
evaluator = Evaluator(logger)
# Get the set of tasks which will be used for saving models
# and annealing learning rate.
eval_tasks = EVAL_METRIC2TASKS[optim_args.metric_name]
# Instantiate the saver class for saving model checkpoints.
saver = ModelSaver(save_dir=logger_args.save_dir,
iters_per_save=logger_args.iters_per_save,
max_ckpts=logger_args.max_ckpts,
metric_name=optim_args.metric_name,
maximize_metric=optim_args.maximize_metric,
keep_topk=logger_args.keep_topk)
# TODO: JBY: handle threshold for fine tuning
if model_args.fine_tuning == 'full': # Fine tune all layers.
pass
else:
# Freeze other layers.
models.PretrainedModel.set_require_grad_for_fine_tuning(
model, model_args.fine_tuning.split(','))
# Instantiate the optimizer class for guiding model training.
optimizer = Optimizer(parameters=model.parameters(),
optim_args=optim_args,
batch_size=data_args.batch_size,
iters_per_print=logger_args.iters_per_print,
iters_per_visual=logger_args.iters_per_visual,
iters_per_eval=logger_args.iters_per_eval,
dataset_len=len(train_loader.dataset),
logger=logger)
if model_args.ckpt_path and not model_args.moco:
# Load the same optimizer as used in the original training.
optimizer.load_optimizer(ckpt_path=model_args.ckpt_path,
gpu_ids=args.gpu_ids)
model_uncertainty = model_args.model_uncertainty
loss_fn = evaluator.get_loss_fn(
loss_fn_name=optim_args.loss_fn,
model_uncertainty=model_args.model_uncertainty,
mask_uncertain=True,
device=args.device)
# Run training
while not optimizer.is_finished_training():
optimizer.start_epoch()
# TODO: JBY, HACK WARNING # What is the hack?
metrics = None
for inputs, targets in train_loader:
optimizer.start_iter()
if optimizer.global_step and optimizer.global_step % optimizer.iters_per_eval == 0 or len(
train_loader.dataset
) - optimizer.iter < optimizer.batch_size:
# Only evaluate every iters_per_eval examples.
predictions, groundtruth = predictor.predict(valid_loader)
# print("predictions: {}".format(predictions))
metrics, curves = evaluator.evaluate_tasks(
groundtruth, predictions)
# Log metrics to stdout.
logger.log_metrics(metrics)
# Add logger for all the metrics for valid_loader
logger.log_scalars(metrics, optimizer.global_step)
# Get the metric used to save model checkpoints.
average_metric = evaluator.evaluate_average_metric(
metrics, eval_tasks, optim_args.metric_name)
if optimizer.global_step % logger_args.iters_per_save == 0:
# Only save every iters_per_save examples directly
# after evaluation.
print("Save global step: {}".format(optimizer.global_step))
saver.save(iteration=optimizer.global_step,
epoch=optimizer.epoch,
model=model,
optimizer=optimizer,
device=args.device,
metric_val=average_metric)
# Step learning rate scheduler.
optimizer.step_scheduler(average_metric)
with torch.set_grad_enabled(True):
logits, embedding = model(inputs.to(args.device))
loss = loss_fn(logits, targets.to(args.device))
optimizer.log_iter(inputs, logits, targets, loss)
optimizer.zero_grad()
loss.backward()
optimizer.step()
optimizer.end_iter()
optimizer.end_epoch(metrics)
logger.log('=== Training Complete ===')
def test(args):
print ("Stage 1")
model, ckpt_info = ModelSaver.load_model(args.ckpt_path, args.gpu_ids)
print ("Stage 2")
args.start_epoch = ckpt_info['epoch'] + 1
model = model.to(args.device)
print ("Stage 3")
model.eval()
print ('This should be false: {}'.format(model.training))
print ("Stage 4")
data_loader = CTDataLoader(args, phase=args.phase, is_training=False)
#print('data_loader={}'.format(data_loader))
#print('data_loader.dataset={}'.format(data_loader.dataset))
study2slices = defaultdict(list)
study2probs = defaultdict(list)
study2labels = {}
logger = TestLogger(args, len(data_loader.dataset), data_loader.dataset.pixel_dict)
print("Stage 5")
f = open('/projectnb/ece601/kaggle-pulmonary-embolism/meganmp/train/series_list.pkl','rb')
data_labels = pickle.load(f)
# Create list to manually process labels
#with open('positive.txt') as f:
#pos_labels = f.readlines()
#pos_labels = [x.strip() for x in pos_labels]
ispos = [x.is_positive for x in data_labels]
isposidx = [x.study_num for x in data_labels]
label_dict = {}
for i in range(len(ispos)):
label_dict[isposidx[i]] = ispos[i]
for key in label_dict.keys():
print('label_dict={}\t{}'.format(key, label_dict[key]))
# Get model outputs, log to TensorBoard, write masks to disk window-by-window
util.print_err('Writing model outputs to {}...'.format(args.results_dir))
with tqdm(total=len(data_loader.dataset), unit=' windows') as progress_bar:
for i, (inputs, targets_dict) in enumerate(data_loader):
with torch.no_grad():
cls_logits = model.forward(inputs.to(args.device))
cls_probs = torch.sigmoid(cls_logits)
if args.visualize_all:
logger.visualize(inputs, cls_logits, targets_dict=None, phase=args.phase, unique_id=i)
max_probs = cls_probs.to('cpu').numpy()
for study_num, slice_idx, prob in \
zip(targets_dict['study_num'], targets_dict['slice_idx'], list(max_probs)):
#print('targets_dict[studynum]={}'.format(targets_dict['study_num']))
#print('targets_dict[sliceidx]={}'.format(targets_dict['slice_idx']))
# Convert to standard python data types
study_num = study_num #.item()
#study_num = int(study_num)
slice_idx = int(slice_idx)
# Save series num for aggregation
study2slices[study_num].append(slice_idx)
study2probs[study_num].append(prob.item())
series = data_loader.get_series(study_num)
if study_num not in study2labels:
print('study_num={}'.format(study_num))
print('series.is_positive={}'.format(label_dict[study_num]))
study2labels[study_num] = label_dict[study_num]
#if study_num in pos_labels:
#print('DEBUG -------=1?-------------------')
#print('POS LABEL')
#print('study_num={}'.format(study_num))
#study2labels[study_num] = 1
#else:
#print('Not in study2labels. series = {}'.format(study_num))
#print('series.is_positive={}'.format(series.is_positive))
#study2labels[study_num] = int(series.is_positive)
#print('study2labels: {}'.format(study2labels[study_num]))
progress_bar.update(inputs.size(0))
print('study2labels={}'.format(study2labels))
# Combine masks
util.print_err('Combining masks...')
max_probs = []
labels = []
predictions = {}
print("Get max prob")
for study_num in tqdm(study2slices):
# Sort by slice index and get max probability
slice_list, prob_list = (list(t) for t in zip(*sorted(zip(study2slices[study_num], study2probs[study_num]),
key=lambda slice_and_prob: slice_and_prob[0])))
study2slices[study_num] = slice_list
study2probs[study_num] = prob_list
max_prob = max(prob_list)
print('study={}\tmax_prob={}'.format(study_num, max_prob))
max_probs.append(max_prob)
label = study2labels[study_num]
labels.append(label)
predictions[study_num] = {'label':label, 'pred':max_prob}
#Save predictions to file, indexed by study number
print("Saving predictions to pickle files")
with open('{}/preds.pickle'.format(args.results_dir),"wb") as fp:
pickle.dump(predictions,fp)
results_series = [k for k,_ in predictions.items()]
results_pred = [v['pred'] for _,v in predictions.items()]
results_label = [v['label'] for _,v in predictions.items()]
print('roc_auc_score={}'.format(roc_auc_score(results_label, results_pred)))
# Create dataframe summary
TRAIN_CSV = '/projectnb/ece601/kaggle-pulmonary-embolism/rsna-str-pulmonary-embolism-detection/train.csv'
train_df = pd.read_csv(TRAIN_CSV)
train_df = train_df[['SeriesInstanceUID', 'negative_exam_for_pe']]
train_df = train_df.groupby('SeriesInstanceUID').aggregate(list)
train_df['pe_label'] = train_df['negative_exam_for_pe'].apply(lambda x: 0 if 1 in x else 1)
results_dict = {
'series': results_series,
'pred': results_pred
}
results_df = pd.DataFrame.from_dict(results_dict)
results_df = results_df.set_index('series')
results_df = results_df.join(train_df, how='left').reset_index().rename({'index': 'series'})
print('roc_auc_score={}'.format(roc_auc_score(results_df['pe_label'], results_df['pred'])))
# Calculate confusion matrix
results_df['interpretation'] = results_df['pred'].apply(lambda x: 0 if x < 0.5 else 1)
print(results_df.head(10))
tn, fp, fn, tp = confusion_matrix(results_df['pe_label'], results_df['interpretation']).ravel()
print('confusion_matrix: [{} {} {} {}]'.format(tp, fp, fn, tn))
ckpt_path = 'penet_best.pth.tar'
device = 'cuda'
gpu_ids = 1
#map_location = 'cpu'
print("Reading input dicom...")
study = util.dicom_2_npy(input_study)
print('is study empty')
print(study)
# normalize and convert to tensor
print("Formatting input for model...")
study_windows = util.format_img(study)
print("is study window empty")
print(study_windows)
print ("Loading saved model...")
model, ckpt_info = ModelSaver.load_model(ckpt_path, [0])
print ("Sending model to GPU device...")
#start_epoch = ckpt_info['epoch'] + 1
model = model.to(device)
print ("Evaluating study...")
model.eval()
predicted_probabilities = [] # window wise for a single study
with torch.no_grad():
for window in study_windows:
cls_logits = model.forward(window.to(device, dtype=torch.float))
cls_probs = torch.sigmoid(cls_logits).to('cpu').numpy()
predicted_probabilities.append(cls_probs[0][0])
def test(args):
"""Run model testing."""
test_args = args.test_args
logger_args = args.logger_args
# Load the model at ckpt_path.
ckpt_path = test_args.ckpt_path
ckpt_save_dir = Path(ckpt_path).parent
# Get model args from checkpoint and add them to
# command-line specified model args.
model_args, data_args, optim_args, logger_args\
= ModelSaver.get_args(cl_logger_args=logger_args,
ckpt_save_dir=ckpt_save_dir)
model, ckpt_info = ModelSaver.load_model(ckpt_path=ckpt_path,
gpu_ids=args.gpu_ids,
model_args=model_args,
is_training=False)
# Get logger.
logger = Logger(logger_args=logger_args,
data_args=data_args,
optim_args=optim_args,
test_args=test_args)
# Instantiate the Predictor class for obtaining model predictions.
predictor = Predictor(model=model, device=args.device)
phase = test_args.phase
is_test = False
if phase == 'test':
is_test = True
phase = 'valid' # Run valid first to get threshold
print(f"======================{phase}=======================")
# Get phase loader object.
loader = get_loader(phase=phase,
data_args=data_args,
is_training=False,
logger=logger)
# Obtain model predictions.
predictions, groundtruth = predictor.predict(loader)
# Instantiate the evaluator class for evaluating models.
evaluator = Evaluator(logger=logger, tune_threshold=True)
# Get model metrics and curves on the phase dataset.
metrics = evaluator.evaluate(groundtruth, predictions)
# Log metrics to stdout and file.
logger.log_stdout(f"Writing metrics to {logger.metrics_path}.")
logger.log_metrics(metrics, phase=phase)
# Evaluate dense to get back thresholds
dense_loader = get_loader(phase=phase,
data_args=data_args,
is_training=False,
logger=logger)
dense_predictions, dense_groundtruth = predictor.predict(dense_loader)
dense_metrics = evaluator.dense_evaluate(dense_groundtruth,
dense_predictions)
# Log metrics to stdout and file.
logger.log_stdout(f"Writing metrics to {logger.metrics_path}.")
logger.log_metrics(dense_metrics, phase=phase)
if is_test:
phase = 'test'
threshold = metrics['threshold']
print(f"======================{phase}=======================")
# Get phase loader object.
loader = get_loader(phase=phase,
data_args=data_args,
is_training=False,
test_args=test_args,
logger=logger)
# Obtain model predictions.
predictions, groundtruth = predictor.predict(loader)
# Instantiate the evaluator class for evaluating models.
evaluator = Evaluator(logger=logger,
threshold=threshold,
tune_threshold=False)
# Get model metrics and curves on the phase dataset.
metrics = evaluator.evaluate(groundtruth, predictions)
# Log metrics to stdout and file.
logger.log_stdout(f"Writing metrics to {logger.metrics_path}.")
logger.log_metrics(metrics, phase=phase)
# Dense test
phase = 'dense_test'
dense_loader = get_loader(phase=phase,
data_args=data_args,
is_training=False,
test_args=test_args,
logger=logger)
threshold_dense = dense_metrics["threshold_dense"]
threshold_tunef1_dense = dense_metrics["threshold_tunef1_dense"]
dense_predictions, dense_groundtruth = predictor.predict(dense_loader)
dense_metrics = evaluator.dense_evaluate(
dense_groundtruth,
dense_predictions,
threshold=threshold_dense,
threshold_tunef1=threshold_tunef1_dense)
logger.log_stdout(f"Writing metrics to {logger.metrics_path}.")
logger.log_metrics(dense_metrics, phase=phase)