def test(model, epoch, batch_size, data_path, fold, gpu, dicts, model_dir, testing, thresh = None, obs_limit = None): """ Testing loop. Returns metrics on desired fold (validation or test). """ filename = data_path.replace('train', fold) print('\nfile for evaluation: %s' % filename) y, yhat, yhat_raw, hids, losses = [], [], [], [], [] model.eval() gen = datasets.data_generator(filename, dicts, batch_size) for batch_idx, tup in enumerate(gen): if obs_limit: # Early stopping for debugging obs_seen = batch_idx * batch_size if obs_seen > obs_limit: print("Reached {} of test/val set".format(obs_limit)) break data, target, hadm_ids = tup data, target = Variable(torch.LongTensor(data), volatile=True), Variable(torch.FloatTensor(target)) if gpu: data = data.cuda() target = target.cuda() model.zero_grad() output, loss = model(data, target) # Forward pass output = output.data.cpu().numpy() losses.append(loss.data[0]) target_data = target.data.cpu().numpy() #save predictions, target, hadm ids yhat_raw.append(output) output = np.round(output) # Rounds to 0 for <= 0.5, up to one for > 0.5 yhat.append(output) y.append(target_data) hids.extend(hadm_ids) y = np.concatenate(y, axis=0) yhat = np.concatenate(yhat, axis=0) yhat_raw = np.concatenate(yhat_raw, axis=0) print("\nMax Prediction This Epoch:") print(max(yhat_raw)) #write the predictions persistence.write_preds(yhat, model_dir, hids, fold, yhat_raw) metrics = evaluation.all_metrics(yhat_raw, y, thresh) evaluation.print_metrics(metrics) metrics['loss_%s' % fold] = np.float64(np.mean(losses)) #float64 for json serialization return metrics
def test(model, epoch, batch_size, data_path, struc_feats, fold, gpu, dicts, model_dir, testing, test_frac=1, thresh=None): """ Testing loop. Returns metrics """ filename = data_path.replace('train', fold) print('\nfile for evaluation: %s' % filename) y, yhat, yhat_raw, hids, losses = [], [], [], [], [] print(struc_feats.shape[0]) model.eval() gen = datasets.data_generator(filename, dicts, batch_size) for batch_idx, tup in enumerate(gen): data, target, hadm_ids = tup batch_size_safe = min( batch_size, struc_feats.shape[0] - batch_idx * batch_size) # Avoiding going out of range if batch_idx * batch_size > test_frac * struc_feats.shape[0]: print("Reached {} of test/val set".format(test_frac)) break struc_data = struc_feats[ batch_idx * batch_size:batch_idx * batch_size + batch_size_safe].todense( ) # Only need in second index b/c batch_size_safe should be < batch_size only once data, struc_data, target = Variable(torch.LongTensor(data)), Variable( torch.FloatTensor(struc_data)), Variable(torch.FloatTensor(target)) if gpu: data = data.cuda() struc_data = struc_data.cuda() target = target.cuda() model.zero_grad() output, loss = model(data, struc_data, target) # Forward pass output = output.data.cpu().numpy() losses.append(loss.data[0]) target_data = target.data.cpu().numpy() #save predictions, target, hadm ids yhat_raw.append(output) output = np.round( output) # Rounds to 0 for <= 0.5, up to one for > 0.5 yhat.append(output) y.append(target_data) hids.extend(hadm_ids) y = np.concatenate(y, axis=0) yhat = np.concatenate(yhat, axis=0) yhat_raw = np.concatenate(yhat_raw, axis=0) print("\nMax Prediction:") print(max(yhat_raw)) #write the predictions persistence.write_preds(yhat, model_dir, hids, fold, yhat_raw) metrics = evaluation.all_metrics(yhat_raw, y, thresh) evaluation.print_metrics(metrics) metrics['loss_%s' % fold] = np.float64( np.mean(losses)) #float64 for json serialization return metrics
def test(model, epoch, batch_size, data_path, fold, gpu, dicts, model_dir, testing): """ Testing loop. Returns metrics """ filename = data_path.replace('train', fold) print('\nfile for evaluation: %s' % filename) y, yhat, yhat_raw, hids, losses = [], [], [], [], [] model.eval() gen = datasets.data_generator(filename, dicts, batch_size) for batch_idx, tup in enumerate(gen): data, target, hadm_ids = tup data, target = Variable(torch.LongTensor(data), volatile=True), Variable( torch.FloatTensor(target)) if gpu: data = data.cuda() target = target.cuda() model.zero_grad() output, loss = model(data, target) # Forward pass output = output.data.cpu().numpy() losses.append(loss.data[0]) target_data = target.data.cpu().numpy() #save predictions, target, hadm ids yhat_raw.append(output) output = np.round( output) # Rounds to 0 for <= 0.5, up to one for > 0.5 yhat.append(output) y.append(target_data) hids.extend(hadm_ids) y = np.concatenate(y, axis=0) yhat = np.concatenate(yhat, axis=0) yhat_raw = np.concatenate(yhat_raw, axis=0) print("\nMax Prediction:") print(max(yhat_raw)) # print("y shape: " + str(y.shape)) # print("yhat shape: " + str(yhat.shape)) #write the predictions persistence.write_preds(yhat, model_dir, hids, fold, yhat_raw) metrics = evaluation.all_metrics(yhat_raw, y) evaluation.print_metrics(metrics) metrics['loss_%s' % fold] = np.float64( np.mean(losses)) #float64 for json serialization return metrics
def test(model, epoch, batch_size, data_path, fold, gpu, dicts, samples, model_dir, testing, debug): """ Testing loop. Returns metrics """ filename = data_path.replace('train', fold) print('\nfile for evaluation: %s' % filename) #initialize stuff for saving attention samples if samples: tp_file = open('%s/tp_%s_examples_%d.txt' % (model_dir, fold, epoch), 'w') fp_file = open('%s/fp_%s_examples_%d.txt' % (model_dir, fold, epoch), 'w') y, yhat, yhat_raw, hids, losses = [], [], [], [], [] model.eval() gen = datasets.data_generator(filename, dicts, batch_size) for batch_idx, tup in tqdm(enumerate(gen)): if debug and batch_idx > 50: break data, target, hadm_ids = tup data, target = Variable(torch.LongTensor(data), volatile=True), Variable(torch.FloatTensor(target)) if gpu: data = data.cuda() target = target.cuda() model.zero_grad() output, loss = model(data, target) # Forward pass output = output.data.cpu().numpy() losses.append(loss.data[0]) target_data = target.data.cpu().numpy() #save predictions, target, hadm ids yhat_raw.append(output) output = np.round(output) # Rounds to 0 for <= 0.5, up to one for > 0.5 yhat.append(output) y.append(target_data) hids.extend(hadm_ids) if samples: tp_file.close() fp_file.close() y = np.concatenate(y, axis=0) yhat = np.concatenate(yhat, axis=0) yhat_raw = np.concatenate(yhat_raw, axis=0) print("\nMax Prediction:") print(max(yhat_raw)) # print("y shape: " + str(y.shape)) # print("yhat shape: " + str(yhat.shape)) #write the predictions persistence.write_preds(yhat, model_dir, hids, fold, yhat_raw) metrics = evaluation.all_metrics(yhat, y, yhat_raw=yhat_raw) evaluation.print_metrics(metrics) metrics['loss_%s' % fold] = np.mean(losses) return metrics