def __init__(self, path):
super(Course, self).__init__(path)
self.path = path
utils.parse_json_props(self)
self._modules = {
module_path: Module(module_path)
for module_path in utils.get_dirs(path)
}
def __init__(self, path):
super(Module, self).__init__(path)
self.path = path
utils.parse_json_props(self)
self._lessons = {
item_path: Lesson(item_path)
for item_path in utils.get_dirs(path)
}
def __init__(self, path):
super(Lesson, self).__init__(path)
self.path = path
utils.parse_json_props(self)
self._tasks = {
item_path: Task(item_path)
for item_path in utils.get_dirs(path)
}
def generate_flow_mag(path):
dirs = utils.get_dirs(path);
save_path = path + '_pkls'
## Fork 25 parallel threads to
# process mutliple files at the same time
queue_size = 25;
queues = [None] * queue_size
processes = [None] * queue_size
filenames = [None] * queue_size
idx = 0
for d in dirs:
current_dir = os.path.join(path,d);
files = utils.get_files(current_dir,'.avi',append_base=True);
for vdx_path in files:
save_dir = os.path.join(save_path,d);
if(not os.path.exists(save_dir)):
os.makedirs(save_dir)
name, ext = utils.get_file_name_ext(vdx_path);
file_name = os.path.join(save_dir, name) + '.pkl';
if(not os.path.exists(file_name)):
print('processing ',vdx_path )
vid = imageio.get_reader(vdx_path, 'ffmpeg')
queues[idx] = mp.Queue()
filenames[idx] = file_name ;
processes[idx] = mp.Process(target=video2motion_level, args=(vid,queues[idx]))
processes[idx].start()
idx += 1
else:
flow_mag = utils.pkl_read(file_name)
## Make sure it contains a least one frame
if(flow_mag.shape[0] == 0):
vid = imageio.get_reader(vdx_path, 'ffmpeg')
queues[idx] = mp.Queue()
filenames[idx] = file_name;
processes[idx] = mp.Process(target=video2motion_level, args=(vid, queues[idx]))
processes[idx].start()
idx += 1
if(idx == queue_size):
for i in range(queue_size):
processes[i].join()
flow_mag = queues[i].get()
if(flow_mag is not None):
utils.pkl_write(filenames[i],flow_mag)
print('Done ', filenames[i])
else:
print('Error in ', vdx_path)
idx = 0;
for i in range(idx):
processes[i].join()
flow_mag = queues[i].get()
if(flow_mag is not None):
utils.pkl_write(filenames[i],flow_mag)
print('Done ', filenames[i])
def load_series_data(roots,
metrics,
columns,
roots_post=None,
is_budget=False):
streams = utils.get_dirs("{0}/{1}".format(roots[0], columns[0]))
if roots_post is None:
roots_post = [] * len(roots)
all = {}
for metric in metrics:
series = {}
for col in columns:
for stream in streams:
i = 0
for root in roots:
data_path = "{0}/{1}/{2}/series/{3}.data".format(
root, col, stream, metric)
print("Reading: {0}".format(data_path))
data_file = open(data_path, "r")
data_lines = data_file.readlines()
for line in data_lines:
[row, values] = line.split(",", 1)
if is_budget:
row = row.split("-")[0]
row += roots_post[i]
values = list(
map(lambda x: float(x.replace("\n", "")),
values.split(",")))
filter_values = list(
filter(lambda x: not math.isnan(x), values))
avg = sum(filter_values) / len(filter_values)
values = list(
map(lambda x: avg if math.isnan(x) else x, values))
if row not in series:
series[row] = {}
if col not in series[row]:
series[row][col] = {}
series[row][col][stream] = values
i += 1
all[metric] = series
return {
"data": all,
"metrics": metrics,
"columns": columns,
"streams": streams,
}
def transform_files(in_path,out_path,transform,dirs=False):
utils.make_dir(out_path)
if(dirs):
names=utils.get_dirs(in_path)
else:
names=utils.get_files(in_path)
for name in names:
full_in_path=in_path+name
full_out_path=out_path+name
transform(full_in_path,full_out_path)
def sample_video(dataset_path):
dirs = utils.get_dirs(dataset_path)
print(len(dirs))
sample_activity = np.random.choice(len(dirs))
d = dirs[sample_activity]
current_dir = os.path.join(dataset_path, d)
files = utils.get_files(current_dir, '.avi', append_base=True)
sample_vdx = np.random.choice(len(files))
print(len(files))
current_vdx = files[sample_vdx]
return d, current_vdx
def load_avg_data(roots, metrics, columns, roots_post=None, is_budget=False):
streams = utils.get_dirs("{0}/{1}".format(roots[0], columns[0]))
if roots_post is None:
roots_post = [''] * len(roots)
all = {}
for metric in metrics:
averages = {}
for col in columns:
for stream in streams:
i = 0
for root in roots:
data_path = "{0}/{1}/{2}/averages/{3}.data".format(
root, col, stream, metric)
print("Reading: {0}".format(data_path))
data_file = open(data_path, "r")
data_lines = data_file.readlines()
for line in data_lines:
[row, value] = line.split(",")
if is_budget:
row = row.split("-")[0]
row += roots_post[i]
if row not in averages:
averages[row] = {}
if col not in averages[row]:
averages[row][col] = {}
averages[row][col][stream] = {
"all": float(value.replace("\n", ""))
}
i += 1
all[metric] = averages
return {
"data": all,
"streams": streams,
"metrics": metrics,
"columns": columns # todo: add rows
}
def run(args):
real_path = "list.txt"
real_dirs = sorted(utils.get_dirs(args.path))
with open(real_path, 'r', encoding='utf-8') as f:
dirs = f.readlines()
user_dirs = []
for i in range(len(dirs)):
tmp = dirs[i].split('|')
if tmp[1] == args.settingsActive:
user_dirs.append(tmp[0])
user_dirs = sorted(user_dirs)
if len(user_dirs) == len(real_dirs):
print(','.join(user_dirs))
else:
utils.log('compare', tp='error')
real_dirs.reverse()
user_dirs.reverse()
utils.log('file_dirs', ','.join(real_dirs), 'error')
utils.log('statistical', ','.join(user_dirs), 'error')
def main(_):
model_dir, data_dir = get_dirs(conf, ['exp_name'])
# exp_start_time = datetime.datetime.now().strftime("%A_%b%d-%H%M%S")
# data_dir = "logs/" + conf.exp_name + "_" + exp_start_time
preprocess_conf(conf, model_dir)
env = gym.make(conf.env_name)
env.seed(conf.random_seed)
state_shape = env.observation_space.shape
if type(env.action_space) is gym.spaces.Discrete:
action_shape = env.action_space.n
else:
action_shape = env.action_space.shape[0]
# replay buffer
buffer = ReplayBuffer2(conf.buffer_size)
# building agent
# config = tf.ConfigProto(allow_soft_placement=True)
# config.gpu_options.allow_growth = True
config = tf.ConfigProto(intra_op_parallelism_threads=8,
inter_op_parallelism_threads=8)
with tf.Session(config=config) as sess:
# agent
agent = SoftPolicyGradient(sess, conf, state_shape, action_shape)
# statistic
stat = Statistic(sess, conf, model_dir, data_dir)
if conf.load_model:
stat.load_model()
def var_print():
for var in tf.global_variables():
print(var)
print("printing vars:------------------------------------------------")
var_print()
print(
"printing vars::------------------------------------------------")
start_steps = 1000
episode, global_step, local_step = 0, 0, 0
epi_rewards = 0
total_Q, Q_loss, pi_loss = [], [], []
state = env.reset()
# pbar = tqdm(total=conf.max_steps, dynamic_ncols=True)
while global_step < conf.max_steps:
# interaction with environment
action = agent.sampling_actions(
[state], is_deterministic=False)[0] # [-inf, inf]
next_state, reward, done, info = env.step(
action_converter(env, action))
global_step += 1
local_step += 1
epi_rewards += reward
reward *= conf.reward_scale
buffer.add_transition(state, action, reward, next_state, done)
state = next_state
# train step
if buffer.size() >= conf.batch_size and global_step >= start_steps:
for i in range(conf.num_train_steps):
transitions = buffer.get_transitions(conf.batch_size)
Q, single_Q_loss, single_pi_loss = agent.trainer(
transitions)
total_Q.append(np.mean(Q))
Q_loss.append(single_Q_loss)
pi_loss.append(single_pi_loss)
# evaluate step
if global_step % conf.eval_interval == 0:
ave_epi_rewards = np.mean(eval_step(env, agent))
stat.save_step(global_step, ave_epi_rewards)
print('\n[Evaluation] averaged_epi_rewards: %.3f' %
ave_epi_rewards)
if done:
# save step
all_epi_rewards.append(epi_rewards)
stat.save_step(global_step, epi_rewards, np.mean(total_Q),
np.mean(Q_loss), np.mean(pi_loss))
# pbar.update(local_step)
lenn = len(all_epi_rewards)
fromm = max(lenn - 20, 0)
to = lenn
min_5_ep_ret = min(all_epi_rewards[fromm:to])
# pbar.set_description('Episode: %s, epi_rewards: %.3f, pi_loss: %.3f, Q_loss: %.3f avg_5_epi_rew %.1f' %
# (episode+1, epi_rewards, np.mean(pi_loss), np.mean(Q_loss), sum(all_epi_rewards[fromm:to])/(to-fromm) ) )
print(
'Episode: %s, epi_rewards: %.3f, pi_loss: %.3f, Q_loss: %.3f \tmin_5_epi_rew %.1f'
% (episode + 1, epi_rewards, np.mean(pi_loss),
np.mean(Q_loss), min_5_ep_ret))
threshold = -500.0
if ((to - fromm) > 3 and min_5_ep_ret > threshold):
time_end = time.time()
print("SHI hyperParams have made algo converge (",
threshold, ") in ", (time_end - time_begin) / 1.0,
" s")
stat.save_step(global_step, epi_rewards, np.mean(total_Q),
np.mean(Q_loss), np.mean(pi_loss))
stat.save_model(global_step)
sys.exit()
episode += 1
local_step = 0
epi_rewards = 0
total_Q, Q_loss, pi_loss = [], [], []
state = env.reset()
def main(_):
model_dir, data_dir = get_dirs(conf, ['env_name'])
preprocess_conf(conf, model_dir)
env = gym.make(conf.env_name)
# env.seed(conf.random_seed)
state_shape = env.observation_space.shape
if type(env.action_space) is gym.spaces.Discrete:
action_shape = env.action_space.n
else:
action_shape = env.action_space.shape[0]
# replay buffer
buffer = ReplayBuffer2(conf.buffer_size)
# building agent
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# agent
agent = SoftPolicyGradient(sess, conf, state_shape, action_shape)
# statistic
stat = Statistic(sess, conf, model_dir, data_dir)
if conf.load_model:
stat.load_model()
episode, global_step, local_step = 0, 0, 0
epi_rewards = 0
total_Q, Q_loss, pi_loss = [], [], []
state = env.reset()
pbar = tqdm(total=conf.max_steps, dynamic_ncols=True)
while global_step < conf.max_steps:
# interaction with environment
action = agent.sampling_actions([state], is_deterministic=False)[0] # [-inf, inf]
next_state, reward, done, info = env.step(action_converter(env, action))
global_step += 1
local_step += 1
epi_rewards += reward
reward *= conf.reward_scale
buffer.add_transition(state, action, reward, next_state, done)
state = next_state
# train step
if buffer.size() >= conf.batch_size:
for i in range(conf.num_train_steps):
transitions = buffer.get_transitions(conf.batch_size)
Q, single_Q_loss, single_pi_loss = agent.trainer(transitions)
total_Q.append(np.mean(Q))
Q_loss.append(single_Q_loss)
pi_loss.append(single_pi_loss)
# evaluate step
if global_step % conf.eval_interval == 0:
ave_epi_rewards = np.mean(eval_step(env, agent))
stat.save_step(global_step, ave_epi_rewards)
print('\n[Evaluation] averaged_epi_rewards: %.3f' % ave_epi_rewards)
if done:
# save step
stat.save_step(global_step, epi_rewards, np.mean(total_Q), np.mean(Q_loss), np.mean(pi_loss))
pbar.update(local_step)
pbar.set_description('Episode: %s, epi_rewards: %.3f, pi_loss: %.3f, Q_loss: %.3f' %
(episode+1, epi_rewards, np.mean(pi_loss), np.mean(Q_loss)))
print()
episode += 1
local_step = 0
epi_rewards = 0
total_Q, Q_loss, pi_loss = [], [], []
state = env.reset()
pbar.close()
def main():
# General parameters
net = ['base', 'cifar', 'emb+soft', 'resnet50', 'resnet20',
'local_feat'][0]
database = ['cifar10', 'mnist', 'fashion_mnist', 'skillup'][1]
epochs = 10
learn_rate = 0.01
decay = (learn_rate / epochs) * 0.8
ims_per_id = 8
ids_per_batch = 8
margin = 0.9
embedding_size = 64
squared = False
data_augmentation = False
patience = 25
# built model's parameters
dropout = 0.3
blocks = 3
n_channels = 32
weight_decay = 1e-4 * 0
# dataloader parameters
use_dataloader = True
path = '/home/daniel/proyectos/product_detection/web_market_preproces/duke_from_images'
exp_dir, log_dir, model_weights_path, model_name = get_dirs(database)
tl_object = TripletLoss(ims_per_id=ims_per_id,
ids_per_batch=ids_per_batch,
margin=margin,
squared=squared)
tl_h = TripletLoss(ims_per_id, ids_per_batch, margin, squared)
opt = optimizers.Adam(lr=learn_rate, decay=decay)
data, input_size = get_database(database)
im_size = input_size[:2]
data_gen_args_train = dict(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
10, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range=0.1, # Randomly zoom image
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False)
if not data_augmentation:
data_gen_args_train = {}
model_args = dict(embedding_dim=embedding_size,
input_shape=input_size,
drop=dropout,
blocks=blocks,
n_channels=n_channels,
weight_decay=weight_decay,
layer_limit=173,
patience=patience)
data_loader_args = dict(path=path,
ims_per_id=ims_per_id,
ids_per_batch=ids_per_batch,
target_image_size=im_size,
data_gen_args=data_gen_args_train,
preprocess_unit=True,
data=data)
if database == 'skillup':
dl = FileDataloader(**data_loader_args)
else:
dl = StaticDataloader(**data_loader_args)
model = get_net_object(net, model_args)
model.compile(opt, tl_object.cluster_loss)
if use_dataloader:
model.train_generator(dl, model_weights_path, epochs, log_dir)
else:
model.train(data, model_weights_path, epochs,
ims_per_id * ids_per_batch, log_dir)
model.save_model(model_weights_path)
visualize_embeddings(database=database,
model_dir=exp_dir,
model_name=model_name,
model=model.model)
import sys
sys.path.append('../')
import utils
import os
import numpy as np
if __name__ == '__main__':
path = utils.get_dataset_path('UCF50')
dirs = utils.get_dirs(path)
train_list = []
val_list = []
test_list = []
train_set_percentage = 0.7
val_set_percentage_from_training = 0.2
for d in dirs:
current_dir = os.path.join(path, d)
files = utils.get_files(current_dir, '.avi', append_base=False)
files = [os.path.join(d, f) for f in files]
np.random.shuffle(files)
num_train = int(len(files) * train_set_percentage)
num_val = int(num_train * val_set_percentage_from_training)
train_list.extend(files[0:num_train - num_val])
val_list.extend(files[num_train - num_val:num_train])
test_list.extend(files[num_train:])
print(len(files), len(train_list), len(val_list), len(test_list))
save_path = path + '_lists'
utils.touch_dir(save_path)
utils.txt_write(save_path + '/trainlist.txt', train_list)
utils.txt_write(save_path + '/vallist.txt', val_list)
def parse_year(path, workers=WORKERS):
#@TODO add & parse session metadata
log.info('Parsing year data at "%s", using %d workers' % (path, workers))
sessions = sorted(get_dirs(path))
return dict(Parallel(n_jobs=workers)(delayed(parse_session)(one) for one in sessions))
def read_action_frame(dir_path):
actions_paths=utils.get_dirs(dir_path)
actions_paths=utils.append_path(dir_path, actions_paths)
actions=[read_action(path) for path in actions_paths]
return create_action_frame(actions)
def test_seg_validation(net_name):
# Init
c = color_codes()
options = parse_inputs()
depth = options['blocks']
filters = options['filters']
d_path = options['loo_dir']
v_path = options['val_dir']
seg_path = os.path.join(v_path, 'segmentation')
if not os.path.isdir(seg_path):
os.mkdir(seg_path)
unc_path = os.path.join(v_path, 'uncertainty')
if not os.path.isdir(unc_path):
os.mkdir(unc_path)
p = get_dirs(d_path)[0]
test_patients = filter(lambda p: 'BraTS19' in p, get_dirs(v_path))
_, test_x = get_images(test_patients, True)
print(
'Testing patients = %d' % (
len(test_patients)
)
)
# The sub-regions considered for evaluation are:
# 1) the "enhancing tumor" (ET)
# 2) the "tumor core" (TC)
# 3) the "whole tumor" (WT)
#
# The provided segmentation labels have values of 1 for NCR & NET,
# 2 for ED, 4 for ET, and 0 for everything else.
# The participants are called to upload their segmentation labels
# as a single multi-label file in nifti (.nii.gz) format.
#
# The participants are called to upload 4 nifti (.nii.gz) volumes
# (3 uncertainty maps and 1 multi-class segmentation volume from
# Task 1) onto CBICA's Image Processing Portal format. For example,
# for each ID in the dataset, participants are expected to upload
# following 4 volumes:
# 1. {ID}.nii.gz (multi-class label map)
# 2. {ID}_unc_whole.nii.gz (Uncertainty map associated with whole tumor)
# 3. {ID}_unc_core.nii.gz (Uncertainty map associated with tumor core)
# 4. {ID}_unc_enhance.nii.gz (Uncertainty map associated with enhancing tumor)
for i, (p_i, test_i) in enumerate(zip(test_patients, test_x)):
t_in = time.time()
# unc_i = np.zeros((4,) + test_i.shape[1:])
pred_i = np.zeros((4,) + test_i.shape[1:])
for f in range(5):
model_name = '%s-f%d.mdl' % (net_name, f)
net = BratsSegmentationNet(depth=depth, filters=filters)
net.load_model(os.path.join(d_path, model_name))
#
# unc_i += net.uncertainty([test_i], steps=10)[0] * 0.2
pred_i += net.segment([test_i])[0]
seg_i = np.argmax(pred_i, axis=0)
seg_i[seg_i == 3] = 4
# seg_unc_i = np.argmax(unc_i, axis=0)
# seg_unc_i[seg_unc_i == 3] = 4
tumor_mask = remove_small_regions(
seg_i.astype(np.bool), min_size=30
)
seg_i[log_not(tumor_mask)] = 0
# seg_unc_i[log_not(tumor_mask)] = 0
#
# whole_i = np.sum(unc_i[1:]) * tumor_mask.astype(np.float32)
# core_i = unc_i[1] + unc_i[-1] * tumor_mask.astype(np.float32)
# enhance_i = unc_i[-1] * tumor_mask.astype(np.float32)
#
# seg_unc_i = np.argmax(unc_i, axis=0)
# seg_unc_i[seg_unc_i == 3] = 4
niiname = os.path.join(d_path, p, p + '_seg.nii.gz')
nii = load_nii(niiname)
nii.get_data()[:] = seg_i
save_nii(nii, os.path.join(seg_path, p_i + '.nii.gz'))
# nii.get_data()[:] = seg_unc_i
# save_nii(nii, os.path.join(unc_path, p_i + '.nii.gz'))
# niiname = os.path.join(v_path, p_i, p_i + '_flair.nii.gz')
# nii = load_nii(niiname)
# nii.get_data()[:] = whole_i
# save_nii(
# nii, os.path.join(unc_path, p_i + '_unc_whole.nii.gz')
# )
# nii.get_data()[:] = core_i
# save_nii(
# nii, os.path.join(unc_path, p_i + '_unc_core.nii.gz')
# )
# nii.get_data()[:] = enhance_i
# save_nii(
# nii, os.path.join(unc_path, p_i + '_unc_enhance.nii.gz')
# )
t_s = time_to_string(time.time() - t_in)
print(
'Finished patient %s (%d/%d) %s' % (p_i, i + 1, len(test_x), t_s)
)
def get_seasons(self):
path = os.path.join(config.DATA_ROOT, self.name)
return [Season(name, self) for name in utils.get_dirs(path)]
(opts, args) = op.parse_args()
if not opts.base_dir:
logger.error("You must give base directory")
op.print_help()
exit()
if not opts.config or not os.path.exists(opts.config):
logger.error("No such config file: %s", opts.config)
op.print_help()
exit()
# make the filename as suffix of compiled binaries
suffix = "_" + os.path.splitext(os.path.basename(opts.config))[0]
base_dir = os.path.join(opts.base_dir, "gnu" + suffix)
src_dir, out_dir, log_dir = get_dirs(base_dir)
os.makedirs(src_dir, exist_ok=True)
os.makedirs(out_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
logger.info("base directory : %s", base_dir)
logger.info("source directory : %s", src_dir)
logger.info("output directory : %s", out_dir)
logger.info("log directory : %s", log_dir)
with open(opts.config, "r") as f:
config = yaml.load(f)
opti_list = config["opti"]
arch_list = config["arch"]
compiler_list = config["compiler"]
num_opt = len(opti_list) * len(arch_list) * len(compiler_list)
def Train_No_GAN(opt): # w / o GAN
# ----------------------------------------
# Network training parameters
# ----------------------------------------
# cudnn benchmark
cudnn.benchmark = opt.cudnn_benchmark
# Loss functions
criterion_L1 = torch.nn.L1Loss().cuda()
# Initialize Generator
generatorNet = utils.create_generator(opt)
flownet = utils.create_pwcnet(opt)
# To device
if opt.multi_gpu:
generatorNet = nn.DataParallel(generatorNet)
generatorNet = generatorNet.cuda()
flownet = nn.DataParallel(flownet)
flownet = flownet.cuda()
else:
generatorNet = generatorNet.cuda()
flownet = flownet.cuda()
# Optimizers
optimizer_G = torch.optim.Adam(generatorNet.parameters(), lr = opt.lr_g, betas = (opt.b1, opt.b2), weight_decay = opt.weight_decay)
# Learning rate decrease
def adjust_learning_rate(opt, epoch, iteration, optimizer):
#Set the learning rate to the initial LR decayed by "lr_decrease_factor" every "lr_decrease_epoch" epochs
if opt.lr_decrease_mode == 'epoch':
lr = opt.lr_g * (opt.lr_decrease_factor ** (epoch // opt.lr_decrease_epoch))
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if opt.lr_decrease_mode == 'iter':
lr = opt.lr_g * (opt.lr_decrease_factor ** (iteration // opt.lr_decrease_iter))
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# Save the model if pre_train == True
def save_model(opt, epoch, iteration, len_dataset, generator):
"""Save the model at "checkpoint_interval" and its multiple"""
if opt.multi_gpu == True:
if opt.save_mode == 'epoch':
if (epoch % opt.save_by_epoch == 0) and (iteration % len_dataset == 0):
if opt.save_name_mode:
torch.save(generator.module, 'Pre_%s_epoch%d_bs%d.pth' % (opt.task, epoch, opt.batch_size))
print('The trained model is successfully saved at epoch %d' % (epoch))
if opt.save_mode == 'iter':
if iteration % opt.save_by_iter == 0:
if opt.save_name_mode:
torch.save(generator.module, 'Pre_%s_iter%d_bs%d.pth' % (opt.task, iteration, opt.batch_size))
print('The trained model is successfully saved at iteration %d' % (iteration))
else:
if opt.save_mode == 'epoch':
if (epoch % opt.save_by_epoch == 0) and (iteration % len_dataset == 0):
if opt.save_name_mode:
torch.save(generator, 'Pre_%s_epoch%d_bs%d.pth' % (opt.task, epoch, opt.batch_size))
print('The trained model is successfully saved at epoch %d' % (epoch))
if opt.save_mode == 'iter':
if iteration % opt.save_by_iter == 0:
if opt.save_name_mode:
torch.save(generator, 'Pre_%s_iter%d_bs%d.pth' % (opt.task, iteration, opt.batch_size))
print('The trained model is successfully saved at iteration %d' % (iteration))
# ----------------------------------------
# Network dataset
# ----------------------------------------
# Define the class list
imglist = utils.text_readlines('videocolor_linux.txt')
classlist = utils.get_dirs(opt.baseroot)
'''
imgnumber = len(imglist) - (len(imglist) % opt.batch_size)
imglist = imglist[:imgnumber]
'''
# Define the dataset
trainset = dataset.MultiFramesDataset(opt, imglist, classlist)
print('The overall number of classes:', len(trainset))
# Define the dataloader
dataloader = utils.create_dataloader(trainset, opt)
# ----------------------------------------
# Training
# ----------------------------------------
# Count start time
prev_time = time.time()
# For loop training
for epoch in range(opt.epochs):
for iteration, (in_part, out_part) in enumerate(dataloader):
# Train Generator
optimizer_G.zero_grad()
lstm_state = None
loss_flow = 0
loss_flow_long = 0
loss_L1 = 0
x_0 = in_part[0].cuda()
p_t_0 = in_part[0].cuda()
for iter_frame in range(opt.iter_frames):
# Read data
x_t = in_part[iter_frame].cuda()
y_t = out_part[iter_frame].cuda()
# Initialize the second input and compute flow loss
if iter_frame == 0:
p_t_last = torch.zeros(opt.batch_size, opt.out_channels, opt.resize_h, opt.resize_w).cuda()
elif iter_frame == 1:
x_t_last = in_part[iter_frame - 1].cuda()
p_t_last = p_t.detach()
p_t_0 = p_t.detach()
p_t_last.requires_grad = False
p_t_0.requires_grad = False
# o_t_last_2_t range is [-20, +20]
o_t_last_2_t = pwcnet.PWCEstimate(flownet, x_t, x_t_last)
x_t_warp = pwcnet.PWCNetBackward((x_t_last + 1) / 2, o_t_last_2_t)
# y_t_warp range is [0, 1]
p_t_warp = pwcnet.PWCNetBackward((p_t_last + 1) / 2, o_t_last_2_t)
else:
x_t_last = in_part[iter_frame - 1].cuda()
p_t_last = p_t.detach()
p_t_last.requires_grad = False
# o_t_last_2_t o_t_first_2_t range is [-20, +20]
o_t_last_2_t = pwcnet.PWCEstimate(flownet, x_t, x_t_last)
o_t_first_2_t = pwcnet.PWCEstimate(flownet,x_t, x_0)
# y_t_warp, y_t_warp_long range is [0, 1]
x_t_warp = pwcnet.PWCNetBackward((x_t_last + 1) / 2, o_t_last_2_t)
p_t_warp = pwcnet.PWCNetBackward((p_t_last + 1) / 2, o_t_last_2_t)
x_t_warp_long = pwcnet.PWCNetBackward((x_0 + 1) / 2, o_t_first_2_t)
p_t_warp_long = pwcnet.PWCNetBackward((p_t_0 + 1) / 2, o_t_first_2_t)
# Generator output
p_t, lstm_state = generatorNet(x_t, p_t_last, lstm_state)
lstm_state = utils.repackage_hidden(lstm_state)
if iter_frame == 1:
mask_flow = torch.exp( -opt.mask_para * torch.sum((x_t + 1) / 2 - x_t_warp, dim=1).pow(2) ).unsqueeze(1)
loss_flow += criterion_L1(mask_flow * (p_t + 1) / 2, mask_flow * p_t_warp)
elif iter_frame > 1:
mask_flow = torch.exp( -opt.mask_para * torch.sum((x_t + 1) / 2 - x_t_warp, dim=1).pow(2) ).unsqueeze(1)
loss_flow += criterion_L1(mask_flow * (p_t + 1) / 2, mask_flow * p_t_warp)
mask_flow_long = torch.exp( -opt.mask_para * torch.sum((x_t + 1) / 2 - x_t_warp_long, dim=1).pow(2) ).unsqueeze(1)
loss_flow_long += criterion_L1(mask_flow_long * (p_t + 1) / 2, mask_flow_long * p_t_warp_long)
# Pixel-level loss
loss_L1 += criterion_L1(p_t, y_t)
# Overall Loss and optimize
loss = loss_L1 + opt.lambda_flow * loss_flow + opt.lambda_flow_long * loss_flow_long
loss.backward()
optimizer_G.step()
# Determine approximate time left
iters_done = epoch * len(dataloader) + iteration
iters_left = opt.epochs * len(dataloader) - iters_done
time_left = datetime.timedelta(seconds = iters_left * (time.time() - prev_time))
prev_time = time.time()
# Print log
print("\r[Epoch %d/%d] [Batch %d/%d] [L1 Loss: %.4f] [Flow Loss Short: %.8f] [Flow Loss Long: %.8f] Time_left: %s" %
((epoch + 1), opt.epochs, iteration, len(dataloader), loss_L1.item(), loss_flow.item(), loss_flow_long.item(), time_left))
# Save model at certain epochs or iterations
save_model(opt, (epoch + 1), (iters_done + 1), len(dataloader), generatorNet)
# Learning rate decrease at certain epochs
adjust_learning_rate(opt, (epoch + 1), (iters_done + 1), optimizer_G)
os.path.join(dataset_path + '_lists', 'vallist.txt'))
print('**** Val has ', len(files_list))
save_dir = os.path.join(save_dir, 'val')
max_num_tuplex = 100 ## 20000
elif (current_subset == const.Subset.TEST.value):
files_list = utils.txt_read(
os.path.join(dataset_path + '_lists', 'testlist.txt'))
print('*** Test has ', len(files_list))
save_dir = os.path.join(save_dir, 'test')
utils.touch_dir(save_dir)
tuple_idx = 0
tuple_idx = max(0,
utils.last_tuple_idx(save_dir) -
5) ## As caution, regenerate last 5 tuples
activity_list = sorted(utils.get_dirs(dataset_path))
print('activity_list ', len(activity_list), activity_list)
lbls_file = os.path.join(save_dir, 'lbl.pkl')
if (tuple_idx == 0):
lbls_ary = np.ones(max_num_tuplex, dtype=np.int32) * -1
## Invalid Activity
else:
lbls_ary = utils.pkl_read(lbls_file)
tuple_idx = 0
import time
start_time = time.time()
queue_size = 25
# built model's parameters
dropout = 0.35 # Dropout probability of each layer. Conv layers use SpatialDropout2D
blocks = 6 # Number of (Conv -> Act -> BN -> MaxPool -> Dropout) blocks
n_channels = args[
'channels'] # Number of channels (or feature maps) of the first convolution block.
# the following ones are 1.5 times the number of channels of the previous block
weight_decay = 1e-4 * 0
# dataloader parameters.
# Folder's path where the files query.txt and bounding_box_train.txt are
# query.txt contains the path and the class of test images
# bounding_box_train.txt contains the path and the class of train images
path = args['path']
exp_dir, log_dir, model_weights_path, model_name = get_dirs(database)
print(exp_dir, log_dir, model_weights_path, model_name)
data, input_size = get_database(
database) # if database == 'skillup'. data is None
im_size = input_size[:2]
data_gen_args_train = dict(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=10, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range=0.1, # Randomly zoom image
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
def get_actions(action_dir,cls):
action_files=utils.get_dirs(action_dir)
action_files=utils.append_path(action_dir,action_files)
return [compute_sequence(path,cls) for path in action_files]
(opts, args) = op.parse_args()
if not opts.base_dir:
logger.error("You must give base directory")
op.print_help()
exit()
if not opts.config or not os.path.exists(opts.config):
logger.error("No such config file: %s", opts.config)
op.print_help()
exit()
# make the filename as suffix of compiled binaries
suffix = "_" + os.path.splitext(os.path.basename(opts.config))[0]
base_dir = opts.base_dir
src_dir, out_dir, log_dir = get_dirs(opts.base_dir, suffix)
os.makedirs(src_dir, exist_ok=True)
os.makedirs(out_dir, exist_ok=True)
os.makedirs(log_dir, exist_ok=True)
logger.info("base directory : %s", base_dir)
logger.info("source directory : %s", src_dir)
logger.info("output directory : %s", out_dir)
logger.info("log directory : %s", log_dir)
with open(opts.config, "r") as f:
config = yaml.safe_load(f)
opti_list = config["opti"]
arch_list = config["arch"]
compiler_list = config["compiler"]
num_opt = len(opti_list) * len(arch_list) * len(compiler_list)
def train_test_seg(net_name, n_folds, val_split=0.1):
# Init
c = color_codes()
options = parse_inputs()
depth = options['blocks']
filters = options['filters']
d_path = options['loo_dir']
unc_path = os.path.join(d_path, 'uncertainty')
if not os.path.isdir(unc_path):
os.mkdir(unc_path)
seg_path = os.path.join(d_path, 'segmentation')
if not os.path.isdir(seg_path):
os.mkdir(seg_path)
patients = get_dirs(d_path)
cbica = filter(lambda p: 'CBICA' in p, patients)
tcia = filter(lambda p: 'TCIA' in p, patients)
tmc = filter(lambda p: 'TMC' in p, patients)
b2013 = filter(lambda p: '2013' in p, patients)
for i in range(n_folds):
print(
'%s[%s] %sFold %s(%s%d%s%s/%d)%s' % (
c['c'], strftime("%H:%M:%S"), c['g'],
c['c'], c['b'], i + 1, c['nc'], c['c'], n_folds, c['nc']
)
)
# Training itself
# Data split (using the patient names) for train and validation.
# We also compute the number of batches for both training and
# validation according to the batch size.
''' Training '''
ini_cbica = len(cbica) * i / n_folds
end_cbica = len(cbica) * (i + 1) / n_folds
fold_cbica = cbica[:ini_cbica] + cbica[end_cbica:]
n_fold_cbica = len(fold_cbica)
n_cbica = int(n_fold_cbica * (1 - val_split))
ini_tcia = len(tcia) * i / n_folds
end_tcia = len(tcia) * (i + 1) / n_folds
fold_tcia = tcia[:ini_tcia] + tcia[end_tcia:]
n_fold_tcia = len(fold_tcia)
n_tcia = int(n_fold_tcia * (1 - val_split))
ini_tmc = len(tmc) * i / n_folds
end_tmc = len(tmc) * (i + 1) / n_folds
fold_tmc = tmc[:ini_tmc] + tmc[end_tmc:]
n_fold_tmc = len(fold_tmc)
n_tmc = int(n_fold_tmc * (1 - val_split))
ini_b2013 = len(b2013) * i / n_folds
end_b2013 = len(b2013) * (i + 1) / n_folds
fold_b2013 = b2013[:ini_b2013] + b2013[end_b2013:]
n_fold_b2013 = len(fold_b2013)
n_b2013 = int(n_fold_b2013 * (1 - val_split))
training_n = n_fold_cbica + n_fold_tcia + n_fold_tmc + n_fold_b2013
testing_n = len(patients) - training_n
print(
'Training / testing samples = %d / %d' % (
training_n, testing_n
)
)
# Training
train_cbica = fold_cbica[:n_cbica]
train_tcia = fold_tcia[:n_tcia]
train_tmc = fold_tmc[:n_tmc]
train_b2013 = fold_b2013[:n_b2013]
train_patients = train_cbica + train_tcia + train_tmc + train_b2013
# Validation
val_cbica = fold_cbica[n_cbica:]
val_tcia = fold_tcia[n_tcia:]
val_tmc = fold_tmc[n_tmc:]
val_b2013 = fold_b2013[n_b2013:]
val_patients = val_cbica + val_tcia + val_tmc + val_b2013
model_name = '%s-f%d.mdl' % (net_name, i)
net = BratsSegmentationNet(depth=depth, filters=filters)
# train_seg(net, model_name, train_patients, val_patients)
train_seg(
net, model_name, train_patients, val_patients, dropout=0
)
# model_name = '%s-f%d-R.mdl' % (net_name, i)
# train_seg(
# net, model_name, train_patients, val_patients,
# refine=True, dropout=0.5, lr=1e-2
# )
# Testing data (with GT)
test_cbica = cbica[ini_cbica:end_cbica]
test_tcia = tcia[ini_tcia:end_tcia]
test_tmc = tmc[ini_tmc:end_tmc]
test_b2013 = b2013[ini_b2013:end_b2013]
test_patients = test_cbica + test_tcia + test_tmc + test_b2013
patient_paths = map(lambda p: os.path.join(d_path, p), test_patients)
_, test_x = get_images(test_patients)
print(
'Testing patients (with GT) = %d' % (
len(test_patients)
)
)
# The sub-regions considered for evaluation are:
# 1) the "enhancing tumor" (ET)
# 2) the "tumor core" (TC)
# 3) the "whole tumor" (WT)
#
# The provided segmentation labels have values of 1 for NCR & NET,
# 2 for ED, 4 for ET, and 0 for everything else.
# The participants are called to upload their segmentation labels
# as a single multi-label file in nifti (.nii.gz) format.
#
# The participants are called to upload 4 nifti (.nii.gz) volumes
# (3 uncertainty maps and 1 multi-class segmentation volume from
# Task 1) onto CBICA's Image Processing Portal format. For example,
# for each ID in the dataset, participants are expected to upload
# following 4 volumes:
# 1. {ID}.nii.gz (multi-class label map)
# 2. {ID}_unc_whole.nii.gz (Uncertainty map associated with whole tumor)
# 3. {ID}_unc_core.nii.gz (Uncertainty map associated with tumor core)
# 4. {ID}_unc_enhance.nii.gz (Uncertainty map associated with enhancing tumor)
for p, (path_i, p_i, test_i) in enumerate(zip(
patient_paths, test_patients, test_x
)):
pred_i = net.segment([test_i])[0]
# unc_i = net.uncertainty([test_i], steps=25)[0]
# whole_i = np.sum(unc_i[1:])
# core_i = unc_i[1] + unc_i[-1]
# enhance_i = unc_i[-1]
seg_i = np.argmax(pred_i, axis=0)
seg_i[seg_i == 3] = 4
# seg_unc_i = np.argmax(unc_i, axis=0)
# seg_unc_i[seg_unc_i == 3] = 4
# tumor_mask = remove_small_regions(
# seg_i.astype(np.bool), min_size=30
# )
#
# seg_i[log_not(tumor_mask)] = 0
# seg_unc_i[log_not(tumor_mask)] = 0
#
# whole_i *= tumor_mask.astype(np.float32)
# core_i *= tumor_mask.astype(np.float32)
# enhance_i *= tumor_mask.astype(np.float32)
niiname = os.path.join(path_i, p_i + '_seg.nii.gz')
nii = load_nii(niiname)
seg = nii.get_data()
dsc = map(
lambda label: dsc_seg(seg == label, seg_i == label),
[1, 2, 4]
)
# dsc_unc = map(
# lambda label: dsc_seg(seg == label, seg_unc_i == label),
# [1, 2, 4]
# )
nii.get_data()[:] = seg_i
save_nii(nii, os.path.join(seg_path, p_i + '.nii.gz'))
# nii.get_data()[:] = seg_unc_i
# save_nii(nii, os.path.join(unc_path, p_i + '.nii.gz'))
# niiname = os.path.join(d_path, p_i, p_i + '_flair.nii.gz')
# nii = load_nii(niiname)
# nii.get_data()[:] = whole_i
# save_nii(nii, os.path.join(unc_path, p_i + '_unc_whole.nii.gz'))
# nii.get_data()[:] = core_i
# save_nii(nii, os.path.join(unc_path, p_i + '_unc_core.nii.gz'))
# nii.get_data()[:] = enhance_i
# save_nii(nii, os.path.join(unc_path, p_i + '_unc_enhance.nii.gz'))
print(
'Segmentation - Patient %s (%d/%d): %s' % (
p_i, p, len(test_x), ' / '.join(map(str, dsc))
)
)
def train_test_survival(net_name, n_folds, val_split=0.1):
# Init
c = color_codes()
options = parse_inputs()
epochs = options['epochs']
batch_size = options['batch_size']
patch_size = options['patch_size']
patience = options['patience']
depth = options['blocks']
filters = options['filters']
d_path = options['loo_dir']
patients = get_dirs(d_path)
survival_dict = get_survival_data()
seg_patients = filter(lambda p: p not in survival_dict.keys(), patients)
survival_patients = survival_dict.keys()
low_patients = filter(
lambda p: int(survival_dict[p]['Survival']) < 300,
survival_patients
)
mid_patients = filter(
lambda p: (int(survival_dict[p]['Survival']) >= 300) &
(int(survival_dict[p]['Survival']) < 450),
survival_patients
)
hi_patients = filter(
lambda p: int(survival_dict[p]['Survival']) >= 450,
survival_patients
)
t_survival_dict = get_survival_data(True)
t_survival_patients = t_survival_dict.keys()
test_survivals = np.zeros(len(t_survival_patients))
t_survival_ages = map(lambda v: float(v['Age']), t_survival_dict.values())
''' Segmentation training'''
# The goal here is to pretrain a unique segmentation network for all
# the survival folds. We only do it once. Then we split the survival
# patients accordingly.
net = BratsSurvivalNet(depth_seg=depth, depth_pred=depth, filters=filters)
print(
'Training segmentation samples = %d' % (
len(seg_patients)
)
)
# Training itself
model_name = '%s-init.mdl' % net_name
num_workers = 8
try:
net.base_model.load_model(os.path.join(d_path, model_name))
except IOError:
n_params = sum(
p.numel() for p in net.base_model.parameters() if p.requires_grad
)
print(
'%sStarting segmentation training with a unet%s (%d parameters)' %
(c['c'], c['nc'], n_params)
)
targets = get_labels(seg_patients)
rois, data = get_images(seg_patients)
print('< Training dataset >')
if patch_size is None:
seg_dataset = BBImageDataset(
data, targets, rois, flip=True
)
else:
seg_dataset = BratsDataset(
data, targets, rois, patch_size
)
print('Dataloader creation <with validation>')
train_loader = DataLoader(
seg_dataset, batch_size, True, num_workers=num_workers,
)
targets = get_labels(survival_patients)
rois, data = get_images(survival_patients)
# Validation
print('< Validation dataset >')
val_dataset = BBImageDataset(
data, targets, rois
)
val_loader = DataLoader(
val_dataset, 1, num_workers=num_workers
)
print(
'%s[%s] %sInitial %s%spretraining%s' % (
c['c'], strftime("%H:%M:%S"), c['g'],
c['c'], c['b'], c['nc']
)
)
net.base_model.fit(train_loader, val_loader, epochs=epochs, patience=patience)
net.base_model.save_model(os.path.join(d_path, model_name))
tr_csv = os.path.join(options['loo_dir'], 'survival_results.csv')
val_csv = os.path.join(options['val_dir'], 'survival_results.csv')
with open(tr_csv, 'w') as csvfile, open(val_csv, 'w') as val_csvfile:
csvwriter = csv.writer(csvfile, delimiter=',')
val_csvwriter = csv.writer(val_csvfile, delimiter=',')
# First we'll define the maximum bounding box, for training and testing
masks, _ = get_images(survival_patients)
indices = map(lambda mask: np.where(mask > 0), masks)
min_bb = np.min(
map(
lambda idx: np.min(idx, axis=-1),
indices
),
axis=0
)
max_bb = np.max(
map(
lambda idx: np.max(idx, axis=-1),
indices
),
axis=0
)
bb = map(
lambda (min_i, max_i): slice(min_i, max_i),
zip(min_bb, max_bb)
)
# After that, we can finally train the model to predict the survival.
for i in range(n_folds):
model_name = '%s-init.mdl' % net_name
net = BratsSurvivalNet(depth_seg=depth, depth_pred=depth, filters=filters)
net.base_model.load_model(os.path.join(d_path, model_name))
''' Survival training'''
model_name = '%s_f%d.mdl' % (net_name, i)
print(
'%s[%s] %sFold %s(%s%d%s%s/%d)%s' % (
c['c'], strftime("%H:%M:%S"), c['g'],
c['c'], c['b'], i + 1, c['nc'], c['c'], n_folds, c['nc']
)
)
try:
net.load_model(os.path.join(d_path, model_name))
except IOError:
# Train/Test patient split according to survival classes
hi_ini = len(hi_patients) * i / n_folds
mid_ini = len(mid_patients) * i / n_folds
low_ini = len(low_patients) * i / n_folds
hi_end = len(hi_patients) * (i + 1) / n_folds
mid_end = len(mid_patients) * (i + 1) / n_folds
low_end = len(low_patients) * (i + 1) / n_folds
high_i = hi_patients[:hi_ini] + hi_patients[hi_end:]
mid_i = mid_patients[:mid_ini] + mid_patients[mid_end:]
low_i = low_patients[:low_ini] + low_patients[low_end:]
# Split of the target survivals
hi_survival = map(
lambda h_i: float(survival_dict[h_i]['Survival']), high_i
)
mid_survival = map(
lambda h_i: float(survival_dict[h_i]['Survival']), mid_i
)
low_survival = map(
lambda h_i: float(survival_dict[h_i]['Survival']), low_i
)
# Split of the age feature
hi_ages = map(
lambda h_i: float(survival_dict[h_i]['Age']), high_i
)
mid_ages = map(
lambda h_i: float(survival_dict[h_i]['Age']), mid_i
)
low_ages = map(
lambda h_i: float(survival_dict[h_i]['Age']), low_i
)
# Data split (using numpy) for train and validation.
# We also compute the number of batches for both training and
# validation according to the batch size.
# Train/Validation split
n_hi = len(high_i)
n_mid = len(mid_i)
n_low = len(low_i)
n_hitrain = int(n_hi * (1 - val_split))
n_midtrain = int(n_mid * (1 - val_split))
n_lowtrain = int(n_low * (1 - val_split))
# Training
hi_train = high_i[:n_hitrain]
mid_train = mid_i[:n_midtrain]
low_train = low_i[:n_lowtrain]
train_i = hi_train + mid_train + low_train
hi_train_ages = hi_ages[:n_hitrain]
mid_train_ages = mid_ages[:n_midtrain]
low_train_ages = low_ages[:n_lowtrain]
train_ages = hi_train_ages + mid_train_ages + low_train_ages
hi_train_surv = hi_survival[:n_hitrain]
mid_train_surv = mid_survival[:n_midtrain]
low_train_surv = low_survival[:n_lowtrain]
train_surv = hi_train_surv + mid_train_surv + low_train_surv
print('< Training dataset >')
train_rois, train_data = get_images(train_i)
train_dataset = BBImageValueDataset(
train_data, train_ages, train_surv, train_rois, bb=bb
)
print('Dataloader creation <train>')
train_loader = DataLoader(
train_dataset, 8, True, num_workers=num_workers,
)
# Validation
hi_train = high_i[n_hitrain:]
mid_train = mid_i[n_midtrain:]
low_train = low_i[n_lowtrain:]
val_i = hi_train + mid_train + low_train
hi_train_ages = hi_ages[n_hitrain:]
mid_train_ages = mid_ages[n_midtrain:]
low_train_ages = low_ages[n_lowtrain:]
val_ages = hi_train_ages + mid_train_ages + low_train_ages
hi_train_surv = hi_survival[n_hitrain:]
mid_train_surv = mid_survival[n_midtrain:]
low_train_surv = low_survival[n_lowtrain:]
val_surv = hi_train_surv + mid_train_surv + low_train_surv
print('< Validation dataset >')
val_rois, val_data = get_images(val_i)
val_dataset = BBImageValueDataset(
val_data, val_ages, val_surv, val_rois, bb=bb
)
print('Dataloader creation <val>')
val_loader = DataLoader(
val_dataset, 1, num_workers=num_workers
)
n_train = len(train_dataset)
n_val = len(val_dataset)
n_test = len(survival_patients) - n_train - n_val
print(
'Train / validation / test samples = %d / %d / %d' % (
n_train, n_val, n_test
)
)
net.fit(
train_loader, val_loader, epochs=epochs, patience=patience
)
net.save_model(os.path.join(d_path, model_name))
''' Survival testing '''
# Testing data
high_i = hi_patients[hi_ini:hi_end]
mid_i = mid_patients[mid_ini:mid_end]
low_i = low_patients[low_ini:low_end]
test_patients = high_i + mid_i + low_i
# Split of the target survivals
hi_survival = map(
lambda h_i: float(survival_dict[h_i]['Survival']), high_i
)
mid_survival = map(
lambda h_i: float(survival_dict[h_i]['Survival']), mid_i
)
low_survival = map(
lambda h_i: float(survival_dict[h_i]['Survival']), low_i
)
test_survival = hi_survival + mid_survival + low_survival
# Split of the age feature
hi_ages = map(
lambda h_i: float(survival_dict[h_i]['Age']), high_i
)
mid_ages = map(
lambda h_i: float(survival_dict[h_i]['Age']), mid_i
)
low_ages = map(
lambda h_i: float(survival_dict[h_i]['Age']), low_i
)
test_ages = hi_ages + mid_ages + low_ages
_, test_data = get_images(test_patients)
print(
'Testing patients (with GT) = %d' % (
len(test_patients)
)
)
pred_y = net.predict(test_data, test_ages, bb)
for p, survival_out, survival in zip(
test_patients, pred_y, test_survival
):
print(
'Estimated survival = %f (%f)' % (survival_out, survival)
)
csvwriter.writerow([p, '%f' % float(survival_out)])
_, test_data = get_images(t_survival_patients, True)
print(
'Testing patients = %d' % (
len(t_survival_patients)
)
)
pred_y = net.predict(test_data, t_survival_ages, bb)
test_survivals += np.array(pred_y)
for p, survival_out, s in zip(
t_survival_patients, pred_y, test_survivals
):
print(
'Estimated survival = %f (%f)' % (
survival_out, s / (i + 1)
)
)
test_survivals = test_survivals / n_folds
for p, survival_out in zip(test_patients, test_survivals):
print('Final estimated survival = %f' % survival_out)
val_csvwriter.writerow([p, '%f' % float(survival_out)])
def get_all(cls):
tournaments = []
return [cls(name) for name in utils.get_dirs(config.DATA_ROOT)]
