def __init__(self, path): super(Course, self).__init__(path) self.path = path utils.parse_json_props(self) self._modules = { module_path: Module(module_path) for module_path in utils.get_dirs(path) }
def __init__(self, path): super(Module, self).__init__(path) self.path = path utils.parse_json_props(self) self._lessons = { item_path: Lesson(item_path) for item_path in utils.get_dirs(path) }
def __init__(self, path): super(Lesson, self).__init__(path) self.path = path utils.parse_json_props(self) self._tasks = { item_path: Task(item_path) for item_path in utils.get_dirs(path) }
def generate_flow_mag(path): dirs = utils.get_dirs(path); save_path = path + '_pkls' ## Fork 25 parallel threads to # process mutliple files at the same time queue_size = 25; queues = [None] * queue_size processes = [None] * queue_size filenames = [None] * queue_size idx = 0 for d in dirs: current_dir = os.path.join(path,d); files = utils.get_files(current_dir,'.avi',append_base=True); for vdx_path in files: save_dir = os.path.join(save_path,d); if(not os.path.exists(save_dir)): os.makedirs(save_dir) name, ext = utils.get_file_name_ext(vdx_path); file_name = os.path.join(save_dir, name) + '.pkl'; if(not os.path.exists(file_name)): print('processing ',vdx_path ) vid = imageio.get_reader(vdx_path, 'ffmpeg') queues[idx] = mp.Queue() filenames[idx] = file_name ; processes[idx] = mp.Process(target=video2motion_level, args=(vid,queues[idx])) processes[idx].start() idx += 1 else: flow_mag = utils.pkl_read(file_name) ## Make sure it contains a least one frame if(flow_mag.shape[0] == 0): vid = imageio.get_reader(vdx_path, 'ffmpeg') queues[idx] = mp.Queue() filenames[idx] = file_name; processes[idx] = mp.Process(target=video2motion_level, args=(vid, queues[idx])) processes[idx].start() idx += 1 if(idx == queue_size): for i in range(queue_size): processes[i].join() flow_mag = queues[i].get() if(flow_mag is not None): utils.pkl_write(filenames[i],flow_mag) print('Done ', filenames[i]) else: print('Error in ', vdx_path) idx = 0; for i in range(idx): processes[i].join() flow_mag = queues[i].get() if(flow_mag is not None): utils.pkl_write(filenames[i],flow_mag) print('Done ', filenames[i])
def load_series_data(roots, metrics, columns, roots_post=None, is_budget=False): streams = utils.get_dirs("{0}/{1}".format(roots[0], columns[0])) if roots_post is None: roots_post = [] * len(roots) all = {} for metric in metrics: series = {} for col in columns: for stream in streams: i = 0 for root in roots: data_path = "{0}/{1}/{2}/series/{3}.data".format( root, col, stream, metric) print("Reading: {0}".format(data_path)) data_file = open(data_path, "r") data_lines = data_file.readlines() for line in data_lines: [row, values] = line.split(",", 1) if is_budget: row = row.split("-")[0] row += roots_post[i] values = list( map(lambda x: float(x.replace("\n", "")), values.split(","))) filter_values = list( filter(lambda x: not math.isnan(x), values)) avg = sum(filter_values) / len(filter_values) values = list( map(lambda x: avg if math.isnan(x) else x, values)) if row not in series: series[row] = {} if col not in series[row]: series[row][col] = {} series[row][col][stream] = values i += 1 all[metric] = series return { "data": all, "metrics": metrics, "columns": columns, "streams": streams, }
def transform_files(in_path,out_path,transform,dirs=False): utils.make_dir(out_path) if(dirs): names=utils.get_dirs(in_path) else: names=utils.get_files(in_path) for name in names: full_in_path=in_path+name full_out_path=out_path+name transform(full_in_path,full_out_path)
def sample_video(dataset_path): dirs = utils.get_dirs(dataset_path) print(len(dirs)) sample_activity = np.random.choice(len(dirs)) d = dirs[sample_activity] current_dir = os.path.join(dataset_path, d) files = utils.get_files(current_dir, '.avi', append_base=True) sample_vdx = np.random.choice(len(files)) print(len(files)) current_vdx = files[sample_vdx] return d, current_vdx
def load_avg_data(roots, metrics, columns, roots_post=None, is_budget=False): streams = utils.get_dirs("{0}/{1}".format(roots[0], columns[0])) if roots_post is None: roots_post = [''] * len(roots) all = {} for metric in metrics: averages = {} for col in columns: for stream in streams: i = 0 for root in roots: data_path = "{0}/{1}/{2}/averages/{3}.data".format( root, col, stream, metric) print("Reading: {0}".format(data_path)) data_file = open(data_path, "r") data_lines = data_file.readlines() for line in data_lines: [row, value] = line.split(",") if is_budget: row = row.split("-")[0] row += roots_post[i] if row not in averages: averages[row] = {} if col not in averages[row]: averages[row][col] = {} averages[row][col][stream] = { "all": float(value.replace("\n", "")) } i += 1 all[metric] = averages return { "data": all, "streams": streams, "metrics": metrics, "columns": columns # todo: add rows }
def run(args): real_path = "list.txt" real_dirs = sorted(utils.get_dirs(args.path)) with open(real_path, 'r', encoding='utf-8') as f: dirs = f.readlines() user_dirs = [] for i in range(len(dirs)): tmp = dirs[i].split('|') if tmp[1] == args.settingsActive: user_dirs.append(tmp[0]) user_dirs = sorted(user_dirs) if len(user_dirs) == len(real_dirs): print(','.join(user_dirs)) else: utils.log('compare', tp='error') real_dirs.reverse() user_dirs.reverse() utils.log('file_dirs', ','.join(real_dirs), 'error') utils.log('statistical', ','.join(user_dirs), 'error')
def main(_): model_dir, data_dir = get_dirs(conf, ['exp_name']) # exp_start_time = datetime.datetime.now().strftime("%A_%b%d-%H%M%S") # data_dir = "logs/" + conf.exp_name + "_" + exp_start_time preprocess_conf(conf, model_dir) env = gym.make(conf.env_name) env.seed(conf.random_seed) state_shape = env.observation_space.shape if type(env.action_space) is gym.spaces.Discrete: action_shape = env.action_space.n else: action_shape = env.action_space.shape[0] # replay buffer buffer = ReplayBuffer2(conf.buffer_size) # building agent # config = tf.ConfigProto(allow_soft_placement=True) # config.gpu_options.allow_growth = True config = tf.ConfigProto(intra_op_parallelism_threads=8, inter_op_parallelism_threads=8) with tf.Session(config=config) as sess: # agent agent = SoftPolicyGradient(sess, conf, state_shape, action_shape) # statistic stat = Statistic(sess, conf, model_dir, data_dir) if conf.load_model: stat.load_model() def var_print(): for var in tf.global_variables(): print(var) print("printing vars:------------------------------------------------") var_print() print( "printing vars::------------------------------------------------") start_steps = 1000 episode, global_step, local_step = 0, 0, 0 epi_rewards = 0 total_Q, Q_loss, pi_loss = [], [], [] state = env.reset() # pbar = tqdm(total=conf.max_steps, dynamic_ncols=True) while global_step < conf.max_steps: # interaction with environment action = agent.sampling_actions( [state], is_deterministic=False)[0] # [-inf, inf] next_state, reward, done, info = env.step( action_converter(env, action)) global_step += 1 local_step += 1 epi_rewards += reward reward *= conf.reward_scale buffer.add_transition(state, action, reward, next_state, done) state = next_state # train step if buffer.size() >= conf.batch_size and global_step >= start_steps: for i in range(conf.num_train_steps): transitions = buffer.get_transitions(conf.batch_size) Q, single_Q_loss, single_pi_loss = agent.trainer( transitions) total_Q.append(np.mean(Q)) Q_loss.append(single_Q_loss) pi_loss.append(single_pi_loss) # evaluate step if global_step % conf.eval_interval == 0: ave_epi_rewards = np.mean(eval_step(env, agent)) stat.save_step(global_step, ave_epi_rewards) print('\n[Evaluation] averaged_epi_rewards: %.3f' % ave_epi_rewards) if done: # save step all_epi_rewards.append(epi_rewards) stat.save_step(global_step, epi_rewards, np.mean(total_Q), np.mean(Q_loss), np.mean(pi_loss)) # pbar.update(local_step) lenn = len(all_epi_rewards) fromm = max(lenn - 20, 0) to = lenn min_5_ep_ret = min(all_epi_rewards[fromm:to]) # pbar.set_description('Episode: %s, epi_rewards: %.3f, pi_loss: %.3f, Q_loss: %.3f avg_5_epi_rew %.1f' % # (episode+1, epi_rewards, np.mean(pi_loss), np.mean(Q_loss), sum(all_epi_rewards[fromm:to])/(to-fromm) ) ) print( 'Episode: %s, epi_rewards: %.3f, pi_loss: %.3f, Q_loss: %.3f \tmin_5_epi_rew %.1f' % (episode + 1, epi_rewards, np.mean(pi_loss), np.mean(Q_loss), min_5_ep_ret)) threshold = -500.0 if ((to - fromm) > 3 and min_5_ep_ret > threshold): time_end = time.time() print("SHI hyperParams have made algo converge (", threshold, ") in ", (time_end - time_begin) / 1.0, " s") stat.save_step(global_step, epi_rewards, np.mean(total_Q), np.mean(Q_loss), np.mean(pi_loss)) stat.save_model(global_step) sys.exit() episode += 1 local_step = 0 epi_rewards = 0 total_Q, Q_loss, pi_loss = [], [], [] state = env.reset()
def main(_): model_dir, data_dir = get_dirs(conf, ['env_name']) preprocess_conf(conf, model_dir) env = gym.make(conf.env_name) # env.seed(conf.random_seed) state_shape = env.observation_space.shape if type(env.action_space) is gym.spaces.Discrete: action_shape = env.action_space.n else: action_shape = env.action_space.shape[0] # replay buffer buffer = ReplayBuffer2(conf.buffer_size) # building agent config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # agent agent = SoftPolicyGradient(sess, conf, state_shape, action_shape) # statistic stat = Statistic(sess, conf, model_dir, data_dir) if conf.load_model: stat.load_model() episode, global_step, local_step = 0, 0, 0 epi_rewards = 0 total_Q, Q_loss, pi_loss = [], [], [] state = env.reset() pbar = tqdm(total=conf.max_steps, dynamic_ncols=True) while global_step < conf.max_steps: # interaction with environment action = agent.sampling_actions([state], is_deterministic=False)[0] # [-inf, inf] next_state, reward, done, info = env.step(action_converter(env, action)) global_step += 1 local_step += 1 epi_rewards += reward reward *= conf.reward_scale buffer.add_transition(state, action, reward, next_state, done) state = next_state # train step if buffer.size() >= conf.batch_size: for i in range(conf.num_train_steps): transitions = buffer.get_transitions(conf.batch_size) Q, single_Q_loss, single_pi_loss = agent.trainer(transitions) total_Q.append(np.mean(Q)) Q_loss.append(single_Q_loss) pi_loss.append(single_pi_loss) # evaluate step if global_step % conf.eval_interval == 0: ave_epi_rewards = np.mean(eval_step(env, agent)) stat.save_step(global_step, ave_epi_rewards) print('\n[Evaluation] averaged_epi_rewards: %.3f' % ave_epi_rewards) if done: # save step stat.save_step(global_step, epi_rewards, np.mean(total_Q), np.mean(Q_loss), np.mean(pi_loss)) pbar.update(local_step) pbar.set_description('Episode: %s, epi_rewards: %.3f, pi_loss: %.3f, Q_loss: %.3f' % (episode+1, epi_rewards, np.mean(pi_loss), np.mean(Q_loss))) print() episode += 1 local_step = 0 epi_rewards = 0 total_Q, Q_loss, pi_loss = [], [], [] state = env.reset() pbar.close()
def main(): # General parameters net = ['base', 'cifar', 'emb+soft', 'resnet50', 'resnet20', 'local_feat'][0] database = ['cifar10', 'mnist', 'fashion_mnist', 'skillup'][1] epochs = 10 learn_rate = 0.01 decay = (learn_rate / epochs) * 0.8 ims_per_id = 8 ids_per_batch = 8 margin = 0.9 embedding_size = 64 squared = False data_augmentation = False patience = 25 # built model's parameters dropout = 0.3 blocks = 3 n_channels = 32 weight_decay = 1e-4 * 0 # dataloader parameters use_dataloader = True path = '/home/daniel/proyectos/product_detection/web_market_preproces/duke_from_images' exp_dir, log_dir, model_weights_path, model_name = get_dirs(database) tl_object = TripletLoss(ims_per_id=ims_per_id, ids_per_batch=ids_per_batch, margin=margin, squared=squared) tl_h = TripletLoss(ims_per_id, ids_per_batch, margin, squared) opt = optimizers.Adam(lr=learn_rate, decay=decay) data, input_size = get_database(database) im_size = input_size[:2] data_gen_args_train = dict( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 10, # randomly rotate images in the range (degrees, 0 to 180) zoom_range=0.1, # Randomly zoom image width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=False, # randomly flip images vertical_flip=False) if not data_augmentation: data_gen_args_train = {} model_args = dict(embedding_dim=embedding_size, input_shape=input_size, drop=dropout, blocks=blocks, n_channels=n_channels, weight_decay=weight_decay, layer_limit=173, patience=patience) data_loader_args = dict(path=path, ims_per_id=ims_per_id, ids_per_batch=ids_per_batch, target_image_size=im_size, data_gen_args=data_gen_args_train, preprocess_unit=True, data=data) if database == 'skillup': dl = FileDataloader(**data_loader_args) else: dl = StaticDataloader(**data_loader_args) model = get_net_object(net, model_args) model.compile(opt, tl_object.cluster_loss) if use_dataloader: model.train_generator(dl, model_weights_path, epochs, log_dir) else: model.train(data, model_weights_path, epochs, ims_per_id * ids_per_batch, log_dir) model.save_model(model_weights_path) visualize_embeddings(database=database, model_dir=exp_dir, model_name=model_name, model=model.model)
import sys sys.path.append('../') import utils import os import numpy as np if __name__ == '__main__': path = utils.get_dataset_path('UCF50') dirs = utils.get_dirs(path) train_list = [] val_list = [] test_list = [] train_set_percentage = 0.7 val_set_percentage_from_training = 0.2 for d in dirs: current_dir = os.path.join(path, d) files = utils.get_files(current_dir, '.avi', append_base=False) files = [os.path.join(d, f) for f in files] np.random.shuffle(files) num_train = int(len(files) * train_set_percentage) num_val = int(num_train * val_set_percentage_from_training) train_list.extend(files[0:num_train - num_val]) val_list.extend(files[num_train - num_val:num_train]) test_list.extend(files[num_train:]) print(len(files), len(train_list), len(val_list), len(test_list)) save_path = path + '_lists' utils.touch_dir(save_path) utils.txt_write(save_path + '/trainlist.txt', train_list) utils.txt_write(save_path + '/vallist.txt', val_list)
def parse_year(path, workers=WORKERS): #@TODO add & parse session metadata log.info('Parsing year data at "%s", using %d workers' % (path, workers)) sessions = sorted(get_dirs(path)) return dict(Parallel(n_jobs=workers)(delayed(parse_session)(one) for one in sessions))
def read_action_frame(dir_path): actions_paths=utils.get_dirs(dir_path) actions_paths=utils.append_path(dir_path, actions_paths) actions=[read_action(path) for path in actions_paths] return create_action_frame(actions)
def test_seg_validation(net_name): # Init c = color_codes() options = parse_inputs() depth = options['blocks'] filters = options['filters'] d_path = options['loo_dir'] v_path = options['val_dir'] seg_path = os.path.join(v_path, 'segmentation') if not os.path.isdir(seg_path): os.mkdir(seg_path) unc_path = os.path.join(v_path, 'uncertainty') if not os.path.isdir(unc_path): os.mkdir(unc_path) p = get_dirs(d_path)[0] test_patients = filter(lambda p: 'BraTS19' in p, get_dirs(v_path)) _, test_x = get_images(test_patients, True) print( 'Testing patients = %d' % ( len(test_patients) ) ) # The sub-regions considered for evaluation are: # 1) the "enhancing tumor" (ET) # 2) the "tumor core" (TC) # 3) the "whole tumor" (WT) # # The provided segmentation labels have values of 1 for NCR & NET, # 2 for ED, 4 for ET, and 0 for everything else. # The participants are called to upload their segmentation labels # as a single multi-label file in nifti (.nii.gz) format. # # The participants are called to upload 4 nifti (.nii.gz) volumes # (3 uncertainty maps and 1 multi-class segmentation volume from # Task 1) onto CBICA's Image Processing Portal format. For example, # for each ID in the dataset, participants are expected to upload # following 4 volumes: # 1. {ID}.nii.gz (multi-class label map) # 2. {ID}_unc_whole.nii.gz (Uncertainty map associated with whole tumor) # 3. {ID}_unc_core.nii.gz (Uncertainty map associated with tumor core) # 4. {ID}_unc_enhance.nii.gz (Uncertainty map associated with enhancing tumor) for i, (p_i, test_i) in enumerate(zip(test_patients, test_x)): t_in = time.time() # unc_i = np.zeros((4,) + test_i.shape[1:]) pred_i = np.zeros((4,) + test_i.shape[1:]) for f in range(5): model_name = '%s-f%d.mdl' % (net_name, f) net = BratsSegmentationNet(depth=depth, filters=filters) net.load_model(os.path.join(d_path, model_name)) # # unc_i += net.uncertainty([test_i], steps=10)[0] * 0.2 pred_i += net.segment([test_i])[0] seg_i = np.argmax(pred_i, axis=0) seg_i[seg_i == 3] = 4 # seg_unc_i = np.argmax(unc_i, axis=0) # seg_unc_i[seg_unc_i == 3] = 4 tumor_mask = remove_small_regions( seg_i.astype(np.bool), min_size=30 ) seg_i[log_not(tumor_mask)] = 0 # seg_unc_i[log_not(tumor_mask)] = 0 # # whole_i = np.sum(unc_i[1:]) * tumor_mask.astype(np.float32) # core_i = unc_i[1] + unc_i[-1] * tumor_mask.astype(np.float32) # enhance_i = unc_i[-1] * tumor_mask.astype(np.float32) # # seg_unc_i = np.argmax(unc_i, axis=0) # seg_unc_i[seg_unc_i == 3] = 4 niiname = os.path.join(d_path, p, p + '_seg.nii.gz') nii = load_nii(niiname) nii.get_data()[:] = seg_i save_nii(nii, os.path.join(seg_path, p_i + '.nii.gz')) # nii.get_data()[:] = seg_unc_i # save_nii(nii, os.path.join(unc_path, p_i + '.nii.gz')) # niiname = os.path.join(v_path, p_i, p_i + '_flair.nii.gz') # nii = load_nii(niiname) # nii.get_data()[:] = whole_i # save_nii( # nii, os.path.join(unc_path, p_i + '_unc_whole.nii.gz') # ) # nii.get_data()[:] = core_i # save_nii( # nii, os.path.join(unc_path, p_i + '_unc_core.nii.gz') # ) # nii.get_data()[:] = enhance_i # save_nii( # nii, os.path.join(unc_path, p_i + '_unc_enhance.nii.gz') # ) t_s = time_to_string(time.time() - t_in) print( 'Finished patient %s (%d/%d) %s' % (p_i, i + 1, len(test_x), t_s) )
def get_seasons(self): path = os.path.join(config.DATA_ROOT, self.name) return [Season(name, self) for name in utils.get_dirs(path)]
(opts, args) = op.parse_args() if not opts.base_dir: logger.error("You must give base directory") op.print_help() exit() if not opts.config or not os.path.exists(opts.config): logger.error("No such config file: %s", opts.config) op.print_help() exit() # make the filename as suffix of compiled binaries suffix = "_" + os.path.splitext(os.path.basename(opts.config))[0] base_dir = os.path.join(opts.base_dir, "gnu" + suffix) src_dir, out_dir, log_dir = get_dirs(base_dir) os.makedirs(src_dir, exist_ok=True) os.makedirs(out_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True) logger.info("base directory : %s", base_dir) logger.info("source directory : %s", src_dir) logger.info("output directory : %s", out_dir) logger.info("log directory : %s", log_dir) with open(opts.config, "r") as f: config = yaml.load(f) opti_list = config["opti"] arch_list = config["arch"] compiler_list = config["compiler"] num_opt = len(opti_list) * len(arch_list) * len(compiler_list)
def Train_No_GAN(opt): # w / o GAN # ---------------------------------------- # Network training parameters # ---------------------------------------- # cudnn benchmark cudnn.benchmark = opt.cudnn_benchmark # Loss functions criterion_L1 = torch.nn.L1Loss().cuda() # Initialize Generator generatorNet = utils.create_generator(opt) flownet = utils.create_pwcnet(opt) # To device if opt.multi_gpu: generatorNet = nn.DataParallel(generatorNet) generatorNet = generatorNet.cuda() flownet = nn.DataParallel(flownet) flownet = flownet.cuda() else: generatorNet = generatorNet.cuda() flownet = flownet.cuda() # Optimizers optimizer_G = torch.optim.Adam(generatorNet.parameters(), lr = opt.lr_g, betas = (opt.b1, opt.b2), weight_decay = opt.weight_decay) # Learning rate decrease def adjust_learning_rate(opt, epoch, iteration, optimizer): #Set the learning rate to the initial LR decayed by "lr_decrease_factor" every "lr_decrease_epoch" epochs if opt.lr_decrease_mode == 'epoch': lr = opt.lr_g * (opt.lr_decrease_factor ** (epoch // opt.lr_decrease_epoch)) for param_group in optimizer.param_groups: param_group['lr'] = lr if opt.lr_decrease_mode == 'iter': lr = opt.lr_g * (opt.lr_decrease_factor ** (iteration // opt.lr_decrease_iter)) for param_group in optimizer.param_groups: param_group['lr'] = lr # Save the model if pre_train == True def save_model(opt, epoch, iteration, len_dataset, generator): """Save the model at "checkpoint_interval" and its multiple""" if opt.multi_gpu == True: if opt.save_mode == 'epoch': if (epoch % opt.save_by_epoch == 0) and (iteration % len_dataset == 0): if opt.save_name_mode: torch.save(generator.module, 'Pre_%s_epoch%d_bs%d.pth' % (opt.task, epoch, opt.batch_size)) print('The trained model is successfully saved at epoch %d' % (epoch)) if opt.save_mode == 'iter': if iteration % opt.save_by_iter == 0: if opt.save_name_mode: torch.save(generator.module, 'Pre_%s_iter%d_bs%d.pth' % (opt.task, iteration, opt.batch_size)) print('The trained model is successfully saved at iteration %d' % (iteration)) else: if opt.save_mode == 'epoch': if (epoch % opt.save_by_epoch == 0) and (iteration % len_dataset == 0): if opt.save_name_mode: torch.save(generator, 'Pre_%s_epoch%d_bs%d.pth' % (opt.task, epoch, opt.batch_size)) print('The trained model is successfully saved at epoch %d' % (epoch)) if opt.save_mode == 'iter': if iteration % opt.save_by_iter == 0: if opt.save_name_mode: torch.save(generator, 'Pre_%s_iter%d_bs%d.pth' % (opt.task, iteration, opt.batch_size)) print('The trained model is successfully saved at iteration %d' % (iteration)) # ---------------------------------------- # Network dataset # ---------------------------------------- # Define the class list imglist = utils.text_readlines('videocolor_linux.txt') classlist = utils.get_dirs(opt.baseroot) ''' imgnumber = len(imglist) - (len(imglist) % opt.batch_size) imglist = imglist[:imgnumber] ''' # Define the dataset trainset = dataset.MultiFramesDataset(opt, imglist, classlist) print('The overall number of classes:', len(trainset)) # Define the dataloader dataloader = utils.create_dataloader(trainset, opt) # ---------------------------------------- # Training # ---------------------------------------- # Count start time prev_time = time.time() # For loop training for epoch in range(opt.epochs): for iteration, (in_part, out_part) in enumerate(dataloader): # Train Generator optimizer_G.zero_grad() lstm_state = None loss_flow = 0 loss_flow_long = 0 loss_L1 = 0 x_0 = in_part[0].cuda() p_t_0 = in_part[0].cuda() for iter_frame in range(opt.iter_frames): # Read data x_t = in_part[iter_frame].cuda() y_t = out_part[iter_frame].cuda() # Initialize the second input and compute flow loss if iter_frame == 0: p_t_last = torch.zeros(opt.batch_size, opt.out_channels, opt.resize_h, opt.resize_w).cuda() elif iter_frame == 1: x_t_last = in_part[iter_frame - 1].cuda() p_t_last = p_t.detach() p_t_0 = p_t.detach() p_t_last.requires_grad = False p_t_0.requires_grad = False # o_t_last_2_t range is [-20, +20] o_t_last_2_t = pwcnet.PWCEstimate(flownet, x_t, x_t_last) x_t_warp = pwcnet.PWCNetBackward((x_t_last + 1) / 2, o_t_last_2_t) # y_t_warp range is [0, 1] p_t_warp = pwcnet.PWCNetBackward((p_t_last + 1) / 2, o_t_last_2_t) else: x_t_last = in_part[iter_frame - 1].cuda() p_t_last = p_t.detach() p_t_last.requires_grad = False # o_t_last_2_t o_t_first_2_t range is [-20, +20] o_t_last_2_t = pwcnet.PWCEstimate(flownet, x_t, x_t_last) o_t_first_2_t = pwcnet.PWCEstimate(flownet,x_t, x_0) # y_t_warp, y_t_warp_long range is [0, 1] x_t_warp = pwcnet.PWCNetBackward((x_t_last + 1) / 2, o_t_last_2_t) p_t_warp = pwcnet.PWCNetBackward((p_t_last + 1) / 2, o_t_last_2_t) x_t_warp_long = pwcnet.PWCNetBackward((x_0 + 1) / 2, o_t_first_2_t) p_t_warp_long = pwcnet.PWCNetBackward((p_t_0 + 1) / 2, o_t_first_2_t) # Generator output p_t, lstm_state = generatorNet(x_t, p_t_last, lstm_state) lstm_state = utils.repackage_hidden(lstm_state) if iter_frame == 1: mask_flow = torch.exp( -opt.mask_para * torch.sum((x_t + 1) / 2 - x_t_warp, dim=1).pow(2) ).unsqueeze(1) loss_flow += criterion_L1(mask_flow * (p_t + 1) / 2, mask_flow * p_t_warp) elif iter_frame > 1: mask_flow = torch.exp( -opt.mask_para * torch.sum((x_t + 1) / 2 - x_t_warp, dim=1).pow(2) ).unsqueeze(1) loss_flow += criterion_L1(mask_flow * (p_t + 1) / 2, mask_flow * p_t_warp) mask_flow_long = torch.exp( -opt.mask_para * torch.sum((x_t + 1) / 2 - x_t_warp_long, dim=1).pow(2) ).unsqueeze(1) loss_flow_long += criterion_L1(mask_flow_long * (p_t + 1) / 2, mask_flow_long * p_t_warp_long) # Pixel-level loss loss_L1 += criterion_L1(p_t, y_t) # Overall Loss and optimize loss = loss_L1 + opt.lambda_flow * loss_flow + opt.lambda_flow_long * loss_flow_long loss.backward() optimizer_G.step() # Determine approximate time left iters_done = epoch * len(dataloader) + iteration iters_left = opt.epochs * len(dataloader) - iters_done time_left = datetime.timedelta(seconds = iters_left * (time.time() - prev_time)) prev_time = time.time() # Print log print("\r[Epoch %d/%d] [Batch %d/%d] [L1 Loss: %.4f] [Flow Loss Short: %.8f] [Flow Loss Long: %.8f] Time_left: %s" % ((epoch + 1), opt.epochs, iteration, len(dataloader), loss_L1.item(), loss_flow.item(), loss_flow_long.item(), time_left)) # Save model at certain epochs or iterations save_model(opt, (epoch + 1), (iters_done + 1), len(dataloader), generatorNet) # Learning rate decrease at certain epochs adjust_learning_rate(opt, (epoch + 1), (iters_done + 1), optimizer_G)
os.path.join(dataset_path + '_lists', 'vallist.txt')) print('**** Val has ', len(files_list)) save_dir = os.path.join(save_dir, 'val') max_num_tuplex = 100 ## 20000 elif (current_subset == const.Subset.TEST.value): files_list = utils.txt_read( os.path.join(dataset_path + '_lists', 'testlist.txt')) print('*** Test has ', len(files_list)) save_dir = os.path.join(save_dir, 'test') utils.touch_dir(save_dir) tuple_idx = 0 tuple_idx = max(0, utils.last_tuple_idx(save_dir) - 5) ## As caution, regenerate last 5 tuples activity_list = sorted(utils.get_dirs(dataset_path)) print('activity_list ', len(activity_list), activity_list) lbls_file = os.path.join(save_dir, 'lbl.pkl') if (tuple_idx == 0): lbls_ary = np.ones(max_num_tuplex, dtype=np.int32) * -1 ## Invalid Activity else: lbls_ary = utils.pkl_read(lbls_file) tuple_idx = 0 import time start_time = time.time() queue_size = 25
# built model's parameters dropout = 0.35 # Dropout probability of each layer. Conv layers use SpatialDropout2D blocks = 6 # Number of (Conv -> Act -> BN -> MaxPool -> Dropout) blocks n_channels = args[ 'channels'] # Number of channels (or feature maps) of the first convolution block. # the following ones are 1.5 times the number of channels of the previous block weight_decay = 1e-4 * 0 # dataloader parameters. # Folder's path where the files query.txt and bounding_box_train.txt are # query.txt contains the path and the class of test images # bounding_box_train.txt contains the path and the class of train images path = args['path'] exp_dir, log_dir, model_weights_path, model_name = get_dirs(database) print(exp_dir, log_dir, model_weights_path, model_name) data, input_size = get_database( database) # if database == 'skillup'. data is None im_size = input_size[:2] data_gen_args_train = dict( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization=False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range=10, # randomly rotate images in the range (degrees, 0 to 180) zoom_range=0.1, # Randomly zoom image width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width)
def get_actions(action_dir,cls): action_files=utils.get_dirs(action_dir) action_files=utils.append_path(action_dir,action_files) return [compute_sequence(path,cls) for path in action_files]
(opts, args) = op.parse_args() if not opts.base_dir: logger.error("You must give base directory") op.print_help() exit() if not opts.config or not os.path.exists(opts.config): logger.error("No such config file: %s", opts.config) op.print_help() exit() # make the filename as suffix of compiled binaries suffix = "_" + os.path.splitext(os.path.basename(opts.config))[0] base_dir = opts.base_dir src_dir, out_dir, log_dir = get_dirs(opts.base_dir, suffix) os.makedirs(src_dir, exist_ok=True) os.makedirs(out_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True) logger.info("base directory : %s", base_dir) logger.info("source directory : %s", src_dir) logger.info("output directory : %s", out_dir) logger.info("log directory : %s", log_dir) with open(opts.config, "r") as f: config = yaml.safe_load(f) opti_list = config["opti"] arch_list = config["arch"] compiler_list = config["compiler"] num_opt = len(opti_list) * len(arch_list) * len(compiler_list)
def train_test_seg(net_name, n_folds, val_split=0.1): # Init c = color_codes() options = parse_inputs() depth = options['blocks'] filters = options['filters'] d_path = options['loo_dir'] unc_path = os.path.join(d_path, 'uncertainty') if not os.path.isdir(unc_path): os.mkdir(unc_path) seg_path = os.path.join(d_path, 'segmentation') if not os.path.isdir(seg_path): os.mkdir(seg_path) patients = get_dirs(d_path) cbica = filter(lambda p: 'CBICA' in p, patients) tcia = filter(lambda p: 'TCIA' in p, patients) tmc = filter(lambda p: 'TMC' in p, patients) b2013 = filter(lambda p: '2013' in p, patients) for i in range(n_folds): print( '%s[%s] %sFold %s(%s%d%s%s/%d)%s' % ( c['c'], strftime("%H:%M:%S"), c['g'], c['c'], c['b'], i + 1, c['nc'], c['c'], n_folds, c['nc'] ) ) # Training itself # Data split (using the patient names) for train and validation. # We also compute the number of batches for both training and # validation according to the batch size. ''' Training ''' ini_cbica = len(cbica) * i / n_folds end_cbica = len(cbica) * (i + 1) / n_folds fold_cbica = cbica[:ini_cbica] + cbica[end_cbica:] n_fold_cbica = len(fold_cbica) n_cbica = int(n_fold_cbica * (1 - val_split)) ini_tcia = len(tcia) * i / n_folds end_tcia = len(tcia) * (i + 1) / n_folds fold_tcia = tcia[:ini_tcia] + tcia[end_tcia:] n_fold_tcia = len(fold_tcia) n_tcia = int(n_fold_tcia * (1 - val_split)) ini_tmc = len(tmc) * i / n_folds end_tmc = len(tmc) * (i + 1) / n_folds fold_tmc = tmc[:ini_tmc] + tmc[end_tmc:] n_fold_tmc = len(fold_tmc) n_tmc = int(n_fold_tmc * (1 - val_split)) ini_b2013 = len(b2013) * i / n_folds end_b2013 = len(b2013) * (i + 1) / n_folds fold_b2013 = b2013[:ini_b2013] + b2013[end_b2013:] n_fold_b2013 = len(fold_b2013) n_b2013 = int(n_fold_b2013 * (1 - val_split)) training_n = n_fold_cbica + n_fold_tcia + n_fold_tmc + n_fold_b2013 testing_n = len(patients) - training_n print( 'Training / testing samples = %d / %d' % ( training_n, testing_n ) ) # Training train_cbica = fold_cbica[:n_cbica] train_tcia = fold_tcia[:n_tcia] train_tmc = fold_tmc[:n_tmc] train_b2013 = fold_b2013[:n_b2013] train_patients = train_cbica + train_tcia + train_tmc + train_b2013 # Validation val_cbica = fold_cbica[n_cbica:] val_tcia = fold_tcia[n_tcia:] val_tmc = fold_tmc[n_tmc:] val_b2013 = fold_b2013[n_b2013:] val_patients = val_cbica + val_tcia + val_tmc + val_b2013 model_name = '%s-f%d.mdl' % (net_name, i) net = BratsSegmentationNet(depth=depth, filters=filters) # train_seg(net, model_name, train_patients, val_patients) train_seg( net, model_name, train_patients, val_patients, dropout=0 ) # model_name = '%s-f%d-R.mdl' % (net_name, i) # train_seg( # net, model_name, train_patients, val_patients, # refine=True, dropout=0.5, lr=1e-2 # ) # Testing data (with GT) test_cbica = cbica[ini_cbica:end_cbica] test_tcia = tcia[ini_tcia:end_tcia] test_tmc = tmc[ini_tmc:end_tmc] test_b2013 = b2013[ini_b2013:end_b2013] test_patients = test_cbica + test_tcia + test_tmc + test_b2013 patient_paths = map(lambda p: os.path.join(d_path, p), test_patients) _, test_x = get_images(test_patients) print( 'Testing patients (with GT) = %d' % ( len(test_patients) ) ) # The sub-regions considered for evaluation are: # 1) the "enhancing tumor" (ET) # 2) the "tumor core" (TC) # 3) the "whole tumor" (WT) # # The provided segmentation labels have values of 1 for NCR & NET, # 2 for ED, 4 for ET, and 0 for everything else. # The participants are called to upload their segmentation labels # as a single multi-label file in nifti (.nii.gz) format. # # The participants are called to upload 4 nifti (.nii.gz) volumes # (3 uncertainty maps and 1 multi-class segmentation volume from # Task 1) onto CBICA's Image Processing Portal format. For example, # for each ID in the dataset, participants are expected to upload # following 4 volumes: # 1. {ID}.nii.gz (multi-class label map) # 2. {ID}_unc_whole.nii.gz (Uncertainty map associated with whole tumor) # 3. {ID}_unc_core.nii.gz (Uncertainty map associated with tumor core) # 4. {ID}_unc_enhance.nii.gz (Uncertainty map associated with enhancing tumor) for p, (path_i, p_i, test_i) in enumerate(zip( patient_paths, test_patients, test_x )): pred_i = net.segment([test_i])[0] # unc_i = net.uncertainty([test_i], steps=25)[0] # whole_i = np.sum(unc_i[1:]) # core_i = unc_i[1] + unc_i[-1] # enhance_i = unc_i[-1] seg_i = np.argmax(pred_i, axis=0) seg_i[seg_i == 3] = 4 # seg_unc_i = np.argmax(unc_i, axis=0) # seg_unc_i[seg_unc_i == 3] = 4 # tumor_mask = remove_small_regions( # seg_i.astype(np.bool), min_size=30 # ) # # seg_i[log_not(tumor_mask)] = 0 # seg_unc_i[log_not(tumor_mask)] = 0 # # whole_i *= tumor_mask.astype(np.float32) # core_i *= tumor_mask.astype(np.float32) # enhance_i *= tumor_mask.astype(np.float32) niiname = os.path.join(path_i, p_i + '_seg.nii.gz') nii = load_nii(niiname) seg = nii.get_data() dsc = map( lambda label: dsc_seg(seg == label, seg_i == label), [1, 2, 4] ) # dsc_unc = map( # lambda label: dsc_seg(seg == label, seg_unc_i == label), # [1, 2, 4] # ) nii.get_data()[:] = seg_i save_nii(nii, os.path.join(seg_path, p_i + '.nii.gz')) # nii.get_data()[:] = seg_unc_i # save_nii(nii, os.path.join(unc_path, p_i + '.nii.gz')) # niiname = os.path.join(d_path, p_i, p_i + '_flair.nii.gz') # nii = load_nii(niiname) # nii.get_data()[:] = whole_i # save_nii(nii, os.path.join(unc_path, p_i + '_unc_whole.nii.gz')) # nii.get_data()[:] = core_i # save_nii(nii, os.path.join(unc_path, p_i + '_unc_core.nii.gz')) # nii.get_data()[:] = enhance_i # save_nii(nii, os.path.join(unc_path, p_i + '_unc_enhance.nii.gz')) print( 'Segmentation - Patient %s (%d/%d): %s' % ( p_i, p, len(test_x), ' / '.join(map(str, dsc)) ) )
def train_test_survival(net_name, n_folds, val_split=0.1): # Init c = color_codes() options = parse_inputs() epochs = options['epochs'] batch_size = options['batch_size'] patch_size = options['patch_size'] patience = options['patience'] depth = options['blocks'] filters = options['filters'] d_path = options['loo_dir'] patients = get_dirs(d_path) survival_dict = get_survival_data() seg_patients = filter(lambda p: p not in survival_dict.keys(), patients) survival_patients = survival_dict.keys() low_patients = filter( lambda p: int(survival_dict[p]['Survival']) < 300, survival_patients ) mid_patients = filter( lambda p: (int(survival_dict[p]['Survival']) >= 300) & (int(survival_dict[p]['Survival']) < 450), survival_patients ) hi_patients = filter( lambda p: int(survival_dict[p]['Survival']) >= 450, survival_patients ) t_survival_dict = get_survival_data(True) t_survival_patients = t_survival_dict.keys() test_survivals = np.zeros(len(t_survival_patients)) t_survival_ages = map(lambda v: float(v['Age']), t_survival_dict.values()) ''' Segmentation training''' # The goal here is to pretrain a unique segmentation network for all # the survival folds. We only do it once. Then we split the survival # patients accordingly. net = BratsSurvivalNet(depth_seg=depth, depth_pred=depth, filters=filters) print( 'Training segmentation samples = %d' % ( len(seg_patients) ) ) # Training itself model_name = '%s-init.mdl' % net_name num_workers = 8 try: net.base_model.load_model(os.path.join(d_path, model_name)) except IOError: n_params = sum( p.numel() for p in net.base_model.parameters() if p.requires_grad ) print( '%sStarting segmentation training with a unet%s (%d parameters)' % (c['c'], c['nc'], n_params) ) targets = get_labels(seg_patients) rois, data = get_images(seg_patients) print('< Training dataset >') if patch_size is None: seg_dataset = BBImageDataset( data, targets, rois, flip=True ) else: seg_dataset = BratsDataset( data, targets, rois, patch_size ) print('Dataloader creation <with validation>') train_loader = DataLoader( seg_dataset, batch_size, True, num_workers=num_workers, ) targets = get_labels(survival_patients) rois, data = get_images(survival_patients) # Validation print('< Validation dataset >') val_dataset = BBImageDataset( data, targets, rois ) val_loader = DataLoader( val_dataset, 1, num_workers=num_workers ) print( '%s[%s] %sInitial %s%spretraining%s' % ( c['c'], strftime("%H:%M:%S"), c['g'], c['c'], c['b'], c['nc'] ) ) net.base_model.fit(train_loader, val_loader, epochs=epochs, patience=patience) net.base_model.save_model(os.path.join(d_path, model_name)) tr_csv = os.path.join(options['loo_dir'], 'survival_results.csv') val_csv = os.path.join(options['val_dir'], 'survival_results.csv') with open(tr_csv, 'w') as csvfile, open(val_csv, 'w') as val_csvfile: csvwriter = csv.writer(csvfile, delimiter=',') val_csvwriter = csv.writer(val_csvfile, delimiter=',') # First we'll define the maximum bounding box, for training and testing masks, _ = get_images(survival_patients) indices = map(lambda mask: np.where(mask > 0), masks) min_bb = np.min( map( lambda idx: np.min(idx, axis=-1), indices ), axis=0 ) max_bb = np.max( map( lambda idx: np.max(idx, axis=-1), indices ), axis=0 ) bb = map( lambda (min_i, max_i): slice(min_i, max_i), zip(min_bb, max_bb) ) # After that, we can finally train the model to predict the survival. for i in range(n_folds): model_name = '%s-init.mdl' % net_name net = BratsSurvivalNet(depth_seg=depth, depth_pred=depth, filters=filters) net.base_model.load_model(os.path.join(d_path, model_name)) ''' Survival training''' model_name = '%s_f%d.mdl' % (net_name, i) print( '%s[%s] %sFold %s(%s%d%s%s/%d)%s' % ( c['c'], strftime("%H:%M:%S"), c['g'], c['c'], c['b'], i + 1, c['nc'], c['c'], n_folds, c['nc'] ) ) try: net.load_model(os.path.join(d_path, model_name)) except IOError: # Train/Test patient split according to survival classes hi_ini = len(hi_patients) * i / n_folds mid_ini = len(mid_patients) * i / n_folds low_ini = len(low_patients) * i / n_folds hi_end = len(hi_patients) * (i + 1) / n_folds mid_end = len(mid_patients) * (i + 1) / n_folds low_end = len(low_patients) * (i + 1) / n_folds high_i = hi_patients[:hi_ini] + hi_patients[hi_end:] mid_i = mid_patients[:mid_ini] + mid_patients[mid_end:] low_i = low_patients[:low_ini] + low_patients[low_end:] # Split of the target survivals hi_survival = map( lambda h_i: float(survival_dict[h_i]['Survival']), high_i ) mid_survival = map( lambda h_i: float(survival_dict[h_i]['Survival']), mid_i ) low_survival = map( lambda h_i: float(survival_dict[h_i]['Survival']), low_i ) # Split of the age feature hi_ages = map( lambda h_i: float(survival_dict[h_i]['Age']), high_i ) mid_ages = map( lambda h_i: float(survival_dict[h_i]['Age']), mid_i ) low_ages = map( lambda h_i: float(survival_dict[h_i]['Age']), low_i ) # Data split (using numpy) for train and validation. # We also compute the number of batches for both training and # validation according to the batch size. # Train/Validation split n_hi = len(high_i) n_mid = len(mid_i) n_low = len(low_i) n_hitrain = int(n_hi * (1 - val_split)) n_midtrain = int(n_mid * (1 - val_split)) n_lowtrain = int(n_low * (1 - val_split)) # Training hi_train = high_i[:n_hitrain] mid_train = mid_i[:n_midtrain] low_train = low_i[:n_lowtrain] train_i = hi_train + mid_train + low_train hi_train_ages = hi_ages[:n_hitrain] mid_train_ages = mid_ages[:n_midtrain] low_train_ages = low_ages[:n_lowtrain] train_ages = hi_train_ages + mid_train_ages + low_train_ages hi_train_surv = hi_survival[:n_hitrain] mid_train_surv = mid_survival[:n_midtrain] low_train_surv = low_survival[:n_lowtrain] train_surv = hi_train_surv + mid_train_surv + low_train_surv print('< Training dataset >') train_rois, train_data = get_images(train_i) train_dataset = BBImageValueDataset( train_data, train_ages, train_surv, train_rois, bb=bb ) print('Dataloader creation <train>') train_loader = DataLoader( train_dataset, 8, True, num_workers=num_workers, ) # Validation hi_train = high_i[n_hitrain:] mid_train = mid_i[n_midtrain:] low_train = low_i[n_lowtrain:] val_i = hi_train + mid_train + low_train hi_train_ages = hi_ages[n_hitrain:] mid_train_ages = mid_ages[n_midtrain:] low_train_ages = low_ages[n_lowtrain:] val_ages = hi_train_ages + mid_train_ages + low_train_ages hi_train_surv = hi_survival[n_hitrain:] mid_train_surv = mid_survival[n_midtrain:] low_train_surv = low_survival[n_lowtrain:] val_surv = hi_train_surv + mid_train_surv + low_train_surv print('< Validation dataset >') val_rois, val_data = get_images(val_i) val_dataset = BBImageValueDataset( val_data, val_ages, val_surv, val_rois, bb=bb ) print('Dataloader creation <val>') val_loader = DataLoader( val_dataset, 1, num_workers=num_workers ) n_train = len(train_dataset) n_val = len(val_dataset) n_test = len(survival_patients) - n_train - n_val print( 'Train / validation / test samples = %d / %d / %d' % ( n_train, n_val, n_test ) ) net.fit( train_loader, val_loader, epochs=epochs, patience=patience ) net.save_model(os.path.join(d_path, model_name)) ''' Survival testing ''' # Testing data high_i = hi_patients[hi_ini:hi_end] mid_i = mid_patients[mid_ini:mid_end] low_i = low_patients[low_ini:low_end] test_patients = high_i + mid_i + low_i # Split of the target survivals hi_survival = map( lambda h_i: float(survival_dict[h_i]['Survival']), high_i ) mid_survival = map( lambda h_i: float(survival_dict[h_i]['Survival']), mid_i ) low_survival = map( lambda h_i: float(survival_dict[h_i]['Survival']), low_i ) test_survival = hi_survival + mid_survival + low_survival # Split of the age feature hi_ages = map( lambda h_i: float(survival_dict[h_i]['Age']), high_i ) mid_ages = map( lambda h_i: float(survival_dict[h_i]['Age']), mid_i ) low_ages = map( lambda h_i: float(survival_dict[h_i]['Age']), low_i ) test_ages = hi_ages + mid_ages + low_ages _, test_data = get_images(test_patients) print( 'Testing patients (with GT) = %d' % ( len(test_patients) ) ) pred_y = net.predict(test_data, test_ages, bb) for p, survival_out, survival in zip( test_patients, pred_y, test_survival ): print( 'Estimated survival = %f (%f)' % (survival_out, survival) ) csvwriter.writerow([p, '%f' % float(survival_out)]) _, test_data = get_images(t_survival_patients, True) print( 'Testing patients = %d' % ( len(t_survival_patients) ) ) pred_y = net.predict(test_data, t_survival_ages, bb) test_survivals += np.array(pred_y) for p, survival_out, s in zip( t_survival_patients, pred_y, test_survivals ): print( 'Estimated survival = %f (%f)' % ( survival_out, s / (i + 1) ) ) test_survivals = test_survivals / n_folds for p, survival_out in zip(test_patients, test_survivals): print('Final estimated survival = %f' % survival_out) val_csvwriter.writerow([p, '%f' % float(survival_out)])
def get_all(cls): tournaments = [] return [cls(name) for name in utils.get_dirs(config.DATA_ROOT)]