def train(config_path, result_dir): config = yaml_utils.Config( yaml.load(open(config_path), Loader=yaml.SafeLoader)) result_dir, pattern = make_result_dir_path(config, result_dir) # Setup the training iteration = config.expr_itr main_script = config.main gpu_num = config.gpu_num train_cmd = [ 'mpiexec', '--allow-run-as-root', '-n', gpu_num, 'python', main_script, '--config_path', config_path ] logger.info("Start Training: {} for {} iterations".format( pattern, iteration)) for i, seed in enumerate(SEED[:iteration]): cmd = train_cmd + ['--seed', str(seed), '--process_num', str(i)] ret = subprocess.run(cmd) if ret.returncode == 0: logger.info("Succ Train: Iteration no.{} of {}".format( i + 1, pattern)) else: logger.info("Fail Train: Iteration no.{} of {}".format( i + 1, pattern)) logger.info("ErrorLog: {}".format(ret.stderr)) break logger.info("End Training: {} for {} iterations".format( pattern, iteration))
def test(config_path, result_dir): config = yaml_utils.Config( yaml.load(open(config_path), Loader=yaml.SafeLoader)) result_dir, pattern = make_result_dir_path(config, result_dir) # Setup iteration = config.expr_itr main_script = config.main.replace('trainer', 'tester') test_cmd = ['python', main_script, '--config_path', config_path] logger.info("Start Testing: {} for {} iterations".format( pattern, iteration)) for i in range(iteration): cmd = test_cmd + ['--process_num', str(i)] ret = subprocess.run(cmd) if ret.returncode == 0: logger.info("Succ Test: Iteration no.{} of {}".format( i + 1, pattern)) else: logger.info("Fail Test: Iteration no.{} of {}".format( i + 1, pattern)) logger.info("ErrorLog: {}".format(ret.stderr)) make_summary(result_dir, pattern) logger.info("End Testing: {} for {} iterations".format(pattern, iteration))
def main(): parser = argparse.ArgumentParser(description='Train pre') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/Scatter_plot.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/Scatter_plot', help='Directory to output the result') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) #make output dir result_dir = os.path.join(args.base, args.out) if not os.path.exists(result_dir): os.makedirs(result_dir) print('----- Load dataset -----') # Read path to hr data and lr data # path_pairs = [] # with open(os.path.join(args.base, config.dataset['training_fn'])) as paths_file: # for line in paths_file: # line = line.split() # if not line: continue # path_pairs.append(line[:]) # # lr_path_csv = path_pairs[2][0] # LR # hr_patch_csv = path_pairs[2][1] # HR # lr_patch = pd.read_csv(os.path.join(args.root, lr_path_csv)) # hr_patch = pd.read_csv(os.path.join(args.root, hr_patch_csv)) lr_patch = pd.read_csv( "G:/experiment/input_adjast_result/for_KL/LR_1680.csv") hr_patch = pd.read_csv( "G:/experiment/input_adjast_result/for_KL/HR_1680.csv") lr_patch.describe() hr_patch.describe() #plot plt.scatter(lr_patch['std'], lr_patch['mean'], c='blue') plt.scatter(hr_patch['std'], hr_patch['mean'], c='red') # plt.scatter(lr_patch['std'],lr_patch['mean'],c='blue') plt.ylabel('mean') plt.xlabel('std') plt.show()
def main(): parser = argparse.ArgumentParser(description='Train pre') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/cutting_position.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/cutting_position', help='Directory to output the result') parser.add_argument('--margin', '-m', default=5, help='patch margin (ratio)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) #image and mask path image_path = 'train/denoising/MicroCT.mhd' mask_path = 'train/calc_mask.mhd' #make output dir result_dir = os.path.join(args.base, args.out) if not os.path.exists(result_dir): os.makedirs(result_dir) print('----- Load data -----') sitkhr = sitk.ReadImage(os.path.join(args.root, image_path)) hr = sitk.GetArrayFromImage(sitkhr).astype("float32") sitkmask = sitk.ReadImage(os.path.join(args.root, mask_path)) calc_mask = sitk.GetArrayFromImage(sitkmask).astype("float32") #adjast train = hr[:, 0:1210, :] mask = calc_mask[:, 0:1210, :] print('----- Loop start -----') hr_patch = [] lr_patch = [] for i in range(config.num['number']): i = i + config.num['start_position'] # cut train and mask cut_hr, cut_lr = np.split(train, [i], 0) hr_mask, lr_mask = np.split(mask, [i], 0)
def expr(config_path, result_dir): reporter = SlackWrapper() config = yaml_utils.Config( yaml.load(open(config_path), Loader=yaml.SafeLoader)) result_dir, pattern = make_result_dir_path(config, result_dir) # Setup the training iteration = config.expr_itr main_script = config.main gpu_num = config.gpu_num train_cmd = [ 'mpiexec', '--allow-run-as-root', '-n', gpu_num, 'python', main_script, '--config_path', config_path ] test_cmd = [ 'python', main_script.replace('trainer', 'tester'), '--config_path', config_path ] fail_flag = False logger.info("Start: {} for {} iterations".format(pattern, iteration)) for i, seed in enumerate(SEED[:iteration]): # Training cmd = train_cmd + ['--seed', str(seed), '--process_num', str(i)] ret = subprocess.run(cmd) if ret.returncode == 0: logger.info("Succ Train: Iteration no.{} of {}".format( i + 1, pattern)) else: fail_flag = True logger.info("Fail Train: Iteration no.{} of {}".format( i + 1, pattern)) logger.info("ErrorLog: {}".format(ret.stderr)) break # Testing cmd = test_cmd + ['--process_num', str(i)] ret = subprocess.run(cmd) if ret.returncode == 0: logger.info("Succ Test: Iteration no.{} of {}".format( i + 1, pattern)) else: logger.info("Fail Test: Iteration no.{} of {}".format( i + 1, pattern)) logger.info("ErrorLog: {}".format(ret.stderr)) if fail_flag: msg = "{} was failed while the training.".format(pattern) logger.info(msg) reporter.report_fail(msg) else: summary = make_summary(result_dir, pattern) logger.info("End: {} for {} iterations".format(pattern, iteration)) reporter.report_summary(summary)
def main(): parser = argparse.ArgumentParser(description='Preprocessing for 3d-unet') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--training_list', default='configs/training_list.txt', help='Path to training image list file') parser.add_argument('--validation_list', default='configs/validation_list.txt', help='Path to validation image list file') parser.add_argument('--training_coordinate_list', type=str, default='configs/training_coordinate_list.csv') parser.add_argument('--validation_coordinate_list', type=str, default='configs/validation_coordinate_list.csv') args = parser.parse_args() config = yaml_utils.Config(yaml.load(open(os.path.join(os.path.dirname(__file__), args.config_path)))) def read_data_list_txt(data_list_txt): path_pairs = [] with open(data_list_txt) as paths_file: for line in paths_file: line = line.split() if not line : continue path_pairs.append(line[:]) return path_pairs paths = read_data_list_txt(args.training_list) for i in paths: org = IO.read_mhd_and_raw(os.path.join(args.root, 'data', i[0])) make_coordinate_csv(out_dir = args.root, csv_name = args.training_coordinate_list, input_size = [org.shape[2], org.shape[1], org.shape[0]], interval = [config.patch['interval'], config.patch['interval'],config.patch['interval']], patch_side=[config.patch['patchside'],config.patch['patchside'],config.patch['patchside']]) paths = read_data_list_txt(args.validation_list) for i in paths: org = IO.read_mhd_and_raw(os.path.join(args.root, 'data', i[0])) make_coordinate_csv(out_dir = args.root, csv_name = args.validation_coordinate_list, input_size = [org.shape[2], org.shape[1], org.shape[0]], interval = [config.patch['interval'], config.patch['interval'],config.patch['interval']], patch_side=[config.patch['patchside'],config.patch['patchside'],config.patch['patchside']])
def main(): schema = 'filename\tTop1\tTop5\tF-Score\tEntropy\tPrecision\tRecall\tF-score' parser = argparse.ArgumentParser( description='Target Model Tester \n ({})'.format(schema)) parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--results_dir', type=str, default='./result/', help='directory to save the results to') parser.add_argument('--batchsize', type=int, default=128, help='Batchsize for testing') parser.add_argument('--process_num', type=int, default=0) parser.add_argument('--seed', type=int, default=42) args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(args.config_path), Loader=yaml.SafeLoader)) pattern = "-".join([ config.pattern, config.models['classifier']['name'], config.dataset['dataset_name'] ]) out_path = args.results_dir + '/' + pattern # Model model_path = out_path + '/classifier{}.npz'.format(args.process_num) model = load_pretrained_models(config.models['classifier'], model_path) # Dataset test_dataset = yaml_utils.load_dataset(config, test=True) test_itr = chainer.iterators.SerialIterator(test_dataset, args.batchsize, repeat=False) chainer.cuda.get_device_from_id(0).use() model.to_gpu() # Copy the model to the GPU xp = model.xp pred_labels = [] correct_labels = [] count = 0 with chainer.using_config('train', False): for batch in test_itr: batchsize = len(batch) images = [batch[i][0] for i in range(batchsize)] labels = [batch[i][1] for i in range(batchsize)] x = xp.array(images) result = model(x).data pred_labels.append(chainer.cuda.to_cpu(result)) correct_labels.append(np.array(labels)) count += 1 pred_labels = np.concatenate(pred_labels) correct_labels = np.concatenate(correct_labels) top1 = F.mean(F.accuracy(pred_labels, correct_labels)).data top5 = calc_top5_acc(pred_labels, correct_labels) precision, recall, Fscore, _ = F.classification_summary( pred_labels, correct_labels) out_results = { 'test_{}'.format(args.process_num): { 'accuracy': float(top1), 'top-5 accuracy': float(top5), 'precision': float(F.mean(precision).data), 'recall': float(F.mean(recall).data), 'f-score': float(F.mean(Fscore).data) } } result_path = out_path + '/test_result.yaml' if os.path.exists(result_path): result_yaml = yaml.load(open(result_path, 'r+'), Loader=yaml.SafeLoader) else: result_yaml = {} result_yaml.update(out_results) with open(result_path, mode='w') as f: f.write(yaml.dump(result_yaml, default_flow_style=False)) print('{}\t{}\t{}\t{}\t{}\t{}'.format(pattern, top1, top5, F.mean(precision).data, F.mean(recall).data, F.mean(Fscore).data)) return out_results
def main(): parser = argparse.ArgumentParser(description='Train pre') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/Scatter_plot.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/Scatter_plot', help='Directory to output the result') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) #make output dir result_dir = os.path.join(args.base, args.out) if not os.path.exists(result_dir): os.makedirs(result_dir) print('----- Load dataset -----') # Read path to hr data and lr data path_pairs = [] with open(os.path.join(args.base, config.dataset['training_fn'])) as paths_file: for line in paths_file: line = line.split() if not line: continue path_pairs.append(line[:]) coordinate_csv_path = path_pairs[0][0] # LR coordinate_csv_path2 = path_pairs[0][1] # HR coordinate = pd.read_csv(os.path.join(args.root, coordinate_csv_path), names=("x", "y", "z")).values.tolist() coordinate2 = pd.read_csv(os.path.join(args.root, coordinate_csv_path2), names=("x", "y", "z")).values.tolist() # data load sitklr = sitk.ReadImage(os.path.join(args.root, path_pairs[1][0])) lr = sitk.GetArrayFromImage(sitklr).astype("float32") sitkhr = sitk.ReadImage(os.path.join(args.root, path_pairs[1][1])) hr = sitk.GetArrayFromImage(sitkhr).astype("float32") print('----- data load done -----') print('----- start -----') for i in range(len(coordinate)): x, y, z = coordinate[i] x_s, x_e = x, x + config.patch['patchside'] y_s, y_e = y, y + config.patch['patchside'] z_s, z_e = z, z + config.patch['patchside'] #patch image patch_image = lr[z_s:z_e, y_s:y_e, x_s:x_e] #calc std and mean Std_lr = np.std(patch_image) mean_lr = np.mean(patch_image) df = pd.DataFrame({ 'x': [x], 'y': [y], 'z': [z], 'std': [Std_lr], 'mean': [mean_lr] }) if i == 0: df.to_csv('{}/lr_std_mean.csv'.format(result_dir), index=False, encoding='utf-8', mode='a') df.to_csv('{}/lr_std_mean.csv'.format(result_dir), index=False, header=False, encoding='utf-8', mode='a') print('----- LR done -----') for j in range(len(coordinate2)): x, y, z = coordinate2[j] x_s, x_e = x, x + config.patch['patchside'] y_s, y_e = y, y + config.patch['patchside'] z_s, z_e = z, z + config.patch['patchside'] #patch image patch_image2 = hr[z_s:z_e, y_s:y_e, x_s:x_e] #calc std and mean Std_hr = np.std(patch_image2) mean_hr = np.mean(patch_image2) df = pd.DataFrame({ 'x': [x], 'y': [y], 'z': [z], 'std': [Std_hr], 'mean': [mean_hr] }) if j == 0: df.to_csv('{}/lr_std_mean.csv'.format(result_dir), index=False, encoding='utf-8', mode='a') df.to_csv('{}/hr_std_mean.csv'.format(result_dir), index=False, header=False, encoding='utf-8', mode='a') print('----- HR done -----')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--results_dir', type=str, default='./result/', help='directory to save the results to') parser.add_argument('--resume', type=str, default='', help='path to the snapshot') parser.add_argument('--process_num', type=int, default=0) parser.add_argument('--seed', type=int, default=42) args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(args.config_path), Loader=yaml.SafeLoader)) pattern = "-".join([ config.pattern, config.models['classifier']['name'], config.dataset['dataset_name'] ]) comm = chainermn.create_communicator() device = comm.intra_rank if comm.rank == 0: print('==========================================') print('Num process (COMM_WORLD): {}'.format(MPI.COMM_WORLD.Get_size())) print('Num Minibatch-size: {}'.format(config.batchsize)) print('Num Epoch: {}'.format(config.epoch)) print('==========================================') # Model classifier = load_models(config.models['classifier']) if args.resume: print("Resume training with snapshot:{}".format(args.resume)) chainer.serializers.load_npz(args.resume, classifier) chainer.cuda.get_device_from_id(device).use() classifier.to_gpu() # models = {"classifier": classifier} # Optimizer opt = make_optimizer(classifier, comm, config) opt.add_hook(chainer.optimizer.WeightDecay(5e-4)) # Dataset if comm.rank == 0: dataset = yaml_utils.load_dataset(config) first_size = int(len(dataset) * config.train_val_split_ratio) train, val = chainer.datasets.split_dataset_random(dataset, first_size, seed=args.seed) else: yaml_utils.load_module(config.dataset['dataset_func'], config.dataset['dataset_name']) train, val = None, None train = chainermn.scatter_dataset(train, comm) val = chainermn.scatter_dataset(val, comm) # Iterator train_iterator = chainer.iterators.SerialIterator(train, config.batchsize) val_iterator = chainer.iterators.SerialIterator(val, config.batchsize, repeat=False, shuffle=False) kwargs = config.updater['args'] if 'args' in config.updater else {} kwargs.update({ 'classifier': classifier, 'iterator': train_iterator, 'optimizer': opt, 'device': device, }) # Updater updater = yaml_utils.load_updater_class(config) updater = updater(**kwargs) out = args.results_dir + '/' + pattern if comm.rank == 0: create_result_dir(out, args.config_path, config) # Trainer trainer = training.Trainer(updater, (config.epoch, 'epoch'), out=out) # Evaluator evaluator = ClassifierEvaluator(val_iterator, classifier, device=device) evaluator = chainermn.create_multi_node_evaluator(evaluator, comm) trainer.extend(evaluator) # Learning Rate Schedule (fixed) schedule = [config.epoch * 0.3, config.epoch * 0.6, config.epoch * 0.8] trainer.extend(extensions.ExponentialShift('lr', 0.1), trigger=ManualScheduleTrigger(schedule, 'epoch')) report_keys = [ 'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ] if comm.rank == 0: # Set up logging trainer.extend(extensions.snapshot_object( classifier, 'classifier{}.npz'.format(args.process_num)), trigger=MaxValueTrigger('validation/main/accuracy')) trainer.extend( extensions.LogReport(keys=report_keys, trigger=(config.display_interval, 'epoch'))) trainer.extend(extensions.PrintReport(report_keys), trigger=(config.display_interval, 'epoch')) trainer.extend( extensions.ProgressBar( update_interval=config.progressbar_interval)) # Run the training trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/prediction', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data(snapshot)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--test_list', default='configs/test_list.txt', help='Path to test image list file') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) print('') unet = UNet3D(config.unet['number_of_label']) chainer.serializers.load_npz(args.model, unet) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() unet.to_gpu() xp = unet.xp # Read test list path_pairs = [] with open(os.path.join(args.base, args.test_list)) as paths_file: for line in paths_file: line = line.split() if not line: continue path_pairs.append(line[:]) for i in path_pairs: print(' Org from: {}'.format(i[0])) print(' label from: {}'.format(i[1])) sitkOrg = sitk.ReadImage(os.path.join(args.root, 'data', i[0])) org = sitk.GetArrayFromImage(sitkOrg).astype("float32") # Calculate maximum of number of patch at each side ze, ye, xe = org.shape xm = int(math.ceil((float(xe) / float(config.patch['patchside'])))) ym = int(math.ceil((float(ye) / float(config.patch['patchside'])))) zm = int(math.ceil((float(ze) / float(config.patch['patchside'])))) margin = ((0, config.patch['patchside']), (0, config.patch['patchside']), (0, config.patch['patchside'])) org = np.pad(org, margin, 'edge') org = chainer.Variable( xp.array(org[np.newaxis, np.newaxis, :], dtype=xp.float32)) prediction_map = np.zeros( (ze + config.patch['patchside'], ye + config.patch['patchside'], xe + config.patch['patchside'])) probability_map = np.zeros( (config.unet['number_of_label'], ze + config.patch['patchside'], ye + config.patch['patchside'], xe + config.patch['patchside'])) # Patch loop for s in range(xm * ym * zm): xi = int(s % xm) * config.patch['patchside'] yi = int((s % (ym * xm)) / xm) * config.patch['patchside'] zi = int(s / (ym * xm)) * config.patch['patchside'] # Extract patch from original image patch = org[:, :, zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] with chainer.using_config('train', False), chainer.using_config( 'enable_backprop', False): probability_patch = unet(patch) # Generate probability map probability_patch = probability_patch.data if args.gpu >= 0: probability_patch = chainer.cuda.to_cpu(probability_patch) for ch in range(probability_patch.shape[1]): probability_map[ch, zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] = probability_patch[ 0, ch, :, :, :] prediction_patch = np.argmax(probability_patch, axis=1) prediction_map[zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] = prediction_patch[ 0, :, :, :] print('Save image') probability_map = probability_map[:, :ze, :ye, :xe] prediction_map = prediction_map[:ze, :ye, :xe] # Save prediction map imagePrediction = sitk.GetImageFromArray(prediction_map) imagePrediction.SetSpacing(sitkOrg.GetSpacing()) imagePrediction.SetOrigin(sitkOrg.GetOrigin()) result_dir = os.path.join(args.base, args.out, os.path.dirname(i[0])) if not os.path.exists(result_dir): os.makedirs(result_dir) fn = os.path.splitext(os.path.basename(i[0]))[0] sitk.WriteImage(imagePrediction, '{}/{}.mhd'.format(result_dir, fn)) # Save probability map for ch in range(probability_map.shape[0]): imageProbability = sitk.GetImageFromArray(probability_map[ch, :]) imageProbability.SetSpacing(sitkOrg.GetSpacing()) imageProbability.SetOrigin(sitkOrg.GetOrigin()) sitk.WriteImage( imageProbability, '{}/{}_probability_{}.mhd'.format(result_dir, fn, ch))
def main(): parser = argparse.ArgumentParser(description='Train 3D-Unet') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--out', '-o', default='Results_trM1_ValiM2', help='Directory to output the result') parser.add_argument('--model', '-m', default='UNet3D_200.npz', help='Load model data') parser.add_argument('--resume', '-res', default='', help='Resume the training from snapshot') parser.add_argument('--test_list', default='configs/M3.txt', help='Path to training image list file') parser.add_argument('--test_coordinate_list', type=str, default='configs/test_coordinate_nopad52_1.csv') args = parser.parse_args() config = yaml_utils.Config( yaml.load( open(os.path.join(os.path.dirname(__file__), args.config_path)))) print('GPU: {}'.format(args.gpu)) print('') test = UnetDataset(args.root, args.test_list, args.test_coordinate_list, config.patch['patchside']) unet = UNet3D(2) if (args.gpu >= 0): use_cudnn = True unet.to_gpu() else: unet.to_cpu() chainer.serializers.load_npz(os.path.join(args.root, args.out, args.model), unet) coordi = pd.read_csv(os.path.join(args.root, args.test_coordinate_list), names=("x", "y", "z")).values.tolist() out_side = 52 ResultOut = np.zeros((860, 544, 544), dtype=np.uint8) for index in range(len(test)): t, x = test[index] x = x[np.newaxis, :] x = cp.array(x) print(x.shape) print(index) with chainer.using_config('train', False), chainer.using_config( 'enable_backprop', False): y = unet(x) y = F.softmax(y).data pred_label = np.squeeze(to_cpu(y.argmax(axis=1))) x, y, z = coordi[index] x_s, x_e = (x - int(out_side / 2)), (x + int(out_side / 2)) y_s, y_e = (y - int(out_side / 2)), (y + int(out_side / 2)) z_s, z_e = (z - int(out_side / 2)), (z + int(out_side / 2)) ResultOut[z_s:z_e, y_s:y_e, x_s:x_e] = pred_label io.save_raw(ResultOut, os.path.join(args.root, args.out, "TestResultM3.raw"), np.uint8) print("Test done")
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='../../configs/training.yml', help='path to config file') parser.add_argument('--out', '-o', default='../../work/visualize_hidden_layer', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data(snapshot)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--save_flag', type=bool, default=False, help='Decision flag whether to save criteria image') parser.add_argument('--filename_arg', type=str, default='val_fn', help='Which do you want to use val_fn or test_fn') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) LR_PATCH_SIDE, HR_PATCH_SIDE = config.patch['patchside'], config.patch[ 'patchside'] LR_PATCH_SIZE, HR_PATCH_SIZE = int(LR_PATCH_SIDE**3), int(HR_PATCH_SIDE**3) LR_MIN, LR_MAX = config.patch['lrmin'], config.patch['lrmax'] print('HR PATCH SIZE: {}'.format(HR_PATCH_SIZE)) print('LR PATCH SIZE: {}'.format(LR_PATCH_SIZE)) print('') gen = Generator() chainer.serializers.load_npz(args.model, gen) if args.gpu >= 0: chainer.backends.cuda.set_max_workspace_size(1024 * 1024 * 1024) # 1GB chainer.backends.cuda.get_device_from_id(args.gpu).use() gen.to_gpu() xp = gen.xp def create_result_dir(base_dir, output_dir, config_path, config): """https://github.com/pfnet-research/sngan_projection/blob/master/train.py""" result_dir = os.path.join(base_dir, output_dir) if not os.path.exists(result_dir): os.makedirs(result_dir) def copy_to_result_dir(fn, result_dir): bfn = os.path.basename(fn) shutil.copy(fn, '{}/{}'.format(result_dir, bfn)) copy_to_result_dir(os.path.join(base_dir, config_path), result_dir) copy_to_result_dir( os.path.join(base_dir, '../..', config.network['fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, '../..', config.updater['fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, '../..', config.dataset[args.filename_arg]), result_dir) create_result_dir(args.base, args.out, args.config_path, config) # Read data path_pairs = [] with open( os.path.join(args.base, '../..', config.dataset[args.filename_arg])) as paths_file: for line in paths_file: line = line.split() if not line: continue path_pairs.append(line[:]) print(' Inference start') for i in path_pairs: print(' Tri from: {}'.format(i[0])) print(' Org from: {}'.format(i[1])) #Read data and reshape sitkTri = sitk.ReadImage(os.path.join(args.root, i[0])) tri = sitk.GetArrayFromImage(sitkTri).astype("float32") tri = (tri - LR_MIN) / (LR_MAX - LR_MIN) # Calculate maximum of number of patch at each side ze, ye, xe = tri.shape xm = int(math.ceil((float(xe) / float(config.patch['interval'])))) ym = int(math.ceil((float(ye) / float(config.patch['interval'])))) zm = int(math.ceil((float(ze) / float(config.patch['interval'])))) margin = ((0, config.patch['patchside']), (0, config.patch['patchside']), (0, config.patch['patchside'])) tri = np.pad(tri, margin, 'edge') tri = chainer.Variable( xp.array(tri[np.newaxis, np.newaxis, :], dtype=xp.float32)) ch = 4 sc_map = np.zeros( (ch, ze + config.patch['patchside'], ye + config.patch['patchside'], xe + config.patch['patchside'])) res_map = np.zeros( (ch, ze + config.patch['patchside'], ye + config.patch['patchside'], xe + config.patch['patchside'])) overlap_count = np.zeros( (sc_map.shape[1], sc_map.shape[2], sc_map.shape[3])) # Patch loop print(' #Patches {}'.format(xm * ym * zm)) for s in range(xm * ym * zm): xi = int(s % xm) * config.patch['interval'] yi = int((s % (ym * xm)) / xm) * config.patch['interval'] zi = int(s / (ym * xm)) * config.patch['interval'] # Extract patch from original image patch = tri[:, :, zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] with chainer.using_config('train', False), chainer.using_config( 'enable_backprop', False): sc, res = gen.get_hidden_layer(patch) # Generate probability map sc = sc.array res = res.array if args.gpu >= 0: sc = chainer.backends.cuda.to_cpu(sc) res = chainer.backends.cuda.to_cpu(res) sc_map[:, zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] += np.squeeze(sc) res_map[:, zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] += np.squeeze(res) overlap_count[zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] += 1 print(' Save image') sc_map = sc_map[:, :ze, :ye, :xe] res_map = res_map[:, :ze, :ye, :xe] overlap_count = overlap_count[:ze, :ye, :xe] sc_map[:, ...] /= overlap_count res_map[:, ...] /= overlap_count # Save prediction map result_dir = os.path.join(args.base, args.out) sc_dir = '{}/sc'.format(result_dir) res_dir = '{}/res'.format(result_dir) if not os.path.exists(sc_dir): os.makedirs(sc_dir) if not os.path.exists(res_dir): os.makedirs(res_dir) sc_ims = [] res_ims = [] for loop in range(sc_map.shape[0]): fn = os.path.splitext(os.path.basename(i[0]))[0] scImage = sitk.GetImageFromArray(sc_map[loop, ...]) scImage.SetSpacing(sitkTri.GetSpacing()) scImage.SetOrigin(sitkTri.GetOrigin()) sitk.WriteImage(scImage, '{}/{}_{:04d}.mhd'.format(sc_dir, fn, loop)) resImage = sitk.GetImageFromArray(res_map[loop, ...]) resImage.SetSpacing(sitkTri.GetSpacing()) resImage.SetOrigin(sitkTri.GetOrigin()) sitk.WriteImage(resImage, '{}/{}_{:04d}.mhd'.format(res_dir, fn, loop)) # Rescale for visualization nd_sc = sitk.GetArrayFromImage( sitk.Cast(sitk.RescaleIntensity(scImage), sitk.sitkUInt8))[240, 30:130, 80:180] nd_res = sitk.GetArrayFromImage( sitk.Cast(sitk.RescaleIntensity(resImage), sitk.sitkUInt8))[240, 30:130, 80:180] sc_ims.append(nd_sc) res_ims.append(nd_res) sc_ims = np.asarray(sc_ims).transpose(1, 0, 2).reshape((100, -1)) res_ims = np.asarray(res_ims).transpose(1, 0, 2).reshape((100, -1)) ims = np.vstack((sc_ims, res_ims)) preview_path = '{}/preview.png'.format(result_dir) figure = plt.figure() plt.imshow(ims, cmap='gray', interpolation='none') plt.axis('off') plt.savefig(preview_path, dpi=300) with open('{}/preview.pickle'.format(result_dir), 'wb') as f: pickle.dump(figure, f) plt.show() print(' Inference done')
def main(): parser = argparse.ArgumentParser(description='Train CVAE') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/training.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/training', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data') parser.add_argument('--resume', '-res', default='', help='Resume the training from snapshot') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(config.batchsize)) print('# iteration: {}'.format(config.iteration)) print('Learning Rate: {}'.format(config.adam['alpha'])) print('') #load the dataset print('----- Load dataset -----') train = CvaeDataset(args.root, os.path.join(args.base, config.dataset['training_fn']), config.patch['patchside'], [config.patch['lrmin'], config.patch['lrmax']], augmentation=True) train_iter = chainer.iterators.SerialIterator(train, batch_size=config.batchsize) val = CvaeDataset(args.root, os.path.join(args.base, config.dataset['val_fn']), config.patch['patchside'], [config.patch['lrmin'], config.patch['lrmax']], augmentation=False) val_iter = chainer.iterators.SerialIterator(val, batch_size=config.batchsize, repeat=False, shuffle=False) print('----- Set up model ------') avg_elbo_loss = AvgELBOLoss() if args.gpu >= 0: chainer.backends.cuda.set_max_workspace_size(1024 * 1024 * 1024) # 1GB chainer.backends.cuda.get_device_from_id(args.gpu).use() avg_elbo_loss.to_gpu(args.gpu) print('----- Make optimizer -----') def make_optimizer(model, alpha=0.00001, beta1=0.9, beta2=0.999): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1, beta2=beta2) optimizer.setup(model) return optimizer gen_opt = make_optimizer(model=avg_elbo_loss, alpha=config.adam['alpha'], beta1=config.adam['beta1'], beta2=config.adam['beta2']) print('----- Make updater -----') updater = training.updaters.StandardUpdater(iterator=train_iter, optimizer=gen_opt, device=args.gpu, loss_func=avg_elbo_loss) print('----- Save configs -----') def create_result_dir(base_dir, output_dir, config_path, config): """https://github.com/pfnet-research/sngan_projection/blob/master/train.py""" result_dir = os.path.join(base_dir, output_dir) if not os.path.exists(result_dir): os.makedirs(result_dir) def copy_to_result_dir(fn, result_dir): bfn = os.path.basename(fn) shutil.copy(fn, '{}/{}'.format(result_dir, bfn)) copy_to_result_dir(os.path.join(base_dir, config_path), result_dir) copy_to_result_dir(os.path.join(base_dir, config.network['fn']), result_dir) copy_to_result_dir(os.path.join(base_dir, config.dataset['fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, config.dataset['training_fn']), result_dir) create_result_dir(args.base, args.out, args.config_path, config) print('----- Make trainer -----') trainer = training.Trainer(updater, (config.iteration, 'iteration'), out=os.path.join(args.base, args.out)) print('----- Set up logging -----') snapshot_interval = (config.snapshot_interval, 'iteration') display_interval = (config.display_interval, 'iteration') evaluation_interval = (config.evaluation_interval, 'iteration') trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( avg_elbo_loss.encoder, filename='encoder_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( avg_elbo_loss.decoder, filename='decoder_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport(trigger=display_interval)) # Evaluator trainer.extend(CvaeEvaluator(val_iter, avg_elbo_loss, device=args.gpu), trigger=evaluation_interval) trainer.extend(generate_samples(avg_elbo_loss, os.path.join(args.base, args.out)), trigger=evaluation_interval, priority=extension.PRIORITY_WRITER) # trainer.extend(projection_to_latent_space(avg_elbo_loss, os.path.join(args.base, args.out), train), # trigger=evaluation_interval, # priority=extension.PRIORITY_WRITER) trainer.extend(reconstruction_img(avg_elbo_loss, os.path.join(args.base, args.out), train), trigger=evaluation_interval, priority=extension.PRIORITY_WRITER) # Print selected entries of the log to stdout #trainer.extend(extensions.PrintReport(['epoch', 'iteration', 'gen/loss', 'elapsed_time']), trigger=display_interval) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar(update_interval=10)) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/kl_penalty', 'validation/kl_penalty'], 'iteration', file_name='kl_loss.png', trigger=display_interval)) trainer.extend( extensions.PlotReport(['main/reconstr', 'validation/reconstr'], 'iteration', file_name='reconstr_loss.png', trigger=display_interval)) trainer.extend( extensions.PlotReport(['main/loss', 'validation/loss'], 'iteration', file_name='loss.png', trigger=display_interval)) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training reset_seed(0) trainer.run()
def main(): parser = argparse.ArgumentParser(description='Train CycleGAN') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/training.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/training', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data') parser.add_argument('--model2', '-m2', default='', help='Load model data') parser.add_argument('--resume', '-res', default='', help='Resume the training from snapshot') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(config.batchsize)) print('# iteration: {}'.format(config.iteration)) print('Learning Rate: {}'.format(config.adam['alpha'])) print('') #load the dataset print('----- Load dataset -----') train = CycleganDataset(args.root, os.path.join(args.base, config.dataset['training_fn']), config.patch['patchside'], [config.patch['lrmin'], config.patch['lrmax']], augmentation=True) train_iter = chainer.iterators.MultiprocessIterator( train, batch_size=config.batchsize) print('----- Set up model ------') gen = Generator_SR() gen2 = Generator_SR() disY = Discriminator() # chainer.serializers.load_npz(args.model, gen) # chainer.serializers.load_npz(args.model2, gen2) if args.gpu >= 0: chainer.backends.cuda.set_max_workspace_size(1024 * 1024 * 1024) # 1GB chainer.backends.cuda.get_device_from_id(args.gpu).use() gen.to_gpu() gen2.to_gpu() disY.to_gpu() print('----- Make optimizer -----') def make_optimizer(model, alpha=0.00001, beta1=0.9, beta2=0.999): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1, beta2=beta2) optimizer.setup(model) return optimizer gen_opt = make_optimizer(model=gen, alpha=config.adam['alpha'], beta1=config.adam['beta1'], beta2=config.adam['beta2']) gen2_opt = make_optimizer(model=gen2, alpha=config.adam['alpha'], beta1=config.adam['beta1'], beta2=config.adam['beta2']) disY_opt = make_optimizer(model=disY, alpha=config.adam['alpha'], beta1=config.adam['beta1'], beta2=config.adam['beta2']) print('----- Make updater -----') updater = CinCGANUpdater(models=(gen, gen2, disY), iterator=train_iter, optimizer={ 'gen': gen_opt, 'gen2': gen2_opt, 'disY': disY_opt }, device=args.gpu) print('----- Save configs -----') def create_result_dir(base_dir, output_dir, config_path, config): """https://github.com/pfnet-research/sngan_projection/blob/master/train.py""" result_dir = os.path.join(base_dir, output_dir) if not os.path.exists(result_dir): os.makedirs(result_dir) if not os.path.exists('{}/init'.format(result_dir)): os.makedirs('{}/init'.format(result_dir)) def copy_to_result_dir(fn, result_dir): bfn = os.path.basename(fn) shutil.copy(fn, '{}/{}'.format(result_dir, bfn)) copy_to_result_dir(os.path.join(base_dir, config_path), result_dir) copy_to_result_dir(os.path.join(base_dir, config.network['fn']), result_dir) copy_to_result_dir(os.path.join(base_dir, config.updater['fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, config.dataset['training_fn']), result_dir) create_result_dir(args.base, args.out, args.config_path, config) print('----- Make trainer -----') trainer = training.Trainer(updater, (config.iteration, 'iteration'), out=os.path.join(args.base, args.out)) # Set up logging snapshot_interval = (config.snapshot_interval, 'iteration') display_interval = (config.display_interval, 'iteration') evaluation_interval = (config.evaluation_interval, 'iteration') trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen, filename='gen_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( gen2, filename='gen2_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( disY, filename='disY_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport(trigger=display_interval)) trainer.extend(reconstruct_hr_img(gen, gen2, os.path.join(args.base, args.out), train_iter, train), trigger=evaluation_interval, priority=extension.PRIORITY_WRITER) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar(update_interval=10)) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['gen/loss_gen1'], 'iteration', file_name='gen_loss.png', trigger=display_interval)) trainer.extend( extensions.PlotReport([ 'disY/loss_dis1_fake', 'disY/loss_dis1_real', 'disY/loss_dis1' ], 'iteration', file_name='dis_loss.png', trigger=display_interval)) trainer.extend( extensions.PlotReport(['gen/loss_gen', 'disY/loss_dis1'], 'iteration', file_name='adv_loss.png', trigger=display_interval)) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) print('----- Run the training -----') reset_seed(0) trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/training.yml', help='path to config file') parser.add_argument('--out', '-o', default= 'results/inference/test', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data(snapshot)') parser.add_argument('--model2', '-m2', default='', help='Load model data(snapshot)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--save_flag', type=bool, default=False, help='Decision flag whether to save criteria image') parser.add_argument('--filename_arg', '-F',type=str, default='test_fn', help='Which do you want to use val_fn or test_fn') args = parser.parse_args() config = yaml_utils.Config(yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) LR_PATCH_SIDE, HR_PATCH_SIDE = config.patch['patchside'], config.patch['patchside'] LR_PATCH_SIZE, HR_PATCH_SIZE = int(LR_PATCH_SIDE**3), int(HR_PATCH_SIDE**3) LR_MIN, LR_MAX = config.patch['lrmin'], config.patch['lrmax'] print('HR PATCH SIZE: {}'.format(HR_PATCH_SIZE)) print('LR PATCH SIZE: {}'.format(LR_PATCH_SIZE)) print('') gen = Generator_SR() chainer.serializers.load_npz(args.model, gen) if args.gpu >= 0: chainer.backends.cuda.set_max_workspace_size(1024 * 1024 * 1024) # 1GB chainer.backends.cuda.get_device_from_id(args.gpu).use() gen.to_gpu() xp = gen.xp def create_result_dir(base_dir, output_dir, config_path, config): """https://github.com/pfnet-research/sngan_projection/blob/master/train.py""" result_dir = os.path.join(base_dir, output_dir) if not os.path.exists(result_dir): os.makedirs(result_dir) def copy_to_result_dir(fn, result_dir): bfn = os.path.basename(fn) shutil.copy(fn, '{}/{}'.format(result_dir, bfn)) copy_to_result_dir( os.path.join(base_dir, config_path), result_dir) copy_to_result_dir( os.path.join(base_dir, config.network['fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, config.updater['fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, config.dataset[args.filename_arg]), result_dir) create_result_dir(args.base, args.out, args.config_path, config) # Read data path_pairs = [] with open(os.path.join(args.base, config.dataset[args.filename_arg])) as paths_file: for line in paths_file: line = line.split() if not line : continue path_pairs.append(line[:]) print(' Inference start') for i in path_pairs: print(' Tri from: {}'.format(i[0])) print(' Org from: {}'.format(i[1])) #Read data and reshape sitkTri = sitk.ReadImage(os.path.join(args.root, i[0])) tri = (sitk.GetArrayFromImage(sitkTri).astype("float32")/127.5)-1. # Calculate maximum of number of patch at each side ze, ye, xe = tri.shape xm = int(math.ceil((float(xe)/float(config.patch['interval'])))) ym = int(math.ceil((float(ye)/float(config.patch['interval'])))) zm = int(math.ceil((float(ze)/float(config.patch['interval'])))) edge = 16. margin = ((8, config.patch['patchside']), (8, config.patch['patchside']), (8, config.patch['patchside'])) tri = np.pad(tri, margin, 'edge') tri = chainer.Variable(xp.array(tri[np.newaxis, np.newaxis, :], dtype=xp.float32)) inferred_map = np.zeros((ze+config.patch['patchside'],ye+config.patch['patchside'], xe+config.patch['patchside'])) overlap_count = np.zeros(inferred_map.shape) # Patch loop print(' #Patches {}'.format(xm*ym*zm)) for s in range(xm*ym*zm): xi = int(s%xm)*config.patch['interval'] yi = int((s%(ym*xm))/xm)*config.patch['interval'] zi = int(s/(ym*xm))*config.patch['interval'] # Extract patch from original image patch = tri[:,:,zi:zi+config.patch['patchside']+edge,yi:yi+config.patch['patchside']+edge,xi:xi+config.patch['patchside']+edge] with chainer.using_config('train', False), chainer.using_config('enable_backprop', False): inferred_patch = gen(patch)# T48*48*48 inferred_patch = cropping(inferred_patch, 32) # 32*32*32 # Generate probability map inferred_patch = inferred_patch.data if args.gpu >= 0: inferred_patch = chainer.backends.cuda.to_cpu(inferred_patch) inferred_map[zi:zi+config.patch['patchside'],yi:yi+config.patch['patchside'],xi:xi+config.patch['patchside']] += np.squeeze(inferred_patch) overlap_count[zi:zi+config.patch['patchside'],yi:yi+config.patch['patchside'],xi:xi+config.patch['patchside']] += 1 print(' Save image') inferred_map = inferred_map[:ze,:ye,:xe] overlap_count = overlap_count[:ze,:ye,:xe] inferred_map /= overlap_count inferred_map = ((inferred_map+1.)/2*255) # Save prediction map inferenceImage = sitk.GetImageFromArray(inferred_map) inferenceImage.SetSpacing(sitkTri.GetSpacing()) inferenceImage.SetOrigin(sitkTri.GetOrigin()) result_dir = os.path.join(args.base, args.out) if not os.path.exists(result_dir): os.makedirs(result_dir) fn = os.path.splitext(os.path.basename(i[0]))[0] sitk.WriteImage(inferenceImage, '{}/{}.mhd'.format(result_dir, fn)) print(' Start evaluations ') sitkGt = sitk.ReadImage(os.path.join(args.root, i[1])) gt = sitk.GetArrayFromImage(sitkGt).astype("float") start = time.time() mse_const = calc_mse(gt, inferred_map) psnr_const = calc_psnr(gt, inferred_map) ssim_const = calc_ssim(gt, inferred_map) zncc_const = calc_zncc(gt, inferred_map) diff_spe_const, diff_spe, diff_ang_const, diff_ang = calc_score_on_fft_domain(gt, inferred_map) df = pd.DataFrame({'MSE': [mse_const], 'PSNR':[psnr_const], 'SSIM':[ssim_const], 'ZNCC':[zncc_const], 'MAE-Power':[diff_spe_const], 'MAE-Angle':[diff_ang_const]}) df.to_csv('{}/results.csv'.format(result_dir), index=False, encoding='utf-8', mode='w') print(' Finsish evaluations: {:.3f} [s]'.format(time.time() - start)) if args.save_flag: def array2sitk(arr, spacing=[], origin=[]): if not len(spacing) == arr.ndim or len(origin) == arr.ndim: print("Dimension Error") quit() sitkImg = sitk.GetImageFromArray(arr) sitkImg.SetSpacing(spacing) sitkImg.SetOrigin(origin) return sitkImg diffSpeImage = array2sitk(diff_spe, [1,1,1], [0,0,0]) diffAngImage = array2sitk(diff_ang, [1,1,1], [0,0,0]) sitk.WriteImage(diffSpeImage, '{}/{}-diff_power_spe.mhd'.format(result_dir, fn)) sitk.WriteImage(diffAngImage, '{}/{}-diff_angle.mhd'.format(result_dir, fn)) print(' Inference done')
def main(): parser = argparse.ArgumentParser(description='Train 3D-Unet') parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--out', '-o', default='Results_trM1_ValiM2', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data') parser.add_argument('--resume', '-res', default='', help='Resume the training from snapshot') parser.add_argument('--training_list', default='configs/M1.txt', help='Path to training image list file') parser.add_argument('--training_coordinate_list', type=str, default='configs/M1.csv') parser.add_argument('--validation_list', default='configs/M2.txt', help='Path to validation image list file') parser.add_argument('--validation_coordinate_list', type=str, default='configs/M2.csv') args = parser.parse_args() ''' 'https://stackoverflow.com/questions/21005822/what-does-os-path-abspathos-path-joinos-path-dirname-file-os-path-pardir' ''' config = yaml_utils.Config( yaml.load( open(os.path.join(os.path.dirname(__file__), args.config_path)))) print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(config.batchsize)) print('# iteration: {}'.format(config.iteration)) print('') # Load the datasets train = UnetDataset(args.root, args.training_list, args.training_coordinate_list, config.patch['patchside']) train_iter = chainer.iterators.SerialIterator(train, batch_size=config.batchsize) validation = UnetDataset(args.root, args.validation_list, args.validation_coordinate_list, config.patch['patchside']) validation_iter = chainer.iterators.SerialIterator( validation, batch_size=config.batchsize, repeat=False, shuffle=False) # Set up a neural network to train print('Set up a neural network to train') unet = UNet3D(2) if args.model: chainer.serializers.load_npz(args.model, gen) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() unet.to_gpu() #Set up an optimizer def make_optimizer(model, alpha=0.0001, beta1=0.9, beta2=0.999): optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1, beta2=beta2) optimizer.setup(model) return optimizer opt_unet = make_optimizer(model=unet, alpha=config.adam['alpha'], beta1=config.adam['beta1'], beta2=config.adam['beta2']) #Set up a trainer updater = Unet3DUpdater(models=(unet), iterator=train_iter, optimizer={'unet': opt_unet}, device=args.gpu) def create_result_dir(base, result_dir, config_path, config): """https://github.com/pfnet-research/sngan_projection/blob/master/train.py""" if not os.path.exists(result_dir): os.makedirs(result_dir) def copy_to_result_dir(fn, result_dir): bfn = os.path.basename(fn) shutil.copy(fn, '{}/{}'.format(result_dir, bfn)) copy_to_result_dir(config_path, result_dir) copy_to_result_dir(os.path.join(base, config.unet['fn']), result_dir) copy_to_result_dir(os.path.join(base, config.updater['fn']), result_dir) out = os.path.join(args.root, args.out) config_path = os.path.join(os.path.dirname(__file__), args.config_path) create_result_dir(args.root, out, config_path, config) trainer = training.Trainer(updater, (config.iteration, 'iteration'), out=out) #serializers.load_npz('C:\\Users\\yourb\\Documents\\GitHub\\3D-Unet\\Results_trM1_ValiM2\\snapshot_iter_10500.npz', trainer) # Set up logging snapshot_interval = (config.snapshot_interval, 'iteration') display_interval = (config.display_interval, 'iteration') evaluation_interval = (config.evaluation_interval, 'iteration') trainer.extend(UNet3DEvaluator(validation_iter, unet, device=args.gpu), trigger=evaluation_interval) trainer.extend( extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'), trigger=snapshot_interval) trainer.extend(extensions.snapshot_object( unet, filename=unet.__class__.__name__ + '_{.updater.iteration}.npz'), trigger=snapshot_interval) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport(trigger=display_interval)) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar(update_interval=10)) # Print selected entries of the log to stdout #report_keys = ['epoch', 'iteration', 'unet/loss','unet/dice','vali/unet/loss','vali/unet/dice'] report_keys = ['iteration', 'unet/dice', 'vali/unet/dice'] trainer.extend(extensions.PrintReport(report_keys), trigger=display_interval) # Use linear shift ext_opt_unet = extensions.LinearShift( 'alpha', (config.adam['alpha'], 0.), (config.iteration_decay_start, config.iteration), opt_unet) trainer.extend(ext_opt_unet) # Save two plot images to the result dir if extensions.PlotReport.available(): #trainer.extend(extensions.PlotReport(['unet/loss','vali/unet/loss'], 'iteration', file_name='unet_loss.png',trigger=display_interval)) trainer.extend( extensions.PlotReport(['unet/dice', 'vali/unet/dice'], 'iteration', file_name='unet_dice.png', trigger=display_interval)) if args.resume: # Resume from a snapshot print("Resume training with snapshot:{}".format(args.resume)) chainer.serializers.load_npz(args.resume, trainer) # Run the training chainer.config.autotune = True print('Start training') trainer.run()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/training.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/projection', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data(snapshot)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') parser.add_argument('--num_patches', type=int, default=500, help='number of patches that you want to extract') parser.add_argument('--filename', type=str, default='training_fn', help='Which do you want to use val_fn or test_fn') parser.add_argument('--dataset', help='path to dataset pickle') args = parser.parse_args() if args.dataset: with open(args.dataset, 'rb') as f: dataset = pickle.load(f) out = '{}/{}'.format(args.out, args.filename) output_dir = os.path.join(args.base, out) if not os.path.exists(output_dir): os.makedirs(output_dir) plot(dataset, output_dir) return print('----- Read configs ------') config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) print('----- Load model ------') encoder = Encoder() chainer.serializers.load_npz(args.model, encoder) if args.gpu >= 0: chainer.backends.cuda.set_max_workspace_size(1024 * 1024 * 1024) # 1GB chainer.backends.cuda.get_device_from_id(args.gpu).use() encoder.to_gpu() xp = encoder.xp print('----- Save configs ------') def create_result_dir(base_dir, output_dir, config_path, config): """https://github.com/pfnet-research/sngan_projection/blob/master/train.py""" result_dir = os.path.join(base_dir, output_dir) if not os.path.exists(result_dir): os.makedirs(result_dir) if not os.path.exists('{}/model'.format(result_dir)): os.makedirs('{}/model'.format(result_dir)) def copy_to_result_dir(fn, result_dir): bfn = os.path.basename(fn) shutil.copy(fn, '{}/{}'.format(result_dir, bfn)) copy_to_result_dir(os.path.join(base_dir, config_path), result_dir) copy_to_result_dir(os.path.join(base_dir, config.network['fn']), result_dir) copy_to_result_dir(os.path.join(base_dir, config.dataset['val_fn']), result_dir) copy_to_result_dir( os.path.join(base_dir, config.dataset['training_fn']), result_dir) copy_to_result_dir(os.path.join(base_dir, args.model), '{}/model'.format(result_dir)) out = '{}/{}'.format(args.out, args.filename) create_result_dir(args.base, out, args.config_path, config) output_dir = os.path.join(args.base, out) print('----- Load dataset -----') dataset = CvaeDataset(args.root, os.path.join(args.base, config.dataset[args.filename]), config.patch['patchside'], [config.patch['lrmin'], config.patch['lrmax']], augmentation=False) print('----- Start projection ------') index_list = [] latent_list = [] for n in range(args.num_patches): idx = np.random.randint(0, dataset.__len__()) patch = dataset.get_example(idx) patch = chainer.Variable( xp.array(patch[np.newaxis, :], dtype=xp.float32)) with chainer.using_config('train', False), chainer.using_config( 'enable_backprop', False): mu, _ = encoder(patch) mu = mu.array if args.gpu >= 0: mu = chainer.backends.cuda.to_cpu(mu) mu = np.squeeze(mu, axis=0) latent_list.append(mu) index_list.append(idx) latent_list = np.asarray(latent_list) print('----- Start save -----') np.savetxt('{}/coordinate_idx.csv'.format(output_dir), np.asarray(index_list, dtype=int), delimiter=',') np.savetxt('{}/latent_data.csv'.format(output_dir), latent_list, delimiter=',') with open('{}/latent_data.pickle'.format(output_dir), 'wb') as f: pickle.dump(latent_list, f) print('----- plot results ------') plot(latent_list, output_dir)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/base.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/inference', help='Directory to output the result') parser.add_argument('--model', '-m', default='', help='Load model data(snapshot)') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) print('GPU: {}'.format(args.gpu)) print('') hr_patchside = config.patch['patchside'] config.patch['patchside'] = int(config.patch['patchside'] / config.upsampling_rate) gen = ESPCN(r=config.upsampling_rate) chainer.serializers.load_npz(args.model, gen) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() gen.to_gpu() xp = gen.xp # Read test list path_pairs = [] with open(os.path.join(args.base, config.dataset['test_fn'])) as paths_file: for line in paths_file: line = line.split() if not line: continue path_pairs.append(line[:]) case_list = [] ssim_list = [] psnr_list = [] for i in path_pairs: print(' LR from: {}'.format(i[0])) print(' HR from: {}'.format(i[1])) sitkLR = sitk.ReadImage(os.path.join(args.root, i[0])) lr = sitk.GetArrayFromImage(sitkLR).astype("float32") # Calculate maximum of number of patch at each side ze, ye, xe = lr.shape xm = int(math.ceil((float(xe) / float(config.patch['patchside'])))) ym = int(math.ceil((float(ye) / float(config.patch['patchside'])))) zm = int(math.ceil((float(ze) / float(config.patch['patchside'])))) margin = ((0, config.patch['patchside']), (0, config.patch['patchside']), (0, config.patch['patchside'])) lr = np.pad(lr, margin, 'edge') lr = chainer.Variable( xp.array(lr[np.newaxis, np.newaxis, :], dtype=xp.float32)) zh, yh, xh = ze * config.upsampling_rate, ye * config.upsampling_rate, xe * config.upsampling_rate hr_map = np.zeros( (zh + hr_patchside, yh + hr_patchside, xh + hr_patchside)) # Patch loop for s in range(xm * ym * zm): xi = int(s % xm) * config.patch['patchside'] yi = int((s % (ym * xm)) / xm) * config.patch['patchside'] zi = int(s / (ym * xm)) * config.patch['patchside'] # Extract patch from original image patch = lr[:, :, zi:zi + config.patch['patchside'], yi:yi + config.patch['patchside'], xi:xi + config.patch['patchside']] with chainer.using_config('train', False), chainer.using_config( 'enable_backprop', False): hr_patch = gen(patch) # Generate HR map hr_patch = hr_patch.data if args.gpu >= 0: hr_patch = chainer.cuda.to_cpu(hr_patch) zi, yi, xi = zi * config.upsampling_rate, yi * config.upsampling_rate, xi * config.upsampling_rate hr_map[zi:zi + hr_patchside, yi:yi + hr_patchside, xi:xi + hr_patchside] = hr_patch[0, :, :, :] print('Save image') hr_map = hr_map[:zh, :yh, :xh] # Save HR map inferenceHrImage = sitk.GetImageFromArray(hr_map) lr_spacing = sitkLR.GetSpacing() new_spacing = [i / config.upsampling_rate for i in lr_spacing] inferenceHrImage.SetSpacing(new_spacing) inferenceHrImage.SetOrigin(sitkLR.GetOrigin()) result_dir = os.path.join(args.base, args.out) if not os.path.exists(result_dir): os.makedirs(result_dir) fn = os.path.splitext(os.path.basename(i[0]))[0] sitk.WriteImage(inferenceHrImage, '{}/{}.mhd'.format(result_dir, fn)) # Calc metric case_list.append(os.path.basename(i[0])) # PSNR sitkHR = sitk.ReadImage(os.path.join(args.root, i[1])) hr_gt = sitk.GetArrayFromImage(sitkHR).astype("float") psnr_const = psnr(hr_gt, hr_map, dynamic_range=np.amax(hr_gt) - np.amin(hr_gt)) print('PSNR: {}'.format(psnr_const)) psnr_list.append(psnr_const) # SSIM ssim_const = ssim(hr_gt, hr_map, dynamic_range=np.amax(hr_gt) - np.amin(hr_gt), gaussian_weights=True, use_sample_covariance=False) print('SSIM: {}'.format(ssim_const)) ssim_list.append(ssim_const) df = pd.DataFrame({ 'Case': case_list, 'PSNR': psnr_list, 'SSIM': ssim_list }) df.to_csv('{}/result.csv'.format(result_dir), index=False, encoding="utf-8", mode='w')
def main(): parser = argparse.ArgumentParser(description='Train pre') parser.add_argument('--base', '-B', default=os.path.dirname(os.path.abspath(__file__)), help='base directory path of program files') parser.add_argument('--config_path', type=str, default='configs/cutting_position.yml', help='path to config file') parser.add_argument('--out', '-o', default='results/histogram_base_KL_debug', help='Directory to output the result') parser.add_argument('--root', '-R', default=os.path.dirname(os.path.abspath(__file__)), help='Root directory path of input image') args = parser.parse_args() config = yaml_utils.Config( yaml.load(open(os.path.join(args.base, args.config_path)))) #make output dir result_dir = os.path.join(args.base, args.out) if not os.path.exists(result_dir): os.makedirs(result_dir) mu1 = [-10, 0] mu2 = [10, 0] cov = [[2, 0], [0, 2]] total = 45000 #num of data loop = 100 # num of loop epshiron = [2.22e-6, 2.22e-7, 2.22e-8] num_bins = np.arange(0.1, 5.0, 0.1, dtype=float) #Scatter_plot t1, v1 = np.random.multivariate_normal(mu1, cov, total).T t2, v2 = np.random.multivariate_normal(mu2, cov, total).T plt.scatter(t1, v1, c='cyan') plt.scatter(t2, v2, c='green') plt.ylabel('y') plt.xlabel('x') plt.show() print('----- Start -----') for e in range(len(epshiron)): ep = epshiron[e] kld = np.zeros(loop) x_bin = np.zeros(loop) y_bin = np.zeros(loop) for num_b in range(len(num_bins)): num_bin = num_bins[num_b] for i in range(loop): #make datas x1, y1 = np.random.multivariate_normal(mu1, cov, total).T x2, y2 = np.random.multivariate_normal(mu2, cov, total).T # join x3 = np.hstack((x1, x2)) y3 = np.hstack((y1, y2)) #calc num of bins binx = int( float(abs(int(np.min(x3) - 1)) + int(np.max(x3) + 1)) * num_bin) biny = int( float(abs(int(np.min(y3) - 1)) + int(np.max(y3) + 1)) * num_bin) x_bin[i] = binx y_bin[i] = biny # make two histograms hist_lr, _, _ = np.histogram2d( x1, y1, bins=[binx, biny], range=[[int(np.min(x3) - 1.), int(np.max(x3) + 1.)], [int(np.min(y3) - 1.), int(np.max(y3) + 1.)]]) hist_hr, _, _ = np.histogram2d( x2, y2, bins=[binx, biny], range=[[int(np.min(x3) - 1.), int(np.max(x3) + 1.)], [int(np.min(y3) - 1.), int(np.max(y3) + 1.)]]) if (np.sum(hist_hr) != total or np.sum(hist_lr) != total): sys.exit() #calc histogram base KL divergence kld[i] = KLD(hist_lr, hist_hr, ep) #save info df = pd.DataFrame({ 'ep': [ep], 'num_b': [num_bin], 'KLD_mean': [np.mean(kld)], 'KLD_std': [np.std(kld)], 'binx_mean': [np.mean(x_bin)], 'binx_std': [np.std(x_bin)], 'biny_mean': [np.mean(y_bin)], 'biny_std': [np.std(y_bin)] }) if (e == 0 and num_b == 0.1): df.to_csv('{}/results.csv'.format(result_dir), index=False, encoding='utf-8', mode='a', header=True) df.to_csv('{}/results_another3.csv'.format(result_dir), index=False, encoding='utf-8', mode='a', header=None) print('----- Finish -----')