def transform(cnf, n_iter, skip, test, train, weights_from, test_dir): config = util.load_module(cnf).config runs = {} if train: runs["train"] = config.get("train_dir") if test or test_dir: runs["test"] = test_dir or config.get("test_dir") net = nn.create_net(config) if weights_from is None: net.load_params_from(config.weights_file) print("loaded weights from {}".format(config.weights_file)) else: weights_from = str(weights_from) net.load_params_from(weights_from) print("loaded weights from {}".format(weights_from)) if n_iter > 1: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=config.cnf["sigma"], **config.cnf["aug_params"] ) else: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=0.0, **data.no_augmentation_params ) for run, directory in sorted(runs.items(), reverse=True): print("extracting features for files in {}".format(directory)) tic = time.time() files = data.get_image_files(directory) Xs, Xs2 = None, None for i, (tf, color_vec) in enumerate(zip(tfs, color_vecs), start=1): print("{} transform iter {}".format(run, i)) X = net.transform(files, transform=tf, color_vec=color_vec) if Xs is None: Xs = X Xs2 = X ** 2 else: Xs += X Xs2 += X ** 2 print("took {:6.1f} seconds".format(time.time() - tic)) if i % 5 == 0 or n_iter < 5: std = np.sqrt((Xs2 - Xs ** 2 / i) / (i - 1)) config.save_features(Xs / i, i, skip=skip, test=True if run == "test" else False) config.save_std(std, i, skip=skip, test=True if run == "test" else False) print("saved {} iterations".format(i))
def transform(cnf=cnf, n_iter=n_iter, skip=skip, test=test, train=train, weights_from=weights_from, test_dir=test_dir): config = util.load_module(cnf).config config.cnf['batch_size_train'] = 128 config.cnf['batch_size_test'] = 128 runs = {} if train: runs['train'] = config.get('train_dir') if test or test_dir: runs['test'] = test_dir or config.get('test_dir') net = nn.create_net(config) if weights_from is None: net.load_params_from(config.weights_file) print("loaded weights from {}".format(config.weights_file)) else: weights_from = str(weights_from) net.load_params_from(weights_from) print("loaded weights from {}".format(weights_from)) if n_iter > 1: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=config.cnf['sigma'], **config.cnf['aug_params']) else: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=0.0, **data.no_augmentation_params) ret_val = [] for run, directory in sorted(runs.items(), reverse=True): print("extracting features for files in {}".format(directory)) tic = time.time() files = data.get_image_files(directory) Xs, Xs2 = None, None for i, (tf, color_vec) in enumerate(zip(tfs, color_vecs), start=1): print("{} transform iter {}".format(run, i)) X = net.transform(files[:1000], transform=tf, color_vec=color_vec) ret_val.append(X) return ret_val, net
def estimate_scale(img): return np.maximum(img.shape[0], img.shape[1]) / 85.0 scale_factors = [estimate_scale, 5.0] # combine size-based rescaling + fixed rescaling # augmentation_transforms_test = [] # for flip in [True, False]: # for zoom in [1/1.3, 1/1.2, 1/1.1, 1.0, 1.1, 1.2, 1.3]: # for rot in np.linspace(0.0, 360.0, 5, endpoint=False): # tf = data.build_augmentation_transform(zoom=(zoom, zoom), rotation=rot, flip=flip) # augmentation_transforms_test.append(tf) augmentation_transforms_test = tta.build_quasirandom_transforms(70, **{ 'zoom_range': (1 / 1.4, 1.4), 'rotation_range': (0, 360), 'shear_range': (-10, 10), 'translation_range': (-8, 8), 'do_flip': True, 'allow_stretch': 1.2, }) data_loader = load.ZmuvMultiscaleDataLoader(scale_factors=scale_factors, num_chunks_train=num_chunks_train, patch_sizes=patch_sizes, chunk_size=chunk_size, augmentation_params=augmentation_params, augmentation_transforms_test=augmentation_transforms_test, validation_split_path=validation_split_path) # Conv2DLayer = nn.layers.cuda_convnet.Conv2DCCLayer # MaxPool2DLayer = nn.layers.cuda_convnet.MaxPool2DCCLayer Conv2DLayer = tmp_dnn.Conv2DDNNLayer
def transform(cnf, n_iter, skip, test, train, weights_from, test_dir): config = util.load_module(cnf).config config.cnf['batch_size_train'] = 128 config.cnf['batch_size_test'] = 128 runs = {} if train: runs['train'] = config.get('train_dir') if test or test_dir: runs['test'] = test_dir or config.get('test_dir') net = nn.create_net(config) if weights_from is None: net.load_params_from(config.weights_file) print("loaded weights from {}".format(config.weights_file)) else: weights_from = str(weights_from) net.load_params_from(weights_from) print("loaded weights from {}".format(weights_from)) if n_iter > 1: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=config.cnf['sigma'], **config.cnf['aug_params']) else: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=0.0, **data.no_augmentation_params) for run, directory in sorted(runs.items(), reverse=True): print("extracting features for files in {}".format(directory)) tic = time.time() files = data.get_image_files(directory) Xs, Xs2 = None, None for i, (tf, color_vec) in enumerate(zip(tfs, color_vecs), start=1): print("{} transform iter {}".format(run, i)) X = net.transform(files, transform=tf, color_vec=color_vec) if Xs is None: Xs = X Xs2 = X**2 else: Xs += X Xs2 += X**2 print('took {:6.1f} seconds'.format(time.time() - tic)) if i % 10 == 0 or n_iter < 5: std = np.sqrt((Xs2 - Xs**2 / i) / (i - 1)) config.save_features(Xs / i, i, skip=skip, test=True if run == 'test' else False) config.save_std(std, i, skip=skip, test=True if run == 'test' else False) print('saved {} iterations'.format(i))
def estimate_scale(img): return np.maximum(img.shape[0], img.shape[1]) / 85.0 # augmentation_transforms_test = [] # for flip in [True, False]: # for zoom in [1/1.3, 1/1.2, 1/1.1, 1.0, 1.1, 1.2, 1.3]: # for rot in np.linspace(0.0, 360.0, 5, endpoint=False): # tf = data.build_augmentation_transform(zoom=(zoom, zoom), rotation=rot, flip=flip) # augmentation_transforms_test.append(tf) augmentation_transforms_test = tta.build_quasirandom_transforms( 70, **{ 'zoom_range': (1 / 1.4, 1.4), 'rotation_range': (0, 360), 'shear_range': (-10, 10), 'translation_range': (-8, 8), 'do_flip': True, 'allow_stretch': 1.2, }) data_loader = load.ZmuvRescaledDataLoader( estimate_scale=estimate_scale, num_chunks_train=num_chunks_train, patch_size=patch_size, chunk_size=chunk_size, augmentation_params=augmentation_params, augmentation_transforms_test=augmentation_transforms_test, validation_split_path=validation_split_path) # Conv2DLayer = nn.layers.cuda_convnet.Conv2DCCLayer
from __future__ import division import time import click import numpy as np import nn import data_orig import tta import utils cnf='configs/c_512_5x5_32.py' config = utils.load_module(cnf).config config.cnf['batch_size_train'] = 128 runs = {} runs['train'] = config.get('train_dir') net = nn.create_net(config) weights_from = 'weights/c_512_5x5_32/weights_final.pkl' net.load_params_from(weights_from) tf, color_vecs = tta.build_quasirandom_transforms(1, skip=0, color_sigma=0.0, **data_orig.no_augmentation_params) for i, (tf, color_vec) in enumerate(zip(tfs, color_vecs), start=1): pass
def transform(cnf, exp_run_folder, n_iter, skip, test, train, weights_from, test_dir, fold): config = util.load_module(cnf).config config.cnf['fold'] = fold # <-- used to change the directories for weights_best, weights_epoch and weights_final config.cnf['exp_run_folder'] = exp_run_folder runs = {} if train: runs['train'] = config.get('train_dir') if test or test_dir: runs['test'] = test_dir or config.get('test_dir') folds = yaml.load(open('folds/'+data.settings['protocol']+'.yml')) f0, f1 = fold.split('x') train_list = folds['Fold_' + f0][int(f1)-1] test_list = folds['Fold_' + f0][0 if f1=='2' else 1] net = nn.create_net(config) if weights_from is None: net.load_params_from(config.weights_file) print("loaded weights from {}".format(config.weights_file)) else: weights_from = str(weights_from) net.load_params_from(weights_from) print("loaded weights from {}".format(weights_from)) if n_iter > 1: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=config.cnf['sigma'], **config.cnf['aug_params']) else: tfs, color_vecs = tta.build_quasirandom_transforms( n_iter, skip=skip, color_sigma=0.0, **data.no_augmentation_params) for run, directory in sorted(runs.items(), reverse=True): print("extracting features for files in {}".format(directory)) tic = time.time() if run == 'train': files = data.get_image_files(directory, train_list) else: files = data.get_image_files(directory, test_list) Xs, Xs2 = None, None for i, (tf, color_vec) in enumerate(zip(tfs, color_vecs), start=1): print("{} transform iter {}".format(run, i)) X = net.transform(files, transform=tf, color_vec=color_vec) if Xs is None: Xs = X Xs2 = X**2 else: Xs += X Xs2 += X**2 print('took {:6.1f} seconds'.format(time.time() - tic)) if i % 5 == 0 or n_iter < 5: std = np.sqrt((Xs2 - Xs**2 / i) / (i - 1)) config.save_features_fold(Xs / i, i, skip=skip, fold=fold, test=True if run == 'test' else False) #config.save_std_fold(std, i, skip=skip, fold=fold, # test=True if run == 'test' else False) print('saved {} iterations'.format(i))
scale_factors = [estimate_scale, 2.0, 5.0] # combine size-based rescaling + fixed rescaling # augmentation_transforms_test = [] # for flip in [True, False]: # for zoom in [1/1.3, 1/1.2, 1/1.1, 1.0, 1.1, 1.2, 1.3]: # for rot in np.linspace(0.0, 360.0, 5, endpoint=False): # tf = data.build_augmentation_transform(zoom=(zoom, zoom), rotation=rot, flip=flip) # augmentation_transforms_test.append(tf) augmentation_transforms_test = tta.build_quasirandom_transforms( 70, **{ "zoom_range": (1 / 1.4, 1.4), "rotation_range": (0, 360), "shear_range": (-10, 10), "translation_range": (-8, 8), "do_flip": True, "allow_stretch": 1.2, } ) data_loader = load.ZmuvMultiscaleDataLoader( scale_factors=scale_factors, num_chunks_train=num_chunks_train, patch_sizes=patch_sizes, chunk_size=chunk_size, augmentation_params=augmentation_params, augmentation_transforms_test=augmentation_transforms_test, )