def optimize_a_b_step(data, template, y, a, b, idx_out_train, idx_out_test, pipeline_params, pad_size): test_score_prediction = import_func(**pipeline_params['prediction_func']) count_grads_a_b = import_func(**pipeline_params['count_grads_a_b']) reg = pipeline_params['pipeline_optimization_params'] y_out_train = y[idx_out_train] kwargs = create_kwargs(pipeline_params, data, template, a, b, idx_out_train, False, reg['add_padding'], pad_size) K, da, db = count_dist_matrix_to_template(**kwargs) K_out_train = K[np.ix_(idx_out_train, idx_out_train)] best_params = find_pipeline_params( K_out_train, y_out_train, pipeline_params['ml_params'], pipeline_params['n_jobs'], random_state=pipeline_params['random_state'], scaled=pipeline_params['scaled'], scoring='roc_auc', n_splits=pipeline_params['n_splits'], kernel=pipeline_params['kernel']) test_score, test_loss = test_score_prediction(K=K, y=y, idx_train=idx_out_train, idx_test=idx_out_test, params=best_params) grads_da, grads_db, train_score, train_loss = count_grads_a_b( exp_K=K_out_train, y=y_out_train, da=da[np.ix_(idx_out_train, idx_out_train)], db=db[np.ix_(idx_out_train, idx_out_train)], params=best_params, n_splits=pipeline_params['n_splits'], random_state=pipeline_params['random_state']) gc.collect() return best_params, grads_da, grads_db, train_score, test_score, train_loss, test_loss
def create_template(path_to_data, train_idx, path_to_template, template_name, resolution, sigma=0.01, load_func_template=None): images = [] if isinstance(path_to_data[0], (str, np.string_, np.unicode_)): assert load_func_template is not None, "if data given by full path, you should provide loader" load_images = import_func(**load_func_template) images = load_images(path_to_data[np.ix_(train_idx)]) if isinstance(path_to_data[0], np.ndarray): images = path_to_data[np.ix_(train_idx)] if resolution != 1: images = change_resolution(images, resolution, sigma) images = np.array(images) template = np.mean(images, axis=0) save_nii(template, os.path.join(path_to_template, template_name)) return template
def metric_learning_to_template(PATH): # path to experiment config print("START EXPERIMENT") pipeline_params = json.load(open(PATH, 'r')) # extract params to shorter usage pipeline_main_loop = import_func(**pipeline_params['pipeline_main_loop']) random_state = pipeline_params['random_state'] experiment_name = pipeline_params['experiment_name'] experiment_path = os.path.join(pipeline_params['path_to_exp'], experiment_name) path_to_template = os.path.join(experiment_path, 'templates/') template_name = 'template_0.nii' # create folder and path create_exp_folders(experiment_path, params=pipeline_params) print('experiment name: ', experiment_name) load_data = import_func(**pipeline_params['load_func']) data, y = load_data(**pipeline_params['load_params']) if isinstance(data[0], (str, np.unicode, np.str, np.string_, np.unicode_)): np.savez(os.path.join(experiment_path, 'data_path.npz'), np.array(data)) if pipeline_params['subset'] != 1.: data, y = get_subset(data, y, pipeline_params['subset'], pipeline_params['random_state']) np.savez(os.path.join(experiment_path, 'target.npz'), y) print("Data size: ", data.shape, " target mean: ", y.mean()) # create splits for (train+val) and test idx_out_train, idx_out_test = list( StratifiedShuffleSplit(n_splits=1, test_size=0.3, random_state=random_state).split( np.arange(len(data)), y))[0] splits = { 'train_val': idx_out_train.tolist(), 'test': idx_out_test.tolist() } save_params(experiment_path, 'splits_indices', splits) # create template on train data and save it template = create_template( path_to_data=data, train_idx=idx_out_train, path_to_template=os.path.join(experiment_path, 'templates/'), template_name=template_name, resolution=pipeline_params['resolution'], load_func_template=pipeline_params['load_func_template']) # check if template needs padding if check_for_padding(template): template = pad_template_data_after_loop( template.copy(), os.path.join(path_to_template, template_name), pad_size=pipeline_params['pipeline_optimization_params'] ['pad_size'], ndim=pipeline_params['ndim']) pipeline_params['pipeline_optimization_params']['add_padding'] = True pad_size = pipeline_params['pipeline_optimization_params']['pad_size'] else: pad_size = 0 pipeline_main_loop(data=data, template=template, y=y, idx_out_train=idx_out_train, idx_out_test=idx_out_test, experiment_path=experiment_path, path_to_template=path_to_template, template_name=template_name, pipeline_params=pipeline_params, pad_size=pad_size) print("FINISHED")
def sparse_dot_product_forward(vector, ndim, mat_shape, T, loss, window, params_grad, param_der, n_jobs=10): mat_len = int(np.prod(mat_shape)) # assert ndim * mat_len == len(vector), "not correct shape of vector" derivative_func = import_func(**param_der) deltas = list( itertools.product(range(-window, window + 1), repeat=vector.ndim - 2)) mn, mx = (0, ) * ndim, vector.shape[2:] data, rows, cols = [], [], [] for ax in range(ndim): result = Parallel(n_jobs=n_jobs, temp_folder='~/JOBLIB_TMP_FOLDER/')( delayed(one_der)(i, ax, T, mat_shape, mat_len, deltas, mn, mx, derivative_func, vector, loss, params_grad) for i in tqdm(np.ndindex(*vector.shape[2:]), desc='dJ_der')) loc_data, loc_rows, loc_cols = map(np.concatenate, zip(*result)) data.extend(loc_data) rows.extend(loc_rows) cols.extend(loc_cols) gc.collect() for i in range(min(mat_shape)): I = matrix_to_vec_indices(i, mat_shape) for ax in itertools.combinations(range(ndim), 2): der = derivative_func(i=( T, ax[0], ) + (i, ) * ndim, j=( T, ax[1], ) + (i, ) * ndim, vf=np.copy(vector), loss=loss, **params_grad) # if np.abs(der) > 1e-15: i_loc = I + ax[0] * mat_len j_loc = I + ax[1] * mat_len data.extend([der, der]) rows.extend([i_loc, j_loc]) cols.extend([j_loc, i_loc]) shape = (ndim * mat_len, ndim * mat_len) result = coo_matrix((data, (rows, cols)), shape=shape) regul = np.real(ifftn(params_grad['reg'].regularizer.operator)) r = np.arange(int(ndim * mat_len)) reg = coo_matrix((np.repeat(regul.reshape(-1), ndim), (r, r)), shape=shape) try: return inv(result + reg) except: regul2 = coo_matrix((np.repeat(1e-8, len(r)), (r, r)), shape=shape) return inv(result + reg + regul2)
def pipeline_main_loop_template_only(data, template, y, idx_out_train, idx_out_test, experiment_path, path_to_template, template_name, pipeline_params, pad_size): # initialize learning rate changing strategy lr_change = import_func(**pipeline_params['lr_type']) lr_params = pipeline_params['lr_change_params'][pipeline_params['lr_type'] ['func']] template_updates = pipeline_params['template_updates'] reg = pipeline_params['pipeline_optimization_params'] lr = template_updates['lr'] it = 1 a = pipeline_params['pipeline_optimization_params']['a'] b = pipeline_params['pipeline_optimization_params']['b'] # create a resulting data frame if pipeline_params['kernel']: results = pd.DataFrame(columns=[ "iter", "a", "b", "kernel gamma", "LR C ", "train_score", "train_loss", "test_score", "test_loss", "one_loop_time", "pad_size" ]) else: results = pd.DataFrame(columns=[ "iter", "a", "b", "LR C ", "train_score", "train_loss", "test_score", "test_loss", "one_loop_time", "pad_size" ]) print('For params a {} and b {}'.format(a, b)) test_score_prediction = import_func(**pipeline_params['prediction_func']) count_grads_template = import_func( **pipeline_params['count_grads_template']) add_padding = reg['add_padding'] kwargs = create_kwargs(pipeline_params, data, template, a, b, idx_out_train, True, add_padding, pad_size) y_out_train = y[idx_out_train] while it < pipeline_params['Number_of_iterations']: st = time.time() print('For iter {}'.format(int(it))) K, da, db, dJ = count_dist_matrix_to_template(**kwargs) K_path = os.path.join( os.path.join(pipeline_params['path_to_exp'], pipeline_params['experiment_name']), 'kernel') np.savez(os.path.join(K_path, 'dJ_' + str(it) + '.npz'), dJ) np.savez(os.path.join(K_path, 'kernel_' + str(it) + '.npz'), K) K_out_train = K[np.ix_(idx_out_train, idx_out_train)] best_params = find_pipeline_params( K_out_train, y_out_train, pipeline_params['ml_params'], pipeline_params['n_jobs'], random_state=pipeline_params['random_state'], scaled=pipeline_params['scaled'], scoring='roc_auc', n_splits=pipeline_params['n_splits'], kernel=pipeline_params['kernel']) test_score, test_loss = test_score_prediction(K=K, y=y, idx_train=idx_out_train, idx_test=idx_out_test, params=best_params) grads_da, grads_db, grads_dJ, train_score, train_loss = count_grads_template( exp_K=K_out_train, y=y_out_train, da=da[np.ix_(idx_out_train, idx_out_train)], db=db[np.ix_(idx_out_train, idx_out_train)], dJ=dJ, params=best_params, n_splits=pipeline_params['n_splits'], ndim=pipeline_params['ndim'], random_state=pipeline_params['random_state'], kernel=pipeline_params['kernel']) np.savez(os.path.join(experiment_path, f'grads_dJ/grad_dJ_{it}.npz'), grads_dJ) # delta = preprocess_delta_template(grads_dJ, axis=template_updates['template_axis'], # contour_color=template_updates['color'], # width=template_updates['width'], # ndim=pipeline_params['ndim']) # np.savez(os.path.join(experiment_path, f'grads_dJ/delta_dJ_{it}.npz'), delta) # lr_loc = lr * int(np.max(template))/ np.max(grads_dJ) # normalize on 255, 0.1 * 255 /np.max() lr_loc = template_updates['lr'] * np.max(template) / np.std(grads_dJ) template_name = template_name.split('_')[0] + '_' + str(it) + '.nii' template = update_template(template, path_to_template, template_name, grads_dJ, lr_loc) if check_for_padding(template): template = pad_template_data_after_loop( template.copy(), os.path.join(path_to_template, template_name), pad_size=reg['pad_size'], ndim=pipeline_params['ndim']) kwargs['add_padding'] = True kwargs['pad_size'] += reg['pad_size'] if pipeline_params['kernel']: results.loc[it - 1] = [ it, a, b, best_params['kernel__gamma'], best_params['ml__C'], train_score, train_loss, test_score, test_loss, time.time() - st, pad_size ] else: results.loc[it - 1] = [ it, a, b, best_params['ml__C'], train_score, train_loss, test_score, test_loss, time.time() - st, pad_size ] kwargs['template'] = template it += 1 lr = lr_change(prev_lr=lr, it=it, step=lr_params['step'], decay=lr_params['decay']) results.to_csv(os.path.join(experiment_path, 'results.csv')) gc.collect()
def optimize_template_step(data, template, y, a, b, idx_out_train, idx_out_test, pipeline_params, template_name, path_to_template, pad_size, it, lr): test_score_prediction = import_func(**pipeline_params['prediction_func']) count_grads_template = import_func( **pipeline_params['count_grads_template']) experiment_name = pipeline_params['experiment_name'] experiment_path = os.path.join(pipeline_params['path_to_exp'], experiment_name) template_updates = pipeline_params['template_updates'] reg = pipeline_params['pipeline_optimization_params'] add_padding = reg['add_padding'] y_out_train = y[idx_out_train] kwargs = create_kwargs(pipeline_params, data, template, a, b, idx_out_train, True, add_padding, pad_size) K, da, db, dJ = count_dist_matrix_to_template(**kwargs) K_path = os.path.join( os.path.join(pipeline_params['path_to_exp'], pipeline_params['experiment_name']), 'kernel') np.savez(os.path.join(K_path, 'dJ_' + str(it) + '.npz'), dJ) np.savez(os.path.join(K_path, 'kernel_' + str(it) + '.npz'), K) K_out_train = K[np.ix_(idx_out_train, idx_out_train)] best_params = find_pipeline_params( K_out_train, y_out_train, pipeline_params['ml_params'], pipeline_params['n_jobs'], random_state=pipeline_params['random_state'], scaled=pipeline_params['scaled'], scoring='roc_auc', n_splits=pipeline_params['n_splits'], kernel=pipeline_params['kernel']) test_score, test_loss = test_score_prediction(K=K, y=y, idx_train=idx_out_train, idx_test=idx_out_test, params=best_params) grads_da, grads_db, grads_dJ, train_score, train_loss = count_grads_template( exp_K=K_out_train, y=y_out_train, da=da[np.ix_(idx_out_train, idx_out_train)], db=db[np.ix_(idx_out_train, idx_out_train)], dJ=dJ, params=best_params, n_splits=pipeline_params['n_splits'], ndim=pipeline_params['ndim'], random_state=pipeline_params['random_state'], kernel=pipeline_params['kernel']) np.savez(os.path.join(experiment_path, f'grads_dJ/grad_dJ_{it}.npz'), grads_dJ) # delta = preprocess_delta_template(grads_dJ, axis=template_updates['template_axis'], # contour_color=template_updates['color'], # width=template_updates['width'], # ndim=pipeline_params['ndim']) # np.savez(os.path.join(experiment_path, f'grads_dJ/delta_dJ_{it}.npz'), delta) lr_loc = template_updates['lr'] * np.max(template) / np.std( grads_dJ) # lr * int(np.max(template))/ np.max(grads_dJ) template_name = template_name.split('_')[0] + '_' + str(it) + '.nii' template = update_template(template, path_to_template, template_name, grads_dJ, lr_loc) if check_for_padding(template): template = pad_template_data_after_loop(template.copy(), os.path.join( path_to_template, template_name), pad_size=reg['pad_size'], ndim=pipeline_params['ndim']) add_padding = True pad_size += reg['pad_size'] kwargs['add_padding'] = True kwargs['pad_size'] += reg['pad_size'] gc.collect() return template, best_params, grads_da, grads_db, train_score, test_score, train_loss, test_loss, add_padding, pad_size
def pipeline_main_loop(data, template, y, idx_out_train, idx_out_test, experiment_path, path_to_template, template_name, pipeline_params, pad_size): # initialize learning rate changing strategy lr_change = import_func(**pipeline_params['lr_type']) lr_params = pipeline_params['lr_change_params'][pipeline_params['lr_type'] ['func']] lr = lr_params['init_lr'] it = 1 a_it = [0., pipeline_params['pipeline_optimization_params']['a']] b_it = [0., pipeline_params['pipeline_optimization_params']['b']] mta, vta = 0., 0. mtb, vtb = 0., 0. # start param for optimization optim_template = pipeline_params['start_optim_template'] # create a resulting data frame if pipeline_params['kernel']: results = pd.DataFrame(columns=[ "iter", "a", "b", "kernel gamma", "LR C ", "train_score", "train_loss", "test_score", "test_loss", "one_loop_time", "pad_size" ]) else: results = pd.DataFrame(columns=[ "iter", "a", "b", "LR C ", "train_score", "train_loss", "test_score", "test_loss", "one_loop_time", "pad_size" ]) while (abs(a_it[-1] - a_it[-2]) + abs(b_it[-1] - b_it[-2])) > 1e-10 or \ it < pipeline_params['Number_of_iterations']: st = time.time() print('For iter {}'.format(int(it))) print('For params a {} and b {}'.format(a_it[-1], b_it[-1])) if optim_template: template, best_params, grads_da, grads_db, train_score, test_score, train_loss, test_loss, add_padding, pad_size = optimize_template_step( data.copy(), template, y.copy(), a_it[-1], b_it[-1], idx_out_train, idx_out_test, pipeline_params, template_name, path_to_template, pad_size, it, pipeline_params['template_updates']['lr']) if add_padding: pipeline_params['pipeline_optimization_params'][ 'add_padding'] = add_padding else: best_params, grads_da, grads_db, train_score, test_score, train_loss, test_loss = optimize_a_b_step( data.copy(), template, y.copy(), a_it[-1], b_it[-1], idx_out_train, idx_out_test, pipeline_params, pad_size) adam_grad_da, mta, vta = adam_step(grads_da, mta, vta, it) adam_grad_db, mtb, vtb = adam_step(grads_db, mtb, vtb, it) if pipeline_params['kernel']: results.loc[it - 1] = [ it, a_it[-1], b_it[-1], best_params['kernel__gamma'], best_params['ml__C'], train_score, train_loss, test_score, test_loss, time.time() - st, pad_size ] else: results.loc[it - 1] = [ it, a_it[-1], b_it[-1], best_params['ml__C'], train_score, train_loss, test_score, test_loss, time.time() - st, pad_size ] print("one loop time: ", time.time() - st) a_it += [a_it[-1] - lr * adam_grad_da] # b_it += [b_it[-1] - lr * adam_grad_db] it += 1 lr = lr_change(prev_lr=lr, it=it, step=lr_params['step'], decay=lr_params['decay']) optim_template = optim_template_strategy( it, pipeline_params['step_size_optim_template']) results.to_csv(os.path.join(experiment_path, 'results.csv')) gc.collect()