def benchmark1(): parameters = dict(n_var=200, n_tasks=5, density=0.15, tol=1e-2, n_alphas=5, max_iter=50, min_samples=100, max_samples=150) next_num, cache_dir, gt = create_signals(parameters, output_dir=output_dir) emp_covs, n_samples = empirical_covariances(gt['signals']) max_alpha, _ = compute_alpha_max(emp_covs, n_samples) min_alpha = max_alpha / 100. print(min_alpha, max_alpha) alphas = np.logspace(np.log10(min_alpha), np.log10(max_alpha), parameters['n_alphas'])[::-1] joblib.Parallel(n_jobs=1, verbose=1)( joblib.delayed(save_group_sparse_covariance)( emp_covs, n_samples, alpha, max_iter=parameters['max_iter'], tol=parameters['tol'], debug=False, cache_dir=cache_dir, num=num) for alpha, num in zip(alphas, itertools.count(next_num)))
def sample_precision_space(parameters, number=100): """Launch a large number of times the same estimation, with different starting points. number: int number of samples to generate. """ # Estimation max_iter = 200 # Generate signals next_num, cache_dir, gt = create_signals(parameters, output_dir="_gsc_sensitivity") precisions, topology, signals = (gt["precisions"], gt["topology"], gt["signals"]) emp_covs, n_samples = empirical_covariances(signals) print("alpha max: %.3e" % compute_alpha_max(emp_covs, n_samples)[0]) # Estimate a lot of precision matrices parameters = joblib.Parallel(n_jobs=7, verbose=1)( joblib.delayed(save_group_sparse_covariance)( emp_covs, n_samples, parameters["alpha"], max_iter=max_iter, tol=parameters["tol"], cache_dir=cache_dir, num=n) for n in xrange(next_num, next_num + number))
def benchmark(parameters, output_d="_convergence"): _, _, gt = create_signals(parameters, output_dir=output_d) emp_covs, n_samples = empirical_covariances(gt["signals"]) print("alpha_max: %.3e, %.3e" % compute_alpha_max(emp_covs, n_samples)) sp = ScoreProbe(duality_gap=True) _group_sparse_covariance( emp_covs, n_samples, alpha=parameters["alpha"], tol=parameters["tol"], max_iter=parameters["max_iter"], probe_function=sp, verbose=1) return {"log_lik": np.asarray(sp.log_lik), "objective": np.asarray(sp.objective), "precisions": np.asarray(sp.precisions), "duality_gap": np.asarray(sp.duality_gap), "time": np.asarray(sp.wall_clock)}, gt
def brute_force_study(output_dir="_early_stopping"): """Loop through many values of alpha, and run a full gsc for each. Record information for each iteration using CostProbe, store the obtained values on disk. Plot scores on train and test sets versus wall-clock time. """ parameters = {'n_tasks': 10, 'tol': 1e-3, 'max_iter': 50, "fold_n": 2, "n_alphas": 20} mem = joblib.Memory(".") print("-- Extracting signals ...") signals = [] for n in range(parameters["n_tasks"]): signals.append(mem.cache(region_signals)(n)) signals, test_signals, emp_covs, test_emp_covs, n_samples_norm = \ split_signals(signals, fold_n=parameters["fold_n"]) print("-- Optimizing --") alpha_mx, _ = compute_alpha_max(emp_covs, n_samples_norm) # alphas = np.logspace(-3, -1, 10) alphas = np.logspace(np.log10(alpha_mx / 500), np.log10(alpha_mx), parameters["n_alphas"]) cost_probes = [] t0 = time.time() for alpha in alphas: # Honorio-Samaras cost_probes.append(CostProbe(test_emp_covs)) _, est_precs = utils.timeit(group_sparse_covariance)( signals, alpha, max_iter=parameters['max_iter'], tol=parameters['tol'], verbose=1, debug=False, probe_function=cost_probes[-1]) t1 = time.time() print ('Time spent in loop: %.2fs' % (t1 - t0)) out_filename = os.path.join(output_dir, 'brute_force_study.pickle') pickle.dump([alphas, cost_probes], open(out_filename, "wb")) print("Use plot_early_stopping.py to analyze the generated file:\n" "%s" % out_filename)
def benchmark1(): """Plot different quantities for varying alpha.""" # Signals min_samples, max_samples = 100, 150 # train signals length n_var = 50 n_tasks = 40 density = 0.1 random_state = np.random.RandomState(0) test_samples = 4000 # number of samples for test signals # Estimation n_alphas = 10 max_iter = 200 tol = 1e-3 # Generate signals signals, precisions, topology = \ testing.generate_group_sparse_gaussian_graphs( n_subjects=n_tasks, n_features=n_var, density=density, random_state=random_state, min_n_samples=min_samples, max_n_samples=max_samples) emp_covs, n_samples = empirical_covariances(signals) # Estimate precision matrices alpha_1, _ = compute_alpha_max(emp_covs, n_samples) alpha_0 = 1e-2 * alpha_1 ## alpha_1 = 0.067 ## alpha_0 = 0.044 alphas = np.logspace(np.log10(alpha_0), np.log10(alpha_1), n_alphas)[::-1] parameters = joblib.Parallel(n_jobs=7, verbose=1)( joblib.delayed(group_sparse_covariance)(emp_covs, n_samples, alpha, max_iter=max_iter, tol=tol) for alpha in alphas) # Compute scores test_signals = testing.generate_signals_from_precisions( precisions, min_n_samples=test_samples, max_n_samples=test_samples + 1, random_state=random_state) test_emp_covs, _ = empirical_covariances(test_signals) del test_signals for params in parameters: params["ll_score"], params["pen_score"] = group_sparse_scores( params["precisions"], n_samples, test_emp_covs, params["alpha"]) # Plot graphs alpha, ll_score, pen_score = get_series( parameters, ("alpha", "ll_score", "pen_score")) non_zero = [(p["precisions"][..., 0] != 0).sum() for p in parameters] pl.figure() pl.semilogx(alpha, ll_score, "-+", label="log-likelihood") pl.semilogx(alpha, pen_score, "-+", label="penalized LL") pl.xlabel("alpha") pl.ylabel("score") pl.grid() pl.figure() pl.semilogx(alpha, non_zero, "-+") pl.xlabel("alpha") pl.ylabel("non_zero") pl.grid() pl.figure() pl.loglog(alpha, non_zero, "-+") pl.xlabel("alpha") pl.ylabel("non_zero") pl.grid() pl.figure() pl.imshow(topology, interpolation="nearest") pl.title("true topology") ## precisions = get_series(parameters, ("precisions", )) ## for prec, alpha in zip(precisions, alpha): ## pl.figure() ## pl.imshow(prec[..., 0] != 0, interpolation="nearest") ## pl.title(alpha) pl.show()
def benchmark3(): """Compare group_sparse_covariance result for different initializations. """ ## parameters = {'n_tasks': 10, 'n_var': 50, 'density': 0.15, ## 'alpha': .001, 'tol': 1e-2, 'max_iter': 100} parameters = {'n_var': 40, 'n_tasks': 10, 'density': 0.15, 'alpha': .01, 'tol': 1e-3, 'max_iter': 100} mem = joblib.Memory(".") _, _, gt = create_signals(parameters, output_dir="_prof_group_sparse_covariance") signals = gt["signals"] emp_covs, n_samples = empirical_covariances(signals) print("alpha max: " + str(compute_alpha_max(emp_covs, n_samples))) # With diagonal elements initialization probe1 = ScoreProbe() est_precs1, probe1 = mem.cache(modified_gsc)(signals, parameters, probe1) probe1.comment = "diagonal" # set after execution for joblib not to see it probe1.plot() # With Ledoit-Wolf initialization ld = np.empty(emp_covs.shape) for k in range(emp_covs.shape[-1]): ld[..., k] = np.linalg.inv(ledoit_wolf(signals[k])[0]) probe1 = ScoreProbe() est_precs1, probe1 = utils.timeit(mem.cache(modified_gsc))( signals, parameters, probe=probe1) probe1.comment = "diagonal" # for joblib to ignore this value probe2 = ScoreProbe() parameters["precisions_init"] = ld est_precs2, probe2 = utils.timeit(mem.cache(modified_gsc))( signals, parameters, probe=probe2) probe2.comment = "ledoit-wolf" print("difference between final estimates (max norm) %.2e" % abs(est_precs1 - est_precs2).max()) pl.figure() pl.semilogy(probe1.timings[1:], probe1.max_norm, "+-", label=probe1.comment) pl.semilogy(probe2.timings[1:], probe2.max_norm, "+-", label=probe2.comment) pl.xlabel("Time [s]") pl.ylabel("Max norm") pl.grid() pl.legend(loc="best") pl.figure() pl.plot(probe1.timings, probe1.objective, "+-", label=probe1.comment) pl.plot(probe2.timings, probe2.objective, "+-", label=probe2.comment) pl.xlabel("Time [s]") pl.ylabel("objective") pl.grid() pl.legend(loc="best") pl.show()