def benchmark1():
parameters = dict(n_var=200,
n_tasks=5,
density=0.15,
tol=1e-2,
n_alphas=5,
max_iter=50,
min_samples=100,
max_samples=150)
next_num, cache_dir, gt = create_signals(parameters, output_dir=output_dir)
emp_covs, n_samples = empirical_covariances(gt['signals'])
max_alpha, _ = compute_alpha_max(emp_covs, n_samples)
min_alpha = max_alpha / 100.
print(min_alpha, max_alpha)
alphas = np.logspace(np.log10(min_alpha), np.log10(max_alpha),
parameters['n_alphas'])[::-1]
joblib.Parallel(n_jobs=1, verbose=1)(
joblib.delayed(save_group_sparse_covariance)(
emp_covs, n_samples, alpha, max_iter=parameters['max_iter'],
tol=parameters['tol'], debug=False, cache_dir=cache_dir, num=num)
for alpha, num in zip(alphas, itertools.count(next_num)))
def sample_precision_space(parameters, number=100):
"""Launch a large number of times the same estimation, with different
starting points.
number: int
number of samples to generate.
"""
# Estimation
max_iter = 200
# Generate signals
next_num, cache_dir, gt = create_signals(parameters,
output_dir="_gsc_sensitivity")
precisions, topology, signals = (gt["precisions"], gt["topology"],
gt["signals"])
emp_covs, n_samples = empirical_covariances(signals)
print("alpha max: %.3e" % compute_alpha_max(emp_covs, n_samples)[0])
# Estimate a lot of precision matrices
parameters = joblib.Parallel(n_jobs=7, verbose=1)(
joblib.delayed(save_group_sparse_covariance)(
emp_covs, n_samples, parameters["alpha"], max_iter=max_iter,
tol=parameters["tol"], cache_dir=cache_dir, num=n)
for n in xrange(next_num, next_num + number))
def benchmark(parameters, output_d="_convergence"):
_, _, gt = create_signals(parameters, output_dir=output_d)
emp_covs, n_samples = empirical_covariances(gt["signals"])
print("alpha_max: %.3e, %.3e" % compute_alpha_max(emp_covs, n_samples))
sp = ScoreProbe(duality_gap=True)
_group_sparse_covariance(
emp_covs, n_samples, alpha=parameters["alpha"], tol=parameters["tol"],
max_iter=parameters["max_iter"], probe_function=sp, verbose=1)
return {"log_lik": np.asarray(sp.log_lik),
"objective": np.asarray(sp.objective),
"precisions": np.asarray(sp.precisions),
"duality_gap": np.asarray(sp.duality_gap),
"time": np.asarray(sp.wall_clock)}, gt
def brute_force_study(output_dir="_early_stopping"):
"""Loop through many values of alpha, and run a full gsc for each.
Record information for each iteration using CostProbe, store the
obtained values on disk.
Plot scores on train and test sets versus wall-clock time.
"""
parameters = {'n_tasks': 10, 'tol': 1e-3, 'max_iter': 50, "fold_n": 2,
"n_alphas": 20}
mem = joblib.Memory(".")
print("-- Extracting signals ...")
signals = []
for n in range(parameters["n_tasks"]):
signals.append(mem.cache(region_signals)(n))
signals, test_signals, emp_covs, test_emp_covs, n_samples_norm = \
split_signals(signals, fold_n=parameters["fold_n"])
print("-- Optimizing --")
alpha_mx, _ = compute_alpha_max(emp_covs, n_samples_norm)
# alphas = np.logspace(-3, -1, 10)
alphas = np.logspace(np.log10(alpha_mx / 500), np.log10(alpha_mx),
parameters["n_alphas"])
cost_probes = []
t0 = time.time()
for alpha in alphas:
# Honorio-Samaras
cost_probes.append(CostProbe(test_emp_covs))
_, est_precs = utils.timeit(group_sparse_covariance)(
signals, alpha, max_iter=parameters['max_iter'],
tol=parameters['tol'], verbose=1, debug=False,
probe_function=cost_probes[-1])
t1 = time.time()
print ('Time spent in loop: %.2fs' % (t1 - t0))
out_filename = os.path.join(output_dir, 'brute_force_study.pickle')
pickle.dump([alphas, cost_probes], open(out_filename, "wb"))
print("Use plot_early_stopping.py to analyze the generated file:\n"
"%s" % out_filename)
def benchmark1():
"""Plot different quantities for varying alpha."""
# Signals
min_samples, max_samples = 100, 150 # train signals length
n_var = 50
n_tasks = 40
density = 0.1
random_state = np.random.RandomState(0)
test_samples = 4000 # number of samples for test signals
# Estimation
n_alphas = 10
max_iter = 200
tol = 1e-3
# Generate signals
signals, precisions, topology = \
testing.generate_group_sparse_gaussian_graphs(
n_subjects=n_tasks, n_features=n_var, density=density,
random_state=random_state, min_n_samples=min_samples,
max_n_samples=max_samples)
emp_covs, n_samples = empirical_covariances(signals)
# Estimate precision matrices
alpha_1, _ = compute_alpha_max(emp_covs, n_samples)
alpha_0 = 1e-2 * alpha_1
## alpha_1 = 0.067
## alpha_0 = 0.044
alphas = np.logspace(np.log10(alpha_0), np.log10(alpha_1), n_alphas)[::-1]
parameters = joblib.Parallel(n_jobs=7, verbose=1)(
joblib.delayed(group_sparse_covariance)(emp_covs, n_samples, alpha,
max_iter=max_iter, tol=tol)
for alpha in alphas)
# Compute scores
test_signals = testing.generate_signals_from_precisions(
precisions, min_n_samples=test_samples, max_n_samples=test_samples + 1,
random_state=random_state)
test_emp_covs, _ = empirical_covariances(test_signals)
del test_signals
for params in parameters:
params["ll_score"], params["pen_score"] = group_sparse_scores(
params["precisions"], n_samples, test_emp_covs, params["alpha"])
# Plot graphs
alpha, ll_score, pen_score = get_series(
parameters, ("alpha", "ll_score", "pen_score"))
non_zero = [(p["precisions"][..., 0] != 0).sum() for p in parameters]
pl.figure()
pl.semilogx(alpha, ll_score, "-+", label="log-likelihood")
pl.semilogx(alpha, pen_score, "-+", label="penalized LL")
pl.xlabel("alpha")
pl.ylabel("score")
pl.grid()
pl.figure()
pl.semilogx(alpha, non_zero, "-+")
pl.xlabel("alpha")
pl.ylabel("non_zero")
pl.grid()
pl.figure()
pl.loglog(alpha, non_zero, "-+")
pl.xlabel("alpha")
pl.ylabel("non_zero")
pl.grid()
pl.figure()
pl.imshow(topology, interpolation="nearest")
pl.title("true topology")
## precisions = get_series(parameters, ("precisions", ))
## for prec, alpha in zip(precisions, alpha):
## pl.figure()
## pl.imshow(prec[..., 0] != 0, interpolation="nearest")
## pl.title(alpha)
pl.show()
def benchmark3():
"""Compare group_sparse_covariance result for different initializations.
"""
## parameters = {'n_tasks': 10, 'n_var': 50, 'density': 0.15,
## 'alpha': .001, 'tol': 1e-2, 'max_iter': 100}
parameters = {'n_var': 40, 'n_tasks': 10, 'density': 0.15,
'alpha': .01, 'tol': 1e-3, 'max_iter': 100}
mem = joblib.Memory(".")
_, _, gt = create_signals(parameters,
output_dir="_prof_group_sparse_covariance")
signals = gt["signals"]
emp_covs, n_samples = empirical_covariances(signals)
print("alpha max: " + str(compute_alpha_max(emp_covs, n_samples)))
# With diagonal elements initialization
probe1 = ScoreProbe()
est_precs1, probe1 = mem.cache(modified_gsc)(signals, parameters, probe1)
probe1.comment = "diagonal" # set after execution for joblib not to see it
probe1.plot()
# With Ledoit-Wolf initialization
ld = np.empty(emp_covs.shape)
for k in range(emp_covs.shape[-1]):
ld[..., k] = np.linalg.inv(ledoit_wolf(signals[k])[0])
probe1 = ScoreProbe()
est_precs1, probe1 = utils.timeit(mem.cache(modified_gsc))(
signals, parameters, probe=probe1)
probe1.comment = "diagonal" # for joblib to ignore this value
probe2 = ScoreProbe()
parameters["precisions_init"] = ld
est_precs2, probe2 = utils.timeit(mem.cache(modified_gsc))(
signals, parameters, probe=probe2)
probe2.comment = "ledoit-wolf"
print("difference between final estimates (max norm) %.2e"
% abs(est_precs1 - est_precs2).max())
pl.figure()
pl.semilogy(probe1.timings[1:], probe1.max_norm,
"+-", label=probe1.comment)
pl.semilogy(probe2.timings[1:], probe2.max_norm,
"+-", label=probe2.comment)
pl.xlabel("Time [s]")
pl.ylabel("Max norm")
pl.grid()
pl.legend(loc="best")
pl.figure()
pl.plot(probe1.timings, probe1.objective,
"+-", label=probe1.comment)
pl.plot(probe2.timings, probe2.objective,
"+-", label=probe2.comment)
pl.xlabel("Time [s]")
pl.ylabel("objective")
pl.grid()
pl.legend(loc="best")
pl.show()