def sample_precision_space(parameters, number=100):
"""Launch a large number of times the same estimation, with different
starting points.
number: int
number of samples to generate.
"""
# Estimation
max_iter = 200
# Generate signals
next_num, cache_dir, gt = create_signals(parameters,
output_dir="_gsc_sensitivity")
precisions, topology, signals = (gt["precisions"], gt["topology"],
gt["signals"])
emp_covs, n_samples = empirical_covariances(signals)
print("alpha max: %.3e" % compute_alpha_max(emp_covs, n_samples)[0])
# Estimate a lot of precision matrices
parameters = joblib.Parallel(n_jobs=7, verbose=1)(
joblib.delayed(save_group_sparse_covariance)(
emp_covs, n_samples, parameters["alpha"], max_iter=max_iter,
tol=parameters["tol"], cache_dir=cache_dir, num=n)
for n in xrange(next_num, next_num + number))
def benchmark1():
parameters = dict(n_var=200,
n_tasks=5,
density=0.15,
tol=1e-2,
n_alphas=5,
max_iter=50,
min_samples=100,
max_samples=150)
next_num, cache_dir, gt = create_signals(parameters, output_dir=output_dir)
emp_covs, n_samples = empirical_covariances(gt['signals'])
max_alpha, _ = compute_alpha_max(emp_covs, n_samples)
min_alpha = max_alpha / 100.
print(min_alpha, max_alpha)
alphas = np.logspace(np.log10(min_alpha), np.log10(max_alpha),
parameters['n_alphas'])[::-1]
joblib.Parallel(n_jobs=1, verbose=1)(
joblib.delayed(save_group_sparse_covariance)(
emp_covs, n_samples, alpha, max_iter=parameters['max_iter'],
tol=parameters['tol'], debug=False, cache_dir=cache_dir, num=num)
for alpha, num in zip(alphas, itertools.count(next_num)))
def plot_benchmark1():
"""Plot various quantities obtained for varying values of alpha."""
parameters = dict(n_var=200,
n_tasks=5,
density=0.15,
tol=1e-2,
# max_iter=50,
min_samples=100,
max_samples=150)
cache_dir = get_cache_dir(parameters, output_dir=output_dir)
gt = get_ground_truth(cache_dir)
gt['precisions'] = np.dstack(gt['precisions'])
emp_covs, n_samples = empirical_covariances(gt['signals'])
n_samples /= n_samples.sum()
alpha = []
objective = []
log_likelihood = []
ll_penalized = []
sparsity = []
kl = []
true_covs = np.empty(gt['precisions'].shape)
for k in range(gt['precisions'].shape[-1]):
true_covs[..., k] = np.linalg.inv(gt['precisions'][..., k])
for out in iter_outputs(cache_dir):
alpha.append(out['alpha'])
objective.append(- out['objective'][-1])
ll, llpen = group_sparse_scores(out['precisions'],
n_samples, true_covs, out['alpha'])
log_likelihood.append(ll)
ll_penalized.append(llpen)
sparsity.append(1. * (out['precisions'][..., 0] != 0).sum()
/ out['precisions'].shape[0] ** 2)
kl.append(distance(out['precisions'], gt['precisions']))
gt["true_sparsity"] = (1. * (gt['precisions'][..., 0] != 0).sum()
/ gt['precisions'].shape[0] ** 2)
title = (("n_var: {n_var}, n_tasks: {n_tasks}, "
+ "true sparsity: {true_sparsity:.2f} "
+ "\ntol: {tol:.2e} samples: {min_samples}-{max_samples}").format(
true_sparsity=gt["true_sparsity"],
**parameters))
plot(alpha, objective, label="objective", title=title)
plot(alpha, log_likelihood, label="log-likelihood", new_figure=False)
plot(alpha, ll_penalized, label="penalized L-L", new_figure=False)
plot(alpha, sparsity, label="sparsity", title=title)
pl.hlines(gt["true_sparsity"], min(alpha), max(alpha))
plot(alpha, kl, label="distance", title=title)
pl.show()
def split_signals(signals, fold_n=0):
"""Split signals into train and test sets."""
# Smallest signal is 77-sample long.
# Keep first 50 samples for train set, everything else for test set.
# train_test = [(s[:50, ...], s[50:, ...]) for s in signals]
# Keep first two-third for train set, everything else for test set
folds = [tuple(KFold(s.shape[0], 3)) for s in signals]
train_test = [(s[fold[fold_n][0], ...], s[fold[fold_n][1], ...])
for s, fold in zip(signals, folds)]
signals, test_signals = zip(*train_test)
emp_covs, n_samples = empirical_covariances(signals)
test_emp_covs, test_n_samples = empirical_covariances(test_signals)
n_samples_norm = n_samples.copy()
n_samples_norm /= n_samples_norm.sum()
return signals, test_signals, emp_covs, test_emp_covs, n_samples_norm
def benchmark(parameters, output_d="_convergence"):
_, _, gt = create_signals(parameters, output_dir=output_d)
emp_covs, n_samples = empirical_covariances(gt["signals"])
print("alpha_max: %.3e, %.3e" % compute_alpha_max(emp_covs, n_samples))
sp = ScoreProbe(duality_gap=True)
_group_sparse_covariance(
emp_covs, n_samples, alpha=parameters["alpha"], tol=parameters["tol"],
max_iter=parameters["max_iter"], probe_function=sp, verbose=1)
return {"log_lik": np.asarray(sp.log_lik),
"objective": np.asarray(sp.objective),
"precisions": np.asarray(sp.precisions),
"duality_gap": np.asarray(sp.duality_gap),
"time": np.asarray(sp.wall_clock)}, gt
def singular_cov_case():
"""Check behaviour of algorithm for singular input matrix."""
parameters = {'n_tasks': 10, 'n_var': 40, 'density': 0.15,
'rho': .1, 'tol': 1e-2, 'max_iter': 50,
'min_samples': 10, 'max_samples': 15}
_, _, gt = create_signals(parameters, output_dir=output_dir)
signals = gt["signals"]
emp_covs, _ = empirical_covariances(signals)
# Check that all covariance matrices are singular.
eps = np.finfo(float).eps
for k in range(emp_covs.shape[-1]):
eigvals = np.linalg.eigvalsh(emp_covs[..., k])
assert(abs(eigvals.min()) <= 50 * eps)
_, gsc_precisions = utils.timeit(group_sparse_covariance)(
signals, parameters['rho'], max_iter=parameters['max_iter'],
tol=parameters['tol'], verbose=1, debug=False)
print('found sparsity: {0:.3f}'
''.format(1. * (gsc_precisions[..., 0] != 0).sum()
/ gsc_precisions.shape[0] ** 2))
def benchmark1():
"""Plot different quantities for varying alpha."""
# Signals
min_samples, max_samples = 100, 150 # train signals length
n_var = 50
n_tasks = 40
density = 0.1
random_state = np.random.RandomState(0)
test_samples = 4000 # number of samples for test signals
# Estimation
n_alphas = 10
max_iter = 200
tol = 1e-3
# Generate signals
signals, precisions, topology = \
testing.generate_group_sparse_gaussian_graphs(
n_subjects=n_tasks, n_features=n_var, density=density,
random_state=random_state, min_n_samples=min_samples,
max_n_samples=max_samples)
emp_covs, n_samples = empirical_covariances(signals)
# Estimate precision matrices
alpha_1, _ = compute_alpha_max(emp_covs, n_samples)
alpha_0 = 1e-2 * alpha_1
## alpha_1 = 0.067
## alpha_0 = 0.044
alphas = np.logspace(np.log10(alpha_0), np.log10(alpha_1), n_alphas)[::-1]
parameters = joblib.Parallel(n_jobs=7, verbose=1)(
joblib.delayed(group_sparse_covariance)(emp_covs, n_samples, alpha,
max_iter=max_iter, tol=tol)
for alpha in alphas)
# Compute scores
test_signals = testing.generate_signals_from_precisions(
precisions, min_n_samples=test_samples, max_n_samples=test_samples + 1,
random_state=random_state)
test_emp_covs, _ = empirical_covariances(test_signals)
del test_signals
for params in parameters:
params["ll_score"], params["pen_score"] = group_sparse_scores(
params["precisions"], n_samples, test_emp_covs, params["alpha"])
# Plot graphs
alpha, ll_score, pen_score = get_series(
parameters, ("alpha", "ll_score", "pen_score"))
non_zero = [(p["precisions"][..., 0] != 0).sum() for p in parameters]
pl.figure()
pl.semilogx(alpha, ll_score, "-+", label="log-likelihood")
pl.semilogx(alpha, pen_score, "-+", label="penalized LL")
pl.xlabel("alpha")
pl.ylabel("score")
pl.grid()
pl.figure()
pl.semilogx(alpha, non_zero, "-+")
pl.xlabel("alpha")
pl.ylabel("non_zero")
pl.grid()
pl.figure()
pl.loglog(alpha, non_zero, "-+")
pl.xlabel("alpha")
pl.ylabel("non_zero")
pl.grid()
pl.figure()
pl.imshow(topology, interpolation="nearest")
pl.title("true topology")
## precisions = get_series(parameters, ("precisions", ))
## for prec, alpha in zip(precisions, alpha):
## pl.figure()
## pl.imshow(prec[..., 0] != 0, interpolation="nearest")
## pl.title(alpha)
pl.show()
def benchmark3():
"""Compare group_sparse_covariance result for different initializations.
"""
## parameters = {'n_tasks': 10, 'n_var': 50, 'density': 0.15,
## 'alpha': .001, 'tol': 1e-2, 'max_iter': 100}
parameters = {'n_var': 40, 'n_tasks': 10, 'density': 0.15,
'alpha': .01, 'tol': 1e-3, 'max_iter': 100}
mem = joblib.Memory(".")
_, _, gt = create_signals(parameters,
output_dir="_prof_group_sparse_covariance")
signals = gt["signals"]
emp_covs, n_samples = empirical_covariances(signals)
print("alpha max: " + str(compute_alpha_max(emp_covs, n_samples)))
# With diagonal elements initialization
probe1 = ScoreProbe()
est_precs1, probe1 = mem.cache(modified_gsc)(signals, parameters, probe1)
probe1.comment = "diagonal" # set after execution for joblib not to see it
probe1.plot()
# With Ledoit-Wolf initialization
ld = np.empty(emp_covs.shape)
for k in range(emp_covs.shape[-1]):
ld[..., k] = np.linalg.inv(ledoit_wolf(signals[k])[0])
probe1 = ScoreProbe()
est_precs1, probe1 = utils.timeit(mem.cache(modified_gsc))(
signals, parameters, probe=probe1)
probe1.comment = "diagonal" # for joblib to ignore this value
probe2 = ScoreProbe()
parameters["precisions_init"] = ld
est_precs2, probe2 = utils.timeit(mem.cache(modified_gsc))(
signals, parameters, probe=probe2)
probe2.comment = "ledoit-wolf"
print("difference between final estimates (max norm) %.2e"
% abs(est_precs1 - est_precs2).max())
pl.figure()
pl.semilogy(probe1.timings[1:], probe1.max_norm,
"+-", label=probe1.comment)
pl.semilogy(probe2.timings[1:], probe2.max_norm,
"+-", label=probe2.comment)
pl.xlabel("Time [s]")
pl.ylabel("Max norm")
pl.grid()
pl.legend(loc="best")
pl.figure()
pl.plot(probe1.timings, probe1.objective,
"+-", label=probe1.comment)
pl.plot(probe2.timings, probe2.objective,
"+-", label=probe2.comment)
pl.xlabel("Time [s]")
pl.ylabel("objective")
pl.grid()
pl.legend(loc="best")
pl.show()
def compute_stats(cache_dir):
l = 0 # task to plot
data_fnames = glob.glob(os.path.join(cache_dir, "precisions_*.pickle"))
d = pickle.load(open(data_fnames[0], "rb"))
mean = np.zeros(d["precisions"].shape)
acc2 = mean.copy()
## topo_count = mean.copy()
minimum = float("inf") * np.ones(mean.shape)
maximum = float("-inf") * np.ones(mean.shape)
data_fnames = data_fnames[:5]
for data_fname in data_fnames:
print(data_fname)
d = pickle.load(open(data_fname, "rb"))
mean += d["precisions"]
acc2 += d["precisions"] ** 2
minimum = np.where(d["precisions"] < minimum, d["precisions"], minimum)
maximum = np.where(d["precisions"] > maximum, d["precisions"], maximum)
## topo_count += d["precisions"] != 0
mean /= len(data_fnames)
acc2 /= len(data_fnames)
var = acc2 - mean ** 2
assert var.min() >= -1e-13
var[var < 0] = 0 # remove very small negative values
matshow(var[..., l], title="variance")
std = np.sqrt(var)
assert np.all(np.isreal(std))
matshow(std[..., l], title="std")
matshow(mean[..., l], title="mean")
# matshow(mean[..., l] != 0, title="topology")
# matshow(maximum[..., l], title="maximum")
# matshow(minimum[..., l], title="minimum")
matshow(maximum[..., l] - minimum[..., 0], title="ptp")
mean_no_diag = mean.copy()
for k in range(mean_no_diag.shape[-1]):
mean_no_diag[..., k].flat[::mean_no_diag.shape[0] + 1] = 0
matshow(mean_no_diag[..., l], title="mean without diagonal")
ratio = (std / abs(mean))[..., 0]
ratio[np.logical_not(np.isfinite(ratio))] = 0
matshow(ratio, title="ratio")
# load estimated covariance
gt = pickle.load(
open(os.path.join(cache_dir, "ground_truth.pickle"), "rb"))
emp_covs, n_samples = empirical_covariances(gt["signals"])
rho = 0.02
# Estimate first-order sensitivity
n_samples /= n_samples.sum()
last_m = d["precisions"]
last_m_inv = np.empty(last_m.shape)
for k in range(last_m.shape[-1]):
last_m_inv[..., k] = np.linalg.inv(last_m[..., k])
norms = np.sqrt(np.sum(last_m ** 2, axis=-1))
last_m_normed = np.empty(last_m.shape)
for k in range(last_m.shape[-1]):
last_m_normed[..., k] = last_m[..., k] / norms
# set diagonal to zero
last_m_normed[..., k].flat[::last_m_normed.shape[0] + 1] = 0
derivative = (n_samples * (last_m_inv - emp_covs) - rho * last_m_normed)
derivative[np.logical_not(np.isfinite(derivative))] = 0
derivative = derivative ** 2
# estimate second-order sensibility
sens2 = np.empty(last_m.shape)
for k in range(last_m.shape[-1]):
sens2[..., k] = n_samples[k] * (
np.dot(
np.dot(last_m_inv[..., k],
derivative[..., k]),
last_m_inv[..., k])
)
sens2 = np.abs(sens2 / 2.)
matshow(np.sqrt(derivative[..., l]), title="sensitivity 1")
matshow(np.sqrt(sens2[..., l]), title="sensitivity 2")
matshow(np.sqrt(sens2[..., l] + derivative[..., l]),
title="sensitivity 1+2")
## matshow((last_m - mean)[..., l], title="difference with mean")
## matshow(topo_count[..., l], title="non-zero count")
pl.show()
