def test_creation_and_from_to_x_y(self):
problem, true_parameters = LinearLMEProblem.generate(
groups_sizes=[4, 5, 10],
features_labels=[3, 3, 1, 2],
random_intercept=True,
obs_std=0.1,
seed=42)
x1, y1 = problem.to_x_y()
problem2, _ = LinearLMEProblem.from_x_y(x1, y1)
x2, y2 = problem2.to_x_y()
self.assertTrue(np.all(x1 == x2) and np.all(y1 == y2))
test_problem, true_test_parameters = LinearLMEProblem.generate(
groups_sizes=[3, 4, 5],
features_labels=[3, 3, 1, 2],
random_intercept=True,
beta=true_parameters["beta"],
gamma=true_parameters["gamma"],
true_random_effects=true_parameters["random_effects"],
obs_std=0.1,
seed=43)
self.assertTrue(
np.all(true_parameters["beta"] == true_test_parameters["beta"]) and
np.all(true_parameters["gamma"] == true_test_parameters["gamma"])
and np.all([
np.all(u1 == u2)
for u1, u2 in zip(true_parameters["random_effects"],
true_test_parameters["random_effects"])
]))
def predict(self, x, use_sparse_coefficients=False):
"""
Makes a prediction if .fit(X, y) was called before and throws an error otherwise.
Parameters
----------
x : np.ndarray
Data matrix. Should have the same format as the data which was used for fitting the model:
the number of columns and the columns' labels should be the same. It may contain new groups, in which case
the prediction will be formed using the fixed effects only.
use_sparse_coefficients : bool, default is False
If true then uses sparse coefficients, tbeta and tgamma, for making a prediction, otherwise uses
beta and gamma.
Returns
-------
y : np.ndarray
Models predictions.
"""
check_is_fitted(self, 'coef_')
problem, _ = LinearLMEProblem.from_x_y(x, y=None)
if use_sparse_coefficients:
beta = self.coef_['tbeta']
us = self.coef_['sparse_random_effects']
else:
beta = self.coef_['beta']
us = self.coef_['random_effects']
assert problem.num_fixed_effects == beta.shape[0], \
"Number of fixed effects is not the same to what it was in the train data."
assert problem.num_random_effects == us[0].shape[0], \
"Number of random effects is not the same to what it was in the train data."
group_labels = self.coef_['group_labels']
answers = []
for i, (x, _, z, stds) in enumerate(problem):
label = problem.group_labels[i]
idx_of_this_label_in_train = np.where(group_labels == label)
assert len(
idx_of_this_label_in_train
) <= 1, "Group labels of the classifier contain duplicates."
if len(idx_of_this_label_in_train) == 1:
idx_of_this_label_in_train = idx_of_this_label_in_train[0]
y = x.dot(beta) + z.dot(us[idx_of_this_label_in_train][0])
else:
# If we have not seen this group (so we don't have inferred random effects for this)
# then we make a prediction with "expected" (e.g. zero) random effects
y = x.dot(beta)
answers.append(y)
return np.concatenate(answers)
def test_from_to_xy_preserves_dataset_structure(self):
study_sizes = [20, 15, 10]
num_features = 6
num_random_effects = 4
np.random.seed(42)
x = np.random.rand(
sum(study_sizes) + 1,
1 + (num_features - 1) + 1 + (num_random_effects - 1) + 1)
y = np.random.rand(sum(study_sizes))
x[1:, 0] = np.repeat([0, 1, 2], study_sizes)
x[0, :] = [0] + [1] * (num_features - 1) + [
3
] + [2] * (num_random_effects - 1) + [4]
problem, true_parameters = LinearLMEProblem.from_x_y(x, y)
x2, y2 = problem.to_x_y()
self.assertTrue(np.all(x2 == x),
msg="x is not the same after from/to transformation")
self.assertTrue(np.all(y2 == y),
msg="y is not the same after from/to transformation")
def fit(self,
x: np.ndarray,
y: np.ndarray,
columns_labels: np.ndarray = None,
initial_parameters: dict = None,
warm_start=False,
random_intercept=True,
**kwargs):
"""
Fits a Linear Model with Linear Mixed-Effects to the given data.
Parameters
----------
x : np.ndarray
Data. If columns_labels = None then it's assumed that columns_labels are in the first row of x.
y : np.ndarray
Answers, real-valued array.
columns_labels : np.ndarray
List of column labels. There shall be only one column of group labels and answers STDs,
and overall n columns with fixed effects (1 or 3) and k columns of random effects (2 or 3).
- 1 : fixed effect
- 2 : random effect
- 3 : both fixed and random,
- 0 : groups labels
- 4 : answers standard deviations
initial_parameters : np.ndarray
Dict with possible fields:
- | 'beta0' : np.ndarray, shape = [n],
| Initial estimate of fixed effects. If None then it defaults to an all-ones vector.
- | 'gamma0' : np.ndarray, shape = [k],
| Initial estimate of random effects covariances. If None then it defaults to an all-ones vector.
- | 'tbeta0' : np.ndarray, shape = [n],
| Initial estimate of sparse fixed effects. If None then it defaults to an all-zeros vector.
- | 'tgamma0' : np.ndarray, shape = [k],
| Initial estimate of sparse random covariances. If None then it defaults to an all-zeros vector.
warm_start : bool, default is False
Whether to use previous parameters as initial ones. Overrides initial_parameters if given.
Throws NotFittedError if set to True when not fitted.
random_intercept : bool, default = True
Whether treat the intercept as a random effect.
kwargs :
Not used currently, left here for passing debugging parameters.
Returns
-------
self : LinearLMESparseModel
Fitted regression model.
"""
problem, _ = LinearLMEProblem.from_x_y(
x, y, columns_labels, random_intercept=random_intercept, **kwargs)
if initial_parameters is None:
initial_parameters = {}
beta0 = initial_parameters.get("beta", None)
gamma0 = initial_parameters.get("gamma", None)
tbeta0 = initial_parameters.get("tbeta", None)
tgamma0 = initial_parameters.get("tgamma", None)
_check_input_consistency(problem, beta0, gamma0, tbeta0, tgamma0)
if self.regularization_type == "l2":
oracle = LinearLMEOracleRegularized(problem,
lb=self.lb,
lg=self.lg,
nnz_tbeta=self.nnz_tbeta,
nnz_tgamma=self.nnz_tgamma)
elif self.regularization_type == "loss-weighted":
oracle = LinearLMEOracleW(problem,
lb=self.lb,
lg=self.lg,
nnz_tbeta=self.nnz_tbeta,
nnz_tgamma=self.nnz_tgamma)
else:
raise ValueError("regularization_type is not understood.")
num_fixed_effects = problem.num_fixed_effects
num_random_effects = problem.num_random_effects
assert num_fixed_effects >= self.nnz_tbeta
assert num_random_effects >= self.nnz_tgamma
# old_oracle = OldOracle(problem, lb=self.lb, lg=self.lg, k=self.nnz_tbeta, j=self.nnz_tgamma)
if warm_start:
check_is_fitted(self, 'coef_')
beta = self.coef_["beta"]
gamma = self.coef_["gamma"]
tbeta = self.coef_["tbeta"]
tgamma = self.coef_["tgamma"]
else:
if beta0 is not None:
beta = beta0
else:
beta = np.ones(num_fixed_effects)
if gamma0 is not None:
gamma = gamma0
else:
gamma = np.ones(num_random_effects)
if tbeta0 is not None:
tbeta = tbeta0
else:
tbeta = np.zeros(num_fixed_effects)
if tgamma0 is not None:
tgamma = tgamma0
else:
tgamma = np.zeros(num_random_effects)
if self.initializer == "EM":
beta = oracle.optimal_beta(gamma, tbeta)
us = oracle.optimal_random_effects(beta, gamma)
gamma = np.sum(us**2, axis=0) / oracle.problem.num_groups
# tbeta = oracle.optimal_tbeta(beta)
# tgamma = oracle.optimal_tgamma(tbeta, gamma)
def projected_direction(current_gamma, current_direction):
proj_direction = current_direction.copy()
for j, _ in enumerate(current_gamma):
if current_gamma[j] == 0 and current_direction[j] <= 0:
proj_direction[j] = 0
return proj_direction
loss = oracle.loss(beta, gamma, tbeta, tgamma)
self.logger_ = Logger(self.logger_keys)
prev_tbeta = np.infty
prev_tgamma = np.infty
iteration = 0
while (np.linalg.norm(tbeta - prev_tbeta) > self.tol
and np.linalg.norm(tgamma - prev_tgamma) > self.tol
and iteration < self.n_iter):
if iteration >= self.n_iter:
us = oracle.optimal_random_effects(beta, gamma)
if len(self.logger_keys) > 0:
self.logger_.log(**locals())
self.coef_ = {
"beta": beta,
"gamma": gamma,
"tbeta": tbeta,
"tgamma": tgamma,
"random_effects": us
}
self.logger_.add("converged", 0)
return self
if self.solver == 'pgd':
inner_iteration = 0
beta = oracle.optimal_beta(gamma, tbeta, beta=beta)
gradient_gamma = oracle.gradient_gamma(beta, gamma, tgamma)
direction = projected_direction(gamma, -gradient_gamma)
while (np.linalg.norm(direction) > self.tol_inner
and inner_iteration < self.n_iter_inner):
# gradient_gamma = oracle.gradient_gamma(beta, gamma, tgamma)
# projecting the gradient to the set of constraints
# direction = projected_direction(gamma, -gradient_gamma)
if self.use_line_search:
# line search method
step_len = 0.1
for i, _ in enumerate(gamma):
if direction[i] < 0:
step_len = min(-gamma[i] / direction[i],
step_len)
current_loss = oracle.loss(beta, gamma, tbeta, tgamma)
while (
oracle.loss(beta, gamma + step_len * direction,
tbeta, tgamma) >=
(1 - np.sign(current_loss) * 1e-5) * current_loss):
step_len *= 0.5
if step_len <= 1e-15:
break
else:
# fixed step size
step_len = 1 / iteration
if step_len <= 1e-15:
break
gamma = gamma + step_len * direction
gradient_gamma = oracle.gradient_gamma(beta, gamma, tgamma)
direction = projected_direction(gamma, -gradient_gamma)
inner_iteration += 1
prev_tbeta = tbeta
prev_tgamma = tgamma
tbeta = oracle.optimal_tbeta(beta=beta, gamma=gamma)
tgamma = oracle.optimal_tgamma(tbeta, gamma, beta=beta)
iteration += 1
loss = oracle.loss(beta, gamma, tbeta, tgamma)
if len(self.logger_keys) > 0:
self.logger_.log(locals())
us = oracle.optimal_random_effects(beta, gamma)
sparse_us = oracle.optimal_random_effects(tbeta, tgamma)
per_group_coefficients = get_per_group_coefficients(
beta, us, labels=problem.column_labels)
sparse_per_group_coefficients = get_per_group_coefficients(
tbeta, sparse_us, labels=problem.column_labels)
self.logger_.add('converged', 1)
self.logger_.add('iterations', iteration)
self.coef_ = {
"beta": beta,
"gamma": gamma,
"tbeta": tbeta,
"tgamma": tgamma,
"random_effects": us,
"sparse_random_effects": sparse_us,
"group_labels": np.copy(problem.group_labels),
"per_group_coefficients": per_group_coefficients,
"sparse_per_group_coefficients": sparse_per_group_coefficients,
}
return self