def addPriors(self, prior_list):
"""
Add priors to the object, require prior_list contains priors
"""
self.prior_list = prior_list
(self.C, self.JC, self.c, self.H, self.JH, self.h, self.uprior,
self.gprior, self.lprior, self.C_list, self.JC_list, self.c_list,
self.H_list, self.JH_list, self.h_list, self.id_C_list,
self.id_C_var_list, self.id_H_list, self.id_H_var_list,
self.num_constraints_list,
self.num_regularizers_list) = utils.constructPrior(prior_list, self)
# renew uprior for gamma
if self.uprior is None:
self.uprior = np.array([[-np.inf] * self.k_beta +
[1e-7] * self.k_gamma, [np.inf] * self.k])
else:
uprior_beta = self.uprior[:, self.id_beta]
uprior_gamma = self.uprior[:, self.id_gamma]
uprior_gamma[0] = np.maximum(1e-7, uprior_gamma[0])
uprior_gamma[1] = np.maximum(uprior_gamma[0], uprior_gamma[1])
self.uprior = np.hstack((uprior_beta, uprior_gamma))
self.lt.C, self.lt.JC, self.lt.c = self.C, self.JC, self.c
self.lt.H, self.lt.JH, self.lt.h = self.H, self.JH, self.h
(self.lt.uprior, self.lt.gprior,
self.lt.lprior) = (self.uprior, self.gprior, self.lprior)
self.lt = LimeTr(self.study_sizes,
self.k_beta,
self.k_gamma,
self.obs_mean,
self.F,
self.JF,
self.Z,
self.obs_std,
C=self.C,
JC=self.JC,
c=self.c,
H=self.H,
JH=self.JH,
h=self.h,
uprior=self.uprior,
gprior=self.gprior,
lprior=self.lprior,
inlier_percentage=self.inlier_percentage)
def __init__(self,
obs_mean,
obs_std,
study_sizes,
x_cov_list,
z_cov_list,
spline_list=[],
inlier_percentage=1.0,
rr_random_slope=False):
"""
Initialize the object and pass in the data, require
- obs_mean: observations
- obs_std: standard deviations for the observations
- study_sizes: all study sizes in a list
- x_cov_list: all x cov in a list
- z_cov_list: all z cov in a list
- spline_list: optional, all spline in a list
- inlier_percentage: optional, used for trimming
"""
# pass in data
self.obs_mean = obs_mean
self.obs_std = obs_std
self.study_sizes = study_sizes
self.num_studies = len(study_sizes)
self.num_obs = sum(study_sizes)
# construct x and z "covariates"
self.spline_list = spline_list
self.x_cov_list = x_cov_list
self.z_cov_list = z_cov_list
(self.F, self.JF, self.F_list, self.JF_list, self.id_beta_list,
self.id_spline_beta_list,
self.k_beta_list) = utils.constructXCov(x_cov_list,
spline_list=spline_list)
(self.Z, self.Z_list, self.id_gamma_list, self.id_spline_gamma_list,
self.k_gamma_list) = utils.constructZCov(z_cov_list,
spline_list=spline_list)
self.k_beta = int(sum(self.k_beta_list))
self.k_gamma = int(sum(self.k_gamma_list))
self.k = self.k_beta + self.k_gamma
self.id_beta = slice(0, self.k_beta)
self.id_gamma = slice(self.k_beta, self.k)
# if use the random slope model or not
self.rr_random_slope = rr_random_slope
if rr_random_slope:
valid_x_cov_id = [
i for i in range(len(x_cov_list)) if x_cov_list[i]['cov_type']
in ['log_ratio_spline', 'log_ratio_spline_integral']
]
if len(valid_x_cov_id) == 0:
raise Exception(
"Error: no suitable x cov for random slope model.")
if len(valid_x_cov_id) >= 2:
raise Exception(
"Error: multiple x cov for random slope model.")
x_cov = x_cov_list[valid_x_cov_id[0]]
mat = x_cov['mat']
if x_cov['cov_type'] == 'log_ratio_spline':
scaling = mat[0] - mat[1]
else:
scaling = 0.5 * (mat[0] + mat[1] - mat[2] - mat[3])
self.Z *= scaling.reshape(scaling.size, 1)
# create limetr object
self.inlier_percentage = inlier_percentage
self.lt = LimeTr(self.study_sizes,
self.k_beta,
self.k_gamma,
self.obs_mean,
self.F,
self.JF,
self.Z,
self.obs_std,
inlier_percentage=inlier_percentage)
def optimize(self,
var=None,
S=None,
trim_percentage=0.0,
share_obs_std=True,
fit_fixed=True,
inner_print_level=5,
inner_max_iter=100,
inner_tol=1e-5,
inner_verbose=True,
inner_acceptable_tol=1e-4,
inner_nlp_scaling_min_value=1e-8,
outer_verbose=False,
outer_max_iter=1,
outer_step_size=1,
outer_tol=1e-6,
normalize_Z=False,
build_X=True,
random_seed=0):
"""
Run optimization routine via LimeTr.
Args:
var (numpy.ndarray | None, optional):
One-dimensional array that gives initialization for variables.
If None, the program will first run without random effects
to obtain a starting point.
S (numpy.ndarray | None, optional):
One-dimensional numpy array that gives standard deviation for
each measurement. The size of S should be the same as that of
measurements vector. If None standard deviation of measurement
will be treated as variables and optimized.
trim_percentage (float | 0.0, optional):
A float that gives percentage of datapoints to trim.
Default is 0, i.e. no trimming.
share_obs_std (boolean | True, optional):
A boolean that indicates whether the model should assume data
across studies share the same measurement standard deviation.
fit_fixed (boolean | True, optional):
A boolean that indicates whether to run a fit without random
effects first in order to obtain a good starting point.
inner_print_level (int | 5, optional):
``print_level`` for Ipopt.
inner_max_iter (int | 100, optional):
Maximum number of iterations for inner optimization.
inner_tol (float | 1e-5, optional):
Tolerance level for inner optimization.
inner_verbose (boolean | True, optional):
Verbose option for inner optimization.
inner_acceptable_tol (float | 1e-4, optional):
Acceptable tolerance level for inner optimization.
inner_nlp_scaling_min_value (float | 1e-8, optional):
Min scaling for objective function.
outer_verbose (boolean | False, optional):
Verbose option for outer optimization.
outer_max_iter (int | 1, optional):
Maximum number of iterations for outer optimization. When there
is no trimming, outer optimization is not needed, so the default
is set to be 1.
outer_step_size (float |1.0, optional):
Step size for outer optimization. Used in trimming.
outer_tol (float | 1e-6, optional):
Tolerance level for outer optimization.
normalize_Z (bool | False, optional):
Whether to normalize Z matrix before optimization.
build_X (bool | True, optional):
Whether to explicitly build and store X matrix.
random_seed (int | 0, optional):
random seed for choosing an initial point for optimization. If equals 0
the initial point is chosen to be a vector of 0.01.
"""
self.S = S
self.share_obs_std = share_obs_std
Z_norm = self.buildZ(normalize_Z)
k = self.k_beta + self.k_gamma
if S is None:
if share_obs_std:
k += 1
else:
k += len(self.grouping)
print('n_groups', self.n_groups)
print('k_beta', self.k_beta)
print('k_gamma', self.k_gamma)
print('total number of fixed effects variables', k)
if self.k_gamma == 0:
self.add_re = False
self.k_gamma = 1
k += 1
else:
self.add_re = True
C = []
start = self.k_beta
for ran in self.ran_list:
_, dims = ran
m = np.prod(dims[self.n_grouping_dims:])
c = np.zeros((m - 1, k))
for i in range(m - 1):
c[i, start + i] = 1
c[i, start + i + 1] = -1
C.append(c)
start += m
if len(C) > 0:
self.constraints = np.vstack(C)
assert self.constraints.shape[1] == k
else:
self.constraints = []
C = None
if len(self.constraints) > 0:
def C(var):
return self.constraints.dot(var)
JC = None
if len(self.constraints) > 0:
def JC(var):
return self.constraints
c = None
if len(self.constraints) > 0:
c = np.zeros((2, self.constraints.shape[0]))
self.uprior = np.array(
[[-np.inf] * self.k_beta + [1e-7] * self.k_gamma + [1e-7] *
(k - self.k_beta - self.k_gamma), [np.inf] * k])
if self.global_cov_bounds is not None:
if self.global_intercept:
self.uprior[:, 1:len(self.global_ids) +
1] = self.global_cov_bounds
else:
self.uprior[:, :len(self.global_ids)] = self.global_cov_bounds
self.gprior = None
if self.use_gprior:
assert len(self.ran_eff_gamma_sd) == self.k_gamma
self.gprior = np.array(
[[0] * k, [np.inf] * self.k_beta + self.ran_eff_gamma_sd +
[np.inf] * (k - self.k_beta - self.k_gamma)])
x0 = np.ones(k) * .01
if random_seed != 0:
np.random.seed(random_seed)
x0 = np.random.randn(k) * .01
if var is not None:
if self.add_re is True:
assert len(var) == k
x0 = var
else:
assert len(var) == self.k_beta
x0 = np.append(var, [1e-8])
assert len(x0) == k
if build_X:
self._buildX()
if fit_fixed or self.add_re is False:
uprior_fixed = copy.deepcopy(self.uprior)
uprior_fixed[:, self.k_beta:self.k_beta + self.k_gamma] = 1e-8
if S is None or trim_percentage >= 0.01:
model_fixed = LimeTr(self.grouping,
int(self.k_beta),
int(self.k_gamma),
self.Y,
self.X,
self.JX,
self.Z,
S=S,
C=C,
JC=JC,
c=c,
inlier_percentage=1. - trim_percentage,
share_obs_std=share_obs_std,
uprior=uprior_fixed)
model_fixed.optimize(
x0=x0,
print_level=inner_print_level,
max_iter=inner_max_iter,
tol=inner_tol,
acceptable_tol=inner_acceptable_tol,
nlp_scaling_min_value=inner_nlp_scaling_min_value)
x0 = model_fixed.soln
self.beta_fixed = model_fixed.beta
if self.add_re is False:
self.beta_soln = self.beta_fixed
self.delta_soln = model_fixed.delta
self.gamma_soln = model_fixed.gamma
self.w_soln = model_fixed.w
self.info = model_fixed.info['status_msg']
self.yfit_no_random = model_fixed.F(model_fixed.beta)
return
else:
self.beta_fixed = self._solveBeta(S)
x0 = np.append(self.beta_fixed, [1e-8] * self.k_gamma)
if self.add_re is False:
self.beta_soln = self.beta_fixed
self.yfit_no_random = self.Xm.dot(self.beta_fixed)
return
model = LimeTr(self.grouping,
int(self.k_beta),
int(self.k_gamma),
self.Y,
self.X,
self.JX,
self.Z,
S=S,
C=C,
JC=JC,
c=c,
inlier_percentage=1 - trim_percentage,
share_obs_std=share_obs_std,
uprior=self.uprior,
gprior=self.gprior)
model.fitModel(x0=x0,
inner_print_level=inner_print_level,
inner_max_iter=inner_max_iter,
inner_acceptable_tol=inner_acceptable_tol,
inner_nlp_scaling_min_value=inner_nlp_scaling_min_value,
inner_tol=inner_tol,
outer_verbose=outer_verbose,
outer_max_iter=outer_max_iter,
outer_step_size=outer_step_size,
outer_tol=outer_tol)
self.beta_soln = model.beta
self.gamma_soln = model.gamma
if normalize_Z:
self.gamma_soln /= Z_norm**2
self.delta_soln = model.delta
self.info = model.info
self.w_soln = model.w
self.u_soln = model.estimateRE()
self.solve_status = model.info['status']
self.solve_status_msg = model.info['status_msg']
self.yfit_no_random = model.F(model.beta)
self.yfit = []
Z_split = np.split(self.Z, self.n_groups)
yfit_no_random_split = np.split(self.yfit_no_random, self.n_groups)
for i in range(self.n_groups):
self.yfit.append(yfit_no_random_split[i] +
Z_split[i].dot(self.u_soln[i]))
self.yfit = np.concatenate(self.yfit)
self.model = model
if inner_verbose == True and self.solve_status != 0:
print(self.solve_status_msg)
def fit_model(self, **fit_options):
"""Fitting the model through limetr.
Args:
x0 (np.ndarray): Initial guess for the optimization problem.
inner_print_level (int): If non-zero printing iteration information of the inner problem.
inner_max_iter (int): Maximum inner number of iterations.
inner_tol (float): Tolerance of the inner problem.
outer_verbose (bool): If `True` print out iteration information.
outer_max_iter (int): Maximum outer number of iterations.
outer_step_size (float): Step size of the outer problem.
outer_tol (float): Tolerance of the outer problem.
normalize_trimming_grad (bool): If `True`, normalize the gradient of the outer trimmign problem.
"""
# dimensions
n = self.data.study_sizes
k_beta = self.num_x_vars
k_gamma = self.num_z_vars
# data
y = self.data.obs
s = self.data.obs_se
# create x fun and z mat
x_fun, x_fun_jac = self.create_x_fun()
z_mat = self.create_z_mat()
# scale z_mat
z_scale = np.max(np.abs(z_mat), axis=0)
z_mat /= z_scale
# priors
c_mat, c_vec = self.create_c_mat()
h_mat, h_vec = self.create_h_mat()
c_fun, c_fun_jac = utils.mat_to_fun(c_mat)
h_fun, h_fun_jac = utils.mat_to_fun(h_mat)
uprior = self.create_uprior()
uprior[:, self.num_x_vars:self.num_vars] *= z_scale**2
gprior = self.create_gprior()
gprior[:, self.num_x_vars:self.num_vars] *= z_scale**2
lprior = self.create_lprior()
lprior[:, self.num_x_vars:self.num_vars] *= z_scale**2
if np.isneginf(uprior[0]).all() and np.isposinf(uprior[1]).all():
uprior = None
if np.isposinf(gprior[1]).all():
gprior = None
if np.isposinf(lprior[1]).all():
lprior = None
# create limetr object
self.lt = LimeTr(n,
k_beta,
k_gamma,
y,
x_fun,
x_fun_jac,
z_mat,
S=s,
C=c_fun,
JC=c_fun_jac,
c=c_vec,
H=h_fun,
JH=h_fun_jac,
h=h_vec,
uprior=uprior,
gprior=gprior,
lprior=lprior,
inlier_percentage=self.inlier_pct)
self.lt.fitModel(**fit_options)
self.lt.Z *= z_scale
if hasattr(self.lt, 'gprior'):
self.lt.gprior[:, self.lt.idx_gamma] /= z_scale**2
if hasattr(self.lt, 'uprior'):
self.lt.uprior[:, self.lt.idx_gamma] /= z_scale**2
if hasattr(self.lt, 'lprior'):
self.lt.lprior[:, self.lt.idx_gamma] /= z_scale**2
self.lt.gamma /= z_scale**2
self.beta_soln = self.lt.beta.copy()
self.gamma_soln = self.lt.gamma.copy()
self.w_soln = self.lt.w.copy()
self.u_soln = self.lt.estimateRE()
self.re_soln = {
study: self.u_soln[i]
for i, study in enumerate(self.data.studies)
}
def fit(self,
max_iter=100,
inlier_pct=1.0,
outer_max_iter=100,
outer_step_size=1.0):
"""Optimize the model parameters.
This is a interface to limetr.
Args:
max_iter (int, optional):
Maximum number of iterations.
inlier_pct (float, optional):
How much percentage of the data do you trust.
outer_max_iter (int, optional):
Outer maximum number of iterations.
outer_step_size (float, optional):
Step size of the trimming problem, the larger the step size the faster it will converge,
and the less quality of trimming it will guarantee.
"""
# dimensions for limetr
n = self.cwdata.study_sizes
if n.size == 0:
n = np.full(self.cwdata.num_obs, 1)
k_beta = self.num_vars
k_gamma = 1
y = self.cwdata.obs
s = self.cwdata.obs_se
x = self.design_mat
z = np.ones((self.cwdata.num_obs, 1))
uprior = np.hstack(
(self.prior_beta_uniform, self.prior_gamma_uniform[:, None]))
gprior = np.hstack(
(self.prior_beta_gaussian, self.prior_gamma_gaussian[:, None]))
if self.constraint_mat is None:
cfun = None
jcfun = None
cvec = None
else:
num_constraints = self.constraint_mat.shape[0]
cmat = np.hstack(
(self.constraint_mat, np.zeros((num_constraints, 1))))
cvec = np.array([[-np.inf] * num_constraints,
[0.0] * num_constraints])
def cfun(var):
return cmat.dot(var)
def jcfun(var):
return cmat
def fun(var):
return x.dot(var)
def jfun(beta):
return x
self.lt = LimeTr(n,
k_beta,
k_gamma,
y,
fun,
jfun,
z,
S=s,
gprior=gprior,
uprior=uprior,
C=cfun,
JC=jcfun,
c=cvec,
inlier_percentage=inlier_pct)
self.beta, self.gamma, self.w = self.lt.fitModel(
inner_print_level=5,
inner_max_iter=max_iter,
outer_max_iter=outer_max_iter,
outer_step_size=outer_step_size)
self.fixed_vars = {
var: self.beta[self.var_idx[var]]
for var in self.vars
}
if self.use_random_intercept:
u = self.lt.estimateRE()
self.random_vars = {
sid: u[i]
for i, sid in enumerate(self.cwdata.unique_study_id)
}
else:
self.random_vars = dict()
# compute the posterior distribution of beta
hessian = self.get_beta_hessian()
unconstrained_id = np.hstack([
np.arange(self.lt.k_beta)[self.var_idx[dorm]]
for dorm in self.cwdata.unique_dorms if dorm != self.gold_dorm
])
self.beta_sd = np.zeros(self.lt.k_beta)
self.beta_sd[unconstrained_id] = np.sqrt(
np.diag(np.linalg.inv(hessian)))