class MR_BRT:
def __init__(self,
obs_mean,
obs_std,
study_sizes,
x_cov_list,
z_cov_list,
spline_list=[],
inlier_percentage=1.0,
rr_random_slope=False):
"""
Initialize the object and pass in the data, require
- obs_mean: observations
- obs_std: standard deviations for the observations
- study_sizes: all study sizes in a list
- x_cov_list: all x cov in a list
- z_cov_list: all z cov in a list
- spline_list: optional, all spline in a list
- inlier_percentage: optional, used for trimming
"""
# pass in data
self.obs_mean = obs_mean
self.obs_std = obs_std
self.study_sizes = study_sizes
self.num_studies = len(study_sizes)
self.num_obs = sum(study_sizes)
# construct x and z "covariates"
self.spline_list = spline_list
self.x_cov_list = x_cov_list
self.z_cov_list = z_cov_list
(self.F, self.JF, self.F_list, self.JF_list, self.id_beta_list,
self.id_spline_beta_list,
self.k_beta_list) = utils.constructXCov(x_cov_list,
spline_list=spline_list)
(self.Z, self.Z_list, self.id_gamma_list, self.id_spline_gamma_list,
self.k_gamma_list) = utils.constructZCov(z_cov_list,
spline_list=spline_list)
self.k_beta = int(sum(self.k_beta_list))
self.k_gamma = int(sum(self.k_gamma_list))
self.k = self.k_beta + self.k_gamma
self.id_beta = slice(0, self.k_beta)
self.id_gamma = slice(self.k_beta, self.k)
# if use the random slope model or not
self.rr_random_slope = rr_random_slope
if rr_random_slope:
valid_x_cov_id = [
i for i in range(len(x_cov_list)) if x_cov_list[i]['cov_type']
in ['log_ratio_spline', 'log_ratio_spline_integral']
]
if len(valid_x_cov_id) == 0:
raise Exception(
"Error: no suitable x cov for random slope model.")
if len(valid_x_cov_id) >= 2:
raise Exception(
"Error: multiple x cov for random slope model.")
x_cov = x_cov_list[valid_x_cov_id[0]]
mat = x_cov['mat']
if x_cov['cov_type'] == 'log_ratio_spline':
scaling = mat[0] - mat[1]
else:
scaling = 0.5 * (mat[0] + mat[1] - mat[2] - mat[3])
self.Z *= scaling.reshape(scaling.size, 1)
# create limetr object
self.inlier_percentage = inlier_percentage
self.lt = LimeTr(self.study_sizes,
self.k_beta,
self.k_gamma,
self.obs_mean,
self.F,
self.JF,
self.Z,
self.obs_std,
inlier_percentage=inlier_percentage)
def addPriors(self, prior_list):
"""
Add priors to the object, require prior_list contains priors
"""
self.prior_list = prior_list
(self.C, self.JC, self.c, self.H, self.JH, self.h, self.uprior,
self.gprior, self.lprior, self.C_list, self.JC_list, self.c_list,
self.H_list, self.JH_list, self.h_list, self.id_C_list,
self.id_C_var_list, self.id_H_list, self.id_H_var_list,
self.num_constraints_list,
self.num_regularizers_list) = utils.constructPrior(prior_list, self)
# renew uprior for gamma
if self.uprior is None:
self.uprior = np.array([[-np.inf] * self.k_beta +
[1e-7] * self.k_gamma, [np.inf] * self.k])
else:
uprior_beta = self.uprior[:, self.id_beta]
uprior_gamma = self.uprior[:, self.id_gamma]
uprior_gamma[0] = np.maximum(1e-7, uprior_gamma[0])
uprior_gamma[1] = np.maximum(uprior_gamma[0], uprior_gamma[1])
self.uprior = np.hstack((uprior_beta, uprior_gamma))
self.lt.C, self.lt.JC, self.lt.c = self.C, self.JC, self.c
self.lt.H, self.lt.JH, self.lt.h = self.H, self.JH, self.h
(self.lt.uprior, self.lt.gprior,
self.lt.lprior) = (self.uprior, self.gprior, self.lprior)
self.lt = LimeTr(self.study_sizes,
self.k_beta,
self.k_gamma,
self.obs_mean,
self.F,
self.JF,
self.Z,
self.obs_std,
C=self.C,
JC=self.JC,
c=self.c,
H=self.H,
JH=self.JH,
h=self.h,
uprior=self.uprior,
gprior=self.gprior,
lprior=self.lprior,
inlier_percentage=self.inlier_percentage)
def fitModel(self,
x0=None,
outer_verbose=False,
outer_max_iter=100,
outer_step_size=1.0,
outer_tol=1e-6,
inner_print_level=0,
inner_max_iter=20):
# initialization with gamma set to be zero
gamma_uprior = self.lt.uprior[:, self.lt.idx_gamma].copy()
self.lt.uprior[:, self.lt.idx_gamma] = 1e-6
self.lt.n = np.array([1] * self.num_obs)
norm_z_col = np.linalg.norm(self.lt.Z, axis=0)
self.lt.Z /= norm_z_col
if x0 is None:
if self.lprior is None:
x0 = np.array([1.0] * self.k_beta + [1e-6] * self.k_gamma)
else:
x0 = np.array([1.0] * self.k_beta * 2 +
[1e-6] * self.k_gamma * 2)
else:
if self.lprior is not None:
beta0 = x0[:self.k_beta]
gamma0 = x0[self.k_beta:self.k_beta + self.k_gamma]
x0 = np.hstack((beta0, np.abs(beta0), gamma0, gamma0))
(beta_0, gamma_0,
self.w_soln) = self.lt.fitModel(x0=x0,
outer_verbose=outer_verbose,
outer_max_iter=outer_max_iter,
outer_step_size=outer_step_size,
outer_tol=outer_tol,
inner_print_level=inner_print_level,
inner_max_iter=inner_max_iter)
# print("init obj", self.lt.objective(self.lt.soln))
# fit the model from the initial point
self.lt.uprior[:, self.lt.idx_gamma] = gamma_uprior
self.lt.n = self.study_sizes
if self.lprior is not None:
x0 = np.hstack((beta_0, np.abs(beta_0), gamma_0, gamma_0))
else:
x0 = np.hstack((beta_0, gamma_0))
self.lt.optimize(x0=x0, print_level=inner_print_level, max_iter=100)
self.lt.Z *= norm_z_col
self.lt.gamma /= norm_z_col**2
self.beta_soln = self.lt.beta
self.gamma_soln = self.lt.gamma
# print("final obj", self.lt.objective(self.lt.soln))
# print("------------------------------")
def predictData(self,
pred_x_cov_list,
pred_z_cov_list,
sample_size,
pred_study_sizes=None,
given_beta_samples=None,
given_gamma_samples=None,
ref_point=None,
include_random_effect=True):
# sample solutions
if given_beta_samples is None or given_gamma_samples is None:
beta_samples, gamma_samples = LimeTr.sampleSoln(
self.lt, sample_size=sample_size)
else:
beta_samples = given_beta_samples
gamma_samples = given_gamma_samples
# calculate the beta and gamma post cov
# self.beta_samples_mean = np.mean(beta_samples, axis=0)
# self.gamma_samples_mean = np.mean(gamma_samples, axis=0)
# self.beta_samples_cov = \
# beta_samples.T.dot(beta_samples)/sample_size - \
# np.outer(self.beta_samples_mean, self.beta_samples_mean)
# self.gamma_samples_cov = \
# gamma_samples.T.dot(gamma_samples)/sample_size - \
# np.outer(self.gamma_samples_mean, self.gamma_samples_mean)
# create x cov
(pred_F, pred_JF, pred_F_list, pred_JF_list,
pred_id_beta_list) = utils.constructPredXCov(pred_x_cov_list, self)
# create z cov
(pred_Z, pred_Z_list,
pred_id_gamma_list) = utils.constructPredZCov(pred_z_cov_list, self)
# num of studies
pred_num_obs = pred_Z.shape[0]
# create observation samples
y_samples = np.vstack([pred_F(beta) for beta in beta_samples])
if ref_point is not None:
x_cov_spline_id = [
x_cov['spline_id'] for x_cov in pred_x_cov_list
if 'spline' in x_cov['cov_type']
]
if len(x_cov_spline_id) == 0:
raise Exception("Error: no spline x cov")
if len(x_cov_spline_id) >= 2:
raise Exception("Error: multiple spline x covs")
spline = self.spline_list[x_cov_spline_id[0]]
ref_risk = spline.designMat(np.array([ref_point])).dot(
beta_samples[:,
self.id_spline_beta_list[x_cov_spline_id[0]]].T)
y_samples /= ref_risk.reshape(sample_size, 1)
pred_gamma = np.hstack([
self.gamma_soln[pred_id_gamma_list[i]]
for i in range(len(pred_id_gamma_list))
])
if include_random_effect:
if self.rr_random_slope:
u = np.random.randn(sample_size, self.k_gamma)*\
np.sqrt(self.gamma_soln)
# zu = np.sum(pred_Z*u, axis=1)
zu = u[:, 0]
valid_x_cov_id = [
i for i in range(len(pred_x_cov_list))
if pred_x_cov_list[i]['cov_type'] == 'spline'
]
if len(valid_x_cov_id) == 0:
raise Exception(
"Error: no suitable x cov for random slope model.")
if len(valid_x_cov_id) >= 2:
raise Exception(
"Error: multiple x cov for random slope model.")
mat = pred_x_cov_list[valid_x_cov_id[0]]['mat']
if ref_point is None:
y_samples *= np.exp(np.outer(zu, mat - mat[0]))
else:
y_samples *= np.exp(np.outer(zu, mat - ref_point))
else:
if pred_study_sizes is None:
pred_study_sizes = np.array([1] * pred_num_obs)
else:
assert sum(pred_study_sizes) == pred_num_obs
pred_num_studies = len(pred_study_sizes)
pred_Z_sub = np.split(pred_Z, np.cumsum(pred_study_sizes)[:-1])
u = [
np.random.multivariate_normal(
np.zeros(pred_study_sizes[i]),
(pred_Z_sub[i] * pred_gamma).dot(pred_Z_sub[i].T),
sample_size) for i in range(pred_num_studies)
]
U = np.hstack(u)
if np.any([
'log_ratio' in self.x_cov_list[i]['cov_type']
for i in range(len(self.x_cov_list))
]):
y_samples *= np.exp(U)
else:
y_samples += U
return y_samples, beta_samples, gamma_samples, pred_F, pred_Z
def optimize(self,
var=None,
S=None,
trim_percentage=0.0,
share_obs_std=True,
fit_fixed=True,
inner_print_level=5,
inner_max_iter=100,
inner_tol=1e-5,
inner_verbose=True,
inner_acceptable_tol=1e-4,
inner_nlp_scaling_min_value=1e-8,
outer_verbose=False,
outer_max_iter=1,
outer_step_size=1,
outer_tol=1e-6,
normalize_Z=False,
build_X=True,
random_seed=0):
"""
Run optimization routine via LimeTr.
Args:
var (numpy.ndarray | None, optional):
One-dimensional array that gives initialization for variables.
If None, the program will first run without random effects
to obtain a starting point.
S (numpy.ndarray | None, optional):
One-dimensional numpy array that gives standard deviation for
each measurement. The size of S should be the same as that of
measurements vector. If None standard deviation of measurement
will be treated as variables and optimized.
trim_percentage (float | 0.0, optional):
A float that gives percentage of datapoints to trim.
Default is 0, i.e. no trimming.
share_obs_std (boolean | True, optional):
A boolean that indicates whether the model should assume data
across studies share the same measurement standard deviation.
fit_fixed (boolean | True, optional):
A boolean that indicates whether to run a fit without random
effects first in order to obtain a good starting point.
inner_print_level (int | 5, optional):
``print_level`` for Ipopt.
inner_max_iter (int | 100, optional):
Maximum number of iterations for inner optimization.
inner_tol (float | 1e-5, optional):
Tolerance level for inner optimization.
inner_verbose (boolean | True, optional):
Verbose option for inner optimization.
inner_acceptable_tol (float | 1e-4, optional):
Acceptable tolerance level for inner optimization.
inner_nlp_scaling_min_value (float | 1e-8, optional):
Min scaling for objective function.
outer_verbose (boolean | False, optional):
Verbose option for outer optimization.
outer_max_iter (int | 1, optional):
Maximum number of iterations for outer optimization. When there
is no trimming, outer optimization is not needed, so the default
is set to be 1.
outer_step_size (float |1.0, optional):
Step size for outer optimization. Used in trimming.
outer_tol (float | 1e-6, optional):
Tolerance level for outer optimization.
normalize_Z (bool | False, optional):
Whether to normalize Z matrix before optimization.
build_X (bool | True, optional):
Whether to explicitly build and store X matrix.
random_seed (int | 0, optional):
random seed for choosing an initial point for optimization. If equals 0
the initial point is chosen to be a vector of 0.01.
"""
self.S = S
self.share_obs_std = share_obs_std
Z_norm = self.buildZ(normalize_Z)
k = self.k_beta + self.k_gamma
if S is None:
if share_obs_std:
k += 1
else:
k += len(self.grouping)
print('n_groups', self.n_groups)
print('k_beta', self.k_beta)
print('k_gamma', self.k_gamma)
print('total number of fixed effects variables', k)
if self.k_gamma == 0:
self.add_re = False
self.k_gamma = 1
k += 1
else:
self.add_re = True
C = []
start = self.k_beta
for ran in self.ran_list:
_, dims = ran
m = np.prod(dims[self.n_grouping_dims:])
c = np.zeros((m - 1, k))
for i in range(m - 1):
c[i, start + i] = 1
c[i, start + i + 1] = -1
C.append(c)
start += m
if len(C) > 0:
self.constraints = np.vstack(C)
assert self.constraints.shape[1] == k
else:
self.constraints = []
C = None
if len(self.constraints) > 0:
def C(var):
return self.constraints.dot(var)
JC = None
if len(self.constraints) > 0:
def JC(var):
return self.constraints
c = None
if len(self.constraints) > 0:
c = np.zeros((2, self.constraints.shape[0]))
self.uprior = np.array(
[[-np.inf] * self.k_beta + [1e-7] * self.k_gamma + [1e-7] *
(k - self.k_beta - self.k_gamma), [np.inf] * k])
if self.global_cov_bounds is not None:
if self.global_intercept:
self.uprior[:, 1:len(self.global_ids) +
1] = self.global_cov_bounds
else:
self.uprior[:, :len(self.global_ids)] = self.global_cov_bounds
self.gprior = None
if self.use_gprior:
assert len(self.ran_eff_gamma_sd) == self.k_gamma
self.gprior = np.array(
[[0] * k, [np.inf] * self.k_beta + self.ran_eff_gamma_sd +
[np.inf] * (k - self.k_beta - self.k_gamma)])
x0 = np.ones(k) * .01
if random_seed != 0:
np.random.seed(random_seed)
x0 = np.random.randn(k) * .01
if var is not None:
if self.add_re is True:
assert len(var) == k
x0 = var
else:
assert len(var) == self.k_beta
x0 = np.append(var, [1e-8])
assert len(x0) == k
if build_X:
self._buildX()
if fit_fixed or self.add_re is False:
uprior_fixed = copy.deepcopy(self.uprior)
uprior_fixed[:, self.k_beta:self.k_beta + self.k_gamma] = 1e-8
if S is None or trim_percentage >= 0.01:
model_fixed = LimeTr(self.grouping,
int(self.k_beta),
int(self.k_gamma),
self.Y,
self.X,
self.JX,
self.Z,
S=S,
C=C,
JC=JC,
c=c,
inlier_percentage=1. - trim_percentage,
share_obs_std=share_obs_std,
uprior=uprior_fixed)
model_fixed.optimize(
x0=x0,
print_level=inner_print_level,
max_iter=inner_max_iter,
tol=inner_tol,
acceptable_tol=inner_acceptable_tol,
nlp_scaling_min_value=inner_nlp_scaling_min_value)
x0 = model_fixed.soln
self.beta_fixed = model_fixed.beta
if self.add_re is False:
self.beta_soln = self.beta_fixed
self.delta_soln = model_fixed.delta
self.gamma_soln = model_fixed.gamma
self.w_soln = model_fixed.w
self.info = model_fixed.info['status_msg']
self.yfit_no_random = model_fixed.F(model_fixed.beta)
return
else:
self.beta_fixed = self._solveBeta(S)
x0 = np.append(self.beta_fixed, [1e-8] * self.k_gamma)
if self.add_re is False:
self.beta_soln = self.beta_fixed
self.yfit_no_random = self.Xm.dot(self.beta_fixed)
return
model = LimeTr(self.grouping,
int(self.k_beta),
int(self.k_gamma),
self.Y,
self.X,
self.JX,
self.Z,
S=S,
C=C,
JC=JC,
c=c,
inlier_percentage=1 - trim_percentage,
share_obs_std=share_obs_std,
uprior=self.uprior,
gprior=self.gprior)
model.fitModel(x0=x0,
inner_print_level=inner_print_level,
inner_max_iter=inner_max_iter,
inner_acceptable_tol=inner_acceptable_tol,
inner_nlp_scaling_min_value=inner_nlp_scaling_min_value,
inner_tol=inner_tol,
outer_verbose=outer_verbose,
outer_max_iter=outer_max_iter,
outer_step_size=outer_step_size,
outer_tol=outer_tol)
self.beta_soln = model.beta
self.gamma_soln = model.gamma
if normalize_Z:
self.gamma_soln /= Z_norm**2
self.delta_soln = model.delta
self.info = model.info
self.w_soln = model.w
self.u_soln = model.estimateRE()
self.solve_status = model.info['status']
self.solve_status_msg = model.info['status_msg']
self.yfit_no_random = model.F(model.beta)
self.yfit = []
Z_split = np.split(self.Z, self.n_groups)
yfit_no_random_split = np.split(self.yfit_no_random, self.n_groups)
for i in range(self.n_groups):
self.yfit.append(yfit_no_random_split[i] +
Z_split[i].dot(self.u_soln[i]))
self.yfit = np.concatenate(self.yfit)
self.model = model
if inner_verbose == True and self.solve_status != 0:
print(self.solve_status_msg)
