class MR_BRT: def __init__(self, obs_mean, obs_std, study_sizes, x_cov_list, z_cov_list, spline_list=[], inlier_percentage=1.0, rr_random_slope=False): """ Initialize the object and pass in the data, require - obs_mean: observations - obs_std: standard deviations for the observations - study_sizes: all study sizes in a list - x_cov_list: all x cov in a list - z_cov_list: all z cov in a list - spline_list: optional, all spline in a list - inlier_percentage: optional, used for trimming """ # pass in data self.obs_mean = obs_mean self.obs_std = obs_std self.study_sizes = study_sizes self.num_studies = len(study_sizes) self.num_obs = sum(study_sizes) # construct x and z "covariates" self.spline_list = spline_list self.x_cov_list = x_cov_list self.z_cov_list = z_cov_list (self.F, self.JF, self.F_list, self.JF_list, self.id_beta_list, self.id_spline_beta_list, self.k_beta_list) = utils.constructXCov(x_cov_list, spline_list=spline_list) (self.Z, self.Z_list, self.id_gamma_list, self.id_spline_gamma_list, self.k_gamma_list) = utils.constructZCov(z_cov_list, spline_list=spline_list) self.k_beta = int(sum(self.k_beta_list)) self.k_gamma = int(sum(self.k_gamma_list)) self.k = self.k_beta + self.k_gamma self.id_beta = slice(0, self.k_beta) self.id_gamma = slice(self.k_beta, self.k) # if use the random slope model or not self.rr_random_slope = rr_random_slope if rr_random_slope: valid_x_cov_id = [ i for i in range(len(x_cov_list)) if x_cov_list[i]['cov_type'] in ['log_ratio_spline', 'log_ratio_spline_integral'] ] if len(valid_x_cov_id) == 0: raise Exception( "Error: no suitable x cov for random slope model.") if len(valid_x_cov_id) >= 2: raise Exception( "Error: multiple x cov for random slope model.") x_cov = x_cov_list[valid_x_cov_id[0]] mat = x_cov['mat'] if x_cov['cov_type'] == 'log_ratio_spline': scaling = mat[0] - mat[1] else: scaling = 0.5 * (mat[0] + mat[1] - mat[2] - mat[3]) self.Z *= scaling.reshape(scaling.size, 1) # create limetr object self.inlier_percentage = inlier_percentage self.lt = LimeTr(self.study_sizes, self.k_beta, self.k_gamma, self.obs_mean, self.F, self.JF, self.Z, self.obs_std, inlier_percentage=inlier_percentage) def addPriors(self, prior_list): """ Add priors to the object, require prior_list contains priors """ self.prior_list = prior_list (self.C, self.JC, self.c, self.H, self.JH, self.h, self.uprior, self.gprior, self.lprior, self.C_list, self.JC_list, self.c_list, self.H_list, self.JH_list, self.h_list, self.id_C_list, self.id_C_var_list, self.id_H_list, self.id_H_var_list, self.num_constraints_list, self.num_regularizers_list) = utils.constructPrior(prior_list, self) # renew uprior for gamma if self.uprior is None: self.uprior = np.array([[-np.inf] * self.k_beta + [1e-7] * self.k_gamma, [np.inf] * self.k]) else: uprior_beta = self.uprior[:, self.id_beta] uprior_gamma = self.uprior[:, self.id_gamma] uprior_gamma[0] = np.maximum(1e-7, uprior_gamma[0]) uprior_gamma[1] = np.maximum(uprior_gamma[0], uprior_gamma[1]) self.uprior = np.hstack((uprior_beta, uprior_gamma)) self.lt.C, self.lt.JC, self.lt.c = self.C, self.JC, self.c self.lt.H, self.lt.JH, self.lt.h = self.H, self.JH, self.h (self.lt.uprior, self.lt.gprior, self.lt.lprior) = (self.uprior, self.gprior, self.lprior) self.lt = LimeTr(self.study_sizes, self.k_beta, self.k_gamma, self.obs_mean, self.F, self.JF, self.Z, self.obs_std, C=self.C, JC=self.JC, c=self.c, H=self.H, JH=self.JH, h=self.h, uprior=self.uprior, gprior=self.gprior, lprior=self.lprior, inlier_percentage=self.inlier_percentage) def fitModel(self, x0=None, outer_verbose=False, outer_max_iter=100, outer_step_size=1.0, outer_tol=1e-6, inner_print_level=0, inner_max_iter=20): # initialization with gamma set to be zero gamma_uprior = self.lt.uprior[:, self.lt.idx_gamma].copy() self.lt.uprior[:, self.lt.idx_gamma] = 1e-6 self.lt.n = np.array([1] * self.num_obs) norm_z_col = np.linalg.norm(self.lt.Z, axis=0) self.lt.Z /= norm_z_col if x0 is None: if self.lprior is None: x0 = np.array([1.0] * self.k_beta + [1e-6] * self.k_gamma) else: x0 = np.array([1.0] * self.k_beta * 2 + [1e-6] * self.k_gamma * 2) else: if self.lprior is not None: beta0 = x0[:self.k_beta] gamma0 = x0[self.k_beta:self.k_beta + self.k_gamma] x0 = np.hstack((beta0, np.abs(beta0), gamma0, gamma0)) (beta_0, gamma_0, self.w_soln) = self.lt.fitModel(x0=x0, outer_verbose=outer_verbose, outer_max_iter=outer_max_iter, outer_step_size=outer_step_size, outer_tol=outer_tol, inner_print_level=inner_print_level, inner_max_iter=inner_max_iter) # print("init obj", self.lt.objective(self.lt.soln)) # fit the model from the initial point self.lt.uprior[:, self.lt.idx_gamma] = gamma_uprior self.lt.n = self.study_sizes if self.lprior is not None: x0 = np.hstack((beta_0, np.abs(beta_0), gamma_0, gamma_0)) else: x0 = np.hstack((beta_0, gamma_0)) self.lt.optimize(x0=x0, print_level=inner_print_level, max_iter=100) self.lt.Z *= norm_z_col self.lt.gamma /= norm_z_col**2 self.beta_soln = self.lt.beta self.gamma_soln = self.lt.gamma # print("final obj", self.lt.objective(self.lt.soln)) # print("------------------------------") def predictData(self, pred_x_cov_list, pred_z_cov_list, sample_size, pred_study_sizes=None, given_beta_samples=None, given_gamma_samples=None, ref_point=None, include_random_effect=True): # sample solutions if given_beta_samples is None or given_gamma_samples is None: beta_samples, gamma_samples = LimeTr.sampleSoln( self.lt, sample_size=sample_size) else: beta_samples = given_beta_samples gamma_samples = given_gamma_samples # calculate the beta and gamma post cov # self.beta_samples_mean = np.mean(beta_samples, axis=0) # self.gamma_samples_mean = np.mean(gamma_samples, axis=0) # self.beta_samples_cov = \ # beta_samples.T.dot(beta_samples)/sample_size - \ # np.outer(self.beta_samples_mean, self.beta_samples_mean) # self.gamma_samples_cov = \ # gamma_samples.T.dot(gamma_samples)/sample_size - \ # np.outer(self.gamma_samples_mean, self.gamma_samples_mean) # create x cov (pred_F, pred_JF, pred_F_list, pred_JF_list, pred_id_beta_list) = utils.constructPredXCov(pred_x_cov_list, self) # create z cov (pred_Z, pred_Z_list, pred_id_gamma_list) = utils.constructPredZCov(pred_z_cov_list, self) # num of studies pred_num_obs = pred_Z.shape[0] # create observation samples y_samples = np.vstack([pred_F(beta) for beta in beta_samples]) if ref_point is not None: x_cov_spline_id = [ x_cov['spline_id'] for x_cov in pred_x_cov_list if 'spline' in x_cov['cov_type'] ] if len(x_cov_spline_id) == 0: raise Exception("Error: no spline x cov") if len(x_cov_spline_id) >= 2: raise Exception("Error: multiple spline x covs") spline = self.spline_list[x_cov_spline_id[0]] ref_risk = spline.designMat(np.array([ref_point])).dot( beta_samples[:, self.id_spline_beta_list[x_cov_spline_id[0]]].T) y_samples /= ref_risk.reshape(sample_size, 1) pred_gamma = np.hstack([ self.gamma_soln[pred_id_gamma_list[i]] for i in range(len(pred_id_gamma_list)) ]) if include_random_effect: if self.rr_random_slope: u = np.random.randn(sample_size, self.k_gamma)*\ np.sqrt(self.gamma_soln) # zu = np.sum(pred_Z*u, axis=1) zu = u[:, 0] valid_x_cov_id = [ i for i in range(len(pred_x_cov_list)) if pred_x_cov_list[i]['cov_type'] == 'spline' ] if len(valid_x_cov_id) == 0: raise Exception( "Error: no suitable x cov for random slope model.") if len(valid_x_cov_id) >= 2: raise Exception( "Error: multiple x cov for random slope model.") mat = pred_x_cov_list[valid_x_cov_id[0]]['mat'] if ref_point is None: y_samples *= np.exp(np.outer(zu, mat - mat[0])) else: y_samples *= np.exp(np.outer(zu, mat - ref_point)) else: if pred_study_sizes is None: pred_study_sizes = np.array([1] * pred_num_obs) else: assert sum(pred_study_sizes) == pred_num_obs pred_num_studies = len(pred_study_sizes) pred_Z_sub = np.split(pred_Z, np.cumsum(pred_study_sizes)[:-1]) u = [ np.random.multivariate_normal( np.zeros(pred_study_sizes[i]), (pred_Z_sub[i] * pred_gamma).dot(pred_Z_sub[i].T), sample_size) for i in range(pred_num_studies) ] U = np.hstack(u) if np.any([ 'log_ratio' in self.x_cov_list[i]['cov_type'] for i in range(len(self.x_cov_list)) ]): y_samples *= np.exp(U) else: y_samples += U return y_samples, beta_samples, gamma_samples, pred_F, pred_Z
def optimize(self, var=None, S=None, trim_percentage=0.0, share_obs_std=True, fit_fixed=True, inner_print_level=5, inner_max_iter=100, inner_tol=1e-5, inner_verbose=True, inner_acceptable_tol=1e-4, inner_nlp_scaling_min_value=1e-8, outer_verbose=False, outer_max_iter=1, outer_step_size=1, outer_tol=1e-6, normalize_Z=False, build_X=True, random_seed=0): """ Run optimization routine via LimeTr. Args: var (numpy.ndarray | None, optional): One-dimensional array that gives initialization for variables. If None, the program will first run without random effects to obtain a starting point. S (numpy.ndarray | None, optional): One-dimensional numpy array that gives standard deviation for each measurement. The size of S should be the same as that of measurements vector. If None standard deviation of measurement will be treated as variables and optimized. trim_percentage (float | 0.0, optional): A float that gives percentage of datapoints to trim. Default is 0, i.e. no trimming. share_obs_std (boolean | True, optional): A boolean that indicates whether the model should assume data across studies share the same measurement standard deviation. fit_fixed (boolean | True, optional): A boolean that indicates whether to run a fit without random effects first in order to obtain a good starting point. inner_print_level (int | 5, optional): ``print_level`` for Ipopt. inner_max_iter (int | 100, optional): Maximum number of iterations for inner optimization. inner_tol (float | 1e-5, optional): Tolerance level for inner optimization. inner_verbose (boolean | True, optional): Verbose option for inner optimization. inner_acceptable_tol (float | 1e-4, optional): Acceptable tolerance level for inner optimization. inner_nlp_scaling_min_value (float | 1e-8, optional): Min scaling for objective function. outer_verbose (boolean | False, optional): Verbose option for outer optimization. outer_max_iter (int | 1, optional): Maximum number of iterations for outer optimization. When there is no trimming, outer optimization is not needed, so the default is set to be 1. outer_step_size (float |1.0, optional): Step size for outer optimization. Used in trimming. outer_tol (float | 1e-6, optional): Tolerance level for outer optimization. normalize_Z (bool | False, optional): Whether to normalize Z matrix before optimization. build_X (bool | True, optional): Whether to explicitly build and store X matrix. random_seed (int | 0, optional): random seed for choosing an initial point for optimization. If equals 0 the initial point is chosen to be a vector of 0.01. """ self.S = S self.share_obs_std = share_obs_std Z_norm = self.buildZ(normalize_Z) k = self.k_beta + self.k_gamma if S is None: if share_obs_std: k += 1 else: k += len(self.grouping) print('n_groups', self.n_groups) print('k_beta', self.k_beta) print('k_gamma', self.k_gamma) print('total number of fixed effects variables', k) if self.k_gamma == 0: self.add_re = False self.k_gamma = 1 k += 1 else: self.add_re = True C = [] start = self.k_beta for ran in self.ran_list: _, dims = ran m = np.prod(dims[self.n_grouping_dims:]) c = np.zeros((m - 1, k)) for i in range(m - 1): c[i, start + i] = 1 c[i, start + i + 1] = -1 C.append(c) start += m if len(C) > 0: self.constraints = np.vstack(C) assert self.constraints.shape[1] == k else: self.constraints = [] C = None if len(self.constraints) > 0: def C(var): return self.constraints.dot(var) JC = None if len(self.constraints) > 0: def JC(var): return self.constraints c = None if len(self.constraints) > 0: c = np.zeros((2, self.constraints.shape[0])) self.uprior = np.array( [[-np.inf] * self.k_beta + [1e-7] * self.k_gamma + [1e-7] * (k - self.k_beta - self.k_gamma), [np.inf] * k]) if self.global_cov_bounds is not None: if self.global_intercept: self.uprior[:, 1:len(self.global_ids) + 1] = self.global_cov_bounds else: self.uprior[:, :len(self.global_ids)] = self.global_cov_bounds self.gprior = None if self.use_gprior: assert len(self.ran_eff_gamma_sd) == self.k_gamma self.gprior = np.array( [[0] * k, [np.inf] * self.k_beta + self.ran_eff_gamma_sd + [np.inf] * (k - self.k_beta - self.k_gamma)]) x0 = np.ones(k) * .01 if random_seed != 0: np.random.seed(random_seed) x0 = np.random.randn(k) * .01 if var is not None: if self.add_re is True: assert len(var) == k x0 = var else: assert len(var) == self.k_beta x0 = np.append(var, [1e-8]) assert len(x0) == k if build_X: self._buildX() if fit_fixed or self.add_re is False: uprior_fixed = copy.deepcopy(self.uprior) uprior_fixed[:, self.k_beta:self.k_beta + self.k_gamma] = 1e-8 if S is None or trim_percentage >= 0.01: model_fixed = LimeTr(self.grouping, int(self.k_beta), int(self.k_gamma), self.Y, self.X, self.JX, self.Z, S=S, C=C, JC=JC, c=c, inlier_percentage=1. - trim_percentage, share_obs_std=share_obs_std, uprior=uprior_fixed) model_fixed.optimize( x0=x0, print_level=inner_print_level, max_iter=inner_max_iter, tol=inner_tol, acceptable_tol=inner_acceptable_tol, nlp_scaling_min_value=inner_nlp_scaling_min_value) x0 = model_fixed.soln self.beta_fixed = model_fixed.beta if self.add_re is False: self.beta_soln = self.beta_fixed self.delta_soln = model_fixed.delta self.gamma_soln = model_fixed.gamma self.w_soln = model_fixed.w self.info = model_fixed.info['status_msg'] self.yfit_no_random = model_fixed.F(model_fixed.beta) return else: self.beta_fixed = self._solveBeta(S) x0 = np.append(self.beta_fixed, [1e-8] * self.k_gamma) if self.add_re is False: self.beta_soln = self.beta_fixed self.yfit_no_random = self.Xm.dot(self.beta_fixed) return model = LimeTr(self.grouping, int(self.k_beta), int(self.k_gamma), self.Y, self.X, self.JX, self.Z, S=S, C=C, JC=JC, c=c, inlier_percentage=1 - trim_percentage, share_obs_std=share_obs_std, uprior=self.uprior, gprior=self.gprior) model.fitModel(x0=x0, inner_print_level=inner_print_level, inner_max_iter=inner_max_iter, inner_acceptable_tol=inner_acceptable_tol, inner_nlp_scaling_min_value=inner_nlp_scaling_min_value, inner_tol=inner_tol, outer_verbose=outer_verbose, outer_max_iter=outer_max_iter, outer_step_size=outer_step_size, outer_tol=outer_tol) self.beta_soln = model.beta self.gamma_soln = model.gamma if normalize_Z: self.gamma_soln /= Z_norm**2 self.delta_soln = model.delta self.info = model.info self.w_soln = model.w self.u_soln = model.estimateRE() self.solve_status = model.info['status'] self.solve_status_msg = model.info['status_msg'] self.yfit_no_random = model.F(model.beta) self.yfit = [] Z_split = np.split(self.Z, self.n_groups) yfit_no_random_split = np.split(self.yfit_no_random, self.n_groups) for i in range(self.n_groups): self.yfit.append(yfit_no_random_split[i] + Z_split[i].dot(self.u_soln[i])) self.yfit = np.concatenate(self.yfit) self.model = model if inner_verbose == True and self.solve_status != 0: print(self.solve_status_msg)