def adjust_mean_mtslice(tdata_mt, ref=None):
"""Adjust the mean of the mtslice by the given ref
"""
if ref is None:
ref = tdata_mt.mean_temp[0]
# import pdb; pdb.set_trace()
study_slices = utils.sizes_to_slices(tdata_mt.study_sizes)
for i in range(tdata_mt.num_studies):
obs_mean = tdata_mt.obs_mean[study_slices[i]]
obs_std = tdata_mt.obs_std[study_slices[i]]
cov = tdata_mt.daily_temp[study_slices[i]]
# fit the curve
if tdata_mt.study_sizes[i] >= 5:
spline = xspline.XSpline(np.array([cov.min(), ref,
cov.max()]),
2,
l_linear=True)
else:
spline = xspline.XSpline(np.array([cov.min(), cov.max()]), 1)
beta = utils.fit_spline(obs_mean, obs_std, cov, spline)
ref_lnrr = spline.design_mat(ref).dot(beta)
# adjust the mean
tdata_mt.obs_mean[study_slices[i]] -= ref_lnrr
return tdata_mt
def __init__(self,
beta,
beta_var,
gamma,
random_effects,
mean_temp,
num_beta_spline_knots=6,
num_gamma_spline_knots=6,
beta_spline_degree=3,
gamma_spline_degree=3):
# pass in the data
self.num_mean_temp = mean_temp.size
assert beta.shape == (self.num_mean_temp, 2)
assert gamma.shape == (self.num_mean_temp, 2)
self.beta = beta
self.beta_var = beta_var
self.gamma = gamma
self.mean_temp = mean_temp
self.random_effects = random_effects
# construct the splines
self.min_mean_temp = self.mean_temp.min()
self.max_mean_temp = self.mean_temp.max()
beta_spline_knots = np.linspace(self.min_mean_temp, self.max_mean_temp,
num_beta_spline_knots)
gamma_spline_knots = np.linspace(self.min_mean_temp,
self.max_mean_temp,
num_gamma_spline_knots)
# gamma_spline_knots = np.array([
# self.min_mean_temp,
# 13.0,
# 17.0,
# 22.0,
# self.max_mean_temp
# ])
self.beta_spline = xspline.XSpline(beta_spline_knots,
beta_spline_degree,
l_linear=True,
r_linear=True)
self.gamma_spline = xspline.XSpline(gamma_spline_knots,
gamma_spline_degree,
l_linear=True,
r_linear=True)
# compute the spline bases coefficients
X_beta = self.beta_spline.design_mat(self.mean_temp)
X_gamma = self.gamma_spline.design_mat(self.mean_temp)
self.c_beta = np.linalg.solve(X_beta.T.dot(X_beta), X_beta.T.dot(beta))
self.c_gamma = np.linalg.solve(X_gamma.T.dot(X_gamma),
X_gamma.T.dot(gamma))
def create_spline(self, data: MRData) -> xspline.XSpline:
"""Create spline given current spline parameters.
Args:
data (mrtool.MRData):
The data frame used for storing the data
Returns:
xspline.XSpline: The spline object.
"""
# extract covariate
alt_cov = data.get_covs(self.alt_cov)
ref_cov = data.get_covs(self.ref_cov)
cov_all = np.hstack((alt_cov.ravel(), ref_cov.ravel()))
cov = np.array([min(cov_all), max(cov_all)])
if alt_cov.size != 0:
cov = np.hstack((cov, alt_cov.mean(axis=1)))
if ref_cov.size != 0:
cov = np.hstack((cov, ref_cov.mean(axis=1)))
if self.spline_knots_type == 'frequency':
spline_knots = np.quantile(cov, self.spline_knots_template)
else:
spline_knots = min(cov) + self.spline_knots_template*(max(cov) - min(cov))
self.prior_spline_monotonicity_domain = min(cov) + \
self.prior_spline_monotonicity_domain_template*(max(cov) - min(cov))
self.prior_spline_convexity_domain = min(cov) + \
self.prior_spline_convexity_domain_template*(max(cov) - min(cov))
spline = xspline.XSpline(spline_knots,
self.spline_degree,
l_linear=self.spline_l_linear,
r_linear=self.spline_r_linear)
return spline
def create_spline(self, data: MRData, spline_knots: np.ndarray = None) -> xspline.XSpline:
"""Create spline given current spline parameters.
Args:
data (mrtool.MRData):
The data frame used for storing the data
spline_knots (np.ndarray, optional):
Spline knots, if ``None`` determined by frequency or domain.
Returns:
xspline.XSpline: The spline object.
"""
# extract covariate
alt_cov = data.get_covs(self.alt_cov)
ref_cov = data.get_covs(self.ref_cov)
cov_all = np.hstack((alt_cov.ravel(), ref_cov.ravel()))
cov = np.array([min(cov_all), max(cov_all)])
if alt_cov.size != 0:
cov = np.hstack((cov, alt_cov.mean(axis=1)))
if ref_cov.size != 0:
cov = np.hstack((cov, ref_cov.mean(axis=1)))
cov = np.unique(cov)
if spline_knots is None:
if self.spline_knots_type == 'frequency':
spline_knots = np.quantile(cov, self.spline_knots_template)
else:
spline_knots = cov.min() + self.spline_knots_template*(cov.max() - cov.min())
self.prior_spline_monotonicity_domain = spline_knots[0] + \
self.prior_spline_monotonicity_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_convexity_domain = spline_knots[0] + \
self.prior_spline_convexity_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_derval_gaussian_domain = spline_knots[0] + \
self.prior_spline_derval_gaussian_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_derval_uniform_domain = spline_knots[0] + \
self.prior_spline_derval_uniform_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_der2val_gaussian_domain = spline_knots[0] + \
self.prior_spline_der2val_gaussian_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_der2val_uniform_domain = spline_knots[0] + \
self.prior_spline_der2val_uniform_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_funval_gaussian_domain = spline_knots[0] + \
self.prior_spline_funval_gaussian_domain_template*(spline_knots[-1] - spline_knots[0])
self.prior_spline_funval_uniform_domain = spline_knots[0] + \
self.prior_spline_funval_uniform_domain_template*(spline_knots[-1] - spline_knots[0])
spline = xspline.XSpline(spline_knots,
self.spline_degree,
l_linear=self.spline_l_linear,
r_linear=self.spline_r_linear)
return spline
def adjust_agg_std_mtslice(tdata_mt, ref=None):
"""Adjust std of the aggregate the tdata slices
"""
if ref is None:
ref = tdata_mt.mean_temp[0]
# fit the curve
spline = xspline.XSpline(np.array(
[tdata_mt.daily_temp.min(), ref,
tdata_mt.daily_temp.max()]),
2,
l_linear=True)
beta = utils.fit_spline(tdata_mt.obs_mean, tdata_mt.obs_std,
tdata_mt.daily_temp, spline)
residual = (tdata_mt.obs_mean -
spline.design_mat(tdata_mt.daily_temp).dot(beta))
residual /= tdata_mt.obs_std
# print(np.maximum(1.0, np.std(residual)))
tdata_mt.obs_std *= np.maximum(3.0, np.std(residual))
return tdata_mt
def fit_trend_mtslice(tdata_at_mean_temp, tmrl, inlier_pct=0.9, debug=False):
"""
Return beta (intercept and slope) and gamma (intercept and slope)
with given data
"""
if debug:
print("number of locations at mean temp",
tdata_at_mean_temp.num_studies)
outer_verbose = True
inner_print_level = 5
else:
outer_verbose = False
inner_print_level = 0
# construct the linear mixed effect model
cov = tdata_at_mean_temp.daily_temp
knots = np.array([cov.min(), tmrl, cov.max()])
degree = 1
spline = xspline.XSpline(knots, degree)
l1 = knots[1] - knots[0]
l2 = knots[2] - knots[1]
mat_transform = np.array([[1.0, 0.0, 0.0], [1.0, l1, 0.0], [1.0, l1, l2]])
M = spline.design_mat(cov).dot(mat_transform)
M[:, 1] -= M[:, 0] * l1
M = M[:, 1:]
scale = np.linalg.norm(M, axis=0)
scaled_M = M / scale
# construct the LimeTr object
F = lambda beta: scaled_M.dot(beta)
JF = lambda beta: scaled_M
Z = scaled_M.copy()
n = tdata_at_mean_temp.study_sizes
k_beta = 2
k_gamma = 2
Y = tdata_at_mean_temp.obs_mean
S = tdata_at_mean_temp.obs_std
uprior = np.array([[-np.inf] * k_beta + [1e-7] * k_gamma,
[np.inf] * k_beta + [1.5] * k_gamma])
lt = limetr.LimeTr(n,
k_beta,
k_gamma,
Y,
F,
JF,
Z,
S=S,
uprior=uprior,
inlier_percentage=inlier_pct)
# fit model
MS = M / S.reshape(S.size, 1)
YS = Y / S
beta0 = np.linalg.solve(MS.T.dot(MS), MS.T.dot(YS))
gamma0 = np.array([0.1, 0.1])
(beta, gamma,
trimming_weights) = lt.fitModel(x0=np.hstack((beta0, gamma0)),
outer_step_size=200.0,
outer_verbose=outer_verbose,
inner_print_level=inner_print_level)
# estimate the random effects
random_effects = lt.estimateRE()
# estimate the uncertainty of beta
V = limetr.utils.VarMat(lt.S**2, lt.Z, gamma, lt.n)
beta_var = np.linalg.inv(M.T.dot(V.invDot(M)))
# # scale beta and gamma back
beta /= scale
beta_var /= scale**2
gamma /= scale**2
random_effects /= scale
return beta, beta_var, gamma, random_effects