def population_test_cosinor_pairs(df, pairs, period = 24):
df_res = pd.DataFrame(columns = ['test',
'd_amplitude',
'p(d_amplitude)',
'q(d_amplitude)',
'd_acrophase',
'p(d_acrophase)',
'q(d_acrophase)'], dtype=float)
for pair in pairs:
df_pop1 = df[df.test.str.startswith(pair[0])]
df_pop2 = df[df.test.str.startswith(pair[1])]
res1 = population_fit_cosinor(df_pop1, period = period, plot_on = False)
res2 = population_fit_cosinor(df_pop2, period = period, plot_on = False)
res = population_test_cosinor(res1, res2)
d = {'test': res['test'],
'd_amplitude':res['amp']['pop2'] - res['amp']['pop1'],
'p(d_amplitude)':res['amp']['p_value'],
'd_acrophase1':cosinor.project_acr(res['acr']['pop2']-res['acr']['pop1']),
'p(d_acrophase)':res['acr']['p_value']}
df_res = df_res.append(d, ignore_index=True)
df_res['q(d_amplitude)'] = multi.multipletests(df_res['p(d_amplitude)'], method = 'fdr_bh')[1]
df_res['q(d_acrophase)'] = multi.multipletests(df_res['p(d_acrophase)'], method = 'fdr_bh')[1]
return df_res
def test_cosinor_single(data, period = 24, corrected = True):
rrr = np.cos(2*np.pi*data.x/period)
sss = np.sin(2*np.pi*data.x/period)
data['rrr'] = rrr
data['sss'] = sss
results = smf.ols('y ~ rrr + sss', data).fit()
beta_s = results.params['sss']
beta_r = results.params['rrr']
amp, acr = amp_acr(beta_s, beta_r)
# project acropahse to interval -pi,pi
acr = cosinor.project_acr(acr)
vmat = results.cov_params().loc[['rrr', 'sss'], ['rrr', 'sss']]
indVmat = vmat
a_r = (beta_r**2 + beta_s**2)**(-0.5) * beta_r
a_s = (beta_r**2 + beta_s**2)**(-0.5) * beta_s
b_r = (1 / (1 + (beta_s**2 / beta_r**2))) * (-beta_s / beta_r**2)
b_s = (1 / (1 + (beta_s**2 / beta_r**2))) * (1 / beta_r)
if corrected:
b_r = -b_r
b_s = -b_s
jac = np.array([[a_r, a_s], [b_r, b_s]])
cov_trans = np.dot(np.dot(jac, indVmat), np.transpose(jac))
se_trans_only =np.sqrt(np.diag(cov_trans))
zt = abs(stats.norm.ppf((1-0.95)/2))
trans_names = [results.params.index.values[0]] + ['amp', 'acr']
coef_trans = np.array([results.params.iloc[0], amp, acr])
se_trans = np.concatenate((np.sqrt(np.diag(results.cov_params().loc[['Intercept'], ['Intercept']])),se_trans_only))
lower_CI_trans = coef_trans - np.abs(zt * se_trans)
upper_CI_trans = coef_trans + np.abs(zt * se_trans)
p_value_trans = 2 * stats.norm.cdf(-np.abs(coef_trans/se_trans))
statistics= {'parameters': trans_names,
'values': coef_trans,
'SE': se_trans,
'CI': (lower_CI_trans, upper_CI_trans),
'p-values': p_value_trans,
'F-test': results.f_pvalue}
return results, amp, acr, statistics
def test_cosinor_pairs_independent(df,
pairs,
period=24,
period2=None,
df_best_models=-1):
period1 = period
if not period2:
period2 = period
df_results = pd.DataFrame(columns=[
'test', 'p1', 'q1', 'p2', 'q2', 'period1', 'period2', 'amplitude1',
'amplitude2', 'd_amplitude', 'p(d_amplitude)', 'q(d_amplitude)',
'CI(d_amplitude)', 'acrophase1', 'acrophase2', 'd_acrophase',
'p(d_acrophase)', 'q(d_acrophase)', 'CI(d_acrophase)'
],
dtype=float)
for test1, test2 in pairs:
X1, Y1 = np.array(df[df.test == test1].x), np.array(
df[df.test == test1].y)
X2, Y2 = np.array(df[df.test == test2].x), np.array(
df[df.test == test2].y)
if type(df_best_models) == int:
pass
else:
period1 = df_best_models[df_best_models.test ==
test1].period.iloc[0]
period2 = df_best_models[df_best_models.test ==
test2].period.iloc[0]
fit_results1, amp1, acr1, stats1 = fit_cosinor(X1,
Y1,
period=period1,
plot_on=False)
fit_results2, amp2, acr2, stats2 = fit_cosinor(X2,
Y2,
period=period2,
plot_on=False)
k = len(X1) + len(X2)
k1 = len(X1)
k2 = len(X2)
DoF = k - (len(fit_results1.params) + len(fit_results2.params))
DoF1 = k1 - len(fit_results1.params)
DoF2 = k2 - len(fit_results2.params)
t = abs(stats.t.ppf(0.05 / 2, df=DoF))
d_amp = amp2 - amp1
d_acr = cosinor.project_acr(acr2 - acr1)
CI_amp1 = stats1['CI'][0][-2], stats1['CI'][1][-2]
CI_amp2 = stats2['CI'][0][-2], stats2['CI'][1][-2]
se_amp = cosinor.get_se_diff_from_CIs(CI_amp1,
CI_amp2,
DoF1,
DoF2,
t_test=True,
angular=False,
CI_type="se",
n1=k1,
n2=k2,
DoF=DoF,
biased=True)
dev_amp = t * se_amp
T0 = d_amp / se_amp
p_val_amp = 2 * (1 - stats.t.cdf(abs(T0), DoF))
CI_amp = [d_amp - dev_amp, d_amp + dev_amp]
CI_acr1 = stats1['CI'][0][-1], stats1['CI'][1][-1]
CI_acr2 = stats2['CI'][0][-1], stats2['CI'][1][-1]
se_acr = cosinor.get_se_diff_from_CIs(CI_acr1,
CI_acr2,
DoF1,
DoF2,
t_test=True,
angular=True,
CI_type="se",
n1=k1,
n2=k2,
DoF=DoF,
biased=True)
dev_acr = se_acr * t
T0 = d_acr / se_acr
p_val_acr = 2 * (1 - stats.t.cdf(abs(T0), DoF))
CI_acr = [d_acr - dev_acr, d_acr + dev_acr]
d = {
'test': test1 + ' vs. ' + test2,
'p1': fit_results1.f_pvalue,
'p2': fit_results2.f_pvalue,
'period1': period1,
'period2': period2,
'amplitude1': amp1,
'amplitude2': amp2,
'd_amplitude': d_amp,
'p(d_amplitude)': p_val_amp,
'CI(d_amplitude)': CI_amp,
'acrophase1': acr1,
'acrophase2': acr2,
'd_acrophase': d_acr,
'p(d_acrophase)': p_val_acr,
'CI(d_acrophase)': CI_acr
}
df_results = df_results.append(d, ignore_index=True)
df_results['q1'] = multi.multipletests(df_results['p1'],
method='fdr_bh')[1]
df_results['q2'] = multi.multipletests(df_results['p2'],
method='fdr_bh')[1]
df_results['q(d_amplitude)'] = multi.multipletests(
df_results['p(d_amplitude)'], method='fdr_bh')[1]
df_results['q(d_acrophase)'] = multi.multipletests(
df_results['p(d_acrophase)'], method='fdr_bh')[1]
return df_results
def population_test_cosinor_pairs_independent(df,
pairs,
period=24,
period2=None):
period1 = period
if not period2:
period2 = period
df_res = pd.DataFrame(columns=[
'test', 'amplitude1', 'amplitude2', 'd_amplitude', 'p(d_amplitude)',
'q(d_amplitude)', 'CI(d_amplitude)', 'acrophase1', 'acrophase2',
'd_acrophase', 'p(d_acrophase)', 'q(d_acrophase)', 'CI(d_acrophase)'
],
dtype=float)
for pair in pairs:
df_pop1 = df[df.test.str.startswith(pair[0])]
df_pop2 = df[df.test.str.startswith(pair[1])]
k1 = len(df_pop1.test.unique())
k2 = len(df_pop2.test.unique())
DoF1 = k1 - 1
DoF2 = k2 - 1
DoF = k1 + k2 - 2
t = abs(stats.t.ppf(0.05 / 2, df=DoF))
res1 = population_fit_cosinor(df_pop1, period=period1, plot_on=False)
res2 = population_fit_cosinor(df_pop2, period=period2, plot_on=False)
#amplitude ('amp')
param = 'amp'
amp1, amp2 = res1['means'][-2], res2['means'][-2]
CI1 = res1['confint'][param][0], res1['confint'][param][1]
CI2 = res2['confint'][param][0], res2['confint'][param][1]
se = cosinor.get_se_diff_from_CIs(CI1,
CI2,
DoF1,
DoF2,
t_test=True,
angular=False,
CI_type="se",
n1=k1,
n2=k2,
DoF=DoF,
biased=True)
dev = se * t
d_amp = amp2 - amp1
CI_amp = [d_amp - dev, d_amp + dev]
T0 = d_amp / se
p_val_amp = 2 * (1 - stats.t.cdf(abs(T0), DoF))
#acrophase ('acr')
param = 'acr'
acr1, acr2 = res1['means'][-1], res2['means'][-1]
CI1 = res1['confint'][param][0], res1['confint'][param][1]
CI2 = res2['confint'][param][0], res2['confint'][param][1]
se = cosinor.get_se_diff_from_CIs(CI1,
CI2,
DoF1,
DoF2,
t_test=True,
angular=True,
CI_type="se",
n1=k1,
n2=k2,
DoF=DoF,
biased=True)
dev = se * t
d_acr = cosinor.project_acr(acr2 - acr1)
CI_acr = [d_acr - dev, d_acr + dev]
T0 = d_acr / se
p_val_acr = 2 * (1 - stats.t.cdf(abs(T0), DoF))
d = {
'test': pair[0] + ' vs ' + pair[1],
'amplitude1': amp1,
'amplitude2': amp2,
'd_amplitude': d_amp,
'p(d_amplitude)': p_val_amp,
'CI(d_amplitude)': CI_amp,
'acrophase1': acr1,
'acrophase2': acr2,
'd_acrophase': d_acr,
'p(d_acrophase)': p_val_acr,
'CI(d_acrophase)': CI_acr
}
df_res = df_res.append(d, ignore_index=True)
df_res['q(d_amplitude)'] = multi.multipletests(df_res['p(d_amplitude)'],
method='fdr_bh')[1]
df_res['q(d_acrophase)'] = multi.multipletests(df_res['p(d_acrophase)'],
method='fdr_bh')[1]
return df_res
def test_cosinor_pair(data, period, corrected=True):
rrr = np.cos(2 * np.pi * data.x / period)
sss = np.sin(2 * np.pi * data.x / period)
data['rrr'] = rrr
data['sss'] = sss
results = smf.ols('y ~ group + rrr + sss + group:rrr + group:sss',
data).fit()
beta_s = np.array([results.params['sss'], results.params['group:sss']])
beta_r = np.array([results.params['rrr'], results.params['group:rrr']])
groups_s = beta_s.copy()
groups_s[1] += groups_s[0]
groups_r = beta_r.copy()
groups_r[1] += groups_r[0]
amp, acr = amp_acr(groups_s, groups_r)
vmat = results.cov_params().loc[['rrr', 'group:rrr', 'sss', 'group:sss'],
['rrr', 'group:rrr', 'sss', 'group:sss'], ]
indexmat = np.eye(4)
indexmat[1, 0] = 1
indexmat[3, 2] = 1
indVmat = np.dot(np.dot(indexmat, vmat), np.transpose(indexmat))
a_r = (groups_r**2 + groups_s**2)**(-0.5) * groups_r
a_s = (groups_r**2 + groups_s**2)**(-0.5) * groups_s
b_r = (1 / (1 + (groups_s**2 / groups_r**2))) * (-groups_s / groups_r**2)
b_s = (1 / (1 + (groups_s**2 / groups_r**2))) * (1 / groups_r)
if corrected:
b_r = -b_r
b_s = -b_s
jac = np.array((np.concatenate((np.diag(a_r)[0, :], np.diag(a_s)[0, :])),
np.concatenate((np.diag(a_r)[1, :], np.diag(a_s)[1, :])),
np.concatenate((np.diag(b_r)[0, :], np.diag(b_s)[0, :])),
np.concatenate((np.diag(b_r)[1, :], np.diag(b_s)[1, :]))))
cov_trans = np.dot(np.dot(jac, indVmat), np.transpose(jac))
se_trans_only = np.sqrt(np.diag(cov_trans))
zt = abs(stats.norm.ppf((1 - 0.95) / 2))
coef_raw = results.params
raw_names = list(coef_raw.index.values)
coef_raw = coef_raw.values
se_raw = np.sqrt(np.diag(results.cov_params()))
lower_CI_raw = coef_raw - zt * se_raw
upper_CI_raw = coef_raw + zt * se_raw
p_value_raw = 2 * stats.norm.cdf(-np.abs(coef_raw / se_raw))
statistics_raw = {
'parameters': raw_names,
'values': coef_raw,
'SE': se_raw,
'CI': (lower_CI_raw, upper_CI_raw),
'p-values': p_value_raw
}
trans_names = list(results.params.index.values[:2]) + [
'amp', 'group:amp', 'acr', 'group:acr'
]
coef_trans = np.concatenate((np.array(results.params.iloc[0:2]), amp, acr))
# correct acropahse
coef_trans[-1] = cosinor.project_acr(coef_trans[-1])
coef_trans[-2] = cosinor.project_acr(coef_trans[-2])
se_trans = np.concatenate((np.sqrt(
np.diag(results.cov_params().loc[['Intercept', 'group'],
['Intercept', 'group']])),
se_trans_only))
lower_CI_trans = coef_trans - zt * se_trans
upper_CI_trans = coef_trans + zt * se_trans
p_value_trans = 2 * stats.norm.cdf(-np.abs(coef_trans / se_trans))
statistics_trans = {
'parameters': trans_names,
'values': coef_trans,
'SE': se_trans,
'CI': (lower_CI_trans, upper_CI_trans),
'p-values': p_value_trans
}
diff_est_amp = coef_trans[3] - coef_trans[2]
idx = np.array([-1, 1, 0, 0])
diff_var_amp = np.dot(np.dot(idx, cov_trans),
np.transpose(idx[np.newaxis]))
glob_chi_amp = diff_est_amp * (1 / diff_var_amp) * diff_est_amp
ind_Z_amp = diff_est_amp / np.sqrt(np.diag(diff_var_amp))
interval_amp = np.array(
(diff_est_amp - 1.96 * np.sqrt(np.diag(diff_var_amp)),
diff_est_amp + 1.96 * np.sqrt(np.diag(diff_var_amp))))
df_amp = 1
global_p_value_amp = 1 - stats.chi2.cdf(glob_chi_amp, df_amp)
ind_p_value_amp = 2 * stats.norm.cdf(-abs(ind_Z_amp))
diff_est_acr = coef_trans[5] - coef_trans[4]
diff_est_acr = cosinor.project_acr(diff_est_acr)
idx = np.array([0, 0, -1, 1])
diff_var_acr = np.dot(np.dot(idx, cov_trans),
np.transpose(idx[np.newaxis]))
glob_chi_acr = diff_est_acr * (1 / diff_var_acr) * diff_est_acr
ind_Z_acr = diff_est_acr / np.sqrt(np.diag(diff_var_acr))
interval_acr = np.array(
(diff_est_acr - 1.96 * np.sqrt(np.diag(diff_var_acr)),
diff_est_acr + 1.96 * np.sqrt(np.diag(diff_var_acr))))
df_acr = 1
global_p_value_acr = 1 - stats.chi2.cdf(glob_chi_acr, df_acr)
ind_p_value_acr = 2 * stats.norm.cdf(-abs(ind_Z_acr))
global_test_amp = {
'name': 'global test of amplitude change',
'statistics': glob_chi_amp.flatten(),
'df': df_amp,
'p_value': global_p_value_amp.flatten()
}
ind_test_amp = {
'name': 'individual test of amplitude change',
'statistics': ind_Z_amp.flatten(),
'df': np.nan,
'value': diff_est_amp,
'conf_int': interval_amp.flatten(),
'p_value': ind_p_value_amp.flatten()
}
global_test_acr = {
'name': 'global test of acrophase shift',
'statistics': glob_chi_acr.flatten(),
'df': df_acr,
'p_value': global_p_value_acr.flatten()
}
ind_test_acr = {
'name': 'individual test of acrophase shift',
'statistics': ind_Z_acr.flatten(),
'df': np.nan,
'value': diff_est_acr,
'conf_int': interval_acr.flatten(),
'p_value': ind_p_value_acr.flatten()
}
return results, statistics_raw, statistics_trans, global_test_amp, ind_test_amp, global_test_acr, ind_test_acr