def population_test_cosinor_pairs(df, pairs, period = 24): df_res = pd.DataFrame(columns = ['test', 'd_amplitude', 'p(d_amplitude)', 'q(d_amplitude)', 'd_acrophase', 'p(d_acrophase)', 'q(d_acrophase)'], dtype=float) for pair in pairs: df_pop1 = df[df.test.str.startswith(pair[0])] df_pop2 = df[df.test.str.startswith(pair[1])] res1 = population_fit_cosinor(df_pop1, period = period, plot_on = False) res2 = population_fit_cosinor(df_pop2, period = period, plot_on = False) res = population_test_cosinor(res1, res2) d = {'test': res['test'], 'd_amplitude':res['amp']['pop2'] - res['amp']['pop1'], 'p(d_amplitude)':res['amp']['p_value'], 'd_acrophase1':cosinor.project_acr(res['acr']['pop2']-res['acr']['pop1']), 'p(d_acrophase)':res['acr']['p_value']} df_res = df_res.append(d, ignore_index=True) df_res['q(d_amplitude)'] = multi.multipletests(df_res['p(d_amplitude)'], method = 'fdr_bh')[1] df_res['q(d_acrophase)'] = multi.multipletests(df_res['p(d_acrophase)'], method = 'fdr_bh')[1] return df_res
def test_cosinor_single(data, period = 24, corrected = True): rrr = np.cos(2*np.pi*data.x/period) sss = np.sin(2*np.pi*data.x/period) data['rrr'] = rrr data['sss'] = sss results = smf.ols('y ~ rrr + sss', data).fit() beta_s = results.params['sss'] beta_r = results.params['rrr'] amp, acr = amp_acr(beta_s, beta_r) # project acropahse to interval -pi,pi acr = cosinor.project_acr(acr) vmat = results.cov_params().loc[['rrr', 'sss'], ['rrr', 'sss']] indVmat = vmat a_r = (beta_r**2 + beta_s**2)**(-0.5) * beta_r a_s = (beta_r**2 + beta_s**2)**(-0.5) * beta_s b_r = (1 / (1 + (beta_s**2 / beta_r**2))) * (-beta_s / beta_r**2) b_s = (1 / (1 + (beta_s**2 / beta_r**2))) * (1 / beta_r) if corrected: b_r = -b_r b_s = -b_s jac = np.array([[a_r, a_s], [b_r, b_s]]) cov_trans = np.dot(np.dot(jac, indVmat), np.transpose(jac)) se_trans_only =np.sqrt(np.diag(cov_trans)) zt = abs(stats.norm.ppf((1-0.95)/2)) trans_names = [results.params.index.values[0]] + ['amp', 'acr'] coef_trans = np.array([results.params.iloc[0], amp, acr]) se_trans = np.concatenate((np.sqrt(np.diag(results.cov_params().loc[['Intercept'], ['Intercept']])),se_trans_only)) lower_CI_trans = coef_trans - np.abs(zt * se_trans) upper_CI_trans = coef_trans + np.abs(zt * se_trans) p_value_trans = 2 * stats.norm.cdf(-np.abs(coef_trans/se_trans)) statistics= {'parameters': trans_names, 'values': coef_trans, 'SE': se_trans, 'CI': (lower_CI_trans, upper_CI_trans), 'p-values': p_value_trans, 'F-test': results.f_pvalue} return results, amp, acr, statistics
def test_cosinor_pairs_independent(df, pairs, period=24, period2=None, df_best_models=-1): period1 = period if not period2: period2 = period df_results = pd.DataFrame(columns=[ 'test', 'p1', 'q1', 'p2', 'q2', 'period1', 'period2', 'amplitude1', 'amplitude2', 'd_amplitude', 'p(d_amplitude)', 'q(d_amplitude)', 'CI(d_amplitude)', 'acrophase1', 'acrophase2', 'd_acrophase', 'p(d_acrophase)', 'q(d_acrophase)', 'CI(d_acrophase)' ], dtype=float) for test1, test2 in pairs: X1, Y1 = np.array(df[df.test == test1].x), np.array( df[df.test == test1].y) X2, Y2 = np.array(df[df.test == test2].x), np.array( df[df.test == test2].y) if type(df_best_models) == int: pass else: period1 = df_best_models[df_best_models.test == test1].period.iloc[0] period2 = df_best_models[df_best_models.test == test2].period.iloc[0] fit_results1, amp1, acr1, stats1 = fit_cosinor(X1, Y1, period=period1, plot_on=False) fit_results2, amp2, acr2, stats2 = fit_cosinor(X2, Y2, period=period2, plot_on=False) k = len(X1) + len(X2) k1 = len(X1) k2 = len(X2) DoF = k - (len(fit_results1.params) + len(fit_results2.params)) DoF1 = k1 - len(fit_results1.params) DoF2 = k2 - len(fit_results2.params) t = abs(stats.t.ppf(0.05 / 2, df=DoF)) d_amp = amp2 - amp1 d_acr = cosinor.project_acr(acr2 - acr1) CI_amp1 = stats1['CI'][0][-2], stats1['CI'][1][-2] CI_amp2 = stats2['CI'][0][-2], stats2['CI'][1][-2] se_amp = cosinor.get_se_diff_from_CIs(CI_amp1, CI_amp2, DoF1, DoF2, t_test=True, angular=False, CI_type="se", n1=k1, n2=k2, DoF=DoF, biased=True) dev_amp = t * se_amp T0 = d_amp / se_amp p_val_amp = 2 * (1 - stats.t.cdf(abs(T0), DoF)) CI_amp = [d_amp - dev_amp, d_amp + dev_amp] CI_acr1 = stats1['CI'][0][-1], stats1['CI'][1][-1] CI_acr2 = stats2['CI'][0][-1], stats2['CI'][1][-1] se_acr = cosinor.get_se_diff_from_CIs(CI_acr1, CI_acr2, DoF1, DoF2, t_test=True, angular=True, CI_type="se", n1=k1, n2=k2, DoF=DoF, biased=True) dev_acr = se_acr * t T0 = d_acr / se_acr p_val_acr = 2 * (1 - stats.t.cdf(abs(T0), DoF)) CI_acr = [d_acr - dev_acr, d_acr + dev_acr] d = { 'test': test1 + ' vs. ' + test2, 'p1': fit_results1.f_pvalue, 'p2': fit_results2.f_pvalue, 'period1': period1, 'period2': period2, 'amplitude1': amp1, 'amplitude2': amp2, 'd_amplitude': d_amp, 'p(d_amplitude)': p_val_amp, 'CI(d_amplitude)': CI_amp, 'acrophase1': acr1, 'acrophase2': acr2, 'd_acrophase': d_acr, 'p(d_acrophase)': p_val_acr, 'CI(d_acrophase)': CI_acr } df_results = df_results.append(d, ignore_index=True) df_results['q1'] = multi.multipletests(df_results['p1'], method='fdr_bh')[1] df_results['q2'] = multi.multipletests(df_results['p2'], method='fdr_bh')[1] df_results['q(d_amplitude)'] = multi.multipletests( df_results['p(d_amplitude)'], method='fdr_bh')[1] df_results['q(d_acrophase)'] = multi.multipletests( df_results['p(d_acrophase)'], method='fdr_bh')[1] return df_results
def population_test_cosinor_pairs_independent(df, pairs, period=24, period2=None): period1 = period if not period2: period2 = period df_res = pd.DataFrame(columns=[ 'test', 'amplitude1', 'amplitude2', 'd_amplitude', 'p(d_amplitude)', 'q(d_amplitude)', 'CI(d_amplitude)', 'acrophase1', 'acrophase2', 'd_acrophase', 'p(d_acrophase)', 'q(d_acrophase)', 'CI(d_acrophase)' ], dtype=float) for pair in pairs: df_pop1 = df[df.test.str.startswith(pair[0])] df_pop2 = df[df.test.str.startswith(pair[1])] k1 = len(df_pop1.test.unique()) k2 = len(df_pop2.test.unique()) DoF1 = k1 - 1 DoF2 = k2 - 1 DoF = k1 + k2 - 2 t = abs(stats.t.ppf(0.05 / 2, df=DoF)) res1 = population_fit_cosinor(df_pop1, period=period1, plot_on=False) res2 = population_fit_cosinor(df_pop2, period=period2, plot_on=False) #amplitude ('amp') param = 'amp' amp1, amp2 = res1['means'][-2], res2['means'][-2] CI1 = res1['confint'][param][0], res1['confint'][param][1] CI2 = res2['confint'][param][0], res2['confint'][param][1] se = cosinor.get_se_diff_from_CIs(CI1, CI2, DoF1, DoF2, t_test=True, angular=False, CI_type="se", n1=k1, n2=k2, DoF=DoF, biased=True) dev = se * t d_amp = amp2 - amp1 CI_amp = [d_amp - dev, d_amp + dev] T0 = d_amp / se p_val_amp = 2 * (1 - stats.t.cdf(abs(T0), DoF)) #acrophase ('acr') param = 'acr' acr1, acr2 = res1['means'][-1], res2['means'][-1] CI1 = res1['confint'][param][0], res1['confint'][param][1] CI2 = res2['confint'][param][0], res2['confint'][param][1] se = cosinor.get_se_diff_from_CIs(CI1, CI2, DoF1, DoF2, t_test=True, angular=True, CI_type="se", n1=k1, n2=k2, DoF=DoF, biased=True) dev = se * t d_acr = cosinor.project_acr(acr2 - acr1) CI_acr = [d_acr - dev, d_acr + dev] T0 = d_acr / se p_val_acr = 2 * (1 - stats.t.cdf(abs(T0), DoF)) d = { 'test': pair[0] + ' vs ' + pair[1], 'amplitude1': amp1, 'amplitude2': amp2, 'd_amplitude': d_amp, 'p(d_amplitude)': p_val_amp, 'CI(d_amplitude)': CI_amp, 'acrophase1': acr1, 'acrophase2': acr2, 'd_acrophase': d_acr, 'p(d_acrophase)': p_val_acr, 'CI(d_acrophase)': CI_acr } df_res = df_res.append(d, ignore_index=True) df_res['q(d_amplitude)'] = multi.multipletests(df_res['p(d_amplitude)'], method='fdr_bh')[1] df_res['q(d_acrophase)'] = multi.multipletests(df_res['p(d_acrophase)'], method='fdr_bh')[1] return df_res
def test_cosinor_pair(data, period, corrected=True): rrr = np.cos(2 * np.pi * data.x / period) sss = np.sin(2 * np.pi * data.x / period) data['rrr'] = rrr data['sss'] = sss results = smf.ols('y ~ group + rrr + sss + group:rrr + group:sss', data).fit() beta_s = np.array([results.params['sss'], results.params['group:sss']]) beta_r = np.array([results.params['rrr'], results.params['group:rrr']]) groups_s = beta_s.copy() groups_s[1] += groups_s[0] groups_r = beta_r.copy() groups_r[1] += groups_r[0] amp, acr = amp_acr(groups_s, groups_r) vmat = results.cov_params().loc[['rrr', 'group:rrr', 'sss', 'group:sss'], ['rrr', 'group:rrr', 'sss', 'group:sss'], ] indexmat = np.eye(4) indexmat[1, 0] = 1 indexmat[3, 2] = 1 indVmat = np.dot(np.dot(indexmat, vmat), np.transpose(indexmat)) a_r = (groups_r**2 + groups_s**2)**(-0.5) * groups_r a_s = (groups_r**2 + groups_s**2)**(-0.5) * groups_s b_r = (1 / (1 + (groups_s**2 / groups_r**2))) * (-groups_s / groups_r**2) b_s = (1 / (1 + (groups_s**2 / groups_r**2))) * (1 / groups_r) if corrected: b_r = -b_r b_s = -b_s jac = np.array((np.concatenate((np.diag(a_r)[0, :], np.diag(a_s)[0, :])), np.concatenate((np.diag(a_r)[1, :], np.diag(a_s)[1, :])), np.concatenate((np.diag(b_r)[0, :], np.diag(b_s)[0, :])), np.concatenate((np.diag(b_r)[1, :], np.diag(b_s)[1, :])))) cov_trans = np.dot(np.dot(jac, indVmat), np.transpose(jac)) se_trans_only = np.sqrt(np.diag(cov_trans)) zt = abs(stats.norm.ppf((1 - 0.95) / 2)) coef_raw = results.params raw_names = list(coef_raw.index.values) coef_raw = coef_raw.values se_raw = np.sqrt(np.diag(results.cov_params())) lower_CI_raw = coef_raw - zt * se_raw upper_CI_raw = coef_raw + zt * se_raw p_value_raw = 2 * stats.norm.cdf(-np.abs(coef_raw / se_raw)) statistics_raw = { 'parameters': raw_names, 'values': coef_raw, 'SE': se_raw, 'CI': (lower_CI_raw, upper_CI_raw), 'p-values': p_value_raw } trans_names = list(results.params.index.values[:2]) + [ 'amp', 'group:amp', 'acr', 'group:acr' ] coef_trans = np.concatenate((np.array(results.params.iloc[0:2]), amp, acr)) # correct acropahse coef_trans[-1] = cosinor.project_acr(coef_trans[-1]) coef_trans[-2] = cosinor.project_acr(coef_trans[-2]) se_trans = np.concatenate((np.sqrt( np.diag(results.cov_params().loc[['Intercept', 'group'], ['Intercept', 'group']])), se_trans_only)) lower_CI_trans = coef_trans - zt * se_trans upper_CI_trans = coef_trans + zt * se_trans p_value_trans = 2 * stats.norm.cdf(-np.abs(coef_trans / se_trans)) statistics_trans = { 'parameters': trans_names, 'values': coef_trans, 'SE': se_trans, 'CI': (lower_CI_trans, upper_CI_trans), 'p-values': p_value_trans } diff_est_amp = coef_trans[3] - coef_trans[2] idx = np.array([-1, 1, 0, 0]) diff_var_amp = np.dot(np.dot(idx, cov_trans), np.transpose(idx[np.newaxis])) glob_chi_amp = diff_est_amp * (1 / diff_var_amp) * diff_est_amp ind_Z_amp = diff_est_amp / np.sqrt(np.diag(diff_var_amp)) interval_amp = np.array( (diff_est_amp - 1.96 * np.sqrt(np.diag(diff_var_amp)), diff_est_amp + 1.96 * np.sqrt(np.diag(diff_var_amp)))) df_amp = 1 global_p_value_amp = 1 - stats.chi2.cdf(glob_chi_amp, df_amp) ind_p_value_amp = 2 * stats.norm.cdf(-abs(ind_Z_amp)) diff_est_acr = coef_trans[5] - coef_trans[4] diff_est_acr = cosinor.project_acr(diff_est_acr) idx = np.array([0, 0, -1, 1]) diff_var_acr = np.dot(np.dot(idx, cov_trans), np.transpose(idx[np.newaxis])) glob_chi_acr = diff_est_acr * (1 / diff_var_acr) * diff_est_acr ind_Z_acr = diff_est_acr / np.sqrt(np.diag(diff_var_acr)) interval_acr = np.array( (diff_est_acr - 1.96 * np.sqrt(np.diag(diff_var_acr)), diff_est_acr + 1.96 * np.sqrt(np.diag(diff_var_acr)))) df_acr = 1 global_p_value_acr = 1 - stats.chi2.cdf(glob_chi_acr, df_acr) ind_p_value_acr = 2 * stats.norm.cdf(-abs(ind_Z_acr)) global_test_amp = { 'name': 'global test of amplitude change', 'statistics': glob_chi_amp.flatten(), 'df': df_amp, 'p_value': global_p_value_amp.flatten() } ind_test_amp = { 'name': 'individual test of amplitude change', 'statistics': ind_Z_amp.flatten(), 'df': np.nan, 'value': diff_est_amp, 'conf_int': interval_amp.flatten(), 'p_value': ind_p_value_amp.flatten() } global_test_acr = { 'name': 'global test of acrophase shift', 'statistics': glob_chi_acr.flatten(), 'df': df_acr, 'p_value': global_p_value_acr.flatten() } ind_test_acr = { 'name': 'individual test of acrophase shift', 'statistics': ind_Z_acr.flatten(), 'df': np.nan, 'value': diff_est_acr, 'conf_int': interval_acr.flatten(), 'p_value': ind_p_value_acr.flatten() } return results, statistics_raw, statistics_trans, global_test_amp, ind_test_amp, global_test_acr, ind_test_acr