def test_correct_resample_interpolate(self): (res, freq) = dt_freq(self.lmp.df_adjust.index) assert (res, freq) == (1., 'D') assert self.can_adjusted.index.size == self.values_to_adjust.index.size # candidate before correction np.testing.assert_equal(np.unique(self.values_to_adjust.values), np.array([10., 11.])) # candidate after correction np.testing.assert_equal(np.unique(self.can_adjusted.values), np.array([50., 51.])) corrections_interpolated = self.lmp.adjust_obj.adjustments # interpolated M to D assert corrections_interpolated.index.size == self.values_to_adjust.index.size assert all(corrections_interpolated == 40.) # 50-40 resp. 51-11 m0 = self.lmp.get_model_params(0) m1 = self.lmp.get_model_params(1) np.testing.assert_almost_equal(m0['slope'], 1.0) np.testing.assert_almost_equal(m0['inter'], -20.10928961) np.testing.assert_almost_equal(m1['slope'], 1.0) np.testing.assert_almost_equal(m1['inter'], 19.8907103) # also get corrections from interpolated M values - must be same for the # synthetic case. assert np.alltrue( np.equal(self.can_adjusted_direct.values, self.can_adjusted.values)) assert np.alltrue( np.equal(self.can_adjusted_nocore.values, self.can_adjusted.values))
def test_correct_resample_interpolate(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ (res, freq) = dt_freq(self.hom.ref_regress.df_model.index) assert (res, freq) == (1., 'M') assert self.can_adjusted.index.size == self.values_to_adjust.index.size corrections = self.hom.adjust_obj.adjustments # interpolated M to D assert all(self.can_adjusted == (self.values_to_adjust - corrections)) # todo: this should be + plot_corrections = self.hom.plot_adjustments() model = self.hom.get_model_params() # the 1 model that counts assert model['poly_order'] == 2 np.testing.assert_almost_equal(model['coef_0'], 1.1555392089) np.testing.assert_almost_equal(model['coef_1'], -0.010662915) np.testing.assert_almost_equal(model['inter'], 0.22256330678) np.testing.assert_almost_equal(model['r2'], 0.68610786923) np.testing.assert_almost_equal(model['n_input'], 51) np.testing.assert_almost_equal(model['filter_p'], 5.)
def test_correct_no_bins(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ (res, freq) = dt_freq(self.hom.ref_regress.df_model.index) assert (res, freq) == (1., 'D') assert self.can_adjusted.index.size == self.values_to_adjust.index.size corrections = self.hom.adjust_obj.adjustments # interpolated M to D assert all(self.can_adjusted == (self.values_to_adjust - corrections)) # todo: this should be + plot_corrections = self.hom.plot_adjustments() model = self.hom.ref_regress.get_model_params() assert model['poly_order'] == 2 np.testing.assert_almost_equal(model['coef_0'], 0.2889355292) np.testing.assert_almost_equal(model['coef_1'], 0.0240520552301) np.testing.assert_almost_equal(model['inter'], 5.0994331) np.testing.assert_almost_equal(model['r2'], 0.4534750300) np.testing.assert_almost_equal(model['n_input'], 1160) np.testing.assert_almost_equal(model['filter_p'], 5.)
def setUpClass(cls): ts = read_test_data(707393) ts_full = ts.rename(columns={'CCI_44_COMBINED': 'candidate', 'MERRA2': 'reference'}).loc['2007-01-01':].copy(True) ts_full['candidate_original'] = ts_full['candidate'] # keep original # introduce some breaks to correct cls.breaktimes = np.array([datetime(2012,7,1), datetime(2010,1,15)]) can_biased = ts_full.loc[:, 'candidate'].copy(True) can_biased[cls.breaktimes[0]:] += 0.1 # first break can_biased.loc[:cls.breaktimes[1]] -= 0.1 # second break ts_full.loc[:, 'candidate'] = can_biased ts_full['flags'] = 0. # all are good in the example test_kwargs = dict([('test_resample', ('M', 0.3)), ('mean_test', 'wilkoxon'), ('var_test', 'scipy_fligner_killeen'), ('alpha', 0.01), ('test_check_min_data', 5), ('test_check_spearR_sig', [0., 1.])]) adjmodel_kwargs = dict([('regress_resample', ('M', 0.3)), ('filter', None), ('poly_orders', [1, 2]), ('select_by', 'R'), ('cdf_types', None)]) adjfct_kwargs = {'alpha': 0.4, 'use_separate_cdf': False, 'from_bins': False} adjcheck_kwargs = {'adjust_check_fix_temp_coverage': False, 'adjust_check_min_group_range': 365, 'adjust_check_pearsR_sig': (0., 1.)} cls.ts_full = ts_full.copy(True) cls.src = TsRelMultiBreak(candidate=cls.ts_full['candidate'], reference=cls.ts_full['reference'], breaktimes=cls.breaktimes, adjustment_method='HOM', candidate_flags=(cls.ts_full['flags'], [0]), full_period_bias_corr_method='cdf_match', sub_period_bias_corr_method='linreg', base_breaktime=None, HSP_init_breaktest=True, models_from_hsp=True, adjust_within='breaks', input_resolution='D', test_kwargs=test_kwargs, adjmodel_kwargs=adjmodel_kwargs, adjcheck_kwargs=adjcheck_kwargs, create_model_plots=True, frame_ts_figure=True, frame_tsstats_plots=True) (res, freq) = dt_freq(cls.src.df_original.index) assert (res, freq) == (1., 'D') cls.candidate_adjusted = cls.src.adjust_all(**adjfct_kwargs) assert cls.src.candidate_has_changed()
def setUp(self): (res, freq) = dt_freq(self.qcm.df_original.index) assert (res, freq) == (1., 'D') self.values_to_adjust = self.ts_full.loc[ datetime(2000, 1, 1):self.qcm.breaktime, 'can'].dropna() self.can_adjusted = self.qcm.adjust(self.values_to_adjust, interpolation_method='cubic')
def setUp(self): (res, freq) = dt_freq(self.hom.df_original.index) assert (res, freq) == (1., 'D') self.values_to_adjust = self.ts_full.loc[ datetime(2000, 1, 1):self.hom.breaktime, 'can'].dropna() # correction from core has only impact if values to adjust are not the same # as used to create the first model self.can_adjusted = self.hom.adjust( self.values_to_adjust, use_separate_cdf=False, alpha=0.6, from_bins=False)
def test_correct_no_bins(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ (res, freq) = dt_freq(self.hom.ref_regress.df_model.index) assert (res, freq) == (1., 'D') assert self.can_adjusted.index.size == self.values_to_adjust.index.size corrections = self.hom.adjust_obj.adjustments # interpolated M to D # todo: this should be + assert all(self.can_adjusted == (self.values_to_adjust - corrections)) model = self.hom.ref_regress.get_model_params() assert model['poly_order'] == 2
def setUpClass(cls): ts = read_test_data(707393) ts_full = ts.rename(columns={'CCI_44_COMBINED': 'candidate', 'MERRA2': 'reference'}).loc['2007-01-01':].copy(True) ts_full['candidate_original'] = ts_full['candidate'] # keep original # introduce some breaks to correct cls.breaktimes = np.array([datetime(2012,7,1), datetime(2010,1,15)]) can_biased = ts_full.loc[:, 'candidate'].copy(True) can_biased[cls.breaktimes[0]:] += 0.1 # first break can_biased.loc[:cls.breaktimes[1]] -= 0.1 # second break ts_full.loc[:, 'candidate'] = can_biased ts_full['flags'] = 0. # all are good in the example test_kwargs = dict([('test_resample', ('M', 0.3)), ('mean_test', 'wilkoxon'), ('var_test', 'scipy_fligner_killeen'), ('alpha', 0.01), ('test_check_min_data', 5), ('test_check_spearR_sig', [0., 1.])]) cls.ts_full = ts_full.copy(True) cls.src = TsRelMultiBreak(candidate=cls.ts_full['candidate'], reference=cls.ts_full['reference'], breaktimes=cls.breaktimes, adjustment_method=None, candidate_flags=(cls.ts_full['flags'], [0]), full_period_bias_corr_method='cdf_match', sub_period_bias_corr_method='linreg', base_breaktime=None, HSP_init_breaktest=False, models_from_hsp=False, adjust_within='frames', input_resolution='D', test_kwargs=test_kwargs, adjmodel_kwargs={}, adjcheck_kwargs={}, create_model_plots=False, frame_ts_figure=False, frame_tsstats_plots=False) (res, freq) = dt_freq(cls.src.df_original.index) assert (res, freq) == (1., 'D') cls.src.test_all() assert cls.src.candidate_has_changed() == False
def test_correct_direct(self): # do not resample for the models (res, freq) = dt_freq(self.lmp.df_adjust.index) assert (res, freq) == (1., 'D') assert self.can_adjusted_noresample.index.size == self.values_to_adjust.index.size corrections_interpolated = self.lmp.adjust_obj.adjustments # D, not interpolated assert corrections_interpolated.index.size == 366. m0 = self.lmp.get_model_params( 0) # models stay the same as for the previous test m1 = self.lmp.get_model_params(1) np.testing.assert_almost_equal(m0['slope'], 0.79388864747) np.testing.assert_almost_equal(m0['inter'], 2.126399268) np.testing.assert_almost_equal(m1['slope'], 1.009376751) np.testing.assert_almost_equal(m1['inter'], -0.1921845042)
def test_correct_direct(self): # do not resample for the models (res, freq) = dt_freq(self.lmp.df_adjust.index) assert (res, freq) == (1., 'M') values_to_adjust = self.ts_full.loc[ datetime(2000, 1, 1):self.lmp.breaktime, 'can'] assert self.can_adjusted_noresample.index.size == values_to_adjust.index.size corrections_interpolated = self.lmp.adjust_obj.adjustments # D, not interpolated assert corrections_interpolated.index.size == 366. m0 = self.lmp.get_model_params( 0) # models stay the same as for the previous test m1 = self.lmp.get_model_params(1) np.testing.assert_almost_equal(m0['slope'], 0.867307299) np.testing.assert_almost_equal(m0['inter'], 1.12528662) np.testing.assert_almost_equal(m1['slope'], 0.9175243965) np.testing.assert_almost_equal(m1['inter'], 1.392644610)
def test_correct_resample_interpolate(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ (res, freq) = dt_freq(self.lmp.df_adjust.index) assert (res, freq) == (1., 'M') assert self.can_adjusted.index.size == self.values_to_adjust.index.size corrections_interpolated = self.lmp.adjust_obj.adjustments # interpolated M to D assert corrections_interpolated.index.size == 366. m0 = self.lmp.get_model_params(0) m1 = self.lmp.get_model_params(1) np.testing.assert_almost_equal(m0['slope'], 0.867307299) np.testing.assert_almost_equal(m0['inter'], 1.12528662) np.testing.assert_almost_equal(m1['slope'], 0.9175243965) np.testing.assert_almost_equal(m1['inter'], 1.392644610)
def setUp(self): (res, freq) = dt_freq(self.lmp.df_original.index) assert (res, freq) == (1., 'D') self.values_to_adjust = self.ts_full.loc[ datetime(2000, 1, 1):self.lmp.breaktime, 'can'] # correction from core has only impact if values to adjust are not the same # as used to create the first model self.can_adjusted = self.lmp.adjust( self.values_to_adjust, corrections_from_core=True, resample_corrections=True, interpolation_method='linear', values_to_adjust_freq='D') # interpolation from M to D self.can_adjusted_noresample = self.lmp.adjust( self.values_to_adjust, corrections_from_core=True, resample_corrections=False, interpolation_method='linear', values_to_adjust_freq='D')
def test_correct_resample_interpolate(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ (res, freq) = dt_freq(self.lmp.df_adjust.index) assert (res, freq) == (1., 'D') assert self.can_adjusted.index.size == self.values_to_adjust.index.size corrections_interpolated = self.lmp.adjust_obj.adjustments # interpolated M to D assert corrections_interpolated.index.size == 366. m0 = self.lmp.get_model_params(0) m1 = self.lmp.get_model_params(1) np.testing.assert_almost_equal(m0['slope'], 0.79388864747) np.testing.assert_almost_equal(m0['inter'], 2.126399268) np.testing.assert_almost_equal(m1['slope'], 1.009376751) np.testing.assert_almost_equal(m1['inter'], -0.1921845042)
def setUp(self): (res, freq) = dt_freq(self.src.df_original.index) assert (res, freq) == (1., 'D') self.src.test_and_adjust()
def test_adjusted_data(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ testresults_ifirst, models_ifirst, testresults_ilast, \ models_ilast, group_stats, group_metrics, metrics_change, \ checkstats = self.src.get_results() assert self.src.testresult['mean']['stats'][ 'zval'] == testresults_ilast['zval_MEAN'] (res, freq) = dt_freq(self.src.adjust_obj.df_original.index) assert (res, freq) == (1., 'D') # before correction assert testresults_ifirst['h_MEAN'] == 1. np.testing.assert_almost_equal(testresults_ifirst['zval_MEAN'], -7.5498487236321) np.testing.assert_almost_equal(testresults_ifirst['pval_MEAN'], 0.) assert testresults_ifirst['h_VAR'] == 0. np.testing.assert_almost_equal(testresults_ifirst['z_VAR'], 0.012002612) np.testing.assert_almost_equal(testresults_ifirst['pval_VAR'], 0.9127611636) assert testresults_ifirst['error_code_test'] == 0. assert testresults_ifirst['n0'] == 30 assert testresults_ifirst['n1'] == 72 np.testing.assert_almost_equal(testresults_ifirst['frame_spearmanR'], 0.58956409632967) np.testing.assert_almost_equal(testresults_ifirst['frame_corrPval'], 0.) # after correction assert testresults_ilast['h_MEAN'] == 0. # break removed assert testresults_ilast['h_VAR'] == testresults_ifirst['h_VAR'] assert testresults_ilast['error_code_test'] == testresults_ifirst[ 'error_code_test'] assert testresults_ilast['n0'] == testresults_ifirst['n0'] assert testresults_ilast['n1'] == testresults_ifirst['n1'] # there was only one iteration, therefore first == last for k, v in models_ifirst['model0'].items(): np.testing.assert_almost_equal(models_ilast['model0'][k], v) for k, v in models_ifirst['model1'].items(): np.testing.assert_almost_equal(models_ilast['model1'][k], v) m0 = models_ifirst['model0'] # model parameters np.testing.assert_almost_equal(m0['n_quantiles'], 4) np.testing.assert_almost_equal(m0[0.125], -0.042636302715681) np.testing.assert_almost_equal(m0[1.0], 0.196129642521334) m1 = models_ifirst['model1'] np.testing.assert_almost_equal(m1['n_quantiles'], 4) np.testing.assert_almost_equal(m1[0.125], 0.059842589741188) np.testing.assert_almost_equal(m1[1.0], 0.30814550359433) assert compare_stats(group_stats, group_metrics, metrics_change) assert compare_metrics(group_stats, group_metrics, metrics_change) # some checkstats np.testing.assert_almost_equal( checkstats['n0'], testresults_ifirst['n0']) # both resampled np.testing.assert_almost_equal( checkstats['n1'], testresults_ifirst['n1']) # both resampled assert checkstats['error_code_adjust'] == 0 assert checkstats['error_code_test'] == 0 assert checkstats['THRES_R_pearson'] == 0.5
def test_adjusted_data(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ testresults_ifirst, models_ifirst, testresults_ilast, \ models_ilast, group_stats, group_metrics, metrics_change, \ checkstats = self.src.get_results() assert self.src.testresult['mean']['stats'][ 'zval'] == testresults_ilast['zval_MEAN'] (res, freq) = dt_freq(self.src.adjust_obj.df_original.index) assert (res, freq) == (1., 'D') # before correction assert testresults_ifirst['h_MEAN'] == 1. np.testing.assert_almost_equal(testresults_ifirst['zval_MEAN'], -7.5498487236321) np.testing.assert_almost_equal(testresults_ifirst['pval_MEAN'], 0.) assert testresults_ifirst['h_VAR'] == 0. np.testing.assert_almost_equal(testresults_ifirst['z_VAR'], 0.012002612) np.testing.assert_almost_equal(testresults_ifirst['pval_VAR'], 0.9127611636) assert testresults_ifirst['error_code_test'] == 0. assert testresults_ifirst['n0'] == 30 assert testresults_ifirst['n1'] == 72 np.testing.assert_almost_equal(testresults_ifirst['frame_spearmanR'], 0.58956409632967) np.testing.assert_almost_equal(testresults_ifirst['frame_corrPval'], 0.) # after correction assert testresults_ilast['h_MEAN'] == 0. # break removed assert testresults_ilast['h_VAR'] == testresults_ifirst['h_VAR'] assert testresults_ilast['error_code_test'] == testresults_ifirst[ 'error_code_test'] assert testresults_ilast['n0'] == testresults_ifirst['n0'] assert testresults_ilast['n1'] == testresults_ifirst['n1'] # there was only one iteration, therefore first == last for k, v in models_ifirst.items(): np.testing.assert_almost_equal(models_ilast[k], v) # model parameters np.testing.assert_almost_equal(models_ifirst['poly_order'], 2) np.testing.assert_almost_equal(models_ifirst['coef_0'], 1.23908388632) np.testing.assert_almost_equal(models_ifirst['coef_1'], -1.13139529834) np.testing.assert_almost_equal(models_ifirst['inter'], 0.1198528480961279) np.testing.assert_almost_equal(models_ifirst['r2'], 0.5432003747255678) np.testing.assert_almost_equal(models_ifirst['filter_p'], np.nan) np.testing.assert_almost_equal(models_ifirst['sse'], 2.211062267887628) np.testing.assert_almost_equal(models_ifirst['n_input'], 1542) # both resampled # some group stats assert compare_stats(group_stats, group_metrics, metrics_change) assert compare_metrics(group_stats, group_metrics, metrics_change) # some checkstats np.testing.assert_almost_equal( checkstats['n0'], testresults_ifirst['n0']) # both resampled np.testing.assert_almost_equal( checkstats['n1'], testresults_ifirst['n1']) # both resampled assert checkstats['error_code_adjust'] == 0 assert checkstats['error_code_test'] == 0 assert checkstats['THRES_R_pearson'] == 0.5
def test_adjusted_data(self): """ The model is calculated from daily values. Corrections are derived for monthly resampled values and then interpolated to the target daily resolution of the values to adjust. """ testresults_ifirst, models_ifirst, testresults_ilast, \ models_ilast, group_stats, group_metrics, metrics_change, \ checkstats = self.src.get_results() assert self.src.testresult['mean']['stats'][ 'zval'] == testresults_ilast['zval_MEAN'] (res, freq) = dt_freq(self.src.adjust_obj.df_adjust.index) assert (res, freq) == (1., 'M') # before correction assert testresults_ifirst['h_MEAN'] == 1. np.testing.assert_almost_equal(testresults_ifirst['zval_MEAN'], -7.5498487236321) np.testing.assert_almost_equal(testresults_ifirst['pval_MEAN'], 0.) assert testresults_ifirst['h_VAR'] == 0. np.testing.assert_almost_equal(testresults_ifirst['z_VAR'], 0.012002612) np.testing.assert_almost_equal(testresults_ifirst['pval_VAR'], 0.9127611636) assert testresults_ifirst['error_code_test'] == 0. assert testresults_ifirst['n0'] == 30 assert testresults_ifirst['n1'] == 72 np.testing.assert_almost_equal(testresults_ifirst['frame_spearmanR'], 0.58956409632967) np.testing.assert_almost_equal(testresults_ifirst['frame_corrPval'], 0.) # after correction assert testresults_ilast['h_MEAN'] == 0. # break removed assert testresults_ilast['h_VAR'] == testresults_ifirst['h_VAR'] assert testresults_ilast['error_code_test'] == testresults_ifirst[ 'error_code_test'] assert testresults_ilast['n0'] == testresults_ifirst['n0'] assert testresults_ilast['n1'] == testresults_ifirst['n1'] # there was only one iteration, therefore first == last assert models_ifirst == models_ilast # model parameters m0 = models_ifirst['model0'] m1 = models_ifirst['model1'] np.testing.assert_almost_equal(m0['slope'], 0.49615028455) np.testing.assert_almost_equal(m0['inter'], 0.12478766038) np.testing.assert_almost_equal(m0['s02'], 0.0006490746439413) np.testing.assert_almost_equal(m0['std_error'], 0.0543443828232) np.testing.assert_almost_equal(m0['r_squared'], 0.748545649337) np.testing.assert_almost_equal(m0['p_value'], 0.) np.testing.assert_almost_equal(m0['sum_squared_residuals'], 0.0181740900303) np.testing.assert_almost_equal( m0['n_input'], testresults_ifirst['n0']) # both resampled np.testing.assert_almost_equal(m1['slope'], 0.45355634119) np.testing.assert_almost_equal(m1['inter'], 0.24941273901) np.testing.assert_almost_equal(m1['s02'], 0.000523507763) np.testing.assert_almost_equal(m1['std_error'], 0.0365909511) np.testing.assert_almost_equal(m1['r_squared'], 0.6870023018) np.testing.assert_almost_equal(m1['p_value'], 0.) np.testing.assert_almost_equal(m1['sum_squared_residuals'], 0.03664554343) np.testing.assert_almost_equal( m1['n_input'], testresults_ifirst['n1']) # both resampled # some group stats assert compare_stats(group_stats, group_metrics, metrics_change) assert compare_metrics(group_stats, group_metrics, metrics_change) # some checkstats np.testing.assert_almost_equal( checkstats['n0'], testresults_ifirst['n0']) # both resampled np.testing.assert_almost_equal( checkstats['n1'], testresults_ifirst['n1']) # both resampled assert checkstats['error_code_adjust'] == 0 assert checkstats['error_code_test'] == 0 assert checkstats['THRES_R_pearson'] == 0.5