def test_correct_resample_interpolate(self):
(res, freq) = dt_freq(self.lmp.df_adjust.index)
assert (res, freq) == (1., 'D')
assert self.can_adjusted.index.size == self.values_to_adjust.index.size
# candidate before correction
np.testing.assert_equal(np.unique(self.values_to_adjust.values),
np.array([10., 11.]))
# candidate after correction
np.testing.assert_equal(np.unique(self.can_adjusted.values),
np.array([50., 51.]))
corrections_interpolated = self.lmp.adjust_obj.adjustments # interpolated M to D
assert corrections_interpolated.index.size == self.values_to_adjust.index.size
assert all(corrections_interpolated == 40.) # 50-40 resp. 51-11
m0 = self.lmp.get_model_params(0)
m1 = self.lmp.get_model_params(1)
np.testing.assert_almost_equal(m0['slope'], 1.0)
np.testing.assert_almost_equal(m0['inter'], -20.10928961)
np.testing.assert_almost_equal(m1['slope'], 1.0)
np.testing.assert_almost_equal(m1['inter'], 19.8907103)
# also get corrections from interpolated M values - must be same for the
# synthetic case.
assert np.alltrue(
np.equal(self.can_adjusted_direct.values,
self.can_adjusted.values))
assert np.alltrue(
np.equal(self.can_adjusted_nocore.values,
self.can_adjusted.values))
def test_correct_resample_interpolate(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
(res, freq) = dt_freq(self.hom.ref_regress.df_model.index)
assert (res, freq) == (1., 'M')
assert self.can_adjusted.index.size == self.values_to_adjust.index.size
corrections = self.hom.adjust_obj.adjustments # interpolated M to D
assert all(self.can_adjusted == (self.values_to_adjust - corrections)) # todo: this should be +
plot_corrections = self.hom.plot_adjustments()
model = self.hom.get_model_params() # the 1 model that counts
assert model['poly_order'] == 2
np.testing.assert_almost_equal(model['coef_0'], 1.1555392089)
np.testing.assert_almost_equal(model['coef_1'], -0.010662915)
np.testing.assert_almost_equal(model['inter'], 0.22256330678)
np.testing.assert_almost_equal(model['r2'], 0.68610786923)
np.testing.assert_almost_equal(model['n_input'], 51)
np.testing.assert_almost_equal(model['filter_p'], 5.)
def test_correct_no_bins(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
(res, freq) = dt_freq(self.hom.ref_regress.df_model.index)
assert (res, freq) == (1., 'D')
assert self.can_adjusted.index.size == self.values_to_adjust.index.size
corrections = self.hom.adjust_obj.adjustments # interpolated M to D
assert all(self.can_adjusted == (self.values_to_adjust - corrections)) # todo: this should be +
plot_corrections = self.hom.plot_adjustments()
model = self.hom.ref_regress.get_model_params()
assert model['poly_order'] == 2
np.testing.assert_almost_equal(model['coef_0'], 0.2889355292)
np.testing.assert_almost_equal(model['coef_1'], 0.0240520552301)
np.testing.assert_almost_equal(model['inter'], 5.0994331)
np.testing.assert_almost_equal(model['r2'], 0.4534750300)
np.testing.assert_almost_equal(model['n_input'], 1160)
np.testing.assert_almost_equal(model['filter_p'], 5.)
def setUpClass(cls):
ts = read_test_data(707393)
ts_full = ts.rename(columns={'CCI_44_COMBINED': 'candidate',
'MERRA2': 'reference'}).loc['2007-01-01':].copy(True)
ts_full['candidate_original'] = ts_full['candidate'] # keep original
# introduce some breaks to correct
cls.breaktimes = np.array([datetime(2012,7,1), datetime(2010,1,15)])
can_biased = ts_full.loc[:, 'candidate'].copy(True)
can_biased[cls.breaktimes[0]:] += 0.1 # first break
can_biased.loc[:cls.breaktimes[1]] -= 0.1 # second break
ts_full.loc[:, 'candidate'] = can_biased
ts_full['flags'] = 0. # all are good in the example
test_kwargs = dict([('test_resample', ('M', 0.3)),
('mean_test', 'wilkoxon'),
('var_test', 'scipy_fligner_killeen'),
('alpha', 0.01),
('test_check_min_data', 5),
('test_check_spearR_sig', [0., 1.])])
adjmodel_kwargs = dict([('regress_resample', ('M', 0.3)),
('filter', None),
('poly_orders', [1, 2]),
('select_by', 'R'),
('cdf_types', None)])
adjfct_kwargs = {'alpha': 0.4,
'use_separate_cdf': False,
'from_bins': False}
adjcheck_kwargs = {'adjust_check_fix_temp_coverage': False,
'adjust_check_min_group_range': 365,
'adjust_check_pearsR_sig': (0., 1.)}
cls.ts_full = ts_full.copy(True)
cls.src = TsRelMultiBreak(candidate=cls.ts_full['candidate'],
reference=cls.ts_full['reference'],
breaktimes=cls.breaktimes,
adjustment_method='HOM',
candidate_flags=(cls.ts_full['flags'], [0]),
full_period_bias_corr_method='cdf_match',
sub_period_bias_corr_method='linreg',
base_breaktime=None,
HSP_init_breaktest=True,
models_from_hsp=True,
adjust_within='breaks',
input_resolution='D',
test_kwargs=test_kwargs,
adjmodel_kwargs=adjmodel_kwargs,
adjcheck_kwargs=adjcheck_kwargs,
create_model_plots=True,
frame_ts_figure=True,
frame_tsstats_plots=True)
(res, freq) = dt_freq(cls.src.df_original.index)
assert (res, freq) == (1., 'D')
cls.candidate_adjusted = cls.src.adjust_all(**adjfct_kwargs)
assert cls.src.candidate_has_changed()
def setUp(self):
(res, freq) = dt_freq(self.qcm.df_original.index)
assert (res, freq) == (1., 'D')
self.values_to_adjust = self.ts_full.loc[
datetime(2000, 1, 1):self.qcm.breaktime, 'can'].dropna()
self.can_adjusted = self.qcm.adjust(self.values_to_adjust,
interpolation_method='cubic')
def setUp(self):
(res, freq) = dt_freq(self.hom.df_original.index)
assert (res, freq) == (1., 'D')
self.values_to_adjust = self.ts_full.loc[
datetime(2000, 1, 1):self.hom.breaktime, 'can'].dropna()
# correction from core has only impact if values to adjust are not the same
# as used to create the first model
self.can_adjusted = self.hom.adjust(
self.values_to_adjust, use_separate_cdf=False, alpha=0.6, from_bins=False)
def test_correct_no_bins(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
(res, freq) = dt_freq(self.hom.ref_regress.df_model.index)
assert (res, freq) == (1., 'D')
assert self.can_adjusted.index.size == self.values_to_adjust.index.size
corrections = self.hom.adjust_obj.adjustments # interpolated M to D
# todo: this should be +
assert all(self.can_adjusted == (self.values_to_adjust - corrections))
model = self.hom.ref_regress.get_model_params()
assert model['poly_order'] == 2
def setUpClass(cls):
ts = read_test_data(707393)
ts_full = ts.rename(columns={'CCI_44_COMBINED': 'candidate',
'MERRA2': 'reference'}).loc['2007-01-01':].copy(True)
ts_full['candidate_original'] = ts_full['candidate'] # keep original
# introduce some breaks to correct
cls.breaktimes = np.array([datetime(2012,7,1), datetime(2010,1,15)])
can_biased = ts_full.loc[:, 'candidate'].copy(True)
can_biased[cls.breaktimes[0]:] += 0.1 # first break
can_biased.loc[:cls.breaktimes[1]] -= 0.1 # second break
ts_full.loc[:, 'candidate'] = can_biased
ts_full['flags'] = 0. # all are good in the example
test_kwargs = dict([('test_resample', ('M', 0.3)),
('mean_test', 'wilkoxon'),
('var_test', 'scipy_fligner_killeen'),
('alpha', 0.01),
('test_check_min_data', 5),
('test_check_spearR_sig', [0., 1.])])
cls.ts_full = ts_full.copy(True)
cls.src = TsRelMultiBreak(candidate=cls.ts_full['candidate'],
reference=cls.ts_full['reference'],
breaktimes=cls.breaktimes,
adjustment_method=None,
candidate_flags=(cls.ts_full['flags'], [0]),
full_period_bias_corr_method='cdf_match',
sub_period_bias_corr_method='linreg',
base_breaktime=None,
HSP_init_breaktest=False,
models_from_hsp=False,
adjust_within='frames',
input_resolution='D',
test_kwargs=test_kwargs,
adjmodel_kwargs={},
adjcheck_kwargs={},
create_model_plots=False,
frame_ts_figure=False,
frame_tsstats_plots=False)
(res, freq) = dt_freq(cls.src.df_original.index)
assert (res, freq) == (1., 'D')
cls.src.test_all()
assert cls.src.candidate_has_changed() == False
def test_correct_direct(self):
# do not resample for the models
(res, freq) = dt_freq(self.lmp.df_adjust.index)
assert (res, freq) == (1., 'D')
assert self.can_adjusted_noresample.index.size == self.values_to_adjust.index.size
corrections_interpolated = self.lmp.adjust_obj.adjustments # D, not interpolated
assert corrections_interpolated.index.size == 366.
m0 = self.lmp.get_model_params(
0) # models stay the same as for the previous test
m1 = self.lmp.get_model_params(1)
np.testing.assert_almost_equal(m0['slope'], 0.79388864747)
np.testing.assert_almost_equal(m0['inter'], 2.126399268)
np.testing.assert_almost_equal(m1['slope'], 1.009376751)
np.testing.assert_almost_equal(m1['inter'], -0.1921845042)
def test_correct_direct(self):
# do not resample for the models
(res, freq) = dt_freq(self.lmp.df_adjust.index)
assert (res, freq) == (1., 'M')
values_to_adjust = self.ts_full.loc[
datetime(2000, 1, 1):self.lmp.breaktime, 'can']
assert self.can_adjusted_noresample.index.size == values_to_adjust.index.size
corrections_interpolated = self.lmp.adjust_obj.adjustments # D, not interpolated
assert corrections_interpolated.index.size == 366.
m0 = self.lmp.get_model_params(
0) # models stay the same as for the previous test
m1 = self.lmp.get_model_params(1)
np.testing.assert_almost_equal(m0['slope'], 0.867307299)
np.testing.assert_almost_equal(m0['inter'], 1.12528662)
np.testing.assert_almost_equal(m1['slope'], 0.9175243965)
np.testing.assert_almost_equal(m1['inter'], 1.392644610)
def test_correct_resample_interpolate(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
(res, freq) = dt_freq(self.lmp.df_adjust.index)
assert (res, freq) == (1., 'M')
assert self.can_adjusted.index.size == self.values_to_adjust.index.size
corrections_interpolated = self.lmp.adjust_obj.adjustments # interpolated M to D
assert corrections_interpolated.index.size == 366.
m0 = self.lmp.get_model_params(0)
m1 = self.lmp.get_model_params(1)
np.testing.assert_almost_equal(m0['slope'], 0.867307299)
np.testing.assert_almost_equal(m0['inter'], 1.12528662)
np.testing.assert_almost_equal(m1['slope'], 0.9175243965)
np.testing.assert_almost_equal(m1['inter'], 1.392644610)
def setUp(self):
(res, freq) = dt_freq(self.lmp.df_original.index)
assert (res, freq) == (1., 'D')
self.values_to_adjust = self.ts_full.loc[
datetime(2000, 1, 1):self.lmp.breaktime, 'can']
# correction from core has only impact if values to adjust are not the same
# as used to create the first model
self.can_adjusted = self.lmp.adjust(
self.values_to_adjust,
corrections_from_core=True,
resample_corrections=True,
interpolation_method='linear',
values_to_adjust_freq='D') # interpolation from M to D
self.can_adjusted_noresample = self.lmp.adjust(
self.values_to_adjust,
corrections_from_core=True,
resample_corrections=False,
interpolation_method='linear',
values_to_adjust_freq='D')
def test_correct_resample_interpolate(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
(res, freq) = dt_freq(self.lmp.df_adjust.index)
assert (res, freq) == (1., 'D')
assert self.can_adjusted.index.size == self.values_to_adjust.index.size
corrections_interpolated = self.lmp.adjust_obj.adjustments # interpolated M to D
assert corrections_interpolated.index.size == 366.
m0 = self.lmp.get_model_params(0)
m1 = self.lmp.get_model_params(1)
np.testing.assert_almost_equal(m0['slope'], 0.79388864747)
np.testing.assert_almost_equal(m0['inter'], 2.126399268)
np.testing.assert_almost_equal(m1['slope'], 1.009376751)
np.testing.assert_almost_equal(m1['inter'], -0.1921845042)
def setUp(self):
(res, freq) = dt_freq(self.src.df_original.index)
assert (res, freq) == (1., 'D')
self.src.test_and_adjust()
def test_adjusted_data(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
testresults_ifirst, models_ifirst, testresults_ilast, \
models_ilast, group_stats, group_metrics, metrics_change, \
checkstats = self.src.get_results()
assert self.src.testresult['mean']['stats'][
'zval'] == testresults_ilast['zval_MEAN']
(res, freq) = dt_freq(self.src.adjust_obj.df_original.index)
assert (res, freq) == (1., 'D')
# before correction
assert testresults_ifirst['h_MEAN'] == 1.
np.testing.assert_almost_equal(testresults_ifirst['zval_MEAN'],
-7.5498487236321)
np.testing.assert_almost_equal(testresults_ifirst['pval_MEAN'], 0.)
assert testresults_ifirst['h_VAR'] == 0.
np.testing.assert_almost_equal(testresults_ifirst['z_VAR'],
0.012002612)
np.testing.assert_almost_equal(testresults_ifirst['pval_VAR'],
0.9127611636)
assert testresults_ifirst['error_code_test'] == 0.
assert testresults_ifirst['n0'] == 30
assert testresults_ifirst['n1'] == 72
np.testing.assert_almost_equal(testresults_ifirst['frame_spearmanR'],
0.58956409632967)
np.testing.assert_almost_equal(testresults_ifirst['frame_corrPval'],
0.)
# after correction
assert testresults_ilast['h_MEAN'] == 0. # break removed
assert testresults_ilast['h_VAR'] == testresults_ifirst['h_VAR']
assert testresults_ilast['error_code_test'] == testresults_ifirst[
'error_code_test']
assert testresults_ilast['n0'] == testresults_ifirst['n0']
assert testresults_ilast['n1'] == testresults_ifirst['n1']
# there was only one iteration, therefore first == last
for k, v in models_ifirst['model0'].items():
np.testing.assert_almost_equal(models_ilast['model0'][k], v)
for k, v in models_ifirst['model1'].items():
np.testing.assert_almost_equal(models_ilast['model1'][k], v)
m0 = models_ifirst['model0']
# model parameters
np.testing.assert_almost_equal(m0['n_quantiles'], 4)
np.testing.assert_almost_equal(m0[0.125], -0.042636302715681)
np.testing.assert_almost_equal(m0[1.0], 0.196129642521334)
m1 = models_ifirst['model1']
np.testing.assert_almost_equal(m1['n_quantiles'], 4)
np.testing.assert_almost_equal(m1[0.125], 0.059842589741188)
np.testing.assert_almost_equal(m1[1.0], 0.30814550359433)
assert compare_stats(group_stats, group_metrics, metrics_change)
assert compare_metrics(group_stats, group_metrics, metrics_change)
# some checkstats
np.testing.assert_almost_equal(
checkstats['n0'], testresults_ifirst['n0']) # both resampled
np.testing.assert_almost_equal(
checkstats['n1'], testresults_ifirst['n1']) # both resampled
assert checkstats['error_code_adjust'] == 0
assert checkstats['error_code_test'] == 0
assert checkstats['THRES_R_pearson'] == 0.5
def test_adjusted_data(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
testresults_ifirst, models_ifirst, testresults_ilast, \
models_ilast, group_stats, group_metrics, metrics_change, \
checkstats = self.src.get_results()
assert self.src.testresult['mean']['stats'][
'zval'] == testresults_ilast['zval_MEAN']
(res, freq) = dt_freq(self.src.adjust_obj.df_original.index)
assert (res, freq) == (1., 'D')
# before correction
assert testresults_ifirst['h_MEAN'] == 1.
np.testing.assert_almost_equal(testresults_ifirst['zval_MEAN'],
-7.5498487236321)
np.testing.assert_almost_equal(testresults_ifirst['pval_MEAN'], 0.)
assert testresults_ifirst['h_VAR'] == 0.
np.testing.assert_almost_equal(testresults_ifirst['z_VAR'],
0.012002612)
np.testing.assert_almost_equal(testresults_ifirst['pval_VAR'],
0.9127611636)
assert testresults_ifirst['error_code_test'] == 0.
assert testresults_ifirst['n0'] == 30
assert testresults_ifirst['n1'] == 72
np.testing.assert_almost_equal(testresults_ifirst['frame_spearmanR'],
0.58956409632967)
np.testing.assert_almost_equal(testresults_ifirst['frame_corrPval'],
0.)
# after correction
assert testresults_ilast['h_MEAN'] == 0. # break removed
assert testresults_ilast['h_VAR'] == testresults_ifirst['h_VAR']
assert testresults_ilast['error_code_test'] == testresults_ifirst[
'error_code_test']
assert testresults_ilast['n0'] == testresults_ifirst['n0']
assert testresults_ilast['n1'] == testresults_ifirst['n1']
# there was only one iteration, therefore first == last
for k, v in models_ifirst.items():
np.testing.assert_almost_equal(models_ilast[k], v)
# model parameters
np.testing.assert_almost_equal(models_ifirst['poly_order'], 2)
np.testing.assert_almost_equal(models_ifirst['coef_0'], 1.23908388632)
np.testing.assert_almost_equal(models_ifirst['coef_1'], -1.13139529834)
np.testing.assert_almost_equal(models_ifirst['inter'],
0.1198528480961279)
np.testing.assert_almost_equal(models_ifirst['r2'], 0.5432003747255678)
np.testing.assert_almost_equal(models_ifirst['filter_p'], np.nan)
np.testing.assert_almost_equal(models_ifirst['sse'], 2.211062267887628)
np.testing.assert_almost_equal(models_ifirst['n_input'],
1542) # both resampled
# some group stats
assert compare_stats(group_stats, group_metrics, metrics_change)
assert compare_metrics(group_stats, group_metrics, metrics_change)
# some checkstats
np.testing.assert_almost_equal(
checkstats['n0'], testresults_ifirst['n0']) # both resampled
np.testing.assert_almost_equal(
checkstats['n1'], testresults_ifirst['n1']) # both resampled
assert checkstats['error_code_adjust'] == 0
assert checkstats['error_code_test'] == 0
assert checkstats['THRES_R_pearson'] == 0.5
def test_adjusted_data(self):
"""
The model is calculated from daily values.
Corrections are derived for monthly resampled values and then interpolated
to the target daily resolution of the values to adjust.
"""
testresults_ifirst, models_ifirst, testresults_ilast, \
models_ilast, group_stats, group_metrics, metrics_change, \
checkstats = self.src.get_results()
assert self.src.testresult['mean']['stats'][
'zval'] == testresults_ilast['zval_MEAN']
(res, freq) = dt_freq(self.src.adjust_obj.df_adjust.index)
assert (res, freq) == (1., 'M')
# before correction
assert testresults_ifirst['h_MEAN'] == 1.
np.testing.assert_almost_equal(testresults_ifirst['zval_MEAN'],
-7.5498487236321)
np.testing.assert_almost_equal(testresults_ifirst['pval_MEAN'], 0.)
assert testresults_ifirst['h_VAR'] == 0.
np.testing.assert_almost_equal(testresults_ifirst['z_VAR'],
0.012002612)
np.testing.assert_almost_equal(testresults_ifirst['pval_VAR'],
0.9127611636)
assert testresults_ifirst['error_code_test'] == 0.
assert testresults_ifirst['n0'] == 30
assert testresults_ifirst['n1'] == 72
np.testing.assert_almost_equal(testresults_ifirst['frame_spearmanR'],
0.58956409632967)
np.testing.assert_almost_equal(testresults_ifirst['frame_corrPval'],
0.)
# after correction
assert testresults_ilast['h_MEAN'] == 0. # break removed
assert testresults_ilast['h_VAR'] == testresults_ifirst['h_VAR']
assert testresults_ilast['error_code_test'] == testresults_ifirst[
'error_code_test']
assert testresults_ilast['n0'] == testresults_ifirst['n0']
assert testresults_ilast['n1'] == testresults_ifirst['n1']
# there was only one iteration, therefore first == last
assert models_ifirst == models_ilast
# model parameters
m0 = models_ifirst['model0']
m1 = models_ifirst['model1']
np.testing.assert_almost_equal(m0['slope'], 0.49615028455)
np.testing.assert_almost_equal(m0['inter'], 0.12478766038)
np.testing.assert_almost_equal(m0['s02'], 0.0006490746439413)
np.testing.assert_almost_equal(m0['std_error'], 0.0543443828232)
np.testing.assert_almost_equal(m0['r_squared'], 0.748545649337)
np.testing.assert_almost_equal(m0['p_value'], 0.)
np.testing.assert_almost_equal(m0['sum_squared_residuals'],
0.0181740900303)
np.testing.assert_almost_equal(
m0['n_input'], testresults_ifirst['n0']) # both resampled
np.testing.assert_almost_equal(m1['slope'], 0.45355634119)
np.testing.assert_almost_equal(m1['inter'], 0.24941273901)
np.testing.assert_almost_equal(m1['s02'], 0.000523507763)
np.testing.assert_almost_equal(m1['std_error'], 0.0365909511)
np.testing.assert_almost_equal(m1['r_squared'], 0.6870023018)
np.testing.assert_almost_equal(m1['p_value'], 0.)
np.testing.assert_almost_equal(m1['sum_squared_residuals'],
0.03664554343)
np.testing.assert_almost_equal(
m1['n_input'], testresults_ifirst['n1']) # both resampled
# some group stats
assert compare_stats(group_stats, group_metrics, metrics_change)
assert compare_metrics(group_stats, group_metrics, metrics_change)
# some checkstats
np.testing.assert_almost_equal(
checkstats['n0'], testresults_ifirst['n0']) # both resampled
np.testing.assert_almost_equal(
checkstats['n1'], testresults_ifirst['n1']) # both resampled
assert checkstats['error_code_adjust'] == 0
assert checkstats['error_code_test'] == 0
assert checkstats['THRES_R_pearson'] == 0.5
