def test_hamed_rao_modification_test(NoTrendData, TrendData,
arbitrary_1d_data):
# check with no trend data
NoTrendRes = mk.hamed_rao_modification_test(NoTrendData)
assert NoTrendRes.trend == 'no trend'
assert NoTrendRes.h == False
assert NoTrendRes.p == 1.0
assert NoTrendRes.z == 0
assert NoTrendRes.Tau == 0.0
assert NoTrendRes.s == 0.0
assert NoTrendRes.slope == 0.0
# check with trendy data
TrendRes = mk.hamed_rao_modification_test(TrendData)
assert TrendRes.trend == 'increasing'
assert TrendRes.h == True
assert TrendRes.p == 0.0
assert TrendRes.Tau == 1.0
assert TrendRes.s == 64620.0
np.testing.assert_allclose(TrendRes.slope, 1.0, rtol=1e-02)
# check with arbitrary data
result = mk.hamed_rao_modification_test(arbitrary_1d_data)
assert result.trend == 'decreasing'
assert result.h == True
assert result.p == 0.00011372459883540742
assert result.z == -3.859273515045842
assert result.Tau == -0.03153167653875869
assert result.s == -1959.0
assert result.var_s == 257403.38678462413
assert result.slope == -0.0064516129032258064
def test_hamed_rao_modification_test_lag3(NoTrendData, TrendData,
arbitrary_1d_data):
# check with no trend data
NoTrendRes = mk.hamed_rao_modification_test(NoTrendData, lag=3)
assert NoTrendRes.trend == 'no trend'
assert NoTrendRes.h == False
assert NoTrendRes.p == 1.0
assert NoTrendRes.z == 0
assert NoTrendRes.Tau == 0.0
assert NoTrendRes.s == 0.0
assert NoTrendRes.slope == 0.0
# check with trendy data
TrendRes = mk.hamed_rao_modification_test(TrendData, lag=3)
assert TrendRes.trend == 'increasing'
assert TrendRes.h == True
assert TrendRes.p == 0.0
assert TrendRes.Tau == 1.0
assert TrendRes.s == 64620.0
np.testing.assert_allclose(TrendRes.slope, 1.0, rtol=1e-02)
# check with arbitrary data
result = mk.hamed_rao_modification_test(arbitrary_1d_data, lag=3)
assert result.trend == 'no trend'
assert result.h == False
assert result.p == 0.603684460662274
assert result.z == -0.5191093899188985
assert result.Tau == -0.03153167653875869
assert result.s == -1959.0
assert result.var_s == 14226812.425138814
assert result.slope == -0.0064516129032258064
def test_hamed_rao_modification_test_lag3(NoTrendData, TrendData, arbitrary_1d_data):
# check with no trend data
NoTrendRes = mk.hamed_rao_modification_test(NoTrendData, lag=3)
assert NoTrendRes.trend == 'no trend'
assert NoTrendRes.h == False
assert NoTrendRes.p == 1.0
assert NoTrendRes.z == 0
assert NoTrendRes.Tau == 0.0
assert NoTrendRes.s == 0.0
# check with trendy data
TrendRes = mk.hamed_rao_modification_test(TrendData, lag=3)
assert TrendRes.trend == 'increasing'
assert TrendRes.h == True
assert TrendRes.p == 0.0
assert TrendRes.Tau == 1.0
assert TrendRes.s == 64620.0
# check with arbitrary data
result = mk.hamed_rao_modification_test(arbitrary_1d_data, lag=3)
assert result.trend == 'no trend'
assert result.h == False
assert result.p == 0.6037112685123898
assert result.z == -0.5190709455046154
assert result.Tau == -0.03153167653875869
assert result.s == -1959.0
assert result.var_s == 14228919.889368296
def test_hamed_rao_modification_test(NoTrendData, TrendData, arbitrary_1d_data):
# check with no trend data
NoTrendRes = mk.hamed_rao_modification_test(NoTrendData)
assert NoTrendRes.trend == 'no trend'
assert NoTrendRes.h == False
assert NoTrendRes.p == 1.0
assert NoTrendRes.z == 0
assert NoTrendRes.Tau == 0.0
assert NoTrendRes.s == 0.0
# check with trendy data
TrendRes = mk.hamed_rao_modification_test(TrendData)
assert TrendRes.trend == 'increasing'
assert TrendRes.h == True
assert TrendRes.p == 0.0
assert TrendRes.Tau == 1.0
assert TrendRes.s == 64620.0
# check with arbitrary data
result = mk.hamed_rao_modification_test(arbitrary_1d_data)
assert result.trend == 'decreasing'
assert result.h == True
assert result.p == 0.00012203829241275166
assert result.z == -3.8419950613710894
assert result.Tau == -0.03153167653875869
assert result.s == -1959.0
assert result.var_s == 259723.81316716125
def test_residuals(model, timeperiod, reg):
"""
Test for a residual trend, applying a Mann-Kendall-test
Parameters
----------
model : GLMObject
Best model
timeperiod : np.array
considered years (not used here)
Returns
-------
float
slope in residuals
float
p-value
"""
res_trend = mk.original_test(model.resid_response, alpha=0.1)
fig, ax = plt.subplots(figsize=(12, 8))
sm.graphics.tsa.plot_acf(model.resid_response, lags=39, ax=ax)
ax.set_xlabel('lag')
ax.set_title('Autocorrelation {}'.format(reg))
#fig.savefig('/home/insauer/projects/NC_Submission/Climada_papers/Test/AutocorrResidualsGMT_{}.png'.format(reg),bbox_inches = 'tight',dpi =600)
alt_trend_test = mk.hamed_rao_modification_test(model.resid_response)
return res_trend.slope, res_trend.p, alt_trend_test.trend, alt_trend_test.p
def test_residuals(model, timeperiod, reg):
"""
Test for a residual trend, applying a Mann-Kendall-test
Parameters
----------
model : GLMObject
Best model
timeperiod : np.array
considered years (not used here)
Returns
-------
float
slope in residuals
float
p-value
"""
res_trend = mk.original_test(model.resid_response, alpha=0.1)
fig, ax = plt.subplots(figsize=(12, 8))
alt_trend_test = mk.hamed_rao_modification_test(model.resid_response)
return res_trend.slope, res_trend.p, alt_trend_test.trend, alt_trend_test.p
def mk_test(y):
#teste de mann-kendall
print('Resultado do Teste Mann-Kendall')
mkteste = pymannkendall.hamed_rao_modification_test(y)
mksaida = pd.Series(mkteste[0:10],
index=[
'Movimento Tendência', 'Há Tendência?', 'Valor p',
'Teste normalizado', 'Coeficiente Kendall',
'Mand-Kendall Score', 'Variância S', 'Sen Slope',
'Interceptar'
])
print(mksaida)
def doTheAnnMeanFlowTrends(pastStats, currentStats = None):
tSeries = []
for key in pastStats:
#tSeries.extend(pastStats[key][:12])
tSeries.append(pastStats[key][12])
#print(pastStats[key])
if currentStats is not None:
for key in currentStats:
tSeries.append(currentStats[key][12])
#print(len(tSeries), sum(tSeries)/len(tSeries))
#print(tSeries)
return [pmk.original_test(tSeries), pmk.hamed_rao_modification_test(tSeries)]
MKT = mk.original_test(Birth_data, alpha=0.05)
print(MKT)
# IMPORTING SECOND DATA SET
Shampoo_data = pd.read_csv("shampoo.csv",
parse_dates=['Month'],
index_col='Month')
# SUMMARY STATISTICS
head_shampoo = Shampoo_data.head()
Summary_shampoo = Shampoo_data.describe()
print(head_shampoo)
print(Summary_shampoo)
# TREND TEST 1
MKT1 = mk.hamed_rao_modification_test(Shampoo_data)
print(MKT1)
# TREND TEST 2
MKT2 = mk.yue_wang_modification_test(Shampoo_data)
print(MKT2)
# TREND TEST 3
MKT3 = mk.trend_free_pre_whitening_modification_test(Shampoo_data)
print(MKT3)
# IMPORTING THIRD DATA
Malaria_data = pd.read_csv("Malaria_data.csv",
parse_dates=['Date'],
index_col='Date')
def add_element(self, value):
'''
Add new element to the statistic
'''
#reset parameters if change was detected:
if self.in_concept_change:
self.reset()
#append elements:
self.instance_memory.append(value)
if len(self.instance_memory) == self.min_instances:
self.sample_count = 1
if len(self.instance_memory) > self.min_instances:
self.instance_count += 1
#start drift detection: >> min_instances have to be reached, then always perform test once, after that perform test every i_th instance (instances_step)
if len(self.instance_memory) >= self.min_instances and ((self.instance_count == self.instances_step) or (self.sample_count == 1)):
if self.test_type == 'original_mk':
#call corresponding test from package:
print('Perform MK test')
results_tuple = mk.original_test(self.instance_memory, self.alpha)
print('MK test ended')
if self.test_type == 'hamed_rao_mod':
#call corresponding test from package:
results_tuple = mk.hamed_rao_modification_test(self.instance_memory, self.alpha)
if self.test_type == 'yue_wang_mod':
#call corresponding test from package:
results_tuple = mk.yue_wang_modification_test(self.instance_memory, self.alpha)
if self.test_type == 'trend_free_pre_whitening_mod':
#call corresponding test from package:
results_tuple = mk.trend_free_pre_whitening_modification_test(self.instance_memory, self.alpha)
if self.test_type == 'pre_whitening_mod':
#call corresponding test from package:
results_tuple = mk.pre_whitening_modification_test(self.instance_memory, self.alpha)
if self.test_type == 'seasonal':
#call corresponding test from package:
results_tuple = mk.seasonal_test(self.instance_memory, period = self.period, alpha = self.alpha)
#reset counter every time a test was performed:
self.sample_count = 0
self.instance_count = 0
#assign results:
self.p_value = results_tuple[2]
self.sens_slope = results_tuple[-1]
self.trend = results_tuple[0]
if self.p_value < self.alpha and np.abs(self.sens_slope) > self.slope_threshold:
self.in_concept_change = True
else:
self.in_concept_change = False
TREND = []
for index, row in list_nom.iterrows():
data = pd.read_csv(path_m + row[0] + '_MONTH_' + varin + '_' +
indice + '_' + str(yearmin) + '_' +
str(yearmax) + '_' +
str('{:02d}'.format(month)) + '.csv',
skiprows=2)
data = data.rename(columns={
data.columns[1]: "var"
}).set_index('datetime')
if (valeur['name'] == 'Original Mann-Kendall test'):
trend, h, p, z, Tau, s, var_s, slope, intercept = mk.original_test(
data)
elif (valeur['name'] == 'Hamed and Rao Modified MK Test'):
trend, h, p, z, Tau, s, var_s, slope, intercept = mk.hamed_rao_modification_test(
data)
elif (valeur['name'] == 'Yue and Wang Modified MK Test'):
trend, h, p, z, Tau, s, var_s, slope, intercept = mk.yue_wang_modification_test(
data)
elif (valeur['name'] ==
'Modified MK test using Pre-Whitening method'):
trend, h, p, z, Tau, s, var_s, slope, intercept = mk.pre_whitening_modification_test(
data)
elif (valeur['name'] ==
'Modified MK test using Trend free Pre-Whitening method'):
trend, h, p, z, Tau, s, var_s, slope, intercept = mk.trend_free_pre_whitening_modification_test(
data)
if (p >= 0.1): # aucune tendance n'a été détectée
slope = np.nan
TREND.append(slope * 10)
a = fin.readlines()
temp = ""
temp1 = np.zeros(len(a))
temp2 = np.zeros(len(a))
temp3 = np.zeros(len(a))
temp4 = np.zeros(len(a))
#temp5=np.zeros(len(a))
for i in range(1, len(a)):
temp = a[i].split()
temp1[i] = float(temp[0])
temp2[i] = float(temp[4])
temp3[i] = float(temp[3])
temp4[i] = float(temp[2])
#temp5[i]=float(temp[4])
kt, pkt = ss.kendalltau(temp1, temp2)
print("Kendall tau and pvalue are", kt, pkt)
result = mk.hamed_rao_modification_test(temp2)
print(result, "RR")
kt, pkt = ss.kendalltau(temp1, temp3)
print("Kendall tau and pvalue are", kt, pkt)
result = mk.hamed_rao_modification_test(temp3)
print(result, "Det")
kt, pkt = ss.kendalltau(temp1, temp4)
print("Kendall tau and pvalue are", kt, pkt)
result = mk.hamed_rao_modification_test(temp4)
print(result, "Lam")
#result=mk.hamed_rao_modification_test(temp5)
#print(result, "ratio")
sdftccnoifl = sdf.groupby(['scenario']).tcacnoifl.sum() print(sdftccnoifl) #%% # Test trends import pymannkendall as mk scen = 'neg01' reg = 'sa' lt = 'ftcratio' mkt = df1[df1['scenario'].isin([scen])] mkt = mkt[mkt['region'].isin([reg])] mkt = mkt[mkt['losstype'].isin([lt])] print mk.original_test(mkt.vals) print mk.hamed_rao_modification_test(mkt.vals) print mk.yue_wang_modification_test(mkt.vals) print mk.sens_slope(mkt.vals) #%% #------------------------------------------------------------ # BY ECOREGION #------------------------------------------------------------ #%% # Input dir for csvs idir = 'G:/My Drive/Projects/ifl_corridors/statsfiles/wwf_lybytc2000_fluxes/' #%% flist = glob.glob(os.path.join(idir, '*wwf_lybytc2000_flux_sums.csv')) flist = [pd.read_csv(f) for f in flist]
