def test_hamed_rao_modification_test(NoTrendData, TrendData, arbitrary_1d_data): # check with no trend data NoTrendRes = mk.hamed_rao_modification_test(NoTrendData) assert NoTrendRes.trend == 'no trend' assert NoTrendRes.h == False assert NoTrendRes.p == 1.0 assert NoTrendRes.z == 0 assert NoTrendRes.Tau == 0.0 assert NoTrendRes.s == 0.0 assert NoTrendRes.slope == 0.0 # check with trendy data TrendRes = mk.hamed_rao_modification_test(TrendData) assert TrendRes.trend == 'increasing' assert TrendRes.h == True assert TrendRes.p == 0.0 assert TrendRes.Tau == 1.0 assert TrendRes.s == 64620.0 np.testing.assert_allclose(TrendRes.slope, 1.0, rtol=1e-02) # check with arbitrary data result = mk.hamed_rao_modification_test(arbitrary_1d_data) assert result.trend == 'decreasing' assert result.h == True assert result.p == 0.00011372459883540742 assert result.z == -3.859273515045842 assert result.Tau == -0.03153167653875869 assert result.s == -1959.0 assert result.var_s == 257403.38678462413 assert result.slope == -0.0064516129032258064
def test_hamed_rao_modification_test_lag3(NoTrendData, TrendData, arbitrary_1d_data): # check with no trend data NoTrendRes = mk.hamed_rao_modification_test(NoTrendData, lag=3) assert NoTrendRes.trend == 'no trend' assert NoTrendRes.h == False assert NoTrendRes.p == 1.0 assert NoTrendRes.z == 0 assert NoTrendRes.Tau == 0.0 assert NoTrendRes.s == 0.0 assert NoTrendRes.slope == 0.0 # check with trendy data TrendRes = mk.hamed_rao_modification_test(TrendData, lag=3) assert TrendRes.trend == 'increasing' assert TrendRes.h == True assert TrendRes.p == 0.0 assert TrendRes.Tau == 1.0 assert TrendRes.s == 64620.0 np.testing.assert_allclose(TrendRes.slope, 1.0, rtol=1e-02) # check with arbitrary data result = mk.hamed_rao_modification_test(arbitrary_1d_data, lag=3) assert result.trend == 'no trend' assert result.h == False assert result.p == 0.603684460662274 assert result.z == -0.5191093899188985 assert result.Tau == -0.03153167653875869 assert result.s == -1959.0 assert result.var_s == 14226812.425138814 assert result.slope == -0.0064516129032258064
def test_hamed_rao_modification_test_lag3(NoTrendData, TrendData, arbitrary_1d_data): # check with no trend data NoTrendRes = mk.hamed_rao_modification_test(NoTrendData, lag=3) assert NoTrendRes.trend == 'no trend' assert NoTrendRes.h == False assert NoTrendRes.p == 1.0 assert NoTrendRes.z == 0 assert NoTrendRes.Tau == 0.0 assert NoTrendRes.s == 0.0 # check with trendy data TrendRes = mk.hamed_rao_modification_test(TrendData, lag=3) assert TrendRes.trend == 'increasing' assert TrendRes.h == True assert TrendRes.p == 0.0 assert TrendRes.Tau == 1.0 assert TrendRes.s == 64620.0 # check with arbitrary data result = mk.hamed_rao_modification_test(arbitrary_1d_data, lag=3) assert result.trend == 'no trend' assert result.h == False assert result.p == 0.6037112685123898 assert result.z == -0.5190709455046154 assert result.Tau == -0.03153167653875869 assert result.s == -1959.0 assert result.var_s == 14228919.889368296
def test_hamed_rao_modification_test(NoTrendData, TrendData, arbitrary_1d_data): # check with no trend data NoTrendRes = mk.hamed_rao_modification_test(NoTrendData) assert NoTrendRes.trend == 'no trend' assert NoTrendRes.h == False assert NoTrendRes.p == 1.0 assert NoTrendRes.z == 0 assert NoTrendRes.Tau == 0.0 assert NoTrendRes.s == 0.0 # check with trendy data TrendRes = mk.hamed_rao_modification_test(TrendData) assert TrendRes.trend == 'increasing' assert TrendRes.h == True assert TrendRes.p == 0.0 assert TrendRes.Tau == 1.0 assert TrendRes.s == 64620.0 # check with arbitrary data result = mk.hamed_rao_modification_test(arbitrary_1d_data) assert result.trend == 'decreasing' assert result.h == True assert result.p == 0.00012203829241275166 assert result.z == -3.8419950613710894 assert result.Tau == -0.03153167653875869 assert result.s == -1959.0 assert result.var_s == 259723.81316716125
def test_residuals(model, timeperiod, reg): """ Test for a residual trend, applying a Mann-Kendall-test Parameters ---------- model : GLMObject Best model timeperiod : np.array considered years (not used here) Returns ------- float slope in residuals float p-value """ res_trend = mk.original_test(model.resid_response, alpha=0.1) fig, ax = plt.subplots(figsize=(12, 8)) sm.graphics.tsa.plot_acf(model.resid_response, lags=39, ax=ax) ax.set_xlabel('lag') ax.set_title('Autocorrelation {}'.format(reg)) #fig.savefig('/home/insauer/projects/NC_Submission/Climada_papers/Test/AutocorrResidualsGMT_{}.png'.format(reg),bbox_inches = 'tight',dpi =600) alt_trend_test = mk.hamed_rao_modification_test(model.resid_response) return res_trend.slope, res_trend.p, alt_trend_test.trend, alt_trend_test.p
def test_residuals(model, timeperiod, reg): """ Test for a residual trend, applying a Mann-Kendall-test Parameters ---------- model : GLMObject Best model timeperiod : np.array considered years (not used here) Returns ------- float slope in residuals float p-value """ res_trend = mk.original_test(model.resid_response, alpha=0.1) fig, ax = plt.subplots(figsize=(12, 8)) alt_trend_test = mk.hamed_rao_modification_test(model.resid_response) return res_trend.slope, res_trend.p, alt_trend_test.trend, alt_trend_test.p
def mk_test(y): #teste de mann-kendall print('Resultado do Teste Mann-Kendall') mkteste = pymannkendall.hamed_rao_modification_test(y) mksaida = pd.Series(mkteste[0:10], index=[ 'Movimento Tendência', 'Há Tendência?', 'Valor p', 'Teste normalizado', 'Coeficiente Kendall', 'Mand-Kendall Score', 'Variância S', 'Sen Slope', 'Interceptar' ]) print(mksaida)
def doTheAnnMeanFlowTrends(pastStats, currentStats = None): tSeries = [] for key in pastStats: #tSeries.extend(pastStats[key][:12]) tSeries.append(pastStats[key][12]) #print(pastStats[key]) if currentStats is not None: for key in currentStats: tSeries.append(currentStats[key][12]) #print(len(tSeries), sum(tSeries)/len(tSeries)) #print(tSeries) return [pmk.original_test(tSeries), pmk.hamed_rao_modification_test(tSeries)]
MKT = mk.original_test(Birth_data, alpha=0.05) print(MKT) # IMPORTING SECOND DATA SET Shampoo_data = pd.read_csv("shampoo.csv", parse_dates=['Month'], index_col='Month') # SUMMARY STATISTICS head_shampoo = Shampoo_data.head() Summary_shampoo = Shampoo_data.describe() print(head_shampoo) print(Summary_shampoo) # TREND TEST 1 MKT1 = mk.hamed_rao_modification_test(Shampoo_data) print(MKT1) # TREND TEST 2 MKT2 = mk.yue_wang_modification_test(Shampoo_data) print(MKT2) # TREND TEST 3 MKT3 = mk.trend_free_pre_whitening_modification_test(Shampoo_data) print(MKT3) # IMPORTING THIRD DATA Malaria_data = pd.read_csv("Malaria_data.csv", parse_dates=['Date'], index_col='Date')
def add_element(self, value): ''' Add new element to the statistic ''' #reset parameters if change was detected: if self.in_concept_change: self.reset() #append elements: self.instance_memory.append(value) if len(self.instance_memory) == self.min_instances: self.sample_count = 1 if len(self.instance_memory) > self.min_instances: self.instance_count += 1 #start drift detection: >> min_instances have to be reached, then always perform test once, after that perform test every i_th instance (instances_step) if len(self.instance_memory) >= self.min_instances and ((self.instance_count == self.instances_step) or (self.sample_count == 1)): if self.test_type == 'original_mk': #call corresponding test from package: print('Perform MK test') results_tuple = mk.original_test(self.instance_memory, self.alpha) print('MK test ended') if self.test_type == 'hamed_rao_mod': #call corresponding test from package: results_tuple = mk.hamed_rao_modification_test(self.instance_memory, self.alpha) if self.test_type == 'yue_wang_mod': #call corresponding test from package: results_tuple = mk.yue_wang_modification_test(self.instance_memory, self.alpha) if self.test_type == 'trend_free_pre_whitening_mod': #call corresponding test from package: results_tuple = mk.trend_free_pre_whitening_modification_test(self.instance_memory, self.alpha) if self.test_type == 'pre_whitening_mod': #call corresponding test from package: results_tuple = mk.pre_whitening_modification_test(self.instance_memory, self.alpha) if self.test_type == 'seasonal': #call corresponding test from package: results_tuple = mk.seasonal_test(self.instance_memory, period = self.period, alpha = self.alpha) #reset counter every time a test was performed: self.sample_count = 0 self.instance_count = 0 #assign results: self.p_value = results_tuple[2] self.sens_slope = results_tuple[-1] self.trend = results_tuple[0] if self.p_value < self.alpha and np.abs(self.sens_slope) > self.slope_threshold: self.in_concept_change = True else: self.in_concept_change = False
TREND = [] for index, row in list_nom.iterrows(): data = pd.read_csv(path_m + row[0] + '_MONTH_' + varin + '_' + indice + '_' + str(yearmin) + '_' + str(yearmax) + '_' + str('{:02d}'.format(month)) + '.csv', skiprows=2) data = data.rename(columns={ data.columns[1]: "var" }).set_index('datetime') if (valeur['name'] == 'Original Mann-Kendall test'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.original_test( data) elif (valeur['name'] == 'Hamed and Rao Modified MK Test'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.hamed_rao_modification_test( data) elif (valeur['name'] == 'Yue and Wang Modified MK Test'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.yue_wang_modification_test( data) elif (valeur['name'] == 'Modified MK test using Pre-Whitening method'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.pre_whitening_modification_test( data) elif (valeur['name'] == 'Modified MK test using Trend free Pre-Whitening method'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.trend_free_pre_whitening_modification_test( data) if (p >= 0.1): # aucune tendance n'a été détectée slope = np.nan TREND.append(slope * 10)
a = fin.readlines() temp = "" temp1 = np.zeros(len(a)) temp2 = np.zeros(len(a)) temp3 = np.zeros(len(a)) temp4 = np.zeros(len(a)) #temp5=np.zeros(len(a)) for i in range(1, len(a)): temp = a[i].split() temp1[i] = float(temp[0]) temp2[i] = float(temp[4]) temp3[i] = float(temp[3]) temp4[i] = float(temp[2]) #temp5[i]=float(temp[4]) kt, pkt = ss.kendalltau(temp1, temp2) print("Kendall tau and pvalue are", kt, pkt) result = mk.hamed_rao_modification_test(temp2) print(result, "RR") kt, pkt = ss.kendalltau(temp1, temp3) print("Kendall tau and pvalue are", kt, pkt) result = mk.hamed_rao_modification_test(temp3) print(result, "Det") kt, pkt = ss.kendalltau(temp1, temp4) print("Kendall tau and pvalue are", kt, pkt) result = mk.hamed_rao_modification_test(temp4) print(result, "Lam") #result=mk.hamed_rao_modification_test(temp5) #print(result, "ratio")
sdftccnoifl = sdf.groupby(['scenario']).tcacnoifl.sum() print(sdftccnoifl) #%% # Test trends import pymannkendall as mk scen = 'neg01' reg = 'sa' lt = 'ftcratio' mkt = df1[df1['scenario'].isin([scen])] mkt = mkt[mkt['region'].isin([reg])] mkt = mkt[mkt['losstype'].isin([lt])] print mk.original_test(mkt.vals) print mk.hamed_rao_modification_test(mkt.vals) print mk.yue_wang_modification_test(mkt.vals) print mk.sens_slope(mkt.vals) #%% #------------------------------------------------------------ # BY ECOREGION #------------------------------------------------------------ #%% # Input dir for csvs idir = 'G:/My Drive/Projects/ifl_corridors/statsfiles/wwf_lybytc2000_fluxes/' #%% flist = glob.glob(os.path.join(idir, '*wwf_lybytc2000_flux_sums.csv')) flist = [pd.read_csv(f) for f in flist]