def test_trend_free_pre_whitening_modification_test(NoTrendData, TrendData, arbitrary_1d_data): # check with no trend data NoTrendRes = mk.trend_free_pre_whitening_modification_test(NoTrendData) assert NoTrendRes.trend == 'no trend' assert NoTrendRes.h == False assert NoTrendRes.p == 1.0 assert NoTrendRes.z == 0 assert NoTrendRes.Tau == 0.0 assert NoTrendRes.s == 0.0 # check with trendy data TrendRes = mk.trend_free_pre_whitening_modification_test(TrendData) assert TrendRes.trend == 'increasing' assert TrendRes.h == True assert TrendRes.p == 0.0 np.testing.assert_allclose(TrendRes.Tau, 1.0, rtol=1e-02) np.testing.assert_allclose(TrendRes.slope, 1.0, rtol=1e-02) # check with arbitrary data result = mk.trend_free_pre_whitening_modification_test(arbitrary_1d_data) assert result.trend == 'no trend' assert result.h == False assert result.p == 0.7741578265217384 assert result.z == -0.2869405688895601 assert result.Tau == -0.010262885262885263 assert result.s == -634.0 assert result.var_s == 4866576.0 assert result.slope == -0.004174019670423232
def test_trend_free_pre_whitening_modification_test(NoTrendData, TrendData, arbitrary_1d_data): # check with no trend data NoTrendRes = mk.trend_free_pre_whitening_modification_test(NoTrendData) assert NoTrendRes.trend == 'no trend' assert NoTrendRes.h == False assert NoTrendRes.p == 1.0 assert NoTrendRes.z == 0 assert NoTrendRes.Tau == 0.0 # check with trendy data TrendRes = mk.trend_free_pre_whitening_modification_test(TrendData) assert TrendRes.trend == 'increasing' assert TrendRes.h == True assert TrendRes.p == 0.0 assert TrendRes.Tau == 1.0 # check with arbitrary data result = mk.trend_free_pre_whitening_modification_test(arbitrary_1d_data) assert result.trend == 'no trend' assert result.h == False assert result.p == 0.7755465706913385 assert result.z == -0.28512735834365455 assert result.Tau == -0.010198135198135198 assert result.s == -630.0 assert result.var_s == 4866576.0
# SUMMARY STATISTICS head_shampoo = Shampoo_data.head() Summary_shampoo = Shampoo_data.describe() print(head_shampoo) print(Summary_shampoo) # TREND TEST 1 MKT1 = mk.hamed_rao_modification_test(Shampoo_data) print(MKT1) # TREND TEST 2 MKT2 = mk.yue_wang_modification_test(Shampoo_data) print(MKT2) # TREND TEST 3 MKT3 = mk.trend_free_pre_whitening_modification_test(Shampoo_data) print(MKT3) # IMPORTING THIRD DATA Malaria_data = pd.read_csv("Malaria_data.csv", parse_dates=['Date'], index_col='Date') # SUMMARY STATISTICS Malaria_head = Malaria_data.head() Malaria_summary = Malaria_data.describe() print(Malaria_head) print(Malaria_summary) #TREND TEST MKT4 = mk.hamed_rao_modification_test(Malaria_data)
def add_element(self, value): ''' Add new element to the statistic ''' #reset parameters if change was detected: if self.in_concept_change: self.reset() #append elements: self.instance_memory.append(value) if len(self.instance_memory) == self.min_instances: self.sample_count = 1 if len(self.instance_memory) > self.min_instances: self.instance_count += 1 #start drift detection: >> min_instances have to be reached, then always perform test once, after that perform test every i_th instance (instances_step) if len(self.instance_memory) >= self.min_instances and ((self.instance_count == self.instances_step) or (self.sample_count == 1)): if self.test_type == 'original_mk': #call corresponding test from package: print('Perform MK test') results_tuple = mk.original_test(self.instance_memory, self.alpha) print('MK test ended') if self.test_type == 'hamed_rao_mod': #call corresponding test from package: results_tuple = mk.hamed_rao_modification_test(self.instance_memory, self.alpha) if self.test_type == 'yue_wang_mod': #call corresponding test from package: results_tuple = mk.yue_wang_modification_test(self.instance_memory, self.alpha) if self.test_type == 'trend_free_pre_whitening_mod': #call corresponding test from package: results_tuple = mk.trend_free_pre_whitening_modification_test(self.instance_memory, self.alpha) if self.test_type == 'pre_whitening_mod': #call corresponding test from package: results_tuple = mk.pre_whitening_modification_test(self.instance_memory, self.alpha) if self.test_type == 'seasonal': #call corresponding test from package: results_tuple = mk.seasonal_test(self.instance_memory, period = self.period, alpha = self.alpha) #reset counter every time a test was performed: self.sample_count = 0 self.instance_count = 0 #assign results: self.p_value = results_tuple[2] self.sens_slope = results_tuple[-1] self.trend = results_tuple[0] if self.p_value < self.alpha and np.abs(self.sens_slope) > self.slope_threshold: self.in_concept_change = True else: self.in_concept_change = False
if (valeur['name'] == 'Original Mann-Kendall test'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.original_test( data) elif (valeur['name'] == 'Hamed and Rao Modified MK Test'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.hamed_rao_modification_test( data) elif (valeur['name'] == 'Yue and Wang Modified MK Test'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.yue_wang_modification_test( data) elif (valeur['name'] == 'Modified MK test using Pre-Whitening method'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.pre_whitening_modification_test( data) elif (valeur['name'] == 'Modified MK test using Trend free Pre-Whitening method'): trend, h, p, z, Tau, s, var_s, slope, intercept = mk.trend_free_pre_whitening_modification_test( data) if (p >= 0.1): # aucune tendance n'a été détectée slope = np.nan TREND.append(slope * 10) trend = pd.DataFrame({'trend': TREND}) df = pd.concat([list_nom, trend, list_latlon], axis=1) pathlib.Path(path_trend_m + cle + '/').mkdir(parents=True, exist_ok=True) df.to_csv(path_trend_m + cle + '/INDICES_MONTH_' + valeur['code'] + '_' + varin + '_' + indice + '_' + str(yearmin) + '_' + str(yearmax) + '_' + str('{:02d}'.format(month)) + '.csv') fig = plt.figure(figsize=(30, 18), frameon=True)
def __init__(self, prepared_list, station_names, basin_name): self.prepared_list = prepared_list self.station_names = station_names self.basin_name = basin_name import subprocess import sys import os reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) installed_packages = [r.decode().split('==')[0] for r in reqs.split()] needed_packages = [ 'pandas', 'numpy', 'datetime', 'tqdm', 'pymannkendall', 'scipy' ] def install(package): subprocess.check_call( [sys.executable, "-m", "pip", "install", package]) for package in needed_packages: if package in installed_packages: pass else: install(package) import pandas as pd import numpy as np from datetime import datetime from tqdm import trange, tqdm import pymannkendall as mk from scipy import stats df_T = self.prepared_list def autocorr(x, t=1): return np.corrcoef(np.array([x[:-t], x[t:]]))[0][1] def running_mean(x, N): x = np.array(x) cumsum = np.cumsum(np.insert(x, 0, 0)) t = (cumsum[N:] - cumsum[:-N]) / float(N) return t[::N] def Hodges(inputdata): l_avgs = running_mean(inputdata, 2) hl_est = np.nanmedian(l_avgs) return hl_est def Pettitt2(signal): T = len(signal) U = [[[np.sign(signal[i] - signal[j]) for j in range(t + 1, T)] for i in range(t)] for t in range(T)] U_sum = [[sum(U[i][j]) for j in range(len(U[i]))] for i in range(len(U))] U_sum = [sum(U_sum[i]) for i in range(len(U_sum))] loc = np.argmax(np.abs(U_sum)) K = max(np.abs(U_sum)) p = 2 * np.exp(-6 * K**2 / (T**3 + T**2)) return (loc, p) def Buishand(inputdata): inputdata_mean = np.mean(inputdata) n = inputdata.shape[0] k = range(n) Sk = [np.sum(inputdata[0:x + 1] - inputdata_mean) for x in k] sigma = np.sqrt( np.sum((inputdata - np.mean(inputdata))**2) / (n - 1)) U = np.sum((Sk[0:(n - 2)] / sigma)**2) / (n * (n + 1)) Ska = np.abs(Sk) S = np.max(Ska) K = list(Ska).index(S) + 1 Skk = (Sk / sigma) return (K) ### Number of observations monthes = [ 'Январь', 'Февраль', 'Март', 'Апрель', 'Май', 'Июнь', 'Июль', 'Август', 'Сентябрь', 'Октябрь', 'Ноябрь', 'Декабрь', 'Среднегодовое' ] time_of_obs = [ pd.Series([len(df_T[i][j]) for j in monthes], index=monthes) for i in range(len(df_T)) ] missings = [ pd.Series([sum(df_T[i][j].isna()) for j in monthes], index=monthes) for i in range(len(df_T)) ] actual_obs = [ pd.Series([ time_of_obs[i][j] - missings[i][j] for j in range(len(time_of_obs[i])) ], index=monthes) for i in range(len(time_of_obs)) ] #mean MEAN_T = [ df_T[i][monthes].mean(axis=0) for i in trange(len(df_T), desc='AVG calc') ] #cv CV_T = [ df_T[i][monthes].std(axis=0) / df_T[i][monthes].mean(axis=0) for i in trange(len(df_T), desc='CV calc') ] import pymannkendall as mk # https://github.com/mmhs013/pyMannKendall #mk MK_T_005 = [[ np.array(mk.original_test(df_T[i][j], alpha=0.05))[[0]] if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN] for j in monthes ] for i in trange(len(df_T), desc='MK calc 0.05')] MK_T_01 = [[ np.array(mk.original_test(df_T[i][j], alpha=0.1))[[0, 2, 4]] if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN, np.NaN, np.NaN] for j in monthes ] for i in trange(len(df_T), desc='MK calc 0.1')] MK_T_trend_005 = [ pd.Series([MK_T_005[i][j][0] for j in range(len(MK_T_005[i]))], index=monthes) for i in trange(len(MK_T_005), desc='MK trend 0.05') ] MK_T_trend_005 = [ pd.Series([ '0' if MK_T_trend_005[i][j] == 'no trend' else '1' if MK_T_trend_005[i][j] == 'increasing' else '-1' for j in range(len(MK_T_trend_005[i])) ], index=monthes) for i in range(len(MK_T_trend_005)) ] MK_T_trend_01 = [ pd.Series([MK_T_01[i][j][0] for j in range(len(MK_T_01[i]))], index=monthes) for i in trange(len(MK_T_01), desc='MK trend 0.1') ] MK_T_trend_01 = [ pd.Series([ '0' if MK_T_trend_01[i][j] == 'no trend' else '1' if MK_T_trend_01[i][j] == 'increasing' else '-1' for j in range(len(MK_T_trend_01[i])) ], index=monthes) for i in range(len(MK_T_trend_01)) ] MK_T_p = [ pd.Series([MK_T_01[i][j][1] for j in range(len(MK_T_01[i]))], index=monthes) for i in trange(len(MK_T_01), desc='MK p') ] MK_T_tau = [ pd.Series([MK_T_01[i][j][2] for j in range(len(MK_T_01[i]))], index=monthes) for i in trange(len(MK_T_01), desc='MK tau') ] #spearman https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.spearmanr.html SRC_T = [[ stats.spearmanr(df_T[i][j].dropna(), range(len( df_T[i][j].dropna()))) if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN, np.NaN] for j in monthes ] for i in trange(len(df_T), desc='SRC calc')] SRC_T_rho = [ pd.Series([SRC_T[i][j][0] for j in range(len(SRC_T[i]))], index=monthes) for i in trange(len(SRC_T), desc='SRC rho') ] SRC_T_rho_p = [ pd.Series([SRC_T[i][j][1] for j in range(len(SRC_T[i]))], index=monthes) for i in trange(len(SRC_T), desc='SRC p') ] #THEIL-Sen https://docs.scipy.org/doc/scipy-0.15.1/reference/generated/scipy.stats.mstats.theilslopes.html THEIL_T = [[ stats.theilslopes(df_T[i][j].dropna(), range(len(df_T[i][j].dropna())), 0.90) if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN] for j in monthes ] for i in trange(len(df_T), desc='THEIL calc')] THEIL_T_S = [ pd.Series([THEIL_T[i][j][0] for j in range(len(THEIL_T[i]))], index=monthes) for i in trange(len(THEIL_T), desc='THEIL SLOPE') ] #AR1 AR1_T = [ pd.Series([ autocorr(df_T[i][j].dropna()) if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else np.NaN for j in monthes ], index=monthes) for i in trange(len(df_T), desc='AR calc') ] #TFPW TFPW_T_005 = [[ np.array( mk.trend_free_pre_whitening_modification_test(df_T[i][j], alpha=0.05))[[0]] if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN] for j in monthes ] for i in trange(len(df_T), desc='TFPW calc 0.05')] TFPW_T_01 = [[ np.array( mk.trend_free_pre_whitening_modification_test( df_T[i][j], alpha=0.1))[[0, 2, 4, 7]] if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN] * 4 for j in monthes ] for i in trange(len(df_T), desc='TFPW calc 0.1')] TFPW_T_trend_01 = [ pd.Series([TFPW_T_01[i][j][0] for j in range(len(TFPW_T_01[i]))], index=monthes) for i in trange(len(TFPW_T_01), desc='TFPW_T trend') ] TFPW_T_trend_01 = [ pd.Series([ '0' if TFPW_T_trend_01[i][j] == 'no trend' else '1' if TFPW_T_trend_01[i][j] == 'increasing' else '-1' for j in range(len(TFPW_T_trend_01[i])) ], index=monthes) for i in range(len(TFPW_T_trend_01)) ] TFPW_T_trend_005 = [ pd.Series([TFPW_T_005[i][j][0] for j in range(len(TFPW_T_005[i]))], index=monthes) for i in trange(len(TFPW_T_005), desc='TFPW_T trend') ] TFPW_T_trend_005 = [ pd.Series([ '0' if TFPW_T_trend_005[i][j] == 'no trend' else '1' if TFPW_T_trend_005[i][j] == 'increasing' else '-1' for j in range(len(TFPW_T_trend_005[i])) ], index=monthes) for i in range(len(TFPW_T_trend_005)) ] TFPW_T_p = [ pd.Series([TFPW_T_01[i][j][1] for j in range(len(TFPW_T_01[i]))], index=monthes) for i in trange(len(TFPW_T_01), desc='TFPW_T p') ] TFPW_T_tau = [ pd.Series([TFPW_T_01[i][j][2] for j in range(len(TFPW_T_01[i]))], index=monthes) for i in trange(len(TFPW_T_01), desc='TFPW_T tau') ] TFPW_T_S = [ pd.Series([TFPW_T_01[i][j][3] for j in range(len(TFPW_T_01[i]))], index=monthes, dtype='float64') for i in trange(len(TFPW_T_01), desc='TFPW_T SLOPE') ] # Pettitt Pettitt_T = [[ np.array(Pettitt2(df_T[i][j].dropna().to_numpy())) if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else [np.NaN, np.NaN] for j in monthes ] for i in trange(len(df_T), desc='Pettitt calc') if len(df_T[i]) > 2] Pettitt_T_pos = [ pd.Series([Pettitt_T[i][j][0] for j in range(len(Pettitt_T[i]))], index=monthes) for i in trange(len(Pettitt_T), desc='Pettitt_T pos') ] Pettitt_T_posY = [ pd.Series([df_T[j].date[i] for i in Pettitt_T_pos[j]], index=monthes) for j in trange(len(Pettitt_T_pos), desc='YEAR OF PETTITT') ] Pettitt_T_p = [ pd.Series([Pettitt_T[i][j][1] for j in range(len(Pettitt_T[i]))], index=monthes) for i in trange(len(Pettitt_T), desc='Pettitt_T p') ] Pettitt_T_005 = [ pd.Series([ '1' if Pettitt_T_p[j][i] > 0.05 else '0' for i in range(len(Pettitt_T_p[j])) ], index=monthes) for j in range(len(Pettitt_T_p)) ] Pettitt_T_01 = [ pd.Series([ '1' if Pettitt_T_p[j][i] > 0.1 else '0' for i in range(len(Pettitt_T_p[j])) ], index=monthes) for j in range(len(Pettitt_T_p)) ] # Buishand Buishand_T = [ pd.Series([ Buishand(df_T[i][j].dropna().to_numpy()) if sum(df_T[i][j].isna()) < len(df_T[i][j]) - 2 else np.NaN for j in monthes ], index=monthes) for i in trange(len(df_T), desc='Buishand calc') ] Buishand_TY = [ pd.Series([df_T[j].date[i] for i in Buishand_T[j]], index=monthes) for j in trange(len(Buishand_T), desc='YEAR OF BUISHAND') ] # Hodges-Lehmann Estimator Hodges_T = [ pd.Series( [Hodges(df_T[i][j].dropna().to_numpy()) for j in monthes], index=monthes) for i in trange(len(df_T), desc='Hodges calc') ] # Change in data CHANGE = [ pd.Series([ THEIL_T_S[i][j] * actual_obs[i][j] if np.abs(AR1_T[i][j]) < 0.20 else TFPW_T_S[i][j] * actual_obs[i][j] for j in range(len(actual_obs[i])) ], index=monthes) for i in trange(len(actual_obs), desc='CHANGE CALC') ] CHANGE_percent = [ pd.Series([ THEIL_T_S[i][j] * 100 * actual_obs[i][j] / abs(MEAN_T[i][j]) if np.abs(AR1_T[i][j]) < 0.20 else TFPW_T_S[i][j] * 100 * actual_obs[i][j] / abs(MEAN_T[i][j]) for j in range(len(actual_obs[i])) ], index=monthes) for i in trange(len(actual_obs), desc='CHANGE PERCENT CALC') ] index_names = [ 'Start', 'End', 'N_obs', 'N_miss', 'Obs', 'Mean', #'CV', 'MK_trend_005', 'MK_trend_01', #'MK_p', 'MK_tau', #'SRC_rho', 'SRC_rho_p', 'THEIL_S', #'AR1', #'TFPW_trend_005', 'TFPW_trend_01', 'TFPW_p', 'TFPW_tau','TFPW_S', #'Pettitt_Y', 'Pettitt_p', 'Pettitt_005', 'Pettitt_01', #'Buishand_Y', #'Hodges', 'Change', 'Change_proc' ] start_year = [ pd.Series(df_T[i].date.min(), index=monthes) for i in range(len(df_T)) ] last_year = [ pd.Series(df_T[i].date.max(), index=monthes) for i in range(len(df_T)) ] time_of_obs = [ pd.Series([len(df_T[i][j]) for j in monthes], index=monthes) for i in range(len(df_T)) ] missings = [ pd.Series([sum(df_T[i][j].isna()) for j in monthes], index=monthes) for i in range(len(df_T)) ] observed_gauges_u = self.station_names test = [ pd.concat( [ start_year[i], last_year[i], time_of_obs[i], missings[i], actual_obs[i], MEAN_T[i].astype('float64').apply( '{:.3f}'.format ), #CV_T[i].astype('float64').apply('{:.2f}'.format), MK_T_trend_005[i], MK_T_trend_01[ i], #MK_T_p[i].astype('float64').apply('{:.3f}'.format), MK_T_tau[i].astype('float64').apply('{:.2f}'.format), #SRC_T_rho[i].astype('float64').apply('{:.2f}'.format), SRC_T_rho_p[i].astype('float64').apply('{:.3f}'.format), THEIL_T_S[i].astype('float64').apply('{:.3f}'.format), #AR1_T[i].astype('float64').apply('{:.2f}'.format), #TFPW_T_trend_005[i], TFPW_T_trend_01[i], TFPW_T_p[i].astype('float64').apply('{:.3f}'.format), TFPW_T_tau[i].astype('float64').apply('{:.2f}'.format), TFPW_T_S[i].astype('float64').apply('{:.3f}'.format), #Pettitt_T_posY[i], Pettitt_T_p[i].astype('float64').apply('{:.3f}'.format), Pettitt_T_005[i], Pettitt_T_01[i], #Buishand_TY[i], #Hodges_T[i].astype('float64').apply('{:.2f}'.format), CHANGE[i].astype('float64').apply('{:.2f}'.format), CHANGE_percent[i].astype('float64').apply('{:.3}'.format) ], axis=1).T for i in trange(len(df_T)) ] test = [test[i].set_index([index_names]) for i in range(len(test))] FOR_WHAT = self.basin_name path_for_data = os.getcwd() + r'\STAT\{}'.format(FOR_WHAT) + '_stat' if not os.path.exists(path_for_data): os.makedirs(path_for_data) for i in trange(len(test)): test[i].to_csv(path_for_data + '\{}.txt'.format(observed_gauges_u[i]), sep="\t", index=True) tryout = pd.concat(test) tryout.to_csv(path_for_data + '\MEGA_FILE.txt', sep='\t', index=True) test_trend = [ test[k].iloc[np.r_[7:9, 15:17]] for k in range(len(test)) ] MEGA_trend = pd.concat(test_trend) MEGA_trend.to_csv(path_for_data + '\MEGA_TREND.txt', sep='\t', index=True) test_change = [test[k].iloc[np.r_[-2]] for k in range(len(test))] MEGA_change = pd.concat(test_change) MEGA_change.to_csv(path_for_data + '\MEGA_change.txt', sep='\t', index=True)