subtitles = np.array([[r'$corr(SST_{t}' + rf', {t}_t)$']]) else: subtitles = np.array([[r'$parcorr(SST_{t-lag}, mx2t_t\ |\ SST_{t-1},mx2t_{t-1})$']]) kwrgs_plot = {'row_dim':'split', 'col_dim':'lag', 'aspect':2, 'hspace':-.57, 'wspace':-.22, 'size':4, 'cbar_vert':-.02, 'subtitles':subtitles, 'units':'Corr. Coeff. [-]', 'zoomregion':(130,260,-10,60), 'map_proj':ccrs.PlateCarree(central_longitude=220), 'x_ticks':np.array([]), 'y_ticks':np.array([]), 'drawbox':[(0,0), sst_green_bb], 'clim':(-.6,.6)} rg.plot_maps_corr(var='SST', save=save, min_detect_gc=min_detect_gc, kwrgs_plot=kwrgs_plot) rg.list_for_MI[1].selbox = sst_green_bb #%% Get PDO df_PDO, PDO_patterns = climate_indices.PDO(rg.list_precur_pp[1][1], None) #rg.df_splits) from func_models import standardize_on_train # summerdates = core_pp.get_subdates(dates, start_end_TVdate) df_PDOsplit = df_PDO.loc[0]#.loc[summerdates] # standardize = preprocessing.StandardScaler() # standardize.fit(df_PDOsplit[df_PDOsplit['TrainIsTrue'].values].values.reshape(-1,1)) # df_PDOsplit = pd.DataFrame(standardize.transform(df_PDOsplit['PDO'].values.reshape(-1,1)), # index=df_PDOsplit.index, columns=['PDO']) df_PDOsplit = df_PDOsplit[['PDO']].apply(standardize_on_train, args=[df_PDO.loc[0]['TrainIsTrue']], result_type='broadcast') # Butter Lowpass yr = 2 dates = df_PDOsplit.index
#%% # df_States_lin = (df_States_lin - df_States_lin.mean(0)) / df_States_lin.std(0) FG = sns.FacetGrid(df_States_loess.stack().reset_index(), col='variable', col_wrap=4) for ia, ax in enumerate(FG.fig.axes): ax.plot(df_States[cols[ia]], label=cols[ia]) ax.set_title(cols[ia]) # ax.axhline(y=0) ; ax.set_ylim(-3,3) #%% rg_mistake.pp_TV(name_ds='Soy_Yield', ext_annual_to_mon=False, detrend=True) rg_mistake.pp_precursors() df_PDO, patterns = climate_indices.PDO(rg_mistake.list_precur_pp[0][1]) df_PDO = df_PDO.groupby(df_PDO.loc[0].index.year).mean() df_PDO.index = rg_mistake.df_fullts.index df_Pac = functions_pp.load_hdf5('/Users/semvijverberg/surfdrive/output_paper3/hindcast/USDA_Soy_hindcast/bimonthly_random_20_s1/pandas_dfs_04-06-21_10hr.h5') df_Pac = df_Pac['df_data'] df_Pac = df_Pac[['Soy_Yield'] + [k for k in df_Pac.columns if '..1..sst' in k]] df_Pac = df_Pac.mean(axis=0, level=1).mean(axis=1) # mean over folds, mean over months March-October df_Pac = pd.DataFrame(df_Pac, columns=['eastern Pacific']) df_merge = rg_mistake.df_fullts.merge(df_PDO[['PDO']], left_index=True, right_index=True) df_merge = df_merge.merge(df_Pac, left_index=True, right_index=True) df_merge = df_merge.merge(rg_always.df_fullts.rename({rg_always.df_fullts.columns[0]:'new'}, axis=1), left_index=True, right_index=True) df_merge = (df_merge - df_merge.mean(axis=0)) / df_merge.std(0)
name_or_cluster_label = 'z500' name_ds = f'0..0..{name_or_cluster_label}_sp' start_end_date = ('1-1', start_end_TVdate[-1]) filepath_df_PDOs = os.path.join(path_data, 'df_PDOs_monthly.h5') filepath_df_ENSO = os.path.join(path_data, 'df_ENSOs_monthly.h5') #%% Get PDO and apply low-pass filter if 'parcorr' == exper: try: df_PDOs = functions_pp.load_hdf5(filepath_df_PDOs)['df_data'] except: SST_pp_filepath = user_dir + '/surfdrive/ERA5/input_raw/preprocessed/sst_1979-2020_jan_dec_monthly_1.0deg.nc' if 'df_ENSO' not in globals(): df_PDO, PDO_patterns = climate_indices.PDO(SST_pp_filepath, None) PDO_plot_kwrgs = {'units':'[-]', 'cbar_vert':-.1, # 'zoomregion':(130,260,20,60), 'map_proj':ccrs.PlateCarree(central_longitude=220), 'y_ticks':np.array([25,40,50,60]), 'x_ticks':np.arange(130, 280, 25), 'clevels':np.arange(-.6,.61,.075), 'clabels':np.arange(-.6,.61,.3), 'subtitles':np.array([['PDO loading pattern']])} fig = plot_maps.plot_corr_maps(PDO_patterns[0], **PDO_plot_kwrgs) filepath = os.path.join(path_out_main, 'PDO_pattern') fig.savefig(filepath + '.pdf', bbox_inches='tight') fig.savefig(filepath + '.png', bbox_inches='tight') # summerdates = core_pp.get_subdates(dates, start_end_TVdate) df_PDOsplit = df_PDO.loc[0]#.loc[summerdates]
lags_store = lags_i[:2] for l, lag in enumerate(lags_i): print(f'lag {l}') df_data_ts = ts_lag(actor, lag) df_data_ts = df_data_ts.merge(df_splits, left_index=True, right_index=True) df_data_lag = add_spactov(dict_ds, lag, df_data_ts, actor) if l == 0: # PDO and ENSO do not depend on lag filepath = os.path.join(ex['path_pp'], ex['filename_precur']) # if ex['datafolder'] == 'EC': # df_PDO, PDO_patterns = climate_indices.PDO(actor.precur_arr, df_splits) # else: print('calculating PDO') df_PDO, PDO_patterns = climate_indices.PDO(filepath, df_splits) df_data_lag = df_PDO.merge(df_data_lag, left_index=True, right_index=True) print('calculating ENSO') df_ENSO_34 = climate_indices.ENSO_34(filepath, df_splits) df_data_lag = df_ENSO_34.merge(df_data_lag, left_index=True, right_index=True) df_data_lag = add_RV(df_data_lag, RV) dict_of_dfs = {'df_data': df_data_lag} fname = '{}_{}_lag_{}_{}.h5'.format(ex['datafolder'], today, lag, ex['hash']) file_path = os.path.join(ex['path_data_out'], fname) functions_pp.store_hdf_df(dict_of_dfs, file_path)
#%% Get PDO and apply low-pass filter import climate_indices # filepath_df_PDOs = os.path.join(data_dir, 'df_PDOs_daily.h5') filepath_df_PDOs = os.path.join(data_dir, 'df_PDOs.h5') try: df_PDOs = functions_pp.load_hdf5(filepath_df_PDOs)['df_data'] except: SST_pp_filepath = user_dir + '/surfdrive/ERA5/input_raw/preprocessed/sst_1979-2020_1jan_31dec_daily_1.0deg.nc' if 'df_PDOsplit' not in globals(): # get PDO timeseries, using the same training sample as for RGDR df_PDO, PDO_patterns = climate_indices.PDO(SST_pp_filepath, rg.df_splits) # summerdates = core_pp.get_subdates(dates, start_end_TVdate) # df_PDOsplit = df_PDO.loc[0]#.loc[summerdates] # # standardize = preprocessing.StandardScaler() # # standardize.fit(df_PDOsplit[df_PDOsplit['TrainIsTrue'].values].values.reshape(-1,1)) # # df_PDOsplit = pd.DataFrame(standardize.transform(df_PDOsplit['PDO'].values.reshape(-1,1)), # # index=df_PDOsplit.index, columns=['PDO']) # df_PDOsplit = df_PDOsplit[['PDO']].apply(standardize_on_train, # args=[df_PDO.loc[0]['TrainIsTrue']], # result_type='broadcast') # # Butter Lowpass # dates = df_PDOsplit.index # freqraw = (dates[1] - dates[0]).days # ls = ['solid', 'dotted', 'dashdot', 'dashed'] # fig, ax = plt.subplots(1,1, figsize=(10,5))
if 'cmap' in kwrgs_plot_c.keys(): kwrgs_plot_c.pop('cmap') kwrgs_plot_c.update({'cbar_vert':-.14, 'units':'[-] Corr. Coef.', 'size':2, # 'subtitles':[['$t=$'+'July-Aug'+' mean']], 'subtitles':[[r'$corr(SST_{MJ}, T^{E}_{JA})$']], 'clevels':np.arange(-.6,.61,.075), 'clabels':np.arange(-.6,.61,.3)}) g = plot_maps.plot_corr_maps(corr.sel(months='July-Aug').where(~np.isnan(labels.sel(months='July-Aug'))), **kwrgs_plot_c) fig_path = os.path.join(rg.path_outsub1, 'Corr_map_julyaugust')+rg.figext g.fig.savefig(fig_path, bbox_inches='tight') #%% Plot PDO import climate_indices df_PDO, PDO_patterns = climate_indices.PDO(rg.list_precur_pp[0][1],None) PDO_patterns *= -1 PDO_patterns.name = 'sst' #%% kwrgs_plot_p = kwrgs_plot_c.copy() # if 'cmap' in kwrgs_plot_p.keys(): kwrgs_plot_p.pop('cmap') kwrgs_plot_p.update({'units':'[-] EOF loading pattern', 'zoomregion':(110,250,20,70), 'subtitles':[['PDO pattern (negative phase)']]}) g = plot_maps.plot_corr_maps(PDO_patterns[0], **kwrgs_plot_p) fig_path = os.path.join(rg.path_outsub1, 'PDO_pattern')+rg.figext g.fig.savefig(fig_path, bbox_inches='tight') #%% # df = df_test_b.stack().reset_index(level=1)