def extract_pseudo(NN=2000,dir='/work/bb0962/work3/member_relax_3_big/post/',name='member_relax_3_T_2M_ts_monmean_1995.nc',var='T_2M'):
'''
:param nn: number of observations
:return: PO, lon, lat, rlon, rlat pseudo obs and their locations in rotated and regular grid
'''
import numpy as np
import random
random.seed(777)
from RMSE_MAPS_INGO import read_data_from_mistral as rdfm
from CCLM_OUTS import rand_station_locations as rsl
s, t = rsl(N=7000, sed=777)
#TT=t.values()
TT=t
#SS=s.values()
SS=s
from rotgrid import Rotgrid
mapping = Rotgrid(-165.0, 46.0, 0, 0)
for i in range(0, NN):
# print(t.values()[i])
(TT[i], SS[i]) = mapping.transform(TT[i], SS[i])
points=np.zeros((NN,3))
points[:, 1] = SS[0:NN]
points[:, 2] = TT[0:NN]
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(dir, name, var)
print(t_o.shape)
Interp_Vals=np.zeros((NN,12))
Interp_Vals_dirty=np.zeros((NN,12))
noise=np.zeros((NN,12))
from scipy.interpolate import RegularGridInterpolator as RegInt
z=range(0,12)
my_interpolating_function = RegInt((z,rlat_o, rlon_o), t_o, method='nearest')
for i in range(0,12):
points[:, 0] = np.zeros(NN)+i
Interp_Vals[:,i] = my_interpolating_function(points)
for k in range(0,NN):
np.random.seed(777+k)
#noise[k,:] = np.random.normal(0, np.sqrt(np.var(Interp_Vals[k,:])/200), 12)
noise[k,:] = np.random.normal(0, .3, 12)
Interp_Vals_dirty[k,:] = Interp_Vals[k,:] + noise[k,:]
return(Interp_Vals_dirty, Interp_Vals, TT[0:NN], SS[0:NN], t_o, rlon_o, rlat_o)
for i in range(0, month_length):
result = np.zeros((Forecast_3.shape[0], Forecast_3.shape[1]))
# for member in range(0,no_members):
member = no_members - 1
fil = '333333' + str(member) + '_' + str(inflation) + '_' + str(
no_members) + '/inst/' + 'fi' + str(member) + str(i) + '.csv'
result = result + np.array(list(csv.reader(open(fil, "rb"),
delimiter=','))).astype('float')
result_IO[i, :, :] = np.squeeze(t_f[i, :, :]) + (result / no_members)
# plot the ensemble Analysis RMSE :
pdf_name = 'Ensemble_RMSE_last_m100_l20.pdf'
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir='NETCDFS_CCLM/03/member_relax_3_big_00/post/', name=name_2, var=Vari)
dext_lon = t_o.shape[2] - (2 * buffer)
dext_lat = t_o.shape[1] - (2 * buffer)
start_lon = (buffer + 4)
start_lat = (buffer - 4)
forecast = result_IO
obs = t_o[0:month_length, buffer:buffer + dext_lat, buffer:buffer + dext_lon]
RMSE = np.zeros((forecast.shape[1], forecast.shape[2]))
RMSE_TIME_SERIES = np.zeros(forecast.shape[0])
RMSE_TIME_SERIES_Forecast = np.zeros(forecast.shape[0])
for i in range(0, forecast.shape[1]):
for j in range(0, forecast.shape[2]):
forecast_resh = np.squeeze(forecast[:, i, j])
obs_resh = np.squeeze(obs[:, i, j])
RMSE[i, j] = mean_squared_error(
obs_resh,
from RMSE_MAPS_INGO import read_data_from_mistral as rdfm
DIR='/home/fallah/Documents/DATA_ASSIMILATION/Bijan/CODES/Optimal_Interpolation/optiminterp/inst/'
octave.run(DIR+"run_IO.m")
# ============================================= NAMELIST ==========================================
## read forecast :
SEAS='DJF'
NN=1000#number of observations should be read from previous funcions!!!!
#NN=600
#month_length=12
month_length=20
name_1 = 'member04_relax_3_T_2M_ts_splitseas_1984_2014_' + SEAS + '.nc'
name_2 = 'member_relax_3_T_2M_ts_splitseas_1984_2014_' + SEAS + '.nc'
t_f, lat_f, lon_f, rlat_f, rlon_f =rdfm(dir='/work/bb0962/work4/member04_relax_3_big/post/',
name=name_1,
var='T_2M')
# =================================================================================================
print(t_f.shape)
## add correction to forecast :
result_IO = t_f[0:month_length,:,:] - t_f[0:month_length,:,:]
import os.path
import csv
import numpy
from sklearn.metrics import mean_squared_error
if os.path.isfile(DIR+'fi'+str(month_length-1)+'.csv')==True:
for i in range(0,month_length):
print(i)
fil=DIR + 'fi' + str(i) + '.csv'
result=numpy.array(list(csv.reader(open(fil,"rb"),delimiter=','))).astype('float')
def extract_pseudo(
NN=2000,
dir='/work/bb1029/b324045/work5/03/member_relax_3_big_00/post/',
name='member_relax_3_big_00_T_2M_ts_splitseas_1979_2015_DJF.nc',
var='T_2M',
month_length=20,
buffer=20):
'''
:param nn: number of observations, no: number of members
:return: PO, lon, lat, rlon, rlat pseudo obs and their locations in rotated and regular grid
'''
#import math
import numpy as np
import matplotlib.pyplot as plt
#import random
#import scipy.spatial as spatial
#random.seed(770)
from RMSE_MAPS_INGO import read_data_from_mistral as rdfm
from CCLM_OUTS import rand_station_locations as rsl
s, t = rsl(N=7000, sed=770)
#TT=t.values()
TT = t
#SS=s.values()
SS = s
from rotgrid import Rotgrid
mapping = Rotgrid(
-165.0, 46.0, 0,
0) # TDOD: this line has to be adopted according to the CCLM domain!!!
for i in range(0, NN):
(TT[i], SS[i]) = mapping.transform(TT[i], SS[i])
points = np.zeros((NN, 3))
#TODO: data-thining
# thin the data to let one obs in each grid:
#fert_ok = 0
poi = np.array([SS[0:NN], TT[0:NN]])
points[:, 1] = SS[0:NN]
points[:, 2] = TT[0:NN]
# t_o_1, lat_o_1, lon_o_1, rlat_o_1, rlon_o_1 = rdfm #added for correcting the edges for 4x36 years run !!!!
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(dir, name, var)
#t_o = t_o.data
#t_o[t_o==-9999]=float('nan')
#t_o[np.isnan(t_o)]=np.nanmean(t_o)
Interp_Vals = np.zeros((NN, month_length))
Interp_Vals_dirty = np.zeros((NN, month_length))
noise = np.zeros((NN, month_length))
from scipy.interpolate import RegularGridInterpolator as RegInt
z = range(0, month_length)
my_interpolating_function = RegInt(
(z, rlat_o[buffer:-buffer], rlon_o[buffer:-buffer]),
t_o[0:month_length, buffer:-buffer, buffer:-buffer],
method='nearest')
for i in range(0, month_length):
points[:, 0] = np.zeros(NN) + i
Interp_Vals[:, i] = my_interpolating_function(points)
for k in range(0, NN):
noise[k, :] = np.random.normal(
0, .3, month_length) # for sesonal values T_2M summer JJA
Interp_Vals_dirty[k, :] = Interp_Vals[k, :] + noise[k, :]
return (Interp_Vals_dirty, Interp_Vals, TT[0:NN], SS[0:NN],
t_o[0:month_length, :, :], rlon_o, rlat_o)
t_f_DJF = t_f_JJA - t_f_JJA
for month in range(
0, month_length): # Reading the ensemble forecast for each month!
t_f_JJA[month, :, :] = pd.read_csv(DIR1 + 'test01_5.00_1.7_500_19_1.0_20' +
'/' + 'Trash/SEASON_MEAN' + str(month) +
'_' + SEAS1 + '.csv',
header=None)
t_f_DJF[month, :, :] = pd.read_csv(DIR2 + 'test01_5.00_2.1_500_19_1.0_20' +
'/' + 'Trash/SEASON_MEAN' + str(month) +
'_' + SEAS2 + '.csv',
header=None)
t_f_JJA = np.array(t_f_JJA)
t_f_DJF = np.array(t_f_DJF)
name_1 = 'member_relax_3_big_00_' + VAR + '_ts_splitseas_1990_1999_' + SEAS1 + '.nc'
t_o_JJA, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir='NETCDFS_CCLM/03/member_relax_3_big_00/post/', name=name_1, var=VAR)
name_2 = 'member_relax_3_big_00_' + VAR + '_ts_splitseas_1990_1999_' + SEAS2 + '.nc'
t_o_DJF, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir='NETCDFS_CCLM/03/member_relax_3_big_00/post/', name=name_2, var=VAR)
dext_lon = t_o_DJF.shape[2] - (2 * buffer)
dext_lat = t_o_DJF.shape[1] - (2 * buffer)
start_lon = (buffer + 4)
start_lat = (buffer - 4)
obs_JJA = t_o_JJA[0:10, buffer:buffer + dext_lat, buffer:buffer + dext_lon]
RMSE_TIME_SERIES_Forecast_JJA = np.zeros(obs_JJA.shape[0])
obs_DJF = t_o_DJF[0:10, buffer:buffer + dext_lat, buffer:buffer + dext_lon]
RMSE_TIME_SERIES_Forecast_DJF = np.zeros(obs_DJF.shape[0])
# run octave from python (IO code )
from oct2py import octave
import numpy as np
from RMSE_MAPS_INGO import read_data_from_mistral as rdfm
DIR = '/home/fallah/Documents/DATA_ASSIMILATION/Bijan/CODES/Optimal_Interpolation/optiminterp/inst/'
octave.run(DIR + "run_IO.m")
## read forecast :
NN = 1000 #number of observations should be read from previous funcions!!!!
#NN=600
month_length = 12
t_f, lat_f, lon_f, rlat_f, rlon_f = rdfm(
dir='/work/bb0962/work3/member04_relax_3_big/post/',
name='member04_relax_3_T_2M_ts_monmean_1995.nc',
var='T_2M')
print(t_f.shape)
## add correction to forecast :
result_IO = t_f - t_f
import os.path
import csv
import numpy
from sklearn.metrics import mean_squared_error
if os.path.isfile(DIR + 'fi' + str(month_length - 1) + '.csv') == True:
for i in range(0, month_length):
print(i)
fil = DIR + 'fi' + str(i) + '.csv'
result = numpy.array(list(csv.reader(open(fil, "rb"),
delimiter=','))).astype('float')
result_IO[i, :, :] = np.squeeze(t_f[i, :, :]) + result
#print(max(result_IO[i,:,:]))
#print(max(result))
buf = 20
name_2 = 'member_relax_3_big_00_T_2M_ts_splitseas_1990_1999_' + SEAS + '.nc'
PDF = 'Stations.pdf'
Plot_CCLM(dir_mistral='NETCDFS_CCLM/03/member_relax_3_big_00/post/',
name=name_2,
bcolor='black',
var='T_2M',
flag='FALSE',
color_map='TRUE',
alph=1,
grids='FALSE',
grids_color='red',
rand_obs='TRUE',
NN=NN)
col = ['k', 'r', 'b', 'g', 'm']
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir='NETCDFS_CCLM/03/member_relax_3_big_00/post/', name=name_2, var='T_2M')
for i in [1, 3]:
direc = i
name_1 = 'member_relax_' + str(direc) + '_big_0' + str(
shift) + '_' + Vari + '_ts_splitseas_1990_1999_' + SEAS + '.nc'
# ==============================================================================================
t_f, lat_f, lon_f, rlat_f, rlon_f = rdfm(
dir='NETCDFS_CCLM/0' + str(direc) + '/member_relax_' + str(direc) +
'_big_0' + str(shift) + '/post/',
name=name_1,
var='T_2M')
plt.hlines(y=min(rlat_f),
SEAS = 'DJF'
#SEAS='JJA'
Vari = 'T_2M'
#Vari = 'TOT_PREC'
buffer = 20
#tg_0.44deg_rot_v15.0_JJA_1979_2015_remapbil.nc
name_2 = 'tg_0.44deg_rot_v15.0_' + SEAS + '_1979_2015_remapbil.nc'
PDF1 = 'ENSEMBLE_RMSE_' + SEAS + '_' + Vari + '.pdf'
PDF2 = 'ENSEMBLE_SPREAD_' + SEAS + '_' + Vari + '.pdf'
timesteps = 10 # number of the seasons (years)
start_time = 0
#t_o, lat_o, lon_o, rlat_o, rlon_o =rdfm(dir='/work/bb1029/b324045/work5/03/member_relax_3_big_00/post/', # the observation (default run without shifting)
# name=name_2,
# var=Vari)
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir='NETCDFS_CCLM/eobs/', # the observation (default run without shifting)
name=name_2,
var=Vari)
##TODO: make it a function:
#def f(x):
# if x==-9999:
# return float('NaN')
# else:
# return x
#f2 = np.vectorize(f)
#t_o= f2(t_o)
#print(t_o)
#t_o=np.array(t_o.data)
#print(t_o)
#t_o[t_o<-900]=float('NaN')
#t_o[np.isnan(t_o)]=np.nanmean(np.nanmean(t_o,axis=(0,1)))
NN = 1000
SEAS = 'DJF'
#SEAS='JJA'
Vari = 'T_2M'
#Vari = 'TOT_PREC'
month_length = 20 # number of the seasons (years)
start_time = 0
here = "path_dir" + "/"
no_members = 20
buffer = 20
# -----------------------------------------------------------------------------
name_2 = 'member_relax_3_big_00_' + Vari + '_ts_splitseas_1990_1999_' + SEAS + '.nc'
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir=
'NETCDFS_CCLM/03/member_relax_3_big_00/post/', # the observation (default run without shifting)
name=name_2,
var=Vari)
forecast = genfromtxt(here + 'NAMES' + '/' + "Trash/Forecast_0_1_1_" + SEAS +
".csv",
delimiter=",")
dumm2 = np.zeros((forecast.shape[0], forecast.shape[1]))
time_series = np.zeros((month_length, no_members))
time_series_min = np.zeros(month_length)
time_series_max = np.zeros(month_length)
time_series_mean = np.zeros(month_length)
time_series_Nature = np.zeros(month_length)
for ii in range(0, month_length):
counter = 0
for kk in range(1, 6):
NN = 500
SEAS = 'DJF'
#SEAS='JJA'
Vari = 'T_2M'
#Vari = 'TOT_PREC'
month_length = 36 # number of the seasons (years)
start_time = 0
here = "." + "/"
no_members = 4
buffer = 20
# -----------------------------------------------------------------------------
name_2 = 'member_relax_3_big_00_' + Vari + '_ts_splitseas_1979_2015_' + SEAS + '.nc'
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir=
'NETCDFS_CCLM/03/member_relax_3_big_00/post/', # the observation (default run without shifting)
name=name_2,
var=Vari)
name_2 = 'member_relax_1_big_04_' + Vari + '_ts_splitseas_1979_2015_' + SEAS + '.nc'
t_1, lat_1, lon_1, rlat_1, rlon_1 = rdfm(
dir='NETCDFS_CCLM/01/member_relax_1_big_04/post/', name=name_2, var=Vari)
name_2 = 'member_relax_2_big_04_' + Vari + '_ts_splitseas_1979_2015_' + SEAS + '.nc'
t_2, lat_2, lon_2, rlat_2, rlon_2 = rdfm(
dir='NETCDFS_CCLM/02/member_relax_2_big_04/post/', name=name_2, var=Vari)
name_2 = 'member_relax_3_big_04_' + Vari + '_ts_splitseas_1979_2015_' + SEAS + '.nc'
t_3, lat_3, lon_3, rlat_3, rlon_3 = rdfm(
dir='NETCDFS_CCLM/03/member_relax_3_big_04/post/', name=name_2, var=Vari)
name_2 = 'member_relax_4_big_04_' + Vari + '_ts_splitseas_1979_2015_' + SEAS + '.nc'
t_4, lat_4, lon_4, rlat_4, rlon_4 = rdfm(
dir='NETCDFS_CCLM/04/member_relax_4_big_04/post/', name=name_2, var=Vari)
fil = DIR + 'fi' + str(member) + str(i) + '.csv'
result = np.array(list(csv.reader(open(fil, "rb"),
delimiter=','))).astype('float')
result_IO[i, :, :] = np.squeeze(t_f[i, :, :]) + result
# plot differences
pdf_name = 'last_m100_l20_' + str(member) + '.pdf'
#t_o, lat_o, lon_o, rlat_o, rlon_o =rdfm(dir='/work/bb1029/b324045/work5/03/member_relax_3_big_00/post/', # the observation (default run without shifting)
# name=name_2,
# var=Vari)
#t_o, lat_o, lon_o, rlat_o, rlon_o =rdfm(dir='NETCDFS_CCLM/03/member_relax_3_big_00/post/', # the observation (default run without shifting)
# name=name_2,
# var=Vari)
t_o, lat_o, lon_o, rlat_o, rlon_o = rdfm(
dir='/NETCDFS_CCLM/eobs/', # the observation (default run without shifting)
name=name_2,
var=Vari)
dext_lon = t_o.shape[2] - (2 * buffer)
dext_lat = t_o.shape[1] - (2 * buffer)
start_lon = (buffer + 4)
start_lat = (buffer - 4)
##TODO: make it a function:
#def f(x):
# if x==-9999:
# return float('NaN')
# else:
# return x
#f2 = np.vectorize(f)
#t_o= f2(t_o)