def SWOT_observation_error(): """observation error of WSE depending on the L*W of each pixel used sigma*(1/l)*(1/w) l=k*L, w=q*W Rodrigaz et al 2017: According to CaMa k=0.25, q=0.85""" fname=pm.DA_dir()+"/out/"+pm.experiment()+"/assim_out/obs/obs_err.bin" if os.path.isfile(fname): return 0 else: nx,ny,gsize = pm.map_dimension() k=1.00 # assume nearest part to the unit catchment q=1.00 # used 1.0 -> river width variability is 30% ovs_err = 0.10 rivlen=np.fromfile(pm.CaMa_dir()+"/map/glb_15min/rivlen.bin",np.float32).reshape(ny,nx) rivwth=np.fromfile(pm.CaMa_dir()+"/map/glb_15min/rivwth_gwdlr.bin",np.float32).reshape(ny,nx) nextx=(np.fromfile(pm.CaMa_dir()+"/map/glb_15min/nextxy.bin",np.int32).reshape(2,ny,nx)[0]!=-9999)*1.0 rivlen=1.0 #rivlen*1.0e-3 #used as one kilometer rivwth=rivwth*1.0e-3 area=(k*rivlen)*(q*rivwth) obs_err=ovs_err*(1/(k*rivlen+1.0e-20))*(1/(q*rivwth+1.0e-20))*nextx #obs_err=pm.ovs_err()*(1/(area+1.0e-20))*nextx # if water area < 1.0 km2 -> 0.25 obs_err=obs_err*(area>=1.0)*1.0+0.25*(1/(k*rivlen+1.0e-20))*(1/(q*rivwth+1.0e-20))*nextx*(area<1.0)*1.0 obs_err=ma.masked_where(area<0.625,obs_err).filled(0.25) # 25cm for < 1km^2 water area obs_err=obs_err*nextx obs_err=obs_err.astype(np.float32) # obs_err0=obs_err[iy,ix] fname=pm.DA_dir()+"/out/"+pm.experiment()+"/assim_out/obs/obs_err.bin" obs_err.tofile(fname) return 0
def err_rand(ix,iy): """make random values to add to true values""" nx,ny,gsize = pm.map_dimension() fname=pm.DA_dir()+"/out/"+pm.experiment()+"/assim_out/obs/obs_err.bin" obs_err=np.fromfile(fname,np.float32).reshape(ny,nx) obs_err=obs_err*((obs_err<=0.25)*1.0) + 0.25*((obs_err>0.25)*1.0) rand = np.random.normal(0.0,obs_err[iy,ix],1) return rand
def write_txt(inputlist):
yyyy = inputlist[0]
mm = inputlist[1]
dd = inputlist[2]
print("write text file: ", yyyy, mm, dd)
# obs_dir="/cluster/data6/menaka/HydroWeb"
# if pm.obs_name() == "HydroWeb":
# obs_dir="/cluster/data6/menaka/HydroWeb"
# HydroWeb_dir=inputlist[3]
target_dt = datetime.date(int(yyyy), int(mm), int(dd))
txtfile = pm.DA_dir() + "/out/" + pm.experiment(
) + "/assim_out/obs/" + yyyy + mm + dd + ".txt"
# print txtfile
# pnum=len(lname)
# print (pnum)
# print pnum
# == read relevant observation data ==
# print (yyyy,mm,dd,pm.obs_name())
if pm.obs_name() == "HydroWeb":
xlist, ylist, l_wse, m_wse, s_wse, l_sat = HydroWeb_data(yyyy, mm, dd)
if pm.obs_name() == "SWOT":
xlist, ylist, l_wse, m_wse, s_wse, l_sat = swot_data(yyyy, mm, dd)
#--------------
pnum = len(xlist)
# print ('xlist:',pnum, "l_wse:",len(l_wse))
with open(txtfile, "w") as txtf:
# for point in np.arange(pnum):
# # == read relevant observation data ==
# if pm.obs_name() == "HydroWeb":
# # == for HydroWeb data ==
# wseo, mean_wse, std_wse = read_HydroWeb(yyyy,mm,dd,lname[point],lEGM08[point],lEGM96[point],leledif[point])
# else:
# wseo, mean_wse, std_wse = read_HydroWeb(yyyy,mm,dd,lname[point],lEGM08[point],lEGM96[point],leledif[point])
# print (point, lname[point], wseo)
# if wseo == -9999.0:
# continue
# iix=xlist[point]
# iiy=ylist[point]
# mean_wse=np.mean(np.array(lwse))
# std_wse=np.std(np.array(lwse))
# sat=satellite[point]
# pnum=len(xlist)
for point in np.arange(pnum):
iix = xlist[point]
iiy = ylist[point]
wseo = l_wse[point]
mean_wse = m_wse[point]
std_wse = s_wse[point]
sat = l_sat[point]
line = "%04d %04d %10.4f %10.4f %10.4f %s\n" % (
iix, iiy, wseo, mean_wse, std_wse, sat)
txtf.write(line)
print(line)
return 0
def cal_monthly_mean_ens(ens_num): #start_year,end_year,ens_num,months=24):
# calc monthly mean value for two years
start_year=pm.spinup_end_year()
end_year,end_month,end_date=pm.starttime()
#start_year=pm.start_year() #inputlist[0]
#end_year=pm.end_year() #inputlist[1]
#ens_num=inputlist[2]
months=24 #inputlist[3]
#threshold=inputlist[4]
runname=pm.runname(pm.mode())
if runname=="E2O":
nx=1440
ny=720
threshold=0.1
else:
nx=360
ny=180
threshold=1.0e-8
#os.system("rm -Rf ./CaMa_in/ELSE_GPCC/mean_month/*")
mkdir("./CaMa_in/"+runname+"/mean_month")
#os.system("rm -Rf ./CaMa_in/ELSE_KIM2009/mean_month/*")
#threshold=0.1
start_dt=datetime.date(start_year,1,1)
end_dt=datetime.date(end_year,12,31)
ens_char="%03d"%(ens_num)
for month in np.arange(months):
ynow=int(start_year+int(month/12))
ychar="%04d"%(ynow)
mchar="%02d"%((month%12)+1)
#print ychar, mchar
roff_mon=np.zeros([ny,nx],np.float32)#.reshape([180,360])
count=np.zeros([ny,nx],np.float32)#.reshape([180,360])
for day in np.arange(month*30,(month+1)*30):
day_num=day-month*30
running_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (running_dt.year)
mm='%02d' % (running_dt.month)
dd='%02d' % (running_dt.day)
roff=np.fromfile(pm.DA_dir()+"/inp/"+runname+"/Roff/Roff__"+str(yyyy)+str(mm)+str(dd)+ens_char+".one",np.float32).reshape([ny,nx])
roff_mon=roff_mon+roff*(roff>threshold)
count=count+(roff>threshold)
roff_mean=roff_mon/(count+1e-20)
roff_mean=roff_mean.astype(np.float32)
roff_mean=roff_mean+threshold
roff_mean.tofile("./CaMa_in/"+runname+"/mean_month/mean_"+ychar+mchar+ens_char+".bin")
def make_anomaly_data(mode=pm.mode()):
# make directory for mean sfcelv
mkdir("./assim_out/mean_sfcelv/")
# copy the anomaly files
if mode == 1:
# for mean
iname = pm.DA_dir() + "/dat/mean_sfcelv_E2O_" + pm.mapname(
) + "_1980-2014.bin"
oname = "./assim_out/mean_sfcelv/mean_sfcelv.bin"
os.system("cp " + iname + " " + oname)
# for std
iname = pm.DA_dir() + "/dat/std_sfcelv_E2O_" + pm.mapname(
) + "_1980-2014.bin"
oname = "./assim_out/mean_sfcelv/std_sfcelv.bin"
os.system("cp " + iname + " " + oname)
if mode == 2:
# for mean
iname = pm.DA_dir() + "/dat/mean_sfcelv_E2O_" + pm.mapname(
) + "_1980-2014.bin"
oname = "./assim_out/mean_sfcelv/mean_sfcelv.bin"
os.system("cp " + iname + " " + oname)
# for std
iname = pm.DA_dir() + "/dat/std_sfcelv_E2O_" + pm.mapname(
) + "_1980-2014.bin"
oname = "./assim_out/mean_sfcelv/std_sfcelv.bin"
os.system("cp " + iname + " " + oname)
if mode == 3:
# for mean
iname = pm.DA_dir() + "/dat/mean_sfcelv_VIC_BC_" + pm.mapname(
) + "_1979-2013.bin"
oname = "./assim_out/mean_sfcelv/mean_sfcelv.bin"
os.system("cp " + iname + " " + oname)
# for std
iname = pm.DA_dir() + "/dat/std_sfcelv_VIC_BC_" + pm.mapname(
) + "_1979-2013.bin"
oname = "./assim_out/mean_sfcelv/std_sfcelv.bin"
os.system("cp " + iname + " " + oname)
return 0
def save_statistic():
# copy mean and std of simulated WSE
# for anomaly and normalized assimilations
mkdir("./assim_out/mean_sfcelv/")
# if pm.input()=="E2O":
# iname = pm.DA_dir()+"/dat/mean_sfcelv_"+pm.input()+"_"+pm.mapname()+"_2000-2010"+ens_char+".bin
# oname = "./assim_out/mean_sfcelv/meansfcelv"+ens+".bin"
# os.system("cp -r "+pm.DA_dir()+"/dat/mean_sfcelv_"+pm.input()+"_"+pm.mapname()+"_2000-2014.bin ./assim_out/mean_sfcelv/mean_sfcelv.bin")
# os.system("cp -r "+pm.DA_dir()+"/dat/std_sfcelv_"+pm.input()+"_"+pm.mapname()+"_2000-2014.bin ./assim_out/mean_sfcelv/std_sfcelv.bin")
# print ("cp -r "+pm.DA_dir()+"/dat/mean_sfcelv_"+pm.input()+"_"+pm.mapname()+"_2000-2014.bin ./assim_out/mean_sfcelv/mean_sfcelv.bin")
# if pm.input()=="VIC_BC":
# os.system("cp -r "+pm.DA_dir()+"/dat/mean_sfcelv_"+pm.input()+"_"+pm.mapname()+"_1979-2013.bin ./assim_out/mean_sfcelv/mean_sfcelv.bin")
# os.system("cp -r "+pm.DA_dir()+"/dat/std_sfcelv_"+pm.input()+"_"+pm.mapname()+"_1979-2013.bin ./assim_out/mean_sfcelv/std_sfcelv.bin")
# print ("cp -r "+pm.DA_dir()+"/dat/mean_sfcelv_"+pm.input()+"_"+pm.mapname()+"_1979-2013.bin ./assim_out/mean_sfcelv/mean_sfcelv.bin")
# #===========
# # mean
# inputlist=[]
# for ens in np.arange(1,pm.ens_mem(pm.mode())+1):
# ens_char="%03d"%(ens)
# if pm.input()=="E2O":
# iname = pm.DA_dir()+"/dat/mean_"+pm.stat_name()+"_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/mean_sfcelv_E2O_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/mean_sfcelv_cal_E2O_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# oname = "./assim_out/mean_sfcelv/meansfcelvC"+ens_char+".bin"
# if pm.input()=="VIC_BC":
# iname = pm.DA_dir()+"/dat/mean_"+pm.stat_name()+"_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/mean_sfcelv_VIC_BC_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/mean_sfcelv_cal_VIC_BC_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# oname = "./assim_out/mean_sfcelv/meansfcelvC"+ens_char+".bin"
# inputlist.append([iname,oname])
#===========
# mean
inputlist = []
for ens in np.arange(1, pm.ens_mem(pm.mode()) + 1):
ens_char = "%03d" % (ens)
iname = pm.DA_dir() + "/dat/mean_" + pm.stat_name(
) + "_" + ens_char + ".bin"
oname = "./assim_out/mean_sfcelv/meansfcelvC" + ens_char + ".bin"
inputlist.append([iname, oname])
# do parallel
p = Pool(pm.para_nums())
p.map(copy_stat, inputlist)
p.terminate()
# #===========
# # std
# inputlist=[]
# for ens in np.arange(1,pm.ens_mem(pm.mode())+1):
# ens_char="%03d"%(ens)
# if pm.input()=="E2O":
# iname = pm.DA_dir()+"/dat/std_"+pm.stat_name()+"_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/std_sfcelv_E2O_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/std_sfcelv_cal_E2O_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# oname = "./assim_out/mean_sfcelv/stdsfcelvC"+ens_char+".bin"
# if pm.input()=="VIC_BC":
# iname = pm.DA_dir()+"/dat/std_"+pm.stat_name()+"_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/std_sfcelv_VIC_BC_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# # iname = pm.DA_dir()+"/dat/std_sfcelv_cal_VIC_BC_"+pm.mapname()+"_2000-2010_"+ens_char+".bin"
# oname = "./assim_out/mean_sfcelv/stdsfcelvC"+ens_char+".bin"
# inputlist.append([iname,oname])
#===========
# std
inputlist = []
for ens in np.arange(1, pm.ens_mem(pm.mode()) + 1):
ens_char = "%03d" % (ens)
iname = pm.DA_dir() + "/dat/std_" + pm.stat_name(
) + "_" + ens_char + ".bin"
oname = "./assim_out/mean_sfcelv/stdsfcelvC" + ens_char + ".bin"
inputlist.append([iname, oname])
# do parallel
p = Pool(pm.para_nums()) #*cpu_nums())
p.map(copy_stat, inputlist)
p.terminate()
return 0
import sys
from mpl_toolkits.basemap import Basemap
import os
import calendar
#assim_out="assim_out_E2O_womc"
#assim_out="assim_out_E2O_wmc"
#assim_out="assim_out_biased_womc"
#assim_out="assim_out_ECMWF_womc_baised_if_fixed1.10"
#sys.path.append('../'+assim_out+'/')
os.system("ln -sf ../gosh/params.py params.py")
import params as pm
experiment = "E2O_wmc_06"
#assim_out=pm.DA_dir()+"/out/"+pm.experiment()+"/assim_out"
assim_out = pm.DA_dir() + "/out/" + experiment + "/assim_out"
print assim_out
#----
def mk_dir(sdir):
try:
os.makedirs(sdir)
except:
pass
#----
mk_dir(assim_out + "/fig")
mk_dir(assim_out + "/fig/AI")
#----
def prepare_input_old():
# spinup start_year
# simulation end_year
start_year=pm.spinup_end_year()
end_year,end_month,end_date=pm.endtime()
#start_year=pm.start_year()
#end_year=pm.end_year()
start_dt=datetime.date(start_year,1,1)
last_dt=datetime.date(end_year,end_month,end_date)
start=0
last=int((last_dt-start_dt).days)
#--------------
# E2O
if pm.input()=="E2O": # Earth2Observe
#print "E2O"
distopen=pm.distopen(1)
diststd=pm.diststd(1)
true_run=pm.true_run(1) # for true ensemble
runname=pm.runname(1) # get runoff name
# make courrpted/assimilated runoff
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff*"))!=0):
#print "Roff_CORR available"
pass
# corrupted input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR")
#print "E2O/Roff"
# # # dist std for ensembles
# # distopen_ranges={}
# # #open_list=np.setdiff1d(np.arange(1,7+1),[true_run])
# # open_list=np.arange(1,7+1)
# # random_runs=random.sample(open_list,k=2)
# # mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/assim_out/runoff_error")
# # f=open(pm.DA_dir()+"/out/"+pm.experiment()+"/assim_out/runoff_error/ensemble.txt","w")
# # for runens in open_list:
# # ens_char="%03d"%(runens)
# # diststd_num=3
# # distopen_range=rd.normal(1,diststd,diststd_num)
# # distopen_range=np.sort(distopen_range)
# # distopen_range=distopen_range.astype(np.float32)
# # distopen_ranges[ens_char]=distopen_range#[0.25,1.00,1.25]
# # line="%s %3.2f %3.2f %3.2f\n"%(ens_char,distopen_range[0],distopen_range[1],distopen_range[2])
# # f.write(line)
# # f.close()
# # #print "L1141"
# # inputlist=[]
# # for day in np.arange(start,last):
# # target_dt=start_dt+datetime.timedelta(days=day)
# # yyyy='%04d' % (target_dt.year)
# # mm='%02d' % (target_dt.month)
# # dd='%02d' % (target_dt.day)
# # ens_num=1
# # for runens in open_list: #np.arange(1,7+1):
# # #print runens
# # #if runens!=true_run:
# # run_num="%03d"%(runens)
# # iname=pm.DA_dir()+"/inp/"+runname+"/Roff/Roff__"+yyyy+mm+dd+run_num+".one"
# # distopens=distopen_ranges[run_num]
# # for dist in distopens:
# # ens_char="C%03d"%(ens_num)
# # oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one"
# # inputlist.append([iname,oname,str(abs(dist))])
# # ens_num=ens_num+1
distopen_ranges={}
# fname="random_ensemble_E2O.txt"
fname="random_ensemble_E2O_49.txt"
with open(pm.DA_dir()+"/dat/"+fname,"r") as f:
lines=f.readlines()
for line in lines:
line = filter(None,re.split(" ", line))
# print line
runens = int(line[0])
rndnum = float(line[1].strip("\n"))
ens_char="C%03d"%(runens)
distopen_ranges[ens_char]=rndnum
#print "L1141"
inputlist=[]
open_list=np.arange(1,7+1)
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
ens_num=1
for runens in open_list: #np.arange(1,7+1):
#print runens
#if runens!=true_run:
run_num="%03d"%(runens)
iname=pm.DA_dir()+"/inp/"+runname+"/Roff/Roff__"+yyyy+mm+dd+run_num+".one"
# distopens=distopen_ranges[run_num]
# for dist in distopens:
for _ in np.arange(1,7+1):
ens_char="C%03d"%(ens_num)
dist=distopen_ranges[ens_char]
oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one"
inputlist.append([iname,oname,str(abs(dist))])
ens_num=ens_num+1
# do parallel
p=Pool(pm.para_nums())
p.map(copy_runoff,inputlist)
p.terminate()
#---------
# ERA20CM
if pm.input()=="ERA20CM": #ECMWF ERA20CM
distopen=pm.distopen(2)
diststd=pm.diststd(2)
true_run=pm.true_run(2) # for true ensemble
runname=pm.runname(2) # get runoff name
# make courrpted runoff
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff*"))!=0):
pass
# corrupted input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR")
#--
# dist std for ensembles
distopen_ranges={}
#open_list=np.setdiff1d(np.arange(1,10+1),[true_run])
open_list=np.arange(1,10+1)
random_runs=random.sample(open_list,k=2)
for runens in open_list:
ens_char="%03d"%(runens)
diststd_num=2
distopen_range=rd.normal(1,diststd,diststd_num)
distopen_range=np.sort(distopen_range)
distopen_range=distopen_range.astype(np.float32)
distopen_ranges[ens_char]=distopen_range#[0.25,1.00,1.25]
#
inputlist=[]
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
ens_num=1
for runens in open_list: #np.arange(1,10+1):
run_num="%03d"%(runens)
iname=pm.DA_dir()+"/inp/"+runname+"/Roff/Roff__"+yyyy+mm+dd+run_num+".one"
distopens=distopen_ranges[run_num]
for dist in distopens:
ens_char="C%03d"%(ens_num)
oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one"
inputlist.append([iname,oname,str(dist)])
ens_num=ens_num+1
# do parallel
p=Pool(pm.para_nums())
p.map(copy_runoff,inputlist)
p.terminate()
#---------
# VIC BC
if pm.input()=="VIC_BC": #VIC BC
nXX,nYY=1440,720
distopen=1.0 #pm.distopen(3)
diststd=0.25 #pm.diststd(3)
true_run=pm.true_run(3) # for true ensemble
runname=pm.runname(3) # get runoff name
# make courrpted runoff
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff*"))!=0):
pass
# corrupted input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR")
#--
# dist std for ensembles
#distopen_ranges={}
# for 12 months
distopen_range=np.zeros([12,pm.ens_mem(),nYY,nXX],np.float32)
# fname="../../dat/std_runoff_E2O_1980-2000.bin"
# #print fname
# std_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
# fname="../../dat/mean_runoff_E2O_1980-2000.bin"
# #print "L1481",fname
# mean_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
# #print mean_runoff
# std_runoff=ma.masked_where(std_runoff==-9999.0,std_runoff).filled(0.0)
# mean_runoff=ma.masked_where(mean_runoff==-9999.0,mean_runoff).filled(0.0)
for mon in range(1,12+1): # for 12 months
mm="%02d"%(mon)
fname=pm.DA_dir()+"/dat/std_month_runoff_E2O_"+mm+".bin"
#print fname
std_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
fname=pm.DA_dir()+"/dat/mean_month_runoff_E2O_"+mm+".bin"
#print "L1481",fname
mean_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
#print mean_runoff
std_runoff=ma.masked_where(std_runoff==-9999.0,std_runoff).filled(0.0)
mean_runoff=ma.masked_where(mean_runoff==-9999.0,mean_runoff).filled(0.0)
for iXX in range(nXX):
for iYY in range(nYY):
#distopen_range[:,iYY,iXX]=np.sort(rd.normal(distopen,std_runoff[iYY,iXX],pm.ens_mem()))
#Log-normal model
#sk=np.sort(rd.normal(distopen,diststd,pm.ens_mem()))
sk=np.sort(rd.normal(distopen,diststd,pm.ens_mem()))
beta=0.0
#E=std_runoff[iYY,iXX]/(mean_runoff[iYY,iXX]+1.0e-20)
E=diststd
#distopen_range[mon,:,iYY,iXX]=((1+beta)/math.sqrt(E**2+1))*np.exp(math.sqrt(math.log(E**2+1))*sk)
distopen_range[mon-1,:,iYY,iXX]=np.sort(rd.normal(distopen,E,pm.ens_mem()))
# distopen_range[mon-1,:,iYY,iXX]=rd.normal(distopen,E,pm.ens_mem())
#distopen_range[:,iYY,iXX]=np.sort(rd.normal(distopen,diststd,pm.ens_mem()))
#----------
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
mon=int(target_dt.month)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
iname=pm.DA_dir()+"/inp/"+runname+"/Roff/Roff____"+yyyy+mm+dd+".one"
#print iname
roff=np.fromfile(iname,np.float32).reshape(nYY,nXX)
#roff=np.ones([nYY,nXX],np.float32)*-9999.0
fname=pm.DA_dir()+"/dat/std_runoff_E2O_1980-2000.bin"
#print fname
std_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
fname=pm.DA_dir()+"/dat/mean_runoff_E2O_1980-2000.bin"
#print "L1481",fname
mean_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
#print mean_runoff
std_runoff=ma.masked_where(std_runoff==-9999.0,std_runoff).filled(0.0)
mean_runoff=ma.masked_where(mean_runoff==-9999.0,mean_runoff).filled(0.0)
for ens_num in np.arange(pm.ens_mem()):
ens_char="C%03d"%(ens_num+1)
roffc=roff+distopen_range[mon-1,ens_num,:,:]*mean_runoff #*std_runoff #
roffc=roff*distopen_range[mon-1,ens_num,:,:] #*mean_runoff
oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one"
roffc.tofile(oname)
#--------------
# -25% biased runoff experiment
if pm.input()=="ELSE": # Kim 2009/E2O/ERA20CM
#print "-25% biased runoff experiment", pm.true_run(3), pm.runname(3)
distopen=pm.distopen(3)
diststd=pm.diststd(3)
true_run=pm.true_run(3) # for true ensemble
runname=pm.runname(3) # get runoff name
# copy for TRUE simulation
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE/Roff*"))!=0):
pass
# true input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE")
#print true_run
inputlist=[]
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
ens_char="T000"
true_char="%03d"%(true_run)
if runname=="ELSE_KIM2009":
nx,ny=360,180
prefix="Roff____"
suffix=".one"
if runname=="E2O":
nx,ny=1440,720
prefix="Roff__"
suffix="%03d.one"%(true_run)
if runname=="ERA20CM":
nx,ny=360,180
prefix="Roff__"
suffix="%03d.one"%(true_run)
iname=pm.DA_dir()+"/inp/"+runname+"/Roff/"+prefix+yyyy+mm+dd+suffix
oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE/Roff__"+yyyy+mm+dd+ens_char+".one"
inputlist.append([iname,oname,"1.00"])
# do parallel
p=Pool(pm.para_nums())
p.map(copy_runoff,inputlist)
p.terminate()
# calculate mothly mean
cal_monthly_mean(start_year,end_year,24)
# calculate monthly total
cal_monthly_total(start_year,end_year,24)
# make courrpted runoff
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff*"))!=0):
pass
# corrupted input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR")
inputlist=[]
std=rd.normal(0,diststd,pm.ens_mem())
std=std.astype(np.float32)
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
# make random values
#std=np.fromfile("./CaMa_out/randlist.bin",np.float32)
#std=rd.normal(0,diststd,pm.ens_mem())
#std=std.astype(np.float32)
#print std
ifile=np.fromfile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE/Roff__"+yyyy+mm+dd+"T000.one",np.float32).reshape(ny,nx)
roff_mean=np.fromfile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/mean_month/mean_"+yyyy+mm+".bin",np.float32).reshape(ny,nx)
roff_total=np.fromfile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/total_month/total_"+yyyy+mm+".bin",np.float32).reshape(ny,nx)
for ens in np.arange(1,pm.ens_mem()+1):
ens_char="C%03d"%(ens)
#print pm.distopen(),std[ens-1]
ofile=ifile*distopen + roff_mean*std[ens-1]#*10.0
#ofile=ifile*(distopen + std[ens-1])
#ofile=ifile*distopen + roff_total*std[ens-1]
#ofile=ifile*distopen + roff_total*std[ens-1]*0.75
ofile.astype(np.float32)
ofile.tofile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one")
# do parallel
#p=Pool(pm.para_nums())
#p.map(copy_runoff,inputlist)
#p.terminate()
#--------------
# blind runoff experiment
if pm.input()=="ELSE_KIM2009": # differnt yeaer
#distopen=1.0 #0.75 #pm.distopen()
#diststd=0.5 #pm.diststd()
distopen=pm.distopen(4)
diststd=pm.diststd(4)
true_run=pm.true_run(4) # for true ensemble
runname=pm.runname(4) # get runoff name
# copy for TRUE simulation
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE/Roff*"))!=0):
pass
# true input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE")
inputlist=[]
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
ens_char="T000"
iname=pm.DA_dir()+"/inp/"+runname+"/Roff/Roff____"+yyyy+mm+dd+".one"
oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_TRUE/Roff__"+yyyy+mm+dd+ens_char+".one"
inputlist.append([iname,oname,"1.00"])
# do parallel
p=Pool(pm.para_nums())
p.map(copy_runoff,inputlist)
p.terminate()
# calculate mothly mean
cal_monthly_mean(start_year,end_year,24)
# calculate monthly total
cal_monthly_total(start_year,end_year,24)
# make courrpted runoff
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff*"))!=0):
pass
# corrupted input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR")
inputlist=[]
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
# make random values
#std=np.fromfile("./CaMa_out/randlist.bin",np.float32)
std=rd.normal(0,diststd,pm.ens_mem())
std=std.astype(np.float32)
#print std
ifile=np.fromfile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff/Roff____"+yyyy+mm+dd+".one",np.float32).reshape(ny,nx)
roff_mean=np.fromfile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/mean_month/mean_"+yyyy+mm+".bin",np.float32).reshape(ny,nx)
roff_total=np.fromfile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/total_month/total_"+yyyy+mm+".bin",np.float32).reshape(ny,nx)
for ens in np.arange(1,pm.ens_mem()+1):
ens_char="C%03d"%(ens)
#print pm.distopen(),std[ens-1]
ofile=ifile*distopen + roff_mean*std[ens-1]*10.0
#ofile=ifile*(distopen + std[ens-1])
#ofile=ifile*distopen + roff_total*std[ens-1]
#ofile=ifile*distopen + roff_total*std[ens-1]*0.75
ofile.astype(np.float32)
ofile.tofile(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one")
# do parallel
#p=Pool(pm.para_nums())
#p.map(copy_runoff,inputlist)
#p.terminate()
#---------
# ERA5
if pm.input()=="ERA5": #ERA5-Land
nXX,nYY=3600,1800
distopen=pm.distopen(5)
diststd=pm.diststd(5)
true_run=pm.true_run(3) # for true ensemble
runname=pm.runname(3) # get runoff name
# make courrpted runoff
if(len(glob.glob(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff*"))!=0):
pass
# corrupted input file is not ready
# need preparation
# make directories
mkdir(pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR")
#--
# dist std for ensembles
#distopen_ranges={}
# for 12 months
distopen_range=np.zeros([12,pm.ens_mem(),nYY,nXX],np.float32)
# fname="../../dat/std_runoff_E2O_1980-2000.bin"
# #print fname
# std_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
# fname="../../dat/mean_runoff_E2O_1980-2000.bin"
# #print "L1481",fname
# mean_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
# #print mean_runoff
# std_runoff=ma.masked_where(std_runoff==-9999.0,std_runoff).filled(0.0)
# mean_runoff=ma.masked_where(mean_runoff==-9999.0,mean_runoff).filled(0.0)
for mon in range(1,12+1): # for 12 months
mm="%02d"%(mon)
fname=pm.DA_dir()+"/dat/std_month_runoff_E2O_"+mm+".bin"
#print fname
std_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
fname=pm.DA_dir()+"/dat/mean_month_runoff_E2O_"+mm+".bin"
#print "L1481",fname
mean_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
#print mean_runoff
std_runoff=ma.masked_where(std_runoff==-9999.0,std_runoff).filled(0.0)
mean_runoff=ma.masked_where(mean_runoff==-9999.0,mean_runoff).filled(0.0)
for iXX in range(nXX):
for iYY in range(nYY):
#distopen_range[:,iYY,iXX]=np.sort(rd.normal(distopen,std_runoff[iYY,iXX],pm.ens_mem()))
#Log-normal model
#sk=np.sort(rd.normal(distopen,diststd,pm.ens_mem()))
sk=np.sort(rd.normal(distopen,diststd,pm.ens_mem()))
beta=0.0
#E=std_runoff[iYY,iXX]/(mean_runoff[iYY,iXX]+1.0e-20)
E=diststd
#distopen_range[mon,:,iYY,iXX]=((1+beta)/math.sqrt(E**2+1))*np.exp(math.sqrt(math.log(E**2+1))*sk)
distopen_range[mon-1,:,iYY,iXX]=np.sort(rd.normal(distopen,E,pm.ens_mem()))
# distopen_range[mon-1,:,iYY,iXX]=rd.normal(distopen,E,pm.ens_mem())
#distopen_range[:,iYY,iXX]=np.sort(rd.normal(distopen,diststd,pm.ens_mem()))
#----------
for day in np.arange(start,last):
target_dt=start_dt+datetime.timedelta(days=day)
mon=int(target_dt.month)
yyyy='%04d' % (target_dt.year)
mm='%02d' % (target_dt.month)
dd='%02d' % (target_dt.day)
iname=pm.DA_dir()+"/inp/"+runname+"/Roff/Roff____"+yyyy+mm+dd+".sixmin"
#print iname
roff=np.fromfile(iname,np.float32).reshape(nYY,nXX)
#roff=np.ones([nYY,nXX],np.float32)*-9999.0
fname=pm.DA_dir()+"/dat/std_runoff_E2O_1980-2000.bin"
#print fname
std_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
fname=pm.DA_dir()+"/dat/mean_runoff_E2O_1980-2000.bin"
#print "L1481",fname
mean_runoff=np.fromfile(fname,np.float32).reshape(nYY,nXX)
#print mean_runoff
std_runoff=ma.masked_where(std_runoff==-9999.0,std_runoff).filled(0.0)
mean_runoff=ma.masked_where(mean_runoff==-9999.0,mean_runoff).filled(0.0)
for ens_num in np.arange(pm.ens_mem()):
ens_char="C%03d"%(ens_num+1)
roffc=roff+distopen_range[mon-1,ens_num,:,:]*mean_runoff #*std_runoff #
roffc=roff*distopen_range[mon-1,ens_num,:,:] #*mean_runoff
oname=pm.DA_dir()+"/out/"+pm.experiment()+"/CaMa_in/"+runname+"/Roff_CORR/Roff__"+yyyy+mm+dd+ens_char+".one"
roffc.tofile(oname)
return 0
os.remove(dst)
os.symlink(src,dst)
else:
raise
#--
def make_yearlist(syear,smon,sday,N=365):
start_dt=datetime.date(syear,smon,sday)
fname="year_day.txt"
f=open(fname,"w")
for day in np.arange(0,N):
nw_dt = start_dt + datetime.timedelta(days=day)
yyyy,mm,dd = nw_dt.year, nw_dt.month, nw_dt.day
yyyymmdd = "%04d%02d%02d\n"%(yyyy,mm,dd)
f.write(yyyymmdd)
f.close()
#--
slink("../gosh/params.py", "params.py")
import params as pm
# define values
N=366 #2004
exp="E2O_womc_anomalyDA_if3.0"
outdir=pm.DA_dir()+"/out/"+exp
# mkdir stat
mk_dir(outdir+"/assim_out/stat")
# make list of days
#make_yearlist(2004,1,1,N=366)
# compile
#os.system("ifort calc_stat.f90 -o calc_stat -O3 -assume byterecl -heap-arrays -g -traceback -free -parallel")
# run calc_stat
os.system("./calc_stat "+str(N)+" "+str(pm.ens_mem())+" "+outdir+" "+pm.CaMa_dir())
