import numpy as np
import dapper as dpr
from dapper.mods.KS import Model, Tplot
from dapper.mods.Lorenz96 import LPs
from dapper.tools.localization import nd_Id_localization
from dapper.tools.math import Id_Obs
KS = Model(dt=0.5)
Nx = KS.Nx
# nRepeat=10
t = dpr.Chronology(KS.dt,
dkObs=2,
KObs=2 * 10**4,
BurnIn=2 * 10**3,
Tplot=Tplot)
Dyn = {'M': Nx, 'model': KS.step, 'linear': KS.dstep_dx, 'noise': 0}
X0 = dpr.GaussRV(mu=KS.x0, C=0.001)
Obs = Id_Obs(Nx)
Obs['noise'] = 1
Obs['localizer'] = nd_Id_localization((Nx, ), (4, ))
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0)
HMM.liveplotters = LPs(np.arange(Nx))
####################
# Settings not taken from anywhere
import dapper as dpr
from dapper.mods.LotkaVolterra import step, dstep_dx, x0, LP_setup, Tplot
# dt has been chosen after noting that
# using dt up to 0.7 does not change the chaotic properties much,
# as adjudged with eye-ball and Lyapunov measures.
t = dpr.Chronology(0.5, dtObs=10, T=1000, BurnIn=Tplot, Tplot=Tplot)
Nx = len(x0)
Dyn = {'M': Nx, 'model': step, 'linear': dstep_dx, 'noise': 0}
X0 = dpr.GaussRV(mu=x0, C=0.01**2)
jj = [1, 3]
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = 0.04**2
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0, LP=LP_setup(jj))
####################
# Suggested tuning
####################
# Not carefully tuned:
# xps += EnKF_N(N=6)
# xps += ExtKF(infl=1.02)
"""Setup parameters for twin experiments."""
import numpy as np
import dapper as dpr
from dapper.mods.Ikeda import step, x0, Tplot, LPs
t = dpr.Chronology(1, dkObs=1, KObs=1000, Tplot=Tplot, BurnIn=4 * Tplot)
Nx = len(x0)
Dyn = {'M': Nx, 'model': step, 'noise': 0}
X0 = dpr.GaussRV(C=.1, mu=x0)
jj = np.arange(Nx) # obs_inds
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = .1 # dpr.GaussRV(C=CovMat(1*eye(Nx)))
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0)
HMM.liveplotters = LPs(jj)
####################
# Suggested tuning
####################
statistically converged numbers.
More interesting settings: mods.Lorenz84.harder
"""
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz63 import LPs
from dapper.mods.Lorenz84 import dstep_dx, step, x0
Nx = len(x0)
Ny = Nx
day = 0.05/6 * 24 # coz dt=0.05 <--> 6h in "model time scale"
t = dpr.Chronology(0.05, dkObs=1, T=200*day, BurnIn=10*day)
Dyn = {
'M': Nx,
'model': step,
'linear': dstep_dx,
'noise': 0
}
# X0 = dpr.GaussRV(C=0.01,M=Nx) # Decreased from Pajonk's C=1.
X0 = dpr.GaussRV(C=0.01, mu=x0)
jj = np.arange(Nx)
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = 0.1
import numpy as np
import dapper as dpr
import dapper.tools.utils as utils
from dapper.mods.LorenzUV import model_instance
LUV = model_instance(nU=36, J=10, F=10)
nU = LUV.nU
################
# Full
################
t = dpr.Chronology(dt=0.005, dtObs=0.05, T=4**3, BurnIn=6)
Dyn = {
'M': LUV.M,
'model': dpr.with_rk4(LUV.dxdt, autonom=True),
'noise': 0,
'linear': LUV.dstep_dx,
}
X0 = dpr.GaussRV(mu=LUV.x0, C=0.01)
R = 1.0
jj = np.arange(nU)
Obs = dpr.partial_Id_Obs(LUV.M, jj)
Obs['noise'] = R
# Smaller version.
# => convenient for debugging, e.g., scripts/test_iEnKS.py
import numpy as np
import dapper as dpr
from dapper.mods.LA import Fmat, homogeneous_1D_cov
from dapper.mods.Lorenz96 import LPs
tseq = dpr.Chronology(dt=1, dkObs=5, T=300, BurnIn=-1, Tplot=100)
Nx = 100
# def step(x,t,dt):
# return np.roll(x,1,axis=x.ndim-1)
Fm = Fmat(Nx, -1, 1, tseq.dt)
def step(x, t, dt):
assert dt == tseq.dt
return x @ Fm.T
Dyn = {'M': Nx, 'model': step, 'linear': lambda x, t, dt: Fm, 'noise': 0}
X0 = dpr.GaussRV(mu=np.zeros(Nx),
C=homogeneous_1D_cov(Nx, Nx / 8, kind='Gauss'))
Ny = 4
jj = dpr.linspace_int(Nx, Ny)
Obs = dpr.partial_Id_Obs(Nx, jj)
"""Concerns figure 4 of Todter and Ahrens (2015):
'A Second-Order Exact Ensemble Square Root Filter
for Nonlinear Data Assimilation'"""
import numpy as np
import dapper as dpr
import dapper.tools.randvars as RVs
from dapper.mods.Lorenz96 import step
from dapper.tools.localization import nd_Id_localization
t = dpr.Chronology(0.05, dkObs=2, T=4**5, BurnIn=20)
Nx = 80
Dyn = {'M': Nx, 'model': step, 'noise': 0}
X0 = dpr.GaussRV(M=Nx, C=0.001)
jj = np.arange(0, Nx, 2)
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['localizer'] = nd_Id_localization((Nx, ), (1, ), jj)
# Obs['noise'] = RVs.LaplaceRV(C=1,M=len(jj))
Obs['noise'] = RVs.LaplaceParallelRV(C=1, M=len(jj))
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0)
####################
# Suggested tuning
####################
# rmse.a
############################
# Time series, model, initial condition
############################
model = model_config("sakov2008", {})
Dyn = {
'M': np.prod(shape),
'model': model.step,
'noise': 0,
}
# Considering that I have 8GB mem on the Mac, and the estimate:
# ≈ (8 bytes/float)*(129² float/stat)*(7 stat/k) * K,
# it should be possible to run experiments of length (K) < 8000.
t = dpr.Chronology(dt=model.prms['dtout'], dkObs=1, T=1500, BurnIn=250)
# In my opinion the burn in should be 400.
# Sakov also used 10 repetitions.
X0 = dpr.RV(M=Dyn['M'], file=sample_filename)
############################
# Observation settings
############################
# This will look like satellite tracks when plotted in 2D
Ny = 300
jj = dpr.linspace_int(Dyn['M'], Ny)
# Want: random_offset(t1)==random_offset(t2) if t1==t2.
# Solutions: (1) use caching (ensure maxsize=inf) or (2) stream seeding.
from dapper.mods.LorenzUV import model_instance
LUV = model_instance()
nU = LUV.nU
# Wilks2005 uses dt=1e-4 with RK4 for the full model,
# and dt=5e-3 with RK2 for the forecast/truncated model.
# As berry2014linear notes, this is possible coz
# "numerical stiffness disappears when fast processes are removed".
################
# Full
################
# t = dpr.Chronology(dt=0.001,dtObs=0.05,T=4**3,BurnIn=6) # allows using rk2
t = dpr.Chronology(dt=0.005, dtObs=0.05, T=4**3, BurnIn=6) # requires rk4
Dyn = {
'M': LUV.M,
'model': dpr.with_rk4(LUV.dxdt, autonom=True),
'noise': 0,
'linear': LUV.dstep_dx,
}
X0 = dpr.GaussRV(mu=LUV.x0, C=0.01)
R = 0.1
jj = np.arange(nU)
Obs = dpr.partial_Id_Obs(LUV.M, jj)
Obs['noise'] = R
"""Reproduce results from fig2 of Raanes, Patrick Nima, Alberto Carrassi, and Laurent Bertino (2015): "Extending the square root method to account for additive forecast noise in ensemble methods." """ import numpy as np import dapper as dpr from dapper.mods.LA import Fmat, sinusoidal_sample from dapper.mods.Lorenz96 import LPs from dapper.tools.math import center, tsvd # Burn-in allows damp*x and x+noise balance out tseq = dpr.Chronology(dt=1, dkObs=5, T=500, BurnIn=60, Tplot=100) Nx = 1000 Ny = 40 jj = dpr.linspace_int(Nx, Ny) Obs = dpr.partial_Id_Obs(Nx, jj) Obs['noise'] = 0.01 ################# # Noise setup # ################# # Instead of sampling model noise from sinusoidal_sample(), # we will replicate it below by a covariance matrix approach. # But, for strict equivalence, one would have to use # uniform (i.e. not Gaussian) random numbers. wnumQ = 25
"""Reproduce results from Table 1 of Sakov, Oliver, Bertino (2012):
'An Iterative EnKF for Strongly Nonlinear Systems'"""
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz63 import LPs, Tplot, dstep_dx, step, x0
t = dpr.Chronology(0.01, dkObs=25, KObs=1000, Tplot=Tplot, BurnIn=4 * Tplot)
Nx = len(x0)
Dyn = {'M': Nx, 'model': step, 'linear': dstep_dx, 'noise': 0}
X0 = dpr.GaussRV(C=2, mu=x0)
jj = np.arange(Nx) # obs_inds
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = 2 # dpr.GaussRV(C=CovMat(2*eye(Nx)))
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0)
HMM.liveplotters = LPs(jj)
####################
# Suggested tuning
####################
# from dapper.mods.Lorenz63.sakov2012 import HMM # rmse.a:
# xps += Climatology() # 7.6
# xps += OptInterp() # 1.25
# xps += Var3D(xB=0.1) # 1.04
# Settings not taken from anywhere
import dapper as dpr
from dapper.mods.DoublePendulum import step, x0, LP_setup, dstep_dx
t = dpr.Chronology(0.01, dkObs=100, T=30, BurnIn=10)
Dyn = {
'M': len(x0),
'model': step,
'noise': 0,
'linear': dstep_dx,
}
X0 = dpr.GaussRV(mu=x0, C=0.01**2)
jj = [0, 2]
Obs = dpr.partial_Id_Obs(len(x0), jj)
Obs['noise'] = 0.1**2
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0, LP=LP_setup(jj))
####################
# Suggested tuning
####################
# from dapper.mods.DoublePendulum.settings101 import HMM # Expct rmse.a:
# HMM.t.dkObs = anything
# xps += Climatology() # 5
# xps += OptInterp() # 2.5
"""A land-ocean setup for Lorenz-96 from Anderson's 2009 Tellus A article:
'Spatially and temporally varying adaptive covariance inflation for ensemble filters'
"""
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz96.sakov2008 import X0, Dyn, LPs, Nx, Tplot
from dapper.tools.localization import localization_setup, pairwise_distances
from dapper.tools.viz import xtrema
t = dpr.Chronology(0.05,
dtObs=0.05,
KObs=4000,
Tplot=Tplot,
BurnIn=2000 * 0.05)
# Define obs sites
obs_sites = 0.395 + 0.01 * np.arange(1, 21)
obs_sites *= 40
# Surrounding inds
ii_below = obs_sites.astype(int)
ii_above = ii_below + 1
# Linear-interpolation weights
w_above = obs_sites - ii_below
w_below = 1 - w_above
# Define obs matrix
H = np.zeros((20, 40))
H[np.arange(20), ii_below] = w_below
H[np.arange(20), ii_above] = w_above
# Measure obs-state distances
"""From Fig. 1 of Bocquet 2010 'Beyond Gaussian Statistical Modeling
in Geophysical Data Assimilation'."""
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz96 import step
t = dpr.Chronology(0.05, dkObs=1, T=4**3, BurnIn=20)
Nx = 10
Dyn = {'M': Nx, 'model': step, 'noise': 0}
X0 = dpr.GaussRV(M=Nx, C=0.001)
jj = np.arange(0, Nx, 2)
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = 1.5
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0)
####################
# Suggested tuning
####################
# Why are these benchmarks superior to those in the article?
# We use, in the EnKF,
# - inflation instead of additive noise ?
# - Sqrt instead of perturbed obs
# - random orthogonal rotations.
# The particle filters are also probably better tuned:
# - jitter covariance proportional to ensemble (weighted) cov
"""Reproduce experiments from
'Application of a hybrid EnKF-OI to ocean forecasting'
by F. Counillon, P. Sakov, and L. Bertino (2009)."""
import dapper as dpr
from dapper.mods.QG import model_config
from dapper.mods.QG.sakov2008 import HMM
dt = 1.25 * 10 # 10 steps between obs (also requires dkObs=1)
HMM.t = dpr.Chronology(dt=dt, dkObs=1, T=1000 * dt, BurnIn=10 * dt)
HMM.Dyn.model = model_config("counillon2009_ens", {
"dtout": dt,
'RKH2': 2.0e-11
}).step
truth_model = model_config("counillon2009_truth", {"dtout": dt}).step
####################
# Suggested tuning
####################
# Reproduce Table 1 results.
# - Note that Counillon et al:
# - Report forecast rmse's (but they are pretty close to analysis rmse anyways).
# - Use enkf-matlab which has a bug which cause them to report the
# wrong localization radius (see mods/QG/sakov2008.py).
# Eg. enkf-matlab radius 15 (resp 25) corresponds to
# DAPPER radius 10.6 (resp 17.7).
# R = 17.7 # equiv. to R=25 in enkf-matlab
# from dapper.mods.QG.counillon2009 import HMM, truth_model # rmse.f:
# xps += LETKF(mp=True, N=25,infl=1.15,taper='Gauss',loc_rad=R) # 1.11
'Optimal Sites for Supplementary Weather Observations:
Simulation with a Small Model',
except that the observations are from the entire state.
This HMM is used (with small variations) in many DA papers,
some of which are mentioned below.
"""
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz96 import LPs, Tplot, dstep_dx, step, x0
from dapper.tools.localization import nd_Id_localization
# Sakov uses K=300000, BurnIn=1000*0.05
t = dpr.Chronology(0.05, dkObs=1, KObs=1000, Tplot=Tplot, BurnIn=2*Tplot)
Nx = 40
x0 = x0(Nx)
Dyn = {
'M': Nx,
'model': step,
'linear': dstep_dx,
'noise': 0
}
X0 = dpr.GaussRV(mu=x0, C=0.001)
jj = np.arange(Nx) # obs_inds
Obs = dpr.partial_Id_Obs(Nx, jj)
# "Toward a nonlinear ensemble filter for high-dimensional systems."
# J. Geophys. Res. 108, 8775.
# - LEI, J. & BICKEL, P. (2011).
# "A moment matching ensemble filter for nonlinear non-Gaussian data assimilation."
# Mon. Weather Rev. 139, 3964–73
# - FREI, M. & KUNSCH H. R. (2013).
# "Mixture ensemble Kalman filters"
# Comp. Statist. Data Anal. 58, 127–38.
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz96 import dstep_dx, step
from dapper.tools.localization import nd_Id_localization
t = dpr.Chronology(0.05, dtObs=0.4, T=4**5, BurnIn=20)
Nx = 40
Dyn = {'M': Nx, 'model': step, 'linear': dstep_dx, 'noise': 0}
X0 = dpr.GaussRV(M=Nx, C=0.001)
jj = 1 + np.arange(0, Nx, 2)
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = 0.5
Obs['localizer'] = nd_Id_localization((Nx, ), (2, ), jj)
HMM = dpr.HiddenMarkovModel(Dyn, Obs, t, X0)
####################
# Suggested tuning
"""Reproduce raanes'2016 ("EnRTS and EnKS")"""
import dapper as dpr
from dapper.mods.Lorenz96.sakov2008 import HMM
HMM.t = dpr.Chronology(0.01, dkObs=15, T=4**5, BurnIn=20)
# from dapper.mods.Lorenz96.raanes2016 import HMM
# xps += EnKS ('Sqrt',N=25,infl=1.08,rot=False,Lag=12)
# xps += EnRTS('Sqrt',N=25,infl=1.08,rot=False,cntr=0.99)
# ...
# print_averages(xps,avrgs,[],['rmse.u'])
inspired by MWR 1998 by E. N. Lorenz and K. A. Emanuel:
"Optimal Sites for Supplementary Weather Observations: Simulation with a Small Model"
"""
import numpy as np
import dapper as dpr
from dapper.mods.Lorenz96.sakov2008 import X0, Dyn, LPs, Nx, Tplot
from dapper.tools.localization import nd_Id_localization
# Use small dt to "cope with" ocean sector blow up
# (due to spatially-constant infl)
OneYear = 0.05 * (24 / 6) * 365
t = dpr.Chronology(0.005,
dtObs=0.05,
T=110 * OneYear,
Tplot=Tplot,
BurnIn=10 * OneYear)
land_sites = np.arange(Nx // 2)
ocean_sites = np.arange(Nx // 2, Nx)
jj = land_sites
Obs = dpr.partial_Id_Obs(Nx, jj)
Obs['noise'] = 1
Obs['localizer'] = nd_Id_localization((Nx, ), (1, ), jj)
HMM = dpr.HiddenMarkovModel(Dyn,
Obs,
t,
X0,
# The settings of Pajonk are too easy:
# the average value of trHK is 0.013.
# Here we increase dkObs to make the DA problem more difficult.
import dapper as dpr
from dapper.mods.Lorenz84.pajonk2012 import HMM
HMM.t = dpr.Chronology(0.05, dkObs=10, T=4**5, BurnIn=20)
####################
# Suggested tuning
####################
# xps += ExtKF(infl=8)
# xps += EnKF ('Sqrt',N=10,infl=1.05)
# xps += EnKF ('Sqrt',N=100,rot=True,infl=1.01)
# xps += EnKF_N (N=4)
# xps += PartFilt(N=100, NER=0.4) # add reg!
# xps += PartFilt(N=1000, NER=0.1) # add reg!