def export_html_diags(self, htmlfile, **kwargs):
"""Make and export diagnostics as an html file
Parameters
----------
htmlfile: string
Output html file name
**kwargs
All other params are passed to :meth:`plot_diags`
Return
------
string
Html file name
"""
# Get figures
figs = self.plot_diags(**kwargs)
figs = {'Ensemble diagnostics': figs}
# Figure paths
figs = dicttree_relpath(figs, os.path.dirname(htmlfile))
# Render with template
checkdir(htmlfile)
render_and_export_html_template('dict2tree.html',
htmlfile,
title='Ensemble diagnostics',
content=figs)
self.created(htmlfile)
return htmlfile
def write_usages(app):
for name in status_iterator(
app.config.sonathelps_commands,
bold("generating helps... "),
length=len(app.config.sonathelps_commands),
):
# Get help string
opts = app.config.sonathelps_commands[name]
with capture() as out:
try:
sonat_main(opts)
except SystemExit:
pass
# Write help as rst
help_file = app.config.sonathelps_help_pat.format(name)
if help_file:
help_file = os.path.join(app.builder.srcdir, help_file)
checkdir(help_file, asfile=True)
prog = ' '.join(['sonat'] + opts[:-1])
f = open(help_file, 'w')
f.write(opt2rst(out[0], prog=prog))
f.close()
# Write usage
usage_file = app.config.sonathelps_usage_pat.format(name)
if usage_file:
usage_file = os.path.join(app.builder.srcdir, usage_file)
checkdir(usage_file, asfile=True)
f = open(usage_file, 'w')
f.write(out[0])
f.close()
def write_options(app):
app.info(bold('writing config options to rst... '), nonl=True)
config_directives = os.path.join(app.env.srcdir,
app.config.sonatconfig_options_file)
checkdir(config_directives)
f = open(config_directives, 'w')
f.write(SONAT_CFGM.get_rst(mode='specs'))
f.close()
app.info('done into ' + os.path.basename(config_directives))
def write_ncdumps(app):
for name in app.builder.status_iterator(
app.config.sonatncdumph_ncfiles,
bold("generating list of ncdump -h... "),
length=len(app.config.sonatncdumph_ncfiles),
):
txtfile = app.config.sonatncdumph_filepat.format(name)
txtfile = os.path.join(app.builder.srcdir, txtfile)
checkdir(txtfile, asfile=True)
ncfile = app.config.sonatncdumph_ncfiles[name]
if not os.path.isabs(ncfile):
ncfile = os.path.join(app.builder.srcdir, ncfile)
os.system('ncdump -h {ncfile} > {txtfile}'.format(**locals()))
def write_content(app):
app.info(bold('writing default config to rst... '), nonl=True)
config_rstfile = os.path.join(app.env.srcdir,
app.config.sonatconfig_content_file)
checkdir(config_rstfile)
s = StringIO()
SONAT_CFGM.defaults().write(s)
s = s.getvalue()
s = s.replace(os.environ['USER'], '${USER}')
f = open(config_rstfile, 'w')
f.write(s)
f.close()
app.info('done into ' + os.path.basename(config_rstfile))
def arm_sa_from_cfg(cfg, sa_names=None):
"""Perform ARM sensitivity analyses
Parameters
----------
cfg: configobj/dict
sa_names: strings, None
Force the list of registered sensitivity analyserrs
Return
------
string
Path to HTML file
"""
# Config
cfga = cfg['arm']
cfgas = cfga['sa']
cfge = cfg['ens']
cfgo = cfg['obs']
cfgop = cfgo['plots']
lon = get_cfg_xminmax(cfg)
lat = get_cfg_yminmax(cfg)
ncensfile = get_cfg_path(cfg, 'ens', 'ncensfile')
varnames = cfge['varnames'] or None
norms = get_cfg_norms(cfg)
htmlfile = get_cfg_path(cfg, ['arm', 'sa'], 'htmlfile')
# Init
logger = init_from_cfg(cfg)
# Load ensemble
ens = Ensemble.from_file(ncensfile,
varnames=varnames,
logger=logger,
lon=lon,
lat=lat)
# Load obs manager
obs = load_obs_from_cfg(cfg)
# Init ARM
arm = ARM(ens, obs, norms=norms, logger=logger)
# Bathy
bathy = read_bathy_from_cfg(cfg, logger)
if bathy is not None:
arm.set_bathy(bathy)
# SA names
if sa_names is None:
sa_names = list_arm_sensitivity_analysers()
# Loop on analysers
res = OrderedDict()
for sa_name in sa_names:
# Format paths
rebase_cfg_paths(cfg, ['arm', 'sa', sa_name])
# Config
kwargs = cfgas[sa_name].dict()
activate = kwargs.pop('activate')
if not activate:
continue
# Setup analyser
sa = get_arm_sensitivity_analyser(sa_name, arm, logger=logger)
# Run and export
res.update(
sa.plot(lon=lon,
lat=lat,
obs_color=cfgop['colorcycle'],
obs_marker=cfgop['markercycle'],
**kwargs))
# Paths
checkdir(htmlfile)
res = dicttree_relpath(res, os.path.dirname(htmlfile))
res = {"ARM sensitivity analyses": res}
# Render with template
render_and_export_html_template('dict2tree.html',
htmlfile,
title="ARM sensitivity analyses",
content=res)
arm.created(htmlfile)
return htmlfile
xisize = 1. * xsize / dpi
yisize = xisize * aspect
sonat_size = 145. * xisize / 8.
o_size = 50 * xisize / 8.
figfiles = []
P.figure(figsize=(xisize, yisize), dpi=dpi)
kw = dict(ha='center', va='center', color=color, weight='bold')
ts = P.figtext(.5, .41, 'SONAT', size=sonat_size, **kw)
to = P.figtext(.302, .49, 'O', size=o_size, **kw)
shadow = ''
bgcolor = 'transp'
figfiles.append(figpat.format(**locals()))
checkdir(figfiles[-1], asfile=True)
P.savefig(figfiles[-1], transparent=True)
bgcolor = 'white'
figfiles.append(figpat.format(**locals()))
P.savefig(figfiles[-1])
bgcolor = 'blue'
figfiles.append(figpat.format(**locals()))
P.savefig(figfiles[-1], facecolor="#2882B9")
shadow = True
add_shadow(ts, width=9, alpha=.5, xoffset=6, yoffset=-6)
add_shadow(to, width=9, alpha=.5, xoffset=6, yoffset=-6)
P.axis('off')
shadow = '-shadow'
def plot_diags(
self,
mean=True,
variance=True,
kurtosis=True,
skew=True,
skewtest=True,
kurtosistest=True,
normaltest=True,
titlepat='{var_name} - {diag_long_name} - {slice_loc}',
surf=None,
bottom=None,
horiz_sections=None, #points=None,
zonal_sections=None,
merid_sections=None,
fmtlonlat='{:.2f}',
fmtdep='{:04.0f}m',
figpat_slice='sonat.ens.{diag_name}_{var_name}_{slice_type}_{slice_loc}.png',
figpat_generic='sonat.ens.{diag_name}.png',
show=False,
savefig=True,
props=None,
**kwargs):
"""Create figures for diagnostics
Diagnostic arguments are passed to :meth:`get_diags`,
like ``variance=True``.
Parameters
----------
surf: bool
bottom: bool
horiz_sections: floats
List of depths for horizontal slices. All by default.
It accepts scalars or list of floats and of the two special values
"bottom" and "surf". In the latter case, these variable must
be explicitely included in the ensemble, like "temp_surf". A depth
of 0 is not equivalent to "surf", despite results may be similar,
or equivalent when the sea level is always 0.
In the case of floats, the
ensemble must contain 3d variables.
zonal_sections: None, list, floats
Latitudes
merid_sections: None, list, floats
Longitudes
props: dict, None
Dictionary of graphic properties passed as keywords to plotting
functions. The keys must be valid diagnostics names such as
"mean", or one of the follow plotting function: map, curve.
Return
------
dict
A tree of :class:`dict` with the following branch types,
with all keys capitalised:
- Diagnostic
- Variable
- Type of slice
- Location of slice
"""
# Diags
diags = self.get_diags(mean=mean,
variance=variance,
kurtosis=kurtosis,
skew=skew,
skewtest=skewtest,
kurtosistest=kurtosistest,
normaltest=normaltest)
# Inits
self.verbose('Plotting ensemble diagnostics')
figs = OrderedDict()
if props is None:
props = {}
kwprops = dict_merge(props, DEFAULT_PLOT_KWARGS_PER_ENS_DIAG)
# Loop on diags
figs = OrderedDict()
for diag_name, diag_var in diags.items():
diag_long_name = diag_name.title().replace('_', ' ')
self.debug('Plotting ens diag: ' + diag_long_name)
# Explained variance
if diag_name == 'spectrum':
self.debug(' Variable: spectrum')
figfile = figpat_generic.format(**locals())
checkdir(figfile)
kw = kwprops.get(diag_name, {})
dict_check_defaults(kw,
xmin=.5,
xmax=len(diag_var) + .5,
width=.5,
xminor_locator=False,
ymax=1.1 * diag_var.max(),
savefig=figfile,
fig='new',
xlabel='%(xlong_name)s',
xlocator=MultipleLocator(1),
**DEFAULT_PLOT_KWARGS)
p = bar(diag_var, **kw)
self.created(figfile)
figs[diag_long_name] = figfile
# Plot other sliced variables
else:
ff = self.plot_fields(
diag_var,
surf=surf,
bottom=bottom,
horiz_sections=horiz_sections,
zonal_sections=zonal_sections,
merid_sections=merid_sections,
# points=points,
figpat=figpat_slice,
savefig=savefig,
titlepat=titlepat,
fmtlonlat=fmtlonlat,
fmtdep=fmtdep,
subst={
'diag_long_name': diag_long_name,
'diag_name': diag_name
},
props=props,
)
if ff:
figs[diag_long_name] = ff
return figs
def generate_pseudo_ensemble(ncpat,
varnames=None,
nrens=50,
enrich=2.,
norms=None,
getmodes=False,
logger=None,
asdicts=False,
anomaly=True,
ncensfile=None,
**kwargs):
"""Generate a static pseudo-ensemble from a single simulation
Parameters
----------
ncpat: string
netcdf file name or pattern
nrens: int
Ensemble size
enrich: float
Enrichment factor
getmodes: bool
Get also EOFs end eigen values
**kwargs:
Extra parameters are passed to :func:`load_model_at_dates`
Return
------
list (or dict) of arrays:
variables with their name as keys
dict: eofs, ev and variance, optional
eofs: list (or dict) of arrays(nmodes, ...), optional
EOFs
ev: array(nmodes), optional
Eigen values
var: array
Variance
"""
# Logger
kwlog = kwfilter(kwargs, 'logger_')
if logger is None:
logger = get_logger(**kwlog)
logger.verbose('Generating pseudo-ensemble')
# Ensembe size
enrich = max(enrich, 1.)
nt = int(nrens * enrich)
logger.debug(
' enrich={enrich}, nt={nt}, ncpat={ncpat}, varnames={varnames}'.
format(**locals()))
# Read variables
logger.debug('Reading the model at {} dates'.format(nt))
data = load_model_at_regular_dates(ncpat,
varnames=varnames,
nt=nt,
asdict=False,
**kwargs)
single = not isinstance(data, list)
# Norms
if isinstance(norms, dict):
norms = var_prop_dict2list(data, norms)
# Enrichment
witheofs = nrens != nt
if witheofs:
logger.debug('Computing reduced rank ensemble with EOFs analysis')
# Stack packed variables together
stacker = Stacker(data, norms=norms, logger=logger)
meanstate = N.zeros(stacker.ns)
states = N.asfortranarray(stacker.stacked_data.copy())
# Compute EOFs
stddev, svals, svecs, status = f_eofcovar(dim_fields=stacker.ns,
offsets=1,
remove_mstate=0,
do_mv=0,
states=states,
meanstate=meanstate)
if status != 0:
raise SONATError('Error while calling fortran eofcovar routine')
neof = svals.size # computed
neofr = nrens - 1 # retained
svals = svals[:neofr] * N.sqrt(
(neof - 1.) / neof) # to be consistent with total variance
svecs = svecs[:, :neofr]
# Generate ensemble
sens = f_sampleens(svecs, svals, meanstate, flag=0)
# Unstack
ens = stacker.unstack(sens,
format=2,
rescale='norm' if anomaly else True)
if getmodes:
# Modes
mode_axis = create_axis(N.arange(1, neofr + 1, dtype='i'),
id='mode')
eofs = stacker.unstack(svecs,
firstdims=mode_axis,
id='{id}_eof',
rescale=False,
format=1)
svals = MV2.array(svals,
axes=[mode_axis],
id='ev',
attributes={'long_name': 'Eigen values'})
svals.total_variance = float(stacker.ns)
# Variance
vv = stacker.format_arrays([d.var(axis=0) for d in stacker.datas],
id='{id}_variance',
mode=1)
variance = stacker.unmap(vv)
else: # No enrichment -> take the anomaly if requested
logger.debug('Getting the anomaly to build the ensemble')
ens = data
if anomaly:
if single:
ens[:] = ens.asma() - ens.asma().mean(axis=0)
else:
for i, e in enumerate(ens):
ens[i][:] = e.asma() - e.asma().mean(axis=0)
# Finalize
getmodes = getmodes and witheofs
member_axis = create_axis(N.arange(nrens, dtype='i'),
id='member',
long_name='Member')
if single:
ens.setAxis(0, member_axis)
else:
for var in ens:
var.setAxis(0, member_axis)
# Dump to file
if ncensfile:
logger.debug('Dump the ensemble to netcdf')
checkdir(ncensfile)
f = cdms2.open(ncensfile, 'w')
ensvars = list(ens) if not single else [ens]
if getmodes:
if single:
ensvars.append(eofs)
ensvars.append(variance)
else:
ensvars.extend(eofs)
ensvars.extend(variance)
ensvars.append(svals)
for var in ensvars:
f.write(var)
f.close()
logger.created(ncensfile)
# As dicts
if asdicts:
if single:
ens = OrderedDict([(ens.id, ens)])
if getmodes:
eofs = OrderedDict([(eofs.id, eofs)])
variance = OrderedDict([(variance.id, variance)])
else:
ens = OrderedDict([(var.id, var) for var in ens])
if getmodes:
eofs = OrderedDict([(var.id, var) for var in eofs])
variance = OrderedDict([(var.id, var) for var in variance])
# Return
if not getmodes:
return ens
return ens, dict(eofs=eofs, eigenvalues=svals, variance=variance)
def plot_fields(self,
variables,
surf=None,
bottom=None,
horiz_sections=False,
points=None,
zonal_sections=None,
merid_sections=None,
titlepat="{var_name} - {slice_loc}",
props=None,
subst={},
figpat='sonat.ens.{var_name}_{slice_type}_{slice_loc}.png',
fmtlonlat=u'{:.2f}°{}',
fmtdep='{:04.0f}m',
savefig=True,
obs=None,
close=True,
show=False,
**kwargs):
"""Plot one or several platform like variables
Parameters
----------
depths: string, floats
List of depths for horizontal slices. All by default.
It accepts scalars or list of floats and of the two special values
"bottom" and "surf". In the latter case, these variable must
be explicitely included in the ensemble, like "temp_surf". A depth
of 0 is not equivalent to "surf", despite results may be similar,
or equivalent when the sea level is always 0.
In the case of floats, the
ensemble must contain 3d variables.
props: dict, None
Dictionary of graphic properties passed as keywords to plotting
functions. The keys must be valid diagnostics names such as
"mean", or one of the follow plotting function: map, curve.
"""
# Init
if props is None:
props = {}
kwprops = dict_merge(props, DEFAULT_PLOT_KWARGS_PER_ENS_DIAG)
kwprops.update(fig='new') # TODO: must be more flexible like for obs
figs = OrderedDict()
kwobs = kwfilter(kwargs, 'obs')
kwobs.update(surf=False,
bottom=False,
zonal_sections=None,
merid_sections=None,
horiz_sections=None)
# Loop on variables
for variable in ArgList(variables).get():
var_short_name, var_depth, diag_name = split_varname(variable)
diag_name = getattr(variable, 'sonat_ens_diag', diag_name)
var_name = (var_short_name if var_depth is None else
(var_short_name + '_' + var_depth))
varname = var_name
id = variable.id
self.debug(' Variable: ' + id)
order = variable.getOrder()
if diag_name:
kw = kwprops.get(diag_name, {})
else:
kw = {}
toplot = []
# Maps
if (horiz_sections is not False or surf is not False
or bottom is not False):
# Surf/bottom/3D
if ((var_depth == 'surf' and surf is not False) or
(var_depth == 'bottom' and bottom is not False)): # 2D
slice_loc = var_depth
toplot.append(
dict(slice_loc=slice_loc,
kw=kw,
keys=(var_short_name, 'map', slice_loc),
obs_plot={var_depth: True},
slice_type='map'))
elif (variable.getLevel() is not None
and horiz_sections is not False
and horiz_sections is not None): # 3D
# Surf and bottom
for ok, slice_loc in [(surf, 'surf'), (bottom, 'bottom')]:
if ok is True:
toplot.append(
dict(
slice_loc=slice_loc,
slice_type='map',
keys=(var_short_name, 'map', slice_loc),
kw=dict_merge(dict(depth=slice_loc), kw),
obs_plot={slice_loc: True},
))
# Loop on 3D depths specs
if horiz_sections is True:
depths = variable.getLevel()
else:
depths = horiz_sections
if N.isscalar(depths):
depths = [depths]
depths = list(depths)
for depth in depths:
if isinstance(depth, str):
slice_loc = depth
else:
slice_loc = fmtdep.format(abs(depth))
toplot.append(
dict(
slice_loc=slice_loc,
slice_type='map',
keys=(var_name, 'map', slice_loc),
kw=dict_merge(dict(depth=depth), kw),
obs_plot={'horiz_sections': depth},
))
# Zonal sections
if zonal_sections is not None and zonal_sections is not False:
slice_type = 'zonal'
if N.isscalar(zonal_sections):
zonal_sections = [zonal_sections]
for lat in zonal_sections:
slice_loc = latlab(lat, no_symbol=True)
toplot.append(
dict(
slice_loc=slice_loc,
slice_type=slice_type,
keys=(var_name, slice_type, slice_loc),
kw=dict_merge(dict(lat=lat), kw),
obs_plot={'zonal_sections': lat},
))
# Meridional sections
if merid_sections is not None and merid_sections is not False:
slice_type = 'merid'
if N.isscalar(merid_sections):
merid_sections = [merid_sections]
for lon in merid_sections:
slice_loc = lonlab(lon, no_symbol=True)
toplot.append(
dict(
slice_loc=slice_loc,
slice_type=slice_type,
keys=(var_name, slice_type, slice_loc),
kw=dict_merge(dict(lon=lon), kw),
obs_plot={'merid_sections': lon},
))
# # Points
# if points:
# slice_type = 'point'
# if isinstance(points, tuple):
# points = [points]
# for lon, lat in points:
# slice_loc = (lonlab(lon, no_symbol=True) + '-' +
# latlab(lat, no_symbol=True))
# toplot.append(dict(
# slice_loc=slice_loc,
# slice_type=slice_type,
# keys=(var_name, slice_type, slice_loc),
# kw=dict_merge(dict(lon=lon, lat=lat), kw),
# obs_plot={'points':(lon, lat)},
# ))
# Make all pending plots for this variable
for spec in toplot:
# Get specs
slice_loc = spec['slice_loc']
slice_type = spec['slice_type']
obs_plot = spec['obs_plot']
keys = map(str.title, spec['keys'])
kw = kwargs.copy()
kw.update(spec['kw'])
self.debug(' Location: ' + slice_loc)
# Setup
dfmt = locals()
dfmt.update(subst)
title = titlepat.format(**dfmt)
kw.update(title=title, savefig=False, close=False)
dict_check_defaults(kw,
fig='new',
**DEFAULT_PLOT_KWARGS_PER_ENS_DIAG)
# Plot
p = plot_gridded_var(variable, **kw) # Action !
# Plot obs locations on 2D plots only
if obs and isinstance(p, Plot2D):
#FIXME: should we pass lon/lat/level from variable?
kwo = kwobs.copy()
kwo.update(obs_plot,
full3d=False,
full2d=False,
plotter=p,
savefig=False,
close=False,
title=False,
colorbar=False)
# obs.set_cached_plot('locations', slice_type, slice_loc, p)
obs.plot('locations', **kwo)
# Finalise
if savefig:
figfile = figpat.format(**dfmt)
checkdir(figfile)
p.savefig(figfile)
close = True
self.created(figfile)
kw = {keys[-1]: figfile, '__class__': OrderedDict}
dicttree_set(figs, *keys[:-1], **kw)
if not show and close:
p.close()
if show:
P.show()
elif close:
P.close()
return figs