def main(project_info):
"""Diagnostics and plotting script for Southern Hemisphere radiation."""
# ESMValProject provides some easy methods to access information in
# project_info but you can also access it directly (as in main.py)
# First we get some general configurations (ESMValProject)
# and then we read in the model configuration file
E = ESMValProject(project_info)
config_file = E.get_configfile()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
# A-laue_ax+
diag_script = E.get_diag_script_name()
res = E.write_references(
diag_script, # diag script name
["A_maek_ja"], # authors
["A_eval_ma", "A_jone_co"], # contributors
[""], # diag_references
[""], # obs_references
["P_embrace"], # proj_references
project_info,
verbosity,
False)
# A-laue_ax-
modelconfig = ConfigParser.ConfigParser()
modelconfig.read(config_file)
E.ensure_directory(plot_dir)
# Check which parts of the code to run
if (modelconfig.getboolean('general', 'plot_scatter')):
info("Starting scatter plot", verbosity, 2)
process_scatter(E, modelconfig)
def endElement(self, name):
""" @brief default SAX endElement event handler
@param name default SAX endElement argument
"""
def replaceAttag(tbr):
pat = re.compile('@\\{..*\\}')
if pat.search(tbr):
patlist = pat.findall(tbr)
patlist = [item[2:-1] for item in patlist]
tmp = list(set(patlist) & set(self.conf.pathCollection.keys()))
for t in tmp:
tbr = re.sub("@{"+t+"}", self.conf.getPathByID(t), tbr)
return tbr
info("endElement: " + name, verbosity, required_verbosity=12)
if self.include is not None:
if name == self.current_tag.__class__.__name__:
self.project_info[name] = getattr(self.current_tag, 'closing_tag')(replaceAttag(self.str), self.attributes)
elif name == 'include':
pass
elif name == 'namelist':
pass
else:
self.current_tag.add_nml_entry(name, replaceAttag(self.str), self.attributes)
else:
if name == self.current_tag.__class__.__name__:
self.project_info[name] = getattr(self.current_tag, 'closing_tag')(self.str, self.attributes)
elif name == 'include':
pass
elif name == 'namelist':
pass
else:
self.current_tag.add_nml_entry(name, self.str, self.attributes)
def write_env_projinfo(self, project_info):
""" @brief Writes XML-file information to env. variables
@param project_info Current namelist in dictionary format
Information between Python and NCL is (partially) exchanged
through environment variables. This function write the
project_info-dictionary content to environment variables
prefixed with "ESMValTool_".
"""
verbosity = project_info['GLOBAL']['verbosity']
for section_key in ['GLOBAL', 'RUNTIME', 'TEMPORARY']:
if section_key in project_info.keys():
for key in project_info[section_key]:
info("writing key to env. variable, key=" + key,
verbosity,
required_verbosity=11)
# Check and fail on duplicate entries
if "ESMValTool_" + key in os.environ:
raise writeProjinfoError("Environment variable " +
"'ESMValTool_" + key +
"' already defined")
os.environ["ESMValTool_" + key] = \
str(project_info[section_key][key])
def startElement(self, name, attr):
""" @brief default SAX startElement event handler
@param name default SAX startElement argument
@param attr default SAX startElement attribute
"""
info("startElement: " + name, verbosity, required_verbosity=12)
if name == 'include':
self.include = attr['href']
self.conf = configPrivateHandler()
p = xml.sax.make_parser()
p.setContentHandler(self.conf)
p.parse(self.include)
if name in vars(nml_tags):
self.current_tag = vars(nml_tags)[name]()
else:
info("clearing str '" + self.str + "'",
verbosity,
required_verbosity=12)
self.attributes = {}
self.str = ""
if attr:
for attr_key in attr.keys():
self.attributes[attr_key.lower()] = attr[attr_key]
def select_base_vars(self, variables, model, currProject, project_info):
""" @brief Determine base variables to be read from input file(s)
@param variables - base variables provided by variable_defs script
@param model - current model
@param currProject - current project
@param project_info - project_info-dictionary
@return base_vars - variables to be read from input file(s)
"""
verbosity = project_info['GLOBAL']['verbosity']
for base_var in variables:
# Check if this variable should be excluded for this model-id
if self.id_is_explicitly_excluded(base_var, model):
continue
# first try: use base variables provided by variable_defs script
os.environ['__ESMValTool_base_var'] = base_var.var
infile = reformat.infile(currProject, project_info, base_var,
model)
precomputed = currProject.get_cf_fullpath(project_info, model,
base_var.fld,
base_var.var,
base_var.mip,
base_var.exp)
if (len(glob.glob(infile) + glob.glob(precomputed)) == 0):
info(
" No input files found for " + base_var.var + " (" +
base_var.fld + ") as " + infile, verbosity, 1)
base_var.var = base_var.var0
base_var.fld = base_var.fld0
# try again with input variable = base variable (non derived)
infile = reformat.infile(currProject, project_info, base_var,
model)
precomputed = currProject.get_cf_fullpath(
project_info, model, base_var.fld, base_var.var,
base_var.mip, base_var.exp)
if (len(glob.glob(infile) + glob.glob(precomputed)) == 0):
raise exceptions.IOError(2, "No input files found in ",
infile)
else:
info(
" Using " + base_var.var + " (" + base_var.fld +
") as " + infile, verbosity, 1)
base_vars = [base_var]
break # discard other base vars
else:
base_vars = variables
return base_vars
def climate(currProject, currDiag, project_info, model, field, base_variable,
variable_attributes):
""" @brief Wrapper to call ncl script that calculates climatology
@param currProject An instance of the current project
@param project_info Dictionary with all info from the namelist
@param currDiag Current diagnostic
"""
infilename = currProject.get_cf_fullpath(project_info, model, field,
base_variable,
variable_attributes)
cfield_m, cfield_s, cfield_a = currDiag.get_climate_field_type(field)
file_monthly_clim = currProject.get_cf_fullpath(project_info, model,
cfield_m, base_variable,
variable_attributes)
file_season_clim = currProject.get_cf_fullpath(project_info, model,
cfield_s, base_variable,
variable_attributes)
file_annual_clim = currProject.get_cf_fullpath(project_info, model,
cfield_a, base_variable,
variable_attributes)
if ((not os.path.isfile(file_monthly_clim)
or not os.path.isfile(file_season_clim)
or not os.path.isfile(file_annual_clim))
or project_info['GLOBAL']['force_processing']):
verbosity = project_info['GLOBAL']['verbosity']
project_info['TEMPORARY'] = {}
project_info['TEMPORARY']['infilename'] = infilename
project_info['TEMPORARY']['mfile'] = file_monthly_clim
project_info['TEMPORARY']['sfile'] = file_season_clim
project_info['TEMPORARY']['afile'] = file_annual_clim
project_info['TEMPORARY']['base_variable'] = base_variable
info(" Calling climate.py", verbosity, required_verbosity=1)
info(" INFILE = " + infilename, 1, verbosity)
info(" OUTFILES = " + file_monthly_clim, 1, verbosity)
info(" " + file_season_clim, 1, verbosity)
info(" " + file_annual_clim, 1, verbosity)
exit_on_warning = project_info['GLOBAL']['exit_on_warning']
executable = "./interface_scripts/climate.ncl"
projects.run_executable(executable, project_info, verbosity,
exit_on_warning)
del (project_info['TEMPORARY'])
def endElement(self, element_name):
""" @brief default SAX endElement event handler
@param name default SAX endElement argument
"""
info("endElement: " + element_name, verbosity, required_verbosity=12)
if element_name == 'ESGF':
pass
elif element_name in ESGFConfig.valid_node_names\
or element_name == "USER_CACHE":
self.node = None
else:
self.current_tag.add_ESGF_entry(
element_name,
self.string.strip(),
self.attributes,
self.node
)
def main(project_info):
"""Diagnostics and plotting script for Tropical Variability Equatorial.
We use ts as a proxy for Sea Surface Temperature. """
# ESMValProject provides some easy methods to access information in
# project_info but you can also access it directly (as in main.py)
# We need to ensure that needed files for precipitation and temperature
# are present before we can proceed
E = ESMValProject(project_info)
config_file = E.get_configfile()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
# A-laue_ax+
diag_script = E.get_diag_script_name()
res = E.write_references(
diag_script, # diag script name
["A_maek_ja"], # authors
["A_eval_ma", "A_jone_co"], # contributors
["D_li14jclim"], # diag_references
[""], # obs_references
["P_embrace"], # proj_references
project_info,
verbosity,
False)
# A-laue_ax-
modelconfig = ConfigParser.ConfigParser()
modelconfig.read(config_file)
E.ensure_directory(plot_dir)
# Here we check and process only desired parts of the diagnostics
if (modelconfig.getboolean('general', 'plot_equatorial')):
info("Starting to plot equatorial means", verbosity, 2)
process_equatorial_means(E, modelconfig)
post_process(E)
def main(project_info):
"""Preprocessing script for Tropical Variability equatorial divergence
winds. """
# ESMValProject provides some easy methods to access information in
# project_info but you can also access it directly (as in main.py)
# First we get some general configurations (ESMValProject)
# and then we read in the model configuration file
E = ESMValProject(project_info)
config_file = E.get_configfile()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
# A-laue_ax+
diag_script = E.get_diag_script_name()
res = E.write_references(
diag_script, # diag script name
["A_maek_ja"], # authors
["A_eval_ma", "A_jone_co"], # contributors
["D_li14jclim"], # diag_references
[""], # obs_references
["P_embrace"], # proj_references
project_info,
verbosity,
False)
# A-laue_ax-
modelconfig = ConfigParser.ConfigParser()
modelconfig.read(config_file)
E.ensure_directory(plot_dir)
# Here we check and process only desired parts of the diagnostics
if (modelconfig.getboolean('general', 'plot_equatorial')):
info("Starting to gather values for equatorial divergence plots",
verbosity, 2)
process_divergence(E, modelconfig)
def main(project_info):
"""Diagnostics and plotting script for Southern Hemisphere radiation."""
# ESMValProject provides some easy methods to access information in
# project_info but you can also access it directly (as in main.py)
# First we get some general configurations (ESMValProject)
# and then we read in the model configuration file
E = ESMValProject(project_info)
config_file = E.get_configfile()
datakeys = E.get_currVars()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
# A-laue_ax+
diag_script = E.get_diag_script_name()
res = E.write_references(diag_script, # diag script name
["A_maek_ja"], # authors
["A_eval_ma", "A_jone_co"], # contributors
[""], # diag_references
[""], # obs_references
["P_embrace"], # proj_references
project_info,
verbosity,
False)
# A-laue_ax-
modelconfig = ConfigParser.ConfigParser()
modelconfig.read(config_file)
E.ensure_directory(plot_dir)
# Check at which stage of the program we are
clouds = False
fluxes = False
radiation = False
if ('clt' in datakeys or 'clivi' in datakeys or 'clwvi' in datakeys):
clouds = True
elif ('hfls' in datakeys or 'hfss' in datakeys):
fluxes = True
else:
radiation = True
# Check which parts of the code to run
if (modelconfig.getboolean('general', 'plot_clouds') and clouds is True):
info("Starting to plot clouds", verbosity, 2)
process_clouds(E, modelconfig)
if (modelconfig.getboolean('general', 'plot_fluxes') and fluxes is True):
info("Starting to plot turbulent fluxes", verbosity, 2)
process_fluxes(E, modelconfig)
if (modelconfig.getboolean('general', 'plot_radiation') and radiation is True):
info("Starting to plot radiation graphs", verbosity, 2)
process_radiation(E, modelconfig)
def process_zonal_means(E, modelconfig):
"""Main script for plotting zonal means. Outputs variable specific plots
containing all models for that variable (including observations). """
experiment = 'zonal_means'
datakeys = E.get_currVars()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
areas = modelconfig.get(experiment, 'areas').split()
plot_grid = modelconfig.getboolean('general', 'plot_grid')
# We don't do the zonal means for winds (equatorial elsewhere)
if 'ua' in datakeys:
datakeys.remove('ua')
for area in areas:
plt.clf()
fig, axs = plt.subplots(2, 2, figsize=(12, 8))
fig.subplots_adjust(top=0.87)
fig.subplots_adjust(right=0.67)
fig.subplots_adjust(hspace=0.2)
fig.subplots_adjust(wspace=0.4)
for datakey in datakeys:
col = datakeys.index(datakey)
# We output a file for each variable
model_filenames = E.get_clim_model_filenames(variable=datakey)
# Start looping and generating subplots
ymin, ymax = False, False
lon_min = modelconfig.getint(experiment + '_' + area, 'lon_min')
lon_max = modelconfig.getint(experiment + '_' + area, 'lon_max')
lon_area = E.get_ticks_labels(np.array([lon_min, lon_max]), 'lons')
ymin = -9999
ymax = -9999
sd = np.zeros(len(model_filenames))
mindex = 0
for model in model_filenames:
# read in specified data and concatenate
datafile = nc.Dataset(model_filenames[model], 'r')
# A-laue_ax+
E.add_to_filelist(model_filenames[model])
# A-laue_ax-
data_units = datafile.variables[datakey].units
lats, data = E.get_model_data(modelconfig, experiment, area,
datakey, datafile)
data = E.average_data(data, 1)
datafile.close()
norm = data.mean()
# get y min and max iteratively
if (ymin == -9999):
ymin = data.min()
ymax = data.max()
else:
ymin = np.min([data.min(), ymin])
ymax = np.max([data.max(), ymax])
sd[mindex] = (data / norm).std()
mindex += 1
# read model specific plotting style
color, dashes, width = E.get_model_plot_style(model)
# plotting custom dashes requires some extra effort (else)
# with empty dashes the format is default
if (len(dashes) == 0):
axs[0, col].plot(lats,
data,
color=color,
linewidth=width,
label=model)
axs[1, col].plot(lats,
data / norm,
color=color,
linewidth=width,
label=model)
else:
line1, = axs[0, col].plot(lats,
data,
'--',
color=color,
linewidth=width,
label=model)
line1.set_dashes(dashes)
line2, = axs[1, col].plot(lats,
data / norm,
color=color,
linewidth=width,
label=model)
line2.set_dashes(dashes)
# Now to make the plot pretty i.e. headers, ticks etc.
# axs[-1, col].yaxis.set_label_position("right")
axs[0, col].grid(plot_grid)
axs[1, col].grid(plot_grid)
area_key = experiment + '_' + area
xmin = modelconfig.getint(area_key, 'lat_min')
xmax = modelconfig.getint(area_key, 'lat_max')
axs[0, col].set_xlim(xmin, xmax)
axs[1, col].set_xlim(xmin, xmax)
xticks = E.get_ticks(5, np.array([xmin, xmax]))
labels = E.get_ticks_labels(xticks, 'lats')
axs[0, col].xaxis.set_ticks(xticks)
axs[0, col].set_xticklabels(labels)
axs[1, col].xaxis.set_ticks(xticks)
axs[1, col].set_xticklabels(labels)
yticks = E.get_ticks(8, np.array([ymin, ymax]))
axs[0, col].yaxis.set_ticks(yticks)
axs[1, col].yaxis.set_ticks([1.0])
axs[1, col].set_yticklabels(['$\mu$'])
#axs[1, col].yaxis.set_ticks([1.0-sd.mean(), 1.0, 1.0+sd.mean()])
#axs[1, col].set_yticklabels(['$\mu$-$\sigma$', '$\mu$', '$\mu$+$\sigma$'])
if (datakey == 'pr' or datakey == 'pr-mmday'):
axs[0, col].set_title("Precipitation")
axs[0, col].set_ylabel("[" + data_units + "]")
axs[1, col].set_ylabel("normalized")
elif (datakey == 'ts'):
axs[0, col].set_title("Temperature")
axs[0, 0].set_ylabel("[" + data_units + "]")
axs[1, 0].set_ylabel("normalized")
plt.suptitle(area + " ocean [" + lon_area[0] + ":" + lon_area[1] +
"] seasonal mean",
fontsize=16)
# Adding the legend
handles0, labels0 = axs[0, 0].get_legend_handles_labels()
handles1, labels1 = axs[0, 1].get_legend_handles_labels()
handles_all = handles0 + handles1
labels_all = labels0 + labels1
labels_sorted = zip(*sorted(zip(labels_all, handles_all)))[0]
handles_sorted = zip(*sorted(zip(labels_all, handles_all)))[1]
handles = []
labels = []
for item in xrange(len(labels_sorted)):
if (labels_sorted[item] not in labels):
labels.append(labels_sorted[item])
handles.append(handles_sorted[item])
plt.legend(handles,
labels,
loc='center left',
bbox_to_anchor=(0.7, 0.5),
bbox_transform=plt.gcf().transFigure)
# And save the plot
diag_name = E.get_diag_script_name()
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
specifier = area + '-seasonal-mean'
variable = "".join(E.get_currVars())
output_file = E.get_plot_output_filename(diag_name=diag_name,
variable=variable,
specifier=specifier)
plt.savefig(os.path.join(output_dir, output_file))
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
def read_pr_sm_topo(project_info, model):
"""
;; Arguments
;; project_info: dictionary
;; all info from namelist
;;
;; Return
;; pr: iris cube [time, lat, lon]
;; precipitation time series
;; sm: iris cube [time, lat, lon]
;; soil moisture time series
;; topo: array [lat, lon]
;; topography
;; lon: array [lon]
;; longitude
;; lat: array [lat]
;; latitude
;; time: iris cube coords
;; time info of cube
;; time_bnds_1: float
;; first time_bnd of time series
;;
;;
;; Description
;; Read cmip5 input data for computing the diagnostic
;;
"""
import projects
E = ESMValProject(project_info)
verbosity = E.get_verbosity()
#-------------------------
# Read model info
#-------------------------
currProject = getattr(vars()['projects'], model.split_entries()[0])()
model_info = model.split_entries()
mip = currProject.get_model_mip(model)
exp = currProject.get_model_exp(model)
start_year = currProject.get_model_start_year(model)
end_year = currProject.get_model_end_year(model)
years = range(int(start_year), int(end_year) + 1)
'''
#-------------------------
# Read model info
#-------------------------
model_name = model_info[1]
time_step = model_info[2]
exp_fam = model_info[3]
model_run = model_info[4]
year_start = model_info[5]
year_end = model_info[6]
filedir = model_info[7]
years = range(int(year_start), int(year_end)+1)
'''
#-------------------------
# Input data directories
#-------------------------
currDiag = project_info['RUNTIME']['currDiag']
pr_index = currDiag.get_variables().index('pr')
pr_field = currDiag.get_field_types()[pr_index]
sm_index = currDiag.get_variables().index('mrsos')
sm_field = currDiag.get_field_types()[sm_index]
indir = currProject.get_cf_outpath(project_info, model)
in_file = currProject.get_cf_outfile(project_info, model, pr_field, 'pr', mip, exp)
pr_files = [os.path.join(indir, in_file)]
in_file = currProject.get_cf_outfile(project_info, model, sm_field, 'mrsos', mip, exp)
sm_files = [os.path.join(indir, in_file)]
'''
#-------------------------
# Input data directories
#-------------------------
pr_files = []
sm_files = []
for yy in years:
Patt = filedir+'pr_'+time_step+'_'+model_name+'_'+exp_fam+'_'+\
model_run+'_'+str(yy)+'*.nc'
pr_files.append(glob.glob(Patt))
Patt = filedir+'mrsos_'+time_step+'_'+model_name+'_'+exp_fam+'_'+\
model_run+'_'+str(yy)+'*.nc'
sm_files.append(glob.glob(Patt))
pr_files = [l[0] for l in pr_files if len(l)>0]
pr_files = sorted(pr_files)
sm_files = [l[0] for l in sm_files if len(l)>0]
sm_files = sorted(sm_files)
'''
#----------------------
# Read in precipitation
#----------------------
pr_list = []
for pr_file in pr_files:
info('Reading precipitation from ' + pr_file, verbosity, required_verbosity=1)
pr = iris.load(pr_file)[0]
for at_k in pr.attributes.keys():
pr.attributes.pop(at_k)
pr_list.append(pr)
pr = iris.cube.CubeList(pr_list)
pr = pr.concatenate()[0]
# Convert longitude from 0_360 to -180_180
pr = coord_change([pr])[0]
# Add metadata: day, month, year
add_month(pr, 'time')
add_day_of_month(pr, 'time', name='dom')
add_year(pr, 'time')
# Convert units to kg m-2 hr-1
pr.convert_units('kg m-2 hr-1')
#-----------------------
# Read in soil moisture
#-----------------------
sm_list = []
for sm_file in sm_files:
info('Reading soil moisture from ' + sm_file, verbosity, required_verbosity=1)
sm = iris.load(sm_file)[0]
for at_k in sm.attributes.keys():
sm.attributes.pop(at_k)
sm_list.append(sm)
sm = iris.cube.CubeList(sm_list)
sm = sm.concatenate()[0]
# Convert longitude from 0_360 to -180_180
sm = coord_change([sm])[0]
# Add metadata: day, month, year
add_month(sm, 'time')
add_day_of_month(sm, 'time', name='dom')
add_year(sm, 'time')
#----------------------------------------------
# Constrain pr and sm data to latitude 60S_60N
#----------------------------------------------
latconstraint = iris.Constraint(latitude=lambda cell: -59.0 <= cell <= 59.0)
pr = pr.extract(latconstraint)
sm = sm.extract(latconstraint)
#---------------------------------------------------
# Read in grid info: latitude, longitude, timestamp
#---------------------------------------------------
lon = sm.coords('longitude')[0].points
lat = sm.coords('latitude')[0].points
time = sm.coords('time')
# --------------------------------------
# Convert missing data (if any) to -999.
# --------------------------------------
try:
sm.data.set_fill_value(-999)
sm.data.data[sm.data.mask] = -999.
except:
info('no missing data conversion', verbosity, required_verbosity=1)
#----------------------
# Read in topography
#----------------------
# Topography map specs:
# latitude 60S_60N
# longitude 180W_180E
# model resolution
#ftopo = currProject.get_cf_fx_file(project_info, model)
#dt = '>f4'
#topo = (np.fromfile(ftopo, dtype=dt)).reshape(len(lat), len(lon))
topo = get_topo(project_info, lon, lat, model)
#----------------------
# Read in time bounds
#----------------------
indir, infiles = currProject.get_cf_infile(project_info, model, pr_field, 'pr', mip, exp)
Patt = os.path.join(indir, infiles)
pr_files = sorted(glob.glob(Patt))
ncf = nc4.Dataset(pr_files[0])
time_bnds_1 = ncf.variables['time_bnds'][0][0]
time_bnds_1 = time_bnds_1 - int(time_bnds_1)
ncf.close()
#-----------------------------------------------
# Return input data to compute sm_pr diagnostic
#-----------------------------------------------
return pr, sm, topo, lon, lat, time, time_bnds_1
def main(project_info):
"""
;; Description
;; Main fuction
;; Call all callable fuctions to
;; read CMIP5 data, compute and plot diagnostic
"""
E = ESMValProject(project_info)
plot_dir = E.get_plot_dir()
work_dir = E.get_work_dir()
verbosity = E.get_verbosity()
fileout = work_dir
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
for model in project_info['MODELS']:
info(model, verbosity, required_verbosity=1)
if not os.path.exists(work_dir+'/sample_events'):
os.makedirs(work_dir+'/sample_events')
if not os.path.exists(work_dir+'/event_output'):
os.makedirs(work_dir+'/event_output')
# --------------------------------------
# Get input data to compute diagnostic
# --------------------------------------
# Read cmip5 model data
pr, sm, topo, lon, lat, time, time_bnds_1 = read_pr_sm_topo(project_info, model)
# --------------------------------------
# Get sm monthly climatology
# at selected local time: 6:00 am
# --------------------------------------
smclim = get_smclim(sm, lon, time)
# -------------------------------
# Compute diagnostic per month
# -------------------------------
samplefileout = fileout + 'sample_events/'
for mn in np.arange(1, 13):
# -------------------------------------------------
# Create montly arrays required by fortran routines
# -------------------------------------------------
prbef, smbef, praft, smaft, \
monthlypr, monthlysm, days_per_year = get_monthly_input(project_info, mn, time,
lon, lat, time_bnds_1, pr, sm, fileout,
samplefileout, model,
verbosity)
# -----------------------
# Run fortran routines
# -----------------------
info('Executing global_rain_sm for month ' + str(mn), verbosity, required_verbosity=1)
grs.global_rain_sm(np.asfortranarray(monthlypr),
np.asfortranarray(prbef),
np.asfortranarray(praft),
np.asfortranarray(monthlysm),
np.asfortranarray(smbef),
np.asfortranarray(smaft),
np.asfortranarray(smclim[mn - 1, :, :]),
np.asfortranarray(topo),
np.asfortranarray(lon),
np.asfortranarray(mn),
fileout, days_per_year)
info('Executing sample_events for month ' + str(mn), verbosity, required_verbosity=1)
se.sample_events(np.asfortranarray(monthlysm),
np.asfortranarray(smbef),
np.asfortranarray(smaft),
np.asfortranarray(lon),
np.asfortranarray(lat),
np.asfortranarray(mn),
fileout, days_per_year, samplefileout)
# ---------------------------------------------------
# Compute p_values (as in Fig. 3, Taylor et al 2012)
# --------------------------------------------------
info('Computing diagnostic', verbosity, required_verbosity=1)
xs, ys, p_vals = get_p_val(samplefileout)
# --------------------------------------------------
# Save diagnostic to netCDF file and plot
# --------------------------------------------------
write_nc(fileout, xs, ys, p_vals, project_info, model)
plot_diagnostic(fileout, plot_dir, project_info, model)
# --------------------------------------------------
# Remove temporary folders
# --------------------------------------------------
shutil.rmtree(str(fileout) + 'event_output')
shutil.rmtree(str(fileout) + 'sample_events')
def get_topo(project_info, longi, lati, model):
"""
;; Arguments
;; in_dir: dir
;; directory with input file "topo_var_5x5.gra"
;; longi: array [lon]
;; longitude in degrees east
;; lati: array [lat]
;; latitude
;; Return
;; topo: array [lat, lon]
;; topography ranges in model grid
;;
;; Description
;; Computes topography information
;;
"""
import projects
E = ESMValProject(project_info)
verbosity = E.get_verbosity()
#-------------------------
# Read model info
#-------------------------
currProject = getattr(vars()['projects'], model.split_entries()[0])()
model_info = model.split_entries()
# load topo max-min topography(m)
# within 1.25x1.25 area
nx = 1440
ny = 480
ftopo = currProject.get_cf_fx_file(project_info, model)
info('topo file: ' + ftopo , verbosity, required_verbosity=1)
dt = '>f4'
topo = (np.fromfile(ftopo, dtype=dt)).reshape(4, ny, nx)
topo_range = np.transpose(topo[3,:,:])
# Average onto model grid
nxo = len(longi)
nyo = len(lati)
lon1 = longi[0]
lat1 = lati[0]
dlon = np.abs(longi[0]-longi[1])
dlat = np.abs(lati[0]-lati[1])
topo_model = np.zeros((nxo, nyo), dtype = 'f4')
n = np.zeros((nxo, nyo), dtype = 'f4')
for j in range(0, ny):
for i in range(0, nx):
lon = i*0.25-179.875
lat = j*0.25-59.875
jj = int(round((lat-lat1)/dlat))
ii = int(round((lon-lon1)/dlon))
if(ii==nxo):
ii = 0
if(jj==nyo):
jj = nyo -1
if(topo_range[i,j]!=-999.):
topo_model[ii,jj] = topo_model[ii,jj] + topo_range[i,j]
n[ii,jj] = n[ii,jj] + 1.
topo_model[n>0] = topo_model[n>0]/n[n>0]
topo_model[n==0] = -999.
topo_model[topo_model==-999] = 0
return np.transpose(topo_model)
def get_monthly_input(project_info, mn, time, lon, lat,
time_bnds_1, pr, sm, fileout,
samplefileout, model,
verbosity):
"""
;; Arguments
;; mn: int
;; month, values from 1 to 12
;; time: iris cube coords
;; time info of cube
;; lon: array [lon]
;; longitude
;; lat: array [lat]
;; latitude
;; time_bnds_1: float
;; first time_bnd of time series
;; pr: iris cube [time, lat, lon]
;; 3-hourly precipitation time series
;; sm: iris cube [time, lat, lon]
;; 3-hourly soil moisture time series
;; fileout: dir
;; output directory
;; samplefileout: dir
;; temporary outpout directory used by fortran routines
;;
;; Return
;; prbef: array [year, day time steps (=8), lat, lon]
;; 3-hourly precipitation in the previous day
;; smbef: array [year, day time steps (=8), lat, lon]
;; 3-hourly soil moisture in the previous day
;; praf: array [year, day time steps (=8), lat, lon]
;; 3-hourly precipitation in the following day
;; smaft: array [year, day time steps (=8), lat, lon]
;; 3-hourly soil moisture in the following day
;; monthlypr: array [year, days in month * day time steps, lat, lon]
;; 3-hourly precipitation in month mn for the whole analysis period
;; monthlysm: array [year, days in month * day time steps, lat, lon]
;; 3-hourly soil moisture in month mn for the whole analysis period
;; days_per_year: int
;; number of days per year
;;
;; Description
;; Prepare monthly input data for fortran routines
;;
"""
import projects
E = ESMValProject(project_info)
verbosity = E.get_verbosity()
#-------------------------
# Read model info
#-------------------------
model = project_info['MODELS'][0]
currProject = getattr(vars()['projects'], model.split_entries()[0])()
model_info = model.split_entries()
# --------------------------------------
# Get grid and calendar info
# --------------------------------------
utimedate = time[0].units
first_year = int(model_info[5])
last_year = int(model_info[6])
nyr = last_year - first_year + 1
years = np.arange(nyr) + first_year
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
calendar = utimedate.calendar
nx = len(lon)
ny = len(lat)
days_permonth_360 = [30 for i in range(0, 12)]
days_permonth_noleap = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
days_permonth_leap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
if calendar == '360_day':
days_per_year = 360
days_permonth = days_permonth_360
nts = [8 * dpm for dpm in days_permonth_360]
elif any([calendar == '365_day', calendar == 'noleap']):
days_per_year = 365
days_permonth = days_permonth_noleap
nts = [8 * dpm for dpm in days_permonth_noleap]
elif any([calendar == 'gregorian', calendar == 'standard']):
days_per_year = 365
days_permonth = days_permonth_noleap
nts = [8 * dpm for dpm in days_permonth_noleap]
# Leap days are considered by get_monthly_input()
else:
error('Missing calendar info')
# --------------------------------------
# Create pr, sm before and after arrays
# --------------------------------------
prbef = np.zeros((nyr, 8, ny, nx), dtype='f4')
praft = np.zeros((nyr, 8, ny, nx), dtype='f4')
smbef = np.zeros((nyr, 8, ny, nx), dtype='f4')
smaft = np.zeros((nyr, 8, ny, nx), dtype='f4')
try:
os.mkdir(os.path.join(samplefileout, "5x5G_mon" + str(mn).zfill(2)), stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise exc
pass
try:
os.mkdir(os.path.join(fileout, "event_output", "mon" + str(mn).zfill(2)), stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise exc
pass
nt = nts[mn - 1]
monthlypr_list = [np.zeros((nt, ny, nx), dtype='f4') for y in years]
monthlysm_list = [np.zeros((nt, ny, nx), dtype='f4') for y in years]
for yr, year in enumerate(years):
info('month, year: ' + str(mn) + ", " + str(year), verbosity, required_verbosity=1)
if all([mn == 2, any([calendar == 'gregorian', calendar == 'standard'])]):
monthlysm_list[yr][:,:,:] = (
sm.extract(iris.Constraint(month=months[mn-1],year=year, dom = range(1,29) )).data)
monthlypr_list[yr][:,:,:] = (
pr.extract(iris.Constraint(month=months[mn-1],year=year, dom = range(1,29) )).data)
else:
try:
monthlypr_list[yr][:,:,:] = (
pr.extract(iris.Constraint(month=months[mn-1],year=year)).data)
except:
try:
monthlypr_list[yr][0,:,:] = -999.
monthlypr_list[yr][1::,:,:] = (
pr.extract(iris.Constraint(month=months[mn-1],year=year)).data)
except:
info('omitted pr: ' + str(mn) + ", " + str(year), verbosity, required_verbosity=1)
try:
monthlysm_list[yr][:,:,:] = (
sm.extract(iris.Constraint(month=months[mn-1],year=year)).data)
except:
try:
monthlysm_list[yr][0,:,:] = -999.
monthlysm_list[yr][1::,:,:] = (
sm.extract(iris.Constraint(month=months[mn-1],year=year)).data)
except:
info('omitted sm: ' + str(mn) + ", " + str(year), verbosity, required_verbosity=1)
# last day of previous month
if all([mn == 1, year == first_year]):
prbef[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999.
smbef[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999.
elif (mn == 1):
prbef[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year-1,month='Dec',dom=days_permonth[-1])).data)
smbef[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year-1,month='Dec',dom=days_permonth[-1])).data)
else:
if any([calendar == '360_day', calendar == '365_day', calendar == 'noleap']):
prbef[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data)
smbef[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data)
elif any([calendar == 'gregorian', calendar == 'standard']):
if cal.isleap(year):
prbef[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth_leap[mn-2])).data)
smbef[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth_leap[mn-2])).data)
else:
prbef[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data)
smbef[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year,month=months[mn-2],dom=days_permonth[mn-2])).data)
# first day of following month
if (mn == 12 and year == last_year):
praft[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999.
smaft[yr,:,:,:] = np.zeros( (8, ny, nx) ) - 999.
elif (mn == 12):
praft[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year+1,month='Jan',dom=1)).data)
smaft[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year+1,month='Jan',dom=1)).data)
else:
if any([calendar == '360_day', calendar == '365_day', calendar == 'noleap']):
praft[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data)
smaft[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data)
elif any([calendar == 'gregorian', calendar == 'standard']):
if all([cal.isleap(year),mn == 2]):
praft[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year,month=months[1],dom=29)).data)
smaft[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year,month=months[1],dom=29)).data)
else:
praft[yr,:,:,:] = (
pr.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data)
smaft[yr,:,:,:] = (
sm.extract(iris.Constraint(year=year,month=months[mn],dom=1)).data)
monthlypr = np.vstack(tuple(monthlypr_list))
monthlysm = np.vstack(tuple(monthlysm_list))
monthlypr = monthlypr.reshape(nyr, nt, ny, nx)
monthlysm = monthlysm.reshape(nyr, nt, ny, nx)
if time_bnds_1 == 0.0625:
monthlypr[:, 0:-1, :, :] = monthlypr[:, 1::, :, :]
monthlypr[:, -1, :, :] = -9999.
prbef[:, 0:-1, :, :] = prbef[:, 1::, :, :]
praft[:, 0:-1, :, :] = praft[:, 1::, :, :]
prbef[:, -1, :, :] = -9999.
praft[:, -1, :, :] = -9999.
return prbef, smbef, praft, smaft, monthlypr, monthlysm, days_per_year
input_xml_full_path = args[0]
# Parse input namelist into project_info-dictionary.
Project = xml_parsers.namelistHandler()
parser = xml.sax.make_parser()
parser.setContentHandler(Project)
parser.parse(input_xml_full_path)
# Project_info is a dictionary with all info from the namelist.
project_info = Project.project_info
if options.reformat:
if 'REFORMAT' not in project_info.keys():
error('No REFORMAT tag specified in {0}'.format(input_xml_full_path))
if len(project_info['REFORMAT']) == 0:
info('No reformat script specified', 1, 1)
print_header({}, options.reformat)
for k, v in project_info['REFORMAT'].iteritems():
if not os.path.exists(v):
error('Path {0} does not exist'.format(v))
projects.run_executable(v, project_info, 1, False, write_di=False)
sys.exit(0)
verbosity = project_info['GLOBAL']['verbosity']
climo_dir = project_info['GLOBAL']['climo_dir']
exit_on_warning = project_info['GLOBAL'].get('exit_on_warning', False)
# Additional entries to 'project_info'. The 'project_info' construct
# is one way by which Python passes on information to the NCL-routines.
project_info['RUNTIME'] = {}
def process_equatorial_means(E, modelconfig):
"""Main script for plotting equatorial means. Outputs one image with
four subplots (precipitation, temperature, equatorial winds and divergence).
TODO: This script is not too flexible - since we're using ncl for preprocessing
we cant exclude multiple obs files - so we can use only two different
observation models (so they can exclude one another). """
experiment = 'equatorial'
datakeys = E.get_currVars()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
work_dir = E.get_work_dir()
areas = modelconfig.get(experiment, 'areas').split()
plot_grid = modelconfig.getboolean('general', 'plot_grid')
ua_key = datakeys[0]
# Get the model paths etc. including the obs file for winds
models = E.get_clim_model_filenames(ua_key)
# Get other variables and check data existance
datakeys.extend(check_data_existance(E, modelconfig))
# Looping over areas etc.
for area in areas:
area_key = experiment + '_' + area
lat_min = modelconfig.getint(area_key, 'lat_min')
lat_max = modelconfig.getint(area_key, 'lat_max')
lon_min = modelconfig.getint(area_key, 'lon_min')
lon_max = modelconfig.getint(area_key, 'lon_max')
lat_area = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats')
# Initial figure config
fig, axs = plt.subplots(4, 1, figsize=(12, 11))
fig.subplots_adjust(top=0.88)
fig.subplots_adjust(right=0.67)
fig.subplots_adjust(hspace=0.4)
for datakey in datakeys:
# Output precipitation / temperature and winds
# Adjust titles accordingly
suptitle = area + " ocean equatorial ["
suptitle += lat_area[0] + ":" + lat_area[1] + "] mean"
plt.suptitle(suptitle, fontsize=16)
if (datakey == 'pr' or datakey == 'pr-mmday'):
row = 0
title = "Precipitation"
elif (datakey == 'ts'):
row = 1
title = "Temperature"
elif (datakey == 'ua' or datakey == 'ua-1000'):
row = 2
title = "Eastward wind"
elif (datakey == 'divergence'):
row = 3
title = "Wind divergence"
# The data is handled differently for winds and the rest
# For winds we have to extract the data for each model
# For the rest we only have one file / datakey
scaling = ''
if (datakey == 'ua' or datakey == 'ua-1000'):
for model in models:
# extract the model specific data
datafile = nc.Dataset(models[model], 'r')
# A-laue_ax+
E.add_to_filelist(models[model])
# A-laue_ax-
data_units = datafile.variables[datakey].units
lats, lons, data = E.get_model_data(
modelconfig, experiment, area, datakey, datafile)
datafile.close()
lons = E.ensure_looping(lons)
# Take the means and get the plot style
data = E.average_data(data, 2)
color, dashes, width = E.get_model_plot_style(model)
if (len(dashes) == 0):
axs[row].plot(lons,
data,
color=color,
linewidth=width,
label=model)
else:
line, = axs[row].plot(lons,
data,
'--',
color=color,
linewidth=width,
label=model)
line.set_dashes(dashes)
ymin = modelconfig.getint(area_key, 'wind_min')
ymax = modelconfig.getint(area_key, 'wind_max')
yticks = E.get_ticks(8, np.array([ymin, ymax]))
# Next the datakeys with all data in one file (per datakey)
# These will be read from external files previously written by
# TropicalVariability.py
elif (datakey == 'pr' or datakey == 'pr-mmday' or datakey == 'ts'):
base_name = work_dir + 'TropicalVariability_'\
+ area_key + '_' + datakey + '_model_'
datafiles = glob(base_name + '*')
key_models = []
for dfile in datafiles:
start = dfile.find('_model_') + 7
key_models.append(str(dfile)[start:-4])
# now we can do the loop over the models
for model in key_models:
# extract model specific data
alldata = np.load(base_name + model + '.npy')
lons = alldata[0]
data = alldata[1]
color, dashes, width = E.get_model_plot_style(model)
lons = E.ensure_looping(lons)
# get the style and plot it out
if (len(dashes) == 0):
axs[row].plot(lons,
data,
color=color,
linewidth=width,
label=model)
else:
line, = axs[row].plot(lons,
data,
'--',
color=color,
linewidth=width,
label=model)
line.set_dashes(dashes)
# Some additional figure config
if (datakey == 'pr' or datakey == 'pr-mmday'):
ymin = modelconfig.getint(area_key, 'prec_min')
ymax = modelconfig.getint(area_key, 'prec_max')
yticks = E.get_ticks(6, np.array([ymin, ymax]))
if (datakey == 'pr'):
data_units = 'kg/m2s'
else:
data_units = 'mm/day'
elif (datakey == 'ts'):
ymin = modelconfig.getint(area_key, 'temp_min')
ymax = modelconfig.getint(area_key, 'temp_max')
yticks = E.get_ticks(6, np.array([ymin, ymax]))
data_units = 'K'
# Next the wind divergence that is also read from external files
# previously written by TropicalVariability_wind.py
elif (datakey == 'divergence'):
base_name = work_dir + 'TropicalVariability_'\
+ area_key + '_' + datakey + '_model_'
datafiles = glob(base_name + '*')
key_models = []
for dfile in datafiles:
start = dfile.find('_model_') + 7
key_models.append(str(dfile)[start:-4])
# now we can do the loop over the models
for model in key_models:
# extract model specific data
alldata = np.load(base_name + model + '.npy')
lons = alldata[0]
data = 1E6 * alldata[1]
color, dashes, width = E.get_model_plot_style(model)
lons = E.ensure_looping(lons)
# get the style and plot it out
if (len(dashes) == 0):
axs[row].plot(lons,
data,
color=color,
linewidth=width,
label=model)
else:
line, = axs[row].plot(lons,
data,
'--',
color=color,
linewidth=width,
label=model)
line.set_dashes(dashes)
ymin = modelconfig.getint(area_key, 'div_min')
ymax = modelconfig.getint(area_key, 'div_max')
yticks = E.get_ticks(6, np.array([ymin, ymax]))
data_units = '1/s'
scaling = '1E6'
# This part is the same for all subplots
axs[row].set_ylim(ymin, ymax)
axs[row].yaxis.set_ticks(yticks)
xticks = E.get_ticks(9, np.array([lon_min, lon_max]))
labels = E.get_ticks_labels(xticks, 'lons')
axs[row].xaxis.set_ticks(xticks)
axs[row].set_xticklabels(labels)
if (lon_min < lon_max):
axs[row].set_xlim(lon_min, lon_max)
else:
axs[row].set_xlim(lon_min, 360 + lon_max)
if (len(scaling) > 0):
axs[row].set_title(title + ' ' + scaling + ' * [' +
data_units + ']')
else:
axs[row].set_title(title + ' [' + data_units + ']')
axs[row].grid(plot_grid)
# Adding the legend
handles0, labels0 = axs[0].get_legend_handles_labels()
handles1, labels1 = axs[1].get_legend_handles_labels()
handles2, labels2 = axs[2].get_legend_handles_labels()
handles3, labels3 = axs[3].get_legend_handles_labels()
handles_all = handles0 + handles1 + handles2 + handles3
labels_all = labels0 + labels1 + labels2 + labels3
# Sort both lists based on labels
labels_sorted = zip(*sorted(zip(labels_all, handles_all)))[0]
handles_sorted = zip(*sorted(zip(labels_all, handles_all)))[1]
# Now we remove duplicates
handles = []
labels = []
for item in xrange(len(labels_sorted)):
if (labels_sorted[item] not in labels):
labels.append(labels_sorted[item])
handles.append(handles_sorted[item])
# Plotting the legend at its proper place
plt.legend(handles,
labels,
loc='center left',
bbox_to_anchor=(0.7, 0.5),
bbox_transform=plt.gcf().transFigure)
# Saving the plot and printing the image location
diag_name = E.get_diag_script_name()
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
plt.suptitle(area + " ocean mean wind direction and strength",
fontsize=16)
specifier = '-'.join([area, experiment, 'mean'])
variable = "".join(E.get_currVars())
output_file = E.get_plot_output_filename(diag_name=diag_name,
variable=variable,
specifier=specifier)
plt.savefig(os.path.join(output_dir, output_file))
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
def process_scatter(E, modelconfig):
"""
Main script for gathering cloud vs radiation values and plotting them.
"""
config_file = E.get_configfile()
experiment = 'SouthernHemisphere'
areas = modelconfig.get(experiment, 'scatter_areas').split()
seasons = modelconfig.get(experiment, 'seasons').split()
plot_grid = modelconfig.getboolean('general', 'plot_background_grid')
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
work_dir = E.get_work_dir()
datakeys = E.get_currVars()
# Check which cloud diagnostic we are working with
if ('clt' in datakeys):
cl_key = 'clt'
elif ('clivi' in datakeys):
cl_key = 'clivi'
elif ('clwvi' in datakeys):
cl_key = 'clwvi'
# Extract cloud observations
datakeys.remove(cl_key)
cl_obs, cl_loc, cl_models = E.get_clim_model_and_obs_filenames(cl_key)
cl_obsfile = nc.Dataset(cl_loc, 'r')
# A-laue_ax+
E.add_to_filelist(cl_loc)
# A-laue_ax-
# Basic structure of the figure is same for all
# plots - we clear it at intervals
for area in areas:
# Get area specific configuration and observations
coords = E.get_area_coordinates(modelconfig, experiment, area)
area_key = experiment + '_scatter_' + area
cl_lats, cl_lons, cl_data = E.get_model_data(modelconfig, experiment,
area, cl_key, cl_obsfile)
cl_data = E.average_data(cl_data, 'annual')
for datakey in datakeys:
# Extract radiation observations
rd_obs, rd_loc, rd_models = E.get_clim_model_and_obs_filenames(
datakey)
rd_obsfile = nc.Dataset(rd_loc, 'r')
# A-laue_ax+
E.add_to_filelist(rd_loc)
# A-laue_ax-
rd_lats, rd_lons, rd_data = E.get_model_data(modelconfig,
experiment,
area,
datakey,
rd_obsfile,
extend='lons')
# Check that we have similar model sets
E.check_model_instances(cl_models, rd_models)
# Average and interpolate to cloud obsevation grid
rd_data = E.average_data(rd_data, 'annual')
rd_data = interpolate_3d(rd_data, rd_lats, rd_lons, cl_lats,
cl_lons)
for model in rd_models:
# Get model and area specific values
cl_datafile = nc.Dataset(cl_models[model], 'r')
rd_datafile = nc.Dataset(rd_models[model], 'r')
# A-laue_ax+
E.add_to_filelist(cl_models[model])
E.add_to_filelist(rd_models[model])
# A-laue_ax-
mcl_lats, mcl_lons, mcl_data = E.get_model_data(
modelconfig, experiment, area, cl_key, cl_datafile)
mrd_lats, mrd_lons, mrd_data = E.get_model_data(modelconfig,
experiment,
area,
datakey,
rd_datafile,
extend='lons')
cl_datafile.close()
rd_datafile.close()
# Average and interpolate model data to cloud data grid
mcl_data = E.average_data(mcl_data, 'annual')
mrd_data = E.average_data(mrd_data, 'annual')
mrd_data = interpolate_3d(mrd_data, mrd_lats, mrd_lons,
mcl_lats, mcl_lons)
# One plot for all seasons
plt.clf()
fig, axs = plt.subplots(2, (len(seasons)),
figsize=(4 * len(seasons), 8))
fig.subplots_adjust(top=0.8)
fig.subplots_adjust(bottom=0.15)
fig.subplots_adjust(right=0.75)
fig.subplots_adjust(hspace=0.4)
fig.subplots_adjust(wspace=0.4)
for season in seasons:
col = seasons.index(season)
# Mask unwanted seasonal values
obs_cl = E.extract_seasonal_mean_values(modelconfig,
cl_data,
experiment,
season,
monthly=True)
obs_rd = E.extract_seasonal_mean_values(modelconfig,
rd_data,
experiment,
season,
monthly=True)
data_cl = E.extract_seasonal_mean_values(modelconfig,
mcl_data,
experiment,
season,
monthly=True)
data_rd = E.extract_seasonal_mean_values(modelconfig,
mrd_data,
experiment,
season,
monthly=True)
# Calculate scatterplot values
obs_out = calculate_scatterplot_values(
modelconfig, area, cl_key, obs_cl, obs_rd)
model_out = calculate_scatterplot_values(modelconfig,
area,
cl_key,
data_cl,
data_rd,
bins=obs_out[0])
centers = np.zeros(len(obs_out[0]) - 1)
for i in xrange(len(centers)):
centers[i] = (obs_out[0][i + 1] + obs_out[0][i]) / 2.
# Plot and make it pretty
modelcolor, dashes, width = E.get_model_plot_style(model)
obs_label = cl_obs + ', \n' + rd_obs
axs[0, col].scatter(model_out[2],
model_out[3],
facecolors=modelcolor,
edgecolors=modelcolor,
label=model)
axs[0, col].scatter(obs_out[2],
obs_out[3],
facecolors='none',
edgecolors='black',
label=obs_label)
if (len(dashes) == 0):
axs[1, col].plot(centers,
obs_out[1],
color=modelcolor,
linewidth=width,
label=model)
else:
line1, = axs[1, col].plot(centers,
model_out[1],
'--',
color=modelcolor,
linewidth=width,
label=model)
line1.set_dashes(dashes)
axs[1, col].plot(centers,
obs_out[1],
color='black',
label=obs_label)
# If we get nan as values we convert it to zero
obs_x = np.ma.masked_array(obs_out[2])
obs_y = np.ma.masked_array(obs_out[3])
model_x = np.ma.masked_array(model_out[2])
model_y = np.ma.masked_array(model_out[3])
rmse = (np.mean((obs_x - model_x)**2) + np.mean(
(obs_y - model_y)**2))**0.5
rmse = round(rmse, 1)
xmin = int(obs_out[0][0])
xmax = int(np.ceil(obs_out[0][-1]))
# Make it prettier
axs[0,
col].set_title(season + " (RMSE: " + str(rmse) + ")",
fontsize=12)
axs[0, col].set_xlim(xmin, xmax)
axs[0, col].locator_params(axis='x', nbins=5)
axs[0, col].locator_params(axis='y', nbins=5)
axs[0, col].grid(plot_grid)
axs[1, col].set_xlim(xmin, xmax)
axs[1, col].locator_params(axis='x', nbins=5)
axs[1, col].locator_params(axis='y', nbins=5)
axs[1, col].grid(plot_grid)
xlabel = E.get_title_basename(cl_key)
ylabel = E.get_title_basename(datakey)
axs[0, 0].set_xlabel(xlabel)
axs[0, 0].set_ylabel('Radiation')
axs[1, 0].set_xlabel('Cloud cover percentage')
axs[1, 0].set_ylabel('Proportion of values')
# Title and legend
suptitle = ylabel + " sensitivity to " + xlabel
plt.suptitle(suptitle, fontsize=20)
handles0, labels0 = axs[0, 0].get_legend_handles_labels()
handles1, labels1 = axs[1, 0].get_legend_handles_labels()
handles = handles0 + handles1
labels = labels0 + labels1
plt.legend(handles,
labels,
loc='center left',
bbox_to_anchor=(0.78, 0.5),
bbox_transform=plt.gcf().transFigure)
# Define ouput name and save
variable = cl_key + '-' + datakey
diag_name = E.get_diag_script_name()
output_file = E.get_plot_output_filename(diag_name=diag_name,
variable=variable,
model=model)
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
plt.savefig(os.path.join(output_dir, output_file))
plt.clf()
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
cl_obsfile.close()
rd_obsfile.close()
def process_divergence(E, modelconfig):
"""This script preprocessess equatorial divergence values.
The data is extracted and saved to a npy-file. Plotting is done in
TropicalVariability_EQ.py. """
experiment = 'equatorial'
datakeys = E.get_currVars()
verbosity = E.get_verbosity()
plot_dir = E.get_plot_dir()
work_dir = E.get_work_dir()
areas = modelconfig.get(experiment, 'areas').split()
if ('ua' in datakeys[0]):
ua_key = datakeys[0]
va_key = datakeys[1]
elif ('ua' in datakeys[1]):
ua_key = datakeys[1]
va_key = datakeys[0]
else:
print("PY ERROR: Unexpected variables for divergence.")
print("PY ERROR: I was expecting only 'ua' and 'va' (or similar).")
print("PY ERROR: What I got was: " + datakeys[0] + ", " + datakeys[1])
print("PY ERROR: Stopping the script and exiting")
sys.exit()
# Starting to extract required areas
for area in areas:
area_key = experiment + '_' + area
lat_min = modelconfig.getint(area_key, 'lat_min')
lat_max = modelconfig.getint(area_key, 'lat_max')
lon_min = modelconfig.getint(area_key, 'lon_min')
lon_max = modelconfig.getint(area_key, 'lon_max')
# create a python array file for each area, model and datakey
# overwrite if exists
base_name = work_dir\
+ 'TropicalVariability_'\
+ area_key\
+ '_divergence_model_'
ua_models = E.get_clim_model_filenames(variable=ua_key)
va_models = E.get_clim_model_filenames(variable=va_key)
E.check_model_instances(ua_models, va_models)
# Get obs model name and extract these first
for model in ua_models:
if (E.get_model_id(model) == 'obs'):
obs = model
out_file = base_name + model + '.npy'
ua_datafile = nc.Dataset(ua_models[model], 'r')
va_datafile = nc.Dataset(va_models[model], 'r')
# A-laue_ax+
E.add_to_filelist(ua_models[model])
E.add_to_filelist(va_models[model])
# A-laue_ax-
# Add ghost layers for ua longtitudes
# va doesn't need them. Lats and lons are same for both
ulats, ulons, ua_data = E.get_model_data(modelconfig,
experiment,
area,
ua_key,
ua_datafile,
extend='lons')
vlats, vlons, va_data = E.get_model_data(modelconfig,
experiment,
area,
va_key,
va_datafile,
extend='lats')
ulons = E.ensure_looping(ulons)
vlons = E.ensure_looping(vlons)
ua_datafile.close()
va_datafile.close()
ua_obs, ua_derv = calculate_derivatives(
E, ulats, ulons, ua_data, 'ua')
va_obs, va_derv = calculate_derivatives(
E, vlats, vlons, va_data, 'va')
diverg = np.zeros(len(vlons))
for lon in xrange(diverg.shape[0]):
diverg[lon] = np.mean(ua_derv[:, lon] + va_derv[:, lon])
# Now we have everything so only output needed
outdata = vlons, diverg
np.save(out_file, outdata)
obscolor, obsdashes, obswidth = E.get_model_plot_style(model)
obslats, obslons = ulats, vlons
del ua_models[obs]
# Now we do the same for all models and plot the wind vector field
for model in ua_models:
out_file = base_name + model + '.npy'
ua_datafile = nc.Dataset(ua_models[model], 'r')
va_datafile = nc.Dataset(va_models[model], 'r')
# A-laue_ax+
E.add_to_filelist(ua_models[model])
E.add_to_filelist(va_models[model])
# A-laue_ax-
# Add ghost layers for ua longtitudes
# va doesn't need them. Lats and lons are same for both
ulats, ulons, ua_data = E.get_model_data(modelconfig,
experiment,
area,
ua_key,
ua_datafile,
extend='lons')
vlats, vlons, va_data = E.get_model_data(modelconfig,
experiment,
area,
va_key,
va_datafile,
extend='lats')
ulons = E.ensure_looping(ulons)
vlons = E.ensure_looping(vlons)
ua_datafile.close()
va_datafile.close()
ua_mean, ua_derv = calculate_derivatives(E, ulats, ulons, ua_data,
'ua')
va_mean, va_derv = calculate_derivatives(E, vlats, vlons, va_data,
'va')
diverg = np.zeros(len(vlons))
for lon in xrange(diverg.shape[0]):
diverg[lon] = np.mean(ua_derv[:, lon] + va_derv[:, lon])
# Now we have everything so only output needed
outdata = vlons, diverg
np.save(out_file, outdata)
# Interpolate model data to obs grid
ua_mean = interpolate_data_grid(ua_mean, ulats, vlons, obslats,
obslons)
va_mean = interpolate_data_grid(va_mean, ulats, vlons, obslats,
obslons)
color, dashes, width = E.get_model_plot_style(model)
# Ensure proper handling for lons
if (lon_max < lon_min):
lon_max += 360
# Start the plots
plt.clf()
fig, axs = plt.subplots(2,
1,
figsize=(5 + (lon_max - lon_min) / 10, 10))
fig.subplots_adjust(hspace=0.3)
for ax in axs.flat:
map_ax = Basemap(ax=ax,
fix_aspect=False,
llcrnrlat=obslats[0],
urcrnrlat=obslats[-1],
llcrnrlon=obslons[0],
urcrnrlon=obslons[-1])
map_ax.drawcoastlines()
Q0 = axs[0].quiver(obslons,
obslats,
va_mean,
ua_mean,
color=color,
pivot='middle',
units='xy',
scale=5.0,
width=0.1,
headwidth=4,
headlength=3,
headaxislength=4)
Q1 = axs[1].quiver(obslons,
obslats,
va_obs,
ua_obs,
color=obscolor,
pivot='middle',
units='xy',
scale=5.0,
width=0.1,
headwidth=4,
headlength=3,
headaxislength=4)
qk0 = plt.quiverkey(Q0, 1.05, 0.5, 5, r'$5 \frac{m}{s}$')
qk1 = plt.quiverkey(Q1, 1.05, 0.5, 5, r'$5 \frac{m}{s}$')
# Ticks, labels, header etc.
xticks = E.get_ticks(10, np.array([lon_min, lon_max]))
yticks = E.get_ticks(5, np.array([lat_min, lat_max]))
xlabels = E.get_ticks_labels(xticks, 'lons')
ylabels = E.get_ticks_labels(yticks, 'lats')
axs[0].set_title(model)
axs[0].set_xlim(lon_min, lon_max)
axs[0].set_ylim(lat_min, lat_max)
axs[0].xaxis.set_ticks(xticks)
axs[0].yaxis.set_ticks(yticks)
axs[0].set_xticklabels(xlabels)
axs[0].set_yticklabels(ylabels)
axs[1].set_title(obs)
axs[1].set_xlim(lon_min, lon_max)
axs[1].set_ylim(lat_min, lat_max)
axs[1].xaxis.set_ticks(xticks)
axs[1].yaxis.set_ticks(yticks)
axs[1].set_xticklabels(xlabels)
axs[1].set_yticklabels(ylabels)
# Get output filename and save the figure
diag_name = E.get_diag_script_name()
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
plt.suptitle(area + " ocean mean wind direction and strength",
fontsize=16)
variable_string = ''.join([ua_key, va_key])
output_file = E.get_plot_output_filename(diag_name=diag_name,
model=model,
variable=variable_string,
specifier=area)
plt.savefig(os.path.join(output_dir, output_file))
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
def process_radiation_maps(E, modelconfig, datakey, specifier):
"""
Main script for gathering radiation seasonal
map values and plotting them.
"""
config_file = E.get_configfile()
experiment = 'SouthernHemisphere'
areas = modelconfig.get(experiment, 'areas').split()
seasons = modelconfig.get(experiment, 'seasons').split()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
work_dir = E.get_work_dir()
datakeycs = datakey + 'cs'
# Check that we have similar model sets (obs should be the same)
obsmodel, obs_loc, models = E.get_clim_model_and_obs_filenames(datakey)
obsmodel, obscs_loc, modelscs = E.get_clim_model_and_obs_filenames(datakeycs)
E.check_model_instances(models, modelscs)
# Loop over areas and seasons - generate season specific maps
obsfile = nc.Dataset(obs_loc, 'r')
obscsfile = nc.Dataset(obscs_loc, 'r')
# A-laue_ax+
E.add_to_filelist(obs_loc)
E.add_to_filelist(obscs_loc)
# A-laue_ax-
# Basic structure of the figure is same for all plots,
# we clear it at intervals
for area in areas:
# Get area specific configuration and observations
coords = E.get_area_coordinates(modelconfig, experiment, area)
xticks = E.get_ticks(5, coords[2:])
yticks = E.get_ticks(3, coords[:2])
xlabels = E.get_ticks_labels(xticks, 'lons')
ylabels = E.get_ticks_labels(yticks, 'lats')
area_key = experiment + '_' + area
obslats, obslons, obsdata = E.get_model_data(modelconfig,
experiment,
area,
datakey,
obsfile)
obscslats, obscslons, obscsdata = E.get_model_data(modelconfig,
experiment,
area,
datakeycs,
obscsfile)
for model in models:
# Get model and area specific values
datafile = nc.Dataset(models[model], 'r')
datafilecs = nc.Dataset(modelscs[model], 'r')
# A-laue_ax+
E.add_to_filelist(models[model])
E.add_to_filelist(modelscs[model])
# A-laue_ax-
mlats, mlons, mdata = E.get_model_data(modelconfig,
experiment,
area,
datakey,
datafile)
mcslats, mcslons, mcsdata = E.get_model_data(modelconfig,
experiment,
area,
datakeycs,
datafilecs)
units = datafile.variables[datakey].units
for season in seasons:
# Extract seasonal specific values from area values
data = E.extract_seasonal_mean_values(modelconfig,
mdata,
experiment,
season)
datacs = E.extract_seasonal_mean_values(modelconfig,
mcsdata,
experiment,
season)
obs = E.extract_seasonal_mean_values(modelconfig,
obsdata,
experiment,
season)
obscs = E.extract_seasonal_mean_values(modelconfig,
obscsdata,
experiment,
season)
# Interpolate to obs grid
data = interpolate_data_grid(data,
mlats,
mlons,
obslats,
obslons)
datacs = interpolate_data_grid(datacs,
mcslats,
mcslons,
obslats,
obslons)
obscs = interpolate_data_grid(obscs,
obscslats,
obscslons,
obslats,
obslons)
# Get contour specific plotting limits
a, b, c, d, e, f = get_contour_config(modelconfig,
config_file,
area_key,
datakey)
lev, diff, diff2, cm_model, cm_diff, cm_dev = a, b, c, d, e, f
# Plot the figures
fig, axs = plt.subplots(3, 2, figsize=(21, 15))
fig.subplots_adjust(hspace=1.0)
# Same Basemap (coastlines) for all subplots
# This is what usually takes time to process
for ax in axs.flat:
map_ax = Basemap(ax=ax,
fix_aspect=False,
llcrnrlat=coords[0], urcrnrlat=coords[1],
llcrnrlon=coords[2], urcrnrlon=coords[3])
map_ax.drawcoastlines()
# Plot the contours and colourmaps
dash = ' - ' # Could also use u"\u2014" for long dash
c00 = axs[0, 0].contourf(obslons,
obslats,
data,
levels=lev,
cmap=cm_model,
extend='both',
latlon=True)
cax00 = fig.add_axes([0.15, 0.69, 0.3, 0.01])
cbar00 = plt.colorbar(c00, cax=cax00, orientation='horizontal')
cbar00.set_label(units)
axs[0, 0].set_title(model)
c01 = axs[0, 1].contourf(obslons,
obslats,
datacs,
levels=lev,
cmap=cm_model,
extend='both',
latlon=True)
cax01 = fig.add_axes([0.575, 0.69, 0.3, 0.01])
cbar01 = plt.colorbar(c01, cax=cax01, orientation='horizontal')
cbar01.set_label(units)
axs[0, 1].set_title(model + ' (clear sky)')
c10 = axs[1, 0].contourf(obslons,
obslats,
data - obs,
levels=diff,
cmap=cm_diff,
extend='both',
latlon=True)
cax10 = fig.add_axes([0.15, 0.37, 0.3, 0.01])
cbar10 = plt.colorbar(c10, cax=cax10, orientation='horizontal')
cbar10.set_label(units)
axs[1, 0].set_title(model + dash + obsmodel)
c11 = axs[1, 1].contourf(obslons,
obslats,
datacs - obscs,
levels=diff,
cmap=cm_diff,
extend='both',
latlon=True)
cax11 = fig.add_axes([0.575, 0.37, 0.3, 0.01])
cbar11 = plt.colorbar(c11, cax=cax11, orientation='horizontal')
cbar11.set_label(units)
axs[1, 1].set_title(model + ' (cs)' + dash + obsmodel + ' (cs)')
c20 = axs[2, 0].contourf(obslons,
obslats,
data - datacs,
levels=diff2,
cmap=cm_dev,
extend='both',
latlon=True)
cax20 = fig.add_axes([0.15, 0.05, 0.3, 0.01])
cbar20 = plt.colorbar(c20, cax=cax20, orientation='horizontal')
cbar20.set_label(units)
axs[2, 0].set_title(model + dash + model + ' (cs)')
c21 = axs[2, 1].contourf(obslons,
obslats,
obs - obscs,
levels=diff2,
cmap=cm_dev,
extend='both',
latlon=True)
cax21 = fig.add_axes([0.575, 0.05, 0.3, 0.01])
cbar21 = plt.colorbar(c21, cax=cax21, orientation='horizontal')
cbar21.set_label(units)
axs[2, 1].set_title(obsmodel + dash + obsmodel + ' (cs)')
# Labels
for row in xrange(3):
for col in xrange(2):
axs[row, col].xaxis.set_ticks(xticks)
axs[row, col].set_xticklabels(xlabels)
axs[row, col].yaxis.set_ticks(yticks)
axs[row, col].set_yticklabels(ylabels)
# Title and filename based on variables
suptitle = E.get_title_basename(datakey) + " for months: " + season
plt.suptitle(suptitle, fontsize=20)
variable = datakey
specifier = specifier + "-" + season
diag_name = E.get_diag_script_name()
output_file = E.get_plot_output_filename(variable=variable,
specifier=specifier,
model=model)
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
plt.savefig(os.path.join(output_dir, output_file))
plt.clf()
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
datafile.close()
datafilecs.close()
obsfile.close()
obscsfile.close()
def process_scatterplot(E, modelconfig):
"""Main script for scatterplots. Outputs model specific scatterplots with
specified observations. Also prints out comparable statistics. """
experiment = 'scatter'
config_file = E.get_configfile()
datakeys = E.get_currVars()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
plot_grid = modelconfig.getboolean('general', 'plot_grid')
areas = modelconfig.get(experiment, 'areas').split()
seasons = modelconfig.get(experiment, 'seasons').split()
# Scatterplots are only for temperature/precipitation
if 'ua' in datakeys:
datakeys = datakeys.remove('ua')
# We extract the explicit precipitation datakey since precip can be in
# mmday or kg/m2s - this affects the datakey (pr or pr-mmday)
# For consistency we do this also for ts
for datakey in datakeys:
if (datakey == 'ts'):
ts_key = datakey
elif (datakey == 'pr' or datakey == 'pr-mmday'):
pr_key = datakey
# Get the required filenames and check consistency of model filenames
# The model keys should be same - the key values (paths) are different
pr_model, pr_obs_loc, pr_models = E.get_clim_model_and_obs_filenames(
pr_key)
ts_model, ts_obs_loc, ts_models = E.get_clim_model_and_obs_filenames(
ts_key)
E.check_model_instances(pr_models, ts_models)
# Initialize observations
pr_obs_file = nc.Dataset(pr_obs_loc, 'r')
ts_obs_file = nc.Dataset(ts_obs_loc, 'r')
# A-laue_ax+
E.add_to_filelist(pr_obs_loc)
E.add_to_filelist(ts_obs_loc)
# A-laue_ax-
# We are outputting model specific scatterplots with areas and seasons
# So some looping to do. Remember that keys in pr_models and ts_models
# are the same (we checked this) - the paths differ.
for model in pr_models:
# Initializing model specific files
pr_file = nc.Dataset(pr_models[model], 'r')
ts_file = nc.Dataset(ts_models[model], 'r')
# A-laue_ax+
E.add_to_filelist(pr_models[model])
E.add_to_filelist(ts_models[model])
# A-laue_ax-
pr_units = pr_file.variables[pr_key].units
ts_units = ts_file.variables[ts_key].units
# Extract model specific plotting style (we only need the colour)
modelcolor, dashes, width = E.get_model_plot_style(model)
plt.clf()
fig, axs = plt.subplots(len(areas),
len(seasons),
figsize=(3 * len(seasons),
1 + 3 * (len(areas))))
fig.subplots_adjust(top=0.83)
fig.subplots_adjust(right=0.85)
fig.subplots_adjust(hspace=0.4)
fig.subplots_adjust(wspace=0.3)
# Next we extract the required values for each area
for area in areas:
pr_data = E.get_model_data(modelconfig, experiment, area, pr_key,
pr_file)
ts_data = E.get_model_data(modelconfig, experiment, area, ts_key,
ts_file)
pr_obs = E.get_model_data(modelconfig, experiment, area, pr_key,
pr_obs_file)
ts_obs = E.get_model_data(modelconfig, experiment, area, ts_key,
ts_obs_file)
row = areas.index(area)
area_key = experiment + '_' + area
lat_min = modelconfig.getint(area_key, 'lat_min')
lat_max = modelconfig.getint(area_key, 'lat_max')
lon_min = modelconfig.getint(area_key, 'lon_min')
lon_max = modelconfig.getint(area_key, 'lon_max')
lat_area = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats')
lon_area = E.get_ticks_labels(np.array([lon_min, lon_max]), 'lons')
c_area = "[" + lat_area[0] + ":" + lat_area[1] + ", "\
+ lon_area[0] + ":" + lon_area[1] + "]"
# And one more loop for different seasons
for season in seasons:
col = seasons.index(season)
pr_season = get_scatterplot_values(modelconfig, season,
pr_data)
ts_season = get_scatterplot_values(modelconfig, season,
ts_data)
pr_obs_se = get_scatterplot_values(modelconfig, season, pr_obs)
ts_obs_se = get_scatterplot_values(modelconfig, season, ts_obs)
# plot
axs[row, col].scatter(ts_season,
pr_season,
facecolors=modelcolor,
edgecolors=modelcolor,
label='modelled')
axs[row, col].scatter(ts_obs_se,
pr_obs_se,
facecolors='none',
edgecolors='black',
label='observed')
# statitics: errors in precip/temp and both
mse_pr = np.mean((pr_season - pr_obs_se)**2)
mse_ts = np.mean((ts_season - ts_obs_se)**2)
mse = (mse_pr + mse_ts)**0.5
rmse = np.round(mse, 2)
if (row == 0):
axs[row, col].set_title(season + "\n rmse: " + str(rmse))
else:
axs[row, col].set_title("rmse: " + str(rmse))
# Get specific plotting limits
if modelconfig.getboolean(experiment, 'seasonal_limits'):
# If we use user defined limits
limits = get_scatterplot_limits(modelconfig, config_file,
area, season)
if (len(limits) == 4):
axs[row, col].set_xlim(limits[0], limits[1])
axs[row, col].set_ylim(limits[2], limits[3])
xticks = E.get_ticks(5, limits[:2])
yticks = E.get_ticks(5, limits[2:])
else:
xticks = E.get_ticks(5, ts_season, ts_obs_se)
yticks = E.get_ticks(5, pr_season, pr_obs_se)
# Setting the ticks (where to place the numbers
axs[row, col].xaxis.set_ticks(xticks)
axs[row, col].yaxis.set_ticks(yticks)
else:
# Let the Gods decide
ts_min = int(np.min(ts_obs_se - 0.7))
ts_max = int(np.max(ts_obs_se + 1.7))
pr_min = int(np.min(pr_obs_se - 1.7))
pr_max = int(np.max(pr_obs_se + 2.7))
axs[row, col].set_xlim(ts_min, ts_max)
axs[row, col].set_ylim(pr_min, pr_max)
if (ts_max - ts_min > 4):
axs[row, col].locator_params(axis='x', nbins=5)
else:
axs[row, col].locator_params(axis='x',
nbins=ts_max - ts_min)
if (pr_max - pr_min > 6):
axs[row, col].locator_params(axis='y', nbins=7)
else:
axs[row, col].locator_params(axis='y',
nbins=pr_max - pr_min)
axs[row, -1].yaxis.set_label_position("right")
axs[row, -1].set_ylabel(area + "\n" + c_area,
rotation=0,
horizontalalignment='left')
axs[row, col].grid(plot_grid)
# Closing the model datafiles
pr_file.close()
ts_file.close()
axs[-1, 0].set_ylabel("Precipitation [" + pr_units + "]")
axs[-1, 0].set_xlabel("SST [" + ts_units + "]")
plt.suptitle("Seasonal (annual/monthly) mean values of " +
"SST and precipitation for \n" + model +
" (coloured) vs " + ts_model + " (sst obs) and " +
pr_model + " (pr obs)",
fontsize=20)
# Saving the plot and letting the user know what just happened
variable = ''.join([ts_key, pr_key])
diag_name = E.get_diag_script_name()
output_file = E.get_plot_output_filename(diag_name=diag_name,
variable=variable,
model=model,
specifier=experiment)
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
plt.savefig(os.path.join(output_dir, output_file))
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
# After all the loops remember to close the observation files aswell
pr_obs_file.close()
ts_obs_file.close()
def process_mean_plots(E, modelconfig, datakey, orientation, mean_name):
"""Generates latitudal / lontitudal / monthly mean plots.
Seasonal values are extracted for lat/lon plots. """
experiment = 'SouthernHemisphere'
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
areas = []
plot_grid = modelconfig.getboolean('general', 'plot_background_grid')
# Extract required keys based on datakey
if (orientation == 'monthly'):
if (modelconfig.getboolean('general', 'plot_monthly_averages')):
areas += modelconfig.get(experiment, 'areas').split()
if (modelconfig.getboolean('general', 'plot_sub_areas')):
areas += modelconfig.get(experiment, 'sub_areas').split()
seasons = ['monthly']
else:
areas += modelconfig.get(experiment, 'areas').split()
seasons = modelconfig.get(experiment, 'seasons').split()
# Observations location
obsmodel, obs_loc, models = E.get_clim_model_and_obs_filenames(datakey)
obsfile = nc.Dataset(obs_loc, 'r')
# A-laue_ax+
E.add_to_filelist(obs_loc)
# A-laue_ax-
scale_cloud = False
# Ugly fix for scaling cloud ice/liquid water path values
if (datakey == 'clivi' or datakey == 'clwvi'):
scale_cloud = True
scale = 1E3
scale_str = ' * 1E3'
for area in areas:
# Value configurations
area_key = experiment + '_' + area
lat_min = modelconfig.getint(area_key, 'lat_min')
lat_max = modelconfig.getint(area_key, 'lat_max')
lon_min = modelconfig.getint(area_key, 'lon_min')
lon_max = modelconfig.getint(area_key, 'lon_max')
lat_area = E.get_ticks_labels(np.array([lat_min, lat_max]), 'lats')
# First extract the observations and then start looping over seasons
obslats, obslons, obsdata = E.get_model_data(modelconfig,
experiment,
area,
datakey,
obsfile)
for season in seasons:
# Plot layout configuration
plt.clf()
fig, axs = plt.subplots(2, 1, figsize=(15, 10))
fig.subplots_adjust(top=0.9)
fig.subplots_adjust(right=0.67)
fig.subplots_adjust(hspace=0.35)
if (orientation == 'lats'):
odata = E.extract_seasonal_mean_values(modelconfig,
obsdata,
experiment,
season,
monthly=True)
odata = E.average_data(odata, 1)
xobs = obslats
elif (orientation == 'lons'):
odata = E.extract_seasonal_mean_values(modelconfig,
obsdata,
experiment,
season,
monthly=True)
odata = E.average_data(odata, 2)
xobs = obslons
elif (orientation == 'monthly'):
odata = E.average_data(obsdata, 'monthly')
xobs = np.arange(0, 12, 1)
# Bad cloud values fix
if (scale_cloud):
odata = odata * scale
# Plot model values to first graph
ocolor, odashes, owidth = E.get_model_plot_style(obsmodel)
if (len(odashes) == 0):
axs[0].plot(xobs,
odata,
color=ocolor,
linewidth=owidth,
label=obsmodel + ' (obs)')
else:
line1, = axs[0].plot(xobs,
odata,
'--',
color=ocolor,
linewidth=owidth,
label=obsmodel + ' (obs)')
line1.set_dashes(odashes)
multimodelmean_initialized = False
for model in models:
# Read in model specific data
datafile = nc.Dataset(models[model], 'r')
# A-laue_ax+
E.add_to_filelist(models[model])
# A-laue_ax-
data_units = datafile.variables[datakey].units
if (orientation == 'monthly'):
lats, lons, data = E.get_model_data(modelconfig,
experiment,
area,
datakey,
datafile)
else:
lats, lons, data = E.get_model_data(modelconfig,
experiment,
area,
datakey,
datafile,
extend=orientation)
datafile.close()
# Bad cloud values fix
if (scale_cloud):
data = data * scale
if (data_units == 'kg m-2'):
data_units = r'$\mu$m'
else:
data_units += scale_str
# Process depending on orientation
if (orientation == 'lats'):
data = E.extract_seasonal_mean_values(modelconfig,
data,
experiment,
season,
monthly=True)
data = E.average_data(data, 1)
xdata = lats
interpolate = interp1d(xdata, data, kind='cubic', bounds_error=False)
idata = interpolate(xobs)
elif (orientation == 'lons'):
data = E.extract_seasonal_mean_values(modelconfig,
data,
experiment,
season,
monthly=True)
data = E.average_data(data, 2)
xdata = lons
interpolate = interp1d(xdata, data, kind='cubic', bounds_error=False)
idata = interpolate(xobs)
elif (orientation == 'monthly'):
data = E.average_data(data, 'monthly')
xdata = xobs
idata = data
# plotting custom dashes requires some extra effort (else)
# with empty dashes the format is default
rmse = round(np.mean((idata - odata) ** 2) ** 0.5, 1)
mlabel = model + " (RMSE: " + str(rmse) + ")"
color, dashes, width = E.get_model_plot_style(model)
if (len(dashes) == 0):
axs[0].plot(xdata,
data, color=color,
linewidth=width,
label=mlabel)
axs[1].plot(xobs,
idata - odata,
color=color,
linewidth=width,
label=mlabel)
else:
line1, = axs[0].plot(xdata,
data,
'--',
color=color,
linewidth=width,
label=mlabel)
line1.set_dashes(dashes)
line2, = axs[1].plot(xobs,
idata - odata,
color=color,
linewidth=width,
label=mlabel)
line2.set_dashes(dashes)
# Next the multimodel mean. We can just sum it up and divide
if not (multimodelmean_initialized):
multimodelmean_initialized = True
mmmean = idata
else:
mmmean += idata
# Plot the multimodel mean out if required
if (len(models) > 1):
mmmean = mmmean / len(models)
rmse = round(np.mean((mmmean - odata) ** 2) ** 0.5, 1)
mlabel = "Multimodel mean (RMSE: " + str(rmse) + ")"
color, dashes, width = E.get_model_plot_style('model_mean')
if (len(dashes) == 0):
axs[0].plot(xobs,
mmmean,
color=color,
linewidth=width,
label=mlabel)
axs[1].plot(xobs,
mmmean - odata,
color=color,
linewidth=width,
label=mlabel)
else:
line1, = axs[0].plot(xobs,
mmmean,
'--',
color=color,
linewidth=width,
label=mlabel)
line1.set_dashes(dashes)
line2, = axs[1].plot(xobs,
mmmean - odata,
color=color,
linewidth=width,
label=mlabel)
line2.set_dashes(dashes)
# Now to make the plot pretty i.e. headers, ticks, title etc.
suptitle = E.get_title_basename(datakey)
specifier = mean_name
if (orientation == 'lats'):
title = "Latitudinal mean"
specifier += '-latitudinal-mean'
x_min = lat_min
x_max = lat_max
xticks = E.get_ticks(8, np.array([x_min, x_max]))
lon_info = E.get_ticks_labels(np.array([lon_min, lon_max]),
'lons')
if (int(lon_max) == 360):
suptitle += " (lons [" + lon_info[0] + ":360" + "])"
else:
suptitle += " (lons [" + lon_info[0] + ":" + lon_info[1] + "])"
elif (orientation == 'lons'):
title = "Longitudinal mean"
specifier += '-longitudinal-mean'
x_min = lon_min
x_max = lon_max
xticks = E.get_ticks(8, np.array([x_min, x_max]))
lat_info = E.get_ticks_labels(np.array([lat_min, lat_max]),
'lats')
suptitle += " (lats [" + lat_info[0] + ":" + lat_info[1] + "])"
elif (orientation == 'monthly'):
title = "Monthly mean"
specifier += '-yearly-cycle-' + area
x_min = 0
x_max = 11
xticks = xobs
lat_info = E.get_ticks_labels(np.array([lat_min, lat_max]),
'lats')
lon_info = E.get_ticks_labels(np.array([lon_min, lon_max]),
'lons')
suptitle += " (lats [" + lat_info[0] + ":" + lat_info[1] + "], "
if (lon_max == 360):
suptitle += " lons [" + lon_info[0] + ":360" + "])"
else:
suptitle += " lons [" + lon_info[0] + ":" + lon_info[1] + "])"
if (season == 'monthly'):
pass
else:
suptitle += " for " + season
specifier = specifier + "-" + season
plt.suptitle(suptitle, fontsize=20)
labels = E.get_ticks_labels(xticks, orientation)
axs[0].grid(plot_grid)
axs[0].set_xlim(x_min, x_max)
axs[0].xaxis.set_ticks(xticks)
axs[0].set_xticklabels(labels)
axs[0].locator_params(axis='y', nbins=5)
axs[0].set_title(title + " [" + data_units + "]")
axs[1].grid(plot_grid)
axs[1].set_xlim(x_min, x_max)
axs[1].xaxis.set_ticks(xticks)
axs[1].set_xticklabels(labels)
axs[1].locator_params(axis='y', nbins=5)
axs[1].set_title(title + " (model - obs) [" + data_units + "]")
# Saving the plot and letting the user know what just happened
diag_name = E.get_diag_script_name()
output_file = E.get_plot_output_filename(diag_name=diag_name,
variable=datakey,
specifier=specifier)
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
handles, labels = axs[0].get_legend_handles_labels()
plt.legend(handles,
labels,
loc='center left',
bbox_to_anchor=(0.7, 0.5),
bbox_transform=plt.gcf().transFigure)
plt.savefig(os.path.join(output_dir, output_file))
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
obsfile.close()
def main(_argv):
print("Start")
info("OpenCV version: " + CV_VERSION)
# Default parameters:
args = Struct(
ImageFile=
"/Users/shahargino/Documents/ImageProcessing/Bracelet_decoder/Database/180717/24.png",
PreprocessCvcSel="V",
PreprocessMode="Legacy",
PreprocessGaussKernel=(5, 5),
PreprocessThreshBlockSize=19,
PreprocessThreshweight=7,
PreprocessMorphKernel=(3, 3),
PreprocessMedianBlurKernel=13,
PreprocessCannyThr=80,
imgEnhancementEn=False,
MinPixelWidth=7,
MaxPixelWidth=30,
MinPixelHeight=7,
MaxPixelHeight=30,
MinAspectRatio=0.6,
MaxAspectRatio=2.5,
MinPixelArea=150,
MaxPixelArea=600,
MinExtent=0.4,
MaxExtent=0.9,
MaxDrift=2.5,
MarksRows=3,
MarksCols=10,
ROI=(0, 0),
PerspectiveMode=0,
FindContoursMode="Legacy",
HoughParams=(-1, -1, -1, -1, -1, -1),
debugMode=False)
# -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- ..
# User-Arguments parameters (overrides Defaults):
try:
opts, user_args = getopt(_argv, "hvi:", [
"PreprocessCvcSel=", "PreprocessMode=", "PreprocessGaussKernel=",
"PreprocessThreshBlockSize=", "PreprocessThreshweight=",
"PreprocessMorphKernel=", "PreprocessMedianBlurKernel=",
"PreprocessCannyThr=", "ROI=", "MinPixelWidth=", "MaxPixelWidth=",
"MinPixelHeight=", "MaxPixelHeight=", "MinAspectRatio=",
"MaxAspectRatio=", "MinPixelArea=", "MaxPixelArea=", "MinExtent=",
"MaxExtent=", "MaxDrift=", "MarksRows=", "MarksCols=",
"imgEnhancementEn", "PerspectiveMode=", "FindContoursMode=",
"HoughParams=", "debug", "version"
])
for opt, user_arg in opts:
if opt == '-h':
usage()
exit()
elif opt in "-i":
args.ImageFile = user_arg
elif opt in "--PreprocessCvcSel":
args.PreprocessCvcSel = user_arg
elif opt in "--PreprocessMode":
args.PreprocessMode = user_arg
elif opt in "--PreprocessGaussKernel":
args.PreprocessGaussKernel = literal_eval(user_arg)
elif opt in "--PreprocessThreshBlockSize":
args.PreprocessThreshBlockSize = int(user_arg)
elif opt in "--PreprocessThreshweight":
args.PreprocessThreshweight = int(user_arg)
elif opt in "--PreprocessMorphKernel":
args.PreprocessMorphKernel = literal_eval(user_arg)
elif opt in "--PreprocessMedianBlurKernel":
args.PreprocessMedianBlurKernel = int(user_arg)
elif opt in "--PreprocessCannyThr":
args.PreprocessCannyThr = int(user_arg)
elif opt in "--ROI":
args.ROI = literal_eval(user_arg)
elif opt in "--MinPixelWidth":
args.MinPixelWidth = float(user_arg)
elif opt in "--MaxPixelWidth":
args.MaxPixelWidth = float(user_arg)
elif opt in "--MinPixelHeight":
args.MinPixelHeight = float(user_arg)
elif opt in "--MaxPixelHeight":
args.MaxPixelHeight = float(user_arg)
elif opt in "--MinAspectRatio":
args.MinAspectRatio = float(user_arg)
elif opt in "--MaxAspectRatio":
args.MaxAspectRatio = float(user_arg)
elif opt in "--MinPixelArea":
args.MinPixelArea = float(user_arg)
elif opt in "--MaxPixelArea":
args.MaxPixelArea = float(user_arg)
elif opt in "--MinExtent":
args.MinExtent = float(user_arg)
elif opt in "--MaxExtent":
args.MaxExtent = float(user_arg)
elif opt in "--MaxDrift":
args.MaxDrift = float(user_arg)
elif opt in "--MarksRows":
args.MarksRows = int(user_arg)
elif opt in "--MarksCols":
args.MarksCols = int(user_arg)
elif opt in "--imgEnhancementEn":
args.imgEnhancementEn = True
elif opt in "--PerspectiveMode":
args.PerspectiveMode = int(user_arg)
elif opt in "--FindContoursMode":
args.FindContoursMode = user_arg
elif opt in "--HoughParams":
args.HoughParams = literal_eval(user_arg)
elif opt in "--debug":
args.debugMode = True
elif opt in "--version" or opt == '-v':
info("BraceletDecoder version: %s" % __version__)
exit()
except GetoptError as e:
error(str(e))
usage()
exit(2)
# -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- .. -- ..
# Load input image:
frame_orig = imread(args.ImageFile)
height, width, _ = frame_orig.shape
# Create a working environment (debugMode):
if args.debugMode:
envpath = datetime.now().strftime(
"results_%d%m%y_%H%M%S_") + args.ImageFile.split('/')[-1].replace(
'.', '_')
if not path.exists(envpath):
makedirs(envpath)
cwd = getcwd()
chdir(envpath)
# Resizing preparations:
if args.PreprocessMode == "Legacy":
resizingVec = (1600, 1200)
if (width < height):
resizingVec = resizingVec[::-1]
elif args.PreprocessMode == "BlurAndCanny":
resizingVec = (1120, 840)
else:
error("Unsupported PreprocessMode mode: %s" % args.PreprocessMode)
# Resizing and ROI cropping:
if args.PreprocessMode == "Legacy":
imgResized = resize(frame_orig, resizingVec)
imgCropped = crop_roi_from_image(imgResized, args.ROI)
image = imgCropped
elif args.PreprocessMode == "BlurAndCanny":
imgCropped = crop_roi_from_image(frame_orig, args.ROI)
imgResized = resize(imgCropped, resizingVec)
image = imgResized
# Image enhancement:
if (args.imgEnhancementEn):
imgEnhanced = imageEnhancement(image, 2, (8, 8), 3, args.debugMode)
else:
imgEnhanced = image
# Preprocess:
frame_gray, frame_thresh = preprocess(
imgEnhanced, args.PreprocessCvcSel, args.PreprocessMode,
args.PreprocessGaussKernel, args.PreprocessThreshBlockSize,
args.PreprocessThreshweight, args.PreprocessMorphKernel,
args.PreprocessMedianBlurKernel, args.PreprocessCannyThr)
# Find bracelet marks:
possible_marks, rotation_angle = find_possible_marks(
frame_thresh, args.MinPixelWidth, args.MaxPixelWidth,
args.MinPixelHeight, args.MaxPixelHeight, args.MinAspectRatio,
args.MaxAspectRatio, args.MinPixelArea, args.MaxPixelArea,
args.MinExtent, args.MaxExtent, args.MaxDrift, args.PerspectiveMode,
args.FindContoursMode, args.HoughParams, args.debugMode)
# Encode marks:
code = decode_marks(possible_marks, args.MarksRows, args.MarksCols,
frame_thresh.shape, rotation_angle, args.debugMode)
info("Code = %s" % code)
if args.debugMode:
imwrite("frame_orig.jpg", draw_roi(frame_orig, args.ROI))
imwrite("frame_gray.jpg", frame_gray)
imwrite("frame_thresh.jpg", frame_thresh)
# Restore path:
chdir(cwd)
def process_simple_maps(E, modelconfig, datakey, specifier):
""" Main script for gathering seasonal map values and plotting them. """
config_file = E.get_configfile()
experiment = 'SouthernHemisphere'
areas = modelconfig.get(experiment, 'areas').split()
seasons = modelconfig.get(experiment, 'seasons').split()
plot_dir = E.get_plot_dir()
verbosity = E.get_verbosity()
work_dir = E.get_work_dir()
# Get the observations
obsmodel, obs_loc, models = E.get_clim_model_and_obs_filenames(datakey)
obsfile = nc.Dataset(obs_loc, 'r')
# A-laue_ax+
E.add_to_filelist(obs_loc)
# A-laue_ax-
scale_cloud = False
# Ugly fix for scaling cloud ice/liquid water path values
if (datakey == 'clivi' or datakey == 'clwvi'):
scale_cloud = True
scale = 1E3
scale_str = ' * 1E3'
# Basic structure of the figure is same for all plots,
# we clear it at intervals
for area in areas:
# Get area specific configuration and observations
coords = E.get_area_coordinates(modelconfig, experiment, area)
xticks = E.get_ticks(5, coords[2:])
yticks = E.get_ticks(3, coords[:2])
xlabels = E.get_ticks_labels(xticks, 'lons')
ylabels = E.get_ticks_labels(yticks, 'lats')
area_key = experiment + '_' + area
obslats, obslons, obsdata = E.get_model_data(modelconfig,
experiment,
area,
datakey,
obsfile)
if (scale_cloud):
obsdata = obsdata * scale
for model in models:
# Get model and area specific values
datafile = nc.Dataset(models[model], 'r')
# A-laue_ax+
E.add_to_filelist(models[model])
# A-laue_ax-
mlats, mlons, mdata = E.get_model_data(modelconfig,
experiment,
area,
datakey,
datafile)
units = datafile.variables[datakey].units
if (scale_cloud):
mdata = mdata * scale
if (units == 'kg m-2'):
units = r'$\mu$m'
else:
units += scale_str
for season in seasons:
# Extract seasonal specific values from area values
data = E.extract_seasonal_mean_values(modelconfig,
mdata,
experiment,
season)
obs = E.extract_seasonal_mean_values(modelconfig,
obsdata,
experiment,
season)
# Interpolate to obs grid
data = interpolate_data_grid(data,
mlats,
mlons,
obslats,
obslons)
# Get contour specific plotting limits
lev, diff, cm_model, cm_diff = get_contour_config(modelconfig,
config_file,
area_key,
datakey)
# Plot the figures
fig, axs = plt.subplots(2, 1, figsize=(15, 10))
fig.subplots_adjust(top=0.9)
fig.subplots_adjust(hspace=0.5)
# Same Basemap (coastlines) for all subplots
# This is what usually takes time to process
for ax in axs.flat:
map_ax = Basemap(ax=ax,
fix_aspect=False,
llcrnrlat=coords[0], urcrnrlat=coords[1],
llcrnrlon=coords[2], urcrnrlon=coords[3])
map_ax.drawcoastlines()
# Plot the contours and colourmaps
dash = ' - ' # Could also use u"\u2014" for long dash
c0 = axs[0].contourf(obslons, obslats, data,
levels=lev, cmap=cm_model,
extend='both', latlon=True)
cax0 = fig.add_axes([0.36, 0.525, 0.3, 0.01])
cbar0 = plt.colorbar(c0, cax=cax0, orientation='horizontal')
cbar0.set_label(units)
axs[0].set_title(model)
c1 = axs[1].contourf(obslons, obslats, data - obs,
levels=diff, cmap=cm_diff,
extend='both', latlon=True)
cax1 = fig.add_axes([0.36, 0.05, 0.3, 0.01])
cbar1 = plt.colorbar(c1, cax=cax1, orientation='horizontal')
cbar1.set_label(units)
axs[1].set_title(model + dash + obsmodel)
# Labels
for row in xrange(2):
axs[row].xaxis.set_ticks(xticks)
axs[row].set_xticklabels(xlabels)
axs[row].yaxis.set_ticks(yticks)
axs[row].set_yticklabels(ylabels)
# Title and filename based on variables
suptitle = E.get_title_basename(datakey) + " for months: " + season
plt.suptitle(suptitle, fontsize=20)
variable = datakey
specifier = specifier + "-" + season
diag_name = E.get_diag_script_name()
output_file = E.get_plot_output_filename(diag_name=diag_name,
variable=variable,
specifier=specifier,
model=model)
# Save the image and let the user know what happened
output_dir = os.path.join(plot_dir, diag_name)
E.ensure_directory(output_dir)
plt.savefig(os.path.join(output_dir, output_file))
plt.clf()
info("", verbosity, 1)
info("Created image: ", verbosity, 1)
info(output_file, verbosity, 1)
datafile.close()
obsfile.close()
def cmor_reformat(currProject, project_info, variable, model):
model_name = currProject.get_model_name(model)
project_name = currProject.get_project_name(model)
project_basename = currProject.get_project_basename()
project_info['RUNTIME']['model'] = model_name
project_info['RUNTIME']['project'] = project_name
project_info['RUNTIME']['project_basename'] = project_basename
verbosity = project_info["GLOBAL"]["verbosity"]
exit_on_warning = project_info['GLOBAL'].get('exit_on_warning', False)
# Variable put in environment to be used for the (optional)
# wildcard syntax in the model path, ".../${VARIABLE}/..."
# in the namelist
os.environ['__ESMValTool_base_var'] = variable.var
# Build input and output file names
indir, infile = currProject.get_cf_infile(project_info, model,
variable.fld, variable.var,
variable.mip, variable.exp)
fullpath = currProject.get_cf_fullpath(project_info, model, variable.fld,
variable.var, variable.mip,
variable.exp)
# print "indir = %s" % indir
# print "infile = %s" % infile
# print "fullpath = %s" % fullpath
if (not os.path.isdir(os.path.dirname(fullpath))):
os.makedirs(os.path.dirname(fullpath))
# Area file name for ocean grids
areafile_path = currProject.get_cf_areafile(project_info, model)
# Land-mask file name for land variables
lmaskfile_path = currProject.get_cf_lmaskfile(project_info, model)
omaskfile_path = currProject.get_cf_omaskfile(project_info, model)
# Porosity file name for land variables
porofile_path = currProject.get_cf_porofile(project_info, model)
# Additional grid file names for ocean grids, if available (ECEARTH)
hgridfile_path = False
zgridfile_path = False
lsmfile_path = False
if hasattr(currProject, "get_cf_hgridfile"):
hgridfile_path = currProject.get_cf_hgridfile(project_info, model)
if hasattr(currProject, "get_cf_zgridfile"):
zgridfile_path = currProject.get_cf_zgridfile(project_info, model)
if hasattr(currProject, "get_cf_lsmfile"):
lsmfile_path = \
currProject.get_cf_lsmfile(project_info, model, variable.fld)
# General fx file name entry
fx_file_path = False
if hasattr(currProject, "get_cf_fx_file"):
fx_file_path = currProject.get_cf_fx_file(project_info, model)
project, name, ensemble, start_year, end_year, dir\
= currProject.get_cf_sections(model)
info("project is " + project, verbosity, required_verbosity=4)
info("ensemble is " + ensemble, verbosity, required_verbosity=4)
info("dir is " + dir, verbosity, required_verbosity=4)
# Check if the current project has a specific reformat routine,
# otherwise use default
if (os.path.isdir("reformat_scripts/" + project)):
which_reformat = project
else:
which_reformat = 'default'
reformat_script = os.path.join("reformat_scripts", which_reformat,
"reformat_" + which_reformat + "_main.ncl")
# Set enviroment variables
project_info['TEMPORARY'] = {}
project_info['TEMPORARY']['indir_path'] = indir
project_info['TEMPORARY']['outfile_fullpath'] = fullpath
project_info['TEMPORARY']['infile_path'] = os.path.join(indir, infile)
project_info['TEMPORARY']['areafile_path'] = areafile_path
project_info['TEMPORARY']['lmaskfile_path'] = lmaskfile_path
project_info['TEMPORARY']['omaskfile_path'] = omaskfile_path
project_info['TEMPORARY']['porofile_path'] = porofile_path
project_info['TEMPORARY']['start_year'] = start_year
project_info['TEMPORARY']['end_year'] = end_year
project_info['TEMPORARY']['ensemble'] = ensemble
project_info['TEMPORARY']['variable'] = variable.var
project_info['TEMPORARY']['field'] = variable.fld
# FX file path
if fx_file_path:
project_info['TEMPORARY']['fx_file_path'] = fx_file_path
# Special cases
if 'realm' in currProject.get_model_sections(model):
project_info['TEMPORARY']['realm'] = \
currProject.get_model_sections(model)["realm"]
if 'shift_year' in currProject.get_model_sections(model):
project_info['TEMPORARY']['shift_year'] = \
currProject.get_model_sections(model)["shift_year"]
if 'case_name' in currProject.get_model_sections(model):
project_info['TEMPORARY']['case_name'] = \
currProject.get_model_sections(model)["case_name"]
if hgridfile_path and zgridfile_path:
project_info['TEMPORARY']['hgridfile_path'] = hgridfile_path
project_info['TEMPORARY']['zgridfile_path'] = zgridfile_path
if lsmfile_path:
project_info['TEMPORARY']['lsmfile_path'] = lsmfile_path
# Execute the ncl reformat script
if ((not os.path.isfile(project_info['TEMPORARY']['outfile_fullpath']))
or project_info['GLOBAL']['force_processing']):
info(" Calling " + reformat_script + " to check/reformat model data",
verbosity,
required_verbosity=1)
projects.run_executable(reformat_script, project_info, verbosity,
exit_on_warning)
if 'NO_REFORMAT' in reformat_script:
pass
else:
if (not os.path.isfile(project_info['TEMPORARY']['outfile_fullpath'])):
raise exceptions.IOError(
2, "Expected reformatted file isn't available: ",
project_info['TEMPORARY']['outfile_fullpath'])
del (project_info['TEMPORARY'])
