def percentile_plotting(inifilename=None, ini_dict=None, verbose=1):
'''
Top-level plotting routine to generate percentile plots.
:param inifilename: String giving filename of ini file.
:param ini_dict: Dictionary of the ini file.
:param verbose: Level of print output:
* 0 = No extra printing
* 1 = Standard level (recommended)
* 2 = Extra level for debugging
:returns: (ini_dict, percs_md_list)
* **ini_dict** - Dictionary of a :class:`inifile` object
* **percs_md_list** - Model Data list - each item will be name
in :class:`adaq_data` format
**Method:**
* Reads in inifile
* Reads in model data
* Calculates the percentile of the ensemble members
* Plots data using style given in inifile
>>> ini_dict, percs_md_list = percentile_plotting()
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Current version of python: ...
[...]
Current version of iris: ...
Beginning at ...
Reading inifile .../adaqscripts/ens_percentile_plot.ini
Getting model data for mogreps-g_name at ...
Plotting gridded fields
Saved figure \
.../Fieldplot_mogreps-g_name_95TH_PERCENTILE_OF_VOLCANIC_ASH_AIR\
_CONCENTRATION_FromFL100toFL125_201905141600.png
...
Finished at ...
...
'''
print(' ')
print('Current version of python:', sys.version)
print('Current version of iris:', iris.__version__)
print('Beginning at ', datetime.datetime.now())
if ini_dict is None:
ini_dict = adaqcode.inifile.get_inidict(
inifilename=inifilename,
defaultfilename='adaqscripts/ens_percentile_plot.ini')
check_ini_dict(ini_dict)
percentiles = ini_dict.get('percentile_list', [10, 50, 90])
percentiles = [float(v) for v in percentiles]
md_list = adaqcode.adaq_functions.get_models(ini_dict)
# Check for presence of short_name_list and populate if not there
if 'short_name_list' not in ini_dict:
short_name_list = []
for md in md_list:
short_name_list.extend([
cube.attributes['short_name'] for cube in md.gridded_cube_list
])
ini_dict['short_name_list'] = short_name_list
if len(ini_dict['short_name_list']) > 1:
warnings.warn('Plotting ' + str(len(ini_dict['short_name_list'])) +
' fields may take some time')
if 'units' in ini_dict:
for md in md_list:
for ngc in md.gridded_cube_list:
try:
ngc.convert_units(ini_dict['units'])
except ValueError:
print("Can't convert units from ", ngc.units, \
" to ", ini_dict['units'])
percs_md_list = []
for md in md_list: # List of models
percs_md = adaqcode.adaq_data.ADAQData()
for ngc in md.gridded_cube_list: # List of variables
ngc.coord('realization').guess_bounds()
percs_ngc = ngc.collapsed(['realization'],
iris.analysis.PERCENTILE,
percent=percentiles,
fast_percentile_method=True)
for percentile in percentiles:
perc_ngc = percs_ngc.extract(
iris.Constraint(percentile_over_realization=percentile))
perc_ngc.attributes['short_name'] = str(int(percentile)) + \
'TH_PERCENTILE_OF_' + ngc.attributes['short_name']
perc_ngc.attributes['Quantity'] = str(int(percentile)) + \
'th percentile of ' + ngc.attributes['Quantity']
perc_ngc.attributes['Percentile'] = percentile
perc_ngc.data = numpy.ma.masked_equal(perc_ngc.data, 0)
percs_md.gridded_cube_list.append(perc_ngc)
percs_md_list.append(percs_md)
del md_list # Just to free up memory on the system.
short_name_to_plot_list = []
for percs_md in percs_md_list:
short_name_to_plot_list.extend([
cube.attributes['short_name']
for cube in percs_md.gridded_cube_list
])
for short_name in short_name_to_plot_list:
plot_md_gridded_fields(ini_dict,
percs_md_list,
short_name,
defaults='NAME',
verbose=verbose)
print(' ')
print('Finished at ', datetime.datetime.now())
return ini_dict, percs_md_list
def aq_stats_and_plots(ini_dict,
sites_data,
od,
md_list,
od_stats,
md_list_stats,
verbose=1):
"""
Function to call all required statistics and plotting routines,
depending on values set in ini_dict.
:param ini_dict: Dictionary of a :class:`inifile` object.
:param sites_data: numpy ndarray containing site information data
from a :class:`sites_info.SitesInfo` object.
:param od: observation data - an :class:`adaq_data.ADAQData` object
with data in sites_cube_list
:param md_list: model data - a list of :class:`adaq_data.ADAQData` objects
with data in sites_cube_list
:param od_stats: observation data which has been matched to model data
ready for statistic calculations - an
:class:`adaq_data.ADAQData` object with data in
sites_cube_list
:param md_list_stats: model data which has been matched to observation data-
a list of :class:`adaq_data.ADAQData` objects with data in
sites_cube_list
:param verbose: Level of print output:
* 0 = No extra printing
* 1 = Standard level (recommended)
* 2 = Extra level for debugging
"""
# Calculate statistics and save to csv file
#If 'calc_stats' not in ini_dict then set to a default of False
#Only calculate stats if 'calc_stats'=True
if ini_dict.get('calc_stats', False):
adaq_functions.calc_stats(ini_dict,
od_stats,
md_list_stats,
thresholds=STATS_THRESHOLDS,
nc_append=True)
# Produce species-specific plots
for short_name in ini_dict['short_name_list']:
print('Short_name:', short_name)
#--- Field plot data
if ini_dict['fieldplot']:
#Set up contour levels
if ini_dict['fixed_levels_list'] is None:
if ini_dict['cmap'] == 'CAMS':
#Get levels from CAMS_LEVELS if available
contour_levels = adaqcode.aq_indices.CAMS_LEVELS.get(
short_name, None)
else:
#By default take from DAQI_LEVELS if available
contour_levels = adaqcode.aq_indices.DAQI_LEVELS.get(
short_name, None)
# Check units - for CAMS or DAQI levels, should be in ug/m3
chem_units = ini_dict.get('chem_units', 'ug/m3')
aerosol_units = ini_dict.get('aerosol_units', 'ug/m3')
if chem_units != 'ug/m3' or aerosol_units != 'ug/m3':
cube = md_list[0].extract(short_name=short_name,
gridded=True,
singlecube=True)
if cube and cube.units != 'ug/m3':
contour_levels = None
else:
contour_levels = ini_dict['fixed_levels_list']
ini_dict['levels_list'] = contour_levels
if ini_dict.get('contours', False):
adaq_plotting.plot_md_gridded_fields(ini_dict,
md_list,
short_name,
defaults='AQ',
plot_subdir='gridded_fields',
verbose=verbose)
# Maps of DAQI sites data
if 'DAQI' in short_name and ini_dict.get('daqi', False) and \
ini_dict.get('daqi_site_maps', False):
#Increase default marker size to 40 (instead of 20)
ini_dict['marker_size'] = ini_dict.get('marker_size', 40)
adaq_plotting.plot_sitescube_maps(ini_dict,
od,
md_list,
short_name,
SITE_CLASSIFICATIONS,
plot_subdir='gridded_fields',
tight=True,
verbose=verbose)
#--- Site specific data
# Histogram plots - only if set to True in inifile
if ini_dict.get('histograms', False):
#Force binsize to 2 for main AQ species
if short_name in ['O3', 'NO2', 'SO2', 'PM10', 'PM2.5']:
binsize = 2.
if short_name in ['O3', 'NO2', 'SO2'] and \
ini_dict.get('chem_units', 'ug/m3') == 'cm-3':
#Automatically calculate binsize
binsize = None
elif short_name[-4:] == 'DAQI':
binsize = 1
else:
#Automatically calculate binsize
binsize = None
adaq_plotting.plot_histogram(ini_dict,
od_stats,
md_list_stats,
short_name,
binsize=binsize,
maxperc=99.5)
# Quantile-Quantile plots - only if set to True in inifile
if ini_dict.get('qqplots', False):
adaq_plotting.plot_qq(ini_dict, od_stats, md_list_stats,
short_name)
# Soccer plots - only if set to True in inifile
if ini_dict.get('soccer_plots', False):
adaq_plotting.plot_soccer_plots(ini_dict, od_stats, md_list_stats,
short_name, SITE_CLASSIFICATIONS)
# Diurnal plots
if ini_dict.get('diurnal', False):
#Doesn't make sense to produce hourly plots for daily values
if short_name[-4:] != 'DAQI':
adaq_plotting.plot_diurnal(ini_dict, od_stats, md_list_stats,
short_name)
# Timeseries of statistics
if ini_dict.get('timeseries_of_stats', False):
threshold = STATS_THRESHOLDS.get(short_name, None)
adaq_plotting.plot_timeseries_of_stats(ini_dict,
od_stats,
md_list_stats,
short_name,
threshold=threshold)
#Timeseries of aggregate statistics across sites
if ini_dict.get('timeseries_of_aggregate_stats', False):
for stat in ini_dict.get('timeseries_of_aggregate_stats_list', []):
adaq_plotting.plot_timeseries_aggregate_stats(
ini_dict, [od_stats] + md_list_stats,
short_name,
stat=stat,
linestyles=['--'] + ['-'] * len(md_list_stats))
# Produce site-specific timeseries plots
if ini_dict.get('tseries', False):
#Only loop through all sites if 'tseries' set to True
#- set up in aq_get_data()
for site in sites_data:
print('Site:', site['abbrev'])
#Plot time-series - only if set to True in inifile
if ini_dict.get('timeseries', False):
for short_name in ini_dict['short_name_list']:
print('Short_name:', short_name)
adaq_plotting.plot_timeseries(ini_dict,
od,
md_list,
short_name,
site,
verbose=verbose)
#Produce multiple-species timeseries
if ini_dict.get('timeseries_multiple_short_names', False):
tseries_multiple_sn_dict = ini_dict.get(
'timeseries_multiple_short_names_dict', {})
for group, sname_list in tseries_multiple_sn_dict.items():
print('Multiple short_names group:', group)
adaq_plotting.plot_tseries_multiple_snames(ini_dict,
od,
md_list,
site,
sname_list,
group,
verbose=verbose)
def field_plotting(inifilename=None, ini_dict=None, verbose=1):
'''
Top-level plotting routine to generate field plots
:param inifilename: String giving filename of ini file.
:param verbose: Level of print output:
* 0 = No extra printing
* 1 = Standard level (recommended)
* 2 = Extra level for debugging
:returns: (ini_dict, sites_data, od, md_list)
* **ini_dict** - Dictionary of a :class:`inifile` object
* **md_list** - Model Data list - each item will be name
in :class:`adaq_data` format
**Method:**
* Reads in inifile
* Reads in model data
* Plots data using style given in inifile
>>> ini_dict, md_list = field_plotting()
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Current version of python: ...
[...]
Current version of iris: ...
Beginning at ...
Reading inifile .../adaqscripts/name_field_plot.ini
Getting model data for name at ...
<class '...NAMEData'> - Subclass of ADAQData - Contains:
sites_cube_list:
< No cubes >
gridded_cube_list:
0: VOLCANIC_ASH_AIR_CONCENTRATION / (g/m3) (latitude: 480; longitude: 640)
trajectory_cube_list:
< No cubes >
Plotting gridded fields
Saved figure \
.../Fieldplot_name_VOLCANIC_ASH_AIR_CONCENTRATION_FromFL025to\
FL050_201105230000.png
Finished at ...
'''
print('Current version of python:', sys.version)
print('Current version of iris:', iris.__version__)
print('Beginning at ', datetime.datetime.now())
# Read ini file
if ini_dict is None:
ini_dict = adaqcode.inifile.get_inidict(
inifilename=inifilename,
defaultfilename='adaqscripts/name_field_plot.ini')
# Read in model data
md_list = adaqcode.adaq_functions.get_models(ini_dict)
if 'percent' in ini_dict and ini_dict['percent']:
md_list = adaqcode.adaq_functions.percent_contribution(md_list)
# Check for presence of short_name_list and populate if not there
if 'short_name_list' not in ini_dict:
short_name_list = []
for md in md_list:
short_name_list.extend([
cube.attributes['short_name'] for cube in md.gridded_cube_list
])
ini_dict['short_name_list'] = short_name_list
for md in md_list:
print(md)
if 'units' in ini_dict:
for ngc in md.gridded_cube_list:
try:
ngc.convert_units(ini_dict['units'])
except ValueError:
print("Can't convert units from ", ngc.units, \
" to ", ini_dict['units'])
# Plot data
for short_name in ini_dict['short_name_list']:
plot_md_gridded_fields(ini_dict,
md_list,
short_name,
defaults='NAME',
verbose=verbose)
print('Finished at ', datetime.datetime.now())
return ini_dict, md_list
def field_diff_plotting(inifilename=None, ini_dict=None, verbose=1):
'''
Top-level plotting routine to generate field plots of both
case1 and case2 together with difference and relative difference plots.
:param inifilename: String giving filename of ini file.
:param ini_dict: Dictionary of properties required for plot type.
:param verbose: Level of print output:
* 0 = No extra printing
* 1 = Standard level (recommended)
* 2 = Extra level for debugging
:returns: (ini_dict)
* **ini_dict** - Dictionary of a :class:`inifile` object
**Method:**
* Reads in inifile, case1 model data and case2 model data
* Calculates the differences between the two.
* Plots both case1 and case 2 with the difference and relative difference in a montage.
>>> ini_dict = field_diff_plotting()
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Current version of python: ...
[...]
Current version of iris: ...
Beginning at ...
Reading inifile .../adaqscripts/name_field_plot_diff.ini
Getting model data for global-name at ...
Getting model data for global-name at ...
*** The two cubes are different ***
max_abs_reldiff: 99.98737
reldiff_tolerance: 5
***********************************
Plotting gridded fields
...
Saved figure .../case1/Fieldplot_global-name_NO2_EULERIAN_CONCENTRATION\
_From0_0to100_0magl_201506140000.png
Plotting gridded fields
...
Saved figure .../case2/Fieldplot_global-name_NO2_EULERIAN_CONCENTRATION\
_From0_0to100_0magl_201506140000.png
Plotting gridded fields
...
Saved figure .../diff/Fieldplot_global-name_NO2_EULERIAN_CONCENTRATION\
_From0_0to100_0magl_201506140000.png
Plotting gridded fields
...
Saved figure .../reldiff/Fieldplot_global-name_NO2_EULERIAN_CONCENTRATION\
_From0_0to100_0magl_201506140000.png
Plotting montage
...
Saved figure .../montage/Fieldplot_global-name_NO2_EULERIAN_CONCENTRATION\
_From0_0to100_0magl_201506140000.png
Finished at ...
'''
print('Current version of python:', sys.version)
print('Current version of iris:', iris.__version__)
print('Beginning at ', datetime.datetime.now())
# Read ini file and check it
if ini_dict is None:
ini_dict = inifile.get_inidict(
inifilename=inifilename,
defaultfilename='adaqscripts/name_field_plot_diff.ini')
check_ini_dict(ini_dict)
# Get Data
try:
ini_dict['models_dir_list'] = ini_dict['models_dir_case1_list']
except:
raise ValueError("models_dir_case1_list has not be set in inifile")
md_list_case1 = adaq_functions.get_models(ini_dict)
try:
ini_dict['models_dir_list'] = ini_dict['models_dir_case2_list']
except:
raise ValueError("models_dir_case2_list has not be set in inifile")
md_list_case2 = adaq_functions.get_models(ini_dict)
# Change the units of both md_lists
if 'units' in ini_dict:
md_list_case1 = convert_units(md_list_case1, ini_dict)
md_list_case2 = convert_units(md_list_case2, ini_dict)
# Check for presence of short_name_list and populate if not there
if 'short_name_list' not in ini_dict:
short_name_list = []
for md_case1 in md_list_case1:
short_name_list.extend([
cube.attributes['short_name']
for cube in md_case1.gridded_cube_list
])
print('short_name_list: ', short_name_list)
ini_dict['short_name_list'] = short_name_list
check_shape(md_list_case1, md_list_case2)
diff_count, abs_diff_levels, rel_diff_levels, md_list_diff, md_list_reldiff = \
differencing_cubes(ini_dict, md_list_case1, md_list_case2)
if diff_count > 0:
for short_name in ini_dict['short_name_list']:
# Get directories ready
if 'plot_dir_case1' not in ini_dict:
ini_dict['plot_dir_case1'] = '/tmp/plot_dir_case1'
if 'plot_dir_case2' not in ini_dict:
ini_dict['plot_dir_case2'] = '/tmp/plot_dir_case2'
if 'plot_dir_diff' not in ini_dict:
ini_dict['plot_dir_diff'] = '/tmp/plot_dir_diff'
if 'plot_dir_reldiff' not in ini_dict:
ini_dict['plot_dir_reldiff'] = '/tmp/plot_dir_reldiff'
shell_commands.call_shell('rm -f ' + ini_dict['plot_dir_case1'] +
'/*')
shell_commands.call_shell('rm -f ' + ini_dict['plot_dir_case2'] +
'/*')
shell_commands.call_shell('rm -f ' + ini_dict['plot_dir_diff'] +
'/*')
shell_commands.call_shell('rm -f ' + ini_dict['plot_dir_reldiff'] +
'/*')
# Get cbar_labels ready
if 'cbar_label' in ini_dict:
cbar_label_diff = 'Absolute Difference in ' + ini_dict[
'cbar_label']
cbar_label_reldiff = 'Relative Difference in ' + ini_dict[
'cbar_label']
else:
cbar_label_diff = 'Absolute Difference in ' + short_name
cbar_label_reldiff = 'Relative Difference in ' + short_name
# Plot the four graphics individually
# Case 1
ini_dict['plot_dir'] = ini_dict['plot_dir_case1']
ini_dict['suptitle'] = 'case1: ' + ini_dict['suptitle_case1']
plot_md_gridded_fields(ini_dict,
md_list_case1,
short_name,
defaults='NAME',
verbose=verbose)
# Case 2
ini_dict['plot_dir'] = ini_dict['plot_dir_case2']
ini_dict['suptitle'] = 'case2: ' + ini_dict['suptitle_case2']
plot_md_gridded_fields(ini_dict,
md_list_case2,
short_name,
defaults='NAME',
verbose=verbose)
# Abs Diff
if 'abs_diff_levels_list' in ini_dict:
ini_dict['levels_list'] = ini_dict['abs_diff_levels_list']
else:
ini_dict['levels_list'] = abs_diff_levels
ini_dict['plot_dir'] = ini_dict['plot_dir_diff']
ini_dict['suptitle'] = 'Absolute Difference = case2 - case1'
ini_dict['cmap'] = 'PiYG'
ini_dict['cbar_label'] = cbar_label_diff
plot_md_gridded_fields(ini_dict,
md_list_diff,
short_name,
defaults='NAME',
verbose=verbose)
# Rel Diff
if 'rel_diff_levels_list' in ini_dict:
ini_dict['levels_list'] = ini_dict['rel_diff_levels_list']
else:
ini_dict['levels_list'] = rel_diff_levels
ini_dict['plot_dir'] = ini_dict['plot_dir_reldiff']
ini_dict['suptitle'] = 'Relative Difference = ' \
'(case2-case1)*100/(case2+case1)'
ini_dict['cmap'] = 'bwr'
ini_dict['cbar_label'] = cbar_label_reldiff
plot_md_gridded_fields(ini_dict,
md_list_reldiff,
short_name,
defaults='NAME',
verbose=verbose)
# Create the montage
print('Plotting montage')
rc = shell_commands.call_shell(
"for file in $(ls " + ini_dict['plot_dir_diff'] +
") ; do montage " + ini_dict['plot_dir_case1'] + "/$file " +
ini_dict['plot_dir_case2'] + "/$file " +
ini_dict['plot_dir_diff'] + "/$file " +
ini_dict['plot_dir_reldiff'] + "/$file -geometry +2+2 " +
ini_dict['plot_dir_montage'] + "/$file ; "
"echo Saved figure " + ini_dict['plot_dir_montage'] +
"/$file ; done")
if rc != 0:
raise ValueError("Error with script to produce montage")
print('Finished at ', datetime.datetime.now())
if diff_count > 0 and ini_dict.get('fail_if_diff', False):
sys.exit(1)
return ini_dict
def exceedance_plotting(inifilename=None, ini_dict=None, verbose=1):
'''
Top-level plotting routine to generate exceedance plots
:param inifilename: String giving filename of ini file.
:param ini_dict: Dictionary of the ini file.
:param verbose: Level of print output:
* 0 = No extra printing
* 1 = Standard level (recommended)
* 2 = Extra level for debugging
:returns: (ini_dict, count_md_list)
* **ini_dict** - Dictionary of a :class:`inifile` object
* **count_md_list** - Model Data list - each item will be name
in :class:`adaq_data` format
**Method:**
* Reads in inifile
* Reads in model data
* Calculates the number of ensemble members exceeding the
threshold.
* Plots data using style given in inifile
>>> ini_dict, count_md_list = exceedance_plotting()
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Current version of python: ...
[...]
Current version of iris: ...
Beginning at ...
Reading inifile .../adaqscripts/ens_exceedance_plot.ini
Getting model data for mogreps-g_name at ...
Plotting gridded fields
Saved figure \
.../Fieldplot_mogreps-g_name_MEM_EXCEED_VOLCANIC_ASH_AIR_\
CONCENTRATION_OF_200_FromFL100toFL125_201905141600.png
...
Finished at ...
...
'''
print(' ')
print('Current version of python:', sys.version)
print('Current version of iris:', iris.__version__)
print('Beginning at ', datetime.datetime.now())
if ini_dict is None:
ini_dict = adaqcode.inifile.get_inidict(
inifilename=inifilename,
defaultfilename='adaqscripts/ens_exceedance_plot.ini')
check_ini_dict(ini_dict)
md_list = adaqcode.adaq_functions.get_models(ini_dict)
# Check for presence of short_name_list and populate if not there
if 'short_name_list' not in ini_dict:
short_name_list = []
for md in md_list:
short_name_list.extend([cube.attributes['short_name']
for cube in md.gridded_cube_list])
ini_dict['short_name_list'] = short_name_list
if len(ini_dict['short_name_list']) > 1:
warnings.warn('Plotting '+str(len(ini_dict['short_name_list'])) +
' fields may take some time')
if 'percent' in ini_dict and ini_dict['percent']:
md_list = adaqcode.adaq_functions.percent_contribution(md_list)
for md in md_list:
for ngc in md.gridded_cube_list:
try:
ngc.convert_units(ini_dict['units'])
except ValueError:
print("Can't convert units from ", ngc.units, \
" to ", ini_dict['units'])
count_md_list = []
for md in md_list: # List of models
count_md = adaqcode.adaq_data.ADAQData()
for ngc in md.gridded_cube_list: # List of variables
ngc.coord('realization').guess_bounds()
for threshold in ini_dict['threshold_list']:
count_ngc = ngc.collapsed(['realization'],
iris.analysis.COUNT,
function=lambda values: values > float(threshold))
count_ngc.attributes['Quantity'] = '# members exceeding an '+ \
ngc.attributes['Quantity']+ \
' of '+str(threshold)+' '+ini_dict['units']
count_ngc.attributes['Threshold'] = threshold
count_ngc.attributes['short_name'] = 'MEM_EXCEED_'+ \
ngc.attributes['short_name']+'_'+ \
'OF_'+str(threshold)
count_ngc.rename(count_ngc.attributes['short_name'])
count_ngc.data = numpy.ma.masked_equal(count_ngc.data, 0)
count_md.gridded_cube_list.append(count_ngc)
count_md_list.append(count_md)
del md_list # Just to free up memory on the system.
short_name_to_plot_list = []
for count_md in count_md_list:
short_name_to_plot_list.extend([cube.attributes['short_name']
for cube in count_md.gridded_cube_list])
for short_name in short_name_to_plot_list:
plot_md_gridded_fields(ini_dict, count_md_list, short_name,
defaults='NAME', verbose=verbose)
print(' ')
print('Finished at ', datetime.datetime.now())
return ini_dict, count_md_list