def to_csv(self, filename):
"""Save data to comma separated value (csv) file.
Limitation: Currently works with 1D arrays only.
Parameters
----------
filename : str
A string for output csv filename.
See also
--------
to_txt Save records to text file and with more control and better
performance.
TODO: Auto generate filename
"""
def fields_view(arr, fields):
"""Create columns views of a structured array.
Parameters
----------
arr : array
numpy structured array
fields : list of str
list of fields to view in specific order
"""
# dtype2 = np.dtype(
# {name: arr.dtype.fields[name] for name in fields})
dtype2 = np.dtype(
dict((name, arr.dtype.fields[name]) for name in fields))
return np.ndarray(arr.shape, dtype2, arr, 0, arr.strides)
if self.mdb_extract or self.data is None:
data = self.extract()
else:
log.info('Skip MDB_EXTRACT.')
data = self.data
if data is None:
return
# save data to csv
with open(filename, 'wb') as fh:
write = writer(fh)
write.writerow(self.elements)
write.writerows(fields_view(data, self.elements))
log.info('File saved: %s', filename)
def to_txt(self, filename, delimiter=',', fmt=None):
"""Save data to a text file.
Limitation: Currently works with 1D arrays only.
Preferred over to_csv() for speed and control.
Parameters
----------
filename : str
A string for output text filename.
delimiter : str, optional
String or character separating columns.
fmt : str or sequence of strs, optional
Format specifier(s) for requested elements. This should exclude
Date/Time and coordinate fields.
See also
--------
to_csv Save records to comma-separated text file.
TODO: Auto-generate filename
"""
el_header = delimiter.join(self.elements)
add_n = len(self.elements) - len(Query.elements)
add_fmt = ['%g'] * add_n if fmt is None else list(fmt)
el_fmt = Query.fmt + add_fmt
if self.mdb_extract or self.data is None:
data = self.extract()
else:
log.info('Skip MDB_EXTRACT.')
data = self.data
if data is not None:
mkdirp(os.path.dirname(filename))
np.savetxt(filename,
data,
delimiter=delimiter,
header=el_header,
fmt=el_fmt,
comments='')
log.info('File saved: %s', filename)
def to_dump(self, filename):
"""Dump a pickle of the extracted array to the specified file.
The array can be read back with `pickle.load()` or `numpy.load()`.
Parameters
----------
filename : str
A string for output dump filename.
"""
if self.mdb_extract or self.data is None:
data = self.extract()
else:
log.info('Skip MDB_EXTRACT.')
data = self.data
if data is not None:
mkdirp(os.path.dirname(filename))
data.dump(filename)
log.info('File saved: %s', filename)
def plot_sataod_arabian_peninsula():
"""Plot SATAOD over the Arabian Peninsula."""
from ypylib.utils import seqdate, log
save_dir = os.path.expandvars('$SCRATCH/dust/20150820-20150920')
for date in seqdate('2015-08-20', '2015-09-20',
in_fmt='%Y-%m-%d', out_fmt='%Y%m%d'):
log.info(date)
plt = plot_sataod(
AREA=['38N', '12.5N', '34E', '67E'],
START='/'.join([date, '0000Z']), STOP='/'.join([date, '2359Z']),
plt_type='scatter', markersize=5,
vmin=0, vmax=2,
gspacing=(5, 5),
map_res='l',
drawcountries=True,
title='MODIS aerosol optical depth (DT+DB): ' + date,
cb_on=True, cb_title='AOD at 550nm []')
plt.savefig('/'.join([save_dir, 'SATAOD_' + date + '.png']))
plt.close()
def plot(self,
element,
index=None,
fix_unknown=False,
show=False,
valid_min=None,
valid_max=None,
**kw):
"""Plot requested data on a cylindrical map.
Parameters
----------
element : str
A string specifying the field to show on map. Usually one from the
request elements.
index : int
index of second dimension when retrieving multi-dim array
eg, if extracted field is (N, d), where N is number of records
and d is number of wavelengths then index=1 will plot the field
corresponding to the second wavelength, and so on...
show : bool, optional
True displays the plot (default is False which return a pyplot
object)
valid_min : real
Valid minimum physical value in the array. Setting this will mask
any values < valid_min
valid_max : real
Valid maximim physical value in the array. Setting this will mask
any values > valid_max
Keywords:
See ypylib.XYZ.mapdata() keyword parameters for adjusting plot
style.
Returns
-------
matplotlib.pyplot : object when ``show=False``
"""
# extract data from metdb
if self.mdb_extract is False and self.data is not None:
data = self.data
log.info('Skip MDB_EXTRACT.')
else:
data = self.extract(fix_unknown=fix_unknown)
if data is None:
return
# show data on map
host = ['[{}]'.format(self.hostname), ''][self.hostname is None]
if isinstance(element, str) is False:
log.warning('ELEMENT should be a string, but given\n%s', element)
return
if index is None or len(data[element].shape) == 1:
plt_arr = data[element]
element_str = element
else:
plt_arr = data[element][:, index]
element_str = '{} [{}]'.format(element, str(index))
# vmin = kw.get('vmin', plt_arr.min())
# vmax = kw.get('vmax', plt_arr.max())
vmin = [valid_min, plt_arr.min()][valid_min is None]
vmax = [valid_max, plt_arr.max()][valid_max is None]
if np.ma.is_masked(vmin) and np.ma.is_masked(vmax):
log.warning('No valid data points to plot.')
return
if len(plt_arr.shape) > 1:
log.warning('Cant decide which array to plot,')
log.info('Please give me an index value (e.g. index=1).')
return
w = (plt_arr >= vmin) & (plt_arr <= vmax)
if sum(w) == 0:
log.warning('Number of constrained data points = 0. Abort plot.')
return
xyz = XYZ(data['LNGD'][w], data['LTTD'][w], plt_arr[w])
# w = (data[element] >= vmin) & (data[element] <= vmax)
# xyz = XYZ(data['LNGD'][w], data['LTTD'][w], data[element][w])
# update plot title
title = kw.get(
'title', '{}:{} {}\n{}-{}'.format(self.subtype, element_str, host,
self.start, self.stop))
kw.update({'title': title})
# print(self.extent)
if self.area:
kw.update({'limit': self.limit})
# warnings.simplefilter("ignore", UserWarning)
res = xyz.mapdata(**kw)
if show:
res.show()
else:
return res
def extract(self, verbose=False, fix_unknown=False):
"""Extract obs from MetDB.
Parameters
----------
verbose : bool
increase verbosity
fix_unknown : bool, optional
Set data types for unknown elements e.g., merged back model fields
to 32bit float.
Returns
-------
numpy ndarray matching request
"""
try:
data = obs(Query.contact,
self.subtype,
self.keywords,
self.elements,
hostname=self.hostname)
except ValueError as verr:
if fix_unknown:
elements = self.elements[len(Query.elements):]
log.warning("Setting dtypes for %s as 'float'", elements)
for e in elements:
self.set_element_dtype(e, 'float')
data = obs(Query.contact,
self.subtype,
self.keywords,
self.elements,
hostname=self.hostname)
else:
log.error(
'%s\n** Hint for unknown elements: use '
'extract(fix_unknow=True) OR call '
'set_element_dtype(element, dtype) prior '
'to extract() to bypass this error **', verr)
raise (verr)
except IOError as err:
if verbose:
log.info(' %s:%s\n %s - %s\n %s', self.hostname, self.ddict,
self.start, self.stop, sorted(self.elements))
raise (err)
if self.constrain:
# single field constrain
# key = list(self.constrain.keys())[0]
# value = list(self.constrain.values())[0]
# data = data[data[key] == value]
# iterate through all constained fields
for key, val in self.constrain.items():
data = data[data[key] == val]
if len(data) == 0:
log.error('Constrain results with no data.')
print(self.constrain)
return
if self.keep:
self.data = data
return data
def plotx(filename,
ax=None, cb_on=False, cb_bounds=None, cb_nticks=None, cb_unit='',
cb_width=0.02, ccolor='w', coastline=True, cres='110m',
dataset_path=None, draw_countries=False, dust_rgb=False,
dust_quality=None, extent=None, figsize=(6, 6), gl_font=None,
glw=0.5, list_all=False, MDI=None, projection=None, quick=False,
satellite='MSG', save_fig=None, show_path=False, show_ticks=False,
stride=(1, 1), tag_fig=None, tag_col='k', title=None, xglocs=None,
yglocs=None, **kw):
"""Display 2D arrays from geostationary Slotstore file.
Args:
filename (str): Slotstore filename. An hdf5 file containing full-disc
arrays of geostationary satellite or equivalent model data.
ax (cartopy.mpl.geoaxes.GeoAxesSubplot, optional): Plot data on a
specific geo axis.
cb_on (bool, optional): Add colour bar to the plot.
cb_bounds (array_like, optional): Specify discrete bounds for colorbar.
cb_nticks (int, optional): Number of ticks to show in the colorbar.
cb_unit (str, optional): Show data unit string on colorbar.
cb_width (float, optional): Colour bar width.
ccolor (str, optional): Coastline/boundary colour. Defaults to white.
coastline (bool, optional): Show/hide coastlines. Defaults to True.
Setting this to False will show the plot as a plain 2D array.
cres (str, optional): Resolution of cartopy coastline/country boundary.
Defaults to '110m'. Accepted values are '10m', '50m', '110m'.
dataset_path (str, optional): Full path to 2D array data in hdf5.
draw_countries (bool, optional): Show country boundaries.
dust_rgb (bool, optional): Plot Dust RGB, require BT at 3 SEVIRI IR
window channels in the file. Note: dataset_path and dust_rgb
keywords are mutually exclusive.
dust_quality (int, optional): Mask plot based on dust confidence flag
(value range: 1-7).
extent (sequence, optional): Longitude and Latitude bounds to plot the
data (lon0, lon1, lat0, lat1)
figsize (tuple, optional): Figure size. Default is (6, 6)
gl_font (dict, optional): Font properties for grid labels default is
{'family': 'monospace', 'size': 8, 'color': '#333333'}
glw (float, optional): Width of grid lines.
list_all (bool, optional): Show list of all available dataset in the
file (no plot) and exit.
MDI (number, optional): Missing Data Indicator.
projection (cartopy crs, optional): Cartopy projection to use for
mapping - one from the following
Geostationary(), PlateCarree(), Mercator().
quick (bool, optional): Quick plot. Equivalent to stride=(10, 10).
satellite (str, optional): Satellite ID. Defaults to 'MSG'. Accepted
satellite IDs are: 'GOES16', 'GOES-E', 'GOES17', 'GOES-W',
'IODC'|'MSG_IODC', 'HIM8'.
save_fig (str, optional): Save plot to a named file.
show_path (bool, optional): Show filename and dataset path on plot.
show_ticks (bool, optional): plot ticks and tick labels for non-mapped
display.
stride (tuple, optional): Skip pixels in x,y dimension, Default (1, 1)
is to show at full resolution.
tag_fig (str, optional): Add optional string to top left corner of
figure. Useful for figures for articles.
tag_col (str, optional): Colour of tag_fig text.
title (str, optional): Figure title. Defaults to variable name from
dataset_path (or 'DustRGB').
xglocs (array_like, optional): Locations of x grid-lines.
yglocs (array_like, optional): Locations of y grid-lines.
**kw: accepted `imshow` keywords (see imshow).
Returns:
matplotlib.pyplot or None: plot object or None if figure saved to disc.
Raises:
NotImplementedError: If Projection is not one of the following
Geostationary, PlateCarree, Mercator
ValueError: If dataset_path is missing or incorrect or ambiguous.
"""
import matplotlib.pyplot as plt
import cartopy.feature as cfeature
import cartopy.mpl.gridliner as cgrid
from matplotlib import colors, ticker
# defaults for MSG 0-deg service
idx = 1 # index of datetime string in hdf filename
c_lon = 0 # central geostationary longitude
def_extent = (-80, 80, -80, 80) # default map extent
sat = satellite.upper()
if sat in ('IODC', 'MSG_IODC'):
# update defaults for IODC service
idx = 2
c_lon = 41.5
def_extent = (-40, 120, -80, 80)
if sat == 'HIM8':
# update defaults for Himawari service
c_lon = 140.7
def_extent = (60, 220, -80, 80)
if sat in ('GOES16', 'GOES-E'):
c_lon = -75
def_extent = (-155, 5, -80, 80)
if sat == 'GOES15':
c_lon = -135
# def_extent = (-155, 5, -80, 80)
if sat in ('GOES17', 'GOES-W'):
c_lon = -137
# def_extent = (-155, 5, -80, 80)
h5 = h5Parse(filename)
if list_all:
h5.ls()
return
if quick:
stride = (10, 10)
apply_dust_quality = False
if dust_quality:
dconf = '/Product/GM/DustAOD/DustConfidence'
try:
dset = h5.get_data(dconf)
dust_qual = dset[dconf][::-1, ::-1][::stride[0], ::stride[1]]
apply_dust_quality = True
except KeyError:
log.warn('Dust_quality not available - filter wont be applied')
if gl_font is None:
gl_font = dict(family='monospace', size=8, color='#333333')
# Get dust RGB array
if dust_rgb and dataset_path is None:
dataset_path = 'DustRGB'
cb_on = False
plot_array = get_dust_rgb(h5, stride=stride, satellite=satellite)
elif dataset_path and dust_rgb is False:
dset = h5.get_data(dataset_path)
if len(list(dset.keys())) == 0:
return
plot_array = dset[dataset_path][::-1, ::-1][::stride[0], ::stride[1]]
# print plot_array.dtype
if plot_array.dtype in (
np.int8, np.int16, np.intc, np.uint8, np.uint16, np.intp):
MDI = [MDI, IMDI][MDI is None]
plot_array = np.ma.masked_equal(plot_array, MDI)
if apply_dust_quality:
plot_array = np.ma.masked_less(dust_qual, dust_quality)
else:
MDI = [MDI, RMDI][MDI is None]
plot_array[plot_array <= (MDI + RMDItol)] = np.nan
if apply_dust_quality:
plot_array = np.ma.where(
dust_qual >= dust_quality, plot_array, np.nan)
elif dataset_path and dust_rgb:
raise ValueError('dust_rgb must be False if dataset_path is given')
else:
raise ValueError(
'Either dust_rgb or dataset_path must be specified.\n\n'
' msgview(filename, [dataset_path|dust_rgb=True], ..) OR\n'
' msgview(filename, list_all=True) to see all datasets.\n')
# Get MSG datetime string from hdf5 filename
msg_datetime = os.path.basename(filename).split('_')[idx]
title_str = [msg_datetime, os.path.basename(dataset_path)]
# Reassign 3-channel DustMask values for readable legend/title
if title_str[1] in 'Dust':
title_str[1] = 'DustMask'
plot_array[plot_array < -0.5] = 0
plot_array[plot_array == 300] = 1
plot_array[plot_array == 400] = 2
plot_array[plot_array == 500] = 3
# Rename DustHeightError to DustPressureError
if title_str[1] in 'DustHeightError':
title_str[1] = 'DustPressureError'
if title is None:
title = '{} {}'.format(title_str[0], title_str[1])
# Start plot
if ax:
projection = ax.projection
else:
plt.figure(figsize=figsize)
projection = [projection, ccrs.Geostationary()][projection is None]
# Normalise colour based on discrete colour bounds (cb_bounds)?
if cb_bounds:
bounds = np.array(cb_bounds)
norm = colors.BoundaryNorm(boundaries=bounds, ncolors=len(bounds))
else:
norm = None
# combine plot_array and imshow keywords
plot_array_kw = {
'extent': geo_extent(satellite=satellite),
'origin': 'upper',
'norm': None,
'interpolation': None}
plot_array_kw.update(**kw)
gridline_kw = dict(alpha=0.5, xlocs=xglocs, ylocs=yglocs, linestyle='-',
linewidth=glw)
if coastline:
if projection in (ccrs.PlateCarree(),
ccrs.PlateCarree(central_longitude=0),
ccrs.PlateCarree(central_longitude=180),
ccrs.Mercator(),
ccrs.Mercator(central_longitude=0),
ccrs.Mercator(central_longitude=180)):
# Plot using rectangular projection
if ax is None:
ax = plt.axes(projection=projection)
extent = [extent, def_extent][extent is None]
ax.set_extent(extent, ccrs.PlateCarree())
gl = ax.gridlines(draw_labels=True, **gridline_kw)
gl.xlabels_top = False
gl.ylabels_right = False
gl.xformatter = cgrid.LONGITUDE_FORMATTER
gl.yformatter = cgrid.LATITUDE_FORMATTER
gl.xlabel_style, gl.ylabel_style = gl_font, gl_font
im = ax.imshow(plot_array, transform=ccrs.Geostationary(c_lon),
**plot_array_kw)
elif projection == ccrs.Geostationary():
# Plot in Geostationary projection
if ax is None:
ax = plt.axes(projection=ccrs.Geostationary(c_lon))
ax.set_global()
gl = ax.gridlines(**gridline_kw)
im = ax.imshow(plot_array, **plot_array_kw)
if extent:
ax.set_extent(extent, ccrs.PlateCarree())
else:
raise NotImplementedError(
'{} projection not implemented.\n'
'Please use one from the following:\n'
'\tGeostationary\n\tPlateCarree or\n\tMercator\n'.format(
projection.__class__))
ax.coastlines(resolution=cres, lw=0.5, color=ccolor)
else:
if extent:
# TODO: implement geo-to-pixel for cut outs without map
log.warning('Ignored extent. For geographic subsets please set '
'coastline=True.')
ax_pos = [0.05, 0.1, 0.8, 0.8]
if show_ticks:
ax_pos = [0.1, 0.1, 0.8, 0.8]
if ax is None:
ax = plt.axes(ax_pos)
# Add country borders
if draw_countries:
ax.add_feature(cfeature.BORDERS, lw=0.4, edgecolor=ccolor)
pos = ax.get_position()
# Tag a figure with additional string at top-left (useful for figure
# panel numbering)
if tag_fig:
# title = '{} {}'.format(tag_fig, title)
ax.text(0.01, 0.95, tag_fig, color=tag_col, fontsize=14,
transform=ax.transAxes)
# Draw title text
ax.set_title(title, fontsize="13", loc='left')
# Hide plot ticks and tick labels?
if show_ticks is False:
for ticx, ticy in list(
zip(ax.xaxis.get_major_ticks(), ax.yaxis.get_major_ticks())):
ticx.tick1On = ticx.tick2On = False
ticx.label1On = ticx.label2On = False
ticy.tick1On = ticy.tick2On = False
ticy.label1On = ticy.label2On = False
# Show image without coastline - simple 2D array imshow()
if coastline is False:
im = plt.imshow(plot_array, norm=norm, interpolation=None, **kw)
ax.grid(lw=0.5, alpha=0.5)
# Attach vertical colour bar
if cb_on:
cax = plt.axes([pos.x1, pos.y0, cb_width, pos.height])
cb = plt.colorbar(mappable=im, cax=cax, format='%g')
cb.ax.tick_params(direction='in')
cb.ax.set_title('{}'.format(cb_unit))
if cb_nticks:
tick_locator = ticker.MaxNLocator(nbins=cb_nticks)
cb.locator = tick_locator
cb.update_ticks()
if title_str[1] in 'DustMask':
cb.ax.set_yticklabels(
['Dust', 'Ice cloud', 'Low cloud/\nSurface', 'Uncertain'],
rotation='90', va='bottom')
if show_path:
# Show filename and HDF5 dataset path on plot
textstr = '{}\n{}'.format(dataset_path, filename)
plt.gcf().text(0.99, 0.01, textstr, fontsize=8, ha='right')
if save_fig:
# save figure as file to disk
plt.savefig(save_fig)
plt.close()
log.info('%s saved.', save_fig)
else:
# return pyplot object
return plt
def __init__(self, satellite, channel_resolution=None,
apply_SEVIRI_grid_correction=False):
"""Define satellite-specifics.
Args:
satellite (str): Satellite ID. Defaults to 'MSG'.
channel_resolution (str, optional): Used to indicate channels with
atypical resolution.
apply_SEVIRI_grid_correction (bool, optional): SEVIRI grid
correction to be applied for MSG data prior to 6 Dec 2017.
"""
self.satellite = satellite
self.channel_resolution = channel_resolution
self.apply_SEVIRI_grid_correction = apply_SEVIRI_grid_correction
self.sweep = 'y'
if self.satellite[:3].upper() in ('MSG', 'IOD'):
if self.channel_resolution == 'HRV':
self.cfac = -40927014
self.lfac = 40927014
self.nc = 11136
self.nl = 11136
self.coff = 5566
self.loff = 5566
correction_offset = 1.5
else:
self.cfac = -13642337
self.lfac = 13642337
self.nc = 3712
self.nl = 3712
self.coff = 1856
self.loff = 1856
correction_offset = 0.5
if self.apply_SEVIRI_grid_correction is True:
log.info("Applying SEVIRI grid correction")
self.coff += correction_offset
self.loff += correction_offset
if satellite.upper() in ('MSG_IODC', 'IODC'):
self.sub_lon = 41.5
elif satellite == 'MSG_RSS':
self.sub_lon = 9.5
else:
self.sub_lon = 0
# Intermediate coords deltas
self.dx = 2**16 / self.cfac
self.dy = 2**16 / self.lfac
self.req = REQ
self.rpol = RPOL
self.satellite_height = 35785831
elif self.satellite.upper() == 'HIM8':
self.cfac = 20466275
self.lfac = -20466275
self.nc = 5500
self.nl = 5500
self.coff = 2750.5
self.loff = 2750.5
self.sub_lon = 140.7
# Intermediate coords deltas
self.dx = 2**16 / self.cfac
self.dy = 2**16 / self.lfac
# WGS84 = different from MSG
self.req = 6378137
self.rpol = 6356752.3
self.satellite_height = 35785831
elif self.satellite.upper() in ('GOES16', 'GOES17'):
self.nc = 5424
self.nl = 5424
self.coff = 2712.5
self.loff = 2712.5
if satellite.upper() == 'GOES16':
self.sub_lon = -75.0
elif satellite.upper() == 'GOES17':
self.sub_lon = -137.0
# Intermediate coords deltas
self.dx = np.rad2deg(0.000056)
self.dy = np.rad2deg(-0.000056)
# WGS84 = different from MSG
self.req = 6378137
self.rpol = 6356752.3
self.satellite_height = 35786023
# 'sweep_angle_axis' (CF conventions 1.7)
self.sweep = 'x'
elif self.satellite.upper() == 'GOES15':
self.cfac = 10216334
self.lfac = -10216334
self.nc = 2816
self.nl = 2816
self.coff = 1408.5
self.loff = 1408.5
# GOES West
self.sub_lon = -135.0
# Intermediate coords deltas
self.dx = 2**16 / self.cfac
self.dy = 2**16 / self.lfac
# WGS84 = different from MSG
self.req = 6378137
self.rpol = 6356752.3
self.satellite_height = 35785831