def test_convert(self):
self.assertEqual(TimeLike.convert('2017-04-19'), datetime(2017, 4, 19))
self.assertEqual(TimeLike.convert(datetime(2017, 4, 19)),
datetime(2017, 4, 19))
self.assertEqual(TimeLike.convert(date(2017, 4, 19)),
datetime(2017, 4, 19, 12))
self.assertEqual(TimeLike.convert(' '), None)
self.assertEqual(TimeLike.convert(None), None)
def test_accepts(self):
self.assertTrue(TimeLike.accepts(None))
self.assertTrue(TimeLike.accepts(''))
self.assertTrue(TimeLike.accepts(' '))
self.assertTrue(TimeLike.accepts('2001-01-01'))
self.assertTrue(TimeLike.accepts(datetime(2001, 1, 1)))
self.assertTrue(TimeLike.accepts(date(2001, 1, 1)))
self.assertFalse(TimeLike.accepts('4.3'))
self.assertFalse(TimeLike.accepts('2001-01-01,2001-02-01,'))
self.assertFalse(TimeLike.accepts([datetime(2001, 1, 1)]))
def test_accepts(self):
self.assertTrue(TimeLike.accepts(None))
self.assertTrue(TimeLike.accepts(''))
self.assertTrue(TimeLike.accepts(' '))
self.assertTrue(TimeLike.accepts('2001-01-01'))
self.assertTrue(TimeLike.accepts(datetime(2001, 1, 1)))
self.assertTrue(TimeLike.accepts(date(2001, 1, 1)))
self.assertFalse(TimeLike.accepts('4.3'))
self.assertFalse(TimeLike.accepts('2001-01-01,2001-02-01,'))
self.assertFalse(TimeLike.accepts([datetime(2001, 1, 1)]))
def plot_contour(ds: xr.Dataset,
var: VarName.TYPE,
time: TimeLike.TYPE = None,
indexers: DictLike.TYPE = None,
title: str = None,
filled: bool = True,
properties: DictLike.TYPE = None,
file: str = None) -> Figure:
"""
Create a contour plot of a variable given by dataset *ds* and variable name *var*.
:param ds: the dataset containing the variable to plot
:param var: the variable's name
:param time: time slice index to plot, can be a string "YYYY-MM-DD" or an integer number
:param indexers: Optional indexers into data array of *var*. The *indexers* is a dictionary
or a comma-separated string of key-value pairs that maps the variable's dimension names
to constant labels. e.g. "layer=4".
:param title: an optional title
:param filled: whether the regions between two contours shall be filled
:param properties: optional plot properties for Python matplotlib,
e.g. "bins=512, range=(-1.5, +1.5), label='Sea Surface Temperature'"
For full reference refer to
https://matplotlib.org/api/lines_api.html and
https://matplotlib.org/devdocs/api/_as_gen/matplotlib.patches.Patch.html#matplotlib.patches.Patch
:param file: path to a file in which to save the plot
:return: a matplotlib figure object or None if in IPython mode
"""
var_name = VarName.convert(var)
if not var_name:
raise ValueError("Missing value for 'var'")
var = ds[var_name]
time = TimeLike.convert(time)
indexers = DictLike.convert(indexers) or {}
properties = DictLike.convert(properties) or {}
figure = plt.figure(figsize=(8, 4))
ax = figure.add_subplot(111)
var_data = _get_var_data(var, indexers, time=time)
if filled:
var_data.plot.contourf(ax=ax, **properties)
else:
var_data.plot.contour(ax=ax, **properties)
if title:
ax.set_title(title)
figure.tight_layout()
if file:
figure.savefig(file)
return figure if not in_notebook() else None
def sel(ds: DatasetLike.TYPE,
point: PointLike.TYPE = None,
time: TimeLike.TYPE = None,
indexers: DictLike.TYPE = None,
method: str = 'nearest') -> xr.Dataset:
"""
Return a new dataset with each array indexed by tick labels along the specified dimension(s).
This is a wrapper for the ``xarray.sel()`` function.
For documentation refer to xarray documentation at
http://xarray.pydata.org/en/stable/generated/xarray.Dataset.sel.html#xarray.Dataset.sel
:param ds: The dataset from which to select.
:param point: Optional geographic point given by longitude and latitude
:param time: Optional time
:param indexers: Keyword arguments with names matching dimensions and values given by scalars,
slices or arrays of tick labels. For dimensions with multi-index, the indexer may also be
a dict-like object with keys matching index level names.
:param method: Method to use for inexact matches:
* None: only exact matches
* ``pad`` / ``ffill``: propagate last valid index value forward
* ``backfill`` / ``bfill``: propagate next valid index value backward
* ``nearest`` (default): use nearest valid index value
:return: A new Dataset with the same contents as this dataset, except each variable and dimension
is indexed by the appropriate indexers. In general, each variable's data will be a view of the
variable's data in this dataset.
"""
ds = DatasetLike.convert(ds)
point = PointLike.convert(point)
time = TimeLike.convert(time)
indexers = DictLike.convert(indexers)
indexers = dict(indexers or {})
if point is not None:
indexers.setdefault('lon', point.x)
indexers.setdefault('lat', point.y)
if time is not None:
indexers.setdefault('time', time)
# Filter out non-existent coordinates
indexers = {
name: value
for name, value in indexers.items() if name in ds.coords
}
return ds.sel(method=method, **indexers)
def sel(ds: DatasetLike.TYPE,
point: PointLike.TYPE = None,
time: TimeLike.TYPE = None,
indexers: DictLike.TYPE = None,
method: str = 'nearest') -> xr.Dataset:
"""
Return a new dataset with each array indexed by tick labels along the specified dimension(s).
This is a wrapper for the ``xarray.sel()`` function.
For documentation refer to xarray documentation at
http://xarray.pydata.org/en/stable/generated/xarray.Dataset.sel.html#xarray.Dataset.sel
:param ds: The dataset from which to select.
:param point: Optional geographic point given by longitude and latitude
:param time: Optional time
:param indexers: Keyword arguments with names matching dimensions and values given by scalars,
slices or arrays of tick labels. For dimensions with multi-index, the indexer may also be
a dict-like object with keys matching index level names.
:param method: Method to use for inexact matches:
* None: only exact matches
* ``pad`` / ``ffill``: propagate last valid index value forward
* ``backfill`` / ``bfill``: propagate next valid index value backward
* ``nearest`` (default): use nearest valid index value
:return: A new Dataset with the same contents as this dataset, except each variable and dimension
is indexed by the appropriate indexers. In general, each variable's data will be a view of the
variable's data in this dataset.
"""
ds = DatasetLike.convert(ds)
point = PointLike.convert(point)
time = TimeLike.convert(time)
indexers = DictLike.convert(indexers)
indexers = dict(indexers or {})
if point is not None:
indexers.setdefault('lon', point.x)
indexers.setdefault('lat', point.y)
if time is not None:
indexers.setdefault('time', time)
# Filter out non-existent coordinates
indexers = {name: value for name, value in indexers.items() if name in ds.coords}
return ds.sel(method=method, **indexers)
def test_json(self):
self.assertEqual(TimeLike.to_json(datetime(2017, 4, 19)), '2017-04-19')
self.assertEqual(TimeLike.from_json('2017-04-19'),
datetime(2017, 4, 19))
def test_format(self):
self.assertEqual(TimeLike.format(None), '')
self.assertEqual(TimeLike.format(datetime(2017, 4, 19)), '2017-04-19')
def _make_local(self,
local_ds: LocalDataSource,
time_range: TimeRangeLike.TYPE = None,
region: PolygonLike.TYPE = None,
var_names: VarNamesLike.TYPE = None,
monitor: Monitor = Monitor.NONE):
local_id = local_ds.id
time_range = TimeRangeLike.convert(time_range)
region = PolygonLike.convert(region)
var_names = VarNamesLike.convert(var_names)
time_range, region, var_names = self._apply_make_local_fixes(
time_range, region, var_names)
compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL',
NETCDF_COMPRESSION_LEVEL)
compression_enabled = True if compression_level > 0 else False
do_update_of_verified_time_coverage_start_once = True
verified_time_coverage_start = None
verified_time_coverage_end = None
encoding_update = dict()
if compression_enabled:
encoding_update.update({
'zlib': True,
'complevel': compression_level
})
if region or var_names:
protocol = _ODP_PROTOCOL_OPENDAP
else:
protocol = _ODP_PROTOCOL_HTTP
local_path = os.path.join(local_ds.data_store.data_store_path,
local_id)
if not os.path.exists(local_path):
os.makedirs(local_path)
selected_file_list = self._find_files(time_range)
if not selected_file_list:
msg = 'CCI Open Data Portal data source "{}"\ndoes not seem to have any datasets'.format(
self.id)
if time_range is not None:
msg += ' in given time range {}'.format(
TimeRangeLike.format(time_range))
raise DataAccessError(msg)
try:
if protocol == _ODP_PROTOCOL_OPENDAP:
do_update_of_variables_meta_info_once = True
do_update_of_region_meta_info_once = True
files = self._get_urls_list(selected_file_list, protocol)
monitor.start('Sync ' + self.id, total_work=len(files))
for idx, dataset_uri in enumerate(files):
child_monitor = monitor.child(work=1)
file_name = os.path.basename(dataset_uri)
local_filepath = os.path.join(local_path, file_name)
time_coverage_start = selected_file_list[idx][1]
time_coverage_end = selected_file_list[idx][2]
try:
child_monitor.start(label=file_name, total_work=1)
remote_dataset = xr.open_dataset(dataset_uri)
if var_names:
remote_dataset = remote_dataset.drop([
var_name for var_name in
remote_dataset.data_vars.keys()
if var_name not in var_names
])
if region:
remote_dataset = normalize_impl(remote_dataset)
remote_dataset = subset_spatial_impl(
remote_dataset, region)
geo_lon_min, geo_lat_min, geo_lon_max, geo_lat_max = region.bounds
remote_dataset.attrs[
'geospatial_lat_min'] = geo_lat_min
remote_dataset.attrs[
'geospatial_lat_max'] = geo_lat_max
remote_dataset.attrs[
'geospatial_lon_min'] = geo_lon_min
remote_dataset.attrs[
'geospatial_lon_max'] = geo_lon_max
if do_update_of_region_meta_info_once:
local_ds.meta_info['bbox_maxx'] = geo_lon_max
local_ds.meta_info['bbox_minx'] = geo_lon_min
local_ds.meta_info['bbox_maxy'] = geo_lat_max
local_ds.meta_info['bbox_miny'] = geo_lat_min
do_update_of_region_meta_info_once = False
if compression_enabled:
for sel_var_name in remote_dataset.variables.keys(
):
remote_dataset.variables.get(
sel_var_name).encoding.update(
encoding_update)
remote_dataset.to_netcdf(local_filepath)
child_monitor.progress(work=1,
msg=str(time_coverage_start))
finally:
if do_update_of_variables_meta_info_once:
variables_info = local_ds.meta_info.get(
'variables', [])
local_ds.meta_info['variables'] = [
var_info for var_info in variables_info
if var_info.get('name') in remote_dataset.
variables.keys() and var_info.get(
'name') not in remote_dataset.dims.keys()
]
do_update_of_variables_meta_info_once = False
local_ds.add_dataset(
os.path.join(local_id, file_name),
(time_coverage_start, time_coverage_end))
if do_update_of_verified_time_coverage_start_once:
verified_time_coverage_start = time_coverage_start
do_update_of_verified_time_coverage_start_once = False
verified_time_coverage_end = time_coverage_end
child_monitor.done()
else:
outdated_file_list = []
for file_rec in selected_file_list:
filename, _, _, file_size, url = file_rec
dataset_file = os.path.join(local_path, filename)
# todo (forman, 20160915): must perform better checks on dataset_file if it is...
# ... outdated or incomplete or corrupted.
# JSON also includes "checksum" and "checksum_type" fields.
if not os.path.isfile(dataset_file) or (
file_size
and os.path.getsize(dataset_file) != file_size):
outdated_file_list.append(file_rec)
if outdated_file_list:
with monitor.starting('Sync ' + self.id,
len(outdated_file_list)):
bytes_to_download = sum(
[file_rec[3] for file_rec in outdated_file_list])
dl_stat = _DownloadStatistics(bytes_to_download)
file_number = 1
for filename, coverage_from, coverage_to, file_size, url in outdated_file_list:
dataset_file = os.path.join(local_path, filename)
sub_monitor = monitor.child(work=1.0)
# noinspection PyUnusedLocal
def reporthook(block_number, read_size,
total_file_size):
dl_stat.handle_chunk(read_size)
sub_monitor.progress(work=read_size,
msg=str(dl_stat))
sub_monitor_msg = "file %d of %d" % (
file_number, len(outdated_file_list))
with sub_monitor.starting(sub_monitor_msg,
file_size):
urllib.request.urlretrieve(
url[protocol],
filename=dataset_file,
reporthook=reporthook)
file_number += 1
local_ds.add_dataset(
os.path.join(local_id, filename),
(coverage_from, coverage_to))
if do_update_of_verified_time_coverage_start_once:
verified_time_coverage_start = coverage_from
do_update_of_verified_time_coverage_start_once = False
verified_time_coverage_end = coverage_to
except OSError as e:
raise DataAccessError(
"Copying remote data source failed: {}".format(e),
source=self) from e
local_ds.meta_info['temporal_coverage_start'] = TimeLike.format(
verified_time_coverage_start)
local_ds.meta_info['temporal_coverage_end'] = TimeLike.format(
verified_time_coverage_end)
local_ds.save(True)
def plot_map(ds: xr.Dataset,
var: VarName.TYPE = None,
indexers: DictLike.TYPE = None,
time: TimeLike.TYPE = None,
region: PolygonLike.TYPE = None,
projection: str = 'PlateCarree',
central_lon: float = 0.0,
title: str = None,
properties: DictLike.TYPE = None,
file: str = None) -> Figure:
"""
Create a geographic map plot for the variable given by dataset *ds* and variable name *var*.
Plots the given variable from the given dataset on a map with coastal lines.
In case no variable name is given, the first encountered variable in the
dataset is plotted. In case no *time* is given, the first time slice
is taken. It is also possible to set extents of the plot. If no extents
are given, a global plot is created.
The plot can either be shown using pyplot functionality, or saved,
if a path is given. The following file formats for saving the plot
are supported: eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg,
svgz, tif, tiff
:param ds: the dataset containing the variable to plot
:param var: the variable's name
:param indexers: Optional indexers into data array of *var*. The *indexers* is a dictionary
or a comma-separated string of key-value pairs that maps the variable's dimension names
to constant labels. e.g. "layer=4".
:param time: time slice index to plot, can be a string "YYYY-MM-DD" or an integer number
:param region: Region to plot
:param projection: name of a global projection, see http://scitools.org.uk/cartopy/docs/v0.15/crs/projections.html
:param central_lon: central longitude of the projection in degrees
:param title: an optional title
:param properties: optional plot properties for Python matplotlib,
e.g. "bins=512, range=(-1.5, +1.5)"
For full reference refer to
https://matplotlib.org/api/lines_api.html and
https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.contourf.html
:param file: path to a file in which to save the plot
:return: a matplotlib figure object or None if in IPython mode
"""
if not isinstance(ds, xr.Dataset):
raise NotImplementedError(
'Only gridded datasets are currently supported')
var_name = None
if not var:
for key in ds.data_vars.keys():
var_name = key
break
else:
var_name = VarName.convert(var)
var = ds[var_name]
time = TimeLike.convert(time)
indexers = DictLike.convert(indexers) or {}
properties = DictLike.convert(properties) or {}
extents = None
region = PolygonLike.convert(region)
if region:
lon_min, lat_min, lon_max, lat_max = region.bounds
if not _check_bounding_box(lat_min, lat_max, lon_min, lon_max):
raise ValueError(
'Provided plot extents do not form a valid bounding box '
'within [-180.0,+180.0,-90.0,+90.0]')
extents = [lon_min, lon_max, lat_min, lat_max]
# See http://scitools.org.uk/cartopy/docs/v0.15/crs/projections.html#
if projection == 'PlateCarree':
proj = ccrs.PlateCarree(central_longitude=central_lon)
elif projection == 'LambertCylindrical':
proj = ccrs.LambertCylindrical(central_longitude=central_lon)
elif projection == 'Mercator':
proj = ccrs.Mercator(central_longitude=central_lon)
elif projection == 'Miller':
proj = ccrs.Miller(central_longitude=central_lon)
elif projection == 'Mollweide':
proj = ccrs.Mollweide(central_longitude=central_lon)
elif projection == 'Orthographic':
proj = ccrs.Orthographic(central_longitude=central_lon)
elif projection == 'Robinson':
proj = ccrs.Robinson(central_longitude=central_lon)
elif projection == 'Sinusoidal':
proj = ccrs.Sinusoidal(central_longitude=central_lon)
elif projection == 'NorthPolarStereo':
proj = ccrs.NorthPolarStereo(central_longitude=central_lon)
elif projection == 'SouthPolarStereo':
proj = ccrs.SouthPolarStereo(central_longitude=central_lon)
else:
raise ValueError('illegal projection: "%s"' % projection)
figure = plt.figure(figsize=(8, 4))
ax = plt.axes(projection=proj)
if extents:
ax.set_extent(extents)
else:
ax.set_global()
ax.coastlines()
var_data = _get_var_data(var,
indexers,
time=time,
remaining_dims=('lon', 'lat'))
var_data.plot.contourf(ax=ax, transform=proj, **properties)
if title:
ax.set_title(title)
figure.tight_layout()
if file:
figure.savefig(file)
return figure if not in_notebook() else None
def test_json(self):
self.assertEqual(TimeLike.to_json(datetime(2017, 4, 19)), '2017-04-19')
self.assertEqual(TimeLike.from_json('2017-04-19'), datetime(2017, 4, 19))
def test_format(self):
self.assertEqual(TimeLike.format(None), '')
self.assertEqual(TimeLike.format(datetime(2017, 4, 19)), '2017-04-19')
def test_convert(self):
self.assertEqual(TimeLike.convert('2017-04-19'), datetime(2017, 4, 19))
self.assertEqual(TimeLike.convert(datetime(2017, 4, 19)), datetime(2017, 4, 19))
self.assertEqual(TimeLike.convert(date(2017, 4, 19)), datetime(2017, 4, 19, 12))
self.assertEqual(TimeLike.convert(' '), None)
self.assertEqual(TimeLike.convert(None), None)
def plot_map(ds: xr.Dataset,
var: VarName.TYPE = None,
indexers: DictLike.TYPE = None,
time: TimeLike.TYPE = None,
region: PolygonLike.TYPE = None,
projection: str = 'PlateCarree',
central_lon: float = 0.0,
title: str = None,
contour_plot: bool = False,
properties: DictLike.TYPE = None,
file: str = None) -> object:
"""
Create a geographic map plot for the variable given by dataset *ds* and variable name *var*.
Plots the given variable from the given dataset on a map with coastal lines.
In case no variable name is given, the first encountered variable in the
dataset is plotted. In case no *time* is given, the first time slice
is taken. It is also possible to set extents of the plot. If no extents
are given, a global plot is created.
The plot can either be shown using pyplot functionality, or saved,
if a path is given. The following file formats for saving the plot
are supported: eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg,
svgz, tif, tiff
:param ds: the dataset containing the variable to plot
:param var: the variable's name
:param indexers: Optional indexers into data array of *var*. The *indexers* is a dictionary
or a comma-separated string of key-value pairs that maps the variable's dimension names
to constant labels. e.g. "layer=4".
:param time: time slice index to plot, can be a string "YYYY-MM-DD" or an integer number
:param region: Region to plot
:param projection: name of a global projection, see http://scitools.org.uk/cartopy/docs/v0.15/crs/projections.html
:param central_lon: central longitude of the projection in degrees
:param title: an optional title
:param contour_plot: If true plot a filled contour plot of data, otherwise plots a pixelated colormesh
:param properties: optional plot properties for Python matplotlib,
e.g. "bins=512, range=(-1.5, +1.5)"
For full reference refer to
https://matplotlib.org/api/lines_api.html and
https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.contourf.html
:param file: path to a file in which to save the plot
:return: a matplotlib figure object or None if in IPython mode
"""
if not isinstance(ds, xr.Dataset):
raise ValidationError('Only gridded datasets are currently supported.')
var_name = None
if not var:
for key in ds.data_vars.keys():
var_name = key
break
else:
var_name = VarName.convert(var)
var = ds[var_name]
time = TimeLike.convert(time)
indexers = DictLike.convert(indexers) or {}
properties = DictLike.convert(properties) or {}
extents = None
bounds = handle_plot_polygon(region)
if bounds:
lon_min, lat_min, lon_max, lat_max = bounds
extents = [lon_min, lon_max, lat_min, lat_max]
if len(ds.lat) < 2 or len(ds.lon) < 2:
# Matplotlib can not plot datasets with less than these dimensions with
# contourf and pcolormesh methods
raise ValidationError('The minimum dataset spatial dimensions to create a map'
' plot are (2,2)')
# See http://scitools.org.uk/cartopy/docs/v0.15/crs/projections.html#
if projection == 'PlateCarree':
proj = ccrs.PlateCarree(central_longitude=central_lon)
elif projection == 'LambertCylindrical':
proj = ccrs.LambertCylindrical(central_longitude=central_lon)
elif projection == 'Mercator':
proj = ccrs.Mercator(central_longitude=central_lon)
elif projection == 'Miller':
proj = ccrs.Miller(central_longitude=central_lon)
elif projection == 'Mollweide':
proj = ccrs.Mollweide(central_longitude=central_lon)
elif projection == 'Orthographic':
proj = ccrs.Orthographic(central_longitude=central_lon)
elif projection == 'Robinson':
proj = ccrs.Robinson(central_longitude=central_lon)
elif projection == 'Sinusoidal':
proj = ccrs.Sinusoidal(central_longitude=central_lon)
elif projection == 'NorthPolarStereo':
proj = ccrs.NorthPolarStereo(central_longitude=central_lon)
elif projection == 'SouthPolarStereo':
proj = ccrs.SouthPolarStereo(central_longitude=central_lon)
else:
raise ValidationError('illegal projection: "%s"' % projection)
figure = plt.figure(figsize=(8, 4))
ax = plt.axes(projection=proj)
if extents:
ax.set_extent(extents, ccrs.PlateCarree())
else:
ax.set_global()
ax.coastlines()
var_data = get_var_data(var, indexers, time=time, remaining_dims=('lon', 'lat'))
# transform keyword is for the coordinate our data is in, which in case of a
# 'normal' lat/lon dataset is PlateCarree.
if contour_plot:
var_data.plot.contourf(ax=ax, transform=ccrs.PlateCarree(), subplot_kws={'projection': proj}, **properties)
else:
var_data.plot.pcolormesh(ax=ax, transform=ccrs.PlateCarree(), subplot_kws={'projection': proj}, **properties)
if title:
ax.set_title(title)
figure.tight_layout()
if file:
try:
figure.savefig(file)
except MemoryError:
raise MemoryError('Not enough memory to save the plot. Try using a different file format'
' or enabling contour_plot.')
return figure if not in_notebook() else ax