def test_roll(chunks, shift, axis):
x = np.random.randint(10, size=(4, 6))
a = da.from_array(x, chunks=chunks)
if _maybe_len(shift) != _maybe_len(axis):
with pytest.raises(TypeError if axis is None else ValueError):
da.roll(a, shift, axis)
else:
assert_eq(np.roll(x, shift, axis), da.roll(a, shift, axis))
def test_roll(chunks, shift, axis):
x = np.random.randint(10, size=(4, 6))
a = da.from_array(x, chunks=chunks)
if _maybe_len(shift) != _maybe_len(axis):
with pytest.raises(TypeError if axis is None else ValueError):
da.roll(a, shift, axis)
else:
assert_eq(np.roll(x, shift, axis), da.roll(a, shift, axis))
def test_roll(chunks, shift, axis):
x = np.random.randint(10, size=(4, 6))
a = da.from_array(x, chunks=chunks)
if _maybe_len(shift) != _maybe_len(axis):
with pytest.raises(TypeError if axis is None else ValueError):
da.roll(a, shift, axis)
else:
if (_maybe_len(shift) > 1
and LooseVersion(np.__version__) < LooseVersion("1.12.0")):
pytest.skip("NumPy %s doesn't support multiple axes with `roll`."
" Need NumPy 1.12.0 or greater." % np.__version__)
assert_eq(np.roll(x, shift, axis), da.roll(a, shift, axis))
def access_roll(x):
#da.roll extracts an array where all values are shifted left, right, up or down
l_roll = da.roll(x, 1, axis=0)
r_roll = da.roll(x, -1, axis=0)
return l_roll + r_roll + da.roll(x,1,axis=1) + da.roll(x,-1,axis=1) + \
da.roll(l_roll, 1, axis=1) + da.roll(l_roll, -1, axis=1) + \
da.roll(r_roll, 1, axis=1) + da.roll(r_roll, -1, axis=1)
def _remove_bad_pixels(dask_array, bad_pixel_array):
"""Replace values in bad pixels with mean of neighbors.
Parameters
----------
dask_array : Dask array
Must be at least two dimensions
bad_pixel_array : array-like
Must either have the same shape as dask_array,
or the same shape as the two last dimensions of dask_array.
Returns
-------
data_output : Dask array
Examples
--------
>>> import pyxem.utils.dask_tools as dt
>>> s = pxm.dummy_data.dummy_data.get_dead_pixel_signal(lazy=True)
>>> dead_pixels = dt._find_dead_pixels(s.data)
>>> data_output = dt._remove_bad_pixels(s.data, dead_pixels)
"""
if len(dask_array.shape) < 2:
raise ValueError("dask_array {0} must be at least 2 dimensions".format(
dask_array.shape))
if bad_pixel_array.shape == dask_array.shape:
pass
elif bad_pixel_array.shape == dask_array.shape[-2:]:
temp_array = da.zeros_like(dask_array)
bad_pixel_array = da.add(temp_array, bad_pixel_array)
else:
raise ValueError(
"bad_pixel_array {0} must either 2-D and have the same shape "
"as the two last dimensions in dask_array {1}. Or be "
"the same shape as dask_array {2}".format(bad_pixel_array.shape,
dask_array.shape[-2:],
dask_array.shape))
dif0 = da.roll(dask_array, shift=1, axis=-2)
dif1 = da.roll(dask_array, shift=-1, axis=-2)
dif2 = da.roll(dask_array, shift=1, axis=-1)
dif3 = da.roll(dask_array, shift=-1, axis=-1)
dif = (dif0 + dif1 + dif2 + dif3) / 4
dif = dif * bad_pixel_array
data_output = da.multiply(dask_array, da.logical_not(bad_pixel_array))
data_output = data_output + dif
return data_output
def test_roll(chunks, shift, axis):
x = np.random.randint(10, size=(4, 6))
a = da.from_array(x, chunks=chunks)
if _maybe_len(shift) != _maybe_len(axis):
with pytest.raises(TypeError if axis is None else ValueError):
da.roll(a, shift, axis)
else:
if (_maybe_len(shift) > 1 and
LooseVersion(np.__version__) < LooseVersion("1.12.0")):
pytest.skip(
"NumPy %s doesn't support multiple axes with `roll`."
" Need NumPy 1.12.0 or greater." % np.__version__
)
assert_eq(np.roll(x, shift, axis), da.roll(a, shift, axis))
def _find_hot_pixels(dask_array, threshold_multiplier=500, mask_array=None):
"""Find single pixels which have much larger values compared to neighbors.
Finds pixel which has very large value difference compared to its
neighbors. The functions looks at both at the direct neighbors
(x-1, y), and also the diagonal neighbors (x-1, y-1).
Experimental function, so use with care.
Parameters
----------
dask_array : Dask array
Must be have 4 dimensions.
threshold_multiplier : scaler
Used to threshold the dif.
mask_array : NumPy array, optional
Array with bool values. The True values will be masked
(i.e. ignored). Must have the same shape as the two
last dimensions in dask_array.
"""
if len(dask_array.shape) < 2:
raise ValueError("dask_array must have at least 2 dimensions")
dask_array = dask_array.astype("float64")
dif0 = da.roll(dask_array, shift=1, axis=-2)
dif1 = da.roll(dask_array, shift=-1, axis=-2)
dif2 = da.roll(dask_array, shift=1, axis=-1)
dif3 = da.roll(dask_array, shift=-1, axis=-1)
dif4 = da.roll(dask_array, shift=(1, 1), axis=(-2, -1))
dif5 = da.roll(dask_array, shift=(-1, 1), axis=(-2, -1))
dif6 = da.roll(dask_array, shift=(1, -1), axis=(-2, -1))
dif7 = da.roll(dask_array, shift=(-1, -1), axis=(-2, -1))
dif = dif0 + dif1 + dif2 + dif3 + dif4 + dif5 + dif6 + dif7
dif = dif - (dask_array * 8)
if mask_array is not None:
data = _mask_array(dask_array, mask_array=mask_array)
else:
data = dask_array
data_mean = data.mean() * threshold_multiplier
data_threshold = dif < -data_mean
if mask_array is not None:
mask_array = np.invert(mask_array).astype(np.float64)
data_threshold = data_threshold * mask_array
return data_threshold
def vertical_integration(var_x, var_y):
"""
Vertical integration.
Perform a non-cyclic centered finite-difference to integrate
variable x with respect to variable y along pressure levels.
Parameters
----------
x: iris.cube.Cube
Cube of variable x.
y: iris.cube.Cube
Cube of variable y.
Returns
-------
dxdy: iris.cube.Cube
Cube of variable integrated along pressure levels.
"""
plevs = var_x.shape[1]
dxdy_0 = (
(var_x[:, 1, :, :].lazy_data() - var_x[:, 0, :, :].lazy_data()) /
(var_y[:, 1, :, :].lazy_data() - var_y[:, 0, :, :].lazy_data()))
dxdy_centre = ((var_x[:, 2:plevs, :, :].lazy_data() -
var_x[:, 0:plevs - 2, :, :].lazy_data()) /
(var_y[:, 2:plevs, :, :].lazy_data() -
var_y[:, 0:plevs - 2, :, :].lazy_data()))
dxdy_end = ((var_x[:, plevs - 1, :, :].lazy_data() -
var_x[:, plevs - 2, :, :].lazy_data()) /
(var_y[:, plevs - 1, :, :].lazy_data() -
var_y[:, plevs - 2, :, :].lazy_data()))
bounds = [dxdy_end, dxdy_0]
stacked_bounds = da.stack(bounds, axis=1)
total = [dxdy_centre, stacked_bounds]
# Concatenate arrays where the last slice is dxdy_0
dxdy = da.concatenate(total, axis=1)
# Move dxdy_0 to the beggining of the array
dxdy = da.roll(dxdy, 1, axis=1)
return dxdy
def fix_pattern_order(data, shift=-1, overwrite=False, corrupt_idx=(0, 0),
overwrite_idx=(-1, -1)):
"""Shift the patterns a number of steps equal to `shift` using
`numpy.roll` or the dask equivalent. If a pattern specified by
`corrupt_idx` is corrupted this pattern can be overwritten by
another pattern specified by `overwrite_idx` before shifting, if
the data is not lazy.
Parameters
----------
data : array_like
Two-dimensional array containing signal data.
shift : int, optional
Number of steps to shift patterns.
overwrite : bool, optional
Whether to overwrite a pattern or not.
corrupt_idx : tuple, optional
Index of corrupted pattern.
overwrite_idx : tuple, optional
Index of pattern to overwrite corrupted pattern with.
Returns
-------
data : array_like
Two-dimensional array containing shifted data.
"""
# Overwrite patterns before shifting
if overwrite:
# Check if lazy
if isinstance(data, da.Array):
raise ValueError("Cannot overwrite data in dask array.")
data[corrupt_idx] = data[overwrite_idx]
# Shift patterns
sx, sy = data.shape[2:]
shift_by = shift * sx * sy
if isinstance(data, da.Array):
data = da.roll(data, shift=shift_by)
else:
data = np.roll(data, shift=shift_by)
return data
def roll(field, to_roll, axes_order, **kwargs):
from dask.array import roll
indeces = []
shifts = []
for axis, shift in to_roll.items():
axis_indeces = []
last_index = -1
count = axes_order.count(axis)
for i in range(count):
axis_indeces.append(axes_order.index(axis, last_index + 1))
last_index = axis_indeces[-1]
if count > 1:
axis_order = kwargs[key + "_order"]
else:
axis_order = [0]
for idx, pos in zip(axis_indeces, axis_order):
indeces.append(idx)
shifts.append(shift[pos])
return roll(field, tuple(shifts), axis=tuple(indeces))
def _extract_variable(in_files, var, cfg, out_dir):
logger.info("CMORizing variable '%s' from input files '%s'",
var['short_name'], ', '.join(in_files))
attributes = deepcopy(cfg['attributes'])
attributes['mip'] = var['mip']
cmor_table = CMOR_TABLES[attributes['project_id']]
definition = cmor_table.get_variable(var['mip'], var['short_name'])
cube = _load_cube(in_files, var)
utils.set_global_atts(cube, attributes)
# Set correct names
cube.var_name = definition.short_name
# cube.standard_name = definition.standard_name
cube.long_name = definition.long_name
# Fix units
cube.units = definition.units
# Fix data type
cube.data = cube.core_data().astype('float32')
# Roll longitude
cube.coord('longitude').points = cube.coord('longitude').points + 180.
nlon = len(cube.coord('longitude').points)
cube.data = da.roll(cube.core_data(), int(nlon / 2), axis=-1)
# Fix coordinates
cube = _fix_coordinates(cube, definition)
cube.coord('latitude').attributes = None
cube.coord('longitude').attributes = None
cube = _fix_time_monthly(cube)
logger.debug("Saving cube\n%s", cube)
utils.save_variable(cube, cube.var_name, out_dir, attributes)
logger.info("Finished CMORizing %s", ', '.join(in_files))
def _roll_cube_data(cube, shift, axis):
"""Roll a cube data on specified axis."""
cube.data = da.roll(cube.core_data(), shift, axis=axis)
return cube
def fix_metadata(self, cubes):
"""Fix metadata.
Remove unnecessary spaces in metadat and rename ``var_name`` of
latitude and longitude and fix longitude boundary description may be
wrong (lons=[0, ..., 356.25]; on_bnds=[[-1.875, 1.875], ..., [354.375,
360]]).
Parameters
----------
cubes : iris.cube.CubeList
Cubes to fix.
Returns
-------
iris.cube.Cube
"""
coords_to_change = {
'latitude': 'lat',
'longitude': 'lon',
}
for cube in cubes:
strip_cube_metadata(cube)
for (std_name, var_name) in coords_to_change.items():
try:
coord = cube.coord(std_name)
except iris.exceptions.CoordinateNotFoundError:
pass
else:
coord.var_name = var_name
time_units = cube.attributes.get('parent_time_units')
if time_units is not None:
cube.attributes['parent_time_units'] = time_units.replace(
' (noleap)', '')
for cube in cubes:
coord_names = [cor.standard_name for cor in cube.coords()]
if 'longitude' in coord_names:
lon_coord = cube.coord('longitude')
if lon_coord.ndim == 1 and lon_coord.has_bounds():
lon_bnds = lon_coord.bounds.copy()
# atmos & land
if lon_coord.points[0] == 0. and \
lon_coord.points[-1] == 356.25 and \
lon_bnds[-1][-1] == 360.:
lon_bnds[-1][-1] = 358.125
lon_coord.bounds = lon_bnds
# ocean & seaice
if lon_coord.points[0] == -0.9375:
lon_dim = cube.coord_dims('longitude')[0]
cube.data = da.roll(cube.core_data(), -1, axis=lon_dim)
lon_points = np.roll(lon_coord.core_points(), -1)
lon_bounds = np.roll(lon_coord.core_bounds(), -1,
axis=0)
lon_points[-1] += 360.0
lon_bounds[-1] += 360.0
lon_coord.points = lon_points
lon_coord.bounds = lon_bounds
return cubes
def _rmatvec(self, x):
if self.reshape:
x = da.reshape(x, self.dims)
y = da.roll(x, shift=-self.shift, axis=self.dir)
y = y.rechunk(x.chunks)
return y.ravel()