def migration(population, total, k, min_density, csx, csy, **kwargs):
"""
'relocation of a portion of a population from one region to another (migration dispersal)'
:param da.Array population: population to redistribute
:param da.Array k: Carrying Capacity - target location weight and distribution
:param float min_density: Each element may have a minimum possible density when relocating population
kwargs:
source_weight da.Array: Proportion of population to remove from an element. Bounded by `0` and `1`
target_weight da.Array: Multiplier for carrying capacity at destination
:return: Redistributed population
"""
if min_density is None:
min_density = 0
source_weight = da.clip(
kwargs.get("source_weight", da.ones_like(population)), 0, 1)
target_weight = da.clip(
kwargs.get("target_weight", da.ones_like(population)), 0, 1)
# Remove using the source weight
removed = population * source_weight
new_population = population - removed
# Weighting is a combination of k and the target weights
target_locations = k * target_weight
def redistribute(pop, target, default):
# Redistribute population proportionally at the target locations
redistribution = target / target.sum()
# If there are no target locations, all redistribution values will be inf.
# In this case, use the default
return da.where(
da.isinf(redistribution) | da.isnan(redistribution), default,
pop * redistribution)
new_population += redistribute(removed.sum(), target_locations, removed)
# Remove locations where density is not met
density_not_met = (target_locations >
0) & (new_population / target_locations < min_density)
removed = da.where(density_not_met, new_population, 0)
new_population -= removed
new_population += redistribute(
removed.sum(), da.where(density_not_met, 0, target_locations), removed)
return new_population
def test_clip():
# clip internally calls dd.Series.clip
s = pd.Series(np.random.randint(1, 100, size=20))
ds = dd.from_pandas(s, 3)
# applying Dask ufunc doesn't trigger computation
assert isinstance(da.clip(ds, 5, 50), dd.Series)
assert_eq(da.clip(ds, 5, 50), np.clip(s, 5, 50))
# applying Dask ufunc doesn't trigger computation
assert isinstance(np.clip(ds, 5, 50), dd.Series)
assert_eq(np.clip(ds, 5, 50), np.clip(s, 5, 50))
# applying Dask ufunc to normal Series triggers computation
assert isinstance(da.clip(s, 5, 50), pd.Series)
assert_eq(da.clip(s, 5, 50), np.clip(s, 5, 50))
df = pd.DataFrame(np.random.randint(1, 100, size=(20, 2)),
columns=['A', 'B'])
ddf = dd.from_pandas(df, 3)
# applying Dask ufunc doesn't trigger computation
assert isinstance(da.clip(ddf, 5.5, 40.5), dd.DataFrame)
assert_eq(da.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5))
# applying Dask ufunc doesn't trigger computation
assert isinstance(np.clip(ddf, 5.5, 40.5), dd.DataFrame)
assert_eq(np.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5))
# applying Dask ufunc to normal DataFrame triggers computation
assert isinstance(da.clip(df, 5.5, 40.5), pd.DataFrame)
assert_eq(da.clip(df, 5.5, 40.5), np.clip(df, 5.5, 40.5))
def test_clip():
# clip internally calls dd.Series.clip
s = pd.Series(np.random.randint(1, 100, size=20))
ds = dd.from_pandas(s, 3)
# applying Dask ufunc doesn't trigger computation
assert isinstance(da.clip(ds, 5, 50), dd.Series)
assert_eq(da.clip(ds, 5, 50), np.clip(s, 5, 50))
# applying Dask ufunc doesn't trigger computation
assert isinstance(np.clip(ds, 5, 50), dd.Series)
assert_eq(np.clip(ds, 5, 50), np.clip(s, 5, 50))
# applying Dask ufunc to normal Series triggers computation
assert isinstance(da.clip(s, 5, 50), pd.Series)
assert_eq(da.clip(s, 5, 50), np.clip(s, 5, 50))
df = pd.DataFrame(np.random.randint(1, 100, size=(20, 2)),
columns=['A', 'B'])
ddf = dd.from_pandas(df, 3)
# applying Dask ufunc doesn't trigger computation
assert isinstance(da.clip(ddf, 5.5, 40.5), dd.DataFrame)
assert_eq(da.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5))
# applying Dask ufunc doesn't trigger computation
assert isinstance(np.clip(ddf, 5.5, 40.5), dd.DataFrame)
assert_eq(np.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5))
# applying Dask ufunc to normal DataFrame triggers computation
assert isinstance(da.clip(df, 5.5, 40.5), pd.DataFrame)
assert_eq(da.clip(df, 5.5, 40.5), np.clip(df, 5.5, 40.5))
def discretize(x_data, n_state, min_=0.0, max_=1.0, chunks=()):
"""Primitive discretization of a microstructure.
Args:
x_data: the data to discrtize
n_state: the number of local states
min_: the minimum local state
max_: the maximum local state
Returns:
the discretized microstructure
>>> discretize(da.random.random((12, 9), chunks=(3, 9)),
... 3,
... chunks=(1,)).chunks
((3, 3, 3, 3), (9,), (1, 1, 1))
>>> discretize(np.array([[0, 1], [0.5, 0.5]]), 3, chunks=(1,)).chunks
((2,), (2,), (1, 1, 1))
>>> discretize(np.array([[0, 1], [0.5, 0.5]]), 3, chunks=(1,)).compute()
array([[[ 1., 0., 0.],
[ 0., 0., 1.]],
<BLANKLINE>
[[ 0., 1., 0.],
[ 0., 1., 0.]]])
"""
return da.maximum(
discretize_nomax(
da.clip(x_data, min_, max_),
da.linspace(min_, max_, n_state, chunks=chunks or (n_state, ))), 0)
def rescale_amplitude_range(self, darray, min_val, max_val, preview=None):
"""
Description
-----------
Clip the seismic data to specified values
Parameters
----------
darray : Array-like, acceptable inputs include Numpy, HDF5, or Dask Arrays
min_val : Number, min clip value
max_val : Numer, max clip value
Keywork Arguments
-----------------
preview : str, enables or disables preview mode and specifies direction
Acceptable inputs are (None, 'inline', 'xline', 'z')
Optimizes chunk size in different orientations to facilitate rapid
screening of algorithm output
Returns
-------
result : Dask Array
"""
darray, chunks_init = self.create_array(darray, preview=preview)
result = da.clip(darray, min_val, max_val)
return (result)
def relative_amplitude_change(self, darray, preview=None):
"""
Description
-----------
Compute the Relative Amplitude Change of the input data
Parameters
----------
darray : Array-like, acceptable inputs include Numpy, HDF5, or Dask Arrays
Keywork Arguments
-----------------
preview : str, enables or disables preview mode and specifies direction
Acceptable inputs are (None, 'inline', 'xline', 'z')
Optimizes chunk size in different orientations to facilitate rapid
screening of algorithm output
Returns
-------
result : Dask Array
"""
darray, chunks_init = self.create_array(darray, preview=preview)
env = self.envelope(darray)
env_prime = sp().first_derivative(env, axis=-1)
result = env_prime / env
result = da.clip(result, -1, 1)
return (result)
def concat_bands(self):
import dask.array as da
self.logger.info('Concatenate RGB bands...')
self._arrays['rgb'] = da.dstack((self._arrays['r'], self._arrays['g'],
self._arrays['b']))
self._arrays['rgb'] = da.clip(self._arrays['rgb'], 0, 1)
del self._arrays['r'], self._arrays['g'], self._arrays['b']
def clip(cube, minimum=None, maximum=None):
"""
Clip values at a specified minimum and/or maximum value
Values lower than minimum are set to minimum and values
higher than maximum are set to maximum.
Parameters
----------
cube: iris.cube.Cube
iris cube to be clipped
minimum: float
lower threshold to be applied on input cube data.
maximum: float
upper threshold to be applied on input cube data.
Returns
-------
iris.cube.Cube
clipped cube.
"""
if minimum is None and maximum is None:
raise ValueError("Either minimum, maximum or both have to be\
specified.")
elif minimum is not None and maximum is not None:
if maximum < minimum:
raise ValueError("Maximum should be equal or larger than minimum.")
cube.data = da.clip(cube.core_data(), minimum, maximum)
return cube
def slerp(self, val, low, high):
"""Code from https://github.com/soumith/dcgan.torch/issues/14"""
omega = da.arccos(da.clip(da.dot(low / da.linalg.norm(low), high.transpose() / da.linalg.norm(high)), -1, 1))
so = da.sin(omega)
if so == 0:
return (1.0-val) * low + val * high # L'Hopital's rule/LERP
return da.sin((1.0 - val) * omega) / so * low + da.sin(val * omega) / so * high
def func(band_data, index=None):
idx = np.array(kwargs['idx']) / 255
sc = np.array(kwargs['sc']) / 255
band_data *= .01
# Interpolate band on [0,1] using "lazy" arrays (put calculations off until the end).
band_data = xr.DataArray(da.clip(band_data.data.map_blocks(np.interp, xp=idx, fp=sc), 0, 1),
coords=band_data.coords, dims=band_data.dims, name=band_data.name,
attrs=band_data.attrs)
return band_data
def func(band_data, xp, fp, index=None):
# Interpolate band on [0,1] using "lazy" arrays (put calculations off until the end).
band_data = xr.DataArray(da.clip(
band_data.data.map_blocks(np.interp, xp=xp, fp=fp), 0, 1),
coords=band_data.coords,
dims=band_data.dims,
name=band_data.name,
attrs=band_data.attrs)
return band_data
def test_clip(pandas, min, max):
dask = dd.from_pandas(pandas, 3)
pandas_type = pandas.__class__
dask_type = dask.__class__
# clip internally calls dd.Series.clip
# applying Dask ufunc doesn't trigger computation
assert isinstance(da.clip(dask, min, max), dask_type)
assert_eq(da.clip(dask, min, max), np.clip(pandas, min, max))
# applying Numpy ufunc doesn't trigger computation
assert isinstance(np.clip(dask, min, max), dask_type)
assert_eq(np.clip(dask, min, max), np.clip(pandas, min, max))
# applying Dask ufunc to normal pandas objects triggers computation
assert isinstance(da.clip(pandas, min, max), pandas_type)
assert_eq(da.clip(pandas, min, max), np.clip(pandas, min, max))
def test_clip(pandas, min, max):
dask = dd.from_pandas(pandas, 3)
pandas_type = pandas.__class__
dask_type = dask.__class__
# clip internally calls dd.Series.clip
# applying Dask ufunc doesn't trigger computation
assert isinstance(da.clip(dask, min, max), dask_type)
assert_eq(da.clip(dask, min, max), np.clip(pandas, min, max))
# applying Numpy ufunc doesn't trigger computation
assert isinstance(np.clip(dask, min, max), dask_type)
assert_eq(np.clip(dask, min, max), np.clip(pandas, min, max))
# applying Dask ufunc to normal pandas objects triggers computation
assert isinstance(da.clip(pandas, min, max), pandas_type)
assert_eq(da.clip(pandas, min, max), np.clip(pandas, min, max))
def _transform_col(self, X_col: da.Array, quantiles: ArrayLike,
inverse: bool) -> ArrayLike:
output_distribution = self.output_distribution
if not inverse:
lower_bound_x = quantiles[0]
upper_bound_x = quantiles[-1]
lower_bound_y = 0
upper_bound_y = 1
else:
lower_bound_x = 0
upper_bound_x = 1
lower_bound_y = quantiles[0]
upper_bound_y = quantiles[-1]
# for inverse transform, match a uniform distribution
if output_distribution == "normal":
X_col = X_col.map_blocks(stats.norm.cdf)
# else output distribution is already a uniform distribution
if output_distribution == "normal":
lower_bounds_idx = X_col - BOUNDS_THRESHOLD < lower_bound_x
upper_bounds_idx = X_col + BOUNDS_THRESHOLD > upper_bound_x
if output_distribution == "uniform":
lower_bounds_idx = X_col == lower_bound_x
upper_bounds_idx = X_col == upper_bound_x
if not inverse:
# See the note in scikit-learn. This trick is to avoid
# repeated extreme values
X_col = 0.5 * (
X_col.map_blocks(np.interp, quantiles, self.references_) -
(-X_col).map_blocks(np.interp, -quantiles[::-1],
-self.references_[::-1]))
else:
X_col = X_col.map_blocks(np.interp, self.references_, quantiles)
X_col[upper_bounds_idx] = upper_bound_y
X_col[lower_bounds_idx] = lower_bound_y
if not inverse:
if output_distribution == "normal":
X_col = X_col.map_blocks(stats.norm.ppf)
# find the value to clip the data to avoid mapping to
# infinity. Clip such that the inverse transform will be
# consistent
clip_min = stats.norm.ppf(BOUNDS_THRESHOLD - np.spacing(1))
clip_max = stats.norm.ppf(1 -
(BOUNDS_THRESHOLD - np.spacing(1)))
X_col = da.clip(X_col, clip_min, clip_max)
# else output distribution is uniform and the ppf is the
# identity function so we let X_col unchanged
return X_col
def get_phaang(cos_vza, cos_sza, sin_vza, sin_sza, cos_raa):
"""
Gets the phase angle
"""
cos_phase_angle = da.clip(
cos_vza * cos_sza + sin_vza * sin_sza * cos_raa, -1, 1)
phase_angle = da.arccos(cos_phase_angle)
sin_phase_angle = da.sin(phase_angle)
return cos_phase_angle, phase_angle, sin_phase_angle
def _transform_col(self, X_col, quantiles, inverse):
if self.output_distribution == 'normal':
output_distribution = 'norm'
else:
output_distribution = self.output_distribution
output_distribution = getattr(stats, output_distribution)
if not inverse:
lower_bound_x = quantiles[0]
upper_bound_x = quantiles[-1]
lower_bound_y = 0
upper_bound_y = 1
else:
lower_bound_x = 0
upper_bound_x = 1
lower_bound_y = quantiles[0]
upper_bound_y = quantiles[-1]
X_col = X_col.map_blocks(output_distribution.cdf)
lower_bounds_idx = (X_col - skdata.BOUNDS_THRESHOLD <
lower_bound_x)
upper_bounds_idx = (X_col + skdata.BOUNDS_THRESHOLD >
upper_bound_x)
if not inverse:
# See the note in scikit-learn. This trick is to avoid
# repeated extreme values
X_col = 0.5 * (
X_col.map_blocks(np.interp, quantiles, self.references_) -
(-X_col).map_blocks(np.interp, -quantiles[::-1],
-self.references_[::-1])
)
else:
X_col = X_col.map_blocks(np.interp, self.references_, quantiles)
X_col[upper_bounds_idx] = upper_bound_y
X_col[lower_bounds_idx] = lower_bound_y
if not inverse:
X_col = X_col.map_blocks(output_distribution.ppf)
clip_min = output_distribution.ppf(skdata.BOUNDS_THRESHOLD -
np.spacing(1))
clip_max = output_distribution.ppf(1 - (skdata.BOUNDS_THRESHOLD -
np.spacing(1)))
X_col = da.clip(X_col, clip_min, clip_max)
return X_col
def get_dense_mdm(self) -> np.ndarray:
r"""
Form the Mapped Distance Matrix as a dense Numpy array.
Returns
-------
np.ndarray
Mapped Distance Matrix as a Numpy array.
"""
if self.mode == 'radial':
distances = da.linalg.norm(self.samples1[:, None, :] - self.samples2[None, ...], axis=-1)
elif self.mode == 'zonal':
distances = da.clip(da.sum(self.samples1[:, None, :] * self.samples2[None, ...], axis=-1), a_min=-1,
a_max=1)
else:
raise ValueError(f'Unsupported mode {self.mode}.')
mapped_distance_matrix = self.function(distances)
return mapped_distance_matrix.astype(self.dtype)
def convert_dtype(in_data, min_val, max_val, to_dtype):
"""
Description
-----------
Convience function to read file and create a pointer to data on disk
Parameters
----------
in_data : Dask Array, data to convert
min_val : number, lower clip
max_val : number, upper clip
to_dtype : NumPy dtype
Acceptable formats include (np.int8, np.float16, np.float32)
Returns
-------
out : Dask Array, converted data
"""
# Check if data is already in correct format
if in_data.dtype == to_dtype:
return (in_data)
else:
in_data = da.clip(in_data, min_val, max_val)
if to_dtype == np.int8:
in_data = ((in_data - min_val) / (max_val - min_val))
dtype = np.iinfo(np.int8)
in_data *= dtype.max - dtype.min
in_data -= dtype.min
out = in_data.astype(np.int8)
elif to_dtype == np.float16:
out = in_data.astype(np.float16)
elif to_dtype == np.int32:
out = in_data.astype(np.float32)
else:
raise Exception('Not a valid dtype')
return (out)
def single_channel_percentile_norm(data: da.core.Array,
min_p: float = 50.0,
max_p: float = 99.8,
**kwargs) -> da.core.Array:
# Enforce shape
if len(data.shape) > 4:
raise exceptions.InvalidShapeError(len(data.shape), 4)
# Get the norm by values
norm_by = da.percentile(data.flatten(), [min_p, min_p]).compute()
# Norm
normed = (data - norm_by[0]) / (norm_by[1] - norm_by[0])
# Clip any values outside of 0 and 1
clipped = da.clip(normed, 0, 1)
# Scale them between 0 and 255
return clipped * 255
def get_dask_mdm(self) -> da.core.Array:
r"""
Form the Mapped Distance Matrix as a dask array.
Returns
-------
da.core.Array
Mapped Distance Matrix as a Dask array.
"""
samples_da = da.from_array(self.samples1[:, None, :], chunks=self.chunks)
knots_da = da.from_array(self.samples2[None, ...], chunks=self.chunks)
if self.mode == 'radial':
distances_da = da.linalg.norm(samples_da - knots_da, axis=-1)
elif self.mode == 'zonal':
distances_da = da.clip(da.sum(samples_da * knots_da, axis=-1), a_min=-1, a_max=1)
else:
raise ValueError(f'Unsupported mode {self.mode}.')
mapped_distance_matrix = da.map_blocks(self.function, distances_da, dtype=self.dtype)
return mapped_distance_matrix
def get_overlap(cos1, cos2, tan1, tan2, sin3, distance, hb, m_pi):
"""
Applies the HB ratio transformation
"""
OverlapInfo = namedtuple('OverlapInfo', 'tvar sint overlap temp')
temp = (1.0 / cos1) + (1.0 / cos2)
cost = da.clip(
hb * da.sqrt(distance * distance +
tan1 * tan1 * tan2 * tan2 * sin3 * sin3) / temp, -1,
1)
tvar = da.arccos(cost)
sint = da.sin(tvar)
overlap = 1.0 / m_pi * (tvar - sint * cost) * temp
overlap = da.maximum(overlap, 0)
return OverlapInfo(tvar=tvar, sint=sint, overlap=overlap, temp=temp)
def discretize(x_data, n_state=2, min_=0.0, max_=1.0, chunks=None):
"""Primitive discretization of a microstructure.
Args:
x_data: the data to discretize
n_state: the number of local states
min_: the minimum local state
max_: the maximum local state
chunks: chunks size for state axis
Returns:
the discretized microstructure
>>> discretize(da.random.random((12, 9), chunks=(3, 9)),
... 3,
... chunks=1).chunks
((3, 3, 3, 3), (9,), (1, 1, 1))
>>> discretize(np.array([[0, 1], [0.5, 0.5]]), 3, chunks=1).chunks
((2,), (2,), (1, 1, 1))
>>> assert np.allclose(
... discretize(
... np.array([[0, 1], [0.5, 0.5]]),
... 3,
... chunks=1
... ).compute(),
... [[[1, 0, 0], [0, 0, 1]], [[0, 1, 0], [0, 1, 0]]]
... )
"""
return da.maximum(
discretize_nomax(
da.clip(x_data, min_, max_),
da.linspace(min_, max_, n_state, chunks=(chunks or n_state,)),
),
0,
)
def apply_geometric_transform(img: da.Array, dark: da.Array, flat: da.Array,
cof_dist: Dict[str, List[float]]) -> da.Array:
"""Apply geometric transformation to array
Parameters
----------
img
Input image
flat
flat field image
cof_dist
Distortion coefficients
Returns
-------
distortion corrected array
"""
norm = da.flipud((img - dark) / da.clip(flat, 1e-6, 1e6))
masked = delayed(mask_image)(norm)
masked = da.from_delayed(masked, shape=norm.shape, dtype="float32")
matrix = get_matrix(masked.shape, cof_dist)
new = delayed(warp)(masked, matrix)
res = da.from_delayed(new, shape=img.shape, dtype="float32")
return res
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
c = da.from_array(z, chunks=(2,))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a ** b, x ** y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a ** 2, x ** 2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2 ** b, 2 ** y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b/10), np.arcsin(y/10))
assert eq(da.arccos(b/10), np.arccos(y/10))
assert eq(da.arctan(b/10), np.arctan(y/10))
assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
assert eq(da.arccosh(b*10), np.arccosh(y*10))
assert eq(da.arctanh(b/10), np.arctanh(y/10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def get_reflectance(self, sun_zenith, sat_zenith, azidiff, bandname, redband=None):
"""Get the reflectance from the three sun-sat angles"""
# Get wavelength in nm for band:
if isinstance(bandname, float):
LOG.warning('A wavelength is provided instead of band name - ' +
'disregard the relative spectral responses and assume ' +
'it is the effective wavelength: %f (micro meter)', bandname)
wvl = bandname * 1000.0
else:
wvl = self.get_effective_wavelength(bandname)
wvl = wvl * 1000.0
rayl, wvl_coord, azid_coord, satz_sec_coord, sunz_sec_coord = self.get_reflectance_lut()
# force dask arrays
compute = False
if HAVE_DASK and not isinstance(sun_zenith, Array):
compute = True
sun_zenith = from_array(sun_zenith, chunks=sun_zenith.shape)
sat_zenith = from_array(sat_zenith, chunks=sat_zenith.shape)
azidiff = from_array(azidiff, chunks=azidiff.shape)
if redband is not None:
redband = from_array(redband, chunks=redband.shape)
clip_angle = rad2deg(arccos(1. / sunz_sec_coord.max()))
sun_zenith = clip(sun_zenith, 0, clip_angle)
sunzsec = 1. / cos(deg2rad(sun_zenith))
clip_angle = rad2deg(arccos(1. / satz_sec_coord.max()))
sat_zenith = clip(sat_zenith, 0, clip_angle)
satzsec = 1. / cos(deg2rad(sat_zenith))
shape = sun_zenith.shape
if not(wvl_coord.min() < wvl < wvl_coord.max()):
LOG.warning(
"Effective wavelength for band %s outside 400-800 nm range!",
str(bandname))
LOG.info(
"Set the rayleigh/aerosol reflectance contribution to zero!")
if HAVE_DASK:
chunks = sun_zenith.chunks if redband is None else redband.chunks
res = zeros(shape, chunks=chunks)
return res.compute() if compute else res
else:
return zeros(shape)
idx = np.searchsorted(wvl_coord, wvl)
wvl1 = wvl_coord[idx - 1]
wvl2 = wvl_coord[idx]
fac = (wvl2 - wvl) / (wvl2 - wvl1)
raylwvl = fac * rayl[idx - 1, :, :, :] + (1 - fac) * rayl[idx, :, :, :]
tic = time.time()
smin = [sunz_sec_coord[0], azid_coord[0], satz_sec_coord[0]]
smax = [sunz_sec_coord[-1], azid_coord[-1], satz_sec_coord[-1]]
orders = [
len(sunz_sec_coord), len(azid_coord), len(satz_sec_coord)]
f_3d_grid = atleast_2d(raylwvl.ravel())
if HAVE_DASK and isinstance(smin[0], Array):
# compute all of these at the same time before passing to the interpolator
# otherwise they are computed separately
smin, smax, orders, f_3d_grid = da.compute(smin, smax, orders, f_3d_grid)
minterp = MultilinearInterpolator(smin, smax, orders)
minterp.set_values(f_3d_grid)
if HAVE_DASK:
ipn = map_blocks(self._do_interp, minterp, sunzsec, azidiff,
satzsec, dtype=raylwvl.dtype, chunks=azidiff.chunks)
else:
ipn = self._do_interp(minterp, sunzsec, azidiff, satzsec)
LOG.debug("Time - Interpolation: {0:f}".format(time.time() - tic))
ipn *= 100
res = ipn
if redband is not None:
res = where(redband < 20., res,
(1 - (redband - 20) / 80) * res)
res = clip(res, 0, 100)
if compute:
res = res.compute()
return res
def _estimate_poissonian_noise_variance(dc, gain_factor, gain_offset,
correlation_factor):
variance = (dc * gain_factor + gain_offset) * correlation_factor
# The lower bound of the variance is the gaussian noise.
variance = da.clip(variance, gain_offset * correlation_factor, np.inf)
return variance
def get_reflectance(self, sun_zenith, sat_zenith, azidiff, bandname, redband=None):
"""Get the reflectance from the three sun-sat angles"""
# Get wavelength in nm for band:
if isinstance(bandname, float):
LOG.warning('A wavelength is provided instead of band name - ' +
'disregard the relative spectral responses and assume ' +
'it is the effective wavelength: %f (micro meter)', bandname)
wvl = bandname * 1000.0
else:
wvl = self.get_effective_wavelength(bandname)
if wvl is None:
LOG.error("Can't get effective wavelength for band %s on platform %s and sensor %s",
str(bandname), self.platform_name, self.sensor)
return None
else:
wvl = wvl * 1000.0
rayl, wvl_coord, azid_coord, satz_sec_coord, sunz_sec_coord = \
self.get_reflectance_lut()
# force dask arrays
compute = False
if HAVE_DASK and not isinstance(sun_zenith, Array):
compute = True
sun_zenith = from_array(sun_zenith, chunks=sun_zenith.shape)
sat_zenith = from_array(sat_zenith, chunks=sat_zenith.shape)
azidiff = from_array(azidiff, chunks=azidiff.shape)
if redband is not None:
redband = from_array(redband, chunks=redband.shape)
clip_angle = rad2deg(arccos(1. / sunz_sec_coord.max()))
sun_zenith = clip(sun_zenith, 0, clip_angle)
sunzsec = 1. / cos(deg2rad(sun_zenith))
clip_angle = rad2deg(arccos(1. / satz_sec_coord.max()))
sat_zenith = clip(sat_zenith, 0, clip_angle)
satzsec = 1. / cos(deg2rad(sat_zenith))
shape = sun_zenith.shape
if not(wvl_coord.min() < wvl < wvl_coord.max()):
LOG.warning(
"Effective wavelength for band %s outside 400-800 nm range!",
str(bandname))
LOG.info(
"Set the rayleigh/aerosol reflectance contribution to zero!")
if HAVE_DASK:
chunks = sun_zenith.chunks if redband is None \
else redband.chunks
res = zeros(shape, chunks=chunks)
return res.compute() if compute else res
else:
return zeros(shape)
idx = np.searchsorted(wvl_coord, wvl)
wvl1 = wvl_coord[idx - 1]
wvl2 = wvl_coord[idx]
fac = (wvl2 - wvl) / (wvl2 - wvl1)
raylwvl = fac * rayl[idx - 1, :, :, :] + (1 - fac) * rayl[idx, :, :, :]
tic = time.time()
smin = [sunz_sec_coord[0], azid_coord[0], satz_sec_coord[0]]
smax = [sunz_sec_coord[-1], azid_coord[-1], satz_sec_coord[-1]]
orders = [
len(sunz_sec_coord), len(azid_coord), len(satz_sec_coord)]
f_3d_grid = atleast_2d(raylwvl.ravel())
if HAVE_DASK and isinstance(smin[0], Array):
# compute all of these at the same time before passing to the interpolator
# otherwise they are computed separately
smin, smax, orders, f_3d_grid = da.compute(smin, smax, orders, f_3d_grid)
minterp = MultilinearInterpolator(smin, smax, orders)
minterp.set_values(f_3d_grid)
def _do_interp(minterp, sunzsec, azidiff, satzsec):
interp_points2 = np.vstack((sunzsec.ravel(),
180 - azidiff.ravel(),
satzsec.ravel()))
res = minterp(interp_points2)
return res.reshape(sunzsec.shape)
if HAVE_DASK:
ipn = map_blocks(_do_interp, minterp, sunzsec, azidiff,
satzsec, dtype=raylwvl.dtype,
chunks=azidiff.chunks)
else:
ipn = _do_interp(minterp, sunzsec, azidiff, satzsec)
LOG.debug("Time - Interpolation: {0:f}".format(time.time() - tic))
ipn *= 100
res = ipn
if redband is not None:
res = where(redband < 20., res,
(1 - (redband - 20) / 80) * res)
res = clip(res, 0, 100)
if compute:
res = res.compute()
return res
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2, ))
b = da.from_array(y, chunks=(2, ))
c = da.from_array(z, chunks=(2, ))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a**b, x**y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a**2, x**2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2**b, 2**y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b / 10), np.arcsin(y / 10))
assert eq(da.arccos(b / 10), np.arccos(y / 10))
assert eq(da.arctan(b / 10), np.arctan(y / 10))
assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10))
assert eq(da.arccosh(b * 10), np.arccosh(y * 10))
assert eq(da.arctanh(b / 10), np.arctanh(y / 10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def _estimate_poissonian_noise_variance(dc, gain_factor, gain_offset,
correlation_factor):
variance = (dc * gain_factor + gain_offset) * correlation_factor
# The lower bound of the variance is the gaussian noise.
variance = da.clip(variance, gain_offset * correlation_factor, np.inf)
return variance
